Merge branch 'master' into aressem/kill-mbuild

586 files changed, 11074 insertions, 4666 deletions

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 276fed18ac5..0466e97ec4f 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -20,6 +20,8 @@ include(build_settings.cmake)
 # Enable CTest unit testing
 enable_testing()
 
+vespa_install_data(valgrind-suppressions.txt etc/vespa)
+
 # Include vespa config definitions in every target
 include_directories(BEFORE ${CMAKE_BINARY_DIR}/configdefinitions/src)
 
@@ -45,6 +47,7 @@ add_subdirectory(container-disc)
 add_subdirectory(container-jersey2)
 add_subdirectory(container-messagebus)
 add_subdirectory(container-search)
+add_subdirectory(container-search-gui)
 add_subdirectory(container-search-and-docproc)
 add_subdirectory(clustercontroller-apps)
 add_subdirectory(clustercontroller-apputil)
diff --git a/README.md b/README.md
index d84ebbe4456..acbeb6ffcec 100644
--- a/README.md
+++ b/README.md
@@ -15,8 +15,8 @@ Travis-CI build status: [![Build Status](https://travis-ci.org/vespa-engine/vesp
 ## Get started developing
 
 ### Setup build environment
-C++ building is supported on CentOS 7. The Java source can be built on any platform having Java 8 and Maven installed. 
-We recommend using the following environment: [Create C++ dev environment on CentOS using VirtualBox and Vagrant](vagrant/README.md).
+C++ and Java building is supported on CentOS 7. The Java source can also be built on any platform having Java 8 and Maven installed. 
+We recommend using the following environment: [Create C++ / Java dev environment on CentOS using VirtualBox and Vagrant](vagrant/README.md).
 You can also setup CentOS 7 natively and install the following build dependencies:
 
     sudo yum-config-manager --add-repo https://copr.fedorainfracloud.org/coprs/g/vespa/vespa/repo/epel-7/group_vespa-vespa-epel-7.repo
diff --git a/application/pom.xml b/application/pom.xml
index db298451a8b..10d2a14721e 100644
--- a/application/pom.xml
+++ b/application/pom.xml
@@ -46,6 +46,11 @@
     </dependency>
     <dependency>
       <groupId>com.yahoo.vespa</groupId>
+      <artifactId>container-search-gui</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>com.yahoo.vespa</groupId>
       <artifactId>zkfacade</artifactId>
       <version>${project.version}</version>
     </dependency>
@@ -81,6 +86,17 @@
       <artifactId>icu4j</artifactId>
     </dependency>
     <dependency>
+      <groupId>com.optimaize.languagedetector</groupId>
+      <artifactId>language-detector</artifactId>
+      <exclusions>
+        <exclusion>
+          <!-- We want to get this via jdisc-core -->
+          <groupId>org.slf4j</groupId>
+          <artifactId>slf4j-api</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+    <dependency>
       <groupId>commons-collections</groupId>
       <artifactId>commons-collections</artifactId>
     </dependency>
@@ -97,6 +113,10 @@
       <artifactId>commons-exec</artifactId>
     </dependency>
     <dependency>
+      <groupId>org.apache.opennlp</groupId>
+      <artifactId>opennlp-tools</artifactId>
+    </dependency>
+    <dependency>
       <groupId>org.hdrhistogram</groupId>
       <artifactId>HdrHistogram</artifactId>
     </dependency>
diff --git a/bootstrap.sh b/bootstrap.sh
index 491f707d9f8..163834a78b1 100755
--- a/bootstrap.sh
+++ b/bootstrap.sh
@@ -26,7 +26,7 @@ else
 fi
 
 mvn_install() {
-    mvn --quiet --batch-mode --threads 1.5C --no-snapshot-updates clean install -Dmaven.test.skip=true -Dmaven.javadoc.skip=true "$@"
+    mvn --quiet --batch-mode --no-snapshot-updates clean install -Dmaven.test.skip=true -Dmaven.javadoc.skip=true "$@"
 }
 
 # Generate vtag map
@@ -48,17 +48,17 @@ $top/dist/getversion.pl -M $top > $top/dist/vtag.map
 echo "Downloading all dependencies. This may take a few minutes with an empty Maven cache."
 (
   cd container-dependency-versions
-  mvn_install
+  mvn_install --threads 1.5C
 )
 (
   cd parent
-  mvn_install
+  mvn_install --threads 1.5C
 )
-mvn_install -N
+mvn_install --threads 1.5C -N
 
 # and build plugins first:
 echo "Building Vespa Maven plugins."
-mvn_install -f maven-plugins/pom.xml
+mvn_install --threads 1 -f maven-plugins/pom.xml
 
 # now everything else should just work with normal maven dependency resolution:
 
diff --git a/bundle-plugin-test/src/test/java/com/yahoo/BundleIT.java b/bundle-plugin-test/src/test/java/com/yahoo/BundleIT.java
index cc6526d27c3..ccee4844b49 100644
--- a/bundle-plugin-test/src/test/java/com/yahoo/BundleIT.java
+++ b/bundle-plugin-test/src/test/java/com/yahoo/BundleIT.java
@@ -64,8 +64,8 @@ public class BundleIT {
     }
 
     @Test
-    @Ignore
-    public void require_that_manifest_version_matches_pom_version() {
+    @Ignore  // TODO Vespa 7: Should we fix this? Why not?
+    public void require_that_bundle_version_matches_pom_version() {
         assertThat(mainAttributes.getValue("Bundle-Version"), is("5.1.0"));
     }
 
diff --git a/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/AssembleContainerPluginMojo.java b/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/AssembleContainerPluginMojo.java
index b5fac517c9d..30ffc54225f 100644
--- a/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/AssembleContainerPluginMojo.java
+++ b/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/AssembleContainerPluginMojo.java
@@ -9,10 +9,12 @@ import org.apache.maven.execution.MavenSession;
 import org.apache.maven.model.Build;
 import org.apache.maven.plugin.AbstractMojo;
 import org.apache.maven.plugin.MojoExecutionException;
+import org.apache.maven.plugins.annotations.Component;
 import org.apache.maven.plugins.annotations.Mojo;
 import org.apache.maven.plugins.annotations.Parameter;
 import org.apache.maven.plugins.annotations.ResolutionScope;
 import org.apache.maven.project.MavenProject;
+import org.apache.maven.project.MavenProjectHelper;
 import org.codehaus.plexus.archiver.jar.JarArchiver;
 
 import java.io.File;
@@ -37,6 +39,9 @@ public class AssembleContainerPluginMojo extends AbstractMojo {
     @Parameter(defaultValue = "${project}")
     private MavenProject project = null;
 
+    @Component
+    private MavenProjectHelper projectHelper;
+
     @Parameter(defaultValue = "${session}", readonly = true, required = true)
     private MavenSession session = null;
 
@@ -46,6 +51,12 @@ public class AssembleContainerPluginMojo extends AbstractMojo {
     @Parameter(alias = "UseCommonAssemblyIds", defaultValue = "false")
     private boolean useCommonAssemblyIds = false;
 
+    @Parameter(alias = "AttachBundle", defaultValue = "false")
+    private boolean attachBundleToArtifact;
+
+    @Parameter(alias = "BundleClassifier", defaultValue = "bundle")
+    private String bundleClassifierName;
+
     @Override
     public void execute() throws MojoExecutionException {
         Map<Dependencies, String> jarSuffixes = new EnumMap<Dependencies, String>(Dependencies.class);
@@ -76,6 +87,13 @@ public class AssembleContainerPluginMojo extends AbstractMojo {
         addClassesDirectory(jarWithDependencies);
         addDependencies(jarWithDependencies);
         createArchive(jarFiles.get(Dependencies.WITH), jarWithDependencies);
+
+        if (attachBundleToArtifact) {
+            projectHelper.attachArtifact(project,
+                                         project.getArtifact().getType(),
+                                         bundleClassifierName,
+                                         jarFiles.get(Dependencies.WITH));
+        }
     }
 
     private File jarFileInBuildDirectory(String name, String suffix) {
diff --git a/config-model-api/pom.xml b/config-model-api/pom.xml
index 743164afb16..109093beda9 100644
--- a/config-model-api/pom.xml
+++ b/config-model-api/pom.xml
@@ -72,7 +72,6 @@
     <dependency>
       <groupId>com.google.guava</groupId>
       <artifactId>guava-testlib</artifactId>
-      <version>17.0</version>
       <scope>test</scope>
     </dependency>
   </dependencies>
diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelFactory.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelFactory.java
index 6d78e7d7756..b2d9244cc9d 100644
--- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelFactory.java
+++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelFactory.java
@@ -1,7 +1,6 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.config.model.api;
 
-import com.yahoo.config.model.api.ValidationParameters.IgnoreValidationErrors;
 import com.yahoo.config.provision.Version;
 
 /**
@@ -31,29 +30,8 @@ public interface ModelFactory {
      * of a {@link Model} and the {@link ModelContext} can be done in this method.
      *
      * @param modelContext an instance of {@link ModelContext}, containing dependencies for creating a {@link Model}
-     * @param ignoreValidationErrors true if validation errors should not trigger exceptions
-     * @return a {@link ModelCreateResult} instance.
-     * @deprecated use {@link #createAndValidateModel(ModelContext, ValidationParameters)}
-     */
-    @SuppressWarnings("DeprecatedIsStillUsed")
-    @Deprecated
-    // TODO: Remove when 6.259 is latest version in use
-    default ModelCreateResult createAndValidateModel(ModelContext modelContext, boolean ignoreValidationErrors) {
-        return createAndValidateModel(modelContext, new ValidationParameters(ignoreValidationErrors
-                                                                                     ? IgnoreValidationErrors.FALSE
-                                                                                     : IgnoreValidationErrors.TRUE));
-    }
-
-    /**
-     * Creates an instance of a {@link Model}. The resulting instance will be used to serve config. Any validation
-     * of a {@link Model} and the {@link ModelContext} can be done in this method.
-     *
-     * @param modelContext an instance of {@link ModelContext}, containing dependencies for creating a {@link Model}
      * @param validationParameters validation parameters
      * @return a {@link ModelCreateResult} instance.
      */
-    // TODO: Remove default implementation when 6.259 is latest version in use
-    default ModelCreateResult createAndValidateModel(ModelContext modelContext, ValidationParameters validationParameters) {
-        return createAndValidateModel(modelContext, validationParameters.ignoreValidationErrors());
-    }
+    ModelCreateResult createAndValidateModel(ModelContext modelContext, ValidationParameters validationParameters);
 }
diff --git a/config-model/pom.xml b/config-model/pom.xml
index bab2f37e667..e78c7f5c0fd 100644
--- a/config-model/pom.xml
+++ b/config-model/pom.xml
@@ -43,7 +43,6 @@
     <dependency>
       <groupId>com.google.guava</groupId>
       <artifactId>guava-testlib</artifactId>
-      <version>17.0</version>
       <scope>test</scope>
     </dependency>
     <dependency>
@@ -53,7 +52,6 @@
     <dependency>
       <groupId>com.google.guava</groupId>
       <artifactId>guava</artifactId>
-      <version>13.0.1</version>
       <scope>provided</scope>
     </dependency>
     <dependency>
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java
index a69c524e863..5855dedc415 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java
@@ -932,8 +932,8 @@ public class RankProfile implements Serializable, Cloneable {
     public static class Macro implements Serializable, Cloneable {
 
         private final String name;
-        private String textualExpression=null;
-        private RankingExpression expression=null;
+        private String textualExpression = null;
+        private RankingExpression expression = null;
         private List<String> formalParams = new ArrayList<>();
 
         /** True if this should be inlined into calling expressions. Useful for very cheap macros. */
@@ -998,6 +998,7 @@ public class RankProfile implements Serializable, Cloneable {
     }
 
     public static final class DiversitySettings {
+
         private String attribute = null;
         private int minGroups = 0;
         private double cutoffFactor = 10;
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java
index 07b79c4f604..b11d5962713 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java
@@ -212,8 +212,9 @@ public class RawRankProfile implements RankProfilesConfig.Producer {
         private Map<String, String> deriveMacroProperties(Map<String, ExpressionFunction> eMacros) {
             SerializationContext context = new SerializationContext(eMacros);
             for (Map.Entry<String, ExpressionFunction> e : eMacros.entrySet()) {
-                String script = e.getValue().getBody().getRoot().toString(new StringBuilder(), context, null, null).toString();
-                context.addFunctionSerialization(RankingExpression.propertyName(e.getKey()), script);
+                String expression = e.getValue().getBody().getRoot().toString(new StringBuilder(), context, null, null).toString();
+                context.addFunctionSerialization(RankingExpression.propertyName(e.getKey()), expression);
+
             }
             return context.serializedFunctions();
         }
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/Configserver.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/Configserver.java
index e8f1f59310b..2a32549b6bf 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/admin/Configserver.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/Configserver.java
@@ -1,16 +1,8 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.model.admin;
 
-import java.io.BufferedReader;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-
-import java.util.logging.Logger;
-
 import com.yahoo.config.model.api.ConfigServerSpec;
-import com.yahoo.log.LogLevel;
 import com.yahoo.config.model.producer.AbstractConfigProducer;
-import static com.yahoo.vespa.defaults.Defaults.getDefaults;
 import com.yahoo.vespa.model.AbstractService;
 
 /**
@@ -26,11 +18,13 @@ import com.yahoo.vespa.model.AbstractService;
  */
 public class Configserver extends AbstractService {
     private static final long serialVersionUID = 1L;
-    private static final int defaultPort = 19070;
-    private static final Logger log = Logger.getLogger(Configserver.class.getName());
+    public static final int defaultRpcPort = 19070;
 
-    public Configserver(AbstractConfigProducer parent, String name) {
+    private final int rpcPort;
+
+    public Configserver(AbstractConfigProducer parent, String name, int rpcPort) {
         super(parent, name);
+        this.rpcPort = rpcPort;
         portsMeta.on(0).tag("rpc").tag("config");
         portsMeta.on(1).tag("http").tag("config").tag("state");
         setProp("clustertype", "admin");
@@ -41,16 +35,7 @@ public class Configserver extends AbstractService {
      * Returns the desired base port for this service.
      */
     public int getWantedPort() {
-        try {
-            // TODO: Provide configserver port as argument when creating this service instead
-            Process process = new ProcessBuilder(getDefaults().underVespaHome("bin/vespa-print-default"), "configserver_rpc_port").start();
-            InputStream in = process.getInputStream();
-            BufferedReader reader = new BufferedReader(new InputStreamReader(in));
-            return Integer.parseInt(reader.readLine().trim());
-        } catch (Exception exception) {
-            log.log(LogLevel.DEBUG, "Error reading port from script, using " + defaultPort);
-            return defaultPort;
-        }
+        return rpcPort;
     }
 
     /**
@@ -84,7 +69,7 @@ public class Configserver extends AbstractService {
         return getRelativePort(1);
     }
 
-    public ConfigServerSpec getConfigServerSpec() {
+    ConfigServerSpec getConfigServerSpec() {
         return new Spec(getHostName(), getConfigServerRpcPort(), getConfigServerHttpPort(), ZooKeepersConfigProvider.zkPort);
     }
 
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java
index b7c9116dc60..a7a93e75293 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java
@@ -130,6 +130,10 @@ public class VespaMetricSet {
         metrics.add(new Metric("http.status.4xx.rate"));
         metrics.add(new Metric("http.status.5xx.rate"));
 
+        metrics.add(new Metric("jdisc.http.request.uri_length.average"));
+        metrics.add(new Metric("jdisc.http.request.uri_length.max"));
+        metrics.add(new Metric("jdisc.http.request.content_size.average"));
+        metrics.add(new Metric("jdisc.http.request.content_size.max"));
         return metrics;
     }
 
@@ -169,6 +173,7 @@ public class VespaMetricSet {
         metrics.add(new Metric("active_queries.average", "active_queries"));
         metrics.add(new Metric("feed.latency.average"));
         metrics.add(new Metric("queries.rate", "queries"));
+        metrics.add(new Metric("query_container_latency.average"));
         metrics.add(new Metric("query_latency.average", "mean_query_latency"));
         metrics.add(new Metric("query_latency.max", "max_query_latency"));
         metrics.add(new Metric("query_latency.95percentile", "95p_query_latency"));
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java
index 96e120cae92..297c4a6e9b9 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java
@@ -34,7 +34,7 @@ public abstract class DomAdminBuilderBase extends VespaDomBuilder.DomConfigProdu
     private static final String DEFAULT_CLUSTER_NAME = "vespa";
 
     private final ApplicationType applicationType;
-    private final List<ConfigServerSpec> configServerSpecs;
+    protected final List<ConfigServerSpec> configServerSpecs;
     private final FileRegistry fileRegistry;
     protected final boolean multitenant;
 
@@ -52,7 +52,7 @@ public abstract class DomAdminBuilderBase extends VespaDomBuilder.DomConfigProdu
             HostSystem hostSystem = parent.getHostSystem();
             HostResource host = new HostResource(Host.createConfigServerHost(hostSystem, spec.getHostName()));
             hostSystem.addBoundHost(host);
-            Configserver configserver = new Configserver(parent, spec.getHostName());
+            Configserver configserver = new Configserver(parent, spec.getHostName(), spec.getConfigServerPort());
             configserver.setHostResource(host);
             configserver.setBasePort(configserver.getWantedPort());
             configserver.initService();
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java
index da8c73c0958..75cd755a91d 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java
@@ -23,6 +23,7 @@ import org.w3c.dom.Element;
 
 import java.util.List;
 import java.util.ArrayList;
+import java.util.Objects;
 import java.util.logging.Level;
 
 /**
@@ -102,7 +103,7 @@ public class DomAdminV2Builder extends DomAdminBuilderBase {
         return cluster;
     }
 
-    // Extra stupid because configservers tag is voluntary
+    // Extra stupid because configservers tag is optional
     private List<Configserver> getConfigServers(AbstractConfigProducer parent, Element adminE) {
         SimpleConfigProducer configServers = new SimpleConfigProducer(parent, "configservers");
         List<Configserver> cfgs = new ArrayList<>();
@@ -114,7 +115,7 @@ public class DomAdminV2Builder extends DomAdminBuilderBase {
             } else {
                 parent.deployLogger().log(LogLevel.INFO, "Specifying configserver without parent element configservers in services.xml is deprecated");
             }
-            Configserver cfgs0 = new ConfigserverBuilder(0).build(configServers, configserverE);
+            Configserver cfgs0 = new ConfigserverBuilder(0, configServerSpecs).build(configServers, configserverE);
             cfgs0.setProp("index", 0);
             cfgs.add(cfgs0);
             return cfgs;
@@ -122,7 +123,7 @@ public class DomAdminV2Builder extends DomAdminBuilderBase {
         // configservers tag in use
         int i = 0;
         for (Element configserverE : XML.getChildren(configserversE, "configserver")) {
-            Configserver cfgsrv = new ConfigserverBuilder(i).build(configServers, configserverE);
+            Configserver cfgsrv = new ConfigserverBuilder(i, configServerSpecs).build(configServers, configserverE);
             cfgsrv.setProp("index", i);
             cfgs.add(cfgsrv);
             i++;
@@ -161,16 +162,21 @@ public class DomAdminV2Builder extends DomAdminBuilderBase {
     }
 
     private static class ConfigserverBuilder extends DomConfigProducerBuilder<Configserver> {
-        int i;
+        private final int i;
+        private final int rpcPort;
 
-        public ConfigserverBuilder(int i) {
+        public ConfigserverBuilder(int i, List<ConfigServerSpec> configServerSpec) {
             this.i = i;
+            Objects.requireNonNull(configServerSpec);
+            if (configServerSpec.size() > 0)
+                this.rpcPort = configServerSpec.get(0).getConfigServerPort();
+            else
+                this.rpcPort = Configserver.defaultRpcPort;
         }
 
         @Override
-        protected Configserver doBuild(AbstractConfigProducer parent,
-                                       Element spec) {
-            return new Configserver(parent, "configserver." + i);
+        protected Configserver doBuild(AbstractConfigProducer parent, Element spec) {
+            return new Configserver(parent, "configserver." + i, rpcPort);
         }
     }
 
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomComponentBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomComponentBuilder.java
index da643c687d9..81f19f00fbb 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomComponentBuilder.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomComponentBuilder.java
@@ -36,7 +36,7 @@ public class DomComponentBuilder extends VespaDomBuilder.DomConfigProducerBuilde
 
     private Component buildComponent(Element spec) {
         BundleInstantiationSpecification bundleSpec =
-                BundleInstantiationSpecificationBuilder.build(spec, false).nestInNamespace(namespace);
+                BundleInstantiationSpecificationBuilder.build(spec).nestInNamespace(namespace);
 
         return new Component<Component<?, ?>, ComponentModel>(new ComponentModel(bundleSpec));
     }
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomFilterBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomFilterBuilder.java
index 2c6364c4c34..74cc2b8421c 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomFilterBuilder.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomFilterBuilder.java
@@ -13,6 +13,6 @@ import org.w3c.dom.Element;
 public class DomFilterBuilder extends VespaDomBuilder.DomConfigProducerBuilder<Component> {
     @Override
     protected Component doBuild(AbstractConfigProducer ancestor, Element element) {
-        return new HttpFilter(BundleInstantiationSpecificationBuilder.build(element, false));
+        return new HttpFilter(BundleInstantiationSpecificationBuilder.build(element));
     }
 }
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomHandlerBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomHandlerBuilder.java
index 23f65fd9fb1..28a4b9dfea5 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomHandlerBuilder.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomHandlerBuilder.java
@@ -12,18 +12,8 @@ import org.w3c.dom.Element;
 
 /**
  * @author gjoranv
- * @since 5.1.6
  */
 public class DomHandlerBuilder extends VespaDomBuilder.DomConfigProducerBuilder<Handler> {
-    private final boolean legacyMode;
-
-    public DomHandlerBuilder(boolean legacyMode) {
-        this.legacyMode = legacyMode;
-    }
-
-    public DomHandlerBuilder() {
-        this(false);
-    }
 
     @Override
     protected Handler doBuild(AbstractConfigProducer ancestor, Element handlerElement) {
@@ -41,7 +31,7 @@ public class DomHandlerBuilder extends VespaDomBuilder.DomConfigProducerBuilder<
     }
 
     protected Handler<? super Component<?, ?>> getHandler(Element handlerElement) {
-        BundleInstantiationSpecification bundleSpec = BundleInstantiationSpecificationBuilder.build(handlerElement, legacyMode);
+        BundleInstantiationSpecification bundleSpec = BundleInstantiationSpecificationBuilder.build(handlerElement);
         return new Handler<>(
                 new ComponentModel(bundleSpec));
     }
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/ServletBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/ServletBuilder.java
index 3d779289340..d2d9e9d2ca6 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/ServletBuilder.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/ServletBuilder.java
@@ -27,7 +27,7 @@ public class ServletBuilder extends VespaDomBuilder.DomConfigProducerBuilder<Ser
     }
 
     private SimpleComponent createServletComponent(Element servletElement) {
-        ComponentModel componentModel = new ComponentModel(BundleInstantiationSpecificationBuilder.build(servletElement, false));
+        ComponentModel componentModel = new ComponentModel(BundleInstantiationSpecificationBuilder.build(servletElement));
         return new SimpleComponent(componentModel);
     }
 
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/ChainedComponentModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/ChainedComponentModelBuilder.java
index 700b042a745..68b37d938a0 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/ChainedComponentModelBuilder.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/ChainedComponentModelBuilder.java
@@ -15,7 +15,7 @@ public class ChainedComponentModelBuilder extends GenericChainedComponentModelBu
 
     public ChainedComponentModelBuilder(Element spec) {
         super(spec);
-        bundleInstantiationSpec = BundleInstantiationSpecificationBuilder.build(spec, false);
+        bundleInstantiationSpec = BundleInstantiationSpecificationBuilder.build(spec);
     }
 
     public ChainedComponentModel build() {
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/GUIHandler.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/GUIHandler.java
new file mode 100644
index 00000000000..025075be8fd
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/GUIHandler.java
@@ -0,0 +1,27 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.container.search;
+
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.container.bundle.BundleInstantiationSpecification;
+import com.yahoo.osgi.provider.model.ComponentModel;
+import com.yahoo.vespa.model.container.component.Handler;
+
+
+/**
+ * @author  Henrik Høiness
+ */
+
+public class GUIHandler extends Handler<AbstractConfigProducer<?>> {
+        public static final String BUNDLE = "container-search-gui";
+        public static final String CLASS = "com.yahoo.search.query.gui.GUIHandler";
+        public static final String BINDING = "*/querybuilder/*";
+
+        public GUIHandler() {
+            super(new ComponentModel(bundleSpec(CLASS, BUNDLE)));
+        }
+
+        public static BundleInstantiationSpecification bundleSpec(String className, String bundle) {
+            return BundleInstantiationSpecification.getFromStrings(className, className, bundle);
+        }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilder.java
index 69ee9048b28..2828fcf09d0 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilder.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilder.java
@@ -4,11 +4,10 @@ package com.yahoo.vespa.model.container.xml;
 import com.yahoo.config.model.builder.xml.XmlHelper;
 import com.yahoo.container.bundle.BundleInstantiationSpecification;
 import com.yahoo.component.ComponentSpecification;
-import com.yahoo.processing.handler.ProcessingHandler;
-import com.yahoo.search.handler.SearchHandler;
 import org.w3c.dom.Element;
 
-import java.util.*;
+import java.util.Arrays;
+import java.util.List;
 
 /**
  * This object builds a bundle instantiation spec from an XML element.
@@ -17,14 +16,13 @@ import java.util.*;
  */
 public class BundleInstantiationSpecificationBuilder {
 
-    public static BundleInstantiationSpecification build(Element spec, boolean legacyMode) {
+    public static BundleInstantiationSpecification build(Element spec) {
         ComponentSpecification id = XmlHelper.getIdRef(spec);
         ComponentSpecification classId = getComponentSpecification(spec, "class");
         ComponentSpecification bundle = getComponentSpecification(spec, "bundle");
 
         BundleInstantiationSpecification instSpec = new BundleInstantiationSpecification(id, classId, bundle);
-        if ( ! legacyMode) // TODO: Remove?
-            validate(instSpec);
+        validate(instSpec);
 
         return bundle == null ? setBundleForKnownClass(instSpec) : instSpec;
     }
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java
index a007c4765c0..d81026c54d1 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java
@@ -23,6 +23,7 @@ import com.yahoo.config.provision.NodeType;
 import com.yahoo.config.provision.Rotation;
 import com.yahoo.config.provision.Zone;
 import com.yahoo.container.jdisc.config.MetricDefaultsConfig;
+import com.yahoo.osgi.provider.model.ComponentModel;
 import com.yahoo.search.rendering.RendererRegistry;
 import com.yahoo.text.XML;
 import com.yahoo.vespa.defaults.Defaults;
@@ -47,6 +48,7 @@ import com.yahoo.vespa.model.container.IdentityProvider;
 import com.yahoo.vespa.model.container.SecretStore;
 import com.yahoo.vespa.model.container.component.Component;
 import com.yahoo.vespa.model.container.component.FileStatusHandlerComponent;
+import com.yahoo.vespa.model.container.component.Handler;
 import com.yahoo.vespa.model.container.component.chain.ProcessingHandler;
 import com.yahoo.vespa.model.container.docproc.ContainerDocproc;
 import com.yahoo.vespa.model.container.docproc.DocprocChains;
@@ -55,6 +57,7 @@ import com.yahoo.vespa.model.container.http.xml.HttpBuilder;
 import com.yahoo.vespa.model.container.jersey.xml.RestApiBuilder;
 import com.yahoo.vespa.model.container.processing.ProcessingChains;
 import com.yahoo.vespa.model.container.search.ContainerSearch;
+import com.yahoo.vespa.model.container.search.GUIHandler;
 import com.yahoo.vespa.model.container.search.PageTemplates;
 import com.yahoo.vespa.model.container.search.QueryProfiles;
 import com.yahoo.vespa.model.container.search.SemanticRules;
@@ -378,6 +381,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> {
             cluster.setSearch(buildSearch(cluster, searchElement, queryProfiles, semanticRules));
 
             addSearchHandler(cluster, searchElement);
+            addGUIHandler(cluster);
             validateAndAddConfiguredComponents(cluster, searchElement, "renderer", ContainerModelBuilder::validateRendererElement);
         }
     }
@@ -670,6 +674,13 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> {
         cluster.addComponent(searchHandler);
     }
 
+    private void addGUIHandler(ContainerCluster cluster) {
+        Handler<?> guiHandler = new GUIHandler();
+        guiHandler.addServerBindings("http://"+GUIHandler.BINDING, "https://"+GUIHandler.BINDING);
+        cluster.addComponent(guiHandler);
+    }
+
+
     private String[] serverBindings(Element searchElement, String... defaultBindings) {
         List<Element> bindings = XML.getChildren(searchElement, "binding");
         if (bindings.isEmpty())
diff --git a/config-model/src/main/python/ES_Vespa_parser.py b/config-model/src/main/python/ES_Vespa_parser.py
new file mode 100644
index 00000000000..477b0db4744
--- /dev/null
+++ b/config-model/src/main/python/ES_Vespa_parser.py
@@ -0,0 +1,222 @@
+# Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+import json
+import argparse
+import os, sys
+
+# Parsing Elastic Search documents to Vespa documents
+# Example of usage:  python ES_Vespa_parser.py my_index.json my_index_mapping.json
+__author__ = 'henrhoi'
+
+
+class ElasticSearchParser:
+    document_file = None
+    mapping_file = None
+    application_name = None
+    search_definitions = {}
+    path = ""
+    _all = True
+    all_mappings = {}
+    no_index = []
+    types = []
+
+    def __init__(self):
+        parser = argparse.ArgumentParser()
+        parser.add_argument("documents_path", help="location of file with documents to be parsed", type=str)
+        parser.add_argument("mappings_path", help="location of file with mappings", type=str)
+        parser.add_argument("--application_name", help="name of application", default="application_name", type=str)
+        args = parser.parse_args()
+
+        self.document_file = args.documents_path
+        self.mapping_file = args.mappings_path
+        self.application_name = args.application_name
+
+    def main(self):
+        self.path = os.getcwd() + "/application/"
+        try:
+            os.mkdir(self.path, 0o777)
+            print(" > Created folder '" + self.path + "'")
+        except OSError:
+            print(" > Folder '" + self.path + "' already existed")
+
+        try:
+            os.makedirs(self.path + "searchdefinitions/", 0o777)
+            print(" > Created folder '" + self.path + "searchdefinitions/" + "'")
+        except OSError:
+            print(" > Folder '" + self.path + "searchdefinitions/" + "' already existed")
+
+        self.parse()
+        self.createServices_xml()
+        self.createHosts_xml()
+
+    def getMapping(self, type):
+        unparsed_mapping_file = open(self.mapping_file, "r")
+        type_mapping = {}
+        for line in unparsed_mapping_file:
+            data = json.loads(line)
+            index = list(data.keys())[0]
+            mappings = data[index]["mappings"][type]["properties"]
+
+            # Checking if some fields could be no-index
+            try:
+                _all_enabled = data[index]["mappings"][type]["_all"]["enabled"]
+                if not _all_enabled:
+                    self._all = False
+                    print(" > All fields in the document type '" + type + "' is not searchable. Go inside " + self.path + type + ".sd to add which fields that should be searchable")
+            except KeyError:
+                print(" > All fields in the document type '" + type + "' is searchable")
+
+            self.walk(mappings, type_mapping, "properties")
+
+        unparsed_mapping_file.close()
+        if type not in self.search_definitions:
+            self.search_definitions[type] = True
+            self.types.append(type)
+            self.createSearchDefinition(type, type_mapping)
+
+        # Adding mapping to global map with mappings
+        self.all_mappings[type] = type_mapping
+        return type_mapping
+
+    def parse(self):
+        file_path = self.path + "documents" + ".json"
+        unparsed_document_file = open(self.document_file, "r")
+        vespa_docs = open(file_path, "w")
+
+        for line in unparsed_document_file:
+            data = json.loads(line)
+            type = data["_type"]
+
+            parsed_data = {
+                "put": "id:"+self.application_name+":" + type + "::" + data["_id"],
+                "fields": {}
+            }
+
+            # Checking for already existing mapping for a type, if not create a new
+            if type in self.all_mappings:
+                mapping = self.all_mappings[type]
+            else:
+                mapping = self.getMapping(type)
+
+            for key, item in mapping.items():
+                try:
+                    parsed_data["fields"][key] = data["_source"][key]
+                except KeyError:
+                    continue
+
+            json.dump(parsed_data, vespa_docs)
+            vespa_docs.write("\n")
+
+        vespa_docs.close()
+        unparsed_document_file.close()
+        print(" > Parsed all documents '" + ", ".join(self.types) + "'" + "' at '" + file_path + "'")
+
+    def createSearchDefinition(self, type, type_mapping):
+        file_path = self.path + "searchdefinitions/" + type + ".sd"
+        new_sd = open(file_path, "w")
+        new_sd.write("search " + type + " {\n")
+        new_sd.write("    document " + type + " {\n")
+
+        for key, item in type_mapping.items():
+            new_sd.write("        field " + key + " type " + self.get_type(item) + " {\n")
+            new_sd.write("            indexing: " + self.get_indexing(key, self.get_type(item)) + "\n")
+            new_sd.write("        }\n")
+
+        new_sd.write("    }\n")
+        new_sd.write("}\n")
+        new_sd.close()
+        print(" > Created search definition for '" + type + "' at '" + file_path + "'")
+
+    def createServices_xml(self):
+        file_path = self.path + "services.xml"
+        new_services = open(file_path, "w")
+        template = ("<?xml version='1.0' encoding='UTF-8'?>"
+                    "<services version='1.0'>\n\n"
+                    "  <container id='default' version='1.0'>\n"
+                    "    <search/>\n"
+                    "    <document-api/>\n"
+                    "    <nodes>\n"
+                    "      <node hostalias='node1'/>\n"
+                    "    </nodes>\n"
+                    "  </container>\n\n"
+                    "  <content id='content' version='1.0'>\n"
+                    "    <redundancy>1</redundancy>\n"
+                    "    <search>\n"
+                    "      <visibility-delay>1.0</visibility-delay>\n"
+                    "    </search>\n"
+                    "    <documents>\n")
+
+        for i in range(0, len(self.types)):
+            template += "      <document mode='index' type='" + self.types[i] + "'/>\n"
+
+        template += ("    </documents>\n"
+                     "    <nodes>\n"
+                     "      <node hostalias='node1' distribution-key=\"0\"/>\n"
+                     "    </nodes>\n"
+                     "    <engine>\n"
+                     "      <proton>\n"
+                     "        <searchable-copies>1</searchable-copies>\n"
+                     "      </proton>\n"
+                     "    </engine>\n"
+                     "  </content>\n\n"
+                     "</services>")
+
+        new_services.write(template)
+        new_services.close()
+        print(" > Created services.xml at '" + file_path + "'")
+
+    def createHosts_xml(self):
+        file_path = self.path + "hosts.xml"
+        new_hosts = open(file_path, "w")
+        template = ("<?xml version=\"1.0\" encoding=\"utf-8\" ?>\n"
+                    "<hosts>\n"
+                    "  <host name=\"localhost\">\n"
+                    "    <alias>node1</alias>\n"
+                    "  </host>\n"
+                    "</hosts>")
+
+        new_hosts.write(template)
+        new_hosts.close()
+        print(" > Created hosts.xml at '" + file_path + "'")
+
+    def get_type(self, type):
+        return {
+            "text": "string",
+            "keyword": "string",
+            "date": "string",
+            "long": "long",
+            "double": "double",
+            "boolean": "string",
+            "ip": "text",
+            "byte": "byte",
+            "float": "float",
+
+        }[type]
+
+    def get_indexing(self, key, key_type):
+        if not self._all:
+            return "summary"
+
+        if key not in self.no_index:
+            if key_type == "string":
+                return "summary | index"
+            else:
+                return "summary | attribute"
+
+        return "summary"
+
+    def walk(self, node, mapping, parent):
+        for key, item in node.items():
+            if isinstance(item, dict):
+                self.walk(item, mapping, key)
+            elif key == "type":
+                mapping[parent] = item
+            elif key == "include_in_all":
+                if not item:  # Field should not be searchable
+                    self.no_index.append(parent)
+            elif key == "index" and parent != "properties":
+                if item == "no":  # Field should not be searchable
+                    self.no_index.append(parent)
+
+
+if __name__ == '__main__':
+    ElasticSearchParser().main()
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilderTest.java
index c58f3308ced..a651bbb7772 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilderTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilderTest.java
@@ -53,7 +53,7 @@ public class BundleInstantiationSpecificationBuilderTest {
         InputStream xmlStream = IOUtils.toInputStream(xml);
         Element component = XmlHelper.getDocumentBuilder().parse(xmlStream).getDocumentElement();
 
-        BundleInstantiationSpecification spec = BundleInstantiationSpecificationBuilder.build(component, false);
+        BundleInstantiationSpecification spec = BundleInstantiationSpecificationBuilder.build(component);
         assertThat(spec.bundle, is(ComponentSpecification.fromString(expectedBundle)));
     }
 }
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/SearchBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/SearchBuilderTest.java
index cc8733fc9e2..30f1df6a394 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/SearchBuilderTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/SearchBuilderTest.java
@@ -8,6 +8,8 @@ import com.yahoo.searchdefinition.parser.ParseException;
 import com.yahoo.vespa.model.VespaModel;
 import com.yahoo.vespa.model.container.Container;
 import com.yahoo.vespa.model.container.ContainerCluster;
+import com.yahoo.vespa.model.container.component.Handler;
+import com.yahoo.vespa.model.container.search.GUIHandler;
 import com.yahoo.vespa.model.test.utils.ApplicationPackageUtils;
 import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithMockPkg;
 import org.junit.Test;
@@ -32,6 +34,31 @@ public class SearchBuilderTest extends ContainerModelBuilderTestBase {
         return root.getConfig(ChainsConfig.class, "default/component/com.yahoo.search.handler.SearchHandler");
     }
 
+    @Test
+    public void gui_search_handler_is_always_included_when_search_is_specified() throws Exception{
+        Element clusterElem = DomBuilderTest.parse(
+                "<jdisc id='default' version='1.0'>",
+                "  <search />",
+                nodesXml,
+                "</jdisc>");
+
+        createModel(root, clusterElem);
+
+        String discBindingsConfig = root.getConfig(JdiscBindingsConfig.class, "default").toString();
+        assertThat(discBindingsConfig, containsString(GUIHandler.BINDING));
+
+        ContainerCluster cluster = (ContainerCluster)root.getChildren().get("default");
+
+        GUIHandler guiHandler = null;
+        for (Handler<?> handler : cluster.getHandlers()) {
+            if (handler instanceof GUIHandler) {
+                guiHandler = (GUIHandler) handler;
+            }
+        }
+        if (guiHandler == null) fail();
+    }
+
+
 
     @Test
     public void search_handler_bindings_can_be_overridden() throws Exception {
diff --git a/config-proxy/src/main/sh/vespa-config-ctl.sh b/config-proxy/src/main/sh/vespa-config-ctl.sh
index 669882c11fd..a670e69cdbf 100755
--- a/config-proxy/src/main/sh/vespa-config-ctl.sh
+++ b/config-proxy/src/main/sh/vespa-config-ctl.sh
@@ -115,7 +115,7 @@ case $1 in
         echo "Starting config proxy using $configsources as config source(s)"
         vespa-runserver -r 10 -s configproxy -p $P_CONFIG_PROXY -- \
             java ${jvmopts} \
-                 -XX:OnOutOfMemoryError="kill -9 %p" $(getJavaOptionsIPV46) \
+                 -XX:+ExitOnOutOfMemoryError $(getJavaOptionsIPV46) \
                  -Dproxyconfigsources="${configsources}" ${userargs} \
                  -cp $cp com.yahoo.vespa.config.proxy.ProxyServer 19090
 
diff --git a/config/pom.xml b/config/pom.xml
index 612541899b2..e1e04e17033 100755
--- a/config/pom.xml
+++ b/config/pom.xml
@@ -65,13 +65,11 @@
     <dependency>
       <groupId>com.google.guava</groupId>
       <artifactId>guava</artifactId>
-      <version>13.0.1</version>
       <scope>provided</scope>
     </dependency>
     <dependency>
       <groupId>com.google.guava</groupId>
       <artifactId>guava-testlib</artifactId>
-      <version>17.0</version>
       <scope>test</scope>
     </dependency>
     <dependency>
diff --git a/config/src/apps/vespa-get-config/getconfig.cpp b/config/src/apps/vespa-get-config/getconfig.cpp
index 4c42779ce91..9e58c9e3777 100644
--- a/config/src/apps/vespa-get-config/getconfig.cpp
+++ b/config/src/apps/vespa-get-config/getconfig.cpp
@@ -49,7 +49,8 @@ GetConfig::usage()
     fprintf(stderr, "usage: %s -n name -i configId\n", _argv[0]);
     fprintf(stderr, "-n name           (config name, including namespace, on the form <namespace>.<name>)\n");
     fprintf(stderr, "-i configId       (config id, optional)\n");
-    fprintf(stderr, "-j                (output config as json)\n");
+    fprintf(stderr, "-j                (output config as json, optional)\n");
+    fprintf(stderr, "-l                (output config in legacy cfg format, optional)\n");
     fprintf(stderr, "-a schema         (config def schema file, optional)\n");
     fprintf(stderr, "-v defVersion     (config definition version, optional, deprecated)\n");
     fprintf(stderr, "-m defMd5         (definition md5sum, optional)\n");
@@ -135,6 +136,9 @@ GetConfig::Main()
         case 'j':
             printAsJson = true;
             break;
+        case 'l':
+            printAsJson = false;
+            break;
         case 'm':
             defMD5 = optArg;
             break;
diff --git a/config/src/main/java/com/yahoo/config/subscription/ConfigURI.java b/config/src/main/java/com/yahoo/config/subscription/ConfigURI.java
index 537aff096a9..850dbcc1bf4 100644
--- a/config/src/main/java/com/yahoo/config/subscription/ConfigURI.java
+++ b/config/src/main/java/com/yahoo/config/subscription/ConfigURI.java
@@ -10,9 +10,9 @@ import com.yahoo.config.subscription.impl.JRTConfigRequester;
  * object to simplify parameter passing.
  *
  * @author lulf
- * @since 5.1
  */
 public class ConfigURI {
+
     private String configId;
     private ConfigSource source;
 
@@ -54,4 +54,5 @@ public class ConfigURI {
     public static ConfigURI createFromIdAndSource(String configId, ConfigSource source) {
         return new ConfigURI(configId, source);
     }
+
 }
diff --git a/config/src/main/java/com/yahoo/config/subscription/FileSource.java b/config/src/main/java/com/yahoo/config/subscription/FileSource.java
index d69e2429dbb..27129853eae 100644
--- a/config/src/main/java/com/yahoo/config/subscription/FileSource.java
+++ b/config/src/main/java/com/yahoo/config/subscription/FileSource.java
@@ -5,15 +5,15 @@ import java.io.File;
 
 /**
  * Source specifying config from one local file
- * @author vegardh
- * @since 5.1
  *
+ * @author vegardh
  */
 public class FileSource implements ConfigSource {
+
     private final File file;
 
     public FileSource(File file) {
-        if (!file.isFile()) throw new IllegalArgumentException("Not an ordinary file: "+file);
+        if ( ! file.isFile()) throw new IllegalArgumentException("Not an ordinary file: "+file);
         this.file = file;
     }
 
diff --git a/config/src/main/java/com/yahoo/vespa/config/ConfigFileFormat.java b/config/src/main/java/com/yahoo/vespa/config/ConfigFileFormat.java
index da51df143d2..751f6578575 100644
--- a/config/src/main/java/com/yahoo/vespa/config/ConfigFileFormat.java
+++ b/config/src/main/java/com/yahoo/vespa/config/ConfigFileFormat.java
@@ -13,9 +13,9 @@ import java.util.Stack;
 
 /**
  * @author lulf
- * @since 5.1
  */
 public class ConfigFileFormat implements SlimeFormat, ObjectTraverser {
+
     private final InnerCNode root;
     private DataOutputStream out = null;
     private Stack<Node> nodeStack;
diff --git a/config/src/vespa/config/common/configparser.cpp b/config/src/vespa/config/common/configparser.cpp
index 6784f0793e9..4d9eac41af6 100644
--- a/config/src/vespa/config/common/configparser.cpp
+++ b/config/src/vespa/config/common/configparser.cpp
@@ -14,7 +14,7 @@ void ConfigParser::throwNoDefaultValue(const vespalib::stringref & key) {
 }
 
 vespalib::string
-ConfigParser::deQuote(const vespalib::stringref & source)
+ConfigParser::deQuote(const vespalib::string & source)
 {
     const char *src = source.c_str();
     const char *s = src;
diff --git a/config/src/vespa/config/common/configparser.h b/config/src/vespa/config/common/configparser.h
index 2f6cdd9cf39..77b5bf7ddaa 100644
--- a/config/src/vespa/config/common/configparser.h
+++ b/config/src/vespa/config/common/configparser.h
@@ -23,7 +23,7 @@ private:
     static std::vector<vsvector> splitArray( const vsvector & config);
     static std::map<vespalib::string, vsvector> splitMap( const vsvector & config);
 
-    static vespalib::string deQuote(const vespalib::stringref & source);
+    static vespalib::string deQuote(const vespalib::string & source);
     static void throwNoDefaultValue(const vespalib::stringref & key);
 
     template<typename T>
diff --git a/configd/src/apps/sentinel/config-handler.cpp b/configd/src/apps/sentinel/config-handler.cpp
index 67216954f51..325de3c232c 100644
--- a/configd/src/apps/sentinel/config-handler.cpp
+++ b/configd/src/apps/sentinel/config-handler.cpp
@@ -133,6 +133,9 @@ ConfigHandler::terminate()
     for (int retry = 0; retry < 10 && doWork(); ++retry) {
         LOG(warning, "some services refuse to terminate cleanly, sending KILL");
         terminateServices(false, true);
+        tv.tv_sec = 0;
+        tv.tv_usec = 200000;
+        select(0, nullptr, nullptr, nullptr, &tv);
     }
     return !doWork();
 }
diff --git a/configd/src/apps/sentinel/service.cpp b/configd/src/apps/sentinel/service.cpp
index 5633c356bc7..e38328975dc 100644
--- a/configd/src/apps/sentinel/service.cpp
+++ b/configd/src/apps/sentinel/service.cpp
@@ -106,27 +106,36 @@ Service::terminate(bool catchable, bool dumpState)
         runPreShutdownCommand();
         LOG(debug, "%s: terminate(%s)", name().c_str(), catchable ? "cleanly" : "NOW");
         resetRestartPenalty();
+        kill(_pid, SIGCONT); // if it was stopped for some reason
         if (catchable) {
-            setState(TERMINATING);
-            int ret = kill(_pid, SIGTERM);
-            LOG(debug, "%s: kill -SIGTERM %d: %s", name().c_str(), (int)_pid,
-                ret == 0 ? "OK" : strerror(errno));
-            return ret;
+            if (_state != TERMINATING) {
+                int ret = kill(_pid, SIGTERM);
+                if (ret == 0) {
+                    setState(TERMINATING);
+                } else {
+                    LOG(warning, "%s: kill -SIGTERM %d failed: %s",
+                        name().c_str(), (int)_pid, strerror(errno));
+                }
+                return ret;
+            }
+            // already sent SIGTERM
+            return 0;
         } else {
             if (dumpState && _state != KILLING) {
-                vespalib::string pstackCmd = make_string("pstack %d > %s/%s.pstack.%d",
+                vespalib::string pstackCmd = make_string("pstack %d > %s/%s.pstack.%d 2>&1",
                                                          _pid, getVespaTempDir().c_str(), name().c_str(), _pid);
-                LOG(info, "%s:%d failed to stop. Stack dumped at %s", name().c_str(), _pid, pstackCmd.c_str());
+                LOG(info, "%s:%d failed to stop. Stack dumping with %s", name().c_str(), _pid, pstackCmd.c_str());
                 int pstackRet = system(pstackCmd.c_str());
                 if (pstackRet != 0) {
                     LOG(warning, "'%s' failed with return value %d", pstackCmd.c_str(), pstackRet);
                 }
             }
             setState(KILLING);
-            kill(_pid, SIGCONT); // if it was stopped for some reason
             int ret = kill(_pid, SIGKILL);
-            LOG(debug, "%s: kill -SIGKILL %d: %s", name().c_str(), (int)_pid,
-                ret == 0 ? "OK" : strerror(errno));
+            if (ret != 0) {
+                LOG(warning, "%s: kill -SIGKILL %d failed: %s",
+                    name().c_str(), (int)_pid, strerror(errno));
+            }
             return ret;
         }
     }
diff --git a/configdefinitions/src/vespa/configserver.def b/configdefinitions/src/vespa/configserver.def
index daaf8906d24..70df8ce5164 100644
--- a/configdefinitions/src/vespa/configserver.def
+++ b/configdefinitions/src/vespa/configserver.def
@@ -55,7 +55,7 @@ ztsUrl string default=""
 nodeAdminInContainer bool default=true
 
 # Maintainers
-maintainerIntervalMinutes int default=30
+maintainerIntervalMinutes int default=60
 # TODO: Default set to a high value (1 year) => maintainer will not run, change when maintainer verified out in prod
 tenantsMaintainerIntervalMinutes int default=525600
 
diff --git a/configserver/pom.xml b/configserver/pom.xml
index c4b25b463af..2aeecb1ba2f 100644
--- a/configserver/pom.xml
+++ b/configserver/pom.xml
@@ -101,6 +101,13 @@
       <scope>provided</scope>
     </dependency>
     <dependency>
+      <!-- To get all necessary test deps. -->
+      <groupId>com.yahoo.vespa</groupId>
+      <artifactId>container-test</artifactId>
+      <version>${project.version}</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
       <groupId>com.yahoo.vespa</groupId>
       <artifactId>vespa_jersey2</artifactId>
       <version>${project.version}</version>
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
index 4380cf5f047..8a02383ad0b 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
@@ -6,7 +6,6 @@ import com.google.inject.Inject;
 import com.yahoo.cloud.config.ConfigserverConfig;
 import com.yahoo.component.Version;
 import com.yahoo.component.Vtag;
-import com.yahoo.concurrent.DaemonThreadFactory;
 import com.yahoo.config.FileReference;
 import com.yahoo.config.application.api.ApplicationFile;
 import com.yahoo.config.application.api.ApplicationMetaData;
@@ -62,17 +61,10 @@ import java.time.Clock;
 import java.time.Duration;
 import java.time.Instant;
 import java.util.Arrays;
-import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
-import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import java.util.concurrent.TimeUnit;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 import java.util.stream.Collectors;
@@ -97,7 +89,6 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
     private final Clock clock;
     private final DeployLogger logger = new SilentDeployLogger();
     private final ConfigserverConfig configserverConfig;
-    private final Environment environment;
     private final FileDistributionStatus fileDistributionStatus;
 
     @Inject
@@ -139,7 +130,6 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
         this.httpProxy = httpProxy;
         this.clock = clock;
         this.configserverConfig = configserverConfig;
-        this.environment = Environment.from(configserverConfig.environment());
         this.fileDistributionStatus = fileDistributionStatus;
     }
 
@@ -250,7 +240,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
 
         // Keep manually deployed tenant applications on the latest version, don't change version otherwise
         // TODO: Remove this and always use version from session once controller starts upgrading manual deployments
-        Version version = decideVersion(application, environment, newSession.getVespaVersion(), bootstrap);
+        Version version = decideVersion(application, zone().environment(), newSession.getVespaVersion(), bootstrap);
                 
         return Optional.of(Deployment.unprepared(newSession, this, hostProvisioner, tenant, timeout, clock,
                                                  false /* don't validate as this is already deployed */, version,
@@ -292,15 +282,14 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
      * @throws RuntimeException if the delete transaction fails. This method is exception safe.
      */
     public boolean delete(ApplicationId applicationId) {
-        Zone zone = new Zone(Environment.from(configserverConfig.environment()), RegionName.from(configserverConfig.region()));
         List<String> hostedZonesToUseDeleteApplication =
                 Arrays.asList("dev.us-east-1", "dev.corp-us-east-1", "test.us-east-1",
                               "prod.corp-us-east-1", "prod.aws-us-east-1a", "prod.aws-us-west-1b");
-        boolean useDeleteApplicationLegacyInThisZone = !hostedZonesToUseDeleteApplication.contains(zone.toString());
+        boolean useDeleteApplicationLegacyInThisZone = !hostedZonesToUseDeleteApplication.contains(zone().toString());
 
         // TODO: Use deleteApplication() in all zones
         if (configserverConfig.deleteApplicationLegacy() ||
-                (configserverConfig.hostedVespa() && SystemName.from(configserverConfig.system()) == SystemName.main &&
+                (configserverConfig.hostedVespa() && zone().system() == SystemName.main &&
                         useDeleteApplicationLegacyInThisZone)) {
             return deleteApplicationLegacy(applicationId);
         } else {
@@ -463,7 +452,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
         }
     }
 
-    private Set<ApplicationId> listApplications() {
+    Set<ApplicationId> listApplications() {
         return tenantRepository.getAllTenants().stream()
                 .flatMap(tenant -> tenant.getApplicationRepo().listApplications().stream())
                 .collect(Collectors.toSet());
@@ -561,10 +550,20 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
         return session.getSessionId();
     }
 
-    public void deleteOldSessions() {
+    public void deleteExpiredLocalSessions() {
         listApplications().forEach(app -> tenantRepository.getTenant(app.tenant()).getLocalSessionRepo().purgeOldSessions());
     }
 
+    public int deleteExpiredRemoteSessions(Duration expiryTime) {
+        return listApplications()
+                .stream()
+                .map(app -> tenantRepository.getTenant(app.tenant()).getRemoteSessionRepo()
+                        // TODO: Delete in all zones
+                        .deleteExpiredSessions(expiryTime, zone().system() == SystemName.cd))
+                .mapToInt(i -> i)
+                .sum();
+    }
+
     // ---------------- Tenant operations ----------------------------------------------------------------
 
     public Set<TenantName> deleteUnusedTenants(Duration ttlForUnusedTenant, Instant now) {
@@ -608,7 +607,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
         return getLocalSession(tenant, sessionId).getMetaData();
     }
 
-    ConfigserverConfig configserverConfig() {
+    public ConfigserverConfig configserverConfig() {
         return configserverConfig;
     }
 
@@ -677,53 +676,6 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
         }
     }
 
-    boolean redeployAllApplications(Duration maxDuration, Duration sleepBetweenRetries) throws InterruptedException {
-        Instant end = Instant.now().plus(maxDuration);
-        Set<ApplicationId> applicationsNotRedeployed = listApplications();
-        do {
-            applicationsNotRedeployed = redeployApplications(applicationsNotRedeployed);
-            if ( ! applicationsNotRedeployed.isEmpty()) {
-                Thread.sleep(sleepBetweenRetries.toMillis());
-            }
-        } while ( ! applicationsNotRedeployed.isEmpty() && Instant.now().isBefore(end));
-
-        if ( ! applicationsNotRedeployed.isEmpty()) {
-            log.log(LogLevel.ERROR, "Redeploying applications not finished after " + maxDuration +
-                    ", exiting, applications that failed redeployment: " + applicationsNotRedeployed);
-            return false;
-        }
-        return true;
-    }
-
-    // Returns the set of applications that failed to redeploy
-    private Set<ApplicationId> redeployApplications(Set<ApplicationId> applicationIds) throws InterruptedException {
-        ExecutorService executor = Executors.newFixedThreadPool(configserverConfig.numParallelTenantLoaders(),
-                                                                new DaemonThreadFactory("redeploy apps"));
-        // Keep track of deployment per application
-        Map<ApplicationId, Future<?>> futures = new HashMap<>();
-        Set<ApplicationId> failedDeployments = new HashSet<>();
-
-        for (ApplicationId appId : applicationIds) {
-            Optional<com.yahoo.config.provision.Deployment> deploymentOptional = deployFromLocalActive(appId, true /* bootstrap */);
-            if ( ! deploymentOptional.isPresent()) continue;
-
-            futures.put(appId, executor.submit(deploymentOptional.get()::activate));
-        }
-
-        for (Map.Entry<ApplicationId, Future<?>> f : futures.entrySet()) {
-            try {
-                f.getValue().get();
-            } catch (ExecutionException e) {
-                ApplicationId app = f.getKey();
-                log.log(LogLevel.WARNING, "Redeploying " + app + " failed, will retry", e);
-                failedDeployments.add(app);
-            }
-        }
-        executor.shutdown();
-        executor.awaitTermination(365, TimeUnit.DAYS); // Timeout should never happen
-        return failedDeployments;
-    }
-
     private LocalSession getExistingSession(Tenant tenant, ApplicationId applicationId) {
         TenantApplications applicationRepo = tenant.getApplicationRepo();
         return getLocalSession(tenant, applicationRepo.getSessionIdForApplication(applicationId));
@@ -768,4 +720,10 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
         return deployLog;
     }
 
+    public Zone zone() {
+        return new Zone(SystemName.from(configserverConfig.system()),
+                        Environment.from(configserverConfig.environment()),
+                        RegionName.from(configserverConfig.region()));
+    }
+
 }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java
index 50052ac856c..46be61964ce 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java
@@ -2,8 +2,11 @@
 package com.yahoo.vespa.config.server;
 
 import com.google.inject.Inject;
+import com.yahoo.cloud.config.ConfigserverConfig;
 import com.yahoo.component.AbstractComponent;
 import com.yahoo.concurrent.DaemonThreadFactory;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.Deployment;
 import com.yahoo.container.handler.VipStatus;
 import com.yahoo.container.jdisc.state.StateMonitor;
 import com.yahoo.log.LogLevel;
@@ -12,8 +15,16 @@ import com.yahoo.vespa.config.server.version.VersionState;
 
 import java.time.Duration;
 import java.time.Instant;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
 
 /**
  * Main component that bootstraps and starts config server threads.
@@ -39,6 +50,7 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable
     private final VersionState versionState;
     private final StateMonitor stateMonitor;
     private final VipStatus vipStatus;
+    private final ConfigserverConfig configserverConfig;
     private final Duration maxDurationOfRedeployment;
     private final Duration sleepTimeWhenRedeployingFails;
     private final RedeployingApplicationsFails exitIfRedeployingApplicationsFails;
@@ -63,8 +75,9 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable
         this.stateMonitor = stateMonitor;
         this.serverThread = new Thread(this, "configserver main");
         this.vipStatus = vipStatus;
-        this.maxDurationOfRedeployment = Duration.ofSeconds(applicationRepository.configserverConfig().maxDurationOfBootstrap());
-        this.sleepTimeWhenRedeployingFails = Duration.ofSeconds(applicationRepository.configserverConfig().sleepTimeWhenRedeployingFails());
+        this.configserverConfig = applicationRepository.configserverConfig();
+        this.maxDurationOfRedeployment = Duration.ofSeconds(configserverConfig.maxDurationOfBootstrap());
+        this.sleepTimeWhenRedeployingFails = Duration.ofSeconds(configserverConfig.sleepTimeWhenRedeployingFails());
         this.exitIfRedeployingApplicationsFails = exitIfRedeployingApplicationsFails;
         rpcServerExecutor = Executors.newSingleThreadExecutor(new DaemonThreadFactory("config server RPC server"));
         initializing(); // Initially take server out of rotation
@@ -91,7 +104,7 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable
             log.log(LogLevel.INFO, "Configserver upgrading from " + versionState.storedVersion() + " to "
                     + versionState.currentVersion() + ". Redeploying all applications");
             try {
-                if ( ! applicationRepository.redeployAllApplications(maxDurationOfRedeployment, sleepTimeWhenRedeployingFails)) {
+                if ( ! redeployAllApplications()) {
                     redeployingApplicationsFailed();
                     return; // Status will not be set to 'up' since we return here
                 }
@@ -163,5 +176,52 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable
         if (exitIfRedeployingApplicationsFails == RedeployingApplicationsFails.EXIT_JVM) System.exit(1);
     }
 
+    private boolean redeployAllApplications() throws InterruptedException {
+        Instant end = Instant.now().plus(maxDurationOfRedeployment);
+        Set<ApplicationId> applicationsNotRedeployed = applicationRepository.listApplications();
+        do {
+            applicationsNotRedeployed = redeployApplications(applicationsNotRedeployed);
+            if ( ! applicationsNotRedeployed.isEmpty()) {
+                Thread.sleep(sleepTimeWhenRedeployingFails.toMillis());
+            }
+        } while ( ! applicationsNotRedeployed.isEmpty() && Instant.now().isBefore(end));
+
+        if ( ! applicationsNotRedeployed.isEmpty()) {
+            log.log(LogLevel.ERROR, "Redeploying applications not finished after " + maxDurationOfRedeployment +
+                    ", exiting, applications that failed redeployment: " + applicationsNotRedeployed);
+            return false;
+        }
+        return true;
+    }
+
+    // Returns the set of applications that failed to redeploy
+    private Set<ApplicationId> redeployApplications(Set<ApplicationId> applicationIds) throws InterruptedException {
+        ExecutorService executor = Executors.newFixedThreadPool(configserverConfig.numParallelTenantLoaders(),
+                                                                new DaemonThreadFactory("redeploy apps"));
+        // Keep track of deployment per application
+        Map<ApplicationId, Future<?>> futures = new HashMap<>();
+        Set<ApplicationId> failedDeployments = new HashSet<>();
+
+        for (ApplicationId appId : applicationIds) {
+            Optional<Deployment> deploymentOptional = applicationRepository.deployFromLocalActive(appId, true /* bootstrap */);
+            if ( ! deploymentOptional.isPresent()) continue;
+
+            futures.put(appId, executor.submit(deploymentOptional.get()::activate));
+        }
+
+        for (Map.Entry<ApplicationId, Future<?>> f : futures.entrySet()) {
+            try {
+                f.getValue().get();
+            } catch (ExecutionException e) {
+                ApplicationId app = f.getKey();
+                log.log(LogLevel.WARNING, "Redeploying " + app + " failed, will retry", e);
+                failedDeployments.add(app);
+            }
+        }
+        executor.shutdown();
+        executor.awaitTermination(365, TimeUnit.DAYS); // Timeout should never happen
+        return failedDeployments;
+    }
+
 }
 
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandler.java
index 8ad7a85619c..7c962e5b6be 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandler.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandler.java
@@ -13,7 +13,7 @@ import com.yahoo.vespa.config.server.tenant.TenantRepository;
 import java.util.Optional;
 
 /**
- * HTTP handler for a v2 getConfig operation
+ * HTTP handler for a v1 getConfig operation
  *
  * @author Ulf Lilleengen
  */
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/SessionsMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/SessionsMaintainer.java
index 53f2553161f..a0cbf4e4845 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/SessionsMaintainer.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/SessionsMaintainer.java
@@ -14,13 +14,22 @@ import java.time.Duration;
  * @author hmusum
  */
 public class SessionsMaintainer extends Maintainer {
+    private final boolean hostedVespa;
 
     SessionsMaintainer(ApplicationRepository applicationRepository, Curator curator, Duration interval) {
         super(applicationRepository, curator, interval);
+        this.hostedVespa = applicationRepository.configserverConfig().hostedVespa();
     }
 
     @Override
     protected void maintain() {
-        applicationRepository.deleteOldSessions();
+        applicationRepository.deleteExpiredLocalSessions();
+
+        // Expired remote sessions are not expected to exist, they should have been deleted when
+        // a deployment happened or when the application was deleted. We still see them from time to time,
+        // probably due to some race or another bug
+        Duration expiryTime = Duration.ofDays(30);
+        if (hostedVespa)
+            applicationRepository.deleteExpiredRemoteSessions(expiryTime);
     }
 }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ZooKeeperDataMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ZooKeeperDataMaintainer.java
index 625665312fd..d01181638c6 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ZooKeeperDataMaintainer.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ZooKeeperDataMaintainer.java
@@ -6,22 +6,27 @@ import com.yahoo.vespa.config.server.ApplicationRepository;
 import com.yahoo.vespa.curator.Curator;
 
 import java.time.Duration;
+import java.util.Arrays;
+import java.util.List;
 
 /**
- * Removes unused zookeeper data (for now only data used by old file distribution code is removed)
+ * Removes unused zookeeper data
  *
  * @author hmusum
  */
 public class ZooKeeperDataMaintainer extends Maintainer {
 
+    private static final List<String> pathsToDelete = Arrays.asList(
+            "/vespa/filedistribution", // Path to file distribution data used before Vespa 6.213
+            "/vespa/config" //  Path to config data used before Vespa 6
+    );
+
     ZooKeeperDataMaintainer(ApplicationRepository applicationRepository, Curator curator, Duration interval) {
         super(applicationRepository, curator, interval);
     }
 
     @Override
     protected void maintain() {
-        curator.delete(Path.fromString("/vespa/filedistribution"));
-        curator.delete(Path.fromString("/vespa/config"));
-        curator.delete(Path.fromString("/provision/v1/defaultFlavor"));
+        pathsToDelete.forEach(path -> curator.delete(Path.fromString(path)));
     }
 }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java
index 7bb77fdcbff..3eece4f5640 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java
@@ -138,27 +138,13 @@ public abstract class ModelsBuilder<MODELRESULT extends ModelResult> {
                                                            allocatedHosts.asOptional(),
                                                            now);
         allocatedHosts.set(latestModelVersion.getModel().allocatedHosts()); // Update with additional clusters allocated
-        
+        List<MODELRESULT> allApplicationVersions = new ArrayList<>(Collections.singletonList(latestModelVersion));
+
         if (latestModelVersion.getModel().skipOldConfigModels(now))
-            return Collections.singletonList(latestModelVersion);
+            return allApplicationVersions;
 
         // load old model versions
-        List<MODELRESULT> allApplicationVersions = new ArrayList<>();
-        allApplicationVersions.add(latestModelVersion);
-
-        // TODO: Enable for all zones
-        if (configserverConfig.buildMinimalSetOfConfigModels() &&
-                (SystemName.from(configserverConfig.system()) == SystemName.cd ||
-                        Arrays.asList(Environment.dev, Environment.test, Environment.staging).contains(zone().environment()) ||
-                        Arrays.asList("corp-us-east-1", "ap-southeast-1").contains(zone().region().value())))
-            versions = keepThoseUsedOn(allocatedHosts.get(), versions);
-
-        // Make sure we build wanted version if we are building models for this major version and we are on hosted vespa
-        // If not on hosted vespa, we do not want to try to build this version, since we have only one version (the latest)
-        Version wantedVersion = Version.fromIntValues(wantedNodeVespaVersion.getMajor(), wantedNodeVespaVersion.getMinor(), wantedNodeVespaVersion.getMicro());
-        if (hosted && wantedVersion.getMajor() == latest.getMajor())
-            versions.add(wantedVersion);
-
+        versions = versionsToBuild(versions, wantedNodeVespaVersion, allocatedHosts.get());
         // TODO: We use the allocated hosts from the newest version when building older model versions.
         // This is correct except for the case where an old model specifies a cluster which the new version
         // does not. In that case we really want to extend the set of allocated hosts to include those of that
@@ -179,6 +165,23 @@ public abstract class ModelsBuilder<MODELRESULT extends ModelResult> {
         return allApplicationVersions;
     }
 
+    private Set<Version> versionsToBuild(Set<Version> versions, com.yahoo.component.Version wantedVersion, AllocatedHosts allocatedHosts) {
+        // TODO: Enable for all zones
+        if (configserverConfig.buildMinimalSetOfConfigModels() &&
+                (SystemName.from(configserverConfig.system()) == SystemName.cd ||
+                        Arrays.asList(Environment.dev, Environment.test, Environment.staging).contains(zone().environment()) ||
+                        Arrays.asList("corp-us-east-1", "ap-southeast-1", "us-central-1").contains(zone().region().value())))
+            versions = keepThoseUsedOn(allocatedHosts, versions);
+
+        // Make sure we build wanted version if we are building models for this major version and we are on hosted vespa
+        // If not on hosted vespa, we do not want to try to build this version, since we have only one version (the latest)
+        Version wanted = Version.fromIntValues(wantedVersion.getMajor(), wantedVersion.getMinor(), wantedVersion.getMicro());
+        if (hosted && wantedVersion.getMajor() == findLatest(versions).getMajor())
+            versions.add(wanted);
+
+        return versions;
+    }
+
     private Set<Version> filterByMajorVersion(int majorVersion, Set<Version> versions) {
         Set<Version> filteredVersions = versions.stream().filter(v -> v.getMajor() == majorVersion).collect(Collectors.toSet());
         if (filteredVersions.isEmpty())
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSession.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSession.java
index 70ea20d5b4d..0f9f8b72de1 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSession.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSession.java
@@ -134,11 +134,6 @@ public class LocalSession extends Session implements Comparable<LocalSession> {
         return lhsId.compareTo(rhsId);
     }
 
-    // in seconds
-    public long getCreateTime() {
-        return zooKeeperClient.readCreateTime();
-    }
-
     public void waitUntilActivated(TimeoutBudget timeoutBudget) {
         zooKeeperClient.getActiveWaiter().awaitCompletion(timeoutBudget.timeLeft());
     }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java
index c47d5791a1b..dbccba395a2 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java
@@ -132,4 +132,9 @@ public class RemoteSession extends Session {
         }
     }
 
+    public void delete() {
+        Transaction transaction = zooKeeperClient.deleteTransaction();
+        transaction.commit();
+    }
+
 }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionRepo.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionRepo.java
index d16d5a17518..39e08a328e5 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionRepo.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionRepo.java
@@ -1,6 +1,8 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.config.server.session;
 
+import java.time.Duration;
+import java.time.Instant;
 import java.util.*;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
@@ -48,6 +50,7 @@ public class RemoteSessionRepo extends SessionRepo<RemoteSession> implements Nod
     private final RemoteSessionFactory remoteSessionFactory;
     private final Map<Long, RemoteSessionStateWatcher> sessionStateWatchers = new HashMap<>();
     private final ReloadHandler reloadHandler;
+    private final TenantName tenantName;
     private final MetricUpdater metrics;
     private final Curator.DirectoryCache directoryCache;
     private final TenantApplications applicationRepo;
@@ -56,20 +59,21 @@ public class RemoteSessionRepo extends SessionRepo<RemoteSession> implements Nod
      * @param curator              a {@link Curator} instance.
      * @param remoteSessionFactory a {@link com.yahoo.vespa.config.server.session.RemoteSessionFactory}
      * @param reloadHandler        a {@link com.yahoo.vespa.config.server.ReloadHandler}
-     * @param tenant               a {@link TenantName} instance.
+     * @param tenantName           a {@link TenantName} instance.
      * @param applicationRepo      a {@link TenantApplications} instance.
      */
     public RemoteSessionRepo(Curator curator,
                              RemoteSessionFactory remoteSessionFactory,
                              ReloadHandler reloadHandler,
-                             TenantName tenant,
+                             TenantName tenantName,
                              TenantApplications applicationRepo,
                              MetricUpdater metricUpdater) {
         this.curator = curator;
-        this.sessionsPath = TenantRepository.getSessionsPath(tenant);
+        this.sessionsPath = TenantRepository.getSessionsPath(tenantName);
         this.applicationRepo = applicationRepo;
         this.remoteSessionFactory = remoteSessionFactory;
         this.reloadHandler = reloadHandler;
+        this.tenantName = tenantName;
         this.metrics = metricUpdater;
         initializeSessions();
         this.directoryCache = curator.createDirectoryCache(sessionsPath.getAbsolute(), false, false, pathChildrenExecutor);
@@ -82,12 +86,37 @@ public class RemoteSessionRepo extends SessionRepo<RemoteSession> implements Nod
         this.curator = null;
         this.remoteSessionFactory = null;
         this.reloadHandler = null;
+        this.tenantName = tenantName;
         this.sessionsPath = TenantRepository.getSessionsPath(tenantName);
         this.metrics = null;
         this.directoryCache = null;
         this.applicationRepo = null;
     }
 
+    public List<Long> getSessions() {
+        return getSessionList(curator.getChildren(sessionsPath));
+    }
+
+    public int deleteExpiredSessions(Duration expiryTime, boolean deleteFromZooKeeper) {
+        int deleted = 0;
+        for (long sessionId : getSessions()) {
+            RemoteSession session = getSession(sessionId);
+            Instant created = Instant.ofEpochSecond(session.getCreateTime());
+            if (sessionHasExpired(created, expiryTime)) {
+                log.log(LogLevel.INFO, "Remote session " + sessionId + " for " + tenantName + " has expired");
+                if (deleteFromZooKeeper) {
+                    session.delete();
+                    deleted++;
+                }
+            }
+        }
+        return deleted;
+    }
+
+    private boolean sessionHasExpired(Instant created, Duration expiryTime) {
+        return (created.plus(expiryTime).isBefore(Instant.now()));
+    }
+
     private void loadActiveSession(RemoteSession session) {
         tryReload(session.ensureApplicationLoaded(), session.logPre());
     }
@@ -113,7 +142,7 @@ public class RemoteSessionRepo extends SessionRepo<RemoteSession> implements Nod
 
     // TODO: Add sessions in parallel
     private void initializeSessions() throws NumberFormatException {
-        getSessionList(curator.getChildren(sessionsPath)).forEach(this::sessionAdded);
+        getSessions().forEach(this::sessionAdded);
     }
 
     private synchronized void sessionsChanged() throws NumberFormatException {
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/Session.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/Session.java
index e98931b0573..64ecc510fe9 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/Session.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/Session.java
@@ -67,4 +67,9 @@ public abstract class Session {
         return TenantRepository.logPre(getTenant());
     }
 
+    // in seconds
+    public long getCreateTime() {
+        return zooKeeperClient.readCreateTime();
+    }
+
 }
diff --git a/configserver/src/main/sh/start-configserver b/configserver/src/main/sh/start-configserver
index 03ce136e13c..8a1cbd0f0a8 100755
--- a/configserver/src/main/sh/start-configserver
+++ b/configserver/src/main/sh/start-configserver
@@ -161,7 +161,7 @@ vespa-run-as-vespa-user vespa-runserver -s configserver -r 30 -p $pidfile -- \
 	-Xms128m -Xmx2048m \
         -XX:+PreserveFramePointer \
 	-XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=${VESPA_HOME}/var/crash \
-	-XX:OnOutOfMemoryError='kill -9 %p' \
+	-XX:+ExitOnOutOfMemoryError \
 	$jvmargs \
 	-Djava.library.path=${VESPA_HOME}/lib64 \
 	-Djava.awt.headless=true \
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java
index b0320dad88b..6148c276088 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java
@@ -245,12 +245,11 @@ public class ApplicationRepositoryTest {
     @Test
     public void testDeletingInactiveSessions() {
         ManualClock clock = new ManualClock(Instant.now());
-        ConfigserverConfig configserverConfig = new ConfigserverConfig(new ConfigserverConfig.Builder()
-                                                                               .configServerDBDir(Files.createTempDir()
-                                                                                                          .getAbsolutePath())
-                                                                               .configDefinitionsDir(Files.createTempDir()
-                                                                                                             .getAbsolutePath())
-                                                                               .sessionLifetime(60));
+        ConfigserverConfig configserverConfig =
+                new ConfigserverConfig(new ConfigserverConfig.Builder()
+                                               .configServerDBDir(Files.createTempDir().getAbsolutePath())
+                                               .configDefinitionsDir(Files.createTempDir().getAbsolutePath())
+                                               .sessionLifetime(60));
         DeployTester tester = new DeployTester(configserverConfig, clock);
         tester.deployApp("src/test/apps/app", "myapp", Instant.now()); // session 2 (numbering starts at 2)
 
@@ -274,11 +273,14 @@ public class ApplicationRepositoryTest {
 
         clock.advance(Duration.ofHours(1)); // longer than session lifetime
 
-        // All sessions except 3 should be removed after the call to deleteOldSessions
-        tester.applicationRepository().deleteOldSessions();
+        // All sessions except 3 should be removed after the call to deleteExpiredLocalSessions
+        tester.applicationRepository().deleteExpiredLocalSessions();
         final Collection<LocalSession> sessions = tester.tenant().getLocalSessionRepo().listSessions();
         assertEquals(1, sessions.size());
         assertEquals(3, new ArrayList<>(sessions).get(0).getSessionId());
+
+        // There should be no expired remote sessions in the common case
+        assertEquals(0, applicationRepository.deleteExpiredRemoteSessions(Duration.ofSeconds(0)));
     }
 
     private PrepareResult prepareAndActivateApp(File application) throws IOException {
diff --git a/container-core/pom.xml b/container-core/pom.xml
index d8473c8b541..d51930b5059 100644
--- a/container-core/pom.xml
+++ b/container-core/pom.xml
@@ -95,9 +95,19 @@
       <version>${project.version}</version>
       <exclusions>
         <exclusion>
+          <!-- Pulled in by language-detector in scope compile -->
+          <groupId>com.google.guava</groupId>
+          <artifactId>guava</artifactId>
+        </exclusion>
+        <exclusion>
           <groupId>log4j</groupId>
           <artifactId>log4j</artifactId>
         </exclusion>
+        <exclusion>
+          <!-- Pulled in by language-detector in scope compile -->
+          <groupId>org.slf4j</groupId>
+          <artifactId>slf4j-api</artifactId>
+        </exclusion>
       </exclusions>
     </dependency>
     <dependency>
diff --git a/container-core/src/main/java/com/yahoo/container/config/StatisticsRequestHandler.java b/container-core/src/main/java/com/yahoo/container/config/StatisticsRequestHandler.java
index 1e5c22f4023..e0b6392b64a 100644
--- a/container-core/src/main/java/com/yahoo/container/config/StatisticsRequestHandler.java
+++ b/container-core/src/main/java/com/yahoo/container/config/StatisticsRequestHandler.java
@@ -20,8 +20,8 @@ import java.util.concurrent.Executor;
  * Handler of statistics http requests. Temporary hack as a step towards a more
  * general network interface.
  *
- * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
- * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ * @author Steinar Knutsen
+ * @author Einar M R Rosenvinge
  */
 public class StatisticsRequestHandler extends ThreadedHttpRequestHandler {
 
diff --git a/container-core/src/main/java/com/yahoo/container/core/config/HandlersConfigurerDi.java b/container-core/src/main/java/com/yahoo/container/core/config/HandlersConfigurerDi.java
index e2c6da6fab8..55d7de90f33 100644
--- a/container-core/src/main/java/com/yahoo/container/core/config/HandlersConfigurerDi.java
+++ b/container-core/src/main/java/com/yahoo/container/core/config/HandlersConfigurerDi.java
@@ -86,11 +86,7 @@ public class HandlersConfigurerDi {
         osgiWrapper = new OsgiWrapper(osgiFramework, vespaContainer.getBundleLoader());
 
         container = new Container(subscriberFactory, configId, deconstructor, osgiWrapper);
-        try {
-            getNewComponentGraph(discInjector, false);
-        } catch (InterruptedException e) {
-            throw new RuntimeException("Interrupted while setting up handlers for the first time.");
-        }
+        getNewComponentGraph(discInjector, false);
     }
 
     private static class OsgiWrapper extends OsgiImpl implements com.yahoo.container.di.Osgi {
@@ -139,10 +135,10 @@ public class HandlersConfigurerDi {
     /**
      * Wait for new config to arrive and produce the new graph
      */
-    public void getNewComponentGraph(Injector discInjector, boolean restartOnRedeploy) throws InterruptedException {
-        currentGraph = container.getNewComponentGraph(currentGraph, createFallbackInjector(vespaContainer, discInjector), restartOnRedeploy);
-
-        assert (currentGraph.getInstance(RegistriesHack.class) != null); // TODO: Remove, seems quite pointless?
+    public void getNewComponentGraph(Injector discInjector, boolean restartOnRedeploy) {
+        currentGraph = container.getNewComponentGraph(currentGraph,
+                                                      createFallbackInjector(vespaContainer, discInjector),
+                                                      restartOnRedeploy);
     }
 
     @SuppressWarnings("deprecation")
diff --git a/container-core/src/main/java/com/yahoo/container/core/config/testutil/HandlersConfigurerTestWrapper.java b/container-core/src/main/java/com/yahoo/container/core/config/testutil/HandlersConfigurerTestWrapper.java
index 033a5819585..42780f75a6c 100644
--- a/container-core/src/main/java/com/yahoo/container/core/config/testutil/HandlersConfigurerTestWrapper.java
+++ b/container-core/src/main/java/com/yahoo/container/core/config/testutil/HandlersConfigurerTestWrapper.java
@@ -111,11 +111,7 @@ public class HandlersConfigurerTestWrapper {
 
     public void reloadConfig() {
         configurer.reloadConfig(++lastGeneration);
-        try {
-            configurer.getNewComponentGraph(Guice.createInjector(), false);
-        } catch (InterruptedException e) {
-           throw new RuntimeException(e);
-        }
+        configurer.getNewComponentGraph(Guice.createInjector(), false);
     }
 
     public void shutdown() {
diff --git a/container-core/src/main/java/com/yahoo/container/handler/AccessLogRequestHandler.java b/container-core/src/main/java/com/yahoo/container/handler/AccessLogRequestHandler.java
index 241ae269fc9..c50dee43eaf 100644
--- a/container-core/src/main/java/com/yahoo/container/handler/AccessLogRequestHandler.java
+++ b/container-core/src/main/java/com/yahoo/container/handler/AccessLogRequestHandler.java
@@ -22,6 +22,7 @@ import java.util.concurrent.Executor;
  * @author dybis
  */
 public class AccessLogRequestHandler extends ThreadedHttpRequestHandler {
+
     private final CircularArrayAccessLogKeeper circularArrayAccessLogKeeper;
     private final JsonFactory jsonFactory = new JsonFactory();
 
@@ -53,4 +54,5 @@ public class AccessLogRequestHandler extends ThreadedHttpRequestHandler {
             }
         };
     }
+
 }
diff --git a/container-core/src/main/java/com/yahoo/container/handler/test/MockService.java b/container-core/src/main/java/com/yahoo/container/handler/test/MockService.java
index 99d28b9bcf1..7bd18c519eb 100644
--- a/container-core/src/main/java/com/yahoo/container/handler/test/MockService.java
+++ b/container-core/src/main/java/com/yahoo/container/handler/test/MockService.java
@@ -39,12 +39,10 @@ import java.util.logging.Logger;
  * for descriptions of the format.
  *
  * @author lulf
- * @since 5.1.21
  */
 @Beta
 public class MockService extends LoggingRequestHandler {
 
-    private final static Logger log = Logger.getLogger(MockService.class.getName());
     private MockServiceHandler handler;
 
     /**
diff --git a/container-core/src/main/java/com/yahoo/restapi/Path.java b/container-core/src/main/java/com/yahoo/restapi/Path.java
index 65f0bab9a8d..7f78572f2d7 100644
--- a/container-core/src/main/java/com/yahoo/restapi/Path.java
+++ b/container-core/src/main/java/com/yahoo/restapi/Path.java
@@ -19,7 +19,7 @@ import java.util.stream.Collectors;
  * If the path spec ends with /{*}, it will match urls with any rest path.
  * The rest path (not including the trailing slash) will be available as getRest().
  * 
- * Note that for convenience in common use this has state which is changes as a side effect of each matches
+ * Note that for convenience in common use this has state which changes as a side effect of each matches
  * invocation. It is therefore for single thread use.
  * 
  * @author bratseth
@@ -40,6 +40,8 @@ public class Path {
     }
 
     /**
+     * Parses the path according to pathSpec - must be called prior to {@link #get}
+     *
      * Returns whether this path matches the given template string.
      * If the given template has placeholders, their values (accessible by get) are reset by calling this,
      * whether or not the path matches the given template.
diff --git a/container-dependencies-enforcer/pom.xml b/container-dependencies-enforcer/pom.xml
index 9929f267c47..bad78ce4182 100644
--- a/container-dependencies-enforcer/pom.xml
+++ b/container-dependencies-enforcer/pom.xml
@@ -104,8 +104,8 @@
                                                 <include>org.apache.felix:org.apache.felix.framework:[${felix.version}]:jar:provided</include>
                                                 <include>org.apache.felix:org.apache.felix.log:[1.0.1]:jar:provided</include>
                                                 <include>org.apache.felix:org.apache.felix.main:[${felix.version}]:jar:provided</include>
-                                                <include>org.apache.httpcomponents:httpclient:[4.3.6]:jar:provided</include>
-                                                <include>org.apache.httpcomponents:httpcore:[4.3.3]:jar:provided</include>
+                                                <include>org.apache.httpcomponents:httpclient:[${apache.httpclient.version}]:jar:provided</include>
+                                                <include>org.apache.httpcomponents:httpcore:[${apache.httpcore.version}]:jar:provided</include>
                                                 <include>org.bouncycastle:bcpkix-jdk15on:[${bouncycastle.version}]:jar:provided</include>
                                                 <include>org.bouncycastle:bcprov-jdk15on:[${bouncycastle.version}]:jar:provided</include>
                                                 <include>org.eclipse.jetty:jetty-http:[${jetty.version}]:jar:provided</include>
diff --git a/container-dependency-versions/pom.xml b/container-dependency-versions/pom.xml
index ec4e9eae5fb..ccb8a9c311c 100644
--- a/container-dependency-versions/pom.xml
+++ b/container-dependency-versions/pom.xml
@@ -238,12 +238,12 @@
             <dependency>
                 <groupId>org.apache.httpcomponents</groupId>
                 <artifactId>httpclient</artifactId>
-                <version>4.3.6</version>
+                <version>${apache.httpclient.version}</version>
             </dependency>
             <dependency>
                 <groupId>org.apache.httpcomponents</groupId>
                 <artifactId>httpcore</artifactId>
-                <version>4.3.3</version>
+                <version>${apache.httpcore.version}</version>
             </dependency>
             <dependency>
                 <groupId>org.bouncycastle</groupId>
@@ -417,6 +417,12 @@
             </dependency>
             <dependency>
                 <!-- NOT provided from jdisc runtime -->
+                <groupId>com.google.guava</groupId>
+                <artifactId>guava-testlib</artifactId>
+                <version>${guava.version}</version>
+            </dependency>
+            <dependency>
+                <!-- NOT provided from jdisc runtime -->
                 <groupId>org.eclipse.jetty</groupId>
                 <artifactId>jetty-continuation</artifactId>
                 <version>${jetty.version}</version>
@@ -452,6 +458,8 @@
     </dependencyManagement>
 
     <properties>
+        <apache.httpclient.version>4.4.1</apache.httpclient.version>
+        <apache.httpcore.version>4.4.1</apache.httpcore.version>
         <bouncycastle.version>1.58</bouncycastle.version>
         <felix.version>5.6.10</felix.version>
         <findbugs.version>1.3.9</findbugs.version>
diff --git a/container-dev/pom.xml b/container-dev/pom.xml
index fe5f852a226..27e2435d825 100644
--- a/container-dev/pom.xml
+++ b/container-dev/pom.xml
@@ -189,7 +189,7 @@
       <version>${project.version}</version>
     </dependency>
 
-    <!-- Dependencies below are added explicitly to exclude transitive deps that are not provided runtime by the container,
+    <!-- NOTE: Dependencies below are added explicitly to exclude transitive deps that are not provided runtime by the container,
          and hence make them invisible to user projects' build classpath.
          Excluded artifacts should be added explicitly to the application module to make then visible in users' test classpath. -->
     <dependency>
@@ -205,6 +205,21 @@
     </dependency>
     <dependency>
       <groupId>com.yahoo.vespa</groupId>
+      <artifactId>linguistics</artifactId>
+      <version>${project.version}</version>
+      <exclusions>
+        <exclusion>
+          <groupId>com.optimaize.languagedetector</groupId>
+          <artifactId>language-detector</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.apache.opennlp</groupId>
+          <artifactId>opennlp-tools</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+    <dependency>
+      <groupId>com.yahoo.vespa</groupId>
       <artifactId>predicate-search-core</artifactId>
       <version>${project.version}</version>
       <exclusions>
@@ -214,6 +229,6 @@
         </exclusion>
       </exclusions>
     </dependency>
-
+    <!-- DO NOT add dependencies here, but above the NOTE! -->
   </dependencies>
 </project>
diff --git a/container-di/src/main/java/com/yahoo/container/di/ConfigRetriever.java b/container-di/src/main/java/com/yahoo/container/di/ConfigRetriever.java
index fe315c0eba5..337997f8ac2 100644
--- a/container-di/src/main/java/com/yahoo/container/di/ConfigRetriever.java
+++ b/container-di/src/main/java/com/yahoo/container/di/ConfigRetriever.java
@@ -47,13 +47,14 @@ public final class ConfigRetriever {
     /**
      * Loop forever until we get config
      */
-    public ConfigSnapshot getConfigs(Set<ConfigKey<? extends ConfigInstance>> componentConfigKeys, long leastGeneration,
-            boolean restartOnRedeploy) {
+    public ConfigSnapshot getConfigs(Set<ConfigKey<? extends ConfigInstance>> componentConfigKeys,
+                                     long leastGeneration,
+                                     boolean restartOnRedeploy) {
         while (true) {
-            if (!Sets.intersection(componentConfigKeys, bootstrapKeys).isEmpty()) {
-                throw new IllegalArgumentException(
-                        "Component config keys [" + componentConfigKeys + "] overlaps with bootstrap config keys [" + bootstrapKeys + "]");
-            }
+            if (!Sets.intersection(componentConfigKeys, bootstrapKeys).isEmpty())
+                throw new IllegalArgumentException("Component config keys [" + componentConfigKeys +
+                                                   "] overlaps with bootstrap config keys [" + bootstrapKeys + "]");
+
             log.log(DEBUG, "getConfigs: " + componentConfigKeys);
             Set<ConfigKey<? extends ConfigInstance>> allKeys = new HashSet<>(componentConfigKeys);
             allKeys.addAll(bootstrapKeys);
@@ -75,8 +76,9 @@ public final class ConfigRetriever {
     /**
      * Try to get config just once
      */
-    public Optional<ConfigSnapshot> getConfigsOnce(Set<ConfigKey<? extends ConfigInstance>> componentConfigKeys, long leastGeneration,
-            boolean restartOnRedeploy) {
+    Optional<ConfigSnapshot> getConfigsOnce(Set<ConfigKey<? extends ConfigInstance>> componentConfigKeys,
+                                            long leastGeneration,
+                                            boolean restartOnRedeploy) {
         if (!Sets.intersection(componentConfigKeys, bootstrapKeys).isEmpty()) {
             throw new IllegalArgumentException(
                     "Component config keys [" + componentConfigKeys + "] overlaps with bootstrap config keys [" + bootstrapKeys + "]");
@@ -114,7 +116,7 @@ public final class ConfigRetriever {
                 } else {
                     // This should not be a normal case, and hence a warning to allow investigation.
                     log.warning("Did not get same generation for bootstrap (" + newestBootstrapGeneration + ") and components configs ("
-                            + newestComponentGeneration + ").");
+                                + newestComponentGeneration + ").");
                     return Optional.empty();
                 }
             }
diff --git a/container-di/src/main/java/com/yahoo/container/di/Container.java b/container-di/src/main/java/com/yahoo/container/di/Container.java
index 7c58120d858..fb427bcf8ae 100644
--- a/container-di/src/main/java/com/yahoo/container/di/Container.java
+++ b/container-di/src/main/java/com/yahoo/container/di/Container.java
@@ -73,7 +73,6 @@ public class Container {
     }
 
     public ComponentGraph getNewComponentGraph(ComponentGraph oldGraph, Injector fallbackInjector, boolean restartOnRedeploy) {
-
         try {
             ComponentGraph newGraph = getConfigAndCreateGraph(oldGraph, fallbackInjector, restartOnRedeploy);
             newGraph.reuseNodes(oldGraph);
@@ -87,11 +86,11 @@ public class Container {
         }
     }
 
-    public ComponentGraph getNewComponentGraph(ComponentGraph oldGraph) {
+    ComponentGraph getNewComponentGraph(ComponentGraph oldGraph) {
         return getNewComponentGraph(oldGraph, Guice.createInjector(), false);
     }
 
-    public ComponentGraph getNewComponentGraph() {
+    ComponentGraph getNewComponentGraph() {
         return getNewComponentGraph(new ComponentGraph(), Guice.createInjector(), false);
     }
 
@@ -125,10 +124,8 @@ public class Container {
         }
     }
 
-    public ComponentGraph getConfigAndCreateGraph(ComponentGraph graph, Injector fallbackInjector, boolean restartOnRedeploy) {
-
+    private ComponentGraph getConfigAndCreateGraph(ComponentGraph graph, Injector fallbackInjector, boolean restartOnRedeploy) {
         ConfigSnapshot snapshot;
-
         while (true) {
             snapshot = configurer.getConfigs(graph.configKeys(), leastGeneration, restartOnRedeploy);
 
diff --git a/container-disc/pom.xml b/container-disc/pom.xml
index c112466bbd4..ea3425d302d 100644
--- a/container-disc/pom.xml
+++ b/container-disc/pom.xml
@@ -169,6 +169,7 @@
                 configdefinitions-jar-with-dependencies.jar,
                 container-jersey2-jar-with-dependencies.jar,
                 container-search-and-docproc-jar-with-dependencies.jar,
+                container-search-gui-jar-with-dependencies.jar,
                 docprocs-jar-with-dependencies.jar,
                 jdisc-security-filters-jar-with-dependencies.jar,
                 jdisc_http_service-jar-with-dependencies.jar,
diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java b/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java
index 181d0018278..69a525a2c54 100644
--- a/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java
+++ b/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java
@@ -203,7 +203,7 @@ public final class ConfiguredApplication implements Application {
                     // Block until new config arrives, and it should be applied
                     configurer.getNewComponentGraph(builder.guiceModules().activate(), qrConfig.restartOnDeploy());
                     intitializeAndActivateContainer(builder);
-                } catch (ConfigInterruptedException | InterruptedException e) {
+                } catch (ConfigInterruptedException e) {
                     break;
                 } catch (Exception | LinkageError e) { // LinkageError: OSGi problems
                     log.log(Level.SEVERE,
@@ -256,13 +256,12 @@ public final class ConfiguredApplication implements Application {
     }
 
     private void configureComponents(Injector discInjector) {
-        configurer = new HandlersConfigurerDi(
-                subscriberFactory,
-                Container.get(),
-                configId,
-                new Deconstructor(true),
-                discInjector,
-                osgiFramework);
+        configurer = new HandlersConfigurerDi(subscriberFactory,
+                                              Container.get(),
+                                              configId,
+                                              new Deconstructor(true),
+                                              discInjector,
+                                              osgiFramework);
     }
 
     private void setupGuiceBindings(GuiceRepository modules) {
diff --git a/container-disc/src/main/sh/vespa-start-container-daemon.sh b/container-disc/src/main/sh/vespa-start-container-daemon.sh
index 88c8d133d03..e6219ab0467 100755
--- a/container-disc/src/main/sh/vespa-start-container-daemon.sh
+++ b/container-disc/src/main/sh/vespa-start-container-daemon.sh
@@ -193,7 +193,7 @@ exec_jsvc () {
         -XX:MaxJavaStackTraceDepth=1000000 \
         -XX:+HeapDumpOnOutOfMemoryError \
         -XX:HeapDumpPath="${VESPA_HOME}/var/crash" \
-        -XX:OnOutOfMemoryError='kill -9 %p' \
+        -XX:+ExitOnOutOfMemoryError \
         -Djava.library.path="${VESPA_HOME}/lib64" \
         -Djava.awt.headless=true \
         -Djavax.net.ssl.keyStoreType=JKS \
@@ -265,7 +265,7 @@ exec $numactlcmd $envcmd java \
         -XX:MaxJavaStackTraceDepth=1000000 \
         -XX:+HeapDumpOnOutOfMemoryError \
         -XX:HeapDumpPath="${VESPA_HOME}/var/crash" \
-        -XX:OnOutOfMemoryError='kill -9 %p' \
+        -XX:+ExitOnOutOfMemoryError \
         -Djava.library.path="${VESPA_HOME}/lib64" \
         -Djava.awt.headless=true \
         -Djavax.net.ssl.keyStoreType=JKS \
diff --git a/container-integration-test/.gitignore b/container-integration-test/.gitignore
new file mode 100644
index 00000000000..3cc25b51fc4
--- /dev/null
+++ b/container-integration-test/.gitignore
@@ -0,0 +1,2 @@
+/pom.xml.build
+/target
diff --git a/container-integration-test/OWNERS b/container-integration-test/OWNERS
new file mode 100644
index 00000000000..3b2ba1ede81
--- /dev/null
+++ b/container-integration-test/OWNERS
@@ -0,0 +1 @@
+gjoranv
diff --git a/container-integration-test/README.md b/container-integration-test/README.md
new file mode 100644
index 00000000000..8abe3a889ea
--- /dev/null
+++ b/container-integration-test/README.md
@@ -0,0 +1,9 @@
+<!-- Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+
+# Integration tests for JDisc components
+
+This module contains integration tests for container components.
+
+Tests that use the `application` framework cannot be added to the same maven
+module as the component itself because that will usually create a cycle in the
+dependency graph.
diff --git a/container-integration-test/pom.xml b/container-integration-test/pom.xml
new file mode 100644
index 00000000000..4b4d5de21eb
--- /dev/null
+++ b/container-integration-test/pom.xml
@@ -0,0 +1,44 @@
+<?xml version="1.0"?>
+<!-- Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
+                             http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>com.yahoo.vespa</groupId>
+        <artifactId>parent</artifactId>
+        <version>6-SNAPSHOT</version>
+        <relativePath>../parent/pom.xml</relativePath>
+    </parent>
+    <artifactId>container-integration-test</artifactId>
+    <packaging>jar</packaging>
+    <version>6-SNAPSHOT</version>
+    <dependencies>
+        <dependency>
+            <groupId>com.yahoo.vespa</groupId>
+            <artifactId>container-search-gui</artifactId>
+            <version>${project.version}</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.yahoo.vespa</groupId>
+            <artifactId>application</artifactId>
+            <version>${project.version}</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-surefire-plugin</artifactId>
+            </plugin>
+        </plugins>
+    </build>
+</project>
\ No newline at end of file
diff --git a/container-search-gui/src/test/java/com/yahoo/search/query/gui/GUIHandlerTest.java b/container-integration-test/src/test/java/com/yahoo/search/query/gui/GUIHandlerTest.java
index ec515a1bf4a..5fd73afe800 100644
--- a/container-search-gui/src/test/java/com/yahoo/search/query/gui/GUIHandlerTest.java
+++ b/container-integration-test/src/test/java/com/yahoo/search/query/gui/GUIHandlerTest.java
@@ -5,13 +5,12 @@ import com.yahoo.application.Networking;
 import com.yahoo.application.container.JDisc;
 import com.yahoo.application.container.handler.Request;
 import com.yahoo.application.container.handler.Response;
+
 import org.junit.After;
-import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
-
 import java.io.IOException;
-
+import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 
 
@@ -25,7 +24,7 @@ public class GUIHandlerTest {
 
     @Before
     public void startContainer() {
-        container = JDisc.fromServicesXml(servicesXml(), Networking.enable);
+        container = JDisc.fromServicesXml(servicesXml(), Networking.disable);
     }
 
     @After
@@ -64,8 +63,8 @@ public class GUIHandlerTest {
 
     private void assertResponse(Request.Method method, String path, String expectedStartString, String expectedContentType, int expectedStatusCode) throws IOException {
         Response response = container.handleRequest(new Request("http://localhost:8080" + path, new byte[0], method));
-        Assert.assertEquals("Status code", expectedStatusCode, response.getStatus());
-        Assert.assertEquals(expectedContentType, response.getHeaders().getFirst("Content-Type"));
+        assertEquals("Status code", expectedStatusCode, response.getStatus());
+        assertEquals(expectedContentType, response.getHeaders().getFirst("Content-Type"));
         if(expectedStartString != null){
             assertTrue(response.getBodyAsString().startsWith(expectedStartString));
         }
diff --git a/container-search-gui/.gitignore b/container-search-gui/.gitignore
index 7f4ebc3a7c6..af784e084e1 100644
--- a/container-search-gui/.gitignore
+++ b/container-search-gui/.gitignore
@@ -1,31 +1,2 @@
-.classpath
-.project
-.settings
-/.cache
-/.classpath
-/.emacs.desktop
-/.nbintdb
-/.project
-/.settings
-/.version
-/QueryAccessLog*
-/accessLog.log
-/build
-/bundles
-/libexec
-/nbproject
 /pom.xml.build
-/prelude.iml
-/prelude.ipr
-/prelude.iws
-/staging
-/target
-/temp
-/testLogFileG.txt
-/testLogs
-/test_yapache_access_log
-/testlogsG
-/testng.out.log
-/tmp
-null.log
-tmp
+/target
\ No newline at end of file
diff --git a/container-search-gui/CMakeLists.txt b/container-search-gui/CMakeLists.txt
new file mode 100644
index 00000000000..95476154443
--- /dev/null
+++ b/container-search-gui/CMakeLists.txt
@@ -0,0 +1,2 @@
+# Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+install_fat_java_artifact(container-search-gui)
\ No newline at end of file
diff --git a/container-search-gui/pom.xml b/container-search-gui/pom.xml
index f2ca70dd1f3..12f425f8b4b 100644
--- a/container-search-gui/pom.xml
+++ b/container-search-gui/pom.xml
@@ -4,46 +4,85 @@
          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
                              http://maven.apache.org/xsd/maven-4.0.0.xsd">
-  <modelVersion>4.0.0</modelVersion>
-  <parent>
-    <groupId>com.yahoo.vespa</groupId>
-    <artifactId>parent</artifactId>
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>com.yahoo.vespa</groupId>
+        <artifactId>parent</artifactId>
+        <version>6-SNAPSHOT</version>
+        <relativePath>../parent/pom.xml</relativePath>
+    </parent>
+    <artifactId>container-search-gui</artifactId>
+    <packaging>container-plugin</packaging>
     <version>6-SNAPSHOT</version>
-    <relativePath>../parent/pom.xml</relativePath>
-  </parent>
-  <artifactId>container-search-gui</artifactId>
-  <packaging>jar</packaging>
-  <version>6-SNAPSHOT</version>
-  <dependencies>
-    <dependency>
-      <groupId>com.yahoo.vespa</groupId>
-      <artifactId>application</artifactId>
-      <version>6-SNAPSHOT</version>
-    </dependency>
-    <dependency>
-      <groupId>junit</groupId>
-      <artifactId>junit</artifactId>
-      <scope>test</scope>
-    </dependency>
-  </dependencies>
-  <build>
-    <plugins>
-      <plugin>
-        <groupId>org.apache.maven.plugins</groupId>
-        <artifactId>maven-compiler-plugin</artifactId>
-        <configuration>
-          <compilerArgs>
-            <arg>-Xlint:all</arg>
-            <arg>-Xlint:-rawtypes</arg>
-            <arg>-Xlint:-unchecked</arg>
-            <arg>-Xlint:-serial</arg>
-            <arg>-Xlint:-deprecation</arg>
-            <arg>-Xlint:-dep-ann</arg>
-            <arg>-Xlint:-cast</arg>
-            <arg>-Werror</arg>
-          </compilerArgs>
-        </configuration>
-      </plugin>
-    </plugins>
-  </build>
-</project>
\ No newline at end of file
+    <dependencies>
+        <dependency>
+            <groupId>com.google.inject</groupId>
+            <artifactId>guice</artifactId>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.yahoo.vespa</groupId>
+            <artifactId>yolean</artifactId>
+            <version>${project.version}</version>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.yahoo.vespa</groupId>
+            <artifactId>jdisc_http_service</artifactId>
+            <version>${project.version}</version>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.yahoo.vespa</groupId>
+            <artifactId>jdisc_core</artifactId>
+            <version>${project.version}</version>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.yahoo.vespa</groupId>
+            <artifactId>container-core</artifactId>
+            <version>${project.version}</version>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.yahoo.vespa</groupId>
+            <artifactId>container-search-and-docproc</artifactId>
+            <version>${project.version}</version>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.yahoo.vespa</groupId>
+            <artifactId>configdefinitions</artifactId>
+            <version>${project.version}</version>
+            <scope>provided</scope>
+        </dependency>
+    </dependencies>
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>com.googlecode.maven-download-plugin</groupId>
+                <artifactId>download-maven-plugin</artifactId>
+                <version>1.3.0</version>
+                <executions>
+                    <execution>
+                        <phase>prepare-package</phase>
+                        <goals>
+                            <goal>wget</goal>
+                        </goals>
+                        <configuration>
+                            <url>https://docs.vespa.ai/documentation/reference/search-api-reference.html</url>
+                            <outputFileName>search-api-reference.html</outputFileName>
+                            <outputDirectory>${project.build.outputDirectory}/gui/_includes/</outputDirectory>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <groupId>com.yahoo.vespa</groupId>
+                <artifactId>bundle-plugin</artifactId>
+                <extensions>true</extensions>
+            </plugin>
+        </plugins>
+    </build>
+
+</project>
diff --git a/container-search-gui/src/main/java/com/yahoo/search/query/gui/GUIHandler.java b/container-search-gui/src/main/java/com/yahoo/search/query/gui/GUIHandler.java
index 0f4eebd7fd7..45a616ce473 100644
--- a/container-search-gui/src/main/java/com/yahoo/search/query/gui/GUIHandler.java
+++ b/container-search-gui/src/main/java/com/yahoo/search/query/gui/GUIHandler.java
@@ -2,22 +2,39 @@
 package com.yahoo.search.query.gui;
 
 import com.google.inject.Inject;
+
+import com.yahoo.container.QrSearchersConfig;
 import com.yahoo.container.jdisc.HttpRequest;
 import com.yahoo.container.jdisc.HttpResponse;
 import com.yahoo.container.jdisc.LoggingRequestHandler;
-
-
+import com.yahoo.prelude.IndexModel;
+import com.yahoo.prelude.fastsearch.CacheControl;
+import com.yahoo.prelude.querytransform.RecallSearcher;
+import com.yahoo.search.Query;
+import com.yahoo.search.query.Model;
+import com.yahoo.search.query.Presentation;
+import com.yahoo.search.query.Ranking;
+import com.yahoo.search.query.ranking.Diversity;
+import com.yahoo.search.query.ranking.MatchPhase;
 import com.yahoo.search.query.restapi.ErrorResponse;
+import com.yahoo.search.yql.MinimalQueryInserter;
+import com.yahoo.vespa.config.search.RankProfilesConfig;
+import com.yahoo.search.config.IndexInfoConfig;
 import com.yahoo.yolean.Exceptions;
 
-import java.io.File;
+import org.json.JSONException;
+import org.json.JSONObject;
+import java.io.ByteArrayInputStream;
 import java.io.IOException;
+import java.io.InputStream;
 import java.io.OutputStream;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
 import java.util.logging.Level;
 
+
 /**
  * Takes requests on /querybuilder
  *
@@ -26,9 +43,15 @@ import java.util.logging.Level;
 
 public class GUIHandler extends LoggingRequestHandler {
 
+    private final IndexModel indexModel;
+    private final RankProfilesConfig rankProfilesConfig;
+
     @Inject
-    public GUIHandler(Context parentCtx) {
-        super(parentCtx);
+    public GUIHandler(Context parentContext, IndexInfoConfig indexInfo, QrSearchersConfig clusters, RankProfilesConfig rankProfilesConfig) {
+        super(parentContext);
+        this.indexModel = new IndexModel(indexInfo, clusters);
+        this.rankProfilesConfig = rankProfilesConfig;
+
     }
 
     @Override
@@ -47,75 +70,141 @@ public class GUIHandler extends LoggingRequestHandler {
     }
 
     private HttpResponse handleGET(HttpRequest request) {
-         com.yahoo.restapi.Path path = new com.yahoo.restapi.Path(request.getUri().getPath());
+        com.yahoo.restapi.Path path = new com.yahoo.restapi.Path(request.getUri().getPath());
         if (path.matches("/querybuilder/")) {
-            return new FileResponse("_includes/index.html");
+            return new FileResponse("_includes/index.html", null, null);
         }
         if (!path.matches("/querybuilder/{*}") ) {
-            return ErrorResponse.notFoundError("Nothing at " + path);
+            return ErrorResponse.notFoundError("Nothing at path:" + path);
         }
         String filepath = path.getRest();
-        if (!isValidPath(GUIHandler.class.getClassLoader().getResource("gui").getFile()+"/"+filepath)){
-            return ErrorResponse.notFoundError("Nothing at " + path);
+        if (!isValidPath(filepath) && !filepath.equals("config.json")){
+            return ErrorResponse.notFoundError("Nothing at path:" + filepath);
         }
-        return new FileResponse(filepath);
+        return new FileResponse(filepath, indexModel, rankProfilesConfig);
     }
 
     private static boolean isValidPath(String path) {
-        File file = new File(path);
-        return file.exists();
+        InputStream in = GUIHandler.class.getClassLoader().getResourceAsStream("gui/"+path);
+        boolean isValid = (in != null);
+        if(isValid){
+            try { in.close(); } catch (IOException e) {/* Problem with closing inputstream */}
+        }
+
+        return isValid;
     }
 
     private static class FileResponse extends HttpResponse {
 
-        private final Path path;
+        private final String path;
+        private final IndexModel indexModel;
+        private final RankProfilesConfig rankProfilesConfig;
 
-        public FileResponse(String relativePath) {
+        public FileResponse(String relativePath, IndexModel indexModel, RankProfilesConfig rankProfilesConfig) {
             super(200);
-            this.path = Paths.get(GUIHandler.class.getClassLoader().getResource("gui").getFile(), relativePath);
+            this.path = relativePath;
+            this.indexModel = indexModel;
+            this.rankProfilesConfig = rankProfilesConfig;
+
         }
 
         @Override
         public void render(OutputStream out) throws IOException {
-            byte[] data = Files.readAllBytes(path);
-            out.write(data);
+            InputStream is;
+            if (this.path.equals("config.json")){
+                String json = "{}";
+                try { json = getGUIConfig(); } catch (JSONException e) { /*Something happened while parsing JSON */ }
+                is = new ByteArrayInputStream(json.getBytes());
+            } else{
+                is = GUIHandler.class.getClassLoader().getResourceAsStream("gui/"+this.path);
+            }
+            byte[] buf = new byte[1024];
+            int numRead;
+            while ( (numRead = is.read(buf) ) >= 0) {
+                out.write(buf, 0, numRead);
+            }
         }
 
         @Override
         public String getContentType() {
-            if (path.toString().endsWith(".css")) {
+            if (path.endsWith(".css")) {
                 return "text/css";
-            } else if (path.toString().endsWith(".js")) {
+            } else if (path.endsWith(".js")) {
                 return "application/javascript";
-            } else if (path.toString().endsWith(".html")) {
+            } else if (path.endsWith(".html")) {
                 return "text/html";
-            }else if (path.toString().endsWith(".php")) {
+            } else if (path.endsWith(".php")) {
                 return "text/php";
-            }else if (path.toString().endsWith(".svg")) {
+            } else if (path.endsWith(".svg")) {
                 return "image/svg+xml";
-            }else if (path.toString().endsWith(".eot")) {
+            } else if (path.endsWith(".eot")) {
                 return "application/vnd.ms-fontobject";
-            }else if (path.toString().endsWith(".ttf")) {
+            } else if (path.endsWith(".ttf")) {
                 return "font/ttf";
-            }else if (path.toString().endsWith(".woff")) {
+            } else if (path.endsWith(".woff")) {
                 return "font/woff";
-            }else if (path.toString().endsWith(".woff2")) {
+            } else if (path.endsWith(".woff2")) {
                 return "font/woff2";
-            }else if (path.toString().endsWith(".otf")) {
+            } else if (path.endsWith(".otf")) {
                 return "font/otf";
-            }else if (path.toString().endsWith(".png")) {
+            } else if (path.endsWith(".png")) {
                 return "image/png";
-            }else if (path.toString().endsWith(".xml")) {
+            } else if (path.endsWith(".xml")) {
                 return "application/xml";
-            }else if (path.toString().endsWith(".ico")) {
+            } else if (path.endsWith(".ico")) {
                 return "image/x-icon";
-            }else if (path.toString().endsWith(".json")) {
+            } else if (path.endsWith(".json")) {
                 return "application/json";
-            }else if (path.toString().endsWith(".ttf")) {
+            } else if (path.endsWith(".ttf")) {
                 return "font/ttf";
             }
             return "text/html";
         }
-    }
 
-}
+        private String getGUIConfig() throws JSONException {
+            JSONObject json = new JSONObject();
+            json.put("ranking_properties", Arrays.asList("propertyname"));
+            json.put("ranking_features", Arrays.asList("featurename"));
+
+            List<String> sources = new ArrayList<>();
+
+            try {
+                sources = new ArrayList<>(indexModel.getMasterClusters().keySet());
+            } catch (NullPointerException ex){ /* clusters are not set */ }
+            json.put("model_sources", sources);
+
+            List<String> rankProfiles = new ArrayList<>();
+            try {
+                rankProfilesConfig.rankprofile().forEach(rankProfile -> rankProfiles.add(rankProfile.name()));
+            } catch (NullPointerException ex){ /* rankprofiles are not set*/ }
+            json.put("ranking_profile", rankProfiles);
+
+
+            // Creating map from parent to children for GUI: parameter --> child-parameters
+            HashMap<String, List<String>> childMap = new HashMap<>();
+            childMap.put(Model.MODEL, Arrays.asList(Model.DEFAULT_INDEX, Model.ENCODING, Model.LANGUAGE, Model.QUERY_STRING, Model.RESTRICT, Model.SEARCH_PATH, Model.SOURCES, Model.TYPE));
+            childMap.put(Ranking.RANKING, Arrays.asList(Ranking.LOCATION, Ranking.FEATURES, Ranking.LIST_FEATURES, Ranking.PROFILE, Ranking.PROPERTIES, Ranking.SORTING, Ranking.FRESHNESS, Ranking.QUERYCACHE, Ranking.MATCH_PHASE));
+            childMap.put(Ranking.RANKING +"."+ Ranking.MATCH_PHASE, Arrays.asList(MatchPhase.MAX_HITS, MatchPhase.ATTRIBUTE, MatchPhase.ASCENDING, Ranking.DIVERSITY));
+            childMap.put(Ranking.RANKING +"."+ Ranking.MATCH_PHASE +"."+Ranking.DIVERSITY, Arrays.asList(Diversity.ATTRIBUTE, Diversity.MINGROUPS));
+            childMap.put(Presentation.PRESENTATION, Arrays.asList(Presentation.BOLDING, Presentation.FORMAT, Presentation.SUMMARY, "template", Presentation.TIMING ));
+            childMap.put("trace", Arrays.asList("timestamps"));
+            childMap.put("tracelevel", Arrays.asList("rules"));
+            childMap.put("metrics", Arrays.asList("ignore"));
+            childMap.put("collapse", Arrays.asList("summary"));
+            childMap.put("pos", Arrays.asList("ll", "radius", "bb", "attribute"));
+            childMap.put("streaming", Arrays.asList("userid", "groupname", "selection", "priority", "maxbucketspervisitor"));
+            childMap.put("rules", Arrays.asList("off", "rulebase"));
+            json.put("childMap", childMap);
+
+            List<String> levelZeroParameters = Arrays.asList(MinimalQueryInserter.YQL.toString(), Query.HITS.toString(), Query.OFFSET.toString(),
+                                "queryProfile", Query.NO_CACHE.toString(), Query.GROUPING_SESSION_CACHE.toString(),
+                                Query.SEARCH_CHAIN.toString(), Query.TIMEOUT.toString(), "trace", "tracelevel",
+                                Query.TRACE_LEVEL.toString(), Model.MODEL, Ranking.RANKING, "collapse", "collapsesize","collapsefield",
+                                Presentation.PRESENTATION, "pos", "streaming", "rules", RecallSearcher.recallName.toString(), "user",
+                                CacheControl.nocachewrite.toString(), "metrics", "");
+            json.put("levelZeroParameters", levelZeroParameters);
+
+            return json.toString();
+        }
+    }
+}
\ No newline at end of file
diff --git a/container-search-gui/src/main/resources/gui/_includes/css/vespa.css b/container-search-gui/src/main/resources/gui/_includes/css/vespa.css
index 24cfd32cbb2..c63690581c6 100644
--- a/container-search-gui/src/main/resources/gui/_includes/css/vespa.css
+++ b/container-search-gui/src/main/resources/gui/_includes/css/vespa.css
@@ -109,6 +109,13 @@ section h3.section-subheading {
     margin-top: 0;
 }
 
+header .help-title {
+    color: #FFC43C;
+    text-transform: uppercase;
+    font-weight: bold;
+}
+
+
 .text-muted {
     color: #303030;
 }
@@ -242,7 +249,7 @@ header .methodselector{
   -webkit-transition:.3s;
   transition:.3s
 }
-
+header .intro-help:hover {background-color: #4EA2D6 !important;}
 header .methodselector:hover {background-color: #79B4D8;}
 header .button:hover {background-color: #79B4D8;}
 header .removeRow:hover {background-color: #79B4D8;}
@@ -480,12 +487,28 @@ header .intro-copy {
     padding-left: 50px;
     transform: translateX(-50%);
 }
+
+header .intro-help {
+    display: inline-block;
+    margin-top: 15px;
+    width: 134px;
+    height: 45px;
+
+    border-top-left-radius: 29%;
+    border-top-right-radius: 26%;
+    border-bottom-right-radius: 27%;
+    border-bottom-left-radius: 29%;
+
+}
+
 header .intro-refresh {
     display: inline-block;
     margin-top: 15px;
+    margin-left: 25px;
     transform: translateX(-50%);
 }
 
+
 @media (min-height: 1150px) {
     header .intro-down-arrow {
         /* Hard code top: 1150 (height of top section) - 20 (bottom) - 36 (image height) = 1094 */
diff --git a/container-search-gui/src/main/resources/gui/_includes/index.html b/container-search-gui/src/main/resources/gui/_includes/index.html
index ca3302dd404..a441e302720 100644
--- a/container-search-gui/src/main/resources/gui/_includes/index.html
+++ b/container-search-gui/src/main/resources/gui/_includes/index.html
@@ -84,7 +84,7 @@
                 	<option class="options" value="POST">POST</option>
                 	<option class="options" value="GET">GET</option>
               </select>
-              <input type="text" class="textbox" name="value" value="http://httpbin.org/post" id="url" size="30">
+              <input type="text" class="textbox" name="value" value="http://localhost:8080/search/" id="url" size="30">
               <button class="button" onclick="startSending();" id="send">Send</button>
 
               <br/>
@@ -116,6 +116,30 @@
                     <div class="intro-refresh" onclick="refresh();" style="cursor:pointer;">
                       <img src="/querybuilder/img/reload.svg" height="30" width="30" />
                     </div>
+                    </br>
+                    </br>
+                    <div id="helpbutton" class="intro-help" onclick="toggleHelp();" style="cursor:pointer;">
+                      <img src="/querybuilder/img/features-help.png" height="47" width="138" style="margin-left: -1px;"/>
+                    </div>
+                    <div id="help" style="display: none;">
+                    <div class="intro-param" id="help">
+                      </br>
+                      <div class="help-title">Features</div>
+                      <span> ○  Autocompletion of YQL-syntax</span> </br>
+                      <span> ○  Drop-down lists of all valid parameters</span> </br>
+                      <span> ○  Sending both POST and GET-requests to <i>Vespa</i></span> </br>
+                      <span> ○  Easy access to the <a href="https://docs.vespa.ai/documentation/reference/search-api-reference.html">documentation</a> of each parameter</span> </br>
+                      <span> ○  Conversion of POST- to GET-query</span> </br>
+                      <span> ○  Pasting already built JSON-query</span> </br>
+                      <span> ○  View and copy the response of queries</span> </br>
+                      <span> ○  View the finished JSON-query as you build </span> </br>
+                    </div>
+                    </br>
+                    <div class="intro-param" id="help">
+                      <div class="help-title">Help</div>
+                      <span> If you find errors, spelling mistakes, faulty pieces of code or want to improve the querybuilder, please submit a pull request or create an <a href="https://github.com/vespa-engine/vespa/issues">issue</a>.</span> </br>
+                    </div>
+                  </div>
                   </div>
               </div>
             </div>
@@ -174,7 +198,6 @@
 
       <script language="javascript" type="text/javascript" src="/querybuilder/editarea/edit_area/edit_area_full.js"></script>
       <script language="javascript" type="text/javascript">
-
       function fEALoaded() {
         $('#frame_'+window.yqlID).contents().find('.area_toolbar').hide();
         var iframe = document.getElementById("frame_"+window.yqlID);
@@ -184,50 +207,31 @@
 
 
 
-
-
       <SCRIPT language="javascript">
-
-        const CONFIG = $.getJSON('/querybuilder/gui_variables.json', function(data){window.CONFIG = data;});
+        const CONFIG = $.getJSON('/querybuilder/config.json', function(data){window.CONFIG = data;});
         method = "POST";
       	var number = 0;
       	var childno = {};
       	var json = JSON.parse("{}");
         var searchApiReference = null;
-
-      	var possible = ["yql", "hits", "offset", "queryProfile", "nocache", "groupingSessionCache", "searchChain", "timeout", "trace","tracelevel","traceLevel", "",
-      , "model", "ranking", "collapse","collapsesize","collapsesize","presentation", "pos", "streaming", "rules", "recall", "user", "nocachewrite", "metrics"];
-
+        var possible = null;
       	var usedProps = [];
       	var removedIndexes = [0];
-      	var childrenProps = {
-      		"model" : ["defaultIndex", "encoding", "language", "queryString", "restrict", "searchPath", "sources", "type"],
-      		"ranking" : ["location", "features", "listFeatures", "profile", "properties", "sorting", "freshness", "queryCache", "matchPhase"],
-      		"ranking.matchPhase" : ["maxHits", "attribute", "ascending", "diversity"],
-      		"ranking.matchPhase.diversity" : ["attribute", "minGroups"],
-      		"presentation" : ["bolding", "format", "summary", "template", "timing"],
-          "trace" : ["timestamps"],
-          "tracelevel" : ["rules"],
-          "metrics" : ["ignore"],
-      		"collapse":["summary"],
-      		"pos" : ["ll", "radius", "bb", "attribute"],
-      		"streaming" : ["userid", "groupname", "selection", "priority", "maxbucketspervisitor"],
-      		"rules" : ["off", "rulebase"]
-      	};
-
+        var childrenProps = null;
         window.onload = function() {
-          // Adding variables from configuration file
-          if (window.CONFIG.hasOwnProperty("featurename")){
-            childrenProps["ranking.features"] = window.CONFIG.featurename;
-          }
-          if (window.CONFIG.hasOwnProperty("propertyname")){
-            childrenProps["ranking.properties"] = window.CONFIG.propertyname;
-          }
-
-          addNewRow();
-          getSearchApiReference();
+          setTimeout(function(){
+            possible = window.CONFIG.levelZeroParameters;
+            childrenProps = window.CONFIG.childMap;
+            if (window.CONFIG.hasOwnProperty("ranking_features")){
+              childrenProps["ranking.features"] = window.CONFIG.ranking_features;
+            }
+            if (window.CONFIG.hasOwnProperty("ranking_properties")){
+              childrenProps["ranking.properties"] = window.CONFIG.ranking_properties;
+            }
+            addNewRow();
+            getSearchApiReference();
+          }, 250);
         };
-
         var stringType = ["yql", "queryProfile", "searchChain", "model.defaultIndex", "model.encoding", "model.language",
          "model.queryString", "model.searchPath", "model.type", "ranking.features", "ranking.profile", "ranking.properties", "ranking.sorting", "ranking.matchPhase.diversity.attribute",
          "ranking.matchPhase.attribute","pos.attribute", "presentation.summary", "collapse.summary",
@@ -243,6 +247,19 @@
         var yqlID = "v1";
 
 
+
+      function toggleHelp(){
+        var div = document.getElementById("help");
+        var buttonDiv = document.getElementById("helpbutton");
+        if (div.style.display === "none") {
+            div.style.display = "block";
+            buttonDiv.style.backgroundColor = '#4EA2D6';
+        } else {
+            div.style.display = "none";
+            buttonDiv.style.backgroundColor = 'transparent';
+        }
+      }
+
       function updateFields(){
       	var temp = number;
       	while (temp > 0){
@@ -260,7 +277,6 @@
       		temp -= 1;
       	}
       }
-
       function pasteJSON(){
         var button = document.getElementById("pasteJSON");
         if (button.innerHTML === "Press CMD+V"){
@@ -275,31 +291,24 @@
           document.getElementById('response').addEventListener('paste', handlePaste);
         }
       }
-
       function handlePaste (e) {
           var clipboardData, pastedData;
-
           // Stop data actually being pasted into div
           e.stopPropagation();
           e.preventDefault();
-
           // Get pasted data via clipboard API
           clipboardData = e.clipboardData || window.clipboardData;
           pastedData = clipboardData.getData('Text');
-
           // Do whatever with pasteddata
           alert("Converting JSON: \n\n "+pastedData);
-
           //Removing eventlistener
           var oldResponse = document.getElementById('response');
           var newResponse = oldResponse.cloneNode(true);
           oldResponse.parentNode.replaceChild(newResponse, oldResponse);
-
           //Returning to old button and
           pasteJSON();
           convertPastedJSON(pastedData);
       }
-
       function convertPastedJSON(pastedData){
         try {
           document.getElementById("request").innerHTML = "</br> <div class=\"intro-param\">Construct a query by adding parameters or pasting a JSON.</div>";
@@ -311,7 +320,6 @@
           alert("Could not parse JSON, with error-message: \n\n"+err.message);
         }
       }
-
       function buildFromJSON(obj,parentNo) {
         Object.keys(obj).forEach(function(key) {
           if ( typeof(eval('obj.'+key)) === "object" ) {
@@ -334,7 +342,6 @@
           }
         });
       }
-
       function copyURL(){
         generateJSON();
         var urlMap = dotNotate(window.json);
@@ -346,7 +353,6 @@
         document.body.removeChild(el);
         return url;
       }
-
       function buildURL(map){
         var url = document.getElementById("url").value + "?";
         var parameters = "";
@@ -355,19 +361,15 @@
         });
         parameters = parameters.substr(1)
         return url+parameters
-
       }
-
       function encodeToURI(string){
         string = encodeURIComponent(string);
         string = string.replace(/%20/g, '+');
         return string;
       }
-
         function dotNotate(obj,target,prefix) {
           target = target || {},
           prefix = prefix || "";
-
           Object.keys(obj).forEach(function(key) {
             if ( typeof(obj[key]) === "object" ) {
               dotNotate(obj[key],target,prefix + key + ".");
@@ -375,22 +377,18 @@
               return target[prefix + key] = obj[key];
             }
           });
-
           return target;
         }
-
       function copyToClipboard(id){
         id.select();
         document.execCommand('copy');
         clearSelection();
       }
-
       function clearSelection()
       {
        if (window.getSelection) {window.getSelection().removeAllRanges();}
        else if (document.selection) {document.selection.empty();}
       }
-
       function chooseMethod(method) {
       	var selectBox = document.getElementById("method");
       	var selectedMethod = selectBox.options[selectBox.selectedIndex].value;
@@ -399,7 +397,6 @@
       		changeDiv(selectedMethod);
       	}
       }
-
       function changeDiv(selectedMethod){
       	if (selectedMethod === "GET"){
       		document.getElementById("request").innerHTML = '</br><textarea class=\"responsebox\" cols=70 rows=4 id=\"url2\">'+copyURL();+'</textarea>';
@@ -410,12 +407,11 @@
       		number = 0;
         	changeVisibility();
           let jsonString = JSON.stringify(window.json);
-          if (jsonString === "{}" || jsonString=== '{"":""}'){
+          if (jsonString === "{}" || jsonString=== '{"":""}'){
         		addNewRow();
           } else {
             buildFromJSON(window.json, 0);
           }
-
       	}
       }
       function stripMyHTML(html)
@@ -427,7 +423,6 @@
          }
          return tmp.innerHTML.replace('<code>','').replace('</code>','');
       }
-
       function addNewRow(key, value){
       	number += 1;
       	const temp = number;
@@ -443,6 +438,10 @@
       	newInputVal.type = "text";
       	newInputVal.id = "v"+temp;
         newInputVal.classList.add("propvalue");
+        newInputVal.setAttribute("list", "val"+temp);
+        var newDatalist2 = document.createElement("datalist");
+      	newDatalist2.id = "val"+temp;
+        //newDatalist2.style = "display: none;";
       	var newButton = document.createElement("button");
       	newButton.id = "b"+temp;
       	newButton.innerHTML = "-";
@@ -450,37 +449,29 @@
         newButton.classList.add("removeRow");
       	var br = document.createElement("br");
       	br.id = "br"+temp
-
         var img = document.createElement("img");
         img.src = "/querybuilder/img/information.svg";
         img.height = "15";
         img.width = "15";
         img.classList.add("information");
-
         var span = document.createElement("span");
         span.id = "span"+temp;
         span.innerHTML = stripMyHTML('Choose a parameter for information');
-
         var a = document.createElement("a");
         a.href = "#";
         a.classList.add("tip");
         a.id = "inf"+temp
-
         a.appendChild(img);
         a.appendChild(span);
-
-
       	var div = document.createElement("div");
       	div.id = number;
       	div.appendChild(newInput);
       	div.appendChild(newDatalist);
-
         div.appendChild(a);
-
       	div.appendChild(newInputVal);
+        div.appendChild(newDatalist2);
       	div.appendChild(newButton);
       	div.appendChild(br);
-
       	var bigdiv = document.getElementById("request");
       	bigdiv.appendChild(div);
       	updateFields();
@@ -489,7 +480,7 @@
           newInput.value = key;
           keySelected(temp, value)
         }
-        if (value || value2==="false"){
+        if (value || value2==="false"){
           if (key === "yql"){
             editAreaLoader.setValue(window.yqlID, value);
           } else {
@@ -499,7 +490,6 @@
         generateJSON();
         return temp;
       }
-
       function showInformation(no, key){
         var a = document.getElementById("inf"+no);
         if(validKey(no, key)){
@@ -514,7 +504,6 @@
         a.style = "visibility: hidden;"
         }
       }
-
       function validKey(no, possibleKey){
         if (contains(possible, possibleKey)){return true;}
         for (var key in childrenProps){
@@ -524,7 +513,19 @@
         }
         return false;
       }
-
+      function checkConfigOptions(key, no){
+        var jsonID = key.replace(/\./g , "_");
+        var datalist = document.getElementById("val"+no);
+        datalist.innerHTML = "";
+        if (window.CONFIG.hasOwnProperty(jsonID)){
+          var optionlist = eval("window.CONFIG."+jsonID);
+          optionlist.forEach(function(item){
+            var option = document.createElement("option");
+            option.value = item;
+            datalist.appendChild(option);
+          });
+        }
+      }
       	function keySelected(no, value){
       		var key = document.getElementById("i"+no).value;
           showInformation(no, key);
@@ -536,6 +537,8 @@
               editAreaLoader.delete_instance(window.yqlID);
             }
       			var button = document.createElement("button");
+            //var datalist = document.getElementById("val"+no); //Hide value-datalist
+            //datalist.style = "display:none;";
       			button.id="propb"+no
       			button.innerHTML=" +   Add property";
       			button.onclick = function(){addChildProp(no);};
@@ -549,6 +552,7 @@
       				newInputVal.type = "text";
       				newInputVal.id = "v"+no;
               newInputVal.classList.add("propvalue");
+              newInputVal.setAttribute("list", "val"+no);
               var parent = button.parentNode;
               showType(newInputVal, no);
               if (key === "yql"){
@@ -586,11 +590,11 @@
                 newInputVal.type = "text";
                 newInputVal.id = "v"+no;
                 newInputVal.classList.add("propvalue");
+                newInputVal.setAttribute("list", "val"+no);
                 showType(newInputVal, no);
                 var parent = inputval.parentNode;
                 parent.replaceChild(newInputVal, inputval);
               }
-
               if (key === "yql"){
                 newInputVal = document.createElement("textarea");
                 newInputVal.id = "v"+no;
@@ -619,7 +623,6 @@
               if(value){
                 inputval.value = value;
               }
-
             }
       		}
           if (!validKey(no, key)){
@@ -627,14 +630,17 @@
             //keyInput.style = "border-width: 1px; border-color: red;"
             var valueInput = document.getElementById("v"+no);
             valueInput.placeholder = "Possible invalid parameter";
+            //Removes possible options for for former parameter
+            var datalist = document.getElementById("val"+no);
+            datalist.innerHTML = "";
           }
           if (validKey(no, key)){
+            // Check if datalist should be visible and add options
+            checkConfigOptions(fullKey, no);
             var keyInput = document.getElementById("i"+no);
             //keyInput.style = "border-width: 0px;"
           }
-
       	}
-
       function showType(inputVal, no){
         var key = document.getElementById("i"+no).value;
         var key = getFullName(no, key);
@@ -663,7 +669,6 @@
           inputVal.placeholder = "";
         }
       }
-
       function removeRow(no){
       	var div = document.getElementById(no);
       	var key = document.getElementById("i"+no).value;
@@ -676,7 +681,6 @@
       	updateFields();
         generateJSON();
       }
-
       function removeChildRow(no){
       	var a = "" + no;
         var parentNo = a.split(".").slice(0, -1).join(".");
@@ -688,30 +692,29 @@
       	updateChildrenFields(parentNo);
         generateJSON();
       }
-
       function countDots(s1) {
           return ( s1.match( /\./g ) || [] ).length;
         }
-
       function addChildProp(no, key, value){
         generateJSON();
       	childno[no] += 1;
       	var temp = ""+no+"."+childno[no];
       	childno[temp] = 0;
       	var parentNode = document.getElementById(no);
-
       	var newInput = document.createElement("input");
       	newInput.id = "i"+temp;
       	newInput.setAttribute("list", "prop"+temp);
       	newInput.onchange = function(){keySelected(temp);};
         newInput.classList.add("input")
-
       	var newDatalist = document.createElement("datalist");
       	newDatalist.id = "prop"+temp;
       	newInputVal = document.createElement("input");
       	newInputVal.type = "text";
       	newInputVal.id = "v"+temp;
         newInputVal.classList.add("propvalue");
+        newInputVal.setAttribute("list", "val"+temp);
+        var newDatalist2 = document.createElement("datalist");
+        newDatalist2.id = "val"+temp;
       	var newButton = document.createElement("button");
       	newButton.id = "b"+temp;
       	newButton.innerHTML = "-";
@@ -725,13 +728,11 @@
       	b.innerHTML = ' ↳   ';
       	var margin = 20*(temp).length;
       	b.style = "padding-left:"+margin+"px;";
-
         var img = document.createElement("img");
         img.src = "/querybuilder/img/information.svg";
         img.height = "15";
         img.width = "15";
         img.classList.add("information");
-
         var span = document.createElement("span");
         span.id = "span"+temp;
         span.innerHTML = stripMyHTML('Choose a parameter for information');
@@ -746,6 +747,7 @@
       	div.appendChild(newDatalist);
         div.append(a);
       	div.appendChild(newInputVal);
+        div.appendChild(newDatalist2);
       	div.appendChild(newButton);
       	div.appendChild(br);
       	parentNode.appendChild(div);
@@ -760,7 +762,6 @@
         }
         return temp;
       }
-
       function updateChildrenFields(parentNo){
       	parentKey = document.getElementById("i"+parentNo).value;
       	var temp = parseInt(childno[parentNo]);
@@ -780,7 +781,6 @@
       		temp -= 1;
       	}
       }
-
       function contains(a, obj) {
           for (var i = 0; i < a.length; i++) {
               if (a[i] === obj) {
@@ -789,18 +789,15 @@
           }
           return false;
       }
-
       function refresh(){
       	document.location.reload(true);
       }
-
       function generateJSON(){
       	json = JSON.parse("{}");
       	buildJSON(json, number, 0);
         var textarea = document.getElementById("jsonquery");
         textarea.innerHTML = JSON.stringify(window.json, undefined, 4);
       }
-
       function buildJSON(parent, no, thresh){
       	var temp = no;
         if (countDots(""+temp) > 0){
@@ -824,7 +821,7 @@
                   value = false;
               }}
               var fullKey = getFullName(temp, key);
-              if (contains(integerType, fullKey) || contains(longType, fullKey)){
+              if (contains(integerType, fullKey) || contains(longType, fullKey)){
                 value = parseInt(value);
                 value = isNaN(value) ? 0 : value;
               }
@@ -855,13 +852,10 @@
           }
       	}
       }
-
       function findUsedProps(){
       	usedProps = [];
       	addUsedProps(window.number, 0);
-
       }
-
       function addUsedProps(no, thresh){
         var temp = no;
         if (countDots(""+temp) > 0){
@@ -895,7 +889,6 @@
           }
         }
       }
-
       function startSending(){
       	if (method === "POST"){
       		var url = document.getElementById("url").value;
@@ -912,8 +905,6 @@
           window.open(url);
       	}
       }
-
-
       function changeVisibility() {
           var x = document.getElementById("url");
       		var y = document.getElementById("addRow");
@@ -931,7 +922,6 @@
               a.style.display = "none";
           }
       }
-
       function showJSON(){
         var textarea = document.getElementById("jsonquery");
         var copyJSON = document.getElementById("copyJSON");
@@ -950,7 +940,6 @@
             copyURL.style.display = "none";
         }
       }
-
       function getSearchApiReference(){
         var div = document.getElementById("div");
         var object = document.createElement("object");
@@ -965,7 +954,6 @@
           }, 20);
         }, 150);
       }
-
       function changeInformation(no,key){
         if (key===""){return "Choose a parameter for information"}
         if (getFullName(no,key) in childrenProps){return "Add parameters under the parent ''" + getFullName(no,key) + "'"}
@@ -974,7 +962,6 @@
         var tabletext = stripMyHTML(ref.getElementById(refId).nextElementSibling.outerHTML);
         return tabletext;
       }
-
       function getFullName(no, key){
           var name = key;
           no = ""+no;
diff --git a/container-search-gui/src/main/resources/gui/_includes/search-api-reference.html b/container-search-gui/src/main/resources/gui/_includes/search-api-reference.html
deleted file mode 100644
index 195c9dc2663..00000000000
--- a/container-search-gui/src/main/resources/gui/_includes/search-api-reference.html
+++ /dev/null
@@ -1,1011 +0,0 @@
----
-# Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-title: "Vespa Search API reference"
----
-
-<p>
-All the search request parameters listed below can be set in query
-profiles. The first four blocks of properties are also modeled as
-query profile types. These types can be referred from query profiles
-(and inheriting types) to provide type checking on the parameters.
-</p><p>
-These parameters often have both a full name - which includes the
-path from the root query profile - and one or more abbreviated
-names. Both names can be used in search requests, while only full
-names can be used in query profiles. The full names are case
-sensitive, while the abbreviated names are case insensitive.
-</p><p>
-The parameters modeled as query profiles are also available through
-get methods as Java objects from the Query to Searcher components.
-</p>
-
-
-
-<h2>Index</h2>
-
-<dt>Query</dt>
-<dd>
-  <ul>
-    <li><a href="#yql">yql</a></li>
-  </ul>
-</dd>
-</dl>
-
-<dl>
-<dt>Native Execution Parameters</dt>
-<dd>
-  <ul>
-    <li><a href="#hits">hits</a> [<em>count</em>]</li>
-    <li><a href="#offset">offset </a>[<em>start</em>]</li>
-    <li><a href="#queryProfile">queryProfile</a></li>
-    <li><a href="#nocache">nocache</a></li>
-    <li><a href="#groupingSessionCache">groupingSessionCache</a></li>
-    <li><a href="#searchChain">searchChain</a></li>
-    <li><a href="#timeout">timeout</a></li>
-    <li><a href="#tracelevel">tracelevel</a></li>
-    <li><a href="#trace.timestamps">trace.timestamps</a></li>
-  </ul>
-</dd>
-
-<dt>Query Model Parameters</dt>
-<dd>
-  <ul>
-    <li><a href="#model.defaultIndex">model.defaultIndex</a> [<em>def-idx, default-index</em>]</li>
-    <li><a href="#model.encoding">model.encoding</a> [<em>encoding</em>]</li>
-    <li><a href="#model.filter">model.filter</a> [<em>filter</em>]</li>
-    <li><a href="#model.language">model.language</a> [<em>lang, language</em>]</li>
-    <li><a href="#model.queryString">model.queryString</a> [<em>query</em>]</li>
-    <li><a href="#model.restrict">model.restrict</a> [<em>restrict</em>]</li>
-    <li><a href="#model.searchPath">model.searchPath</a> [<em>path</em>]</li>
-    <li><a href="#model.sources">model.sources</a> [<em>search, sources</em>]</li>
-   <li><a href="#model.type">model.type</a> [<em>type</em>]</li>
-  </ul>
-</dd>
-
-<dt>Ranking</dt>
-<dd>
-  <ul>
-    <li><a href="#ranking.location">ranking.location</a> [<em>location</em>]</li>
-    <li><a href="#ranking.features">ranking.features</a> [<em>rankfeature</em>]</li>
-    <li><a href="#ranking.listFeatures">ranking.listFeatures</a> [<em>rankfeatures</em>]</li>
-    <li><a href="#ranking.profile">ranking.profile</a> [<em>ranking</em>]</li>
-    <li><a href="#ranking.properties">ranking.properties</a> [<em>rankproperty</em>]</li>
-    <li><a href="#ranking.sorting">ranking.sorting</a> [<em>sorting</em>]</li>
-    <li><a href="#ranking.freshness">ranking.freshness</a></li>
-    <li><a href="#ranking.queryCache">ranking.queryCache</a></li>
-    <li><a href="#ranking.matchPhase">ranking.matchPhase</a></li>
-  </ul>
-</dd>
-
-<dt>Presentation</dt>
-<dd>
-  <ul>
-    <li><a href="#presentation.bolding">presentation.bolding</a> [<em>bolding</em>]</li>
-    <li><a href="#presentation.format">presentation.format</a> [<em>format</em>]</li>
-    <li><a href="#presentation.template">presentation.template</a></li>
-    <li><a href="#presentation.summary">presentation.summary</a> [<em>summary</em>]</li>
-    <li><a href="#presentation.timing">presentation.timing</a></li>
-  </ul>
-</dd>
-
-<dt>Grouping</dt>
-<dd>
-  <ul>
-    <li><a href="#select">select</a></li>
-    <li><a href="#collapse.summary">collapse.summary</a></li>
-    <li><a href="#collapsefield">collapsefield</a></li>
-    <li><a href="#collapsesize">collapsesize</a></li>
-  </ul>
-</dd>
-
-<dt>Geographical Searches</dt>
-<dd>
-  <ul>
-    <li><a href="#pos.ll">pos.ll</a></li>
-    <li><a href="#pos.radius">pos.radius</a>,</li>
-    <li><a href="#pos.attribute">pos.attribute</a></li>
-    <li><a href="#pos.bb">pos.bb</a></li>
-  </ul>
-</dd>
-
-<dt>Streaming Search</dt>
-<dd>
-  <ul>
-    <li><a href="#streaming.userid">streaming.userid</a></li>
-    <li><a href="#streaming.groupname">streaming.groupname</a></li>
-    <li><a href="#streaming.selection">streaming.selection</a></li>
-    <li><a href="#streaming.priority">streaming.priority</a></li>
-    <li><a href="#streaming.maxbucketspervisitor">streaming.maxbucketspervisitor</a></li>
-  </ul>
-</dd>
-
-<dt>Semantic Rules</dt>
-<dd>
-  <ul>
-    <li><a href="#rules.off">rules.off</a></li>
-    <li><a href="#rules.rulebase">rules.rulebase</a></li>
-    <li><a href="#tracelevel.rules">tracelevel.rules</a></li>
-  </ul>
-</dd>
-
-<dt>Other</dt>
-<dd>
-  <ul>
-    <li><a href="#recall">recall</a></li>
-    <li><a href="#user">user</a></li>
-    <li><a href="#nocachewrite">nocachewrite</a></li>
-    <li><a href="#hitcountestimate">hitcountestimate</a></li>
-    <li><a href="#metrics.ignore">metrics.ignore</a></li>
-  </ul>
-</dd>
-</dl>
-
-
-<h2 id="query">Query</h2>
-<h3 id="yql">yql</h3>
-<table class="table table-striped">
-<tr><td>Alias</td><td></td></tr>
-<tr><td>Values</td><td>String</td></tr>
-<tr><td>Default</td><td>None</td></tr>
-</table>
-<p>
-The YQL query will be parsed and executed in the backend.
-Only simple YQL programs are supported, refer to
-<a href="../query-language.html">YQL</a> for details.
-</p>
-
-
-
-<h2 id="native-execution-parameters">Native Execution Parameters</h2>
-<p>
-These parameters are defined in the <code>native</code> query profile type.
-</p>
-
-
-<h3 id="hits">hits</h3>
-<table class="table table-striped">
-<tr><td>Alias</td><td>count</td></tr>
-<tr><td>Values</td>
-    <td>
-      A positive integer, or 0. The sum of <a href="#offset">offset</a> and
-      <a href="#hits">hits</a> should be lower than the configured maxoffset
-      value, and will be adjusted to fit. See also comment
-      at <code>offset</code>.
-    </td>
-</tr>
-<tr><td>Default</td><td>10</td></tr>
-</table>
-<p>
-The maximum number of hits to return from the result set.
-Must be lower than <code>maxHits</code>, which is either set in a
-<a href="#queryProfile">query profile</a>, or default 400.
-<!-- ToDo: link to def file or code where this is definied -->
-</p>
-
-
-<h3 id="offset">offset</h3>
-<table class="table table-striped">
-  <tr><td>Alias</td><td>start</td></tr>
-  <tr><td>Values</td>
-    <td>
-      A positive integer, including 0.
-    </td>
-  </tr>
-  <tr><td>Default</td><td>0</td></tr>
-</table>
-<p>
-The index of the first hit to return from the result set.
-Must be lower than <code>maxOffset</code>, which is either set in a
-<a href="#queryProfile">query profile</a>, or default 1000.
-<!-- ToDo: link to def file or code where this is definied -->
-</p>
-
-
-<h3 id="queryProfile">queryProfile</h3>
-<table class="table table-striped">
-  <tr><td>Alias</td><td><em>None</em></td></tr>
-  <tr><td>Values</td>
-    <td>
-      A query profile id  -  name:version, where version can be omitted
-      or partially specified, e.g "myprofile:2.1"
-    </td>
-  </tr>
-  <tr><td>Default</td><td><code>default</code></td></tr>
-</table>
-<p>
-A <a href="../query-profiles.html">query profile</a> has default properties for a query.
-The default query profile is named <em>default</em> - example:
-<pre>
-&lt;query-profile id="default"&gt;
-  &lt;field name="maxHits"&gt;10&lt;/field&gt;
-  &lt;field name="maxOffset"&gt;1000&lt;/field&gt;
-&lt;/query-profile&gt;
-</pre>
-</p>
-
-
-<h3 id="nocache">nocache</h3>
-<table class="table table-striped">
-<tr><td>Alias</td><td></td></tr>
-<tr><td>Values</td>
-    <td>
-      True or false
-    </td>
-</tr>
-<tr><td>Default</td><td>false</td></tr>
-</table>
-<p>
-Set to true to avoid the result being fetched from cache, and avoid
-writing the result to cache after fetching it.
-</p>
-
-
-<h3 id="groupingSessionCache">groupingSessionCache</h3>
-<table class="table table-striped">
-<tr><td>Alias</td><td></td></tr>
-<tr><td>Values</td>
-    <td>
-      True or false
-    </td>
-</tr>
-<tr><td>Default</td><td>false</td></tr>
-</table>
-<p>
-Set to true to store intermediate grouping results in the search back ends when
-using multi level grouping expressions in order to speed up grouping at a
-potential loss of accuracy. See the <a
-href="grouping-syntax.html#sessionCache">grouping reference</a> for more
-details.
-</p>
-
-
-<h3 id="searchChain">searchChain</h3>
-<table class="table table-striped">
-<tr><td>Alias</td><td></td></tr>
-<tr><td>Values</td>
-    <td>
-      A search chain id  -  name:version, where version can be
-      omitted or partially specified, e.g "mychain:2.1.3".
-    </td>
-</tr>
-<tr><td>Default</td><td><code>default</code></td></tr>
-</table>
-<p>
-The search chain initially invoked when processing this query. This
-search chain may invoke other chains.
-</p>
-
-
-<h3 id="timeout">timeout</h3>
-<table class="table table-striped">
-<tr><td>Alias</td><td></td></tr>
-<tr><td>Values</td>
-    <td>
-      Positive floating point number with an optional unit. Default unit
-      is seconds (s), valid unit strings are e.g. <em>ms</em> and <em>s</em>. To set
-      a timeout of one minute, the argument could be set to <em>60 s</em>.
-      Space between the number and the unit is optional.
-    </td>
-</tr>
-<tr><td>Default</td><td>Undefined, but guaranteed to be at least 5000 milliseconds. This default can be overridden by configuring timeout in a <a href="../query-profiles.html">query profile.</a></td></tr>
-</table>
-<p>The query timeout.</p>
-
-
-<h3 id="tracelevel">tracelevel</h3>
-<table class="table table-striped">
-<tr><td>Alias</td><td></td></tr>
-<tr><td>Values</td>
-    <td>
-      Any positive number
-    </td>
-</tr>
-<tr><td>Default</td><td><em>No tracing</em></td></tr>
-</table>
-<p>
-Set to a positive number to collect trace information for debugging
-when running a query. Higher numbers give
-progressively more detail on query transformations and searcher
-execution.
-</p>
-
-<h3 id="trace.timestamps">trace.timestamps</h3>
-<table class="table table-striped">
-<tr><td>Alias</td><td></td></tr>
-<tr><td>Values</td>
-    <td>
-      true or false
-    </td>
-</tr>
-<tr><td>Default</td><td><em>No timestamps in trace</em></td></tr>
-</table>
-<p>
-Enable it to get timing information already at <a href="#tracelevel">tracelevel=1</a> which is useful for debugging latency spent at different components in the search chain without rendering a lot of string data which is associated with higher trace levels.
-</p>
-
-
-
-<h2 id="query-model">Query Model Parameters</h2>
-
-<h3 id="model.defaultIndex">model.defaultIndex [<em>def-idx, default-index</em>]</h3>
-<table class="table table-striped">
-<tr><td>Alias</td><td>def-idx, default-index</td></tr>
-<tr><td>Values</td><td>An index name</td></tr>
-<tr><td>Default</td><td><code>default</code></td></tr>
-</table>
-<p>
-The field which is searched for query terms which doesn't explicitly specify an index.
-</p>
-
-
-<h3 id="model.encoding">model.encoding [<em>encoding</em>]</h3>
-<table class="table table-striped">
-<tr><td>Alias</td><td>encoding</td></tr>
-<tr><td>Values</td><td>Encoding names or aliases defined in the <a href="http://www.iana.org/assignments/character-sets">IANA character sets</a></td></tr>
-<tr><td>Default</td><td>utf-8</td></tr>
-</table>
-<p> Sets the encoding to use when returning a result. The encodings <em>big5</em>,
-<em>euc-jp</em>, <em>euc-kr</em>, <em>gb2312</em>, <em>iso-2022-jp</em> and <em>shift-jis</em>
-also influences how tokenization is done in the absence of an explicit language setting.
-</p><p>
-The query is always encoded as UTF-8, independently of how the result will be encoded.
-</p>
-
-
-<h3 id="model.filter">model.filter [<em>filter</em>]</h3>
-<table class="table table-striped">
-<tr><td>Alias</td><td>filter</td></tr>
-<tr><td>Values</td><td>Any allowed collection of filter terms</td></tr>
-<tr><td>Default</td><td><em>Not set</em></td></tr>
-</table>
-<p>
-Sets a filter to be combined with the query. Typical use of a filter
-is to add machine generated or preferences based filter terms to a raw
-user query. The filter is parsed the same way as a query of type any,
-the full syntax is available. The positive terms (preceded by +) and
-phrases act as AND filters, the negative terms (preceded by -) act as
-NOT filters, while the unprefixed terms will be used to RANK the
-results. Unless the query has no positive terms, the filter will only
-restrict and influence ranking of the result set, never cause more
-matches than the query.
-</p>
-
-
-<h3 id="model.language">model.language [<em>lang, language</em>]</h3>
-<table class="table table-striped">
-<tr><td>Alias</td><td>language, lang</td></tr>
-<tr><td>Values</td><td>Ref. RFC 3066</td></tr>
-<tr><td>Default</td><td></em>Unspecified</em></td></tr>
-</table>
-<p>
-  Informs Vespa about the natural language of the query. Please see
-  <a href="../linguistics.html">linguistics</a> for details.
-  This attribute should always be set when it is known. If this
-  parameter is not set, it will be guessed from the query and encoding, and
-  default to english if it cannot be guessed.
-</p>
-
-
-<h3 id="model.queryString">model.queryString [<em>query</em>]</h3>
-<table class="table table-striped">
-<tr><td>Alias</td><td>query</td></tr>
-<tr><td>Values</td><td>Any HTTP encoded legal Vespa query language string</td></tr>
-<tr><td>Default</td><td><em>Not set</em></td></tr>
-</table>
-<p>
-The <a href="simple-query-language-reference.html">Simple Vespa Query Language</a> query string
-specifying which documents to match in this query.
-</p>
-
-
-<h3 id="model.restrict">model.restrict [<em>restrict</em>]</h3>
-<table class="table table-striped">
-<tr><td>Alias</td><td>restrict</td></tr>
-<tr><td>Values</td><td>A comma delimited list of document type names.</td></tr>
-<tr><td>Default</td><td><em>Search unrestricted</em></td></tr>
-</table>
-<p>
-The document types to restrict the search to when different document
-types share the same search cluster.
-</p>
-
-
-<h3 id="model.searchPath">model.searchPath [<em>path</em>]</h3>
-<table class="table table-striped">
-<tr><td>Alias</td><td>searchpath</td></tr>
-<tr><td>Values</td><td><ul>
-      <li>searchpath::ELEMENT [';' ELEMENT]*</li>
-      <li>ELEMENT::PART ['/' ROW]</li>
-      <li>PART::EXP [',' EXP]*</li>
-      <li>EXP::NUM | RANGE</li>
-      <li>ROW::NUM</li>
-      <li>RANGE::'['NUM ',' NUM ' &gt;'</li>
-  </ul></td></tr>
-<tr><td>Default</td><td><em>Whole cluster</em></td></tr>
-</table>
-<p>
-Specification of which path to send the query to.
-Used to select which set of search nodes in the cluster should be used.
-Only meant for debugging/monitoring.
-</p><p>
-Examples:
-Note that in an indexed content cluster with flat distribution we have 1 implicit row
-and each search node represents a part.
-<ul>
-  <li>'7/3' = part 7, row 3.</li>
-  <li>'7/' = part 7, any row.</li>
-  <li>'7,1,9/0' = parts 1,7 and 9, row 0.</li>
-  <li>'1,[3,9&gt;/0' = parts 1,3,4,5,6,7,8, row 0.</li>
-</ul>
-</p><p>
-In a cluster with a multi-level dispatch setup we must specify a search path element for each level.
-Lets say we have a setup with 2 mid-level dispatch groups, each containing 3 search nodes (and 3 dispatchers):
-<ul>
-  <li>'0/;2/' = dispatch group (part) 0, any of the dispatchers (row); search node (part) 2, any row (of 1 present).</li>
-  <li>'0/1;2/0' = dispatch group (part) 0, dispatcher (row) 1; search node (part) 2, row 0 (of 1 present).</li>
-</ul>
-</p>
-
-
-<h3 id="model.sources">model.sources [<em>search, sources</em>]</h3>
-<table class="table table-striped">
-<tr><td>Alias</td><td>search, sources</td></tr>
-<tr><td>Values</td><td>A comma separated list of search cluster names or other source names</td></tr>
-<tr><td>Default</td><td><em>Search unrestricted</em></td></tr>
-</table>
-<p>
-The names of the sources to search, e.g one or more search clusters and/or federated sources.
-</p>
-
-
-<h3 id="model.type">model.type [<em>type</em>]</h3>
-<table class="table table-striped">
-<tr><td>Alias</td><td>type</td></tr>
-<tr><td>Values</td><td>web, all, any, phrase, yql, adv (deprecated) -
-  refer to <a href="simple-query-language-reference.html">simple query language reference</a></td></tr>
-<tr><td>Default</td><td>all</td></tr>
-</table>
-<p>
-Selects the query language syntax of the <a href="#model.queryString">query</a> parameter.
-</p>
-
-
-
-<h2 id="ranking">Ranking</h2>
-
-<h3 id="ranking.location">ranking.location [<em>location</em>]</h3>
-<table class="table table-striped">
-<tr><td>Alias</td><td>location</td></tr>
-<tr><td>Values</td><td>See <a href="../geo-search.html">Geo search</a></td></tr>
-<tr><td>Default</td><td>None</td></tr>
-</table>
-<p>
-Point (one or two dimensional) location to use as base for location ranking.
-For geographical locations, it is recommended to add the location using <a href="#pos.ll">pos.ll</a>
-<!-- ToDo: Why? -->
-</p>
-
-
-<h3 id="ranking.features">ranking.features.<em>featurename</em> [<em>rankfeature.</em>featurename]</h3>
-<table class="table table-striped">
-<tr><td>Alias</td><td>rankfeature.featurename</td></tr>
-<tr><td>Values</td><td>Any string</td></tr>
-<tr><td>Default</td><td>None</td></tr>
-</table>
-<p>
-Set a rank feature to a value. This works for any key name <code>query(anyname)</code> (query features),
-and also as a way to override all existing (match and document) features.
-Example: <em>query=foo&amp;ranking.features.query(userage)=42&amp;ranking.features.fieldMatch(title)=0.65</em>
-</p>
-
-
-<h3 id="ranking.listFeatures">ranking.listFeatures [<em>rankfeatures</em>]</h3>
-<table class="table table-striped">
-<tr><td>Alias</td><td>rankfeatures</td></tr>
-<tr><td>Values</td><td>boolean</td></tr>
-<tr><td>Default</td><td>false</td></tr>
-</table>
-<p>
-Set to true to request <em>all</em> rank features to be calculated and returned.
-The rank features will be returned in the summary field rankfeatures.
-This option is typically used for MLR training, should not to be used for production.
-</p>
-
-
-<h3 id="ranking.profile">ranking.profile [<em>ranking</em>]</h3>
-<table class="table table-striped">
-<tr><td>Alias</td><td>ranking</td></tr>
-<tr><td>Values</td><td>Any rank profile name</td></tr>
-<tr><td>Default</td><td><code>default</code></td></tr>
-</table>
-<p>
-Sets the name of the rank profile to use for assigning relevancy scores.
-The default rank profile will be used for back-ends which does not have the given rank profile.
-</p>
-
-
-<h3 id="ranking.properties">ranking.properties.<em>propertyname</em> [<em>rankproperty.propertyname</em>]</h3>
-<table class="table table-striped">
-<tr><td>Alias</td><td>rankproperty.propertyname</td></tr>
-<tr><td>Values</td><td>Any string</td></tr>
-<tr><td>Default</td><td><em>None</em></td></tr>
-</table>
-<p>
-Set a rank property that is passed to, and used by a feature executor for this query.
-Example: <em>query=foo&amp;ranking.properties.dotProduct.X={a:1,b:2}</em>
-</p>
-
-
-<h3 id="ranking.sorting">ranking.sorting [<em>sorting</em>]</h3>
-<table class="table table-striped">
-<tr><td>Alias</td><td>sorting</td></tr>
-<tr><td>Values</td><td>A valid <a href="sorting.html">sort specification</a></td></tr>
-<tr><td>Default</td><td>None - order by relevance</td></tr>
-</table>
-<p>
-A specification of how to sort the result.
-Fields you want to sort on must be stored as document attributes in the index structure
-by adding <a href="search-definitions-reference.html#attribute">attribute</a> to the indexing statement.
-</p>
-
-
-<h3 id="ranking.freshness">ranking.freshness</h3>
-<table class="table table-striped">
-<tr><td>Alias</td><td></td></tr>
-<tr><td>Values</td><td><code>[integer]</code>, an absolute time in seconds since epoch, or <code>now-[number]</code>, to use a time [integer] seconds into the past, or <code>now</code> to use the current time</td></tr>
-<tr><td>Default</td><td>None - use the current time on each node.</td></tr>
-</table>
-<p>
-Sets the time which will be used as <em>now</em> during execution.
-</p>
-
-
-<h3 id="ranking.queryCache">ranking.queryCache</h3>
-<table class="table table-striped">
-<tr><td>Alias</td><td></td></tr>
-<tr><td>Values</td><td>boolean</td></tr>
-<tr><td>Default</td><td>false</td></tr>
-</table>
-<p>
-Turns query cache on or off. Search is a two-phase process. If the
-query cache is on, the query is stored on the search nodes between the
-first and second phase, saving network bandwidth and also query setup
-time, at the expense of using more memory.
-</p>
-
-
-<h3 id="ranking.matchPhase">ranking.matchPhase</h3>
-<p>Settings which control Vespa's behavior during the match phase.
-If these are set in the query they will override any match-phase setting
-in the rank profile.</p>
-<dt></dt>
-<dd>
-  <ul>
-    <li><a href="#ranking.matchPhase.maxHits">ranking.matchPhase.maxHits</a> the max number of hits that should be generated during the match phase</li>
-    <li><a href="#ranking.matchPhase.attribute">ranking.matchPhase.attribute</a> the attribute to limit matches by if more than maxHits hits will be generated</li>
-    <li><a href="#ranking.matchPhase.ascending">ranking.matchPhase.ascending</a> whether to keep the documents having the highest (default) or lowest values of the attribute</li>
-    <li><a href="#ranking.matchPhase.diversity.attribute">ranking.matchPhase.diversity.attribute</a> the attribute to use to guarantee diversity.</li>
-    <li><a href="#ranking.matchPhase.diversity.minGroups">ranking.matchPhase.diversity.minGroups</a> the minimum number of groups grouped by the diversity attribute.</li>
-  </ul>
-</dd>
-
-
-<h3 id="ranking.matchPhase.maxHits">ranking.matchPhase.maxHits</h3>
-<table class="table table-striped">
-<tr><td>Alias</td><td></td></tr>
-<tr><td>Values</td><td>long</td></tr>
-<tr><td>Default</td><td>If sorting and not ranking: max(10000, maxhits+maxoffset).
-  Otherwise: <em>none</em>.</td></tr>
-</table>
-<p>
-The max hits the engine should attempt to produce in the match phase on each partition.
-If it is determined during matching that many more hits than this will be generated, the matching will fall back to
-take the best (highest or lowest) values of the attribute given by ranking.matchPhase.attribute.
-</p><p>
-By default, this will be turned on only when sorting is used and grouping is not.
-If sorting is used, the primary sort attribute will be used as the match phase attribute if it has fast-search set.
-In that case the default can be overridden by setting this value explicitly.
-</p>
-
-
-<h3 id="ranking.matchPhase.attribute">ranking.matchPhase.attribute</h3>
-<table class="table table-striped">
-<tr><td>Alias</td><td></td></tr>
-<tr><td>Values</td><td>An attribute name</td></tr>
-<tr><td>Default</td><td><em>none</em></td></tr>
-</table>
-<p>
-The attribute to decide which documents are a match if the match phase
-estimates that there will be more than maxHits matches.
-This attribute should have fast-search set and should correlate with the order
-which would be produced by a full evaluation.
-</p>
-
-
-<h3 id="ranking.matchPhase.ascending">ranking.matchPhase.ascending</h3>
-<table class="table table-striped">
-<tr><td>Alias</td><td></td></tr>
-<tr><td>Values</td><td>boolean</td></tr>
-<tr><td>Default</td><td>false</td></tr>
-</table>
-<p>
-Whether the attribute should be sorted in ascending or descending (default) order
-to determine which documents to keep as matches.
-</p>
-
-
-<h3 id="ranking.matchPhase.diversity.attribute">ranking.matchPhase.diversity.attribute</h3>
-<table class="table table-striped">
-<tr><td>Alias</td><td></td></tr>
-<tr><td>Values</td><td>An attribute name</td></tr>
-<tr><td>Default</td><td>none.</td></tr>
-</table>
-<p>
-The attribute to be used for producing the desired diversity.
-Also see <a href="search-definitions-reference.html#diversity-attribute">attribute</a>.
-</p>
-
-
-<h3 id="ranking.matchPhase.diversity.minGroups">ranking.matchPhase.diversity.minGroups</h3>
-<table class="table table-striped">
-<tr><td>Alias</td><td></td></tr>
-<tr><td>Values</td><td>long</td></tr>
-<tr><td>Default</td><td>none</td></tr>
-</table>
-<p>
-The minimum number of groups that should be returned from the match phase grouped by the diversity attribute.
-Also see <a href="search-definitions-reference.html#diversity-min-groups">min-groups</a>.
-</p>
-
-
-
-<h2 id="presentation">Presentation</h2>
-
-<h3 id="presentation.bolding">presentation.bolding [<em>bolding</em>]</h3>
-<table class="table table-striped">
-<tr><td>Alias</td><td>bolding</td></tr>
-<tr><td>Values</td><td>boolean</td></tr>
-<tr><td>Default</td><td>true</td></tr>
-</table>
-<p>
-Whether or not to bold search terms in <a href="search-definitions-reference.html">search definition</a>
-fields defined with <a href="search-definitions-reference.html#bolding">bolding: on</a>
-or <a href="search-definitions-reference.html#summary">summary: dynamic</a>.
-</p>
-
-
-<h3 id="presentation.format">presentation.format [<em>format</em>]</h3>
-<table class="table table-striped">
-<tr><td>Alias</td><td>format</td></tr>
-<tr><td>Values</td><td>
-  <table class="table table-striped">
-    <tr>
-      <td><em>No value</em> or <code><a href="default-result-format.html">default</a></code></td>
-      <td>The default, builtin JSON format</td>
-    </tr>
-    <tr>
-      <td><code><a href="default-result-format.html">json</a></code></td>
-      <td>Builtin JSON format</td>
-    </tr>
-    <tr>
-      <td><code>xml</code></td>
-      <td>Deprecated, builtin XML format</td>
-    </tr>
-    <tr>
-      <td><code><a href="page-result-format.html">page</a></code></td>
-      <td>Alternative deprecated XML format which is suitable for use with <a href="../page-templates.html">page templates</a>.</td>
-    </tr>
-    <tr>
-      <td><em>Any other value</em></td>
-      <td>A custom <a href="../result-rendering.html">result renderer</a> supplied by the application
-        </tr>
-  </table>
-
-</td></tr>
-<tr><td>Default</td><td>default</td></tr>
-</table>
-
-
-<h3 id="presentation.summary">presentation.summary [<em>summary</em>]</h3>
-<table class="table table-striped">
-<tr><td>Alias</td><td>summary</td></tr>
-<tr><td>Values</td><td>
-  The name of the <a href="../document-summaries.html#summary-classes-in-queries">summary class</a>
-  used to select fields in results.
-</td></tr>
-<tr><td>Default</td><td>The default summary class of the search definition.</td></tr>
-</table>
-
-
-<h3 id="presentation.template">presentation.template</h3>
-<table class="table table-striped">
-<tr><td>Alias</td><td></td></tr>
-<tr><td>Values</td><td>Any id specification of a deployed page template.</td></tr>
-<tr><td>Default</td><td></td></tr>
-</table>
-<p>
-The id of the page template to use for this result. This should be used with the
-<a href="page-result-format.html">page</a> result format.
-</p>
-
-
-<h3 id="presentation.timing">presentation.timing</h3>
-<table class="table table-striped">
-<tr><td>Alias</td><td></td></tr>
-<tr><td>Values</td><td>boolean</td></tr>
-<tr><td>Default</td><td>false</td></tr>
-</table>
-<p>
-Whether a result renderer should try to add optional timing information
-to the rendered page.
-</p>
-
-
-
-<h2 id="">Grouping and Aggregation</h2>
-
-<h3 id="select">select</h3>
-<table class="table table-striped">
-<tr><td>Alias</td><td></td></tr>
-<tr><td>Values</td><td>A valid grouping specification.</td></tr>
-<tr><td>Default</td><td>No grouping</td></tr>
-</table>
-<p>
-Requests specific multi-level result set statistics and/or hit groups to be returned in the result.
-Fields you want to retrieve statistics or hit groups for must be stored as document attributes
-in the index structure by adding attribute to the indexing statement.
-See the <a href="../grouping.html">grouping guide</a>.
-</p>
-
-
-<h3 id="collapsefield">collapsefield</h3>
-<table class="table table-striped">
-<tr><td>Alias</td><td></td></tr>
-<tr><td>Values</td><td>Any document summary field name</td></tr>
-<tr><td>Default</td><td>No field collapsing</td></tr>
-</table>
-<p>
-Collapse (i.e. aggregate) results using this field.
-Collapsing is run in the container, not content node level.
-Define a <em>collapsefield</em> to remove duplicates if the corpus has few duplicates -
-this is more efficient than using <a href="#select">grouping</a>.
-Otherwise, use grouping.
-</p>
-
-
-<h3 id="collapsesize">collapsesize</h3>
-<table class="table table-striped">
-<tr><td>Alias</td><td></td></tr>
-<tr><td>Values</td><td>A positive integer</td></tr>
-<tr><td>Default</td><td>1</td></tr>
-</table>
-<p>The number of hits to keep in each collapsed bucket</p>
-
-
-<h3 id="collapse.summary">collapse.summary</h3>
-<table class="table table-striped">
-  <tr><td>Alias</td><td></td></tr>
-  <tr><td>Values</td><td>A valid name of a document summary class.</td></tr>
-  <tr><td>Default</td><td>Use default summary or attributes.</td></tr>
-</table>
-<p>Use this summary class to fetch the field used for collapsing.</p>
-
-
-
-<h2 id="geographical-searches">Geographical Searches</h2>
-
-<h3 id="pos.ll">pos.ll</h3>
-<table class="table table-striped">
-<tr><td>Alias</td><td></td></tr>
-<tr><td>Values</td>
-  <td>Position given in latitude and longitude - example: <em>S22.4532;W123.9887</em>
-    Refer to <a href="search-definitions-reference.html#type:position">position field</a>
-    for format specification.</td>
-</tr>
-<tr><td>Default</td><td>None</td></tr>
-</table>
-
-
-<h3 id="pos.radius">pos.radius</h3>
-<table class="table table-striped">
-<tr><td>Alias</td><td></td></tr>
-<tr><td>Values</td><td>
-  Radius of the circle used for filtering. Valid units of measurement are km, m and mi. Examples:
-  <ul>
-    <li>pos.radius=100m</li>
-    <li>pos.radius=42mi</li>
-    <li>pos.radius=4km</li>
-  </ul>
-  One can also specify just a number (internal units, micro-degrees), but this is not recommended.
-</td></tr>
-<tr><td>Default</td><td>50km</td></tr>
-</table>
-
-
-<h3 id="pos.bb">pos.bb</h3>
-<table class="table table-striped">
-<tr><td>Alias</td><td></td></tr>
-<tr><td>Values</td><td>
-  Bounding box for positions, given as latitude and longitude boundaries.
-  The four boundaries must be specified as N, S, E, W, with degrees as
-  a decimal fraction.  Degrees south of equator or west of Greenwich are
-  input as negative numbers. Examples:
-  <ul>
-    <li>n=37.44899,s=37.3323,e=-121.98241,w=-122.06566</li>
-    <li>s=40.183868,w=-74.819519,n=40.248291,e=-74.728798</li>
-  </ul>
-</td></tr>
-<tr><td>Default</td><td>None</td></tr>
-</table>
-
-
-<h3 id="pos.attribute">pos.attribute</h3>
-<table class="table table-striped">
-  <tr><td>Alias</td><td></td></tr>
-  <tr><td>Values</td><td>Any attribute that has zcurve encoded positions as a long attribute.</td></tr>
-  <tr><td>Default</td><td>Random choice among the ones declared as position in the searchdefinition.</td></tr>
-</table>
-<p>
-Which attribute to use for the position. Can be both single- or multi-value.
-</p>
-
-
-
-<h2 id="">Streaming Search</h2>
-<p>
-The features in this section applies to <a href="../streaming-search.html">streaming search</a> only.
-</p>
-
-<h3 id="streaming.userid">streaming.userid</h3>
-<table class="table table-striped">
-<tr><td>Alias</td><td></td></tr>
-<tr><td>Values</td><td>An integer in decimal notation in the range [0, 2^64></td></tr>
-<tr><td>Default</td><td>None</td></tr>
-</table>
-<p>
-Restricts streaming search to only stream through documents with document ids having the n=&lt;number&gt;
-modifier and the userid part matches the supplied value. This can be used for grouping documents on a 64 bit integer.
-</p>
-
-
-<h3 id="streaming.groupname">streaming.groupname</h3>
-<table class="table table-striped">
-<tr><td>Alias</td><td></td></tr>
-<tr><td>Values</td><td>A string</td></tr>
-<tr><td>Default</td><td>None</td></tr>
-</table>
-<p>
-Restricts streaming search to only stream through documents with document ids having the g=&lt;groupname&gt;
-modifier and the groupname part matches the supplied value. This can be used for grouping documents on a string.
-</p>
-
-
-<h3 id="streaming.selection">streaming.selection</h3>
-<table class="table table-striped">
-<tr><td>Alias</td><td></td></tr>
-<tr><td>Values</td><td>A string</td></tr>
-<tr><td>Default</td><td>None</td></tr>
-</table>
-<p>
-Restricts streaming search using a <a href="document-select-language.html">document selection</a>.
-This can be used for selecting a subset of documents based on an advanced expression.
-</p>
-
-
-<h3 id="streaming.priority">streaming.priority</h3>
-<table class="table table-striped">
-<tr><td>Alias</td><td></td></tr>
-<tr><td>Values</td><td><a href="services.html#load-types">Priority class</a></td></tr>
-<tr><td>Default</td><td>VERY_HIGH</td></tr>
-</table>
-<p>
-Priority of the streaming search visitor. Having a high priority visitor helps maintain low latencies
-even when the system is under load.
-</p>
-
-
-<h3 id="streaming.maxbucketspervisitor">streaming.maxbucketspervisitor</h3>
-<table class="table table-striped">
-<tr><td>Alias</td><td></td></tr>
-<tr><td>Values</td><td>int</td></tr>
-<tr><td>Default</td><td>1 (if ordering is set), or infinite</td></tr>
-</table>
-<p>
-If set, visit only this many buckets at a time.
-Combine with ordering to reduce visiting time for large users/groups.
-</p>
-
-
-
-<h2 id="semantic-rules">Semantic Rules</h2>
-<p>
-Refer to <a href="semantic-rules.html">semantic rules</a>.
-</p>
-
-<h3 id="rules.off">rules.off</h3>
-<table class="table table-striped">
-<tr><td>Alias</td><td></td></tr>
-<tr><td>Values</td><td>Boolean</td></tr>
-<tr><td>Default</td><td>True</td></tr>
-</table>
-<p>Turn rule evaluation off for this query</p>
-
-
-<h3 id="rules.rulebase">rules.rulebase</h3>
-<table class="table table-striped">
-<tr><td>Alias</td><td></td></tr>
-<tr><td>Values</td><td>String</td></tr>
-<tr><td>Default</td><td>A rule base name</td></tr>
-</table>
-<p>The name of the rule base to use for these queries</p>
-
-
-<h3 id="tracelevel.rules">tracelevel.rules</h3>
-<table class="table table-striped">
-<tr><td>Alias</td><td></td></tr>
-<tr><td>Values</td><td>int</td></tr>
-<tr><td>Default</td><td>1-5 (?)</td></tr>
-</table>
-<p>
-The amount of rule evaluation trace output to show, higher number means more details.
-This is useful to see a trace from rule evaluation
-without having to see trace from all other searchers at the same time.
-</p>
-
-
-
-<h2 id="other">Other</h2>
-
-<h3 id="recall">recall</h3>
-<table class="table table-striped">
-<tr><td>Alias</td><td></td></tr>
-<tr><td>Values</td><td>Any allowed collection of recall terms</td></tr>
-<tr><td>Default</td><td>No recall</td></tr>
-</table>
-<p>
-Sets a recall parameter to be combined with the query.
-This is identical to <a href="#model.filter">filter</a>,
-except that recall terms are not exposed to the ranking framework and thus not ranked.
-As such, one can not use unprefixed terms; they must either by positive or negative.
-</p>
-
-
-<h3 id="user">user</h3>
-<table class="table table-striped">
-<tr><td>Alias</td><td></td></tr>
-<tr><td>Values</td><td>A string</td></tr>
-<tr><td>Default</td><td>None</td></tr>
-</table>
-<p>
-The id of the user making the query. The contents of the argument are made available to the search chain,
-but it triggers no features in Vespa apart from being propagated to the access log.
-</p>
-
-
-<h3 id="nocachewrite">nocachewrite</h3>
-<table class="table table-striped">
-<tr><td>Alias</td><td></td></tr>
-<tr><td>Values</td><td>Boolean</td></tr>
-<tr><td>Default</td><td>False</td></tr>
-</table>
-<p>Set to true to avoid the result being written to cache when fetched.</p>
-
-
-<h3 id="hitcountestimate">hitcountestimate</h3>
-<table class="table table-striped">
-<tr><td>Alias</td><td></td></tr>
-<tr><td>Values</td><td>Boolean</td></tr>
-<tr><td>Default</td><td>False</td></tr>
-</table>
-<p>Make this an estimation query.
-No hits will be returned, and total hit count will be set to an estimate of what executing
-the query as a normal query would give.
-</p>
-
-<h3 id="metrics.ignore">metrics.ignore</h3>
-<table class="table table-striped">
-<tr><td>Alias</td><td></td></tr>
-<tr><td>Values</td><td>Boolean</td></tr>
-<tr><td>Default</td><td>False</td></tr>
-</table>
-<p>Ignore metric collection for this query request, useful for warm up queries</p>
diff --git a/container-search-gui/src/main/resources/gui/editarea/edit_area/plugins/autocompletion/autocompletion.js b/container-search-gui/src/main/resources/gui/editarea/edit_area/plugins/autocompletion/autocompletion.js
index d6984f59899..5531a2a4f69 100644
--- a/container-search-gui/src/main/resources/gui/editarea/edit_area/plugins/autocompletion/autocompletion.js
+++ b/container-search-gui/src/main/resources/gui/editarea/edit_area/plugins/autocompletion/autocompletion.js
@@ -480,7 +480,10 @@ var EditArea_autocompletion= {
 				{

 					var line= "<li><a href=\"#\" class=\"entry\" onmousedown=\"EditArea_autocompletion._select('"+ results[i][1]['replace_with'].replace(new RegExp('"', "g"), "&quot;") +"');return false;\">"+ results[i][1]['comment'];

 					if(results[i][0]['prefix_name'].length>0)

-						line+='<span class="prefix">'+ results[i][0]['prefix_name'] +'</span>';

+						var value = results[i][0]['prefix_name'];

+						value = (value == undefined) ? "" : value;

+						if (value == 'SOURCES' || value == ','){value = 'Source'}

+						line+='<span class="prefix">'+ value +'</span>';

 					line+='</a></li>';

 					lines[lines.length]=line;

 				}

diff --git a/container-search-gui/src/main/resources/gui/editarea/edit_area/reg_syntax/yql.js b/container-search-gui/src/main/resources/gui/editarea/edit_area/reg_syntax/yql.js
index e4e9cd3b32d..0a7dae862f3 100755
--- a/container-search-gui/src/main/resources/gui/editarea/edit_area/reg_syntax/yql.js
+++ b/container-search-gui/src/main/resources/gui/editarea/edit_area/reg_syntax/yql.js
@@ -1,6 +1,19 @@
 /**

 * Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.

 */

+function getSources(bool){

+		var options = [];

+		if (window.CONFIG.hasOwnProperty("model_sources")){

+			var sources = window.CONFIG.model_sources;

+			for (i in sources){

+				option = [sources[i]];

+				options.push(option);

+			}

+			if (bool == true){options.push(["*"])}

+		}

+		return options;

+}

+

 editAreaLoader.load_syntax["yql"] = {

 	'DISPLAY_NAME' : 'YQL'

 	,'QUOTEMARKS' : {1: "'", 2: '"', 3: '`'}

@@ -74,9 +87,23 @@ editAreaLoader.load_syntax["yql"] = {
 						 ['DESC','DESC'],['ASC','ASC'],['ALL','ALL'], ['GROUP','GROUP'],

 						 ['RANGE','RANGE'],['EACH','EACH'],['OUTPUT','OUTPUT'],

 						 ['SUM','SUM'],['LIMIT','LIMIT'],['OFFSET','OFFSET'],

-						 ['TIMEOUT','TIMEOUT'],

-						]

+						 ['TIMEOUT','TIMEOUT']

+					 ]

+			}

+		},

+		"sources": {	// the name of this definition group. It's posisble to have different rules inside the same definition file

+			"REGEXP": { "before_word": "[^a-zA-Z0-9_]|^"	// \\s|\\.|

+						,"possible_words_letters": "[a-zA-Z0-9_]+"

+						,"letter_after_word_must_match": "[^a-zA-Z0-9_]|$"

+						,"prefix_separator": " "

+					}

+			,"CASE_SENSITIVE": false

+			,"MAX_TEXT_LENGTH": 50		// the maximum length of the text being analyzed before the cursor position

+			,"KEYWORDS": {

+					'SOURCES' : getSources(true) ,

+					',' : getSources(false)

 			}

 		}

+

 	}

 };

diff --git a/container-search-gui/src/main/resources/gui/gui_variables.json b/container-search-gui/src/main/resources/gui/gui_variables.json
deleted file mode 100644
index 1b5f411611a..00000000000
--- a/container-search-gui/src/main/resources/gui/gui_variables.json
+++ /dev/null
@@ -1,4 +0,0 @@
-{
-  "propertyname" : ["propertyname"],
-  "featurename" : ["featurename"]
-}
diff --git a/container-search-gui/src/main/resources/gui/img/features-help.png b/container-search-gui/src/main/resources/gui/img/features-help.png
new file mode 100644
index 00000000000..65702f8b91f
--- /dev/null
+++ b/container-search-gui/src/main/resources/gui/img/features-help.png
diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/CacheControl.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/CacheControl.java
index 33e7ad4bccc..dd5b3caf0c5 100644
--- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/CacheControl.java
+++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/CacheControl.java
@@ -18,7 +18,7 @@ import java.util.Optional;
  */
 public class CacheControl {
 
-    private static final CompoundName nocachewrite=new CompoundName("nocachewrite");
+    public static final CompoundName nocachewrite=new CompoundName("nocachewrite");
 
     /** Whether this CacheControl actually should cache hits at all. */
     private final boolean activeCache;
diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java
index 3a2e922f9e0..0d25e71dd76 100644
--- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java
+++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java
@@ -125,13 +125,15 @@ public class FastHit extends Hit {
      * found. The row count is used to decode the part id into a column and a
      * row number: the number of n least significant bits required to hold the
      * highest row number are the row bits, the rest are column bits.
+     *
+     * Note: Remove partId when all dispatching happens from the container dispatcher, not fdispatch
      */
     public void setPartId(int partId) { this.partId = partId; }
 
     /** Returns the index of the node this hit originated at */
     public int getDistributionKey() { return distributionKey; }
 
-    /** Returns the index of the node this hit originated at */
+    /** Sets the index of the node this hit originated at */
     public void setDistributionKey(int distributionKey) { this.distributionKey = distributionKey; }
 
     /**
diff --git a/container-search/src/main/java/com/yahoo/prelude/querytransform/RecallSearcher.java b/container-search/src/main/java/com/yahoo/prelude/querytransform/RecallSearcher.java
index 314ca38f092..060fd598245 100644
--- a/container-search/src/main/java/com/yahoo/prelude/querytransform/RecallSearcher.java
+++ b/container-search/src/main/java/com/yahoo/prelude/querytransform/RecallSearcher.java
@@ -35,7 +35,7 @@ import static com.yahoo.prelude.querytransform.StemmingSearcher.STEMMING;
 @Before({STEMMING, ACCENT_REMOVAL})
 public class RecallSearcher extends Searcher {
 
-    private static final CompoundName recallName=new CompoundName("recall");
+    public static final CompoundName recallName=new CompoundName("recall");
 
     @Override
     public Result search(Query query, Execution execution) {
diff --git a/container-search/src/main/java/com/yahoo/search/Query.java b/container-search/src/main/java/com/yahoo/search/Query.java
index dd3cc853742..cfb1c9a26be 100644
--- a/container-search/src/main/java/com/yahoo/search/Query.java
+++ b/container-search/src/main/java/com/yahoo/search/Query.java
@@ -20,6 +20,7 @@ import com.yahoo.search.query.Presentation;
 import com.yahoo.search.query.Properties;
 import com.yahoo.search.query.QueryTree;
 import com.yahoo.search.query.Ranking;
+import com.yahoo.search.query.Select;
 import com.yahoo.search.query.SessionId;
 import com.yahoo.search.query.Sorting;
 import com.yahoo.search.query.Sorting.AttributeSorter;
@@ -100,7 +101,8 @@ public class Query extends com.yahoo.processing.Request implements Cloneable {
         ADVANCED(3,"adv"),
         WEB(4,"web"),
         PROGRAMMATIC(5, "prog"),
-        YQL(6, "yql");
+        YQL(6, "yql"),
+        SELECT(7, "select");;
 
         private final int intValue;
         private final String stringValue;
@@ -170,6 +172,9 @@ public class Query extends com.yahoo.processing.Request implements Cloneable {
     /** How results of this query should be presented */
     private Presentation presentation = new Presentation(this);
 
+    /** The selection of where-clause and grouping */
+    private Select select = new Select(this);
+
     //---------------- Tracing ----------------------------------------------------
 
     private static Logger log = Logger.getLogger(Query.class.getName());
@@ -188,6 +193,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable {
     public static final CompoundName GROUPING_SESSION_CACHE = new CompoundName("groupingSessionCache");
     public static final CompoundName TIMEOUT = new CompoundName("timeout");
 
+
     private static QueryProfileType argumentType;
     static {
         argumentType = new QueryProfileType("native");
@@ -206,10 +212,12 @@ public class Query extends com.yahoo.processing.Request implements Cloneable {
         argumentType.addField(new FieldDescription(Presentation.PRESENTATION, new QueryProfileFieldType(Presentation.getArgumentType())));
         argumentType.addField(new FieldDescription(Ranking.RANKING, new QueryProfileFieldType(Ranking.getArgumentType())));
         argumentType.addField(new FieldDescription(Model.MODEL, new QueryProfileFieldType(Model.getArgumentType())));
+        argumentType.addField(new FieldDescription(Select.SELECT, new QueryProfileFieldType(Select.getArgumentType())));
         argumentType.freeze();
     }
     public static QueryProfileType getArgumentType() { return argumentType; }
 
+
     /** The aliases of query properties */
     private static Map<String,CompoundName> propertyAliases;
     static {
@@ -218,6 +226,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable {
         addAliases(Ranking.getArgumentType(), propertyAliasesBuilder);
         addAliases(Model.getArgumentType(), propertyAliasesBuilder);
         addAliases(Presentation.getArgumentType(), propertyAliasesBuilder);
+        addAliases(Select.getArgumentType(), propertyAliasesBuilder);
         propertyAliases = ImmutableMap.copyOf(propertyAliasesBuilder);
     }
     private static void addAliases(QueryProfileType arguments, Map<String, CompoundName> aliases) {
@@ -237,6 +246,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable {
         registry.register(Query.getArgumentType().unfrozen());
         registry.register(Ranking.getArgumentType().unfrozen());
         registry.register(Model.getArgumentType().unfrozen());
+        registry.register(Select.getArgumentType().unfrozen());
         registry.register(Presentation.getArgumentType().unfrozen());
         registry.register(DefaultProperties.argumentType.unfrozen());
     }
@@ -381,6 +391,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable {
         setFrom(properties,Model.getArgumentType(), context);
         setFrom(properties,Presentation.getArgumentType(), context);
         setFrom(properties,Ranking.getArgumentType(), context);
+        setFrom(properties, Select.getArgumentType(), context);
     }
 
     /**
@@ -511,11 +522,15 @@ public class Query extends com.yahoo.processing.Request implements Cloneable {
      * Get the appropriate timeout for the query.
      *
      * @return timeout in milliseconds
-     **/
+     */
     public long getTimeLeft() {
         return getTimeout() - getDurationTime();
     }
 
+    /**
+     * @deprecated do not use
+     */
+    @Deprecated // TODO: Remove on Vespa 7
     public boolean requestHasProperty(String name) {
         return httpRequest.hasProperty(name);
     }
@@ -976,6 +991,9 @@ public class Query extends com.yahoo.processing.Request implements Cloneable {
     /** Returns the presentation to be used for this query, never null */
     public Presentation getPresentation() { return presentation; }
 
+    /** Returns the select to be used for this query, never null */
+    public Select getSelect() { return select; }
+
     /** Returns the ranking to be used for this query, never null */
     public Ranking getRanking() { return ranking; }
 
diff --git a/container-search/src/main/java/com/yahoo/search/federation/vespa/VespaSearcher.java b/container-search/src/main/java/com/yahoo/search/federation/vespa/VespaSearcher.java
index b59578ab6a3..fdbee9c8f11 100644
--- a/container-search/src/main/java/com/yahoo/search/federation/vespa/VespaSearcher.java
+++ b/container-search/src/main/java/com/yahoo/search/federation/vespa/VespaSearcher.java
@@ -140,6 +140,8 @@ public class VespaSearcher extends ConfiguredHTTPProviderSearcher {
             return Query.Type.PROGRAMMATIC;
         } else if (providerQueryType == ProviderConfig.QueryType.YQL) {
             return Query.Type.YQL;
+        } else if (providerQueryType == ProviderConfig.QueryType.SELECT) {
+            return Query.Type.SELECT;
         } else {
             throw new RuntimeException("Query type " + providerQueryType + " unsupported.");
         }
diff --git a/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java b/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java
index 7406d492bc8..5ac1f834031 100644
--- a/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java
+++ b/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java
@@ -283,13 +283,16 @@ public class SearchHandler extends LoggingRequestHandler {
 
 
     private HttpSearchResponse handleBody(HttpRequest request){
-        // Find query profile
-        String queryProfileName = request.getProperty("queryProfile");
+
+        Map<String, String> requestMap = requestMapFromRequest(request);
+
+        // Get query profile
+        String queryProfileName = requestMap.getOrDefault("queryProfile", null);
         CompiledQueryProfile queryProfile = queryProfileRegistry.findQueryProfile(queryProfileName);
-        boolean benchmarkOutput = VespaHeaders.benchmarkOutput(request);
 
-        Query query = queryFromRequest(request, queryProfile);
+        Query query = new Query(request, requestMap, queryProfile);
 
+        boolean benchmarkOutput = VespaHeaders.benchmarkOutput(request);
         boolean benchmarkCoverage = VespaHeaders.benchmarkCoverage(benchmarkOutput, request.getJDiscRequest().headers());
 
         // Find and execute search chain if we have a valid query
@@ -558,9 +561,10 @@ public class SearchHandler extends LoggingRequestHandler {
         return searchChainRegistry;
     }
 
+    private Map<String, String> requestMapFromRequest(HttpRequest request) {
 
-    private Query queryFromRequest(HttpRequest request, CompiledQueryProfile queryProfile){
-        if (request.getMethod() == com.yahoo.jdisc.http.HttpRequest.Method.POST && request.getHeader(com.yahoo.jdisc.http.HttpHeaders.Names.CONTENT_TYPE).equals(JSON_CONTENT_TYPE)) {
+        if (request.getMethod() == com.yahoo.jdisc.http.HttpRequest.Method.POST
+            && JSON_CONTENT_TYPE.equals(request.getHeader(com.yahoo.jdisc.http.HttpHeaders.Names.CONTENT_TYPE))) {
             Inspector inspector;
             try {
                 byte[] byteArray = IOUtils.readBytes(request.getData(), 1 << 20);
@@ -576,16 +580,26 @@ public class SearchHandler extends LoggingRequestHandler {
             // Create request-mapping
             Map<String, String> requestMap = new HashMap<>();
             createRequestMapping(inspector, requestMap, "");
-            return new Query(request, requestMap, queryProfile);
 
+            // Throws QueryException if query contains both yql- and select-parameter
+            if (requestMap.containsKey("yql") && (requestMap.containsKey("select.where") || requestMap.containsKey("select.grouping")) ) {
+                throw new QueryException("Illegal query: Query contains both yql- and select-parameter");
+            }
+
+            // Throws QueryException if query contains both query- and select-parameter
+            if (requestMap.containsKey("query") && (requestMap.containsKey("select.where") || requestMap.containsKey("select.grouping")) ) {
+                throw new QueryException("Illegal query: Query contains both query- and select-parameter");
+            }
+
+            return requestMap;
 
         } else {
-            return new Query(request, queryProfile);
+            return request.propertyMap();
 
         }
     }
 
-    public void createRequestMapping(Inspector inspector, Map<String, String> map, String parent){
+    public void createRequestMapping(Inspector inspector, Map<String, String> map, String parent) {
         inspector.traverse((ObjectTraverser) (key, value) -> {
             String qualifiedKey = parent + key;
             switch (value.type()) {
@@ -605,6 +619,10 @@ public class SearchHandler extends LoggingRequestHandler {
                     map.put(qualifiedKey, value.asString());
                     break;
                 case OBJECT:
+                    if (qualifiedKey.equals("select.where") || qualifiedKey.equals("select.grouping")){
+                        map.put(qualifiedKey, value.toString());
+                        break;
+                    }
                     createRequestMapping(value, map, qualifiedKey+".");
                     break;
             }
@@ -612,8 +630,6 @@ public class SearchHandler extends LoggingRequestHandler {
         });
     }
 
-
-
 }
 
 
diff --git a/container-search/src/main/java/com/yahoo/search/query/Model.java b/container-search/src/main/java/com/yahoo/search/query/Model.java
index dc7a61344cb..bd0c229085b 100644
--- a/container-search/src/main/java/com/yahoo/search/query/Model.java
+++ b/container-search/src/main/java/com/yahoo/search/query/Model.java
@@ -78,11 +78,11 @@ public class Model implements Cloneable {
     private String defaultIndex = null;
     private Query.Type type = Query.Type.ALL;
     private Query parent;
-    private Set<String> sources=new LinkedHashSet<>();
-    private Set<String> restrict=new LinkedHashSet<>();
+    private Set<String> sources = new LinkedHashSet<>();
+    private Set<String> restrict = new LinkedHashSet<>();
     private String searchPath;
     private String documentDbName = null;
-    private Execution execution=new Execution(new Execution.Context(null, null, null, null, null));
+    private Execution execution = new Execution(new Execution.Context(null, null, null, null, null));
 
     public Model(Query query) {
         setParent(query);
@@ -101,7 +101,7 @@ public class Model implements Cloneable {
      */
     @Deprecated
     public void traceLanguage() {
-        if (getParent().getTraceLevel()<2) return;
+        if (getParent().getTraceLevel() < 2) return;
         if (language != null) {
             getParent().trace("Language " + getLanguage() + " specified directly as a parameter", false, 2);
         }
@@ -235,9 +235,9 @@ public class Model implements Cloneable {
      * Passing null causes this to be set to an empty string.
      */
     public void setQueryString(String queryString) {
-        if (queryString==null) queryString="";
+        if (queryString == null) queryString="";
         this.queryString = queryString;
-        queryTree=null; // Cause parsing of the new query string next time the tree is accessed
+        queryTree = null; // Cause parsing of the new query string next time the tree is accessed
     }
 
     /**
@@ -249,7 +249,7 @@ public class Model implements Cloneable {
     public String getQueryString() { return queryString; }
 
     /**
-     * Returns the query as an object structure.
+     * Returns the query as an object structure. Remember to have the correct Query.Type set.
      * This causes parsing of the query string if it has changed since this was last called
      * (i.e query parsing is lazy)
      */
diff --git a/container-search/src/main/java/com/yahoo/search/query/Select.java b/container-search/src/main/java/com/yahoo/search/query/Select.java
new file mode 100644
index 00000000000..3ffc6bddb24
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/Select.java
@@ -0,0 +1,110 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query;
+
+import com.yahoo.processing.request.CompoundName;
+import com.yahoo.search.Query;
+import com.yahoo.search.grouping.GroupingRequest;
+import com.yahoo.search.query.parser.ParserEnvironment;
+import com.yahoo.search.query.parser.ParserFactory;
+import com.yahoo.search.query.profile.types.FieldDescription;
+import com.yahoo.search.query.profile.types.QueryProfileType;
+import com.yahoo.search.yql.VespaGroupingStep;
+
+
+
+/**
+ * The parameters defining the where-clause and groping of a query
+ *
+ * @author henrhoi
+ */
+public class Select implements Cloneable {
+
+    /** The type representing the property arguments consumed by this */
+    private static final QueryProfileType argumentType;
+    private static final CompoundName argumentTypeName;
+
+    public static final String SELECT = "select";
+    public static final String WHERE = "where";
+    public static final String GROUPING = "grouping";
+
+
+    private static Model model;
+    private Query parent;
+    private String where = "";
+    private String grouping = "";
+
+    static {
+        argumentType = new QueryProfileType(SELECT);
+        argumentType.setStrict(true);
+        argumentType.setBuiltin(true);
+        argumentType.addField(new FieldDescription(WHERE, "string", "where"));
+        argumentType.addField(new FieldDescription(GROUPING, "string", "grouping"));
+        argumentType.freeze();
+        argumentTypeName=new CompoundName(argumentType.getId().getName());
+    }
+
+    public static QueryProfileType getArgumentType() { return argumentType; }
+
+    public Select(String where, String grouping){
+        this.where = where;
+        this.grouping = grouping;
+    }
+
+    public Select(Query query) {
+        setParent(query);
+        model = query.getModel();
+    }
+
+
+    /** Returns the query owning this, never null */
+    private Query getParent() { return parent; }
+
+
+    /** Assigns the query owning this */
+    public void setParent(Query parent) {
+        if (parent==null) throw new NullPointerException("A query models owner cannot be null");
+        this.parent = parent;
+    }
+
+
+    /** Set the where-clause for the query. Must be a JSON-string, with the format described in the Select Reference doc - https://docs.vespa.ai/documentation/reference/select-reference.html. */
+    public void setWhere(String where) {
+        this.where = where;
+        model.setType(SELECT);
+
+        // Setting the queryTree to null
+        model.setQueryString(null);
+    }
+
+
+    /** Returns the where-clause in the query */
+    public String getWhereString(){
+        return this.where;
+    }
+
+
+    /** Set the grouping-string for the query. Must be a JSON-string, with the format described in the Select Reference doc - https://docs.vespa.ai/documentation/reference/select-reference.html. */
+    public void setGrouping(String grouping){
+        this.grouping = grouping;
+        SelectParser parser = (SelectParser) ParserFactory.newInstance(Query.Type.SELECT, new ParserEnvironment());
+
+        for (VespaGroupingStep step : parser.getGroupingSteps(grouping)) {
+            GroupingRequest.newInstance(parent)
+                    .setRootOperation(step.getOperation())
+                    .continuations().addAll(step.continuations());
+        }
+    }
+
+
+    /** Returns the grouping in the query */
+    public String getGroupingString(){
+        return this.grouping;
+    }
+
+
+    @Override
+    public String toString() {
+        return "where: [" + where + "], grouping: [" + grouping+ "]";
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/SelectParser.java b/container-search/src/main/java/com/yahoo/search/query/SelectParser.java
new file mode 100644
index 00000000000..13ebacb62ef
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/query/SelectParser.java
@@ -0,0 +1,1185 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.query;
+
+
+import com.google.common.base.Preconditions;
+import com.yahoo.collections.LazyMap;
+import com.yahoo.language.Language;
+import com.yahoo.language.process.Normalizer;
+import com.yahoo.prelude.IndexFacts;
+import com.yahoo.prelude.query.AndItem;
+import com.yahoo.prelude.query.CompositeItem;
+import com.yahoo.prelude.query.DotProductItem;
+import com.yahoo.prelude.query.EquivItem;
+import com.yahoo.prelude.query.ExactStringItem;
+import com.yahoo.prelude.query.IntItem;
+import com.yahoo.prelude.query.Item;
+import com.yahoo.prelude.query.Limit;
+import com.yahoo.prelude.query.NearItem;
+import com.yahoo.prelude.query.NotItem;
+import com.yahoo.prelude.query.ONearItem;
+import com.yahoo.prelude.query.OrItem;
+import com.yahoo.prelude.query.PhraseItem;
+import com.yahoo.prelude.query.PredicateQueryItem;
+import com.yahoo.prelude.query.PrefixItem;
+import com.yahoo.prelude.query.QueryException;
+import com.yahoo.prelude.query.RangeItem;
+import com.yahoo.prelude.query.RankItem;
+import com.yahoo.prelude.query.RegExpItem;
+import com.yahoo.prelude.query.SameElementItem;
+import com.yahoo.prelude.query.SegmentingRule;
+import com.yahoo.prelude.query.Substring;
+import com.yahoo.prelude.query.SubstringItem;
+import com.yahoo.prelude.query.SuffixItem;
+import com.yahoo.prelude.query.TaggableItem;
+import com.yahoo.prelude.query.WandItem;
+import com.yahoo.prelude.query.WeakAndItem;
+import com.yahoo.prelude.query.WeightedSetItem;
+import com.yahoo.prelude.query.WordAlternativesItem;
+import com.yahoo.prelude.query.WordItem;
+import com.yahoo.search.grouping.request.GroupingOperation;
+import com.yahoo.search.query.parser.Parsable;
+import com.yahoo.search.query.parser.Parser;
+import com.yahoo.search.query.parser.ParserEnvironment;
+import com.yahoo.search.yql.VespaGroupingStep;
+import com.yahoo.slime.ArrayTraverser;
+import com.yahoo.slime.Inspector;
+import com.yahoo.slime.ObjectTraverser;
+import com.yahoo.vespa.config.SlimeUtils;
+import edu.umd.cs.findbugs.annotations.NonNull;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+import static com.yahoo.slime.Type.ARRAY;
+import static com.yahoo.slime.Type.DOUBLE;
+import static com.yahoo.slime.Type.LONG;
+import static com.yahoo.slime.Type.OBJECT;
+import static com.yahoo.slime.Type.STRING;
+
+/**
+ * The Select query language.
+ *
+ * This class will be parsing the Select parameters, and will be used when the query has the SELECT-type.
+ *
+ * @author henrhoi
+ */
+
+
+public class SelectParser implements Parser {
+
+    Parsable query;
+    private final IndexFacts indexFacts;
+    private final Map<Integer, TaggableItem> identifiedItems = LazyMap.newHashMap();
+    private final List<ConnectedItem> connectedItems = new ArrayList<>();
+    private final Normalizer normalizer;
+    private final ParserEnvironment environment;
+    private IndexFacts.Session indexFactsSession;
+
+
+
+    /** YQL parameters and functions */
+
+    private static final String DESCENDING_HITS_ORDER = "descending";
+    private static final String ASCENDING_HITS_ORDER = "ascending";
+    private static final Integer DEFAULT_TARGET_NUM_HITS = 10;
+    private static final String ORIGIN_LENGTH = "length";
+    private static final String ORIGIN_OFFSET = "offset";
+    private static final String ORIGIN = "origin";
+    private static final String ORIGIN_ORIGINAL = "original";
+    private static final String CONNECTION_ID = "id";
+    private static final String CONNECTION_WEIGHT = "weight";
+    private static final String CONNECTIVITY = "connectivity";
+    private static final String ANNOTATIONS = "annotations";
+    private static final String NFKC = "nfkc";
+    private static final String USER_INPUT_LANGUAGE = "language";
+    private static final String ACCENT_DROP = "accentDrop";
+    private static final String ALTERNATIVES = "alternatives";
+    private static final String AND_SEGMENTING = "andSegmenting";
+    private static final String DISTANCE = "distance";
+    private static final String DOT_PRODUCT = "dotProduct";
+    private static final String EQUIV = "equiv";
+    private static final String FILTER = "filter";
+    private static final String HIT_LIMIT = "hitLimit";
+    private static final String IMPLICIT_TRANSFORMS = "implicitTransforms";
+    private static final String LABEL = "label";
+    private static final String NEAR = "near";
+    private static final String NORMALIZE_CASE = "normalizeCase";
+    private static final String ONEAR = "onear";
+    private static final String PHRASE = "phrase";
+    private static final String PREDICATE = "predicate";
+    private static final String PREFIX = "prefix";
+    private static final String RANKED = "ranked";
+    private static final String RANK = "rank";
+    private static final String SAME_ELEMENT = "sameElement";
+    private static final String SCORE_THRESHOLD = "scoreThreshold";
+    private static final String SIGNIFICANCE = "significance";
+    private static final String STEM = "stem";
+    private static final String SUBSTRING = "substring";
+    private static final String SUFFIX = "suffix";
+    private static final String TARGET_NUM_HITS = "targetNumHits";
+    private static final String THRESHOLD_BOOST_FACTOR = "thresholdBoostFactor";
+    private static final String UNIQUE_ID = "id";
+    private static final String USE_POSITION_DATA = "usePositionData";
+    private static final String WAND = "wand";
+    private static final String WEAK_AND = "weakAnd";
+    private static final String WEIGHTED_SET = "weightedSet";
+    private static final String WEIGHT = "weight";
+    private static final String AND = "and";
+    private static final String AND_NOT = "and_not";
+    private static final String OR = "or";
+    private static final String EQ = "equals";
+    private static final String RANGE = "range";
+    private static final String CONTAINS = "contains";
+    private static final String MATCHES = "matches";
+    private static final String CALL = "call";
+    private static final List<String> FUNCTION_CALLS = Arrays.asList(WAND, WEIGHTED_SET, DOT_PRODUCT, PREDICATE, RANK, WEAK_AND);
+
+    /**************************************/
+
+
+
+    public SelectParser(ParserEnvironment environment) {
+        indexFacts = environment.getIndexFacts();
+        normalizer = environment.getLinguistics().getNormalizer();
+
+        this.environment = environment;
+    }
+
+
+    @Override
+    public QueryTree parse(Parsable query) {
+        indexFactsSession = indexFacts.newSession(query.getSources(), query.getRestrict());
+        connectedItems.clear();
+        identifiedItems.clear();
+        this.query = query;
+
+        return buildTree();
+    }
+
+
+
+    private QueryTree buildTree() {
+        Inspector inspector = SlimeUtils.jsonToSlime(this.query.getSelect().getWhereString().getBytes()).get();
+        if (inspector.field("error_message").valid()){
+            throw new QueryException("Illegal query: "+inspector.field("error_message").asString() + ", at: "+ new String(inspector.field("offending_input").asData(), StandardCharsets.UTF_8));
+        }
+
+        Item root = walkJson(inspector);
+        connectItems();
+        QueryTree newTree = new QueryTree(root);
+
+        return newTree;
+    }
+
+
+    private Item walkJson(Inspector inspector){
+        final Item[] item = {null};
+        inspector.traverse((ObjectTraverser) (key, value) -> {
+            String type = (FUNCTION_CALLS.contains(key)) ? CALL : key;
+
+            switch (type) {
+
+                case AND:
+                    item[0] = buildAnd(key, value);
+                    break;
+                case AND_NOT:
+                    item[0] = buildNotAnd(key, value);
+                    break;
+                case OR:
+                    item[0] = buildOr(key, value);
+                    break;
+                case EQ:
+                    item[0] = buildEquals(key, value);
+                    break;
+                case RANGE:
+                    item[0] = buildRange(key, value);
+                    break;
+                case CONTAINS:
+                    item[0] = buildTermSearch(key, value);
+                    break;
+                case MATCHES:
+                    item[0] = buildRegExpSearch(key, value);
+                    break;
+                case CALL:
+                    item[0] = buildFunctionCall(key, value);
+                    break;
+                default:
+                    throw newUnexpectedArgumentException(key, AND, CALL, CONTAINS, EQ, OR, RANGE, AND_NOT);
+            }
+        });
+        return item[0];
+    }
+
+
+    public List<VespaGroupingStep> getGroupingSteps(String grouping){
+        List<VespaGroupingStep> groupingSteps = new ArrayList<>();
+        List<String> groupingOperations = getOperations(grouping);
+        for (String groupingString : groupingOperations){
+            GroupingOperation groupingOperation = GroupingOperation.fromString(groupingString);
+            VespaGroupingStep groupingStep = new VespaGroupingStep(groupingOperation);
+            groupingSteps.add(groupingStep);
+        }
+        return groupingSteps;
+    }
+
+    private List<String> getOperations(String grouping) {
+        List<String> operations = new ArrayList<>();
+        Inspector inspector = SlimeUtils.jsonToSlime(grouping.getBytes()).get();
+        if (inspector.field("error_message").valid()){
+            throw new QueryException("Illegal query: "+inspector.field("error_message").asString() + ", at: "+ new String(inspector.field("offending_input").asData(), StandardCharsets.UTF_8));
+        }
+
+        inspector.traverse( (ArrayTraverser) (key, value) -> {
+            String groupingString = value.toString();
+            groupingString = groupingString.replace(" ", "").replace("\"", "").replace("\'", "").replace(":{", "(").replace(":", "(").replace("}", ")").replace(",", ")");
+            groupingString = groupingString.substring(1, groupingString.length());
+            operations.add(groupingString);
+        });
+
+        return operations;
+
+    }
+
+
+    @NonNull
+    private Item buildFunctionCall(String key, Inspector value) {
+        switch (key) {
+            case WAND:
+                return buildWand(key, value);
+            case WEIGHTED_SET:
+                return buildWeightedSet(key, value);
+            case DOT_PRODUCT:
+                return buildDotProduct(key, value);
+            case PREDICATE:
+                return buildPredicate(key, value);
+            case RANK:
+                return buildRank(key, value);
+            case WEAK_AND:
+                return buildWeakAnd(key, value);
+            default:
+                throw newUnexpectedArgumentException(key, DOT_PRODUCT, RANK, WAND, WEAK_AND, WEIGHTED_SET, PREDICATE);
+        }
+    }
+
+
+    private void addItemsFromInspector(CompositeItem item, Inspector inspector){
+        if (inspector.type() == ARRAY){
+            inspector.traverse((ArrayTraverser) (index, new_value) -> {
+                item.addItem(walkJson(new_value));
+            });
+
+        } else if (inspector.type() == OBJECT){
+            if (inspector.field("children").valid()){
+                inspector.field("children").traverse((ArrayTraverser) (index, new_value) -> {
+                    item.addItem(walkJson(new_value));
+                });
+            }
+
+        }
+    }
+
+
+    private Inspector getChildren(Inspector inspector){
+        if (inspector.type() == ARRAY){
+            return inspector;
+
+        } else if (inspector.type() == OBJECT){
+            if (inspector.field("children").valid()){
+                return inspector.field("children");
+            }
+            if (inspector.field(1).valid()){
+                return inspector.field(1);
+            }
+        }
+        return null;
+    }
+
+
+    private HashMap<Integer, Inspector> getChildrenMap(Inspector inspector){
+        HashMap<Integer, Inspector> children = new HashMap<>();
+            if (inspector.type() == ARRAY){
+                inspector.traverse((ArrayTraverser) (index, new_value) -> {
+                    children.put(index, new_value);
+                });
+
+            } else if (inspector.type() == OBJECT){
+                if (inspector.field("children").valid()){
+                    inspector.field("children").traverse((ArrayTraverser) (index, new_value) -> {
+                        children.put(index, new_value);
+                    });
+                }
+            }
+        return children;
+    }
+
+
+    private Inspector getAnnotations(Inspector inspector){
+        if (inspector.type() == OBJECT && inspector.field("attributes").valid()){
+            return inspector.field("attributes");
+        }
+        return null;
+    }
+
+
+    private HashMap<String, Inspector> getAnnotationMapFromAnnotationInspector(Inspector annotation){
+        HashMap<String, Inspector> attributes = new HashMap<>();
+        if (annotation.type() == OBJECT){
+            annotation.traverse((ObjectTraverser) (index, new_value) -> {
+                attributes.put(index, new_value);
+            });
+        }
+        return attributes;
+    }
+
+
+    private HashMap<String, Inspector> getAnnotationMap(Inspector inspector){
+        HashMap<String, Inspector> attributes = new HashMap<>();
+        if (inspector.type() == OBJECT && inspector.field("attributes").valid()){
+            inspector.field("attributes").traverse((ObjectTraverser) (index, new_value) -> {
+                attributes.put(index, new_value);
+            });
+        }
+        return attributes;
+    }
+
+
+    private <T> T getAnnotation(String annotationName, HashMap<String, Inspector> annotations, Class<T> expectedClass, T defaultValue) {
+        return (annotations.get(annotationName) == null) ? defaultValue : expectedClass.cast(annotations.get(annotationName).asString());
+    }
+
+
+    private Boolean getBoolAnnotation(String annotationName, HashMap<String, Inspector> annotations, Boolean defaultValue) {
+        if (annotations != null){
+            Inspector annotation = annotations.getOrDefault(annotationName, null);
+            if (annotation != null){
+                return annotation.asBool();
+            }
+        }
+        return defaultValue;
+    }
+
+
+    private Integer getIntegerAnnotation(String annotationName, HashMap<String, Inspector> annotations, Integer defaultValue) {
+        if (annotations != null){
+            Inspector annotation = annotations.getOrDefault(annotationName, null);
+            if (annotation != null){
+                return (int)annotation.asLong();
+            }
+        }
+        return defaultValue;
+    }
+
+
+    private Double getDoubleAnnotation(String annotationName, HashMap<String, Inspector> annotations, Double defaultValue) {
+        if (annotations != null){
+            Inspector annotation = annotations.getOrDefault(annotationName, null);
+            if (annotation != null){
+                return annotation.asDouble();
+            }
+        }
+        return defaultValue;
+    }
+
+
+    private Inspector getAnnotationAsInspectorOrNull(String annotationName, HashMap<String, Inspector> annotations) {
+        return annotations.get(annotationName);
+    }
+
+
+    @NonNull
+    private CompositeItem buildAnd(String key, Inspector value) {
+        AndItem andItem = new AndItem();
+        addItemsFromInspector(andItem, value);
+
+        return andItem;
+    }
+
+
+    @NonNull
+    private CompositeItem buildNotAnd(String key, Inspector value) {
+        NotItem notItem = new NotItem();
+        addItemsFromInspector(notItem, value);
+
+        return notItem;
+    }
+
+
+    @NonNull
+    private CompositeItem buildOr(String key, Inspector value) {
+        OrItem orItem = new OrItem();
+        addItemsFromInspector(orItem, value);
+        return orItem;
+    }
+
+
+    @NonNull
+    private CompositeItem buildWeakAnd(String key, Inspector value) {
+        WeakAndItem weakAnd = new WeakAndItem();
+        addItemsFromInspector(weakAnd, value);
+        Inspector annotations = getAnnotations(value);
+
+        if (annotations != null){
+            annotations.traverse((ObjectTraverser) (annotation_name, annotation_value) -> {
+                if (TARGET_NUM_HITS.equals(annotation_name)){
+                    weakAnd.setN((int)(annotation_value.asDouble()));
+                }
+                if (SCORE_THRESHOLD.equals(annotation_name)){
+                    weakAnd.setScoreThreshold((int)(annotation_value.asDouble()));
+                }
+            });
+        }
+
+        return weakAnd;
+    }
+
+
+    @NonNull
+    private <T extends TaggableItem> T leafStyleSettings(Inspector annotations, @NonNull T out) {
+        {
+            if (annotations != null) {
+                Inspector  itemConnectivity= getAnnotationAsInspectorOrNull(CONNECTIVITY, getAnnotationMapFromAnnotationInspector(annotations));
+                if (itemConnectivity != null) {
+                    Integer[] id = {null};
+                    Double[] weight = {null};
+                    itemConnectivity.traverse((ObjectTraverser) (key, value) -> {
+                        switch (key){
+                            case CONNECTION_ID:
+                                id[0] = (int) value.asLong();
+                                break;
+                            case CONNECTION_WEIGHT:
+                                weight[0] = value.asDouble();
+                                break;
+                        }
+                    });
+                    connectedItems.add(new ConnectedItem(out, id[0], weight[0]));
+                }
+
+                annotations.traverse((ObjectTraverser) (annotation_name, annotation_value) -> {
+
+                    if (SIGNIFICANCE.equals(annotation_name)) {
+                        if (annotation_value != null) {
+                            out.setSignificance(annotation_value.asDouble());
+                        }
+                    }
+                    if (UNIQUE_ID.equals(annotation_name)) {
+                        if (annotation_value != null) {
+                            out.setUniqueID((int)annotation_value.asLong());
+                            identifiedItems.put((int)annotation_value.asLong(), out);
+                        }
+                    }
+                });
+            }
+        }
+        {
+            Item leaf = (Item) out;
+            if (annotations != null) {
+                Inspector itemAnnotations = getAnnotationAsInspectorOrNull(ANNOTATIONS, getAnnotationMapFromAnnotationInspector(annotations));
+                if (itemAnnotations != null) {
+                    itemAnnotations.traverse((ObjectTraverser) (key, value) -> {
+                        leaf.addAnnotation(key, value.asString());
+                    });
+                }
+
+                annotations.traverse((ObjectTraverser) (annotation_name, annotation_value) -> {
+                    if (FILTER.equals(annotation_name)) {
+                        if (annotation_value != null) {
+                            leaf.setFilter(annotation_value.asBool());
+                        }
+                    }
+                    if (RANKED.equals(annotation_name)) {
+                        if (annotation_value != null) {
+                            leaf.setRanked(annotation_value.asBool());
+                        }
+                    }
+                    if (LABEL.equals(annotation_name)) {
+                        if (annotation_value != null) {
+                            leaf.setLabel(annotation_value.asString());
+                        }
+                    }
+                    if (WEIGHT.equals(annotation_name)) {
+                        if (annotation_value != null) {
+                            leaf.setWeight((int)annotation_value.asDouble());
+                        }
+                    }
+                });
+            }
+            if (out instanceof IntItem && annotations != null) {
+                IntItem number = (IntItem) out;
+                Integer hitLimit = getCappedRangeSearchParameter(annotations);
+                if (hitLimit != null) {
+                    number.setHitLimit(hitLimit);
+                }
+
+            }
+        }
+
+        return out;
+    }
+
+
+    private Integer getCappedRangeSearchParameter(Inspector annotations) {
+        final Integer[] hitLimit = {null};
+        annotations.traverse((ObjectTraverser) (annotation_name, annotation_value) -> {
+            if (HIT_LIMIT.equals(annotation_name)) {
+                if (annotation_value != null) {
+                    hitLimit[0] = (int)(annotation_value.asDouble());
+                }
+            }
+        });
+        final Boolean[] ascending = {null};
+        final Boolean[] descending = {null};
+
+        if (hitLimit[0] != null) {
+            annotations.traverse((ObjectTraverser) (annotation_name, annotation_value) -> {
+                if (ASCENDING_HITS_ORDER.equals(annotation_name)) {
+                    ascending[0] = annotation_value.asBool();
+                }
+                if (DESCENDING_HITS_ORDER.equals(annotation_name)) {
+                    descending[0] = annotation_value.asBool();
+                }
+
+            });
+            Preconditions.checkArgument(ascending[0] == null || descending[0] == null,
+                    "Settings for both ascending and descending ordering set, only one of these expected.");
+
+            if (Boolean.TRUE.equals(descending[0]) || Boolean.FALSE.equals(ascending[0])) {
+                hitLimit[0] = hitLimit[0] * -1;
+            }
+        }
+        return hitLimit[0];
+    }
+
+
+    @NonNull
+    private Item buildRange(String key, Inspector value) {
+        HashMap<Integer, Inspector> children = getChildrenMap(value);
+        Inspector annotations = getAnnotations(value);
+
+        final boolean[] equals = {false};
+
+        String field;
+        Inspector boundInspector;
+        if (children.get(0).type() == STRING){
+            field = children.get(0).asString();
+            boundInspector = children.get(1);
+        } else {
+            field = children.get(1).asString();
+            boundInspector = children.get(0);
+        }
+
+        final Number[] bounds = {null, null};
+        final String[] operators = {null, null};
+        boundInspector.traverse((ObjectTraverser) (operator, bound) -> {
+            if (bound.type() == STRING) {
+                throw new IllegalArgumentException("Expected operator LITERAL, got READ_FIELD.");
+            }
+            if (operator.equals("=")) {
+                bounds[0] = (bound.type() == DOUBLE) ? Number.class.cast(bound.asDouble()) : Number.class.cast(bound.asLong());
+                operators[0] = operator;
+                equals[0] = true;
+            }
+            if (operator.equals(">=") || operator.equals(">")){
+                bounds[0] = (bound.type() == DOUBLE) ? Number.class.cast(bound.asDouble()) : Number.class.cast(bound.asLong());
+                operators[0] = operator;
+            } else if (operator.equals("<=") || operator.equals("<")){
+                bounds[1] = (bound.type() == DOUBLE) ? Number.class.cast(bound.asDouble()) : Number.class.cast(bound.asLong());
+                operators[1] = operator;
+            }
+
+        });
+        IntItem range = null;
+        if (equals[0]){
+            range = new IntItem(bounds[0].toString(), field);
+        } else if (operators[0]==null || operators[1]==null){
+            Integer index = (operators[0] == null) ? 1 : 0;
+            switch (operators[index]){
+                case ">=":
+                    range = buildGreaterThanOrEquals(field, bounds[index].toString());
+                    break;
+                case ">":
+                    range = buildGreaterThan(field, bounds[index].toString());
+                    break;
+                case "<":
+                    range = buildLessThan(field, bounds[index].toString());
+                    break;
+                case "<=":
+                    range = buildLessThanOrEquals(field, bounds[index].toString());
+                    break;
+            }
+        }
+        else {
+            range = instantiateRangeItem(bounds[0], bounds[1], field, operators[0].equals(">"), operators[1].equals("<"));
+        }
+
+        return leafStyleSettings(annotations, range);
+    }
+
+    @NonNull
+    private IntItem buildGreaterThanOrEquals(String field, String bound) {
+        return new IntItem("[" + bound + ";]", field);
+
+    }
+
+
+    @NonNull
+    private IntItem buildLessThanOrEquals(String field, String bound) {
+        return new IntItem("[;" + bound + "]", field);
+    }
+
+
+    @NonNull
+    private IntItem buildGreaterThan(String field, String bound) {
+        return new IntItem(">" + bound, field);
+
+    }
+
+
+    @NonNull
+    private IntItem buildLessThan(String field, String bound) {
+        return new IntItem("<" + bound, field);
+    }
+
+
+    @NonNull
+    private IntItem instantiateRangeItem(Number lowerBound, Number upperBound, String field, boolean bounds_left_open, boolean bounds_right_open) {
+        Preconditions.checkArgument(lowerBound != null && upperBound != null && field != null,
+                "Expected 3 NonNull-arguments");
+
+        if (!bounds_left_open && !bounds_right_open) {
+            return new RangeItem(lowerBound, upperBound, field);
+        } else {
+            Limit from;
+            Limit to;
+            if (bounds_left_open && bounds_right_open) {
+                from = new Limit(lowerBound, false);
+                to = new Limit(upperBound, false);
+            } else if (bounds_left_open) {
+                from = new Limit(lowerBound, false);
+                to = new Limit(upperBound, true);
+            } else {
+                from = new Limit(lowerBound, true);
+                to = new Limit(upperBound, false);
+            }
+            return new IntItem(from, to, field);
+        }
+    }
+
+
+    @NonNull
+    private Item buildEquals(String key, Inspector value) {
+         return buildRange(key, value);
+    }
+
+
+    @NonNull
+    private Item buildWand(String key, Inspector value) {
+        HashMap<String, Inspector> annotations = getAnnotationMap(value);
+        HashMap<Integer, Inspector> children = getChildrenMap(value);
+
+        Preconditions.checkArgument(children.size() == 2, "Expected 2 arguments, got %s.", children.size());
+        Integer target_num_hits= getIntegerAnnotation(TARGET_NUM_HITS, annotations, DEFAULT_TARGET_NUM_HITS);
+
+        WandItem out = new WandItem(children.get(0).asString(), target_num_hits);
+
+        Double scoreThreshold = getDoubleAnnotation(SCORE_THRESHOLD, annotations, null);
+
+        if (scoreThreshold != null) {
+            out.setScoreThreshold(scoreThreshold);
+        }
+
+        Double thresholdBoostFactor = getDoubleAnnotation(THRESHOLD_BOOST_FACTOR, annotations, null);
+        if (thresholdBoostFactor != null) {
+            out.setThresholdBoostFactor(thresholdBoostFactor);
+        }
+        return fillWeightedSet(value, children, out);
+    }
+
+
+    @NonNull
+    private WeightedSetItem fillWeightedSet(Inspector value, HashMap<Integer, Inspector> children, @NonNull WeightedSetItem out) {
+        addItems(children, out);
+
+        return leafStyleSettings(getAnnotations(value), out);
+    }
+
+
+    private static void addItems(HashMap<Integer, Inspector> children, WeightedSetItem out) {
+        switch (children.get(1).type()) {
+            case OBJECT:
+                addStringItems(children, out);
+                break;
+            case ARRAY:
+                addLongItems(children, out);
+                break;
+            default:
+                throw newUnexpectedArgumentException(children.get(1).type(), ARRAY, OBJECT);
+        }
+    }
+
+
+    private static void addStringItems(HashMap<Integer, Inspector> children, WeightedSetItem out) {
+        //{"a":1, "b":2}
+        children.get(1).traverse((ObjectTraverser) (key, value) -> {
+            if (value.type() == STRING){
+                throw new IllegalArgumentException("Expected operator LITERAL, got READ_FIELD.");
+            }
+            out.addToken(key, (int)value.asLong());
+        });
+    }
+
+
+    private static void addLongItems(HashMap<Integer, Inspector> children, WeightedSetItem out) {
+        //[[11,1], [37,2]]
+        children.get(1).traverse((ArrayTraverser) (index, pair) -> {
+            List<Integer> pairValues = new ArrayList<>();
+            pair.traverse((ArrayTraverser) (pairIndex, pairValue) -> {
+                pairValues.add((int)pairValue.asLong());
+            });
+            Preconditions.checkArgument(pairValues.size() == 2,
+                    "Expected item and weight, got %s.", pairValues);
+            out.addToken(pairValues.get(0).longValue(), pairValues.get(1));
+        });
+    }
+
+
+    @NonNull
+    private Item buildRegExpSearch(String key, Inspector value) {
+        assertHasOperator(key, MATCHES);
+        HashMap<Integer, Inspector> children = getChildrenMap(value);
+        String field = children.get(0).asString();
+        String wordData = children.get(1).asString();
+        RegExpItem regExp = new RegExpItem(field, true, wordData);
+        return leafStyleSettings(getAnnotations(value), regExp);
+    }
+
+
+    @NonNull
+    private Item buildWeightedSet(String key, Inspector value) {
+        HashMap<Integer, Inspector> children = getChildrenMap(value);
+        String field = children.get(0).asString();
+        Preconditions.checkArgument(children.size() == 2, "Expected 2 arguments, got %s.", children.size());
+        return fillWeightedSet(value, children, new WeightedSetItem(field));
+    }
+
+
+    @NonNull
+    private Item buildDotProduct(String key, Inspector value) {
+        HashMap<Integer, Inspector> children = getChildrenMap(value);
+        String field = children.get(0).asString();
+        Preconditions.checkArgument(children.size() == 2, "Expected 2 arguments, got %s.", children.size());
+        return fillWeightedSet(value, children, new DotProductItem(field));
+    }
+
+
+    @NonNull
+    private Item buildPredicate(String key, Inspector value) {
+        HashMap<Integer, Inspector> children = getChildrenMap(value);
+        String field = children.get(0).asString();
+        Inspector args = children.get(1);
+
+        Preconditions.checkArgument(children.size() == 3, "Expected 3 arguments, got %s.", children.size());
+
+        PredicateQueryItem item = new PredicateQueryItem();
+        item.setIndexName(field);
+
+        List<Inspector> argumentList = valueListFromInspector(getChildren(value));
+
+        // Adding attributes
+        argumentList.get(1).traverse((ObjectTraverser) (attrKey, attrValue) -> {
+            if (attrValue.type() == ARRAY){
+                List<Inspector> attributes = valueListFromInspector(attrValue);
+                attributes.forEach( (attribute) -> item.addFeature(attrKey, attribute.asString()));
+            } else {
+                item.addFeature(attrKey, attrValue.asString());
+            }
+        });
+
+        // Adding range attributes
+        argumentList.get(2).traverse((ObjectTraverser) (attrKey, attrValue) -> item.addRangeFeature(attrKey, (int)attrValue.asDouble()));
+
+        return leafStyleSettings(getAnnotations(value), item);
+    }
+
+
+    @NonNull
+    private CompositeItem buildRank(String key, Inspector value) {
+        RankItem rankItem = new RankItem();
+        addItemsFromInspector(rankItem, value);
+        return rankItem;
+    }
+
+
+    @NonNull
+    private Item buildTermSearch(String key, Inspector value) {
+        HashMap<Integer, Inspector> children = getChildrenMap(value);
+        String field = children.get(0).asString();
+
+        return instantiateLeafItem(field, key, value);
+    }
+
+
+    private String getInspectorKey(Inspector inspector){
+        String[] actualKey = {""};
+        if (inspector.type() == OBJECT){
+            inspector.traverse((ObjectTraverser) (key, value) -> {
+                actualKey[0] = key;
+
+            });
+        }
+        return actualKey[0];
+    }
+
+
+    @NonNull
+    private Item instantiateLeafItem(String field, String key, Inspector value) {
+        List<Inspector> possibleLeafFunction = valueListFromInspector(value);
+        String possibleLeafFunctionName = (possibleLeafFunction.size() > 1) ? getInspectorKey(possibleLeafFunction.get(1)) : "";
+        if (FUNCTION_CALLS.contains(key)) {
+            return instantiateCompositeLeaf(field, key, value);
+        } else if(!possibleLeafFunctionName.equals("")){
+           return instantiateCompositeLeaf(field, possibleLeafFunctionName, valueListFromInspector(value).get(1).field(possibleLeafFunctionName));
+        } else {
+            return instantiateWordItem(field, key, value);
+        }
+    }
+
+
+    @NonNull
+    private Item instantiateCompositeLeaf(String field, String key, Inspector value) {
+        switch (key) {
+            case SAME_ELEMENT:
+                return instantiateSameElementItem(field, key, value);
+            case PHRASE:
+                return instantiatePhraseItem(field, key, value);
+            case NEAR:
+                return instantiateNearItem(field, key, value);
+            case ONEAR:
+                return instantiateONearItem(field, key, value);
+            case EQUIV:
+                return instantiateEquivItem(field, key, value);
+            case ALTERNATIVES:
+                return instantiateWordAlternativesItem(field, key, value);
+            default:
+                throw newUnexpectedArgumentException(key, EQUIV, NEAR, ONEAR, PHRASE, SAME_ELEMENT);
+        }
+    }
+
+
+    @NonNull
+    private Item instantiateWordItem(String field, String key, Inspector value) {
+        String wordData = getChildrenMap(value).get(1).asString();
+        return instantiateWordItem(field, wordData, key, value, false, decideParsingLanguage(value, wordData));
+    }
+
+
+    @NonNull
+    private Item instantiateWordItem(String field, String rawWord, String key, Inspector value, boolean exactMatch, Language language) {
+        String wordData = rawWord;
+        HashMap<String, Inspector> annotations = getAnnotationMap(value);
+
+        if (getBoolAnnotation(NFKC, annotations, Boolean.FALSE)) {
+            // NOTE: If this is set to FALSE (default), we will still NFKC normalize text data
+            // during tokenization/segmentation, as that is always turned on also on the indexing side.
+            wordData = normalizer.normalize(wordData);
+        }
+        boolean fromQuery = getBoolAnnotation(IMPLICIT_TRANSFORMS, annotations,  Boolean.TRUE);
+        boolean prefixMatch = getBoolAnnotation(PREFIX, annotations, Boolean.FALSE);
+        boolean suffixMatch = getBoolAnnotation(SUFFIX, annotations,  Boolean.FALSE);
+        boolean substrMatch = getBoolAnnotation(SUBSTRING,annotations, Boolean.FALSE);
+
+        Preconditions.checkArgument((prefixMatch ? 1 : 0)
+                        + (substrMatch ? 1 : 0) + (suffixMatch ? 1 : 0) < 2,
+                "Only one of prefix, substring and suffix can be set.");
+        @NonNull
+        final TaggableItem wordItem;
+
+        if (exactMatch) {
+            wordItem = new ExactStringItem(wordData, fromQuery);
+        } else if (prefixMatch) {
+            wordItem = new PrefixItem(wordData, fromQuery);
+        } else if (suffixMatch) {
+            wordItem = new SuffixItem(wordData, fromQuery);
+        } else if (substrMatch) {
+            wordItem = new SubstringItem(wordData, fromQuery);
+        } else {
+            wordItem = new WordItem(wordData, fromQuery);
+        }
+
+        if (wordItem instanceof WordItem) {
+            prepareWord(field, value, (WordItem) wordItem);
+        }
+        if (language != Language.ENGLISH)
+            ((Item)wordItem).setLanguage(language);
+
+        return (Item) leafStyleSettings(getAnnotations(value), wordItem);
+    }
+
+
+    private Language decideParsingLanguage(Inspector value, String wordData) {
+        String languageTag = getAnnotation(USER_INPUT_LANGUAGE, getAnnotationMap(value), String.class, null);
+
+        Language language = Language.fromLanguageTag(languageTag);
+        if (language != Language.UNKNOWN) return language;
+
+        Optional<Language> explicitLanguage = query.getExplicitLanguage();
+        if (explicitLanguage.isPresent()) return explicitLanguage.get();
+
+        return Language.ENGLISH;
+    }
+
+
+    private void prepareWord(String field, Inspector value, WordItem wordItem) {
+        wordItem.setIndexName(field);
+        wordStyleSettings(value, wordItem);
+    }
+
+
+    private void wordStyleSettings(Inspector value, WordItem out) {
+        HashMap<String, Inspector> annotations = getAnnotationMap(value);
+
+        Substring origin = getOrigin(getAnnotations(value));
+        if (origin != null) {
+            out.setOrigin(origin);
+        }
+        if (annotations !=  null){
+            Boolean usePositionData = Boolean.getBoolean(getAnnotation(USE_POSITION_DATA, annotations, String.class, null));
+            if (usePositionData != null) {
+                out.setPositionData(usePositionData);
+            }
+            Boolean stem = getBoolAnnotation(STEM, annotations, null);
+            if (stem != null) {
+                out.setStemmed(!stem);
+            }
+
+            Boolean normalizeCase = getBoolAnnotation(NORMALIZE_CASE, annotations, null);
+            if (normalizeCase != null) {
+                out.setLowercased(!normalizeCase);
+            }
+            Boolean accentDrop = getBoolAnnotation(ACCENT_DROP, annotations, null);
+            if (accentDrop != null) {
+                out.setNormalizable(accentDrop);
+            }
+            Boolean andSegmenting = getBoolAnnotation(AND_SEGMENTING, annotations,  null);
+            if (andSegmenting != null) {
+                if (andSegmenting) {
+                    out.setSegmentingRule(SegmentingRule.BOOLEAN_AND);
+                } else {
+                    out.setSegmentingRule(SegmentingRule.PHRASE);
+                }
+            }
+        }
+    }
+
+
+    private Substring getOrigin(Inspector annotations) {
+        if (annotations != null) {
+            Inspector origin = getAnnotationAsInspectorOrNull(ORIGIN, getAnnotationMapFromAnnotationInspector(annotations));
+            if (origin == null) {
+                return null;
+            }
+            final String[] original = {null};
+            final Integer[] offset = {null};
+            final Integer[] length = {null};
+
+            origin.traverse((ObjectTraverser) (key, value) -> {
+                switch (key) {
+                    case (ORIGIN_ORIGINAL):
+                        original[0] = value.asString();
+                        break;
+                    case (ORIGIN_OFFSET):
+                        offset[0] = (int) value.asDouble();
+                        break;
+                    case (ORIGIN_LENGTH):
+                        length[0] = (int) value.asDouble();
+                        break;
+                }
+
+
+            });
+            return new Substring(offset[0], length[0] + offset[0], original[0]);
+        }
+        return null;
+    }
+
+
+    @NonNull
+    private Item instantiateSameElementItem(String field, String key, Inspector value) {
+        assertHasOperator(key, SAME_ELEMENT);
+
+        SameElementItem sameElement = new SameElementItem(field);
+        // All terms below sameElement are relative to this.
+        getChildren(value).traverse((ArrayTraverser) (index, term) -> {
+            sameElement.addItem(walkJson(term));
+        });
+
+        return sameElement;
+    }
+
+
+    @NonNull
+    private Item instantiatePhraseItem(String field, String key, Inspector value) {
+        assertHasOperator(key, PHRASE);
+        HashMap<String, Inspector> annotations = getAnnotationMap(value);
+
+        PhraseItem phrase = new PhraseItem();
+        phrase.setIndexName(field);
+        HashMap<Integer, Inspector> children = getChildrenMap(value);
+
+        for (Inspector word :  children.values())
+            if (word.type() == STRING) phrase.addItem(new WordItem(word.asString()));
+            else if (word.type() == OBJECT && word.field(PHRASE).valid()) {
+                    phrase.addItem(instantiatePhraseItem(field, key, getChildren(word)));
+            }
+        return leafStyleSettings(getAnnotations(value), phrase);
+    }
+
+
+    @NonNull
+    private Item instantiateNearItem(String field, String key, Inspector value) {
+        assertHasOperator(key, NEAR);
+
+        NearItem near = new NearItem();
+        near.setIndexName(field);
+
+        HashMap<Integer, Inspector> children = getChildrenMap(value);
+
+        for (Inspector word :  children.values()){
+            near.addItem(new WordItem(word.asString(), field));
+        }
+
+        Integer distance = getIntegerAnnotation(DISTANCE, getAnnotationMap(value), null);
+
+        if (distance != null) {
+            near.setDistance((int)distance);
+        }
+        return near;
+    }
+
+
+    @NonNull
+    private Item instantiateONearItem(String field, String key, Inspector value) {
+        assertHasOperator(key, ONEAR);
+
+        NearItem onear = new ONearItem();
+        onear.setIndexName(field);
+        HashMap<Integer, Inspector> children = getChildrenMap(value);
+
+        for (Inspector word :  children.values()){
+            onear.addItem(new WordItem(word.asString(), field));
+        }
+
+        Integer distance = getIntegerAnnotation(DISTANCE, getAnnotationMap(value), null);
+        if (distance != null) {
+            onear.setDistance(distance);
+        }
+        return onear;
+    }
+
+
+    @NonNull
+    private Item instantiateEquivItem(String field, String key, Inspector value) {
+
+        HashMap<Integer, Inspector> children = getChildrenMap(value);
+        Preconditions.checkArgument(children.size() >= 2, "Expected 2 or more arguments, got %s.", children.size());
+
+        EquivItem equiv = new EquivItem();
+        equiv.setIndexName(field);
+
+        for (Inspector word :  children.values()){
+            if (word.type() == STRING || word.type() == LONG || word.type() == DOUBLE){
+                equiv.addItem(new WordItem(word.asString(), field));
+            }
+            if (word.type() == OBJECT){
+                word.traverse((ObjectTraverser) (key2, value2) -> {
+                    assertHasOperator(key2, PHRASE);
+                    equiv.addItem(instantiatePhraseItem(field, key2, value2));
+                });
+            }
+        }
+
+        return leafStyleSettings(getAnnotations(value), equiv);
+    }
+
+
+    private Item instantiateWordAlternativesItem(String field, String key, Inspector value) {
+        HashMap<Integer, Inspector> children = getChildrenMap(value);
+        Preconditions.checkArgument(children.size() >= 1, "Expected 1 or more arguments, got %s.", children.size());
+        Preconditions.checkArgument(children.get(0).type() == OBJECT, "Expected OBJECT, got %s.", children.get(0).type());
+
+        List<WordAlternativesItem.Alternative> terms = new ArrayList<>();
+
+        children.get(0).traverse((ObjectTraverser) (keys, values) -> {
+            terms.add(new WordAlternativesItem.Alternative(keys, values.asDouble()));
+        });
+        return leafStyleSettings(getAnnotations(value), new WordAlternativesItem(field, Boolean.TRUE, null, terms));
+    }
+
+
+    //  Not in use yet
+    @NonNull
+    private String getIndex(String field) {
+        Preconditions.checkArgument(indexFactsSession.isIndex(field), "Field '%s' does not exist.", field);
+        //return indexFactsSession.getCanonicName(field);
+        return field;
+    }
+
+
+    private static void assertHasOperator(String key, String expectedKey) {
+        Preconditions.checkArgument(key.equals(expectedKey), "Expected operator %s, got %s.", expectedKey, key);
+    }
+
+
+    private static IllegalArgumentException newUnexpectedArgumentException(Object actual, Object... expected) {
+        StringBuilder out = new StringBuilder("Expected ");
+        for (int i = 0, len = expected.length; i < len; ++i) {
+            out.append(expected[i]);
+            if (i < len - 2) {
+                out.append(", ");
+            } else if (i < len - 1) {
+                out.append(" or ");
+            }
+        }
+        out.append(", got ").append(actual).append(".");
+        return new IllegalArgumentException(out.toString());
+    }
+
+
+    private List<Inspector> valueListFromInspector(Inspector inspector){
+        List<Inspector> inspectorList = new ArrayList<>();
+        inspector.traverse((ArrayTraverser) (key, value) -> inspectorList.add(value));
+        return inspectorList;
+    }
+
+
+    private void connectItems() {
+        for (ConnectedItem entry : connectedItems) {
+            TaggableItem to = identifiedItems.get(entry.toId);
+            Preconditions.checkNotNull(to,
+                    "Item '%s' was specified to connect to item with ID %s, which does not "
+                            + "exist in the query.", entry.fromItem,
+                    entry.toId);
+            entry.fromItem.setConnectivity((Item) to, entry.weight);
+        }
+    }
+
+
+    private static final class ConnectedItem {
+
+        final double weight;
+        final int toId;
+        final TaggableItem fromItem;
+
+        ConnectedItem(TaggableItem fromItem, int toId, double weight) {
+            this.weight = weight;
+            this.toId = toId;
+            this.fromItem = fromItem;
+        }
+    }
+
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/query/parser/Parsable.java b/container-search/src/main/java/com/yahoo/search/query/parser/Parsable.java
index e5941a90b83..64fb201fe21 100644
--- a/container-search/src/main/java/com/yahoo/search/query/parser/Parsable.java
+++ b/container-search/src/main/java/com/yahoo/search/query/parser/Parsable.java
@@ -3,6 +3,7 @@ package com.yahoo.search.query.parser;
 
 import com.yahoo.language.Language;
 import com.yahoo.search.query.Model;
+import com.yahoo.search.query.Select;
 
 import java.util.Collection;
 import java.util.HashSet;
@@ -36,6 +37,7 @@ public final class Parsable {
     private String defaultIndexName;
     private Language language; // TODO: Initialize to UNKNOWN
     private Optional<Language> explicitLanguage = Optional.empty();
+    private Select select;
 
     /** If this is set it will be used to determine the language, if not set explicitly */
     private Optional<Model> model = Optional.empty();
@@ -133,6 +135,15 @@ public final class Parsable {
         return this;
     }
 
+    public Parsable setSelect(Select select){
+        this.select = select;
+        return this;
+    }
+
+    public Select getSelect(){
+        return this.select;
+    }
+
     public static Parsable fromQueryModel(Model model) {
         return new Parsable()
                 .setModel(model)
@@ -141,7 +152,11 @@ public final class Parsable {
                 .setExplicitLanguage(Optional.ofNullable(model.getLanguage()))
                 .setDefaultIndexName(model.getDefaultIndex())
                 .addSources(model.getSources())
-                .addRestricts(model.getRestrict());
+                .addRestricts(model.getRestrict())
+                .setSelect(model.getParent().getSelect());
     }
 
+
+
+
 }
diff --git a/container-search/src/main/java/com/yahoo/search/query/parser/ParserFactory.java b/container-search/src/main/java/com/yahoo/search/query/parser/ParserFactory.java
index 8d008abaac2..69d46527255 100644
--- a/container-search/src/main/java/com/yahoo/search/query/parser/ParserFactory.java
+++ b/container-search/src/main/java/com/yahoo/search/query/parser/ParserFactory.java
@@ -3,6 +3,7 @@ package com.yahoo.search.query.parser;
 
 import com.yahoo.prelude.query.parser.*;
 import com.yahoo.search.Query;
+import com.yahoo.search.query.SelectParser;
 import com.yahoo.search.yql.YqlParser;
 
 /**
@@ -40,6 +41,8 @@ public final class ParserFactory {
                 return new ProgrammaticParser();
             case YQL:
                 return new YqlParser(environment);
+            case SELECT:
+                return new SelectParser(environment);
             default:
                 throw new UnsupportedOperationException(type.toString());
         }
diff --git a/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java b/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java
index 0aea5e96161..71002166b11 100644
--- a/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java
+++ b/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java
@@ -113,6 +113,10 @@ public class QueryProperties extends Properties {
                 if (key.get(1).equals(Ranking.PROPERTIES)) return ranking.getProperties().get(key.rest().rest().toString());
             }
         }
+        else if (key.size()==2 && key.first().equals(Select.SELECT)) {
+            if (key.last().equals(Select.WHERE)) return query.getSelect().getWhereString();
+            if (key.last().equals(Select.GROUPING)) return query.getSelect().getGroupingString();
+        }
         else if (key.size()==2 && key.first().equals(Presentation.PRESENTATION)) {
             if (key.last().equals(Presentation.BOLDING)) return query.getPresentation().getBolding();
             if (key.last().equals(Presentation.SUMMARY)) return query.getPresentation().getSummary();
@@ -247,6 +251,13 @@ public class QueryProperties extends Properties {
                 else if ( ! key.last().equals(Presentation.REPORT_COVERAGE)) // TODO: Change this line to "else" on Vespa 7
                     throwIllegalParameter(key.last(), Presentation.PRESENTATION);
             }
+            else if (key.size()==2 && key.first().equals(Select.SELECT)) {
+                if (key.last().equals(Select.WHERE)){
+                    query.getSelect().setWhere(asString(value, ""));
+                } else if (key.last().equals(Select.GROUPING)) {
+                    query.getSelect().setGrouping(asString(value, ""));
+                }
+            }
             else if (key.first().equals("rankfeature") || key.first().equals("featureoverride") ) { // featureoverride is deprecated
                 setRankingFeature(query, key.rest().toString(), toSpecifiedType(key.rest().toString(), value, profileRegistry.getTypeRegistry().getComponent("features")));
             } else if (key.first().equals("rankproperty")) {
diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/PhaseNames.java b/container-search/src/main/java/com/yahoo/search/searchchain/PhaseNames.java
index 1b50ae20183..ec79979387b 100644
--- a/container-search/src/main/java/com/yahoo/search/searchchain/PhaseNames.java
+++ b/container-search/src/main/java/com/yahoo/search/searchchain/PhaseNames.java
@@ -7,7 +7,7 @@ package com.yahoo.search.searchchain;
  * a searcher should depend on some explicit functionality, not these
  * checkpoints.
  *
- * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ * @author Steinar Knutsen
  */
 public final class PhaseNames {
     private PhaseNames() {
diff --git a/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java
index 6bad032600c..e0e9042e1a3 100644
--- a/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java
+++ b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java
@@ -789,7 +789,7 @@ public class YqlParser implements Parser {
             OperatorNode<ExpressionOperator> groupingAst = ast.<List<OperatorNode<ExpressionOperator>>> getArgument(2).get(0);
             GroupingOperation groupingOperation = GroupingOperation.fromString(groupingAst.<String> getArgument(0));
             VespaGroupingStep groupingStep = new VespaGroupingStep(groupingOperation);
-            List<String> continuations = getAnnotation(groupingAst, "continuations", List.class, 
+            List<String> continuations = getAnnotation(groupingAst, "continuations", List.class,
                                                        Collections.emptyList(), "grouping continuations");
             for (String continuation : continuations) {
                 groupingStep.continuations().add(Continuation.fromString(continuation));
diff --git a/container-search/src/main/resources/configdefinitions/provider.def b/container-search/src/main/resources/configdefinitions/provider.def
index 79b09913b49..f9ab305b114 100644
--- a/container-search/src/main/resources/configdefinitions/provider.def
+++ b/container-search/src/main/resources/configdefinitions/provider.def
@@ -35,7 +35,7 @@ yca.ttl int default=0
 yca.retry int default=0
 
 # The form of the serialized query.
-queryType enum { LEGACY, PROGRAMMATIC, YQL } default=LEGACY
+queryType enum { LEGACY, PROGRAMMATIC, YQL, SELECT } default=LEGACY
 
 # How to do pinging against a backend.
 pingOption enum { DISABLE, NORMAL, YCA } default=NORMAL
diff --git a/container-search/src/test/java/com/yahoo/search/handler/test/JSONSearchHandlerTestCase.java b/container-search/src/test/java/com/yahoo/search/handler/test/JSONSearchHandlerTestCase.java
index eea58d5444e..e85a945cc67 100644
--- a/container-search/src/test/java/com/yahoo/search/handler/test/JSONSearchHandlerTestCase.java
+++ b/container-search/src/test/java/com/yahoo/search/handler/test/JSONSearchHandlerTestCase.java
@@ -11,7 +11,10 @@ import com.yahoo.io.IOUtils;
 import com.yahoo.net.HostName;
 import com.yahoo.search.handler.SearchHandler;
 import com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase;
+import com.yahoo.slime.ArrayTraverser;
 import com.yahoo.slime.Inspector;
+import com.yahoo.slime.ObjectTraverser;
+import com.yahoo.slime.Type;
 import com.yahoo.vespa.config.SlimeUtils;
 import org.json.JSONObject;
 import org.junit.After;
@@ -21,8 +24,10 @@ import org.junit.Test;
 import org.junit.rules.TemporaryFolder;
 import java.io.File;
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Map;
 import java.util.HashMap;
+import java.util.stream.Collectors;
 
 import static com.yahoo.jdisc.http.HttpRequest.Method.GET;
 import static org.hamcrest.CoreMatchers.containsString;
@@ -338,6 +343,35 @@ public class JSONSearchHandlerTestCase {
     }
 
 
+    @Test
+    public void testSelectParameter() throws Exception {
+        JSONObject json = new JSONObject();
+
+        JSONObject select = new JSONObject();
+
+            JSONObject where = new JSONObject();
+            where.put("where", "where");
+
+            JSONObject grouping = new JSONObject();
+            grouping.put("grouping", "grouping");
+
+        select.put("where", where);
+        select.put("grouping", grouping);
+
+        json.put("select", select);
+
+
+        // Create mapping
+        Inspector inspector = SlimeUtils.jsonToSlime(json.toString().getBytes("utf-8")).get();
+        Map<String, String> map = new HashMap<>();
+        searchHandler.createRequestMapping(inspector, map, "");
+
+        JSONObject processedWhere = new JSONObject(map.get("select.where"));
+        assertEquals(where.toString(), processedWhere.toString());
+
+        JSONObject processedGrouping = new JSONObject(map.get("select.grouping"));
+        assertEquals(grouping.toString(), processedGrouping.toString());
+    }
 
     @Test
     public void testRequestMapping() throws Exception {
diff --git a/container-search/src/test/java/com/yahoo/select/SelectParserTestCase.java b/container-search/src/test/java/com/yahoo/select/SelectParserTestCase.java
new file mode 100644
index 00000000000..031ba386ad4
--- /dev/null
+++ b/container-search/src/test/java/com/yahoo/select/SelectParserTestCase.java
@@ -0,0 +1,779 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.select;
+
+import com.yahoo.prelude.query.AndItem;
+import com.yahoo.prelude.query.ExactStringItem;
+import com.yahoo.prelude.query.Item;
+import com.yahoo.prelude.query.PhraseItem;
+import com.yahoo.prelude.query.PrefixItem;
+import com.yahoo.prelude.query.RegExpItem;
+import com.yahoo.prelude.query.SegmentingRule;
+import com.yahoo.prelude.query.Substring;
+import com.yahoo.prelude.query.SubstringItem;
+import com.yahoo.prelude.query.SuffixItem;
+import com.yahoo.prelude.query.WeakAndItem;
+import com.yahoo.prelude.query.WordAlternativesItem;
+import com.yahoo.prelude.query.WordItem;
+import com.yahoo.search.Query;
+import com.yahoo.search.federation.ProviderConfig;
+import com.yahoo.search.query.QueryTree;
+import com.yahoo.search.query.Select;
+import com.yahoo.search.query.SelectParser;
+import com.yahoo.search.query.parser.Parsable;
+import com.yahoo.search.query.parser.ParserEnvironment;
+import com.yahoo.search.yql.VespaGroupingStep;
+import org.apache.http.client.utils.URIBuilder;
+import org.json.JSONException;
+import org.json.JSONObject;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+
+/**
+ * Specification for the conversion of Select expressions to Vespa search queries.
+ *
+ * @author henrhoi
+ */
+
+public class SelectParserTestCase {
+
+    private final SelectParser parser = new SelectParser(new ParserEnvironment());
+
+
+    /** WHERE TESTS */
+
+    @Test
+    public void test_contains() throws Exception {
+        JSONObject json = new JSONObject();
+        List<String> contains = Arrays.asList("default", "foo");
+        json.put("contains", contains);
+        assertParse(json.toString(), "default:foo");
+    }
+
+    @Test
+    public void test() {
+        assertParse("{'contains' : ['title', 'madonna']}",
+                "title:madonna");
+    }
+
+
+    @Test
+    public void testDottedFieldNames() {
+        assertParse("{ 'contains' : ['my.nested.title', 'madonna']}",
+                "my.nested.title:madonna");
+    }
+
+
+
+    @Test
+    public void testOr() throws Exception {
+        JSONObject json_two_or = new JSONObject();
+        JSONObject json_three_or = new JSONObject();
+        List<String> contains1 = Arrays.asList("title", "madonna");
+        List<String> contains2 = Arrays.asList("title", "saint");
+        List<String> contains3 = Arrays.asList("title", "angel");
+
+        JSONObject contains_json1 = new JSONObject();
+        JSONObject contains_json2 = new JSONObject();
+        JSONObject contains_json3 = new JSONObject();
+        contains_json1.put("contains", contains1);
+        contains_json2.put("contains", contains2);
+        contains_json3.put("contains", contains3);
+
+        json_two_or.put("or", Arrays.asList(contains_json1, contains_json2));
+        json_three_or.put("or", Arrays.asList(contains_json1, contains_json2, contains_json3));
+
+        assertParse(json_two_or.toString(), "OR title:madonna title:saint");
+        assertParse(json_three_or.toString(), "OR title:madonna title:saint title:angel");
+    }
+
+    @Test
+    public void testAnd() throws Exception{
+        JSONObject json_two_and = new JSONObject();
+        JSONObject json_three_and = new JSONObject();
+        List<String> contains1 = Arrays.asList("title", "madonna");
+        List<String> contains2 = Arrays.asList("title", "saint");
+        List<String> contains3 = Arrays.asList("title", "angel");
+
+        JSONObject contains_json1 = new JSONObject();
+        JSONObject contains_json2 = new JSONObject();
+        JSONObject contains_json3 = new JSONObject();
+        contains_json1.put("contains", contains1);
+        contains_json2.put("contains", contains2);
+        contains_json3.put("contains", contains3);
+
+        json_two_and.put("and", Arrays.asList(contains_json1, contains_json2));
+        json_three_and.put("and", Arrays.asList(contains_json1, contains_json2, contains_json3));
+
+        assertParse(json_two_and.toString(), "AND title:madonna title:saint");
+        assertParse(json_three_and.toString(), "AND title:madonna title:saint title:angel");
+    }
+
+    @Test
+    public void testAndNot() throws JSONException {
+        JSONObject json_and_not = new JSONObject();
+        List<String> contains1 = Arrays.asList("title", "madonna");
+        List<String> contains2 = Arrays.asList("title", "saint");
+
+        JSONObject contains_json1 = new JSONObject();
+        JSONObject contains_json2 = new JSONObject();
+        contains_json1.put("contains", contains1);
+        contains_json2.put("contains", contains2);
+
+        json_and_not.put("and_not", Arrays.asList(contains_json1, contains_json2));
+
+        assertParse(json_and_not.toString(),
+                "+title:madonna -title:saint");
+    }
+
+
+    @Test
+    public void testLessThan() throws JSONException {
+        JSONObject range_json = new JSONObject();
+        JSONObject operators = new JSONObject();
+        operators.put("<", 500);
+
+        List<Object> range = Arrays.asList("price", operators);
+
+        range_json.put("range", range);
+
+        assertParse(range_json.toString(),
+                "price:<500");
+    }
+
+    @Test
+    public void testGreaterThan() throws JSONException {
+        JSONObject range_json = new JSONObject();
+        JSONObject operators = new JSONObject();
+        operators.put(">", 500);
+
+        List<Object> range = Arrays.asList("price", operators);
+
+        range_json.put("range", range);
+
+        assertParse(range_json.toString(),
+                "price:>500");
+    }
+
+
+    @Test
+    public void testLessThanOrEqual() throws JSONException {
+        JSONObject range_json = new JSONObject();
+        JSONObject operators = new JSONObject();
+        operators.put("<=", 500);
+
+        List<Object> range = Arrays.asList("price", operators);
+
+        range_json.put("range", range);
+
+        assertParse(range_json.toString(),
+                "price:[;500]");
+    }
+
+    @Test
+    public void testGreaterThanOrEqual() throws JSONException {
+        JSONObject range_json = new JSONObject();
+        JSONObject operators = new JSONObject();
+        operators.put(">=", 500);
+
+        List<Object> range = Arrays.asList("price", operators);
+
+        range_json.put("range", range);
+
+        assertParse(range_json.toString(),
+                "price:[500;]");
+    }
+
+    @Test
+    public void testEquality() throws JSONException {
+        JSONObject range_json = new JSONObject();
+        JSONObject operators = new JSONObject();
+        operators.put("=", 500);
+
+        List<Object> range = Arrays.asList("price", operators);
+
+        range_json.put("range", range);
+
+        assertParse(range_json.toString(),
+                "price:500");
+    }
+
+    @Test
+    public void testNegativeLessThan() throws JSONException {
+        JSONObject range_json = new JSONObject();
+        JSONObject operators = new JSONObject();
+        operators.put("<", -500);
+
+        List<Object> range = Arrays.asList("price", operators);
+
+        range_json.put("range", range);
+
+        assertParse(range_json.toString(),
+                "price:<-500");
+    }
+
+    @Test
+    public void testNegativeGreaterThan() throws JSONException {
+        JSONObject range_json = new JSONObject();
+        JSONObject operators = new JSONObject();
+        operators.put(">", -500);
+
+        List<Object> range = Arrays.asList("price", operators);
+
+        range_json.put("range", range);
+
+        assertParse(range_json.toString(),
+                "price:>-500");
+    }
+
+    @Test
+    public void testNegativeLessThanOrEqual() throws JSONException {
+        JSONObject range_json = new JSONObject();
+        JSONObject operators = new JSONObject();
+        operators.put("<=", -500);
+
+        List<Object> range = Arrays.asList("price", operators);
+
+        range_json.put("range", range);
+
+        assertParse(range_json.toString(),
+                "price:[;-500]");
+    }
+
+    @Test
+    public void testNegativeGreaterThanOrEqual() throws JSONException {
+        JSONObject range_json = new JSONObject();
+        JSONObject operators = new JSONObject();
+        operators.put(">=", -500);
+
+        List<Object> range = Arrays.asList("price", operators);
+
+        range_json.put("range", range);
+
+        assertParse(range_json.toString(),
+                "price:[-500;]");
+    }
+
+    @Test
+    public void testNegativeEquality() throws JSONException {
+        JSONObject range_json = new JSONObject();
+        JSONObject operators = new JSONObject();
+        operators.put("=", -500);
+
+        List<Object> range = Arrays.asList("price", operators);
+
+        range_json.put("range", range);
+
+        assertParse(range_json.toString(),
+                "price:-500");
+    }
+
+    @Test
+    public void testAnnotatedLessThan() {
+        String jsonString = "{ \"range\": { \"children\" : [\"price\", {\"<\" : -500}], \"attributes\" : {\"filter\" : true} } }";
+        assertParse(jsonString, "|price:<-500");
+    }
+
+    @Test
+    public void testAnnotatedGreaterThan() {
+        String jsonString = "{ \"range\": { \"children\" : [\"price\", {\">\" : 500}], \"attributes\" : {\"filter\" : true} } }";
+        assertParse(jsonString, "|price:>500");
+    }
+
+    @Test
+    public void testAnnotatedLessThanOrEqual() {
+        String jsonString = "{ \"range\": { \"children\" : [\"price\", {\"<=\" : -500}], \"attributes\" : {\"filter\" : true} } }";
+        assertParse(jsonString, "|price:[;-500]");
+    }
+
+    @Test
+    public void testAnnotatedGreaterThanOrEqual() {
+        String jsonString = "{ \"range\": { \"children\" : [\"price\", {\">=\" : 500}], \"attributes\" : {\"filter\" : true} } }";
+        assertParse(jsonString, "|price:[500;]");
+    }
+
+
+    @Test
+    public void testAnnotatedEquality() {
+        String jsonString = "{ \"range\": { \"children\" : [\"price\", {\"=\" : -500}], \"attributes\" : {\"filter\" : true} } }";
+        assertParse(jsonString, "|price:-500");
+    }
+
+    @Test
+    public void testTermAnnotations() {
+        assertEquals("merkelapp",
+                getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"colors\"], \"attributes\" : {\"label\" : \"merkelapp\"} } }").getLabel());
+        assertEquals("another",
+                getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"colors\"], \"attributes\" : {\"annotations\" : {\"cox\" : \"another\"} } } }").getAnnotation("cox"));
+        assertEquals(23.0, getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"colors\"], \"attributes\" : {\"significance\" : 23.0 } } }").getSignificance(), 1E-6);
+        assertEquals(150, getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"colors\"], \"attributes\" : {\"weight\" : 150 } } }").getWeight());
+        assertFalse(getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"colors\"], \"attributes\" : {\"usePositionData\" : false } } }").usePositionData());
+        assertTrue(getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"colors\"], \"attributes\" : {\"filter\" : true } } }").isFilter());
+        assertFalse(getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"colors\"], \"attributes\" : {\"ranked\" : false } } }").isRanked());
+        Substring origin = getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"colors\"], \"attributes\" : {\"origin\": {\"original\": \"abc\", \"offset\": 1, \"length\": 2}} } }").getOrigin();
+        assertEquals("abc", origin.string);
+        assertEquals(1, origin.start);
+        assertEquals(3, origin.end);
+    }
+
+
+    @Test
+    public void testSameElement() {
+        assertParse("{ \"contains\": [ \"baz\", {\"sameElement\" : [ { \"contains\" : [\"f1\", \"a\"] }, { \"contains\" : [\"f2\", \"b\"] } ]} ] }",
+                "baz:{f1:a f2:b}");
+
+        assertParse("{ \"contains\": [ \"baz\", {\"sameElement\" : [ { \"contains\" : [\"f1\", \"a\"] }, {\"range\":[\"f2\",{\"=\":10}] } ]} ] }",
+                "baz:{f1:a f2:10}");
+
+        assertParse("{ \"contains\": [ \"baz\", {\"sameElement\" : [ { \"contains\" : [\"key\", \"a\"] }, {\"range\":[\"value.f2\",{\"=\":10}] } ]} ] }",
+                "baz:{key:a value.f2:10}");
+    }
+
+    @Test
+    public void testPhrase() {
+        assertParse("{ \"contains\": [ \"baz\", {\"phrase\" : [ \"a\", \"b\"] } ] }",
+                "baz:\"a b\"");
+    }
+
+    @Test
+    public void testNestedPhrase() {
+        assertParse("{ \"contains\": [ \"baz\", {\"phrase\" : [ \"a\", \"b\", {\"phrase\" : [ \"c\", \"d\"] }] } ] }",
+                "baz:\"a b c d\"");
+    }
+
+    @Test
+    public void testStemming() {
+        assertTrue(getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"colors\"], \"attributes\" : {\"stem\" : false} } }").isStemmed());
+        assertFalse(getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"colors\"], \"attributes\" : {\"stem\" : true} } }").isStemmed());
+        assertFalse(getRootWord("{ \"contains\": [\"baz\", \"colors\"] }").isStemmed());
+    }
+
+    @Test
+    public void testRaw() {
+        Item root = parseWhere("{ \"contains\":[ \"baz\", \"yoni jo dima\" ] }").getRoot();
+        assertTrue(root instanceof WordItem);
+        assertFalse(root instanceof ExactStringItem);
+        assertEquals("yoni jo dima", ((WordItem)root).getWord());
+
+        root = parseWhere("{ \"contains\": { \"children\" : [\"baz\", \"yoni jo dima\"], \"attributes\" : {\"grammar\" : \"raw\"} } }").getRoot();
+        assertTrue(root instanceof WordItem);
+        assertFalse(root instanceof ExactStringItem);
+        assertEquals("yoni jo dima", ((WordItem)root).getWord());
+    }
+
+    @Test
+    public void testAccentDropping() {
+        assertFalse(getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"colors\"], \"attributes\" : {\"accentDrop\" : false} } }").isNormalizable());
+        assertTrue(getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"colors\"], \"attributes\" : {\"accentDrop\" : true} } }").isNormalizable());
+        assertTrue(getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"colors\"] } }").isNormalizable());
+    }
+
+    @Test
+    public void testCaseNormalization() {
+        assertTrue(getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"colors\"], \"attributes\" : {\"normalizeCase\" : false} } }").isLowercased());
+        assertFalse(getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"colors\"], \"attributes\" : {\"normalizeCase\" : true} } }").isLowercased());
+        assertFalse(getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"colors\"] } }").isLowercased());
+    }
+
+    @Test
+    public void testSegmentingRule() {
+        assertEquals(SegmentingRule.PHRASE,
+                getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"colors\"], \"attributes\" : {\"andSegmenting\" : false} } }").getSegmentingRule());
+        assertEquals(SegmentingRule.BOOLEAN_AND,
+                getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"colors\"], \"attributes\" : {\"andSegmenting\" : true} } }").getSegmentingRule());
+        assertEquals(SegmentingRule.LANGUAGE_DEFAULT,
+                getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"colors\"] } }").getSegmentingRule());
+    }
+
+    @Test
+    public void testNfkc() {
+        assertEquals("a\u030a", getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"a\\u030a\"], \"attributes\" : {\"nfkc\" : false} } }").getWord());
+        assertEquals("\u00e5", getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"a\\u030a\"], \"attributes\" : {\"nfkc\" : true} } }").getWord());
+        assertEquals("No NKFC by default", "a\u030a", getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"a\\u030a\"] } } ").getWord());
+    }
+
+    @Test
+    public void testImplicitTransforms() {
+        assertFalse(getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"cox\"], \"attributes\" : {\"implicitTransforms\" : false} } }").isFromQuery());
+        assertTrue(getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"cox\"], \"attributes\" : {\"implicitTransforms\" : true} } }").isFromQuery());
+        assertTrue(getRootWord("{ \"contains\": { \"children\" : [\"baz\", \"cox\"] } }").isFromQuery());
+    }
+
+    @Test
+    public void testConnectivity() {
+        QueryTree parsed = parseWhere("{ \"and\": [ {\"contains\" : { \"children\" : [\"title\", \"madonna\"], \"attributes\" : {\"id\": 1, \"connectivity\": {\"id\": 3, \"weight\": 7.0}} } }, " +
+                "{ \"contains\" : { \"children\" : [\"title\", \"saint\"], \"attributes\" : {\"id\": 2} } }, " +
+                "{ \"contains\" : { \"children\" : [\"title\", \"angel\"], \"attributes\" : {\"id\": 3} } } ] }");
+        assertEquals("AND title:madonna title:saint title:angel", parsed.toString());
+
+        AndItem root = (AndItem)parsed.getRoot();
+        WordItem first = (WordItem)root.getItem(0);
+        WordItem second = (WordItem)root.getItem(1);
+        WordItem third = (WordItem)root.getItem(2);
+        assertTrue(first.getConnectedItem() == third);
+        assertEquals(first.getConnectivity(), 7.0d, 1E-6);
+        assertNull(second.getConnectedItem());
+
+        assertParseFail("{ \"and\": [ {\"contains\" : { \"children\" : [\"title\", \"madonna\"], \"attributes\" : {\"id\": 1, \"connectivity\": {\"id\": 4, \"weight\": 7.0}} } }, " +
+                "{ \"contains\" : { \"children\" : [\"title\", \"saint\"], \"attributes\" : {\"id\": 2} } }, " +
+                "{ \"contains\" : { \"children\" : [\"title\", \"angel\"], \"attributes\" : {\"id\": 3} } } ] }",
+                new NullPointerException("Item 'title:madonna' was specified to connect to item with ID 4, " +
+                "which does not exist in the query."));
+    }
+
+    @Test
+    public void testAnnotatedPhrase() {
+        QueryTree parsed = parseWhere("{ \"contains\": [\"baz\", { \"phrase\": { \"children\": [\"a\", \"b\"], \"attributes\": { \"label\": \"hello world\" } } }] }");
+        assertEquals("baz:\"a b\"", parsed.toString());
+        PhraseItem phrase = (PhraseItem)parsed.getRoot();
+        assertEquals("hello world", phrase.getLabel());
+    }
+
+    @Test
+    public void testRange() {
+        QueryTree parsed = parseWhere("{ \"range\": [\"baz\", { \">=\": 1, \"<=\": 8 }] }");
+        assertEquals("baz:[1;8]", parsed.toString());
+    }
+
+    @Test
+    public void testNegativeRange() {
+        QueryTree parsed = parseWhere("{ \"range\": [\"baz\", { \">=\": -8, \"<=\": -1 }] }");
+        assertEquals("baz:[-8;-1]", parsed.toString());
+    }
+
+    @Test
+    public void testRangeIllegalArguments() {
+        assertParseFail("{ \"range\": [\"baz\", { \">=\": \"cox\", \"<=\": -1 }] }",
+                new IllegalArgumentException("Expected operator LITERAL, got READ_FIELD."));
+    }
+
+    @Test
+    public void testNear() {
+        assertParse("{ \"contains\": [\"description\", { \"near\": [\"a\", \"b\"] }] }",
+                "NEAR(2) description:a description:b");
+        assertParse("{ \"contains\": [\"description\", { \"near\": { \"children\": [\"a\", \"b\"], \"attributes\": { \"distance\": 100 } } } ] }",
+                "NEAR(100) description:a description:b");
+    }
+
+    @Test
+    public void testOrderedNear() {
+        assertParse("{ \"contains\": [\"description\", { \"onear\": [\"a\", \"b\"] }] }",
+                "ONEAR(2) description:a description:b");
+        assertParse("{ \"contains\": [\"description\", { \"onear\": { \"children\": [\"a\", \"b\"], \"attributes\": { \"distance\": 100 } } } ] }",
+                "ONEAR(100) description:a description:b");
+    }
+
+    @Test
+    public void testWand() {
+        assertParse("{ \"wand\": [\"description\", { \"a\": 1, \"b\": 2 }] }",
+                "WAND(10,0.0,1.0) description{[1]:\"a\",[2]:\"b\"}");
+        assertParse("{ \"wand\": { \"children\": [\"description\", { \"a\": 1, \"b\": 2 }], \"attributes\": { \"scoreThreshold\": 13.3, \"targetNumHits\": 7, \"thresholdBoostFactor\": 2.3 } } }",
+                "WAND(7,13.3,2.3) description{[1]:\"a\",[2]:\"b\"}");
+    }
+
+    @Test
+    public void testNumericWand() {
+        String numWand = "WAND(10,0.0,1.0) description{[1]:\"11\",[2]:\"37\"}";
+        assertParse("{ \"wand\" : [\"description\", [[11,1], [37,2]] ]}", numWand);
+        assertParseFail("{ \"wand\" : [\"description\", 12] }",
+                new IllegalArgumentException("Expected ARRAY or OBJECT, got LONG."));
+    }
+
+    @Test
+    public void testWeightedSet() {
+        assertParse("{ \"weightedSet\" : [\"description\", {\"a\":1, \"b\":2} ]}",
+                "WEIGHTEDSET description{[1]:\"a\",[2]:\"b\"}");
+        assertParseFail("{ \"weightedSet\" : [\"description\", {\"a\":\"g\", \"b\":2} ]}",
+                new IllegalArgumentException("Expected operator LITERAL, got READ_FIELD."));
+        assertParseFail("{ \"weightedSet\" : [\"description\" ]}",
+                new IllegalArgumentException("Expected 2 arguments, got 1."));
+    }
+
+    @Test
+    public void testDotProduct() {
+        assertParse("{ \"dotProduct\" : [\"description\", {\"a\":1, \"b\":2} ]}",
+                "DOTPRODUCT description{[1]:\"a\",[2]:\"b\"}");
+        assertParse("{ \"dotProduct\" : [\"description\", {\"a\":2} ]}",
+                "DOTPRODUCT description{[2]:\"a\"}");
+    }
+
+    @Test
+    public void testPredicate() {
+        assertParse("{ \"predicate\" : [\"predicate_field\", {\"gender\":\"male\", \"hobby\":[\"music\", \"hiking\"]}, {\"age\":23} ]}",
+                "PREDICATE_QUERY_ITEM gender=male, hobby=music, hobby=hiking, age:23");
+        assertParse("{ \"predicate\" : [\"predicate_field\", 0, \"void\" ]}",
+                "PREDICATE_QUERY_ITEM ");
+    }
+
+    @Test
+    public void testRank() {
+        assertParse("{ \"rank\": [{ \"contains\": [\"a\", \"A\"] }, { \"contains\": [\"b\", \"B\"] } ] }",
+                "RANK a:A b:B");
+        assertParse("{ \"rank\": [{ \"contains\": [\"a\", \"A\"] }, { \"contains\": [\"b\", \"B\"] }, { \"contains\": [\"c\", \"C\"] } ] }",
+                "RANK a:A b:B c:C");
+        assertParse("{ \"rank\": [{ \"contains\": [\"a\", \"A\"] }, { \"or\": [{ \"contains\": [\"b\", \"B\"] }, { \"contains\": [\"c\", \"C\"] }] }] }",
+                "RANK a:A (OR b:B c:C)");
+    }
+
+    @Test
+    public void testWeakAnd() {
+        assertParse("{ \"weakAnd\": [{ \"contains\": [\"a\", \"A\"] }, { \"contains\": [\"b\", \"B\"] } ] }",
+                "WAND(100) a:A b:B");
+        assertParse("{ \"weakAnd\": { \"children\" : [{ \"contains\": [\"a\", \"A\"] }, { \"contains\": [\"b\", \"B\"] } ], \"attributes\" : {\"targetNumHits\": 37} }}",
+                "WAND(37) a:A b:B");
+
+        QueryTree tree = parseWhere("{ \"weakAnd\": { \"children\" : [{ \"contains\": [\"a\", \"A\"] }, { \"contains\": [\"b\", \"B\"] } ], \"attributes\" : {\"scoreThreshold\": 41}}}");
+        assertEquals("WAND(100) a:A b:B", tree.toString());
+        assertEquals(WeakAndItem.class, tree.getRoot().getClass());
+        assertEquals(41, ((WeakAndItem)tree.getRoot()).getScoreThreshold());
+    }
+
+    @Test
+    public void testEquiv() {
+        assertParse("{ \"contains\" : [\"fieldName\", {\"equiv\" : [\"A\",\"B\"]}]}",
+                "EQUIV fieldName:A fieldName:B");
+
+        assertParse("{ \"contains\" : [\"fieldName\", {\"equiv\" : [\"ny\",{\"phrase\" : [ \"new\",\"york\" ] } ] } ] }",
+                "EQUIV fieldName:ny fieldName:\"new york\"");
+
+        assertParseFail("{ \"contains\" : [\"fieldName\", {\"equiv\" : [\"ny\"] } ] }",
+                new IllegalArgumentException("Expected 2 or more arguments, got 1."));
+        assertParseFail("{ \"contains\" : [\"fieldName\", {\"equiv\" : [\"ny\",{\"nalle\" : [ \"void\" ] } ] } ] }",
+                new IllegalArgumentException("Expected operator phrase, got nalle."));
+        assertParseFail("{ \"contains\" : [\"fieldName\", {\"equiv\" : [\"ny\", 42]}]}",
+                new IllegalArgumentException("Word item word can not be empty"));
+    }
+
+    @Test
+    public void testAffixItems() {
+        assertRootClass("{ \"contains\" : { \"children\" : [\"baz\", \"colors\"], \"attributes\" : {\"suffix\": true} } }",
+                SuffixItem.class);
+
+
+        assertRootClass("{ \"contains\" : { \"children\" : [\"baz\", \"colors\"], \"attributes\" : {\"prefix\": true} } }",
+                PrefixItem.class);
+        assertRootClass("{ \"contains\" : { \"children\" : [\"baz\", \"colors\"], \"attributes\" : {\"substring\": true} } }",
+                SubstringItem.class);
+        assertParseFail("{ \"contains\" : { \"children\" : [\"baz\", \"colors\"], \"attributes\" : {\"suffix\": true, \"prefix\" : true} } }",
+                new IllegalArgumentException("Only one of prefix, substring and suffix can be set."));
+        assertParseFail("{ \"contains\" : { \"children\" : [\"baz\", \"colors\"], \"attributes\" : {\"suffix\": true, \"substring\" : true} } }",
+                new IllegalArgumentException("Only one of prefix, substring and suffix can be set."));
+    }
+
+    @Test
+    public void testLongNumberInSimpleExpression() {
+        assertParse("{ \"range\" : [ \"price\", { \"=\" : 8589934592 }]}",
+                "price:8589934592");
+    }
+
+    @Test
+    public void testNegativeLongNumberInSimpleExpression() {
+        assertParse("{ \"range\" : [ \"price\", { \"=\" : -8589934592 }]}",
+                "price:-8589934592");
+    }
+
+    @Test
+    public void testNegativeHitLimit() {
+        assertParse(
+                "{ \"range\" : { \"children\":[ \"foo\", { \">=\" : 0, \"<=\" : 1 }], \"attributes\" : {\"hitLimit\": -38 } } }",
+                "foo:[0;1;-38]");
+    }
+
+    @Test
+    public void testRangeSearchHitPopulationOrdering() {
+        assertParse("{ \"range\" : { \"children\":[ \"foo\", { \">=\" : 0, \"<=\" : 1 }], \"attributes\" : {\"hitLimit\": 38 ,\"ascending\": true} } }", "foo:[0;1;38]");
+        assertParse("{ \"range\" : { \"children\":[ \"foo\", { \">=\" : 0, \"<=\" : 1 }], \"attributes\" : {\"hitLimit\": 38 ,\"ascending\": false} } }", "foo:[0;1;-38]");
+        assertParse("{ \"range\" : { \"children\":[ \"foo\", { \">=\" : 0, \"<=\" : 1 }], \"attributes\" : {\"hitLimit\": 38 ,\"descending\": true} } }", "foo:[0;1;-38]");
+        assertParse("{ \"range\" : { \"children\":[ \"foo\", { \">=\" : 0, \"<=\" : 1 }], \"attributes\" : {\"hitLimit\": 38 ,\"descending\": false} } }", "foo:[0;1;38]");
+
+        boolean gotExceptionFromParse = false;
+        try {
+            parseWhere("{ \"range\" : { \"children\":[ \"foo\", { \">=\" : 0, \"<=\" : 1 }], \"attributes\" : {\"hitLimit\": 38, \"ascending\": true, \"descending\": false} } }");
+        } catch (IllegalArgumentException e) {
+            assertTrue("Expected information about abuse of settings.",
+                    e.getMessage().contains("both ascending and descending ordering set"));
+            gotExceptionFromParse = true;
+        }
+        assertTrue(gotExceptionFromParse);
+    }
+
+    // NB: Uses operator-keys to set bounds, not annotations
+    @Test
+    public void testOpenIntervals() {
+        assertParse("{ \"range\" : { \"children\":[ \"title\", { \">=\" : 0.0, \"<=\" : 500.0 }] } }" +
+                        "select * from sources * where range(title, 0.0, 500.0);",
+                "title:[0.0;500.0]");
+        assertParse(
+                "{ \"range\" : { \"children\":[ \"title\", { \">\" : 0.0, \"<\" : 500.0 }] } }",
+                "title:<0.0;500.0>");
+        assertParse(
+                "{ \"range\" : { \"children\":[ \"title\", { \">\" : 0.0, \"<=\" : 500.0 }]  } }",
+                "title:<0.0;500.0]");
+        assertParse(
+                "{ \"range\" : { \"children\":[ \"title\", { \">=\" : 0.0, \"<\" : 500.0 }] } }",
+                "title:[0.0;500.0>");
+    }
+
+    @Test
+    public void testRegexp() {
+        QueryTree x = parseWhere("{ \"matches\" : [\"foo\", \"a b\"]}");
+        Item root = x.getRoot();
+        assertSame(RegExpItem.class, root.getClass());
+        assertEquals("a b", ((RegExpItem) root).stringValue());
+    }
+
+    @Test
+    public void testWordAlternatives() {
+        QueryTree x = parseWhere("{\"contains\" : [\"foo\", {\"alternatives\" : [{\"trees\": 1.0, \"tree\": 0.7}]}]}");
+        Item root = x.getRoot();
+        assertSame(WordAlternativesItem.class, root.getClass());
+        WordAlternativesItem alternatives = (WordAlternativesItem) root;
+        checkWordAlternativesContent(alternatives);
+    }
+
+    /** GROUPING TESTS */
+
+    @Test
+    public void testGrouping(){
+        String grouping = "[ { \"all\" : { \"group\" : \"time.year(a)\", \"each\" : { \"output\" : \"count()\" } } } ]";
+        String expected = "[[]all(group(time.year(a)) each(output(count())))]";
+        assertGrouping(expected, parseGrouping(grouping));
+    }
+
+
+    @Test
+    public void testMultipleGroupings() {
+        String grouping = "[ { \"all\" : { \"group\" : \"a\", \"each\" : { \"output\" : \"count()\"}}}, { \"all\" : { \"group\" : \"b\", \"each\" : { \"output\" : \"count()\"}}} ]";
+        String expected = "[[]all(group(a) each(output(count()))), []all(group(b) each(output(count())))]";
+
+        assertGrouping(expected, parseGrouping(grouping));
+    }
+
+
+
+    /** OTHER TESTS */
+
+    @Test
+    public void testOverridingOtherQueryTree() {
+        Query query = new Query("?query=default:query");
+        assertEquals("default:query", query.getModel().getQueryTree().toString());
+        assertEquals(Query.Type.ALL, query.getModel().getType());
+
+        query.getSelect().setWhere("{\"contains\" : [\"default\", \"select\"] }");
+        assertEquals("default:select", query.getModel().getQueryTree().toString());
+        assertEquals(Query.Type.SELECT, query.getModel().getType());
+    }
+
+
+    @Test
+    public void testOverridingWhereQueryTree() {
+        Query query = new Query();
+        query.getSelect().setWhere("{\"contains\" : [\"default\", \"select\"] }");
+        assertEquals("default:select", query.getModel().getQueryTree().toString());
+        assertEquals(Query.Type.SELECT, query.getModel().getType());
+
+        query.getModel().setQueryString("default:query");
+        query.getModel().setType("all");
+        assertEquals("default:query", query.getModel().getQueryTree().toString());
+        assertEquals(Query.Type.ALL, query.getModel().getType());
+    }
+
+
+
+
+    /** Assert-methods */
+    private void assertParse(String where, String expectedQueryTree) {
+        String queryTree = parseWhere(where).toString();
+        assertEquals(expectedQueryTree, queryTree);
+    }
+
+    private void assertParseFail(String where, Throwable expectedException) {
+        try {
+            parseWhere(where).toString();
+        } catch (Throwable t) {
+            assertEquals(expectedException.getClass(), t.getClass());
+            assertEquals(expectedException.getMessage(), t.getMessage());
+            return;
+        }
+        fail("Parse succeeded: " + where);
+    }
+
+    private void assertRootClass(String where, Class<? extends Item> expectedRootClass) {
+        assertEquals(expectedRootClass, parseWhere(where).getRoot().getClass());
+    }
+
+    private void assertGrouping(String expected, List<VespaGroupingStep> steps) {
+        List<String> actual = new ArrayList<>(steps.size());
+        for (VespaGroupingStep step : steps) {
+            actual.add(step.continuations().toString() +
+                    step.getOperation());
+        }
+        assertEquals(expected, actual.toString());
+    }
+
+
+
+
+    /** Parse-methods*/
+
+    private QueryTree parseWhere(String where) {
+        Select select = new Select(where, "");
+
+        return parser.parse(new Parsable().setSelect(select));
+    }
+
+    private List<VespaGroupingStep> parseGrouping(String grouping) {
+
+        return parser.getGroupingSteps(grouping);
+    }
+
+    private QueryTree parse(String where, String grouping) {
+        Select select = new Select(where, grouping);
+
+        return parser.parse(new Parsable().setSelect(select));
+    }
+
+
+
+
+
+    /** Other methods */
+    private WordItem getRootWord(String yqlQuery) {
+        Item root = parseWhere(yqlQuery).getRoot();
+        assertTrue(root instanceof WordItem);
+        return (WordItem)root;
+    }
+
+    private void checkWordAlternativesContent(WordAlternativesItem alternatives) {
+        boolean seenTree = false;
+        boolean seenForest = false;
+        final String forest = "trees";
+        final String tree = "tree";
+        assertEquals(2, alternatives.getAlternatives().size());
+        for (WordAlternativesItem.Alternative alternative : alternatives.getAlternatives()) {
+            if (tree.equals(alternative.word)) {
+                assertFalse("Duplicate term introduced", seenTree);
+                seenTree = true;
+                assertEquals(.7d, alternative.exactness, 1e-15d);
+            } else if (forest.equals(alternative.word)) {
+                assertFalse("Duplicate term introduced", seenForest);
+                seenForest = true;
+                assertEquals(1.0d, alternative.exactness, 1e-15d);
+            } else {
+                fail("Unexpected term: " + alternative.word);
+            }
+        }
+    }
+
+
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/EnvironmentResource.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/EnvironmentResource.java
index 812c4a60288..bea496c19d7 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/EnvironmentResource.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/EnvironmentResource.java
@@ -36,8 +36,10 @@ public interface EnvironmentResource {
 
     String API_PATH = "environment";
 
+    String APPLICATION_TEST_ZIP = "applicationTestZip";
     String APPLICATION_ZIP = "applicationZip";
     String DEPLOY_OPTIONS = "deployOptions";
+    String SUBMIT_OPTIONS = "submitOptions";
 
     @POST
     @Path("{environmentId}/region/{regionId}/instance/{instanceId}/deploy")
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/SubmitOptions.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/SubmitOptions.java
new file mode 100644
index 00000000000..fb97a3c1004
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/SubmitOptions.java
@@ -0,0 +1,55 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.application.v4.model;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.yahoo.vespa.hosted.controller.api.identifiers.GitBranch;
+import com.yahoo.vespa.hosted.controller.api.identifiers.GitCommit;
+import com.yahoo.vespa.hosted.controller.api.identifiers.GitRepository;
+
+import java.util.Objects;
+
+/**
+ * Additional options to be sent along the application package and the application test package
+ * when submitting an application to the controller
+ *
+ * @author freva
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class SubmitOptions {
+
+    public GitRepository repository;
+    public GitBranch branch;
+    public GitCommit commit;
+
+    public static SubmitOptions from(String repository, String branch, String commit) {
+        SubmitOptions options = new SubmitOptions();
+        options.repository = new GitRepository(repository);
+        options.branch = new GitBranch(branch);
+        options.commit = new GitCommit(commit);
+        return options;
+    }
+
+    @Override
+    public String toString() {
+        return "SubmitOptions{" +
+                "repository=" + repository +
+                ", branch=" + branch +
+                ", commit=" + commit +
+                '}';
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        SubmitOptions that = (SubmitOptions) o;
+        return Objects.equals(repository, that.repository) &&
+                Objects.equals(branch, that.branch) &&
+                Objects.equals(commit, that.commit);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(repository, branch, commit);
+    }
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/SubmitResult.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/SubmitResult.java
new file mode 100644
index 00000000000..a7f136cf3ce
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/SubmitResult.java
@@ -0,0 +1,37 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.application.v4.model;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+
+import java.util.Objects;
+
+/**
+ * Represents the response from application submit request
+ *
+ * @author freva
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class SubmitResult {
+
+    public String version;
+
+    @Override
+    public String toString() {
+        return "SubmitResult{" +
+                "version='" + version + '\'' +
+                '}';
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        SubmitResult that = (SubmitResult) o;
+        return Objects.equals(version, that.version);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(version);
+    }
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/BuildService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/BuildService.java
index e91a5909f80..56c2ee8da6b 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/BuildService.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/BuildService.java
@@ -20,11 +20,6 @@ public interface BuildService {
      */
     JobState stateOf(BuildJob buildJob);
 
-    /**
-     * Returns whether the given build job should be performed by this build service.
-     */
-    default boolean builds(BuildJob buildJob) { return true; }
-
     enum JobState {
 
         /** Job is not running, and may be triggered. */
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/InvalidTokenException.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/InvalidTokenException.java
deleted file mode 100644
index 967af1c553f..00000000000
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/InvalidTokenException.java
+++ /dev/null
@@ -1,11 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.controller.api.integration.athenz;
-
-/**
- * @author bjorncs
- */
-public class InvalidTokenException extends RuntimeException {
-    public InvalidTokenException(String message) {
-        super(message);
-    }
-}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZmsClient.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZmsClient.java
index e8bc16ca271..3630748b10a 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZmsClient.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZmsClient.java
@@ -3,8 +3,6 @@ package com.yahoo.vespa.hosted.controller.api.integration.athenz;
 
 import com.yahoo.vespa.athenz.api.AthenzDomain;
 import com.yahoo.vespa.athenz.api.AthenzIdentity;
-import com.yahoo.vespa.athenz.api.AthenzPublicKey;
-import com.yahoo.vespa.athenz.api.AthenzService;
 import com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId;
 
 import java.util.List;
@@ -33,8 +31,4 @@ public interface ZmsClient {
 
     List<AthenzDomain> getDomainList(String prefix);
 
-    AthenzPublicKey getPublicKey(AthenzService service, String keyId);
-
-    List<AthenzPublicKey> getPublicKeys(AthenzService service);
-
 }
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZmsKeystore.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZmsKeystore.java
deleted file mode 100644
index b3dc9fd4fe1..00000000000
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZmsKeystore.java
+++ /dev/null
@@ -1,18 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.controller.api.integration.athenz;
-
-import com.yahoo.vespa.athenz.api.AthenzService;
-
-import java.security.PublicKey;
-import java.util.Optional;
-
-/**
- * @author bjorncs
- */
-public interface ZmsKeystore {
-
-    Optional<PublicKey> getPublicKey(AthenzService service, String keyId);
-
-    default void preloadKeys(AthenzService service) { /* Default implementation is noop */ }
-
-}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/ArtifactRepository.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/ArtifactRepository.java
index b44e7f6f5e7..117dbd38a3b 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/ArtifactRepository.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/ArtifactRepository.java
@@ -21,10 +21,10 @@ public interface ArtifactRepository {
     /** Returns the system application package of the given version. */
     byte[] getSystemApplicationPackage(ApplicationId application, ZoneId zone, Version version);
 
-    /** Stores the given tester application fat jar of the given version. */
-    void putTesterJar(ApplicationId tester, String applicationVersion, byte[] fatTestJar);
+    /** Stores the given tester application package of the given version. Does NOT contain the services.xml. */
+    void putTesterPackage(ApplicationId tester, String applicationVersion, byte[] testerPackage);
 
-    /** Returns the tester application fat jar of the given version. */
-    byte[] getTesterJar(ApplicationId tester, String applicationVersion);
+    /** Returns the tester application package of the given version. Does NOT contain the services.xml. */
+    byte[] getTesterPackage(ApplicationId tester, String applicationVersion);
 
 }
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobType.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobType.java
index c9c73c3844a..3d8fe06e9b0 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobType.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobType.java
@@ -24,6 +24,7 @@ public enum JobType {
     productionApSoutheast1 ("production-ap-southeast-1"   , ZoneId.from("prod"   , "ap-southeast-1") , null                                                          ),
     productionEuWest1      ("production-eu-west-1"        , ZoneId.from("prod"   , "eu-west-1")      , null                                                          ),
     productionAwsUsEast1a  ("production-aws-us-east-1a"   , ZoneId.from("prod"   , "aws-us-east-1a") , null                                                          ),
+    productionAwsUsWest1b  ("production-aws-us-west-1b"   , ZoneId.from("prod"   , "aws-us-west-1b") , null                                                          ),
     productionCdAwsUsEast1a("production-cd-aws-us-east-1a", null                                                       , ZoneId.from("prod"    , "cd-aws-us-east-1a")),
     productionCdUsCentral1 ("production-cd-us-central-1"  , null                                                       , ZoneId.from("prod"    , "cd-us-central-1")  ),
     productionCdUsCentral2 ("production-cd-us-central-2"  , null                                                       , ZoneId.from("prod"    , "cd-us-central-2")  );
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/Testers.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/TesterCloud.java
index dccc0e47ceb..f1d07fc9097 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/Testers.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/TesterCloud.java
@@ -7,7 +7,7 @@ import java.net.URI;
  *
  * @author jonmv
  */
-public interface Testers {
+public interface TesterCloud {
 
     /** Signals the tester to run its tests. */
     void startTests(URI testerUrl, Suite suite, byte[] config);
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockTesters.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockTesterCloud.java
index 021e4d7f293..c2199c284f3 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockTesters.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockTesterCloud.java
@@ -1,16 +1,14 @@
 package com.yahoo.vespa.hosted.controller.api.integration.stubs;
 
-import com.yahoo.vespa.hosted.controller.api.integration.deployment.Testers;
+import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud;
 
 import java.net.URI;
 import java.util.Arrays;
 
-import static com.yahoo.vespa.hosted.controller.api.integration.deployment.Testers.Status.FAILURE;
-import static com.yahoo.vespa.hosted.controller.api.integration.deployment.Testers.Status.NOT_STARTED;
-import static com.yahoo.vespa.hosted.controller.api.integration.deployment.Testers.Status.RUNNING;
-import static com.yahoo.vespa.hosted.controller.api.integration.deployment.Testers.Status.SUCCESS;
+import static com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud.Status.NOT_STARTED;
+import static com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud.Status.RUNNING;
 
-public class MockTesters implements Testers {
+public class MockTesterCloud implements TesterCloud {
 
     private byte[] logs = new byte[0];
     private Status status = NOT_STARTED;
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java
index 790d6d00035..f8be9f55b84 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java
@@ -26,8 +26,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.routing.GlobalRoutingSe
 import com.yahoo.vespa.hosted.controller.api.integration.routing.RotationStatus;
 import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingGenerator;
 import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry;
-import com.yahoo.vespa.hosted.controller.deployment.DelegatingBuildService;
-import com.yahoo.vespa.hosted.controller.deployment.InternalBuildService;
 import com.yahoo.vespa.hosted.controller.deployment.JobController;
 import com.yahoo.vespa.hosted.controller.persistence.CuratorDb;
 import com.yahoo.vespa.hosted.controller.rotation.Rotation;
@@ -126,8 +124,7 @@ public class Controller extends AbstractComponent {
                                                           configServer,
                                                           Objects.requireNonNull(artifactRepository, "ArtifactRepository cannot be null"),
                                                           Objects.requireNonNull(routingGenerator, "RoutingGenerator cannot be null"),
-                                                          new DelegatingBuildService(Objects.requireNonNull(buildService, "BuildService cannot be null"),
-                                                                                     new InternalBuildService(jobController)),
+                                                          Objects.requireNonNull(buildService, "BuildService cannot be null"),
                                                           clock);
         tenantController = new TenantController(this, curator, athenzClientFactory);
 
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/filter/NTokenValidator.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/filter/NTokenValidator.java
deleted file mode 100644
index 4dcca519058..00000000000
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/filter/NTokenValidator.java
+++ /dev/null
@@ -1,81 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.controller.athenz.filter;
-
-import com.yahoo.athenz.auth.token.PrincipalToken;
-import com.yahoo.log.LogLevel;
-import com.yahoo.vespa.athenz.api.AthenzDomain;
-import com.yahoo.vespa.athenz.api.AthenzPrincipal;
-import com.yahoo.vespa.athenz.api.NToken;
-import com.yahoo.vespa.athenz.utils.AthenzIdentities;
-import com.yahoo.vespa.hosted.controller.api.integration.athenz.InvalidTokenException;
-import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZmsKeystore;
-
-import java.security.PublicKey;
-import java.time.Duration;
-import java.util.Optional;
-import java.util.logging.Logger;
-
-import static com.yahoo.vespa.athenz.utils.AthenzIdentities.ZMS_ATHENZ_SERVICE;
-
-
-/**
- * Validates the content of an NToken:
- *   1) Verifies that the token is signed by the sys.auth.zms service (by validating the signature)
- *   2) Verifies that the token is not expired
- *
- * @author bjorncs
- */
-// TODO Move to vespa-athenz
-class NTokenValidator {
-
-        // Max allowed skew in token timestamp (only for creation, not expiry timestamp)
-    private static final long ALLOWED_TIMESTAMP_OFFSET = Duration.ofMinutes(5).getSeconds();
-
-    private static final Logger log = Logger.getLogger(NTokenValidator.class.getName());
-
-    private final ZmsKeystore keystore;
-
-    NTokenValidator(ZmsKeystore keystore) {
-        this.keystore = keystore;
-    }
-
-    void preloadPublicKeys() {
-        keystore.preloadKeys(ZMS_ATHENZ_SERVICE);
-    }
-
-    AthenzPrincipal validate(NToken token) throws InvalidTokenException {
-        PrincipalToken principalToken = new PrincipalToken(token.getRawToken());
-        PublicKey zmsPublicKey = getPublicKey(principalToken.getKeyId())
-                .orElseThrow(() -> new InvalidTokenException("NToken has an unknown keyId"));
-        validateSignatureAndExpiration(principalToken, zmsPublicKey);
-        return new AthenzPrincipal(
-                AthenzIdentities.from(
-                        new AthenzDomain(principalToken.getDomain()),
-                        principalToken.getName()),
-                token);
-    }
-
-    private Optional<PublicKey> getPublicKey(String keyId) throws InvalidTokenException {
-        try {
-            return keystore.getPublicKey(ZMS_ATHENZ_SERVICE, keyId);
-        } catch (Exception e) {
-            logDebug(e.getMessage());
-            throw new InvalidTokenException("Failed to retrieve public key");
-        }
-    }
-
-    private static void validateSignatureAndExpiration(PrincipalToken token,
-                                                       PublicKey zmsPublicKey) throws InvalidTokenException {
-        StringBuilder errorMessageBuilder = new StringBuilder();
-        if (!token.validate(zmsPublicKey, (int) ALLOWED_TIMESTAMP_OFFSET, true, errorMessageBuilder)) {
-            String message = "NToken is expired or has invalid signature: " + errorMessageBuilder.toString();
-            logDebug(message);
-            throw new InvalidTokenException(message);
-        }
-    }
-
-    private static void logDebug(String message) {
-        log.log(LogLevel.DEBUG, "Failed to validate NToken: " + message);
-    }
-
-}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/filter/UserAuthWithAthenzPrincipalFilter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/filter/UserAuthWithAthenzPrincipalFilter.java
index b801c038bd8..26cd9f2e9b8 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/filter/UserAuthWithAthenzPrincipalFilter.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/filter/UserAuthWithAthenzPrincipalFilter.java
@@ -4,19 +4,19 @@ package com.yahoo.vespa.hosted.controller.athenz.filter;
 import com.google.inject.Inject;
 import com.yahoo.jdisc.Response;
 import com.yahoo.jdisc.http.filter.DiscFilterRequest;
+import com.yahoo.jdisc.http.filter.security.athenz.AthenzPrincipalFilter;
+import com.yahoo.jdisc.http.filter.security.athenz.AthenzPrincipalFilterConfig;
 import com.yahoo.jdisc.http.filter.security.cors.CorsFilterConfig;
 import com.yahoo.log.LogLevel;
 import com.yahoo.vespa.athenz.api.AthenzPrincipal;
 import com.yahoo.vespa.athenz.api.AthenzUser;
 import com.yahoo.vespa.athenz.api.NToken;
 import com.yahoo.vespa.hosted.controller.api.identifiers.UserId;
-import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZmsKeystore;
 import com.yahoo.vespa.hosted.controller.athenz.config.AthenzConfig;
 import com.yahoo.yolean.chain.After;
 
 import java.security.Principal;
 import java.util.Optional;
-import java.util.concurrent.Executor;
 import java.util.logging.Logger;
 import java.util.stream.Stream;
 
@@ -38,13 +38,10 @@ public class UserAuthWithAthenzPrincipalFilter extends AthenzPrincipalFilter {
     private final String principalHeaderName;
 
     @Inject
-    public UserAuthWithAthenzPrincipalFilter(ZmsKeystore zmsKeystore,
-                                             Executor executor,
-                                             AthenzConfig athenzConfig,
-                                             CorsFilterConfig corsConfig) {
-        super(zmsKeystore, executor, athenzConfig, corsConfig);
+    public UserAuthWithAthenzPrincipalFilter(AthenzPrincipalFilterConfig filterConfig, AthenzConfig athenzConfig, CorsFilterConfig corsConfig) {
+        super(filterConfig, corsConfig);
         this.userAuthenticationPassThruAttribute = athenzConfig.userAuthenticationPassThruAttribute();
-        this.principalHeaderName = athenzConfig.principalHeaderName();
+        this.principalHeaderName = filterConfig.principalHeaderName();
     }
 
     @Override
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/ZmsClientImpl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/ZmsClientImpl.java
index 67191d4c09d..6179d9891fd 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/ZmsClientImpl.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/ZmsClientImpl.java
@@ -1,22 +1,18 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.athenz.impl;
 
-import com.yahoo.athenz.auth.util.Crypto;
 import com.yahoo.athenz.zms.DomainList;
 import com.yahoo.athenz.zms.ProviderResourceGroupRoles;
-import com.yahoo.athenz.zms.PublicKeyEntry;
-import com.yahoo.athenz.zms.ServiceIdentity;
 import com.yahoo.athenz.zms.Tenancy;
 import com.yahoo.athenz.zms.TenantRoleAction;
 import com.yahoo.athenz.zms.ZMSClient;
 import com.yahoo.athenz.zms.ZMSClientException;
 import com.yahoo.log.LogLevel;
-import com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId;
 import com.yahoo.vespa.athenz.api.AthenzDomain;
-import com.yahoo.vespa.hosted.controller.api.integration.athenz.ApplicationAction;
 import com.yahoo.vespa.athenz.api.AthenzIdentity;
-import com.yahoo.vespa.athenz.api.AthenzPublicKey;
 import com.yahoo.vespa.athenz.api.AthenzService;
+import com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId;
+import com.yahoo.vespa.hosted.controller.api.integration.athenz.ApplicationAction;
 import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZmsClient;
 import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZmsException;
 import com.yahoo.vespa.hosted.controller.athenz.config.AthenzConfig;
@@ -130,28 +126,6 @@ public class ZmsClientImpl implements ZmsClient {
                 });
     }
 
-    @Override
-    public AthenzPublicKey getPublicKey(AthenzService service, String keyId) {
-        log("getPublicKeyEntry(domain=%s, service=%s, keyId=%s)", service.getDomain().getName(), service.getName(), keyId);
-        return getOrThrow(() -> {
-            PublicKeyEntry entry = zmsClient.getPublicKeyEntry(service.getDomain().getName(), service.getName(), keyId);
-            return fromYbase64EncodedKey(entry.getKey(), keyId);
-        });
-    }
-
-    @Override
-    public List<AthenzPublicKey> getPublicKeys(AthenzService service) {
-        log("getServiceIdentity(domain=%s, service=%s)", service.getDomain().getName(), service.getName());
-        return getOrThrow(() -> {
-            ServiceIdentity serviceIdentity = zmsClient.getServiceIdentity(service.getDomain().getName(), service.getName());
-            return toAthenzPublicKeys(serviceIdentity.getPublicKeys());
-        });
-    }
-
-    private static AthenzPublicKey fromYbase64EncodedKey(String encodedKey, String keyId) {
-        return new AthenzPublicKey(Crypto.loadPublicKey(Crypto.ybase64DecodeString(encodedKey)), keyId);
-    }
-
     private static List<TenantRoleAction> createTenantRoleActions() {
         return Arrays.stream(ApplicationAction.values())
                 .map(action -> new TenantRoleAction().setAction(action.name()).setRole(action.roleName))
@@ -162,12 +136,6 @@ public class ZmsClientImpl implements ZmsClient {
         return domains.stream().map(AthenzDomain::new).collect(toList());
     }
 
-    private static List<AthenzPublicKey> toAthenzPublicKeys(List<PublicKeyEntry> publicKeys) {
-        return publicKeys.stream()
-                .map(entry -> fromYbase64EncodedKey(entry.getKey(), entry.getId()))
-                .collect(toList());
-    }
-
     private boolean hasAccess(String action, String resource, AthenzIdentity identity) {
         log("getAccess(action=%s, resource=%s, principal=%s)", action, resource, identity);
         return getOrThrow(
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/ZmsKeystoreImpl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/ZmsKeystoreImpl.java
deleted file mode 100644
index 4b194651439..00000000000
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/ZmsKeystoreImpl.java
+++ /dev/null
@@ -1,120 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.controller.athenz.impl;
-
-import com.google.inject.Inject;
-import com.yahoo.log.LogLevel;
-import com.yahoo.vespa.athenz.api.AthenzPublicKey;
-import com.yahoo.vespa.athenz.api.AthenzService;
-import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzClientFactory;
-import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZmsException;
-import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZmsKeystore;
-
-import java.security.PublicKey;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Optional;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.logging.Logger;
-
-/**
- *  Downloads and caches public keys for Athens services.
- *
- * @author bjorncs
- */
-public class ZmsKeystoreImpl implements ZmsKeystore {
-    private static final Logger log = Logger.getLogger(ZmsKeystoreImpl.class.getName());
-
-    private final Map<FullKeyId, PublicKey> cachedKeys = new ConcurrentHashMap<>();
-    private final AthenzClientFactory athenzClientFactory;
-
-    @Inject
-    public ZmsKeystoreImpl(AthenzClientFactory factory) {
-        this.athenzClientFactory = factory;
-    }
-
-    @Override
-    public Optional<PublicKey> getPublicKey(AthenzService service, String keyId)  {
-        FullKeyId fullKeyId = new FullKeyId(service, keyId);
-        PublicKey cachedKey = cachedKeys.get(fullKeyId);
-        if (cachedKey != null) {
-            return Optional.of(cachedKey);
-        }
-        Optional<PublicKey> downloadedKey = downloadPublicKey(fullKeyId);
-        downloadedKey.ifPresent(key -> {
-            log.log(LogLevel.INFO, "Adding key " + fullKeyId + " to the cache");
-            cachedKeys.put(fullKeyId, key);
-        });
-        return downloadedKey;
-    }
-
-    @Override
-    public void preloadKeys(AthenzService service) {
-        try {
-            log.log(LogLevel.INFO, "Downloading keys for " + service);
-            List<AthenzPublicKey> publicKeys = athenzClientFactory.createZmsClientWithServicePrincipal()
-                    .getPublicKeys(service);
-            for (AthenzPublicKey publicKey : publicKeys) {
-                FullKeyId fullKeyId = new FullKeyId(service, publicKey.getKeyId());
-                log.log(LogLevel.DEBUG, "Adding key " + fullKeyId + " to the cache");
-                cachedKeys.put(fullKeyId, publicKey.getPublicKey());
-            }
-            log.log(LogLevel.INFO, "Successfully downloaded keys for " + service);
-        } catch (ZmsException e) {
-            log.log(LogLevel.WARNING, "Failed to download keys for " + service + ": " + e.getMessage());
-        }
-    }
-
-    private Optional<PublicKey> downloadPublicKey(FullKeyId fullKeyId) {
-        try {
-            log.log(LogLevel.INFO, "Downloading key " + fullKeyId);
-            AthenzPublicKey publicKey = athenzClientFactory.createZmsClientWithServicePrincipal()
-                    .getPublicKey(fullKeyId.service, fullKeyId.keyId);
-            return Optional.of(publicKey.getPublicKey());
-        } catch (ZmsException e) {
-            if (e.getCode() == 404) { // Key does not exist
-                log.log(LogLevel.INFO, "Key " + fullKeyId + " not found");
-                return Optional.empty();
-            }
-            String msg = String.format("Unable to retrieve public key from Athens (%s): %s", fullKeyId, e.getMessage());
-            throw createException(msg, e);
-        }
-    }
-
-    private static RuntimeException createException(String message, Exception cause) {
-        log.log(LogLevel.ERROR, message);
-        return new RuntimeException(message, cause);
-    }
-
-    private static class FullKeyId {
-        private final AthenzService service;
-        private final String keyId;
-
-        private FullKeyId(AthenzService service, String keyId) {
-            this.service = service;
-            this.keyId = keyId;
-        }
-
-        @Override
-        public boolean equals(Object o) {
-            if (this == o) return true;
-            if (o == null || getClass() != o.getClass()) return false;
-            FullKeyId fullKeyId1 = (FullKeyId) o;
-            return Objects.equals(service, fullKeyId1.service) &&
-                    Objects.equals(keyId, fullKeyId1.keyId);
-        }
-
-        @Override
-        public int hashCode() {
-            return Objects.hash(service, keyId);
-        }
-
-        @Override
-        public String toString() {
-            return "FullKeyId{" +
-                    "service=" + service +
-                    ", keyId='" + keyId + '\'' +
-                    '}';
-        }
-    }
-}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/ZmsClientMock.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/ZmsClientMock.java
index 3ee2655108a..5e8674ce637 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/ZmsClientMock.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/ZmsClientMock.java
@@ -5,8 +5,6 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId;
 import com.yahoo.vespa.athenz.api.AthenzDomain;
 import com.yahoo.vespa.hosted.controller.api.integration.athenz.ApplicationAction;
 import com.yahoo.vespa.athenz.api.AthenzIdentity;
-import com.yahoo.vespa.athenz.api.AthenzPublicKey;
-import com.yahoo.vespa.athenz.api.AthenzService;
 import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZmsClient;
 import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZmsException;
 
@@ -96,16 +94,6 @@ public class ZmsClientMock implements ZmsClient {
         return new ArrayList<>(athenz.domains.keySet());
     }
 
-    @Override
-    public AthenzPublicKey getPublicKey(AthenzService service, String keyId) {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public List<AthenzPublicKey> getPublicKeys(AthenzService service) {
-        throw new UnsupportedOperationException();
-    }
-
     private AthenzDbMock.Domain getDomainOrThrow(AthenzDomain domainName, boolean verifyVespaTenant) {
         AthenzDbMock.Domain domain = Optional.ofNullable(athenz.domains.get(domainName))
                 .orElseThrow(() -> zmsException(400, "Domain '%s' not found", domainName));
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DelegatingBuildService.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DelegatingBuildService.java
deleted file mode 100644
index d2159841c9d..00000000000
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DelegatingBuildService.java
+++ /dev/null
@@ -1,30 +0,0 @@
-package com.yahoo.vespa.hosted.controller.deployment;
-
-import com.yahoo.vespa.hosted.controller.api.integration.BuildService;
-
-/**
- * Sends build jobs to an internal build system whenever it accepts them, or to an external one otherwise.
- *
- * @author jonmv
- */
-public class DelegatingBuildService implements BuildService {
-
-    private final BuildService external;
-    private final BuildService internal;
-
-    public DelegatingBuildService(BuildService external, BuildService internal) {
-        this.external = external;
-        this.internal = internal;
-    }
-
-    @Override
-    public void trigger(BuildJob buildJob) {
-        (internal.builds(buildJob) ? internal : external).trigger(buildJob);
-    }
-
-    @Override
-    public JobState stateOf(BuildJob buildJob) {
-        return (internal.builds(buildJob) ? internal : external).stateOf(buildJob);
-    }
-
-}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentSteps.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentSteps.java
index 51c1e61ecb2..f60b7400219 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentSteps.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentSteps.java
@@ -6,7 +6,6 @@ import com.yahoo.config.provision.Environment;
 import com.yahoo.config.provision.SystemName;
 import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId;
 import com.yahoo.vespa.hosted.controller.application.Deployment;
-import com.yahoo.vespa.hosted.controller.application.DeploymentJobs;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
 import com.yahoo.vespa.hosted.controller.application.JobStatus;
 
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java
index e3b4b4cef8c..72bc1c61eae 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java
@@ -69,11 +69,13 @@ public class DeploymentTrigger {
     private final Controller controller;
     private final Clock clock;
     private final BuildService buildService;
+    private final JobController jobs;
 
     public DeploymentTrigger(Controller controller, BuildService buildService, Clock clock) {
         this.controller = Objects.requireNonNull(controller, "controller cannot be null");
-        this.buildService = Objects.requireNonNull(buildService, "buildService cannot be null");
         this.clock = Objects.requireNonNull(clock, "clock cannot be null");
+        this.buildService = Objects.requireNonNull(buildService, "buildService cannot be null");
+        this.jobs = controller.jobController();
     }
 
     public DeploymentSteps steps(DeploymentSpec spec) {
@@ -161,14 +163,22 @@ public class DeploymentTrigger {
      * Attempts to trigger the given job for the given application and returns the outcome.
      *
      * If the build service can not find the given job, or claims it is illegal to trigger it,
-     * the project id is removed from the application owning the job, to prevent further trigger attemps.
+     * the project id is removed from the application owning the job, to prevent further trigger attempts.
      */
     public boolean trigger(Job job) {
         log.log(LogLevel.INFO, String.format("Triggering %s: %s", job, job.triggering));
         try {
-            buildService.trigger(job);
-            applications().lockOrThrow(job.applicationId(), application ->
-                    applications().store(application.withJobTriggering(job.jobType, job.triggering)));
+            applications().lockOrThrow(job.applicationId(), application -> {
+                if (application.get().deploymentJobs().builtInternally())
+                    jobs.start(job.applicationId(), job.jobType, new Versions(job.triggering.platform(),
+                                                                              job.triggering.application(),
+                                                                              job.triggering.sourcePlatform(),
+                                                                              job.triggering.sourceApplication()));
+                else
+                    buildService.trigger(job);
+
+                applications().store(application.withJobTriggering(job.jobType, job.triggering));
+            });
             return true;
         }
         catch (RuntimeException e) {
@@ -184,6 +194,9 @@ public class DeploymentTrigger {
     public List<JobType> forceTrigger(ApplicationId applicationId, JobType jobType, String user) {
         Application application = applications().require(applicationId);
         if (jobType == component) {
+            if (application.deploymentJobs().builtInternally())
+                throw new IllegalArgumentException(applicationId + " has no component job we can trigger.");
+
             buildService.trigger(BuildJob.of(applicationId, application.deploymentJobs().projectId().getAsLong(), jobType.jobName()));
             return singletonList(component);
         }
@@ -339,6 +352,7 @@ public class DeploymentTrigger {
         if (!jobStatus.get().lastCompleted().isPresent()) return true; // Never completed
         if (!jobStatus.get().firstFailing().isPresent()) return true; // Should not happen as firstFailing should be set for an unsuccessful job
         if (!versions.targetsMatch(jobStatus.get().lastCompleted().get())) return true; // Always trigger as targets have changed
+        if (application.deploymentSpec().upgradePolicy() == DeploymentSpec.UpgradePolicy.canary) return true; // Don't throttle canaries
 
         Instant firstFailing = jobStatus.get().firstFailing().get().at();
         Instant lastCompleted = jobStatus.get().lastCompleted().get().at();
@@ -376,6 +390,10 @@ public class DeploymentTrigger {
     }
 
     private JobState jobStateOf(Application application, JobType jobType) {
+        if (application.deploymentJobs().builtInternally()) {
+            Optional<RunStatus> run = controller.jobController().last(application.id(), jobType);
+            return run.isPresent() && ! run.get().hasEnded() ? JobState.running : JobState.idle;
+        }
         return buildService.stateOf(BuildJob.of(application.id(),
                                                 application.deploymentJobs().projectId().getAsLong(),
                                                 jobType.jobName()));
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalBuildService.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalBuildService.java
deleted file mode 100644
index 18b62f5ea0f..00000000000
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalBuildService.java
+++ /dev/null
@@ -1,39 +0,0 @@
-package com.yahoo.vespa.hosted.controller.deployment;
-
-import com.yahoo.vespa.hosted.controller.api.integration.BuildService;
-import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
-
-import java.util.Optional;
-
-/**
- * Wraps a JobController as a BuildService.
- *
- * Shall be inlined when the {@link DelegatingBuildService} delegates all jobs to it.
- *
- * @author jonmv
- */
-public class InternalBuildService implements BuildService {
-
-    private final JobController jobs;
-
-    public InternalBuildService(JobController jobs) {
-        this.jobs = jobs;
-    }
-
-    @Override
-    public void trigger(BuildJob buildJob) {
-        jobs.start(buildJob.applicationId(), JobType.fromJobName(buildJob.jobName()));
-    }
-
-    @Override
-    public JobState stateOf(BuildJob buildJob) {
-        Optional<RunStatus> run = jobs.last(buildJob.applicationId(), JobType.fromJobName(buildJob.jobName()));
-        return run.isPresent() && ! run.get().hasEnded() ? JobState.running : JobState.idle;
-    }
-
-    @Override
-    public boolean builds(BuildJob buildJob) {
-        return jobs.builds(buildJob.applicationId());
-    }
-
-}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
index 77b6baa6bea..5c848ba279b 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
@@ -20,7 +20,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.configserver.PrepareRes
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.ServiceConvergence;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId;
-import com.yahoo.vespa.hosted.controller.api.integration.deployment.Testers;
+import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud;
 import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId;
 import com.yahoo.vespa.hosted.controller.application.ApplicationPackage;
 import com.yahoo.vespa.hosted.controller.application.ApplicationVersion;
@@ -54,7 +54,6 @@ import static com.yahoo.vespa.hosted.controller.api.integration.configserver.Con
 import static com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException.ErrorCode.APPLICATION_LOCK_FAILURE;
 import static com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException.ErrorCode.OUT_OF_CAPACITY;
 import static com.yahoo.vespa.hosted.controller.api.integration.configserver.Node.State.active;
-import static com.yahoo.vespa.hosted.controller.api.integration.configserver.Node.State.dirty;
 import static com.yahoo.vespa.hosted.controller.api.integration.configserver.Node.State.reserved;
 import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.failed;
 import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.succeeded;
@@ -85,12 +84,12 @@ public class InternalStepRunner implements StepRunner {
     }
 
     private final Controller controller;
-    private final Testers testers;
+    private final TesterCloud testerCloud;
     private final ThreadLocal<ByteArrayLogger> logger = new ThreadLocal<>();
 
-    public InternalStepRunner(Controller controller, Testers testers) {
+    public InternalStepRunner(Controller controller, TesterCloud testerCloud) {
         this.controller = controller;
-        this.testers = testers;
+        this.testerCloud = testerCloud;
     }
 
     @Override
@@ -302,9 +301,9 @@ public class InternalStepRunner implements StepRunner {
         Optional<URI> testerEndpoint = testerEndpoint(id);
         if (testerEndpoint.isPresent()) {
             logger.get().log("Starting tests ...");
-            testers.startTests(testerEndpoint.get(),
-                               Testers.Suite.of(id.type()),
-                               testConfig(id.application(), zone(id.type()), controller.system(), endpoints));
+            testerCloud.startTests(testerEndpoint.get(),
+                                   TesterCloud.Suite.of(id.type()),
+                                   testConfig(id.application(), zone(id.type()), controller.system(), endpoints));
             return succeeded;
         }
 
@@ -322,7 +321,7 @@ public class InternalStepRunner implements StepRunner {
                 .orElseThrow(() -> new NoSuchElementException("Endpoint for tester vanished again before tests were complete!"));
 
         Status status;
-        switch (testers.getStatus(testerEndpoint)) {
+        switch (testerCloud.getStatus(testerEndpoint)) {
             case NOT_STARTED:
                 throw new IllegalStateException("Tester reports tests not started, even though they should have!");
             case RUNNING:
@@ -340,7 +339,7 @@ public class InternalStepRunner implements StepRunner {
             default:
                 throw new AssertionError("Unknown status!");
         }
-        logger.get().log(new String(testers.getLogs(testerEndpoint))); // TODO jvenstad: Replace with something less hopeless!
+        logger.get().log(new String(testerCloud.getLogs(testerEndpoint))); // TODO jvenstad: Replace with something less hopeless!
         return status;
     }
 
@@ -404,12 +403,14 @@ public class InternalStepRunner implements StepRunner {
                                                                   .lastTriggered().get()
                                                                   .application();
 
-        byte[] testJar = controller.applications().artifacts().getTesterJar(testerOf(id.application()), version.id());
-        byte[] servicesXml = servicesXml();
+        byte[] testPackage = controller.applications().artifacts().getTesterPackage(testerOf(id.application()), version.id());
+        byte[] servicesXml = servicesXml(controller.system());
 
-        // TODO hakonhall: Assemble!
-
-        throw new AssertionError();
+        try (ZipBuilder zipBuilder = new ZipBuilder(testPackage.length + servicesXml.length + 1000)) {
+            zipBuilder.add(testPackage);
+            zipBuilder.add("services.xml", servicesXml);
+            return new ApplicationPackage(zipBuilder.toByteArray());
+        }
     }
 
     /** Returns all endpoints for all current deployments of the given real application. */
@@ -434,9 +435,47 @@ public class InternalStepRunner implements StepRunner {
     }
 
     /** Returns the generated services.xml content for the tester application. */
-    private byte[] servicesXml() {
-        //TODO hakonhall: Create!
-        return "".getBytes();
+    static byte[] servicesXml(SystemName systemName) {
+        String domain = systemName == SystemName.main ? "vespa.vespa" : "vespa.vespa.cd";
+
+        String servicesXml = "<?xml version='1.0' encoding='UTF-8'?>\n" +
+                "<services xmlns:deploy='vespa' version='1.0'>\n" +
+                "    <container version='1.0' id='default'>\n" +
+                "\n" +
+                "        <component id=\"com.yahoo.vespa.hosted.testrunner.TestRunner\" bundle=\"vepsa-testrunner-components\">\n" +
+                "            <config name=\"com.yahoo.vespa.hosted.testrunner.test-runner\">\n" +
+                "                <artifactsPath>artifacts</artifactsPath>\n" +
+                "            </config>\n" +
+                "        </component>\n" +
+                "\n" +
+                "        <handler id=\"com.yahoo.vespa.hosted.testrunner.TestRunnerHandler\" bundle=\"vespa-testrunner-components\">\n" +
+                "            <binding>http://*/tester/v1/*</binding>\n" +
+                "        </handler>\n" +
+                "\n" +
+                "        <http>\n" +
+                "            <filtering>\n" +
+                "                <request-chain id=\"testrunner-api\">\n" +
+                "                    <filter id='authz-filter' class='com.yahoo.jdisc.http.filter.security.athenz.AthenzAuthorizationFilter' bundle=\"jdisc-security-filters\">\n" +
+                "                        <config name=\"jdisc.http.filter.security.athenz.athenz-authorization-filter\">\n" +
+                "                            <credentialsToVerify>TOKEN_ONLY</credentialsToVerify>\n" +
+                "                            <roleTokenHeaderName>Yahoo-Role-Auth</roleTokenHeaderName>\n" +
+                "                        </config>\n" +
+                "                        <component id=\"com.yahoo.jdisc.http.filter.security.athenz.StaticRequestResourceMapper\" bundle=\"jdisc-security-filters\">\n" +
+                "                            <config name=\"jdisc.http.filter.security.athenz.static-request-resource-mapper\">\n" +
+                "                                <resourceName>" + domain + ":tester-application</resourceName>\n" +
+                "                                <action>deploy</action>\n" +
+                "                            </config>\n" +
+                "                        </component>\n" +
+                "                    </filter>\n" +
+                "                </request-chain>\n" +
+                "            </filtering>\n" +
+                "        </http>\n" +
+                "\n" +
+                "        <nodes count=\"1\" flavor=\"d-2-8-50\" />\n" +
+                "    </container>\n" +
+                "</services>\n";
+
+        return servicesXml.getBytes();
     }
 
     /** Returns the config for the tests to run for the given job. */
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
index 9ced75303ff..06af36a2e26 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
@@ -1,6 +1,7 @@
 package com.yahoo.vespa.hosted.controller.deployment;
 
 import com.google.common.collect.ImmutableMap;
+import com.yahoo.component.Version;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.vespa.curator.Lock;
 import com.yahoo.vespa.hosted.controller.Application;
@@ -169,14 +170,14 @@ public class JobController {
     }
 
     /** Registers the given application, such that it may have deployment jobs run here. */
-    void register(ApplicationId id) {
+    public void register(ApplicationId id) {
         controller.applications().lockIfPresent(id, application ->
                 controller.applications().store(application.withBuiltInternally(true)));
     }
 
     /** Accepts and stores a new application package and test jar pair under a generated application version key. */
     public ApplicationVersion submit(ApplicationId id, SourceRevision revision,
-                                     byte[] applicationPackage, byte[] applicationTestJar) {
+                                     byte[] applicationPackage, byte[] applicationTestPackage) {
         AtomicReference<ApplicationVersion> version = new AtomicReference<>();
         controller.applications().lockOrThrow(id, application -> {
             controller.applications().store(application.withBuiltInternally(true));
@@ -185,14 +186,17 @@ public class JobController {
             version.set(ApplicationVersion.from(revision, run));
 
             // TODO smorgrav: Store the pair.
-
-            notifyOfNewSubmission(id, revision, run);
+//            controller.applications().artifacts().putApplicationPackage(id, version.toString(), applicationPackage);
+//            controller.applications().artifacts().putTesterPackage(
+//                    InternalStepRunner.testerOf(id), version.toString(), applicationTestPackage);
+//
+//            notifyOfNewSubmission(id, revision, run);
         });
         return version.get();
     }
 
     /** Orders a run of the given type, or throws an IllegalStateException if that job type is already running. */
-    public void start(ApplicationId id, JobType type) {
+    public void start(ApplicationId id, JobType type, Versions versions) {
         controller.applications().lockIfPresent(id, application -> {
             if ( ! application.get().deploymentJobs().builtInternally())
                 throw new IllegalArgumentException(id + " is not built here!");
@@ -203,7 +207,7 @@ public class JobController {
                     throw new IllegalStateException("Can not start " + type + " for " + id + "; it is already running!");
 
                 RunId newId = new RunId(id, type, last.map(run -> run.id().number()).orElse(0L) + 1);
-                curator.writeLastRun(RunStatus.initial(newId, controller.clock().instant()));
+                curator.writeLastRun(RunStatus.initial(newId, versions, controller.clock().instant()));
             });
         });
     }
@@ -265,7 +269,7 @@ public class JobController {
     private void notifyOfNewSubmission(ApplicationId id, SourceRevision revision, long number) {
         DeploymentJobs.JobReport report = new DeploymentJobs.JobReport(id,
                                                                        JobType.component,
-                                                                       Long.MAX_VALUE, // TODO jvenstad: Clean up this!
+                                                                       1,
                                                                        number,
                                                                        Optional.of(revision),
                                                                        Optional.empty());
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunResult.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunResult.java
index aaf43097908..c4aac48503f 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunResult.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunResult.java
@@ -1,7 +1,7 @@
 package com.yahoo.vespa.hosted.controller.deployment;
 
 /**
- * Outcomes of jobs run by an {@link InternalBuildService}.
+ * Outcomes of jobs run by a {@link JobController}.
  *
  * @author jonmv
  */
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java
index a91cc905add..c30b1aee7f1 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java
@@ -1,7 +1,9 @@
 package com.yahoo.vespa.hosted.controller.deployment;
 
 import com.google.common.collect.ImmutableList;
+import com.yahoo.component.Version;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId;
+import com.yahoo.vespa.hosted.controller.application.ApplicationVersion;
 
 import java.time.Instant;
 import java.util.Collections;
@@ -16,7 +18,7 @@ import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.unfinishe
 import static java.util.Objects.requireNonNull;
 
 /**
- * Immutable class containing status information for a deployment job run by an {@link InternalBuildService}.
+ * Immutable class containing status information for a deployment job run by a {@link JobController}.
  *
  * @author jonmv
  */
@@ -24,24 +26,26 @@ public class RunStatus {
 
     private final RunId id;
     private final Map<Step, Step.Status> steps;
+    private final Versions versions;
     private final Instant start;
     private final Optional<Instant> end;
     private final boolean aborted;
-    // TODO jvenstad: Add a Versions object and a reason String. Requires shortcutting of triggering of these runs.
 
     // For deserialisation only -- do not use!
-    public RunStatus(RunId id, Map<Step, Step.Status> steps, Instant start, Optional<Instant> end, boolean aborted) {
+    public RunStatus(RunId id, Map<Step, Step.Status> steps, Versions versions,
+                     Instant start, Optional<Instant> end, boolean aborted) {
         this.id = id;
         this.steps = Collections.unmodifiableMap(new EnumMap<>(steps));
+        this.versions = versions;
         this.start = start;
         this.end = end;
         this.aborted = aborted;
     }
 
-    public static RunStatus initial(RunId id, Instant now) {
+    public static RunStatus initial(RunId id, Versions versions, Instant now) {
         EnumMap<Step, Step.Status> steps = new EnumMap<>(Step.class);
         JobProfile.of(id.type()).steps().forEach(step -> steps.put(step, unfinished));
-        return new RunStatus(id, steps, requireNonNull(now), Optional.empty(), false);
+        return new RunStatus(id, steps, requireNonNull(versions), requireNonNull(now), Optional.empty(), false);
     }
 
     public RunStatus with(Step.Status status, LockedStep step) {
@@ -50,21 +54,21 @@ public class RunStatus {
 
         EnumMap<Step, Step.Status> steps = new EnumMap<>(this.steps);
         steps.put(step.get(), requireNonNull(status));
-        return new RunStatus(id, steps, start, end, aborted);
+        return new RunStatus(id, steps, versions, start, end, aborted);
     }
 
     public RunStatus finished(Instant now) {
         if (hasEnded())
             throw new AssertionError("This step ended at " + end.get() + " -- it can't be ended again!");
 
-        return new RunStatus(id, new EnumMap<>(steps), start, Optional.of(now), aborted);
+        return new RunStatus(id, new EnumMap<>(steps), versions, start, Optional.of(now), aborted);
     }
 
     public RunStatus aborted() {
         if (hasEnded())
             throw new AssertionError("This step ended at " + end.get() + " -- it can't be aborted now!");
 
-        return new RunStatus(id, new EnumMap<>(steps), start, end, true);
+        return new RunStatus(id, new EnumMap<>(steps), versions, start, end, true);
     }
 
     /** Returns the id of this run. */
@@ -72,15 +76,23 @@ public class RunStatus {
         return id;
     }
 
-    /** Returns an unmodifiable view of the status of all steps in this run. */
+    /** Returns an unmodifiable view of the status of all steps in this run.
+     * TODO maybe reflect in the signature that the map is a EnumMap or at least behaves as a sorted map?
+     * */
     public Map<Step, Step.Status> steps() {
         return steps;
     }
 
     /** Returns the final result of this run, if it has ended. */
     public Optional<RunResult> result() {
-        // TODO jvenstad: To implement, or not ... If so, base on status.
-        throw new AssertionError();
+
+        // No result of not finished yet
+        if ( ! hasEnded()) return Optional.empty();
+
+        // If any steps has failed - then we need to figure out what - for now return fixed error result
+        if (hasFailed()) return Optional.of(RunResult.testError);
+
+        return Optional.of(RunResult.success);
     }
 
     /** Returns the instant at which this run began. */
@@ -108,6 +120,11 @@ public class RunStatus {
         return end.isPresent();
     }
 
+    /** Returns the target, and possibly source, versions for this run. */
+    public Versions versions() {
+        return versions;
+    }
+
     @Override
     public boolean equals(Object o) {
         if (this == o) return true;
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java
index eeb85dc2506..f734dde9440 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java
@@ -2,8 +2,6 @@ package com.yahoo.vespa.hosted.controller.deployment;
 
 import com.google.common.collect.ImmutableList;
 
-import java.util.Arrays;
-import java.util.Collections;
 import java.util.List;
 
 /**
@@ -68,7 +66,6 @@ public enum Step {
 
     public List<Step> prerequisites() { return prerequisites; }
 
-
     public enum Status {
 
         /** Step still has unsatisfied finish criteria -- it may not even have started. */
@@ -78,7 +75,7 @@ public enum Step {
         failed,
 
         /** Step succeeded and subsequent steps may now start. */
-        succeeded;
+        succeeded
 
     }
 
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Versions.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Versions.java
index bf58bac177c..e0f7a955a80 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Versions.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Versions.java
@@ -9,8 +9,11 @@ import com.yahoo.vespa.hosted.controller.application.Change;
 import com.yahoo.vespa.hosted.controller.application.Deployment;
 import com.yahoo.vespa.hosted.controller.application.JobStatus;
 
+import java.util.Objects;
 import java.util.Optional;
 
+import static java.util.Objects.requireNonNull;
+
 /**
  * Source and target versions for an application.
  *
@@ -26,10 +29,13 @@ public class Versions {
 
     public Versions(Version targetPlatform, ApplicationVersion targetApplication, Optional<Version> sourcePlatform,
                     Optional<ApplicationVersion> sourceApplication) {
-        this.targetPlatform = targetPlatform;
-        this.targetApplication = targetApplication;
-        this.sourcePlatform = sourcePlatform;
-        this.sourceApplication = sourceApplication;
+        if (sourcePlatform.isPresent() ^ sourceApplication.isPresent())
+            throw new IllegalArgumentException("Sources must both be present or absent.");
+
+        this.targetPlatform = requireNonNull(targetPlatform);
+        this.targetApplication = requireNonNull(targetApplication);
+        this.sourcePlatform = requireNonNull(sourcePlatform);
+        this.sourceApplication = requireNonNull(sourceApplication);
     }
 
     /** Target platform version for this */
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/ZipBuilder.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/ZipBuilder.java
new file mode 100644
index 00000000000..e3ede999c11
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/ZipBuilder.java
@@ -0,0 +1,65 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.deployment;
+
+import org.apache.commons.io.IOUtils;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipInputStream;
+import java.util.zip.ZipOutputStream;
+
+/**
+ * Utility class to build zipped content by adding already zipped byte content or
+ * adding new unzipped entries.
+ *
+ * @author freva
+ */
+public class ZipBuilder implements AutoCloseable {
+
+    private final ByteArrayOutputStream byteArrayOutputStream;
+    private final ZipOutputStream zipOutputStream;
+
+    public ZipBuilder(int initialSize) {
+        byteArrayOutputStream = new ByteArrayOutputStream(initialSize);
+        zipOutputStream = new ZipOutputStream(byteArrayOutputStream);
+    }
+
+    public void add(byte[] zippedContent) {
+        try (ZipInputStream zin = new ZipInputStream(new ByteArrayInputStream(zippedContent))) {
+            for (ZipEntry entry = zin.getNextEntry(); entry != null; entry = zin.getNextEntry()) {
+                zipOutputStream.putNextEntry(entry);
+                IOUtils.copy(zin, zipOutputStream);
+                zipOutputStream.closeEntry();
+            }
+        } catch (IOException e) {
+            throw new UncheckedIOException("Failed to add zipped content", e);
+        }
+    }
+
+    public void add(String entryName, byte[] content) {
+        try {
+            zipOutputStream.putNextEntry(new ZipEntry(entryName));
+            zipOutputStream.write(content);
+            zipOutputStream.closeEntry();
+        } catch (IOException e) {
+            throw new UncheckedIOException("Failed to add entry " + entryName, e);
+        }
+    }
+
+    /** @return zipped byte array */
+    public byte[] toByteArray() {
+        return byteArrayOutputStream.toByteArray();
+    }
+
+    @Override
+    public void close() {
+        try {
+            zipOutputStream.close();
+        } catch (IOException e) {
+            throw new UncheckedIOException("Failed to close zip output stream", e);
+        }
+    }
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
index 7fa16a02649..e663b0154d7 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
@@ -4,13 +4,12 @@ package com.yahoo.vespa.hosted.controller.maintenance;
 import com.yahoo.component.AbstractComponent;
 import com.yahoo.jdisc.Metric;
 import com.yahoo.vespa.hosted.controller.Controller;
-import com.yahoo.vespa.hosted.controller.api.integration.deployment.Testers;
+import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud;
 import com.yahoo.vespa.hosted.controller.api.integration.dns.NameService;
 import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryClientInterface;
 import com.yahoo.vespa.hosted.controller.api.integration.organization.OwnershipIssues;
 import com.yahoo.vespa.hosted.controller.api.integration.organization.DeploymentIssues;
 import com.yahoo.vespa.hosted.controller.api.integration.chef.Chef;
-import com.yahoo.vespa.hosted.controller.deployment.DummyStepRunner;
 import com.yahoo.vespa.hosted.controller.deployment.InternalStepRunner;
 import com.yahoo.vespa.hosted.controller.maintenance.config.MaintainerConfig;
 import com.yahoo.vespa.hosted.controller.persistence.CuratorDb;
@@ -45,7 +44,7 @@ public class ControllerMaintenance extends AbstractComponent {
 
     @SuppressWarnings("unused") // instantiated by Dependency Injection
     public ControllerMaintenance(MaintainerConfig maintainerConfig, Controller controller, CuratorDb curator,
-                                 JobControl jobControl, Metric metric, Chef chefClient, Testers testers,
+                                 JobControl jobControl, Metric metric, Chef chefClient, TesterCloud testerCloud,
                                  DeploymentIssues deploymentIssues, OwnershipIssues ownershipIssues,
                                  NameService nameService, NodeRepositoryClientInterface nodeRepositoryClient) {
         Duration maintenanceInterval = Duration.ofMinutes(maintainerConfig.intervalMinutes());
@@ -63,7 +62,7 @@ public class ControllerMaintenance extends AbstractComponent {
         applicationOwnershipConfirmer = new ApplicationOwnershipConfirmer(controller, Duration.ofHours(12), jobControl, ownershipIssues);
         dnsMaintainer = new DnsMaintainer(controller, Duration.ofHours(12), jobControl, nameService);
         systemUpgrader = new SystemUpgrader(controller, Duration.ofMinutes(1), jobControl);
-        jobRunner = new JobRunner(controller, Duration.ofSeconds(30), jobControl, new InternalStepRunner(controller, testers));
+        jobRunner = new JobRunner(controller, Duration.ofSeconds(30), jobControl, new InternalStepRunner(controller, testerCloud));
     }
 
     public Upgrader upgrader() { return upgrader; }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java
index 7dbf1a2c05e..9fa67cb5f89 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java
@@ -2,7 +2,6 @@ package com.yahoo.vespa.hosted.controller.maintenance;
 
 import com.yahoo.log.LogLevel;
 import com.yahoo.vespa.hosted.controller.Controller;
-import com.yahoo.vespa.hosted.controller.deployment.InternalBuildService;
 import com.yahoo.vespa.hosted.controller.deployment.JobController;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId;
 import com.yahoo.vespa.hosted.controller.deployment.RunStatus;
@@ -20,9 +19,8 @@ import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.logging.Logger;
 
 /**
- * Advances the set of {@link RunStatus}es for an {@link InternalBuildService}.
+ * Advances the set of {@link RunStatus}es for a {@link JobController}.
  *
- * @see JobController
  * @author jonmv
  */
 public class JobRunner extends Maintainer {
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java
index 7df60278390..153f4e327a0 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java
@@ -1,5 +1,6 @@
 package com.yahoo.vespa.hosted.controller.persistence;
 
+import com.yahoo.component.Version;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.slime.ArrayTraverser;
 import com.yahoo.slime.Cursor;
@@ -8,9 +9,12 @@ import com.yahoo.slime.ObjectTraverser;
 import com.yahoo.slime.Slime;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId;
+import com.yahoo.vespa.hosted.controller.application.ApplicationVersion;
+import com.yahoo.vespa.hosted.controller.application.SourceRevision;
 import com.yahoo.vespa.hosted.controller.deployment.RunStatus;
 import com.yahoo.vespa.hosted.controller.deployment.Step;
 import com.yahoo.vespa.hosted.controller.deployment.Step.Status;
+import com.yahoo.vespa.hosted.controller.deployment.Versions;
 
 import java.time.Instant;
 import java.util.EnumMap;
@@ -47,6 +51,13 @@ public class RunSerializer {
     private static final String startField = "start";
     private static final String endField = "end";
     private static final String abortedField = "aborted";
+    private static final String versionsField = "versions";
+    private static final String platformVersionField = "platform";
+    private static final String repositoryField = "repository";
+    private static final String branchField = "branch";
+    private static final String commitField = "commit";
+    private static final String buildField = "build";
+    private static final String sourceField = "source";
 
     RunStatus runFromSlime(Slime slime) {
         return runFromSlime(slime.get());
@@ -71,6 +82,7 @@ public class RunSerializer {
                                        JobType.fromJobName(runObject.field(jobTypeField).asString()),
                                        runObject.field(numberField).asLong()),
                              steps,
+                             versionsFromSlime(runObject.field(versionsField)),
                              Instant.ofEpochMilli(runObject.field(startField).asLong()),
                              Optional.of(runObject.field(endField))
                                      .filter(Inspector::valid)
@@ -78,6 +90,25 @@ public class RunSerializer {
                              runObject.field(abortedField).asBool());
     }
 
+    private Versions versionsFromSlime(Inspector versionsObject) {
+        Version targetPlatformVersion = Version.fromString(versionsObject.field(platformVersionField).asString());
+        ApplicationVersion targetApplicationVersion = ApplicationVersion.from(new SourceRevision(versionsObject.field(repositoryField).asString(),
+                                                                                                 versionsObject.field(branchField).asString(),
+                                                                                                 versionsObject.field(commitField).asString()),
+                                                                              versionsObject.field(buildField).asLong());
+        Optional<Version> sourcePlatformVersion = versionsObject.field(sourceField).valid()
+                ? Optional.of(Version.fromString(versionsObject.field(sourceField).field(platformVersionField).asString()))
+                : Optional.empty();
+        Optional<ApplicationVersion> sourceApplicationVersion = versionsObject.field(sourceField).valid()
+                ? Optional.of(ApplicationVersion.from(new SourceRevision(versionsObject.field(repositoryField).asString(),
+                                                                         versionsObject.field(branchField).asString(),
+                                                                         versionsObject.field(commitField).asString()),
+                                                      versionsObject.field(buildField).asLong()))
+                : Optional.empty();
+
+        return new Versions(targetPlatformVersion, targetApplicationVersion, sourcePlatformVersion, sourceApplicationVersion);
+    }
+
     Slime toSlime(Iterable<RunStatus> runs) {
         Slime slime = new Slime();
         Cursor runArray = slime.setArray();
@@ -98,59 +129,84 @@ public class RunSerializer {
         runObject.setLong(startField, run.start().toEpochMilli());
         run.end().ifPresent(end -> runObject.setLong(endField, end.toEpochMilli()));
         if (run.isAborted()) runObject.setBool(abortedField, true);
+
         Cursor stepsObject = runObject.setObject(stepsField);
         run.steps().forEach((step, status) -> stepsObject.setString(valueOf(step), valueOf(status)));
+
+        Cursor versionsObject = runObject.setObject(versionsField);
+        toSlime(run.versions().targetPlatform(), run.versions().targetApplication(), versionsObject);
+        run.versions().sourcePlatform().ifPresent(sourcePlatformVersion -> {
+            toSlime(sourcePlatformVersion,
+                    run.versions().sourceApplication()
+                   .orElseThrow(() -> new IllegalArgumentException("Source versions must be both present or absent.")),
+                    versionsObject.setObject(sourceField));
+        });
+    }
+
+    private void toSlime(Version platformVersion, ApplicationVersion applicationVersion, Cursor versionsObject) {
+        versionsObject.setString(platformVersionField, platformVersion.toString());
+        SourceRevision targetSourceRevision = applicationVersion.source()
+                .orElseThrow(() -> new IllegalArgumentException("Source revision must be present in target application version."));
+        versionsObject.setString(repositoryField, targetSourceRevision.repository());
+        versionsObject.setString(branchField, targetSourceRevision.branch());
+        versionsObject.setString(commitField, targetSourceRevision.commit());
+        versionsObject.setLong(buildField, applicationVersion.buildNumber()
+                              .orElseThrow(() -> new IllegalArgumentException("Build number must be present in target application version.")));
     }
 
     static String valueOf(Step step) {
         switch (step) {
-            case deployInitialReal  : return "DIR";
-            case installInitialReal : return "IIR";
-            case deployReal         : return "DR" ;
-            case installReal        : return "IR" ;
-            case deactivateReal     : return "DAR";
-            case deployTester       : return "DT" ;
-            case installTester      : return "IT" ;
-            case deactivateTester   : return "DAT";
-            case startTests         : return "ST" ;
-            case endTests           : return "ET" ;
-            case report             : return "R"  ;
+            case deployInitialReal  : return "deployInitialReal";
+            case installInitialReal : return "installInitialReal";
+            case deployReal         : return "deployReal";
+            case installReal        : return "installReal";
+            case deactivateReal     : return "deactivateReal";
+            case deployTester       : return "deployTester";
+            case installTester      : return "installTester";
+            case deactivateTester   : return "deactivateTester";
+            case startTests         : return "startTests";
+            case endTests           : return "endTests";
+            case report             : return "report";
+
             default                 : throw new AssertionError("No value defined for '" + step + "'!");
         }
     }
 
     static Step stepOf(String step) {
         switch (step) {
-            case "DIR" : return deployInitialReal ;
-            case "IIR" : return installInitialReal;
-            case "DR"  : return deployReal        ;
-            case "IR"  : return installReal       ;
-            case "DAR" : return deactivateReal    ;
-            case "DT"  : return deployTester      ;
-            case "IT"  : return installTester     ;
-            case "DAT" : return deactivateTester  ;
-            case "ST"  : return startTests        ;
-            case "ET"  : return endTests          ;
-            case "R"   : return report            ;
-            default    : throw new IllegalArgumentException("No step defined by '" + step + "'!");
+            case "deployInitialReal"  : return deployInitialReal;
+            case "installInitialReal" : return installInitialReal;
+            case "deployReal"         : return deployReal;
+            case "installReal"        : return installReal;
+            case "deactivateReal"     : return deactivateReal;
+            case "deployTester"       : return deployTester;
+            case "installTester"      : return installTester;
+            case "deactivateTester"   : return deactivateTester;
+            case "startTests"         : return startTests;
+            case "endTests"           : return endTests;
+            case "report"             : return report;
+
+            default                   : throw new IllegalArgumentException("No step defined by '" + step + "'!");
         }
     }
 
     static String valueOf(Status status) {
         switch (status) {
-            case unfinished : return "U";
-            case failed     : return "F";
-            case succeeded  : return "S";
-            default  : throw new AssertionError("No value defined for '" + status + "'!");
+            case unfinished : return "unfinished";
+            case failed     : return "failed";
+            case succeeded  : return "succeeded";
+
+            default         : throw new AssertionError("No value defined for '" + status + "'!");
         }
     }
 
     static Status statusOf(String status) {
         switch (status) {
-            case "U" : return unfinished;
-            case "F" : return failed    ;
-            case "S" : return succeeded ;
-            default  : throw new IllegalArgumentException("No status defined by '" + status + "'!");
+            case "unfinished" : return unfinished;
+            case "failed"     : return failed;
+            case "succeeded"  : return succeeded;
+
+            default           : throw new IllegalArgumentException("No status defined by '" + status + "'!");
         }
     }
 
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
index b8f2ccad879..9e45526e5e4 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
@@ -49,6 +49,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZmsException;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.Log;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
+import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId;
 import com.yahoo.vespa.hosted.controller.api.integration.organization.User;
 import com.yahoo.vespa.hosted.controller.api.integration.routing.RotationStatus;
 import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId;
@@ -63,6 +64,8 @@ import com.yahoo.vespa.hosted.controller.application.DeploymentJobs;
 import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics;
 import com.yahoo.vespa.hosted.controller.application.JobStatus;
 import com.yahoo.vespa.hosted.controller.application.SourceRevision;
+import com.yahoo.vespa.hosted.controller.deployment.DeploymentSteps;
+import com.yahoo.vespa.hosted.controller.deployment.RunStatus;
 import com.yahoo.vespa.hosted.controller.restapi.ErrorResponse;
 import com.yahoo.vespa.hosted.controller.restapi.MessageResponse;
 import com.yahoo.restapi.Path;
@@ -88,6 +91,7 @@ import java.security.Principal;
 import java.time.DayOfWeek;
 import java.time.Duration;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -168,6 +172,12 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
         if (path.matches("/application/v4/tenant/{tenant}")) return tenant(path.get("tenant"), request);
         if (path.matches("/application/v4/tenant/{tenant}/application")) return applications(path.get("tenant"), request);
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}")) return application(path.get("tenant"), path.get("application"), request);
+        if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job"))
+            return JobControllerApiHandlerHelper.jobTypeResponse(jobTypes(path), latestRuns(path), request.getUri());
+        if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}"))
+            return JobControllerApiHandlerHelper.runStatusResponse(controller.jobController().runs(appIdFromPath(path), jobTypeFromPath(path)), request.getUri());
+        if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}/run/{number}"))
+            return JobControllerApiHandlerHelper.runDetailsResponse(controller.jobController(), runIdFromPath(path));
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}")) return deployment(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request);
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/service")) return services(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request);
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/service/{service}/{*}")) return service(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), path.get("service"), path.getRest(), request);
@@ -193,6 +203,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/promote")) return promoteApplication(path.get("tenant"), path.get("application"), request);
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying")) return deploy(path.get("tenant"), path.get("application"), request);
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/jobreport")) return notifyJobCompletion(path.get("tenant"), path.get("application"), request);
+        if (path.matches("/application/v4/tenant/{tenant}/application/{application}/submit")) return submit(path.get("tenant"), path.get("application"), request);
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}")) return deploy(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request);
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/deploy")) return deploy(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); // legacy synonym of the above
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/restart")) return restart(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request);
@@ -1204,4 +1215,48 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
                 message + ": No NToken provided"));
     }
 
+    private ApplicationId appIdFromPath(Path path) {
+        return ApplicationId.from(path.get("tenant"), path.get("application"), path.get("instance"));
+    }
+
+    private JobType jobTypeFromPath(Path path) {
+        return JobType.fromJobName(path.get("jobtype"));
+    }
+
+    private RunId runIdFromPath(Path path) {
+        long number = Long.parseLong(path.get("number"));
+        return new RunId(appIdFromPath(path), jobTypeFromPath(path), number);
+    }
+
+    private List<JobType> jobTypes(Path path) {
+        ApplicationId appId = appIdFromPath(path);
+        DeploymentSpec deploymentSpec = controller.applications().get(appId).get().deploymentSpec();
+        DeploymentSteps deploymentSteps = new DeploymentSteps(deploymentSpec, controller::system);
+        return deploymentSteps.jobs();
+    }
+
+    private Map<JobType, RunStatus> latestRuns(Path path) {
+        Map<JobType, RunStatus> jobMap = new HashMap<>();
+        ApplicationId appId = appIdFromPath(path);
+        controller.jobController().jobs(appId)
+                .forEach(jobType -> jobMap.put(jobType, controller.jobController()
+                        .last(appId, jobType)
+                        .orElseThrow(() -> new RuntimeException(String.format("Job %s for application %s appears in " +
+                                        "the list of previously ran jobs, but no status of the last execution found",
+                                jobType.jobName(), appId.toShortString())))));
+
+        return jobMap;
+    }
+
+    private HttpResponse submit(String tenant, String application, HttpRequest request) {
+        Map<String, byte[]> dataParts = new MultipartParser().parse(request);
+        Inspector submitOptions = SlimeUtils.jsonToSlime(dataParts.get(EnvironmentResource.SUBMIT_OPTIONS)).get();
+        SourceRevision sourceRevision = toSourceRevision(submitOptions).orElseThrow(() ->
+                new IllegalArgumentException("Must specify 'repository', 'branch', and 'commit'"));
+
+        return JobControllerApiHandlerHelper.submitResponse(controller.jobController(), tenant, application,
+                sourceRevision,
+                dataParts.get(EnvironmentResource.APPLICATION_ZIP),
+                dataParts.get(EnvironmentResource.APPLICATION_TEST_ZIP));
+    }
 }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java
new file mode 100644
index 00000000000..281e59ef19f
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java
@@ -0,0 +1,122 @@
+package com.yahoo.vespa.hosted.controller.restapi.application;
+
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.container.jdisc.HttpResponse;
+import com.yahoo.slime.Cursor;
+import com.yahoo.slime.Slime;
+import com.yahoo.vespa.hosted.controller.NotExistsException;
+import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
+import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId;
+import com.yahoo.vespa.hosted.controller.application.ApplicationVersion;
+import com.yahoo.vespa.hosted.controller.application.SourceRevision;
+import com.yahoo.vespa.hosted.controller.deployment.JobController;
+import com.yahoo.vespa.hosted.controller.deployment.RunDetails;
+import com.yahoo.vespa.hosted.controller.deployment.RunStatus;
+import com.yahoo.vespa.hosted.controller.deployment.Step;
+import com.yahoo.vespa.hosted.controller.restapi.SlimeJsonResponse;
+
+import java.net.URI;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+/**
+ * Implements the REST API for the job controller delegated from the Application API.
+ *
+ * @see JobController
+ * @see ApplicationApiHandler
+ */
+class JobControllerApiHandlerHelper {
+
+    /**
+     * @return Response with all job types that have recorded runs for the application _and_ the status for the last run of that type
+     */
+    static HttpResponse jobTypeResponse(List<JobType> sortedJobs, Map<JobType, RunStatus> lastStatus, URI baseUriForJobs) {
+        Slime slime = new Slime();
+        Cursor responseObject = slime.setObject();
+        Cursor jobArray = responseObject.setArray("jobs");
+
+        sortedJobs.forEach(jobType ->
+                jobTypeToSlime(jobArray.addObject(), jobType, Optional.ofNullable(lastStatus.get(jobType)), baseUriForJobs));
+        return new SlimeJsonResponse(slime);
+    }
+
+    private static void jobTypeToSlime(Cursor cursor, JobType jobType, Optional<RunStatus> runStatus, URI baseUriForJobs) {
+        Cursor jobObject = cursor.setObject(jobType.jobName());
+
+        // Url that are specific to the jobtype
+        String jobTypePath = baseUriForJobs.getPath() + "/" + jobType.jobName();
+        URI baseUriForJobType = baseUriForJobs.resolve(jobTypePath);
+        jobObject.setString("url", baseUriForJobType.toString());
+
+        // Add the last run status for the jobtype if present
+        runStatus.ifPresent(status -> {
+            Cursor lastObject = jobObject.setObject("last");
+            runStatusToSlime(lastObject, status, baseUriForJobType);
+        });
+    }
+
+    /**
+     * @return Response with the runstatuses for a specific jobtype
+     */
+    static HttpResponse runStatusResponse(Map<RunId, RunStatus> runStatuses, URI baseUriForJobType) {
+        Slime slime = new Slime();
+        Cursor cursor = slime.setObject();
+
+        runStatuses.forEach((runid, runstatus) -> runStatusToSlime(cursor.setObject(Long.toString(runid.number())), runstatus, baseUriForJobType));
+
+        return new SlimeJsonResponse(slime);
+    }
+
+    private static void runStatusToSlime(Cursor cursor, RunStatus runStatus, URI baseUriForJobType) {
+        runStatus.result().ifPresent(result -> cursor.setString("result", result.name()));
+        runStatus.end().ifPresent(instant -> cursor.setString("end", instant.toString()));
+
+        Cursor stepsArray = cursor.setArray("steps");
+        runStatus.steps().forEach((step, status) -> {
+            Cursor stepObject = stepsArray.addObject();
+            stepObject.setString(step.name(), status.name());
+        });
+
+        cursor.setString("start", runStatus.start().toString());
+        cursor.setLong("id", runStatus.id().number());
+        String logsPath = baseUriForJobType.getPath() + "/run/" + runStatus.id().number();
+        cursor.setString("logs", baseUriForJobType.resolve(logsPath).toString());
+    }
+
+    /**
+     * @return Response with logs from a single run
+     */
+    static HttpResponse runDetailsResponse(JobController jobController, RunId runId) {
+        Slime slime = new Slime();
+        Cursor logsObject = slime.setObject();
+
+        RunDetails runDetails = jobController.details(runId).orElseThrow(() ->
+                new NotExistsException(String.format(
+                        "No run details exist for application: %s, job type: %s, number: %d",
+                        runId.application().toShortString(), runId.type().jobName(), runId.number())));
+        for (Step step : Step.values()) {
+            runDetails.get(step).ifPresent(stepLog -> logsObject.setString(step.name(), stepLog));
+        }
+
+        return new SlimeJsonResponse(slime);
+    }
+
+    /**
+     * Unpack payload and submit to job controller. Defaults instance to 'default' and renders the
+     * application version on success.
+     *
+     * @return Response with the new application version
+     */
+    static HttpResponse submitResponse(JobController jobController, String tenant, String application,
+                                       SourceRevision sourceRevision, byte[] appPackage, byte[] testPackage) {
+        ApplicationVersion version = jobController.submit(ApplicationId.from(tenant, application, "default"),
+                sourceRevision, appPackage, testPackage);
+
+        Slime slime = new Slime();
+        Cursor responseObject = slime.setObject();
+        responseObject.setString("version", version.id());
+        return new SlimeJsonResponse(slime);
+    }
+}
+
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/athenz/filter/AthenzTestUtils.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/athenz/filter/AthenzTestUtils.java
deleted file mode 100644
index 40b38254dda..00000000000
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/athenz/filter/AthenzTestUtils.java
+++ /dev/null
@@ -1,22 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.controller.athenz.filter;
-
-import java.security.KeyPair;
-import java.security.KeyPairGenerator;
-import java.security.NoSuchAlgorithmException;
-
-/**
- * @author bjorncs
- */
-public class AthenzTestUtils {
-    public static KeyPair generateRsaKeypair() {
-        try {
-            KeyPairGenerator keyGen = KeyPairGenerator.getInstance("RSA");
-            keyGen.initialize(512);
-            return keyGen.genKeyPair();
-        } catch (NoSuchAlgorithmException e) {
-            throw new RuntimeException(e);
-        }
-    }
-
-}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java
index af7261149ad..95eb9117c5a 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java
@@ -480,7 +480,6 @@ public class DeploymentTriggerTest {
                 .environment(Environment.prod)
                 .region("us-central-1")
                 .region("eu-west-1")
-                .upgradePolicy("canary")
                 .build();
         tester.deployCompletely(application, applicationPackage);
 
@@ -497,7 +496,7 @@ public class DeploymentTriggerTest {
         tester.clock().advance(Duration.ofHours(1));
 
         Version v1 = new Version("7.1");
-        tester.upgradeSystem(v1); // Downgrade, but it works, since the app is a canary.
+        tester.upgradeSystem(v1); // Downgrade to cut down on the amount of code.
         assertEquals(Change.of(v1), app.get().change());
 
         // 7.1 proceeds 'til the last job, where it fails; us-central-1 is skipped, as current change is strictly dominated by what's deployed there.
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/IntegerationStepRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/IntegerationStepRunnerTest.java
new file mode 100644
index 00000000000..d45f5cc98f8
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/IntegerationStepRunnerTest.java
@@ -0,0 +1,34 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.deployment;
+
+import com.yahoo.config.provision.SystemName;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author freva
+ */
+public class IntegerationStepRunnerTest {
+
+    @Test
+    public void generates_correct_services_xml_test() {
+        assertFile("test_runner_services.xml-cd", new String(InternalStepRunner.servicesXml(SystemName.cd)));
+    }
+
+    private void assertFile(String resourceName, String actualContent) {
+        try {
+            Path path = Paths.get(getClass().getClassLoader().getResource(resourceName).getPath());
+            String expectedContent = new String(Files.readAllBytes(path));
+            assertEquals(expectedContent, actualContent);
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+    }
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ZipBuilderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ZipBuilderTest.java
new file mode 100644
index 00000000000..ce81e4e6dbd
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ZipBuilderTest.java
@@ -0,0 +1,63 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.deployment;
+
+import org.apache.commons.io.IOUtils;
+import org.junit.Test;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipInputStream;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author freva
+ */
+public class ZipBuilderTest {
+
+    @Test
+    public void test() {
+        Map<String, String> expected = new HashMap<>();
+        expected.put("dir/myfile", "my content");
+        expected.put("rootfile", "this is root");
+        expected.put("dir/newfile", "new file");
+        expected.put("dir/dir2/file", "nested file");
+
+        try (ZipBuilder zipBuilder1 = new ZipBuilder(100);
+             ZipBuilder zipBuilder2 = new ZipBuilder(1000)) {
+
+            // Add the entries to both zip builders one by one
+            Iterator<Map.Entry<String, String>> entries = expected.entrySet().iterator();
+            for (int i = 0; entries.hasNext(); i++) {
+                Map.Entry<String, String> entry = entries.next();
+                (i % 2 == 0 ? zipBuilder1 : zipBuilder2)
+                        .add(entry.getKey(), entry.getValue().getBytes(StandardCharsets.UTF_8));
+            }
+
+            // Add the zipped data from zip1 to zip2
+            zipBuilder2.add(zipBuilder1.toByteArray());
+
+            Map<String, String> actual = unzipToMap(zipBuilder2.toByteArray());
+            assertEquals(expected, actual);
+        }
+    }
+
+    Map<String, String> unzipToMap(byte[] zippedContent) {
+        Map<String, String> contents = new HashMap<>();
+        try (ZipInputStream zin = new ZipInputStream(new ByteArrayInputStream(zippedContent))) {
+            for (ZipEntry entry = zin.getNextEntry(); entry != null; entry = zin.getNextEntry()) {
+                if (entry.isDirectory()) continue;
+                contents.put(entry.getName(), IOUtils.toString(zin, StandardCharsets.UTF_8));
+            }
+        } catch (IOException e) {
+            throw new UncheckedIOException("Failed to read zipped content", e);
+        }
+        return contents;
+    }
+}
\ No newline at end of file
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ArtifactRepositoryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ArtifactRepositoryMock.java
index c722d30c885..3f800ad9a56 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ArtifactRepositoryMock.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ArtifactRepositoryMock.java
@@ -52,12 +52,12 @@ public class ArtifactRepositoryMock extends AbstractComponent implements Artifac
     }
 
     @Override
-    public byte[] getTesterJar(ApplicationId tester, String applicationVersion) {
+    public byte[] getTesterPackage(ApplicationId tester, String applicationVersion) {
         throw new AssertionError();
     }
 
     @Override
-    public void putTesterJar(ApplicationId tester, String applicationVersion, byte[] fatTestJar) {
+    public void putTesterPackage(ApplicationId tester, String applicationVersion, byte[] testerPackage) {
         throw new AssertionError();
     }
 
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java
index e084e9aa46d..e5416672bb7 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java
@@ -1,8 +1,10 @@
 package com.yahoo.vespa.hosted.controller.maintenance;
 
+import com.yahoo.component.Version;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.vespa.hosted.controller.TestIdentities;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId;
+import com.yahoo.vespa.hosted.controller.application.ApplicationVersion;
 import com.yahoo.vespa.hosted.controller.application.SourceRevision;
 import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester;
 import com.yahoo.vespa.hosted.controller.deployment.JobController;
@@ -10,6 +12,7 @@ import com.yahoo.vespa.hosted.controller.deployment.RunStatus;
 import com.yahoo.vespa.hosted.controller.deployment.Step;
 import com.yahoo.vespa.hosted.controller.deployment.Step.Status;
 import com.yahoo.vespa.hosted.controller.deployment.StepRunner;
+import com.yahoo.vespa.hosted.controller.deployment.Versions;
 import org.junit.Test;
 
 import java.time.Duration;
@@ -40,7 +43,6 @@ import static com.yahoo.vespa.hosted.controller.deployment.Step.deactivateReal;
 import static com.yahoo.vespa.hosted.controller.deployment.Step.deactivateTester;
 import static com.yahoo.vespa.hosted.controller.deployment.Step.deployReal;
 import static com.yahoo.vespa.hosted.controller.deployment.Step.deployTester;
-import static com.yahoo.vespa.hosted.controller.deployment.Step.installInitialReal;
 import static com.yahoo.vespa.hosted.controller.deployment.Step.installReal;
 import static com.yahoo.vespa.hosted.controller.deployment.Step.installTester;
 import static com.yahoo.vespa.hosted.controller.deployment.Step.report;
@@ -56,6 +58,14 @@ import static org.junit.Assert.fail;
  */
 public class JobRunnerTest {
 
+    static final Versions versions = new Versions(Version.fromString("1.2.3"),
+                                                  ApplicationVersion.from(new SourceRevision("repo",
+                                                                                             "branch",
+                                                                                             "bada55"),
+                                                                          321),
+                                                  Optional.empty(),
+                                                  Optional.empty());
+
     @Test
     public void multiThreadedExecutionFinishes() throws InterruptedException {
         DeploymentTester tester = new DeploymentTester();
@@ -67,15 +77,15 @@ public class JobRunnerTest {
                                          Executors.newFixedThreadPool(32), notifying(stepRunner, latch));
 
         ApplicationId id = tester.createApplication("real", "tenant", 1, 1L).id();
-        jobs.submit(id, new SourceRevision("repo", "branch", "bada55"), new byte[0], new byte[0]);
+        jobs.submit(id, versions.targetApplication().source().get(), new byte[0], new byte[0]);
 
-        jobs.start(id, systemTest);
+        jobs.start(id, systemTest, versions);
         try {
-            jobs.start(id, systemTest);
+            jobs.start(id, systemTest, versions);
             fail("Job is already running, so this should not be allowed!");
         }
         catch (IllegalStateException e) { }
-        jobs.start(id, stagingTest);
+        jobs.start(id, stagingTest, versions);
 
         assertTrue(jobs.last(id, systemTest).get().steps().values().stream().allMatch(unfinished::equals));
         runner.maintain();
@@ -100,10 +110,10 @@ public class JobRunnerTest {
                                          inThreadExecutor(), mappedRunner(outcomes));
 
         ApplicationId id = tester.createApplication("real", "tenant", 1, 1L).id();
-        jobs.submit(id, new SourceRevision("repo", "branch", "bada55"), new byte[0], new byte[0]);
+        jobs.submit(id, versions.targetApplication().source().get(), new byte[0], new byte[0]);
         Supplier<RunStatus> run = () -> jobs.last(id, systemTest).get();
 
-        jobs.start(id, systemTest);
+        jobs.start(id, systemTest, versions);
         RunId first = run.get().id();
 
         Map<Step, Status> steps = run.get().steps();
@@ -155,7 +165,7 @@ public class JobRunnerTest {
         assertTrue(run.get().isAborted());
 
         // A new run is attempted.
-        jobs.start(id, systemTest);
+        jobs.start(id, systemTest, versions);
         assertEquals(first.number() + 1, run.get().id().number());
 
         // Run fails on tester deployment -- remaining run-always steps succeed, and the run finishes.
@@ -171,7 +181,7 @@ public class JobRunnerTest {
         assertEquals(2, jobs.runs(id, systemTest).size());
 
         // Start a third run, then unregister and wait for data to be deleted.
-        jobs.start(id, systemTest);
+        jobs.start(id, systemTest, versions);
         jobs.unregister(id);
         runner.maintain();
         assertFalse(jobs.last(id, systemTest).isPresent());
@@ -188,10 +198,10 @@ public class JobRunnerTest {
                                          Executors.newFixedThreadPool(32), waitingRunner(barrier));
 
         ApplicationId id = tester.createApplication("real", "tenant", 1, 1L).id();
-        jobs.submit(id, new SourceRevision("repo", "branch", "bada55"), new byte[0], new byte[0]);
+        jobs.submit(id, versions.targetApplication().source().get(), new byte[0], new byte[0]);
 
         RunId runId = new RunId(id, systemTest, 1);
-        jobs.start(id, systemTest);
+        jobs.start(id, systemTest, versions);
         runner.maintain();
         barrier.await();
         try {
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json
index d659bd9fff0..9b3de20acc4 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json
@@ -5,17 +5,31 @@
     "number": 112358,
     "start": 1196676930000,
     "steps": {
-      "DIR": "U",
-      "IIR": "F",
-      "DR":  "S",
-      "IR":  "U",
-      "DAR": "F",
-      "DT":  "S",
-      "IT":  "U",
-      "DAT": "F",
-      "ST":  "S",
-      "ET":  "U",
-      "R":   "F"
+      "deployInitialReal": "unfinished",
+      "installInitialReal": "failed",
+      "deployReal": "succeeded",
+      "installReal": "unfinished",
+      "deactivateReal": "failed",
+      "deployTester": "succeeded",
+      "installTester": "unfinished",
+      "deactivateTester": "failed",
+      "startTests": "succeeded",
+      "endTests": "unfinished",
+      "report": "failed"
+    },
+    "versions": {
+      "platform": "1.2.3",
+      "repository": "git@github.com:user/repo.git",
+      "branch": "master",
+      "commit": "f00bad",
+      "build": 123,
+      "source": {
+        "platform": "1.2.3",
+        "repository": "git@github.com:user/repo.git",
+        "branch": "master",
+        "commit": "badb17",
+        "build": 122
+      }
     }
   }
 ]
\ No newline at end of file
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java
index de645cff96c..639cfad9958 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java
@@ -81,7 +81,7 @@ public class ControllerContainerTest {
             "  <component id='com.yahoo.vespa.hosted.controller.maintenance.JobControl'/>\n" +
             "  <component id='com.yahoo.vespa.hosted.controller.integration.RoutingGeneratorMock'/>\n" +
             "  <component id='com.yahoo.vespa.hosted.controller.integration.ArtifactRepositoryMock'/>\n" +
-            "  <component id='com.yahoo.vespa.hosted.controller.api.integration.stubs.MockTesters'/>\n" +
+            "  <component id='com.yahoo.vespa.hosted.controller.api.integration.stubs.MockTesterCloud'/>\n" +
             "  <handler id='com.yahoo.vespa.hosted.controller.restapi.application.ApplicationApiHandler'>\n" +
             "    <binding>http://*/application/v4/*</binding>\n" +
             "  </handler>\n" +
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java
new file mode 100644
index 00000000000..f350cc8627b
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java
@@ -0,0 +1,168 @@
+package com.yahoo.vespa.hosted.controller.restapi.application;
+
+import com.yahoo.component.Version;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.TenantName;
+import com.yahoo.container.jdisc.HttpResponse;
+import com.yahoo.vespa.hosted.controller.ControllerTester;
+import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
+import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId;
+import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockLogStore;
+import com.yahoo.vespa.hosted.controller.application.ApplicationVersion;
+import com.yahoo.vespa.hosted.controller.application.SourceRevision;
+import com.yahoo.vespa.hosted.controller.deployment.JobController;
+import com.yahoo.vespa.hosted.controller.deployment.RunStatus;
+import com.yahoo.vespa.hosted.controller.deployment.Step;
+import com.yahoo.vespa.hosted.controller.deployment.Versions;
+import org.json.JSONException;
+import org.json.JSONObject;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.net.URI;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+import static org.junit.Assert.fail;
+
+public class JobControllerApiHandlerHelperTest {
+
+    private final ApplicationId appId = ApplicationId.from("vespa", "music", "default");
+    private final Instant start = Instant.parse("2018-06-27T10:12:35Z");
+    static final Versions versions = new Versions(Version.fromString("1.2.3"),
+                                                  ApplicationVersion.from(new SourceRevision("repo",
+                                                                                             "branch",
+                                                                                             "bada55"),
+                                                                          321),
+                                                  Optional.empty(),
+                                                  Optional.empty());
+
+    private static Step lastStep = Step.values()[Step.values().length - 1];
+
+    @Test
+    public void jobTypeResponse() {
+        Map<JobType, RunStatus> jobMap = new HashMap<>();
+        List<JobType> jobList = new ArrayList<>();
+        jobMap.put(JobType.systemTest, createStatus(JobType.systemTest, 1, 30, lastStep, Step.Status.succeeded));
+        jobList.add(JobType.systemTest);
+        jobMap.put(JobType.productionApNortheast1, createStatus(JobType.productionApNortheast1, 1, 60, lastStep, Step.Status.succeeded));
+        jobList.add(JobType.productionApNortheast1);
+        jobMap.put(JobType.productionUsWest1, createStatus(JobType.productionUsWest1, 1, 60, Step.startTests, Step.Status.failed));
+        jobList.add(JobType.productionUsWest1);
+
+        URI jobUrl = URI.create("https://domain.tld/application/v4/tenant/sometenant/application/someapp/instance/usuallydefault/job");
+
+        HttpResponse response = JobControllerApiHandlerHelper.jobTypeResponse(jobList, jobMap, jobUrl);
+        assertFile(response, "job/job-type-response.json");
+    }
+
+    @Test
+    public void runStatusResponse() {
+        Map<RunId, RunStatus> statusMap = new HashMap<>();
+        RunStatus status;
+
+        status = createStatus(JobType.systemTest, 3, 30, lastStep, Step.Status.succeeded);
+        statusMap.put(status.id(), status);
+
+        status = createStatus(JobType.systemTest, 2, 56, Step.installReal, Step.Status.failed);
+        statusMap.put(status.id(), status);
+
+        status = createStatus(JobType.systemTest, 1, 44, lastStep, Step.Status.succeeded);
+        statusMap.put(status.id(), status);
+
+        URI jobTypeUrl = URI.create("https://domain.tld/application/v4/tenant/sometenant/application/someapp/instance/usuallydefault/job/systemtest");
+
+        HttpResponse response = JobControllerApiHandlerHelper.runStatusResponse(statusMap, jobTypeUrl);
+        assertFile(response, "job/run-status-response.json");
+    }
+
+    @Test
+    public void runDetailsResponse() {
+        ControllerTester tester = new ControllerTester();
+        MockLogStore logStore = new MockLogStore();
+        JobController jobController = new JobController(tester.controller(), logStore);
+        RunId runId = new RunId(appId, JobType.systemTest, 42);
+        tester.curator().writeHistoricRuns(
+                runId.application(),
+                runId.type(),
+                Collections.singleton(createStatus(JobType.systemTest, 42, 44, lastStep, Step.Status.succeeded)));
+
+        logStore.append(runId, Step.deployTester.name(), "INFO\t1234567890\tSUCCESS".getBytes());
+        logStore.append(runId, Step.installTester.name(), "INFO\t1234598760\tSUCCESS".getBytes());
+        logStore.append(runId, Step.deactivateTester.name(), "INFO\t1234678901\tERROR: Something went wrong".getBytes());
+
+        HttpResponse response = JobControllerApiHandlerHelper.runDetailsResponse(jobController, runId);
+        assertFile(response, "job/run-details-response.json");
+    }
+
+    @Test
+    public void submitResponse() {
+        ControllerTester tester = new ControllerTester();
+        tester.createTenant("tenant", "domain", 1L);
+        tester.createApplication(TenantName.from("tenant"), "application", "default", 1L);
+
+        JobController jobController = new JobController(tester.controller(), new MockLogStore());
+        jobController.register(ApplicationId.from("tenant", "application", "default"));
+
+        HttpResponse response = JobControllerApiHandlerHelper.submitResponse(
+                jobController, "tenant", "application", new SourceRevision("repository", "branch", "commit"), new byte[0], new byte[0]);
+        compare(response, "{\"version\":\"1.0.1-commit\"}");
+    }
+
+
+    private RunStatus createStatus(JobType type, long runid, long duration, Step lastStep, Step.Status lastStepStatus) {
+        RunId runId = new RunId(appId, type, runid);
+
+        Map<Step, Step.Status> stepStatusMap = new HashMap<>();
+        for (Step step : Step.values()) {
+            if (step.ordinal() < lastStep.ordinal()) {
+                stepStatusMap.put(step, Step.Status.succeeded);
+            } else if (step.equals(lastStep)) {
+                stepStatusMap.put(step, lastStepStatus);
+            } else {
+                stepStatusMap.put(step, Step.Status.unfinished);
+            }
+        }
+
+        Optional<Instant> end = Optional.empty();
+        if (lastStepStatus == Step.Status.failed || stepStatusMap.get(lastStep) == Step.Status.succeeded) {
+            end = Optional.of(start.plusSeconds(duration));
+        }
+
+        return new RunStatus(runId, stepStatusMap, versions, start, end, false);
+    }
+
+    private void compare(HttpResponse response, String expected) {
+        try {
+            ByteArrayOutputStream baos = new ByteArrayOutputStream();
+            response.render(baos);
+            String actual = new String(baos.toByteArray());
+
+            JSONObject actualJSON = new JSONObject(actual);
+            JSONObject expectedJSON = new JSONObject(expected);
+            Assert.assertEquals(expectedJSON.toString(), actualJSON.toString());
+        } catch (IOException | JSONException e) {
+            fail();
+        }
+    }
+
+    private void assertFile(HttpResponse response, String resourceName) {
+        try {
+            Path path = Paths.get(getClass().getClassLoader().getResource(resourceName).getPath());
+            String expected = new String(Files.readAllBytes(path));
+            compare(response, expected);
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+}
diff --git a/controller-server/src/test/resources/job/job-type-response.json b/controller-server/src/test/resources/job/job-type-response.json
new file mode 100644
index 00000000000..2c51f0d9275
--- /dev/null
+++ b/controller-server/src/test/resources/job/job-type-response.json
@@ -0,0 +1,145 @@
+{
+  "jobs":[
+    {
+      "system-test":{
+        "last":{
+          "result":"success",
+          "start":"2018-06-27T10:12:35Z",
+          "end":"2018-06-27T10:13:05Z",
+          "id":1,
+          "steps":[
+            {
+              "deployInitialReal":"succeeded"
+            },
+            {
+              "installInitialReal":"succeeded"
+            },
+            {
+              "deployReal":"succeeded"
+            },
+            {
+              "installReal":"succeeded"
+            },
+            {
+              "deployTester":"succeeded"
+            },
+            {
+              "installTester":"succeeded"
+            },
+            {
+              "startTests":"succeeded"
+            },
+            {
+              "endTests":"succeeded"
+            },
+            {
+              "deactivateReal":"succeeded"
+            },
+            {
+              "deactivateTester":"succeeded"
+            },
+            {
+              "report":"succeeded"
+            }
+          ],
+          "logs":"https://domain.tld/application/v4/tenant/sometenant/application/someapp/instance/usuallydefault/job/system-test/run/1"
+        },
+        "url":"https://domain.tld/application/v4/tenant/sometenant/application/someapp/instance/usuallydefault/job/system-test"
+      }
+    },
+    {
+      "production-ap-northeast-1":{
+        "last":{
+          "result":"success",
+          "start":"2018-06-27T10:12:35Z",
+          "end":"2018-06-27T10:13:35Z",
+          "id":1,
+          "steps":[
+            {
+              "deployInitialReal":"succeeded"
+            },
+            {
+              "installInitialReal":"succeeded"
+            },
+            {
+              "deployReal":"succeeded"
+            },
+            {
+              "installReal":"succeeded"
+            },
+            {
+              "deployTester":"succeeded"
+            },
+            {
+              "installTester":"succeeded"
+            },
+            {
+              "startTests":"succeeded"
+            },
+            {
+              "endTests":"succeeded"
+            },
+            {
+              "deactivateReal":"succeeded"
+            },
+            {
+              "deactivateTester":"succeeded"
+            },
+            {
+              "report":"succeeded"
+            }
+          ],
+          "logs":"https://domain.tld/application/v4/tenant/sometenant/application/someapp/instance/usuallydefault/job/production-ap-northeast-1/run/1"
+        },
+        "url":"https://domain.tld/application/v4/tenant/sometenant/application/someapp/instance/usuallydefault/job/production-ap-northeast-1"
+      }
+    },
+    {
+      "production-us-west-1":{
+        "last":{
+          "result":"testError",
+          "start":"2018-06-27T10:12:35Z",
+          "end":"2018-06-27T10:13:35Z",
+          "id":1,
+          "steps":[
+            {
+              "deployInitialReal":"succeeded"
+            },
+            {
+              "installInitialReal":"succeeded"
+            },
+            {
+              "deployReal":"succeeded"
+            },
+            {
+              "installReal":"succeeded"
+            },
+            {
+              "deployTester":"succeeded"
+            },
+            {
+              "installTester":"succeeded"
+            },
+            {
+              "startTests":"failed"
+            },
+            {
+              "endTests":"unfinished"
+            },
+            {
+              "deactivateReal":"unfinished"
+            },
+            {
+              "deactivateTester":"unfinished"
+            },
+            {
+              "report":"unfinished"
+            }
+          ],
+          "logs":"https://domain.tld/application/v4/tenant/sometenant/application/someapp/instance/usuallydefault/job/production-us-west-1/run/1"
+        },
+        "url":"https://domain.tld/application/v4/tenant/sometenant/application/someapp/instance/usuallydefault/job/production-us-west-1"
+      }
+    }
+  ]
+}
diff --git a/controller-server/src/test/resources/job/run-details-response.json b/controller-server/src/test/resources/job/run-details-response.json
new file mode 100644
index 00000000000..3ba9bff049e
--- /dev/null
+++ b/controller-server/src/test/resources/job/run-details-response.json
@@ -0,0 +1,5 @@
+{
+  "deployTester":"INFO\t1234567890\tSUCCESS",
+  "installTester":"INFO\t1234598760\tSUCCESS",
+  "deactivateTester":"INFO\t1234678901\tERROR: Something went wrong"
+}
diff --git a/controller-server/src/test/resources/job/run-status-response.json b/controller-server/src/test/resources/job/run-status-response.json
new file mode 100644
index 00000000000..c3041440fd2
--- /dev/null
+++ b/controller-server/src/test/resources/job/run-status-response.json
@@ -0,0 +1,128 @@
+{
+  "1":{
+    "result":"success",
+    "start":"2018-06-27T10:12:35Z",
+    "end":"2018-06-27T10:13:19Z",
+    "id":1,
+    "steps":[
+      {
+        "deployInitialReal":"succeeded"
+      },
+      {
+        "installInitialReal":"succeeded"
+      },
+      {
+        "deployReal":"succeeded"
+      },
+      {
+        "installReal":"succeeded"
+      },
+      {
+        "deployTester":"succeeded"
+      },
+      {
+        "installTester":"succeeded"
+      },
+      {
+        "startTests":"succeeded"
+      },
+      {
+        "endTests":"succeeded"
+      },
+      {
+        "deactivateReal":"succeeded"
+      },
+      {
+        "deactivateTester":"succeeded"
+      },
+      {
+        "report":"succeeded"
+      }
+    ],
+    "logs":"https://domain.tld/application/v4/tenant/sometenant/application/someapp/instance/usuallydefault/job/systemtest/run/1"
+  },
+  "2":{
+    "result":"testError",
+    "start":"2018-06-27T10:12:35Z",
+    "end":"2018-06-27T10:13:31Z",
+    "id":2,
+    "steps":[
+      {
+        "deployInitialReal":"succeeded"
+      },
+      {
+        "installInitialReal":"succeeded"
+      },
+      {
+        "deployReal":"succeeded"
+      },
+      {
+        "installReal":"failed"
+      },
+      {
+        "deployTester":"unfinished"
+      },
+      {
+        "installTester":"unfinished"
+      },
+      {
+        "startTests":"unfinished"
+      },
+      {
+        "endTests":"unfinished"
+      },
+      {
+        "deactivateReal":"unfinished"
+      },
+      {
+        "deactivateTester":"unfinished"
+      },
+      {
+        "report":"unfinished"
+      }
+    ],
+    "logs":"https://domain.tld/application/v4/tenant/sometenant/application/someapp/instance/usuallydefault/job/systemtest/run/2"
+  },
+  "3":{
+    "result":"success",
+    "start":"2018-06-27T10:12:35Z",
+    "end":"2018-06-27T10:13:05Z",
+    "id":3,
+    "steps":[
+      {
+        "deployInitialReal":"succeeded"
+      },
+      {
+        "installInitialReal":"succeeded"
+      },
+      {
+        "deployReal":"succeeded"
+      },
+      {
+        "installReal":"succeeded"
+      },
+      {
+        "deployTester":"succeeded"
+      },
+      {
+        "installTester":"succeeded"
+      },
+      {
+        "startTests":"succeeded"
+      },
+      {
+        "endTests":"succeeded"
+      },
+      {
+        "deactivateReal":"succeeded"
+      },
+      {
+        "deactivateTester":"succeeded"
+      },
+      {
+        "report":"succeeded"
+      }
+    ],
+    "logs":"https://domain.tld/application/v4/tenant/sometenant/application/someapp/instance/usuallydefault/job/systemtest/run/3"
+  }
+}
diff --git a/controller-server/src/test/resources/test_runner_services.xml-cd b/controller-server/src/test/resources/test_runner_services.xml-cd
new file mode 100644
index 00000000000..c539478eec8
--- /dev/null
+++ b/controller-server/src/test/resources/test_runner_services.xml-cd
@@ -0,0 +1,36 @@
+<?xml version='1.0' encoding='UTF-8'?>
+<services xmlns:deploy='vespa' version='1.0'>
+    <container version='1.0' id='default'>
+
+        <component id="com.yahoo.vespa.hosted.testrunner.TestRunner" bundle="vepsa-testrunner-components">
+            <config name="com.yahoo.vespa.hosted.testrunner.test-runner">
+                <artifactsPath>artifacts</artifactsPath>
+            </config>
+        </component>
+
+        <handler id="com.yahoo.vespa.hosted.testrunner.TestRunnerHandler" bundle="vespa-testrunner-components">
+            <binding>http://*/tester/v1/*</binding>
+        </handler>
+
+        <http>
+            <filtering>
+                <request-chain id="testrunner-api">
+                    <filter id='authz-filter' class='com.yahoo.jdisc.http.filter.security.athenz.AthenzAuthorizationFilter' bundle="jdisc-security-filters">
+                        <config name="jdisc.http.filter.security.athenz.athenz-authorization-filter">
+                            <credentialsToVerify>TOKEN_ONLY</credentialsToVerify>
+                            <roleTokenHeaderName>Yahoo-Role-Auth</roleTokenHeaderName>
+                        </config>
+                        <component id="com.yahoo.jdisc.http.filter.security.athenz.StaticRequestResourceMapper" bundle="jdisc-security-filters">
+                            <config name="jdisc.http.filter.security.athenz.static-request-resource-mapper">
+                                <resourceName>vespa.vespa.cd:tester-application</resourceName>
+                                <action>deploy</action>
+                            </config>
+                        </component>
+                    </filter>
+                </request-chain>
+            </filtering>
+        </http>
+
+        <nodes count="1" flavor="d-2-8-50" />
+    </container>
+</services>
diff --git a/defaults/src/apps/printdefault/printdefault.cpp b/defaults/src/apps/printdefault/printdefault.cpp
index 33c47dbcb20..be6b989fd70 100644
--- a/defaults/src/apps/printdefault/printdefault.cpp
+++ b/defaults/src/apps/printdefault/printdefault.cpp
@@ -4,6 +4,30 @@
 #include <stdio.h>
 #include <string.h>
 
+void dumpAllVars() {
+    printf("VESPA_HOME = '%s'\n", vespa::Defaults::vespaHome());
+    std::string v = vespa::Defaults::underVespaHome("foo");
+    printf("underVespaHome(foo) = '%s'\n", v.c_str());
+    printf("VESPA_USER = '%s'\n", vespa::Defaults::vespaUser());
+    printf("VESPA_HOSTNAME = '%s'\n", vespa::Defaults::vespaHostname());
+    printf("web service port = %d\n", vespa::Defaults::vespaWebServicePort());
+    printf("VESPA_PORT_BASE = %d\n", vespa::Defaults::vespaPortBase());
+    printf("config server rpc port = %d\n", vespa::Defaults::vespaConfigServerRpcPort());
+    size_t count = 0;
+    for (std::string vv : vespa::Defaults::vespaConfigServerHosts()) {
+        ++count;
+        printf("config server host %zu = '%s'\n", count, vv.c_str());
+    }
+    count = 0;
+    for (std::string vv : vespa::Defaults::vespaConfigServerRestUrls()) {
+        ++count;
+        printf("config server rest URL %zu = '%s'\n", count, vv.c_str());
+    }
+    v = vespa::Defaults::vespaConfigProxyRpcAddr();
+    printf("config proxy RPC addr = '%s'\n", v.c_str());
+    printf("vespa version = '%s'\n", V_TAG_COMPONENT);
+}
+
 int main(int argc, char **argv) {
     if (argc != 2) {
         fprintf(stderr, "usage: %s <variable>\n", argv[0]);
@@ -15,6 +39,8 @@ int main(int argc, char **argv) {
     }
     if (strcmp(argv[1], "home") == 0) {
         printf("%s\n", vespa::Defaults::vespaHome());
+    } else if (strcmp(argv[1], "everything") == 0) {
+        dumpAllVars();
     } else if (strcmp(argv[1], "user") == 0) {
         printf("%s\n", vespa::Defaults::vespaUser());
     } else if (strcmp(argv[1], "hostname") == 0) {
diff --git a/defaults/src/main/java/com/yahoo/vespa/defaults/Defaults.java b/defaults/src/main/java/com/yahoo/vespa/defaults/Defaults.java
index af92a183443..0fce5d654fb 100644
--- a/defaults/src/main/java/com/yahoo/vespa/defaults/Defaults.java
+++ b/defaults/src/main/java/com/yahoo/vespa/defaults/Defaults.java
@@ -27,20 +27,25 @@ public class Defaults {
     private final String vespaHost;
     private final int vespaWebServicePort;
     private final int vespaPortBase;
+    private final int vespaPortConfigServerRpc;
+    private final int vespaPortConfigServerHttp;
+    private final int vespaPortConfigProxyRpc;
 
     private Defaults() {
-        vespaHome = findVespaHome();
-        vespaUser = findVespaUser();
-        vespaHost = findVespaHostname();
-        vespaWebServicePort = findVespaWebServicePort();
-        vespaPortBase = 19000; // TODO
+        vespaHome = findVespaHome("/opt/vespa");
+        vespaUser = findVespaUser("vespa");
+        vespaHost = findVespaHostname("localhost");
+        vespaWebServicePort = findWebServicePort(8080);
+        vespaPortBase = findVespaPortBase(19000);
+        vespaPortConfigServerRpc = findConfigServerPort(vespaPortBase + 70);
+        vespaPortConfigServerHttp = vespaPortConfigServerRpc + 1;
+        vespaPortConfigProxyRpc = findConfigProxyPort(vespaPortBase + 90);
     }
-
-    static private String findVespaHome() {
+    static private String findVespaHome(String defHome) {
         Optional<String> vespaHomeEnv = Optional.ofNullable(System.getenv("VESPA_HOME"));
         if ( ! vespaHomeEnv.isPresent() || vespaHomeEnv.get().trim().isEmpty()) {
-            log.info("VESPA_HOME not set, using /opt/vespa");
-            return "/opt/vespa";
+            log.info("VESPA_HOME not set, using " + defHome);
+            return defHome;
         }
         String vespaHome = vespaHomeEnv.get().trim();
         if (vespaHome.endsWith("/")) {
@@ -50,38 +55,50 @@ public class Defaults {
         return vespaHome;
     }
 
-    static private String findVespaHostname() {
+    static private String findVespaHostname(String defHost) {
         Optional<String> vespaHostEnv = Optional.ofNullable(System.getenv("VESPA_HOSTNAME"));
         if (vespaHostEnv.isPresent() && ! vespaHostEnv.get().trim().isEmpty()) {
             return vespaHostEnv.get().trim();
         }
-        return "localhost";
+        return defHost;
     }
 
-    static private String findVespaUser() {
+    static private String findVespaUser(String defUser) {
         Optional<String> vespaUserEnv = Optional.ofNullable(System.getenv("VESPA_USER"));
         if (! vespaUserEnv.isPresent()) {
-            log.fine("VESPA_USER not set, using vespa");
-            return "vespa";
+            log.fine("VESPA_USER not set, using "+defUser);
+            return defUser;
         }
         return vespaUserEnv.get().trim();
     }
 
-    static private int findVespaWebServicePort() {
-        Optional<String> vespaWebServicePortString = Optional.ofNullable(System.getenv("VESPA_WEB_SERVICE_PORT"));
-        if ( ! vespaWebServicePortString.isPresent() || vespaWebServicePortString.get().trim().isEmpty()) {
-            log.info("VESPA_WEB_SERVICE_PORT not set, using 8080");
-            return 8080;
+    static private int findPort(String varName, int defaultPort) {
+        Optional<String> port = Optional.ofNullable(System.getenv(varName));
+        if ( ! port.isPresent() || port.get().trim().isEmpty()) {
+            log.fine("" + varName + " not set, using " + defaultPort);
+            return defaultPort;
         }
         try {
-            return Integer.parseInt(vespaWebServicePortString.get());
-        }
-        catch (NumberFormatException e) {
-            throw new IllegalArgumentException("VESPA_WEB_SERVICE_PORT must be an integer, was '" +
-                                               vespaWebServicePortString.get() + "'");
+            return Integer.parseInt(port.get());
+        } catch (NumberFormatException e) {
+            throw new IllegalArgumentException("must be an integer, was '" +
+                                               port.get() + "'");
         }
     }
 
+    static private int findVespaPortBase(int defaultPort) {
+        return findPort("VESPA_PORT_BASE", defaultPort);
+    }
+    static private int findConfigServerPort(int defaultPort) {
+        return findPort("port_configserver_rpc", defaultPort);
+    }
+    static private int findConfigProxyPort(int defaultPort) {
+        return findPort("port_configproxy_rpc", defaultPort);
+    }
+    static private int findWebServicePort(int defaultPort) {
+        return findPort("VESPA_WEB_SERVICE_PORT", defaultPort);
+    }
+
     /**
      * Get the username to own directories, files and processes
      * @return the vespa user name
@@ -135,6 +152,15 @@ public class Defaults {
      */
     public int vespaPortBase() { return vespaPortBase; }
 
+    /** @return port number used by cloud config server (for its RPC protocol) */
+    public int vespaConfigServerRpcPort() { return vespaPortConfigServerRpc; }
+
+    /** @return port number used by cloud config server (REST api on HTTP) */
+    public int vespaConfigServerHttpPort() { return vespaPortConfigServerHttp; }
+
+    /** @return port number used by config proxy server (RPC protocol) */
+    public int vespaConfigProxyRpcPort() { return vespaPortConfigProxyRpc; }
+
     /** Returns the defaults of this runtime environment */
     public static Defaults getDefaults() { return defaults; }
 
diff --git a/defaults/src/test/java/com/yahoo/vespa/defaults/DefaultsTestCase.java b/defaults/src/test/java/com/yahoo/vespa/defaults/DefaultsTestCase.java
index 75da8e1146f..07d3c39fc9c 100644
--- a/defaults/src/test/java/com/yahoo/vespa/defaults/DefaultsTestCase.java
+++ b/defaults/src/test/java/com/yahoo/vespa/defaults/DefaultsTestCase.java
@@ -22,4 +22,29 @@ public class DefaultsTestCase {
         assertEquals("vespa", Defaults.getDefaults().vespaUser());
     }
 
+    @Test
+    public void testPortsArePositive() {
+        Defaults d = Defaults.getDefaults();
+        assertEquals(true, d.vespaPortBase() > 0);
+        assertEquals(true, d.vespaWebServicePort() > 0);
+        assertEquals(true, d.vespaConfigServerRpcPort() > 0);
+        assertEquals(true, d.vespaConfigServerHttpPort() > 0);
+        assertEquals(true, d.vespaConfigProxyRpcPort() > 0);
+    }
+
+    @Test
+    public void dumpAllVars() {
+        Defaults d = Defaults.getDefaults();
+        System.out.println("vespa user = '" + d.vespaUser() + "'");
+        System.out.println("vespa hostname = '" + d.vespaHostname() + "'");
+        System.out.println("vespa home = '" + d.vespaHome() + "'");
+        System.out.println("underVespaHome(foo) = '" + d.underVespaHome("foo") + "'");
+
+        System.out.println("web service port = '" + d.vespaWebServicePort() + "'");
+        System.out.println("vespa port base = '" + d.vespaPortBase() + "'");
+        System.out.println("config server RPC port = '" + d.vespaConfigServerRpcPort() + "'");
+        System.out.println("config server HTTP port = '" + d.vespaConfigServerHttpPort() + "'");
+        System.out.println("config proxy RPC port = '" + d.vespaConfigProxyRpcPort() + "'");
+    }
+
 }
diff --git a/defaults/src/vespa/defaults.cpp b/defaults/src/vespa/defaults.cpp
index dcb5e38584d..428dcfa6ce5 100644
--- a/defaults/src/vespa/defaults.cpp
+++ b/defaults/src/vespa/defaults.cpp
@@ -15,15 +15,15 @@ namespace {
 
 #define HOST_BUF_SZ 1024
 
-const char *defaultHome = "/opt/vespa";
-const char *defaultUser = "vespa";
-const char *defaultHost = "localhost";
-int defaultWebServicePort = 8080;
-int defaultPortBase = 19000;
-int defaultPortConfigServerRpc = 19070;
-int defaultPortConfigServerHttp = 19071;
-int defaultPortConfigProxyRpc = 19090;
-const char *defaultConfigServers = "localhost";
+const char *defaultHome = 0;
+const char *defaultUser = 0;
+const char *defaultHost = 0;
+int defaultWebServicePort = 0;
+int defaultPortBase = 0;
+int defaultPortConfigServerRpc = 0;
+int defaultPortConfigServerHttp = 0;
+int defaultPortConfigProxyRpc = 0;
+const char *defaultConfigServers = 0;
 
 std::atomic<bool> initialized(false);
 
@@ -42,8 +42,7 @@ long getNumFromEnv(const char *envName)
     return -1;
 }
 
-void findDefaults() {
-    if (initialized) return;
+const char *findVespaHome(const char *defHome) {
     const char *env = getenv("VESPA_HOME");
     if (env != NULL && *env != '\0') {
         DIR *dp = NULL;
@@ -51,47 +50,72 @@ void findDefaults() {
             dp = opendir(env);
         }
         if (dp != NULL) {
-            defaultHome = env;
-            // fprintf(stderr, "debug\tVESPA_HOME is '%s'\n", defaultHome);
+            // fprintf(stderr, "debug\tVESPA_HOME is '%s'\n", env);
             closedir(dp);
+            return env;
         } else {
             fprintf(stderr, "warning\tbad VESPA_HOME '%s' (ignored)\n", env);
         }
     }
-    env = getenv("VESPA_USER");
+    return defHome;
+}
+
+const char *findVespaUser(const char *defUser) {
+    const char *env = getenv("VESPA_USER");
     if (env != NULL && *env != '\0') {
         if (getpwnam(env) == 0) {
             fprintf(stderr, "warning\tbad VESPA_USER '%s' (ignored)\n", env);
         } else {
-            defaultUser = env;
+            return env;
         }
     }
-    env = getenv("VESPA_HOSTNAME");
+    return defUser;
+}
+
+const char *findHostname(const char *defHost) {
+    const char *env = getenv("VESPA_HOSTNAME");
     if (env != NULL && *env != '\0') {
-        defaultHost = env;
+        return env;
     }
+    return defHost;
+}
+
+int findWebServicePort(int defPort) {
     long p = getNumFromEnv("VESPA_WEB_SERVICE_PORT");
     if (p > 0) {
         // fprintf(stderr, "debug\tdefault web service port is '%ld'\n", p);
-        defaultWebServicePort = p;
+        return p;
     }
-    p = getNumFromEnv("VESPA_PORT_BASE");
+    return defPort;
+}
+
+int findVespaPortBase(int defPort) {
+    long p = getNumFromEnv("VESPA_PORT_BASE");
     if (p > 0) {
         // fprintf(stderr, "debug\tdefault port base is '%ld'\n", p);
-        defaultPortBase = p;
+        return p;
     }
-    p = getNumFromEnv("port_configserver_rpc");
+    return defPort;
+}
+
+int findConfigServerPort(int defPort) {
+    long p = getNumFromEnv("port_configserver_rpc");
     if (p > 0) {
-        defaultPortConfigServerRpc = p;
-        defaultPortConfigServerHttp = p+1;
+        return p;
     }
-    p = getNumFromEnv("port_configproxy_rpc");
+    return defPort;
+}
+
+int findConfigProxyPort(int defPort) {
+    long p = getNumFromEnv("port_configproxy_rpc");
     if (p > 0) {
-        defaultPortConfigProxyRpc = p;
-    } else {
-        defaultPortConfigProxyRpc = defaultPortBase + 90;
+        return p;
     }
-    env = getenv("VESPA_CONFIGSERVERS");
+    return defPort;
+}
+
+const char *findConfigServers(const char *defServers) {
+    const char *env = getenv("VESPA_CONFIGSERVERS");
     if (env == NULL || *env == '\0') {
         env = getenv("services__addr_configserver");
     }
@@ -103,14 +127,26 @@ void findDefaults() {
     }
     if (env != NULL && *env != '\0') {
         // fprintf(stderr, "debug\tdefault configserver(s) is '%s'\n", env);
-        defaultConfigServers = env;
+        return env;
     }
-    initialized = true;
+    return defServers;
 }
 
-}
+void findDefaults() {
+    if (initialized) return;
 
-namespace vespa {
+    defaultHome = findVespaHome("/opt/vespa");
+    defaultUser = findVespaUser("vespa");
+    defaultHost = findHostname("localhost");
+    defaultWebServicePort = findWebServicePort(8080);
+    defaultPortBase = findVespaPortBase(19000);
+    defaultPortConfigServerRpc = findConfigServerPort(defaultPortBase + 70);
+    defaultPortConfigServerHttp = defaultPortConfigServerRpc + 1;
+    defaultPortConfigProxyRpc = findConfigProxyPort(defaultPortBase + 90);
+    defaultConfigServers = findConfigServers("localhost");
+
+    initialized = true;
+}
 
 std::string myPath(const char *argv0)
 {
@@ -137,6 +173,10 @@ std::string myPath(const char *argv0)
     return argv0;
 }
 
+}
+
+namespace vespa {
+
 void
 Defaults::bootstrap(const char *argv0)
 {
@@ -241,6 +281,7 @@ Defaults::vespaConfigServerHosts()
 int
 Defaults::vespaConfigServerRpcPort()
 {
+    findDefaults();
     return defaultPortConfigServerRpc;
 }
 
diff --git a/dist/vespa.spec b/dist/vespa.spec
index 0e3cabcea80..bd2bf798e58 100644
--- a/dist/vespa.spec
+++ b/dist/vespa.spec
@@ -56,7 +56,7 @@ BuildRequires: gtest-devel
 BuildRequires: gmock-devel
 %endif
 %if 0%{?fc29}
-BuildRequires: llvm4.0-devel >= 4.0
+BuildRequires: llvm3.9-devel >= 3.9.1
 BuildRequires: boost-devel >= 1.66
 BuildRequires: gtest-devel
 BuildRequires: gmock-devel
@@ -130,10 +130,10 @@ Requires: llvm4.0-libs >= 4.0
 %define _vespa_llvm_include_directory /usr/include/llvm4.0
 %endif
 %if 0%{?fc29}
-Requires: llvm4.0-libs >= 4.0
-%define _vespa_llvm_version 4.0
-%define _vespa_llvm_link_directory /usr/lib64/llvm4.0/lib
-%define _vespa_llvm_include_directory /usr/include/llvm4.0
+Requires: llvm3.9-libs >= 3.9.1
+%define _vespa_llvm_version 3.9
+%define _vespa_llvm_link_directory /usr/lib64/llvm3.9/lib
+%define _vespa_llvm_include_directory /usr/include/llvm3.9
 %endif
 %define _extra_link_directory /opt/vespa-cppunit/lib%{?_vespa_llvm_link_directory:;%{_vespa_llvm_link_directory}}%{?_vespa_gtest_link_directory:;%{_vespa_gtest_link_directory}}
 %define _extra_include_directory /opt/vespa-cppunit/include%{?_vespa_llvm_include_directory:;%{_vespa_llvm_include_directory}}%{?_vespa_gtest_include_directory:;%{_vespa_gtest_include_directory}}
@@ -162,7 +162,7 @@ source %{_devtoolset_enable} || true
 source %{_rhmaven35_enable} || true
 %endif
 sh bootstrap.sh java
-mvn --batch-mode -nsu -T 2C install -Dmaven.test.skip=true -Dmaven.javadoc.skip=true
+mvn --batch-mode -nsu -T 1  install -Dmaven.test.skip=true -Dmaven.javadoc.skip=true
 cmake3 -DCMAKE_INSTALL_PREFIX=%{_prefix} \
        -DJAVA_HOME=/usr/lib/jvm/java-openjdk \
        -DEXTRA_LINK_DIRECTORY="%{_extra_link_directory}" \
diff --git a/docker-api/pom.xml b/docker-api/pom.xml
index 6118865a6e0..87761a17f70 100644
--- a/docker-api/pom.xml
+++ b/docker-api/pom.xml
@@ -128,31 +128,11 @@
                 <groupId>com.yahoo.vespa</groupId>
                 <artifactId>bundle-plugin</artifactId>
                 <extensions>true</extensions>
+                <configuration>
+                    <attachBundleToArtifact>true</attachBundleToArtifact>
+               </configuration>
             </plugin>
             <plugin>
-                <groupId>org.codehaus.mojo</groupId>
-                <artifactId>build-helper-maven-plugin</artifactId>
-                <executions>
-                    <execution>
-                        <id>attach-artifacts</id>
-                        <phase>package</phase>
-                        <goals>
-                            <goal>attach-artifact</goal>
-                        </goals>
-                        <configuration>
-                            <artifacts>
-                                <artifact>
-                                    <file>target/${project.artifactId}-jar-with-dependencies.jar</file>
-                                    <type>jar</type>
-                                    <classifier>bundle</classifier>
-                                </artifact>
-                            </artifacts>
-                        </configuration>
-                    </execution>
-                </executions>
-            </plugin>
-
-            <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-compiler-plugin</artifactId>
             </plugin>
diff --git a/document/src/main/java/com/yahoo/document/DocumentUpdate.java b/document/src/main/java/com/yahoo/document/DocumentUpdate.java
index 2a9ab9e6169..70c5410534e 100644
--- a/document/src/main/java/com/yahoo/document/DocumentUpdate.java
+++ b/document/src/main/java/com/yahoo/document/DocumentUpdate.java
@@ -1,11 +1,14 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.document;
 
+import com.yahoo.document.datatypes.FieldValue;
 import com.yahoo.document.fieldpathupdate.FieldPathUpdate;
 import com.yahoo.document.serialization.DocumentSerializerFactory;
 import com.yahoo.document.serialization.DocumentUpdateReader;
 import com.yahoo.document.serialization.DocumentUpdateWriter;
+import com.yahoo.document.update.AssignValueUpdate;
 import com.yahoo.document.update.FieldUpdate;
+import com.yahoo.document.update.ValueUpdate;
 import com.yahoo.io.GrowableByteBuffer;
 
 import java.util.ArrayList;
@@ -94,6 +97,12 @@ public class DocumentUpdate extends DocumentOperation implements Iterable<FieldP
         docId = id;
     }
 
+    private void verifyType(Document doc) {
+        if (!documentType.equals(doc.getDataType())) {
+            throw new IllegalArgumentException(
+                    "Document " + doc.getId() + " with type " + doc.getDataType() + " must have same type as update, which is type " + documentType);
+        }
+    }
     /**
      * Applies this document update.
      *
@@ -102,10 +111,7 @@ public class DocumentUpdate extends DocumentOperation implements Iterable<FieldP
      * @throws IllegalArgumentException if the document does not have the same document type as this update
      */
     public DocumentUpdate applyTo(Document doc) {
-        if (!documentType.equals(doc.getDataType())) {
-            throw new IllegalArgumentException(
-                    "Document " + doc + " must have same type as update, which is type " + documentType);
-        }
+        verifyType(doc);
 
         for (FieldUpdate fieldUpdate : fieldUpdates) {
             fieldUpdate.applyTo(doc);
@@ -117,6 +123,30 @@ public class DocumentUpdate extends DocumentOperation implements Iterable<FieldP
     }
 
     /**
+     * Prune away any field update that will not modify any field in the document.
+     * @param doc document to check against
+     * @return a reference to itself
+     * @throws IllegalArgumentException if the document does not have the same document type as this update
+     */
+    public DocumentUpdate prune(Document doc) {
+        verifyType(doc);
+
+        for (Iterator<FieldUpdate> iter = fieldUpdates.iterator(); iter.hasNext();) {
+            FieldUpdate update = iter.next();
+            if (!update.isEmpty()) {
+                ValueUpdate last = update.getValueUpdate(update.size() - 1);
+                if (last instanceof AssignValueUpdate) {
+                    FieldValue currentValue = doc.getFieldValue(update.getField());
+                    if ((currentValue != null) && (currentValue.compareTo(last.getValue()) == 0)) {
+                        iter.remove();
+                    }
+                }
+            }
+        }
+        return this;
+    }
+
+    /**
      * Get an unmodifiable list of all field updates that this document update specifies.
      *
      * @return a list of all FieldUpdates in this DocumentUpdate
diff --git a/document/src/main/java/com/yahoo/document/datatypes/Raw.java b/document/src/main/java/com/yahoo/document/datatypes/Raw.java
index 2a5383705df..23ed0cee23e 100644
--- a/document/src/main/java/com/yahoo/document/datatypes/Raw.java
+++ b/document/src/main/java/com/yahoo/document/datatypes/Raw.java
@@ -17,15 +17,16 @@ import java.util.Arrays;
 /**
  * FieldValue which encapsulates a Raw value
  *
- * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ * @author Einar M R Rosenvinge
  */
 public final class Raw extends FieldValue {
+
     private static class Factory extends PrimitiveDataType.Factory {
         public FieldValue create() {
             return new Raw();
         }
     }
-    public static PrimitiveDataType.Factory getFactory() { return new Factory(); }
+
     public static final int classId = registerClass(Ids.document + 16, Raw.class);
     private ByteBuffer value;
 
@@ -42,6 +43,8 @@ public final class Raw extends FieldValue {
         value.position(0);
     }
 
+    public static PrimitiveDataType.Factory getFactory() { return new Factory(); }
+
     public ByteBuffer getByteBuffer() {
         return value;
     }
@@ -136,11 +139,11 @@ public final class Raw extends FieldValue {
     }
 
     /* (non-Javadoc)
-      * @see com.yahoo.document.datatypes.FieldValue#deserialize(com.yahoo.document.Field, com.yahoo.document.serialization.FieldReader)
-      */
-
+     * @see com.yahoo.document.datatypes.FieldValue#deserialize(com.yahoo.document.Field, com.yahoo.document.serialization.FieldReader)
+     */
     @Override
     public void deserialize(Field field, FieldReader reader) {
         reader.read(field, this);
     }
+
 }
diff --git a/document/src/test/java/com/yahoo/document/DocumentUpdateTestCase.java b/document/src/test/java/com/yahoo/document/DocumentUpdateTestCase.java
index 4392ea932b2..4f3d7d3b820 100644
--- a/document/src/test/java/com/yahoo/document/DocumentUpdateTestCase.java
+++ b/document/src/test/java/com/yahoo/document/DocumentUpdateTestCase.java
@@ -651,6 +651,64 @@ public class DocumentUpdateTestCase {
         assertNull(doc.getFieldValue(tensorField));
     }
 
+    @Test
+    public void testThatNonIdenticalAssignCanNotBePrunedAway() {
+        Field field = docType.getField("strfoo");
+        String expected = "some other value";
+        Document doc = createDocument();
+        doc.setFieldValue(field, "some value");
+        DocumentUpdate update = new DocumentUpdate(docType, new DocumentId(documentId));
+        update.addFieldUpdate(FieldUpdate.createAssign(field, new StringFieldValue(expected)));
+        update.prune(doc);
+        assertEquals(1, update.size());
+        update.applyTo(doc);
+        assertEquals(expected, doc.getFieldValue(field).getWrappedValue());
+    }
+
+    @Test
+    public void testThatIdenticalAssignCanBePrunedAway() {
+        Field field = docType.getField("strfoo");
+        String expected = "some value";
+        Document doc = createDocument();
+        doc.setFieldValue(field, "some value");
+        DocumentUpdate update = new DocumentUpdate(docType, new DocumentId(documentId));
+        update.addFieldUpdate(FieldUpdate.createAssign(field,new StringFieldValue(expected)));
+        update.prune(doc);
+        assertEquals(0, update.size());
+        update.applyTo(doc);
+        assertEquals(expected, doc.getFieldValue(field).getWrappedValue());
+    }
+
+    @Test
+    public void testThatIdenticalAssignCanBePrunedAwayIfLast() {
+        Field field = docType.getField("strfoo");
+        String expected = "some value";
+        Document doc = createDocument();
+        doc.setFieldValue(field, "some value");
+        DocumentUpdate update = new DocumentUpdate(docType, new DocumentId(documentId));
+        update.addFieldUpdate(FieldUpdate.createClearField(field));
+        update.addFieldUpdate(FieldUpdate.createAssign(field, new StringFieldValue(expected)));
+        update.prune(doc);
+        assertEquals(0, update.size());
+        update.applyTo(doc);
+        assertEquals(expected, doc.getFieldValue(field).getWrappedValue());
+    }
+
+    @Test
+    public void testThatIdenticalAssignCanNotBePrunedAwayIfNotLast() {
+        Field field = docType.getField("strfoo");
+        String expected = "some random value";
+        Document doc = createDocument();
+        doc.setFieldValue(field, "some value");
+        DocumentUpdate update = new DocumentUpdate(docType, new DocumentId(documentId));
+        update.addFieldUpdate(FieldUpdate.createAssign(field, new StringFieldValue("some value")));
+        update.addFieldUpdate(FieldUpdate.createAssign(field, new StringFieldValue(expected)));
+        update.prune(doc);
+        assertEquals(1, update.size());
+        update.applyTo(doc);
+        assertEquals(expected, doc.getFieldValue(field).getWrappedValue());
+    }
+
     private static TensorFieldValue createTensorFieldValue(String tensor) {
         return new TensorFieldValue(Tensor.from(tensor));
     }
diff --git a/document/src/tests/documentupdatetestcase.cpp b/document/src/tests/documentupdatetestcase.cpp
index eea5dd4decf..b4d98f0dd21 100644
--- a/document/src/tests/documentupdatetestcase.cpp
+++ b/document/src/tests/documentupdatetestcase.cpp
@@ -64,6 +64,7 @@ struct DocumentUpdateTest : public CppUnit::TestFixture {
   void testThatCreateIfNonExistentFlagIsSerializedAndDeserialized();
   void array_element_update_can_be_roundtrip_serialized();
   void array_element_update_applies_to_specified_element();
+  void array_element_update_for_invalid_index_is_ignored();
 
   CPPUNIT_TEST_SUITE(DocumentUpdateTest);
   CPPUNIT_TEST(testSimpleUsage);
@@ -91,6 +92,7 @@ struct DocumentUpdateTest : public CppUnit::TestFixture {
   CPPUNIT_TEST(testThatCreateIfNonExistentFlagIsSerializedAndDeserialized);
   CPPUNIT_TEST(array_element_update_can_be_roundtrip_serialized);
   CPPUNIT_TEST(array_element_update_applies_to_specified_element);
+  CPPUNIT_TEST(array_element_update_for_invalid_index_is_ignored);
   CPPUNIT_TEST_SUITE_END();
 
 };
@@ -1050,4 +1052,17 @@ void DocumentUpdateTest::array_element_update_applies_to_specified_element() {
     CPPUNIT_ASSERT_EQUAL(vespalib::string("blarg"), (*result_array)[2].getAsString());
 }
 
+void DocumentUpdateTest::array_element_update_for_invalid_index_is_ignored() {
+    ArrayUpdateFixture f;
+
+    ArrayFieldValue array_value(f.array_field.getDataType());
+    array_value.add("jerry");
+    f.doc->setValue(f.array_field, array_value);
+
+    f.update->applyTo(*f.doc); // MapValueUpdate for index 1, which does not exist
+
+    auto result_array = f.doc->getAs<ArrayFieldValue>(f.array_field);
+    CPPUNIT_ASSERT_EQUAL(array_value, *result_array);
+}
+
 }  // namespace document
diff --git a/document/src/tests/fieldpathupdatetestcase.cpp b/document/src/tests/fieldpathupdatetestcase.cpp
index 80a63a86b92..f5db5831912 100644
--- a/document/src/tests/fieldpathupdatetestcase.cpp
+++ b/document/src/tests/fieldpathupdatetestcase.cpp
@@ -33,6 +33,7 @@ struct FieldPathUpdateTestCase : public CppUnit::TestFixture {
     void tearDown() override;
 
     void testWhereClause();
+    void testBrokenWhereClause();
     void testNoIterateMapValues();
     void testRemoveField();
     void testApplyRemoveEntireListField();
@@ -69,9 +70,11 @@ struct FieldPathUpdateTestCase : public CppUnit::TestFixture {
     void testSerializeAssignMath();
     void testReadSerializedFile();
     void testGenerateSerializedFile();
+    void array_element_update_for_invalid_index_is_ignored();
 
     CPPUNIT_TEST_SUITE(FieldPathUpdateTestCase);
     CPPUNIT_TEST(testWhereClause);
+    CPPUNIT_TEST(testBrokenWhereClause);
     CPPUNIT_TEST(testNoIterateMapValues);
     CPPUNIT_TEST(testRemoveField);
     CPPUNIT_TEST(testApplyRemoveEntireListField);
@@ -108,6 +111,7 @@ struct FieldPathUpdateTestCase : public CppUnit::TestFixture {
     CPPUNIT_TEST(testSerializeAssignMath);
     CPPUNIT_TEST(testReadSerializedFile);
     CPPUNIT_TEST(testGenerateSerializedFile);
+    CPPUNIT_TEST(array_element_update_for_invalid_index_is_ignored);
     CPPUNIT_TEST_SUITE_END();
 private:
     DocumentUpdate::UP
@@ -354,6 +358,17 @@ FieldPathUpdateTestCase::testWhereClause()
 }
 
 void
+FieldPathUpdateTestCase::testBrokenWhereClause()
+{
+    DocumentTypeRepo repo(getRepoConfig());
+    Document::UP doc(createTestDocument(repo));
+    std::string where = "l1s1.structmap.value.smap{$x} == \"dicaprio\"";
+    TestFieldPathUpdate update("l1s1.structmap.value.smap{$x}", where);
+    update.applyTo(*doc);
+    CPPUNIT_ASSERT_EQUAL(std::string(""), update._str);
+}
+
+void
 FieldPathUpdateTestCase::testNoIterateMapValues()
 {
     DocumentTypeRepo repo(getRepoConfig());
@@ -1183,4 +1198,23 @@ FieldPathUpdateTestCase::testGenerateSerializedFile()
     close(fd);
 }
 
+void FieldPathUpdateTestCase::array_element_update_for_invalid_index_is_ignored() {
+    auto doc = std::make_unique<Document>(_foobar_type, DocumentId("id::foobar::1"));
+    doc->setRepo(*_repo);
+    auto& field = doc->getType().getField("strarray");
+
+    ArrayFieldValue str_array(field.getDataType());
+    str_array.add(StringFieldValue("jerry"));
+    doc->setValue("strarray", str_array);
+
+    DocumentUpdate docUp(*_repo, _foobar_type, DocumentId("id::foobar::1"));
+    docUp.addFieldPathUpdate(FieldPathUpdate::CP(
+            new AssignFieldPathUpdate(*doc->getDataType(), "strarray[1]", "", StringFieldValue("george"))));
+    docUp.applyTo(*doc);
+
+    // Doc is unmodified.
+    auto new_arr = doc->getAs<ArrayFieldValue>(field);
+    CPPUNIT_ASSERT_EQUAL(str_array, *new_arr);
+}
+
 }
diff --git a/document/src/tests/primitivefieldvaluetest.cpp b/document/src/tests/primitivefieldvaluetest.cpp
index c4cdc16c3ba..3ccb5aa714b 100644
--- a/document/src/tests/primitivefieldvaluetest.cpp
+++ b/document/src/tests/primitivefieldvaluetest.cpp
@@ -195,7 +195,7 @@ void deserialize(const ByteBuffer &buffer, T &value) {
 
         CPPUNIT_ASSERT_EQUAL(size_t(3), value2.getValueRef().size());
         // Zero termination
-        CPPUNIT_ASSERT(*(value2.getValueRef().c_str() + value2.getValueRef().size()) == '\0');
+        CPPUNIT_ASSERT(*(value2.getValueRef().data() + value2.getValueRef().size()) == '\0');
     }
 
 }
@@ -226,7 +226,7 @@ PrimitiveFieldValueTest::testRaw()
             value.toXml("  "));
 
     value.setValue("grmpf", 4);
-    CPPUNIT_ASSERT(strncmp("grmpf", value.getValueRef().c_str(),
+    CPPUNIT_ASSERT(strncmp("grmpf", value.getValueRef().data(),
                            value.getValueRef().size()) == 0);
 }
 
diff --git a/document/src/vespa/document/base/documentid.cpp b/document/src/vespa/document/base/documentid.cpp
index d2eae01922d..c3ba8fea29d 100644
--- a/document/src/vespa/document/base/documentid.cpp
+++ b/document/src/vespa/document/base/documentid.cpp
@@ -18,7 +18,7 @@ DocumentId::DocumentId()
 DocumentId::DocumentId(vespalib::stringref id)
     : Printable(),
       _globalId(),
-      _id(IdString::createIdString(id.c_str(), id.size()).release())
+      _id(IdString::createIdString(id.data(), id.size()).release())
 {
 }
 
diff --git a/document/src/vespa/document/base/documentid.h b/document/src/vespa/document/base/documentid.h
index 4611de73741..a4b01cdad82 100644
--- a/document/src/vespa/document/base/documentid.h
+++ b/document/src/vespa/document/base/documentid.h
@@ -41,10 +41,15 @@ public:
      * Parse the given document identifier given as string, and create an
      * identifier object from it.
      *
+     * Precondition: `id` MUST be null-terminated.
+     *
      * @throws IdParseException If the identifier given is invalid.
      */
     explicit DocumentId(vespalib::stringref id);
 
+    /**
+     * Precondition: `id` MUST be null-terminated.
+     */
     void set(vespalib::stringref id);
 
     /**
diff --git a/document/src/vespa/document/base/field.cpp b/document/src/vespa/document/base/field.cpp
index 5a3fe6c1935..578294df7b0 100644
--- a/document/src/vespa/document/base/field.cpp
+++ b/document/src/vespa/document/base/field.cpp
@@ -78,7 +78,7 @@ Field::calculateIdV7()
     ost << getName();
     ost << _dataType->getId();
 
-    int newId = vespalib::BobHash::hash(ost.str().c_str(), ost.str().length(), 0);
+    int newId = vespalib::BobHash::hash(ost.str().data(), ost.str().length(), 0);
     // Highest bit is reserved to tell 7-bit id's from 31-bit ones
     if (newId < 0) newId = -newId;
     validateId(newId);
@@ -91,7 +91,7 @@ Field::validateId(int newId) {
         throw vespalib::IllegalArgumentException(vespalib::make_string(
                     "Attempt to set the id of %s to %d failed, values from "
                     "100 to 127 are reserved for internal use",
-                    getName().c_str(), newId));
+                    getName().data(), newId));
     }
 
     if ((newId & 0x80000000) != 0) // Highest bit must not be set
@@ -99,7 +99,7 @@ Field::validateId(int newId) {
         throw vespalib::IllegalArgumentException(vespalib::make_string(
                     "Attempt to set the id of %s to %d"
                     " failed, negative id values are illegal",
-                    getName().c_str(), newId));
+                    getName().data(), newId));
     }
 }
 
diff --git a/document/src/vespa/document/base/fieldpath.cpp b/document/src/vespa/document/base/fieldpath.cpp
index 6625f9ae2a5..d1603ca5c09 100644
--- a/document/src/vespa/document/base/fieldpath.cpp
+++ b/document/src/vespa/document/base/fieldpath.cpp
@@ -139,7 +139,7 @@ FieldPathEntry::visitMembers(vespalib::ObjectVisitor &visitor) const
 vespalib::string FieldPathEntry::parseKey(vespalib::stringref & key)
 {
     vespalib::string v;
-    const char *c = key.c_str();
+    const char *c = key.data();
     const char *e = c + key.size();
     for(;(c < e) && isspace(c[0]); c++);
     if ((c < e) && (c[0] == '{')) {
@@ -156,7 +156,8 @@ vespalib::string FieldPathEntry::parseKey(vespalib::stringref & key)
             if ((c < e) && (c[0] == '"')) {
                 c++;
             } else {
-                throw IllegalArgumentException(make_string("Escaped key '%s' is incomplete. No matching '\"'", key.c_str()), VESPA_STRLOC);
+                throw IllegalArgumentException(make_string("Escaped key '%s' is incomplete. No matching '\"'",
+                                                           vespalib::string(key).c_str()), VESPA_STRLOC);
             }
         } else {
             const char * start = c;
@@ -169,10 +170,12 @@ vespalib::string FieldPathEntry::parseKey(vespalib::stringref & key)
         if ((c < e) && (c[0] == '}')) {
             key = c+1;
         } else {
-            throw IllegalArgumentException(make_string("Key '%s' is incomplete. No matching '}'", key.c_str()), VESPA_STRLOC);
+            throw IllegalArgumentException(make_string("Key '%s' is incomplete. No matching '}'",
+                                                       vespalib::string(key).c_str()), VESPA_STRLOC);
         }
     } else {
-        throw IllegalArgumentException(make_string("key '%s' does not start with '{'", key.c_str()), VESPA_STRLOC);
+        throw IllegalArgumentException(make_string("key '%s' does not start with '{'",
+                                                   vespalib::string(key).c_str()), VESPA_STRLOC);
     }
     return v;
 }
diff --git a/document/src/vespa/document/base/idstring.cpp b/document/src/vespa/document/base/idstring.cpp
index 9c64ac6a648..175fb653542 100644
--- a/document/src/vespa/document/base/idstring.cpp
+++ b/document/src/vespa/document/base/idstring.cpp
@@ -84,9 +84,9 @@ void reportTooShortDocId(const char * id, size_t sz)
 
 uint64_t getAsNumber(const stringref & s, const char* part) {
     char* errPos = NULL;
-    uint64_t value = strtoull(s.c_str(), &errPos, 10);
+    uint64_t value = strtoull(s.data(), &errPos, 10);
 
-    if (s.c_str() + s.size() != errPos) {
+    if (s.data() + s.size() != errPos) {
         reportError(s, part);
     }
     return value;
@@ -96,9 +96,9 @@ void
 getOrderDocBits(const stringref& scheme, uint16_t & widthBits, uint16_t & divisionBits)
 {
     const char* parenPos = reinterpret_cast<const char*>(
-            memchr(scheme.c_str(), '(', scheme.size()));
+            memchr(scheme.data(), '(', scheme.size()));
     const char* endParenPos = reinterpret_cast<const char*>(
-            memchr(scheme.c_str(), ')', scheme.size()));
+            memchr(scheme.data(), ')', scheme.size()));
 
     if (parenPos == NULL || endParenPos == NULL || endParenPos < parenPos) {
         reportError(scheme);
@@ -198,13 +198,13 @@ IdString::Offsets::Offsets(uint32_t maxComponents, uint32_t namespaceOffset, con
 {
     _offsets[0] = namespaceOffset;
     size_t index(1);
-    const char * s(id.c_str() + namespaceOffset);
-    const char * e(id.c_str() + id.size());
+    const char * s(id.data() + namespaceOffset);
+    const char * e(id.data() + id.size());
     for(s=fmemchr(s, e);
         (s != NULL) && (index < maxComponents);
         s = fmemchr(s+1, e))
     {
-        _offsets[index++] = s - id.c_str() + 1;
+        _offsets[index++] = s - id.data() + 1;
     }
     _numComponents = index;
     for (;index < VESPA_NELEMS(_offsets); index++) {
@@ -276,14 +276,14 @@ union LocationUnion {
 
 IdString::LocationType makeLocation(const stringref &s) {
     LocationUnion location;
-    fastc_md5sum(reinterpret_cast<const unsigned char*>(s.c_str()), s.size(), location._key);
+    fastc_md5sum(reinterpret_cast<const unsigned char*>(s.data()), s.size(), location._key);
     return location._location[0];
 }
 
 uint64_t parseNumber(const stringref &number) {
     char* errPos = NULL;
     errno = 0;
-    uint64_t n = strtoul(number.c_str(), &errPos, 10);
+    uint64_t n = strtoul(number.data(), &errPos, 10);
     if (*errPos) {
         throw IdParseException(
                 "'n'-value must be a 64-bit number. It was " +
@@ -396,7 +396,7 @@ GroupDocIdString::locationFromGroupName(vespalib::stringref name)
 }
 
 OrderDocIdString::OrderDocIdString(const stringref & rawId) :
-    IdString(4, static_cast<const char *>(memchr(rawId.c_str(), ':', rawId.size())) - rawId.c_str() + 1, rawId),
+    IdString(4, static_cast<const char *>(memchr(rawId.data(), ':', rawId.size())) - rawId.data() + 1, rawId),
     _widthBits(0),
     _divisionBits(0),
     _ordering(getAsNumber(rawId.substr(offset(2), offset(3) - offset(2) - 1), "ordering"))
diff --git a/document/src/vespa/document/base/idstring.h b/document/src/vespa/document/base/idstring.h
index 05554a68ba3..b1a14cfec94 100644
--- a/document/src/vespa/document/base/idstring.h
+++ b/document/src/vespa/document/base/idstring.h
@@ -24,7 +24,7 @@ public:
     static const vespalib::string & getTypeName(Type t);
 
     /** @throws document::IdParseException If parsing of id scheme failed. */
-    static IdString::UP createIdString(const vespalib::stringref & id) { return createIdString(id.c_str(), id.size()); }
+    static IdString::UP createIdString(const vespalib::stringref & id) { return createIdString(id.data(), id.size()); }
     static IdString::UP createIdString(const char *id, size_t sz);
 
     ~IdString() {}
diff --git a/document/src/vespa/document/datatype/arraydatatype.cpp b/document/src/vespa/document/datatype/arraydatatype.cpp
index 5ced6053b05..b49599ac620 100644
--- a/document/src/vespa/document/datatype/arraydatatype.cpp
+++ b/document/src/vespa/document/datatype/arraydatatype.cpp
@@ -60,7 +60,8 @@ ArrayDataType::onBuildFieldPath(FieldPath & path, const vespalib::stringref & re
             if (remainFieldName[1] == '$') {
                 path.insert(path.begin(), std::make_unique<FieldPathEntry>(getNestedType(), remainFieldName.substr(2, endPos - 2)));
             } else {
-                path.insert(path.begin(), std::make_unique<FieldPathEntry>(getNestedType(), atoi(remainFieldName.substr(1, endPos - 1).c_str())));
+                // FIXME C++17 range-safe from_chars() instead of atoi()
+                path.insert(path.begin(), std::make_unique<FieldPathEntry>(getNestedType(), atoi(remainFieldName.substr(1, endPos - 1).data())));
             }
         }
     } else {
diff --git a/document/src/vespa/document/datatype/datatype.h b/document/src/vespa/document/datatype/datatype.h
index 247d72db665..5857157c218 100644
--- a/document/src/vespa/document/datatype/datatype.h
+++ b/document/src/vespa/document/datatype/datatype.h
@@ -125,6 +125,7 @@ public:
      * This takes a . separated fieldname and gives you back the path of
      * fields you have to apply to get to your leaf.
      * @param remainFieldName. The remaining part of the fieldname that you want the path of.
+     *                         MUST be null-terminated.
      * @return pointer to field path or null if an error occured
      */
     void buildFieldPath(FieldPath & fieldPath, const vespalib::stringref & remainFieldName) const;
diff --git a/document/src/vespa/document/datatype/documenttype.cpp b/document/src/vespa/document/datatype/documenttype.cpp
index c7eaf42b50b..2be5acaf3db 100644
--- a/document/src/vespa/document/datatype/documenttype.cpp
+++ b/document/src/vespa/document/datatype/documenttype.cpp
@@ -96,7 +96,7 @@ DocumentType::addField(const Field& field)
     } else if (!_ownedFields.get()) {
         throw vespalib::IllegalStateException(make_string(
                         "Cannot add field %s to a DocumentType that does not "
-                        "own its fields.", field.getName().c_str()), VESPA_STRLOC);
+                        "own its fields.", field.getName().data()), VESPA_STRLOC);
     }
     _ownedFields->addField(field);
 }
diff --git a/document/src/vespa/document/datatype/mapdatatype.cpp b/document/src/vespa/document/datatype/mapdatatype.cpp
index 5c940b1af6e..4598b96d970 100644
--- a/document/src/vespa/document/datatype/mapdatatype.cpp
+++ b/document/src/vespa/document/datatype/mapdatatype.cpp
@@ -73,7 +73,7 @@ MapDataType::buildFieldPathImpl(FieldPath & path, const DataType &dataType,
             *fv = keyValue;
             path.insert(path.begin(), std::make_unique<FieldPathEntry>(valueType, dataType, std::move(fv)));
         }
-    } else if (memcmp(remainFieldName.c_str(), "key", 3) == 0) {
+    } else if (memcmp(remainFieldName.data(), "key", 3) == 0) {
         size_t endPos = 3;
         if (remainFieldName[endPos] == '.') {
             endPos++;
@@ -82,7 +82,7 @@ MapDataType::buildFieldPathImpl(FieldPath & path, const DataType &dataType,
         keyType.buildFieldPath(path, remainFieldName.substr(endPos));
 
         path.insert(path.begin(), std::make_unique<FieldPathEntry>(dataType, keyType, valueType, true, false));
-    } else if (memcmp(remainFieldName.c_str(), "value", 5) == 0) {
+    } else if (memcmp(remainFieldName.data(), "value", 5) == 0) {
         size_t endPos = 5;
         if (remainFieldName[endPos] == '.') {
             endPos++;
diff --git a/document/src/vespa/document/datatype/referencedatatype.cpp b/document/src/vespa/document/datatype/referencedatatype.cpp
index d02793edd3a..8f0c6a0a4c3 100644
--- a/document/src/vespa/document/datatype/referencedatatype.cpp
+++ b/document/src/vespa/document/datatype/referencedatatype.cpp
@@ -36,7 +36,7 @@ ReferenceDataType* ReferenceDataType::clone() const {
 void ReferenceDataType::onBuildFieldPath(FieldPath &, const vespalib::stringref& remainingFieldName) const {
     if ( ! remainingFieldName.empty() ) {
         throw IllegalArgumentException(make_string("Reference data type does not support further field recursion: '%s'",
-                                                   remainingFieldName.c_str()), VESPA_STRLOC);
+                                                   vespalib::string(remainingFieldName).c_str()), VESPA_STRLOC);
     }
 
 }
diff --git a/document/src/vespa/document/datatype/structdatatype.cpp b/document/src/vespa/document/datatype/structdatatype.cpp
index 0abde69c2a1..9ff0b7e0b0a 100644
--- a/document/src/vespa/document/datatype/structdatatype.cpp
+++ b/document/src/vespa/document/datatype/structdatatype.cpp
@@ -83,7 +83,7 @@ StructDataType::addField(const Field& field)
     vespalib::string error = containsConflictingField(field);
     if (error != "") {
         throw IllegalArgumentException(make_string("Failed to add field '%s' to struct '%s': %s",
-                                                   field.getName().c_str(), getName().c_str(),
+                                                   field.getName().data(), getName().c_str(),
                                                    error.c_str()), VESPA_STRLOC);
     }
     if (hasField(field.getName())) {
diff --git a/document/src/vespa/document/datatype/structureddatatype.cpp b/document/src/vespa/document/datatype/structureddatatype.cpp
index 41c0cd6f4e7..604b4cad045 100644
--- a/document/src/vespa/document/datatype/structureddatatype.cpp
+++ b/document/src/vespa/document/datatype/structureddatatype.cpp
@@ -80,7 +80,8 @@ StructuredDataType::onBuildFieldPath(FieldPath & path, const vespalib::stringref
         path.insert(path.begin(), std::make_unique<FieldPathEntry>(fp));
     } else {
         throw FieldNotFoundException(currFieldName, make_string("Invalid field path '%s', no field named '%s'",
-                                                   remainFieldName.c_str(), currFieldName.c_str()));
+                                                                vespalib::string(remainFieldName).c_str(),
+                                                                vespalib::string(currFieldName).c_str()));
     }
 }
 
diff --git a/document/src/vespa/document/fieldvalue/arrayfieldvalue.cpp b/document/src/vespa/document/fieldvalue/arrayfieldvalue.cpp
index f3239553fa9..a09d5a25dd2 100644
--- a/document/src/vespa/document/fieldvalue/arrayfieldvalue.cpp
+++ b/document/src/vespa/document/fieldvalue/arrayfieldvalue.cpp
@@ -186,9 +186,6 @@ ArrayFieldValue::iterateSubset(int startPos, int endPos,
 {
     fieldvalue::ModificationStatus retVal = ModificationStatus::NOT_MODIFIED;
 
-    LOG(spam, "iterateSubset(start=%d, end=%d, variable='%s')",
-        startPos, endPos, variable.c_str());
-
     std::vector<int> indicesToRemove;
 
     for (int i = startPos; i <= endPos && i < static_cast<int>(_array->size()); ++i) {
diff --git a/document/src/vespa/document/fieldvalue/document.cpp b/document/src/vespa/document/fieldvalue/document.cpp
index 47366b388a9..51ba135b826 100644
--- a/document/src/vespa/document/fieldvalue/document.cpp
+++ b/document/src/vespa/document/fieldvalue/document.cpp
@@ -32,12 +32,13 @@ void documentTypeError(const vespalib::stringref & name) __attribute__((noinline
 void throwTypeMismatch(vespalib::stringref type, vespalib::stringref docidType) __attribute__((noinline));
 
 void documentTypeError(const vespalib::stringref & name) {
-    throw IllegalArgumentException(make_string("Cannot generate a document with non-document type %s.", name.c_str()), VESPA_STRLOC);
+    throw IllegalArgumentException(make_string("Cannot generate a document with non-document type %s.",
+                                               vespalib::string(name).c_str()), VESPA_STRLOC);
 }
 
 void throwTypeMismatch(vespalib::stringref type, vespalib::stringref docidType) {
     throw IllegalArgumentException(make_string("Trying to create a document with type %s that don't match the id (type %s).",
-                                               type.c_str(), docidType.c_str()),
+                                               vespalib::string(type).c_str(), vespalib::string(docidType).c_str()),
                                    VESPA_STRLOC);
 }
 
diff --git a/document/src/vespa/document/fieldvalue/literalfieldvalue.cpp b/document/src/vespa/document/fieldvalue/literalfieldvalue.cpp
index 747c789c6cd..5614330a495 100644
--- a/document/src/vespa/document/fieldvalue/literalfieldvalue.cpp
+++ b/document/src/vespa/document/fieldvalue/literalfieldvalue.cpp
@@ -79,7 +79,7 @@ LiteralFieldValueB::fastCompare(const FieldValue& other) const
 void
 LiteralFieldValueB::printXml(XmlOutputStream& out) const
 {
-    out << XmlContentWrapper(_value.c_str(), _value.size());
+    out << XmlContentWrapper(_value.data(), _value.size());
 }
 
 void
@@ -106,7 +106,7 @@ LiteralFieldValueB::getAsString() const
 std::pair<const char*, size_t>
 LiteralFieldValueB::getAsRaw() const
 {
-    return std::make_pair(_value.c_str(), _value.size());
+    return std::make_pair(_value.data(), _value.size());
 }
 
 void
diff --git a/document/src/vespa/document/fieldvalue/literalfieldvalue.h b/document/src/vespa/document/fieldvalue/literalfieldvalue.h
index d72e734b6a6..2f9050eb13c 100644
--- a/document/src/vespa/document/fieldvalue/literalfieldvalue.h
+++ b/document/src/vespa/document/fieldvalue/literalfieldvalue.h
@@ -54,7 +54,7 @@ public:
         _value = _backing;
         _altered = true;
     }
-    size_t hash() const override final { return vespalib::hashValue(_value.c_str()); }
+    size_t hash() const override final { return vespalib::hashValue(_value.data(), _value.size()); }
     void setValue(const char* val, size_t size) { setValue(stringref(val, size)); }
 
     int compare(const FieldValue& other) const override;
@@ -76,7 +76,7 @@ public:
 protected:
     void syncBacking() const __attribute__((noinline));
     void sync() const {
-        if (__builtin_expect(_backing.c_str() != _value.c_str(), false)) {
+        if (__builtin_expect(_backing.data() != _value.data(), false)) {
             syncBacking();
         }
     }
diff --git a/document/src/vespa/document/fieldvalue/numericfieldvalue.hpp b/document/src/vespa/document/fieldvalue/numericfieldvalue.hpp
index 881cdb4a7e3..c3ef6781706 100644
--- a/document/src/vespa/document/fieldvalue/numericfieldvalue.hpp
+++ b/document/src/vespa/document/fieldvalue/numericfieldvalue.hpp
@@ -84,9 +84,10 @@ NumericFieldValue<Number>::operator=(const vespalib::stringref & value)
         // so detect these in front.
     if ((value.size() > 2) && (value[0] == '0') && ((value[1] | 0x20) == 'x')) {
         char* endp;
-            // It is safe to assume that all hex numbers can be contained within
-            // 64 bit unsigned value.
-        unsigned long long val = strtoull(value.c_str(), &endp, 16);
+        // It is safe to assume that all hex numbers can be contained within
+        // 64 bit unsigned value.
+        // FIXME C++17 range-safe from_chars() instead of strtoull()
+        unsigned long long val = strtoull(value.data(), &endp, 16);
         if (*endp == '\0') {
                 // Allow numbers to be specified in range max signed to max
                 // unsigned. These become negative numbers.
diff --git a/document/src/vespa/document/fieldvalue/rawfieldvalue.cpp b/document/src/vespa/document/fieldvalue/rawfieldvalue.cpp
index 6e415471cd0..5d4abe658c1 100644
--- a/document/src/vespa/document/fieldvalue/rawfieldvalue.cpp
+++ b/document/src/vespa/document/fieldvalue/rawfieldvalue.cpp
@@ -15,13 +15,13 @@ void
 RawFieldValue::printXml(XmlOutputStream& out) const
 {
     out << XmlBase64Content()
-        << XmlContentWrapper(_value.c_str(), _value.size());
+        << XmlContentWrapper(_value.data(), _value.size());
 }
 
 void
 RawFieldValue::print(std::ostream& out, bool, const std::string&) const
 {
-    StringUtil::printAsHex(out, _value.c_str(), _value.size());
+    StringUtil::printAsHex(out, _value.data(), _value.size());
 }
 
 } // document
diff --git a/document/src/vespa/document/fieldvalue/structuredfieldvalue.cpp b/document/src/vespa/document/fieldvalue/structuredfieldvalue.cpp
index 92d4e4788fb..c193b0919ea 100644
--- a/document/src/vespa/document/fieldvalue/structuredfieldvalue.cpp
+++ b/document/src/vespa/document/fieldvalue/structuredfieldvalue.cpp
@@ -71,7 +71,7 @@ void StructuredFieldValue::setFieldValue(const Field & field, const FieldValue &
         throw IllegalArgumentException(
                 "Cannot assign value of type " + value.getDataType()->toString()
                 + "with value : '" + value.toString()
-                + "' to field " + field.getName().c_str() + " of type "
+                + "' to field " + field.getName() + " of type "
                 + field.getDataType().toString() + ".", VESPA_STRLOC);
     }
     setFieldValue(field, FieldValue::UP(value.clone()));
diff --git a/document/src/vespa/document/select/simpleparser.cpp b/document/src/vespa/document/select/simpleparser.cpp
index 462243f342c..349c1c17362 100644
--- a/document/src/vespa/document/select/simpleparser.cpp
+++ b/document/src/vespa/document/select/simpleparser.cpp
@@ -24,7 +24,7 @@ bool icmp(char c, char l)
 bool IdSpecParser::parse(const vespalib::stringref & s)
 {
     bool retval(false);
-    size_t pos(eatWhite(s.c_str(), s.size()));
+    size_t pos(eatWhite(s.data(), s.size()));
     if (pos+1 < s.size()) {
         if (icmp(s[pos], 'i') && icmp(s[pos+1],'d')) {
             pos += 2;
@@ -77,7 +77,7 @@ bool IdSpecParser::parse(const vespalib::stringref & s)
 bool OperatorParser::parse(const vespalib::stringref & s)
 {
     bool retval(false);
-    size_t pos(eatWhite(s.c_str(), s.size()));
+    size_t pos(eatWhite(s.data(), s.size()));
     if (pos+1 < s.size()) {
         retval = true;
         if (s[pos] == '=') {
@@ -122,7 +122,7 @@ bool StringParser::parse(const vespalib::stringref & s)
 {
     bool retval(false);
     setRemaining(s);
-    size_t pos(eatWhite(s.c_str(), s.size()));
+    size_t pos(eatWhite(s.data(), s.size()));
     if (pos + 1 < s.size()) {
         if (s[pos++] == '"') {
             vespalib::string str;
@@ -146,7 +146,7 @@ bool StringParser::parse(const vespalib::stringref & s)
 bool IntegerParser::parse(const vespalib::stringref & s)
 {
     bool retval(false);
-    size_t pos(eatWhite(s.c_str(), s.size()));
+    size_t pos(eatWhite(s.data(), s.size()));
     if (pos < s.size()) {
         char * err(NULL);
         errno = 0;
diff --git a/document/src/vespa/document/serialization/vespadocumentserializer.cpp b/document/src/vespa/document/serialization/vespadocumentserializer.cpp
index 08fddbaad41..a309fdd3500 100644
--- a/document/src/vespa/document/serialization/vespadocumentserializer.cpp
+++ b/document/src/vespa/document/serialization/vespadocumentserializer.cpp
@@ -513,7 +513,10 @@ void VespaDocumentSerializer::write(const MapValueUpdate &value)
 
 namespace {
 
-void writeStringWithZeroTermination(nbostream & os, stringref s)
+// We must ensure that string passed is always zero-terminated, so take in
+// string instead of stringref. No extra allocs; function only ever called with
+// string arguments.
+void writeStringWithZeroTermination(nbostream & os, const vespalib::string& s)
 {
     uint32_t sz(s.size() + 1);
     os << sz;
diff --git a/document/src/vespa/document/update/addvalueupdate.cpp b/document/src/vespa/document/update/addvalueupdate.cpp
index 051ffcf8b2a..e5a99b49a9e 100644
--- a/document/src/vespa/document/update/addvalueupdate.cpp
+++ b/document/src/vespa/document/update/addvalueupdate.cpp
@@ -43,7 +43,7 @@ AddValueUpdate::checkCompatibility(const Field& field) const
         const CollectionDataType& type(static_cast<const CollectionDataType&>(field.getDataType()));
         if (!type.getNestedType().isValueType(*_value)) {
             throw IllegalArgumentException("Cannot add value of type " + _value->getDataType()->toString() +
-                                           " to field " + field.getName().c_str() + " of container type " +
+                                           " to field " + field.getName() + " of container type " +
                                            field.getDataType().toString(), VESPA_STRLOC);
         }
     } else {
diff --git a/document/src/vespa/document/update/arithmeticvalueupdate.cpp b/document/src/vespa/document/update/arithmeticvalueupdate.cpp
index 9ae7dd17a52..3af9350062e 100644
--- a/document/src/vespa/document/update/arithmeticvalueupdate.cpp
+++ b/document/src/vespa/document/update/arithmeticvalueupdate.cpp
@@ -35,7 +35,7 @@ ArithmeticValueUpdate::checkCompatibility(const Field& field) const
     if ( ! field.getDataType().inherits(NumericDataType::classId)) {
         throw IllegalArgumentException(vespalib::make_string(
                 "Can not perform arithmetic update on non-numeric field '%s'.",
-                field.getName().c_str()), VESPA_STRLOC);	
+                field.getName().data()), VESPA_STRLOC);
     }
 }
 
diff --git a/document/src/vespa/document/update/fieldpathupdate.cpp b/document/src/vespa/document/update/fieldpathupdate.cpp
index e7d5824b6e0..fa7a8b38aba 100644
--- a/document/src/vespa/document/update/fieldpathupdate.cpp
+++ b/document/src/vespa/document/update/fieldpathupdate.cpp
@@ -3,7 +3,9 @@
 #include <vespa/document/datatype/datatype.h>
 #include <vespa/document/fieldvalue/document.h>
 #include <vespa/document/fieldvalue/iteratorhandler.h>
+#include <vespa/document/select/constant.h>
 #include <vespa/document/select/parser.h>
+#include <vespa/document/select/parsing_failed_exception.h>
 #include <vespa/document/util/serializableexceptions.h>
 #include <vespa/vespalib/objects/nbostream.h>
 #include <ostream>
@@ -11,6 +13,7 @@
 #include <vespa/log/log.h>
 LOG_SETUP(".document.update.fieldpathupdate");
 
+using document::select::ParsingFailedException;
 using vespalib::make_string;
 using vespalib::IllegalArgumentException;
 
@@ -26,8 +29,13 @@ std::unique_ptr<select::Node>
 parseDocumentSelection(vespalib::stringref query, const DocumentTypeRepo& repo)
 {
     BucketIdFactory factory;
-    select::Parser parser(repo, factory);
-    return parser.parse(query);
+    try {
+        select::Parser parser(repo, factory);
+        return parser.parse(query);
+    } catch (const ParsingFailedException &e) {
+        LOG(warning, "Failed to parse selection for field path update: %s", e.getMessage().c_str());
+        return std::make_unique<select::Constant>(false);
+    }
 }
 
 }  // namespace
diff --git a/document/src/vespa/document/update/mapvalueupdate.cpp b/document/src/vespa/document/update/mapvalueupdate.cpp
index ff1063a3dda..be970b3c30a 100644
--- a/document/src/vespa/document/update/mapvalueupdate.cpp
+++ b/document/src/vespa/document/update/mapvalueupdate.cpp
@@ -46,7 +46,7 @@ MapValueUpdate::checkCompatibility(const Field& field) const
 	    if (_key->getClass().id() != IntFieldValue::classId) {
             throw IllegalArgumentException(vespalib::make_string(
                     "Key for field '%s' is of wrong type (expected '%s', was '%s').",
-                    field.getName().c_str(), DataType::INT->toString().c_str(),
+                    field.getName().data(), DataType::INT->toString().c_str(),
                     _key->getDataType()->toString().c_str()), VESPA_STRLOC);
         }
     } else if (field.getDataType().getClass().id() == WeightedSetDataType::classId) {
@@ -54,7 +54,7 @@ MapValueUpdate::checkCompatibility(const Field& field) const
         if (!type.getNestedType().isValueType(*_key)) {
             throw IllegalArgumentException(vespalib::make_string(
                     "Key for field '%s' is of wrong type (expected '%s', was '%s').",
-                    field.getName().c_str(), DataType::INT->toString().c_str(),
+                    field.getName().data(), DataType::INT->toString().c_str(),
                     _key->getDataType()->toString().c_str()), VESPA_STRLOC);
         }
     } else {
@@ -71,9 +71,9 @@ MapValueUpdate::applyTo(FieldValue& value) const
         ArrayFieldValue& val(static_cast<ArrayFieldValue&>(value));
         int32_t index = _key->getAsInt();
         if (index < 0 || static_cast<uint32_t>(index) >= val.size()) {
-            throw IllegalStateException(vespalib::make_string(
-                    "Tried to update element %i in an array of %zu elements",
-		            index, val.size()), VESPA_STRLOC);
+            // Silently ignoring updates with index out of bounds matches
+            // behavior of functionally identical fieldpath updates.
+            return true;
         }
         if (!_update->applyTo(val[_key->getAsInt()])) {
             val.remove(_key->getAsInt());
diff --git a/document/src/vespa/document/update/removevalueupdate.cpp b/document/src/vespa/document/update/removevalueupdate.cpp
index 28c69652a0e..fdbee3cb394 100644
--- a/document/src/vespa/document/update/removevalueupdate.cpp
+++ b/document/src/vespa/document/update/removevalueupdate.cpp
@@ -44,7 +44,7 @@ RemoveValueUpdate::checkCompatibility(const Field& field) const
             throw IllegalArgumentException(
                     "Cannot remove value of type "
                     + _key->getDataType()->toString() + " from field "
-                    + field.getName().c_str() + " of container type "
+                    + field.getName() + " of container type "
                     + field.getDataType().toString(), VESPA_STRLOC);
         }
     } else {
diff --git a/documentgen-test/pom.xml b/documentgen-test/pom.xml
index d009ef3d592..fb6515c3f35 100644
--- a/documentgen-test/pom.xml
+++ b/documentgen-test/pom.xml
@@ -20,12 +20,19 @@
       <scope>test</scope>
     </dependency>
     <dependency>
+      <!-- TODO: Explicitly list deps instead - container-dev is for bundle development. -->
       <groupId>com.yahoo.vespa</groupId>
       <artifactId>container-dev</artifactId>
       <version>${project.version}</version>
       <scope>compile</scope>
     </dependency>
     <dependency>
+      <!-- TODO: Excluded from container-dev. Remove when deps are explicitly listed.  -->
+      <groupId>com.yahoo.vespa</groupId>
+      <artifactId>linguistics</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+    <dependency>
       <groupId>com.yahoo.vespa</groupId>
       <artifactId>config-model</artifactId>
       <version>${project.version}</version>
diff --git a/eval/src/tests/eval/compiled_function/compiled_function_test.cpp b/eval/src/tests/eval/compiled_function/compiled_function_test.cpp
index 0e9806d5381..151a4cf5dd5 100644
--- a/eval/src/tests/eval/compiled_function/compiled_function_test.cpp
+++ b/eval/src/tests/eval/compiled_function/compiled_function_test.cpp
@@ -178,7 +178,7 @@ TEST("dump ir code to verify lazy casting") {
     Function function = Function::parse({"a", "b"}, "12==2+if(a==3&&a<10||b,10,5)");
     LLVMWrapper wrapper;
     size_t id = wrapper.make_function(function.num_params(), PassParams::SEPARATE, function.root(), {});
-    wrapper.compile(true); // dump module before compiling it
+    wrapper.compile(llvm::dbgs()); // dump module before compiling it
     using fun_type = double (*)(double, double);
     fun_type fun = (fun_type) wrapper.get_function_address(id);
     EXPECT_EQUAL(0.0, fun(0.0, 0.0));
diff --git a/eval/src/vespa/eval/eval/llvm/llvm_wrapper.cpp b/eval/src/vespa/eval/eval/llvm/llvm_wrapper.cpp
index f314f8a69cb..71ca74f4167 100644
--- a/eval/src/vespa/eval/eval/llvm/llvm_wrapper.cpp
+++ b/eval/src/vespa/eval/eval/llvm/llvm_wrapper.cpp
@@ -48,8 +48,7 @@ double vespalib_eval_forest_proxy(Forest::eval_function eval_forest, const Fores
     }
 }
 
-namespace vespalib {
-namespace eval {
+namespace vespalib::eval {
 
 using namespace nodes;
 
@@ -84,6 +83,19 @@ struct FunctionBuilder : public NodeVisitor, public NodeTraverser {
     std::vector<gbdt::Forest::UP> &forests;
     std::vector<PluginState::UP> &plugin_state;
 
+    llvm::FunctionType *make_call_1_fun_t() {
+        std::vector<llvm::Type*> param_types;
+        param_types.push_back(builder.getDoubleTy());
+        return llvm::FunctionType::get(builder.getDoubleTy(), param_types, false);
+    }
+
+    llvm::FunctionType *make_call_2_fun_t() {
+        std::vector<llvm::Type*> param_types;
+        param_types.push_back(builder.getDoubleTy());
+        param_types.push_back(builder.getDoubleTy());
+        return llvm::FunctionType::get(builder.getDoubleTy(), param_types, false);
+    }
+
     llvm::PointerType *make_eval_forest_funptr_t() {
         std::vector<llvm::Type*> param_types;
         param_types.push_back(builder.getVoidTy()->getPointerTo());
@@ -320,9 +332,7 @@ struct FunctionBuilder : public NodeVisitor, public NodeTraverser {
         make_call_1(llvm::Intrinsic::getDeclaration(&module, id, builder.getDoubleTy()));
     }
     void make_call_1(const char *name) {
-        make_call_1(dynamic_cast<llvm::Function*>(module.getOrInsertFunction(name,
-                                builder.getDoubleTy(),
-                                builder.getDoubleTy(), nullptr)));
+        make_call_1(llvm::dyn_cast<llvm::Function>(module.getOrInsertFunction(name, make_call_1_fun_t())));
     }
 
     void make_call_2(llvm::Function *fun) {
@@ -337,10 +347,7 @@ struct FunctionBuilder : public NodeVisitor, public NodeTraverser {
         make_call_2(llvm::Intrinsic::getDeclaration(&module, id, builder.getDoubleTy()));
     }
     void make_call_2(const char *name) {
-        make_call_2(dynamic_cast<llvm::Function*>(module.getOrInsertFunction(name,
-                                builder.getDoubleTy(),
-                                builder.getDoubleTy(),
-                                builder.getDoubleTy(), nullptr)));
+        make_call_2(llvm::dyn_cast<llvm::Function>(module.getOrInsertFunction(name, make_call_2_fun_t())));
     }
 
     //-------------------------------------------------------------------------
@@ -659,11 +666,11 @@ LLVMWrapper::make_forest_fragment(size_t num_params, const std::vector<const Nod
 }
 
 void
-LLVMWrapper::compile(bool dump_module)
+LLVMWrapper::compile(llvm::raw_ostream * dumpStream)
 {
     std::lock_guard<std::recursive_mutex> guard(_global_llvm_lock);
-    if (dump_module) {
-        _module->dump();
+    if (dumpStream) {
+        _module->print(*dumpStream, nullptr);
     }
     _engine.reset(llvm::EngineBuilder(std::move(_module)).setOptLevel(llvm::CodeGenOpt::Aggressive).create());
     assert(_engine && "llvm jit not available for your platform");
@@ -687,5 +694,4 @@ LLVMWrapper::~LLVMWrapper() {
     _context.reset();
 }
 
-} // namespace vespalib::eval
-} // namespace vespalib
+}
diff --git a/eval/src/vespa/eval/eval/llvm/llvm_wrapper.h b/eval/src/vespa/eval/eval/llvm/llvm_wrapper.h
index 6860be922f4..db29771ee9e 100644
--- a/eval/src/vespa/eval/eval/llvm/llvm_wrapper.h
+++ b/eval/src/vespa/eval/eval/llvm/llvm_wrapper.h
@@ -21,8 +21,7 @@ extern "C" {
     double vespalib_eval_elu(double a);
 };
 
-namespace vespalib {
-namespace eval {
+namespace vespalib::eval {
 
 /**
  * Simple interface used to track and clean up custom state. This is
@@ -52,6 +51,7 @@ private:
 
     static std::recursive_mutex _global_llvm_lock;
 
+    void compile(llvm::raw_ostream * dumpStream);
 public:
     LLVMWrapper();
     LLVMWrapper(LLVMWrapper &&rhs) = default;
@@ -60,11 +60,11 @@ public:
                          const gbdt::Optimize::Chain &forest_optimizers);
     size_t make_forest_fragment(size_t num_params, const std::vector<const nodes::Node *> &fragment);
     const std::vector<gbdt::Forest::UP> &get_forests() const { return _forests; }
-    void compile(bool dump_module = false);
+    void compile(llvm::raw_ostream & dumpStream) { compile(&dumpStream); }
+    void compile() { compile(nullptr); }
     void *get_function_address(size_t function_id);
     ~LLVMWrapper();
 };
 
-} // namespace vespalib::eval
-} // namespace vespalib
+}
 
diff --git a/eval/src/vespa/eval/tensor/dense/dense_tensor_view.cpp b/eval/src/vespa/eval/tensor/dense/dense_tensor_view.cpp
index 80548d33e72..f2ddcb38698 100644
--- a/eval/src/vespa/eval/tensor/dense/dense_tensor_view.cpp
+++ b/eval/src/vespa/eval/tensor/dense/dense_tensor_view.cpp
@@ -67,7 +67,7 @@ checkDimensions(const DenseTensorView &lhs, const DenseTensorView &rhs,
                                                 "dense tensor %s, "
                                                 "lhs dimensions = '%s', "
                                                 "rhs dimensions = '%s'",
-                                                operation.c_str(),
+                                                operation.data(),
                                                 dimensionsAsString(lhs.fast_type()).c_str(),
                                                 dimensionsAsString(rhs.fast_type()).c_str()));
     }
diff --git a/eval/src/vespa/eval/tensor/sparse/sparse_tensor_unsorted_address_builder.h b/eval/src/vespa/eval/tensor/sparse/sparse_tensor_unsorted_address_builder.h
index 681bdabc5eb..7e69adb5804 100644
--- a/eval/src/vespa/eval/tensor/sparse/sparse_tensor_unsorted_address_builder.h
+++ b/eval/src/vespa/eval/tensor/sparse/sparse_tensor_unsorted_address_builder.h
@@ -53,7 +53,7 @@ class SparseTensorUnsortedAddressBuilder
     ElementStringRef
     append(vespalib::stringref str)
     {
-        const char *cstr = str.c_str();
+        const char *cstr = str.data();
         uint32_t start = _elementStrings.size();
         _elementStrings.insert(_elementStrings.end(),
                                cstr, cstr + str.size() + 1);
diff --git a/functions.cmake b/functions.cmake
index d1634baa5c3..1dd5edb6fb2 100644
--- a/functions.cmake
+++ b/functions.cmake
@@ -423,6 +423,14 @@ function(vespa_install_script)
     endif()
 endfunction()
 
+function(vespa_install_data)
+    if(ARGC GREATER 2)
+        install(FILES ${ARGV0} RENAME ${ARGV1} PERMISSIONS OWNER_READ OWNER_WRITE GROUP_READ WORLD_READ DESTINATION ${ARGV2})
+    else()
+        install(FILES ${ARGV0} PERMISSIONS OWNER_READ OWNER_WRITE GROUP_READ WORLD_READ DESTINATION ${ARGV1})
+    endif()
+endfunction()
+
 function(vespa_workaround_gcc_bug_67055 SOURCE_FILE)
     if(CMAKE_COMPILER_IS_GNUCC)
         execute_process(COMMAND ${CMAKE_CPP_COMPILER} -dumpversion OUTPUT_VARIABLE GCC_VERSION)
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/filter/AthenzPrincipalFilter.java b/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/athenz/AthenzPrincipalFilter.java
index 5166f53c6d2..ad6c82138e1 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/filter/AthenzPrincipalFilter.java
+++ b/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/athenz/AthenzPrincipalFilter.java
@@ -1,5 +1,5 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.controller.athenz.filter;
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.jdisc.http.filter.security.athenz;
 
 import com.google.inject.Inject;
 import com.yahoo.jdisc.Response;
@@ -9,15 +9,14 @@ import com.yahoo.jdisc.http.filter.security.cors.CorsRequestFilterBase;
 import com.yahoo.vespa.athenz.api.AthenzPrincipal;
 import com.yahoo.vespa.athenz.api.NToken;
 import com.yahoo.vespa.athenz.utils.AthenzIdentities;
-import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZmsKeystore;
-import com.yahoo.vespa.hosted.controller.athenz.config.AthenzConfig;
+import com.yahoo.vespa.athenz.utils.ntoken.NTokenValidator;
 
+import java.nio.file.Paths;
 import java.security.cert.X509Certificate;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Optional;
 import java.util.Set;
-import java.util.concurrent.Executor;
 
 
 /**
@@ -30,31 +29,24 @@ import java.util.concurrent.Executor;
  *
  * @author bjorncs
  */
-// TODO bjorncs: Move this class to vespa-athenz bundle
 public class AthenzPrincipalFilter extends CorsRequestFilterBase {
 
     private final NTokenValidator validator;
     private final String principalTokenHeader;
 
-    /**
-     * @param executor to preload the ZMS public keys with
-     */
     @Inject
-    public AthenzPrincipalFilter(ZmsKeystore zmsKeystore,
-                                 Executor executor,
-                                 AthenzConfig athenzConfig,
-                                 CorsFilterConfig corsConfig) {
-        this(new NTokenValidator(zmsKeystore), executor, athenzConfig.principalHeaderName(), new HashSet<>(corsConfig.allowedUrls()));
+    public AthenzPrincipalFilter(AthenzPrincipalFilterConfig athenzPrincipalFilterConfig, CorsFilterConfig corsConfig) {
+        this(new NTokenValidator(Paths.get(athenzPrincipalFilterConfig.athenzConfFile())),
+             athenzPrincipalFilterConfig.principalHeaderName(),
+             new HashSet<>(corsConfig.allowedUrls()));
     }
 
     AthenzPrincipalFilter(NTokenValidator validator,
-                          Executor executor,
                           String principalTokenHeader,
                           Set<String> corsAllowedUrls) {
         super(corsAllowedUrls);
         this.validator = validator;
         this.principalTokenHeader = principalTokenHeader;
-        executor.execute(validator::preloadPublicKeys);
     }
 
     @Override
diff --git a/jdisc-security-filters/src/main/resources/configdefinitions/athenz-principal-filter.def b/jdisc-security-filters/src/main/resources/configdefinitions/athenz-principal-filter.def
new file mode 100644
index 00000000000..59e481b0d80
--- /dev/null
+++ b/jdisc-security-filters/src/main/resources/configdefinitions/athenz-principal-filter.def
@@ -0,0 +1,8 @@
+# Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+namespace=jdisc.http.filter.security.athenz
+
+# Principal header name
+principalHeaderName             string  default="Athenz-Principal-Auth"
+
+# Path to athenz.conf file
+athenzConfFile                  string
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/athenz/filter/AthenzPrincipalFilterTest.java b/jdisc-security-filters/src/test/java/com/yahoo/jdisc/http/filter/security/athenz/AthenzPrincipalFilterTest.java
index 301fc461b6f..be5ab9c1d77 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/athenz/filter/AthenzPrincipalFilterTest.java
+++ b/jdisc-security-filters/src/test/java/com/yahoo/jdisc/http/filter/security/athenz/AthenzPrincipalFilterTest.java
@@ -1,7 +1,6 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.controller.athenz.filter;
+package com.yahoo.jdisc.http.filter.security.athenz;
 
-import com.yahoo.application.container.handler.Request;
 import com.yahoo.jdisc.Response;
 import com.yahoo.jdisc.handler.ContentChannel;
 import com.yahoo.jdisc.handler.ReadableContentChannel;
@@ -14,8 +13,7 @@ import com.yahoo.vespa.athenz.api.NToken;
 import com.yahoo.vespa.athenz.tls.KeyAlgorithm;
 import com.yahoo.vespa.athenz.tls.KeyUtils;
 import com.yahoo.vespa.athenz.tls.X509CertificateBuilder;
-import com.yahoo.vespa.hosted.controller.api.integration.athenz.InvalidTokenException;
-import com.yahoo.vespa.hosted.controller.restapi.ApplicationRequestToDiscFilterRequestWrapper;
+import com.yahoo.vespa.athenz.utils.ntoken.NTokenValidator;
 import org.junit.Before;
 import org.junit.Test;
 
@@ -28,20 +26,22 @@ import java.security.KeyPair;
 import java.security.cert.X509Certificate;
 import java.time.Duration;
 import java.time.Instant;
-import java.util.HashMap;
-import java.util.Map;
 import java.util.Objects;
-import java.util.Optional;
 import java.util.Set;
 
 import static com.yahoo.jdisc.Response.Status.UNAUTHORIZED;
 import static com.yahoo.vespa.athenz.tls.SignatureAlgorithm.SHA256_WITH_RSA;
+import static java.util.Collections.emptyList;
 import static java.util.Collections.singleton;
 import static java.util.Collections.singletonList;
 import static java.util.stream.Collectors.joining;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
+import static org.hamcrest.CoreMatchers.containsString;
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.CoreMatchers.notNullValue;
+import static org.junit.Assert.assertThat;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
 
 /**
  * @author bjorncs
@@ -55,98 +55,115 @@ public class AthenzPrincipalFilterTest {
     private static final String ORIGIN = "http://localhost";
     private static final Set<String> CORS_ALLOWED_URLS = singleton(ORIGIN);
 
-    private NTokenValidatorMock validator;
-    private ResponseHandlerMock responseHandler;
+    private NTokenValidator validator;
 
     @Before
     public void before() {
-        this.validator = new NTokenValidatorMock();
-        this.responseHandler = new ResponseHandlerMock();
+        validator = mock(NTokenValidator.class);
     }
 
     @Test
     public void valid_ntoken_is_accepted() {
-        Request request = defaultRequest();
-
+        DiscFilterRequest request = createRequestMock();
         AthenzPrincipal principal = new AthenzPrincipal(IDENTITY, NTOKEN);
-        validator.add(NTOKEN, principal);
+        when(request.getHeader(ATHENZ_PRINCIPAL_HEADER)).thenReturn(NTOKEN.getRawToken());
+        when(request.getClientCertificateChain()).thenReturn(emptyList());
+        when(validator.validate(NTOKEN)).thenReturn(principal);
+
+        AthenzPrincipalFilter filter = new AthenzPrincipalFilter(validator, ATHENZ_PRINCIPAL_HEADER, CORS_ALLOWED_URLS);
+        filter.filter(request, new ResponseHandlerMock());
 
-        AthenzPrincipalFilter filter = new AthenzPrincipalFilter(validator, Runnable::run, ATHENZ_PRINCIPAL_HEADER, CORS_ALLOWED_URLS);
-        DiscFilterRequest filterRequest = new ApplicationRequestToDiscFilterRequestWrapper(request);
-        filter.filter(filterRequest, new ResponseHandlerMock());
+        verify(request).setUserPrincipal(principal);
+    }
 
-        assertEquals(principal, filterRequest.getUserPrincipal());
+    private DiscFilterRequest createRequestMock() {
+        DiscFilterRequest request = mock(DiscFilterRequest.class);
+        when(request.getHeader("Origin")).thenReturn(ORIGIN);
+        return request;
     }
 
     @Test
     public void missing_token_and_certificate_is_unauthorized() {
-        AthenzPrincipalFilter filter = new AthenzPrincipalFilter(validator, Runnable::run, ATHENZ_PRINCIPAL_HEADER, CORS_ALLOWED_URLS);
-        DiscFilterRequest filterRequest = new ApplicationRequestToDiscFilterRequestWrapper(new Request("/"));
-        filter.filter(filterRequest, responseHandler);
+        DiscFilterRequest request = createRequestMock();
+        when(request.getHeader(ATHENZ_PRINCIPAL_HEADER)).thenReturn(null);
+        when(request.getClientCertificateChain()).thenReturn(emptyList());
+
+        ResponseHandlerMock responseHandler = new ResponseHandlerMock();
+
+        AthenzPrincipalFilter filter = new AthenzPrincipalFilter(validator, ATHENZ_PRINCIPAL_HEADER, CORS_ALLOWED_URLS);
+        filter.filter(request, responseHandler);
 
         assertUnauthorized(responseHandler, "Unable to authenticate Athenz identity");
     }
 
     @Test
     public void invalid_token_is_unauthorized() {
-        Request request = defaultRequest();
+        DiscFilterRequest request = createRequestMock();
+        String errorMessage = "Invalid token";
+        when(request.getHeader(ATHENZ_PRINCIPAL_HEADER)).thenReturn(NTOKEN.getRawToken());
+        when(request.getClientCertificateChain()).thenReturn(emptyList());
+        when(validator.validate(NTOKEN)).thenThrow(new NTokenValidator.InvalidTokenException(errorMessage));
 
-        AthenzPrincipalFilter filter = new AthenzPrincipalFilter(validator, Runnable::run, ATHENZ_PRINCIPAL_HEADER, CORS_ALLOWED_URLS);
-        DiscFilterRequest filterRequest = new ApplicationRequestToDiscFilterRequestWrapper(request);
-        filter.filter(filterRequest, responseHandler);
+        ResponseHandlerMock responseHandler = new ResponseHandlerMock();
+
+        AthenzPrincipalFilter filter = new AthenzPrincipalFilter(validator, ATHENZ_PRINCIPAL_HEADER, CORS_ALLOWED_URLS);
+        filter.filter(request, responseHandler);
 
-        String errorMessage = "Invalid token";
         assertUnauthorized(responseHandler, errorMessage);
     }
 
     @Test
     public void certificate_is_accepted() {
-        AthenzPrincipalFilter filter = new AthenzPrincipalFilter(validator, Runnable::run, ATHENZ_PRINCIPAL_HEADER, CORS_ALLOWED_URLS);
-        DiscFilterRequest filterRequest = new ApplicationRequestToDiscFilterRequestWrapper(new Request("/"), singletonList(CERTIFICATE));
-        filter.filter(filterRequest, responseHandler);
+        DiscFilterRequest request = createRequestMock();
+        when(request.getHeader(ATHENZ_PRINCIPAL_HEADER)).thenReturn(null);
+        when(request.getClientCertificateChain()).thenReturn(singletonList(CERTIFICATE));
+
+        ResponseHandlerMock responseHandler = new ResponseHandlerMock();
+
+        AthenzPrincipalFilter filter = new AthenzPrincipalFilter(validator, ATHENZ_PRINCIPAL_HEADER, CORS_ALLOWED_URLS);
+        filter.filter(request, responseHandler);
 
         AthenzPrincipal expectedPrincipal = new AthenzPrincipal(IDENTITY);
-        assertEquals(expectedPrincipal, filterRequest.getUserPrincipal());
+        verify(request).setUserPrincipal(expectedPrincipal);
     }
 
     @Test
     public void both_ntoken_and_certificate_is_accepted() {
-        Request request = defaultRequest();
-
+        DiscFilterRequest request = createRequestMock();
         AthenzPrincipal principalWithToken = new AthenzPrincipal(IDENTITY, NTOKEN);
-        validator.add(NTOKEN, principalWithToken);
+        when(request.getHeader(ATHENZ_PRINCIPAL_HEADER)).thenReturn(NTOKEN.getRawToken());
+        when(request.getClientCertificateChain()).thenReturn(singletonList(CERTIFICATE));
+        when(validator.validate(NTOKEN)).thenReturn(principalWithToken);
 
-        AthenzPrincipalFilter filter = new AthenzPrincipalFilter(validator, Runnable::run, ATHENZ_PRINCIPAL_HEADER, CORS_ALLOWED_URLS);
-        DiscFilterRequest filterRequest = new ApplicationRequestToDiscFilterRequestWrapper(request, singletonList(CERTIFICATE));
-        filter.filter(filterRequest, responseHandler);
+        ResponseHandlerMock responseHandler = new ResponseHandlerMock();
 
-        assertEquals(principalWithToken, filterRequest.getUserPrincipal());
+        AthenzPrincipalFilter filter = new AthenzPrincipalFilter(validator, ATHENZ_PRINCIPAL_HEADER, CORS_ALLOWED_URLS);
+        filter.filter(request, responseHandler);
+
+        verify(request).setUserPrincipal(principalWithToken);
     }
 
     @Test
     public void conflicting_ntoken_and_certificate_is_unauthorized() {
-        Request request = defaultRequest();
-        validator.add(NTOKEN, new AthenzPrincipal(IDENTITY));
-
+        DiscFilterRequest request = createRequestMock();
         AthenzUser conflictingIdentity = AthenzUser.fromUserId("mallory");
-        DiscFilterRequest filterRequest = new ApplicationRequestToDiscFilterRequestWrapper(request, singletonList(createSelfSignedCertificate(conflictingIdentity)));
-        AthenzPrincipalFilter filter = new AthenzPrincipalFilter(validator, Runnable::run, ATHENZ_PRINCIPAL_HEADER, CORS_ALLOWED_URLS);
-        filter.filter(filterRequest, responseHandler);
+        when(request.getHeader(ATHENZ_PRINCIPAL_HEADER)).thenReturn(NTOKEN.getRawToken());
+        when(request.getClientCertificateChain())
+                .thenReturn(singletonList(createSelfSignedCertificate(conflictingIdentity)));
+        when(validator.validate(NTOKEN)).thenReturn(new AthenzPrincipal(IDENTITY));
 
-        assertUnauthorized(responseHandler, "Identity in principal token does not match x509 CN");
-    }
+        ResponseHandlerMock responseHandler = new ResponseHandlerMock();
 
-    private static Request defaultRequest() {
-        Request request = new Request("/");
-        request.getHeaders().add("Origin", ORIGIN);
-        request.getHeaders().add(ATHENZ_PRINCIPAL_HEADER, NTOKEN.getRawToken());
-        return request;
+        AthenzPrincipalFilter filter = new AthenzPrincipalFilter(validator, ATHENZ_PRINCIPAL_HEADER, CORS_ALLOWED_URLS);
+        filter.filter(request, responseHandler);
+
+        assertUnauthorized(responseHandler, "Identity in principal token does not match x509 CN");
     }
 
     private static void assertUnauthorized(ResponseHandlerMock responseHandler, String expectedMessageSubstring) {
-        assertNotNull(responseHandler.response);
-        assertEquals(UNAUTHORIZED, responseHandler.response.getStatus());
-        assertTrue(responseHandler.getResponseContent().contains(expectedMessageSubstring));
+        assertThat(responseHandler.response, notNullValue());
+        assertThat(responseHandler.response.getStatus(), equalTo(UNAUTHORIZED));
+        assertThat(responseHandler.getResponseContent(), containsString(expectedMessageSubstring));
     }
 
     private static class ResponseHandlerMock implements ResponseHandler {
@@ -171,29 +188,6 @@ public class AthenzPrincipalFilterTest {
 
     }
 
-    private static class NTokenValidatorMock extends NTokenValidator {
-
-        private final Map<NToken, AthenzPrincipal> validTokens = new HashMap<>();
-
-        NTokenValidatorMock() {
-            super((service, keyId) -> Optional.empty());
-        }
-
-        public NTokenValidatorMock add(NToken token, AthenzPrincipal principal) {
-            validTokens.put(token, principal);
-            return this;
-        }
-
-        @Override
-        AthenzPrincipal validate(NToken token) throws InvalidTokenException {
-            if (!validTokens.containsKey(token)) {
-                throw new InvalidTokenException("Invalid token");
-            }
-            return validTokens.get(token);
-        }
-
-    }
-
     private static X509Certificate createSelfSignedCertificate(AthenzIdentity identity) {
         KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.RSA, 512);
         X500Principal x500Name = new X500Principal("CN="+ identity.getFullName());
diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/JettyHttpServer.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/JettyHttpServer.java
index f149d6eb34d..452d1aed874 100644
--- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/JettyHttpServer.java
+++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/JettyHttpServer.java
@@ -365,12 +365,11 @@ public class JettyHttpServer extends AbstractServerProvider {
         return statisticsHandler;
     }
 
-    @SuppressWarnings("deprecation")
     private GzipHandler newGzipHandler(ServerConfig serverConfig) {
         GzipHandler gzipHandler = new GzipHandlerWithVaryHeaderFixed();
         gzipHandler.setCompressionLevel(serverConfig.responseCompressionLevel());
-        gzipHandler.setCheckGzExists(false);  // TODO: will be removed without replacement in Jetty 10
-        gzipHandler.setIncludedMethods("GET", "POST");
+        gzipHandler.setInflateBufferSize(8 * 1024);
+        gzipHandler.setIncludedMethods("GET", "POST", "PUT", "PATCH");
         return gzipHandler;
     }
 
diff --git a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpServerTest.java b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpServerTest.java
index 7c79897ac27..30d5f9e657a 100644
--- a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpServerTest.java
+++ b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpServerTest.java
@@ -65,7 +65,7 @@ import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
 /**
- * @author bakksjo
+ * @author Oyvind Bakksjo
  * @author Simon Thoresen Hult
  */
 public class HttpServerTest {
@@ -466,6 +466,20 @@ public class HttpServerTest {
         assertThat(driver.close(), is(true));
     }
 
+    @Test
+    public void requireThatGzipEncodingRequestsAreAutomaticallyDecompressed() throws Exception {
+        final TestDriver driver = TestDrivers.newInstance(new ParameterPrinterRequestHandler());
+        final String requestContent = generateContent('a', 30);
+        final ResponseValidator response =
+                driver.client().newPost("/status.html")
+                        .addHeader(CONTENT_TYPE, APPLICATION_X_WWW_FORM_URLENCODED)
+                        .setGzipContent(requestContent)
+                        .execute();
+        response.expectStatusCode(is(OK))
+                .expectContent(startsWith('{' + requestContent + "=[]}"));
+        assertThat(driver.close(), is(true));
+    }
+
     private static RequestHandler mockRequestHandler() {
         final RequestHandler mockRequestHandler = mock(RequestHandler.class);
         when(mockRequestHandler.refer()).thenReturn(References.NOOP_REFERENCE);
diff --git a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/SimpleHttpClient.java b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/SimpleHttpClient.java
index 79a2dc5a2fb..1836a73d2fd 100644
--- a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/SimpleHttpClient.java
+++ b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/SimpleHttpClient.java
@@ -6,12 +6,14 @@ import org.apache.http.Header;
 import org.apache.http.HttpEntity;
 import org.apache.http.HttpResponse;
 import org.apache.http.client.HttpClient;
+import org.apache.http.client.entity.GzipCompressingEntity;
 import org.apache.http.client.methods.HttpGet;
 import org.apache.http.client.methods.HttpPost;
 import org.apache.http.client.methods.HttpUriRequest;
 import org.apache.http.config.Registry;
 import org.apache.http.config.RegistryBuilder;
 import org.apache.http.conn.socket.ConnectionSocketFactory;
+import org.apache.http.conn.ssl.NoopHostnameVerifier;
 import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
 import org.apache.http.entity.ByteArrayEntity;
 import org.apache.http.entity.StringEntity;
@@ -59,7 +61,7 @@ public class SimpleHttpClient {
         if (sslContext != null) {
             SSLConnectionSocketFactory sslConnectionFactory = new SSLConnectionSocketFactory(
                     sslContext,
-                    SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER);
+                    NoopHostnameVerifier.INSTANCE);
             builder.setSSLSocketFactory(sslConnectionFactory);
 
             Registry<ConnectionSocketFactory> registry = RegistryBuilder.<ConnectionSocketFactory>create()
@@ -154,6 +156,11 @@ public class SimpleHttpClient {
             return this;
         }
 
+        public RequestExecutor setGzipContent(String content) {
+            this.entity = new GzipCompressingEntity(new StringEntity(content, StandardCharsets.UTF_8));
+            return this;
+        }
+
         public RequestExecutor setBinaryContent(final byte[] content) {
             this.entity = new ByteArrayEntity(content);
             return this;
diff --git a/jrt/tests/com/yahoo/jrt/AbortTest.java b/jrt/tests/com/yahoo/jrt/AbortTest.java
index e3a1685c9e2..9093158162d 100644
--- a/jrt/tests/com/yahoo/jrt/AbortTest.java
+++ b/jrt/tests/com/yahoo/jrt/AbortTest.java
@@ -19,8 +19,8 @@ public class AbortTest {
     public void setUp() throws ListenFailedException {
         server   = new Supervisor(new Transport());
         client   = new Supervisor(new Transport());
-        acceptor = server.listen(new Spec(Test.PORT));
-        target   = client.connect(new Spec("localhost", Test.PORT));
+        acceptor = server.listen(new Spec(0));
+        target   = client.connect(new Spec("localhost", acceptor.port()));
         server.addMethod(new Method("test", "i", "i", this, "rpc_test"));
         barrier = new Test.Barrier();
     }
diff --git a/jrt/tests/com/yahoo/jrt/BackTargetTest.java b/jrt/tests/com/yahoo/jrt/BackTargetTest.java
index 3524d47dc1f..ade24f40c55 100644
--- a/jrt/tests/com/yahoo/jrt/BackTargetTest.java
+++ b/jrt/tests/com/yahoo/jrt/BackTargetTest.java
@@ -22,8 +22,8 @@ public class BackTargetTest {
     public void setUp() throws ListenFailedException {
         server   = new Supervisor(new Transport());
         client   = new Supervisor(new Transport());
-        acceptor = server.listen(new Spec(Test.PORT));
-        target   = client.connect(new Spec("localhost", Test.PORT));
+        acceptor = server.listen(new Spec(0));
+        target   = client.connect(new Spec("localhost", acceptor.port()));
 
         server.addMethod(new Method("inc", "", "", this, "server_inc"));
         server.addMethod(new Method("sample_target", "", "", this,
diff --git a/jrt/tests/com/yahoo/jrt/ConnectTest.java b/jrt/tests/com/yahoo/jrt/ConnectTest.java
index 64ed3f5f954..efe4a018c70 100644
--- a/jrt/tests/com/yahoo/jrt/ConnectTest.java
+++ b/jrt/tests/com/yahoo/jrt/ConnectTest.java
@@ -9,12 +9,12 @@ public class ConnectTest {
     public void testConnect() throws ListenFailedException {
         Test.Orb server   = new Test.Orb(new Transport());
         Test.Orb client   = new Test.Orb(new Transport());
-        Acceptor acceptor = server.listen(new Spec(Test.PORT));
+        Acceptor acceptor = server.listen(new Spec(0));
 
         assertTrue(server.checkLifeCounts(0, 0));
         assertTrue(client.checkLifeCounts(0, 0));
 
-        Target target = client.connect(new Spec("localhost", Test.PORT));
+        Target target = client.connect(new Spec("localhost", acceptor.port()));
 
         for (int i = 0; i < 100; i++) {
             if (client.initCount == 1 && server.initCount == 1) {
diff --git a/jrt/tests/com/yahoo/jrt/DetachTest.java b/jrt/tests/com/yahoo/jrt/DetachTest.java
index 1c1256f0689..808d029b5a5 100644
--- a/jrt/tests/com/yahoo/jrt/DetachTest.java
+++ b/jrt/tests/com/yahoo/jrt/DetachTest.java
@@ -20,8 +20,8 @@ public class DetachTest {
     public void setUp() throws ListenFailedException {
         server   = new Test.Orb(new Transport());
         client   = new Test.Orb(new Transport());
-        acceptor = server.listen(new Spec(Test.PORT));
-        target   = client.connect(new Spec("localhost", Test.PORT));
+        acceptor = server.listen(new Spec(0));
+        target   = client.connect(new Spec("localhost", acceptor.port()));
 
         server.addMethod(new Method("d_inc", "i", "i", this,
                                     "rpc_detach_inc"));
diff --git a/jrt/tests/com/yahoo/jrt/EchoTest.java b/jrt/tests/com/yahoo/jrt/EchoTest.java
index 240feda8423..5acd8221a9b 100644
--- a/jrt/tests/com/yahoo/jrt/EchoTest.java
+++ b/jrt/tests/com/yahoo/jrt/EchoTest.java
@@ -19,8 +19,8 @@ public class EchoTest {
     public void setUp() throws ListenFailedException {
         server   = new Supervisor(new Transport());
         client   = new Supervisor(new Transport());
-        acceptor = server.listen(new Spec(Test.PORT));
-        target   = client.connect(new Spec("localhost", Test.PORT));
+        acceptor = server.listen(new Spec(0));
+        target   = client.connect(new Spec("localhost", acceptor.port()));
         server.addMethod(new Method("echo", "*", "*", this, "rpc_echo"));
         refValues = new Values();
         byte[]   dataValue   = { 1, 2, 3, 4 };
diff --git a/jrt/tests/com/yahoo/jrt/InvokeAsyncTest.java b/jrt/tests/com/yahoo/jrt/InvokeAsyncTest.java
index d744fcf8f29..cdc52e9441a 100644
--- a/jrt/tests/com/yahoo/jrt/InvokeAsyncTest.java
+++ b/jrt/tests/com/yahoo/jrt/InvokeAsyncTest.java
@@ -21,8 +21,8 @@ public class InvokeAsyncTest {
     public void setUp() throws ListenFailedException {
         server   = new Supervisor(new Transport());
         client   = new Supervisor(new Transport());
-        acceptor = server.listen(new Spec(Test.PORT));
-        target   = client.connect(new Spec("localhost", Test.PORT));
+        acceptor = server.listen(new Spec(0));
+        target   = client.connect(new Spec("localhost", acceptor.port()));
         server.addMethod(new Method("concat", "ss", "s", this, "rpc_concat")
                          .methodDesc("Concatenate 2 strings")
                          .paramDesc(0, "str1", "a string")
diff --git a/jrt/tests/com/yahoo/jrt/InvokeErrorTest.java b/jrt/tests/com/yahoo/jrt/InvokeErrorTest.java
index 6fe92be97e5..4e810b71fb6 100644
--- a/jrt/tests/com/yahoo/jrt/InvokeErrorTest.java
+++ b/jrt/tests/com/yahoo/jrt/InvokeErrorTest.java
@@ -21,8 +21,8 @@ public class InvokeErrorTest {
     public void setUp() throws ListenFailedException {
         server   = new Supervisor(new Transport());
         client   = new Supervisor(new Transport());
-        acceptor = server.listen(new Spec(Test.PORT));
-        target   = client.connect(new Spec("localhost", Test.PORT));
+        acceptor = server.listen(new Spec(0));
+        target   = client.connect(new Spec("localhost", acceptor.port()));
         server.addMethod(new Method("test", "iib", "i", this, "rpc_test"));
         server.addMethod(new Method("test_barrier", "iib", "i", this,
                                     "rpc_test_barrier"));
diff --git a/jrt/tests/com/yahoo/jrt/InvokeSyncTest.java b/jrt/tests/com/yahoo/jrt/InvokeSyncTest.java
index 37856c0db81..931001804aa 100644
--- a/jrt/tests/com/yahoo/jrt/InvokeSyncTest.java
+++ b/jrt/tests/com/yahoo/jrt/InvokeSyncTest.java
@@ -26,8 +26,8 @@ public class InvokeSyncTest {
     public void setUp() throws ListenFailedException {
         server   = new Supervisor(new Transport());
         client   = new Supervisor(new Transport());
-        acceptor = server.listen(new Spec(Test.PORT));
-        target   = client.connect(new Spec("localhost", Test.PORT));
+        acceptor = server.listen(new Spec(0));
+        target   = client.connect(new Spec("localhost", acceptor.port()));
         server.addMethod(new Method("concat", "ss", "s", this, "rpc_concat")
                          .methodDesc("Concatenate 2 strings")
                          .paramDesc(0, "str1", "a string")
@@ -74,12 +74,12 @@ public class InvokeSyncTest {
     public void testRpcInvoker() throws IOException {
         ByteArrayOutputStream baos = new ByteArrayOutputStream();
         System.setOut(new PrintStream(baos));
-        RpcInvoker.main(new String[] {"-h", "localhost:"+Test.PORT, "concat", "s:foo", "s:bar"});
+        RpcInvoker.main(new String[] {"-h", "localhost:"+acceptor.port(), "concat", "s:foo", "s:bar"});
         baos.flush();
         assertEquals(baos.toString(), "foobar\n");
         baos.reset();
         System.setOut(new PrintStream(baos));
-        RpcInvoker.main(new String[] {"-h", "localhost:"+Test.PORT, "alltypes", "b:1", "h:2", "i:3", "l:4", "f:5.0", "d:6.0", "s:baz"});
+        RpcInvoker.main(new String[] {"-h", "localhost:"+acceptor.port(), "alltypes", "b:1", "h:2", "i:3", "l:4", "f:5.0", "d:6.0", "s:baz"});
         baos.flush();
         assertEquals(baos.toString(), "This was alltypes. The string param was: baz\n");
     }
diff --git a/jrt/tests/com/yahoo/jrt/InvokeVoidTest.java b/jrt/tests/com/yahoo/jrt/InvokeVoidTest.java
index d0c0d9c728e..25e86a16445 100644
--- a/jrt/tests/com/yahoo/jrt/InvokeVoidTest.java
+++ b/jrt/tests/com/yahoo/jrt/InvokeVoidTest.java
@@ -19,8 +19,8 @@ public class InvokeVoidTest {
     public void setUp() throws ListenFailedException {
         server   = new Test.Orb(new Transport());
         client   = new Test.Orb(new Transport());
-        acceptor = server.listen(new Spec(Test.PORT));
-        target   = client.connect(new Spec("localhost", Test.PORT));
+        acceptor = server.listen(new Spec(0));
+        target   = client.connect(new Spec("localhost", acceptor.port()));
 
         server.addMethod(new Method("set", "i", "", this, "rpc_set")
                          .methodDesc("Set the stored value")
diff --git a/jrt/tests/com/yahoo/jrt/ListenTest.java b/jrt/tests/com/yahoo/jrt/ListenTest.java
index 88fd6f22f24..5e3bde08a20 100644
--- a/jrt/tests/com/yahoo/jrt/ListenTest.java
+++ b/jrt/tests/com/yahoo/jrt/ListenTest.java
@@ -24,36 +24,39 @@ public class ListenTest {
     @org.junit.Test
     public void testListen() {
         try {
-            Acceptor a = server.listen(new Spec(Test.PORT));
-            assertEquals(Test.PORT, a.port());
+            Acceptor a = server.listen(new Spec(0));
+            assertTrue(a.port() > 0);
             a.shutdown().join();
             assertEquals(-1, a.port());
         } catch (ListenFailedException e) {
             assertTrue(false);
         }
         try {
-            Acceptor a = server.listen(new Spec(null, Test.PORT));
-            assertEquals(Test.PORT, a.port());
+            Acceptor a = server.listen(new Spec(null, 0));
+            assertTrue(a.port() > 0);
             a.shutdown().join();
             assertEquals(-1, a.port());
         } catch (ListenFailedException e) {
             assertTrue(false);
         }
         try {
-            Acceptor a = server.listen(new Spec("tcp/" + Test.PORT));
-            assertEquals(Test.PORT, a.port());
+            Acceptor a = server.listen(new Spec("tcp/" + 0));
+            assertTrue(a.port() > 0);
             a.shutdown().join();
             assertEquals(-1, a.port());
         } catch (ListenFailedException e) {
             assertTrue(false);
         }
         try {
-            Acceptor a = server.listen(new Spec(Test.PORT_0));
-            Acceptor b = server.listen(new Spec(Test.PORT_1));
-            Acceptor c = server.listen(new Spec(Test.PORT_2));
-            assertEquals(Test.PORT_0, a.port());
-            assertEquals(Test.PORT_1, b.port());
-            assertEquals(Test.PORT_2, c.port());
+            Acceptor a = server.listen(new Spec(0));
+            Acceptor b = server.listen(new Spec(0));
+            Acceptor c = server.listen(new Spec(0));
+            assertTrue(a.port() > 0);
+            assertTrue(b.port() > 0);
+            assertTrue(c.port() > 0);
+            assertTrue(a.port() != b.port());
+            assertTrue(a.port() != c.port());
+            assertTrue(b.port() != c.port());
             a.shutdown().join();
             assertEquals(-1, a.port());
             b.shutdown().join();
@@ -71,35 +74,21 @@ public class ListenTest {
             Acceptor a = server.listen(new Spec("bogus"));
             assertTrue(false);
         } catch (ListenFailedException e) {}
-
-        try {
-            Acceptor a = server.listen(new Spec(Test.PORT));
-            assertEquals(Test.PORT, a.port());
-            // try {
-            //  Acceptor b = server.listen(new Spec(Test.PORT));
-            //  assertTrue(false);
-            // } catch (ListenFailedException e) {}
-            a.shutdown().join();
-            assertEquals(-1, a.port());
-        } catch (ListenFailedException e) {
-            assertTrue(false);
-        }
     }
 
     @org.junit.Test
-    public void testListenAnyPort() {
+    public void testListenSamePort() {
         try {
             Acceptor a = server.listen(new Spec("tcp/0"));
             assertTrue(a.port() > 0);
-            // try {
-            //  Acceptor b = server.listen(new Spec(a.port()));
-            //  assertTrue(false);
-            // } catch (ListenFailedException e) {}
+            try {
+                Acceptor b = server.listen(new Spec(a.port()));
+                assertTrue(false);
+            } catch (ListenFailedException e) {}
             a.shutdown().join();
             assertEquals(-1, a.port());
         } catch (ListenFailedException e) {
             assertTrue(false);
         }
     }
-
 }
diff --git a/jrt/tests/com/yahoo/jrt/MandatoryMethodsTest.java b/jrt/tests/com/yahoo/jrt/MandatoryMethodsTest.java
index c61cae18c09..0bf83240338 100644
--- a/jrt/tests/com/yahoo/jrt/MandatoryMethodsTest.java
+++ b/jrt/tests/com/yahoo/jrt/MandatoryMethodsTest.java
@@ -23,8 +23,8 @@ public class MandatoryMethodsTest {
     public void setUp() throws ListenFailedException {
         server   = new Supervisor(new Transport());
         client   = new Supervisor(new Transport());
-        acceptor = server.listen(new Spec(Test.PORT));
-        target   = client.connect(new Spec("localhost", Test.PORT));
+        acceptor = server.listen(new Spec(0));
+        target   = client.connect(new Spec("localhost", acceptor.port()));
     }
 
     @After
diff --git a/jrt/tests/com/yahoo/jrt/SessionTest.java b/jrt/tests/com/yahoo/jrt/SessionTest.java
index afe14bfb908..1a788c4038e 100644
--- a/jrt/tests/com/yahoo/jrt/SessionTest.java
+++ b/jrt/tests/com/yahoo/jrt/SessionTest.java
@@ -115,8 +115,8 @@ public class SessionTest implements SessionHandler {
         server.setSessionHandler(this);
         client   = new Test.Orb(new Transport());
         client.setSessionHandler(this);
-        acceptor = server.listen(new Spec(Test.PORT));
-        target   = client.connect(new Spec("localhost", Test.PORT),
+        acceptor = server.listen(new Spec(0));
+        target   = client.connect(new Spec("localhost", acceptor.port()),
                                   new Session());
 
         server.addMethod(new Method("set", "i", "", this,
@@ -425,7 +425,7 @@ public class SessionTest implements SessionHandler {
         assertEquals(1, client.downCount);
         assertEquals(1, client.finiCount);
 
-        target = client.connect(new Spec("localhost", Test.PORT),
+        target = client.connect(new Spec("localhost", acceptor.port()),
                                 new Session());
 
         waitState(0, 2, 1, 2, 2, 2, -1, 2, 2);
@@ -441,7 +441,7 @@ public class SessionTest implements SessionHandler {
 
         client.transport().shutdown().join();
 
-        target = client.connect(new Spec("localhost", Test.PORT),
+        target = client.connect(new Spec("localhost", acceptor.port()),
                                 new Session());
 
         waitState(0, 2, 1, 2, 2, 3, oldClientLive, 3, 3);
diff --git a/jrt/tests/com/yahoo/jrt/Test.java b/jrt/tests/com/yahoo/jrt/Test.java
index b538d5729e5..efde0c57912 100644
--- a/jrt/tests/com/yahoo/jrt/Test.java
+++ b/jrt/tests/com/yahoo/jrt/Test.java
@@ -8,19 +8,6 @@ public class Test {
     @org.junit.Test
     public void testNothing() {}
 
-    // www.random.org [2000, 9999]
-    public static final int PORT   = 9741;
-    public static final int PORT_0 = 5069;
-    public static final int PORT_1 = 4935;
-    public static final int PORT_2 = 8862;
-    public static final int PORT_3 = 4695;
-    public static final int PORT_4 = 6975;
-    public static final int PORT_5 = 7186;
-    public static final int PORT_6 = 7694;
-    public static final int PORT_7 = 3518;
-    public static final int PORT_8 = 3542;
-    public static final int PORT_9 = 4954;
-
     /**
      * Supervisor extension with some extra statistics used for
      * testing.
diff --git a/jrt/tests/com/yahoo/jrt/TimeoutTest.java b/jrt/tests/com/yahoo/jrt/TimeoutTest.java
index d1eb2a9895a..61822554a65 100644
--- a/jrt/tests/com/yahoo/jrt/TimeoutTest.java
+++ b/jrt/tests/com/yahoo/jrt/TimeoutTest.java
@@ -20,8 +20,8 @@ public class TimeoutTest {
     public void setUp() throws ListenFailedException {
         server   = new Supervisor(new Transport());
         client   = new Supervisor(new Transport());
-        acceptor = server.listen(new Spec(Test.PORT));
-        target   = client.connect(new Spec("localhost", Test.PORT));
+        acceptor = server.listen(new Spec(0));
+        target   = client.connect(new Spec("localhost", acceptor.port()));
         server.addMethod(new Method("concat", "ss", "s", this, "rpc_concat")
                          .methodDesc("Concatenate 2 strings")
                          .paramDesc(0, "str1", "a string")
diff --git a/jrt/tests/com/yahoo/jrt/WatcherTest.java b/jrt/tests/com/yahoo/jrt/WatcherTest.java
index 591a55662f0..3926b859875 100644
--- a/jrt/tests/com/yahoo/jrt/WatcherTest.java
+++ b/jrt/tests/com/yahoo/jrt/WatcherTest.java
@@ -36,8 +36,8 @@ public class WatcherTest {
     public void setUp() throws ListenFailedException {
         server   = new Supervisor(new Transport());
         client   = new Supervisor(new Transport());
-        acceptor = server.listen(new Spec(Test.PORT));
-        target   = client.connect(new Spec("localhost", Test.PORT));
+        acceptor = server.listen(new Spec(0));
+        target   = client.connect(new Spec("localhost", acceptor.port()));
     }
 
     @After
diff --git a/linguistics/pom.xml b/linguistics/pom.xml
index e4aa7c3049e..f743348dde3 100644
--- a/linguistics/pom.xml
+++ b/linguistics/pom.xml
@@ -62,6 +62,14 @@
       <scope>provided</scope>
       <classifier>no_aop</classifier>
     </dependency>
+    <dependency>
+      <groupId>org.apache.opennlp</groupId>
+      <artifactId>opennlp-tools</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>com.optimaize.languagedetector</groupId>
+      <artifactId>language-detector</artifactId>
+    </dependency>
   </dependencies>
   <build>
     <plugins>
diff --git a/linguistics/src/main/java/com/yahoo/language/Linguistics.java b/linguistics/src/main/java/com/yahoo/language/Linguistics.java
index 5e28213d524..035de415aa7 100644
--- a/linguistics/src/main/java/com/yahoo/language/Linguistics.java
+++ b/linguistics/src/main/java/com/yahoo/language/Linguistics.java
@@ -41,7 +41,12 @@ public interface Linguistics {
         CHARACTER_CLASSES
     }
 
-    /** The same as new com.yahoo.language.simple.SimpleLinguistics(). Prefer using that directly. */
+    /**
+     * The same as new com.yahoo.language.simple.SimpleLinguistics(). Prefer using that directly.
+     *
+     * @deprecated use new com.yahoo.language.simple.SimpleLinguistics()
+     */
+    @Deprecated // TODO: Remove this field on Vespa 7
     Linguistics SIMPLE = new SimpleLinguistics();
 
     /**
diff --git a/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpLinguistics.java b/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpLinguistics.java
new file mode 100644
index 00000000000..12de309a2d3
--- /dev/null
+++ b/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpLinguistics.java
@@ -0,0 +1,11 @@
+package com.yahoo.language.opennlp;
+
+import com.yahoo.language.process.Tokenizer;
+import com.yahoo.language.simple.SimpleLinguistics;
+
+public class OpenNlpLinguistics extends SimpleLinguistics {
+    @Override
+    public Tokenizer getTokenizer() {
+        return new OpenNlpTokenizer(getNormalizer(), getTransformer());
+    }
+}
diff --git a/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpTokenizer.java b/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpTokenizer.java
new file mode 100644
index 00000000000..5d5f5cbfba9
--- /dev/null
+++ b/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpTokenizer.java
@@ -0,0 +1,135 @@
+package com.yahoo.language.opennlp;
+
+import com.yahoo.language.Language;
+import com.yahoo.language.LinguisticsCase;
+import com.yahoo.language.process.*;
+import com.yahoo.language.simple.*;
+import opennlp.tools.stemmer.Stemmer;
+import opennlp.tools.stemmer.snowball.SnowballStemmer;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+public class OpenNlpTokenizer implements Tokenizer {
+    private final static int SPACE_CODE = 32;
+    private final Normalizer normalizer;
+    private final Transformer transformer;
+    private final SimpleTokenizer simpleTokenizer;
+
+    public OpenNlpTokenizer() {
+        this(new SimpleNormalizer(), new SimpleTransformer());
+    }
+
+    public OpenNlpTokenizer(Normalizer normalizer, Transformer transformer) {
+        this.normalizer = normalizer;
+        this.transformer = transformer;
+        simpleTokenizer = new SimpleTokenizer(normalizer, transformer);
+    }
+
+    @Override
+    public Iterable<Token> tokenize(String input, Language language, StemMode stemMode, boolean removeAccents) {
+        if (input.isEmpty()) return Collections.emptyList();
+        Stemmer stemmer = getStemmerForLanguage(language, stemMode);
+        if (stemmer == null) {
+            return simpleTokenizer.tokenize(input, language, stemMode, removeAccents);
+        }
+
+        List<Token> tokens = new ArrayList<>();
+        int nextCode = input.codePointAt(0);
+        TokenType prevType = SimpleTokenType.valueOf(nextCode);
+        for (int prev = 0, next = Character.charCount(nextCode); next <= input.length(); ) {
+            nextCode = next < input.length() ? input.codePointAt(next) : SPACE_CODE;
+            TokenType nextType = SimpleTokenType.valueOf(nextCode);
+            if (!prevType.isIndexable() || !nextType.isIndexable()) {
+                String original = input.substring(prev, next);
+                String token = processToken(original, language, stemMode, removeAccents, stemmer);
+                tokens.add(new SimpleToken(original).setOffset(prev)
+                        .setType(prevType)
+                        .setTokenString(token));
+                prev = next;
+                prevType = nextType;
+            }
+            next += Character.charCount(nextCode);
+        }
+        return tokens;
+    }
+
+    private Stemmer getStemmerForLanguage(Language language, StemMode stemMode) {
+        if (language == null || Language.ENGLISH.equals(language) || StemMode.NONE.equals(stemMode)) {
+            return null;
+        }
+        SnowballStemmer.ALGORITHM alg;
+        switch (language) {
+            case DANISH:
+                alg = SnowballStemmer.ALGORITHM.DANISH;
+                break;
+            case DUTCH:
+                alg = SnowballStemmer.ALGORITHM.DUTCH;
+                break;
+            case FINNISH:
+                alg = SnowballStemmer.ALGORITHM.FINNISH;
+                break;
+            case FRENCH:
+                alg = SnowballStemmer.ALGORITHM.FRENCH;
+                break;
+            case GERMAN:
+                alg = SnowballStemmer.ALGORITHM.GERMAN;
+                break;
+            case HUNGARIAN:
+                alg = SnowballStemmer.ALGORITHM.HUNGARIAN;
+                break;
+            case IRISH:
+                alg = SnowballStemmer.ALGORITHM.IRISH;
+                break;
+            case ITALIAN:
+                alg = SnowballStemmer.ALGORITHM.ITALIAN;
+                break;
+            case NORWEGIAN_BOKMAL:
+            case NORWEGIAN_NYNORSK:
+                alg = SnowballStemmer.ALGORITHM.NORWEGIAN;
+                break;
+            case PORTUGUESE:
+                alg = SnowballStemmer.ALGORITHM.PORTUGUESE;
+                break;
+            case ROMANIAN:
+                alg = SnowballStemmer.ALGORITHM.ROMANIAN;
+                break;
+            case RUSSIAN:
+                alg = SnowballStemmer.ALGORITHM.RUSSIAN;
+                break;
+            case SPANISH:
+                alg = SnowballStemmer.ALGORITHM.SPANISH;
+                break;
+            case SWEDISH:
+                alg = SnowballStemmer.ALGORITHM.SWEDISH;
+                break;
+            case TURKISH:
+                alg = SnowballStemmer.ALGORITHM.TURKISH;
+                break;
+            case ENGLISH:
+                alg = SnowballStemmer.ALGORITHM.ENGLISH;
+                break;
+            default:
+                return null;
+
+        }
+        return new SnowballStemmer(alg);
+    }
+
+    private String processToken(String token, Language language, StemMode stemMode, boolean removeAccents,
+                                Stemmer stemmer) {
+        token = normalizer.normalize(token);
+        token = LinguisticsCase.toLowerCase(token);
+        if (removeAccents)
+            token = transformer.accentDrop(token, language);
+        if (stemMode != StemMode.NONE) {
+            token = doStemming(token, stemmer);
+        }
+        return token;
+    }
+
+    private String doStemming(String token, Stemmer stemmer) {
+        return stemmer.stem(token).toString();
+    }
+}
diff --git a/linguistics/src/main/java/com/yahoo/language/opennlp/package-info.java b/linguistics/src/main/java/com/yahoo/language/opennlp/package-info.java
new file mode 100644
index 00000000000..2bdd315418f
--- /dev/null
+++ b/linguistics/src/main/java/com/yahoo/language/opennlp/package-info.java
@@ -0,0 +1,5 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.language.opennlp;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/linguistics/src/main/java/com/yahoo/language/simple/SimpleDetector.java b/linguistics/src/main/java/com/yahoo/language/simple/SimpleDetector.java
index e6ce4eddb59..2b31f95675b 100644
--- a/linguistics/src/main/java/com/yahoo/language/simple/SimpleDetector.java
+++ b/linguistics/src/main/java/com/yahoo/language/simple/SimpleDetector.java
@@ -1,17 +1,30 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.language.simple;
 
+import com.google.common.base.Optional;
+import com.optimaize.langdetect.LanguageDetector;
+import com.optimaize.langdetect.LanguageDetectorBuilder;
+import com.optimaize.langdetect.i18n.LdLocale;
+import com.optimaize.langdetect.ngram.NgramExtractors;
+import com.optimaize.langdetect.profiles.LanguageProfile;
+import com.optimaize.langdetect.profiles.LanguageProfileReader;
+import com.optimaize.langdetect.text.CommonTextObjectFactories;
+import com.optimaize.langdetect.text.TextObject;
+import com.optimaize.langdetect.text.TextObjectFactory;
 import com.yahoo.language.Language;
 import com.yahoo.language.detect.Detection;
 import com.yahoo.language.detect.Detector;
 import com.yahoo.language.detect.Hint;
 import com.yahoo.text.Utf8;
 
+import java.io.IOException;
 import java.nio.ByteBuffer;
+import java.util.List;
+import java.util.Locale;
 
 /**
- * Includes functionality for determining the langCode from a sample or from the encoding. Currently only Chinese,
- * Japanese and Korean are supported.  There are two ways to guess a String's langCode, by encoding and by character
+ * Includes functionality for determining the langCode from a sample or from the encoding.
+ * There are two ways to guess a String's langCode, by encoding and by character
  * set.  If the encoding is available this is a very good indication of the langCode.  If the encoding is not available,
  * then the actual characters in the string can be used to make an educated guess at the String's langCode.  Recall a
  * String in Java is unicode. Therefore, we can simply look at the unicode blocks of the characters in the string.
@@ -21,8 +34,40 @@ import java.nio.ByteBuffer;
  * character blocks, so if there are no definitive signs of Japanese then it is assumed that the String is Chinese.
  *
  * @author Rich Pito
+ * @author bjorncs
  */
 public class SimpleDetector implements Detector {
+    static private TextObjectFactory textObjectFactory;
+    static private LanguageDetector languageDetector;
+
+    static {
+        // origin: https://github.com/optimaize/language-detector
+        //load all languages:
+        List<LanguageProfile> languageProfiles;
+        try {
+            languageProfiles = new LanguageProfileReader().readAllBuiltIn();
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+
+        //build language detector:
+        languageDetector = LanguageDetectorBuilder.create(NgramExtractors.standard())
+                .withProfiles(languageProfiles)
+                .build();
+
+        //create a text object factory
+        textObjectFactory = CommonTextObjectFactories.forDetectingOnLargeText();
+    }
+
+    private final boolean enableOptimaize;
+
+    public SimpleDetector() {
+        this.enableOptimaize = true;
+    }
+
+    public SimpleDetector(SimpleLinguisticsConfig.Detector detector) {
+        this.enableOptimaize = detector.enableOptimaize();
+    }
 
     @Override
     public Detection detect(byte[] input, int offset, int length, Hint hint) {
@@ -41,11 +86,11 @@ public class SimpleDetector implements Detector {
         return new Detection(guessLanguage(input), Utf8.getCharset().name(), false);
     }
 
-    public static Language guessLanguage(byte[] buf, int offset, int length) {
+    public Language guessLanguage(byte[] buf, int offset, int length) {
         return guessLanguage(Utf8.toString(buf, offset, length));
     }
 
-    public static Language guessLanguage(String input) {
+    public Language guessLanguage(String input) {
         if (input == null || input.length() == 0) {
             return Language.UNKNOWN;
         }
@@ -109,10 +154,26 @@ public class SimpleDetector implements Detector {
                 return Language.THAI;
             }
         }
+        if (enableOptimaize && Language.UNKNOWN.equals(soFar)){
+            return detectLangOptimaize(input);
+        }
         // got to the end, so return the current best guess
         return soFar;
     }
 
+    private static Language detectLangOptimaize(String input) {
+        if (input == null || input.length() == 0) {
+            return Language.UNKNOWN;
+        }
+        TextObject textObject = textObjectFactory.forText(input);
+        Optional<LdLocale> lang = languageDetector.detect(textObject);
+        if (lang.isPresent()) {
+            String language = lang.get().getLanguage();
+            return Language.fromLocale(new Locale(language));
+        }
+        return Language.UNKNOWN;
+    }
+
     private boolean isTrailingOctet(byte i) {
         return ((i >>> 6) & 3) == 2;
     }
diff --git a/linguistics/src/main/java/com/yahoo/language/simple/SimpleLinguistics.java b/linguistics/src/main/java/com/yahoo/language/simple/SimpleLinguistics.java
index ad855a18088..cdfd5b4cb58 100644
--- a/linguistics/src/main/java/com/yahoo/language/simple/SimpleLinguistics.java
+++ b/linguistics/src/main/java/com/yahoo/language/simple/SimpleLinguistics.java
@@ -1,6 +1,7 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.language.simple;
 
+import com.google.inject.Inject;
 import com.yahoo.collections.Tuple2;
 import com.yahoo.component.Version;
 import com.yahoo.language.Linguistics;
@@ -19,15 +20,35 @@ import com.yahoo.language.process.Transformer;
  * Factory of pure Java linguistic processor implementations.
  *
  * @author bratseth
+ * @author bjorncs
  */
 public class SimpleLinguistics implements Linguistics {
 
     // Threadsafe instances
-    private final static Normalizer normalizer = new SimpleNormalizer();
-    private final static Transformer transformer = new SimpleTransformer();
-    private final static Detector detector = new SimpleDetector();
-    private final static CharacterClasses characterClasses = new CharacterClasses();
-    private final static GramSplitter gramSplitter = new GramSplitter(characterClasses);
+    private final Normalizer normalizer;
+    private final Transformer transformer;
+    private final Detector detector;
+    private final CharacterClasses characterClasses;
+    private final GramSplitter gramSplitter;
+
+    @Inject
+    public SimpleLinguistics() {
+        CharacterClasses characterClasses = new CharacterClasses();
+        this.normalizer = new SimpleNormalizer();
+        this.transformer = new SimpleTransformer();
+        this.detector = new SimpleDetector();
+        this.characterClasses = new CharacterClasses();
+        this.gramSplitter = new GramSplitter(characterClasses);
+    }
+
+    public SimpleLinguistics(SimpleLinguisticsConfig config) {
+        CharacterClasses characterClasses = new CharacterClasses();
+        this.normalizer = new SimpleNormalizer();
+        this.transformer = new SimpleTransformer();
+        this.detector = new SimpleDetector(config.detector());
+        this.characterClasses = new CharacterClasses();
+        this.gramSplitter = new GramSplitter(characterClasses);
+    }
 
     @Override
     public Stemmer getStemmer() { return new StemmerImpl(getTokenizer()); }
diff --git a/linguistics/src/main/resources/configdefinitions/simple-linguistics.def b/linguistics/src/main/resources/configdefinitions/simple-linguistics.def
new file mode 100644
index 00000000000..d5e7ced7419
--- /dev/null
+++ b/linguistics/src/main/resources/configdefinitions/simple-linguistics.def
@@ -0,0 +1,6 @@
+# Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+namespace=language.simple
+
+# Enable Optimaize language detector
+detector.enableOptimaize bool  default=true
+
diff --git a/linguistics/src/test/java/com/yahoo/language/opennlp/OpenNlpTokenizationTestCase.java b/linguistics/src/test/java/com/yahoo/language/opennlp/OpenNlpTokenizationTestCase.java
new file mode 100644
index 00000000000..914e3817568
--- /dev/null
+++ b/linguistics/src/test/java/com/yahoo/language/opennlp/OpenNlpTokenizationTestCase.java
@@ -0,0 +1,237 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.language.opennlp;
+
+import com.yahoo.language.Language;
+import com.yahoo.language.process.StemMode;
+import com.yahoo.language.process.Token;
+import com.yahoo.language.process.Tokenizer;
+import org.junit.Test;
+
+import java.util.*;
+
+import static com.yahoo.language.LinguisticsCase.toLowerCase;
+import static org.hamcrest.CoreMatchers.is;
+import static org.junit.Assert.*;
+
+/**
+ * Test of tokenization, with stemming and accent removal
+ *
+ * @author <a href="mailto:mathiasm@yahoo-inc.com">Mathias Mølster Lidal</a>
+ */
+public class OpenNlpTokenizationTestCase {
+
+    private final Tokenizer tokenizer = new OpenNlpTokenizer();
+
+    @Test
+    public void testTokenizer() {
+        assertTokenize("This is a test, 123",
+                       Arrays.asList("this", "is", "a", "test", "123"),
+                       Arrays.asList("This", " ", "is", " ", "a", " ", "test", ",", " ", "123"));
+    }
+
+    @Test
+    public void testUnderScoreTokenization() {
+        assertTokenize("ugcapi_1", Language.ENGLISH, StemMode.SHORTEST, true, Arrays.asList("ugcapi", "1"), null);
+    }
+
+    @Test
+    public void testPhrasesWithPunctuation() {
+        assertTokenize("PHY_101.html a space/time or space-time course", Language.ENGLISH, StemMode.NONE,
+                       false,
+                       Arrays.asList("phy", "101", "html", "a", "space", "time", "or", "space", "time", "course"),
+                       null);
+        assertTokenize("PHY_101.", Language.ENGLISH, StemMode.NONE, false, Arrays.asList("phy", "101"), null);
+        assertTokenize("101.3", Language.ENGLISH, StemMode.NONE, false, Arrays.asList("101", "3"), null);
+    }
+
+    @Test
+    public void testDoubleWidthTokenization() {
+        // "sony"
+        assertTokenize("\uFF53\uFF4F\uFF4E\uFF59", Language.ENGLISH, StemMode.NONE, false,
+                       Arrays.asList("sony"), null);
+        assertTokenize("\uFF53\uFF4F\uFF4E\uFF59", Language.ENGLISH, StemMode.SHORTEST, false,
+                       Arrays.asList("sony"), null);
+        // "SONY"
+        assertTokenize("\uFF33\uFF2F\uFF2E\uFF39", Language.ENGLISH, StemMode.NONE, false,
+                       Arrays.asList("sony"), null);
+        assertTokenize("\uFF33\uFF2F\uFF2E\uFF39", Language.ENGLISH, StemMode.SHORTEST, false,
+                       Arrays.asList("sony"), null);
+        // "on"
+        assertTokenize("\uFF4F\uFF4E", Language.ENGLISH, StemMode.NONE, false,
+                       Arrays.asList("on"), null);
+        assertTokenize("\uFF4F\uFF4E", Language.ENGLISH, StemMode.SHORTEST, false,
+                       Arrays.asList("on"), null);
+        // "ON"
+        assertTokenize("\uFF2F\uFF2E", Language.ENGLISH, StemMode.NONE, false,
+                       Arrays.asList("on"), null);
+        assertTokenize("\uFF2F\uFF2E", Language.ENGLISH, StemMode.SHORTEST, false,
+                       Arrays.asList("on"), null);
+        assertTokenize("наименование", Language.RUSSIAN, StemMode.SHORTEST, false,
+                Arrays.asList("наименован"), null);
+    }
+
+    @Test
+    public void testLargeTextTokenization() {
+        StringBuilder sb = new StringBuilder();
+        String s = "teststring ";
+        for (int i = 0; i < 100000; i++) {
+            sb.append(s);
+        }
+
+        String input = sb.toString();
+
+        int numTokens = 0;
+        List<Long> pos = new ArrayList<>();
+        for (Token t : tokenizer.tokenize(input, Language.ENGLISH, StemMode.NONE, false)) {
+            numTokens++;
+            if ((numTokens % 100) == 0) {
+                pos.add(t.getOffset());
+            }
+        }
+
+        assertEquals("Check that all tokens have been tokenized", numTokens, 200000);
+        assertTrue("Increasing token pos", assertMonoIncr(pos));
+    }
+
+    @Test
+    public void testLargeTokenGuard() {
+        StringBuilder str = new StringBuilder();
+        for (int i = 0; i < 128 * 256; i++) {
+            str.append("ab");
+        }
+        Iterator<Token> it = tokenizer.tokenize(str.toString(), Language.ENGLISH, StemMode.NONE, false).iterator();
+        assertTrue(it.hasNext());
+        assertNotNull(it.next().getTokenString());
+        assertFalse(it.hasNext());
+    }
+
+    @Test
+    public void testTokenIterator() {
+        Iterator<Token> it = tokenizer.tokenize("", Language.ENGLISH, StemMode.NONE, false).iterator();
+        assertFalse(it.hasNext());
+        try {
+            it.next();
+            fail();
+        } catch (NoSuchElementException e) {
+            // success
+        }
+
+        it = tokenizer.tokenize("", Language.ENGLISH, StemMode.NONE, false).iterator();
+        assertFalse(it.hasNext());
+
+        it = tokenizer.tokenize("one two three", Language.ENGLISH, StemMode.NONE, false).iterator();
+        assertNotNull(it.next());
+        assertNotNull(it.next());
+        assertNotNull(it.next());
+        assertNotNull(it.next());
+        assertNotNull(it.next());
+        assertFalse(it.hasNext());
+    }
+
+    @Test
+    public void testGetOffsetLength() {
+        String input = "Deka-Chef Weber r\u00e4umt Kommunikationsfehler ein";
+        long[] expOffset = { 0, 4, 5, 9, 10, 15, 16, 21, 22, 42, 43 };
+        int[] len = { 4, 1, 4, 1, 5, 1, 5, 1, 20, 1, 3 };
+
+        int idx = 0;
+        for (Token token : tokenizer.tokenize(input, Language.GERMAN, StemMode.SHORTEST, false)) {
+            assertThat("Token offset for token #" + idx, token.getOffset(), is(expOffset[idx]));
+            assertThat("Token len for token #" + idx, token.getOrig().length(), is(len[idx]));
+            idx++;
+        }
+    }
+
+    @Test
+    public void testRecursiveDecompose() {
+        for (Token t : tokenizer.tokenize("\u00a510%", Language.ENGLISH, StemMode.SHORTEST, false)) {
+            recurseDecompose(t);
+        }
+    }
+
+    @Test
+    public void testIndexability() {
+        String input = "tafsirnya\u0648\u0643\u064F\u0646\u0652";
+        for (StemMode stemMode : new StemMode[] { StemMode.NONE,
+                StemMode.SHORTEST }) {
+            for (Language l : new Language[] { Language.INDONESIAN,
+                    Language.ENGLISH, Language.ARABIC }) {
+                for (boolean accentDrop : new boolean[] { true, false }) {
+                    for (Token token : tokenizer.tokenize(input,
+                            l, stemMode, accentDrop)) {
+                        if (token.getTokenString().length() == 0) {
+                            assertFalse(token.isIndexable());
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    private void recurseDecompose(Token t) {
+        assertTrue(t.getOffset() >= 0);
+        assertTrue(t.getOrig().length() >= 0);
+
+        int numComp = t.getNumComponents();
+        for (int i = 0; i < numComp; i++) {
+            Token comp = t.getComponent(i);
+            recurseDecompose(comp);
+        }
+    }
+
+    private boolean assertMonoIncr(Iterable<Long> n) {
+        long trailing = -1;
+        for (long i : n) {
+            if (i < trailing) {
+                return false;
+            }
+            trailing = i;
+        }
+        return true;
+    }
+
+    private void assertTokenize(String input, List<String> indexed, List<String> orig) {
+        assertTokenize(input, Language.ENGLISH, StemMode.NONE, false, indexed, orig);
+    }
+
+    /**
+     * <p>Compare the results of running an input string through the tokenizer with an "index" truth, and an optional
+     * "orig" truth.</p>
+     *
+     * @param input      The text to process, passed to tokenizer.
+     * @param language   The language tag, passed to tokenizer.
+     * @param stemMode   If stemMode != NONE, test will silently succeed if tokenizer does not do stemming.
+     * @param accentDrop Passed to the tokenizer.
+     * @param indexed    Compared to the "TokenString" result from the tokenizer.
+     * @param orig       Compared to the "Orig" result from the tokenizer.
+     */
+    private void assertTokenize(String input, Language language, StemMode stemMode, boolean accentDrop,
+                                List<String> indexed, List<String> orig) {
+        int i = 0;
+        int j = 0;
+        for (Token token : tokenizer.tokenize(input, language, stemMode, accentDrop)) {
+            // System.err.println("got token orig '"+token.getOrig()+"'");
+            // System.err.println("got token stem '"+token.getTokenString(stemMode)+"'");
+            if (token.getNumComponents() > 0) {
+                for (int comp = 0; comp < token.getNumComponents(); comp++) {
+                    Token t = token.getComponent(comp);
+                    if (t.getType().isIndexable()) {
+                        assertThat("comp index: " + i, toLowerCase(t.getTokenString()), is(indexed.get(i++)));
+                    }
+                }
+            } else {
+                if (token.getType().isIndexable()) {
+                    assertThat("exp index: " + i, toLowerCase(token.getTokenString()), is(indexed.get(i++)));
+                }
+            }
+            if (orig != null) {
+                assertThat("orig index: " + j, token.getOrig(), is(orig.get(j++)));
+            }
+        }
+        assertThat("indexed length", i, is(indexed.size()));
+        if (orig != null) {
+            assertThat("orig length", j, is(orig.size()));
+        }
+    }
+
+}
diff --git a/linguistics/src/test/java/com/yahoo/language/process/TokenizationTestCase.java b/linguistics/src/test/java/com/yahoo/language/process/TokenizationTestCase.java
index e36d90b3206..72dd6f8ce58 100644
--- a/linguistics/src/test/java/com/yahoo/language/process/TokenizationTestCase.java
+++ b/linguistics/src/test/java/com/yahoo/language/process/TokenizationTestCase.java
@@ -64,6 +64,8 @@ public class TokenizationTestCase {
                        Arrays.asList("on"), null);
         assertTokenize("\uFF2F\uFF2E", Language.ENGLISH, StemMode.SHORTEST, false,
                        Arrays.asList("on"), null);
+
+
     }
 
     @Test
diff --git a/linguistics/src/test/java/com/yahoo/language/simple/SimpleDetectorTestCase.java b/linguistics/src/test/java/com/yahoo/language/simple/SimpleDetectorTestCase.java
index f9912f6b7a2..1905c6d98a9 100644
--- a/linguistics/src/test/java/com/yahoo/language/simple/SimpleDetectorTestCase.java
+++ b/linguistics/src/test/java/com/yahoo/language/simple/SimpleDetectorTestCase.java
@@ -50,6 +50,11 @@ public class SimpleDetectorTestCase {
         // a string from http://www.columbia.edu/kermit/utf8.html that says "I can eat glass (and it doesn't hurt me)".
         assertLanguage(Language.KOREAN, "\ub098\ub294 \uc720\ub9ac\ub97c \uba39\uc744 \uc218 \uc788\uc5b4\uc694. " +
                                         "\uadf8\ub798\ub3c4 \uc544\ud504\uc9c0 \uc54a\uc544\uc694");
+
+        // from https://ru.wikipedia.org/wiki/%D0%A0%D0%BE%D1%81%D1%81%D0%B8%D1%8F
+        assertLanguage(Language.RUSSIAN, "Материал из Википедии — свободной энциклопедии");
+        // https://he.wikipedia.org/wiki/Yahoo!
+        assertLanguage(Language.HEBREW, "אתר יאהו! הוא אחד מאתרי האינטרנט הפופולריים ביותר בעולם, עם מעל 500 מיליון כניסות בכל יום");
     }
 
     @Test
diff --git a/logforwarder/src/apps/vespa-logforwarder-start/child-handler.cpp b/logforwarder/src/apps/vespa-logforwarder-start/child-handler.cpp
index 6ed209cbe5b..0588ce5d2e7 100644
--- a/logforwarder/src/apps/vespa-logforwarder-start/child-handler.cpp
+++ b/logforwarder/src/apps/vespa-logforwarder-start/child-handler.cpp
@@ -6,6 +6,8 @@
 #include <unistd.h>
 #include <sys/types.h>
 #include <sys/wait.h>
+#include <vector>
+#include <string>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".child-handler");
@@ -15,13 +17,18 @@ ChildHandler::ChildHandler() : _childRunning(false) {}
 namespace {
 
 void
-runSplunk(const vespalib::string &prefix, const char *a1, const char *a2 = 0)
+runSplunk(const vespalib::string &prefix, std::vector<const char *> args)
 {
-    const char *argv[] = { 0, a1, a2, 0 };
     vespalib::string path = prefix + "/bin/splunk";
-    argv[0] = path.c_str();
-    LOG(debug, "starting splunk forwarder with command: '%s' '%s' '%s'",
-        argv[0], argv[1], argv[2]);
+    args.insert(args.begin(), path.c_str());
+    std::string dbg = "";
+    for (const char *arg : args) {
+        dbg.append(" '");
+        dbg.append(arg);
+        dbg.append("'");
+    }
+    LOG(debug, "starting splunk forwarder with command: %s", dbg.c_str());
+    args.push_back(nullptr);
     fflush(stdout);
     pid_t child = fork();
     if (child == -1) {
@@ -33,10 +40,10 @@ runSplunk(const vespalib::string &prefix, const char *a1, const char *a2 = 0)
         char *cenv = const_cast<char *>(env.c_str()); // safe cast
         putenv(cenv);
         LOG(debug, "added to environment: '%s'", cenv);
-        char **cargv = const_cast<char **>(argv); // safe cast
-        execv(argv[0], cargv);
+        char **cargv = const_cast<char **>(args.data()); // safe cast
+        execv(cargv[0], cargv);
         // if execv fails:
-        perror(argv[0]);
+        perror(cargv[0]);
         exit(1);
     }
     LOG(debug, "child running with pid %d", (int)child);
@@ -69,19 +76,19 @@ void
 ChildHandler::startChild(const vespalib::string &prefix)
 {
     if (! _childRunning) {
-        runSplunk(prefix, "start", "--accept-license");
+        runSplunk(prefix, {"start", "--answer-yes", "--no-prompt", "--accept-license"});
         _childRunning = true;
         // it is possible that splunk was already running, and
         // then the above won't do anything, so we need to
         // *also* do the restart below, after a small delay.
         sleep(1);
     }
-    runSplunk(prefix, "restart");
+    runSplunk(prefix, {"restart"});
 }
 
 void
 ChildHandler::stopChild(const vespalib::string &prefix)
 {
-    runSplunk(prefix, "stop");
+    runSplunk(prefix, {"stop"});
     _childRunning = false;
 }
diff --git a/logserver/bin/logserver-start.sh b/logserver/bin/logserver-start.sh
index c9e0551648e..b1082ddc6a7 100755
--- a/logserver/bin/logserver-start.sh
+++ b/logserver/bin/logserver-start.sh
@@ -76,7 +76,7 @@ cd $ROOT || { echo "Cannot cd to $ROOT" 1>&2; exit 1; }
 
 addopts="-server -Xms64m -Xmx256m -XX:MaxDirectMemorySize=76m -XX:MaxJavaStackTraceDepth=1000000"
 
-oomopt="-XX:OnOutOfMemoryError=kill -9 %p"
+oomopt="-XX:+ExitOnOutOfMemoryError"
 
 jar="-jar $ROOT/lib/jars/logserver-jar-with-dependencies.jar"
 
diff --git a/messagebus/src/vespa/messagebus/network/rpcsend.cpp b/messagebus/src/vespa/messagebus/network/rpcsend.cpp
index 04cccd59903..87c87173ec7 100644
--- a/messagebus/src/vespa/messagebus/network/rpcsend.cpp
+++ b/messagebus/src/vespa/messagebus/network/rpcsend.cpp
@@ -207,12 +207,12 @@ RPCSend::decode(vespalib::stringref protocolName, const vespalib::Version & vers
             }
         } else {
             error = Error(ErrorCode::DECODE_ERROR,
-                          make_string("Protocol '%s' failed to decode routable.", protocolName.c_str()));
+                          make_string("Protocol '%s' failed to decode routable.", vespalib::string(protocolName).c_str()));
         }
 
     } else {
         error = Error(ErrorCode::UNKNOWN_PROTOCOL,
-                      make_string("Protocol '%s' is not known by %s.", protocolName.c_str(), _serverIdent.c_str()));
+                      make_string("Protocol '%s' is not known by %s.", vespalib::string(protocolName).c_str(), _serverIdent.c_str()));
     }
     return reply;
 }
@@ -263,7 +263,7 @@ RPCSend::invoke(FRT_RPCRequest *req)
     if (protocol == nullptr) {
         replyError(req, params->getVersion(), params->getTraceLevel(),
                    Error(ErrorCode::UNKNOWN_PROTOCOL, make_string("Protocol '%s' is not known by %s.",
-                                                                  params->getProtocol().c_str(), _serverIdent.c_str())));
+                                                                  vespalib::string(params->getProtocol()).c_str(), _serverIdent.c_str())));
         return;
     }
     if (protocol->requireSequencing() || !_net->allowDispatchForDecode()) {
@@ -284,7 +284,7 @@ RPCSend::doRequest(FRT_RPCRequest *req, const IProtocol * protocol, std::unique_
     if ( ! routable ) {
         replyError(req, params->getVersion(), params->getTraceLevel(),
                    Error(ErrorCode::DECODE_ERROR,
-                         make_string("Protocol '%s' failed to decode routable.", params->getProtocol().c_str())));
+                         make_string("Protocol '%s' failed to decode routable.", vespalib::string(params->getProtocol()).c_str())));
         return;
     }
     if (routable->isReply()) {
diff --git a/messagebus/src/vespa/messagebus/routing/routeparser.cpp b/messagebus/src/vespa/messagebus/routing/routeparser.cpp
index 3bc9b57d1e7..0fb90f0d585 100644
--- a/messagebus/src/vespa/messagebus/routing/routeparser.cpp
+++ b/messagebus/src/vespa/messagebus/routing/routeparser.cpp
@@ -34,8 +34,9 @@ RouteParser::createTcpDirective(const stringref &str)
     if (posS == string::npos || posS == posP + 1) {
         return IHopDirective::SP(); // no port
     }
+    // FIXME C++17 range-safe from_chars() instead of atoi()
     return IHopDirective::SP(new TcpDirective(str.substr(0, posP),
-                                              atoi(str.substr(posP + 1, posS - 1).c_str()),
+                                              atoi(str.substr(posP + 1, posS - 1).data()),
                                               str.substr(posS + 1)));
 }
 
@@ -104,7 +105,7 @@ RouteParser::createHop(stringref str)
             return Hop().addDirective(createErrorDirective(
                             vespalib::make_string(
                                     "Failed to completely parse '%s'.",
-                                    str.c_str())));
+                                    vespalib::string(str).c_str())));
         } else if (str[at] == '[') {
             ++depth;
         } else if (str[at] == ']') {
diff --git a/metrics/src/vespa/metrics/countmetric.cpp b/metrics/src/vespa/metrics/countmetric.cpp
index a19b14f9545..0c2504b2077 100644
--- a/metrics/src/vespa/metrics/countmetric.cpp
+++ b/metrics/src/vespa/metrics/countmetric.cpp
@@ -14,7 +14,7 @@ AbstractCountMetric::logWarning(const char* msg, const char * op) const
 {
     vespalib::asciistream ost;
     ost << msg << " in count metric " << getPath() << " op " << op << ". Resetting it.";
-    LOG(warning, "%s", ost.str().c_str());
+    LOG(warning, "%s", ost.str().data());
 }
 
 void
diff --git a/metrics/src/vespa/metrics/valuemetric.cpp b/metrics/src/vespa/metrics/valuemetric.cpp
index 04b442829ed..48baa59c7a0 100644
--- a/metrics/src/vespa/metrics/valuemetric.cpp
+++ b/metrics/src/vespa/metrics/valuemetric.cpp
@@ -19,7 +19,7 @@ AbstractValueMetric::logWarning(const char* msg, const char * op) const
 {
     vespalib::asciistream ost;
     ost << msg << " in value metric " << getPath() << " op " << op << ". Resetting it.";
-    LOG(warning, "%s", ost.str().c_str());
+    LOG(warning, "%s", ost.str().data());
 }
 
 void
diff --git a/model-inference/pom.xml b/model-inference/pom.xml
new file mode 100644
index 00000000000..ad258d1edf4
--- /dev/null
+++ b/model-inference/pom.xml
@@ -0,0 +1,78 @@
+<?xml version="1.0"?>
+<!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
+                             http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <groupId>com.yahoo.vespa</groupId>
+    <artifactId>parent</artifactId>
+    <version>6-SNAPSHOT</version>
+    <relativePath>../parent/pom.xml</relativePath>
+  </parent>
+  <artifactId>model-inference</artifactId>
+  <version>6-SNAPSHOT</version>
+  <packaging>container-plugin</packaging>
+  <dependencies>
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>com.yahoo.vespa</groupId>
+      <artifactId>component</artifactId>
+      <version>${project.version}</version>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>com.yahoo.vespa</groupId>
+      <artifactId>configdefinitions</artifactId>
+      <version>${project.version}</version>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>com.yahoo.vespa</groupId>
+      <artifactId>vespajlib</artifactId>
+      <version>${project.version}</version>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>com.yahoo.vespa</groupId>
+      <artifactId>searchlib</artifactId>
+      <version>${project.version}</version>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>com.google.guava</groupId>
+      <artifactId>guava</artifactId>
+      <scope>provided</scope>
+    </dependency>
+  </dependencies>
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>com.yahoo.vespa</groupId>
+        <artifactId>bundle-plugin</artifactId>
+        <extensions>true</extensions>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-jar-plugin</artifactId>
+        <configuration>
+          <archive>
+            <manifestEntries>
+              <Bundle-SymbolicName>${project.artifactId}</Bundle-SymbolicName>
+              <Vespa-Version>${project.version}</Vespa-Version>
+            </manifestEntries>
+          </archive>
+        </configuration>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-compiler-plugin</artifactId>
+      </plugin>
+    </plugins>
+  </build>
+</project>
diff --git a/model-inference/src/main/java/ai/vespa/models/evaluation/FunctionEvaluator.java b/model-inference/src/main/java/ai/vespa/models/evaluation/FunctionEvaluator.java
new file mode 100644
index 00000000000..4acd6e483b4
--- /dev/null
+++ b/model-inference/src/main/java/ai/vespa/models/evaluation/FunctionEvaluator.java
@@ -0,0 +1,57 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package ai.vespa.models.evaluation;
+
+import com.yahoo.searchlib.rankingexpression.ExpressionFunction;
+import com.yahoo.searchlib.rankingexpression.evaluation.TensorValue;
+import com.yahoo.tensor.Tensor;
+import com.yahoo.tensor.TensorType;
+
+/**
+ * An evaluator which can be used to evaluate a single function once.
+ *
+ * @author bratseth
+ */
+// This wraps all access to the context and the ranking expression to avoid incorrect usage
+public class FunctionEvaluator {
+
+    private final ExpressionFunction function;
+    private final LazyArrayContext context;
+    private boolean evaluated = false;
+
+    FunctionEvaluator(ExpressionFunction function, LazyArrayContext context) {
+        this.function = function;
+        this.context = context;
+    }
+
+    /**
+     * Binds the given variable referred in this expression to the given value.
+     *
+     * @param name the variable to bind
+     * @param value the value this becomes bound to
+     * @return this for chaining
+     */
+    public FunctionEvaluator bind(String name, Tensor value) {
+        if (evaluated)
+            throw new IllegalStateException("You cannot bind a value in a used evaluator");
+        context.put(name, new TensorValue(value));
+        return this;
+    }
+
+    /**
+     * Binds the given variable referred in this expression to the given value.
+     * This is equivalent to <code>bind(name, Tensor.Builder.of(TensorType.empty).cell(value).build())</code>
+     *
+     * @param name the variable to bind
+     * @param value the value this becomes bound to
+     * @return this for chaining
+     */
+    public FunctionEvaluator bind(String name, double value) {
+        return bind(name, Tensor.Builder.of(TensorType.empty).cell(value).build());
+    }
+
+    public Tensor evaluate() {
+        evaluated = true;
+        return function.getBody().evaluate(context).asTensor();
+    }
+
+}
diff --git a/model-inference/src/main/java/ai/vespa/models/evaluation/FunctionReference.java b/model-inference/src/main/java/ai/vespa/models/evaluation/FunctionReference.java
new file mode 100644
index 00000000000..3b50cef6e2e
--- /dev/null
+++ b/model-inference/src/main/java/ai/vespa/models/evaluation/FunctionReference.java
@@ -0,0 +1,76 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package ai.vespa.models.evaluation;
+
+import java.util.Objects;
+import java.util.Optional;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * A reference to a function.
+ * The function may be
+ * - free: Callable from users of models, or
+ * - bound: Representing a specific invocation from another ranking expression.
+ * In bound functions, any arguments are replaced by the values supplied in the function invocation.
+ * Function references has a serial form (textual representation) used in ranking expressions received in ranking
+ * expression configurations.
+ *
+ * This is immutable.
+ *
+ * @author bratseth
+ */
+class FunctionReference {
+
+    private static final Pattern referencePattern =
+            Pattern.compile("rankingExpression\\(([a-zA-Z0-9_]+)(@[a-f0-9]+\\.[a-f0-9]+)?\\)(\\.rankingScript)?");
+
+    /** The name of the function referenced */
+    private final String name;
+
+    /** The id of the specific invocation of the function, or null if it is free */
+    private final String instance;
+
+    private FunctionReference(String name, String instance) {
+        this.name = name;
+        this.instance = instance;
+    }
+
+    /** Returns the name of the function referenced */
+    String functionName() { return name; }
+
+    boolean isFree() {
+        return instance == null;
+    }
+
+    String serialForm() {
+        return "rankingExpression(" + name + (instance != null ? instance : "") + ")";
+    }
+
+    @Override
+    public String toString() { return "reference to function '" + name + "'" +
+                                      ( instance != null ? " instance '" + instance + "'" : ""); }
+
+    @Override
+    public int hashCode() { return Objects.hash(name, instance); }
+
+    @Override
+    public boolean equals(Object o) {
+        if (o == this) return true;
+        if ( ! (o instanceof FunctionReference)) return false;
+        FunctionReference other = (FunctionReference)o;
+        if ( ! Objects.equals(this.name, other.name)) return false;
+        if ( ! Objects.equals(this.instance, other.instance)) return false;
+        return true;
+    }
+
+    /** Returns a function reference from the given serial form, or empty if the string is not a valid reference */
+    static Optional<FunctionReference> fromSerial(String serialForm) {
+        Matcher expressionMatcher = referencePattern.matcher(serialForm);
+        if ( ! expressionMatcher.matches()) return Optional.empty();
+
+        String name = expressionMatcher.group(1);
+        String instance = expressionMatcher.group(2);
+        return Optional.of(new FunctionReference(name, instance));
+    }
+
+}
diff --git a/model-inference/src/main/java/ai/vespa/models/evaluation/LazyArrayContext.java b/model-inference/src/main/java/ai/vespa/models/evaluation/LazyArrayContext.java
new file mode 100644
index 00000000000..2dcfd204077
--- /dev/null
+++ b/model-inference/src/main/java/ai/vespa/models/evaluation/LazyArrayContext.java
@@ -0,0 +1,213 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package ai.vespa.models.evaluation;
+
+import com.google.common.collect.ImmutableMap;
+import com.yahoo.searchlib.rankingexpression.ExpressionFunction;
+import com.yahoo.searchlib.rankingexpression.RankingExpression;
+import com.yahoo.searchlib.rankingexpression.Reference;
+import com.yahoo.searchlib.rankingexpression.evaluation.Context;
+import com.yahoo.searchlib.rankingexpression.evaluation.ContextIndex;
+import com.yahoo.searchlib.rankingexpression.evaluation.DoubleValue;
+import com.yahoo.searchlib.rankingexpression.evaluation.Value;
+import com.yahoo.searchlib.rankingexpression.rule.CompositeNode;
+import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode;
+import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode;
+import com.yahoo.tensor.TensorType;
+
+import java.util.Arrays;
+import java.util.LinkedHashSet;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * An array context supporting functions invocations implemented as lazy values.
+ *
+ * @author bratseth
+ */
+final class LazyArrayContext extends Context implements ContextIndex {
+
+    private final IndexedBindings indexedBindings;
+
+    private LazyArrayContext(IndexedBindings indexedBindings) {
+        this.indexedBindings = indexedBindings.copy(this);
+    }
+
+    /**
+     * Create a fast lookup, lazy context for an expression.
+     *
+     * @param expression the expression to create a context for
+     */
+    LazyArrayContext(RankingExpression expression, Map<FunctionReference, ExpressionFunction> functions, Model model) {
+        this.indexedBindings = new IndexedBindings(expression, functions, this, model);
+    }
+
+    /**
+     * Puts a value by name.
+     * The value will be frozen if it isn't already.
+     *
+     * @throws IllegalArgumentException if the name is not present in the ranking expression this was created with, and
+     *         ignoredUnknownValues is false
+     */
+    @Override
+    public void put(String name, Value value) {
+        put(requireIndexOf(name), value);
+    }
+
+    /** Same as put(index,DoubleValue.frozen(value)) */
+    public final void put(int index, double value) {
+        put(index, DoubleValue.frozen(value));
+    }
+
+    /**
+     * Puts a value by index.
+     * The value will be frozen if it isn't already.
+     */
+    public void put(int index, Value value) {
+        indexedBindings.set(index, value.freeze());
+    }
+
+    @Override
+    public TensorType getType(Reference reference) {
+        // TODO: Add type information so we do not need to evaluate to get this
+        return get(requireIndexOf(reference.toString())).type();
+    }
+
+    /** Perform a slow lookup by name */
+    @Override
+    public Value get(String name) {
+        return get(requireIndexOf(name));
+    }
+
+    /** Perform a fast lookup by index */
+    @Override
+    public Value get(int index) {
+        return indexedBindings.get(index);
+    }
+
+    @Override
+    public double getDouble(int index) {
+        double value = get(index).asDouble();
+        if (value == Double.NaN)
+            throw new UnsupportedOperationException("Value at " + index + " has no double representation");
+        return value;
+    }
+
+    @Override
+    public int getIndex(String name) {
+        return requireIndexOf(name);
+    }
+
+    @Override
+    public int size() {
+        return indexedBindings.names().size();
+    }
+
+    @Override
+    public Set<String> names() { return indexedBindings.names(); }
+
+    private Integer requireIndexOf(String name) {
+        Integer index = indexedBindings.indexOf(name);
+        if (index == null)
+            throw new IllegalArgumentException("Value '" + name + "' can not be bound in " + this);
+        return index;
+    }
+
+    /**
+     * Creates a copy of this context suitable for evaluating against the same ranking expression
+     * in a different thread or for re-binding free variables.
+     */
+    LazyArrayContext copy() {
+        return new LazyArrayContext(indexedBindings);
+    }
+
+    private static class IndexedBindings {
+
+        /** The mapping from variable name to index */
+        private final ImmutableMap<String, Integer> nameToIndex;
+
+        /** The current values set, pre-converted to doubles */
+        private final Value[] values;
+
+        private IndexedBindings(ImmutableMap<String, Integer> nameToIndex, Value[] values) {
+            this.nameToIndex = nameToIndex;
+            this.values = values;
+        }
+
+        /**
+         * Creates indexed bindings for the given expressions.
+         * The given expression and functions may be inspected but cannot be stored.
+         */
+        IndexedBindings(RankingExpression expression,
+                        Map<FunctionReference, ExpressionFunction> functions,
+                        LazyArrayContext owner,
+                        Model model) {
+            Set<String> bindTargets = new LinkedHashSet<>();
+            extractBindTargets(expression.getRoot(), functions, bindTargets);
+
+            values = new Value[bindTargets.size()];
+            Arrays.fill(values, DoubleValue.zero);
+
+            int i = 0;
+            ImmutableMap.Builder<String, Integer> nameToIndexBuilder = new ImmutableMap.Builder<>();
+            for (String variable : bindTargets)
+                nameToIndexBuilder.put(variable,i++);
+            nameToIndex = nameToIndexBuilder.build();
+
+            for (Map.Entry<FunctionReference, ExpressionFunction> function : functions.entrySet()) {
+                Integer index = nameToIndex.get(function.getKey().serialForm());
+                if (index != null) // Referenced in this, so bind it
+                    values[index] = new LazyValue(function.getKey(), owner, model);
+            }
+        }
+
+        private void extractBindTargets(ExpressionNode node,
+                                        Map<FunctionReference, ExpressionFunction> functions,
+                                        Set<String> bindTargets) {
+            if (isFunctionReference(node)) {
+                FunctionReference reference = FunctionReference.fromSerial(node.toString()).get();
+                bindTargets.add(reference.serialForm());
+
+                extractBindTargets(functions.get(reference).getBody().getRoot(), functions, bindTargets);
+            }
+            else if (isConstant(node)) {
+                // Ignore
+            }
+            else if (node instanceof ReferenceNode) {
+                bindTargets.add(node.toString());
+            }
+            else if (node instanceof CompositeNode) {
+                CompositeNode cNode = (CompositeNode)node;
+                for (ExpressionNode child : cNode.children())
+                    extractBindTargets(child, functions, bindTargets);
+            }
+        }
+
+        private boolean isFunctionReference(ExpressionNode node) {
+            if ( ! (node instanceof ReferenceNode)) return false;
+
+            ReferenceNode reference = (ReferenceNode)node;
+            return reference.getName().equals("rankingExpression") && reference.getArguments().size() == 1;
+        }
+
+        private boolean isConstant(ExpressionNode node) {
+            if ( ! (node instanceof ReferenceNode)) return false;
+
+            ReferenceNode reference = (ReferenceNode)node;
+            return reference.getName().equals("value") && reference.getArguments().size() == 1;
+        }
+
+        Value get(int index) { return values[index]; }
+        void set(int index, Value value) { values[index] = value; }
+        Set<String> names() { return nameToIndex.keySet(); }
+        Integer indexOf(String name) { return nameToIndex.get(name); }
+
+        IndexedBindings copy(Context context) {
+            Value[] valueCopy = new Value[values.length];
+            for (int i = 0; i < values.length; i++)
+                valueCopy[i] = values[i] instanceof LazyValue ? ((LazyValue)values[i]).copyFor(context) : values[i];
+            return new IndexedBindings(nameToIndex, valueCopy);
+        }
+
+    }
+
+}
diff --git a/model-inference/src/main/java/ai/vespa/models/evaluation/LazyValue.java b/model-inference/src/main/java/ai/vespa/models/evaluation/LazyValue.java
new file mode 100644
index 00000000000..4a1ee22d288
--- /dev/null
+++ b/model-inference/src/main/java/ai/vespa/models/evaluation/LazyValue.java
@@ -0,0 +1,154 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package ai.vespa.models.evaluation;
+
+import com.yahoo.searchlib.rankingexpression.ExpressionFunction;
+import com.yahoo.searchlib.rankingexpression.evaluation.Context;
+import com.yahoo.searchlib.rankingexpression.evaluation.Value;
+import com.yahoo.searchlib.rankingexpression.rule.Function;
+import com.yahoo.searchlib.rankingexpression.rule.TruthOperator;
+import com.yahoo.tensor.Tensor;
+import com.yahoo.tensor.TensorType;
+
+/**
+ * A Value which is computed from an expression when first requested.
+ * This is not multithread safe.
+ *
+ * @author bratseth
+ */
+class LazyValue extends Value {
+
+    /** The reference to the function computing the value of this */
+    private final FunctionReference function;
+
+    /** The context used to compute the function of this */
+    private final Context context;
+
+    /** The model this is part of */
+    private final Model model;
+
+    private Value computedValue = null;
+
+    public LazyValue(FunctionReference function, Context context, Model model) {
+        this.function = function;
+        this.context = context;
+        this.model = model;
+    }
+
+    private Value computedValue() {
+        if (computedValue == null)
+            computedValue = model.requireReferencedFunction(function).getBody().evaluate(context);
+        return computedValue;
+    }
+
+    @Override
+    public TensorType type() {
+        return computedValue().type(); // TODO: Keep type information in this/ExpressionFunction to avoid computing here
+    }
+
+    @Override
+    public double asDouble() {
+        return computedValue().asDouble();
+    }
+
+    @Override
+    public Tensor asTensor() {
+        return computedValue().asTensor();
+    }
+
+    @Override
+    public boolean hasDouble() {
+        return type().rank() == 0;
+    }
+
+    @Override
+    public boolean asBoolean() {
+        return computedValue().asBoolean();
+    }
+
+    @Override
+    public Value negate() {
+        return computedValue().negate();
+    }
+
+    @Override
+    public Value add(Value value) {
+        return computedValue().add(value);
+    }
+
+    @Override
+    public Value subtract(Value value) {
+        return computedValue().subtract(value);
+    }
+
+    @Override
+    public Value multiply(Value value) {
+        return computedValue().multiply(value);
+    }
+
+    @Override
+    public Value divide(Value value) {
+        return computedValue().divide(value);
+    }
+
+    @Override
+    public Value modulo(Value value) {
+        return computedValue().modulo(value);
+    }
+
+    @Override
+    public Value and(Value value) {
+        return computedValue().and(value);
+    }
+
+    @Override
+    public Value or(Value value) {
+        return computedValue().or(value);
+    }
+
+    @Override
+    public Value not() {
+        return computedValue().not();
+    }
+
+    @Override
+    public Value power(Value value) {
+        return computedValue().power(value);
+    }
+
+    @Override
+    public Value compare(TruthOperator operator, Value value) {
+        return computedValue().compare(operator, value);
+    }
+
+    @Override
+    public Value function(Function function, Value value) {
+        return computedValue().function(function, value);
+    }
+
+    @Override
+    public Value asMutable() {
+        return computedValue().asMutable();
+    }
+
+    @Override
+    public String toString() {
+        return "value of " + function;
+    }
+
+    @Override
+    public boolean equals(Object other) {
+        if (other == this) return true;
+        if (!(other instanceof Value)) return false;
+        return computedValue().equals(other);
+    }
+
+    @Override
+    public int hashCode() {
+        return computedValue().hashCode();
+    }
+
+    LazyValue copyFor(Context context) {
+        return new LazyValue(this.function, context, model);
+    }
+
+}
diff --git a/model-inference/src/main/java/ai/vespa/models/evaluation/Model.java b/model-inference/src/main/java/ai/vespa/models/evaluation/Model.java
new file mode 100644
index 00000000000..ca739195867
--- /dev/null
+++ b/model-inference/src/main/java/ai/vespa/models/evaluation/Model.java
@@ -0,0 +1,132 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package ai.vespa.models.evaluation;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.yahoo.searchlib.rankingexpression.ExpressionFunction;
+import com.yahoo.searchlib.rankingexpression.evaluation.ContextIndex;
+import com.yahoo.searchlib.rankingexpression.evaluation.ExpressionOptimizer;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+/**
+ * A named collection of functions
+ *
+ * @author bratseth
+ */
+public class Model {
+
+    private final String name;
+
+    /** Free functions */
+    private final ImmutableList<ExpressionFunction> functions;
+
+    /** Instances of each usage of the above function, where variables (if any) are replaced by their bindings */
+    private final ImmutableMap<FunctionReference, ExpressionFunction> referencedFunctions;
+
+    /** Context prototypes, indexed by function name (as all invocations of the same function share the same context prototype) */
+    private final ImmutableMap<String, LazyArrayContext> contextPrototypes;
+
+    private final ExpressionOptimizer expressionOptimizer = new ExpressionOptimizer();
+
+    public Model(String name, Collection<ExpressionFunction> functions) {
+        this(name, functions, Collections.emptyMap());
+    }
+
+    Model(String name, Collection<ExpressionFunction> functions, Map<FunctionReference, ExpressionFunction> referencedFunctions) {
+        // TODO: Optimize functions
+        this.name = name;
+        this.functions = ImmutableList.copyOf(functions);
+
+        ImmutableMap.Builder<String, LazyArrayContext> contextBuilder = new ImmutableMap.Builder<>();
+        for (ExpressionFunction function : functions) {
+            try {
+                contextBuilder.put(function.getName(), new LazyArrayContext(function.getBody(), referencedFunctions, this));
+            }
+            catch (RuntimeException e) {
+                throw new IllegalArgumentException("Could not prepare an evaluation context for " + function, e);
+            }
+        }
+        this.contextPrototypes = contextBuilder.build();
+
+        ImmutableMap.Builder<FunctionReference, ExpressionFunction> functionsBuilder = new ImmutableMap.Builder<>();
+        for (Map.Entry<FunctionReference, ExpressionFunction> function : referencedFunctions.entrySet()) {
+            ExpressionFunction optimizedFunction = optimize(function.getValue(),
+                                                            contextPrototypes.get(function.getKey().functionName()));
+            functionsBuilder.put(function.getKey(), optimizedFunction);
+        }
+        this.referencedFunctions = functionsBuilder.build();
+    }
+
+    /** Returns an optimized version of the given function */
+    private ExpressionFunction optimize(ExpressionFunction function, ContextIndex context) {
+        // Note: Optimization is in-place but we do not depend on that outside this method
+        expressionOptimizer.optimize(function.getBody(), context);
+        return function;
+    }
+
+    public String name() { return name; }
+
+    /** Returns an immutable list of the free functions of this */
+    public List<ExpressionFunction> functions() { return functions; }
+
+    /** Returns the given function, or throws a IllegalArgumentException if it does not exist */
+    ExpressionFunction requireFunction(String name) {
+        ExpressionFunction function = function(name);
+        if (function == null)
+            throw new IllegalArgumentException("No function named '" + name + "' in " + this + ". Available functions: " +
+                                               functions.stream().map(f -> f.getName()).collect(Collectors.joining(", ")));
+        return function;
+    }
+
+    /** Returns the given function, or throws a IllegalArgumentException if it does not exist */
+    private LazyArrayContext requireContextProprotype(String name) {
+        LazyArrayContext context = contextPrototypes.get(name);
+        if (context == null) // Implies function is not present
+            throw new IllegalArgumentException("No function named '" + name + "' in " + this + ". Available functions: " +
+                                               functions.stream().map(f -> f.getName()).collect(Collectors.joining(", ")));
+        return context;
+    }
+
+    /** Returns the function withe the given name, or null if none */ // TODO: Parameter overloading?
+    ExpressionFunction function(String name) {
+        for (ExpressionFunction function : functions)
+            if (function.getName().equals(name))
+                return function;
+        return null;
+    }
+
+    /** Returns an immutable map of the referenced function instances of this */
+    Map<FunctionReference, ExpressionFunction> referencedFunctions() { return referencedFunctions; }
+
+    /** Returns the given referred function, or throws a IllegalArgumentException if it does not exist */
+    ExpressionFunction requireReferencedFunction(FunctionReference reference) {
+        ExpressionFunction function = referencedFunctions.get(reference);
+        if (function == null)
+            throw new IllegalArgumentException("No " + reference + " in " + this + ". References: " +
+                                               referencedFunctions.keySet().stream()
+                                                                           .map(FunctionReference::serialForm)
+                                                                           .collect(Collectors.joining(", ")));
+        return function;
+    }
+
+    /**
+     * Returns an evaluator which can be used to evaluate the given function in a single thread once.
+
+     * Usage:
+     * <code>Tensor result = model.evaluatorOf("myFunction").bind("foo", value).bind("bar", value).evaluate()</code>
+     *
+     * @throws IllegalArgumentException if the function is not present
+     */
+    public FunctionEvaluator evaluatorOf(String function) {  // TODO: Parameter overloading?
+        return new FunctionEvaluator(requireFunction(function), requireContextProprotype(function).copy());
+    }
+
+    @Override
+    public String toString() { return "model '" + name + "'"; }
+
+}
diff --git a/model-inference/src/main/java/ai/vespa/models/evaluation/ModelsEvaluator.java b/model-inference/src/main/java/ai/vespa/models/evaluation/ModelsEvaluator.java
new file mode 100644
index 00000000000..b36e06e5505
--- /dev/null
+++ b/model-inference/src/main/java/ai/vespa/models/evaluation/ModelsEvaluator.java
@@ -0,0 +1,46 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package ai.vespa.models.evaluation;
+
+import com.google.common.collect.ImmutableMap;
+import com.yahoo.vespa.config.search.RankProfilesConfig;
+
+import java.util.Map;
+import java.util.stream.Collectors;
+
+/**
+ * Evaluates machine-learned models added to Vespa applications and available as config form.
+ * Usage:
+ * <code>Tensor result = evaluator.bind("foo", value).bind("bar", value").evaluate()</code>
+ *
+ * @author bratseth
+ */
+public class ModelsEvaluator {
+
+    private final ImmutableMap<String, Model> models;
+
+    public ModelsEvaluator(RankProfilesConfig config) {
+        models = ImmutableMap.copyOf(new RankProfilesConfigImporter().importFrom(config));
+    }
+
+    /** Returns the models of this as an immutable map */
+    public Map<String, Model> models() { return models; }
+
+    /**
+     * Returns a function which can be used to evaluate the given function in the given model
+     *
+     * @throws IllegalArgumentException if the function or model is not present
+     */
+    public FunctionEvaluator evaluatorOf(String modelName, String functionName) {
+        return requireModel(modelName).evaluatorOf(functionName);
+    }
+
+    /** Returns the given model, or throws a IllegalArgumentException if it does not exist */
+    Model requireModel(String name) {
+        Model model = models.get(name);
+        if (model == null)
+            throw new IllegalArgumentException("No model named '" + name + ". Available models: " +
+                                               models.keySet().stream().collect(Collectors.joining(", ")));
+        return model;
+    }
+
+}
diff --git a/model-inference/src/main/java/ai/vespa/models/evaluation/RankProfilesConfigImporter.java b/model-inference/src/main/java/ai/vespa/models/evaluation/RankProfilesConfigImporter.java
new file mode 100644
index 00000000000..bfd6342218a
--- /dev/null
+++ b/model-inference/src/main/java/ai/vespa/models/evaluation/RankProfilesConfigImporter.java
@@ -0,0 +1,86 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package ai.vespa.models.evaluation;
+
+import com.yahoo.searchlib.rankingexpression.ExpressionFunction;
+import com.yahoo.searchlib.rankingexpression.RankingExpression;
+import com.yahoo.searchlib.rankingexpression.parser.ParseException;
+import com.yahoo.vespa.config.search.RankProfilesConfig;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+/**
+ * Converts RankProfilesConfig instances to RankingExpressions for evaluation
+ *
+ * @author bratseth
+ */
+class RankProfilesConfigImporter {
+
+    /**
+     * Returns a map of the models contained in this config, indexed on name.
+     * The map is modifiable and owned by the caller.
+     */
+    Map<String, Model> importFrom(RankProfilesConfig config) {
+        try {
+            Map<String, Model> models = new HashMap<>();
+            for (RankProfilesConfig.Rankprofile profile : config.rankprofile()) {
+                Model model = importProfile(profile);
+                models.put(model.name(), model);
+            }
+            return models;
+        }
+        catch (ParseException e) {
+            throw new IllegalArgumentException("Could not read rank profiles config - version mismatch?", e);
+        }
+    }
+
+    private Model importProfile(RankProfilesConfig.Rankprofile profile) throws ParseException {
+        List<ExpressionFunction> functions = new ArrayList<>();
+        Map<FunctionReference, ExpressionFunction> referencedFunctions = new HashMap<>();
+        ExpressionFunction firstPhase = null;
+        ExpressionFunction secondPhase = null;
+        for (RankProfilesConfig.Rankprofile.Fef.Property property : profile.fef().property()) {
+            Optional<FunctionReference> reference = FunctionReference.fromSerial(property.name());
+            if ( reference.isPresent()) {
+                List<String> arguments = new ArrayList<>(); // TODO: Arguments?
+                RankingExpression expression = new RankingExpression(reference.get().functionName(), property.value());
+
+                if (reference.get().isFree()) // make available in model under configured name
+                    functions.add(new ExpressionFunction(reference.get().functionName(), arguments, expression)); //
+
+                // Make all functions, bound or not available under the name they are referenced by in expressions
+                referencedFunctions.put(reference.get(), new ExpressionFunction(reference.get().serialForm(), arguments, expression));
+            }
+            else if (property.name().equals("vespa.rank.firstphase")) { // Include in addition to macros
+                firstPhase = new ExpressionFunction("firstphase", new ArrayList<>(),
+                                                    new RankingExpression("first-phase", property.value()));
+            }
+            else if (property.name().equals("vespa.rank.secondphase")) { // Include in addition to macros
+                secondPhase = new ExpressionFunction("secondphase", new ArrayList<>(),
+                                                     new RankingExpression("second-phase", property.value()));
+            }
+        }
+        if (functionByName("firstphase", functions) == null && firstPhase != null) // may be already included, depending on body
+            functions.add(firstPhase);
+        if (functionByName("secondphase", functions) == null && secondPhase != null) // may be already included, depending on body
+            functions.add(secondPhase);
+
+        try {
+            return new Model(profile.name(), functions, referencedFunctions);
+        }
+        catch (RuntimeException e) {
+            throw new IllegalArgumentException("Could not load model '" + profile.name() + "'", e);
+        }
+    }
+
+    private ExpressionFunction functionByName(String name, List<ExpressionFunction> functions) {
+        for (ExpressionFunction function : functions)
+            if (function.getName().equals(name))
+                return function;
+        return null;
+    }
+
+}
diff --git a/model-inference/src/test/java/ai/vespa/models/evaluation/ModelsEvaluatorTest.java b/model-inference/src/test/java/ai/vespa/models/evaluation/ModelsEvaluatorTest.java
new file mode 100644
index 00000000000..60cf0d25ded
--- /dev/null
+++ b/model-inference/src/test/java/ai/vespa/models/evaluation/ModelsEvaluatorTest.java
@@ -0,0 +1,51 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package ai.vespa.models.evaluation;
+
+import com.yahoo.config.subscription.ConfigGetter;
+import com.yahoo.config.subscription.FileSource;
+import com.yahoo.tensor.Tensor;
+import com.yahoo.vespa.config.search.RankProfilesConfig;
+import org.junit.Test;
+
+import java.io.File;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author bratseth
+ */
+public class ModelsEvaluatorTest {
+
+    private static final double delta = 0.00000000001;
+
+    private ModelsEvaluator createModels() {
+        String configPath = "src/test/resources/config/rankexpression/rank-profiles.cfg";
+        RankProfilesConfig config = new ConfigGetter<>(new FileSource(new File(configPath)), RankProfilesConfig.class).getConfig("");
+        return new ModelsEvaluator(config);
+    }
+
+    @Test
+    public void testTensorEvaluation() {
+        ModelsEvaluator models = createModels();
+        FunctionEvaluator function = models.evaluatorOf("macros", "fourtimessum");
+        function.bind("var1", Tensor.from("{{x:0}:3,{x:1}:5}"));
+        function.bind("var2", Tensor.from("{{x:0}:7,{x:1}:11}"));
+        assertEquals(Tensor.from("{{x:0}:40.0,{x:1}:64.0}"), function.evaluate());
+    }
+
+    @Test
+    public void testEvaluationDependingOnMacroTakingArguments() {
+        ModelsEvaluator models = createModels();
+        FunctionEvaluator function = models.evaluatorOf("macros", "secondphase");
+        function.bind("match", 3);
+        function.bind("rankBoost", 5);
+        assertEquals(32.0, function.evaluate().asDouble(), delta);
+    }
+
+    // TODO: Test argument-less function
+    // TODO: Test that binding nonexisting variable doesn't work
+    // TODO: Test that rebinding doesn't work
+    // TODO: Test with nested macros
+    // TODO: Test TF/ONNX model
+
+}
diff --git a/model-inference/src/test/java/ai/vespa/models/evaluation/RankProfilesImporterTest.java b/model-inference/src/test/java/ai/vespa/models/evaluation/RankProfilesImporterTest.java
new file mode 100644
index 00000000000..d45372fc7da
--- /dev/null
+++ b/model-inference/src/test/java/ai/vespa/models/evaluation/RankProfilesImporterTest.java
@@ -0,0 +1,60 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package ai.vespa.models.evaluation;
+
+import com.yahoo.config.subscription.ConfigGetter;
+import com.yahoo.config.subscription.FileSource;
+import com.yahoo.searchlib.rankingexpression.ExpressionFunction;
+import com.yahoo.vespa.config.search.RankProfilesConfig;
+import org.junit.Test;
+
+import java.io.File;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
+/**
+ * Tests instantiating models from rank-profiles configs.
+ *
+ * @author bratseth
+ */
+public class RankProfilesImporterTest {
+
+    @Test
+    public void testImporting() {
+        String configPath = "src/test/resources/config/rankexpression/rank-profiles.cfg";
+        RankProfilesConfig config = new ConfigGetter<>(new FileSource(new File(configPath)), RankProfilesConfig.class).getConfig("");
+        Map<String, Model> models = new RankProfilesConfigImporter().importFrom(config);
+        assertEquals(18, models.size());
+
+        Model macros = models.get("macros");
+        assertNotNull(macros);
+        assertEquals("macros", macros.name());
+        assertEquals(4, macros.functions().size());
+        assertFunction("fourtimessum", "4 * (var1 + var2)", macros);
+        assertFunction("firstphase", "match + fieldMatch(title) + rankingExpression(myfeature)", macros);
+        assertFunction("secondphase", "rankingExpression(fourtimessum@5cf279212355b980.67f1e87166cfef86)", macros);
+        assertFunction("myfeature",
+                       "70 * fieldMatch(title).completeness * pow(0 - fieldMatch(title).earliness,2) + " +
+                       "30 * pow(0 - fieldMatch(description).earliness,2)",
+                       macros);
+        assertEquals(4, macros.referencedFunctions().size());
+        assertBoundFunction("rankingExpression(fourtimessum@5cf279212355b980.67f1e87166cfef86)",
+                            "4 * (match + rankBoost)", macros);
+    }
+
+    private void assertFunction(String name, String expression, Model model) {
+        ExpressionFunction function = model.function(name);
+        assertNotNull(function);
+        assertEquals(name, function.getName());
+        assertEquals(expression, function.getBody().getRoot().toString());
+    }
+
+    private void assertBoundFunction(String name, String expression, Model model) {
+        ExpressionFunction function = model.referencedFunctions().get(FunctionReference.fromSerial(name).get());
+        assertNotNull("Function '" + name + "' is present", function);
+        assertEquals(name, function.getName());
+        assertEquals(expression, function.getBody().getRoot().toString());
+    }
+
+}
diff --git a/model-inference/src/test/resources/config/rankexpression/rank-profiles.cfg b/model-inference/src/test/resources/config/rankexpression/rank-profiles.cfg
new file mode 100644
index 00000000000..f5652c31d2a
--- /dev/null
+++ b/model-inference/src/test/resources/config/rankexpression/rank-profiles.cfg
@@ -0,0 +1,296 @@
+rankprofile[0].name "default"
+rankprofile[0].fef.property[0].name "foo"
+rankprofile[0].fef.property[0].value "bar, baz"
+rankprofile[0].fef.property[1].name "foo"
+rankprofile[0].fef.property[1].value "foobar"
+rankprofile[0].fef.property[2].name "qux"
+rankprofile[0].fef.property[2].value "quux"
+rankprofile[0].fef.property[3].name "foo.bar"
+rankprofile[0].fef.property[3].value "foo.bar"
+rankprofile[0].fef.property[4].name "foo.bar.baz"
+rankprofile[0].fef.property[4].value "123"
+rankprofile[0].fef.property[5].name "foo(bar).baz.2"
+rankprofile[0].fef.property[5].value "123.4"
+rankprofile[0].fef.property[6].name "foo(bar).baz.qux"
+rankprofile[0].fef.property[6].value "foo(bar)"
+rankprofile[0].fef.property[7].name "nud"
+rankprofile[0].fef.property[7].value "ity"
+rankprofile[0].fef.property[8].name "vespa.rank.firstphase"
+rankprofile[0].fef.property[8].value "classicRank"
+rankprofile[0].fef.property[9].name "vespa.rank.secondphase"
+rankprofile[0].fef.property[9].value "rankingExpression(secondphase)"
+rankprofile[0].fef.property[10].name "rankingExpression(secondphase).rankingScript"
+rankprofile[0].fef.property[10].value "4"
+rankprofile[0].fef.property[11].name "vespa.dump.feature"
+rankprofile[0].fef.property[11].value "attribute(foo1).out"
+rankprofile[0].fef.property[12].name "vespa.dump.feature"
+rankprofile[0].fef.property[12].value "attribute(bar1)"
+rankprofile[0].fef.property[13].name "vespa.dump.feature"
+rankprofile[0].fef.property[13].value "attribute(foo2).out"
+rankprofile[0].fef.property[14].name "vespa.dump.feature"
+rankprofile[0].fef.property[14].value "attribute(bar2).out"
+rankprofile[0].fef.property[15].name "vespa.dump.feature"
+rankprofile[0].fef.property[15].value "attribute(foo3).out"
+rankprofile[0].fef.property[16].name "vespa.dump.feature"
+rankprofile[0].fef.property[16].value "attribute(bar3).out"
+rankprofile[0].fef.property[17].name "vespa.dump.feature"
+rankprofile[0].fef.property[17].value "attribute(foo4).out"
+rankprofile[0].fef.property[18].name "vespa.dump.feature"
+rankprofile[0].fef.property[18].value "attribute(bar4).out"
+rankprofile[0].fef.property[19].name "vespa.hitcollector.heapsize"
+rankprofile[0].fef.property[19].value "10"
+rankprofile[0].fef.property[20].name "vespa.hitcollector.arraysize"
+rankprofile[0].fef.property[20].value "20"
+rankprofile[0].fef.property[21].name "vespa.hitcollector.rankscoredroplimit"
+rankprofile[0].fef.property[21].value "-0.5"
+rankprofile[0].fef.property[22].name "vespa.dump.ignoredefaultfeatures"
+rankprofile[0].fef.property[22].value "true"
+rankprofile[1].name "unranked"
+rankprofile[1].fef.property[0].name "vespa.rank.firstphase"
+rankprofile[1].fef.property[0].value "value(0)"
+rankprofile[1].fef.property[1].name "vespa.hitcollector.heapsize"
+rankprofile[1].fef.property[1].value "0"
+rankprofile[1].fef.property[2].name "vespa.hitcollector.arraysize"
+rankprofile[1].fef.property[2].value "0"
+rankprofile[1].fef.property[3].name "vespa.dump.ignoredefaultfeatures"
+rankprofile[1].fef.property[3].value "true"
+rankprofile[2].name "static"
+rankprofile[2].fef.property[0].name "vespa.rank.firstphase"
+rankprofile[2].fef.property[0].value "attribute"
+rankprofile[2].fef.property[1].name "vespa.rank.secondphase"
+rankprofile[2].fef.property[1].value "rankingExpression(secondphase)"
+rankprofile[2].fef.property[2].name "rankingExpression(secondphase).rankingScript"
+rankprofile[2].fef.property[2].value "10 + feature(arg1).out.out"
+rankprofile[2].fef.property[3].name "vespa.summary.feature"
+rankprofile[2].fef.property[3].value "attribute(foo1).out"
+rankprofile[2].fef.property[4].name "vespa.summary.feature"
+rankprofile[2].fef.property[4].value "attribute(bar1)"
+rankprofile[2].fef.property[5].name "vespa.summary.feature"
+rankprofile[2].fef.property[5].value "attribute(foo2).out"
+rankprofile[2].fef.property[6].name "vespa.summary.feature"
+rankprofile[2].fef.property[6].value "attribute(bar2).out"
+rankprofile[2].fef.property[7].name "vespa.summary.feature"
+rankprofile[2].fef.property[7].value "attribute(foo3).out"
+rankprofile[2].fef.property[8].name "vespa.summary.feature"
+rankprofile[2].fef.property[8].value "attribute(bar3).out"
+rankprofile[2].fef.property[9].name "vespa.summary.feature"
+rankprofile[2].fef.property[9].value "attribute(foo4).out"
+rankprofile[2].fef.property[10].name "vespa.summary.feature"
+rankprofile[2].fef.property[10].value "attribute(bar4).out"
+rankprofile[3].name "overflow"
+rankprofile[3].fef.property[0].name "vespa.rank.firstphase"
+rankprofile[3].fef.property[0].value "rankingExpression(firstphase)"
+rankprofile[3].fef.property[1].name "rankingExpression(firstphase).rankingScript"
+rankprofile[3].fef.property[1].value "feature1(argument1,argument2,argument3,argument4).output + feature2(argument1,argument2,argument3,argument4).output + feature3(argument1,argument2,argument3,argument4).output + feature4(argument1,argument2,argument3,argument4).output + feature5(argument1,argument2,argument3,argument4).output + feature6(argument1,argument2,argument3,argument4).output + feature7(argument1,argument2,argument3,argument4).output + feature8(argument1,argument2,argument3,argument4).output + feature9(argument1,argument2,argument3,argument4).output + feature10(argument1,argument2,argument3,argument4).output + feature11(argument1,argument2,argument3,argument4).output + feature12(argument1,argument2,argument3,argument4).output + feature13(argument1,argument2,argument3,argument4).output + feature14(argument1,argument2,argument3,argument4).output + feature15(argument1,argument2,argument3,argument4).output + feature16(argument1,argument2,argument3,argument4).output + feature17(argument1,argument2,argument3,argument4).output + feature18(argument1,argument2,argument3,argument4).output + feature19(argument1,argument2,argument3,argument4).output + feature20(argument1,argument2,argument3,argument4).output + feature21(argument1,argument2,argument3,argument4).output + feature22(argument1,argument2,argument3,argument4).output + feature23(argument1,argument2,argument3,argument4).output + feature24(argument1,argument2,argument3,argument4).output + feature25(argument1,argument2,argument3,argument4).output + feature26(argument1,argument2,argument3,argument4).output + feature27(argument1,argument2,argument3,argument4).output + feature28(argument1,argument2,argument3,argument4).output + feature29(argument1,argument2,argument3,argument4).output + feature30(argument1,argument2,argument3,argument4).output + feature31(argument1,argument2,argument3,argument4).output + feature32(argument1,argument2,argument3,argument4).output + feature33(argument1,argument2,argument3,argument4).output + feature34(argument1,argument2,argument3,argument4).output + feature35(argument1,argument2,argument3,argument4).output + feature36(argument1,argument2,argument3,argument4).output + feature37(argument1,argument2,argument3,argument4).output + feature38(argument1,argument2,argument3,argument4).output + feature39(argument1,argument2,argument3,argument4).output + feature40(argument1,argument2,argument3,argument4).output + feature41(argument1,argument2,argument3,argument4).output + feature42(argument1,argument2,argument3,argument4).output + feature43(argument1,argument2,argument3,argument4).output + feature44(argument1,argument2,argument3,argument4).output + feature45(argument1,argument2,argument3,argument4).output + feature46(argument1,argument2,argument3,argument4).output + feature47(argument1,argument2,argument3,argument4).output + feature48(argument1,argument2,argument3,argument4).output + feature49(argument1,argument2,argument3,argument4).output + feature50(argument1,argument2,argument3,argument4).output + feature51(argument1,argument2,argument3,argument4).output + feature52(argument1,argument2,argument3,argument4).output + feature53(argument1,argument2,argument3,argument4).output + feature54(argument1,argument2,argument3,argument4).output + feature55(argument1,argument2,argument3,argument4).output + feature56(argument1,argument2,argument3,argument4).output + feature57(argument1,argument2,argument3,argument4).output + feature58(argument1,argument2,argument3,argument4).output + feature59(argument1,argument2,argument3,argument4).output + feature60(argument1,argument2,argument3,argument4).output + feature61(argument1,argument2,argument3,argument4).output + feature62(argument1,argument2,argument3,argument4).output + feature63(argument1,argument2,argument3,argument4).output + feature64(argument1,argument2,argument3,argument4).output + feature65(argument1,argument2,argument3,argument4).output + feature66(argument1,argument2,argument3,argument4).output + feature67(argument1,argument2,argument3,argument4).output + feature68(argument1,argument2,argument3,argument4).output + feature69(argument1,argument2,argument3,argument4).output + feature70(argument1,argument2,argument3,argument4).output + feature71(argument1,argument2,argument3,argument4).output + feature72(argument1,argument2,argument3,argument4).output + feature73(argument1,argument2,argument3,argument4).output + feature74(argument1,argument2,argument3,argument4).output + feature75(argument1,argument2,argument3,argument4).output + feature76(argument1,argument2,argument3,argument4).output + feature77(argument1,argument2,argument3,argument4).output + feature78(argument1,argument2,argument3,argument4).output + feature79(argument1,argument2,argument3,argument4).output + feature80(argument1,argument2,argument3,argument4).output + feature81(argument1,argument2,argument3,argument4).output + feature82(argument1,argument2,argument3,argument4).output + feature83(argument1,argument2,argument3,argument4).output + feature84(argument1,argument2,argument3,argument4).output + feature85(argument1,argument2,argument3,argument4).output + feature86(argument1,argument2,argument3,argument4).output + feature87(argument1,argument2,argument3,argument4).output + feature88(argument1,argument2,argument3,argument4).output + feature89(argument1,argument2,argument3,argument4).output + feature90(argument1,argument2,argument3,argument4).output + feature91(argument1,argument2,argument3,argument4).output + feature92(argument1,argument2,argument3,argument4).output + feature93(argument1,argument2,argument3,argument4).output + feature94(argument1,argument2,argument3,argument4).output + feature95(argument1,argument2,argument3,argument4).output + feature96(argument1,argument2,argument3,argument4).output + feature97(argument1,argument2,argument3,argument4).output + feature98(argument1,argument2,argument3,argument4).output + feature99(argument1,argument2,argument3,argument4).output + feature100(argument1,argument2,argument3,argument4).output + feature101(argument1,argument2,argument3,argument4).output + feature102(argument1,argument2,argument3,argument4).output + feature103(argument1,argument2,argument3,argument4).output + feature104(argument1,argument2,argument3,argument4).output + feature105(argument1,argument2,argument3,argument4).output + feature106(argument1,argument2,argument3,argument4).output + feature107(argument1,argument2,argument3,argument4).output + feature108(argument1,argument2,argument3,argument4).output + feature109(argument1,argument2,argument3,argument4).output + feature110(argument1,argument2,argument3,argument4).output + feature111(argument1,argument2,argument3,argument4).output + feature112(argument1,argument2,argument3,argument4).output + feature113(argument1,argument2,argument3,argument4).output + feature114(argument1,argument2,argument3,argument4).output + feature115(argument1,argument2,argument3,argument4).output + feature116(argument1,argument2,argument3,argument4).output + feature117(argument1,argument2,argument3,argument4).output + feature118(argument1,argument2,argument3,argument4).output + feature119(argument1,argument2,argument3,argument4).output + feature120(argument1,argument2,argument3,argument4).output + feature121(argument1,argument2,argument3,argument4).output + feature122(argument1,argument2,argument3,argument4).output + feature123(argument1,argument2,argument3,argument4).output + feature124(argument1,argument2,argument3,argument4).output + feature125(argument1,argument2,argument3,argument4).output + feature126(argument1,argument2,argument3,argument4).output + feature127(argument1,argument2,argument3,argument4).output + feature128(argument1,argument2,argument3,argument4).output + feature129(argument1,argument2,argument3,argument4).output + feature130(argument1,argument2,argument3,argument4).output + feature131(argument1,argument2,argument3,argument4).output + feature132(argument1,argument2,argument3,argument4).output + feature133(argument1,argument2,argument3,argument4).output + feature134(argument1,argument2,argument3,argument4).output + feature135(argument1,argument2,argument3,argument4).output + feature136(argument1,argument2,argument3,argument4).output + feature137(argument1,argument2,argument3,argument4).output + feature138(argument1,argument2,argument3,argument4).output + feature139(argument1,argument2,argument3,argument4).output + feature140(argument1,argument2,argument3,argument4).output + feature141(argument1,argument2,argument3,argument4).output + feature142(argument1,argument2,argument3,argument4).output + feature143(argument1,argument2,argument3,argument4).output + feature144(argument1,argument2,argument3,argument4).output + feature145(argument1,argument2,argument3,argument4).output + feature146(argument1,argument2,argument3,argument4).output + feature147(argument1,argument2,argument3,argument4).output + feature148(argument1,argument2,argument3,argument4).output + feature149(argument1,argument2,argument3,argument4).output + feature150(argument1,argument2,argument3,argument4).output + feature151(argument1,argument2,argument3,argument4).output + feature152(argument1,argument2,argument3,argument4).output + feature153(argument1,argument2,argument3,argument4).output + feature154(argument1,argument2,argument3,argument4).output + feature155(argument1,argument2,argument3,argument4).output + feature156(argument1,argument2,argument3,argument4).output + feature157(argument1,argument2,argument3,argument4).output + feature158(argument1,argument2,argument3,argument4).output + feature159(argument1,argument2,argument3,argument4).output + feature160(argument1,argument2,argument3,argument4).output + feature161(argument1,argument2,argument3,argument4).output + feature162(argument1,argument2,argument3,argument4).output + feature163(argument1,argument2,argument3,argument4).output + feature164(argument1,argument2,argument3,argument4).output + feature165(argument1,argument2,argument3,argument4).output + feature166(argument1,argument2,argument3,argument4).output + feature167(argument1,argument2,argument3,argument4).output + feature168(argument1,argument2,argument3,argument4).output + feature169(argument1,argument2,argument3,argument4).output + feature170(argument1,argument2,argument3,argument4).output + feature171(argument1,argument2,argument3,argument4).output + feature172(argument1,argument2,argument3,argument4).output + feature173(argument1,argument2,argument3,argument4).output + feature174(argument1,argument2,argument3,argument4).output + feature175(argument1,argument2,argument3,argument4).output + feature176(argument1,argument2,argument3,argument4).output + feature177(argument1,argument2,argument3,argument4).output + feature178(argument1,argument2,argument3,argument4).output + feature179(argument1,argument2,argument3,argument4).output + feature180(argument1,argument2,argument3,argument4).output + feature181(argument1,argument2,argument3,argument4).output + feature182(argument1,argument2,argument3,argument4).output + feature183(argument1,argument2,argument3,argument4).output + feature184(argument1,argument2,argument3,argument4).output + feature185(argument1,argument2,argument3,argument4).output + feature186(argument1,argument2,argument3,argument4).output + feature187(argument1,argument2,argument3,argument4).output + feature188(argument1,argument2,argument3,argument4).output + feature189(argument1,argument2,argument3,argument4).output + feature190(argument1,argument2,argument3,argument4).output + feature191(argument1,argument2,argument3,argument4).output + feature192(argument1,argument2,argument3,argument4).output + feature193(argument1,argument2,argument3,argument4).output + feature194(argument1,argument2,argument3,argument4).output + feature195(argument1,argument2,argument3,argument4).output + feature196(argument1,argument2,argument3,argument4).output + feature197(argument1,argument2,argument3,argument4).output + feature198(argument1,argument2,argument3,argument4).output + feature199(argument1,argument2,argument3,argument4).output + feature200(argument1,argument2,argument3,argument4).output + feature201(argument1,argument2,argument3,argument4).output + feature202(argument1,argument2,argument3,argument4).output + feature203(argument1,argument2,argument3,argument4).output + feature204(argument1,argument2,argument3,argument4).output + feature205(argument1,argument2,argument3,argument4).output + feature206(argument1,argument2,argument3,argument4).output + feature207(argument1,argument2,argument3,argument4).output + feature208(argument1,argument2,argument3,argument4).output + feature209(argument1,argument2,argument3,argument4).output + feature210(argument1,argument2,argument3,argument4).output + feature211(argument1,argument2,argument3,argument4).output + feature212(argument1,argument2,argument3,argument4).output + feature213(argument1,argument2,argument3,argument4).output + feature214(argument1,argument2,argument3,argument4).output + feature215(argument1,argument2,argument3,argument4).output + feature216(argument1,argument2,argument3,argument4).output + feature217(argument1,argument2,argument3,argument4).output + feature218(argument1,argument2,argument3,argument4).output + feature219(argument1,argument2,argument3,argument4).output + feature220(argument1,argument2,argument3,argument4).output + feature221(argument1,argument2,argument3,argument4).output + feature222(argument1,argument2,argument3,argument4).output + feature223(argument1,argument2,argument3,argument4).output + feature224(argument1,argument2,argument3,argument4).output + feature225(argument1,argument2,argument3,argument4).output + feature226(argument1,argument2,argument3,argument4).output + feature227(argument1,argument2,argument3,argument4).output + feature228(argument1,argument2,argument3,argument4).output + feature229(argument1,argument2,argument3,argument4).output + feature230(argument1,argument2,argument3,argument4).output + feature231(argument1,argument2,argument3,argument4).output + feature232(argument1,argument2,argument3,argument4).output + feature233(argument1,argument2,argument3,argument4).output + feature234(argument1,argument2,argument3,argument4).output + feature235(argument1,argument2,argument3,argument4).output + feature236(argument1,argument2,argument3,argument4).output + feature237(argument1,argument2,argument3,argument4).output + feature238(argument1,argument2,argument3,argument4).output + feature239(argument1,argument2,argument3,argument4).output + feature240(argument1,argument2,argument3,argument4).output + feature241(argument1,argument2,argument3,argument4).output + feature242(argument1,argument2,argument3,argument4).output + feature243(argument1,argument2,argument3,argument4).output + feature244(argument1,argument2,argument3,argument4).output + feature245(argument1,argument2,argument3,argument4).output + feature246(argument1,argument2,argument3,argument4).output + feature247(argument1,argument2,argument3,argument4).output + feature248(argument1,argument2,argument3,argument4).output + feature249(argument1,argument2,argument3,argument4).output + feature250(argument1,argument2,argument3,argument4).output + feature251(argument1,argument2,argument3,argument4).output + feature252(argument1,argument2,argument3,argument4).output + feature253(argument1,argument2,argument3,argument4).output + feature254(argument1,argument2,argument3,argument4).output + feature255(argument1,argument2,argument3,argument4).output + feature256(argument1,argument2,argument3,argument4).output + feature257(argument1,argument2,argument3,argument4).output + feature258(argument1,argument2,argument3,argument4).output + feature259(argument1,argument2,argument3,argument4).output + feature260(argument1,argument2,argument3,argument4).output + feature261(argument1,argument2,argument3,argument4).output + feature262(argument1,argument2,argument3,argument4).output + feature263(argument1,argument2,argument3,argument4).output + feature264(argument1,argument2,argument3,argument4).output + feature265(argument1,argument2,argument3,argument4).output + feature266(argument1,argument2,argument3,argument4).output + feature267(argument1,argument2,argument3,argument4).output + feature268(argument1,argument2,argument3,argument4).output + feature269(argument1,argument2,argument3,argument4).output + feature270(argument1,argument2,argument3,argument4).output + feature271(argument1,argument2,argument3,argument4).output + feature272(argument1,argument2,argument3,argument4).output + feature273(argument1,argument2,argument3,argument4).output + feature274(argument1,argument2,argument3,argument4).output + feature275(argument1,argument2,argument3,argument4).output + feature276(argument1,argument2,argument3,argument4).output + feature277(argument1,argument2,argument3,argument4).output + feature278(argument1,argument2,argument3,argument4).output + feature279(argument1,argument2,argument3,argument4).output + feature280(argument1,argument2,argument3,argument4).output + feature281(argument1,argument2,argument3,argument4).output + feature282(argument1,argument2,argument3,argument4).output + feature283(argument1,argument2,argument3,argument4).output + feature284(argument1,argument2,argument3,argument4).output + feature285(argument1,argument2,argument3,argument4).output + feature286(argument1,argument2,argument3,argument4).output + feature287(argument1,argument2,argument3,argument4).output + feature288(argument1,argument2,argument3,argument4).output + feature289(argument1,argument2,argument3,argument4).output + feature290(argument1,argument2,argument3,argument4).output + feature291(argument1,argument2,argument3,argument4).output + feature292(argument1,argument2,argument3,argument4).output + feature293(argument1,argument2,argument3,argument4).output + feature294(argument1,argument2,argument3,argument4).output + feature295(argument1,argument2,argument3,argument4).output + feature296(argument1,argument2,argument3,argument4).output + feature297(argument1,argument2,argument3,argument4).output + feature298(argument1,argument2,argument3,argument4).output + feature299(argument1,argument2,argument3,argument4).output + feature300(argument1,argument2,argument3,argument4).output"
+rankprofile[3].fef.property[2].name "vespa.rank.secondphase"
+rankprofile[3].fef.property[2].value "rankingExpression(secondphase)"
+rankprofile[3].fef.property[3].name "rankingExpression(secondphase).rankingScript"
+rankprofile[3].fef.property[3].value "exp(0) + mysum(attribute(foo),\"attribute( bar )\",\"attribute( \\\"baz\\\" )\")"
+rankprofile[3].fef.property[4].name "vespa.hitcollector.heapsize"
+rankprofile[3].fef.property[4].value "101"
+rankprofile[3].fef.property[5].name "vespa.hitcollector.arraysize"
+rankprofile[3].fef.property[5].value "201"
+rankprofile[3].fef.property[6].name "vespa.hitcollector.rankscoredroplimit"
+rankprofile[3].fef.property[6].value "501.5"
+rankprofile[4].name "duplicates"
+rankprofile[4].fef.property[0].name "fieldMatch(a).proximityLimit"
+rankprofile[4].fef.property[0].value "4"
+rankprofile[4].fef.property[1].name "fieldMatch(a).proximityTable"
+rankprofile[4].fef.property[1].value "0.2"
+rankprofile[4].fef.property[2].name "fieldMatch(a).proximityTable"
+rankprofile[4].fef.property[2].value "0.4"
+rankprofile[4].fef.property[3].name "fieldMatch(a).proximityTable"
+rankprofile[4].fef.property[3].value "0.6"
+rankprofile[4].fef.property[4].name "fieldMatch(a).proximityTable"
+rankprofile[4].fef.property[4].value "0.8"
+rankprofile[4].fef.property[5].name "fieldMatch(a).proximityTable"
+rankprofile[4].fef.property[5].value "1"
+rankprofile[4].fef.property[6].name "fieldMatch(a).proximityTable"
+rankprofile[4].fef.property[6].value "0.8"
+rankprofile[4].fef.property[7].name "fieldMatch(a).proximityTable"
+rankprofile[4].fef.property[7].value "0.6"
+rankprofile[4].fef.property[8].name "fieldMatch(a).proximityTable"
+rankprofile[4].fef.property[8].value "0.4"
+rankprofile[4].fef.property[9].name "fieldMatch(a).proximityTable"
+rankprofile[4].fef.property[9].value "0.2"
+rankprofile[5].name "whitespace1"
+rankprofile[5].fef.property[0].name "vespa.rank.firstphase"
+rankprofile[5].fef.property[0].value "rankingExpression(firstphase)"
+rankprofile[5].fef.property[1].name "rankingExpression(firstphase).rankingScript"
+rankprofile[5].fef.property[1].value "1"
+rankprofile[6].name "whitespace2"
+rankprofile[6].fef.property[0].name "vespa.rank.firstphase"
+rankprofile[6].fef.property[0].value "rankingExpression(firstphase)"
+rankprofile[6].fef.property[1].name "rankingExpression(firstphase).rankingScript"
+rankprofile[6].fef.property[1].value "1"
+rankprofile[7].name "macros"
+rankprofile[7].fef.property[0].name "rankingExpression(fourtimessum).rankingScript"
+rankprofile[7].fef.property[0].value "4 * (var1 + var2)"
+rankprofile[7].fef.property[1].name "rankingExpression(myfeature).rankingScript"
+rankprofile[7].fef.property[1].value "70 * fieldMatch(title).completeness * pow(0 - fieldMatch(title).earliness,2) + 30 * pow(0 - fieldMatch(description).earliness,2)"
+rankprofile[7].fef.property[2].name "rankingExpression(fourtimessum@5cf279212355b980.67f1e87166cfef86).rankingScript"
+rankprofile[7].fef.property[2].value "4 * (match + rankBoost)"
+rankprofile[7].fef.property[3].name "vespa.rank.firstphase"
+rankprofile[7].fef.property[3].value "rankingExpression(firstphase)"
+rankprofile[7].fef.property[4].name "rankingExpression(firstphase).rankingScript"
+rankprofile[7].fef.property[4].value "match + fieldMatch(title) + rankingExpression(myfeature)"
+rankprofile[7].fef.property[5].name "vespa.rank.secondphase"
+rankprofile[7].fef.property[5].value "rankingExpression(fourtimessum@5cf279212355b980.67f1e87166cfef86)"
+rankprofile[7].fef.property[6].name "vespa.summary.feature"
+rankprofile[7].fef.property[6].value "fieldMatch(title)"
+rankprofile[8].name "macros2"
+rankprofile[8].fef.property[0].name "foo"
+rankprofile[8].fef.property[0].value "some, list"
+rankprofile[8].fef.property[1].name "rankingExpression(fourtimessum).rankingScript"
+rankprofile[8].fef.property[1].value "4 * (var1 + var2)"
+rankprofile[8].fef.property[2].name "rankingExpression(myfeature).rankingScript"
+rankprofile[8].fef.property[2].value "70 * fieldMatch(title).completeness * pow(0 - fieldMatch(title).earliness,2) + 30 * pow(0 - fieldMatch(description).earliness,2)"
+rankprofile[8].fef.property[3].name "rankingExpression(mysummaryfeature).rankingScript"
+rankprofile[8].fef.property[3].value "70 * fieldMatch(title).completeness"
+rankprofile[8].fef.property[4].name "rankingExpression(mysummaryfeature2).rankingScript"
+rankprofile[8].fef.property[4].value "71 * fieldMatch(title).completeness"
+rankprofile[8].fef.property[5].name "rankingExpression(fourtimessum@2b1138e8965e7ff5.67f1e87166cfef86).rankingScript"
+rankprofile[8].fef.property[5].value "4 * (match + match)"
+rankprofile[8].fef.property[6].name "vespa.rank.firstphase"
+rankprofile[8].fef.property[6].value "classicRank"
+rankprofile[8].fef.property[7].name "vespa.rank.secondphase"
+rankprofile[8].fef.property[7].value "rankingExpression(secondphase)"
+rankprofile[8].fef.property[8].name "rankingExpression(secondphase).rankingScript"
+rankprofile[8].fef.property[8].value "rankingExpression(fourtimessum@2b1138e8965e7ff5.67f1e87166cfef86) + rankingExpression(mysummaryfeature) + rankingExpression(myfeature)"
+rankprofile[8].fef.property[9].name "vespa.summary.feature"
+rankprofile[8].fef.property[9].value "rankingExpression(mysummaryfeature2)"
+rankprofile[8].fef.property[10].name "vespa.summary.feature"
+rankprofile[8].fef.property[10].value "rankingExpression(mysummaryfeature)"
+rankprofile[9].name "macros3"
+rankprofile[9].fef.property[0].name "rankingExpression(onlyusedinsummaryfeature).rankingScript"
+rankprofile[9].fef.property[0].value "5"
+rankprofile[9].fef.property[1].name "vespa.summary.feature"
+rankprofile[9].fef.property[1].value "rankingExpression(matches(title,rankingExpression(onlyusedinsummaryfeature)))"
+rankprofile[10].name "macros3-inherited"
+rankprofile[10].fef.property[0].name "rankingExpression(onlyusedinsummaryfeature).rankingScript"
+rankprofile[10].fef.property[0].value "5"
+rankprofile[10].fef.property[1].name "vespa.summary.feature"
+rankprofile[10].fef.property[1].value "rankingExpression(matches(title,rankingExpression(onlyusedinsummaryfeature)))"
+rankprofile[11].name "macros-inherited"
+rankprofile[11].fef.property[0].name "foo"
+rankprofile[11].fef.property[0].value "some, list"
+rankprofile[11].fef.property[1].name "rankingExpression(fourtimessum).rankingScript"
+rankprofile[11].fef.property[1].value "4 * (var1 + var2)"
+rankprofile[11].fef.property[2].name "rankingExpression(myfeature).rankingScript"
+rankprofile[11].fef.property[2].value "70 * fieldMatch(title).completeness * pow(0 - fieldMatch(title).earliness,2) + 30 * pow(0 - fieldMatch(description).earliness,2)"
+rankprofile[11].fef.property[3].name "rankingExpression(mysummaryfeature).rankingScript"
+rankprofile[11].fef.property[3].value "80 * fieldMatch(title).completeness"
+rankprofile[11].fef.property[4].name "rankingExpression(mysummaryfeature2).rankingScript"
+rankprofile[11].fef.property[4].value "71 * fieldMatch(title).completeness"
+rankprofile[11].fef.property[5].name "rankingExpression(fourtimessum@2b1138e8965e7ff5.67f1e87166cfef86).rankingScript"
+rankprofile[11].fef.property[5].value "4 * (match + match)"
+rankprofile[11].fef.property[6].name "vespa.rank.firstphase"
+rankprofile[11].fef.property[6].value "rankingExpression(firstphase)"
+rankprofile[11].fef.property[7].name "rankingExpression(firstphase).rankingScript"
+rankprofile[11].fef.property[7].value "20000 * rankingExpression(myfeature) + rankingExpression(mysummaryfeature)"
+rankprofile[11].fef.property[8].name "vespa.rank.secondphase"
+rankprofile[11].fef.property[8].value "rankingExpression(secondphase)"
+rankprofile[11].fef.property[9].name "rankingExpression(secondphase).rankingScript"
+rankprofile[11].fef.property[9].value "rankingExpression(fourtimessum@2b1138e8965e7ff5.67f1e87166cfef86) + rankingExpression(mysummaryfeature) + rankingExpression(myfeature)"
+rankprofile[11].fef.property[10].name "vespa.summary.feature"
+rankprofile[11].fef.property[10].value "rankingExpression(mysummaryfeature2)"
+rankprofile[11].fef.property[11].name "vespa.summary.feature"
+rankprofile[11].fef.property[11].value "rankingExpression(mysummaryfeature)"
+rankprofile[12].name "macros-inherited2"
+rankprofile[12].fef.property[0].name "foo"
+rankprofile[12].fef.property[0].value "some, list"
+rankprofile[12].fef.property[1].name "rankingExpression(fourtimessum).rankingScript"
+rankprofile[12].fef.property[1].value "4 * (var1 + var2)"
+rankprofile[12].fef.property[2].name "rankingExpression(myfeature).rankingScript"
+rankprofile[12].fef.property[2].value "70 * fieldMatch(title).completeness * pow(0 - fieldMatch(title).earliness,2) + 30 * pow(0 - fieldMatch(description).earliness,2)"
+rankprofile[12].fef.property[3].name "rankingExpression(mysummaryfeature).rankingScript"
+rankprofile[12].fef.property[3].value "80 * fieldMatch(title).completeness"
+rankprofile[12].fef.property[4].name "rankingExpression(mysummaryfeature2).rankingScript"
+rankprofile[12].fef.property[4].value "71 * fieldMatch(title).completeness"
+rankprofile[12].fef.property[5].name "rankingExpression(fourtimessum@2b1138e8965e7ff5.67f1e87166cfef86).rankingScript"
+rankprofile[12].fef.property[5].value "4 * (match + match)"
+rankprofile[12].fef.property[6].name "vespa.rank.firstphase"
+rankprofile[12].fef.property[6].value "rankingExpression(firstphase)"
+rankprofile[12].fef.property[7].name "rankingExpression(firstphase).rankingScript"
+rankprofile[12].fef.property[7].value "30000 * rankingExpression(mysummaryfeature) + rankingExpression(myfeature)"
+rankprofile[12].fef.property[8].name "vespa.rank.secondphase"
+rankprofile[12].fef.property[8].value "rankingExpression(secondphase)"
+rankprofile[12].fef.property[9].name "rankingExpression(secondphase).rankingScript"
+rankprofile[12].fef.property[9].value "rankingExpression(fourtimessum@2b1138e8965e7ff5.67f1e87166cfef86) + rankingExpression(mysummaryfeature) + rankingExpression(myfeature)"
+rankprofile[12].fef.property[10].name "vespa.summary.feature"
+rankprofile[12].fef.property[10].value "rankingExpression(mysummaryfeature2)"
+rankprofile[12].fef.property[11].name "vespa.summary.feature"
+rankprofile[12].fef.property[11].value "rankingExpression(mysummaryfeature)"
+rankprofile[13].name "macros-inherited3"
+rankprofile[13].fef.property[0].name "foo"
+rankprofile[13].fef.property[0].value "some, list"
+rankprofile[13].fef.property[1].name "rankingExpression(fourtimessum).rankingScript"
+rankprofile[13].fef.property[1].value "4 * (var1 + var2)"
+rankprofile[13].fef.property[2].name "rankingExpression(myfeature).rankingScript"
+rankprofile[13].fef.property[2].value "700 * fieldMatch(title).completeness"
+rankprofile[13].fef.property[3].name "rankingExpression(mysummaryfeature).rankingScript"
+rankprofile[13].fef.property[3].value "80 * fieldMatch(title).completeness"
+rankprofile[13].fef.property[4].name "rankingExpression(mysummaryfeature2).rankingScript"
+rankprofile[13].fef.property[4].value "71 * fieldMatch(title).completeness"
+rankprofile[13].fef.property[5].name "vespa.rank.firstphase"
+rankprofile[13].fef.property[5].value "rankingExpression(firstphase)"
+rankprofile[13].fef.property[6].name "rankingExpression(firstphase).rankingScript"
+rankprofile[13].fef.property[6].value "30000 * rankingExpression(mysummaryfeature) + rankingExpression(myfeature)"
+rankprofile[13].fef.property[7].name "vespa.rank.secondphase"
+rankprofile[13].fef.property[7].value "rankingExpression(secondphase)"
+rankprofile[13].fef.property[8].name "rankingExpression(secondphase).rankingScript"
+rankprofile[13].fef.property[8].value "40000 * rankingExpression(mysummaryfeature) + rankingExpression(myfeature)"
+rankprofile[13].fef.property[9].name "vespa.summary.feature"
+rankprofile[13].fef.property[9].value "rankingExpression(mysummaryfeature2)"
+rankprofile[13].fef.property[10].name "vespa.summary.feature"
+rankprofile[13].fef.property[10].value "rankingExpression(mysummaryfeature)"
+rankprofile[14].name "macros-refering-macros"
+rankprofile[14].fef.property[0].name "rankingExpression(m1).rankingScript"
+rankprofile[14].fef.property[0].value "700 * fieldMatch(title).completeness"
+rankprofile[14].fef.property[1].name "rankingExpression(m2).rankingScript"
+rankprofile[14].fef.property[1].value "rankingExpression(m1) * 67"
+rankprofile[14].fef.property[2].name "rankingExpression(m4).rankingScript"
+rankprofile[14].fef.property[2].value "703 * fieldMatch(fromfile).completeness"
+rankprofile[14].fef.property[3].name "vespa.rank.secondphase"
+rankprofile[14].fef.property[3].value "rankingExpression(secondphase)"
+rankprofile[14].fef.property[4].name "rankingExpression(secondphase).rankingScript"
+rankprofile[14].fef.property[4].value "40000 * rankingExpression(m2)"
+rankprofile[15].name "macros-refering-macros-inherited"
+rankprofile[15].fef.property[0].name "rankingExpression(m1).rankingScript"
+rankprofile[15].fef.property[0].value "700 * fieldMatch(title).completeness"
+rankprofile[15].fef.property[1].name "rankingExpression(m2).rankingScript"
+rankprofile[15].fef.property[1].value "rankingExpression(m1) * 67"
+rankprofile[15].fef.property[2].name "rankingExpression(m4).rankingScript"
+rankprofile[15].fef.property[2].value "701 * fieldMatch(title).completeness"
+rankprofile[15].fef.property[3].name "rankingExpression(m3).rankingScript"
+rankprofile[15].fef.property[3].value "if (isNan(attribute(nrtgmp)) == 1, 0.0, rankingExpression(m2))"
+rankprofile[15].fef.property[4].name "vespa.rank.secondphase"
+rankprofile[15].fef.property[4].value "rankingExpression(secondphase)"
+rankprofile[15].fef.property[5].name "rankingExpression(secondphase).rankingScript"
+rankprofile[15].fef.property[5].value "3000 * rankingExpression(m2)"
+rankprofile[16].name "macros-refering-macros-inherited2"
+rankprofile[16].fef.property[0].name "rankingExpression(m1).rankingScript"
+rankprofile[16].fef.property[0].value "700 * fieldMatch(title).completeness"
+rankprofile[16].fef.property[1].name "rankingExpression(m2).rankingScript"
+rankprofile[16].fef.property[1].value "rankingExpression(m1) * 67"
+rankprofile[16].fef.property[2].name "rankingExpression(m4).rankingScript"
+rankprofile[16].fef.property[2].value "703 * fieldMatch(fromfile).completeness"
+rankprofile[16].fef.property[3].name "vespa.rank.secondphase"
+rankprofile[16].fef.property[3].value "rankingExpression(secondphase)"
+rankprofile[16].fef.property[4].name "rankingExpression(secondphase).rankingScript"
+rankprofile[16].fef.property[4].value "3002 * rankingExpression(m2)"
+rankprofile[17].name "macros-refering-macros-inherited-two-levels"
+rankprofile[17].fef.property[0].name "rankingExpression(m1).rankingScript"
+rankprofile[17].fef.property[0].value "700 * fieldMatch(title).completeness"
+rankprofile[17].fef.property[1].name "rankingExpression(m2).rankingScript"
+rankprofile[17].fef.property[1].value "rankingExpression(m1) * 67"
+rankprofile[17].fef.property[2].name "rankingExpression(m4).rankingScript"
+rankprofile[17].fef.property[2].value "701 * fieldMatch(title).completeness"
+rankprofile[17].fef.property[3].name "rankingExpression(m3).rankingScript"
+rankprofile[17].fef.property[3].value "if (isNan(attribute(nrtgmp)) == 1, 0.0, rankingExpression(m2))"
+rankprofile[17].fef.property[4].name "rankingExpression(m5).rankingScript"
+rankprofile[17].fef.property[4].value "if (isNan(attribute(glmpfw)) == 1, rankingExpression(m1), rankingExpression(m4))"
+rankprofile[17].fef.property[5].name "vespa.rank.secondphase"
+rankprofile[17].fef.property[5].value "rankingExpression(secondphase)"
+rankprofile[17].fef.property[6].name "rankingExpression(secondphase).rankingScript"
+rankprofile[17].fef.property[6].value "3000 * rankingExpression(m2)"
\ No newline at end of file
diff --git a/model-inference/src/test/resources/config/rankexpression/rankexpression.sd b/model-inference/src/test/resources/config/rankexpression/rankexpression.sd
new file mode 100644
index 00000000000..d3e0057cfe1
--- /dev/null
+++ b/model-inference/src/test/resources/config/rankexpression/rankexpression.sd
@@ -0,0 +1,327 @@
+# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+search rankexpression {
+  
+  document rankexpression {
+
+    field artist type string {
+      indexing: summary | index
+    }
+
+    field title type string {
+      indexing: summary | index
+    }
+
+    field surl type string {
+      indexing: summary
+    }
+
+    field year type int {
+      indexing: summary | attribute
+    }
+
+    field foo1 type int {
+      indexing: attribute
+    }
+
+    field foo2 type int {
+      indexing: attribute
+    }
+
+    field foo3 type int {
+      indexing: attribute
+    }
+
+    field foo4 type int {
+      indexing: attribute
+    }
+
+    field bar1 type int {
+      indexing: attribute
+    }
+
+    field bar2 type int {
+      indexing: attribute
+    }
+
+    field bar3 type int {
+      indexing: attribute
+    }
+
+    field bar4 type int {
+      indexing: attribute
+    }
+
+  }
+
+  rank-profile default {
+      first-phase {
+        expression: classicRank
+        keep-rank-count: 20
+        rank-score-drop-limit: -0.5
+      }
+      second-phase {
+        expression: if(3>2,4,2)
+        rerank-count: 10
+      }
+      rank-features: attribute(foo1).out attribute(bar1)
+      rank-features { attribute(foo2).out attribute(bar2).out }
+      rank-features {
+        attribute(foo3).out attribute(bar3).out }
+      rank-features {
+        attribute(foo4).out
+        attribute(bar4).out 
+      }
+      ignore-default-rank-features
+
+      rank-properties {
+          foo: "bar, baz"
+          qux: "quux"
+          foo: "foobar"
+          foo.bar: "foo.bar"
+          foo.bar.baz: 123
+          foo ( bar ) . baz.2  : 123.4
+          foo(bar).baz.qux: "foo(bar)"
+          "nud":"ity"
+      }
+
+  }
+
+  rank-profile static {
+      first-phase {
+        expression { attribute }
+      }
+      second-phase {
+        expression {
+            file:rankexpression
+        }
+      }
+      summary-features: attribute(foo1).out attribute(bar1)
+      summary-features { attribute(foo2).out attribute(bar2).out }
+      summary-features {
+        attribute(foo3).out attribute(bar3).out }
+      summary-features {
+        attribute(foo4).out
+        attribute(bar4).out
+      }
+  }
+
+  rank-profile overflow {
+      first-phase {
+        expression: file:overflow.expression
+        keep-rank-count: 201
+        rank-score-drop-limit: 501.5
+      }
+      second-phase {
+        expression {
+            exp(0) + 
+            mysum(attribute(foo),
+                "attribute( bar )",
+                "attribute( \"baz\" )") 
+        }
+        rerank-count: 101
+      }
+  }
+
+  rank-profile duplicates {
+    rank-properties {
+      fieldMatch(a).proximityLimit: 4
+      fieldMatch(a).proximityTable: 0.2
+      fieldMatch(a).proximityTable: 0.4
+      fieldMatch(a).proximityTable: 0.6
+      fieldMatch(a).proximityTable: 0.8
+      fieldMatch(a).proximityTable: 1
+      fieldMatch(a).proximityTable: 0.8
+      fieldMatch(a).proximityTable: 0.6
+      fieldMatch(a).proximityTable: 0.4
+      fieldMatch(a).proximityTable: 0.2
+    }
+  }
+
+  rank-profile whitespace1 {
+    first-phase {
+      expression
+        	{
+
+        1
+        }}}
+
+  rank-profile whitespace2 {
+    first-phase
+    {
+      expression		{ 1 }
+    }
+  }
+
+  rank-profile macros {
+    first-phase {
+      expression: match + fieldMatch(title) + myfeature
+    }
+    second-phase {
+      expression: fourtimessum(match,rankBoost)
+    }
+    macro fourtimessum(var1, var2) {
+      expression: 4*(var1+var2)
+    }
+    macro myfeature() {
+       expression {
+         70 * fieldMatch(title).completeness * pow(0 - fieldMatch(title).earliness, 2) +
+         30 * pow(0 - fieldMatch(description).earliness, 2)
+       }
+    }
+    summary-features { 
+      fieldMatch(title)
+    }
+  }
+
+  rank-profile macros2 {
+    first-phase {
+      expression: classicRank
+    }
+    rank-properties {
+      foo: "some, list"
+    }
+    
+    second-phase {
+      expression: fourtimessum(match,match) + mysummaryfeature + myfeature
+    }
+    macro fourtimessum(var1, var2) {
+      expression: 4*(var1+var2)
+    }
+    macro myfeature() {
+       expression {
+         70 * fieldMatch(title).completeness * pow(0 - fieldMatch(title).earliness, 2) +
+         30 * pow(0 - fieldMatch(description).earliness, 2)
+       }
+    }
+    macro mysummaryfeature() {
+       expression {
+         70 * fieldMatch(title).completeness
+       }
+    }
+    macro mysummaryfeature2() {
+       expression {
+         71 * fieldMatch(title).completeness
+       }
+    }    
+    summary-features { 
+      mysummaryfeature
+      rankingExpression(mysummaryfeature2) # Required form earlier
+    }
+  }
+
+  rank-profile macros3 {
+    macro onlyusedinsummaryfeature() {
+      expression: 5
+    }
+    summary-features {
+      rankingExpression(matches(title,rankingExpression(onlyusedinsummaryfeature)))
+    }
+      
+  }
+
+  rank-profile macros3-inherited inherits macros3 {
+    summary-features {
+      rankingExpression(matches(title,rankingExpression(onlyusedinsummaryfeature)))
+    }
+  }
+
+  rank-profile macros-inherited inherits macros2 {
+    macro mysummaryfeature() {
+       expression {
+         80 * fieldMatch(title).completeness
+       }
+    }
+    first-phase {
+      expression {
+        20000 * myfeature + mysummaryfeature
+      }
+    }
+  }
+  
+  rank-profile macros-inherited2 inherits macros-inherited {
+    first-phase {
+      expression {
+        30000 * mysummaryfeature + myfeature
+      }
+    }
+  }
+
+ rank-profile macros-inherited3 inherits macros-inherited2 {
+    macro myfeature() {
+       expression {
+         700 * fieldMatch(title).completeness
+       }
+    }
+    second-phase {
+       expression {
+         40000 * mysummaryfeature + myfeature
+       }
+    }
+  }
+
+  rank-profile macros-refering-macros {
+    macro m2() {
+      expression: m1 * 67
+    }    
+
+    macro m1() {
+      expression {
+         700 * fieldMatch(title).completeness
+       }
+    }
+
+    macro m4() {
+      expression: file:macro.expression
+    }
+    
+    second-phase {
+      expression {
+        40000 * m2
+      }
+    }
+    
+  }
+
+  rank-profile macros-refering-macros-inherited inherits macros-refering-macros {
+    macro m3() {
+      expression {
+        if(isNan(attribute(nrtgmp))==1,
+          0.0,
+          (m2)
+        )
+      }
+    }
+    macro m4() {
+      expression {
+         701 * fieldMatch(title).completeness
+       }
+    }
+    second-phase {
+      expression {
+        3000 * m2
+      }
+    }
+  }
+
+  rank-profile macros-refering-macros-inherited2 inherits macros-refering-macros {
+    second-phase {
+      expression {
+        3002 * m2
+      }
+    }
+  }
+
+  rank-profile macros-refering-macros-inherited-two-levels inherits macros-refering-macros-inherited {
+    macro m5() {
+      expression {
+        if(isNan(attribute(glmpfw))==1,
+          m1,
+          (m4)
+        )
+      }          
+    }
+  }
+
+}
+
+
diff --git a/node-admin/src/main/application/services.xml b/node-admin/src/main/application/services.xml
index 96fe82a5b94..284b356d2ca 100644
--- a/node-admin/src/main/application/services.xml
+++ b/node-admin/src/main/application/services.xml
@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="utf-8" ?>
 <!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
 <services version="1.0" xmlns:preprocess="properties">
-  <jdisc id="node-admin" jetty="true" version="1.0">
+  <container id="node-admin" version="1.0">
     <!-- Please update container test when changing this file -->
     <accesslog type="vespa" fileNamePattern="logs/vespa/node-admin/access.log.%Y%m%d%H%M%S" rotationScheme="date" symlinkName="access.log" />
     <component id="docker-api" class="com.yahoo.vespa.hosted.dockerapi.DockerImpl" bundle="docker-api"/>
@@ -13,5 +13,5 @@
     </config>
 
     <preprocess:include file="variant.xml" required="false"/>
-  </jdisc>
+  </container>
 </services>
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/ConfigServerInfo.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/ConfigServerInfo.java
index 93243f8b8ed..9e94f6ed7e4 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/ConfigServerInfo.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/ConfigServerInfo.java
@@ -1,20 +1,14 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.node.admin.component;
 
-import com.google.common.base.Strings;
-import com.yahoo.vespa.athenz.api.AthenzIdentity;
 import com.yahoo.vespa.athenz.api.AthenzService;
 import com.yahoo.vespa.athenz.utils.AthenzIdentities;
 import com.yahoo.vespa.hosted.node.admin.config.ConfigServerConfig;
-import com.yahoo.vespa.hosted.node.admin.util.KeyStoreOptions;
 
 import java.net.URI;
-import java.nio.file.Paths;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
-import java.util.Optional;
 import java.util.function.Function;
 
 import static java.util.stream.Collectors.toMap;
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/IdempotentTask.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/IdempotentTask.java
index 71e55c36284..5bb5d35e781 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/IdempotentTask.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/IdempotentTask.java
@@ -40,22 +40,4 @@ public interface IdempotentTask<T extends TaskContext> {
      * @throws RuntimeException (or a subclass) if the task is unable to converge.
      */
     boolean converge(T context);
-
-    /**
-     * <p>Converge the task towards some state where it can be suspended. The
-     * TaskContext should provide enough to determine what kind of suspend is wanted, e.g.
-     * suspension of only the task, or the task and the resources/processes it manages.</p>
-     *
-     * <p>convergeSuspend() must be idempotent: it may be called any number of times, or
-     * interrupted at any time e.g. by `kill -9`.</p>
-     *
-     * <p>convergeSuspend() is not thread safe: The caller must ensure there is at most one
-     * invocation of convergeSuspend() at any given time.</p>
-     *
-     * @return false if already converged, i.e. was a no-op
-     * @throws RuntimeException (or a subclass) if the task is unable to suspend.
-     */
-    default boolean convergeSuspend(T context) {
-        return false;
-    }
 }
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/TaskContext.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/TaskContext.java
index 0c49e478d6a..3c44186f78d 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/TaskContext.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/TaskContext.java
@@ -21,8 +21,8 @@ public interface TaskContext {
      *                to bob".
      */
     void recordSystemModification(Logger logger, String message);
-    default void recordSystemModification(Logger logger, String messageFormat, String... args) {
-        recordSystemModification(logger, String.format(messageFormat, (Object[]) args));
+    default void recordSystemModification(Logger logger, String messageFormat, Object... args) {
+        recordSystemModification(logger, String.format(messageFormat, args));
     }
 
     /**
@@ -35,8 +35,8 @@ public interface TaskContext {
      * Do not log a message that is also recorded with recordSystemModification.
      */
     default void log(Logger logger, String message) {}
-    default void log(Logger logger, String messageFormat, String... args) {
-        log(logger, String.format(messageFormat, (Object[]) args));
+    default void log(Logger logger, String messageFormat, Object... args) {
+        log(logger, String.format(messageFormat, args));
     }
 
     /**
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImpl.java
index 12ba777f018..aea44e728ad 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImpl.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImpl.java
@@ -110,41 +110,26 @@ public class ConfigServerApiImpl implements ConfigServerApi {
     private <T> T tryAllConfigServers(CreateRequest requestFactory, Class<T> wantedReturnType) {
         Exception lastException = null;
         for (URI configServer : configServers) {
-            final CloseableHttpResponse response;
-            try {
-                response = client.execute(requestFactory.createRequest(configServer));
-            } catch (Exception e) {
-                // Failure to communicate with a config server is not abnormal, as they are
-                // upgraded at the same time as Docker hosts.
-                if (e.getMessage().indexOf("(Connection refused)") > 0) {
-                    NODE_ADMIN_LOGGER.info("Connection refused to " + configServer + " (upgrading?), will try next");
-                } else {
-                    NODE_ADMIN_LOGGER.warning("Failed to communicate with " + configServer + ", will try next: " + e.getMessage());
-                }
-                lastException = e;
-                continue;
-            }
-
-            try {
-                Optional<HttpException> retryableException = HttpException.handleStatusCode(
-                        response.getStatusLine().getStatusCode(),
-                        "Config server " + configServer);
-                if (retryableException.isPresent()) {
-                    lastException = retryableException.get();
-                    continue;
-                }
+            try (CloseableHttpResponse response = client.execute(requestFactory.createRequest(configServer))) {
+                HttpException.handleStatusCode(
+                        response.getStatusLine().getStatusCode(), "Config server " + configServer);
 
                 try {
                     return mapper.readValue(response.getEntity().getContent(), wantedReturnType);
                 } catch (IOException e) {
-                    throw new RuntimeException("Response didn't contain nodes element, failed parsing?", e);
+                    throw new RuntimeException("Failed parse response from config server", e);
                 }
-            } finally {
-                try {
-                    response.close();
-                } catch (IOException e) {
-                    NODE_ADMIN_LOGGER.warning("Ignoring exception from closing response", e);
+            } catch (HttpException e) {
+                if (!e.isRetryable()) throw e;
+                lastException = e;
+            } catch (Exception e) {
+                // Failure to communicate with a config server is not abnormal during upgrades
+                if (e.getMessage().contains("(Connection refused)")) {
+                    NODE_ADMIN_LOGGER.info("Connection refused to " + configServer + " (upgrading?), will try next");
+                } else {
+                    NODE_ADMIN_LOGGER.warning("Failed to communicate with " + configServer + ", will try next: " + e.getMessage());
                 }
+                lastException = e;
             }
         }
 
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/HttpException.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/HttpException.java
index d0f436d16b6..256fe38ec68 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/HttpException.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/HttpException.java
@@ -2,7 +2,6 @@
 package com.yahoo.vespa.hosted.node.admin.configserver;
 
 import javax.ws.rs.core.Response;
-import java.util.Optional;
 
 /**
  * @author hakonhall
@@ -10,35 +9,34 @@ import java.util.Optional;
 @SuppressWarnings("serial")
 public class HttpException extends RuntimeException {
 
-    public static class NotFoundException extends HttpException {
-
-        public NotFoundException(String message) {
-            super(Response.Status.NOT_FOUND, message);
-        }
+    private final boolean isRetryable;
 
+    private HttpException(int statusCode, String message, boolean isRetryable) {
+        super("HTTP status code " + statusCode + ": " + message);
+        this.isRetryable = isRetryable;
     }
 
-    public static class ForbiddenException extends HttpException {
-
-        public ForbiddenException(String message) {
-            super(Response.Status.FORBIDDEN, message);
-        }
+    private HttpException(Response.Status status, String message, boolean isRetryable) {
+        super(status.toString() + " (" + status.getStatusCode() + "): " + message);
+        this.isRetryable = isRetryable;
+    }
 
+    public boolean isRetryable() {
+        return isRetryable;
     }
 
     /**
-     * Returns empty on success.
-     * Returns an exception if the error is retriable.
-     * Throws an exception on a non-retriable error, like 404 Not Found.
+     * Returns on success.
+     * @throws HttpException for all non-expected status codes.
      */
-    static Optional<HttpException> handleStatusCode(int statusCode, String message) {
+    static void handleStatusCode(int statusCode, String message) {
         Response.Status status = Response.Status.fromStatusCode(statusCode);
         if (status == null) {
-            return Optional.of(new HttpException(statusCode, message));
+            throw new HttpException(statusCode, message, true);
         }
 
         switch (status.getFamily()) {
-            case SUCCESSFUL: return Optional.empty();
+            case SUCCESSFUL: return;
             case CLIENT_ERROR:
                 switch (status) {
                     case FORBIDDEN:
@@ -48,20 +46,24 @@ public class HttpException extends RuntimeException {
                     case CONFLICT:
                         // A response body is assumed to be present, and
                         // will later be interpreted as an error.
-                        return Optional.empty();
+                        return;
                 }
-                throw new HttpException(statusCode, message);
+                throw new HttpException(status, message, false);
         }
 
         // Other errors like server-side errors are assumed to be retryable.
-        return Optional.of(new HttpException(status, message));
+        throw new HttpException(status, message, true);
     }
 
-    private HttpException(int statusCode, String message) {
-        super("HTTP status code " + statusCode + ": " + message);
+    public static class NotFoundException extends HttpException {
+        public NotFoundException(String message) {
+            super(Response.Status.NOT_FOUND, message, false);
+        }
     }
 
-    private HttpException(Response.Status status, String message) {
-        super(status.toString() + " (" + status.getStatusCode() + "): " + message);
+    public static class ForbiddenException extends HttpException {
+        public ForbiddenException(String message) {
+            super(Response.Status.FORBIDDEN, message, false);
+        }
     }
 }
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java
index a7bf22591d4..383c025e2cb 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java
@@ -89,6 +89,13 @@ public class StorageMaintainer {
                 "129600", "--crit", "1", "--coredir", environment.pathInNodeUnderVespaHome("var/crash/processing").toString());
         configs.add(annotatedCheck(node, coredumpSchedule));
 
+        // athenz certificate check
+        Path athenzCertExpiryCheckPath = environment.pathInNodeUnderVespaHome("libexec64/yms/yms_check_athenz_certs");
+        SecretAgentCheckConfig athenzCertExpirySchedule = new SecretAgentCheckConfig("athenz-certificate-expiry", 60,
+                 athenzCertExpiryCheckPath, "--threshold", "20")
+                .withRunAsUser("root");
+        configs.add(annotatedCheck(node, athenzCertExpirySchedule));
+
         if (node.getNodeType() != NodeType.config) {
             // vespa-health
             Path vespaHealthCheckPath = environment.pathInNodeUnderVespaHome("libexec/yms/yms_check_vespa_health");
@@ -200,6 +207,7 @@ public class StorageMaintainer {
         Process duCommand = new ProcessBuilder().command(command).start();
         if (!duCommand.waitFor(60, TimeUnit.SECONDS)) {
             duCommand.destroy();
+            duCommand.waitFor();
             throw new RuntimeException("Disk usage command timed out, aborting.");
         }
         String output = IOUtils.readAll(new InputStreamReader(duCommand.getInputStream()));
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/StoredInteger.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/StoredInteger.java
new file mode 100644
index 00000000000..a815515ac83
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/StoredInteger.java
@@ -0,0 +1,58 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.file;
+
+import com.yahoo.vespa.hosted.node.admin.component.TaskContext;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.file.Files;
+import java.nio.file.NoSuchFileException;
+import java.nio.file.Path;
+import java.util.OptionalInt;
+import java.util.function.Supplier;
+import java.util.logging.Logger;
+
+/**
+ * Class wrapping an integer stored on disk
+ *
+ * @author freva
+ */
+public class StoredInteger implements Supplier<OptionalInt> {
+
+    private static final Logger logger = Logger.getLogger(StoredInteger.class.getName());
+
+    private final Path path;
+    private OptionalInt value;
+    private boolean hasBeenRead = false;
+
+    public StoredInteger(Path path) {
+        this.path = path;
+    }
+
+    @Override
+    public OptionalInt get() {
+        if (!hasBeenRead) {
+            try {
+                String value = new String(Files.readAllBytes(path));
+                this.value = OptionalInt.of(Integer.valueOf(value));
+            } catch (NoSuchFileException e) {
+                this.value = OptionalInt.empty();
+            } catch (IOException e) {
+                throw new UncheckedIOException("Failed to read integer in " + path, e);
+            }
+            hasBeenRead = true;
+        }
+        return value;
+    }
+
+    public void write(TaskContext taskContext, int value) {
+        try {
+            Files.write(path, Integer.toString(value).getBytes());
+            this.value = OptionalInt.of(value);
+            this.hasBeenRead = true;
+            taskContext.log(logger, "Stored new integer in %s: %d", path, value);
+        } catch (IOException e) {
+            throw new UncheckedIOException("Failed to store integer in " + path, e);
+        }
+    }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcessException.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcessException.java
index 9f7aaab2060..cbc8ffbf1b7 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcessException.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcessException.java
@@ -65,7 +65,7 @@ public abstract class ChildProcessException extends RuntimeException {
         if (possiblyHugeOutput.length() <= maxOutputPrefix + maxOutputSuffix + maxOutputSlack) {
             stringBuilder.append(possiblyHugeOutput);
         } else {
-            stringBuilder.append(possiblyHugeOutput.substring(0, maxOutputPrefix))
+            stringBuilder.append(possiblyHugeOutput, 0, maxOutputPrefix)
                     .append("... [")
                     .append(possiblyHugeOutput.length() - maxOutputPrefix - maxOutputSuffix)
                     .append(" chars omitted] ...")
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/Yum.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/Yum.java
index d88c6f4ab33..5d60823d1c5 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/Yum.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/Yum.java
@@ -36,6 +36,10 @@ public class Yum {
         return newYumCommand("install", packages, INSTALL_NOOP_PATTERN);
     }
 
+    /**
+     * @param packages A list of packages, each package being of the form name-1.2.3-1.el7.noarch,
+     *                 if no packages are given, will upgrade all installed packages
+     */
     public GenericYumCommand upgrade(String... packages) {
         return newYumCommand("upgrade", packages, UPGRADE_NOOP_PATTERN);
     }
@@ -70,7 +74,7 @@ public class Yum {
             this.packages = packages;
             this.commandOutputNoopPattern = commandOutputNoopPattern;
 
-            if (packages.isEmpty()) {
+            if (packages.isEmpty() && ! "upgrade".equals(yumCommand)) {
                 throw new IllegalArgumentException("No packages specified");
             }
         }
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RebootTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RebootTest.java
index db14efdd5d2..a1af36f9c21 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RebootTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RebootTest.java
@@ -11,10 +11,6 @@ import com.yahoo.vespa.hosted.provision.Node;
 import org.junit.Ignore;
 import org.junit.Test;
 
-import java.util.Optional;
-
-import static org.hamcrest.core.Is.is;
-import static org.hamcrest.junit.MatcherAssert.assertThat;
 import static org.junit.Assert.assertTrue;
 
 /**
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java
index 91c61623ee7..c348dc4c8b5 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java
@@ -25,6 +25,7 @@ import java.util.stream.Collectors;
 
 import static java.util.Arrays.asList;
 import static org.hamcrest.CoreMatchers.is;
+import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertThat;
 import static org.junit.Assert.assertTrue;
@@ -152,7 +153,7 @@ public class NodeAdminImplTest {
         assertTrue(nodeAdmin.isFrozen());
         assertTrue(nodeAdmin.subsystemFreezeDuration().isZero());
         clock.advance(Duration.ofSeconds(1));
-        assertTrue(nodeAdmin.subsystemFreezeDuration().equals(Duration.ofSeconds(1)));
+        assertEquals(Duration.ofSeconds(1), nodeAdmin.subsystemFreezeDuration());
 
         // Unfreezing floors freeze duration
         assertTrue(nodeAdmin.setFrozen(false)); // Unfreeze everything
@@ -164,7 +165,7 @@ public class NodeAdminImplTest {
         assertTrue(nodeAdmin.setFrozen(true));
         assertTrue(nodeAdmin.subsystemFreezeDuration().isZero());
         clock.advance(Duration.ofSeconds(1));
-        assertTrue(nodeAdmin.subsystemFreezeDuration().equals(Duration.ofSeconds(1)));
+        assertEquals(Duration.ofSeconds(1), nodeAdmin.subsystemFreezeDuration());
     }
 
     @Test
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/MakeDirectoryTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/MakeDirectoryTest.java
index a83f3bbe7d4..b714ab539f6 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/MakeDirectoryTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/MakeDirectoryTest.java
@@ -25,7 +25,7 @@ public class MakeDirectoryTest {
     private final FileSystem fileSystem = TestFileSystem.create();
     private final TestTaskContext context = new TestTaskContext();
 
-    private String path = "/parent/dir";
+    private final String path = "/parent/dir";
     private String permissions = "rwxr----x";
     private String owner = "test-owner";
     private String group = "test-group";
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/CommandLineTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/CommandLineTest.java
index 5bc45d7540e..a5eb0ab059b 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/CommandLineTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/process/CommandLineTest.java
@@ -104,7 +104,6 @@ public class CommandLineTest {
 
     @Test
     public void programFails() {
-        TestChildProcess2 child = new TestChildProcess2(0, "");
         terminal.expectCommand("foo 2>&1", 1, "");
         try {
             commandLine.add("foo").execute();
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTest.java
index d29d8741438..7f37336db70 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTest.java
@@ -9,6 +9,7 @@ import org.junit.Test;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 import static org.mockito.Mockito.mock;
@@ -16,6 +17,7 @@ import static org.mockito.Mockito.mock;
 public class YumTest {
     private final TaskContext taskContext = mock(TaskContext.class);
     private final TestTerminal terminal = new TestTerminal();
+    private final Yum yum = new Yum(terminal);
 
     @Before
     public void tearDown() {
@@ -29,7 +31,6 @@ public class YumTest {
                 0,
                 "foobar\nNothing to do\n");
 
-        Yum yum = new Yum(terminal);
         assertFalse(yum
                 .install("package-1", "package-2")
                 .enableRepo("repo-name")
@@ -43,7 +44,7 @@ public class YumTest {
                 0,
                 "foobar\nNo packages marked for update\n");
 
-        assertFalse(new Yum(terminal)
+        assertFalse(yum
                 .upgrade("package-1", "package-2")
                 .converge(taskContext));
     }
@@ -55,7 +56,7 @@ public class YumTest {
                 0,
                 "foobar\nNo Packages marked for removal\n");
 
-        assertFalse(new Yum(terminal)
+        assertFalse(yum
                 .remove("package-1", "package-2")
                 .converge(taskContext));
     }
@@ -67,7 +68,6 @@ public class YumTest {
                 0,
                 "installing, installing");
 
-        Yum yum = new Yum(terminal);
         assertTrue(yum
                 .install("package-1", "package-2")
                 .converge(taskContext));
@@ -80,7 +80,6 @@ public class YumTest {
                 0,
                 "installing, installing");
 
-        Yum yum = new Yum(terminal);
         assertTrue(yum
                 .install("package-1", "package-2")
                 .enableRepo("repo-name")
@@ -94,7 +93,6 @@ public class YumTest {
                 1,
                 "error");
 
-        Yum yum = new Yum(terminal);
         yum.install("package-1", "package-2")
                 .enableRepo("repo-name")
                 .converge(taskContext);
@@ -112,15 +110,26 @@ public class YumTest {
                         "No package package-2 available.\n" +
                         "Nothing to do\n");
 
-        Yum yum = new Yum(terminal);
         Yum.GenericYumCommand install = yum.install("package-1", "package-2", "package-3");
 
         try {
             install.converge(taskContext);
             fail();
         } catch (Exception e) {
-            assertTrue(e.getCause() != null);
+            assertNotNull(e.getCause());
             assertEquals("Unknown package: package-1", e.getCause().getMessage());
         }
     }
+
+    @Test(expected = IllegalArgumentException.class)
+    public void throwIfNoPackagesSpecified() {
+        yum.install();
+    }
+
+    @Test
+    public void allowToCallUpgradeWithNoPackages() {
+        terminal.expectCommand("yum upgrade --assumeyes 2>&1", 0, "OK");
+
+        yum.upgrade().converge(taskContext);
+    }
 }
\ No newline at end of file
diff --git a/node-maintainer/pom.xml b/node-maintainer/pom.xml
index 743e92f7f7d..f4e1b399cfa 100644
--- a/node-maintainer/pom.xml
+++ b/node-maintainer/pom.xml
@@ -71,7 +71,6 @@
         <dependency>
             <groupId>com.google.guava</groupId>
             <artifactId>guava</artifactId>
-            <version>18.0</version>
         </dependency>
     </dependencies>
 
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java
index 42dfb38e4d3..e3df6199d06 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java
@@ -5,6 +5,7 @@ import com.yahoo.concurrent.DaemonThreadFactory;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.Deployer;
 import com.yahoo.config.provision.Deployment;
+import com.yahoo.log.LogLevel;
 import com.yahoo.transaction.Mutex;
 import com.yahoo.vespa.hosted.provision.Node;
 import com.yahoo.vespa.hosted.provision.NodeRepository;
@@ -15,9 +16,10 @@ import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Optional;
 import java.util.Set;
-import java.util.concurrent.Executor;
-import java.util.concurrent.Executors;
-import java.util.logging.Level;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 
 /**
@@ -27,8 +29,15 @@ import java.util.stream.Collectors;
 public abstract class ApplicationMaintainer extends Maintainer {
 
     private final Deployer deployer;
+    private final List<ApplicationId> pendingDeployments = new CopyOnWriteArrayList<>();
 
-    private final Executor deploymentExecutor = Executors.newCachedThreadPool(new DaemonThreadFactory("node repo application maintainer"));
+    // Use a fixed thread pool to avoid overload on config servers. Resource usage when deploying varies
+    // a lot between applications, so doing one by one avoids issues where one or more resource-demanding
+    // deployments happen simultaneously
+    private final ThreadPoolExecutor deploymentExecutor = new ThreadPoolExecutor(1, 1,
+                                                                                 0L, TimeUnit.MILLISECONDS,
+                                                                                 new LinkedBlockingQueue<>(),
+                                                                                 new DaemonThreadFactory("node repo application maintainer"));
 
     protected ApplicationMaintainer(Deployer deployer, NodeRepository nodeRepository, Duration interval, JobControl jobControl) {
         super(nodeRepository, interval, jobControl);
@@ -37,14 +46,15 @@ public abstract class ApplicationMaintainer extends Maintainer {
 
     @Override
     protected final void maintain() {
-        Set<ApplicationId> applications = applicationsNeedingMaintenance();
-        for (ApplicationId application : applications) {
-            if (canDeployNow(application))
-                deploy(application);
-            throttle(applications.size());
-        }
+        applicationsNeedingMaintenance().forEach(this::deploy);
+    }
+
+    /** Returns the number of deployments that are pending execution */
+    public int pendingDeployments() {
+        return pendingDeployments.size();
     }
 
+    /** Returns whether given application should be deployed at this moment in time */
     protected boolean canDeployNow(ApplicationId application) {
         return true;
     }
@@ -56,18 +66,21 @@ public abstract class ApplicationMaintainer extends Maintainer {
      * even when deployments are slow.
      */
     protected void deploy(ApplicationId application) {
+        if (pendingDeployments.contains(application)) {
+            return;// Avoid queuing multiple deployments for same application
+        }
+        log.log(LogLevel.INFO, application + " will be deployed, last deploy time " +
+                               getLastDeployTime(application));
+        pendingDeployments.add(application);
         deploymentExecutor.execute(() -> deployWithLock(application));
     }
 
     protected Deployer deployer() { return deployer; }
 
-    /** Block in this method until the next application should be maintained */
-    protected abstract void throttle(int applicationCount);
-
-    private Set<ApplicationId> applicationsNeedingMaintenance() {
+    protected Set<ApplicationId> applicationsNeedingMaintenance() {
         return nodesNeedingMaintenance().stream()
-                .map(node -> node.allocation().get().owner())
-                .collect(Collectors.toCollection(LinkedHashSet::new));
+                                        .map(node -> node.allocation().get().owner())
+                                        .collect(Collectors.toCollection(LinkedHashSet::new));
     }
 
     /**
@@ -77,24 +90,45 @@ public abstract class ApplicationMaintainer extends Maintainer {
     protected abstract List<Node> nodesNeedingMaintenance();
 
     /** Redeploy this application. A lock will be taken for the duration of the deployment activation */
-    final void deployWithLock(ApplicationId application) {
-        // An application might change it's state between the time the set of applications is retrieved and the
+    protected final void deployWithLock(ApplicationId application) {
+        // An application might change its state between the time the set of applications is retrieved and the
         // time deployment happens. Lock the application and check if it's still active.
         //
         // Lock is acquired with a low timeout to reduce the chance of colliding with an external deployment.
         try (Mutex lock = nodeRepository().lock(application, Duration.ofSeconds(1))) {
             if ( ! isActive(application)) return; // became inactive since deployment was requested
+            if ( ! canDeployNow(application)) return; // redeployment is no longer needed
             Optional<Deployment> deployment = deployer.deployFromLocalActive(application);
             if ( ! deployment.isPresent()) return; // this will be done at another config server
+            log.log(LogLevel.DEBUG, this.getClass().getSimpleName() + " deploying " + application);
             deployment.get().activate();
         } catch (RuntimeException e) {
-            log.log(Level.WARNING, "Exception on maintenance redeploy", e);
+            log.log(LogLevel.WARNING, "Exception on maintenance redeploy", e);
+        } finally {
+            pendingDeployments.remove(application);
         }
     }
 
+    /** Returns the last time application was deployed. Epoch is returned if the application has never been deployed. */
+    protected final Instant getLastDeployTime(ApplicationId application) {
+        return deployer.lastDeployTime(application).orElse(Instant.EPOCH);
+    }
+
     /** Returns true when application has at least one active node */
     private boolean isActive(ApplicationId application) {
         return ! nodeRepository().getNodes(application, Node.State.active).isEmpty();
     }
 
+    @Override
+    public void deconstruct() {
+        super.deconstruct();
+        this.deploymentExecutor.shutdownNow();
+        try {
+            // Give deployments in progress some time to complete
+            this.deploymentExecutor.awaitTermination(1, TimeUnit.MINUTES);
+        } catch (InterruptedException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
 }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRebooter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRebooter.java
index cc4834d97a6..6b538946e09 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRebooter.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRebooter.java
@@ -35,7 +35,7 @@ public class NodeRebooter extends Maintainer {
 
     @Override
     protected void maintain() {
-        // Reboot candidates: Nodes in long-term states, which we know an safely orchestrate a reboot
+        // Reboot candidates: Nodes in long-term states, which we know can safely orchestrate a reboot
         List<Node> rebootCandidates = nodeRepository().getNodes(NodeType.tenant, Node.State.active, Node.State.ready);
         rebootCandidates.addAll(nodeRepository().getNodes(NodeType.proxy, Node.State.active, Node.State.ready));
         rebootCandidates.addAll(nodeRepository().getNodes(NodeType.host, Node.State.active, Node.State.ready));
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java
index 391807ece95..6985206b78e 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java
@@ -71,7 +71,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent {
         infrastructureVersions = new InfrastructureVersions(nodeRepository.database());
 
         nodeFailer = new NodeFailer(deployer, hostLivenessTracker, serviceMonitor, nodeRepository, durationFromEnv("fail_grace").orElse(defaults.failGrace), clock, orchestrator, throttlePolicyFromEnv("throttle_policy").orElse(defaults.throttlePolicy), metric, jobControl, configserverConfig);
-        periodicApplicationMaintainer = new PeriodicApplicationMaintainer(deployer, nodeRepository, durationFromEnv("periodic_redeploy_interval").orElse(defaults.periodicRedeployInterval), jobControl);
+        periodicApplicationMaintainer = new PeriodicApplicationMaintainer(deployer, nodeRepository, defaults.redeployMaintainerInterval, durationFromEnv("periodic_redeploy_interval").orElse(defaults.periodicRedeployInterval), jobControl);
         operatorChangeApplicationMaintainer = new OperatorChangeApplicationMaintainer(deployer, nodeRepository, clock, durationFromEnv("operator_change_redeploy_interval").orElse(defaults.operatorChangeRedeployInterval), jobControl);
         reservationExpirer = new ReservationExpirer(nodeRepository, clock, durationFromEnv("reservation_expiry").orElse(defaults.reservationExpiry), jobControl);
         retiredExpirer = new RetiredExpirer(nodeRepository, orchestrator, deployer, clock, durationFromEnv("retired_interval").orElse(defaults.retiredInterval), durationFromEnv("retired_expiry").orElse(defaults.retiredExpiry), jobControl);
@@ -130,8 +130,11 @@ public class NodeRepositoryMaintenance extends AbstractComponent {
 
     private static class DefaultTimes {
 
-        /** All applications are redeployed with this period */
+        // TODO: Rename, kept now for compatibility reasons, want to change this and corresponding env variable
+        /** Minimum time to wait between deployments by periodic application maintainer*/
         private final Duration periodicRedeployInterval;
+        /** Time between each run of maintainer that does periodic redeployment */
+        private final Duration redeployMaintainerInterval;
         /** Applications are redeployed after manual operator changes within this time period */
         private final Duration operatorChangeRedeployInterval;
 
@@ -155,6 +158,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent {
         DefaultTimes(Zone zone) {
             failGrace = Duration.ofMinutes(60);
             periodicRedeployInterval = Duration.ofMinutes(30);
+            redeployMaintainerInterval = Duration.ofMinutes(1);
             operatorChangeRedeployInterval = Duration.ofMinutes(1);
             failedExpirerInterval = Duration.ofMinutes(10);
             provisionedExpiry = Duration.ofHours(4);
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainer.java
index 57dee7b0dbc..45db9ea90d9 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainer.java
@@ -53,9 +53,6 @@ public class OperatorChangeApplicationMaintainer extends ApplicationMaintainer {
                 .anyMatch(event -> event.agent() == Agent.operator && event.at().isAfter(instant));
     }
 
-    @Override
-    protected void throttle(int applicationCount) { }
-
     /** 
      * Deploy in the maintenance thread to avoid scheduling multiple deployments of the same application if it takes
      * longer to deploy than the (short) maintenance interval of this
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainer.java
index c0e2d94ebea..8b2d0a55cd8 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainer.java
@@ -6,10 +6,15 @@ import com.yahoo.config.provision.Deployer;
 import com.yahoo.vespa.hosted.provision.Node;
 import com.yahoo.vespa.hosted.provision.NodeRepository;
 
+import java.time.Clock;
 import java.time.Duration;
 import java.time.Instant;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.LinkedHashSet;
 import java.util.List;
-import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
 
 /**
  * The application maintainer regularly redeploys all applications to make sure the node repo and application
@@ -20,24 +25,46 @@ import java.util.Optional;
  */
 public class PeriodicApplicationMaintainer extends ApplicationMaintainer {
 
+    private final Duration minTimeBetweenRedeployments;
+    private final Clock clock;
+    private final Instant start;
+
     public PeriodicApplicationMaintainer(Deployer deployer, NodeRepository nodeRepository, 
-                                         Duration interval, JobControl jobControl) {
+                                         Duration interval, Duration minTimeBetweenRedeployments, JobControl jobControl) {
         super(deployer, nodeRepository, interval, jobControl);
+        this.minTimeBetweenRedeployments = minTimeBetweenRedeployments;
+        this.clock = nodeRepository.clock();
+        this.start = clock.instant();
     }
 
     @Override
-    protected void throttle(int applicationCount) {
-        // Sleep for a length of time that will spread deployment evenly over the maintenance period
-        try { Thread.sleep(interval().toMillis() / applicationCount); } catch (InterruptedException e) { return; }
+    protected boolean canDeployNow(ApplicationId application) {
+        // Don't deploy if a regular deploy just happened
+        return getLastDeployTime(application).isBefore(nodeRepository().clock().instant().minus(minTimeBetweenRedeployments));
     }
 
+    // Returns the applications that need to be redeployed by this config server at this point in time.
     @Override
-    protected boolean canDeployNow(ApplicationId application) {
-        Optional<Instant> lastDeploy = deployer().lastDeployTime(application);
-        if (lastDeploy.isPresent() &&
-            lastDeploy.get().isAfter(nodeRepository().clock().instant().minus(interval())))
-            return false; // Don't deploy if a regular deploy just happened
-        return true;
+    protected Set<ApplicationId> applicationsNeedingMaintenance() {
+        if (waitInitially()) return Collections.emptySet();
+
+        return nodesNeedingMaintenance().stream()
+                                        .map(node -> node.allocation().get().owner())
+                                        .filter(this::shouldBeDeployedOnThisServer)
+                                        .filter(this::canDeployNow)
+                                        .sorted(Comparator.comparing(this::getLastDeployTime))
+                                        .collect(Collectors.toCollection(LinkedHashSet::new));
+    }
+
+    // We only know last deploy time for applications that were deployed on this config server,
+    // the rest will be deployed on another config server
+    protected boolean shouldBeDeployedOnThisServer(ApplicationId application) {
+        return deployer().lastDeployTime(application).isPresent();
+    }
+
+    // TODO: Do not start deploying until some time has gone (ideally only until bootstrap of config server is finished)
+    private boolean waitInitially() {
+        return clock.instant().isBefore(start.plus(minTimeBetweenRedeployments));
     }
 
     @Override
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDeployer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDeployer.java
index 99beab50e16..299dc66c547 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDeployer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDeployer.java
@@ -20,6 +20,7 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
+import java.util.concurrent.locks.ReentrantLock;
 import java.util.stream.Collectors;
 
 /**
@@ -35,6 +36,7 @@ public class MockDeployer implements Deployer {
     public int redeployments = 0;
 
     private final Clock clock;
+    private final ReentrantLock lock = new ReentrantLock();
 
     @Inject
     @SuppressWarnings("unused")
@@ -54,6 +56,10 @@ public class MockDeployer implements Deployer {
         this.applications = applications;
     }
 
+    public ReentrantLock lock() {
+        return lock;
+    }
+
     @Override
     public Optional<Deployment> deployFromLocalActive(ApplicationId id, boolean bootstrap) {
         return deployFromLocalActive(id, Duration.ofSeconds(60));
@@ -61,8 +67,17 @@ public class MockDeployer implements Deployer {
 
     @Override
     public Optional<Deployment> deployFromLocalActive(ApplicationId id, Duration timeout) {
-        lastDeployTimes.put(id, clock.instant());
-        return Optional.of(new MockDeployment(provisioner, applications.get(id)));
+        try {
+            lock.lockInterruptibly();
+        } catch (InterruptedException e) {
+            throw new RuntimeException(e);
+        }
+        try {
+            lastDeployTimes.put(id, clock.instant());
+            return Optional.of(new MockDeployment(provisioner, applications.get(id)));
+        } finally {
+            lock.unlock();
+        }
     }
 
     @Override
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainerTest.java
index 976831d129d..961e9991d71 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainerTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainerTest.java
@@ -50,7 +50,7 @@ public class OperatorChangeApplicationMaintainerTest {
     private Fixture fixture;
 
     @Test
-    public void test_application_maintenance() throws InterruptedException {
+    public void test_application_maintenance() {
         ManualClock clock = new ManualClock();
         Curator curator = new MockCurator();
         Zone zone = new Zone(Environment.prod, RegionName.from("us-east"));
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java
index 360e8de8d11..7e48edfc805 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java
@@ -29,10 +29,10 @@ import com.yahoo.vespa.hosted.provision.provisioning.FlavorConfigBuilder;
 import com.yahoo.vespa.hosted.provision.provisioning.NodeRepositoryProvisioner;
 import com.yahoo.vespa.hosted.provision.testutils.MockDeployer;
 import com.yahoo.vespa.hosted.provision.testutils.MockNameResolver;
+import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
 
-import java.time.Clock;
 import java.time.Duration;
 import java.time.Instant;
 import java.util.ArrayList;
@@ -42,6 +42,7 @@ import java.util.Map;
 import java.util.Optional;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
 
 /**
  * @author bratseth
@@ -52,26 +53,36 @@ public class PeriodicApplicationMaintainerTest {
 
     private NodeRepository nodeRepository;
     private Fixture fixture;
+    private ManualClock clock;
 
     @Before
     public void before() {
         Curator curator = new MockCurator();
         Zone zone = new Zone(Environment.prod, RegionName.from("us-east"));
-        this.nodeRepository = new NodeRepository(nodeFlavors, curator, new ManualClock(), zone,
+        this.clock = new ManualClock();
+        this.nodeRepository = new NodeRepository(nodeFlavors, curator, clock, zone,
                                                  new MockNameResolver().mockAnyLookup(),
                                                  new DockerImage("docker-registry.domain.tld:8080/dist/vespa"),
                                                  true);
         this.fixture = new Fixture(zone, nodeRepository, nodeFlavors, curator);
-    }
 
-    @Test
-    public void test_application_maintenance() {
         createReadyNodes(15, nodeRepository, nodeFlavors);
         createHostNodes(2, nodeRepository, nodeFlavors);
+    }
+
+    @After
+    public void after() {
+        this.fixture.maintainer.deconstruct();
+    }
 
+    @Test(timeout = 60_000)
+    public void test_application_maintenance() {
         // Create applications
         fixture.activate();
 
+        // Exhaust initial wait period
+        clock.advance(Duration.ofMinutes(30).plus(Duration.ofSeconds(1)));
+
         // Fail and park some nodes
         nodeRepository.fail(nodeRepository.getNodes(fixture.app1).get(3).hostname(), Agent.system, "Failing to unit test");
         nodeRepository.fail(nodeRepository.getNodes(fixture.app2).get(0).hostname(), Agent.system, "Failing to unit test");
@@ -103,7 +114,7 @@ public class PeriodicApplicationMaintainerTest {
                      0, fixture.getNodes(Node.State.active).retired().size());
 
         // Cause maintenance deployment which will update the applications with the re-activated nodes
-        ((ManualClock)nodeRepository.clock()).advance(Duration.ofMinutes(35)); // Otherwise redeploys are inhibited
+        clock.advance(Duration.ofMinutes(35)); // Otherwise redeploys are inhibited
         fixture.runApplicationMaintainer();
         assertEquals("Superflous content nodes are retired",
                      reactivatedInApp2, fixture.getNodes(Node.State.active).retired().size());
@@ -111,11 +122,8 @@ public class PeriodicApplicationMaintainerTest {
                      reactivatedInApp1, fixture.getNodes(Node.State.inactive).size());
     }
 
-    @Test
+    @Test(timeout = 60_000)
     public void deleted_application_is_not_reactivated() {
-        createReadyNodes(15, nodeRepository, nodeFlavors);
-        createHostNodes(2, nodeRepository, nodeFlavors);
-
         // Create applications
         fixture.activate();
 
@@ -127,36 +135,81 @@ public class PeriodicApplicationMaintainerTest {
         assertEquals(fixture.wantedNodesApp2, nodeRepository.getNodes(fixture.app2, Node.State.inactive).size());
 
         // Nodes belonging to app2 are inactive after maintenance
-        fixture.runApplicationMaintainer(Optional.of(frozenActiveNodes));
+        fixture.maintainer.setOverriddenNodesNeedingMaintenance(frozenActiveNodes);
+        fixture.runApplicationMaintainer();
         assertEquals("Inactive nodes were incorrectly activated after maintenance", fixture.wantedNodesApp2,
                      nodeRepository.getNodes(fixture.app2, Node.State.inactive).size());
     }
 
-    @Test
+    @Test(timeout = 60_000)
     public void application_deploy_inhibits_redeploy_for_a_while() {
-        ManualClock clock = (ManualClock)nodeRepository.clock();
-        createReadyNodes(15, nodeRepository, nodeFlavors);
-        createHostNodes(2, nodeRepository, nodeFlavors);
-
-        // Create applications
         fixture.activate();
+
+        // Holds off on deployments a while after starting
+        fixture.runApplicationMaintainer();
+        assertFalse("No deployment expected", fixture.deployer.lastDeployTime(fixture.app1).isPresent());
+        assertFalse("No deployment expected", fixture.deployer.lastDeployTime(fixture.app2).isPresent());
+        // Exhaust initial wait period
+        clock.advance(Duration.ofMinutes(30).plus(Duration.ofSeconds(1)));
+
+        // First deployment of applications
         fixture.runApplicationMaintainer();
         Instant firstDeployTime = clock.instant();
         assertEquals(firstDeployTime, fixture.deployer.lastDeployTime(fixture.app1).get());
         assertEquals(firstDeployTime, fixture.deployer.lastDeployTime(fixture.app2).get());
-        ((ManualClock) nodeRepository.clock()).advance(Duration.ofMinutes(5));
+        clock.advance(Duration.ofMinutes(5));
         fixture.runApplicationMaintainer();
-        // Too soo: Not redeployed:
+        // Too soon: Not redeployed:
         assertEquals(firstDeployTime, fixture.deployer.lastDeployTime(fixture.app1).get());
         assertEquals(firstDeployTime, fixture.deployer.lastDeployTime(fixture.app2).get());
 
-        ((ManualClock) nodeRepository.clock()).advance(Duration.ofMinutes(30));
+        clock.advance(Duration.ofMinutes(30));
         fixture.runApplicationMaintainer();
         // Redeployed:
         assertEquals(clock.instant(), fixture.deployer.lastDeployTime(fixture.app1).get());
         assertEquals(clock.instant(), fixture.deployer.lastDeployTime(fixture.app2).get());
     }
 
+    @Test(timeout = 60_000)
+    public void queues_all_eligible_applications_for_deployment() throws Exception {
+        fixture.activate();
+
+        // Exhaust initial wait period
+        clock.advance(Duration.ofMinutes(30).plus(Duration.ofSeconds(1)));
+
+        // Lock deployer to simulate slow deployments
+        fixture.deployer.lock().lockInterruptibly();
+
+        try {
+            // Queues all eligible applications
+            assertEquals(2, fixture.maintainer.applicationsNeedingMaintenance().size());
+            fixture.runApplicationMaintainer(false);
+            assertEquals(2, fixture.maintainer.pendingDeployments());
+
+            // Enough time passes to make applications eligible for another periodic deployment
+            clock.advance(Duration.ofMinutes(30).plus(Duration.ofSeconds(1)));
+            fixture.runApplicationMaintainer(false);
+
+            // Deployments are not re-queued as previous deployments are still pending
+            assertEquals(2, fixture.maintainer.pendingDeployments());
+
+            // Slow deployments complete
+            fixture.deployer.lock().unlock();
+            fixture.runApplicationMaintainer();
+            Instant deployTime = clock.instant();
+            assertEquals(deployTime, fixture.deployer.lastDeployTime(fixture.app1).get());
+            assertEquals(deployTime, fixture.deployer.lastDeployTime(fixture.app2).get());
+
+            // Too soon: Already deployed recently
+            clock.advance(Duration.ofMinutes(5));
+            assertEquals(0, fixture.maintainer.applicationsNeedingMaintenance().size());
+        } finally {
+            if (fixture.deployer.lock().isHeldByCurrentThread()) {
+                fixture.deployer.lock().unlock();
+            }
+        }
+    }
+
     private void createReadyNodes(int count, NodeRepository nodeRepository, NodeFlavors nodeFlavors) {
         List<Node> nodes = new ArrayList<>(count);
         for (int i = 0; i < count; i++)
@@ -180,7 +233,7 @@ public class PeriodicApplicationMaintainerTest {
         final NodeRepository nodeRepository;
         final NodeRepositoryProvisioner provisioner;
         final Curator curator;
-        final Deployer deployer;
+        final MockDeployer deployer;
 
         final ApplicationId app1 = ApplicationId.from(TenantName.from("foo1"), ApplicationName.from("bar"), InstanceName.from("fuz"));
         final ApplicationId app2 = ApplicationId.from(TenantName.from("foo2"), ApplicationName.from("bar"), InstanceName.from("fuz"));
@@ -189,6 +242,8 @@ public class PeriodicApplicationMaintainerTest {
         final int wantedNodesApp1 = 5;
         final int wantedNodesApp2 = 7;
 
+        private final TestablePeriodicApplicationMaintainer maintainer;
+
         Fixture(Zone zone, NodeRepository nodeRepository, NodeFlavors flavors, Curator curator) {
             this.nodeRepository = nodeRepository;
             this.curator = curator;
@@ -200,6 +255,8 @@ public class PeriodicApplicationMaintainerTest {
             apps.put(app2, new MockDeployer.ApplicationContext(app2, clusterApp2,
                                                                Capacity.fromNodeCount(wantedNodesApp2, Optional.of("default"), false, true), 1));
             this.deployer = new MockDeployer(provisioner, nodeRepository.clock(), apps);
+            this.maintainer = new TestablePeriodicApplicationMaintainer(deployer, nodeRepository, Duration.ofDays(1), // Long duration to prevent scheduled runs during test
+                                                                        Duration.ofMinutes(30));
         }
 
         void activate() {
@@ -223,11 +280,12 @@ public class PeriodicApplicationMaintainerTest {
         }
 
         void runApplicationMaintainer() {
-            runApplicationMaintainer(Optional.empty());
+            runApplicationMaintainer(true);
         }
 
-        void runApplicationMaintainer(Optional<List<Node>> overriddenNodesNeedingMaintenance) {
-            new TestablePeriodicApplicationMaintainer(deployer, nodeRepository, Duration.ofMinutes(30), overriddenNodesNeedingMaintenance).run();
+        void runApplicationMaintainer(boolean waitForDeployments) {
+            maintainer.run();
+            while (waitForDeployments && fixture.maintainer.pendingDeployments() != 0);
         }
 
         NodeList getNodes(Node.State ... states) {
@@ -236,28 +294,30 @@ public class PeriodicApplicationMaintainerTest {
 
     }
     
-    public static class TestablePeriodicApplicationMaintainer extends PeriodicApplicationMaintainer {
+    private static class TestablePeriodicApplicationMaintainer extends PeriodicApplicationMaintainer {
 
-        private Optional<List<Node>> overriddenNodesNeedingMaintenance;
-        
-        TestablePeriodicApplicationMaintainer(Deployer deployer, NodeRepository nodeRepository, Duration interval,
-                                              Optional<List<Node>> overriddenNodesNeedingMaintenance) {
-            super(deployer, nodeRepository, interval, new JobControl(nodeRepository.database()));
+        private List<Node> overriddenNodesNeedingMaintenance;
+
+        TestablePeriodicApplicationMaintainer setOverriddenNodesNeedingMaintenance(List<Node> overriddenNodesNeedingMaintenance) {
             this.overriddenNodesNeedingMaintenance = overriddenNodesNeedingMaintenance;
+            return this;
         }
 
-        @Override
-        protected void deploy(ApplicationId application) {
-            deployWithLock(application);
+        TestablePeriodicApplicationMaintainer(Deployer deployer, NodeRepository nodeRepository, Duration interval,
+                                              Duration minTimeBetweenRedeployments) {
+            super(deployer, nodeRepository, interval, minTimeBetweenRedeployments, new JobControl(nodeRepository.database()));
         }
 
-        protected void throttle(int applicationCount) { }
-
         @Override
         protected List<Node> nodesNeedingMaintenance() {
-            if (overriddenNodesNeedingMaintenance.isPresent())
-                return overriddenNodesNeedingMaintenance.get();
-            return super.nodesNeedingMaintenance();
+            return overriddenNodesNeedingMaintenance != null
+                    ? overriddenNodesNeedingMaintenance
+                    : super.nodesNeedingMaintenance();
+        }
+
+        @Override
+        protected boolean shouldBeDeployedOnThisServer(ApplicationId application) {
+            return true;
         }
 
     }
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisioningTest.java
index b6f262775d7..3be56131a05 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisioningTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisioningTest.java
@@ -68,15 +68,15 @@ public class DynamicDockerProvisioningTest {
 
         // Application 1
         ApplicationId application1 = makeApplicationId("t1", "a1");
-        ClusterSpec clusterSpec1 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent"), Version.fromString("6.100"), false);
-        addAndAssignNode(application1, "1a", dockerHosts.get(2).hostname(), flavor, 0, tester);
-        addAndAssignNode(application1, "1b", dockerHosts.get(3).hostname(), flavor, 1, tester);
+        ClusterSpec clusterSpec1 = clusterSpec("myContent.t1.a1");
+        addAndAssignNode(application1, "1a", dockerHosts.get(2).hostname(), clusterSpec1, flavor, 0, tester);
+        addAndAssignNode(application1, "1b", dockerHosts.get(3).hostname(), clusterSpec1, flavor, 1, tester);
 
         // Application 2
         ApplicationId application2 = makeApplicationId("t2", "a2");
-        ClusterSpec clusterSpec2 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent"), Version.fromString("6.100"), false);
-        addAndAssignNode(application2, "2a", dockerHosts.get(2).hostname(), flavor, 0, tester);
-        addAndAssignNode(application2, "2b", dockerHosts.get(3).hostname(), flavor, 1, tester);
+        ClusterSpec clusterSpec2 = clusterSpec("myContent.t2.a2");
+        addAndAssignNode(application2, "2a", dockerHosts.get(2).hostname(), clusterSpec2, flavor, 0, tester);
+        addAndAssignNode(application2, "2b", dockerHosts.get(3).hostname(), clusterSpec2, flavor, 1, tester);
 
         // Redeploy one of the applications
         deployapp(application1, clusterSpec1, flavor, tester, 2);
@@ -115,15 +115,15 @@ public class DynamicDockerProvisioningTest {
 
         // Application 1
         ApplicationId application1 = makeApplicationId("t1", "a1");
-        ClusterSpec clusterSpec1 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent"), Version.fromString("6.100"), false);
-        addAndAssignNode(application1, "1a", dockerHosts.get(0).hostname(), flavor, 0, tester);
-        addAndAssignNode(application1, "1b", dockerHosts.get(1).hostname(), flavor, 1, tester);
+        ClusterSpec clusterSpec1 = clusterSpec("myContent.t1.a1");
+        addAndAssignNode(application1, "1a", dockerHosts.get(0).hostname(), clusterSpec1, flavor, 0, tester);
+        addAndAssignNode(application1, "1b", dockerHosts.get(1).hostname(), clusterSpec1, flavor, 1, tester);
 
         // Application 2
         ApplicationId application2 = makeApplicationId("t2", "a2");
-        ClusterSpec clusterSpec2 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent"), Version.fromString("6.100"), false);
-        addAndAssignNode(application2, "2a", dockerHosts.get(2).hostname(), flavor, 0, tester);
-        addAndAssignNode(application2, "2b", dockerHosts.get(3).hostname(), flavor, 1, tester);
+        ClusterSpec clusterSpec2 = clusterSpec("myContent.t2.a2");
+        addAndAssignNode(application2, "2a", dockerHosts.get(2).hostname(), clusterSpec2, flavor, 0, tester);
+        addAndAssignNode(application2, "2b", dockerHosts.get(3).hostname(), clusterSpec2, flavor, 1, tester);
 
         // Redeploy both applications (to be agnostic on which hosts are picked as spares)
         deployapp(application1, clusterSpec1, flavor, tester, 2);
@@ -162,17 +162,17 @@ public class DynamicDockerProvisioningTest {
 
         // Application 1
         ApplicationId application1 = makeApplicationId("t1", "1");
-        ClusterSpec clusterSpec1 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent"), Version.fromString("6.100"), false);
+        ClusterSpec clusterSpec1 = clusterSpec("myContent.t1.a1");
         String hostParent2 = dockerHosts.get(2).hostname();
         String hostParent3 = dockerHosts.get(3).hostname();
-        addAndAssignNode(application1, "1a", hostParent2, flavorD2, 0, tester);
-        addAndAssignNode(application1, "1b", hostParent3, flavorD2, 1, tester);
+        addAndAssignNode(application1, "1a", hostParent2, clusterSpec1, flavorD2, 0, tester);
+        addAndAssignNode(application1, "1b", hostParent3, clusterSpec1, flavorD2, 1, tester);
 
         // Application 2
         ApplicationId application2 = makeApplicationId("t2", "2");
-        ClusterSpec clusterSpec2 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent"), Version.fromString("6.100"), false);
-        addAndAssignNode(application2, "2a", hostParent2, flavorD1, 0, tester);
-        addAndAssignNode(application2, "2b", hostParent3, flavorD1, 1, tester);
+        ClusterSpec clusterSpec2 = clusterSpec("myContent.t2.a2");
+        addAndAssignNode(application2, "2a", hostParent2, clusterSpec2, flavorD1, 0, tester);
+        addAndAssignNode(application2, "2b", hostParent3, clusterSpec2, flavorD1, 1, tester);
 
         // Assert allocation placement - prior to re-deployment
         assertApplicationHosts(tester.nodeRepository().getNodes(application1), hostParent2, hostParent3);
@@ -217,17 +217,17 @@ public class DynamicDockerProvisioningTest {
 
         // Application 1
         ApplicationId application1 = makeApplicationId("t1", "1");
-        ClusterSpec clusterSpec1 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent"), Version.fromString("6.100"), false);
+        ClusterSpec clusterSpec1 = clusterSpec("myContent.t1.a1");
         String hostParent2 = dockerHosts.get(2).hostname();
         String hostParent3 = dockerHosts.get(3).hostname();
-        addAndAssignNode(application1, "1a", hostParent2, flavorD2, 0, tester);
-        addAndAssignNode(application1, "1b", hostParent3, flavorD2, 1, tester);
+        addAndAssignNode(application1, "1a", hostParent2, clusterSpec1, flavorD2, 0, tester);
+        addAndAssignNode(application1, "1b", hostParent3, clusterSpec1, flavorD2, 1, tester);
 
         // Application 2
         ApplicationId application2 = makeApplicationId("t2", "2");
-        ClusterSpec clusterSpec2 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent"), Version.fromString("6.100"), false);
-        addAndAssignNode(application2, "2a", hostParent2, flavorD1, 0, tester);
-        addAndAssignNode(application2, "2b", hostParent3, flavorD1, 1, tester);
+        ClusterSpec clusterSpec2 = clusterSpec("myContent.t2.a2");
+        addAndAssignNode(application2, "2a", hostParent2, clusterSpec2, flavorD1, 0, tester);
+        addAndAssignNode(application2, "2b", hostParent3, clusterSpec2, flavorD1, 1, tester);
 
         // Assert allocation placement - prior to re-deployment
         assertApplicationHosts(tester.nodeRepository().getNodes(application1), hostParent2, hostParent3);
@@ -275,7 +275,7 @@ public class DynamicDockerProvisioningTest {
      * - Fail host and check redistribution
      */
     @Test
-    public void reloacte_failed_nodes() {
+    public void relocate_failed_nodes() {
         ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.prod, RegionName.from("us-east")), flavorsConfig());
         tester.makeReadyNodes(5, "host-small", NodeType.host, 32);
         deployZoneApp(tester);
@@ -284,20 +284,20 @@ public class DynamicDockerProvisioningTest {
 
         // Application 1
         ApplicationId application1 = makeApplicationId("t1", "a1");
-        ClusterSpec clusterSpec1 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent"), Version.fromString("6.100"), false);
+        ClusterSpec clusterSpec1 = clusterSpec("myContent.t1.a1");
         deployapp(application1, clusterSpec1, flavor, tester, 3);
 
         // Application 2
         ApplicationId application2 = makeApplicationId("t2", "a2");
-        ClusterSpec clusterSpec2 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent"), Version.fromString("6.100"), false);
+        ClusterSpec clusterSpec2 = clusterSpec("myContent.t2.a2");
         deployapp(application2, clusterSpec2, flavor, tester, 2);
 
         // Application 3
         ApplicationId application3 = makeApplicationId("t3", "a3");
-        ClusterSpec clusterSpec3 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent"), Version.fromString("6.100"), false);
+        ClusterSpec clusterSpec3 = clusterSpec("myContent.t3.a3");
         deployapp(application3, clusterSpec3, flavor, tester, 2);
 
-        // App 2 and 3 should have been allocated to the same nodes - fail on of the parent hosts from there
+        // App 2 and 3 should have been allocated to the same nodes - fail one of the parent hosts from there
         String parent = tester.nodeRepository().getNodes(application2).stream().findAny().get().parentHostname().get();
         tester.nodeRepository().failRecursively(parent, Agent.system, "Testing");
 
@@ -315,9 +315,9 @@ public class DynamicDockerProvisioningTest {
             numberOfChildrenStat.put(nofChildren, numberOfChildrenStat.get(nofChildren) + 1);
         }
 
-        assertEquals(3l, (long) numberOfChildrenStat.get(3));
-        assertEquals(1l, (long) numberOfChildrenStat.get(0));
-        assertEquals(1l, (long) numberOfChildrenStat.get(1));
+        assertEquals(3, numberOfChildrenStat.get(3).intValue());
+        assertEquals(1, numberOfChildrenStat.get(0).intValue());
+        assertEquals(1, numberOfChildrenStat.get(1).intValue());
     }
 
     /**
@@ -342,9 +342,9 @@ public class DynamicDockerProvisioningTest {
 
         // Application 1
         ApplicationId application1 = makeApplicationId("t1", "a1");
-        ClusterSpec clusterSpec1 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent"), Version.fromString("6.100"), false);
-        addAndAssignNode(application1, "1a", dockerHosts.get(0).hostname(), flavor, 0, tester);
-        addAndAssignNode(application1, "1b", dockerHosts.get(1).hostname(), flavor, 1, tester);
+        ClusterSpec clusterSpec1 = clusterSpec("myContent.t1.a1");
+        addAndAssignNode(application1, "1a", dockerHosts.get(0).hostname(), clusterSpec1, flavor, 0, tester);
+        addAndAssignNode(application1, "1b", dockerHosts.get(1).hostname(), clusterSpec1, flavor, 1, tester);
 
         // Redeploy both applications (to be agnostic on which hosts are picked as spares)
         deployapp(application1, clusterSpec1, flavor, tester, 2);
@@ -366,11 +366,9 @@ public class DynamicDockerProvisioningTest {
         deployZoneApp(tester);
         Flavor flavor = tester.nodeRepository().getAvailableFlavors().getFlavorOrThrow("d-1");
 
-        //Deploy an application of 6 nodes of 3 nodes in each cluster. We only have 3 docker hosts available
+        //Deploy an application having 6 nodes (3 nodes in 2 groups). We only have 5 docker hosts available
         ApplicationId application1 = tester.makeApplicationId();
-        tester.prepare(application1,
-                ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent"), Version.fromString("6.100"), false),
-                6, 2, flavor.canonicalName());
+        tester.prepare(application1, clusterSpec("myContent.t1.a1"), 6, 2, flavor.canonicalName());
 
         fail("Two groups have been allocated to the same parent host");
     }
@@ -396,9 +394,8 @@ public class DynamicDockerProvisioningTest {
         Flavor flavor = tester.nodeRepository().getAvailableFlavors().getFlavorOrThrow("d-3");
 
         // Deploy initial state (can max deploy 3 nodes due to redundancy requirements)
-        List<HostSpec> hosts = tester.prepare(application1,
-                ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent"), Version.fromString("6.100"), false),
-                3, 1, flavor.canonicalName());
+        ClusterSpec clusterSpec = clusterSpec("myContent.t1.a1");
+        List<HostSpec> hosts = tester.prepare(application1, clusterSpec, 3, 1, flavor.canonicalName());
         tester.activate(application1, ImmutableSet.copyOf(hosts));
 
         DockerHostCapacity capacity = new DockerHostCapacity(tester.nodeRepository().getNodes(Node.State.values()));
@@ -408,17 +405,13 @@ public class DynamicDockerProvisioningTest {
         assertThat(initialSpareCapacity.size(), is(2));
 
         try {
-            hosts = tester.prepare(application1,
-                    ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent"), Version.fromString("6.100"), false),
-                    4, 1, flavor.canonicalName());
+            hosts = tester.prepare(application1, clusterSpec, 4, 1, flavor.canonicalName());
             fail("Was able to deploy with 4 nodes, should not be able to use spare capacity");
         } catch (OutOfCapacityException e) {
         }
 
         tester.fail(hosts.get(0));
-        hosts = tester.prepare(application1,
-                ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent"), Version.fromString("6.100"), false),
-                3, 1, flavor.canonicalName());
+        hosts = tester.prepare(application1, clusterSpec, 3, 1, flavor.canonicalName());
         tester.activate(application1, ImmutableSet.copyOf(hosts));
 
         List<Node> finalSpareCapacity = findSpareCapacity(tester);
@@ -427,16 +420,14 @@ public class DynamicDockerProvisioningTest {
     }
 
     @Test
-    public void non_prod_do_not_have_spares() {
+    public void non_prod_zones_do_not_have_spares() {
         ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.perf, RegionName.from("us-east")), flavorsConfig());
         tester.makeReadyNodes(3, "host-small", NodeType.host, 32);
         deployZoneApp(tester);
         Flavor flavor = tester.nodeRepository().getAvailableFlavors().getFlavorOrThrow("d-3");
 
         ApplicationId application1 = tester.makeApplicationId();
-        List<HostSpec> hosts = tester.prepare(application1,
-                ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent"), Version.fromString("6.100"), false),
-                3, 1, flavor.canonicalName());
+        List<HostSpec> hosts = tester.prepare(application1, clusterSpec("myContent.t1.a1"), 3, 1, flavor.canonicalName());
         tester.activate(application1, ImmutableSet.copyOf(hosts));
 
         List<Node> initialSpareCapacity = findSpareCapacity(tester);
@@ -446,14 +437,12 @@ public class DynamicDockerProvisioningTest {
     @Test(expected = OutOfCapacityException.class)
     public void allocation_should_fail_when_host_is_not_active() {
         ProvisioningTester tester = new ProvisioningTester(new Zone(Environment.prod, RegionName.from("us-east")), flavorsConfig());
-
         tester.makeProvisionedNodes(3, "host-small", NodeType.host, 32);
         deployZoneApp(tester);
 
         ApplicationId application = tester.makeApplicationId();
         Flavor flavor = tester.nodeRepository().getAvailableFlavors().getFlavorOrThrow("d-3");
-        tester.prepare(application, ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent"), Version.fromString("6.100"), false),
-                       2, 1, flavor.canonicalName());
+        tester.prepare(application, clusterSpec("myContent.t2.a2"), 2, 1, flavor.canonicalName());
     }
 
     private ApplicationId makeApplicationId(String tenant, String appName) {
@@ -465,18 +454,16 @@ public class DynamicDockerProvisioningTest {
         tester.activate(id, new HashSet<>(hostSpec));
     }
 
-    private Node addAndAssignNode(ApplicationId id, String hostname, String parentHostname, Flavor flavor, int index, ProvisioningTester tester) {
+    private void addAndAssignNode(ApplicationId id, String hostname, String parentHostname, ClusterSpec clusterSpec, Flavor flavor, int index, ProvisioningTester tester) {
         Node node1a = Node.create("open1", Collections.singleton("127.0.0.100"), new HashSet<>(), hostname, Optional.of(parentHostname), flavor, NodeType.tenant);
-        ClusterSpec clusterSpec = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent"), Version.fromString("6.100"), false).with(Optional.of(ClusterSpec.Group.from(0)));
-        ClusterMembership clusterMembership1 = ClusterMembership.from(clusterSpec, index);
+        ClusterMembership clusterMembership1 = ClusterMembership.from(
+                clusterSpec.with(Optional.of(ClusterSpec.Group.from(0))), index); // Need to add group here so that group is serialized in node allocation
         Node node1aAllocation = node1a.allocate(id, clusterMembership1, Instant.now());
 
         tester.nodeRepository().addNodes(Collections.singletonList(node1aAllocation));
         NestedTransaction transaction = new NestedTransaction().add(new CuratorTransaction(tester.getCurator()));
         tester.nodeRepository().activate(Collections.singletonList(node1aAllocation), transaction);
         transaction.commit();
-
-        return node1aAllocation;
     }
 
     private List<Node> findSpareCapacity(ProvisioningTester tester) {
@@ -506,19 +493,10 @@ public class DynamicDockerProvisioningTest {
     }
 
     private FlavorsConfig flavorsConfig() {
-        FlavorConfigBuilder b = new FlavorConfigBuilder();
-        b.addFlavor("host-large", 6., 6., 6, Flavor.Type.BARE_METAL);
-        b.addFlavor("host-small", 3., 3., 3, Flavor.Type.BARE_METAL);
-        b.addFlavor("d-1", 1, 1., 1, Flavor.Type.DOCKER_CONTAINER);
-        b.addFlavor("d-2", 2, 2., 2, Flavor.Type.DOCKER_CONTAINER);
-        b.addFlavor("d-3", 3, 3., 3, Flavor.Type.DOCKER_CONTAINER);
-        b.addFlavor("d-3-disk", 3, 3., 5, Flavor.Type.DOCKER_CONTAINER);
-        b.addFlavor("d-3-mem", 3, 5., 3, Flavor.Type.DOCKER_CONTAINER);
-        b.addFlavor("d-3-cpu", 5, 3., 3, Flavor.Type.DOCKER_CONTAINER);
-        return b.build();
+        return flavorsConfig(false);
     }
 
-    private List<HostSpec> deployZoneApp(ProvisioningTester tester) {
+    private void deployZoneApp(ProvisioningTester tester) {
         ApplicationId applicationId = tester.makeApplicationId();
         List<HostSpec> list = tester.prepare(applicationId,
                 ClusterSpec.request(ClusterSpec.Type.container,
@@ -528,7 +506,6 @@ public class DynamicDockerProvisioningTest {
                 Capacity.fromRequiredNodeType(NodeType.host),
                 1);
         tester.activate(applicationId, ImmutableSet.copyOf(list));
-        return list;
     }
 
     private boolean isInactiveOrRetired(Node node) {
@@ -540,4 +517,8 @@ public class DynamicDockerProvisioningTest {
 
         return isInactive || isRetired;
     }
+
+    private ClusterSpec clusterSpec(String clusterId) {
+        return ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from(clusterId), Version.fromString("6.100"), false);
+    }
 }
diff --git a/parent/pom.xml b/parent/pom.xml
index 56da33a9c93..51f6bc52da3 100644
--- a/parent/pom.xml
+++ b/parent/pom.xml
@@ -130,7 +130,7 @@
                 <plugin>
                     <groupId>org.apache.maven.plugins</groupId>
                     <artifactId>maven-enforcer-plugin</artifactId>
-                    <version>3.0.0-M1</version>
+                    <version>3.0.0-M2</version>
                 </plugin>
                 <plugin>
                     <groupId>org.apache.maven.plugins</groupId>
@@ -407,11 +407,6 @@
                 <version>${asm.version}</version>
             </dependency>
             <dependency>
-              <groupId>com.google.guava</groupId>
-              <artifactId>guava-testlib</artifactId>
-              <version>18.0</version>
-            </dependency>
-            <dependency>
                 <groupId>com.google.protobuf</groupId>
                 <artifactId>protobuf-java</artifactId>
                 <version>3.4.0</version>
@@ -663,6 +658,16 @@
                 <artifactId>wiremock-standalone</artifactId>
                 <version>2.6.0</version>
             </dependency>
+            <dependency>
+                <groupId>org.apache.opennlp</groupId>
+                <artifactId>opennlp-tools</artifactId>
+                <version>1.8.4</version>
+            </dependency>
+            <dependency>
+                <groupId>com.optimaize.languagedetector</groupId>
+                <artifactId>language-detector</artifactId>
+                <version>0.6</version>
+            </dependency>
         </dependencies>
     </dependencyManagement>
 
diff --git a/persistence/src/vespa/persistence/dummyimpl/dummypersistence.cpp b/persistence/src/vespa/persistence/dummyimpl/dummypersistence.cpp
index d3883744229..25c6b71f7ff 100644
--- a/persistence/src/vespa/persistence/dummyimpl/dummypersistence.cpp
+++ b/persistence/src/vespa/persistence/dummyimpl/dummypersistence.cpp
@@ -528,7 +528,7 @@ DummyPersistence::get(const Bucket& b,
         b.toString().c_str(),
         did.toString().c_str());
     assert(b.getBucketSpace() == FixedBucketSpaces::default_space());
-    BucketContentGuard::UP bc(acquireBucketWithLock(b));
+    BucketContentGuard::UP bc(acquireBucketWithLock(b, LockMode::Shared));
     if (!bc.get()) {
     } else {
         DocEntry::SP entry((*bc)->getEntry(did));
@@ -568,7 +568,7 @@ DummyPersistence::createIterator(
                     "Got invalid/unparseable document selection string");
         }
     }
-    BucketContentGuard::UP bc(acquireBucketWithLock(b));
+    BucketContentGuard::UP bc(acquireBucketWithLock(b, LockMode::Shared));
     if (!bc.get()) {
         return CreateIteratorResult(Result::TRANSIENT_ERROR, "Bucket not found");
     }
@@ -656,7 +656,7 @@ DummyPersistence::iterate(IteratorId id, uint64_t maxByteSize, Context& ctx) con
         it = iter->second.get();
     }
 
-    BucketContentGuard::UP bc(acquireBucketWithLock(it->_bucket));
+    BucketContentGuard::UP bc(acquireBucketWithLock(it->_bucket, LockMode::Shared));
     if (!bc.get()) {
         ctx.trace(9, "finished iterate(); bucket not found");
         return IterateResult(Result::TRANSIENT_ERROR, "Bucket not found");
@@ -942,11 +942,11 @@ DummyPersistence::isActive(const Bucket& b) const
 
 BucketContentGuard::~BucketContentGuard()
 {
-    _persistence.releaseBucketNoLock(_content);
+    _persistence.releaseBucketNoLock(_content, _lock_mode);
 }
 
 BucketContentGuard::UP
-DummyPersistence::acquireBucketWithLock(const Bucket& b) const
+DummyPersistence::acquireBucketWithLock(const Bucket& b, LockMode lock_mode) const
 {
     assert(b.getBucketSpace() == FixedBucketSpaces::default_space());
     vespalib::MonitorGuard lock(_monitor);
@@ -955,28 +955,32 @@ DummyPersistence::acquireBucketWithLock(const Bucket& b) const
     if (it == ncp._content[b.getPartition()].end()) {
         return BucketContentGuard::UP();
     }
-    // Sanity check that SPI-level locking is doing its job correctly.
-    // Atomic CAS might be a bit overkill, but since we "release" the bucket
-    // outside of the mutex, we want to ensure the write is visible across all
-    // threads.
-    bool my_false(false);
-    bool bucketNotInUse(it->second->_inUse.compare_exchange_strong(my_false, true));
-    if (!bucketNotInUse) {
-        LOG(error, "Attempted to acquire %s, but it was already marked as being in use!",
-            b.toString().c_str());
-        LOG_ABORT("should not reach here");
+    if (lock_mode == LockMode::Exclusive) {
+        // Sanity check that SPI-level locking is doing its job correctly.
+        // Atomic CAS might be a bit overkill, but since we "release" the bucket
+        // outside of the mutex, we want to ensure the write is visible across all
+        // threads.
+        bool my_false(false);
+        bool bucketNotInUse(it->second->_inUse.compare_exchange_strong(my_false, true));
+        if (!bucketNotInUse) {
+            LOG(error, "Attempted to acquire %s, but it was already marked as being in use!",
+                b.toString().c_str());
+            LOG_ABORT("dummy persistence bucket locking invariant violation");
+        }
     }
 
-    return BucketContentGuard::UP(new BucketContentGuard(ncp, *it->second));
+    return std::make_unique<BucketContentGuard>(ncp, *it->second, lock_mode);
 }
 
 void
-DummyPersistence::releaseBucketNoLock(const BucketContent& bc) const
+DummyPersistence::releaseBucketNoLock(const BucketContent& bc, LockMode lock_mode) const noexcept
 {
-    bool my_true(true);
-    bool bucketInUse(bc._inUse.compare_exchange_strong(my_true, false));
-    assert(bucketInUse);
-    (void) bucketInUse;
+    if (lock_mode == LockMode::Exclusive) {
+        bool my_true(true);
+        bool bucketInUse(bc._inUse.compare_exchange_strong(my_true, false));
+        assert(bucketInUse);
+        (void) bucketInUse;
+    }
 }
 
 }
diff --git a/persistence/src/vespa/persistence/dummyimpl/dummypersistence.h b/persistence/src/vespa/persistence/dummyimpl/dummypersistence.h
index c93b7fd22c7..c97aab822ac 100644
--- a/persistence/src/vespa/persistence/dummyimpl/dummypersistence.h
+++ b/persistence/src/vespa/persistence/dummyimpl/dummypersistence.h
@@ -24,13 +24,18 @@ class DocumentTypeRepo;
 
 namespace storage::spi::dummy {
 
+enum class LockMode {
+    Exclusive,
+    Shared
+};
+
 struct BucketEntry
 {
     DocEntry::SP entry;
     GlobalId gid;
 
     BucketEntry(DocEntry::SP e, const GlobalId& g)
-        : entry(e),
+        : entry(std::move(e)),
           gid(g)
     { }
 };
@@ -98,30 +103,33 @@ class BucketContentGuard
     BucketContentGuard(const BucketContentGuard&);
     BucketContentGuard& operator=(const BucketContentGuard&);
 public:
-    typedef std::unique_ptr<BucketContentGuard> UP;
+    using UP = std::unique_ptr<BucketContentGuard>;
 
     BucketContentGuard(DummyPersistence& persistence,
-                       BucketContent& content)
+                       BucketContent& content,
+                       LockMode lock_mode)
         : _persistence(persistence),
-          _content(content)
+          _content(content),
+          _lock_mode(lock_mode)
     {
     }
     ~BucketContentGuard();
 
-    BucketContent& getContent() {
+    BucketContent& getContent() noexcept {
         return _content;
     }
 
-    BucketContent* operator->() {
+    BucketContent* operator->() noexcept {
         return &_content;
     }
 
-    BucketContent& operator*() {
+    BucketContent& operator*() noexcept {
         return _content;
     }
 private:
     DummyPersistence& _persistence;
     BucketContent& _content;
+    LockMode _lock_mode;
 };
 
 class DummyPersistence : public AbstractPersistenceProvider
@@ -207,8 +215,8 @@ public:
 private:
     friend class BucketContentGuard;
     // Const since funcs only alter mutable field in BucketContent
-    BucketContentGuard::UP acquireBucketWithLock(const Bucket& b) const;
-    void releaseBucketNoLock(const BucketContent& bc) const;
+    BucketContentGuard::UP acquireBucketWithLock(const Bucket& b, LockMode lock_mode = LockMode::Exclusive) const;
+    void releaseBucketNoLock(const BucketContent& bc, LockMode lock_mode = LockMode::Exclusive) const noexcept;
 
     mutable bool _initialized;
     std::shared_ptr<const document::DocumentTypeRepo> _repo;
diff --git a/persistence/src/vespa/persistence/spi/persistenceprovider.h b/persistence/src/vespa/persistence/spi/persistenceprovider.h
index b5f2fc198c4..96b3d385b87 100644
--- a/persistence/src/vespa/persistence/spi/persistenceprovider.h
+++ b/persistence/src/vespa/persistence/spi/persistenceprovider.h
@@ -232,6 +232,9 @@ struct PersistenceProvider
      * document id. If no versions were found, or the document was removed,
      * the result should be successful, but contain no document (see GetResult).
      *
+     * Concurrency note: may be called concurrently with other read-only
+     * operations.
+     *
      * @param fieldSet A set of fields that should be retrieved.
      * @param id The document id to retrieve.
      */
@@ -253,6 +256,9 @@ struct PersistenceProvider
      * iteration progress and selection criteria. destroyIterator will NOT
      * be called when createIterator returns an error.
      *
+     * Concurrency note: may be called concurrently with other read-only
+     * operations.
+     *
      * @param selection Selection criteria used to limit the subset of
      *   the bucket's documents that will be returned by the iterator. The
      *   provider implementation may use these criteria to optimize its
@@ -323,6 +329,9 @@ struct PersistenceProvider
      * iterator must only set this flag on the result and return without any
      * documents.
      *
+     * Concurrency note: may be called concurrently with other read-only
+     * operations.
+     *
      * @param id An iterator ID returned by a previous call to createIterator
      * @param maxByteSize An indication of the maximum number of bytes that
      * should be returned.
diff --git a/pom.xml b/pom.xml
index c24ddf3f042..b20d038ab3c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -56,6 +56,7 @@
         <module>container-dev</module>
         <module>container-di</module>
         <module>container-disc</module>
+        <module>container-integration-test</module>
         <module>container-jersey2</module>
         <module>container-messagebus</module>
         <module>container-search-and-docproc</module>
@@ -93,6 +94,7 @@
         <module>messagebus-disc</module>
         <module>messagebus</module>
         <module>metrics</module>
+        <module>model-inference</module>
         <module>node-repository</module>
         <module>node-admin</module>
         <module>node-maintainer</module>
diff --git a/searchcommon/src/vespa/searchcommon/attribute/attributecontent.h b/searchcommon/src/vespa/searchcommon/attribute/attributecontent.h
index 6c48b02f357..508a2d04c27 100644
--- a/searchcommon/src/vespa/searchcommon/attribute/attributecontent.h
+++ b/searchcommon/src/vespa/searchcommon/attribute/attributecontent.h
@@ -3,10 +3,9 @@
 #pragma once
 
 #include "iattributevector.h"
-#include <stdint.h>
+#include <cstdint>
 
-namespace search {
-namespace attribute {
+namespace search::attribute {
 
 
 /**
@@ -154,7 +153,6 @@ public:
     }
 };
 
-
 typedef AttributeContent<double>       FloatContent;
 typedef AttributeContent<const char *> ConstCharContent;
 typedef AttributeContent<IAttributeVector::largeint_t> IntegerContent;
@@ -166,7 +164,4 @@ typedef AttributeContent<IAttributeVector::WeightedString>    WeightedStringCont
 typedef AttributeContent<IAttributeVector::WeightedEnum>      WeightedEnumContent;
 typedef IAttributeVector::EnumHandle EnumHandle;
 
-
-} // namespace attribute
-} // namespace search
-
+}
diff --git a/searchcommon/src/vespa/searchcommon/attribute/basictype.cpp b/searchcommon/src/vespa/searchcommon/attribute/basictype.cpp
index c3cf0905b27..67093c686b5 100644
--- a/searchcommon/src/vespa/searchcommon/attribute/basictype.cpp
+++ b/searchcommon/src/vespa/searchcommon/attribute/basictype.cpp
@@ -3,8 +3,7 @@
 #include <vespa/searchcommon/attribute/basictype.h>
 #include <vespa/vespalib/util/exceptions.h>
 
-namespace search {
-namespace attribute {
+namespace search::attribute {
 
 const BasicType::TypeInfo BasicType::_typeTable[BasicType::MAX_TYPE] = {
     { BasicType::NONE,      0,                "none" },
@@ -31,11 +30,8 @@ BasicType::asType(const vespalib::string &t)
             return _typeTable[i]._type;
         }
     }
-    throw vespalib::IllegalStateException(t +
-                                " not recognized as "
-                                "valid attribute data type");
+    throw vespalib::IllegalStateException(t + " not recognized as valid attribute data type");
     return NONE;
 }
 
 }
-}
diff --git a/searchcommon/src/vespa/searchcommon/attribute/basictype.h b/searchcommon/src/vespa/searchcommon/attribute/basictype.h
index d7023aa1c59..3a2a319afd8 100644
--- a/searchcommon/src/vespa/searchcommon/attribute/basictype.h
+++ b/searchcommon/src/vespa/searchcommon/attribute/basictype.h
@@ -4,8 +4,7 @@
 
 #include <vespa/vespalib/stllike/string.h>
 
-namespace search {
-namespace attribute {
+namespace search::attribute {
 
 class BasicType
 {
@@ -66,5 +65,3 @@ class BasicType
 };
 
 }
-}
-
diff --git a/searchcommon/src/vespa/searchcommon/attribute/collectiontype.cpp b/searchcommon/src/vespa/searchcommon/attribute/collectiontype.cpp
index 1d0dcc5328b..d5d33718183 100644
--- a/searchcommon/src/vespa/searchcommon/attribute/collectiontype.cpp
+++ b/searchcommon/src/vespa/searchcommon/attribute/collectiontype.cpp
@@ -3,8 +3,7 @@
 #include <vespa/searchcommon/attribute/collectiontype.h>
 #include <vespa/vespalib/util/exceptions.h>
 
-namespace search {
-namespace attribute {
+namespace search::attribute {
 
 const CollectionType::TypeInfo CollectionType::_typeTable[CollectionType::MAX_TYPE] = {
     { CollectionType::SINGLE, "single" },
@@ -20,11 +19,8 @@ CollectionType::asType(const vespalib::string &t)
             return _typeTable[i]._type;
         }
     }
-    throw vespalib::IllegalStateException(t +
-                             " not recognized as valid attribute "
-                             "collection type");
+    throw vespalib::IllegalStateException(t + " not recognized as valid attribute collection type");
     return SINGLE;
 }
 
 }
-}
diff --git a/searchcommon/src/vespa/searchcommon/attribute/collectiontype.h b/searchcommon/src/vespa/searchcommon/attribute/collectiontype.h
index b5b43083307..2cd46ce2bba 100644
--- a/searchcommon/src/vespa/searchcommon/attribute/collectiontype.h
+++ b/searchcommon/src/vespa/searchcommon/attribute/collectiontype.h
@@ -4,8 +4,7 @@
 
 #include <vespa/vespalib/stllike/string.h>
 
-namespace search {
-namespace attribute {
+namespace search::attribute {
 
 class CollectionType
 {
@@ -74,5 +73,3 @@ class CollectionType
 };
 
 }
-}
-
diff --git a/searchcommon/src/vespa/searchcommon/attribute/config.cpp b/searchcommon/src/vespa/searchcommon/attribute/config.cpp
index 36dd0c94b40..221924a1689 100644
--- a/searchcommon/src/vespa/searchcommon/attribute/config.cpp
+++ b/searchcommon/src/vespa/searchcommon/attribute/config.cpp
@@ -20,10 +20,7 @@ Config::Config() :
 {
 }
 
-Config::Config(BasicType bt,
-               CollectionType ct,
-               bool fastSearch_,
-               bool huge_)
+Config::Config(BasicType bt, CollectionType ct, bool fastSearch_, bool huge_)
     : _basicType(bt),
       _type(ct),
       _fastSearch(fastSearch_),
@@ -41,6 +38,6 @@ Config::Config(BasicType bt,
 
 Config::Config(const Config &) = default;
 Config & Config::operator = (const Config &) = default;
-Config::~Config() {}
+Config::~Config() = default;
 
 }
diff --git a/searchcommon/src/vespa/searchcommon/attribute/config.h b/searchcommon/src/vespa/searchcommon/attribute/config.h
index 8092d620f36..683b45b59e5 100644
--- a/searchcommon/src/vespa/searchcommon/attribute/config.h
+++ b/searchcommon/src/vespa/searchcommon/attribute/config.h
@@ -9,8 +9,7 @@
 #include <vespa/searchcommon/common/compaction_strategy.h>
 #include <vespa/eval/eval/value_type.h>
 
-namespace search {
-namespace attribute {
+namespace search::attribute {
 
 class Config
 {
@@ -120,6 +119,5 @@ private:
     PredicateParams    _predicateParams;
     vespalib::eval::ValueType _tensorType;
 };
-}  // namespace attribute
-}  // namespace search
 
+}
diff --git a/searchcommon/src/vespa/searchcommon/attribute/i_search_context.h b/searchcommon/src/vespa/searchcommon/attribute/i_search_context.h
index d18c4840009..be6b8d213d8 100644
--- a/searchcommon/src/vespa/searchcommon/attribute/i_search_context.h
+++ b/searchcommon/src/vespa/searchcommon/attribute/i_search_context.h
@@ -5,7 +5,6 @@
 #include <vespa/searchcommon/common/range.h>
 #include <vespa/vespalib/stllike/string.h>
 
-
 namespace search::fef { class TermFieldMatchData; }
 namespace search::queryeval { class SearchIterator; }
 namespace search { class QueryTermBase; }
@@ -48,7 +47,7 @@ public:
 
     virtual bool valid() const = 0;
     virtual Int64Range getAsIntegerTerm() const = 0;
-    virtual const QueryTermBase &queryTerm() const = 0;
+    virtual const QueryTermBase * queryTerm() const = 0;
     virtual const vespalib::string &attributeName() const = 0;
 
     int32_t find(DocId docId, int32_t elementId, int32_t &weight) const { return onFind(docId, elementId, weight); }
diff --git a/searchcommon/src/vespa/searchcommon/attribute/iattributecontext.h b/searchcommon/src/vespa/searchcommon/attribute/iattributecontext.h
index 2ba61512338..c283abf3ced 100644
--- a/searchcommon/src/vespa/searchcommon/attribute/iattributecontext.h
+++ b/searchcommon/src/vespa/searchcommon/attribute/iattributecontext.h
@@ -3,11 +3,8 @@
 #pragma once
 
 #include "iattributevector.h"
-#include <vector>
-#include <memory>
 
-namespace search {
-namespace attribute {
+namespace search::attribute {
 
 /**
  * This is an interface used to access all registered attribute vectors.
@@ -53,6 +50,4 @@ public:
     virtual ~IAttributeContext() {}
 };
 
-} // namespace attribute
-} // namespace search
-
+}
diff --git a/searchcommon/src/vespa/searchcommon/attribute/iattributevector.h b/searchcommon/src/vespa/searchcommon/attribute/iattributevector.h
index c432606f7f0..0d3e58331c6 100644
--- a/searchcommon/src/vespa/searchcommon/attribute/iattributevector.h
+++ b/searchcommon/src/vespa/searchcommon/attribute/iattributevector.h
@@ -5,18 +5,18 @@
 #include "collectiontype.h"
 #include "basictype.h"
 #include <vespa/searchcommon/common/iblobconverter.h>
-#include <vespa/vespalib/stllike/string.h>
+#include <vector>
 
 namespace search {
+    class IDocumentWeightAttribute;
+    class QueryTermSimple;
+}
 
-class IDocumentWeightAttribute;
-class QueryTermSimple;
-
-namespace tensor {
-class ITensorAttribute;
+namespace search::tensor {
+    class ITensorAttribute;
 }
 
-namespace attribute {
+namespace search::attribute {
 
 class ISearchContext;
 class SearchContextParams;
@@ -251,6 +251,16 @@ public:
     virtual bool findEnum(const char * value, EnumHandle & e) const = 0;
 
     /**
+     * Finds all enum values matching the given string value.
+     * This method will only have effect if @ref getBasicType() returns BasicType::STRING and
+     * @ref hasEnum() returns true.
+     *
+     * @param value the string value to lookup.
+     * @return vector of EnumHandles, size 0 if no match found.
+     **/
+    virtual std::vector<EnumHandle> findFoldedEnums(const char * value) const = 0;
+
+    /**
      * Given an enum handle, returns the string it refers to.
      * This method will only have effect if @ref getBasicType() returns BasicType::STRING and
      * @ref hasEnum() returns true.
@@ -427,6 +437,4 @@ private:
 
 };
 
-} // namespace fef
-} // namespace search
-
+}
diff --git a/searchcommon/src/vespa/searchcommon/attribute/predicate_params.h b/searchcommon/src/vespa/searchcommon/attribute/predicate_params.h
index 5510da7f54e..6a49bdd4d8e 100644
--- a/searchcommon/src/vespa/searchcommon/attribute/predicate_params.h
+++ b/searchcommon/src/vespa/searchcommon/attribute/predicate_params.h
@@ -4,8 +4,7 @@
 
 #include "persistent_predicate_params.h"
 
-namespace search {
-namespace attribute {
+namespace search::attribute {
 
 /*
  * Parameters for predicate attributes.
@@ -28,5 +27,4 @@ public:
     }
 };
 
-}  // namespace attribute
-}  // namespace search
+}
diff --git a/searchcommon/src/vespa/searchcommon/attribute/status.cpp b/searchcommon/src/vespa/searchcommon/attribute/status.cpp
index 09b1e00a35e..d6d684a9f56 100644
--- a/searchcommon/src/vespa/searchcommon/attribute/status.cpp
+++ b/searchcommon/src/vespa/searchcommon/attribute/status.cpp
@@ -1,9 +1,8 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
-#include <vespa/searchcommon/attribute/status.h>
+#include "status.h"
 
-namespace search {
-namespace attribute {
+namespace search::attribute {
 
 Status::Status(const vespalib::string &)
     : _numDocs              (0),
@@ -42,8 +41,7 @@ Status::Status()
 
 
 vespalib::string
-Status::createName(const vespalib::stringref &index,
-                                    const vespalib::stringref &attr)
+Status::createName(vespalib::stringref index, vespalib::stringref attr)
 {
     vespalib::string name (index);
     name += ".attribute.";
@@ -53,12 +51,8 @@ Status::createName(const vespalib::stringref &index,
 
 
 void
-Status::updateStatistics(uint64_t numValues,
-        uint64_t numUniqueValue,
-        uint64_t allocated,
-        uint64_t used,
-        uint64_t dead,
-        uint64_t onHold)
+Status::updateStatistics(uint64_t numValues, uint64_t numUniqueValue, uint64_t allocated,
+                         uint64_t used, uint64_t dead, uint64_t onHold)
 {
     _numValues       = numValues;
     _numUniqueValues = numUniqueValue;
@@ -71,4 +65,3 @@ Status::updateStatistics(uint64_t numValues,
 }
 
 }
-}
diff --git a/searchcommon/src/vespa/searchcommon/attribute/status.h b/searchcommon/src/vespa/searchcommon/attribute/status.h
index 3f6a12cb88e..dc1ccc4d5d3 100644
--- a/searchcommon/src/vespa/searchcommon/attribute/status.h
+++ b/searchcommon/src/vespa/searchcommon/attribute/status.h
@@ -4,8 +4,7 @@
 
 #include <vespa/vespalib/stllike/string.h>
 
-namespace search {
-namespace attribute {
+namespace search::attribute {
 
 class Status
 {
@@ -14,13 +13,8 @@ public:
     Status(const vespalib::string &name);
     Status();
 
-    void
-    updateStatistics(uint64_t numValues,
-                     uint64_t numUniqueValue,
-                     uint64_t allocated,
-                     uint64_t used,
-                     uint64_t dead,
-                     uint64_t onHold);
+    void updateStatistics(uint64_t numValues, uint64_t numUniqueValue, uint64_t allocated,
+                          uint64_t used, uint64_t dead, uint64_t onHold);
 
     uint64_t getNumDocs()                  const { return _numDocs; }
     uint64_t getNumValues()                const { return _numValues; }
@@ -33,32 +27,18 @@ public:
     uint64_t getLastSyncToken()            const { return _lastSyncToken; }
     uint64_t getUpdateCount()              const { return _updates; }
     uint64_t getNonIdempotentUpdateCount() const { return _nonIdempotentUpdates; }
-    uint32_t
-    getBitVectors() const
-    {
-        return _bitVectors;
-    }
+    uint32_t getBitVectors() const { return _bitVectors; }
 
     void setNumDocs(uint64_t v)                  { _numDocs = v; }
     void incNumDocs()                            { ++_numDocs; }
     void setLastSyncToken(uint64_t v)            { _lastSyncToken = v; }
     void incUpdates(uint64_t v=1)                { _updates += v; }
     void incNonIdempotentUpdates(uint64_t v = 1) { _nonIdempotentUpdates += v; }
-    void
-    incBitVectors()
-    {
-        ++_bitVectors;
-    }
-
-    void
-    decBitVectors()
-    {
-        --_bitVectors;
-    }
+    void incBitVectors() { ++_bitVectors; }
+    void decBitVectors() { --_bitVectors; }
 
     static vespalib::string
-    createName(const vespalib::stringref &index,
-               const vespalib::stringref & attr);
+    createName(vespalib::stringref index, vespalib::stringref attr);
 private:
     uint64_t _numDocs;
     uint64_t _numValues;
@@ -76,5 +56,3 @@ private:
 };
 
 }
-}
-
diff --git a/searchcommon/src/vespa/searchcommon/common/iblobconverter.h b/searchcommon/src/vespa/searchcommon/common/iblobconverter.h
index f9f326b8c69..428a1d7c296 100644
--- a/searchcommon/src/vespa/searchcommon/common/iblobconverter.h
+++ b/searchcommon/src/vespa/searchcommon/common/iblobconverter.h
@@ -5,8 +5,7 @@
 #include <vespa/vespalib/util/buffer.h>
 #include <memory>
 
-namespace search {
-namespace common {
+namespace search::common {
 
 class BlobConverter
 {
@@ -21,5 +20,3 @@ private:
 };
 
 }
-}
-
diff --git a/searchcommon/src/vespa/searchcommon/common/schema.cpp b/searchcommon/src/vespa/searchcommon/common/schema.cpp
index eafe9c28c1b..6cd9d87fa77 100644
--- a/searchcommon/src/vespa/searchcommon/common/schema.cpp
+++ b/searchcommon/src/vespa/searchcommon/common/schema.cpp
@@ -22,7 +22,7 @@ writeFields(vespalib::asciistream & os,
 {
     os << prefix << "[" << fields.size() << "]\n";
     for (size_t i = 0; i < fields.size(); ++i) {
-        fields[i].write(os, vespalib::make_string("%s[%zu].", prefix.c_str(), i));
+        fields[i].write(os, vespalib::make_string("%s[%zu].", prefix.data(), i));
     }
 }
 
@@ -245,7 +245,7 @@ Schema & Schema::operator=(Schema && rhs) = default;
 Schema::~Schema() { }
 
 bool
-Schema::loadFromFile(const vespalib::stringref & fileName)
+Schema::loadFromFile(const vespalib::string & fileName)
 {
     std::ifstream file(fileName.c_str());
     if (!file) {
@@ -284,7 +284,7 @@ Schema::loadFromFile(const vespalib::stringref & fileName)
 }
 
 bool
-Schema::saveToFile(const vespalib::stringref & fileName) const
+Schema::saveToFile(const vespalib::string & fileName) const
 {
     vespalib::asciistream os;
     writeToStream(os, true);
diff --git a/searchcommon/src/vespa/searchcommon/common/schema.h b/searchcommon/src/vespa/searchcommon/common/schema.h
index 374ea840f5c..1c3ab3ccd56 100644
--- a/searchcommon/src/vespa/searchcommon/common/schema.h
+++ b/searchcommon/src/vespa/searchcommon/common/schema.h
@@ -179,7 +179,7 @@ public:
      * @return true if the schema could be loaded.
      **/
     bool
-    loadFromFile(const vespalib::stringref & fileName);
+    loadFromFile(const vespalib::string & fileName);
 
     /**
      * Save this schema to the file with the given name.
@@ -188,7 +188,7 @@ public:
      * @return true if the schema could be saved.
      **/
     bool
-    saveToFile(const vespalib::stringref & fileName) const;
+    saveToFile(const vespalib::string & fileName) const;
 
     vespalib::string toString() const;
 
diff --git a/searchcore/CMakeLists.txt b/searchcore/CMakeLists.txt
index 89e6493cfbc..9d996d96dc7 100644
--- a/searchcore/CMakeLists.txt
+++ b/searchcore/CMakeLists.txt
@@ -129,6 +129,7 @@ vespa_define_module(
     src/tests/proton/proton
     src/tests/proton/proton_config_fetcher
     src/tests/proton/proton_configurer
+    src/tests/proton/proton_disk_layout
     src/tests/proton/reference/gid_to_lid_change_handler
     src/tests/proton/reference/gid_to_lid_change_listener
     src/tests/proton/reference/gid_to_lid_change_registrator
diff --git a/searchcore/src/tests/proton/attribute/attribute_directory/attribute_directory_test.cpp b/searchcore/src/tests/proton/attribute/attribute_directory/attribute_directory_test.cpp
index 37cf201e354..f95ea478ce1 100644
--- a/searchcore/src/tests/proton/attribute/attribute_directory/attribute_directory_test.cpp
+++ b/searchcore/src/tests/proton/attribute/attribute_directory/attribute_directory_test.cpp
@@ -137,6 +137,7 @@ struct Fixture : public DirectoryHandler
         EXPECT_TRUE(hasAttributeDir(dir));
         auto writer = dir->getWriter();
         writer->createInvalidSnapshot(serialNum);
+        vespalib::mkdir(writer->getSnapshotDir(serialNum), false);
         writer->markValidSnapshot(serialNum);
         TEST_DO(assertAttributeDiskDir("foo"));
     }
@@ -162,6 +163,7 @@ struct Fixture : public DirectoryHandler
         auto dir = createFooAttrDir();
         auto writer = dir->getWriter();
         writer->createInvalidSnapshot(serialNum);
+        vespalib::mkdir(writer->getSnapshotDir(serialNum), false);
         writer->markValidSnapshot(serialNum);
     }
 
@@ -208,8 +210,10 @@ TEST_F("Test that we can prune attribute snapshots", Fixture)
     TEST_DO(f.assertNotAttributeDiskDir("foo"));
     auto writer = dir->getWriter();
     writer->createInvalidSnapshot(2);
+    vespalib::mkdir(writer->getSnapshotDir(2), false);
     writer->markValidSnapshot(2);
     writer->createInvalidSnapshot(4);
+    vespalib::mkdir(writer->getSnapshotDir(4), false);
     writer->markValidSnapshot(4);
     writer.reset();
     TEST_DO(f.assertAttributeDiskDir("foo"));
diff --git a/searchcore/src/tests/proton/index/indexmanager_test.cpp b/searchcore/src/tests/proton/index/indexmanager_test.cpp
index 6a098667be8..d88199e6ae8 100644
--- a/searchcore/src/tests/proton/index/indexmanager_test.cpp
+++ b/searchcore/src/tests/proton/index/indexmanager_test.cpp
@@ -662,7 +662,7 @@ TEST_F("requireThatSerialNumberIsReadOnLoad", Fixture) {
 void crippleFusion(uint32_t fusionId) {
     vespalib::asciistream ost;
     ost << index_dir << "/index.flush." << fusionId << "/serial.dat";
-    FastOS_File(ost.str().c_str()).Delete();
+    FastOS_File(ost.str().data()).Delete();
 }
 
 TEST_F("requireThatFailedFusionIsRetried", Fixture) {
diff --git a/searchcore/src/tests/proton/matching/match_loop_communicator/match_loop_communicator_test.cpp b/searchcore/src/tests/proton/matching/match_loop_communicator/match_loop_communicator_test.cpp
index 2bfd907b4a3..b64b2526f06 100644
--- a/searchcore/src/tests/proton/matching/match_loop_communicator/match_loop_communicator_test.cpp
+++ b/searchcore/src/tests/proton/matching/match_loop_communicator/match_loop_communicator_test.cpp
@@ -8,20 +8,21 @@ using namespace proton::matching;
 using vespalib::Box;
 using vespalib::make_box;
 
-typedef MatchLoopCommunicator::Range Range;
-typedef MatchLoopCommunicator::RangePair RangePair;
-typedef MatchLoopCommunicator::feature_t feature_t;
-typedef MatchLoopCommunicator::Matches Matches;
+using Range = MatchLoopCommunicator::Range;
+using RangePair = MatchLoopCommunicator::RangePair;
+using Matches = MatchLoopCommunicator::Matches;
+using Hit = MatchLoopCommunicator::Hit;
+using Hits = MatchLoopCommunicator::Hits;
 
-std::vector<feature_t> makeScores(size_t id) {
+Hits makeScores(size_t id) {
     switch (id) {
-    case 0: return make_box<feature_t>(5.4, 4.4, 3.4, 2.4, 1.4);
-    case 1: return make_box<feature_t>(5.3, 4.3, 3.3, 2.3, 1.3);
-    case 2: return make_box<feature_t>(5.2, 4.2, 3.2, 2.2, 1.2);
-    case 3: return make_box<feature_t>(5.1, 4.1, 3.1, 2.1, 1.1);
-    case 4: return make_box<feature_t>(5.0, 4.0, 3.0, 2.0, 1.0);
+    case 0: return make_box<Hit>({1, 5.4}, {2, 4.4}, {3, 3.4}, {4, 2.4}, {5, 1.4});
+    case 1: return make_box<Hit>({11, 5.3}, {12, 4.3}, {13, 3.3}, {14, 2.3}, {15, 1.3});
+    case 2: return make_box<Hit>({21, 5.2}, {22, 4.2}, {23, 3.2}, {24, 2.2}, {25, 1.2});
+    case 3: return make_box<Hit>({31, 5.1}, {32, 4.1}, {33, 3.1}, {34, 2.1}, {35, 1.1});
+    case 4: return make_box<Hit>({41, 5.0}, {42, 4.0}, {43, 3.0}, {44, 2.0}, {45, 1.0});
     }
-    return Box<feature_t>();
+    return Box<Hit>();
 }
 
 RangePair makeRanges(size_t id) {
@@ -35,43 +36,65 @@ RangePair makeRanges(size_t id) {
     return std::make_pair(Range(-50, -60), Range(60, 50));
 }
 
+void equal(size_t count, const Hits & a, const Hits & b) {
+    EXPECT_EQUAL(count, b.size());
+    for (size_t i(0); i < count; i++) {
+        EXPECT_EQUAL(a[i].first, b[i].first);
+        EXPECT_EQUAL(a[i].second , b[i].second);
+    }
+}
+
+struct EveryOdd : public search::queryeval::IDiversifier {
+    bool accepted(uint32_t docId) override {
+        return docId & 0x01;
+    }
+};
+
 TEST_F("require that selectBest gives appropriate results for single thread", MatchLoopCommunicator(num_threads, 3)) {
-    EXPECT_EQUAL(2u, f1.selectBest(make_box<feature_t>(5, 4)));
-    EXPECT_EQUAL(3u, f1.selectBest(make_box<feature_t>(5, 4, 3)));
-    EXPECT_EQUAL(3u, f1.selectBest(make_box<feature_t>(5, 4, 3, 2)));
+    TEST_DO(equal(2u, make_box<Hit>({1, 5}, {2, 4}), f1.selectBest(make_box<Hit>({1, 5}, {2, 4}))));
+    TEST_DO(equal(3u, make_box<Hit>({1, 5}, {2, 4}, {3, 3}), f1.selectBest(make_box<Hit>({1, 5}, {2, 4}, {3, 3}))));
+    TEST_DO(equal(3u, make_box<Hit>({1, 5}, {2, 4}, {3, 3}), f1.selectBest(make_box<Hit>({1, 5}, {2, 4}, {3, 3}, {4, 2}))));
+}
+
+TEST_F("require that selectBest gives appropriate results for single thread with filter",
+       MatchLoopCommunicator(num_threads, 3, std::make_unique<EveryOdd>()))
+{
+    TEST_DO(equal(1u, make_box<Hit>({1, 5}), f1.selectBest(make_box<Hit>({1, 5}, {2, 4}))));
+    TEST_DO(equal(2u, make_box<Hit>({1, 5}, {3, 3}), f1.selectBest(make_box<Hit>({1, 5}, {2, 4}, {3, 3}))));
+    TEST_DO(equal(3u, make_box<Hit>({1, 5}, {3, 3}, {5, 1}), f1.selectBest(make_box<Hit>({1, 5}, {2, 4}, {3, 3}, {4, 2}, {5, 1}, {6, 0}))));
 }
 
 TEST_MT_F("require that selectBest works with no hits", 10, MatchLoopCommunicator(num_threads, 10)) {
-    EXPECT_EQUAL(0u, f1.selectBest(Box<feature_t>()));
+    EXPECT_TRUE(f1.selectBest(Box<Hit>()).empty());
 }
 
 TEST_MT_F("require that selectBest works with too many hits from all threads", 5, MatchLoopCommunicator(num_threads, 13)) {
     if (thread_id < 3) {
-        EXPECT_EQUAL(3u, f1.selectBest(makeScores(thread_id)));
+        TEST_DO(equal(3u, makeScores(thread_id), f1.selectBest(makeScores(thread_id))));
     } else {
-        EXPECT_EQUAL(2u, f1.selectBest(makeScores(thread_id)));
+        TEST_DO(equal(2u, makeScores(thread_id), f1.selectBest(makeScores(thread_id))));
     }
 }
 
 TEST_MT_F("require that selectBest works with some exhausted threads", 5, MatchLoopCommunicator(num_threads, 22)) {
     if (thread_id < 2) {
-        EXPECT_EQUAL(5u, f1.selectBest(makeScores(thread_id)));
+        TEST_DO(equal(5u, makeScores(thread_id), f1.selectBest(makeScores(thread_id))));
     } else {
-        EXPECT_EQUAL(4u, f1.selectBest(makeScores(thread_id)));
+        TEST_DO(equal(4u, makeScores(thread_id), f1.selectBest(makeScores(thread_id))));
     }
 }
 
 TEST_MT_F("require that selectBest can select all hits from all threads", 5, MatchLoopCommunicator(num_threads, 100)) {
-    EXPECT_EQUAL(5u, f1.selectBest(makeScores(thread_id)));
+    EXPECT_EQUAL(5u, f1.selectBest(makeScores(thread_id)).size());
 }
 
 TEST_MT_F("require that selectBest works with some empty threads", 10, MatchLoopCommunicator(num_threads, 7)) {
     if (thread_id < 2) {
-        EXPECT_EQUAL(2u, f1.selectBest(makeScores(thread_id)));
+        TEST_DO(equal(2u, makeScores(thread_id), f1.selectBest(makeScores(thread_id))));
     } else if (thread_id < 5) {
-        EXPECT_EQUAL(1u, f1.selectBest(makeScores(thread_id)));
+        TEST_DO(equal(1u, makeScores(thread_id), f1.selectBest(makeScores(thread_id))));
     } else {
-        EXPECT_EQUAL(0u, f1.selectBest(makeScores(thread_id)));
+        EXPECT_TRUE(f1.selectBest(makeScores(thread_id)).empty());
     }
 }
 
diff --git a/searchcore/src/tests/proton/matching/matching_stats_test.cpp b/searchcore/src/tests/proton/matching/matching_stats_test.cpp
index 8b58a9e271c..85960de2021 100644
--- a/searchcore/src/tests/proton/matching/matching_stats_test.cpp
+++ b/searchcore/src/tests/proton/matching/matching_stats_test.cpp
@@ -240,6 +240,65 @@ TEST("requireThatPartitionsAreAddedCorrectly") {
     EXPECT_EQUAL(1.0, all1.getPartition(1).wait_time_max());
 }
 
+TEST("requireThatSoftDoomIsSetAndAdded") {
+    MatchingStats stats;
+    MatchingStats stats2;
+    EXPECT_EQUAL(0ul, stats.softDoomed());
+    EXPECT_EQUAL(0.5, stats.softDoomFactor());
+    stats.softDoomFactor(0.7);
+    stats.softDoomed(3);
+    EXPECT_EQUAL(3ul, stats.softDoomed());
+    EXPECT_EQUAL(0.7, stats.softDoomFactor());
+    stats2.add(stats);
+    EXPECT_EQUAL(3ul, stats2.softDoomed());
+    EXPECT_EQUAL(0.5, stats2.softDoomFactor());  // Not affected by add
+}
+
+TEST("requireThatSoftDoomFacorIsComputedCorrectlyForDownAdjustment") {
+    MatchingStats stats;
+    EXPECT_EQUAL(0ul, stats.softDoomed());
+    EXPECT_EQUAL(0.5, stats.softDoomFactor());
+    stats.softDoomed(1);
+    stats.updatesoftDoomFactor(1.0, 0.5, 2.0);
+    EXPECT_EQUAL(1ul, stats.softDoomed());
+    EXPECT_EQUAL(0.47, stats.softDoomFactor());
+    stats.updatesoftDoomFactor(1.0, 0.5, 2.0);
+    EXPECT_EQUAL(1ul, stats.softDoomed());
+    EXPECT_EQUAL(0.44, stats.softDoomFactor());
+    stats.updatesoftDoomFactor(0.0009, 0.5, 2.0);   // hard limits less than 1ms should be ignored
+    EXPECT_EQUAL(1ul, stats.softDoomed());
+    EXPECT_EQUAL(0.44, stats.softDoomFactor());
+    stats.updatesoftDoomFactor(1.0, 0.0009, 2.0);   // soft limits less than 1ms should be ignored
+    EXPECT_EQUAL(1ul, stats.softDoomed());
+    EXPECT_EQUAL(0.44, stats.softDoomFactor());
+    stats.updatesoftDoomFactor(1.0, 0.5, 10.0);      // Prevent changes above 10%
+    EXPECT_EQUAL(1ul, stats.softDoomed());
+    EXPECT_EQUAL(0.396, stats.softDoomFactor());
+}
+
+TEST("requireThatSoftDoomFacorIsComputedCorrectlyForUpAdjustment") {
+    MatchingStats stats;
+    EXPECT_EQUAL(0ul, stats.softDoomed());
+    EXPECT_EQUAL(0.5, stats.softDoomFactor());
+    stats.softDoomed(1);
+    stats.updatesoftDoomFactor(1.0, 0.9, 0.1);
+    EXPECT_EQUAL(1ul, stats.softDoomed());
+    EXPECT_EQUAL(0.508, stats.softDoomFactor());
+    stats.updatesoftDoomFactor(1.0, 0.9, 0.1);
+    EXPECT_EQUAL(1ul, stats.softDoomed());
+    EXPECT_EQUAL(0.516, stats.softDoomFactor());
+    stats.updatesoftDoomFactor(0.0009, 0.9, 0.1);   // hard limits less than 1ms should be ignored
+    EXPECT_EQUAL(1ul, stats.softDoomed());
+    EXPECT_EQUAL(0.516, stats.softDoomFactor());
+    stats.updatesoftDoomFactor(1.0, 0.0009, 0.1);   // soft limits less than 1ms should be ignored
+    EXPECT_EQUAL(1ul, stats.softDoomed());
+    EXPECT_EQUAL(0.516, stats.softDoomFactor());
+    stats.softDoomFactor(0.1);
+    stats.updatesoftDoomFactor(1.0, 0.9, 0.001);      // Prevent changes above 5%
+    EXPECT_EQUAL(1ul, stats.softDoomed());
+    EXPECT_EQUAL(0.105, stats.softDoomFactor());
+}
+
 TEST_MAIN() {
     TEST_RUN_ALL();
 }
diff --git a/searchcore/src/tests/proton/matching/query_test.cpp b/searchcore/src/tests/proton/matching/query_test.cpp
index 61823a17f09..82aab72068d 100644
--- a/searchcore/src/tests/proton/matching/query_test.cpp
+++ b/searchcore/src/tests/proton/matching/query_test.cpp
@@ -63,6 +63,12 @@ using search::queryeval::SearchIterator;
 using search::queryeval::SimpleBlueprint;
 using search::queryeval::SimpleResult;
 using search::queryeval::ParallelWeakAndBlueprint;
+using search::queryeval::RankBlueprint;
+using search::queryeval::AndBlueprint;
+using search::queryeval::IntermediateBlueprint;
+using search::queryeval::AndNotBlueprint;
+using search::queryeval::SourceBlenderBlueprint;
+
 using std::string;
 using std::vector;
 namespace fef_test = search::fef::test;
@@ -106,6 +112,8 @@ class Test : public vespalib::TestApp {
     void requireThatWeakAndBlueprintsAreCreatedCorrectly();
     void requireThatParallelWandBlueprintsAreCreatedCorrectly();
     void requireThatWhiteListBlueprintCanBeUsed();
+    void requireThatRankBlueprintStaysOnTopAfterWhiteListing();
+    void requireThatAndNotBlueprintStaysOnTopAfterWhiteListing();
     void requireThatSameElementTermsAreProperlyPrefixed();
     void requireThatSameElementDoesNotAllocateMatchData();
     void requireThatSameElementIteratorsCanBeBuilt();
@@ -879,6 +887,54 @@ Test::requireThatWhiteListBlueprintCanBeUsed()
     EXPECT_EQUAL(exp, act);
 }
 
+template<typename T1, typename T2>
+void verifyThatRankBlueprintAndAndNotStaysOnTopAfterWhiteListing(QueryBuilder<ProtonNodeTypes> & builder) {
+    builder.addStringTerm("foo", field, field_id, string_weight);
+    builder.addStringTerm("bar", field, field_id, string_weight);
+    builder.addStringTerm("baz", field, field_id, string_weight);
+    std::string stackDump = StackDumpCreator::create(*builder.build());
+    Query query;
+    query.buildTree(stackDump, "", ViewResolver(), plain_index_env);
+    FakeSearchContext context(42);
+    context.addIdx(0).idx(0).getFake()
+            .addResult(field, "foo", FakeResult().doc(1));
+    context.setLimit(42);
+
+    query.setWhiteListBlueprint(std::make_unique<SimpleBlueprint>(SimpleResult()));
+
+    FakeRequestContext requestContext;
+    MatchDataLayout mdl;
+    query.reserveHandles(requestContext, context, mdl);
+    const IntermediateBlueprint * root = dynamic_cast<const T1 *>(query.peekRoot());
+    ASSERT_TRUE(root != nullptr);
+    EXPECT_EQUAL(2u, root->childCnt());
+    const IntermediateBlueprint * second = dynamic_cast<const T2 *>(&root->getChild(0));
+    ASSERT_TRUE(second != nullptr);
+    EXPECT_EQUAL(2u, second->childCnt());
+    const AndBlueprint * first = dynamic_cast<const AndBlueprint *>(&second->getChild(0));
+    ASSERT_TRUE(first != nullptr);
+    EXPECT_EQUAL(2u, first->childCnt());
+    EXPECT_TRUE(dynamic_cast<const SourceBlenderBlueprint *>(&first->getChild(0)));
+    EXPECT_TRUE(dynamic_cast<const SimpleBlueprint *>(&first->getChild(1)));
+    EXPECT_TRUE(dynamic_cast<const SourceBlenderBlueprint *>(&second->getChild(1)));
+    EXPECT_TRUE(dynamic_cast<const SourceBlenderBlueprint *>(&root->getChild(1)));
+}
+
+void Test::requireThatRankBlueprintStaysOnTopAfterWhiteListing() {
+    QueryBuilder<ProtonNodeTypes> builder;
+    builder.addRank(2);
+    builder.addAndNot(2);
+    verifyThatRankBlueprintAndAndNotStaysOnTopAfterWhiteListing<RankBlueprint, AndNotBlueprint>(builder);
+}
+
+void Test::requireThatAndNotBlueprintStaysOnTopAfterWhiteListing() {
+    QueryBuilder<ProtonNodeTypes> builder;
+    builder.addAndNot(2);
+    builder.addRank(2);
+    verifyThatRankBlueprintAndAndNotStaysOnTopAfterWhiteListing<AndNotBlueprint, RankBlueprint>(builder);
+}
+
+
 search::query::Node::UP
 make_same_element_stack_dump(const vespalib::string &prefix, const vespalib::string &term_prefix)
 {
@@ -984,6 +1040,8 @@ Test::Main()
     TEST_CALL(requireThatWeakAndBlueprintsAreCreatedCorrectly);
     TEST_CALL(requireThatParallelWandBlueprintsAreCreatedCorrectly);
     TEST_CALL(requireThatWhiteListBlueprintCanBeUsed);
+    TEST_CALL(requireThatRankBlueprintStaysOnTopAfterWhiteListing);
+    TEST_CALL(requireThatAndNotBlueprintStaysOnTopAfterWhiteListing);
     TEST_CALL(requireThatSameElementTermsAreProperlyPrefixed);
     TEST_CALL(requireThatSameElementDoesNotAllocateMatchData);
     TEST_CALL(requireThatSameElementIteratorsCanBeBuilt);
diff --git a/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp b/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp
index 045c8de9384..dfb1268aaa6 100644
--- a/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp
+++ b/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp
@@ -12,10 +12,11 @@
 #include <vespa/searchcore/proton/server/bootstrapconfig.h>
 #include <vespa/searchcore/proton/server/bootstrapconfigmanager.h>
 #include <vespa/searchcore/proton/server/documentdbconfigmanager.h>
-#include <vespa/searchcore/proton/server/i_document_db_config_owner.h>
+#include <vespa/searchcore/proton/server/document_db_config_owner.h>
 #include <vespa/searchcore/proton/server/proton_config_snapshot.h>
 #include <vespa/searchcore/proton/server/proton_configurer.h>
 #include <vespa/searchcore/proton/server/i_proton_configurer_owner.h>
+#include <vespa/searchcore/proton/server/i_proton_disk_layout.h>
 #include <vespa/searchsummary/config/config-juniperrc.h>
 #include <vespa/searchcore/config/config-ranking-constants.h>
 #include <vespa/vespalib/testkit/testapp.h>
@@ -208,13 +209,13 @@ struct ConfigFixture {
 
 struct MyProtonConfigurerOwner;
 
-struct MyDocumentDBConfigOwner : public IDocumentDBConfigOwner
+struct MyDocumentDBConfigOwner : public DocumentDBConfigOwner
 {
     vespalib::string _name;
     MyProtonConfigurerOwner &_owner;
     MyDocumentDBConfigOwner(const vespalib::string &name,
                             MyProtonConfigurerOwner &owner)
-        : IDocumentDBConfigOwner(),
+        : DocumentDBConfigOwner(),
           _name(name),
           _owner(owner)
     {
@@ -224,28 +225,43 @@ struct MyDocumentDBConfigOwner : public IDocumentDBConfigOwner
     void reconfigure(const DocumentDBConfig::SP & config) override;
 };
 
-struct MyProtonConfigurerOwner : public IProtonConfigurerOwner
+struct MyLog
+{
+    std::vector<vespalib::string> _log;
+
+    MyLog()
+        : _log()
+    {
+    }
+
+    void appendLog(vespalib::string logEntry)
+    {
+        _log.emplace_back(logEntry);
+    }
+};
+
+struct MyProtonConfigurerOwner : public IProtonConfigurerOwner,
+                                 public MyLog
 {
     using InitializeThreads = std::shared_ptr<vespalib::ThreadStackExecutorBase>;
     vespalib::ThreadStackExecutor _executor;
     std::map<DocTypeName, std::shared_ptr<MyDocumentDBConfigOwner>> _dbs;
-    std::vector<vespalib::string> _log;
 
     MyProtonConfigurerOwner()
         : IProtonConfigurerOwner(),
+          MyLog(),
           _executor(1, 128 * 1024),
-          _dbs(),
-          _log()
+          _dbs()
     {
     }
     virtual ~MyProtonConfigurerOwner() { }
 
-    virtual IDocumentDBConfigOwner *addDocumentDB(const DocTypeName &docTypeName,
-                                                  document::BucketSpace bucketSpace,
-                                                  const vespalib::string &configId,
-                                                  const std::shared_ptr<BootstrapConfig> &bootstrapConfig,
-                                                  const std::shared_ptr<DocumentDBConfig> &documentDBConfig,
-                                                  InitializeThreads initializeThreads) override
+    virtual std::shared_ptr<DocumentDBConfigOwner> addDocumentDB(const DocTypeName &docTypeName,
+                                                                 document::BucketSpace bucketSpace,
+                                                                 const vespalib::string &configId,
+                                                                 const std::shared_ptr<BootstrapConfig> &bootstrapConfig,
+                                                                 const std::shared_ptr<DocumentDBConfig> &documentDBConfig,
+                                                                 InitializeThreads initializeThreads) override
     {
         (void) bucketSpace;
         (void) configId;
@@ -257,7 +273,7 @@ struct MyProtonConfigurerOwner : public IProtonConfigurerOwner
         std::ostringstream os;
         os << "add db " << docTypeName.getName() << " " << documentDBConfig->getGeneration();
         _log.push_back(os.str());
-        return db.get();
+        return db;
     }
     virtual void removeDocumentDB(const DocTypeName &docTypeName) override {
         ASSERT_FALSE(_dbs.find(docTypeName) == _dbs.end());
@@ -286,17 +302,48 @@ MyDocumentDBConfigOwner::reconfigure(const DocumentDBConfig::SP & config)
     _owner.reconfigureDocumentDB(_name, config);
 }
 
+struct MyProtonDiskLayout : public IProtonDiskLayout
+{
+    MyLog &_log;
+
+    MyProtonDiskLayout(MyLog &myLog)
+        : _log(myLog)
+    {
+    }
+    void remove(const DocTypeName &docTypeName) override {
+        std::ostringstream os;
+        os << "remove dbdir " << docTypeName.getName();
+        _log.appendLog(os.str());
+    }
+    void initAndPruneUnused(const std::set<DocTypeName> &docTypeNames) override {
+        std::ostringstream os;
+        os << "initial dbs ";
+        bool first = true;
+        for (const auto &docTypeName : docTypeNames) {
+            if (!first) {
+                os << ",";
+            }
+            first = false;
+            os << docTypeName.getName();
+        }
+        _log.appendLog(os.str());
+    }
+};
+
 struct Fixture
 {
     MyProtonConfigurerOwner _owner;
     ConfigFixture _config;
+    std::unique_ptr<IProtonDiskLayout> _diskLayout;
     ProtonConfigurer _configurer;
 
     Fixture()
         : _owner(),
           _config("test"),
-          _configurer(_owner._executor, _owner)
+          _diskLayout(),
+          _configurer(_owner._executor, _owner, _diskLayout)
     {
+        _diskLayout = std::make_unique<MyProtonDiskLayout>(_owner);
     }
     ~Fixture() { }
 
@@ -338,14 +385,14 @@ TEST_F("require that nothing is applied before initial config", Fixture())
 TEST_F("require that initial config is applied", Fixture())
 {
     f.applyInitialConfig();
-    TEST_DO(f1.assertLog({"apply config 2", "add db _alwaysthere_ 2"}));
+    TEST_DO(f1.assertLog({"initial dbs _alwaysthere_", "apply config 2", "add db _alwaysthere_ 2"}));
 }
 
 TEST_F("require that new config is blocked", Fixture())
 {
     f.applyInitialConfig();
     f.reconfigure();
-    TEST_DO(f1.assertLog({"apply config 2", "add db _alwaysthere_ 2"}));
+    TEST_DO(f1.assertLog({"initial dbs _alwaysthere_", "apply config 2", "add db _alwaysthere_ 2"}));
 }
 
 TEST_F("require that new config can be unblocked", Fixture())
@@ -353,14 +400,14 @@ TEST_F("require that new config can be unblocked", Fixture())
     f.applyInitialConfig();
     f.reconfigure();
     f.allowReconfig();
-    TEST_DO(f1.assertLog({"apply config 2", "add db _alwaysthere_ 2", "apply config 3", "reconf db _alwaysthere_ 3"}));
+    TEST_DO(f1.assertLog({"initial dbs _alwaysthere_", "apply config 2", "add db _alwaysthere_ 2", "apply config 3", "reconf db _alwaysthere_ 3"}));
 }
 
 TEST_F("require that initial config is not reapplied due to config unblock", Fixture())
 {
     f.applyInitialConfig();
     f.allowReconfig();
-    TEST_DO(f1.assertLog({"apply config 2", "add db _alwaysthere_ 2"}));
+    TEST_DO(f1.assertLog({"initial dbs _alwaysthere_", "apply config 2", "add db _alwaysthere_ 2"}));
 }
 
 TEST_F("require that we can add document db", Fixture())
@@ -369,7 +416,7 @@ TEST_F("require that we can add document db", Fixture())
     f.allowReconfig();
     f.addDocType("foobar");
     f.reconfigure();
-    TEST_DO(f1.assertLog({"apply config 2", "add db _alwaysthere_ 2", "apply config 3","reconf db _alwaysthere_ 3", "add db foobar 3"}));
+    TEST_DO(f1.assertLog({"initial dbs _alwaysthere_", "apply config 2", "add db _alwaysthere_ 2", "apply config 3","reconf db _alwaysthere_ 3", "add db foobar 3"}));
 }
 
 TEST_F("require that we can remove document db", Fixture())
@@ -379,7 +426,7 @@ TEST_F("require that we can remove document db", Fixture())
     f.allowReconfig();
     f.removeDocType("foobar");
     f.reconfigure();
-    TEST_DO(f1.assertLog({"apply config 2", "add db _alwaysthere_ 2", "add db foobar 2", "apply config 3","reconf db _alwaysthere_ 3", "remove db foobar"}));
+    TEST_DO(f1.assertLog({"initial dbs _alwaysthere_,foobar", "apply config 2", "add db _alwaysthere_ 2", "add db foobar 2", "apply config 3","reconf db _alwaysthere_ 3", "remove db foobar", "remove dbdir foobar"}));
 }
 
 TEST_F("require that document db adds and reconfigs are intermingled", Fixture())
@@ -392,7 +439,7 @@ TEST_F("require that document db adds and reconfigs are intermingled", Fixture()
     f.addDocType("foobar");
     f.addDocType("zbar");
     f.reconfigure();
-    TEST_DO(f1.assertLog({"apply config 2", "add db _alwaysthere_ 2", "add db foobar 2", "apply config 3","reconf db _alwaysthere_ 3", "add db abar 3", "reconf db foobar 3", "add db zbar 3"}));
+    TEST_DO(f1.assertLog({"initial dbs _alwaysthere_,foobar", "apply config 2", "add db _alwaysthere_ 2", "add db foobar 2", "apply config 3","reconf db _alwaysthere_ 3", "add db abar 3", "reconf db foobar 3", "add db zbar 3"}));
 }
 
 TEST_F("require that document db removes are applied at end", Fixture())
@@ -403,7 +450,7 @@ TEST_F("require that document db removes are applied at end", Fixture())
     f.allowReconfig();
     f.removeDocType("abar");
     f.reconfigure();
-    TEST_DO(f1.assertLog({"apply config 2", "add db _alwaysthere_ 2", "add db abar 2", "add db foobar 2", "apply config 3","reconf db _alwaysthere_ 3", "reconf db foobar 3", "remove db abar"}));
+    TEST_DO(f1.assertLog({"initial dbs _alwaysthere_,abar,foobar", "apply config 2", "add db _alwaysthere_ 2", "add db abar 2", "add db foobar 2", "apply config 3","reconf db _alwaysthere_ 3", "reconf db foobar 3", "remove db abar", "remove dbdir abar"}));
 }
 
 TEST_F("require that new configs can be blocked again", Fixture())
@@ -413,7 +460,7 @@ TEST_F("require that new configs can be blocked again", Fixture())
     f.allowReconfig();
     f.disableReconfig();
     f.reconfigure();
-    TEST_DO(f1.assertLog({"apply config 2", "add db _alwaysthere_ 2", "apply config 3", "reconf db _alwaysthere_ 3"}));
+    TEST_DO(f1.assertLog({"initial dbs _alwaysthere_", "apply config 2", "add db _alwaysthere_ 2", "apply config 3", "reconf db _alwaysthere_ 3"}));
 }
 
 TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/searchcore/src/tests/proton/proton_disk_layout/CMakeLists.txt b/searchcore/src/tests/proton/proton_disk_layout/CMakeLists.txt
new file mode 100644
index 00000000000..f63fa21a954
--- /dev/null
+++ b/searchcore/src/tests/proton/proton_disk_layout/CMakeLists.txt
@@ -0,0 +1,9 @@
+# Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchcore_proton_disk_layout_test_app TEST
+    SOURCES
+    proton_disk_layout_test.cpp
+    DEPENDS
+    searchcore_server
+    searchcore_fconfig
+)
+vespa_add_test(NAME searchcore_proton_disk_layout_test_app COMMAND searchcore_proton_disk_layout_test_app)
diff --git a/searchcore/src/tests/proton/proton_disk_layout/proton_disk_layout_test.cpp b/searchcore/src/tests/proton/proton_disk_layout/proton_disk_layout_test.cpp
new file mode 100644
index 00000000000..edb4250ce76
--- /dev/null
+++ b/searchcore/src/tests/proton/proton_disk_layout/proton_disk_layout_test.cpp
@@ -0,0 +1,178 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/searchcore/proton/server/proton_disk_layout.h>
+#include <vespa/searchcore/proton/common/doctypename.h>
+#include <vespa/searchlib/index/dummyfileheadercontext.h>
+#include <vespa/searchlib/transactionlog/translogserver.h>
+#include <vespa/searchlib/transactionlog/translogclient.h>
+#include <vespa/vespalib/io/fileutil.h>
+#include <vespa/vespalib/testkit/testapp.h>
+#include <vespa/vespalib/test/insertion_operators.h>
+#include <vespa/vespalib/util/stringfmt.h>
+
+using search::index::DummyFileHeaderContext;
+using search::transactionlog::TransLogClient;
+using search::transactionlog::TransLogServer;
+using proton::DocTypeName;
+using proton::ProtonDiskLayout;
+
+static constexpr unsigned int tlsPort = 9018;
+
+static const vespalib::string baseDir("testdb");
+static const vespalib::string documentsDir(baseDir + "/documents");
+
+struct FixtureBase
+{
+    FixtureBase() { vespalib::rmdir(baseDir, true); }
+    ~FixtureBase() { vespalib::rmdir(baseDir, true); }
+};
+
+struct DiskLayoutFixture {
+    DummyFileHeaderContext  _fileHeaderContext;
+    TransLogServer          _tls;
+    vespalib::string        _tlsSpec;
+    ProtonDiskLayout        _diskLayout;
+
+    DiskLayoutFixture();
+    ~DiskLayoutFixture();
+
+    void createDirs(const std::set<vespalib::string> &dirs) {
+        for (const auto &dir : dirs) {
+            vespalib::mkdir(documentsDir + "/" + dir, false);
+        }
+    }
+    void createDomains(const std::set<vespalib::string> &domains) {
+        TransLogClient tlc(_tlsSpec);
+        for (const auto &domain : domains) {
+            ASSERT_TRUE(tlc.create(domain));
+        }
+    }
+
+    std::set<vespalib::string> listDomains() {
+        std::vector<vespalib::string> domainVector;
+        TransLogClient tlc(_tlsSpec);
+        ASSERT_TRUE(tlc.listDomains(domainVector));
+        std::set<vespalib::string> domains;
+        for (const auto &domain : domainVector) {
+            domains.emplace(domain);
+        }
+        return domains;
+    }
+
+    std::set<vespalib::string> listDirs() {
+        std::set<vespalib::string> dirs;
+        auto names = vespalib::listDirectory(documentsDir);
+        for (const auto &name : names) {
+            if (vespalib::isDirectory(documentsDir + "/" + name)) {
+                dirs.emplace(name);
+            }
+        }
+        return dirs;
+    }
+
+    void initAndPruneUnused(const std::set<vespalib::string> names)
+    {
+        std::set<DocTypeName> docTypeNames;
+        for (const auto &name: names) {
+            docTypeNames.emplace(name);
+        }
+        _diskLayout.initAndPruneUnused(docTypeNames);
+    }
+
+    void assertDirs(const std::set<vespalib::string> &expDirs) {
+        EXPECT_EQUAL(expDirs, listDirs());
+    }
+
+    void assertDomains(const std::set<vespalib::string> &expDomains)
+    {
+        EXPECT_EQUAL(expDomains, listDomains());
+    }
+};
+
+DiskLayoutFixture::DiskLayoutFixture()
+    : _fileHeaderContext(),
+      _tls("tls", tlsPort, baseDir, _fileHeaderContext),
+      _tlsSpec(vespalib::make_string("tcp/localhost:%u", tlsPort)),
+      _diskLayout(baseDir, _tlsSpec)
+{
+}
+
+DiskLayoutFixture::~DiskLayoutFixture() = default;
+
+struct Fixture : public FixtureBase, public DiskLayoutFixture
+{
+    Fixture()
+        : FixtureBase(),
+          DiskLayoutFixture()
+    {
+    }
+};
+
+TEST_F("require that empty config is ok", Fixture) {
+    TEST_DO(f.assertDirs({}));
+    TEST_DO(f.assertDomains({}));
+}
+
+TEST_F("require that disk layout is preserved", FixtureBase)
+{
+    {
+        DiskLayoutFixture diskLayout;
+        diskLayout.createDirs({"foo", "bar"});
+        diskLayout.createDomains({"bar", "baz"});
+    }
+    {
+        DiskLayoutFixture diskLayout;
+        TEST_DO(diskLayout.assertDirs({"foo", "bar"}));
+        TEST_DO(diskLayout.assertDomains({"bar", "baz"}));
+    }
+}
+
+TEST_F("require that used dir is preserved", Fixture)
+{
+    f.createDirs({"foo"});
+    f.createDomains({"foo"});
+    f.initAndPruneUnused({"foo"});
+    TEST_DO(f.assertDirs({"foo"}));
+    TEST_DO(f.assertDomains({"foo"}));
+}
+
+TEST_F("require that unused dir is removed", Fixture)
+{
+    f.createDirs({"foo"});
+    f.createDomains({"foo"});
+    f.initAndPruneUnused({"bar"});
+    TEST_DO(f.assertDirs({}));
+    TEST_DO(f.assertDomains({}));
+}
+
+TEST_F("require that interrupted remove is completed", Fixture)
+{
+    f.createDirs({"foo.removed"});
+    f.createDomains({"foo"});
+    f.initAndPruneUnused({"foo"});
+    TEST_DO(f.assertDirs({}));
+    TEST_DO(f.assertDomains({}));
+}
+
+TEST_F("require that early interrupted remove is completed", Fixture)
+{
+    f.createDirs({"foo", "foo.removed"});
+    f.createDomains({"foo"});
+    f.initAndPruneUnused({"foo"});
+    TEST_DO(f.assertDirs({}));
+    TEST_DO(f.assertDomains({}));
+}
+
+TEST_F("require that live document db dir remove works", Fixture)
+{
+    f.createDirs({"foo"});
+    f.createDomains({"foo"});
+    f.initAndPruneUnused({"foo"});
+    TEST_DO(f.assertDirs({"foo"}));
+    TEST_DO(f.assertDomains({"foo"}));
+    f._diskLayout.remove(DocTypeName("foo"));
+    TEST_DO(f.assertDirs({}));
+    TEST_DO(f.assertDomains({}));
+}
+
+TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/searchcore/src/tests/proton/reprocessing/attribute_reprocessing_initializer/attribute_reprocessing_initializer_test.cpp b/searchcore/src/tests/proton/reprocessing/attribute_reprocessing_initializer/attribute_reprocessing_initializer_test.cpp
index 7be774f7291..72e558fd25f 100644
--- a/searchcore/src/tests/proton/reprocessing/attribute_reprocessing_initializer/attribute_reprocessing_initializer_test.cpp
+++ b/searchcore/src/tests/proton/reprocessing/attribute_reprocessing_initializer/attribute_reprocessing_initializer_test.cpp
@@ -294,9 +294,9 @@ TEST("require that added attribute aspect with flushed attribute after interrupt
         auto dir = diskLayout->createAttributeDir("a");
         auto writer = dir->getWriter();
         writer->createInvalidSnapshot(INIT_SERIAL_NUM);
-        writer->markValidSnapshot(INIT_SERIAL_NUM);
         auto snapshotdir = writer->getSnapshotDir(INIT_SERIAL_NUM);
         vespalib::mkdir(snapshotdir);
+        writer->markValidSnapshot(INIT_SERIAL_NUM);
         auto av = AttributeFactory::createAttribute(snapshotdir + "/a",
                                                     Config(BasicType::STRING));
         av->save();
diff --git a/searchcore/src/tests/proton/summaryengine/summaryengine.cpp b/searchcore/src/tests/proton/summaryengine/summaryengine.cpp
index 8206eba6350..0a520044985 100644
--- a/searchcore/src/tests/proton/summaryengine/summaryengine.cpp
+++ b/searchcore/src/tests/proton/summaryengine/summaryengine.cpp
@@ -68,7 +68,7 @@ public:
             DocsumReply::Docsum docsum;
             docsum.docid = 10 + i;
             docsum.gid = h.gid;
-            docsum.setData(_reply.c_str(), _reply.size());
+            docsum.setData(_reply.data(), _reply.size());
             retval->docsums.push_back(docsum);
         }
         return retval;
diff --git a/searchcore/src/vespa/searchcore/config/proton.def b/searchcore/src/vespa/searchcore/config/proton.def
index 7578066d93a..ca1ea67d288 100644
--- a/searchcore/src/vespa/searchcore/config/proton.def
+++ b/searchcore/src/vespa/searchcore/config/proton.def
@@ -231,6 +231,9 @@ summary.cache.compression.type enum {NONE, LZ4, ZSTD} default=LZ4
 ## 9 is a reasonable default for both
 summary.cache.compression.level int default=9
 
+## Control if cache entry is updated or ivalidated when changed.
+summary.cache.update_strategy enum {INVALIDATE, UPDATE} default=INVALIDATE
+
 ## Control compression type of the summary while in memory during compaction
 ## NB So far only stragey=LOG honours it.
 summary.log.compact.compression.type enum {NONE, LZ4, ZSTD} default=ZSTD
diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/query.h b/searchcore/src/vespa/searchcore/fdispatch/search/query.h
index 171a68de9ab..4d336d9843e 100644
--- a/searchcore/src/vespa/searchcore/fdispatch/search/query.h
+++ b/searchcore/src/vespa/searchcore/fdispatch/search/query.h
@@ -70,6 +70,6 @@ private:
                             const vespalib::stringref &b)
     {
         return (a.size() == b.size() &&
-                memcmp(a.c_str(), b.c_str(), a.size()) == 0);
+                memcmp(a.data(), b.data(), a.size()) == 0);
     }
 };
diff --git a/searchcore/src/vespa/searchcore/fdispatch/search/search_path.cpp b/searchcore/src/vespa/searchcore/fdispatch/search/search_path.cpp
index d1731b91e6c..90c6fa2155c 100644
--- a/searchcore/src/vespa/searchcore/fdispatch/search/search_path.cpp
+++ b/searchcore/src/vespa/searchcore/fdispatch/search/search_path.cpp
@@ -46,7 +46,7 @@ SearchPath::parsePartList(const vespalib::stringref &partSpec, size_t numNodes)
         }
     } catch (const std::exception & e) {
         LOG(warning, "Failed parsing part of searchpath='%s' with error '%s'. Result might be mumbo jumbo.",
-            partSpec.c_str(), e.what());
+            vespalib::string(partSpec).c_str(), e.what());
     }
 }
 
@@ -97,7 +97,8 @@ void
 SearchPath::parseRow(const vespalib::stringref &rowSpec)
 {
     if (!rowSpec.empty()) {
-        _elements.back().setRow(strtoul(rowSpec.c_str(), NULL, 0));
+        // FIXME C++17 range-safe from_chars() instead of strtoul()
+        _elements.back().setRow(strtoul(rowSpec.data(), nullptr, 0));
     }
 }
 
diff --git a/searchcore/src/vespa/searchcore/grouping/groupingmanager.cpp b/searchcore/src/vespa/searchcore/grouping/groupingmanager.cpp
index 48baba329ca..3ef5a0f63b1 100644
--- a/searchcore/src/vespa/searchcore/grouping/groupingmanager.cpp
+++ b/searchcore/src/vespa/searchcore/grouping/groupingmanager.cpp
@@ -21,9 +21,7 @@ GroupingManager::GroupingManager(GroupingContext & groupingContext)
 {
 }
 
-GroupingManager::~GroupingManager()
-{
-}
+GroupingManager::~GroupingManager() = default;
 
 using search::expression::ExpressionNode;
 using search::expression::AttributeNode;
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_directory.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_directory.cpp
index f2e4ac4905d..2bbc6c99dc0 100644
--- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_directory.cpp
+++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_directory.cpp
@@ -112,6 +112,7 @@ AttributeDirectory::createInvalidSnapshot(SerialNum serialNum)
     if (empty()) {
         vespalib::string dirName(getDirName());
         vespalib::mkdir(dirName, false);
+        vespalib::File::sync(vespalib::dirname(dirName));
     }
     {
         std::lock_guard<std::mutex> guard(_mutex);
@@ -130,6 +131,9 @@ AttributeDirectory::markValidSnapshot(SerialNum serialNum)
         assert(snap.syncToken == serialNum);
         _snapInfo.validateSnapshot(serialNum);
     }
+    vespalib::string snapshotDir(getSnapshotDir(serialNum));
+    vespalib::File::sync(snapshotDir);
+    vespalib::File::sync(dirname(snapshotDir));
     saveSnapInfo();
 }
 
@@ -178,6 +182,7 @@ AttributeDirectory::removeInvalidSnapshots()
         vespalib::rmdir(subDir, true);
     }
     if (!toRemove.empty()) {
+        vespalib::File::sync(getDirName());
         {
             std::lock_guard<std::mutex> guard(_mutex);
             for (const auto &serialNum : toRemove) {
@@ -194,6 +199,7 @@ AttributeDirectory::removeDiskDir()
     if (empty()) {
         vespalib::string dirName(getDirName());
         vespalib::rmdir(dirName, true);
+        vespalib::File::sync(vespalib::dirname(dirName));
         return true;
     }
     return false;
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp
index eb6020b8d5f..442d04d30c8 100644
--- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp
+++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp
@@ -555,12 +555,12 @@ AttributeWriter::update(SerialNum serialNum, const DocumentUpdate &upd, Document
     }
 
     for (const auto &fupd : upd.getUpdates()) {
-        LOG(debug, "Retrieving guard for attribute vector '%s'.", fupd.getField().getName().c_str());
+        LOG(debug, "Retrieving guard for attribute vector '%s'.", fupd.getField().getName().data());
         auto found = _attrMap.find(fupd.getField().getName());
         AttributeVector * attrp = (found != _attrMap.end()) ? found->second.first : nullptr;
         onUpdate.onUpdateField(fupd.getField().getName(), attrp);
         if (attrp == nullptr) {
-            LOG(spam, "Failed to find attribute vector %s", fupd.getField().getName().c_str());
+            LOG(spam, "Failed to find attribute vector %s", fupd.getField().getName().data());
             continue;
         }
         // TODO: Check if we must use > due to multiple entries for same
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attributedisklayout.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attributedisklayout.cpp
index bb2f99d077b..a675927b85f 100644
--- a/searchcore/src/vespa/searchcore/proton/attribute/attributedisklayout.cpp
+++ b/searchcore/src/vespa/searchcore/proton/attribute/attributedisklayout.cpp
@@ -14,6 +14,7 @@ AttributeDiskLayout::AttributeDiskLayout(const vespalib::string &baseDir, Privat
       _dirs()
 {
     vespalib::mkdir(_baseDir, false);
+    vespalib::File::sync(vespalib::dirname(_baseDir));
 }
 
 AttributeDiskLayout::~AttributeDiskLayout()
diff --git a/searchcore/src/vespa/searchcore/proton/common/eventlogger.cpp b/searchcore/src/vespa/searchcore/proton/common/eventlogger.cpp
index f3afab96cf5..78f73742fed 100644
--- a/searchcore/src/vespa/searchcore/proton/common/eventlogger.cpp
+++ b/searchcore/src/vespa/searchcore/proton/common/eventlogger.cpp
@@ -28,7 +28,7 @@ doTransactionLogReplayStart(const string &domainName, SerialNum first, SerialNum
         .appendKey("last").appendInt64(last)
         .endObject();
     jstr.endObject();
-    EV_STATE(eventName.c_str(), jstr.toString().c_str());
+    EV_STATE(eventName.c_str(), jstr.toString().data());
 }
 
 void
@@ -39,7 +39,7 @@ doTransactionLogReplayComplete(const string &domainName, int64_t elapsedTimeMs,
     jstr.appendKey("domain").appendString(domainName);
     jstr.appendKey("time.elapsed.ms").appendInt64(elapsedTimeMs);
     jstr.endObject();
-    EV_STATE(eventName.c_str(), jstr.toString().c_str());
+    EV_STATE(eventName.c_str(), jstr.toString().data());
 }
 
 }
@@ -67,7 +67,7 @@ EventLogger::transactionLogReplayProgress(const string &domainName, float progre
         .appendKey("current").appendInt64(current)
         .endObject();
     jstr.endObject();
-    EV_STATE("transactionlog.replay.progress", jstr.toString().c_str());
+    EV_STATE("transactionlog.replay.progress", jstr.toString().data());
 }
 
 void
@@ -83,7 +83,7 @@ EventLogger::flushInit(const string &name)
     jstr.beginObject();
     jstr.appendKey("name").appendString(name);
     jstr.endObject();
-    EV_STATE("flush.init", jstr.toString().c_str());
+    EV_STATE("flush.init", jstr.toString().data());
 }
 
 void
@@ -105,7 +105,7 @@ EventLogger::flushStart(const string &name, int64_t beforeMemory, int64_t afterM
         .appendKey("current").appendInt64(current)
         .endObject();
     jstr.endObject();
-    EV_STATE("flush.start", jstr.toString().c_str());
+    EV_STATE("flush.start", jstr.toString().data());
 }
 
 void
@@ -121,7 +121,7 @@ EventLogger::flushComplete(const string &name, int64_t elapsedTimeMs,
         LogUtil::logDir(jstr, outputPath, outputPathElems);
     }
     jstr.endObject();
-    EV_STATE("flush.complete", jstr.toString().c_str());
+    EV_STATE("flush.complete", jstr.toString().data());
 }
 
 namespace {
@@ -146,7 +146,7 @@ EventLogger::populateAttributeStart(const std::vector<string> &names)
     jstr.beginObject();
     addNames(jstr, names);
     jstr.endObject();
-    EV_STATE("populate.attribute.start", jstr.toString().c_str());
+    EV_STATE("populate.attribute.start", jstr.toString().data());
 }
 
 void
@@ -157,7 +157,7 @@ EventLogger::populateAttributeComplete(const std::vector<string> &names, int64_t
     addNames(jstr, names);
     jstr.appendKey("documents.populated").appendInt64(documentsPopulated);
     jstr.endObject();
-    EV_STATE("populate.attribute.complete", jstr.toString().c_str());
+    EV_STATE("populate.attribute.complete", jstr.toString().data());
 }
 
 void
@@ -167,7 +167,7 @@ EventLogger::populateDocumentFieldStart(const string &fieldName)
     jstr.beginObject();
     jstr.appendKey("name").appendString(fieldName);
     jstr.endObject();
-    EV_STATE("populate.documentfield.start", jstr.toString().c_str());
+    EV_STATE("populate.documentfield.start", jstr.toString().data());
 }
 
 void
@@ -178,7 +178,7 @@ EventLogger::populateDocumentFieldComplete(const string &fieldName, int64_t docu
     jstr.appendKey("name").appendString(fieldName);
     jstr.appendKey("documents.populated").appendInt64(documentsPopulated);
     jstr.endObject();
-    EV_STATE("populate.documentfield.complete", jstr.toString().c_str());
+    EV_STATE("populate.documentfield.complete", jstr.toString().data());
 }
 
 void
@@ -189,7 +189,7 @@ EventLogger::lidSpaceCompactionComplete(const string &subDbName, uint32_t lidLim
     jstr.appendKey("documentsubdb").appendString(subDbName);
     jstr.appendKey("lidlimit").appendInt64(lidLimit);
     jstr.endObject();
-    EV_STATE("lidspace.compaction.complete", jstr.toString().c_str());
+    EV_STATE("lidspace.compaction.complete", jstr.toString().data());
 }
 
 
@@ -201,7 +201,7 @@ EventLogger::reprocessDocumentsStart(const string &subDb, double visitCost)
     jstr.appendKey("documentsubdb").appendString(subDb);
     jstr.appendKey("visitcost").appendDouble(visitCost);
     jstr.endObject();
-    EV_STATE("reprocess.documents.start", jstr.toString().c_str());
+    EV_STATE("reprocess.documents.start", jstr.toString().data());
 }
     
 
@@ -214,7 +214,7 @@ EventLogger::reprocessDocumentsProgress(const string &subDb, double progress, do
     jstr.appendKey("progress").appendDouble(progress);
     jstr.appendKey("visitcost").appendDouble(visitCost);
     jstr.endObject();
-    EV_STATE("reprocess.documents.progress", jstr.toString().c_str());
+    EV_STATE("reprocess.documents.progress", jstr.toString().data());
 }
     
 
@@ -227,7 +227,7 @@ EventLogger::reprocessDocumentsComplete(const string &subDb, double visitCost, i
     jstr.appendKey("visitcost").appendDouble(visitCost);
     jstr.appendKey("time.elapsed.ms").appendInt64(elapsedTimeMs);
     jstr.endObject();
-    EV_STATE("reprocess.documents.complete", jstr.toString().c_str());
+    EV_STATE("reprocess.documents.complete", jstr.toString().data());
 }
 
 void
@@ -238,7 +238,7 @@ EventLogger::loadAttributeStart(const vespalib::string &subDbName, const vespali
     jstr.appendKey("documentsubdb").appendString(subDbName);
     jstr.appendKey("name").appendString(attrName);
     jstr.endObject();
-    EV_STATE("load.attribute.start", jstr.toString().c_str());
+    EV_STATE("load.attribute.start", jstr.toString().data());
 }
 
 void
@@ -251,7 +251,7 @@ EventLogger::loadAttributeComplete(const vespalib::string &subDbName,
     jstr.appendKey("name").appendString(attrName);
     jstr.appendKey("time.elapsed.ms").appendInt64(elapsedTimeMs);
     jstr.endObject();
-    EV_STATE("load.attribute.complete", jstr.toString().c_str());
+    EV_STATE("load.attribute.complete", jstr.toString().data());
 }
 
 namespace {
@@ -263,7 +263,7 @@ loadComponentStart(const vespalib::string &subDbName, const vespalib::string &co
     jstr.beginObject();
     jstr.appendKey("documentsubdb").appendString(subDbName);
     jstr.endObject();
-    EV_STATE(make_string("load.%s.start", componentName.c_str()).c_str(), jstr.toString().c_str());
+    EV_STATE(make_string("load.%s.start", componentName.c_str()).c_str(), jstr.toString().data());
 }
 
 void
@@ -274,7 +274,7 @@ loadComponentComplete(const vespalib::string &subDbName, const vespalib::string
     jstr.appendKey("documentsubdb").appendString(subDbName);
     jstr.appendKey("time.elapsed.ms").appendInt64(elapsedTimeMs);
     jstr.endObject();
-    EV_STATE(make_string("load.%s.complete", componentName.c_str()).c_str(), jstr.toString().c_str());
+    EV_STATE(make_string("load.%s.complete", componentName.c_str()).c_str(), jstr.toString().data());
 }
 
 }
@@ -314,7 +314,7 @@ EventLogger::transactionLogPruneComplete(const string &domainName, SerialNum pru
         .appendKey("pruned").appendInt64(prunedSerial)
         .endObject();
     jstr.endObject();
-    EV_STATE("transactionlog.prune.complete", jstr.toString().c_str());
+    EV_STATE("transactionlog.prune.complete", jstr.toString().data());
 }
 
 } // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/documentstoreadapter.cpp b/searchcore/src/vespa/searchcore/proton/docsummary/documentstoreadapter.cpp
index 85ae621db48..ea6a16e1547 100644
--- a/searchcore/src/vespa/searchcore/proton/docsummary/documentstoreadapter.cpp
+++ b/searchcore/src/vespa/searchcore/proton/docsummary/documentstoreadapter.cpp
@@ -66,10 +66,10 @@ DocumentStoreAdapter::writeField(const FieldValue &value, ResType type)
                 const LiteralFieldValueB & lfv =
                     static_cast<const LiteralFieldValueB &>(value);
                 vespalib::stringref s = lfv.getValueRef();
-                return writeStringField(s.c_str(), s.size(), type);
+                return writeStringField(s.data(), s.size(), type);
             } else {
                 vespalib::string s = value.getAsString();
-                return writeStringField(s.c_str(), s.size(), type);
+                return writeStringField(s.data(), s.size(), type);
             }
         }
     case RES_DATA:
diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/fieldcache.cpp b/searchcore/src/vespa/searchcore/proton/docsummary/fieldcache.cpp
index 92bb7da55e1..ecc0569cccd 100644
--- a/searchcore/src/vespa/searchcore/proton/docsummary/fieldcache.cpp
+++ b/searchcore/src/vespa/searchcore/proton/docsummary/fieldcache.cpp
@@ -26,7 +26,7 @@ FieldCache::FieldCache(const ResultClass &resClass,
         if (docType.hasField(fieldName)) {
             const Field &field = docType.getField(fieldName);
             LOG(debug, "Caching Field instance for field '%s': %s.%u",
-                fieldName.c_str(), field.getName().c_str(), field.getId());
+                fieldName.c_str(), field.getName().data(), field.getId());
             _cache.push_back(Field::CSP(new Field(field)));
         } else {
             _cache.push_back(Field::CSP());
diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.cpp b/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.cpp
index ead5c1d1ddf..54961d10fd3 100644
--- a/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.cpp
+++ b/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.cpp
@@ -161,7 +161,7 @@ SummaryManager::SummaryManager(vespalib::ThreadExecutor & executor, const LogDoc
                                                    fileHeaderContext, tlSyncer, bucketizer);
 }
 
-SummaryManager::~SummaryManager() {}
+SummaryManager::~SummaryManager() = default;
 
 void
 SummaryManager::putDocument(uint64_t syncToken, search::DocumentIdT lid, const Document & doc)
@@ -204,7 +204,7 @@ IFlushTarget::List SummaryManager::getFlushTargets(searchcorespi::index::IThread
 {
     IFlushTarget::List ret;
     ret.push_back(std::make_shared<SummaryFlushTarget>(getBackingStore(), summaryService));
-    if (dynamic_cast<LogDocumentStore *>(_docStore.get()) != NULL) {
+    if (dynamic_cast<LogDocumentStore *>(_docStore.get()) != nullptr) {
         ret.push_back(std::make_shared<SummaryCompactTarget>(summaryService, getBackingStore()));
     }
     ret.push_back(createShrinkLidSpaceFlushTarget(summaryService, _docStore));
diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/summarymanagerinitializer.cpp b/searchcore/src/vespa/searchcore/proton/docsummary/summarymanagerinitializer.cpp
index c11326090dc..eee0be77c26 100644
--- a/searchcore/src/vespa/searchcore/proton/docsummary/summarymanagerinitializer.cpp
+++ b/searchcore/src/vespa/searchcore/proton/docsummary/summarymanagerinitializer.cpp
@@ -32,7 +32,7 @@ SummaryManagerInitializer(const search::GrowStrategy &grow,
       _result(result)
 { }
 
-SummaryManagerInitializer::~SummaryManagerInitializer() {}
+SummaryManagerInitializer::~SummaryManagerInitializer() = default;
 
 void
 SummaryManagerInitializer::run()
diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.cpp b/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.cpp
index 95b3008985b..0d2c556b4d6 100644
--- a/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.cpp
+++ b/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.cpp
@@ -8,6 +8,7 @@
 #include "tls_stats_factory.h"
 #include <vespa/searchcore/proton/common/eventlogger.h>
 #include <vespa/vespalib/util/jsonwriter.h>
+#include <thread>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".proton.flushengine.flushengine");
@@ -22,8 +23,7 @@ namespace proton {
 namespace {
 
 search::SerialNum
-findOldestFlushedSerial(const IFlushTarget::List &lst,
-                        const IFlushHandler &handler)
+findOldestFlushedSerial(const IFlushTarget::List &lst, const IFlushHandler &handler)
 {
     search::SerialNum ret(handler.getCurrentSerialNumber());
     for (const IFlushTarget::SP & target : lst) {
@@ -33,42 +33,46 @@ findOldestFlushedSerial(const IFlushTarget::List &lst,
     return ret;
 }
 
+void
+logTarget(const char * text, const FlushContext & ctx) {
+    LOG(debug, "Target '%s' %s flush of transactions %" PRIu64 " through %" PRIu64 ".",
+        ctx.getName().c_str(), text,
+        ctx.getTarget()->getFlushedSerialNum() + 1,
+        ctx.getHandler()->getCurrentSerialNumber());
+}
+
 }
 
-FlushEngine::FlushMeta::FlushMeta(const vespalib::string & name, fastos::TimeStamp start, uint32_t id) :
-    _name(name),
-    _start(start),
-    _id(id)
+FlushEngine::FlushMeta::FlushMeta(const vespalib::string & name, fastos::TimeStamp start, uint32_t id)
+    : _name(name),
+      _start(start),
+      _id(id)
 { }
-FlushEngine::FlushMeta::~FlushMeta() { }
+FlushEngine::FlushMeta::~FlushMeta() = default;
 
-FlushEngine::FlushInfo::FlushInfo() :
-    FlushMeta("", fastos::ClockSystem::now(), 0),
-    _target()
+FlushEngine::FlushInfo::FlushInfo()
+    : FlushMeta("", fastos::ClockSystem::now(), 0),
+      _target()
 {
 }
 
-FlushEngine::FlushInfo::~FlushInfo() { }
+FlushEngine::FlushInfo::~FlushInfo() = default;
 
 
-FlushEngine::FlushInfo::FlushInfo(uint32_t taskId,
-                                  const IFlushTarget::SP &target,
-                                  const vespalib::string & destination) :
-    FlushMeta(destination, fastos::ClockSystem::now(), taskId),
-    _target(target)
+FlushEngine::FlushInfo::FlushInfo(uint32_t taskId, const IFlushTarget::SP &target, const vespalib::string & destination)
+    : FlushMeta(destination, fastos::ClockSystem::now(), taskId),
+      _target(target)
 {
 }
 
-FlushEngine::FlushEngine(std::shared_ptr<flushengine::ITlsStatsFactory>
-                         tlsStatsFactory,
-                         IFlushStrategy::SP strategy, uint32_t numThreads,
-                         uint32_t idleIntervalMS)
+FlushEngine::FlushEngine(std::shared_ptr<flushengine::ITlsStatsFactory> tlsStatsFactory,
+                         IFlushStrategy::SP strategy, uint32_t numThreads, uint32_t idleIntervalMS)
     : _closed(false),
       _maxConcurrent(numThreads),
       _idleIntervalMS(idleIntervalMS),
       _taskId(0),
       _threadPool(128 * 1024),
-      _strategy(strategy),
+      _strategy(std::move(strategy)),
       _priorityStrategy(),
       _executor(numThreads, 128 * 1024),
       _lock(),
@@ -78,11 +82,9 @@ FlushEngine::FlushEngine(std::shared_ptr<flushengine::ITlsStatsFactory>
       _setStrategyLock(),
       _strategyLock(),
       _strategyCond(),
-      _tlsStatsFactory(tlsStatsFactory),
+      _tlsStatsFactory(std::move(tlsStatsFactory)),
       _pendingPrune()
-{
-    // empty
-}
+{ }
 
 FlushEngine::~FlushEngine()
 {
@@ -92,7 +94,7 @@ FlushEngine::~FlushEngine()
 FlushEngine &
 FlushEngine::start()
 {
-    if (_threadPool.NewThread(this) == NULL) {
+    if (_threadPool.NewThread(this) == nullptr) {
         throw vespalib::IllegalStateException("Failed to start engine thread.");
     }
     return *this;
@@ -148,10 +150,8 @@ FlushEngine::wait(size_t minimumWaitTimeIfReady)
 }
 
 void
-FlushEngine::Run(FastOS_ThreadInterface *thread, void *arg)
+FlushEngine::Run(FastOS_ThreadInterface *, void *)
 {
-    (void)thread;
-    (void)arg;
     bool shouldIdle = false;
     vespalib::string prevFlushName;
     while (wait(shouldIdle ? _idleIntervalMS : 0)) {
@@ -161,13 +161,14 @@ FlushEngine::Run(FastOS_ThreadInterface *thread, void *arg)
         }
         prevFlushName = flushNextTarget(prevFlushName);
         if ( ! prevFlushName.empty()) {
-            // Sleep at least 10 ms after a successful flush in order to avoid busy loop in case
-            // of strategy error or target error.
-            FastOS_Thread::Sleep(10);
+            // Sleep 1 ms after a successful flush in order to avoid busy loop in case
+            // of strategy or target error.
+            std::this_thread::sleep_for(1ms);
         } else {
             shouldIdle = true;
         }
-        LOG(debug, "Making another wait(idle=%s, timeMS=%d) last was '%s'", shouldIdle ? "true" : "false", shouldIdle ? _idleIntervalMS : 0, prevFlushName.c_str());
+        LOG(debug, "Making another wait(idle=%s, timeMS=%d) last was '%s'",
+            shouldIdle ? "true" : "false", shouldIdle ? _idleIntervalMS : 0, prevFlushName.c_str());
     }
     _executor.sync();
     prune();
@@ -211,18 +212,16 @@ FlushEngine::getTargetList(bool includeFlushingTargets) const
         for (const auto & it : _handlers) {
             IFlushHandler & handler(*it.second);
             search::SerialNum serial(handler.getCurrentSerialNumber());
-            LOG(spam, "Checking FlushHandler '%s' current serial = %ld",
-                handler.getName().c_str(), serial);
+            LOG(spam, "Checking FlushHandler '%s' current serial = %ld", handler.getName().c_str(), serial);
             IFlushTarget::List lst = handler.getFlushTargets();
             for (const IFlushTarget::SP & target : lst) {
-                LOG(spam, "Checking target '%s' with flushedSerialNum = %ld", target->getName().c_str(), target->getFlushedSerialNum());
+                LOG(spam, "Checking target '%s' with flushedSerialNum = %ld",
+                    target->getName().c_str(), target->getFlushedSerialNum());
                 if (!isFlushing(guard, FlushContext::createName(handler, *target)) || includeFlushingTargets) {
-                    ret.push_back(FlushContext::SP(new FlushContext(it.second,
-                                                                    IFlushTarget::SP(new CachedFlushTarget(target)),
-                                                                    serial)));
+                    ret.push_back(std::make_shared<FlushContext>(it.second, std::make_shared<CachedFlushTarget>(target), serial));
                 } else {
                     LOG(debug, "Target '%s' with flushedSerialNum = %ld already has a flush going. Local last serial = %ld.",
-                               target->getName().c_str(), target->getFlushedSerialNum(), serial);
+                        target->getName().c_str(), target->getFlushedSerialNum(), serial);
                 }
             }
         }
@@ -258,17 +257,12 @@ FlushEngine::initNextFlush(const FlushContext::List &lst)
             break;
         }
     }
-    if (ctx.get() != NULL) {
-        LOG(debug, "Target '%s' initiated flush of transactions %" PRIu64 " through %" PRIu64 ".",
-                   ctx->getName().c_str(),
-                   ctx->getTarget()->getFlushedSerialNum() + 1,
-                   ctx->getHandler()->getCurrentSerialNumber());
+    if (ctx) {
+        logTarget("initiated", *ctx);
     }
     return ctx;
 }
 
-
-
 void
 FlushEngine::flushAll(const FlushContext::List &lst)
 {
@@ -276,19 +270,12 @@ FlushEngine::flushAll(const FlushContext::List &lst)
     for (const FlushContext::SP & ctx : lst) {
         if (wait(0)) {
             if (ctx->initFlush()) {
-                LOG(debug, "Target '%s' initiated flush of transactions %" PRIu64 " through %" PRIu64 ".",
-                           ctx->getName().c_str(),
-                           ctx->getTarget()->getFlushedSerialNum() + 1,
-                           ctx->getHandler()->getCurrentSerialNumber());
-                _executor.execute(Task::UP(new FlushTask(initFlush(*ctx), *this, ctx)));
+                logTarget("initiated", *ctx);
+                _executor.execute(std::make_unique<FlushTask>(initFlush(*ctx), *this, ctx));
             } else {
-                LOG(debug, "Target '%s' failed to initiate flush of transactions %" PRIu64 " through %" PRIu64 ".",
-                           ctx->getName().c_str(),
-                           ctx->getTarget()->getFlushedSerialNum() + 1,
-                           ctx->getHandler()->getCurrentSerialNumber());
+                logTarget("failed to initiate", *ctx);
             }
         }
-
     }
 }
 
@@ -311,17 +298,17 @@ FlushEngine::flushNextTarget(const vespalib::string & name)
         return "";
     }
     FlushContext::SP ctx = initNextFlush(lst.first);
-    if (ctx.get() == NULL) {
+    if ( ! ctx) {
         LOG(debug, "All targets refused to flush.");
         return "";
     }
     if ( name == ctx->getName()) {
         LOG(info, "The same target %s out of %ld has been asked to flush again. "
-                  "This might indicate flush logic flaw so I will wait 1s before doing it.",
+                  "This might indicate flush logic flaw so I will wait 100 ms before doing it.",
                   name.c_str(), lst.first.size());
-        FastOS_Thread::Sleep(1000);
+        std::this_thread::sleep_for(100ms);
     }
-    _executor.execute(Task::UP(new FlushTask(initFlush(*ctx), *this, ctx)));
+    _executor.execute(std::make_unique<FlushTask>(initFlush(*ctx), *this, ctx));
     return ctx->getName();
 }
 
@@ -330,12 +317,8 @@ FlushEngine::initFlush(const FlushContext &ctx)
 {
     if (LOG_WOULD_LOG(event)) {
         IFlushTarget::MemoryGain mgain(ctx.getTarget()->getApproxMemoryGain());
-        EventLogger::flushStart(ctx.getName(),
-                                mgain.getBefore(),
-                                mgain.getAfter(),
-                                mgain.gain(),
-                                ctx.getTarget()->getFlushedSerialNum() + 1,
-                                ctx.getHandler()->getCurrentSerialNumber());
+        EventLogger::flushStart(ctx.getName(), mgain.getBefore(), mgain.getAfter(), mgain.gain(),
+                                ctx.getTarget()->getFlushedSerialNum() + 1, ctx.getHandler()->getCurrentSerialNumber());
     }
     return initFlush(ctx.getHandler(), ctx.getTarget());
 }
@@ -350,10 +333,7 @@ FlushEngine::flushDone(const FlushContext &ctx, uint32_t taskId)
     }
     if (LOG_WOULD_LOG(event)) {
         FlushStats stats = ctx.getTarget()->getLastFlushStats();
-        EventLogger::flushComplete(ctx.getName(),
-                                   duration.ms(),
-                                   stats.getPath(),
-                                   stats.getPathElementsToLog());
+        EventLogger::flushComplete(ctx.getName(), duration.ms(), stats.getPath(), stats.getPathElementsToLog());
     }
     LOG(debug, "FlushEngine::flushDone(taskId='%d') took '%f' secs", taskId, duration.sec());
     std::lock_guard<std::mutex> guard(_lock);
@@ -366,8 +346,7 @@ FlushEngine::flushDone(const FlushContext &ctx, uint32_t taskId)
 }
 
 IFlushHandler::SP
-FlushEngine::putFlushHandler(const DocTypeName &docTypeName,
-                             const IFlushHandler::SP &flushHandler)
+FlushEngine::putFlushHandler(const DocTypeName &docTypeName, const IFlushHandler::SP &flushHandler)
 {
     std::lock_guard<std::mutex> guard(_lock);
     IFlushHandler::SP result(_handlers.putHandler(docTypeName, flushHandler));
@@ -379,13 +358,6 @@ FlushEngine::putFlushHandler(const DocTypeName &docTypeName,
 }
 
 IFlushHandler::SP
-FlushEngine::getFlushHandler(const DocTypeName &docTypeName) const
-{
-    std::lock_guard<std::mutex> guard(_lock);
-    return _handlers.getHandler(docTypeName);
-}
-
-IFlushHandler::SP
 FlushEngine::removeFlushHandler(const DocTypeName &docTypeName)
 {
     std::lock_guard<std::mutex> guard(_lock);
@@ -430,7 +402,7 @@ FlushEngine::setStrategy(IFlushStrategy::SP strategy)
         return;
     }
     assert(!_priorityStrategy);
-    _priorityStrategy = strategy;
+    _priorityStrategy = std::move(strategy);
     {
         std::lock_guard<std::mutex> guard(_lock);
         _cond.notify_all();
diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.h b/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.h
index 19175f9ce2a..c1be05ba067 100644
--- a/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.h
+++ b/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.h
@@ -15,7 +15,7 @@ namespace proton {
 
 namespace flushengine { class ITlsStatsFactory; }
 
-class FlushEngine : public FastOS_Runnable
+class FlushEngine final : public FastOS_Runnable
 {
 public:
     class FlushMeta {
@@ -37,9 +37,7 @@ private:
     struct FlushInfo : public FlushMeta
     {
         FlushInfo();
-        FlushInfo(uint32_t taskId,
-                  const IFlushTarget::SP &target,
-                  const vespalib::string &destination);
+        FlushInfo(uint32_t taskId, const IFlushTarget::SP &target, const vespalib::string &destination);
         ~FlushInfo();
 
         IFlushTarget::SP  _target;
@@ -96,14 +94,13 @@ public:
      * @param numThreads The number of worker threads to use.
      * @param idleInterval The interval between when flushes are checked whne there are no one progressing.
      */
-    FlushEngine(std::shared_ptr<flushengine::ITlsStatsFactory>
-                tlsStatsFactory,
+    FlushEngine(std::shared_ptr<flushengine::ITlsStatsFactory> tlsStatsFactory,
                 IFlushStrategy::SP strategy, uint32_t numThreads, uint32_t idleIntervalMS);
 
     /**
      * Destructor. Waits for all pending tasks to complete.
      */
-    ~FlushEngine();
+    ~FlushEngine() override;
 
     /**
      * Observe and reset internal executor stats
@@ -145,19 +142,8 @@ public:
      * @param flushHandler The handler to register.
      * @return The replaced handler, if any.
      */
-    IFlushHandler::SP
-    putFlushHandler(const DocTypeName &docTypeName,
-                    const IFlushHandler::SP &flushHandler);
+    IFlushHandler::SP putFlushHandler(const DocTypeName &docTypeName, const IFlushHandler::SP &flushHandler);
 
-    /**
-     * Returns the flush handler for the given document type. If no handler was
-     * registered, this method returns an empty shared pointer.
-     *
-     * @param docType The document type whose handler to return.
-     * @return The registered handler, if any.
-     */
-    IFlushHandler::SP
-    getFlushHandler(const DocTypeName &docTypeName) const;
 
     /**
      * Removes and returns the flush handler for the given document type. If no
@@ -166,10 +152,8 @@ public:
      * @param docType The document type whose handler to remove.
      * @return The removed handler, if any.
      */
-    IFlushHandler::SP
-    removeFlushHandler(const DocTypeName &docTypeName);
+    IFlushHandler::SP removeFlushHandler(const DocTypeName &docTypeName);
 
-    // Implements FastOS_Runnable.
     void Run(FastOS_ThreadInterface *thread, void *arg) override;
 
     FlushMetaSet getCurrentlyFlushingSet() const;
diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/iflushhandler.h b/searchcore/src/vespa/searchcore/proton/flushengine/iflushhandler.h
index 3a24330f8ec..914de9df30c 100644
--- a/searchcore/src/vespa/searchcore/proton/flushengine/iflushhandler.h
+++ b/searchcore/src/vespa/searchcore/proton/flushengine/iflushhandler.h
@@ -30,9 +30,7 @@ public:
      */
     IFlushHandler(const vespalib::string &name)
         : _name(name)
-    {
-        // empty
-    }
+    { }
 
     /**
      * Virtual destructor required for inheritance.
@@ -76,8 +74,7 @@ public:
      * This method is called to sync tls to stable media, up to and
      * including the given serial number.
      *
-     * @param syncTo The last serial number that has to be persisted to stable
-     *               media.
+     * @param syncTo The last serial number that has to be persisted to stable media.
      */
     virtual void syncTls(SerialNum syncTo) = 0;
 };
diff --git a/searchcore/src/vespa/searchcore/proton/index/index_manager_initializer.cpp b/searchcore/src/vespa/searchcore/proton/index/index_manager_initializer.cpp
index b8f2947c9b4..ad233a66d1f 100644
--- a/searchcore/src/vespa/searchcore/proton/index/index_manager_initializer.cpp
+++ b/searchcore/src/vespa/searchcore/proton/index/index_manager_initializer.cpp
@@ -45,6 +45,7 @@ IndexManagerInitializer::run()
     LOG(debug, "About to create proton::IndexManager with %u index field(s)",
         _schema.getNumIndexFields());
     vespalib::mkdir(_baseDir, false);
+    vespalib::File::sync(vespalib::dirname(_baseDir));
     *_indexManager = std::make_shared<proton::IndexManager>
                     (_baseDir,
                      _warmupCfg,
diff --git a/searchcore/src/vespa/searchcore/proton/matching/attribute_limiter.cpp b/searchcore/src/vespa/searchcore/proton/matching/attribute_limiter.cpp
index 05dc75146c7..ca96033c358 100644
--- a/searchcore/src/vespa/searchcore/proton/matching/attribute_limiter.cpp
+++ b/searchcore/src/vespa/searchcore/proton/matching/attribute_limiter.cpp
@@ -11,8 +11,7 @@ using namespace search::query;
 using vespalib::make_string;
 using vespalib::string;
 
-namespace proton {
-namespace matching {
+namespace proton::matching {
 
 AttributeLimiter::AttributeLimiter(Searchable &searchable_attributes,
                                    const IRequestContext & requestContext,
@@ -35,7 +34,7 @@ AttributeLimiter::AttributeLimiter(Searchable &searchable_attributes,
 {
 }
 
-AttributeLimiter::~AttributeLimiter() {}
+AttributeLimiter::~AttributeLimiter() = default;
 
 namespace {
 
@@ -86,5 +85,4 @@ AttributeLimiter::create_search(size_t want_hits, size_t max_group_size, bool st
     return _blueprint->createSearch(*_match_datas.back(), strictSearch);
 }
 
-} // namespace proton::matching
-} // namespace proton
+}
diff --git a/searchcore/src/vespa/searchcore/proton/matching/attribute_limiter.h b/searchcore/src/vespa/searchcore/proton/matching/attribute_limiter.h
index 95f7bf4e42b..c50a6e0dcb8 100644
--- a/searchcore/src/vespa/searchcore/proton/matching/attribute_limiter.h
+++ b/searchcore/src/vespa/searchcore/proton/matching/attribute_limiter.h
@@ -10,8 +10,7 @@
 #include <vespa/searchlib/fef/matchdata.h>
 #include <mutex>
 
-namespace proton {
-namespace matching {
+namespace proton::matching {
 
 /**
  * This class is responsible for creating attribute-based search
@@ -51,6 +50,4 @@ private:
     DiversityCutoffStrategy                    _diversityCutoffStrategy;
 };
 
-} // namespace proton::matching
-} // namespace proton
-
+}
diff --git a/searchcore/src/vespa/searchcore/proton/matching/i_match_loop_communicator.h b/searchcore/src/vespa/searchcore/proton/matching/i_match_loop_communicator.h
index 04440831045..df24fa9e76b 100644
--- a/searchcore/src/vespa/searchcore/proton/matching/i_match_loop_communicator.h
+++ b/searchcore/src/vespa/searchcore/proton/matching/i_match_loop_communicator.h
@@ -2,19 +2,19 @@
 
 #pragma once
 
-#include <vespa/searchlib/common/feature.h>
 #include <vespa/searchlib/queryeval/scores.h>
 #include <utility>
 #include <cstddef>
+#include <cstdint>
 #include <vector>
 
-namespace proton {
-namespace matching {
+namespace proton::matching {
 
 struct IMatchLoopCommunicator {
-    typedef search::feature_t feature_t;
-    typedef search::queryeval::Scores Range;
-    typedef std::pair<Range, Range> RangePair;
+    using Range = search::queryeval::Scores;
+    using RangePair = std::pair<Range, Range>;
+    using Hit = std::pair<uint32_t, search::feature_t>;
+    using Hits = std::vector<Hit>;
     struct Matches {
         size_t hits;
         size_t docs;
@@ -26,11 +26,9 @@ struct IMatchLoopCommunicator {
         }
     };
     virtual double estimate_match_frequency(const Matches &matches) = 0;
-    virtual size_t selectBest(const std::vector<feature_t> &sortedScores) = 0;
+    virtual Hits selectBest(Hits sortedHits) = 0;
     virtual RangePair rangeCover(const RangePair &ranges) = 0;
     virtual ~IMatchLoopCommunicator() {}
 };
 
-} // namespace matching
-} // namespace proton
-
+}
diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_loop_communicator.cpp b/searchcore/src/vespa/searchcore/proton/matching/match_loop_communicator.cpp
index fa41c73838b..54cffce7f40 100644
--- a/searchcore/src/vespa/searchcore/proton/matching/match_loop_communicator.cpp
+++ b/searchcore/src/vespa/searchcore/proton/matching/match_loop_communicator.cpp
@@ -3,15 +3,17 @@
 #include "match_loop_communicator.h"
 #include <vespa/vespalib/util/priority_queue.h>
 
-namespace proton {
-namespace matching {
+namespace proton:: matching {
 
 MatchLoopCommunicator::MatchLoopCommunicator(size_t threads, size_t topN)
+    : MatchLoopCommunicator(threads, topN, std::unique_ptr<IDiversifier>())
+{}
+MatchLoopCommunicator::MatchLoopCommunicator(size_t threads, size_t topN, std::unique_ptr<IDiversifier> diversifier)
     : _estimate_match_frequency(threads),
-      _selectBest(threads, topN),
+      _selectBest(threads, topN, std::move(diversifier)),
       _rangeCover(threads)
 {}
-MatchLoopCommunicator::~MatchLoopCommunicator() {}
+MatchLoopCommunicator::~MatchLoopCommunicator() = default;
 
 void
 MatchLoopCommunicator::EstimateMatchFrequency::mingle()
@@ -30,22 +32,47 @@ MatchLoopCommunicator::EstimateMatchFrequency::mingle()
     }
 }
 
+MatchLoopCommunicator::SelectBest::SelectBest(size_t n, size_t topN_in, std::unique_ptr<IDiversifier> diversifier)
+    : vespalib::Rendezvous<Hits, Hits>(n),
+      topN(topN_in),
+      _indexes(n, 0),
+      _diversifier(std::move(diversifier))
+{}
+MatchLoopCommunicator::SelectBest::~SelectBest() = default;
+
+template<typename Q, typename F>
+void
+MatchLoopCommunicator::SelectBest::mingle(Q & queue, F && accept) {
+    for (size_t picked = 0; picked < topN && !queue.empty(); ) {
+        uint32_t i = queue.front();
+        const Hit & hit = in(i)[_indexes[i]];
+        if (accept(hit.first)) {
+            out(i).push_back(hit);
+            ++picked;
+        }
+        if (in(i).size() > ++_indexes[i]) {
+            queue.adjust();
+        } else {
+            queue.pop_front();
+        }
+    }
+}
+
 void
 MatchLoopCommunicator::SelectBest::mingle()
 {
     vespalib::PriorityQueue<uint32_t, SelectCmp> queue(SelectCmp(*this));
     for (size_t i = 0; i < size(); ++i) {
         if (!in(i).empty()) {
+            out(i).reserve(std::min(topN, in(i).size()));
+            _indexes[i] = 0;
             queue.push(i);
         }
     }
-    for (size_t picked = 0; picked < topN && !queue.empty(); ++picked) {
-        uint32_t i = queue.front();
-        if (in(i).size() > ++out(i)) {
-            queue.adjust();
-        } else {
-            queue.pop_front();
-        }
+    if (_diversifier) {
+        mingle(queue, [diversifier=_diversifier.get()](uint32_t docId) { return diversifier->accepted(docId);});
+    } else {
+        mingle(queue, [](uint32_t) { return true;});
     }
 }
 
@@ -72,5 +99,4 @@ MatchLoopCommunicator::RangeCover::mingle()
     }
 }
 
-} // namespace matching
-} // namespace proton
+}
diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_loop_communicator.h b/searchcore/src/vespa/searchcore/proton/matching/match_loop_communicator.h
index 3b83cb471f7..e17efd66c78 100644
--- a/searchcore/src/vespa/searchcore/proton/matching/match_loop_communicator.h
+++ b/searchcore/src/vespa/searchcore/proton/matching/match_loop_communicator.h
@@ -3,59 +3,61 @@
 #pragma once
 
 #include "i_match_loop_communicator.h"
+#include <vespa/searchlib/queryeval/idiversifier.h>
 #include <vespa/vespalib/util/rendezvous.h>
 
-namespace proton {
-namespace matching {
+namespace proton::matching {
 
 class MatchLoopCommunicator : public IMatchLoopCommunicator
 {
 private:
+    using IDiversifier = search::queryeval::IDiversifier;
     struct EstimateMatchFrequency : vespalib::Rendezvous<Matches, double> {
-        EstimateMatchFrequency(size_t n)
-            : vespalib::Rendezvous<Matches, double>(n) {}
-        virtual void mingle() override;
+        EstimateMatchFrequency(size_t n) : vespalib::Rendezvous<Matches, double>(n) {}
+        void mingle() override;
     };
-    struct SelectBest : vespalib::Rendezvous<std::vector<feature_t>, size_t> {
+    struct SelectBest : vespalib::Rendezvous<Hits, Hits> {
         size_t topN;
-        SelectBest(size_t n, size_t topN_in)
-            : vespalib::Rendezvous<std::vector<feature_t>, size_t>(n), topN(topN_in) {}
-        virtual void mingle() override;
-        bool cmp(const uint32_t &a, const uint32_t &b) {
-            return (in(a)[out(a)] > in(b)[out(b)]);
+        std::vector<uint32_t> _indexes;
+        std::unique_ptr<IDiversifier> _diversifier;
+        SelectBest(size_t n, size_t topN_in, std::unique_ptr<IDiversifier>);
+        ~SelectBest() override;
+        void mingle() override;
+        template<typename Q, typename F>
+        void mingle(Q & queue, F && accept);
+        bool cmp(uint32_t a, uint32_t b) {
+            return (in(a)[_indexes[a]].second > in(b)[_indexes[b]].second);
         }
     };
     struct SelectCmp {
         SelectBest &sb;
         SelectCmp(SelectBest &sb_in) : sb(sb_in) {}
-        bool operator()(const uint32_t &a, const uint32_t &b) const {
+        bool operator()(uint32_t a, uint32_t b) const {
             return (sb.cmp(a, b));
         }
     };
     struct RangeCover : vespalib::Rendezvous<RangePair, RangePair> {
-        RangeCover(size_t n)
-            : vespalib::Rendezvous<RangePair, RangePair>(n) {}
-        virtual void mingle() override;
+        RangeCover(size_t n) : vespalib::Rendezvous<RangePair, RangePair>(n) {}
+        void mingle() override;
     };
-    EstimateMatchFrequency _estimate_match_frequency;
-    SelectBest             _selectBest;
-    RangeCover             _rangeCover;
+    EstimateMatchFrequency        _estimate_match_frequency;
+    SelectBest                    _selectBest;
+    RangeCover                    _rangeCover;
 
 public:
     MatchLoopCommunicator(size_t threads, size_t topN);
+    MatchLoopCommunicator(size_t threads, size_t topN, std::unique_ptr<IDiversifier>);
     ~MatchLoopCommunicator();
 
-    virtual double estimate_match_frequency(const Matches &matches) override {
+    double estimate_match_frequency(const Matches &matches) override {
         return _estimate_match_frequency.rendezvous(matches);
     }
-    virtual size_t selectBest(const std::vector<feature_t> &sortedScores) override {
-        return _selectBest.rendezvous(sortedScores);
+    Hits selectBest(Hits sortedHits) override {
+        return _selectBest.rendezvous(sortedHits);
     }
-    virtual RangePair rangeCover(const RangePair &ranges) override {
+    RangePair rangeCover(const RangePair &ranges) override {
         return _rangeCover.rendezvous(ranges);
     }
 };
 
-} // namespace matching
-} // namespace proton
-
+}
diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_master.cpp b/searchcore/src/vespa/searchcore/proton/matching/match_master.cpp
index 370c4b930e1..920f84a21b0 100644
--- a/searchcore/src/vespa/searchcore/proton/matching/match_master.cpp
+++ b/searchcore/src/vespa/searchcore/proton/matching/match_master.cpp
@@ -10,8 +10,7 @@
 #include <vespa/log/log.h>
 LOG_SETUP(".proton.matching.match_master");
 
-namespace proton {
-namespace matching {
+namespace proton::matching {
 
 using namespace search::fef;
 using search::queryeval::SearchIterator;
@@ -23,15 +22,15 @@ struct TimedMatchLoopCommunicator : IMatchLoopCommunicator {
     IMatchLoopCommunicator &communicator;
     fastos::StopWatch rerank_time;
     TimedMatchLoopCommunicator(IMatchLoopCommunicator &com) : communicator(com) {}
-    virtual double estimate_match_frequency(const Matches &matches) override {
+    double estimate_match_frequency(const Matches &matches) override {
         return communicator.estimate_match_frequency(matches);
     }
-    virtual size_t selectBest(const std::vector<feature_t> &sortedScores) override {
-        size_t result = communicator.selectBest(sortedScores);
+    Hits selectBest(Hits sortedHits) override {
+        Hits result = communicator.selectBest(std::move(sortedHits));
         rerank_time.start();
         return result;
     }
-    virtual RangePair rangeCover(const RangePair &ranges) override {
+    RangePair rangeCover(const RangePair &ranges) override {
         RangePair result = communicator.rangeCover(ranges);
         rerank_time.stop();
         return result;
@@ -131,18 +130,15 @@ MatchMaster::getFeatureSet(const MatchToolsFactory &matchToolsFactory,
         if (search.seek(docs[i])) {
             uint32_t docId = search.getDocId();
             search.unpack(docId);
-            search::feature_t * f = fs.getFeaturesByIndex(
-                    fs.addDocId(docId));
+            search::feature_t * f = fs.getFeaturesByIndex(fs.addDocId(docId));
             for (uint32_t j = 0; j < featureNames.size(); ++j) {
                 f[j] = resolver.resolve(j).as_number(docId);
             }
         } else {
-            LOG(debug, "getFeatureSet: Did not find hit for docid '%u'. "
-                "Skipping hit", docs[i]);
+            LOG(debug, "getFeatureSet: Did not find hit for docid '%u'. Skipping hit", docs[i]);
         }
     }
     return retval;
 }
 
-} // namespace proton::matching
-} // namespace proton
+}
diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_master.h b/searchcore/src/vespa/searchcore/proton/matching/match_master.h
index b9980023259..4c6463cf75d 100644
--- a/searchcore/src/vespa/searchcore/proton/matching/match_master.h
+++ b/searchcore/src/vespa/searchcore/proton/matching/match_master.h
@@ -8,8 +8,7 @@
 namespace vespalib { class ThreadBundle; }
 namespace search { class FeatureSet; }
 
-namespace proton {
-namespace matching {
+namespace proton::matching {
 
 class MatchToolsFactory;
 class MatchParams;
@@ -37,6 +36,4 @@ public:
     static MatchingStats getStats(MatchMaster && rhs) { return std::move(rhs._stats); }
 };
 
-} // namespace proton::matching
-} // namespace proton
-
+}
diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_params.cpp b/searchcore/src/vespa/searchcore/proton/matching/match_params.cpp
index ddefa7c30c9..549aee20de7 100644
--- a/searchcore/src/vespa/searchcore/proton/matching/match_params.cpp
+++ b/searchcore/src/vespa/searchcore/proton/matching/match_params.cpp
@@ -2,8 +2,7 @@
 
 #include "match_params.h"
 
-namespace proton {
-namespace matching {
+namespace proton::matching {
 
 namespace {
 
@@ -30,8 +29,6 @@ MatchParams::MatchParams(uint32_t          numDocs_in,
       offset(offset_in),
       hits(hits_in),
       rankDropLimit(rankDropLimit_in)
-{
-}
+{ }
 
-} // namespace proton::matching
-} // namespace proton
+}
diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_params.h b/searchcore/src/vespa/searchcore/proton/matching/match_params.h
index 50787516144..c1fd9a4ef32 100644
--- a/searchcore/src/vespa/searchcore/proton/matching/match_params.h
+++ b/searchcore/src/vespa/searchcore/proton/matching/match_params.h
@@ -4,8 +4,7 @@
 
 #include <vespa/searchlib/fef/fef.h>
 
-namespace proton {
-namespace matching {
+namespace proton::matching {
 
 /**
  * Numeric matching parameters. Some of these comes from the config,
@@ -30,6 +29,4 @@ struct MatchParams {
     bool save_rank_scores() const { return ((heapSize + arraySize) != 0); }
 };
 
-} // namespace proton::matching
-} // namespace proton
-
+}
diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_thread.cpp b/searchcore/src/vespa/searchcore/proton/matching/match_thread.cpp
index 1efb74b96ba..9232a15043b 100644
--- a/searchcore/src/vespa/searchcore/proton/matching/match_thread.cpp
+++ b/searchcore/src/vespa/searchcore/proton/matching/match_thread.cpp
@@ -265,12 +265,15 @@ MatchThread::findMatches(MatchTools &tools)
             tools.setup_second_phase();
             DocidRange docid_range = scheduler.total_span(thread_id);
             tools.search().initRange(docid_range.begin, docid_range.end);
-            auto sorted_scores = hits.getSortedHeapScores();
+            auto sorted_hits = hits.getSortedHeapHits();
             WaitTimer select_best_timer(wait_time_s);
-            size_t useHits = communicator.selectBest(sorted_scores);
+            auto kept_hits = communicator.selectBest(std::move(sorted_hits));
             select_best_timer.done();
             DocumentScorer scorer(tools.rank_program(), tools.search());
-            uint32_t reRanked = hits.reRank(scorer, tools.getHardDoom().doom() ? 0 : useHits);
+            if (tools.getHardDoom().doom()) {
+                kept_hits.clear();
+            }
+            uint32_t reRanked = hits.reRank(scorer, std::move(kept_hits));
             thread_stats.docsReRanked(reRanked);
         }
         { // rank scaling
diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_thread.h b/searchcore/src/vespa/searchcore/proton/matching/match_thread.h
index 5c78ee59b1d..8384a45e0e2 100644
--- a/searchcore/src/vespa/searchcore/proton/matching/match_thread.h
+++ b/searchcore/src/vespa/searchcore/proton/matching/match_thread.h
@@ -103,7 +103,7 @@ public:
                 ResultProcessor &rp,
                 vespalib::DualMergeDirector &md,
                 uint32_t distributionKey);
-    virtual void run() override;
+    void run() override;
     const MatchingStats::Partition &get_thread_stats() const { return thread_stats; }
     double get_match_time() const { return match_time_s; }
     PartialResult::UP extract_result() { return std::move(resultContext->result); }
diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_tools.cpp b/searchcore/src/vespa/searchcore/proton/matching/match_tools.cpp
index d52d0fcd841..e7773c94d72 100644
--- a/searchcore/src/vespa/searchcore/proton/matching/match_tools.cpp
+++ b/searchcore/src/vespa/searchcore/proton/matching/match_tools.cpp
@@ -87,16 +87,13 @@ MatchTools::MatchTools(QueryLimiter & queryLimiter,
 {
 }
 
-MatchTools::~MatchTools()
-{
-}
+MatchTools::~MatchTools() = default;
 
 void
 MatchTools::setup_first_phase()
 {
     setup(_rankSetup.create_first_phase_program(),
-          TermwiseLimit::lookup(_queryEnv.getProperties(),
-                                _rankSetup.get_termwise_limit()));
+          TermwiseLimit::lookup(_queryEnv.getProperties(), _rankSetup.get_termwise_limit()));
 }
 
 void
@@ -164,36 +161,34 @@ MatchToolsFactory(QueryLimiter               & queryLimiter,
         double diversity_cutoff_factor = DiversityCutoffFactor::lookup(rankProperties);
         vespalib::string diversity_cutoff_strategy = DiversityCutoffStrategy::lookup(rankProperties);
         if (!limit_attribute.empty() && limit_maxhits > 0) {
-            _match_limiter.reset(new MatchPhaseLimiter(metaStore.getCommittedDocIdLimit(), searchContext.getAttributes(), _requestContext,
+            _match_limiter = std::make_unique<MatchPhaseLimiter>(metaStore.getCommittedDocIdLimit(), searchContext.getAttributes(), _requestContext,
                             limit_attribute, limit_maxhits, !limit_ascending, limit_max_filter_coverage,
                             samplePercentage, postFilterMultiplier,
-                            diversity_attribute, diversity_min_groups,
-                            diversity_cutoff_factor,
-                            AttributeLimiter::toDiversityCutoffStrategy(diversity_cutoff_strategy)));
+                            diversity_attribute, diversity_min_groups, diversity_cutoff_factor,
+                            AttributeLimiter::toDiversityCutoffStrategy(diversity_cutoff_strategy));
         } else if (_rankSetup.hasMatchPhaseDegradation()) {
-            _match_limiter.reset(new MatchPhaseLimiter(metaStore.getCommittedDocIdLimit(), searchContext.getAttributes(), _requestContext,
+            _match_limiter = std::make_unique<MatchPhaseLimiter>(metaStore.getCommittedDocIdLimit(), searchContext.getAttributes(), _requestContext,
                             _rankSetup.getDegradationAttribute(), _rankSetup.getDegradationMaxHits(), !_rankSetup.isDegradationOrderAscending(),
                             _rankSetup.getDegradationMaxFilterCoverage(),
                             _rankSetup.getDegradationSamplePercentage(), _rankSetup.getDegradationPostFilterMultiplier(),
                             _rankSetup.getDiversityAttribute(), _rankSetup.getDiversityMinGroups(),
                             _rankSetup.getDiversityCutoffFactor(),
-                            AttributeLimiter::toDiversityCutoffStrategy(_rankSetup.getDiversityCutoffStrategy())));
+                            AttributeLimiter::toDiversityCutoffStrategy(_rankSetup.getDiversityCutoffStrategy()));
         }
     }
-    if (_match_limiter.get() == nullptr) {
-        _match_limiter.reset(new NoMatchPhaseLimiter());
+    if ( ! _match_limiter) {
+        _match_limiter = std::make_unique<NoMatchPhaseLimiter>();
     }
 }
 
-MatchToolsFactory::~MatchToolsFactory() {}
+MatchToolsFactory::~MatchToolsFactory() = default;
 
 MatchTools::UP
 MatchToolsFactory::createMatchTools() const
 {
     assert(_valid);
-    return MatchTools::UP(
-            new MatchTools(_queryLimiter, _requestContext.getSoftDoom(), _hardDoom, _query, *_match_limiter, _queryEnv,
-                           _mdl, _rankSetup, _featureOverrides));
+    return std::make_unique<MatchTools>(_queryLimiter, _requestContext.getSoftDoom(), _hardDoom, _query,
+                                        *_match_limiter, _queryEnv, _mdl, _rankSetup, _featureOverrides);
 }
 
 }
diff --git a/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp b/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp
index 8de03411ea3..bce13a96da7 100644
--- a/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp
+++ b/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp
@@ -56,9 +56,10 @@ FeatureSet::SP
 findFeatureSet(const DocsumRequest &req, MatchToolsFactory &mtf, bool summaryFeatures)
 {
     std::vector<uint32_t> docs;
-    for (size_t i = 0; i < req.hits.size(); ++i) {
-        if (req.hits[i].docid != search::endDocId) {
-            docs.push_back(req.hits[i].docid);
+    docs.reserve(req.hits.size());
+    for (const auto & hit : req.hits) {
+        if (hit.docid != search::endDocId) {
+            docs.push_back(hit.docid);
         }
     }
     std::sort(docs.begin(), docs.end());
@@ -102,7 +103,7 @@ Matcher::getFeatureSet(const DocsumRequest & req, ISearchContext & searchCtx, IA
         bool searchSessionCached = cache_props.lookup("query").found();
         if (searchSessionCached) {
             SearchSession::SP session(sessionMgr.pickSearch(sessionId));
-            if (session.get()) {
+            if (session) {
                 MatchToolsFactory &mtf = session->getMatchToolsFactory();
                 FeatureSet::SP result = findFeatureSet(req, mtf, summaryFeatures);
                 session->releaseEnumGuards();
@@ -117,7 +118,7 @@ Matcher::getFeatureSet(const DocsumRequest & req, ISearchContext & searchCtx, IA
     if (!mtf->valid()) {
         LOG(warning, "getFeatureSet(%s): query execution failed (invalid query). Returning empty feature set",
                      (summaryFeatures ? "summary features" : "rank features"));
-        return FeatureSet::SP(new FeatureSet());
+        return std::make_shared<FeatureSet>();
     }
     return findFeatureSet(req, *mtf, summaryFeatures);
 }
@@ -136,7 +137,7 @@ Matcher::Matcher(const search::index::Schema &schema, const Properties &props, c
 {
     search::features::setup_search_features(_blueprintFactory);
     search::fef::test::setup_fef_test_plugin(_blueprintFactory);
-    _rankSetup.reset(new search::fef::RankSetup(_blueprintFactory, _indexEnv));
+    _rankSetup = std::make_shared<search::fef::RankSetup>(_blueprintFactory, _indexEnv);
     _rankSetup->configure(); // reads config values from the property map
     if (!_rankSetup->compile()) {
         throw vespalib::IllegalArgumentException("failed to compile rank setup", VESPA_STRLOC);
@@ -224,14 +225,14 @@ Matcher::match(const SearchRequest &request, vespalib::ThreadBundle &threadBundl
             shouldCacheSearchSession = cache_props.lookup("query").found();
             if (shouldCacheGroupingSession) {
                 GroupingSession::UP session(sessionMgr.pickGrouping(sessionId));
-                if (session.get()) {
+                if (session) {
                     return handleGroupingSession(sessionMgr, groupingContext, std::move(session));
                 }
             }
         }
         const Properties *feature_overrides = &request.propertiesMap.featureOverrides();
         if (shouldCacheSearchSession) {
-            owned_objects.feature_overrides.reset(new Properties(*feature_overrides));
+            owned_objects.feature_overrides = std::make_unique<Properties>(*feature_overrides);
             feature_overrides = owned_objects.feature_overrides.get();
         }
         MatchToolsFactory::UP mtf = create_match_tools_factory(request, searchContext, attrContext,
diff --git a/searchcore/src/vespa/searchcore/proton/matching/matching_stats.cpp b/searchcore/src/vespa/searchcore/proton/matching/matching_stats.cpp
index 0bc1807d714..4fb0e1d72e2 100644
--- a/searchcore/src/vespa/searchcore/proton/matching/matching_stats.cpp
+++ b/searchcore/src/vespa/searchcore/proton/matching/matching_stats.cpp
@@ -1,6 +1,7 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include "matching_stats.h"
+#include <cmath>
 
 namespace proton::matching {
 
@@ -13,6 +14,9 @@ MatchingStats::Partition &get_writable_partition(std::vector<MatchingStats::Part
     return state[id];
 }
 
+constexpr double MIN_TIMEOUT_SEC = 0.001;
+constexpr double MAX_CHANGE_FACTOR = 5;
+
 } // namespace proton::matching::<unnamed>
 
 MatchingStats::MatchingStats()
@@ -62,6 +66,7 @@ MatchingStats::add(const MatchingStats &rhs)
     _docsReRanked += rhs._docsReRanked;
     _softDoomed += rhs.softDoomed();
 
+
     _queryCollateralTime.add(rhs._queryCollateralTime);
     _queryLatency.add(rhs._queryLatency);
     _matchTime.add(rhs._matchTime);
@@ -75,10 +80,19 @@ MatchingStats::add(const MatchingStats &rhs)
 
 MatchingStats &
 MatchingStats::updatesoftDoomFactor(double hardLimit, double softLimit, double duration) {
-    if (duration < softLimit) {
-        _softDoomFactor += 0.01*(softLimit - duration)/hardLimit;
-    } else {
-        _softDoomFactor += 0.02*(softLimit - duration)/hardLimit;
+    // The safety capping here should normally not be necessary as all input numbers
+    // will normally be within reasonable values.
+    // It is merely a safety measure to avoid overflow on bad input as can happen with time senstive stuff
+    // in any soft real time system.
+    if ((hardLimit >= MIN_TIMEOUT_SEC) && (softLimit >= MIN_TIMEOUT_SEC)) {
+        double diff = (softLimit - duration)/hardLimit;
+        if (duration < softLimit) {
+            diff = std::min(diff, _softDoomFactor*MAX_CHANGE_FACTOR);
+            _softDoomFactor += 0.01*diff;
+        } else {
+            diff = std::max(diff, -_softDoomFactor*MAX_CHANGE_FACTOR);
+            _softDoomFactor += 0.02*diff;
+        }
     }
     return *this;
 }
diff --git a/searchcore/src/vespa/searchcore/proton/matching/query.cpp b/searchcore/src/vespa/searchcore/proton/matching/query.cpp
index 38affde6075..e550ad8cad7 100644
--- a/searchcore/src/vespa/searchcore/proton/matching/query.cpp
+++ b/searchcore/src/vespa/searchcore/proton/matching/query.cpp
@@ -28,6 +28,9 @@ using search::query::Node;
 using search::query::QueryTreeCreator;
 using search::query::Weight;
 using search::queryeval::AndBlueprint;
+using search::queryeval::AndNotBlueprint;
+using search::queryeval::RankBlueprint;
+using search::queryeval::IntermediateBlueprint;
 using search::queryeval::Blueprint;
 using search::queryeval::IRequestContext;
 using search::queryeval::SearchIterator;
@@ -73,6 +76,27 @@ void AddLocationNode(const string &location_str, Node::UP &query_tree, Location
     }
     query_tree = std::move(new_base);
 }
+
+IntermediateBlueprint *
+asRankOrAndNot(Blueprint * blueprint) {
+    IntermediateBlueprint * rankOrAndNot = dynamic_cast<RankBlueprint*>(blueprint);
+    if (rankOrAndNot == nullptr) {
+        rankOrAndNot = dynamic_cast<AndNotBlueprint*>(blueprint);
+    }
+    return rankOrAndNot;
+}
+
+IntermediateBlueprint *
+lastConsequtiveRankOrAndNot(Blueprint * blueprint) {
+    IntermediateBlueprint * prev = nullptr;
+    IntermediateBlueprint * curr = asRankOrAndNot(blueprint);
+    while (curr != nullptr) {
+        prev =  curr;
+        curr = asRankOrAndNot(&curr->getChild(0));
+    }
+    return prev;
+}
+
 }  // namespace
 
 Query::Query() = default;
@@ -126,10 +150,18 @@ Query::reserveHandles(const IRequestContext & requestContext, ISearchContext &co
     LOG(debug, "original blueprint:\n%s\n", _blueprint->asString().c_str());
     if (_whiteListBlueprint) {
         auto andBlueprint = std::make_unique<AndBlueprint>();
-        (*andBlueprint)
-            .addChild(std::move(_blueprint))
-            .addChild(std::move(_whiteListBlueprint));
-        _blueprint = std::move(andBlueprint);
+        IntermediateBlueprint * rankOrAndNot = lastConsequtiveRankOrAndNot(_blueprint.get());
+        if (rankOrAndNot != nullptr) {
+            (*andBlueprint)
+                    .addChild(rankOrAndNot->removeChild(0))
+                    .addChild(std::move(_whiteListBlueprint));
+            rankOrAndNot->insertChild(0, std::move(andBlueprint));
+        } else {
+            (*andBlueprint)
+                    .addChild(std::move(_blueprint))
+                    .addChild(std::move(_whiteListBlueprint));
+            _blueprint = std::move(andBlueprint);
+        }
         _blueprint->setDocIdLimit(context.getDocIdLimit());
         LOG(debug, "blueprint after white listing:\n%s\n", _blueprint->asString().c_str());
     }
diff --git a/searchcore/src/vespa/searchcore/proton/matching/query.h b/searchcore/src/vespa/searchcore/proton/matching/query.h
index a3b512fc2b7..21365f75133 100644
--- a/searchcore/src/vespa/searchcore/proton/matching/query.h
+++ b/searchcore/src/vespa/searchcore/proton/matching/query.h
@@ -100,6 +100,8 @@ public:
      * @return estimate of hits produced.
      */
     Blueprint::HitEstimate estimate() const;
+
+    const Blueprint * peekRoot() const { return _blueprint.get(); }
 };
 
 }
diff --git a/searchcore/src/vespa/searchcore/proton/matching/result_processor.cpp b/searchcore/src/vespa/searchcore/proton/matching/result_processor.cpp
index 325803d5aa6..dedda1504a5 100644
--- a/searchcore/src/vespa/searchcore/proton/matching/result_processor.cpp
+++ b/searchcore/src/vespa/searchcore/proton/matching/result_processor.cpp
@@ -24,7 +24,7 @@ ResultProcessor::Result::Result(std::unique_ptr<search::engine::SearchReply> rep
       _numFs4Hits(numFs4Hits)
 { }
 
-ResultProcessor::Result::~Result() { }
+ResultProcessor::Result::~Result() = default;
 
 ResultProcessor::Sort::Sort(uint32_t partitionId, const vespalib::Doom & doom, IAttributeContext &ac, const vespalib::string &ss)
     : sorter(FastS_DefaultResultSorter::instance()),
@@ -43,7 +43,7 @@ ResultProcessor::Context::Context(Sort::UP s, PartialResult::UP r, GroupingConte
       groupingSource(grouping.get())
 { }
 
-ResultProcessor::Context::~Context() { }
+ResultProcessor::Context::~Context() = default;
 
 void
 ResultProcessor::GroupingSource::merge(Source &s) {
@@ -75,11 +75,11 @@ ResultProcessor::ResultProcessor(IAttributeContext &attrContext,
       _wasMerged(false)
 {
     if (!_groupingContext.empty()) {
-        _groupingSession.reset(new GroupingSession(sessionId, _groupingContext, attrContext));
+        _groupingSession = std::make_unique<GroupingSession>(sessionId, _groupingContext, attrContext);
     }
 }
 
-ResultProcessor::~ResultProcessor() { }
+ResultProcessor::~ResultProcessor() = default;
 
 void
 ResultProcessor::prepareThreadContextCreation(size_t num_threads)
@@ -95,19 +95,19 @@ ResultProcessor::prepareThreadContextCreation(size_t num_threads)
 ResultProcessor::Context::UP
 ResultProcessor::createThreadContext(const vespalib::Doom & hardDoom, size_t thread_id, uint32_t distributionKey)
 {
-    Sort::UP sort(new Sort(distributionKey, hardDoom, _attrContext, _sortSpec));
-    PartialResult::UP result(new PartialResult((_offset + _hits), sort->hasSortData()));
+    auto sort = std::make_unique<Sort>(distributionKey, hardDoom, _attrContext, _sortSpec);
+    auto result = std::make_unique<PartialResult>((_offset + _hits), sort->hasSortData());
     search::grouping::GroupingContext::UP groupingContext;
-    if (_groupingSession.get() != 0) {
+    if (_groupingSession) {
         groupingContext = _groupingSession->createThreadContext(thread_id, _attrContext);
     }
-    return Context::UP(new Context(std::move(sort), std::move(result), std::move(groupingContext)));
+    return std::make_unique<Context>(std::move(sort), std::move(result), std::move(groupingContext));
 }
 
 ResultProcessor::Result::UP
 ResultProcessor::makeReply(PartialResultUP full_result)
 {
-    search::engine::SearchReply::UP reply(new search::engine::SearchReply());
+    auto reply = std::make_unique<search::engine::SearchReply>();
     const search::IDocumentMetaStore &metaStore = _metaStore;
     search::engine::SearchReply &r = *reply;
     PartialResult &result = *full_result;
@@ -158,7 +158,7 @@ ResultProcessor::makeReply(PartialResultUP full_result)
         assert(sortOffset == sortDataSize);
     }
     numFs4Hits += reply->hits.size();
-    return Result::UP(new Result(std::move(reply), numFs4Hits));
+    return std::make_unique<Result>(std::move(reply), numFs4Hits);
 }
 
 }
diff --git a/searchcore/src/vespa/searchcore/proton/metrics/trans_log_server_metrics.cpp b/searchcore/src/vespa/searchcore/proton/metrics/trans_log_server_metrics.cpp
index 00a7e9b9140..c2624719d81 100644
--- a/searchcore/src/vespa/searchcore/proton/metrics/trans_log_server_metrics.cpp
+++ b/searchcore/src/vespa/searchcore/proton/metrics/trans_log_server_metrics.cpp
@@ -44,6 +44,7 @@ TransLogServerMetrics::considerRemoveDomains(const DomainStats &stats)
     for (auto itr = _domainMetrics.begin(); itr != _domainMetrics.end(); ) {
         const vespalib::string &documentType = itr->first;
         if (stats.find(documentType) == stats.end()) {
+            _parent->unregisterMetric(*itr->second);
             itr = _domainMetrics.erase(itr);
         } else {
             ++itr;
diff --git a/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt
index b371439dd0e..2df34312b52 100644
--- a/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt
+++ b/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt
@@ -13,6 +13,8 @@ vespa_add_library(searchcore_server STATIC
     disk_mem_usage_sampler.cpp
     disk_mem_usage_forwarder.cpp
     docstorevalidator.cpp
+    document_db_config_owner.cpp
+    document_db_directory_holder.cpp
     document_db_explorer.cpp
     document_db_flush_config.cpp
     document_db_maintenance_config.cpp
@@ -73,6 +75,7 @@ vespa_add_library(searchcore_server STATIC
     proton_config_fetcher.cpp
     proton_config_snapshot.cpp
     proton_configurer.cpp
+    proton_disk_layout.cpp
     prune_session_cache_job.cpp
     pruneremoveddocumentsjob.cpp
     putdonecontext.cpp
diff --git a/searchcore/src/vespa/searchcore/proton/server/document_db_config_owner.cpp b/searchcore/src/vespa/searchcore/proton/server/document_db_config_owner.cpp
new file mode 100644
index 00000000000..28492c5d39a
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/document_db_config_owner.cpp
@@ -0,0 +1,21 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "document_db_config_owner.h"
+#include "document_db_directory_holder.h"
+
+namespace proton {
+
+DocumentDBConfigOwner::DocumentDBConfigOwner()
+    : _holder(std::make_shared<DocumentDBDirectoryHolder>())
+{
+}
+
+DocumentDBConfigOwner::~DocumentDBConfigOwner() = default;
+
+std::shared_ptr<DocumentDBDirectoryHolder>
+DocumentDBConfigOwner::getDocumentDBDirectoryHolder()
+{
+    return _holder;
+};
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/document_db_config_owner.h b/searchcore/src/vespa/searchcore/proton/server/document_db_config_owner.h
new file mode 100644
index 00000000000..cc19bc14fab
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/document_db_config_owner.h
@@ -0,0 +1,25 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "i_document_db_config_owner.h"
+
+namespace proton {
+
+class DocumentDBDirectoryHolder;
+
+/*
+ * Abstract class meant to be a base class for DocumentDB where a
+ * directory holder exists until the document db instance is
+ * destroyed.
+ */
+class DocumentDBConfigOwner : public IDocumentDBConfigOwner
+{
+    std::shared_ptr<DocumentDBDirectoryHolder> _holder;
+public:
+    DocumentDBConfigOwner();
+    virtual ~DocumentDBConfigOwner();
+    std::shared_ptr<DocumentDBDirectoryHolder> getDocumentDBDirectoryHolder();
+};
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/document_db_directory_holder.cpp b/searchcore/src/vespa/searchcore/proton/server/document_db_directory_holder.cpp
new file mode 100644
index 00000000000..d0807a3b8a7
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/document_db_directory_holder.cpp
@@ -0,0 +1,33 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "document_db_directory_holder.h"
+#include <mutex>
+#include <condition_variable>
+
+namespace proton {
+
+namespace {
+
+std::mutex mutex;
+std::condition_variable cv;
+
+}
+
+DocumentDBDirectoryHolder::DocumentDBDirectoryHolder()
+{
+}
+
+DocumentDBDirectoryHolder::~DocumentDBDirectoryHolder()
+{
+    std::lock_guard guard(mutex);
+    cv.notify_all();
+}
+
+void
+DocumentDBDirectoryHolder::waitUntilDestroyed(const std::weak_ptr<DocumentDBDirectoryHolder> &holder)
+{
+    std::unique_lock guard(mutex);
+    cv.wait(guard, [&]() { return !holder.lock(); });
+}
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/document_db_directory_holder.h b/searchcore/src/vespa/searchcore/proton/server/document_db_directory_holder.h
new file mode 100644
index 00000000000..65df1b7bd8e
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/document_db_directory_holder.h
@@ -0,0 +1,20 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "i_document_db_config_owner.h"
+
+namespace proton {
+
+/*
+ * class holding onto a document db directory.
+ */
+class DocumentDBDirectoryHolder
+{
+public:
+    DocumentDBDirectoryHolder();
+    ~DocumentDBDirectoryHolder();
+    static void waitUntilDestroyed(const std::weak_ptr<DocumentDBDirectoryHolder> &holder);
+};
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp
index ad7a2d57c89..24d40e15677 100644
--- a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp
@@ -90,7 +90,7 @@ DocumentDB::DocumentDB(const vespalib::string &baseDir,
                        ConfigStore::UP config_store,
                        InitializeThreads initializeThreads,
                        const HwInfo &hwInfo)
-    : IDocumentDBConfigOwner(),
+    : DocumentDBConfigOwner(),
       IReplayConfig(),
       IFeedHandlerOwner(),
       IDocumentSubDBOwner(),
diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.h b/searchcore/src/vespa/searchcore/proton/server/documentdb.h
index e25f1fe66fc..1a64a4013de 100644
--- a/searchcore/src/vespa/searchcore/proton/server/documentdb.h
+++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.h
@@ -10,7 +10,7 @@
 #include "documentsubdbcollection.h"
 #include "executorthreadingservice.h"
 #include "feedhandler.h"
-#include "i_document_db_config_owner.h"
+#include "document_db_config_owner.h"
 #include "i_document_subdb_owner.h"
 #include "i_feed_handler_owner.h"
 #include "i_lid_space_compaction_handler.h"
@@ -53,7 +53,7 @@ namespace matching { class SessionManager; }
  * to ensure that there are never multiple writers. Unless explicitly stated,
  * none of the methods of this class are thread-safe.
  */
-class DocumentDB : public IDocumentDBConfigOwner,
+class DocumentDB : public DocumentDBConfigOwner,
                    public IReplayConfig,
                    public IFeedHandlerOwner,
                    public IDocumentSubDBOwner,
diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.cpp
index d5a9b09ead7..fd1f9f1155d 100644
--- a/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.cpp
@@ -110,7 +110,7 @@ DocumentDBConfig(const DocumentDBConfig &cfg)
       _delayedAttributeAspects(false)
 { }
 
-DocumentDBConfig::~DocumentDBConfig() { }
+DocumentDBConfig::~DocumentDBConfig() = default;
 
 bool
 DocumentDBConfig::operator==(const DocumentDBConfig & rhs) const
diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp
index 80809d76bef..f2230215c3d 100644
--- a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp
@@ -157,13 +157,26 @@ deriveCompression(const T & config) {
     return compression;
 }
 
+DocumentStore::Config::UpdateStrategy
+derive(ProtonConfig::Summary::Cache::UpdateStrategy strategy) {
+    switch (strategy) {
+        case ProtonConfig::Summary::Cache::UpdateStrategy::INVALIDATE:
+            return DocumentStore::Config::UpdateStrategy::INVALIDATE;
+        case ProtonConfig::Summary::Cache::UpdateStrategy::UPDATE:
+            return DocumentStore::Config::UpdateStrategy::UPDATE;
+    }
+    return DocumentStore::Config::UpdateStrategy::INVALIDATE;
+}
+
 DocumentStore::Config
 getStoreConfig(const ProtonConfig::Summary::Cache & cache, const HwInfo & hwInfo)
 {
     size_t maxBytes = (cache.maxbytes < 0)
                       ? (hwInfo.memory().sizeBytes()*std::min(50l, -cache.maxbytes))/100l
                       : cache.maxbytes;
-    return DocumentStore::Config(deriveCompression(cache.compression), maxBytes, cache.initialentries).allowVisitCaching(cache.allowvisitcaching);
+    return DocumentStore::Config(deriveCompression(cache.compression), maxBytes, cache.initialentries)
+            .allowVisitCaching(cache.allowvisitcaching)
+            .updateStrategy(derive(cache.updateStrategy));
 }
 
 LogDocumentStore::Config
diff --git a/searchcore/src/vespa/searchcore/proton/server/documentretriever.cpp b/searchcore/src/vespa/searchcore/proton/server/documentretriever.cpp
index a7bd61e75d4..1a1bf1ed000 100644
--- a/searchcore/src/vespa/searchcore/proton/server/documentretriever.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/documentretriever.cpp
@@ -48,7 +48,7 @@ DocumentRetriever
     LOG(debug, "checking document type '%s' for position fields", docTypeName.getName().c_str());
     for (const document::Field * field : fields) {
         if (field->getDataType().getId() == positionDataTypeId) {
-            LOG(debug, "Field '%s' is a position field", field->getName().c_str());
+            LOG(debug, "Field '%s' is a position field", field->getName().data());
             const vespalib::string & zcurve_name = PositionDataType::getZCurveFieldName(field->getName());
             AttributeGuard::UP attr = attr_manager.getAttribute(zcurve_name);
             if (attr && attr->valid()) {
diff --git a/searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.cpp b/searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.cpp
index 1c2406b2acf..dfe98b44adc 100644
--- a/searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.cpp
@@ -37,11 +37,14 @@ FastAccessFeedView::updateAttributes(SerialNum serialNum, search::DocumentIdT li
 }
 
 void
-FastAccessFeedView::updateAttributes(SerialNum serialNum, Lid lid, FutureDoc doc,
+FastAccessFeedView::updateAttributes(SerialNum serialNum, Lid lid, FutureDoc futureDoc,
                                      bool immediateCommit, OnOperationDoneType onWriteDone)
 {
     if (_attributeWriter->hasStructFieldAttribute()) {
-        _attributeWriter->update(serialNum, *doc.get(), lid, immediateCommit, onWriteDone);
+        const std::unique_ptr<const Document> & doc = futureDoc.get();
+        if (doc) {
+            _attributeWriter->update(serialNum, *doc, lid, immediateCommit, onWriteDone);
+        }
     }
 }
 
diff --git a/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.cpp b/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.cpp
index 76f2ffce93a..72f48df3295 100644
--- a/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.cpp
@@ -229,6 +229,7 @@ FileConfigManager::FileConfigManager(const vespalib::string &baseDir,
       _protonConfig()
 {
     vespalib::mkdir(baseDir, false);
+    vespalib::File::sync(vespalib::dirname(baseDir));
     if (!_info.load())
         _info.save();
     removeInvalid();
@@ -297,6 +298,8 @@ FileConfigManager::saveConfig(const DocumentDBConfig &snapshot,
     bool saveHistorySchemaRes = historySchema.saveToFile(snapDir + "/historyschema.txt");
     assert(saveHistorySchemaRes);
     (void) saveHistorySchemaRes;
+    vespalib::File::sync(snapDir);
+    vespalib::File::sync(_baseDir);
 
     _info.validateSnapshot(serialNum);
 
@@ -402,6 +405,7 @@ FileConfigManager::removeInvalid()
             LOG(warning, "Removing obsolete config directory '%s' failed due to %s", snapDir.c_str(), e.what());
         }
     }
+    vespalib::File::sync(_baseDir);
     for (const auto &serial : toRem) {
         _info.removeSnapshot(serial);
     }
diff --git a/searchcore/src/vespa/searchcore/proton/server/i_proton_configurer_owner.h b/searchcore/src/vespa/searchcore/proton/server/i_proton_configurer_owner.h
index 242e99fffbf..fec8430e41d 100644
--- a/searchcore/src/vespa/searchcore/proton/server/i_proton_configurer_owner.h
+++ b/searchcore/src/vespa/searchcore/proton/server/i_proton_configurer_owner.h
@@ -10,7 +10,7 @@ namespace vespalib { class ThreadStackExecutorBase; }
 
 namespace proton {
 
-class IDocumentDBConfigOwner;
+class DocumentDBConfigOwner;
 
 /*
  * Interface class for owner of a proton configurer, with callback methods
@@ -21,12 +21,12 @@ class IProtonConfigurerOwner
     using InitializeThreads = std::shared_ptr<vespalib::ThreadStackExecutorBase>;
 public:
     virtual ~IProtonConfigurerOwner() { }
-    virtual IDocumentDBConfigOwner *addDocumentDB(const DocTypeName &docTypeName,
-                                                  document::BucketSpace bucketSpace,
-                                                  const vespalib::string &configId,
-                                                  const std::shared_ptr<BootstrapConfig> &bootstrapConfig,
-                                                  const std::shared_ptr<DocumentDBConfig> &documentDBConfig,
-                                                  InitializeThreads initializeThreads) = 0;
+    virtual std::shared_ptr<DocumentDBConfigOwner> addDocumentDB(const DocTypeName &docTypeName,
+                                                                 document::BucketSpace bucketSpace,
+                                                                 const vespalib::string &configId,
+                                                                 const std::shared_ptr<BootstrapConfig> &bootstrapConfig,
+                                                                 const std::shared_ptr<DocumentDBConfig> &documentDBConfig,
+                                                                 InitializeThreads initializeThreads) = 0;
     virtual void removeDocumentDB(const DocTypeName &docTypeName) = 0;
     virtual void applyConfig(const std::shared_ptr<BootstrapConfig> &bootstrapConfig) = 0;
 };
diff --git a/searchcore/src/vespa/searchcore/proton/server/i_proton_disk_layout.h b/searchcore/src/vespa/searchcore/proton/server/i_proton_disk_layout.h
new file mode 100644
index 00000000000..1ee521379eb
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/i_proton_disk_layout.h
@@ -0,0 +1,23 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <set>
+
+namespace proton {
+
+class DocTypeName;
+
+/**
+ * Interface class with utility functions for handling the disk
+ * directory layout for proton instance.
+ */
+class IProtonDiskLayout
+{
+public:
+    virtual ~IProtonDiskLayout() = default;
+    virtual void remove(const DocTypeName &docTypeName) = 0;
+    virtual void initAndPruneUnused(const std::set<DocTypeName> &docTypeNames) = 0;
+};
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/matchview.cpp b/searchcore/src/vespa/searchcore/proton/server/matchview.cpp
index 3162f9a1c45..9af648917c4 100644
--- a/searchcore/src/vespa/searchcore/proton/server/matchview.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/matchview.cpp
@@ -44,13 +44,13 @@ MatchView::MatchView(const Matchers::SP &matchers,
       _docIdLimit(docIdLimit)
 { }
 
-MatchView::~MatchView() { }
+MatchView::~MatchView() = default;
 
 Matcher::SP
 MatchView::getMatcher(const vespalib::string & rankProfile) const
 {
     Matcher::SP retval = _matchers->lookup(rankProfile);
-    if (retval.get() == NULL) {
+    if ( ! retval) {
         throw std::runtime_error(make_string("Failed locating Matcher for rank profile '%s'", rankProfile.c_str()));
     }
     LOG(debug, "Rankprofile = %s has termwise_limit=%f", rankProfile.c_str(), retval->get_termwise_limit());
@@ -60,8 +60,8 @@ MatchView::getMatcher(const vespalib::string & rankProfile) const
 
 MatchContext::UP MatchView::createContext() const {
     IAttributeContext::UP attrCtx = _attrMgr->createContext();
-    ISearchContext::UP searchCtx(new SearchContext(_indexSearchable, _docIdLimit.get()));
-    return MatchContext::UP(new MatchContext(std::move(attrCtx), std::move(searchCtx)));
+    auto searchCtx = std::make_unique<SearchContext>(_indexSearchable, _docIdLimit.get());
+    return std::make_unique<MatchContext>(std::move(attrCtx), std::move(searchCtx));
 }
 
 
diff --git a/searchcore/src/vespa/searchcore/proton/server/memoryflush.cpp b/searchcore/src/vespa/searchcore/proton/server/memoryflush.cpp
index 7d97cc4030a..d72ef274d99 100644
--- a/searchcore/src/vespa/searchcore/proton/server/memoryflush.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/memoryflush.cpp
@@ -213,7 +213,7 @@ MemoryFlush::getFlushTargets(const FlushContext::List &targetList,
             }
             oss << fv[i]->getName();
         }
-        LOG(debug, "getFlushTargets(): %zu sorted targets: [%s]", fv.size(), oss.str().c_str());
+        LOG(debug, "getFlushTargets(): %zu sorted targets: [%s]", fv.size(), oss.str().data());
     }
     return fv;
 }
diff --git a/searchcore/src/vespa/searchcore/proton/server/prepare_restart_handler.cpp b/searchcore/src/vespa/searchcore/proton/server/prepare_restart_handler.cpp
index aef49c51f7a..1c930ef1ddc 100644
--- a/searchcore/src/vespa/searchcore/proton/server/prepare_restart_handler.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/prepare_restart_handler.cpp
@@ -50,8 +50,7 @@ PrepareRestartHandler::performPrepareRestart(const ProtonConfig &protonCfg, std:
 {
     _running = true;
     lock.unlock();
-    auto strategy = std::make_shared<PrepareRestartFlushStrategy>(createPrepareRestartConfig(protonCfg));
-    _flushEngine.setStrategy(strategy);
+    _flushEngine.setStrategy(std::make_shared<PrepareRestartFlushStrategy>(createPrepareRestartConfig(protonCfg)));
     lock.lock();
     _running = false;
     _cond.notify_all();
diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.cpp b/searchcore/src/vespa/searchcore/proton/server/proton.cpp
index 32dc711d5cf..ab0abde6ccd 100644
--- a/searchcore/src/vespa/searchcore/proton/server/proton.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/proton.cpp
@@ -9,6 +9,7 @@
 #include "prepare_restart_handler.h"
 #include "proton.h"
 #include "proton_config_snapshot.h"
+#include "proton_disk_layout.h"
 #include "resource_usage_explorer.h"
 #include "searchhandlerproxy.h"
 #include "simpleflush.h"
@@ -193,7 +194,8 @@ Proton::Proton(const config::ConfigUri & configUri,
       // This executor can only have 1 thread as it is used for
       // serializing startup.
       _executor(1, 128 * 1024),
-      _protonConfigurer(_executor, *this),
+      _protonDiskLayout(),
+      _protonConfigurer(_executor, *this, _protonDiskLayout),
       _protonConfigFetcher(configUri, _protonConfigurer, subscribeTimeout),
       _warmupExecutor(),
       _summaryExecutor(),
@@ -240,9 +242,8 @@ Proton::init(const BootstrapConfig::SP & configSnapshot)
     const HwInfo & hwInfo = configSnapshot->getHwInfo();
 
     setFS4Compression(protonConfig);
-    _diskMemUsageSampler = std::make_unique<DiskMemUsageSampler>
-                           (protonConfig.basedir,
-                            diskMemUsageSamplerConfig(protonConfig, hwInfo));
+    _diskMemUsageSampler = std::make_unique<DiskMemUsageSampler>(protonConfig.basedir,
+                                                                 diskMemUsageSamplerConfig(protonConfig, hwInfo));
 
     _tls = std::make_unique<TLS>(_configUri.createWithNewId(protonConfig.tlsconfigid), _fileHeaderContext);
     _metricsEngine->addMetricsHook(_metricsHook);
@@ -253,6 +254,7 @@ Proton::init(const BootstrapConfig::SP & configSnapshot)
     _distributionKey = protonConfig.distributionkey;
     _summaryEngine= std::make_unique<SummaryEngine>(protonConfig.numsummarythreads);
     _docsumBySlime = std::make_unique<DocsumBySlime>(*_summaryEngine);
+
     IFlushStrategy::SP strategy;
     const ProtonConfig::Flush & flush(protonConfig.flush);
     switch (flush.strategy) {
@@ -269,7 +271,7 @@ Proton::init(const BootstrapConfig::SP & configSnapshot)
         strategy = std::make_shared<SimpleFlush>();
         break;
     }
-    vespalib::mkdir(protonConfig.basedir + "/documents", true);
+    _protonDiskLayout = std::make_unique<ProtonDiskLayout>(protonConfig.basedir, protonConfig.tlsspec);
     vespalib::chdir(protonConfig.basedir);
     _tls->start();
     _flushEngine = std::make_unique<FlushEngine>(std::make_shared<flushengine::TlsStatsFactory>(_tls->getTransLogServer()),
@@ -283,17 +285,15 @@ Proton::init(const BootstrapConfig::SP & configSnapshot)
     LOG(debug, "Start proton server with root at %s and cwd at %s",
         protonConfig.basedir.c_str(), getcwd(tmp, sizeof(tmp)));
 
-    _persistenceEngine.reset(new PersistenceEngine(*this,
-                                                   _diskMemUsageSampler->writeFilter(),
-                                                   protonConfig.visit.defaultserializedsize,
-                                                   protonConfig.visit.ignoremaxbytes));
-
+    _persistenceEngine = std::make_unique<PersistenceEngine>(*this, _diskMemUsageSampler->writeFilter(),
+                                                             protonConfig.visit.defaultserializedsize,
+                                                             protonConfig.visit.ignoremaxbytes);
 
     vespalib::string fileConfigId;
-    _warmupExecutor.reset(new vespalib::ThreadStackExecutor(4, 128*1024));
+    _warmupExecutor = std::make_unique<vespalib::ThreadStackExecutor>(4, 128*1024);
 
     const size_t summaryThreads = deriveCompactionCompressionThreads(protonConfig, hwInfo.cpu());
-    _summaryExecutor.reset(new vespalib::BlockingThreadStackExecutor(summaryThreads, 128*1024, summaryThreads*16));
+    _summaryExecutor = std::make_unique<vespalib::BlockingThreadStackExecutor>(summaryThreads, 128*1024, summaryThreads*16);
     InitializeThreads initializeThreads;
     if (protonConfig.initialize.threads > 0) {
         initializeThreads = std::make_shared<vespalib::ThreadStackExecutor>(protonConfig.initialize.threads, 128 * 1024);
@@ -305,7 +305,7 @@ Proton::init(const BootstrapConfig::SP & configSnapshot)
     _prepareRestartHandler = std::make_unique<PrepareRestartHandler>(*_flushEngine);
     RPCHooks::Params rpcParams(*this, protonConfig.rpcport, _configUri.getConfigId());
     rpcParams.slobrok_config = _configUri.createWithNewId(protonConfig.slobrokconfigid);
-    _rpcHooks.reset(new RPCHooks(rpcParams));
+    _rpcHooks = std::make_unique<RPCHooks>(rpcParams);
 
     waitForInitDone();
 
@@ -354,7 +354,7 @@ Proton::applyConfig(const BootstrapConfig::SP & configSnapshot)
     }
 }
 
-IDocumentDBConfigOwner *
+std::shared_ptr<DocumentDBConfigOwner>
 Proton::addDocumentDB(const DocTypeName &docTypeName,
                       document::BucketSpace bucketSpace,
                       const vespalib::string &configId,
@@ -368,21 +368,21 @@ Proton::addDocumentDB(const DocTypeName &docTypeName,
         if (docType != NULL) {
             LOG(info, "Add document database: doctypename(%s), configid(%s)",
                 docTypeName.toString().c_str(), configId.c_str());
-            return addDocumentDB(*docType, bucketSpace, bootstrapConfig, documentDBConfig, initializeThreads).get();
+            return addDocumentDB(*docType, bucketSpace, bootstrapConfig, documentDBConfig, initializeThreads);
         } else {
 
             LOG(warning,
                 "Did not find document type '%s' in the document manager. "
                 "Skipping creating document database for this type",
                 docTypeName.toString().c_str());
-            return nullptr;
+            return std::shared_ptr<DocumentDBConfigOwner>();
         }
     } catch (const document::DocumentTypeNotFoundException & e) {
         LOG(warning,
             "Did not find document type '%s' in the document manager. "
             "Skipping creating document database for this type",
             docTypeName.toString().c_str());
-        return nullptr;
+        return std::shared_ptr<DocumentDBConfigOwner>();
     }
 }
 
@@ -528,23 +528,11 @@ Proton::addDocumentDB(const document::DocumentType &docType,
         // 1 thread per document type.
         initializeThreads = std::make_shared<vespalib::ThreadStackExecutor>(1, 128 * 1024);
     }
-    DocumentDB::SP ret(new DocumentDB(config.basedir + "/documents",
-                                      documentDBConfig,
-                                      config.tlsspec,
-                                      _queryLimiter,
-                                      _clock,
-                                      docTypeName,
-                                      bucketSpace,
-                                      config,
-                                      *this,
-                                      *_warmupExecutor,
-                                      *_summaryExecutor,
-                                      *_tls->getTransLogServer(),
-                                      *_metricsEngine,
-                                      _fileHeaderContext,
-                                      std::move(config_store),
-                                      initializeThreads,
-                                      bootstrapConfig->getHwInfo()));
+    auto ret = std::make_shared<DocumentDB>(config.basedir + "/documents", documentDBConfig, config.tlsspec,
+                                            _queryLimiter, _clock, docTypeName, bucketSpace, config, *this,
+                                            *_warmupExecutor, *_summaryExecutor, *_tls->getTransLogServer(),
+                                            *_metricsEngine, _fileHeaderContext, std::move(config_store),
+                                            initializeThreads, bootstrapConfig->getHwInfo());
     try {
         ret->start();
     } catch (vespalib::Exception &e) {
@@ -571,10 +559,10 @@ Proton::addDocumentDB(const document::DocumentType &docType,
         // TODO: Fix race with new cluster state setting.
         _persistenceEngine->putHandler(bucketSpace, docTypeName, persistenceHandler);
     }
-    SearchHandlerProxy::SP searchHandler(new SearchHandlerProxy(ret));
+    auto searchHandler = std::make_shared<SearchHandlerProxy>(ret);
     _summaryEngine->putSearchHandler(docTypeName, searchHandler);
     _matchEngine->putSearchHandler(docTypeName, searchHandler);
-    FlushHandlerProxy::SP flushHandler(new FlushHandlerProxy(ret));
+    auto flushHandler = std::make_shared<FlushHandlerProxy>(ret);
     _flushEngine->putFlushHandler(docTypeName, flushHandler);
     _diskMemUsageSampler->notifier().addDiskMemUsageListener(ret->diskMemUsageListener());
     return ret;
@@ -624,7 +612,7 @@ Proton::MonitorReply::UP
 Proton::ping(MonitorRequest::UP request, MonitorClient & client)
 {
     (void) client;
-    MonitorReply::UP reply(new MonitorReply());
+    auto reply = std::make_unique<MonitorReply>();
     MonitorReply &ret = *reply;
 
     BootstrapConfig::SP configSnapshot = getActiveConfigSnapshot();
@@ -807,8 +795,8 @@ struct DocumentDBMapExplorer : vespalib::StateExplorer {
     std::shared_timed_mutex &mutex;
     DocumentDBMapExplorer(const DocumentDBMap &documentDBMap_in, std::shared_timed_mutex &mutex_in)
         : documentDBMap(documentDBMap_in), mutex(mutex_in) {}
-    virtual void get_state(const vespalib::slime::Inserter &, bool) const override {}
-    virtual std::vector<vespalib::string> get_children_names() const override {
+    void get_state(const vespalib::slime::Inserter &, bool) const override {}
+    std::vector<vespalib::string> get_children_names() const override {
         std::shared_lock<std::shared_timed_mutex> guard(mutex);
         std::vector<vespalib::string> names;
         for (const auto &item: documentDBMap) {
@@ -816,14 +804,14 @@ struct DocumentDBMapExplorer : vespalib::StateExplorer {
         }
         return names;
     }
-    virtual std::unique_ptr<vespalib::StateExplorer> get_child(vespalib::stringref name) const override {
+    std::unique_ptr<vespalib::StateExplorer> get_child(vespalib::stringref name) const override {
         typedef std::unique_ptr<StateExplorer> Explorer_UP;
         std::shared_lock<std::shared_timed_mutex> guard(mutex);
         auto result = documentDBMap.find(DocTypeName(vespalib::string(name)));
         if (result == documentDBMap.end()) {
             return Explorer_UP(nullptr);
         }
-        return Explorer_UP(new DocumentDBExplorer(result->second));
+        return std::make_unique<DocumentDBExplorer>(result->second);
     }
 };
 
diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.h b/searchcore/src/vespa/searchcore/proton/server/proton.h
index dd2b42d94cf..6e07ddcecdb 100644
--- a/searchcore/src/vespa/searchcore/proton/server/proton.h
+++ b/searchcore/src/vespa/searchcore/proton/server/proton.h
@@ -37,6 +37,7 @@ namespace proton {
 
 class DiskMemUsageSampler;
 class IDocumentDBReferenceRegistry;
+class IProtonDiskLayout;
 class PrepareRestartHandler;
 class SummaryEngine;
 class DocsumBySlime;
@@ -111,6 +112,7 @@ private:
     std::unique_ptr<vespalib::StateServer>  _stateServer;
     std::unique_ptr<TransportServer>        _fs4Server;
     vespalib::ThreadStackExecutor   _executor;
+    std::unique_ptr<IProtonDiskLayout> _protonDiskLayout;
     ProtonConfigurer                _protonConfigurer;
     ProtonConfigFetcher             _protonConfigFetcher;
     std::unique_ptr<vespalib::ThreadStackExecutorBase> _warmupExecutor;
@@ -130,7 +132,7 @@ private:
     std::mutex                      _nodeUpLock;
     std::set<BucketSpace>           _nodeUp;   // bucketspaces where node is up
 
-    IDocumentDBConfigOwner *
+    std::shared_ptr<DocumentDBConfigOwner>
     addDocumentDB(const DocTypeName & docTypeName, BucketSpace bucketSpace, const vespalib::string & configid,
                   const BootstrapConfig::SP & bootstrapConfig, const std::shared_ptr<DocumentDBConfig> &documentDBConfig,
                   InitializeThreads initializeThreads) override;
diff --git a/searchcore/src/vespa/searchcore/proton/server/proton_configurer.cpp b/searchcore/src/vespa/searchcore/proton/server/proton_configurer.cpp
index 07ed83fd8c6..0b9293a4aab 100644
--- a/searchcore/src/vespa/searchcore/proton/server/proton_configurer.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/proton_configurer.cpp
@@ -4,7 +4,9 @@
 #include "proton_config_snapshot.h"
 #include "bootstrapconfig.h"
 #include "i_proton_configurer_owner.h"
-#include "i_document_db_config_owner.h"
+#include "document_db_config_owner.h"
+#include "document_db_directory_holder.h"
+#include "i_proton_disk_layout.h"
 #include <vespa/vespalib/util/lambdatask.h>
 #include <vespa/vespalib/util/threadstackexecutorbase.h>
 #include <vespa/document/bucket/fixed_bucket_spaces.h>
@@ -38,7 +40,8 @@ getBucketSpace(const BootstrapConfig &bootstrapConfig, const DocTypeName &name)
 
 
 ProtonConfigurer::ProtonConfigurer(vespalib::ThreadStackExecutorBase &executor,
-                                   IProtonConfigurerOwner &owner)
+                                   IProtonConfigurerOwner &owner,
+                                   const std::unique_ptr<IProtonDiskLayout> &diskLayout)
     : IProtonConfigurer(),
       _executor(executor),
       _owner(owner),
@@ -47,7 +50,8 @@ ProtonConfigurer::ProtonConfigurer(vespalib::ThreadStackExecutorBase &executor,
       _activeConfigSnapshot(),
       _mutex(),
       _allowReconfig(false),
-      _componentConfig()
+      _componentConfig(),
+      _diskLayout(diskLayout)
 {
 }
 
@@ -134,6 +138,9 @@ ProtonConfigurer::applyConfig(std::shared_ptr<ProtonConfigSnapshot> configSnapsh
     }
     const auto &bootstrapConfig = configSnapshot->getBootstrapConfig();
     const ProtonConfig &protonConfig = bootstrapConfig->getProtonConfig();
+    if (initialConfig) {
+        pruneInitialDocumentDBDirs(*configSnapshot);
+    }
     _owner.applyConfig(bootstrapConfig);
     for (const auto &ddbConfig : protonConfig.documentdb) {
         DocTypeName docTypeName(ddbConfig.inputdoctypename);
@@ -162,14 +169,28 @@ ProtonConfigurer::configureDocumentDB(const ProtonConfigSnapshot &configSnapshot
     const auto &documentDBConfig = cfgitr->second;
     auto dbitr(_documentDBs.find(docTypeName));
     if (dbitr == _documentDBs.end()) {
-        auto *newdb = _owner.addDocumentDB(docTypeName, bucketSpace, configId, bootstrapConfig, documentDBConfig, initializeThreads);
-        if (newdb != nullptr) {
-            auto insres = _documentDBs.insert(std::make_pair(docTypeName, newdb));
+        auto newdb = _owner.addDocumentDB(docTypeName, bucketSpace, configId, bootstrapConfig, documentDBConfig, initializeThreads);
+        if (newdb) {
+            auto insres = _documentDBs.insert(std::make_pair(docTypeName, std::make_pair(newdb, newdb->getDocumentDBDirectoryHolder())));
             assert(insres.second);
         }
     } else {
-        dbitr->second->reconfigure(documentDBConfig);
+        auto documentDB = dbitr->second.first.lock();
+        assert(documentDB);
+        documentDB->reconfigure(documentDBConfig);
+    }
+}
+
+void
+ProtonConfigurer::pruneInitialDocumentDBDirs(const ProtonConfigSnapshot &configSnapshot)
+{
+    std::set<DocTypeName> docTypeNames;
+    const auto &bootstrapConfig = configSnapshot.getBootstrapConfig();
+    const ProtonConfig &protonConfig = bootstrapConfig->getProtonConfig();
+    for (const auto &ddbConfig : protonConfig.documentdb) {
+        docTypeNames.emplace(ddbConfig.inputdoctypename);
     }
+    _diskLayout->initAndPruneUnused(docTypeNames);
 }
 
 void
@@ -189,6 +210,8 @@ ProtonConfigurer::pruneDocumentDBs(const ProtonConfigSnapshot &configSnapshot)
         auto found(newDocTypes.find(dbitr->first));
         if (found == newDocTypes.end()) {
             _owner.removeDocumentDB(dbitr->first);
+            DocumentDBDirectoryHolder::waitUntilDestroyed(dbitr->second.second);
+            _diskLayout->remove(dbitr->first);
             dbitr = _documentDBs.erase(dbitr);
         } else {
             ++dbitr;
diff --git a/searchcore/src/vespa/searchcore/proton/server/proton_configurer.h b/searchcore/src/vespa/searchcore/proton/server/proton_configurer.h
index 149be3a9e62..c896f12bd4f 100644
--- a/searchcore/src/vespa/searchcore/proton/server/proton_configurer.h
+++ b/searchcore/src/vespa/searchcore/proton/server/proton_configurer.h
@@ -12,9 +12,11 @@
 
 namespace proton {
 
+class DocumentDBDirectoryHolder;
 class IDocumentDBConfigOwner;
 class IProtonConfigurerOwner;
 class BootstrapConfig;
+class IProtonDiskLayout;
 
 /*
  * Class to handle config changes to proton using config snapshots spanning
@@ -22,7 +24,7 @@ class BootstrapConfig;
  */
 class ProtonConfigurer : public IProtonConfigurer
 {
-    using DocumentDBs = std::map<DocTypeName, IDocumentDBConfigOwner *>;
+    using DocumentDBs = std::map<DocTypeName, std::pair<std::weak_ptr<IDocumentDBConfigOwner>, std::weak_ptr<DocumentDBDirectoryHolder>>>;
     using InitializeThreads = std::shared_ptr<vespalib::ThreadStackExecutorBase>;
 
     ExecutorThreadService _executor;
@@ -33,6 +35,7 @@ class ProtonConfigurer : public IProtonConfigurer
     mutable std::mutex _mutex;
     bool _allowReconfig;
     vespalib::SimpleComponentConfigProducer _componentConfig;
+    const std::unique_ptr<IProtonDiskLayout> &_diskLayout;
 
     void performReconfigure();
     bool skipConfig(const ProtonConfigSnapshot *configSnapshot, bool initialConfig);
@@ -42,10 +45,12 @@ class ProtonConfigurer : public IProtonConfigurer
                              const DocTypeName &docTypeName, document::BucketSpace bucketSpace,
                              const vespalib::string &configId, const InitializeThreads &initializeThreads);
     void pruneDocumentDBs(const ProtonConfigSnapshot &configSnapshot);
+    void pruneInitialDocumentDBDirs(const ProtonConfigSnapshot &configSnapshot);
 
 public:
     ProtonConfigurer(vespalib::ThreadStackExecutorBase &executor,
-                     IProtonConfigurerOwner &owner);
+                     IProtonConfigurerOwner &owner,
+                     const std::unique_ptr<IProtonDiskLayout> &diskLayout);
 
     ~ProtonConfigurer();
 
diff --git a/searchcore/src/vespa/searchcore/proton/server/proton_disk_layout.cpp b/searchcore/src/vespa/searchcore/proton/server/proton_disk_layout.cpp
new file mode 100644
index 00000000000..31fd44eec5e
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/proton_disk_layout.cpp
@@ -0,0 +1,120 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "proton_disk_layout.h"
+#include <vespa/vespalib/io/fileutil.h>
+#include <vespa/fastos/file.h>
+#include <vespa/searchcore/proton/common/doctypename.h>
+#include <vespa/searchlib/transactionlog/translogclient.h>
+#include <cassert>
+
+#include <vespa/log/log.h>
+LOG_SETUP(".proton.server.proton_disk_layout");
+
+using search::transactionlog::TransLogClient;
+
+namespace proton {
+
+namespace {
+
+struct DocumentDBDirMeta
+{
+    bool normal;
+    bool removed;
+
+    DocumentDBDirMeta()
+        : normal(false),
+          removed(false)
+    {
+    }
+};
+
+using DocumentDBDirScan = std::map<DocTypeName, DocumentDBDirMeta>;
+
+vespalib::string getDocumentsDir(const vespalib::string &baseDir)
+{
+    return baseDir + "/documents";
+}
+
+vespalib::string removedSuffix(".removed");
+
+vespalib::string getNormalName(const vespalib::string removedName) {
+    return removedName.substr(0, removedName.size() - removedSuffix.size());
+}
+
+vespalib::string getRemovedName(const vespalib::string &normalName)
+{
+    return normalName + removedSuffix;
+}
+
+bool isRemovedName(const vespalib::string &dirName)
+{
+    return dirName.size() > removedSuffix.size() && dirName.substr(dirName.size() - removedSuffix.size()) == removedSuffix;
+}
+
+void scanDir(const vespalib::string documentsDir, DocumentDBDirScan &dirs)
+{
+    auto names = vespalib::listDirectory(documentsDir);
+    for (const auto &name : names) {
+        if (vespalib::isDirectory(documentsDir + "/" + name)) {
+            if (isRemovedName(name)) {
+                dirs[DocTypeName(getNormalName(name))].removed = true;
+            } else {
+                dirs[DocTypeName(name)].normal = true;
+            }
+        }
+    }
+}
+
+}
+
+ProtonDiskLayout::ProtonDiskLayout(const vespalib::string &baseDir, const vespalib::string &tlsSpec)
+    : _baseDir(baseDir),
+      _tlsSpec(tlsSpec)
+{
+    vespalib::mkdir(getDocumentsDir(_baseDir), true);
+}
+
+ProtonDiskLayout::~ProtonDiskLayout() = default;
+
+void
+ProtonDiskLayout::remove(const DocTypeName &docTypeName)
+{
+    vespalib::string documentsDir(getDocumentsDir(_baseDir));
+    vespalib::string name(docTypeName.toString());
+    vespalib::string normalDir(documentsDir + "/" + name);
+    vespalib::string removedDir(documentsDir + "/" + getRemovedName(name));
+    vespalib::rename(normalDir, removedDir, false, false);
+    vespalib::File::sync(documentsDir);
+    TransLogClient tlc(_tlsSpec);
+    if (!tlc.remove(name)) {
+        LOG(fatal, "Failed to remove tls domain %s", name.c_str());
+        LOG_ABORT("Failed to remove tls domain");
+    }
+    vespalib::rmdir(removedDir, true);
+    vespalib::File::sync(documentsDir);
+}
+
+void
+ProtonDiskLayout::initAndPruneUnused(const std::set<DocTypeName> &docTypeNames)
+{
+    vespalib::string documentsDir(getDocumentsDir(_baseDir));
+    DocumentDBDirScan dirs;
+    scanDir(documentsDir, dirs);
+    for (const auto &dir : dirs) {
+        if (dir.second.removed) {
+            // Complete interrupted removal
+            if (dir.second.normal) {
+                vespalib::string name(dir.first.toString());
+                vespalib::string normalDir(documentsDir + "/" + name);
+                vespalib::rmdir(normalDir, true);
+            }
+            remove(dir.first);
+        } else if (docTypeNames.count(dir.first) == 0) {
+            // Remove unused directory
+            remove(dir.first);
+        }
+    }
+}
+
+} // namespace proton
+
diff --git a/searchcore/src/vespa/searchcore/proton/server/proton_disk_layout.h b/searchcore/src/vespa/searchcore/proton/server/proton_disk_layout.h
new file mode 100644
index 00000000000..167a1900b99
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/server/proton_disk_layout.h
@@ -0,0 +1,27 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "i_proton_disk_layout.h"
+#include <vespa/vespalib/stllike/string.h>
+
+namespace proton {
+
+/**
+ * Class with utility functions for handling the disk directory layout
+ * for proton instance.
+ */
+class ProtonDiskLayout : public IProtonDiskLayout
+{
+private:
+    const vespalib::string _baseDir;
+    const vespalib::string _tlsSpec;
+
+public:
+    ProtonDiskLayout(const vespalib::string &baseDir, const vespalib::string &tlsSpec);
+    ~ProtonDiskLayout() override;
+    void remove(const DocTypeName &docTypeName) override;
+    void initAndPruneUnused(const std::set<DocTypeName> &docTypeNames) override;
+};
+
+} // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp
index ad06525a5a3..2d3c204d259 100644
--- a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp
@@ -139,6 +139,7 @@ StoreOnlyDocSubDB::StoreOnlyDocSubDB(const Config &cfg, const Context &ctx)
       _gidToLidChangeHandler(std::make_shared<DummyGidToLidChangeHandler>())
 {
     vespalib::mkdir(_baseDir, false); // Assume parent is created.
+    vespalib::File::sync(vespalib::dirname(_baseDir));
 }
 
 StoreOnlyDocSubDB::~StoreOnlyDocSubDB()
diff --git a/searchcorespi/src/vespa/searchcorespi/index/diskindexcleaner.cpp b/searchcorespi/src/vespa/searchcorespi/index/diskindexcleaner.cpp
index c363a93815b..e0cb8548f0d 100644
--- a/searchcorespi/src/vespa/searchcorespi/index/diskindexcleaner.cpp
+++ b/searchcorespi/src/vespa/searchcorespi/index/diskindexcleaner.cpp
@@ -3,6 +3,7 @@
 #include "diskindexcleaner.h"
 #include "activediskindexes.h"
 #include <vespa/fastos/file.h>
+#include <vespa/vespalib/io/fileutil.h>
 #include <sstream>
 #include <vector>
 
@@ -34,6 +35,11 @@ bool isValidIndex(const string &index_dir) {
     return serial_file.OpenReadOnlyExisting();
 }
 
+void invalidateIndex(const string &index_dir) {
+    vespalib::unlink(index_dir + "/serial.dat");
+    vespalib::File::sync(index_dir);
+}
+
 uint32_t findLastFusionId(const string &base_dir,
                           const vector<string> &indexes) {
     uint32_t fusion_id = 0;
@@ -56,7 +62,8 @@ uint32_t findLastFusionId(const string &base_dir,
 
 void removeDir(const string &dir) {
     LOG(debug, "Removing index dir '%s'", dir.c_str());
-    FastOS_FileInterface::EmptyAndRemoveDirectory(dir.c_str());
+    invalidateIndex(dir);
+    vespalib::rmdir(dir, true);
 }
 
 bool isOldIndex(const string &index, uint32_t last_fusion_id) {
@@ -73,14 +80,18 @@ bool isOldIndex(const string &index, uint32_t last_fusion_id) {
 }
 
 void removeOld(const string &base_dir, const vector<string> &indexes,
-               const ActiveDiskIndexes &active_indexes) {
+               const ActiveDiskIndexes &active_indexes, bool remove) {
     uint32_t last_fusion_id = findLastFusionId(base_dir, indexes);
     for (size_t i = 0; i < indexes.size(); ++i) {
         const string index_dir = base_dir + "/" + indexes[i];
         if (isOldIndex(indexes[i], last_fusion_id) &&
             !active_indexes.isActive(index_dir))
         {
-            removeDir(index_dir);
+            if (remove) {
+                removeDir(index_dir);
+            } else {
+                invalidateIndex(index_dir);
+            }
         }
     }
 }
@@ -99,14 +110,14 @@ void removeInvalid(const string &base_dir, const vector<string> &indexes) {
 void DiskIndexCleaner::clean(const string &base_dir,
                              const ActiveDiskIndexes &active_indexes) {
     vector<string> indexes = readIndexes(base_dir);
-    removeOld(base_dir, indexes, active_indexes);
+    removeOld(base_dir, indexes, active_indexes, false);
     removeInvalid(base_dir, indexes);
 }
 
 void DiskIndexCleaner::removeOldIndexes(
         const string &base_dir, const ActiveDiskIndexes &active_indexes) {
     vector<string> indexes = readIndexes(base_dir);
-    removeOld(base_dir, indexes, active_indexes);
+    removeOld(base_dir, indexes, active_indexes, true);
 }
 
 }
diff --git a/searchcorespi/src/vespa/searchcorespi/index/eventlogger.cpp b/searchcorespi/src/vespa/searchcorespi/index/eventlogger.cpp
index 9d824d69f93..cf1daf983b3 100644
--- a/searchcorespi/src/vespa/searchcorespi/index/eventlogger.cpp
+++ b/searchcorespi/src/vespa/searchcorespi/index/eventlogger.cpp
@@ -19,7 +19,7 @@ EventLogger::diskIndexLoadStart(const vespalib::string &indexDir)
     jstr.appendKey("input");
     LogUtil::logDir(jstr, indexDir, 6);
     jstr.endObject();
-    EV_STATE("diskindex.load.start", jstr.toString().c_str());
+    EV_STATE("diskindex.load.start", jstr.toString().data());
 }
 
 void
@@ -32,7 +32,7 @@ EventLogger::diskIndexLoadComplete(const vespalib::string &indexDir,
     jstr.appendKey("input");
     LogUtil::logDir(jstr, indexDir, 6);
     jstr.endObject();
-    EV_STATE("diskindex.load.complete", jstr.toString().c_str());
+    EV_STATE("diskindex.load.complete", jstr.toString().data());
 }
 
 void
@@ -50,7 +50,7 @@ EventLogger::diskFusionStart(const std::vector<vespalib::string> &sources,
     jstr.appendKey("output");
     LogUtil::logDir(jstr, fusionDir, 6);
     jstr.endObject();
-    EV_STATE("fusion.start", jstr.toString().c_str());
+    EV_STATE("fusion.start", jstr.toString().data());
 }
 
 void
@@ -63,7 +63,7 @@ EventLogger::diskFusionComplete(const vespalib::string &fusionDir,
     jstr.appendKey("output");
     LogUtil::logDir(jstr, fusionDir, 6);
     jstr.endObject();
-    EV_STATE("fusion.complete", jstr.toString().c_str());
+    EV_STATE("fusion.complete", jstr.toString().data());
 }
 
 }
diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp
index a18d24931cb..1e63741084d 100644
--- a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp
+++ b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp
@@ -16,6 +16,7 @@
 #include <vespa/vespalib/util/lambdatask.h>
 #include <sstream>
 #include <vespa/searchcorespi/flush/closureflushtask.h>
+#include <vespa/vespalib/io/fileutil.h>
 #include <vespa/vespalib/util/exceptions.h>
 #include <vespa/vespalib/util/array.hpp>
 #include <vespa/fastos/file.h>
@@ -999,6 +1000,7 @@ IndexMaintainer::runFusion(const FusionSpec &fusion_spec)
             _activeFusionSchema.reset();
             _activeFusionPrunedSchema.reset();
         }
+        vespalib::File::sync(vespalib::dirname(fail_dir));
         return fusion_spec.last_fusion_id;
     }
 
diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexwriteutilities.cpp b/searchcorespi/src/vespa/searchcorespi/index/indexwriteutilities.cpp
index c8b2e81a9c0..ebb316610e1 100644
--- a/searchcorespi/src/vespa/searchcorespi/index/indexwriteutilities.cpp
+++ b/searchcorespi/src/vespa/searchcorespi/index/indexwriteutilities.cpp
@@ -5,6 +5,7 @@
 #include <vespa/searchlib/common/serialnumfileheadercontext.h>
 #include <vespa/searchlib/index/schemautil.h>
 #include <vespa/fastlib/io/bufferedfile.h>
+#include <vespa/vespalib/io/fileutil.h>
 #include <vespa/vespalib/util/exceptions.h>
 #include <sstream>
 #include <unistd.h>
@@ -57,6 +58,7 @@ IndexWriteUtilities::writeSerialNum(SerialNum serialNum,
             tmpFileName.c_str());
     }
     file.Close();
+    vespalib::File::sync(dir);
 
     if (ok) {
         FastOS_File renameFile(tmpFileName.c_str());
@@ -67,6 +69,7 @@ IndexWriteUtilities::writeSerialNum(SerialNum serialNum,
         msg << "Unable to write serial number to '" << dir << "'.";
         throw IllegalStateException(msg.str());
     }
+    vespalib::File::sync(dir);
 }
 
 bool
@@ -94,12 +97,14 @@ IndexWriteUtilities::copySerialNumFile(const vespalib::string &sourceDir,
         return false;
     }
     file.Close();
+    vespalib::File::sync(destDir);
     if (!file.Rename(dest.c_str())) {
         LOG(error,
             "Unable to rename file '%s' to '%s'",
             tmpDest.c_str(), dest.c_str());
         return false;
     }
+    vespalib::File::sync(destDir);
     return true;
 }
 
@@ -151,6 +156,7 @@ IndexWriteUtilities::updateDiskIndexSchema(const vespalib::string &indexDir,
     }
     vespalib::string schemaTmpName = schemaName + ".tmp";
     vespalib::string schemaOrigName = schemaName + ".orig";
+    vespalib::unlink(schemaTmpName);
     if (!newSchema->saveToFile(schemaTmpName)) {
         LOG(error, "Could not save schema to '%s'",
             schemaTmpName.c_str());
@@ -172,6 +178,7 @@ IndexWriteUtilities::updateDiskIndexSchema(const vespalib::string &indexDir,
                 schemaName.c_str(),
                 FastOS_File::getLastErrorString().c_str());
         }
+        vespalib::File::sync(indexDir);
     }
     // XXX: FastOS layer violation
     int renameres = ::rename(schemaTmpName.c_str(), schemaName.c_str());
@@ -183,6 +190,7 @@ IndexWriteUtilities::updateDiskIndexSchema(const vespalib::string &indexDir,
             schemaName.c_str(),
             errString.c_str());
     }
+    vespalib::File::sync(indexDir);
 }
 
 } // namespace index
diff --git a/searchcorespi/src/vespa/searchcorespi/plugin/factoryloader.cpp b/searchcorespi/src/vespa/searchcorespi/plugin/factoryloader.cpp
index 1dac8c7aafd..3180f7ed29d 100644
--- a/searchcorespi/src/vespa/searchcorespi/plugin/factoryloader.cpp
+++ b/searchcorespi/src/vespa/searchcorespi/plugin/factoryloader.cpp
@@ -25,7 +25,8 @@ FactoryLoader::create(const stringref & factory)
     const FastOS_DynamicLibrary & lib = *_libraries.get(factory);
     FuncT registrationMethod = reinterpret_cast<FuncT>(lib.GetSymbol("createIndexManagerFactory"));
     if (registrationMethod == NULL) {
-        throw IllegalArgumentException(make_string("Failed locating symbol 'createIndexManagerFactory' in library '%s' for factory '%s'.", lib.GetLibName(), factory.c_str()));
+        throw IllegalArgumentException(make_string("Failed locating symbol 'createIndexManagerFactory' in library '%s' for factory '%s'.",
+                                                   lib.GetLibName(), vespalib::string(factory).c_str()));
     }
     return IIndexManagerFactory::UP(registrationMethod());
 }
diff --git a/searchlib/pom.xml b/searchlib/pom.xml
index fb9a81b51a5..0202f8510bb 100644
--- a/searchlib/pom.xml
+++ b/searchlib/pom.xml
@@ -17,7 +17,6 @@
     <dependency>
       <groupId>com.google.guava</groupId>
       <artifactId>guava</artifactId>
-      <version>13.0.1</version>
       <scope>provided</scope>
     </dependency>
     <dependency>
diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/ExpressionFunction.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/ExpressionFunction.java
index 9267034e985..c6d8f70fde8 100755
--- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/ExpressionFunction.java
+++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/ExpressionFunction.java
@@ -43,6 +43,11 @@ public class ExpressionFunction {
 
     public RankingExpression getBody() { return body; }
 
+    /** Returns a copy of this with the body changed to the given value */
+    public ExpressionFunction withBody(RankingExpression body) {
+        return new ExpressionFunction(name, arguments, body);
+    }
+
     /**
      * Creates and returns an instance of this function based on the given
      * arguments. If function calls are nested, this call may produce
@@ -110,7 +115,7 @@ public class ExpressionFunction {
 
     @Override
     public String toString() {
-        return name;
+        return "function '" + name + "'";
     }
     
     /**
diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/RankingExpression.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/RankingExpression.java
index f03a8bea583..01ed3b35d4c 100755
--- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/RankingExpression.java
+++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/RankingExpression.java
@@ -35,10 +35,10 @@ import java.util.Map;
  * <h3>Simple usage</h3>
 <pre><code>
 try {
-    MapContext context=new MapContext();
-    context.put("one",1d);
-    RankingExpression expression=new RankingExpression("10*if(i&gt;35,if(i&gt;one,if(i&gt;=670,4,8),if(i&gt;8000,5,3)),if(i==478,90,91))");
-    double result=expression.evaluate(context);
+    MapContext context = new MapContext();
+    context.put("one", 1d);
+    RankingExpression expression = new RankingExpression("10*if(i&gt;35,if(i&gt;one,if(i&gt;=670,4,8),if(i&gt;8000,5,3)),if(i==478,90,91))");
+    double result = expression.evaluate(context);
    }
 catch (ParseException e) {
     throw new RuntimeException(e);
@@ -57,10 +57,10 @@ ArrayContext contextPrototype;
 // Create reusable, gbdt optimized expression and context.
 // The expression is multithread-safe while the context created is not
 try {
-    RankingExpression expression=new RankingExpression("10*if(i&gt;35,if(i&gt;one,if(i&gt;=670,4,8),if(i&gt;8000,5,3)),if(i==478,90,91))");
-    ArrayContext contextPrototype=new ArrayContext(expression);
-    ExpressionOptimizer optimizer=new ExpressionOptimizer(); // Increases evaluation speed of gbdt form expressions by 3-4x
-    OptimizationReport triviaAboutTheOptimization=optimizer.optimize(expression,contextPrototype);
+    RankingExpression expression = new RankingExpression("10*if(i&gt;35,if(i&gt;one,if(i&gt;=670,4,8),if(i&gt;8000,5,3)),if(i==478,90,91))");
+    ArrayContext contextPrototype = new ArrayContext(expression);
+    ExpressionOptimizer optimizer = new ExpressionOptimizer(); // Increases evaluation speed of gbdt form expressions by 3-4x
+    OptimizationReport triviaAboutTheOptimization = optimizer.optimize(expression, contextPrototype);
 }
 catch (ParseException e) {
     throw new RuntimeException(e);
@@ -69,9 +69,9 @@ catch (ParseException e) {
 ...
 
 // Execution (many)
-context=contextPrototype.clone(); // If evaluation is multithreaded - skip this if execution is single-threaded
+context = contextPrototype.clone(); // If evaluation is multithreaded - skip this if execution is single-threaded
 context.put("one",1d);
-double result=expression.evaluate(context);
+double result = expression.evaluate(context);
 </code></pre>
  *
  * @author Simon Thoresen Hult
@@ -276,7 +276,7 @@ public class RankingExpression implements Serializable {
      * Returns the value of evaluating this expression over the given context.
      *
      * @param context The variable bindings to use for this evaluation.
-     * @return The evaluation result.
+     * @return the evaluation result.
      * @throws IllegalArgumentException if there are variables which are not bound in the given map
      */
     public Value evaluate(Context context) {
diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/AbstractArrayContext.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/AbstractArrayContext.java
index ed9fa346c11..41bf827748a 100644
--- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/AbstractArrayContext.java
+++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/AbstractArrayContext.java
@@ -7,8 +7,6 @@ import com.yahoo.searchlib.rankingexpression.rule.CompositeNode;
 import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode;
 import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode;
 
-import java.util.Collections;
-import java.util.HashMap;
 import java.util.LinkedHashSet;
 import java.util.Map;
 import java.util.Set;
@@ -20,19 +18,15 @@ import java.util.Set;
  *
  * @author bratseth
  */
-public abstract class AbstractArrayContext extends Context implements Cloneable {
+public abstract class AbstractArrayContext extends Context implements Cloneable, ContextIndex {
 
     private final boolean ignoreUnknownValues;
 
-    /** The mapping from variable name to index */
-    private final ImmutableMap<String, Integer> nameToIndex;
-
-    /** The current values set, pre-converted to doubles */
-    private double[] doubleValues;
-
     /** The name of the ranking expression this was created for */
     private final String rankingExpressionName;
 
+    private IndexedBindings indexedBindings;
+
     /**
      * Create a fast lookup context for an expression.
      * This instance should be reused indefinitely by a single thread.
@@ -53,44 +47,51 @@ public abstract class AbstractArrayContext extends Context implements Cloneable
     protected AbstractArrayContext(RankingExpression expression, boolean ignoreUnknownValues) {
         this.ignoreUnknownValues = ignoreUnknownValues;
         this.rankingExpressionName = expression.getName();
-        Set<String> variables = new LinkedHashSet<>();
-        extractVariables(expression.getRoot(),variables);
+        this.indexedBindings = new IndexedBindings(expression);
+    }
 
-        doubleValues = new double[variables.size()];
+    protected final Map<String, Integer> nameToIndex() { return indexedBindings.nameToIndex(); }
+    protected final double[] doubleValues() { return indexedBindings.doubleValues(); }
+    protected final boolean ignoreUnknownValues() { return ignoreUnknownValues; }
 
-        int i = 0;
-        ImmutableMap.Builder<String, Integer> nameToIndexBuilder = new ImmutableMap.Builder<>();
-        for (String variable : variables)
-            nameToIndexBuilder.put(variable,i++);
-        nameToIndex = nameToIndexBuilder.build();
+    @Override
+    public Set<String> names() {
+        return indexedBindings.names();
     }
 
-    private void extractVariables(ExpressionNode node,Set<String> variables) {
-        if (node instanceof ReferenceNode) {
-            ReferenceNode fNode=(ReferenceNode)node;
-            if (fNode.getArguments().expressions().size()>0)
-                throw new UnsupportedOperationException("Array lookup is not supported with features having arguments)");
-            variables.add(fNode.toString());
-        }
-        else if (node instanceof CompositeNode) {
-            CompositeNode cNode=(CompositeNode)node;
-            for (ExpressionNode child : cNode.children())
-                extractVariables(child,variables);
-        }
+    /**
+     * Returns the index from a name.
+     *
+     * @throws NullPointerException is this name is not known to this context
+     */
+    @Override
+    public final int getIndex(String name) { return indexedBindings.nameToIndex.get(name); }
+
+    /** Returns the max number of variables which may be set in this */
+    @Override
+    public int size() { return indexedBindings.size(); }
+
+    /** Perform a fast lookup directly of the value as a double. This is faster than get(index).asDouble() */
+    @Override
+    public double getDouble(int index) {
+        return indexedBindings.getDouble(index);
     }
 
-    protected final Map<String, Integer> nameToIndex() { return nameToIndex; }
-    protected final double[] doubleValues() { return doubleValues; }
-    protected final boolean ignoreUnknownValues() { return ignoreUnknownValues; }
+    @Override
+    public String toString() {
+        return "fast lookup context for ranking expression '" + rankingExpressionName +
+                "' [" + size() + " variables]";
+    }
 
     /**
      * Creates a clone of this context suitable for evaluating against the same ranking expression
      * in a different thread (i.e, name name to index map, different value set.
      */
+    @Override
     public AbstractArrayContext clone() {
         try {
-            AbstractArrayContext clone=(AbstractArrayContext)super.clone();
-            clone.doubleValues=new double[nameToIndex.size()];
+            AbstractArrayContext clone = (AbstractArrayContext)super.clone();
+            clone.indexedBindings = indexedBindings.clone();
             return clone;
         }
         catch (CloneNotSupportedException e) {
@@ -98,34 +99,64 @@ public abstract class AbstractArrayContext extends Context implements Cloneable
         }
     }
 
-    public Set<String> names() {
-        return nameToIndex.keySet();
-    }
+    private static class IndexedBindings implements Cloneable {
 
-    /**
-     * Returns the index from a name.
-     *
-     * @throws NullPointerException is this name is not known to this context
-     */
-    public final int getIndex(String name) {
-        return nameToIndex.get(name);
-    }
+        /** The mapping from variable name to index */
+        private final ImmutableMap<String, Integer> nameToIndex;
 
-    /** Returns the max number of variables which may be set in this */
-    public int size() {
-        return doubleValues.length;
-    }
+        /** The current values set, pre-converted to doubles */
+        private double[] doubleValues;
 
-    /** Perform a fast lookup directly of the value as a double. This is faster than get(index).asDouble() */
-    @Override
-    public double getDouble(int index) {
-        return doubleValues[index];
-    }
+        public IndexedBindings(RankingExpression expression) {
+            Set<String> bindTargets = new LinkedHashSet<>();
+            extractBindTargets(expression.getRoot(), bindTargets);
+
+            doubleValues = new double[bindTargets.size()];
+
+            int i = 0;
+            ImmutableMap.Builder<String, Integer> nameToIndexBuilder = new ImmutableMap.Builder<>();
+            for (String variable : bindTargets)
+                nameToIndexBuilder.put(variable,i++);
+            nameToIndex = nameToIndexBuilder.build();
+        }
+
+        private void extractBindTargets(ExpressionNode node, Set<String> bindTargets) {
+            if (node instanceof ReferenceNode) {
+                if (((ReferenceNode)node).getArguments().expressions().size() > 0)
+                    throw new UnsupportedOperationException("Can not bind " + node +
+                                                            ": Array lookup is not supported with features having arguments)");
+                bindTargets.add(node.toString());
+            }
+            else if (node instanceof CompositeNode) {
+                CompositeNode cNode = (CompositeNode)node;
+                for (ExpressionNode child : cNode.children())
+                    extractBindTargets(child, bindTargets);
+            }
+        }
+
+        public Map<String, Integer> nameToIndex() { return nameToIndex; }
+        public double[] doubleValues() { return doubleValues; }
+        public Set<String> names() { return nameToIndex.keySet(); }
+        public int getIndex(String name) { return nameToIndex.get(name); }
+        public int size() { return doubleValues.length; }
+        public double getDouble(int index) { return doubleValues[index]; }
+
+        /**
+         * Creates a clone of this context suitable for evaluating against the same ranking expression
+         * in a different thread (i.e, name name to index map, different value set.
+         */
+        @Override
+        public IndexedBindings clone() {
+            try {
+                IndexedBindings clone = (IndexedBindings)super.clone();
+                clone.doubleValues = new double[nameToIndex.size()];
+                return clone;
+            }
+            catch (CloneNotSupportedException e) {
+                throw new RuntimeException("Programming error");
+            }
+        }
 
-    @Override
-    public String toString() {
-        return "fast lookup context for ranking expression '" + rankingExpressionName +
-                "' [" + doubleValues.length + " variables]";
     }
 
 }
diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/ArrayContext.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/ArrayContext.java
index ee5952d9aea..237c3a1d0b1 100644
--- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/ArrayContext.java
+++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/ArrayContext.java
@@ -119,7 +119,7 @@ public class ArrayContext extends AbstractArrayContext implements Cloneable {
     public ArrayContext clone() {
         ArrayContext clone = (ArrayContext)super.clone();
         clone.values = new Value[nameToIndex().size()];
-        Arrays.fill(values,constantZero);
+        Arrays.fill(values, constantZero);
         return clone;
     }
 
diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/ContextIndex.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/ContextIndex.java
new file mode 100644
index 00000000000..ad6facbf0af
--- /dev/null
+++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/ContextIndex.java
@@ -0,0 +1,26 @@
+package com.yahoo.searchlib.rankingexpression.evaluation;
+
+/**
+ * Indexed context lookup methods.
+ * Any context which implements these methods supports optimizations where map lookups
+ * are replaced by indexed lookups.
+ *
+ * @author bratseth
+ */
+public interface ContextIndex {
+
+    /** Returns the number of bound variables in this */
+    int size();
+
+    /**
+     * Returns the index from a name.
+     *
+     * @throws NullPointerException is this name is not known to this context
+     */
+    int getIndex(String name);
+
+    Value get(int index);
+
+    double getDouble(int index);
+
+}
diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/ExpressionOptimizer.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/ExpressionOptimizer.java
index 836aadd9f70..7060cfc2132 100644
--- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/ExpressionOptimizer.java
+++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/ExpressionOptimizer.java
@@ -44,12 +44,16 @@ public class ExpressionOptimizer {
         return null;
     }
 
-    public OptimizationReport optimize(RankingExpression expression, AbstractArrayContext arrayContext) {
+    public OptimizationReport optimize(RankingExpression expression, ContextIndex contextIndex) {
         OptimizationReport report = new OptimizationReport();
         // Note: Order of optimizations matter
-        gbdtOptimizer.optimize(expression, arrayContext, report);
-        gbdtForestOptimizer.optimize(expression, arrayContext, report);
+        gbdtOptimizer.optimize(expression, contextIndex, report);
+        gbdtForestOptimizer.optimize(expression, contextIndex, report);
         return report;
     }
 
+    public OptimizationReport optimize(RankingExpression expression, AbstractArrayContext arrayContext) {
+        return optimize(expression, (ContextIndex)arrayContext);
+    }
+
 }
diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/Optimizer.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/Optimizer.java
index fd9c02100f7..044b5b589a5 100644
--- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/Optimizer.java
+++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/Optimizer.java
@@ -18,6 +18,6 @@ public abstract class Optimizer {
     /** Returns whether this is enabled */
     public boolean isEnabled() { return enabled; }
 
-    public abstract void optimize(RankingExpression expression, AbstractArrayContext context, OptimizationReport report);
+    public abstract void optimize(RankingExpression expression, ContextIndex context, OptimizationReport report);
 
 }
diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/Value.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/Value.java
index e5a9e6a5ef1..7809cdd4e1b 100644
--- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/Value.java
+++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/Value.java
@@ -115,4 +115,12 @@ public abstract class Value {
             return new DoubleValue(Double.parseDouble(value));
     }
 
+    public static Value of(Tensor tensor) {
+        return new TensorValue(tensor);
+    }
+
+    public static Value of(double scalar) {
+        return new DoubleValue(scalar);
+    }
+
 }
diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/gbdtoptimization/GBDTForestOptimizer.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/gbdtoptimization/GBDTForestOptimizer.java
index 8999be4745a..bb8b91eecab 100644
--- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/gbdtoptimization/GBDTForestOptimizer.java
+++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/gbdtoptimization/GBDTForestOptimizer.java
@@ -2,8 +2,7 @@
 package com.yahoo.searchlib.rankingexpression.evaluation.gbdtoptimization;
 
 import com.yahoo.searchlib.rankingexpression.RankingExpression;
-import com.yahoo.searchlib.rankingexpression.evaluation.AbstractArrayContext;
-import com.yahoo.searchlib.rankingexpression.evaluation.ArrayContext;
+import com.yahoo.searchlib.rankingexpression.evaluation.ContextIndex;
 import com.yahoo.searchlib.rankingexpression.evaluation.OptimizationReport;
 import com.yahoo.searchlib.rankingexpression.evaluation.Optimizer;
 import com.yahoo.searchlib.rankingexpression.rule.ArithmeticNode;
@@ -34,7 +33,7 @@ public class GBDTForestOptimizer extends Optimizer {
      * @param report     the optimization report to which actions of this is logged
      */
     @Override
-    public void optimize(RankingExpression expression, AbstractArrayContext context, OptimizationReport report) {
+    public void optimize(RankingExpression expression, ContextIndex context, OptimizationReport report) {
         if ( ! isEnabled()) return;
 
         this.report = report;
diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/gbdtoptimization/GBDTOptimizer.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/gbdtoptimization/GBDTOptimizer.java
index 74af3e576c1..787818b0f42 100644
--- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/gbdtoptimization/GBDTOptimizer.java
+++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/gbdtoptimization/GBDTOptimizer.java
@@ -33,7 +33,7 @@ public class GBDTOptimizer extends Optimizer {
      * @param report     the optimization report to which actions of this is logged
      */
     @Override
-    public void optimize(RankingExpression expression, AbstractArrayContext context, OptimizationReport report) {
+    public void optimize(RankingExpression expression, ContextIndex context, OptimizationReport report) {
         if (!isEnabled()) return;
 
         this.report = report;
@@ -60,7 +60,7 @@ public class GBDTOptimizer extends Optimizer {
      *
      * @return the optimized expression
      */
-    private ExpressionNode optimize(ExpressionNode node, AbstractArrayContext context) {
+    private ExpressionNode optimize(ExpressionNode node, ContextIndex context) {
         if (node instanceof ArithmeticNode) {
             Iterator<ExpressionNode> childIt = ((ArithmeticNode)node).children().iterator();
             ExpressionNode ret = optimize(childIt.next(), context);
@@ -77,7 +77,7 @@ public class GBDTOptimizer extends Optimizer {
         return node;
     }
 
-    private ExpressionNode createGBDTNode(IfNode cNode, AbstractArrayContext context) {
+    private ExpressionNode createGBDTNode(IfNode cNode,ContextIndex context) {
         List<Double> values = new ArrayList<>();
         try {
             consumeNode(cNode, values, context);
@@ -93,7 +93,7 @@ public class GBDTOptimizer extends Optimizer {
     /**
      * Recursively consume nodes into the value list Returns the number of values produced by this.
      */
-    private int consumeNode(ExpressionNode node, List<Double> values, AbstractArrayContext context) {
+    private int consumeNode(ExpressionNode node, List<Double> values, ContextIndex context) {
         int beforeIndex = values.size();
         if ( node instanceof IfNode) {
             IfNode ifNode = (IfNode)node;
@@ -113,7 +113,7 @@ public class GBDTOptimizer extends Optimizer {
     }
 
     /** Consumes the if condition and return the size of the values resulting, for convenience */
-    private int consumeIfCondition(ExpressionNode condition, List<Double> values, AbstractArrayContext context) {
+    private int consumeIfCondition(ExpressionNode condition, List<Double> values, ContextIndex context) {
         if (condition instanceof ComparisonNode) {
             ComparisonNode comparison = (ComparisonNode)condition;
             if (comparison.getOperator() == TruthOperator.SMALLER)
@@ -138,7 +138,7 @@ public class GBDTOptimizer extends Optimizer {
         return values.size();
     }
 
-    private double getVariableIndex(ExpressionNode node, AbstractArrayContext context) {
+    private double getVariableIndex(ExpressionNode node, ContextIndex context) {
         if (!(node instanceof ReferenceNode)) {
             throw new IllegalArgumentException("Contained a left-hand comparison expression " +
                                                "which was not a feature value but was: " + node);
diff --git a/searchlib/src/tests/attribute/attribute_test.cpp b/searchlib/src/tests/attribute/attribute_test.cpp
index 5148ab2be34..32ac836302a 100644
--- a/searchlib/src/tests/attribute/attribute_test.cpp
+++ b/searchlib/src/tests/attribute/attribute_test.cpp
@@ -1297,6 +1297,9 @@ AttributeTest::testWeightedSet()
             testWeightedSet<IntegerAttribute, AttributeVector::WeightedInt>(ptr, values);
             IAttributeVector::EnumHandle e;
             EXPECT_TRUE(ptr->findEnum("1", e));
+            EXPECT_EQUAL(1u, ptr->findFoldedEnums("1").size());
+            EXPECT_EQUAL(e, ptr->findFoldedEnums("1")[0]);
+
         }
     }
     { // FloatingPointAttribute
@@ -1320,6 +1323,8 @@ AttributeTest::testWeightedSet()
             testWeightedSet<FloatingPointAttribute, AttributeVector::WeightedFloat>(ptr, values);
             IAttributeVector::EnumHandle e;
             EXPECT_TRUE(ptr->findEnum("1", e));
+            EXPECT_EQUAL(1u, ptr->findFoldedEnums("1").size());
+            EXPECT_EQUAL(e, ptr->findFoldedEnums("1")[0]);
         }
     }
     { // StringAttribute
@@ -1345,6 +1350,8 @@ AttributeTest::testWeightedSet()
             testWeightedSet<StringAttribute, AttributeVector::WeightedString>(ptr, values);
             IAttributeVector::EnumHandle e;
             EXPECT_TRUE(ptr->findEnum("string00", e));
+            EXPECT_EQUAL(1u, ptr->findFoldedEnums("StRiNg00").size());
+            EXPECT_EQUAL(e, ptr->findFoldedEnums("StRiNg00")[0]);
         }
     }
 }
diff --git a/searchlib/src/tests/attribute/benchmark/attributesearcher.h b/searchlib/src/tests/attribute/benchmark/attributesearcher.h
index f8cd614c48c..37f33803d27 100644
--- a/searchlib/src/tests/attribute/benchmark/attributesearcher.h
+++ b/searchlib/src/tests/attribute/benchmark/attributesearcher.h
@@ -126,7 +126,7 @@ AttributeFindSearcher<T>::doRun()
         // build simple term query
         vespalib::asciistream ss;
         ss << _values[i % _values.size()].getValue();
-        this->buildTermQuery(_query, _attrPtr->getName(), ss.str().c_str());
+        this->buildTermQuery(_query, _attrPtr->getName(), ss.str().data());
 
         AttributeGuard guard(_attrPtr);
         std::unique_ptr<AttributeVector::SearchContext> searchContext =
@@ -204,7 +204,7 @@ AttributeRangeSearcher::doRun()
         // build simple range term query
         vespalib::asciistream ss;
         ss << "[" << iter.a() << ";" << iter.b() << "]";
-        buildTermQuery(_query, _attrPtr->getName(), ss.str().c_str());
+        buildTermQuery(_query, _attrPtr->getName(), ss.str().data());
 
         AttributeGuard guard(_attrPtr);
         std::unique_ptr<AttributeVector::SearchContext> searchContext =
diff --git a/searchlib/src/tests/attribute/enumstore/enumstore_test.cpp b/searchlib/src/tests/attribute/enumstore/enumstore_test.cpp
index daff432d68d..89dd1cfdab4 100644
--- a/searchlib/src/tests/attribute/enumstore/enumstore_test.cpp
+++ b/searchlib/src/tests/attribute/enumstore/enumstore_test.cpp
@@ -41,6 +41,7 @@ private:
     void testFloatEnumStore(EnumStoreType & es);
     void testFloatEnumStore();
 
+    void testFindFolded();
     void testAddEnum();
     template <typename EnumStoreType>
     void testAddEnum(bool hasPostings);
@@ -275,6 +276,40 @@ EnumStoreTest::testFloatEnumStore()
 }
 
 void
+EnumStoreTest::testFindFolded()
+{
+    StringEnumStore ses(100, false);
+    std::vector<EnumIndex> indices;
+    std::vector<std::string> unique({"", "one", "two", "TWO", "Two", "three"});
+    for (std::string &str : unique) {
+        EnumIndex idx;
+        ses.addEnum(str.c_str(), idx);
+        EXPECT_TRUE(ses.getLastEnum() == indices.size());
+        indices.push_back(idx);
+        ses.incRefCount(idx);
+        EXPECT_EQUAL(1u, ses.getRefCount(idx));
+    }
+    ses.freezeTree();
+    for (uint32_t i = 0; i < indices.size(); ++i) {
+        uint32_t e = ses.getEnum(indices[i]);
+        EXPECT_EQUAL(i, e);
+        EnumIndex idx;
+        EXPECT_TRUE(ses.findIndex(unique[i].c_str(), idx));
+    }
+    EXPECT_EQUAL(1u, ses.findFoldedEnums("").size());
+    EXPECT_EQUAL(0u, ses.findFoldedEnums("foo").size());
+    EXPECT_EQUAL(1u, ses.findFoldedEnums("one").size());
+    EXPECT_EQUAL(3u, ses.findFoldedEnums("two").size());
+    EXPECT_EQUAL(3u, ses.findFoldedEnums("TWO").size());
+    EXPECT_EQUAL(3u, ses.findFoldedEnums("tWo").size());
+    const auto v = ses.findFoldedEnums("Two");
+    EXPECT_EQUAL(std::string("TWO"), ses.getValue(v[0]));
+    EXPECT_EQUAL(std::string("Two"), ses.getValue(v[1]));
+    EXPECT_EQUAL(std::string("two"), ses.getValue(v[2]));
+    EXPECT_EQUAL(1u, ses.findFoldedEnums("three").size());
+}
+
+void
 EnumStoreTest::testAddEnum()
 {
     testAddEnum<StringEnumStore>(false);
@@ -319,6 +354,8 @@ EnumStoreTest::testAddEnum(bool hasPostings)
         uint32_t e = ses.getEnum(indices[i]);
         EXPECT_EQUAL(i, e);
         EXPECT_TRUE(ses.findEnum(unique[i].c_str(), e));
+        EXPECT_EQUAL(1u, ses.findFoldedEnums(unique[i].c_str()).size());
+        EXPECT_EQUAL(e, ses.findFoldedEnums(unique[i].c_str())[0]);
         EXPECT_TRUE(ses.getEnum(datastore::EntryRef(e)) == i);
         EXPECT_TRUE(ses.findIndex(unique[i].c_str(), idx));
         EXPECT_TRUE(idx == indices[i]);
@@ -874,6 +911,7 @@ EnumStoreTest::Main()
     testStringEntry();
     testNumericEntry();
     testFloatEnumStore();
+    testFindFolded();
     testAddEnum();
     testCompaction();
     testReset();
diff --git a/searchlib/src/tests/attribute/imported_search_context/imported_search_context_test.cpp b/searchlib/src/tests/attribute/imported_search_context/imported_search_context_test.cpp
index 0a02824e77a..f8d696b3203 100644
--- a/searchlib/src/tests/attribute/imported_search_context/imported_search_context_test.cpp
+++ b/searchlib/src/tests/attribute/imported_search_context/imported_search_context_test.cpp
@@ -348,7 +348,7 @@ TEST_F("Multiple iterators can be created from the same context", SingleValueFix
 // implemented at all for (single) numeric attributes. Intentional?
 TEST_F("queryTerm() returns term context was created with", WsetValueFixture) {
     auto ctx = f.create_context(word_term("helloworld"));
-    EXPECT_EQUAL(std::string("helloworld"), std::string(ctx->queryTerm().getTerm()));
+    EXPECT_EQUAL(std::string("helloworld"), std::string(ctx->queryTerm()->getTerm()));
 }
 
 struct SearchCacheFixture : Fixture {
diff --git a/searchlib/src/tests/attribute/stringattribute/stringattribute_test.cpp b/searchlib/src/tests/attribute/stringattribute/stringattribute_test.cpp
index b25abdfd2e5..2adfdd135df 100644
--- a/searchlib/src/tests/attribute/stringattribute/stringattribute_test.cpp
+++ b/searchlib/src/tests/attribute/stringattribute/stringattribute_test.cpp
@@ -153,6 +153,8 @@ StringAttributeTest::testMultiValue(Attribute & attr, uint32_t numDocs)
             EXPECT_TRUE(strcmp(attr.get(doc), uniqueStrings[0].c_str()) == 0);
             uint32_t e;
             EXPECT_TRUE(attr.findEnum(uniqueStrings[0].c_str(), e));
+            EXPECT_EQUAL(1u, attr.findFoldedEnums(uniqueStrings[0].c_str()).size());
+            EXPECT_EQUAL(e, attr.findFoldedEnums(uniqueStrings[0].c_str())[0]);
             EXPECT_TRUE(attr.getEnum(doc) == e);
         }
 
diff --git a/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp b/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp
index a22c44e843c..34046f551d9 100644
--- a/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp
+++ b/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp
@@ -282,13 +282,13 @@ TEST("testTruncatedIdxFile"){
     {
         // Files comes from the 'growing test'.
         LogDataStore datastore(executor, TEST_PATH("bug-7257706"), config, GrowStrategy(),
-                               TuneFileSummary(), fileHeaderContext, tlSyncer, NULL);
+                               TuneFileSummary(), fileHeaderContext, tlSyncer, nullptr);
         EXPECT_EQUAL(354ul, datastore.lastSyncToken());
     }
     const char * magic = "mumbo jumbo";
     {
         LogDataStore datastore(executor, "bug-7257706-truncated", config, GrowStrategy(),
-                               TuneFileSummary(), fileHeaderContext, tlSyncer, NULL);
+                               TuneFileSummary(), fileHeaderContext, tlSyncer, nullptr);
         EXPECT_EQUAL(331ul, datastore.lastSyncToken());
         datastore.write(332, 7, magic, strlen(magic));
         datastore.write(333, 8, magic, strlen(magic));
@@ -296,7 +296,7 @@ TEST("testTruncatedIdxFile"){
     }
     {
         LogDataStore datastore(executor, "bug-7257706-truncated", config, GrowStrategy(),
-                               TuneFileSummary(), fileHeaderContext, tlSyncer, NULL);
+                               TuneFileSummary(), fileHeaderContext, tlSyncer, nullptr);
         EXPECT_EQUAL(334ul, datastore.lastSyncToken());
     }
 }
@@ -308,7 +308,7 @@ TEST("testThatEmptyIdxFilesAndDanglingDatFilesAreRemoved") {
     MyTlSyncer tlSyncer;
     LogDataStore datastore(executor, "dangling-test", config,
                            GrowStrategy(), TuneFileSummary(),
-                           fileHeaderContext, tlSyncer, NULL);
+                           fileHeaderContext, tlSyncer, nullptr);
     EXPECT_EQUAL(354ul, datastore.lastSyncToken());
     EXPECT_EQUAL(4096u + 480u, datastore.getDiskHeaderFootprint());
     EXPECT_EQUAL(datastore.getDiskHeaderFootprint() + 94016u, datastore.getDiskFootprint());
@@ -321,7 +321,7 @@ TEST("testThatIncompleteCompactedFilesAreRemoved") {
     MyTlSyncer tlSyncer;
     LogDataStore datastore(executor, "incompletecompact-test", config,
                            GrowStrategy(), TuneFileSummary(),
-                           fileHeaderContext, tlSyncer, NULL);
+                           fileHeaderContext, tlSyncer, nullptr);
     EXPECT_EQUAL(354ul, datastore.lastSyncToken());
     EXPECT_EQUAL(3*(4096u + 480u), datastore.getDiskHeaderFootprint());
     LogDataStore::NameIdSet files = datastore.getAllActiveFiles();
@@ -340,7 +340,7 @@ public:
         _executor(1, 128*1024),
         _tlSyncer(),
         _datastore(_executor, _myDir.getDir(), _config, GrowStrategy(),
-                   TuneFileSummary(), _fileHeaderContext, _tlSyncer, NULL)
+                   TuneFileSummary(), _fileHeaderContext, _tlSyncer, nullptr)
     { }
     ~VisitStore();
     IDataStore & getStore() { return _datastore; }
@@ -414,7 +414,7 @@ makeDocTypeRepoConfig()
 
 
 Document::UP
-makeDoc(const DocumentTypeRepo &repo, uint32_t i, bool extra_field)
+makeDoc(const DocumentTypeRepo &repo, uint32_t i, bool extra_field, size_t numReps=0)
 {
     asciistream idstr;
     idstr << "id:test:test:: " << i;
@@ -424,7 +424,7 @@ makeDoc(const DocumentTypeRepo &repo, uint32_t i, bool extra_field)
     ASSERT_TRUE(doc.get());
     asciistream mainstr;
     mainstr << "static text" << i << " body something";
-    for (uint32_t j = 0; j < 10; ++j) {
+    for (uint32_t j = 0; j < 10+numReps; ++j) {
         mainstr << (j + i * 1000) << " ";
     }
     mainstr << " and end field";
@@ -432,7 +432,6 @@ makeDoc(const DocumentTypeRepo &repo, uint32_t i, bool extra_field)
     if (extra_field) {
         doc->set("extra", "foo");
     }
-
     return doc;
 }
 
@@ -440,11 +439,15 @@ makeDoc(const DocumentTypeRepo &repo, uint32_t i, bool extra_field)
 
 class VisitCacheStore {
 public:
-    VisitCacheStore();
+    using UpdateStrategy=DocumentStore::Config::UpdateStrategy;
+    VisitCacheStore(UpdateStrategy strategy);
     ~VisitCacheStore();
-    IDocumentStore & getStore() { return _datastore; }
+    IDocumentStore & getStore() { return *_datastore; }
     void write(uint32_t id) {
-        write(id, makeDoc(_repo, id, true));
+        write(id, 0);
+    }
+    void write(uint32_t lid, uint32_t numReps) {
+        write(lid, makeDoc(_repo, lid, true, numReps));
     }
     void rewrite(uint32_t id) {
         write(id, makeDoc(_repo, id, false));
@@ -458,7 +461,10 @@ public:
         _inserted.erase(id);
     }
     void verifyRead(uint32_t id) {
-        verifyDoc(*_datastore.read(id, _repo), id);
+        verifyDoc(*_datastore->read(id, _repo), id);
+    }
+    void read(uint32_t id) {
+        *_datastore->read(id, _repo);
     }
     void verifyDoc(const Document & doc, uint32_t id) {
         EXPECT_TRUE(doc == *_inserted[id]);
@@ -468,8 +474,10 @@ public:
     }
     void verifyVisit(const std::vector<uint32_t> & lids, const std::vector<uint32_t> & expected, bool allowCaching) {
         VerifyVisitor vv(*this, expected, allowCaching);
-        _datastore.visit(lids, _repo, vv);
+        _datastore->visit(lids, _repo, vv);
     }
+    void recreate();
+
 private:
     class VerifyVisitor : public IDocumentVisitor {
     public:
@@ -494,7 +502,7 @@ private:
     DummyFileHeaderContext           _fileHeaderContext;
     vespalib::ThreadStackExecutor    _executor;
     MyTlSyncer                       _tlSyncer;
-    LogDocumentStore                 _datastore;
+    std::unique_ptr<LogDocumentStore>  _datastore;
     std::map<uint32_t, Document::UP> _inserted;
     SerialNum                        _serial;
 };
@@ -510,21 +518,33 @@ VisitCacheStore::VerifyVisitor::~VerifyVisitor() {
     EXPECT_EQUAL(_expected.size(), _actual.size());
 }
 
-VisitCacheStore::VisitCacheStore() :
+
+VisitCacheStore::VisitCacheStore(UpdateStrategy strategy) :
     _myDir("visitcache"),
     _repo(makeDocTypeRepoConfig()),
-    _config(DocumentStore::Config(CompressionConfig::LZ4, 1000000, 0).allowVisitCaching(true),
+    _config(DocumentStore::Config(CompressionConfig::LZ4, 1000000, 0)
+                    .allowVisitCaching(true).updateStrategy(strategy),
             LogDataStore::Config().setMaxFileSize(50000).setMaxBucketSpread(3.0)
                     .setFileConfig(WriteableFileChunk::Config(CompressionConfig(), 16384))),
     _fileHeaderContext(),
     _executor(1, 128*1024),
     _tlSyncer(),
-    _datastore(_executor, _myDir.getDir(), _config, GrowStrategy(),
-               TuneFileSummary(), _fileHeaderContext, _tlSyncer, nullptr),
+    _datastore(std::make_unique<LogDocumentStore>(_executor, _myDir.getDir(), _config, GrowStrategy(),
+                                                  TuneFileSummary(), _fileHeaderContext, _tlSyncer, nullptr)),
     _inserted(),
     _serial(1)
 { }
-VisitCacheStore::~VisitCacheStore() {}
+
+VisitCacheStore::~VisitCacheStore() = default;
+
+void
+VisitCacheStore::recreate() {
+    _datastore->flush(_datastore->initFlush(_datastore->tentativeLastSyncToken()));
+    _datastore.reset();
+    _datastore = std::make_unique<LogDocumentStore>(_executor, _myDir.getDir(), _config, GrowStrategy(),
+                                                    TuneFileSummary(), _fileHeaderContext, _tlSyncer, nullptr);
+
+}
 
 void
 verifyCacheStats(CacheStats cs, size_t hits, size_t misses, size_t elements, size_t memory_used) {
@@ -535,8 +555,64 @@ verifyCacheStats(CacheStats cs, size_t hits, size_t misses, size_t elements, siz
     EXPECT_GREATER_EQUAL(memory_used+20,  cs.memory_used);
 }
 
+TEST("test the update cache strategy") {
+    VisitCacheStore vcs(DocumentStore::Config::UpdateStrategy::UPDATE);
+    IDocumentStore & ds = vcs.getStore();
+    for (size_t i(1); i <= 10; i++) {
+        vcs.write(i);
+    }
+    TEST_DO(verifyCacheStats(ds.getCacheStats(), 0, 0, 0, 0));
+    vcs.verifyRead(7);
+    TEST_DO(verifyCacheStats(ds.getCacheStats(), 0, 1, 1, 221));
+    vcs.write(8);
+    TEST_DO(verifyCacheStats(ds.getCacheStats(), 0, 1, 1, 221));
+    vcs.write(7, 17);
+    TEST_DO(verifyCacheStats(ds.getCacheStats(), 0, 1, 1, 282));
+    vcs.verifyRead(7);
+    TEST_DO(verifyCacheStats(ds.getCacheStats(), 1, 1, 1, 282));
+    vcs.remove(8);
+    TEST_DO(verifyCacheStats(ds.getCacheStats(), 1, 1, 1, 282));
+    vcs.remove(7);
+    TEST_DO(verifyCacheStats(ds.getCacheStats(), 1, 1, 0, 0));
+    vcs.write(7);
+    TEST_DO(verifyCacheStats(ds.getCacheStats(), 1, 1, 0, 0));
+    vcs.verifyRead(7);
+    TEST_DO(verifyCacheStats(ds.getCacheStats(), 1, 2, 1, 221));
+    vcs.write(7, 17);
+    TEST_DO(verifyCacheStats(ds.getCacheStats(), 1, 2, 1, 282));
+    vcs.recreate();
+    IDocumentStore & ds2 = vcs.getStore();
+    vcs.verifyRead(7);
+    TEST_DO(verifyCacheStats(ds2.getCacheStats(), 0, 1, 1, 282));
+}
+
+TEST("test the invalidate cache strategy") {
+    VisitCacheStore vcs(DocumentStore::Config::UpdateStrategy::INVALIDATE);
+    IDocumentStore & ds = vcs.getStore();
+    for (size_t i(1); i <= 10; i++) {
+        vcs.write(i);
+    }
+    TEST_DO(verifyCacheStats(ds.getCacheStats(), 0, 0, 0, 0));
+    vcs.verifyRead(7);
+    TEST_DO(verifyCacheStats(ds.getCacheStats(), 0, 1, 1, 221));
+    vcs.write(8);
+    TEST_DO(verifyCacheStats(ds.getCacheStats(), 0, 1, 1, 221));
+    vcs.write(7);
+    TEST_DO(verifyCacheStats(ds.getCacheStats(), 0, 1, 0, 0));
+    vcs.verifyRead(7);
+    TEST_DO(verifyCacheStats(ds.getCacheStats(), 0, 2, 1, 221));
+    vcs.remove(8);
+    TEST_DO(verifyCacheStats(ds.getCacheStats(), 0, 2, 1, 221));
+    vcs.remove(7);
+    TEST_DO(verifyCacheStats(ds.getCacheStats(), 0, 2, 0, 0));
+    vcs.write(7);
+    TEST_DO(verifyCacheStats(ds.getCacheStats(), 0, 2, 0, 0));
+    vcs.verifyRead(7);
+    TEST_DO(verifyCacheStats(ds.getCacheStats(), 0, 3, 1, 221));
+}
+
 TEST("test that the integrated visit cache works.") {
-    VisitCacheStore vcs;
+    VisitCacheStore vcs(DocumentStore::Config::UpdateStrategy::INVALIDATE);
     IDocumentStore & ds = vcs.getStore();
     for (size_t i(1); i <= 100; i++) {
         vcs.write(i);
@@ -546,41 +622,42 @@ TEST("test that the integrated visit cache works.") {
     for (size_t i(1); i <= 100; i++) {
         vcs.verifyRead(i);
     }
-    TEST_DO(verifyCacheStats(ds.getCacheStats(), 0, 100, 100, 20574));
+    constexpr size_t BASE_SZ = 21374;
+    TEST_DO(verifyCacheStats(ds.getCacheStats(), 0, 100, 100, BASE_SZ));
     for (size_t i(1); i <= 100; i++) {
         vcs.verifyRead(i);
     }
-    TEST_DO(verifyCacheStats(ds.getCacheStats(), 100, 100, 100, 20574)); // From the individual cache.
+    TEST_DO(verifyCacheStats(ds.getCacheStats(), 100, 100, 100, BASE_SZ)); // From the individual cache.
 
     vcs.verifyVisit({7,9,17,19,67,88}, false);
-    TEST_DO(verifyCacheStats(ds.getCacheStats(), 100, 100, 100, 20574));
+    TEST_DO(verifyCacheStats(ds.getCacheStats(), 100, 100, 100, BASE_SZ));
     vcs.verifyVisit({7,9,17,19,67,88}, true);
-    TEST_DO(verifyCacheStats(ds.getCacheStats(), 100, 101, 101, 21135));
+    TEST_DO(verifyCacheStats(ds.getCacheStats(), 100, 101, 101, BASE_SZ+557));
     vcs.verifyVisit({7,9,17,19,67,88}, true);
-    TEST_DO(verifyCacheStats(ds.getCacheStats(), 101, 101, 101, 21135));
+    TEST_DO(verifyCacheStats(ds.getCacheStats(), 101, 101, 101, BASE_SZ+557));
     vcs.rewrite(8);
-    TEST_DO(verifyCacheStats(ds.getCacheStats(), 101, 101, 100, 20922)); // From the individual cache.
+    TEST_DO(verifyCacheStats(ds.getCacheStats(), 101, 101, 100, BASE_SZ+328)); // From the individual cache.
     vcs.rewrite(7);
-    TEST_DO(verifyCacheStats(ds.getCacheStats(), 101, 101, 98, 20148)); // From the both caches.
+    TEST_DO(verifyCacheStats(ds.getCacheStats(), 101, 101, 98, BASE_SZ-442)); // From the both caches.
     vcs.verifyVisit({7,9,17,19,67,88}, true);
-    TEST_DO(verifyCacheStats(ds.getCacheStats(), 101, 102, 99, 20732));
+    TEST_DO(verifyCacheStats(ds.getCacheStats(), 101, 102, 99, BASE_SZ+130));
     vcs.verifyVisit({7,9,17,19,67,88,89}, true);
-    TEST_DO(verifyCacheStats(ds.getCacheStats(), 101, 103, 99, 20783));
+    TEST_DO(verifyCacheStats(ds.getCacheStats(), 101, 103, 99, BASE_SZ+201));
     vcs.rewrite(17);
-    TEST_DO(verifyCacheStats(ds.getCacheStats(), 101, 103, 97, 19943));
+    TEST_DO(verifyCacheStats(ds.getCacheStats(), 101, 103, 97, BASE_SZ-657));
     vcs.verifyVisit({7,9,17,19,67,88,89}, true);
-    TEST_DO(verifyCacheStats(ds.getCacheStats(), 101, 104, 98, 20587));
+    TEST_DO(verifyCacheStats(ds.getCacheStats(), 101, 104, 98, BASE_SZ-3));
     vcs.remove(17);
-    TEST_DO(verifyCacheStats(ds.getCacheStats(), 101, 104, 97, 19943));
+    TEST_DO(verifyCacheStats(ds.getCacheStats(), 101, 104, 97, BASE_SZ-657));
     vcs.verifyVisit({7,9,17,19,67,88,89}, {7,9,19,67,88,89}, true);
-    TEST_DO(verifyCacheStats(ds.getCacheStats(), 101, 105, 98, 20526));
+    TEST_DO(verifyCacheStats(ds.getCacheStats(), 101, 105, 98, BASE_SZ-64));
 
     vcs.verifyVisit({41, 42}, true);
-    TEST_DO(verifyCacheStats(ds.getCacheStats(), 101, 106, 99, 20820));
+    TEST_DO(verifyCacheStats(ds.getCacheStats(), 101, 106, 99, BASE_SZ+238));
     vcs.verifyVisit({43, 44}, true);
-    TEST_DO(verifyCacheStats(ds.getCacheStats(), 101, 107, 100, 21124));
+    TEST_DO(verifyCacheStats(ds.getCacheStats(), 101, 107, 100, BASE_SZ+540));
     vcs.verifyVisit({41, 42, 43, 44}, true);
-    TEST_DO(verifyCacheStats(ds.getCacheStats(), 101, 108, 99, 20944));
+    TEST_DO(verifyCacheStats(ds.getCacheStats(), 101, 108, 99, BASE_SZ+362));
 }
 
 TEST("testWriteRead") {
@@ -595,7 +672,7 @@ TEST("testWriteRead") {
         vespalib::ThreadStackExecutor executor(1, 128*1024);
         MyTlSyncer tlSyncer;
         LogDataStore datastore(executor, "empty", config, GrowStrategy(),
-                               TuneFileSummary(), fileHeaderContext, tlSyncer, NULL);
+                               TuneFileSummary(), fileHeaderContext, tlSyncer, nullptr);
         ASSERT_TRUE(datastore.lastSyncToken() == 0);
         size_t headerFootprint = datastore.getDiskHeaderFootprint();
         EXPECT_LESS(0u, headerFootprint);
@@ -606,7 +683,7 @@ TEST("testWriteRead") {
         fetchAndTest(datastore, 0, a[0].c_str(), a[0].size());
         datastore.write(2, 0, a[1].c_str(), a[1].size());
         fetchAndTest(datastore, 0, a[1].c_str(), a[1].size());
-        fetchAndTest(datastore, 1, NULL, 0);
+        fetchAndTest(datastore, 1, nullptr, 0);
         datastore.remove(3, 0);
         fetchAndTest(datastore, 0, "", 0);
 
@@ -632,7 +709,7 @@ TEST("testWriteRead") {
         MyTlSyncer tlSyncer;
         LogDataStore datastore(executor, "empty", config,
                                GrowStrategy(), TuneFileSummary(),
-                               fileHeaderContext, tlSyncer, NULL);
+                               fileHeaderContext, tlSyncer, nullptr);
         size_t headerFootprint = datastore.getDiskHeaderFootprint();
         EXPECT_LESS(0u, headerFootprint);
         EXPECT_EQUAL(4944ul + headerFootprint, datastore.getDiskFootprint());
diff --git a/searchlib/src/tests/hitcollector/hitcollector_test.cpp b/searchlib/src/tests/hitcollector/hitcollector_test.cpp
index 3f49c6969a0..f9f977f4093 100644
--- a/searchlib/src/tests/hitcollector/hitcollector_test.cpp
+++ b/searchlib/src/tests/hitcollector/hitcollector_test.cpp
@@ -1,14 +1,12 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-#include <vespa/log/log.h>
-LOG_SETUP("hitcollector_test");
-#include <vespa/vespalib/testkit/testapp.h>
-
-#include <iostream>
 
+#include <vespa/vespalib/testkit/testapp.h>
 #include <vespa/searchlib/common/bitvector.h>
 #include <vespa/searchlib/fef/fef.h>
 #include <vespa/searchlib/queryeval/hitcollector.h>
-#include <vespa/searchlib/queryeval/scores.h>
+
+#include <vespa/log/log.h>
+LOG_SETUP("hitcollector_test");
 
 using namespace search;
 using namespace search::fef;
@@ -19,8 +17,8 @@ typedef std::map<uint32_t, feature_t> ScoreMap;
 struct BasicScorer : public HitCollector::DocumentScorer
 {
     feature_t _scoreDelta;
-    BasicScorer(feature_t scoreDelta) : _scoreDelta(scoreDelta) {}
-    virtual feature_t score(uint32_t docId) override {
+    explicit BasicScorer(feature_t scoreDelta) : _scoreDelta(scoreDelta) {}
+    feature_t score(uint32_t docId) override {
         return docId + _scoreDelta;
     }
 };
@@ -28,8 +26,8 @@ struct BasicScorer : public HitCollector::DocumentScorer
 struct PredefinedScorer : public HitCollector::DocumentScorer
 {
     ScoreMap _scores;
-    PredefinedScorer(const ScoreMap &scores) : _scores(scores) {}
-    virtual feature_t score(uint32_t docId) override {
+    explicit PredefinedScorer(ScoreMap scores) : _scores(std::move(scores)) {}
+    feature_t score(uint32_t docId) override {
         feature_t retval = default_rank_value;
         auto itr = _scores.find(docId);
         if (itr != _scores.end()) {
@@ -41,38 +39,32 @@ struct PredefinedScorer : public HitCollector::DocumentScorer
 
 void checkResult(const ResultSet & rs, const std::vector<RankedHit> & exp)
 {
-    if (exp.size() > 0) {
+    if ( ! exp.empty()) {
         const RankedHit * rh = rs.getArray();
-        ASSERT_TRUE(rh != NULL);
+        ASSERT_TRUE(rh != nullptr);
         ASSERT_EQUAL(rs.getArrayUsed(), exp.size());
 
         for (uint32_t i = 0; i < exp.size(); ++i) {
-#if 0
-            std::cout << " rh[" << i << "]._docId = " << rh[i]._docId << std::endl;
-            std::cout << "exp[" << i << "]._docId = " << exp[i]._docId << std::endl;
-            std::cout << " rh[" << i << "]._rankValue = " << rh[i]._rankValue << std::endl;
-            std::cout << "exp[" << i << "]._rankValue = " << exp[i]._rankValue << std::endl;
-#endif
             EXPECT_EQUAL(rh[i]._docId, exp[i]._docId);
             EXPECT_EQUAL(rh[i]._rankValue, exp[i]._rankValue);
         }
     } else {
-        ASSERT_TRUE(rs.getArray() == NULL);
+        ASSERT_TRUE(rs.getArray() == nullptr);
     }
 }
 
 void checkResult(ResultSet & rs, BitVector * exp)
 {
-    if (exp != NULL) {
+    if (exp != nullptr) {
         BitVector * bv = rs.getBitOverflow();
-        ASSERT_TRUE(bv != NULL);
+        ASSERT_TRUE(bv != nullptr);
         bv->invalidateCachedCount();
         exp->invalidateCachedCount();
         LOG(info, "bv.hits: %u, exp.hits: %u", bv->countTrueBits(), exp->countTrueBits());
         ASSERT_TRUE(bv->countTrueBits() == exp->countTrueBits());
         EXPECT_TRUE(*bv == *exp);
     } else {
-        ASSERT_TRUE(rs.getBitOverflow() == NULL);
+        ASSERT_TRUE(rs.getBitOverflow() == nullptr);
     }
 }
 
@@ -85,8 +77,8 @@ void testAddHit(uint32_t numDocs, uint32_t maxHitsSize, uint32_t maxHeapSize)
         std::vector<RankedHit> expRh;
 
         std::unique_ptr<ResultSet> rs = hc.getResultSet();
-        TEST_DO(checkResult(*rs.get(), expRh));
-        TEST_DO(checkResult(*rs.get(), NULL));
+        TEST_DO(checkResult(*rs, expRh));
+        TEST_DO(checkResult(*rs, nullptr));
     }
 
     LOG(info, "testAddHit: only ranked hits");
@@ -98,14 +90,14 @@ void testAddHit(uint32_t numDocs, uint32_t maxHitsSize, uint32_t maxHeapSize)
             hc.addHit(i, i + 100);
 
             // build expected result set as we go along
-            expRh.push_back(RankedHit());
+            expRh.emplace_back();
             expRh.back()._docId = i;
             expRh.back()._rankValue = i + 100;
         }
 
         std::unique_ptr<ResultSet> rs = hc.getResultSet();
-        TEST_DO(checkResult(*rs.get(), expRh));
-        TEST_DO(checkResult(*rs.get(), NULL));
+        TEST_DO(checkResult(*rs, expRh));
+        TEST_DO(checkResult(*rs, nullptr));
     }
 
     LOG(info, "testAddHit: both ranked hits and bit vector hits");
@@ -120,15 +112,15 @@ void testAddHit(uint32_t numDocs, uint32_t maxHitsSize, uint32_t maxHeapSize)
             // build expected result set as we go along
             expBv->setBit(i);
             if (i >= (numDocs - maxHitsSize)) {
-                expRh.push_back(RankedHit());
+                expRh.emplace_back();
                 expRh.back()._docId = i;
                 expRh.back()._rankValue = i + 100;
             }
         }
 
         std::unique_ptr<ResultSet> rs = hc.getResultSet();
-        TEST_DO(checkResult(*rs.get(), expRh));
-        TEST_DO(checkResult(*rs.get(), expBv.get()));
+        TEST_DO(checkResult(*rs, expRh));
+        TEST_DO(checkResult(*rs, expBv.get()));
     }
 }
 
@@ -157,23 +149,27 @@ struct Fixture {
         }
     }
     size_t reRank() {
-        return hc.reRank(scorer);
+        return hc.reRank(scorer, hc.getSortedHeapHits());
     }
     size_t reRank(size_t count) {
-        return hc.reRank(scorer, count);
+        auto hits = hc.getSortedHeapHits();
+        if (hits.size() > count) {
+            hits.resize(count);
+        }
+        return hc.reRank(scorer, std::move(hits));
     }
 };
 
 struct AscendingScoreFixture : Fixture {
     AscendingScoreFixture() : Fixture() {}
-    virtual HitRank calculateScore(uint32_t i) override {
+    HitRank calculateScore(uint32_t i) override {
         return i + 100;
     }
 };
 
 struct DescendingScoreFixture : Fixture {
     DescendingScoreFixture() : Fixture() {}
-    virtual HitRank calculateScore(uint32_t i) override {
+    HitRank calculateScore(uint32_t i) override {
         return 100 - i;
     }
 };
@@ -197,8 +193,8 @@ TEST_F("testReRank - ascending", AscendingScoreFixture)
     EXPECT_EQUAL(expRh.size(), 10u);
 
     std::unique_ptr<ResultSet> rs = f.hc.getResultSet();
-    TEST_DO(checkResult(*rs.get(), expRh));
-    TEST_DO(checkResult(*rs.get(), f.expBv.get()));
+    TEST_DO(checkResult(*rs, expRh));
+    TEST_DO(checkResult(*rs, f.expBv.get()));
 }
 
 TEST_F("testReRank - descending", DescendingScoreFixture)
@@ -216,8 +212,8 @@ TEST_F("testReRank - descending", DescendingScoreFixture)
     EXPECT_EQUAL(expRh.size(), 10u);
 
     std::unique_ptr<ResultSet> rs = f.hc.getResultSet();
-    TEST_DO(checkResult(*rs.get(), expRh));
-    TEST_DO(checkResult(*rs.get(), f.expBv.get()));
+    TEST_DO(checkResult(*rs, expRh));
+    TEST_DO(checkResult(*rs, f.expBv.get()));
 }
 
 TEST_F("testReRank - partial", AscendingScoreFixture)
@@ -235,25 +231,24 @@ TEST_F("testReRank - partial", AscendingScoreFixture)
     EXPECT_EQUAL(expRh.size(), 10u);
 
     std::unique_ptr<ResultSet> rs = f.hc.getResultSet();
-    TEST_DO(checkResult(*rs.get(), expRh));
-    TEST_DO(checkResult(*rs.get(), f.expBv.get()));
+    TEST_DO(checkResult(*rs, expRh));
+    TEST_DO(checkResult(*rs, f.expBv.get()));
 }
 
-TEST_F("require that scores for 2nd phase candidates can be retrieved", DescendingScoreFixture)
+TEST_F("require that hits for 2nd phase candidates can be retrieved", DescendingScoreFixture)
 {
     f.addHits();
-    std::vector<feature_t> scores = f.hc.getSortedHeapScores();
+    std::vector<HitCollector::Hit> scores = f.hc.getSortedHeapHits();
     ASSERT_EQUAL(5u, scores.size());
-    EXPECT_EQUAL(100, scores[0]);
-    EXPECT_EQUAL(99, scores[1]);
-    EXPECT_EQUAL(98, scores[2]);
-    EXPECT_EQUAL(97, scores[3]);
-    EXPECT_EQUAL(96, scores[4]);
+    EXPECT_EQUAL(100, scores[0].second);
+    EXPECT_EQUAL(99, scores[1].second);
+    EXPECT_EQUAL(98, scores[2].second);
+    EXPECT_EQUAL(97, scores[3].second);
+    EXPECT_EQUAL(96, scores[4].second);
 }
 
 TEST("require that score ranges can be read and set.") {
-    std::pair<Scores, Scores> ranges =
-        std::make_pair(Scores(1.0, 2.0), Scores(3.0, 4.0));
+    std::pair<Scores, Scores> ranges = std::make_pair(Scores(1.0, 2.0), Scores(3.0, 4.0));
     HitCollector hc(20, 10, 5);
     hc.setRanges(ranges);
     EXPECT_EQUAL(ranges.first.low, hc.getRanges().first.low);
@@ -275,19 +270,19 @@ TEST("testNoHitsToReRank") {
             hc.addHit(i, i + 100);
 
             // build expected result set as we go along
-            expRh.push_back(RankedHit());
+            expRh.emplace_back();
             expRh.back()._docId = i;
             expRh.back()._rankValue = i + 100;
         }
 
         std::unique_ptr<ResultSet> rs = hc.getResultSet();
-        TEST_DO(checkResult(*rs.get(), expRh));
-        TEST_DO(checkResult(*rs.get(), NULL));
+        TEST_DO(checkResult(*rs, expRh));
+        TEST_DO(checkResult(*rs, nullptr));
     }
 }
 
 void testScaling(const std::vector<feature_t> &initScores,
-                 const ScoreMap &finalScores,
+                 ScoreMap finalScores,
                  const std::vector<RankedHit> &expected)
 {
     HitCollector hc(5, 5, 2);
@@ -297,13 +292,13 @@ void testScaling(const std::vector<feature_t> &initScores,
         hc.addHit(i, initScores[i]);
     }
 
-    PredefinedScorer scorer(finalScores);
+    PredefinedScorer scorer(std::move(finalScores));
     // perform second phase ranking
-    EXPECT_EQUAL(2u, hc.reRank(scorer));
+    EXPECT_EQUAL(2u, hc.reRank(scorer, hc.getSortedHeapHits()));
 
     // check results
     std::unique_ptr<ResultSet> rs = hc.getResultSet();
-    TEST_DO(checkResult(*rs.get(), expected));
+    TEST_DO(checkResult(*rs, expected));
 }
 
 TEST("testScaling") {
@@ -332,7 +327,7 @@ TEST("testScaling") {
         finalScores[3] = 300;
         finalScores[4] = 400;
 
-        testScaling(initScores, finalScores, exp);
+        testScaling(initScores, std::move(finalScores), exp);
     }
     { // scale down and adjust up
         exp[0]._rankValue = 200; // scaled
@@ -346,7 +341,7 @@ TEST("testScaling") {
         finalScores[3] = 500;
         finalScores[4] = 600;
 
-        testScaling(initScores, finalScores, exp);
+        testScaling(initScores, std::move(finalScores), exp);
     }
     { // scale up and adjust down
 
@@ -361,7 +356,7 @@ TEST("testScaling") {
         finalScores[3] = 3250;
         finalScores[4] = 4500;
 
-        testScaling(initScores, finalScores, exp);
+        testScaling(initScores, std::move(finalScores), exp);
     }
     { // minimal scale (second phase range = 0 (4 - 4) -> 1)
         exp[0]._rankValue = 1; // scaled
@@ -375,7 +370,7 @@ TEST("testScaling") {
         finalScores[3] = 4;
         finalScores[4] = 4;
 
-        testScaling(initScores, finalScores, exp);
+        testScaling(initScores, std::move(finalScores), exp);
     }
     { // minimal scale (first phase range = 0 (4000 - 4000) -> 1)
         std::vector<feature_t> is(initScores);
@@ -391,7 +386,7 @@ TEST("testScaling") {
         finalScores[3] = 400;
         finalScores[4] = 500;
 
-        testScaling(is, finalScores, exp);
+        testScaling(is, std::move(finalScores), exp);
     }
 }
 
@@ -410,8 +405,8 @@ TEST("testOnlyBitVector") {
 
         std::unique_ptr<ResultSet> rs = hc.getResultSet();
         std::vector<RankedHit> expRh;
-        TEST_DO(checkResult(*rs.get(), expRh));  // no ranked hits
-        TEST_DO(checkResult(*rs.get(), expBv.get())); // only bit vector
+        TEST_DO(checkResult(*rs, expRh));  // no ranked hits
+        TEST_DO(checkResult(*rs, expBv.get())); // only bit vector
     }
 }
 
@@ -433,19 +428,19 @@ TEST_F("require that result set is merged correctly with first phase ranking",
         f.hc.addHit(i, i + 1000);
 
         // build expected result set
-        expRh.push_back(RankedHit());
+        expRh.emplace_back();
         expRh.back()._docId = i;
         // only the maxHitsSize best hits gets a score
         expRh.back()._rankValue = (i < f.numDocs - f.maxHitsSize) ? default_rank_value : i + 1000;
     }
     std::unique_ptr<ResultSet> rs = f.hc.getResultSet();
-    TEST_DO(checkResult(*rs.get(), expRh));
+    TEST_DO(checkResult(*rs, expRh));
 }
 
 void
 addExpectedHitForMergeTest(const MergeResultSetFixture &f, std::vector<RankedHit> &expRh, uint32_t docId)
 {
-    expRh.push_back(RankedHit());
+    expRh.emplace_back();
     expRh.back()._docId = docId;
     if (docId < f.numDocs - f.maxHitsSize) { // only the maxHitsSize best hits gets a score
         expRh.back()._rankValue = default_rank_value;
@@ -466,9 +461,9 @@ TEST_F("require that result set is merged correctly with second phase ranking (d
         f.hc.addHit(i, i + 1000);
         addExpectedHitForMergeTest(f, expRh, i);
     }
-    EXPECT_EQUAL(f.maxHeapSize, f.hc.reRank(scorer));
+    EXPECT_EQUAL(f.maxHeapSize, f.hc.reRank(scorer, f.hc.getSortedHeapHits()));
     std::unique_ptr<ResultSet> rs = f.hc.getResultSet();
-    TEST_DO(checkResult(*rs.get(), expRh));
+    TEST_DO(checkResult(*rs, expRh));
 }
 
 TEST("require that hits can be added out of order") {
@@ -476,7 +471,7 @@ TEST("require that hits can be added out of order") {
     std::vector<RankedHit> expRh;
     // produce expected result in normal order
     for (uint32_t i = 0; i < 5; ++i) {
-        expRh.push_back(RankedHit());
+        expRh.emplace_back();
         expRh.back()._docId = i;
         expRh.back()._rankValue = i + 100;
     }
@@ -485,8 +480,8 @@ TEST("require that hits can be added out of order") {
         hc.addHit(i, i + 100);
     }
     std::unique_ptr<ResultSet> rs = hc.getResultSet();
-    TEST_DO(checkResult(*rs.get(), expRh));
-    TEST_DO(checkResult(*rs.get(), nullptr));
+    TEST_DO(checkResult(*rs, expRh));
+    TEST_DO(checkResult(*rs, nullptr));
 }
 
 TEST("require that hits can be added out of order when passing array limit") {
@@ -495,7 +490,7 @@ TEST("require that hits can be added out of order when passing array limit") {
     // produce expected result in normal order
     const size_t numHits = 150;
     for (uint32_t i = 0; i < numHits; ++i) {
-        expRh.push_back(RankedHit());
+        expRh.emplace_back();
         expRh.back()._docId = i;
         expRh.back()._rankValue = (i < 50) ? default_rank_value : (i + 100);
     }
@@ -504,8 +499,8 @@ TEST("require that hits can be added out of order when passing array limit") {
         hc.addHit(i, i + 100);
     }
     std::unique_ptr<ResultSet> rs = hc.getResultSet();
-    TEST_DO(checkResult(*rs.get(), expRh));
-    TEST_DO(checkResult(*rs.get(), nullptr));
+    TEST_DO(checkResult(*rs, expRh));
+    TEST_DO(checkResult(*rs, nullptr));
 }
 
 TEST("require that hits can be added out of order only after passing array limit") {
@@ -514,7 +509,7 @@ TEST("require that hits can be added out of order only after passing array limit
     // produce expected result in normal order
     const size_t numHits = 150;
     for (uint32_t i = 0; i < numHits; ++i) {
-        expRh.push_back(RankedHit());
+        expRh.emplace_back();
         expRh.back()._docId = i;
         expRh.back()._rankValue = (i < 50) ? default_rank_value : (i + 100);
     }
@@ -527,8 +522,8 @@ TEST("require that hits can be added out of order only after passing array limit
         hc.addHit(i, i + 100);
     }
     std::unique_ptr<ResultSet> rs = hc.getResultSet();
-    TEST_DO(checkResult(*rs.get(), expRh));
-    TEST_DO(checkResult(*rs.get(), nullptr));
+    TEST_DO(checkResult(*rs, expRh));
+    TEST_DO(checkResult(*rs, nullptr));
 }
 
 TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/searchlib/src/tests/stackdumpiterator/stackdumpiteratortest.cpp b/searchlib/src/tests/stackdumpiterator/stackdumpiteratortest.cpp
index 8ad4578b6c1..37a63828667 100644
--- a/searchlib/src/tests/stackdumpiterator/stackdumpiteratortest.cpp
+++ b/searchlib/src/tests/stackdumpiterator/stackdumpiteratortest.cpp
@@ -172,12 +172,12 @@ StackDumpIteratorTest::ShowResult(int testNo,
             delete item;
             break;
         }
-        if (strncmp(item->_indexName.c_str(), idx.c_str(), idx.size()) != 0) {
+        if (strncmp(item->_indexName.c_str(), idx.data(), idx.size()) != 0) {
             results |= ITERATOR_ERROR_WRONG_INDEX;
             delete item;
             break;
         }
-        if (strncmp(item->_term.c_str(), term.c_str(), term.size()) != 0) {
+        if (strncmp(item->_term.c_str(), term.data(), term.size()) != 0) {
             results |= ITERATOR_ERROR_WRONG_TERM;
             delete item;
             break;
diff --git a/searchlib/src/vespa/searchlib/attribute/attribute_header.cpp b/searchlib/src/vespa/searchlib/attribute/attribute_header.cpp
index 4a69e0a827d..828c7c19962 100644
--- a/searchlib/src/vespa/searchlib/attribute/attribute_header.cpp
+++ b/searchlib/src/vespa/searchlib/attribute/attribute_header.cpp
@@ -68,9 +68,7 @@ AttributeHeader::AttributeHeader(const vespalib::string &fileName,
 {
 }
 
-AttributeHeader::~AttributeHeader()
-{
-}
+AttributeHeader::~AttributeHeader() = default;
 
 void
 AttributeHeader::internalExtractTags(const vespalib::GenericHeader &header)
diff --git a/searchlib/src/vespa/searchlib/attribute/attribute_header.h b/searchlib/src/vespa/searchlib/attribute/attribute_header.h
index 3d00396e171..4a0b1e0bee0 100644
--- a/searchlib/src/vespa/searchlib/attribute/attribute_header.h
+++ b/searchlib/src/vespa/searchlib/attribute/attribute_header.h
@@ -10,8 +10,7 @@
 
 namespace vespalib { class GenericHeader; }
 
-namespace search {
-namespace attribute {
+namespace search::attribute {
 
 /**
  * Attribute header class used by attribute savers and attribute initializer
@@ -71,5 +70,4 @@ public:
     void addTags(vespalib::GenericHeader &header) const;
 };
 
-} // namespace search::attribute
-} // namespace search
+}
diff --git a/searchlib/src/vespa/searchlib/attribute/attribute_weighted_set_blueprint.cpp b/searchlib/src/vespa/searchlib/attribute/attribute_weighted_set_blueprint.cpp
index 3ff7db5a184..2eff5a05635 100644
--- a/searchlib/src/vespa/searchlib/attribute/attribute_weighted_set_blueprint.cpp
+++ b/searchlib/src/vespa/searchlib/attribute/attribute_weighted_set_blueprint.cpp
@@ -7,6 +7,8 @@
 #include <vespa/searchlib/common/bitvector.h>
 #include <vespa/vespalib/stllike/hash_map.h>
 #include <vespa/searchlib/fef/matchdatalayout.h>
+#include <vespa/vespalib/objects/visit.h>
+#include <vespa/vespalib/util/stringfmt.h>
 
 namespace search {
 
@@ -36,14 +38,8 @@ class UseStringEnum : public UseAttr
 public:
     UseStringEnum(const IAttributeVector & attr)
         : UseAttr(attr) {}
-    bool mapToken(const ISearchContext &context, int64_t &token) const
-    {
-        attribute::IAttributeVector::EnumHandle handle;
-        if (attribute().findEnum(context.queryTerm().getTerm(), handle)) {
-            token = handle;
-            return true;
-        }
-        return false;
+    auto mapToken(const ISearchContext &context) const {
+        return attribute().findFoldedEnums(context.queryTerm()->getTerm());
     }
     int64_t getToken(uint32_t docId) const {
         return attribute().getEnum(docId);
@@ -56,14 +52,13 @@ class UseInteger : public UseAttr
 {
 public:
     UseInteger(const IAttributeVector & attr) : UseAttr(attr) {}
-    bool mapToken(const ISearchContext &context, int64_t &token) const
-    {
+    std::vector<int64_t> mapToken(const ISearchContext &context) const {
+        std::vector<int64_t> result;
         Int64Range range(context.getAsIntegerTerm());
         if (range.isPoint()) {
-            token = range.lower();
-            return true;
+            result.push_back(range.lower());
         }
-        return false;
+        return result;
     }
     int64_t getToken(uint32_t docId) const {
         return attribute().getInt(docId);
@@ -92,8 +87,7 @@ public:
         : _tfmd(tfmd), _attr(attr), _map(), _weight(0)
     {
         for (size_t i = 0; i < contexts.size(); ++i) {
-            int64_t token(0);
-            if (_attr.mapToken(*contexts[i], token)) {
+            for (int64_t token : _attr.mapToken(*contexts[i])) {
                 _map[token] = weights[i];
             }
         }
@@ -174,11 +168,11 @@ AttributeWeightedSetBlueprint::createLeafSearch(const fef::TermFieldMatchDataArr
         assert(isSingleValue);
         (void) isSingleValue;
         if (isString) {
-            return queryeval::SearchIterator::UP(new AttributeFilter<UseStringEnum>(tfmd, _attr, _weights, _contexts));
+            return std::make_unique<AttributeFilter<UseStringEnum>>(tfmd, _attr, _weights, _contexts);
         } else {
             assert(isInteger);
             (void) isInteger;
-            return queryeval::SearchIterator::UP(new AttributeFilter<UseInteger>(tfmd, _attr, _weights, _contexts));
+            return std::make_unique<AttributeFilter<UseInteger>>(tfmd, _attr, _weights, _contexts);
         }
     }
 }
@@ -187,10 +181,34 @@ void
 AttributeWeightedSetBlueprint::fetchPostings(bool strict)
 {
     if (strict) {
-        for (size_t i = 0; i < _contexts.size(); ++i) {
-            _contexts[i]->fetchPostings(true);
+        for (auto * context : _contexts) {
+            context->fetchPostings(true);
+        }
+    }
+}
+
+void
+AttributeWeightedSetBlueprint::visitMembers(vespalib::ObjectVisitor &visitor) const
+{
+    ComplexLeafBlueprint::visitMembers(visitor);
+    visitor.visitString("attribute", _attr.getName());
+    visitor.openStruct("terms", "TermList");
+    for (size_t i = 0; i < _contexts.size(); ++i) {
+        const ISearchContext * context = _contexts[i];
+        visitor.openStruct(vespalib::make_string("[%zu]", i), "Term");
+        visitor.visitBool("valid", context->valid());
+        if (context-> valid()) {
+            bool isString = (_attr.isStringType() && _attr.hasEnum());
+            if (isString) {
+                visitor.visitString("term", context->queryTerm()->getTerm());
+            } else {
+                visitor.visitInt("term", context->getAsIntegerTerm().lower());
+            }
+            visitor.visitInt("weight", _weights[i]);
         }
+        visitor.closeStruct();
     }
+    visitor.closeStruct();
 }
 
 } // namespace search
diff --git a/searchlib/src/vespa/searchlib/attribute/attribute_weighted_set_blueprint.h b/searchlib/src/vespa/searchlib/attribute/attribute_weighted_set_blueprint.h
index 3b4bbd2d916..d66a544e77a 100644
--- a/searchlib/src/vespa/searchlib/attribute/attribute_weighted_set_blueprint.h
+++ b/searchlib/src/vespa/searchlib/attribute/attribute_weighted_set_blueprint.h
@@ -16,21 +16,21 @@ class AttributeWeightedSetBlueprint : public queryeval::ComplexLeafBlueprint
 private:
     using ISearchContext = attribute::ISearchContext;
     using IAttributeVector = attribute::IAttributeVector;
-    size_t                     _numDocs;
-    size_t                     _estHits;
-    std::vector<int32_t>       _weights;
-    const IAttributeVector    & _attr;
-    std::vector<ISearchContext*> _contexts;
-
-    AttributeWeightedSetBlueprint(const AttributeWeightedSetBlueprint &); // disabled
-    AttributeWeightedSetBlueprint &operator=(const AttributeWeightedSetBlueprint &); // disabled
+    size_t                         _numDocs;
+    size_t                         _estHits;
+    std::vector<int32_t>           _weights;
+    const IAttributeVector       & _attr;
+    std::vector<ISearchContext*>   _contexts;
 
 public:
+    AttributeWeightedSetBlueprint(const AttributeWeightedSetBlueprint &) = delete;
+    AttributeWeightedSetBlueprint &operator=(const AttributeWeightedSetBlueprint &) = delete;
     AttributeWeightedSetBlueprint(const queryeval::FieldSpec &field, const IAttributeVector & attr);
     ~AttributeWeightedSetBlueprint();
     void addToken(std::unique_ptr<ISearchContext> context, int32_t weight);
     queryeval::SearchIterator::UP createLeafSearch(const fef::TermFieldMatchDataArray &tfmda, bool strict) const override;
     void fetchPostings(bool strict) override;
+    void visitMembers(vespalib::ObjectVisitor &visitor) const override;
 };
 
 } // namespace search
diff --git a/searchlib/src/vespa/searchlib/attribute/attributefilewriter.cpp b/searchlib/src/vespa/searchlib/attribute/attributefilewriter.cpp
index b1268aacc4f..415c00cb8fd 100644
--- a/searchlib/src/vespa/searchlib/attribute/attributefilewriter.cpp
+++ b/searchlib/src/vespa/searchlib/attribute/attributefilewriter.cpp
@@ -2,10 +2,10 @@
 
 #include "attributefilewriter.h"
 #include "attributefilebufferwriter.h"
+#include "attribute_header.h"
 #include <vespa/vespalib/data/fileheader.h>
 #include <vespa/searchlib/common/fileheadercontext.h>
 #include <vespa/searchlib/common/tunefileinfo.h>
-#include "attribute_header.h"
 #include <vespa/fastos/file.h>
 
 #include <vespa/log/log.h>
@@ -14,7 +14,6 @@ LOG_SETUP(".searchlib.attribute.attributefilewriter");
 using search::common::FileHeaderContext;
 using vespalib::getLastErrorString;
 
-
 namespace search {
 
 namespace {
@@ -23,8 +22,7 @@ const uint32_t headerAlign = 4096;
 const uint32_t MIN_ALIGNMENT = 4096;
 
 void
-writeDirectIOAligned(FastOS_FileInterface &file, const void *buf,
-                     size_t length)
+writeDirectIOAligned(FastOS_FileInterface &file, const void *buf, size_t length)
 {
     const char * data(static_cast<const char *>(buf));
     size_t remaining(length);
@@ -38,7 +36,6 @@ writeDirectIOAligned(FastOS_FileInterface &file, const void *buf,
     }
 }
 
-
 void
 updateHeader(const vespalib::string &name, uint64_t fileBitSize)
 {
@@ -63,29 +60,22 @@ class FileBackedBufferWriter : public AttributeFileBufferWriter
 {
 public:
     FileBackedBufferWriter(AttributeFileWriter &fileWriter);
+    ~FileBackedBufferWriter() override;
 
-    virtual ~FileBackedBufferWriter();
-
-    virtual void onFlush(size_t nowLen) override;
+    void onFlush(size_t nowLen) override;
 };
 
 
 FileBackedBufferWriter::FileBackedBufferWriter(AttributeFileWriter &fileWriter)
     : AttributeFileBufferWriter(fileWriter)
-{
-}
-
-
-FileBackedBufferWriter::~FileBackedBufferWriter()
-{
-}
+{ }
 
+FileBackedBufferWriter::~FileBackedBufferWriter() = default;
 
 void
 FileBackedBufferWriter::onFlush(size_t nowLen) {
     // Note: Must use const ptr to indicate that buffer is pre-filled.
-    Buffer buf(std::make_unique<BufferBuf>
-               ((const char *) _buf->getFree(), nowLen));
+    auto buf(std::make_unique<BufferBuf>(static_cast<const void *>(_buf->getFree()), nowLen));
     assert(buf->getDataLen() == nowLen);
     assert(buf->getData() == _buf->getFree());
     _fileWriter.writeBuf(std::move(buf));
@@ -93,7 +83,6 @@ FileBackedBufferWriter::onFlush(size_t nowLen) {
 
 }
 
-
 AttributeFileWriter::
 AttributeFileWriter(const TuneFileAttributes &tuneFileAttributes,
                     const FileHeaderContext &fileHeaderContext,
@@ -107,9 +96,7 @@ AttributeFileWriter(const TuneFileAttributes &tuneFileAttributes,
       _fileBitSize(0)
 { }
 
-
-AttributeFileWriter::~AttributeFileWriter() { }
-
+AttributeFileWriter::~AttributeFileWriter() = default;
 
 bool
 AttributeFileWriter::open(const vespalib::string &fileName)
@@ -130,7 +117,6 @@ AttributeFileWriter::open(const vespalib::string &fileName)
     return true;
 }
 
-
 void
 AttributeFileWriter::writeHeader()
 {
@@ -142,7 +128,6 @@ AttributeFileWriter::writeHeader()
     _fileBitSize = headerLen * 8;
 }
 
-
 void
 AttributeFileWriter::addTags(vespalib::GenericHeader &header)
 {
@@ -151,14 +136,12 @@ AttributeFileWriter::addTags(vespalib::GenericHeader &header)
     header.putTag(Tag("desc", _desc));
 }
 
-
 AttributeFileWriter::Buffer
 AttributeFileWriter::allocBuf(size_t size)
 {
     return std::make_unique<BufferBuf>(size, MIN_ALIGNMENT);
 }
 
-
 void
 AttributeFileWriter::writeBuf(Buffer buf)
 {
@@ -168,7 +151,6 @@ AttributeFileWriter::writeBuf(Buffer buf)
     _fileBitSize += bufLen * 8;
 }
 
-
 void
 AttributeFileWriter::close()
 {
@@ -179,13 +161,10 @@ AttributeFileWriter::close()
     }
 }
 
-
 std::unique_ptr<BufferWriter>
 AttributeFileWriter::allocBufferWriter()
 {
     return std::make_unique<FileBackedBufferWriter>(*this);
 }
 
-
-
 } // namespace search
diff --git a/searchlib/src/vespa/searchlib/attribute/attributevector.cpp b/searchlib/src/vespa/searchlib/attribute/attributevector.cpp
index 8b90677a1bc..fbad06821cd 100644
--- a/searchlib/src/vespa/searchlib/attribute/attributevector.cpp
+++ b/searchlib/src/vespa/searchlib/attribute/attributevector.cpp
@@ -508,6 +508,12 @@ int32_t AttributeVector::getWeight(DocId, uint32_t) const { return 1; }
 
 bool AttributeVector::findEnum(const char *, EnumHandle &) const { return false; }
 
+std::vector<search::attribute::IAttributeVector::EnumHandle>
+AttributeVector::findFoldedEnums(const char *) const {
+    std::vector<EnumHandle> empty;
+    return empty;
+}
+
 const char * AttributeVector::getStringFromEnum(EnumHandle) const { return nullptr; }
 
 AttributeVector::SearchContext::SearchContext(const AttributeVector &attr) :
diff --git a/searchlib/src/vespa/searchlib/attribute/attributevector.h b/searchlib/src/vespa/searchlib/attribute/attributevector.h
index 87ef9a41432..cb83b86001b 100644
--- a/searchlib/src/vespa/searchlib/attribute/attributevector.h
+++ b/searchlib/src/vespa/searchlib/attribute/attributevector.h
@@ -500,6 +500,8 @@ public:
 
     // Implements IAttributeVector
     bool findEnum(const char *value, EnumHandle &e) const override;
+    std::vector<EnumHandle> findFoldedEnums(const char *) const override;
+
     const char * getStringFromEnum(EnumHandle e) const override;
 
 ///// Modify API
@@ -536,16 +538,15 @@ public:
         typedef std::unique_ptr<SearchContext> UP;
         ~SearchContext();
 
-        // Implements attribute::ISearchContext
-        virtual unsigned int approximateHits() const override;
-        virtual queryeval::SearchIterator::UP createIterator(fef::TermFieldMatchData *matchData, bool strict) override;
-        virtual void fetchPostings(bool strict) override;
-        virtual bool valid() const override { return false; }
-        virtual Int64Range getAsIntegerTerm() const override { return Int64Range(); }
-        virtual const QueryTermBase &queryTerm() const override {
-            return *static_cast<const QueryTermBase *>(NULL);
+        unsigned int approximateHits() const override;
+        queryeval::SearchIterator::UP createIterator(fef::TermFieldMatchData *matchData, bool strict) override;
+        void fetchPostings(bool strict) override;
+        bool valid() const override { return false; }
+        Int64Range getAsIntegerTerm() const override { return Int64Range(); }
+        const QueryTermBase * queryTerm() const override {
+            return static_cast<const QueryTermBase *>(nullptr);
         }
-        virtual const vespalib::string &attributeName() const override {
+        const vespalib::string &attributeName() const override {
             return _attr.getName();
         }
 
diff --git a/searchlib/src/vespa/searchlib/attribute/attrvector.cpp b/searchlib/src/vespa/searchlib/attribute/attrvector.cpp
index 94b847a02e6..41945dff1fe 100644
--- a/searchlib/src/vespa/searchlib/attribute/attrvector.cpp
+++ b/searchlib/src/vespa/searchlib/attribute/attrvector.cpp
@@ -51,6 +51,19 @@ bool StringDirectAttribute::findEnum(const char * key, EnumHandle & e) const
     return false;
 }
 
+
+// XXX this is not really correct
+std::vector<StringDirectAttribute::EnumHandle>
+StringDirectAttribute::findFoldedEnums(const char *key) const
+{
+    std::vector<EnumHandle> result;
+    EnumHandle handle;
+    if (findEnum(key, handle)) {
+        result.push_back(handle);
+    }
+    return result;
+}
+
 void StringDirectAttribute::onSave(IAttributeSaveTarget & saveTarget)
 {
     assert(!saveTarget.getEnumerated());
diff --git a/searchlib/src/vespa/searchlib/attribute/attrvector.h b/searchlib/src/vespa/searchlib/attribute/attrvector.h
index a6f9a0ebcee..2ba9ed083f0 100644
--- a/searchlib/src/vespa/searchlib/attribute/attrvector.h
+++ b/searchlib/src/vespa/searchlib/attribute/attrvector.h
@@ -152,6 +152,7 @@ protected:
     StringDirectAttribute(const vespalib::string & baseFileName, const Config & c);
     ~StringDirectAttribute();
     bool findEnum(const char * value, EnumHandle & e) const override;
+    std::vector<EnumHandle> findFoldedEnums(const char *) const override;
     void getEnumValue(const EnumHandle * v, uint32_t *e, uint32_t sz) const override {
         for (size_t i(0); i < sz; i++) {
             e[i] = v[i];
diff --git a/searchlib/src/vespa/searchlib/attribute/diversity.cpp b/searchlib/src/vespa/searchlib/attribute/diversity.cpp
index 14e05afa8f0..65a84baa212 100644
--- a/searchlib/src/vespa/searchlib/attribute/diversity.cpp
+++ b/searchlib/src/vespa/searchlib/attribute/diversity.cpp
@@ -1,11 +1,143 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include "diversity.hpp"
+#include "singleenumattribute.h"
+#include "singlenumericattribute.h"
+#include <vespa/vespalib/stllike/hash_map.h>
 
-namespace search {
-namespace attribute {
-namespace diversity {
+using std::make_unique;
+namespace search::attribute::diversity {
 
+template <typename T>
+struct FetchNumberFast {
+    const T * const attr;
+    typedef typename T::LoadedValueType ValueType;
+    FetchNumberFast(const IAttributeVector &attr_in) : attr(dynamic_cast<const T *>(&attr_in)) {}
+    ValueType get(uint32_t docid) const { return attr->getFast(docid); }
+    bool valid() const { return (attr != nullptr); }
+};
+
+struct FetchEnumFast {
+    const SingleValueEnumAttributeBase * const attr;
+    typedef uint32_t ValueType;
+    FetchEnumFast(const IAttributeVector &attr_in) : attr(dynamic_cast<const SingleValueEnumAttributeBase *>(&attr_in)) {}
+    ValueType get(uint32_t docid) const { return attr->getE(docid); }
+    bool valid() const { return (attr != nullptr); }
+};
+
+struct FetchEnum {
+    const IAttributeVector *attr;
+    typedef uint32_t ValueType;
+    FetchEnum(const IAttributeVector & attr_in) : attr(&attr_in) {}
+    ValueType get(uint32_t docid) const { return attr->getEnum(docid); }
+};
+
+struct FetchInteger {
+    const IAttributeVector * attr;
+    typedef int64_t ValueType;
+    FetchInteger(const IAttributeVector & attr_in) : attr(&attr_in) {}
+    ValueType get(uint32_t docid) const { return attr->getInt(docid); }
+};
+
+struct FetchFloat {
+    const IAttributeVector * attr;
+    typedef double ValueType;
+    FetchFloat(const IAttributeVector & attr_in) : attr(&attr_in) {}
+    ValueType get(uint32_t docid) const { return attr->getFloat(docid); }
+};
+
+template <typename Fetcher>
+class DiversityFilterT final : public DiversityFilter {
+private:
+    size_t  _total_count;
+    Fetcher _diversity;
+    size_t  _max_per_group;
+    size_t  _cutoff_max_groups;
+    bool    _cutoff_strict;
+
+    typedef vespalib::hash_map<typename Fetcher::ValueType, uint32_t> Diversity;
+    Diversity _seen;
+public:
+    DiversityFilterT(Fetcher diversity, size_t max_per_group, size_t cutoff_max_groups,
+                     bool cutoff_strict, size_t max_total)
+        : DiversityFilter(max_total), _total_count(0), _diversity(diversity), _max_per_group(max_per_group),
+          _cutoff_max_groups(cutoff_max_groups), _cutoff_strict(cutoff_strict),
+          _seen(std::min(cutoff_max_groups, 10000ul)*3)
+    { }
+
+    bool accepted(uint32_t docId) override;
+private:
+    bool add() {
+        ++_total_count;
+        return true;
+    }
+    bool conditional_add(uint32_t & group_count) {
+        if (group_count  < _max_per_group) {
+            ++group_count;
+            add();
+            return true;
+        }
+        return false;
+    }
+};
+
+template <typename Fetcher>
+bool
+DiversityFilterT<Fetcher>::accepted(uint32_t docId) {
+    if (_total_count < _max_total) {
+        if ((_seen.size() < _cutoff_max_groups) || _cutoff_strict) {
+            typename Fetcher::ValueType group = _diversity.get(docId);
+            if (_seen.size() < _cutoff_max_groups) {
+                return conditional_add(_seen[group]);
+            } else {
+                auto found = _seen.find(group);
+                return (found == _seen.end()) ? add() : conditional_add(found->second);
+            }
+        } else if ( !_cutoff_strict) {
+            return add();
+        }
+    }
+    return false;
 }
+
+std::unique_ptr<DiversityFilter>
+DiversityFilter::create(const IAttributeVector &diversity_attr, size_t wanted_hits,
+                        size_t max_per_group,size_t cutoff_max_groups, bool cutoff_strict)
+{
+    if (diversity_attr.hasEnum()) { // must handle enum first
+        FetchEnumFast fastEnum(diversity_attr);
+        if (fastEnum.valid()) {
+            return make_unique<DiversityFilterT<FetchEnumFast>> (fastEnum, max_per_group, cutoff_max_groups, cutoff_strict, wanted_hits);
+        } else {
+            return make_unique<DiversityFilterT<FetchEnum>>(FetchEnum(diversity_attr), max_per_group, cutoff_max_groups, cutoff_strict, wanted_hits);
+        }
+    } else if (diversity_attr.isIntegerType()) {
+        using FetchInt32Fast = FetchNumberFast<SingleValueNumericAttribute<IntegerAttributeTemplate<int32_t> > >;
+        using FetchInt64Fast = FetchNumberFast<SingleValueNumericAttribute<IntegerAttributeTemplate<int64_t> > >;
+
+        FetchInt32Fast fastInt32(diversity_attr);
+        FetchInt64Fast fastInt64(diversity_attr);
+        if (fastInt32.valid()) {
+            return make_unique<DiversityFilterT<FetchInt32Fast>>(fastInt32, max_per_group, cutoff_max_groups, cutoff_strict, wanted_hits);
+        } else if (fastInt64.valid()) {
+            return make_unique<DiversityFilterT<FetchInt64Fast>>(fastInt64, max_per_group, cutoff_max_groups, cutoff_strict, wanted_hits);
+        } else {
+            return make_unique<DiversityFilterT<FetchInteger>>(FetchInteger(diversity_attr), max_per_group, cutoff_max_groups, cutoff_strict, wanted_hits);
+        }
+    } else if (diversity_attr.isFloatingPointType()) {
+        using FetchFloatFast = FetchNumberFast<SingleValueNumericAttribute<FloatingPointAttributeTemplate<float> > >;
+        using FetchDoubleFast = FetchNumberFast<SingleValueNumericAttribute<FloatingPointAttributeTemplate<double> > >;
+        FetchFloatFast fastFloat(diversity_attr);
+        FetchDoubleFast fastDouble(diversity_attr);
+        if (fastFloat.valid()) {
+            return make_unique<DiversityFilterT<FetchFloatFast>>(fastFloat, max_per_group, cutoff_max_groups, cutoff_strict, wanted_hits);
+        } else if (fastDouble.valid()) {
+            return make_unique<DiversityFilterT<FetchDoubleFast>>(fastDouble, max_per_group, cutoff_max_groups, cutoff_strict, wanted_hits);
+        } else {
+            return make_unique<DiversityFilterT<FetchFloat>>(FetchFloat(diversity_attr), max_per_group, cutoff_max_groups, cutoff_strict, wanted_hits);
+        }
+    }
+    return std::unique_ptr<DiversityFilter>();
 }
+
 }
diff --git a/searchlib/src/vespa/searchlib/attribute/diversity.h b/searchlib/src/vespa/searchlib/attribute/diversity.h
index dff658d99d7..da5df19f9e8 100644
--- a/searchlib/src/vespa/searchlib/attribute/diversity.h
+++ b/searchlib/src/vespa/searchlib/attribute/diversity.h
@@ -2,9 +2,8 @@
 
 #pragma once
 
-#include "singleenumattribute.h"
-#include "singlenumericattribute.h"
-#include <vespa/vespalib/stllike/hash_map.h>
+#include <vespa/searchcommon/attribute/iattributevector.h>
+#include <vespa/searchlib/queryeval/idiversifier.h>
 
 /**
  * This file contains low-level code used to implement diversified
@@ -14,9 +13,7 @@
  * diversified results based on a secondary attribute.
  **/
 
-namespace search {
-namespace attribute {
-namespace diversity {
+namespace search::attribute::diversity {
 
 template <typename ITR>
 class ForwardRange
@@ -61,172 +58,68 @@ public:
     bool has_next() const { return _lower != _upper; }
 };
 
-template <typename T>
-struct FetchNumberFast {
-    const T * const attr;
-    typedef typename T::LoadedValueType ValueType;
-    FetchNumberFast(const IAttributeVector &attr_in) : attr(dynamic_cast<const T *>(&attr_in)) {}
-    ValueType get(uint32_t docid) const { return attr->getFast(docid); }
-    bool valid() const { return (attr != nullptr); }
-};
-
-struct FetchEnumFast {
-    const SingleValueEnumAttributeBase * const attr;
-    typedef uint32_t ValueType;
-    FetchEnumFast(const IAttributeVector &attr_in) : attr(dynamic_cast<const SingleValueEnumAttributeBase *>(&attr_in)) {}
-    ValueType get(uint32_t docid) const { return attr->getE(docid); }
-    bool valid() const { return (attr != nullptr); }
-};
-
-struct FetchEnum {
-    const IAttributeVector &attr;
-    typedef uint32_t ValueType;
-    FetchEnum(const IAttributeVector &attr_in) : attr(attr_in) {}
-    ValueType get(uint32_t docid) const { return attr.getEnum(docid); }
-};
-
-struct FetchInteger {
-    const IAttributeVector &attr;
-    typedef int64_t ValueType;
-    FetchInteger(const IAttributeVector &attr_in) : attr(attr_in) {}
-    ValueType get(uint32_t docid) const { return attr.getInt(docid); }
-};
-
-struct FetchFloat {
-    const IAttributeVector &attr;
-    typedef double ValueType;
-    FetchFloat(const IAttributeVector &attr_in) : attr(attr_in) {}
-    ValueType get(uint32_t docid) const { return attr.getFloat(docid); }
+class DiversityFilter : public queryeval::IDiversifier {
+public:
+    DiversityFilter(size_t max_total) : _max_total(max_total) {}
+    size_t getMaxTotal() const { return _max_total; }
+    static std::unique_ptr<DiversityFilter>
+    create(const IAttributeVector &diversity_attr, size_t wanted_hits,
+           size_t max_per_group,size_t cutoff_max_groups, bool cutoff_strict);
+protected:
+    size_t _max_total;
 };
 
-template <typename Fetcher, typename Result>
-class DiversityFilter {
+template <typename Result>
+class DiversityRecorder {
 private:
-    size_t _total_count;
-    size_t _max_total;
-    const Fetcher &_diversity;
-    size_t _max_per_group;
-    size_t _cutoff_max_groups;
-    bool   _cutoff_strict;
-
-    typedef vespalib::hash_map<typename Fetcher::ValueType, uint32_t> Diversity;
-    Diversity _seen;
+    DiversityFilter & _filter;
     Result &_result;
 public:
-    DiversityFilter(const Fetcher &diversity, size_t max_per_group,
-                    size_t cutoff_max_groups, bool cutoff_strict,
-                    Result &result, size_t max_total)
-        : _total_count(0), _max_total(max_total), _diversity(diversity), _max_per_group(max_per_group),
-          _cutoff_max_groups(cutoff_max_groups), _cutoff_strict(cutoff_strict),
-          _seen(std::min(cutoff_max_groups, 10000ul)*3), _result(result)
+    DiversityRecorder(DiversityFilter & filter, Result &result)
+        : _filter(filter), _result(result)
     { }
+
     template <typename Item>
     void push_back(Item item) {
-        if (_total_count < _max_total) {
-            if ((_seen.size() < _cutoff_max_groups) || _cutoff_strict) {
-                typename Fetcher::ValueType group = _diversity.get(item._key);
-                if (_seen.size() < _cutoff_max_groups) {
-                    conditional_add(_seen[group], item);
-                } else {
-                    auto found = _seen.find(group);
-                    if (found == _seen.end()) {
-                        add(item);
-                    } else {
-                        conditional_add(found->second, item);
-                    }
-                }
-            } else if ( !_cutoff_strict) {
-                add(item);
-            }
-        }
-    }
-private:
-    template <typename Item>
-    void add(Item item) {
-        ++_total_count;
-        _result.push_back(item);
-    }
-    template <typename Item>
-    void conditional_add(uint32_t & group_count, Item item) {
-        if (group_count  < _max_per_group) {
-            ++group_count;
-            add(item);
+        if (_filter.accepted(item._key)) {
+            _result.push_back(item);
         }
     }
+
 };
 
-template <typename DictRange, typename PostingStore, typename Fetcher, typename Result>
-void diversify_3(const DictRange &range_in, const PostingStore &posting, size_t wanted_hits,
-                 const Fetcher &diversity, size_t max_per_group,
-                 size_t cutoff_max_groups, bool cutoff_strict,
+template <typename DictRange, typename PostingStore, typename Result>
+void diversify_2(const DictRange &range_in, const PostingStore &posting, DiversityFilter & filter,
                  Result &result, std::vector<size_t> &fragments)
 {
+
+    DiversityRecorder<Result> recorder(filter, result);
     DictRange range(range_in);
     using DataType = typename PostingStore::DataType;
     using KeyDataType = typename PostingStore::KeyDataType;
-    DiversityFilter<Fetcher, Result> filter(diversity, max_per_group, cutoff_max_groups, cutoff_strict, result, wanted_hits);
-    while (range.has_next() && (result.size() < wanted_hits)) {
+    while (range.has_next() && (result.size() < filter.getMaxTotal())) {
         typename DictRange::Next dict_entry(range);
         posting.foreach_frozen(dict_entry.get().getData(),
                                [&](uint32_t key, const DataType &data)
-                               { filter.push_back(KeyDataType(key, data)); });
+                               { recorder.push_back(KeyDataType(key, data)); });
         if (fragments.back() < result.size()) {
             fragments.push_back(result.size());
         }
     }
 }
 
-template <typename DictRange, typename PostingStore, typename Result>
-void diversify_2(const DictRange &range_in, const PostingStore &posting, size_t wanted_hits,
-                 const IAttributeVector &diversity_attr, size_t max_per_group,
-                 size_t cutoff_max_groups, bool cutoff_strict,
-                 Result &result, std::vector<size_t> &fragments)
-{
-    if (diversity_attr.hasEnum()) { // must handle enum first
-        FetchEnumFast fastEnum(diversity_attr);
-        if (fastEnum.valid()) {
-            diversify_3(range_in, posting, wanted_hits, fastEnum, max_per_group, cutoff_max_groups, cutoff_strict, result, fragments);
-        } else {
-            diversify_3(range_in, posting, wanted_hits, FetchEnum(diversity_attr), max_per_group, cutoff_max_groups, cutoff_strict, result, fragments);
-        }
-    } else if (diversity_attr.isIntegerType()) {
-        FetchNumberFast<SingleValueNumericAttribute<IntegerAttributeTemplate<int32_t> > > fastInt32(diversity_attr);
-        FetchNumberFast<SingleValueNumericAttribute<IntegerAttributeTemplate<int64_t> > > fastInt64(diversity_attr);
-        if (fastInt32.valid()) {
-            diversify_3(range_in, posting, wanted_hits, fastInt32, max_per_group, cutoff_max_groups, cutoff_strict, result, fragments);
-        } else if (fastInt64.valid()) {
-            diversify_3(range_in, posting, wanted_hits, fastInt64, max_per_group, cutoff_max_groups, cutoff_strict, result, fragments);
-        } else {
-            diversify_3(range_in, posting, wanted_hits, FetchInteger(diversity_attr), max_per_group, cutoff_max_groups, cutoff_strict, result, fragments);
-        }
-    } else if (diversity_attr.isFloatingPointType()) {
-        FetchNumberFast<SingleValueNumericAttribute<FloatingPointAttributeTemplate<float> > > fastFloat(diversity_attr);
-        FetchNumberFast<SingleValueNumericAttribute<FloatingPointAttributeTemplate<double> > > fastDouble(diversity_attr);
-        if (fastFloat.valid()) {
-            diversify_3(range_in, posting, wanted_hits, fastFloat, max_per_group, cutoff_max_groups, cutoff_strict, result, fragments);
-        } else if (fastDouble.valid()) {
-            diversify_3(range_in, posting, wanted_hits, fastDouble, max_per_group, cutoff_max_groups, cutoff_strict, result, fragments);
-        } else {
-            diversify_3(range_in, posting, wanted_hits, FetchFloat(diversity_attr), max_per_group, cutoff_max_groups, cutoff_strict, result, fragments);
-        }
-    }
-}
-
 template <typename DictItr, typename PostingStore, typename Result>
 void diversify(bool forward, const DictItr &lower, const DictItr &upper, const PostingStore &posting, size_t wanted_hits,
                const IAttributeVector &diversity_attr, size_t max_per_group,
                size_t cutoff_max_groups, bool cutoff_strict,
                Result &array, std::vector<size_t> &fragments)
 {
+    auto filter = DiversityFilter::create(diversity_attr, wanted_hits, max_per_group, cutoff_max_groups, cutoff_strict);
     if (forward) {
-        diversify_2(ForwardRange<DictItr>(lower, upper), posting, wanted_hits,
-                    diversity_attr, max_per_group, cutoff_max_groups, cutoff_strict, array, fragments);
+        diversify_2(ForwardRange<DictItr>(lower, upper), posting, *filter, array, fragments);
     } else {
-        diversify_2(ReverseRange<DictItr>(lower, upper), posting, wanted_hits,
-                    diversity_attr, max_per_group, cutoff_max_groups, cutoff_strict, array, fragments);
+        diversify_2(ReverseRange<DictItr>(lower, upper), posting, *filter, array, fragments);
     }
 }
 
-} // namespace search::attribute::diversity
-} // namespace search::attribute
-} // namespace search
+}
diff --git a/searchlib/src/vespa/searchlib/attribute/diversity.hpp b/searchlib/src/vespa/searchlib/attribute/diversity.hpp
index 9261bb482f9..698f482dec1 100644
--- a/searchlib/src/vespa/searchlib/attribute/diversity.hpp
+++ b/searchlib/src/vespa/searchlib/attribute/diversity.hpp
@@ -4,9 +4,7 @@
 
 #include "diversity.h"
 
-namespace search {
-namespace attribute {
-namespace diversity {
+namespace search::attribute::diversity {
 
 template <typename ITR>
 ForwardRange<ITR>::ForwardRange(const ForwardRange &) = default;
@@ -18,7 +16,7 @@ ForwardRange<ITR>::ForwardRange(const ITR &lower, const ITR &upper)
 {}
 
 template <typename ITR>
-ForwardRange<ITR>::~ForwardRange() { }
+ForwardRange<ITR>::~ForwardRange() = default;
 
 template <typename ITR>
 ReverseRange<ITR>::ReverseRange(const ReverseRange &) = default;
@@ -31,8 +29,6 @@ ReverseRange<ITR>::ReverseRange(const ITR &lower, const ITR &upper)
 
 
 template <typename ITR>
-ReverseRange<ITR>::~ReverseRange() { }
+ReverseRange<ITR>::~ReverseRange() = default;
 
 }
-}
-}
diff --git a/searchlib/src/vespa/searchlib/attribute/enumstore.h b/searchlib/src/vespa/searchlib/attribute/enumstore.h
index c1f5d278827..3206ea62d73 100644
--- a/searchlib/src/vespa/searchlib/attribute/enumstore.h
+++ b/searchlib/src/vespa/searchlib/attribute/enumstore.h
@@ -166,6 +166,7 @@ public:
     ssize_t deserialize(const void *src, size_t available, Index &idx) override;
     bool foldedChange(const Index &idx1, const Index &idx2) override;
     virtual bool findEnum(Type value, EnumStoreBase::EnumHandle &e) const;
+    virtual std::vector<EnumStoreBase::EnumHandle> findFoldedEnums(Type value) const;
     void addEnum(Type value, Index &newIdx);
     virtual bool findIndex(Type value, Index &idx) const;
     void freeUnusedEnums(bool movePostingidx) override;
diff --git a/searchlib/src/vespa/searchlib/attribute/enumstore.hpp b/searchlib/src/vespa/searchlib/attribute/enumstore.hpp
index 54100bafa1e..794138c7c89 100644
--- a/searchlib/src/vespa/searchlib/attribute/enumstore.hpp
+++ b/searchlib/src/vespa/searchlib/attribute/enumstore.hpp
@@ -200,6 +200,15 @@ EnumStoreT<EntryType>::findEnum(Type value,
 }
 
 template <typename EntryType>
+std::vector<EnumStoreBase::EnumHandle>
+EnumStoreT<EntryType>::findFoldedEnums(Type value) const
+{
+    FoldedComparatorType cmp(*this, value);
+    return _enumDict->findMatchingEnums(cmp);
+}
+
+
+template <typename EntryType>
 bool
 EnumStoreT<EntryType>::findIndex(Type value, Index &idx) const
 {
diff --git a/searchlib/src/vespa/searchlib/attribute/enumstorebase.cpp b/searchlib/src/vespa/searchlib/attribute/enumstorebase.cpp
index 914a56e7abc..61d862b6c4f 100644
--- a/searchlib/src/vespa/searchlib/attribute/enumstorebase.cpp
+++ b/searchlib/src/vespa/searchlib/attribute/enumstorebase.cpp
@@ -507,6 +507,20 @@ EnumStoreDict<Dictionary>::findFrozenIndex(const EnumStoreComparator &cmp,
 
 
 template <typename Dictionary>
+std::vector<EnumStoreBase::EnumHandle>
+EnumStoreDict<Dictionary>::findMatchingEnums(const EnumStoreComparator &cmp) const
+{
+    std::vector<EnumStoreBase::EnumHandle> result;
+    typename Dictionary::ConstIterator itr =
+        _dict.getFrozenView().find(Index(), cmp);
+    while (itr.valid() && !cmp(Index(), itr.getKey())) {
+        result.push_back(itr.getKey().ref());
+        ++itr;
+    }
+    return result;
+}
+
+template <typename Dictionary>
 void
 EnumStoreDict<Dictionary>::onReset()
 {
diff --git a/searchlib/src/vespa/searchlib/attribute/enumstorebase.h b/searchlib/src/vespa/searchlib/attribute/enumstorebase.h
index f74345a8806..9bea2a568e1 100644
--- a/searchlib/src/vespa/searchlib/attribute/enumstorebase.h
+++ b/searchlib/src/vespa/searchlib/attribute/enumstorebase.h
@@ -78,6 +78,9 @@ public:
                                  const EnumStoreComparator *fcmp) = 0;
     virtual bool findIndex(const EnumStoreComparator &cmp, Index &idx) const = 0;
     virtual bool findFrozenIndex(const EnumStoreComparator &cmp, Index &idx) const = 0;
+    virtual std::vector<attribute::IAttributeVector::EnumHandle>
+    findMatchingEnums(const EnumStoreComparator &cmp) const = 0;
+
     virtual void onReset() = 0;
     virtual void onTransferHoldLists(generation_t generation) = 0;
     virtual void onTrimHoldLists(generation_t firstUsed) = 0;
@@ -131,6 +134,9 @@ public:
 
     bool findIndex(const EnumStoreComparator &cmp, Index &idx) const override;
     bool findFrozenIndex(const EnumStoreComparator &cmp, Index &idx) const override;
+    std::vector<attribute::IAttributeVector::EnumHandle>
+    findMatchingEnums(const EnumStoreComparator &cmp) const override;
+
     void onReset() override;
     void onTransferHoldLists(generation_t generation) override;
     void onTrimHoldLists(generation_t firstUsed) override;
diff --git a/searchlib/src/vespa/searchlib/attribute/floatbase.h b/searchlib/src/vespa/searchlib/attribute/floatbase.h
index df156fd9fc5..955b2b252af 100644
--- a/searchlib/src/vespa/searchlib/attribute/floatbase.h
+++ b/searchlib/src/vespa/searchlib/attribute/floatbase.h
@@ -81,6 +81,7 @@ protected:
     Change _defaultValue;
 private:
     bool findEnum(const char *value, EnumHandle &e) const override;
+    std::vector<EnumHandle> findFoldedEnums(const char *value) const override;
     bool isUndefined(DocId doc) const override;
     virtual T get(DocId doc) const = 0;
     virtual T getFromEnum(EnumHandle e) const = 0;
diff --git a/searchlib/src/vespa/searchlib/attribute/floatbase.hpp b/searchlib/src/vespa/searchlib/attribute/floatbase.hpp
index 669c7974e6f..a454fd3e235 100644
--- a/searchlib/src/vespa/searchlib/attribute/floatbase.hpp
+++ b/searchlib/src/vespa/searchlib/attribute/floatbase.hpp
@@ -48,6 +48,18 @@ FloatingPointAttributeTemplate<T>::findEnum(const char *value, EnumHandle &e) co
 }
 
 template<typename T>
+std::vector<EnumStoreBase::EnumHandle>
+FloatingPointAttributeTemplate<T>::findFoldedEnums(const char *value) const
+{
+    std::vector<EnumHandle> result;
+    EnumHandle h;
+    if (findEnum(value, h)) {
+        result.push_back(h);
+    }
+    return result;
+}
+
+template<typename T>
 bool
 FloatingPointAttributeTemplate<T>::isUndefined(DocId doc) const {
     return attribute::isUndefined(get(doc));
diff --git a/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.cpp b/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.cpp
index e6dacde3600..428b14671cd 100644
--- a/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.cpp
+++ b/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.cpp
@@ -99,6 +99,11 @@ bool ImportedAttributeVectorReadGuard::findEnum(const char *value, EnumHandle &e
     return _target_attribute.findEnum(value, e);
 }
 
+std::vector<ImportedAttributeVectorReadGuard::EnumHandle>
+ImportedAttributeVectorReadGuard::findFoldedEnums(const char *value) const {
+    return _target_attribute.findFoldedEnums(value);
+}
+
 const char * ImportedAttributeVectorReadGuard::getStringFromEnum(EnumHandle e) const {
     return _target_attribute.getStringFromEnum(e);
 }
diff --git a/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.h b/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.h
index 2622f091cf1..4cf4d5b64c1 100644
--- a/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.h
+++ b/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.h
@@ -67,6 +67,8 @@ public:
     virtual uint32_t get(DocId docId, WeightedConstChar *buffer, uint32_t sz) const override;
     virtual uint32_t get(DocId docId, WeightedEnum *buffer, uint32_t sz) const override;
     virtual bool findEnum(const char * value, EnumHandle & e) const override;
+    virtual std::vector<EnumHandle> findFoldedEnums(const char *value) const override;
+
     virtual const char * getStringFromEnum(EnumHandle e) const override;
     virtual std::unique_ptr<ISearchContext> createSearchContext(std::unique_ptr<QueryTermSimple> term,
                                                                 const SearchContextParams &params) const override;
diff --git a/searchlib/src/vespa/searchlib/attribute/imported_search_context.cpp b/searchlib/src/vespa/searchlib/attribute/imported_search_context.cpp
index f769e2a6855..44248c97abe 100644
--- a/searchlib/src/vespa/searchlib/attribute/imported_search_context.cpp
+++ b/searchlib/src/vespa/searchlib/attribute/imported_search_context.cpp
@@ -249,7 +249,7 @@ Int64Range ImportedSearchContext::getAsIntegerTerm() const {
     return _target_search_context->getAsIntegerTerm();
 }
 
-const QueryTermBase& ImportedSearchContext::queryTerm() const {
+const QueryTermBase * ImportedSearchContext::queryTerm() const {
     return _target_search_context->queryTerm();
 }
 
diff --git a/searchlib/src/vespa/searchlib/attribute/imported_search_context.h b/searchlib/src/vespa/searchlib/attribute/imported_search_context.h
index cfb5371dbb9..ae6ce181af0 100644
--- a/searchlib/src/vespa/searchlib/attribute/imported_search_context.h
+++ b/searchlib/src/vespa/searchlib/attribute/imported_search_context.h
@@ -59,7 +59,7 @@ public:
     void fetchPostings(bool strict) override;
     bool valid() const override;
     Int64Range getAsIntegerTerm() const override;
-    const QueryTermBase& queryTerm() const override;
+    const QueryTermBase * queryTerm() const override;
     const vespalib::string& attributeName() const override;
 
     using DocId = IAttributeVector::DocId;
diff --git a/searchlib/src/vespa/searchlib/attribute/integerbase.h b/searchlib/src/vespa/searchlib/attribute/integerbase.h
index 8d6c5046070..5dec40fd4da 100644
--- a/searchlib/src/vespa/searchlib/attribute/integerbase.h
+++ b/searchlib/src/vespa/searchlib/attribute/integerbase.h
@@ -98,6 +98,8 @@ protected:
     Change _defaultValue;
 private:
     bool findEnum(const char *value, EnumHandle &e) const override;
+    std::vector<EnumHandle> findFoldedEnums(const char *value) const override;
+
     virtual T get(DocId doc) const = 0;
     virtual T getFromEnum(EnumHandle e) const = 0;
     largeint_t getIntFromEnum(EnumHandle e) const override;
diff --git a/searchlib/src/vespa/searchlib/attribute/integerbase.hpp b/searchlib/src/vespa/searchlib/attribute/integerbase.hpp
index 3866e5a4acb..7b23cd51e92 100644
--- a/searchlib/src/vespa/searchlib/attribute/integerbase.hpp
+++ b/searchlib/src/vespa/searchlib/attribute/integerbase.hpp
@@ -32,6 +32,19 @@ IntegerAttributeTemplate<T>::findEnum(const char *value, EnumHandle &e) const {
     return findEnum(ivalue, e);
 }
 
+
+template<typename T>
+std::vector<EnumStoreBase::EnumHandle>
+IntegerAttributeTemplate<T>::findFoldedEnums(const char *value) const
+{
+    std::vector<EnumHandle> result;
+    EnumHandle h;
+    if (findEnum(value, h)) {
+        result.push_back(h);
+    }
+    return result;
+}
+
 template<typename T>
 largeint_t
 IntegerAttributeTemplate<T>::getIntFromEnum(EnumHandle e) const {
diff --git a/searchlib/src/vespa/searchlib/attribute/multistringattribute.h b/searchlib/src/vespa/searchlib/attribute/multistringattribute.h
index 5e5d33419aa..eb6a8b630de 100644
--- a/searchlib/src/vespa/searchlib/attribute/multistringattribute.h
+++ b/searchlib/src/vespa/searchlib/attribute/multistringattribute.h
@@ -71,6 +71,11 @@ public:
             return this->_enumStore.getValue(indices[0].value());
         }
     }
+
+    std::vector<EnumHandle> findFoldedEnums(const char *value) const override {
+        return this->_enumStore.findFoldedEnums(value);
+    }
+
     const char * getStringFromEnum(EnumHandle e) const override {
         return this->_enumStore.getValue(e);
     }
diff --git a/searchlib/src/vespa/searchlib/attribute/multistringattribute.hpp b/searchlib/src/vespa/searchlib/attribute/multistringattribute.hpp
index 858fe579764..6c459465b51 100644
--- a/searchlib/src/vespa/searchlib/attribute/multistringattribute.hpp
+++ b/searchlib/src/vespa/searchlib/attribute/multistringattribute.hpp
@@ -119,14 +119,14 @@ StringTemplSearchContext(QueryTermSimpleUP qTerm, const AttrType & toBeSearched)
     this->_plsc = static_cast<attribute::IPostingListSearchContext *>(this);
     if (this->valid()) {
         if (this->isPrefix()) {
-            FoldedComparatorType comp(enumStore, queryTerm().getTerm(), true);
+            FoldedComparatorType comp(enumStore, queryTerm()->getTerm(), true);
             lookupRange(comp, comp);
         } else if (this->isRegex()) {
-            vespalib::string prefix(vespalib::Regexp::get_prefix(this->queryTerm().getTerm()));
+            vespalib::string prefix(vespalib::Regexp::get_prefix(this->queryTerm()->getTerm()));
             FoldedComparatorType comp(enumStore, prefix.c_str(), true);
             lookupRange(comp, comp);
         } else {
-            FoldedComparatorType comp(enumStore, queryTerm().getTerm());
+            FoldedComparatorType comp(enumStore, queryTerm()->getTerm());
             lookupTerm(comp);
         }
     }
diff --git a/searchlib/src/vespa/searchlib/attribute/not_implemented_attribute.cpp b/searchlib/src/vespa/searchlib/attribute/not_implemented_attribute.cpp
index 5e6cfa8a847..1dc95c42de8 100644
--- a/searchlib/src/vespa/searchlib/attribute/not_implemented_attribute.cpp
+++ b/searchlib/src/vespa/searchlib/attribute/not_implemented_attribute.cpp
@@ -103,6 +103,12 @@ NotImplementedAttribute::findEnum(const char *, EnumHandle &) const {
     return false;
 }
 
+std::vector<NotImplementedAttribute::EnumHandle>
+NotImplementedAttribute::findFoldedEnums(const char *) const {
+    notImplemented();
+    return std::vector<EnumHandle>();
+}
+
 long
 NotImplementedAttribute::onSerializeForAscendingSort(DocId, void *, long, const common::BlobConverter *) const {
     notImplemented();
diff --git a/searchlib/src/vespa/searchlib/attribute/not_implemented_attribute.h b/searchlib/src/vespa/searchlib/attribute/not_implemented_attribute.h
index 98935c1f155..cbd2ff162b2 100644
--- a/searchlib/src/vespa/searchlib/attribute/not_implemented_attribute.h
+++ b/searchlib/src/vespa/searchlib/attribute/not_implemented_attribute.h
@@ -26,6 +26,8 @@ struct NotImplementedAttribute : AttributeVector {
     uint32_t get(DocId, WeightedConstChar *, uint32_t) const override;
     uint32_t get(DocId, WeightedEnum *, uint32_t) const override;
     bool findEnum(const char *, EnumHandle &) const override;
+    std::vector<EnumHandle> findFoldedEnums(const char *value) const override;
+
     long onSerializeForAscendingSort(DocId, void *, long, const common::BlobConverter *) const override;
     long onSerializeForDescendingSort(DocId, void *, long, const common::BlobConverter *) const override;
     uint32_t clearDoc(DocId) override;
diff --git a/searchlib/src/vespa/searchlib/attribute/posting_list_merger.cpp b/searchlib/src/vespa/searchlib/attribute/posting_list_merger.cpp
index d022e806b91..39ca733eae7 100644
--- a/searchlib/src/vespa/searchlib/attribute/posting_list_merger.cpp
+++ b/searchlib/src/vespa/searchlib/attribute/posting_list_merger.cpp
@@ -15,9 +15,7 @@ PostingListMerger<DataT>::PostingListMerger(uint32_t docIdLimit)
 }
 
 template <typename DataT>
-PostingListMerger<DataT>::~PostingListMerger()
-{
-}
+PostingListMerger<DataT>::~PostingListMerger() = default;
 
 template <typename DataT>
 void
diff --git a/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.h b/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.h
index 07e1d0c2a36..6a2324cef07 100644
--- a/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.h
+++ b/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.h
@@ -283,14 +283,14 @@ StringPostingSearchContext(QueryTermSimpleUP qTerm, bool useBitVector, const Att
 
     if (this->valid()) {
         if (this->isPrefix()) {
-            FoldedComparatorType comp(_enumStore, this->queryTerm().getTerm(), true);
+            FoldedComparatorType comp(_enumStore, this->queryTerm()->getTerm(), true);
             this->lookupRange(comp, comp);
         } else if (this->isRegex()) {
-            vespalib::string prefix(Regexp::get_prefix(this->queryTerm().getTerm()));
+            vespalib::string prefix(Regexp::get_prefix(this->queryTerm()->getTerm()));
             FoldedComparatorType comp(_enumStore, prefix.c_str(), true);
             this->lookupRange(comp, comp);
         } else {
-            FoldedComparatorType comp(_enumStore, this->queryTerm().getTerm());
+            FoldedComparatorType comp(_enumStore, this->queryTerm()->getTerm());
             this->lookupTerm(comp);
         }
         if (this->_uniqueValues == 1u) {
diff --git a/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.hpp b/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.hpp
index cfad425d7ea..cf9450cbcca 100644
--- a/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.hpp
+++ b/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.hpp
@@ -30,7 +30,7 @@ PostingListSearchContextT(const Dictionary &dictionary, uint32_t docIdLimit, uin
 }
 
 template <typename DataT>
-PostingListSearchContextT<DataT>::~PostingListSearchContextT() {}
+PostingListSearchContextT<DataT>::~PostingListSearchContextT() = default;
 
 
 template <typename DataT>
@@ -154,7 +154,7 @@ createPostingIterator(fef::TermFieldMatchData *matchData, bool strict)
 {
     assert(_fetchPostingsDone);
     if (_uniqueValues == 0u) {
-        return SearchIterator::UP(new EmptySearch());
+        return std::make_unique<EmptySearch>();
     }
     if (_merger.hasArray() || _merger.hasBitVector()) { // synthetic results are available
         if (!_merger.emptyArray()) {
@@ -170,7 +170,7 @@ createPostingIterator(fef::TermFieldMatchData *matchData, bool strict)
             }
         }
         if (_merger.hasArray()) {
-            return SearchIterator::UP(new EmptySearch());
+            return std::make_unique<EmptySearch>();
         }
         const BitVector *bv(_merger.getBitVector());
         assert(bv != nullptr);
@@ -181,7 +181,7 @@ createPostingIterator(fef::TermFieldMatchData *matchData, bool strict)
             return BitVectorIterator::create(_gbv, std::min(_gbv->size(), _docIdLimit), *matchData, strict);
         }
         if (!_pidx.valid()) {
-            return SearchIterator::UP(new EmptySearch());
+            return std::make_unique<EmptySearch>();
         }
         const PostingList &postingList = _postingList;
         if (!_frozenRoot.valid()) {
diff --git a/searchlib/src/vespa/searchlib/attribute/singlestringattribute.h b/searchlib/src/vespa/searchlib/attribute/singlestringattribute.h
index 4993b295b37..8d2efcbdc09 100644
--- a/searchlib/src/vespa/searchlib/attribute/singlestringattribute.h
+++ b/searchlib/src/vespa/searchlib/attribute/singlestringattribute.h
@@ -52,6 +52,9 @@ public:
     const char * get(DocId doc) const override {
         return this->_enumStore.getValue(this->_enumIndices[doc]);
     }
+    std::vector<EnumHandle> findFoldedEnums(const char *value) const override {
+        return this->_enumStore.findFoldedEnums(value);
+    }
     const char * getStringFromEnum(EnumHandle e) const override {
         return this->_enumStore.getValue(e);
     }
diff --git a/searchlib/src/vespa/searchlib/attribute/singlestringattribute.hpp b/searchlib/src/vespa/searchlib/attribute/singlestringattribute.hpp
index 84c2ff277d6..4f028e1a478 100644
--- a/searchlib/src/vespa/searchlib/attribute/singlestringattribute.hpp
+++ b/searchlib/src/vespa/searchlib/attribute/singlestringattribute.hpp
@@ -55,14 +55,14 @@ SingleValueStringAttributeT<B>::StringTemplSearchContext::StringTemplSearchConte
     this->_plsc = static_cast<attribute::IPostingListSearchContext *>(this);
     if (this->valid()) {
         if (this->isPrefix()) {
-            FoldedComparatorType comp(enumStore, queryTerm().getTerm(), true);
+            FoldedComparatorType comp(enumStore, queryTerm()->getTerm(), true);
             lookupRange(comp, comp);
         } else if (this->isRegex()) {
-            vespalib::string prefix(vespalib::Regexp::get_prefix(this->queryTerm().getTerm()));
+            vespalib::string prefix(vespalib::Regexp::get_prefix(this->queryTerm()->getTerm()));
             FoldedComparatorType comp(enumStore, prefix.c_str(), true);
             lookupRange(comp, comp);
         } else {
-            FoldedComparatorType comp(enumStore, queryTerm().getTerm());
+            FoldedComparatorType comp(enumStore, queryTerm()->getTerm());
             lookupTerm(comp);
         }
     }
diff --git a/searchlib/src/vespa/searchlib/attribute/stringbase.cpp b/searchlib/src/vespa/searchlib/attribute/stringbase.cpp
index 16a05e5f0a9..04ca9d216a3 100644
--- a/searchlib/src/vespa/searchlib/attribute/stringbase.cpp
+++ b/searchlib/src/vespa/searchlib/attribute/stringbase.cpp
@@ -226,7 +226,7 @@ StringAttribute::StringSearchContext::StringSearchContext(QueryTermSimple::UP qT
     _isPrefix(qTerm->isPrefix()),
     _isRegex(qTerm->isRegex()),
     _queryTerm(std::move(qTerm)),
-    _termUCS4(queryTerm().getUCS4Term()),
+    _termUCS4(queryTerm()->getUCS4Term()),
     _bufferLen(toBeSearched.getMaxValueCount()),
     _buffer()
 {
@@ -247,9 +247,9 @@ StringAttribute::StringSearchContext::valid() const {
     return (_queryTerm.get() && (!_queryTerm->empty()));
 }
 
-const QueryTermBase &
+const QueryTermBase *
 StringAttribute::StringSearchContext::queryTerm() const {
-    return static_cast<const QueryTermBase &>(*_queryTerm);
+    return static_cast<const QueryTermBase *>(_queryTerm.get());
 }
 
 uint32_t StringAttribute::clearDoc(DocId doc)
diff --git a/searchlib/src/vespa/searchlib/attribute/stringbase.h b/searchlib/src/vespa/searchlib/attribute/stringbase.h
index c817332af15..593d8f048f9 100644
--- a/searchlib/src/vespa/searchlib/attribute/stringbase.h
+++ b/searchlib/src/vespa/searchlib/attribute/stringbase.h
@@ -44,6 +44,7 @@ public:
     bool apply(DocId doc, const ArithmeticValueUpdate & op);
     bool applyWeight(DocId doc, const FieldValue & fv, const ArithmeticValueUpdate & wAdjust) override;
     bool findEnum(const char * value, EnumHandle & e) const override = 0;
+    std::vector<EnumHandle> findFoldedEnums(const char *value) const override = 0;
     uint32_t get(DocId doc, largeint_t * v, uint32_t sz) const override;
     uint32_t get(DocId doc, double * v, uint32_t sz) const override;
     uint32_t get(DocId doc, WeightedInt * v, uint32_t sz) const override;
@@ -103,7 +104,7 @@ private:
     protected:
         bool valid() const override;
 
-        const QueryTermBase & queryTerm() const override;
+        const QueryTermBase * queryTerm() const override;
         bool isMatch(const char *src) const {
             if (__builtin_expect(isRegex(), false)) {
                 return getRegex()->match(src);
diff --git a/searchlib/src/vespa/searchlib/common/bitvector.h b/searchlib/src/vespa/searchlib/common/bitvector.h
index fa8ec5b5cdd..7405688f4f7 100644
--- a/searchlib/src/vespa/searchlib/common/bitvector.h
+++ b/searchlib/src/vespa/searchlib/common/bitvector.h
@@ -120,9 +120,13 @@ public:
     }
 
     void setSize(Index sz) {
-        clearBit(size());
+        setBit(sz);  // Need to place the new stop sign first
+        std::atomic_thread_fence(std::memory_order_release);
+        if (sz > _sz) {
+            // Can only remove the old stopsign if it is ahead of the new.
+            clearBit(_sz);
+        }
         _sz = sz;
-        setBit(size());
     }
     void setBit(Index idx) {
         _words[wordNum(idx)] |= mask(idx);
diff --git a/searchlib/src/vespa/searchlib/common/indexmetainfo.cpp b/searchlib/src/vespa/searchlib/common/indexmetainfo.cpp
index 10015bb658a..837c38eb340 100644
--- a/searchlib/src/vespa/searchlib/common/indexmetainfo.cpp
+++ b/searchlib/src/vespa/searchlib/common/indexmetainfo.cpp
@@ -1,6 +1,7 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include "indexmetainfo.h"
+#include <vespa/vespalib/io/fileutil.h>
 #include <vespa/vespalib/util/stringfmt.h>
 #include <vespa/vespalib/util/guard.h>
 #include <cassert>
@@ -310,6 +311,7 @@ IndexMetaInfo::save(const vespalib::string &baseName)
 {
     vespalib::string fileName = makeFileName(baseName);
     vespalib::string newName = fileName + ".new";
+    vespalib::unlink(newName);
     vespalib::FilePointer f(fopen(newName.c_str(), "w"));
     if (!f.valid()) {
         LOG(warning, "could not open file for writing: %s", newName.c_str());
@@ -350,6 +352,7 @@ IndexMetaInfo::save(const vespalib::string &baseName)
             newName.c_str(), fileName.c_str());
         return false;
     }
+    vespalib::File::sync(vespalib::dirname(fileName));
     return true;
 }
 
diff --git a/searchlib/src/vespa/searchlib/common/packets.cpp b/searchlib/src/vespa/searchlib/common/packets.cpp
index b6b0baea92a..90157e318e9 100644
--- a/searchlib/src/vespa/searchlib/common/packets.cpp
+++ b/searchlib/src/vespa/searchlib/common/packets.cpp
@@ -243,7 +243,7 @@ void FS4Properties::set(StringRef & e, const vespalib::stringref & s)
 {
     e.first = _backing.size();
     e.second = s.size();
-    _backing.append(s.c_str(), s.size());
+    _backing.append(s.data(), s.size());
 }
 
 void
diff --git a/searchlib/src/vespa/searchlib/diskindex/dictionarywordreader.cpp b/searchlib/src/vespa/searchlib/diskindex/dictionarywordreader.cpp
index e3c5853d74e..b820a77724b 100644
--- a/searchlib/src/vespa/searchlib/diskindex/dictionarywordreader.cpp
+++ b/searchlib/src/vespa/searchlib/diskindex/dictionarywordreader.cpp
@@ -28,8 +28,8 @@ DictionaryWordReader::~DictionaryWordReader()
 
 
 bool
-DictionaryWordReader::open(const vespalib::stringref &dictionaryName,
-                           const vespalib::stringref & wordMapName,
+DictionaryWordReader::open(const vespalib::string & dictionaryName,
+                           const vespalib::string & wordMapName,
                            const TuneFileSeqRead &tuneFileRead)
 {
     _old2newwordfile.reset(new Fast_BufferedFile(new FastOS_File));
diff --git a/searchlib/src/vespa/searchlib/diskindex/dictionarywordreader.h b/searchlib/src/vespa/searchlib/diskindex/dictionarywordreader.h
index 7204ba14d25..9436c17cdbc 100644
--- a/searchlib/src/vespa/searchlib/diskindex/dictionarywordreader.h
+++ b/searchlib/src/vespa/searchlib/diskindex/dictionarywordreader.h
@@ -106,8 +106,8 @@ public:
     }
 
     bool
-    open(const vespalib::stringref & dictionaryName,
-         const vespalib::stringref & wordMapName,
+    open(const vespalib::string & dictionaryName,
+         const vespalib::string & wordMapName,
          const TuneFileSeqRead &tuneFileRead);
 
     void
diff --git a/searchlib/src/vespa/searchlib/diskindex/fusion.cpp b/searchlib/src/vespa/searchlib/diskindex/fusion.cpp
index e61fe7bab17..2db6025d257 100644
--- a/searchlib/src/vespa/searchlib/diskindex/fusion.cpp
+++ b/searchlib/src/vespa/searchlib/diskindex/fusion.cpp
@@ -218,6 +218,7 @@ Fusion::mergeField(uint32_t id)
     }
     if (!FileKit::createStamp(indexDir +  "/.mergeocc_done"))
         return false;
+    vespalib::File::sync(indexDir);
 
     if (!CleanTmpDirs())
         return false;
diff --git a/searchlib/src/vespa/searchlib/diskindex/indexbuilder.cpp b/searchlib/src/vespa/searchlib/diskindex/indexbuilder.cpp
index 99eced5f97b..b744a68932d 100644
--- a/searchlib/src/vespa/searchlib/diskindex/indexbuilder.cpp
+++ b/searchlib/src/vespa/searchlib/diskindex/indexbuilder.cpp
@@ -296,7 +296,7 @@ FileHandle::open(const vespalib::stringref &dir,
                             index.getSchema(), index.getIndex(),
                             tuneFileWrite, fileHeaderContext)) {
         LOG(error, "Could not open term writer %s for write (%s)",
-            dir.c_str(), getLastErrorString().c_str());
+            vespalib::string(dir).c_str(), getLastErrorString().c_str());
         LOG_ABORT("should not be reached");
     }
 }
@@ -700,6 +700,7 @@ IndexBuilder::close()
     for (FieldHandle & fh : _fields) {
         if (fh.getValid()) {
             fh.close();
+            vespalib::File::sync(fh.getDir());
         }
     }
     if (!docsummary::DocumentSummary::writeDocIdLimit(_prefix, _docIdLimit)) {
diff --git a/searchlib/src/vespa/searchlib/docstore/chunkformat.cpp b/searchlib/src/vespa/searchlib/docstore/chunkformat.cpp
index 37381cfa3f6..93f1d9c0d51 100644
--- a/searchlib/src/vespa/searchlib/docstore/chunkformat.cpp
+++ b/searchlib/src/vespa/searchlib/docstore/chunkformat.cpp
@@ -13,7 +13,7 @@ using vespalib::compression::decompress;
 using vespalib::compression::computeMaxCompressedsize;
 using vespalib::compression::CompressionConfig;
 
-ChunkException::ChunkException(const vespalib::stringref & msg, const vespalib::stringref & location) :
+ChunkException::ChunkException(const vespalib::string & msg, const vespalib::stringref & location) :
     Exception(make_string("Illegal chunk: %s", msg.c_str()), location)
 {
 }
diff --git a/searchlib/src/vespa/searchlib/docstore/chunkformat.h b/searchlib/src/vespa/searchlib/docstore/chunkformat.h
index 9f9580f1f1d..81feaf2a27f 100644
--- a/searchlib/src/vespa/searchlib/docstore/chunkformat.h
+++ b/searchlib/src/vespa/searchlib/docstore/chunkformat.h
@@ -12,7 +12,7 @@ namespace search {
 class ChunkException : public vespalib::Exception
 {
 public:
-    ChunkException(const vespalib::stringref & msg, const vespalib::stringref & location);
+    ChunkException(const vespalib::string & msg, const vespalib::stringref & location);
 };
 
 // This is an interface for implementing a chunk format
diff --git a/searchlib/src/vespa/searchlib/docstore/documentstore.cpp b/searchlib/src/vespa/searchlib/docstore/documentstore.cpp
index 59e92b112fa..6e784bf8c30 100644
--- a/searchlib/src/vespa/searchlib/docstore/documentstore.cpp
+++ b/searchlib/src/vespa/searchlib/docstore/documentstore.cpp
@@ -35,8 +35,7 @@ void
 DocumentVisitorAdapter::visit(uint32_t lid, vespalib::ConstBufferRef buf) {
     if (buf.size() > 0) {
         vespalib::nbostream is(buf.c_str(), buf.size());
-        document::Document::UP doc(new document::Document(_repo, is));
-        _visitor.visit(lid, std::move(doc));
+        _visitor.visit(lid, std::make_unique<document::Document>(_repo, is));
     }
 }
 
@@ -51,34 +50,38 @@ public:
     using Alloc = vespalib::alloc::Alloc;
     typedef std::unique_ptr<Value> UP;
 
-    Value() : _compressedSize(0), _uncompressedSize(0), _compression(CompressionConfig::NONE) {}
-
-    Value(Value &&rhs) :
-            _compressedSize(rhs._compressedSize),
-            _uncompressedSize(rhs._uncompressedSize),
-            _compression(rhs._compression),
-            _buf(std::move(rhs._buf)) {}
-
-    Value(const Value &rhs) :
-            _compressedSize(rhs._compressedSize),
-            _uncompressedSize(rhs._uncompressedSize),
-            _compression(rhs._compression),
-            _buf(Alloc::alloc(rhs.size())) {
+    Value()
+        : _syncToken(0),
+          _compressedSize(0),
+          _uncompressedSize(0),
+          _compression(CompressionConfig::NONE)
+    {}
+
+    Value(uint64_t syncToken)
+        : _syncToken(syncToken),
+          _compressedSize(0),
+          _uncompressedSize(0),
+          _compression(CompressionConfig::NONE)
+    {}
+
+    Value(Value &&rhs) = default;
+    Value &operator=(Value &&rhs) = default;
+
+    Value(const Value &rhs)
+        : _syncToken(rhs._syncToken),
+          _compressedSize(rhs._compressedSize),
+          _uncompressedSize(rhs._uncompressedSize),
+          _compression(rhs._compression),
+          _buf(Alloc::alloc(rhs.size()))
+    {
         memcpy(get(), rhs.get(), size());
     }
 
-    Value &operator=(Value &&rhs) {
-        _buf = std::move(rhs._buf);
-        _compressedSize = rhs._compressedSize;
-        _uncompressedSize = rhs._uncompressedSize;
-        _compression = rhs._compression;
-        return *this;
-    }
-
     void setCompression(CompressionConfig::Type comp, size_t uncompressedSize) {
         _compression = comp;
         _uncompressedSize = uncompressedSize;
     }
+    uint64_t getSyncToken() const { return _syncToken; }
 
     CompressionConfig::Type getCompression() const { return _compression; }
 
@@ -96,7 +99,8 @@ public:
      * Decompress value into temporary buffer and deserialize document from
      * the temporary buffer.
      */
-    document::Document::UP deserializeDocument(const DocumentTypeRepo &repo);
+    document::Document::UP deserializeDocument(const DocumentTypeRepo &repo) const;
+    vespalib::DataBuffer decompressed() const;
 
     size_t size() const { return _compressedSize; }
     bool empty() const { return size() == 0; }
@@ -104,6 +108,7 @@ public:
     const void *get() const { return _buf.get(); }
     void *get() { return _buf.get(); }
 private:
+    uint64_t _syncToken;
     size_t _compressedSize;
     size_t _uncompressedSize;
     CompressionConfig::Type _compression;
@@ -119,7 +124,7 @@ public:
 
     bool read(DocumentIdT key, Value &value) const;
     void visit(const IDocumentStore::LidVector &lids, const DocumentTypeRepo &repo, IDocumentVisitor &visitor) const;
-    void write(DocumentIdT, const Value &) {}
+    void write(DocumentIdT, const Value &);
     void erase(DocumentIdT) {}
     const CompressionConfig &getCompression() const { return _compression; }
     void reconfigure(const CompressionConfig &compression);
@@ -138,8 +143,7 @@ void
 Value::set(vespalib::DataBuffer &&buf, ssize_t len, const CompressionConfig &compression) {
     //Underlying buffer must be identical to allow swap.
     vespalib::DataBuffer compressed(buf.getData(), 0u);
-    CompressionConfig::Type type = compress(compression, vespalib::ConstBufferRef(buf.getData(), len),
-                                            compressed, true);
+    CompressionConfig::Type type = compress(compression, vespalib::ConstBufferRef(buf.getData(), len), compressed, true);
     _compressedSize = compressed.getDataLen();
     if (buf.getData() == compressed.getData()) {
         // Uncompressed so we can just steal the underlying buffer.
@@ -154,16 +158,20 @@ Value::set(vespalib::DataBuffer &&buf, ssize_t len, const CompressionConfig &com
     setCompression(type, len);
 }
 
+vespalib::DataBuffer
+Value::decompressed() const {
+    vespalib::DataBuffer uncompressed(_buf.get(), (size_t) 0);
+    decompress(getCompression(), getUncompressedSize(), vespalib::ConstBufferRef(*this, size()), uncompressed, true);
+    return uncompressed;
+}
 
 document::Document::UP
-Value::deserializeDocument(const DocumentTypeRepo &repo) {
-    vespalib::DataBuffer uncompressed((char *) _buf.get(), (size_t) 0);
-    decompress(getCompression(), getUncompressedSize(), vespalib::ConstBufferRef(*this, size()), uncompressed, true);
+Value::deserializeDocument(const DocumentTypeRepo &repo) const {
+    vespalib::DataBuffer uncompressed(decompressed());
     vespalib::nbostream is(uncompressed.getData(), uncompressed.getDataLen());
-    return document::Document::UP(new document::Document(repo, is));
+    return std::make_unique<document::Document>(repo, is);
 }
 
-
 void
 BackingStore::visit(const IDocumentStore::LidVector &lids, const DocumentTypeRepo &repo,
                     IDocumentVisitor &visitor) const {
@@ -184,6 +192,13 @@ BackingStore::read(DocumentIdT key, Value &value) const {
 }
 
 void
+BackingStore::write(DocumentIdT lid, const Value & value)
+{
+    vespalib::DataBuffer buf = value.decompressed();
+    _backingStore.write(value.getSyncToken(), lid, buf.getData(), buf.getDataLen());
+}
+
+void
 BackingStore::reconfigure(const CompressionConfig &compression) {
     _compression = compression;
 }
@@ -194,8 +209,7 @@ using CacheParams = vespalib::CacheParam<
         vespalib::LruParam<DocumentIdT, docstore::Value>,
         docstore::BackingStore,
         vespalib::zero<DocumentIdT>,
-        vespalib::size<docstore::Value>
->;
+        vespalib::size<docstore::Value> >;
 
 class Cache : public vespalib::cache<CacheParams> {
 public:
@@ -210,6 +224,7 @@ DocumentStore::Config::operator == (const Config &rhs) const {
     return (_maxCacheBytes == rhs._maxCacheBytes) &&
             (_allowVisitCaching == rhs._allowVisitCaching) &&
             (_initialCacheEntries == rhs._initialCacheEntries) &&
+            (_updateStrategy == rhs._updateStrategy) &&
             (_compression == rhs._compression);
 }
 
@@ -226,7 +241,7 @@ DocumentStore::DocumentStore(const Config & config, IDataStore & store)
     _cache->reserveElements(config.getInitialCacheEntries());
 }
 
-DocumentStore::~DocumentStore() {}
+DocumentStore::~DocumentStore() = default;
 
 void
 DocumentStore::reconfigure(const Config & config) {
@@ -282,10 +297,27 @@ DocumentStore::write(uint64_t syncToken, DocumentIdT lid, const document::Docume
 
 void
 DocumentStore::write(uint64_t syncToken, DocumentIdT lid, const vespalib::nbostream & stream) {
-    _backingStore.write(syncToken, lid, stream.peek(), stream.size());
     if (useCache()) {
-        _cache->invalidate(lid);
-        _visitCache->invalidate(lid);
+        switch (_config.updateStrategy()) {
+            case Config::UpdateStrategy::INVALIDATE:
+                _backingStore.write(syncToken, lid, stream.peek(), stream.size());
+                _cache->invalidate(lid);
+                break;
+            case Config::UpdateStrategy::UPDATE:
+                if (_cache->hasKey(lid)) {
+                    Value value(syncToken);
+                    vespalib::DataBuffer buf(stream.size());
+                    buf.writeBytes(stream.peek(), stream.size());
+                    value.set(std::move(buf), stream.size(), _store->getCompression());
+                    _cache->write(lid, std::move(value));
+                } else {
+                    _backingStore.write(syncToken, lid, stream.peek(), stream.size());
+                }
+                break;
+        }
+        _visitCache->invalidate(lid); // The cost and complexity of this updating this is not worth it.
+    } else {
+        _backingStore.write(syncToken, lid, stream.peek(), stream.size());
     }
 }
 
@@ -475,31 +507,22 @@ WrapVisitor(Visitor &visitor,
 
 
 void
-DocumentStore::accept(IDocumentStoreReadVisitor &visitor,
-                      IDocumentStoreVisitorProgress &visitorProgress,
+DocumentStore::accept(IDocumentStoreReadVisitor &visitor, IDocumentStoreVisitorProgress &visitorProgress,
                       const DocumentTypeRepo &repo)
 {
-    WrapVisitor<IDocumentStoreReadVisitor> wrap(visitor, repo,
-                                                _store->getCompression(),
-                                                *this,
-                                                _backingStore.
-                                                tentativeLastSyncToken());
+    WrapVisitor<IDocumentStoreReadVisitor> wrap(visitor, repo, _store->getCompression(), *this,
+                                                _backingStore.tentativeLastSyncToken());
     WrapVisitorProgress wrapVisitorProgress(visitorProgress);
     _backingStore.accept(wrap, wrapVisitorProgress, false);
 }
 
 
 void
-DocumentStore::accept(IDocumentStoreRewriteVisitor &visitor,
-                      IDocumentStoreVisitorProgress &visitorProgress,
+DocumentStore::accept(IDocumentStoreRewriteVisitor &visitor, IDocumentStoreVisitorProgress &visitorProgress,
                       const DocumentTypeRepo &repo)
 {
-    WrapVisitor<IDocumentStoreRewriteVisitor> wrap(visitor,
-                                                   repo,
-                                                   _store->getCompression(),
-                                                   *this,
-                                                   _backingStore.
-                                                   tentativeLastSyncToken());
+    WrapVisitor<IDocumentStoreRewriteVisitor> wrap(visitor, repo, _store->getCompression(), *this,
+                                                   _backingStore.tentativeLastSyncToken());
     WrapVisitorProgress wrapVisitorProgress(visitorProgress);
     _backingStore.accept(wrap, wrapVisitorProgress, true);
 }
diff --git a/searchlib/src/vespa/searchlib/docstore/documentstore.h b/searchlib/src/vespa/searchlib/docstore/documentstore.h
index 08b042d99c5..7bc3ab1c21d 100644
--- a/searchlib/src/vespa/searchlib/docstore/documentstore.h
+++ b/searchlib/src/vespa/searchlib/docstore/documentstore.h
@@ -27,17 +27,20 @@ class DocumentStore : public IDocumentStore
 public:
     class Config {
     public:
+        enum UpdateStrategy {INVALIDATE, UPDATE };
         using CompressionConfig = vespalib::compression::CompressionConfig;
         Config() :
             _compression(CompressionConfig::LZ4, 9, 70),
             _maxCacheBytes(1000000000),
             _initialCacheEntries(0),
+            _updateStrategy(INVALIDATE),
             _allowVisitCaching(false)
         { }
         Config(const CompressionConfig & compression, size_t maxCacheBytes, size_t initialCacheEntries) :
             _compression((maxCacheBytes != 0) ? compression : CompressionConfig::NONE),
             _maxCacheBytes(maxCacheBytes),
             _initialCacheEntries(initialCacheEntries),
+            _updateStrategy(INVALIDATE),
             _allowVisitCaching(false)
         { }
         const CompressionConfig & getCompression() const { return _compression; }
@@ -45,11 +48,14 @@ public:
         size_t getInitialCacheEntries() const { return _initialCacheEntries; }
         bool allowVisitCaching() const { return _allowVisitCaching; }
         Config & allowVisitCaching(bool allow) { _allowVisitCaching = allow; return *this; }
+        Config & updateStrategy(UpdateStrategy strategy) { _updateStrategy = strategy; return *this; }
+        UpdateStrategy updateStrategy() const { return _updateStrategy; }
         bool operator == (const Config &) const;
     private:
         CompressionConfig _compression;
         size_t _maxCacheBytes;
         size_t _initialCacheEntries;
+        UpdateStrategy _updateStrategy;
         bool   _allowVisitCaching;
     };
 
@@ -82,14 +88,10 @@ public:
     CacheStats getCacheStats() const override;
     size_t memoryMeta() const override { return _backingStore.memoryMeta(); }
     const vespalib::string & getBaseDir() const override { return _backingStore.getBaseDir(); }
-    void
-    accept(IDocumentStoreReadVisitor &visitor,
-           IDocumentStoreVisitorProgress &visitorProgress,
-           const document::DocumentTypeRepo &repo) override;
-    void
-    accept(IDocumentStoreRewriteVisitor &visitor,
-           IDocumentStoreVisitorProgress &visitorProgress,
-           const document::DocumentTypeRepo &repo) override;
+    void accept(IDocumentStoreReadVisitor &visitor, IDocumentStoreVisitorProgress &visitorProgress,
+                const document::DocumentTypeRepo &repo) override;
+    void accept(IDocumentStoreRewriteVisitor &visitor, IDocumentStoreVisitorProgress &visitorProgress,
+                const document::DocumentTypeRepo &repo) override;
     double getVisitCost() const override;
     DataStoreStorageStats getStorageStats() const override;
     MemoryUsage getMemoryUsage() const override;
diff --git a/searchlib/src/vespa/searchlib/docstore/logdatastore.cpp b/searchlib/src/vespa/searchlib/docstore/logdatastore.cpp
index 6674952d202..148d77738f9 100644
--- a/searchlib/src/vespa/searchlib/docstore/logdatastore.cpp
+++ b/searchlib/src/vespa/searchlib/docstore/logdatastore.cpp
@@ -296,15 +296,17 @@ LogDataStore::compact(uint64_t syncToken)
     uint64_t usage = getDiskFootprint();
     uint64_t bloat = getDiskBloat();
     LOG(debug, "%s", bloatMsg(bloat, usage).c_str());
-    if (_fileChunks.size() > 1) {
+    const bool doCompact = (_fileChunks.size() > 1);
+    if (doCompact) {
         LOG(info, "%s. Will compact", bloatMsg(bloat, usage).c_str());
         compactWorst(_config.getMaxDiskBloatFactor(), _config.getMaxBucketSpread());
+    }
+    flushActiveAndWait(syncToken);
+    if (doCompact) {
         usage = getDiskFootprint();
         bloat = getDiskBloat();
         LOG(info, "Done compacting. %s", bloatMsg(bloat, usage).c_str());
     }
-
-    flushActiveAndWait(syncToken);
 }
 
 size_t
@@ -932,7 +934,7 @@ LogDataStore::scanDir(const vespalib::string &dir, const vespalib::string &suffi
                                                     base.c_str(), err, getLastErrorString().c_str()));
                 }
             } else {
-                LOG(debug, "Skipping '%s' since it does not end with '%s'", file.c_str(), suffix.c_str());
+                LOG(debug, "Skipping '%s' since it does not end with '%s'", file.data(), suffix.c_str());
             }
         }
     }
diff --git a/searchlib/src/vespa/searchlib/docstore/logdocumentstore.cpp b/searchlib/src/vespa/searchlib/docstore/logdocumentstore.cpp
index e2b29f6bdd6..c285d4323c2 100644
--- a/searchlib/src/vespa/searchlib/docstore/logdocumentstore.cpp
+++ b/searchlib/src/vespa/searchlib/docstore/logdocumentstore.cpp
@@ -26,7 +26,7 @@ LogDocumentStore::LogDocumentStore(vespalib::ThreadExecutor & executor,
                     tuneFileSummary, fileHeaderContext, tlSyncer, bucketizer)
 {}
 
-LogDocumentStore::~LogDocumentStore() {}
+LogDocumentStore::~LogDocumentStore() = default;
 
 void
 LogDocumentStore::reconfigure(const Config & config) {
diff --git a/searchlib/src/vespa/searchlib/docstore/summaryexceptions.cpp b/searchlib/src/vespa/searchlib/docstore/summaryexceptions.cpp
index 30c825bca65..2cfa023e7ed 100644
--- a/searchlib/src/vespa/searchlib/docstore/summaryexceptions.cpp
+++ b/searchlib/src/vespa/searchlib/docstore/summaryexceptions.cpp
@@ -12,7 +12,7 @@ SummaryException::SummaryException(const vespalib::stringref &msg,
                                    FastOS_FileInterface &file,
                                    const vespalib::stringref &location)
     : IoException(make_string("%s : Failing file = '%s'. Reason given by OS = '%s'",
-                              msg.c_str(), file.GetFileName(), file.getLastErrorString().c_str()),
+                              vespalib::string(msg).c_str(), file.GetFileName(), file.getLastErrorString().c_str()),
                   getErrorType(file.GetLastError()), location)
 { }
 
diff --git a/searchlib/src/vespa/searchlib/engine/transportserver.cpp b/searchlib/src/vespa/searchlib/engine/transportserver.cpp
index 005ef473817..c94e36202a1 100644
--- a/searchlib/src/vespa/searchlib/engine/transportserver.cpp
+++ b/searchlib/src/vespa/searchlib/engine/transportserver.cpp
@@ -322,7 +322,7 @@ TransportServer::logPacket(const vespalib::stringref &msg, FNET_Packet *p, FNET_
     } else {
         str = vespalib::make_string("packet { pcode=%u }", p->GetPCODE());
     }
-    LOG(debug, "%s (chid=%u, conn=%u):\n%s", msg.c_str(), chid, conntag, str.c_str());
+    LOG(debug, "%s (chid=%u, conn=%u):\n%s", msg.data(), chid, conntag, str.c_str());
 }
 
 void
diff --git a/searchlib/src/vespa/searchlib/expression/catserializer.cpp b/searchlib/src/vespa/searchlib/expression/catserializer.cpp
index a01f68de1a4..f613bd16a54 100644
--- a/searchlib/src/vespa/searchlib/expression/catserializer.cpp
+++ b/searchlib/src/vespa/searchlib/expression/catserializer.cpp
@@ -17,7 +17,7 @@ using vespalib::stringref;
 CatSerializer & CatSerializer::put(const IFieldBase & field, const stringref & value)
 {
     (void) field;
-    getStream().write(value.c_str(), value.size());
+    getStream().write(value.data(), value.size());
     return *this;
 }
 
diff --git a/searchlib/src/vespa/searchlib/expression/documentfieldnode.cpp b/searchlib/src/vespa/searchlib/expression/documentfieldnode.cpp
index 2055cf9c18c..c7ce9528520 100644
--- a/searchlib/src/vespa/searchlib/expression/documentfieldnode.cpp
+++ b/searchlib/src/vespa/searchlib/expression/documentfieldnode.cpp
@@ -77,7 +77,8 @@ deduceResultNode(const vespalib::stringref & fieldName, const FieldValue & fv, b
         } else if (cInfo.inherits(MapFieldValue::classId)) {
             value = deduceResultNode(fieldName, *static_cast<const MapFieldValue &>(fv).createValue(), preserveAccurateTypes, nestedMultiValue);
         } else {
-            throw std::runtime_error(make_string("Can not deduce correct resultclass for documentfield '%s' in based on class '%s'", fieldName.c_str(), cInfo.name()));
+            throw std::runtime_error(make_string("Can not deduce correct resultclass for documentfield '%s' in based on class '%s'",
+                                                 vespalib::string(fieldName).c_str(), cInfo.name()));
         }
         const Identifiable::RuntimeClass & rInfo = value->getClass();
         if (rInfo.inherits(ResultNodeVector::classId)) {
@@ -97,10 +98,12 @@ deduceResultNode(const vespalib::stringref & fieldName, const FieldValue & fv, b
         } else if (rInfo.inherits(RawResultNode::classId)) {
             value.reset(new RawResultNodeVector());
         } else {
-            throw std::runtime_error(make_string("Can not deduce correct resultclass for documentfield '%s' in based on class '%s'. It nests down to %s which is not expected", fieldName.c_str(), cInfo.name(), rInfo.name()));
+            throw std::runtime_error(make_string("Can not deduce correct resultclass for documentfield '%s' in based on class '%s'. It nests down to %s which is not expected",
+                                                 vespalib::string(fieldName).c_str(), cInfo.name(), rInfo.name()));
         }
     } else {
-        throw std::runtime_error(make_string("Can not deduce correct resultclass for documentfield '%s' in based on class '%s'", fieldName.c_str(), cInfo.name()));
+        throw std::runtime_error(make_string("Can not deduce correct resultclass for documentfield '%s' in based on class '%s'",
+                                             vespalib::string(fieldName).c_str(), cInfo.name()));
     }
     return value;
 }
diff --git a/searchlib/src/vespa/searchlib/features/array_parser.hpp b/searchlib/src/vespa/searchlib/features/array_parser.hpp
index 613fa9ec13e..92abec3aab9 100644
--- a/searchlib/src/vespa/searchlib/features/array_parser.hpp
+++ b/searchlib/src/vespa/searchlib/features/array_parser.hpp
@@ -52,13 +52,13 @@ ArrayParser::parsePartial(const vespalib::string &input, OutputType &output)
                         logWarning(vespalib::make_string(
                                 "Could not parse item '%s' in query vector '%s', skipping. "
                                 "Expected ':' between dimension and component.",
-                                item.c_str(), input.c_str()));
+                                vespalib::string(item).c_str(), input.c_str()));
                         return;
                     }
                 } catch (vespalib::IllegalArgumentException & e) {
                     logWarning(vespalib::make_string(
                             "Could not parse item '%s' in query vector '%s', skipping. "
-                            "Incorrect type of operands", item.c_str(), input.c_str()));
+                            "Incorrect type of operands", vespalib::string(item).c_str(), input.c_str()));
                     return;
                 }
                 if (commaPos != vespalib::string::npos) {
@@ -77,7 +77,7 @@ ArrayParser::parsePartial(const vespalib::string &input, OutputType &output)
                 } catch (vespalib::IllegalArgumentException & e) {
                     logWarning(vespalib::make_string(
                             "Could not parse item[%ld] = '%s' in query vector '%s', skipping. "
-                            "Incorrect type of operands", output.size(), is.c_str(), s.c_str()));
+                            "Incorrect type of operands", output.size(), is.c_str(), vespalib::string(s).c_str()));
                     return;
                 }
             }
diff --git a/searchlib/src/vespa/searchlib/features/attributefeature.cpp b/searchlib/src/vespa/searchlib/features/attributefeature.cpp
index b400233bd99..7e913f96b7f 100644
--- a/searchlib/src/vespa/searchlib/features/attributefeature.cpp
+++ b/searchlib/src/vespa/searchlib/features/attributefeature.cpp
@@ -43,7 +43,7 @@ bool equals(const X & lhs, const Y & rhs) {
 
 template <>
 bool equals<ConstCharPtr, vespalib::stringref>(const ConstCharPtr & lhs, const vespalib::stringref & rhs) {
-    return strcmp(lhs, rhs.c_str()) == 0;
+    return strcmp(lhs, rhs.data()) == 0;
 }
 
 template <typename T>
diff --git a/searchlib/src/vespa/searchlib/features/dotproductfeature.h b/searchlib/src/vespa/searchlib/features/dotproductfeature.h
index 27e96c26538..e3cd662cdd8 100644
--- a/searchlib/src/vespa/searchlib/features/dotproductfeature.h
+++ b/searchlib/src/vespa/searchlib/features/dotproductfeature.h
@@ -100,7 +100,7 @@ public:
     EnumVector(const search::attribute::IAttributeVector * attribute) : _attribute(attribute) {}
     void insert(const vespalib::stringref & label, const vespalib::stringref & value) {
         search::attribute::EnumHandle e;
-        if (_attribute->findEnum(label.c_str(), e)) {
+        if (_attribute->findEnum(label.data(), e)) {
             _vector.push_back(std::make_pair(e, util::strToNum<feature_t>(value)));
         }
     }
diff --git a/searchlib/src/vespa/searchlib/features/internal_max_reduce_prod_join_feature.cpp b/searchlib/src/vespa/searchlib/features/internal_max_reduce_prod_join_feature.cpp
index 90451c01294..9d383e5a03a 100644
--- a/searchlib/src/vespa/searchlib/features/internal_max_reduce_prod_join_feature.cpp
+++ b/searchlib/src/vespa/searchlib/features/internal_max_reduce_prod_join_feature.cpp
@@ -3,16 +3,16 @@
 #include "internal_max_reduce_prod_join_feature.h"
 #include "valuefeature.h"
 #include "weighted_set_parser.h"
+#include "dotproductfeature.h"
 
-#include <vespa/log/log.h>
 #include <vespa/searchlib/attribute/attribute.h>
 #include <vespa/searchlib/attribute/imported_attribute_vector_read_guard.h>
 #include <vespa/searchlib/attribute/multinumericattribute.h>
-#include <vespa/searchlib/features/dotproductfeature.h>
 #include <vespa/searchlib/fef/properties.h>
 #include <vespa/searchlib/fef/featureexecutor.h>
 #include <vespa/searchcommon/common/datatype.h>
 
+#include <vespa/log/log.h>
 LOG_SETUP(".features.internalmaxreduceprodjoin");
 
 using namespace search::attribute;
@@ -20,8 +20,7 @@ using namespace search::fef;
 
 using search::features::dotproduct::wset::IntegerVector;
 
-namespace search {
-namespace features {
+namespace search::features {
 
 /**
  * Executor used when array can be accessed directly
@@ -38,8 +37,7 @@ public:
 };
 
 template <typename BaseType>
-RawExecutor<BaseType>::RawExecutor(const IAttributeVector *attribute,
-                         const IntegerVector &queryVector) :
+RawExecutor<BaseType>::RawExecutor(const IAttributeVector *attribute, const IntegerVector &queryVector) :
         FeatureExecutor(),
         _attribute(attribute),
         _queryVector(queryVector)
@@ -69,7 +67,7 @@ RawExecutor<BaseType>::execute(uint32_t docId)
 {
     using A = IntegerAttributeTemplate<BaseType>;
     const multivalue::Value<BaseType> *values(nullptr);
-    const A *iattr = dynamic_cast<const A *>(_attribute);
+    const A *iattr = static_cast<const A *>(_attribute);
     size_t count = iattr->getRawValues(docId, values);
     outputs().set_number(0, maxProduct(values, count, _queryVector));
 }
@@ -215,8 +213,4 @@ InternalMaxReduceProdJoinBlueprint::createExecutor(const IQueryEnvironment &env,
     return stash.create<SingleZeroValueExecutor>();
 }
 
-
 }
-}
-
-
diff --git a/searchlib/src/vespa/searchlib/features/internal_max_reduce_prod_join_feature.h b/searchlib/src/vespa/searchlib/features/internal_max_reduce_prod_join_feature.h
index 2c576b58345..65dd0ac2082 100644
--- a/searchlib/src/vespa/searchlib/features/internal_max_reduce_prod_join_feature.h
+++ b/searchlib/src/vespa/searchlib/features/internal_max_reduce_prod_join_feature.h
@@ -4,8 +4,7 @@
 
 #include <vespa/searchlib/fef/blueprint.h>
 
-namespace search {
-namespace features {
+namespace search::features {
 
 /**
  * Feature for the specific replacement of the expression:
@@ -41,4 +40,3 @@ public:
 };
 
 }
-}
diff --git a/searchlib/src/vespa/searchlib/features/queryterm.h b/searchlib/src/vespa/searchlib/features/queryterm.h
index c162b041f38..75902c33022 100644
--- a/searchlib/src/vespa/searchlib/features/queryterm.h
+++ b/searchlib/src/vespa/searchlib/features/queryterm.h
@@ -6,8 +6,7 @@
 #include <vespa/searchlib/fef/iqueryenvironment.h>
 #include <vespa/searchlib/fef/itermdata.h>
 
-namespace search {
-namespace features {
+namespace search::features {
 
 /**
  * This class represents a query term with the relevant data. Now also
@@ -58,7 +57,4 @@ public:
                             bool lookupConnectedness = false);
 };
 
-
-} // namespace features
-} // namespace search
-
+}
diff --git a/searchlib/src/vespa/searchlib/features/random_normal_feature.cpp b/searchlib/src/vespa/searchlib/features/random_normal_feature.cpp
index c83ec80f6b5..192fc968324 100644
--- a/searchlib/src/vespa/searchlib/features/random_normal_feature.cpp
+++ b/searchlib/src/vespa/searchlib/features/random_normal_feature.cpp
@@ -8,8 +8,7 @@
 #include <vespa/log/log.h>
 LOG_SETUP(".features.randomnormalfeature");
 
-namespace search {
-namespace features {
+namespace search::features {
 
 RandomNormalExecutor::RandomNormalExecutor(uint64_t seed, double mean, double stddev) :
     search::fef::FeatureExecutor(),
@@ -78,6 +77,4 @@ RandomNormalBlueprint::createExecutor(const search::fef::IQueryEnvironment &, ve
     return stash.create<RandomNormalExecutor>(seed, _mean, _stddev);
 }
 
-
-} // namespace features
-} // namespace search
+}
diff --git a/searchlib/src/vespa/searchlib/features/random_normal_feature.h b/searchlib/src/vespa/searchlib/features/random_normal_feature.h
index 2d2429371d9..adf3d3be63f 100644
--- a/searchlib/src/vespa/searchlib/features/random_normal_feature.h
+++ b/searchlib/src/vespa/searchlib/features/random_normal_feature.h
@@ -6,8 +6,7 @@
 #include <vespa/searchlib/fef/featureexecutor.h>
 #include <vespa/searchlib/util/random_normal.h>
 
-namespace search {
-namespace features {
+namespace search::features {
 
 
 /**
@@ -60,7 +59,4 @@ public:
     fef::FeatureExecutor &createExecutor(const fef::IQueryEnvironment &env, vespalib::Stash &stash) const override;
 };
 
-
-} // namespace features
-} // namespace search
-
+}
diff --git a/searchlib/src/vespa/searchlib/features/random_normal_stable_feature.cpp b/searchlib/src/vespa/searchlib/features/random_normal_stable_feature.cpp
index 5f3cf7fd063..f760f52ee88 100644
--- a/searchlib/src/vespa/searchlib/features/random_normal_stable_feature.cpp
+++ b/searchlib/src/vespa/searchlib/features/random_normal_stable_feature.cpp
@@ -8,8 +8,7 @@
 #include <vespa/log/log.h>
 LOG_SETUP(".features.randomnormalstablefeature");
 
-namespace search {
-namespace features {
+namespace search::features {
 
 RandomNormalStableExecutor::RandomNormalStableExecutor(uint64_t seed, double mean, double stddev) :
     search::fef::FeatureExecutor(),
@@ -78,5 +77,4 @@ RandomNormalStableBlueprint::createExecutor(const search::fef::IQueryEnvironment
 }
 
 
-} // namespace features
-} // namespace search
+}
diff --git a/searchlib/src/vespa/searchlib/features/random_normal_stable_feature.h b/searchlib/src/vespa/searchlib/features/random_normal_stable_feature.h
index 129c929ba3d..ab9a53a2df3 100644
--- a/searchlib/src/vespa/searchlib/features/random_normal_stable_feature.h
+++ b/searchlib/src/vespa/searchlib/features/random_normal_stable_feature.h
@@ -6,8 +6,7 @@
 #include <vespa/searchlib/fef/featureexecutor.h>
 #include <vespa/searchlib/util/random_normal.h>
 
-namespace search {
-namespace features {
+namespace search::features {
 
 /**
  * Implements the executor for the random normal feature outputting a
@@ -61,7 +60,4 @@ public:
     fef::FeatureExecutor &createExecutor(const fef::IQueryEnvironment &env, vespalib::Stash &stash) const override;
 };
 
-
-} // namespace features
-} // namespace search
-
+}
diff --git a/searchlib/src/vespa/searchlib/features/randomfeature.cpp b/searchlib/src/vespa/searchlib/features/randomfeature.cpp
index 937fdb23800..cdedbcadc5e 100644
--- a/searchlib/src/vespa/searchlib/features/randomfeature.cpp
+++ b/searchlib/src/vespa/searchlib/features/randomfeature.cpp
@@ -9,8 +9,7 @@
 #include <vespa/log/log.h>
 LOG_SETUP(".features.randomfeature");
 
-namespace search {
-namespace features {
+namespace search::features {
 
 RandomExecutor::RandomExecutor(uint64_t seed, uint64_t matchSeed) :
     search::fef::FeatureExecutor(),
@@ -81,5 +80,4 @@ RandomBlueprint::createExecutor(const search::fef::IQueryEnvironment &env, vespa
 }
 
 
-} // namespace features
-} // namespace search
+}
diff --git a/searchlib/src/vespa/searchlib/features/randomfeature.h b/searchlib/src/vespa/searchlib/features/randomfeature.h
index f326606009c..8ed0e403836 100644
--- a/searchlib/src/vespa/searchlib/features/randomfeature.h
+++ b/searchlib/src/vespa/searchlib/features/randomfeature.h
@@ -6,8 +6,7 @@
 #include <vespa/searchlib/fef/featureexecutor.h>
 #include <vespa/searchlib/util/rand48.h>
 
-namespace search {
-namespace features {
+namespace search::features {
 
 /**
  * Implements the executor for the random feature outputting a number in the interval [0, 1>.
@@ -23,7 +22,6 @@ public:
     void execute(uint32_t docId) override;
 };
 
-
 /**
  * Implements the blueprint for the random feature.
  */
@@ -45,6 +43,4 @@ public:
     search::fef::FeatureExecutor &createExecutor(const search::fef::IQueryEnvironment &env, vespalib::Stash &stash) const override;
 };
 
-
-} // namespace features
-} // namespace search
+}
diff --git a/searchlib/src/vespa/searchlib/features/rankingexpressionfeature.cpp b/searchlib/src/vespa/searchlib/features/rankingexpressionfeature.cpp
index 4640170e6b9..72865d042e7 100644
--- a/searchlib/src/vespa/searchlib/features/rankingexpressionfeature.cpp
+++ b/searchlib/src/vespa/searchlib/features/rankingexpressionfeature.cpp
@@ -26,8 +26,7 @@ using search::fef::FeatureType;
 using vespalib::ArrayRef;
 using vespalib::ConstArrayRef;
 
-namespace search {
-namespace features {
+namespace search::features {
 
 namespace {
 
@@ -181,9 +180,7 @@ RankingExpressionBlueprint::RankingExpressionBlueprint(rankingexpression::Expres
 {
 }
 
-RankingExpressionBlueprint::~RankingExpressionBlueprint()
-{
-}
+RankingExpressionBlueprint::~RankingExpressionBlueprint() = default;
 
 void
 RankingExpressionBlueprint::visitDumpFeatures(const fef::IIndexEnvironment &,
@@ -305,5 +302,4 @@ RankingExpressionBlueprint::createExecutor(const fef::IQueryEnvironment &env, ve
 
 //-----------------------------------------------------------------------------
 
-} // features
-} // search
+}
diff --git a/searchlib/src/vespa/searchlib/features/rankingexpressionfeature.h b/searchlib/src/vespa/searchlib/features/rankingexpressionfeature.h
index 5da6c9eae87..85514a4def7 100644
--- a/searchlib/src/vespa/searchlib/features/rankingexpressionfeature.h
+++ b/searchlib/src/vespa/searchlib/features/rankingexpressionfeature.h
@@ -8,8 +8,7 @@
 #include <vespa/searchlib/features/rankingexpression/expression_replacer.h>
 #include <vespa/searchlib/features/rankingexpression/intrinsic_expression.h>
 
-namespace search {
-namespace features {
+namespace search::features {
 
 //-----------------------------------------------------------------------------
 
@@ -42,7 +41,4 @@ public:
     fef::FeatureExecutor &createExecutor(const fef::IQueryEnvironment &env, vespalib::Stash &stash) const override;
 };
 
-//-----------------------------------------------------------------------------
-
-} // features
-} // search
+}
diff --git a/searchlib/src/vespa/searchlib/features/raw_score_feature.cpp b/searchlib/src/vespa/searchlib/features/raw_score_feature.cpp
index 02e44e781d4..61355581214 100644
--- a/searchlib/src/vespa/searchlib/features/raw_score_feature.cpp
+++ b/searchlib/src/vespa/searchlib/features/raw_score_feature.cpp
@@ -5,14 +5,14 @@
 
 using namespace search::fef;
 
-namespace search {
-namespace features {
+namespace search::features {
 
 RawScoreExecutor::RawScoreExecutor(const search::fef::IQueryEnvironment &env, uint32_t fieldId)
     : FeatureExecutor(),
       _handles(),
       _md(nullptr)
 {
+    _handles.reserve(env.getNumTerms());
     for (uint32_t i = 0; i < env.getNumTerms(); ++i) {
         search::fef::TermFieldHandle handle = util::getTermFieldHandle(env, i, fieldId);
         if (handle != search::fef::IllegalHandle) {
@@ -25,8 +25,8 @@ void
 RawScoreExecutor::execute(uint32_t docId)
 {
     feature_t output = 0.0;
-    for (uint32_t i = 0; i < _handles.size(); ++i) {
-        const TermFieldMatchData *tfmd = _md->resolveTermField(_handles[i]);
+    for (auto handle : _handles) {
+        const TermFieldMatchData *tfmd = _md->resolveTermField(handle);
         if (tfmd->getDocId() == docId) {
             output += tfmd->getRawScore();
         }
@@ -43,8 +43,7 @@ RawScoreExecutor::handle_bind_match_data(const fef::MatchData &md)
 //-----------------------------------------------------------------------------
 
 bool
-RawScoreBlueprint::setup(const IIndexEnvironment &,
-                         const ParameterList &params)
+RawScoreBlueprint::setup(const IIndexEnvironment &, const ParameterList &params)
 {
     _field = params[0].asField();
     describeOutput("out", "accumulated raw score for the given field");
@@ -57,5 +56,4 @@ RawScoreBlueprint::createExecutor(const IQueryEnvironment &queryEnv, vespalib::S
     return stash.create<RawScoreExecutor>(queryEnv, _field->id());
 }
 
-} // namespace features
-} // namespace search
+}
diff --git a/searchlib/src/vespa/searchlib/features/raw_score_feature.h b/searchlib/src/vespa/searchlib/features/raw_score_feature.h
index 0eceba16ffe..db237f036c4 100644
--- a/searchlib/src/vespa/searchlib/features/raw_score_feature.h
+++ b/searchlib/src/vespa/searchlib/features/raw_score_feature.h
@@ -5,8 +5,7 @@
 #include <vespa/searchlib/fef/blueprint.h>
 #include <vespa/searchlib/fef/featureexecutor.h>
 
-namespace search {
-namespace features {
+namespace search::features {
 
 class RawScoreExecutor : public fef::FeatureExecutor
 {
@@ -40,5 +39,4 @@ public:
     fef::FeatureExecutor &createExecutor(const fef::IQueryEnvironment &env, vespalib::Stash &stash) const override;
 };
 
-} // namespace features
-} // namespace search
+}
diff --git a/searchlib/src/vespa/searchlib/features/weighted_set_parser.hpp b/searchlib/src/vespa/searchlib/features/weighted_set_parser.hpp
index 3199d54bf03..d67b089f5fe 100644
--- a/searchlib/src/vespa/searchlib/features/weighted_set_parser.hpp
+++ b/searchlib/src/vespa/searchlib/features/weighted_set_parser.hpp
@@ -24,13 +24,13 @@ WeightedSetParser::parse(const vespalib::string &input, OutputType &output)
             if (colonPos != vespalib::string::npos) {
                 vespalib::string tmpKey(item.substr(0, colonPos));
                 vespalib::string::size_type start(tmpKey.find_first_not_of(' '));
-                vespalib::stringref key(tmpKey.c_str() + start, colonPos - start);
+                vespalib::stringref key(tmpKey.data() + start, colonPos - start);
                 vespalib::stringref value(item.substr(colonPos+1));
                 output.insert(key, value);
             } else {
                 logWarning(vespalib::make_string(
                         "Could not parse item '%s' in input string '%s', skipping. "
-                    "Expected ':' between key and weight.", item.c_str(), input.c_str()));
+                    "Expected ':' between key and weight.", vespalib::string(item).c_str(), input.c_str()));
             }
             if (commaPos != vespalib::string::npos) {
                 s = s.substr(commaPos+1);
diff --git a/searchlib/src/vespa/searchlib/fef/featureexecutor.h b/searchlib/src/vespa/searchlib/fef/featureexecutor.h
index c8219fada3b..dfc46230e18 100644
--- a/searchlib/src/vespa/searchlib/fef/featureexecutor.h
+++ b/searchlib/src/vespa/searchlib/fef/featureexecutor.h
@@ -4,12 +4,10 @@
 
 #include "handle.h"
 #include "matchdata.h"
-#include <memory>
 #include "number_or_object.h"
 #include <vespa/vespalib/util/arrayref.h>
 
-namespace search {
-namespace fef {
+namespace search::fef {
 
 class FeatureExecutor;
 
@@ -181,8 +179,6 @@ vespalib::eval::Value::CREF FeatureExecutor::Inputs::get_object(size_t idx) cons
     return _inputs[idx].as_object(_docid);
 }
 
-} // namespace fef
-} // namespace search
-
+}
 
 //  LocalWords:  param
diff --git a/searchlib/src/vespa/searchlib/fef/indexproperties.h b/searchlib/src/vespa/searchlib/fef/indexproperties.h
index a2a227b448c..38e0eca7548 100644
--- a/searchlib/src/vespa/searchlib/fef/indexproperties.h
+++ b/searchlib/src/vespa/searchlib/fef/indexproperties.h
@@ -6,10 +6,7 @@
 #include <vector>
 #include <vespa/searchlib/common/feature.h>
 
-namespace search {
-namespace fef {
-
-class Properties;
+namespace search::fef { class Properties; }
 
 /**
  * This namespace is a placeholder for several structs, each representing
@@ -19,7 +16,7 @@ class Properties;
  * instance one should use the property names defined here to perform the lookup.
  * If the property is not present the default value is used.
  **/
-namespace indexproperties {
+namespace search::fef::indexproperties {
 
 namespace eval {
 
@@ -357,8 +354,4 @@ struct QueryFeature {
 
 } // namespace type
 
-
-} // namespace indexproperties
-} // namespace fef
-} // namespace search
-
+}
diff --git a/searchlib/src/vespa/searchlib/fef/iqueryenvironment.h b/searchlib/src/vespa/searchlib/fef/iqueryenvironment.h
index a6ed0058c66..a7f268e5c6b 100644
--- a/searchlib/src/vespa/searchlib/fef/iqueryenvironment.h
+++ b/searchlib/src/vespa/searchlib/fef/iqueryenvironment.h
@@ -3,11 +3,10 @@
 #pragma once
 
 #include "iindexenvironment.h"
+#include "objectstore.h"
 #include <vespa/searchcommon/attribute/iattributecontext.h>
-#include <vespa/searchlib/fef/objectstore.h>
 
-namespace search {
-namespace fef {
+namespace search::fef {
 
 class Location;
 class Properties;
@@ -90,5 +89,4 @@ private:
     ObjectStore _objectStore;
 };
 
-} // namespace fef
-} // namespace search
+}
diff --git a/searchlib/src/vespa/searchlib/fef/itermdata.h b/searchlib/src/vespa/searchlib/fef/itermdata.h
index 94b4ed80cf0..95095dfaff0 100644
--- a/searchlib/src/vespa/searchlib/fef/itermdata.h
+++ b/searchlib/src/vespa/searchlib/fef/itermdata.h
@@ -6,8 +6,7 @@
 #include <vespa/searchlib/query/weight.h>
 #include <cstddef>
 
-namespace search {
-namespace fef {
+namespace search::fef {
 
 /**
  * Interface to static match data for a single unit (term/phrase/etc).
@@ -84,6 +83,4 @@ public:
     void next() { ++_idx; }
 };
 
-} // namespace fef
-} // namespace search
-
+}
diff --git a/searchlib/src/vespa/searchlib/fef/itermfielddata.h b/searchlib/src/vespa/searchlib/fef/itermfielddata.h
index a85998d8d52..bb4ab9c5285 100644
--- a/searchlib/src/vespa/searchlib/fef/itermfielddata.h
+++ b/searchlib/src/vespa/searchlib/fef/itermfielddata.h
@@ -4,8 +4,7 @@
 
 #include "handle.h"
 
-namespace search {
-namespace fef {
+namespace search::fef {
 
 /**
  * Interface to information about a single field that is being
@@ -43,6 +42,4 @@ public:
     virtual TermFieldHandle getHandle() const = 0;
 };
 
-} // namespace fef
-} // namespace search
-
+}
diff --git a/searchlib/src/vespa/searchlib/fef/number_or_object.h b/searchlib/src/vespa/searchlib/fef/number_or_object.h
index 59de864961e..259a1622516 100644
--- a/searchlib/src/vespa/searchlib/fef/number_or_object.h
+++ b/searchlib/src/vespa/searchlib/fef/number_or_object.h
@@ -5,8 +5,7 @@
 #include <vespa/searchlib/common/feature.h>
 #include <vespa/eval/eval/value.h>
 
-namespace search {
-namespace fef {
+namespace search::fef {
 
 /**
  * Storage cell for values passed between feature executors in the
@@ -21,5 +20,4 @@ union NumberOrObject {
     ~NumberOrObject() {}
 };
 
-} // namespace fef
-} // namespace search
+}
diff --git a/searchlib/src/vespa/searchlib/fef/objectstore.h b/searchlib/src/vespa/searchlib/fef/objectstore.h
index b85ce527b8f..a62ca66486c 100644
--- a/searchlib/src/vespa/searchlib/fef/objectstore.h
+++ b/searchlib/src/vespa/searchlib/fef/objectstore.h
@@ -3,8 +3,7 @@
 
 #include <vespa/vespalib/stllike/hash_map.h>
 
-namespace search {
-namespace fef {
+namespace search::fef {
 
 class Anything
 {
@@ -34,4 +33,3 @@ private:
 };
 
 }
-}
diff --git a/searchlib/src/vespa/searchlib/fef/ranksetup.cpp b/searchlib/src/vespa/searchlib/fef/ranksetup.cpp
index 5a16b1b6ba8..eea9188632d 100644
--- a/searchlib/src/vespa/searchlib/fef/ranksetup.cpp
+++ b/searchlib/src/vespa/searchlib/fef/ranksetup.cpp
@@ -20,8 +20,7 @@ public:
 };
 } // namespace <unnamed>
 
-namespace search {
-namespace fef {
+namespace search::fef {
     
 using namespace indexproperties;    
 
@@ -61,7 +60,7 @@ RankSetup::RankSetup(const BlueprintFactory &factory, const IIndexEnvironment &i
       _softTimeoutTailCost(0.1)
 { }
 
-RankSetup::~RankSetup() { }
+RankSetup::~RankSetup() = default;
 
 void
 RankSetup::configure()
@@ -190,5 +189,4 @@ RankSetup::prepareSharedState(const IQueryEnvironment &queryEnv, IObjectStore &o
     }
 }
 
-} // namespace fef
-} // namespace search
+}
diff --git a/searchlib/src/vespa/searchlib/fef/ranksetup.h b/searchlib/src/vespa/searchlib/fef/ranksetup.h
index cccb3d961d5..ef869c87740 100644
--- a/searchlib/src/vespa/searchlib/fef/ranksetup.h
+++ b/searchlib/src/vespa/searchlib/fef/ranksetup.h
@@ -8,8 +8,7 @@
 #include "blueprintresolver.h"
 #include "rank_program.h"
 
-namespace search {
-namespace fef {
+namespace search::fef {
 
 /**
  * A rank setup contains information about how initial and final rank
@@ -403,5 +402,4 @@ public:
     void prepareSharedState(const IQueryEnvironment & queryEnv, IObjectStore & objectStore) const;
 };
 
-} // namespace fef
-} // namespace search
+}
diff --git a/searchlib/src/vespa/searchlib/memoryindex/fieldinverter.cpp b/searchlib/src/vespa/searchlib/memoryindex/fieldinverter.cpp
index 3238243ff0b..957573d0ad7 100644
--- a/searchlib/src/vespa/searchlib/memoryindex/fieldinverter.cpp
+++ b/searchlib/src/vespa/searchlib/memoryindex/fieldinverter.cpp
@@ -256,7 +256,7 @@ FieldInverter::saveWord(const vespalib::stringref word)
 
     char * buf = &_words[0] + wordsSize;
     memset(buf, 0, 4);
-    memcpy(buf + 4, word.c_str(), len);
+    memcpy(buf + 4, word.data(), len);
     uint32_t *lastWord = reinterpret_cast<uint32_t *>(buf + 4 + (len & ~0x3));
     *lastWord &=  (0xffffff >> ((3 - (len & 3)) << 3)); //only on little endian machiness !!
 
diff --git a/searchlib/src/vespa/searchlib/memoryindex/memoryfieldindex.h b/searchlib/src/vespa/searchlib/memoryindex/memoryfieldindex.h
index 92a69277b76..c4583d7f49f 100644
--- a/searchlib/src/vespa/searchlib/memoryindex/memoryfieldindex.h
+++ b/searchlib/src/vespa/searchlib/memoryindex/memoryfieldindex.h
@@ -52,7 +52,7 @@ public:
             if (wordRef.valid()) {
                 return _wordStore.getWord(wordRef);
             }
-            return _word.c_str();
+            return _word.data();
         }
 
     public:
diff --git a/searchlib/src/vespa/searchlib/memoryindex/urlfieldinverter.cpp b/searchlib/src/vespa/searchlib/memoryindex/urlfieldinverter.cpp
index 527de32b956..0f8c05a44b9 100644
--- a/searchlib/src/vespa/searchlib/memoryindex/urlfieldinverter.cpp
+++ b/searchlib/src/vespa/searchlib/memoryindex/urlfieldinverter.cpp
@@ -130,7 +130,7 @@ UrlFieldInverter::processUrlSubField(FieldInverter *inverter,
         LOG(error,
             "Illegal field type %s for URL subfield %s, expected string",
             sfv->getDataType()->getName().c_str(),
-            subField.c_str());
+            vespalib::string(subField).data());
         return;
     }
     const StringFieldValue &value = static_cast<const StringFieldValue &>(*sfv);
diff --git a/searchlib/src/vespa/searchlib/parsequery/parse.cpp b/searchlib/src/vespa/searchlib/parsequery/parse.cpp
index cba69ea474a..60236275f76 100644
--- a/searchlib/src/vespa/searchlib/parsequery/parse.cpp
+++ b/searchlib/src/vespa/searchlib/parsequery/parse.cpp
@@ -63,7 +63,7 @@ ParseItem::ParseItem(ItemType type, const vespalib::stringref & idx, const char
 {
     assert_type(type);
     SetType(type);
-    SetIndex(idx.c_str());
+    SetIndex(idx.data());
     SetTerm(term);
 }
 
diff --git a/searchlib/src/vespa/searchlib/parsequery/simplequerystack.cpp b/searchlib/src/vespa/searchlib/parsequery/simplequerystack.cpp
index ef85835d9b6..36d76629f87 100644
--- a/searchlib/src/vespa/searchlib/parsequery/simplequerystack.cpp
+++ b/searchlib/src/vespa/searchlib/parsequery/simplequerystack.cpp
@@ -159,6 +159,7 @@ SimpleQueryStack::StackbufToString(const vespalib::stringref &theBuf)
     const char *p = theBuf.begin();
     const char *ep = theBuf.end();
     uint64_t tmp(0);
+    int64_t tmpI64(0);
     uint8_t flags(0);
     while (p < ep) {
         vespalib::string metaStr;
@@ -248,22 +249,20 @@ SimpleQueryStack::StackbufToString(const vespalib::stringref &theBuf)
             termRef = p;
             p += termRefLen;
             result.append(make_string("%c/%d:%.*s/%d:%.*s~", _G_ItemName[type],
-                                            idxRefLen, idxRefLen, idxRef,
-                                            termRefLen, termRefLen, termRef));
+                                      idxRefLen, idxRefLen, idxRef, termRefLen, termRefLen, termRef));
             break;
         case ParseItem::ITEM_PURE_WEIGHTED_STRING:
             p += vespalib::compress::Integer::decompressPositive(tmp, p);
             termRefLen = tmp;
             termRef = p;
             p += termRefLen;
-            result.append(make_string("%c/%d:%.*s~", _G_ItemName[type],
-                                            termRefLen, termRefLen, termRef));
+            result.append(make_string("%c/%d:%.*s~", _G_ItemName[type], termRefLen, termRefLen, termRef));
             break;
 
         case ParseItem::ITEM_PURE_WEIGHTED_LONG:
-            tmp = vespalib::nbo::n2h(*reinterpret_cast<const uint64_t *>(p));
+            tmpI64 = vespalib::nbo::n2h(*reinterpret_cast<const int64_t *>(p));
             p += sizeof(uint64_t);
-            result.append(make_string("%c/%lu", _G_ItemName[type], tmp));
+            result.append(make_string("%c/%" PRId64, _G_ItemName[type], tmpI64));
             break;
 
         case ParseItem::ITEM_PHRASE:
@@ -281,13 +280,12 @@ SimpleQueryStack::StackbufToString(const vespalib::stringref &theBuf)
                 uint32_t targetNumHits = tmp;
                 double scoreThreshold = vespalib::nbo::n2h(*reinterpret_cast<const double *>(p)); 
                 p += sizeof(double);
-                double thresholdBoostFactor = vespalib::nbo::n2h(*reinterpret_cast<const double *>(p)); // thresholdBoostFactor
+                double thresholdBoostFactor = vespalib::nbo::n2h(*reinterpret_cast<const double *>(p));
                 p += sizeof(double);
                 result.append(make_string("%c/%d/%d:%.*s(%u,%f,%f)~", _G_ItemName[type], arity, idxRefLen,
-                                                idxRefLen, idxRef, targetNumHits, scoreThreshold, thresholdBoostFactor));
+                                          idxRefLen, idxRef, targetNumHits, scoreThreshold, thresholdBoostFactor));
             } else {
-                result.append(make_string("%c/%d/%d:%.*s~", _G_ItemName[type], arity, idxRefLen,
-                                                idxRefLen, idxRef));
+                result.append(make_string("%c/%d/%d:%.*s~", _G_ItemName[type], arity, idxRefLen, idxRefLen, idxRef));
             }
             break;
 
@@ -297,8 +295,7 @@ SimpleQueryStack::StackbufToString(const vespalib::stringref &theBuf)
             idxRef = p;
             p += idxRefLen;
             size_t feature_count = ReadCompressedPositiveInt(p);
-            result.append(make_string(
-                    "%c/%d:%.*s/%zu(", _G_ItemName[type], idxRefLen, idxRefLen, idxRef, feature_count));
+            result.append(make_string("%c/%d:%.*s/%zu(", _G_ItemName[type], idxRefLen, idxRefLen, idxRef, feature_count));
             for (size_t i = 0; i < feature_count; ++i) {
                 vespalib::string key = ReadString(p);
                 vespalib::string value = ReadString(p);
diff --git a/searchlib/src/vespa/searchlib/parsequery/stackdumpiterator.cpp b/searchlib/src/vespa/searchlib/parsequery/stackdumpiterator.cpp
index 542e7990c1e..1938222b4ac 100644
--- a/searchlib/src/vespa/searchlib/parsequery/stackdumpiterator.cpp
+++ b/searchlib/src/vespa/searchlib/parsequery/stackdumpiterator.cpp
@@ -25,9 +25,9 @@ SimpleQueryStackDumpIterator::SimpleQueryStackDumpIterator(const vespalib::strin
     _currArg2(0),
     _currArg3(0),
     _predicate_query_term(),
-    _currIndexName(NULL),
+    _currIndexName(nullptr),
     _currIndexNameLen(0),
-    _currTerm(NULL),
+    _currTerm(nullptr),
     _currTermLen(0),
     _generatedTerm(),
     _currNum(-1)
@@ -114,9 +114,9 @@ SimpleQueryStackDumpIterator::next()
         _currArity = tmp;
         if (p > _bufEnd) return false;
         _currArg1 = 0;
-        _currIndexName = NULL;
+        _currIndexName = nullptr;
         _currIndexNameLen = 0;
-        _currTerm = NULL;
+        _currTerm = nullptr;
         _currTermLen = 0;
         break;
 
@@ -129,9 +129,9 @@ SimpleQueryStackDumpIterator::next()
         p += vespalib::compress::Integer::decompressPositive(tmp, p);
         _currArg1 = tmp;
         if (p > _bufEnd) return false;
-        _currIndexName = NULL;
+        _currIndexName = nullptr;
         _currIndexNameLen = 0;
-        _currTerm = NULL;
+        _currTerm = nullptr;
         _currTermLen = 0;
         break;
 
@@ -149,7 +149,7 @@ SimpleQueryStackDumpIterator::next()
         _currIndexName = p;
         p += _currIndexNameLen;
         if (p > _bufEnd) return false;
-        _currTerm = NULL;
+        _currTerm = nullptr;
         _currTermLen = 0;
         break;
     case ParseItem::ITEM_SAME_ELEMENT:
@@ -163,7 +163,7 @@ SimpleQueryStackDumpIterator::next()
         _currIndexName = p;
         p += _currIndexNameLen;
         if (p > _bufEnd) return false;
-        _currTerm = NULL;
+        _currTerm = nullptr;
         _currTermLen = 0;
         break;
 
@@ -180,12 +180,12 @@ SimpleQueryStackDumpIterator::next()
         _currArity = 0;
         break;
     case ParseItem::ITEM_PURE_WEIGHTED_LONG:
-        if (p + 8 > _bufEnd) return false;
+        if (p + sizeof(int64_t) > _bufEnd) return false;
         _generatedTerm.clear();
-        _generatedTerm << vespalib::nbo::n2h(*(const uint64_t *)p);
+        _generatedTerm << vespalib::nbo::n2h(*reinterpret_cast<const int64_t *>(p));
         _currTerm = _generatedTerm.c_str();
         _currTermLen = _generatedTerm.size();
-        p += 8;
+        p += sizeof(int64_t);
         if (p > _bufEnd) return false;
 
         _currArg1 = 0;
@@ -197,7 +197,7 @@ SimpleQueryStackDumpIterator::next()
             _currIndexName = p;
             p += _currIndexNameLen;
             _currArity = readCompressedPositiveInt(p);
-            _currTerm = NULL;
+            _currTerm = nullptr;
             _currTermLen = 0;
             if (p > _bufEnd) return false;
         } catch (...) {
@@ -281,7 +281,7 @@ SimpleQueryStackDumpIterator::next()
         } else {
             _currArg1 = 0;
         }
-        _currTerm = NULL;
+        _currTerm = nullptr;
         _currTermLen = 0;
         break;
 
diff --git a/searchlib/src/vespa/searchlib/query/queryterm.cpp b/searchlib/src/vespa/searchlib/query/queryterm.cpp
index ee2e72b41a8..5eabaf35378 100644
--- a/searchlib/src/vespa/searchlib/query/queryterm.cpp
+++ b/searchlib/src/vespa/searchlib/query/queryterm.cpp
@@ -386,16 +386,16 @@ QueryTermSimple::QueryTermSimple(const string & term_, SearchTerm type) :
         }
         _valid = (numParts >= 2) && (numParts < NELEMS(parts));
         if (_valid && numParts > 2) {
-            _rangeLimit = strtol(parts[2].c_str(), NULL, 0);
+            _rangeLimit = strtol(parts[2].data(), nullptr, 0);
             if (numParts > 3) {
                 _valid = (numParts >= 5);
                 if (_valid) {
                     _diversityAttribute = parts[3];
-                    _maxPerGroup = strtoul(parts[4].c_str(), NULL, 0);
+                    _maxPerGroup = strtoul(parts[4].data(), nullptr, 0);
                     if ((_maxPerGroup > 0) && (numParts > 5)) {
                         char *err = nullptr;
-                        size_t cutoffGroups = strtoul(parts[5].c_str(), &err, 0);
-                        if ((err == nullptr) || (size_t(err - parts[5].c_str()) == parts[5].size())) {
+                        size_t cutoffGroups = strtoul(parts[5].data(), &err, 0);
+                        if ((err == nullptr) || (size_t(err - parts[5].data()) == parts[5].size())) {
                             _diversityCutoffGroups = cutoffGroups;
                         }
                         if (numParts > 6) {
diff --git a/searchlib/src/vespa/searchlib/query/tree/stackdumpquerycreator.h b/searchlib/src/vespa/searchlib/query/tree/stackdumpquerycreator.h
index fa42cdac1c0..a208ae357fa 100644
--- a/searchlib/src/vespa/searchlib/query/tree/stackdumpquerycreator.h
+++ b/searchlib/src/vespa/searchlib/query/tree/stackdumpquerycreator.h
@@ -41,7 +41,7 @@ public:
             vespalib::stringref stack = queryStack.getStack();
             LOG(error, "Unable to create query tree from stack dump. Failed at position %ld out of %ld bytes %s",
                        queryStack.getPosition(), stack.size(), builder.error().c_str());
-            LOG(error, "Raw QueryStack = %s", vespalib::HexDump(stack.c_str(), stack.size()).toString().c_str());
+            LOG(error, "Raw QueryStack = %s", vespalib::HexDump(stack.data(), stack.size()).toString().c_str());
             if (LOG_WOULD_LOG(debug)) {
                 vespalib::string query = SimpleQueryStack::StackbufToString(stack);
                 LOG(error, "Error = %s, QueryStack = %s", builder.error().c_str(), query.c_str());
diff --git a/searchlib/src/vespa/searchlib/queryeval/andnotsearch.cpp b/searchlib/src/vespa/searchlib/queryeval/andnotsearch.cpp
index 77f8e454035..5646af9e677 100644
--- a/searchlib/src/vespa/searchlib/queryeval/andnotsearch.cpp
+++ b/searchlib/src/vespa/searchlib/queryeval/andnotsearch.cpp
@@ -4,8 +4,7 @@
 #include "termwise_helper.h"
 #include <vespa/searchlib/common/bitvector.h>
 
-namespace search {
-namespace queryeval {
+namespace search::queryeval {
 
 void
 AndNotSearch::doSeek(uint32_t docid)
@@ -160,5 +159,4 @@ AndNotSearch::or_hits_into(BitVector &result, uint32_t begin_id) {
     result.orWith(*get_hits(begin_id));
 }
 
-}  // namespace queryeval
-}  // namespace search
+}
diff --git a/searchlib/src/vespa/searchlib/queryeval/andnotsearch.h b/searchlib/src/vespa/searchlib/queryeval/andnotsearch.h
index e805448ef98..c29ef9407ad 100644
--- a/searchlib/src/vespa/searchlib/queryeval/andnotsearch.h
+++ b/searchlib/src/vespa/searchlib/queryeval/andnotsearch.h
@@ -2,13 +2,11 @@
 
 #pragma once
 
-#include <vector>
 #include "multisearch.h"
 #include <vespa/searchlib/attribute/attributeiterators.h>
 #include <vespa/searchlib/attribute/singlesmallnumericattribute.h>
 
-namespace search {
-namespace queryeval {
+namespace search::queryeval {
 
 /**
  * A simple implementation of the AndNot search operation.
@@ -97,6 +95,4 @@ private:
     void doUnpack(uint32_t docid) override;
 };
 
-} // namespace queryeval
-} // namespace search
-
+}
diff --git a/searchlib/src/vespa/searchlib/queryeval/blueprint.cpp b/searchlib/src/vespa/searchlib/queryeval/blueprint.cpp
index f9b8a9437dc..c99e07cd355 100644
--- a/searchlib/src/vespa/searchlib/queryeval/blueprint.cpp
+++ b/searchlib/src/vespa/searchlib/queryeval/blueprint.cpp
@@ -18,7 +18,7 @@ namespace search::queryeval {
 
 void maybe_eliminate_self(Blueprint* &self, Blueprint::UP replacement) {
     // replace with replacement
-    if (replacement.get() != nullptr) {
+    if (replacement) {
         Blueprint *tmp = replacement.release();
         tmp->setParent(self->getParent());
         tmp->setSourceId(self->getSourceId());
@@ -41,9 +41,9 @@ Blueprint::HitEstimate
 Blueprint::max(const std::vector<HitEstimate> &data)
 {
     HitEstimate est;
-    for (size_t i = 0; i < data.size(); ++i) {
-        if (est.empty || est.estHits < data[i].estHits) {
-            est = data[i];
+    for (const HitEstimate & hitEst : data) {
+        if (est.empty || est.estHits < hitEst.estHits) {
+            est = hitEst;
         }
     }
     return est;
@@ -69,7 +69,7 @@ Blueprint::State::State(const FieldSpecBaseList &fields_in)
 {
 }
 
-Blueprint::State::~State() { }
+Blueprint::State::~State() = default;
 
 Blueprint::Blueprint()
     : _parent(0),
@@ -79,9 +79,7 @@ Blueprint::Blueprint()
 {
 }
 
-Blueprint::~Blueprint()
-{
-}
+Blueprint::~Blueprint() = default;
 
 Blueprint::UP
 Blueprint::optimize(Blueprint::UP bp) {
@@ -135,7 +133,6 @@ Blueprint::visitMembers(vespalib::ObjectVisitor &visitor) const
         for (size_t i = 0; i < state.numFields(); ++i) {
             const FieldSpecBase &spec = state.field(i);
             visitor.openStruct(vespalib::make_string("[%zu]", i), "Field");
-            // visitor.visitString("name", spec.getName());
             visitor.visitInt("fieldId", spec.getFieldId());
             visitor.visitInt("handle", spec.getHandle());
             visitor.visitBool("isFilter", spec.isFilter());
@@ -180,8 +177,8 @@ void
 IntermediateBlueprint::setDocIdLimit(uint32_t limit)
 {
     Blueprint::setDocIdLimit(limit);
-    for (size_t i = 0; i < _children.size(); ++i) {
-        _children[i]->setDocIdLimit(limit);
+    for (Blueprint * child : _children) {
+        child->setDocIdLimit(limit);
     }
 }
 
@@ -190,8 +187,8 @@ IntermediateBlueprint::calculateEstimate() const
 {
     std::vector<HitEstimate> estimates;
     estimates.reserve(_children.size());
-    for (size_t i = 0; i < _children.size(); ++i) {
-        estimates.push_back(_children[i]->getState().estimate());
+    for (const Blueprint * child : _children) {
+        estimates.push_back(child->getState().estimate());
     }
     return combine(estimates);
 }
@@ -200,8 +197,8 @@ uint32_t
 IntermediateBlueprint::calculate_tree_size() const
 {
     uint32_t nodes = 1;
-    for (size_t i = 0; i < _children.size(); ++i) {
-        nodes += _children[i]->getState().tree_size();
+    for (const Blueprint * child : _children) {
+        nodes += child->getState().tree_size();
     }
     return nodes;
 }
@@ -212,8 +209,8 @@ IntermediateBlueprint::infer_allow_termwise_eval() const
     if (!supports_termwise_children()) {
         return false;
     }
-    for (size_t i = 0; i < _children.size(); ++i) {
-        if (!_children[i]->getState().allow_termwise_eval()) {
+    for (const Blueprint * child : _children) {
+        if (!child->getState().allow_termwise_eval()) {
             return false;
         }
     }
@@ -255,8 +252,8 @@ IntermediateBlueprint::mixChildrenFields() const
 
     Map fieldMap;
     FieldSpecBaseList fieldList;
-    for (size_t i = 0; i < _children.size(); ++i) {
-        const State &childState = _children[i]->getState();
+    for (const Blueprint * child : _children) {
+        const State &childState = child->getState();
         if (!childState.isTermLike()) {
             return fieldList; // empty: non-term-like child
         }
@@ -271,8 +268,8 @@ IntermediateBlueprint::mixChildrenFields() const
             }
         }
     }
-    for (MapPos pos = fieldMap.begin(); pos != fieldMap.end(); ++pos) {
-        fieldList.add(*(pos->second));
+    for (const auto & entry : fieldMap) {
+        fieldList.add(*entry.second);
     }
     return fieldList;
 }
@@ -306,8 +303,8 @@ IntermediateBlueprint::optimize(Blueprint* &self)
 {
     assert(self == this);
     if (should_optimize_children()) {
-        for (size_t i = 0; i < _children.size(); ++i) {
-            _children[i]->optimize(_children[i]);
+        for (auto & child : _children) {
+            child->optimize(child);
         }
     }
     optimize_self();
@@ -328,10 +325,7 @@ IntermediateBlueprint::createSearch(fef::MatchData &md, bool strict) const
     return createIntermediateSearch(subSearches, strict, md);
 }
 
-IntermediateBlueprint::IntermediateBlueprint()
-    : _children()
-{
-}
+IntermediateBlueprint::IntermediateBlueprint() = default;
 
 const Blueprint &
 IntermediateBlueprint::getChild(size_t n) const
@@ -396,8 +390,8 @@ IntermediateBlueprint::fetchPostings(bool strict)
 void
 IntermediateBlueprint::freeze()
 {
-    for (size_t i = 0; i < _children.size(); ++i) {
-        _children[i]->freeze();
+    for (Blueprint * child : _children) {
+        child->freeze();
     }
     freeze_self();
 }
@@ -407,7 +401,7 @@ namespace {
 bool
 areAnyParentsEquiv(const Blueprint * node)
 {
-    return (node == NULL)
+    return (node == nullptr)
            ? false
            : node->isEquiv()
              ? true
@@ -436,7 +430,8 @@ IntermediateBlueprint::calculateUnpackInfo(const fef::MatchData & md) const
                 const Blueprint & child = getChild(i);
                 const State &cs = child.getState();
                 bool canSkipUnpack(canBlueprintSkipUnpack(child, md));
-                LOG(debug, "Child[%ld] has %ld fields. canSkipUnpack='%s'.", i, cs.numFields(), canSkipUnpack ? "true" : "false");
+                LOG(debug, "Child[%ld] has %ld fields. canSkipUnpack='%s'.",
+                    i, cs.numFields(), canSkipUnpack ? "true" : "false");
                 for (size_t j = 0; canSkipUnpack && (j < cs.numFields()); ++j) {
                     if ( ! cs.field(j).resolve(md)->isNotNeeded()) {
                         LOG(debug, "Child[%ld].field(%ld).fieldId=%d need unpack.", i, j, cs.field(j).getFieldId());
@@ -469,7 +464,7 @@ LeafBlueprint::LeafBlueprint(const FieldSpecBaseList &fields, bool allow_termwis
     _state.allow_termwise_eval(allow_termwise_eval);
 }
 
-LeafBlueprint::~LeafBlueprint() { }
+LeafBlueprint::~LeafBlueprint() = default;
 
 void
 LeafBlueprint::fetchPostings(bool strict)
diff --git a/searchlib/src/vespa/searchlib/queryeval/blueprint.h b/searchlib/src/vespa/searchlib/queryeval/blueprint.h
index d9033693371..165f592867a 100644
--- a/searchlib/src/vespa/searchlib/queryeval/blueprint.h
+++ b/searchlib/src/vespa/searchlib/queryeval/blueprint.h
@@ -72,9 +72,9 @@ public:
         size_t numFields() const { return _fields.size(); }
         const FieldSpecBase &field(size_t idx) const { return _fields[idx]; }
         const FieldSpecBase *lookupField(uint32_t fieldId) const {
-            for (size_t i = 0; i < _fields.size(); ++i) {
-                if (_fields[i].getFieldId() == fieldId) {
-                    return &_fields[i];
+            for (const FieldSpecBase & field : _fields) {
+                if (field.getFieldId() == fieldId) {
+                    return &field;
                 }
             }
             return nullptr;
@@ -240,7 +240,7 @@ protected:
 public:
     typedef std::vector<size_t> IndexList;
     IntermediateBlueprint();
-    virtual ~IntermediateBlueprint();
+    ~IntermediateBlueprint() override;
 
     void setDocIdLimit(uint32_t limit) override final;
 
@@ -285,7 +285,7 @@ protected:
 
     LeafBlueprint(const FieldSpecBaseList &fields, bool allow_termwise_eval);
 public:
-    ~LeafBlueprint();
+    ~LeafBlueprint() override;
     const State &getState() const override final { return _state; }
     void setDocIdLimit(uint32_t limit) override final { Blueprint::setDocIdLimit(limit); }
     void fetchPostings(bool strict) override;
diff --git a/searchlib/src/vespa/searchlib/queryeval/fake_search.cpp b/searchlib/src/vespa/searchlib/queryeval/fake_search.cpp
index 3165cd9b68a..099e28f552a 100644
--- a/searchlib/src/vespa/searchlib/queryeval/fake_search.cpp
+++ b/searchlib/src/vespa/searchlib/queryeval/fake_search.cpp
@@ -19,7 +19,7 @@ struct FakeContext : search::attribute::ISearchContext {
     void fetchPostings(bool) override { }
     bool valid() const override { return true; }
     search::Int64Range getAsIntegerTerm() const override { abort(); }
-    const search::QueryTermBase &queryTerm() const override { abort(); }
+    const search::QueryTermBase * queryTerm() const override { abort(); }
     const vespalib::string &attributeName() const override { abort(); }
 };
 
diff --git a/searchlib/src/vespa/searchlib/queryeval/field_spec.h b/searchlib/src/vespa/searchlib/queryeval/field_spec.h
index 7da3d6a8ecc..b274e2c6751 100644
--- a/searchlib/src/vespa/searchlib/queryeval/field_spec.h
+++ b/searchlib/src/vespa/searchlib/queryeval/field_spec.h
@@ -4,12 +4,11 @@
 
 #include <vespa/searchlib/fef/handle.h>
 #include <vespa/searchlib/fef/matchdata.h>
-#include <vector>
 #include <vespa/vespalib/stllike/string.h>
+#include <vector>
 
 namespace search::queryeval {
 
-
 /**
  * Base description of a single field to be searched.
  **/
@@ -61,27 +60,22 @@ private:
 class FieldSpecBaseList
 {
 private:
-    std::vector<FieldSpecBase> _list;
+    using List = std::vector<FieldSpecBase>;
+    List _list;
 
 public:
+    using const_iterator = List::const_iterator;
     FieldSpecBaseList &add(const FieldSpecBase &spec) {
         _list.push_back(spec);
         return *this;
     }
-    bool empty() const {
-        return _list.empty();
-    }
-    size_t size() const {
-        return _list.size();
-    }
-    const FieldSpecBase &operator[](size_t i) const {
-        return _list[i];
-    }
+    bool empty() const { return _list.empty(); }
+    size_t size() const { return _list.size(); }
+    const_iterator begin() const { return _list.begin(); }
+    const_iterator end() const { return _list.end(); }
+    const FieldSpecBase &operator[](size_t i) const { return _list[i]; }
     void clear() { _list.clear(); }
-
-    void swap(FieldSpecBaseList & rhs) {
-        _list.swap(rhs._list);
-    }
+    void swap(FieldSpecBaseList & rhs) { _list.swap(rhs._list); }
 };
 
 /**
@@ -97,19 +91,11 @@ public:
         _list.push_back(spec);
         return *this;
     }
-    bool empty() const {
-        return _list.empty();
-    }
-    size_t size() const {
-        return _list.size();
-    }
-    const FieldSpec &operator[](size_t i) const {
-        return _list[i];
-    }
+    bool empty() const { return _list.empty(); }
+    size_t size() const { return _list.size(); }
+    const FieldSpec &operator[](size_t i) const { return _list[i]; }
     void clear() { _list.clear(); }
-    void swap(FieldSpecList & rhs) {
-        _list.swap(rhs._list);
-    }
+    void swap(FieldSpecList & rhs) { _list.swap(rhs._list); }
 };
 
 }
diff --git a/searchlib/src/vespa/searchlib/queryeval/hitcollector.cpp b/searchlib/src/vespa/searchlib/queryeval/hitcollector.cpp
index f9196bdaef7..2a38d8bf27e 100644
--- a/searchlib/src/vespa/searchlib/queryeval/hitcollector.cpp
+++ b/searchlib/src/vespa/searchlib/queryeval/hitcollector.cpp
@@ -4,8 +4,7 @@
 #include <vespa/searchlib/common/bitvector.h>
 #include <vespa/searchlib/common/sort.h>
 
-namespace search {
-namespace queryeval {
+namespace search::queryeval {
 
 void
 HitCollector::sortHitsByScore(size_t topn)
@@ -53,16 +52,14 @@ HitCollector::HitCollector(uint32_t numDocs,
       _needReScore(false)
 {
     if (_maxHitsSize > 0) {
-        _collector.reset(new RankedHitCollector(*this));
+        _collector = std::make_unique<RankedHitCollector>(*this);
     } else {
-        _collector.reset(new DocIdCollector<false>(*this));
+        _collector = std::make_unique<DocIdCollector<false>>(*this);
     }
     _hits.reserve(maxHitsSize);
 }
 
-HitCollector::~HitCollector()
-{
-}
+HitCollector::~HitCollector() = default;
 
 void
 HitCollector::RankedHitCollector::collect(uint32_t docId, feature_t score)
@@ -113,7 +110,7 @@ HitCollector::RankedHitCollector::collectAndChangeCollector(uint32_t docId, feat
             hc._docIdVector.push_back(hc._hits[i].first);
         }
         hc._docIdVector.push_back(docId);
-        newCollector.reset(new DocIdCollector<true>(hc));
+        newCollector = std::make_unique<DocIdCollector<true>>(hc);
     } else {
         // start using bit vector
         hc._bitVector = BitVector::create(hc._numDocs);
@@ -123,7 +120,7 @@ HitCollector::RankedHitCollector::collectAndChangeCollector(uint32_t docId, feat
             hc._bitVector->setBit(hc._hits[i].first);
         }
         hc._bitVector->setBit(docId);
-        newCollector.reset(new BitVectorCollector<true>(hc));
+        newCollector = std::make_unique<BitVectorCollector<true>>(hc);
     }
     // treat hit vector as a heap
     std::make_heap(hc._hits.begin(), hc._hits.end(), ScoreComparator());
@@ -168,54 +165,40 @@ HitCollector::DocIdCollector<CollectRankedHit>::collectAndChangeCollector(uint32
     std::vector<uint32_t> emptyVector;
     emptyVector.swap(hc._docIdVector);
     hc._bitVector->setBit(docId);
-    hc._collector.reset(new BitVectorCollector<CollectRankedHit>(hc)); // note - self-destruct.
+    hc._collector = std::make_unique<BitVectorCollector<CollectRankedHit>>(hc); // note - self-destruct.
 }
 
-std::vector<feature_t>
-HitCollector::getSortedHeapScores()
+std::vector<HitCollector::Hit>
+HitCollector::getSortedHeapHits()
 {
-    std::vector<feature_t> scores;
+    std::vector<Hit> scores;
     size_t scoresToReturn = std::min(_hits.size(), static_cast<size_t>(_maxReRankHitsSize));
     scores.reserve(scoresToReturn);
     sortHitsByScore(scoresToReturn);
     for (size_t i = 0; i < scoresToReturn; ++i) {
-        scores.push_back(_hits[_scoreOrder[i]].second);
+        scores.push_back(_hits[_scoreOrder[i]]);
     }
     return scores;
 }
 
 size_t
-HitCollector::reRank(DocumentScorer &scorer)
-{
-    return reRank(scorer, _maxReRankHitsSize);
-}
-
-size_t
-HitCollector::reRank(DocumentScorer &scorer, size_t count)
-{
-    size_t hitsToReRank = std::min(_hits.size(), count);
-    if (_hasReRanked || hitsToReRank == 0) {
-        return 0;
-    }
-    sortHitsByScore(hitsToReRank);
-    _reRankedHits.reserve(_reRankedHits.size() + hitsToReRank);
-    for (size_t i(0); i < hitsToReRank; i++) {
-        _reRankedHits.push_back(_hits[_scoreOrder[i]]);
-    }
-
+HitCollector::reRank(DocumentScorer &scorer, std::vector<Hit> hits) {
+    if (hits.empty()) { return 0; }
+    
+    size_t hitsToReRank = hits.size();
     Scores &initScores = _ranges.first;
     Scores &finalScores = _ranges.second;
-    initScores = Scores(_reRankedHits.back().second,
-                        _reRankedHits.front().second);
+    initScores = Scores(hits.back().second, hits.front().second);
     finalScores = Scores(std::numeric_limits<feature_t>::max(),
                          -std::numeric_limits<feature_t>::max());
 
-    std::sort(_reRankedHits.begin(), _reRankedHits.end()); // sort on docId
-    for (auto &hit : _reRankedHits) {
+    std::sort(hits.begin(), hits.end()); // sort on docId
+    for (auto &hit : hits) {
         hit.second = scorer.score(hit.first);
         finalScores.low = std::min(finalScores.low, hit.second);
         finalScores.high = std::max(finalScores.high, hit.second);
     }
+    _reRankedHits = std::move(hits);
     _hasReRanked = true;
     return hitsToReRank;
 }
@@ -335,5 +318,4 @@ HitCollector::getResultSet(HitRank default_value)
     return rs;
 }
 
-} // namespace queryeval
-} // namespace search
+}
diff --git a/searchlib/src/vespa/searchlib/queryeval/hitcollector.h b/searchlib/src/vespa/searchlib/queryeval/hitcollector.h
index 1bf2bc21e95..ef4bf76b66b 100644
--- a/searchlib/src/vespa/searchlib/queryeval/hitcollector.h
+++ b/searchlib/src/vespa/searchlib/queryeval/hitcollector.h
@@ -10,16 +10,14 @@
 #include <vespa/vespalib/util/sort.h>
 #include <vespa/fastos/dynamiclibrary.h>
 
-namespace search {
-
-namespace queryeval {
+namespace search::queryeval {
 
 /**
  * This class is used to store all hits found during parallel query evaluation.
  **/
 class HitCollector {
 public:
-    typedef std::pair<uint32_t, feature_t> Hit;
+    using Hit = std::pair<uint32_t, feature_t>;
 
     /**
      * Interface used to calculate the second phase score for the documents being re-ranked.
@@ -169,18 +167,17 @@ public:
     }
 
     /**
-     * Returns a sorted vector of scores for the hits that are stored
+     * Returns a sorted vector of hits for the hits that are stored
      * in the heap. These are the candidates for re-ranking.
      */
-    std::vector<feature_t> getSortedHeapScores();
+    std::vector<Hit> getSortedHeapHits();
 
     /**
      * Re-ranks the m (=maxHeapSize) best hits by invoking the score()
      * method on the given document scorer. The best m hits are sorted on doc id
      * so that score() is called in doc id order.
      **/
-    size_t reRank(DocumentScorer &scorer);
-    size_t reRank(DocumentScorer &scorer, size_t count);
+    size_t reRank(DocumentScorer &scorer, std::vector<Hit> hits);
 
     std::pair<Scores, Scores> getRanges() const;
     void setRanges(const std::pair<Scores, Scores> &ranges);
@@ -200,6 +197,4 @@ private:
     HitCollector &operator=(const HitCollector &);  // Not implemented
 };
 
-} // namespace queryeval
-} // namespace search
-
+}
diff --git a/searchlib/src/vespa/searchlib/queryeval/idiversifier.h b/searchlib/src/vespa/searchlib/queryeval/idiversifier.h
new file mode 100644
index 00000000000..e77cb959eeb
--- /dev/null
+++ b/searchlib/src/vespa/searchlib/queryeval/idiversifier.h
@@ -0,0 +1,16 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <cstdint>
+
+namespace search::queryeval {
+
+struct IDiversifier {
+    virtual ~IDiversifier() {}
+    /**
+     * Will tell if this document should be kept, and update state for further filtering.
+     */
+    virtual bool accepted(uint32_t docId) = 0;
+};
+}
diff --git a/searchlib/src/vespa/searchlib/queryeval/intermediate_blueprints.cpp b/searchlib/src/vespa/searchlib/queryeval/intermediate_blueprints.cpp
index 9bc9d3161e6..5cb51aa4fd0 100644
--- a/searchlib/src/vespa/searchlib/queryeval/intermediate_blueprints.cpp
+++ b/searchlib/src/vespa/searchlib/queryeval/intermediate_blueprints.cpp
@@ -302,7 +302,7 @@ OrBlueprint::createIntermediateSearch(const MultiSearch::Children &subSearches,
 }
 
 //-----------------------------------------------------------------------------
-WeakAndBlueprint::~WeakAndBlueprint() {}
+WeakAndBlueprint::~WeakAndBlueprint() = default;
 
 Blueprint::HitEstimate
 WeakAndBlueprint::combine(const std::vector<HitEstimate> &data) const
diff --git a/searchlib/src/vespa/searchlib/queryeval/isourceselector.cpp b/searchlib/src/vespa/searchlib/queryeval/isourceselector.cpp
index fa8f465500e..1e0659c92e3 100644
--- a/searchlib/src/vespa/searchlib/queryeval/isourceselector.cpp
+++ b/searchlib/src/vespa/searchlib/queryeval/isourceselector.cpp
@@ -1,8 +1,7 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-#include <vespa/searchlib/queryeval/isourceselector.h>
+#include "isourceselector.h"
 
-namespace search {
-namespace queryeval {
+namespace search::queryeval {
 
 ISourceSelector::ISourceSelector(Source defaultSource) :
     _baseId(0),
@@ -12,5 +11,3 @@ ISourceSelector::ISourceSelector(Source defaultSource) :
 }
 
 }
-
-}
diff --git a/searchlib/src/vespa/searchlib/queryeval/isourceselector.h b/searchlib/src/vespa/searchlib/queryeval/isourceselector.h
index a3eac806558..88a3cb57a8a 100644
--- a/searchlib/src/vespa/searchlib/queryeval/isourceselector.h
+++ b/searchlib/src/vespa/searchlib/queryeval/isourceselector.h
@@ -2,11 +2,9 @@
 
 #pragma once
 
-#include <stdint.h>
 #include <vespa/searchlib/attribute/singlenumericattribute.h>
 
-namespace search {
-namespace queryeval {
+namespace search::queryeval {
 
 typedef uint8_t Source;
 
@@ -91,6 +89,4 @@ private:
     Source   _defaultSource;
 };
 
-} // namespace queryeval
-} // namespace search
-
+}
diff --git a/searchlib/src/vespa/searchlib/queryeval/scores.h b/searchlib/src/vespa/searchlib/queryeval/scores.h
index 4fb2de5d6fa..e8dae898909 100644
--- a/searchlib/src/vespa/searchlib/queryeval/scores.h
+++ b/searchlib/src/vespa/searchlib/queryeval/scores.h
@@ -4,8 +4,7 @@
 
 #include <vespa/searchlib/common/feature.h>
 
-namespace search {
-namespace queryeval {
+namespace search::queryeval {
 
 struct Scores {
     feature_t low;
@@ -16,6 +15,4 @@ struct Scores {
     bool isValid() const { return low <= high; }
 };
 
-}  // namespace queryeval
-}  // namespace search
-
+}
diff --git a/searchlib/src/vespa/searchlib/transactionlog/domain.cpp b/searchlib/src/vespa/searchlib/transactionlog/domain.cpp
index 88c2dd9ecc3..1caff132779 100644
--- a/searchlib/src/vespa/searchlib/transactionlog/domain.cpp
+++ b/searchlib/src/vespa/searchlib/transactionlog/domain.cpp
@@ -3,6 +3,7 @@
 #include "domain.h"
 #include <vespa/vespalib/util/stringfmt.h>
 #include <vespa/vespalib/util/closuretask.h>
+#include <vespa/vespalib/io/fileutil.h>
 #include <vespa/fastos/file.h>
 #include <algorithm>
 #include <thread>
@@ -60,6 +61,7 @@ Domain::Domain(const string &domainName, const string & baseDir, Executor & comm
     _sessionExecutor.sync();
     if (_parts.empty() || _parts.crbegin()->second->isClosed()) {
         _parts[lastPart].reset(new DomainPart(_name, dir(), lastPart, _defaultCrcType, _fileHeaderContext, false));
+        vespalib::File::sync(dir());
     }
 }
 
@@ -294,6 +296,7 @@ void Domain::commit(const Packet & packet)
             _parts[entry.serial()] = dp;
         }
         dp = _parts.rbegin()->second;
+        vespalib::File::sync(dir());
     }
     dp->commit(entry.serial(), packet);
     cleanSessions();
@@ -310,6 +313,7 @@ bool Domain::erase(SerialNum to)
             _parts.erase(it);
         }
         retval = retval && dp->erase(to);
+        vespalib::File::sync(dir());
     }
     if (_parts.begin()->second->range().to() >= to) {
         _parts.begin()->second->erase(to);
diff --git a/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp b/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp
index 4c3c5609a93..bfca137ba06 100644
--- a/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp
+++ b/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp
@@ -336,6 +336,29 @@ void TransLogServer::exportRPC(FRT_Supervisor & supervisor)
     rb.ReturnDesc("syncedto", "Entry synced to");
 }
 
+namespace {
+
+void
+writeDomainDir(std::lock_guard<std::mutex> &guard,
+               vespalib::string dir,
+               vespalib::string domainList,
+               std::map<vespalib::string, std::shared_ptr<Domain>> &domains)
+{
+    (void) guard;
+    vespalib::string domainListTmp(domainList + ".tmp");
+    vespalib::unlink(domainListTmp);
+    std::ofstream domainDir(domainListTmp.c_str(), std::ios::trunc);
+    for (const auto &domainEntry : domains) {
+        domainDir << domainEntry.first << std::endl;
+    }
+    domainDir.close();
+    vespalib::File::sync(domainListTmp);
+    vespalib::rename(domainListTmp, domainList, false, false);
+    vespalib::File::sync(dir);
+}
+
+}
+
 void TransLogServer::createDomain(FRT_RPCRequest *req)
 {
     uint32_t retval(0);
@@ -351,12 +374,9 @@ void TransLogServer::createDomain(FRT_RPCRequest *req)
         try {
             domain = std::make_shared<Domain>(domainName, dir(), _commitExecutor, _sessionExecutor,
                                               _domainPartSize, _defaultCrcType, _fileHeaderContext);
-            {
-                Guard domainGuard(_lock);
-                _domains[domain->name()] = domain;
-            }
-            std::ofstream domainDir(domainList().c_str(), std::ios::app);
-            domainDir << domain->name() << std::endl;
+            Guard domainGuard(_lock);
+            _domains[domain->name()] = domain;
+            writeDomainDir(domainGuard, dir(), domainList(), _domains);
         } catch (const std::exception & e) {
             LOG(warning, "Failed creating %s domain. Exception = %s", domainName, e.what());
             retval = uint32_t(-1);
@@ -385,12 +405,10 @@ void TransLogServer::deleteDomain(FRT_RPCRequest *req)
                 Guard domainGuard(_lock);
                 _domains.erase(domainName);
             }
-            vespalib::rmdir(Domain::getDir(dir(), domainName).c_str(), true);
-            std::ofstream domainDir(domainList().c_str(), std::ios::trunc);
+            vespalib::rmdir(Domain::getDir(dir(), domainName), true);
+            vespalib::File::sync(dir());
             Guard domainGuard(_lock);
-            for (DomainList::const_iterator it(_domains.begin()), mt(_domains.end()); it != mt; it++) {
-                domainDir << it->first << std::endl;
-            }
+            writeDomainDir(domainGuard, dir(), domainList(), _domains);
         } catch (const std::exception & e) {
             msg = make_string("Failed deleting %s domain. Exception = %s", domainName, e.what());
             retval = -1;
diff --git a/searchlib/src/vespa/searchlib/uca/ucaconverter.cpp b/searchlib/src/vespa/searchlib/uca/ucaconverter.cpp
index fffcc782298..b36e9d13cdb 100644
--- a/searchlib/src/vespa/searchlib/uca/ucaconverter.cpp
+++ b/searchlib/src/vespa/searchlib/uca/ucaconverter.cpp
@@ -32,7 +32,7 @@ UcaConverter::UcaConverter(vespalib::stringref locale, vespalib::stringref stren
     Collator *coll(NULL);
     {
         std::lock_guard<std::mutex> guard(_GlobalDirtyICUThreadSafeLock);
-        coll = Collator::createInstance(icu::Locale(locale.c_str()), status);
+        coll = Collator::createInstance(icu::Locale(locale.data()), status);
     }
     if(U_SUCCESS(status)) {
         _collator.reset(coll);
diff --git a/searchlib/src/vespa/searchlib/util/filekit.cpp b/searchlib/src/vespa/searchlib/util/filekit.cpp
index 210a9c30134..7fc4ca8e7bb 100644
--- a/searchlib/src/vespa/searchlib/util/filekit.cpp
+++ b/searchlib/src/vespa/searchlib/util/filekit.cpp
@@ -12,7 +12,7 @@ namespace search {
 using vespalib::getLastErrorString;
 
 bool
-FileKit::createStamp(const vespalib::stringref &name)
+FileKit::createStamp(const vespalib::string &name)
 {
     FastOS_File stamp;
     FastOS_StatInfo statInfo;
@@ -40,7 +40,7 @@ FileKit::createStamp(const vespalib::stringref &name)
 
 
 bool
-FileKit::hasStamp(const vespalib::stringref &name)
+FileKit::hasStamp(const vespalib::string &name)
 {
     FastOS_StatInfo statInfo;
     bool statres;
@@ -57,7 +57,7 @@ FileKit::hasStamp(const vespalib::stringref &name)
 
 
 bool
-FileKit::removeStamp(const vespalib::stringref &name)
+FileKit::removeStamp(const vespalib::string &name)
 {
     FastOS_StatInfo statInfo;
     bool deleteres;
@@ -91,7 +91,7 @@ FileKit::removeStamp(const vespalib::stringref &name)
 
 
 fastos::TimeStamp
-FileKit::getModificationTime(const vespalib::stringref &name)
+FileKit::getModificationTime(const vespalib::string &name)
 {
     FastOS_StatInfo statInfo;
     if (FastOS_File::Stat(name.c_str(), &statInfo)) {
diff --git a/searchlib/src/vespa/searchlib/util/filekit.h b/searchlib/src/vespa/searchlib/util/filekit.h
index 70acf19c70c..929412ee6d8 100644
--- a/searchlib/src/vespa/searchlib/util/filekit.h
+++ b/searchlib/src/vespa/searchlib/util/filekit.h
@@ -12,15 +12,15 @@ class FileKit
 private:
     static bool _syncFiles;
 public:
-    static bool createStamp(const vespalib::stringref &name);
-    static bool hasStamp(const vespalib::stringref &name);
-    static bool removeStamp(const vespalib::stringref &name);
+    static bool createStamp(const vespalib::string &name);
+    static bool hasStamp(const vespalib::string &name);
+    static bool removeStamp(const vespalib::string &name);
 
     /**
      * Returns the modification time of the given file/directory,
      * or time stamp 0 if stating of file/directory fails.
      **/
-    static fastos::TimeStamp getModificationTime(const vespalib::stringref &name);
+    static fastos::TimeStamp getModificationTime(const vespalib::string &name);
 };
 
 }
diff --git a/searchsummary/src/vespa/searchsummary/docsummary/dynamicteaserdfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/dynamicteaserdfw.cpp
index c7eb63a4480..73c62db864c 100644
--- a/searchsummary/src/vespa/searchsummary/docsummary/dynamicteaserdfw.cpp
+++ b/searchsummary/src/vespa/searchsummary/docsummary/dynamicteaserdfw.cpp
@@ -159,7 +159,7 @@ public:
     {
         if (item->_si != NULL) {
             *len = item->_si->getIndexName().size();
-            return item->_si->getIndexName().c_str();
+            return item->_si->getIndexName().data();
         } else {
             *len = item->_data->_indexlen;
             return item->_data->_index;
@@ -221,7 +221,7 @@ JuniperQueryAdapter::Traverse(juniper::IQueryVisitor *v) const
         case search::ParseItem::ITEM_PURE_WEIGHTED_STRING:
             {
                 vespalib::stringref term = iterator.getTerm();
-                v->VisitKeyword(&item, term.c_str(), term.size(), false, isSpecialToken);
+                v->VisitKeyword(&item, term.data(), term.size(), false, isSpecialToken);
             }
             break;
         case search::ParseItem::ITEM_NUMTERM:
@@ -257,7 +257,7 @@ JuniperQueryAdapter::Traverse(juniper::IQueryVisitor *v) const
         case search::ParseItem::ITEM_SUBSTRINGTERM:
             {
                 vespalib::stringref term = iterator.getTerm();
-                v->VisitKeyword(&item, term.c_str(), term.size(), true, isSpecialToken);
+                v->VisitKeyword(&item, term.data(), term.size(), true, isSpecialToken);
             }
             break;
         case search::ParseItem::ITEM_ANY:
diff --git a/searchsummary/src/vespa/searchsummary/docsummary/itokenizer.h b/searchsummary/src/vespa/searchsummary/docsummary/itokenizer.h
index 62f2510c17b..15a367761ec 100644
--- a/searchsummary/src/vespa/searchsummary/docsummary/itokenizer.h
+++ b/searchsummary/src/vespa/searchsummary/docsummary/itokenizer.h
@@ -37,7 +37,7 @@ public:
             _text(textBegin, textEnd - textBegin), _stem(stemBegin, stemEnd - stemBegin), _type(type) {}
         const vespalib::stringref & getText() const { return _text; }
         const vespalib::stringref & getStem() const { return _stem; }
-        bool hasStem() const { return _stem.c_str() != NULL; }
+        bool hasStem() const { return _stem.data() != NULL; }
         Type getType() const { return _type; }
     };
 
diff --git a/searchsummary/src/vespa/searchsummary/docsummary/keywordextractor.cpp b/searchsummary/src/vespa/searchsummary/docsummary/keywordextractor.cpp
index 3a60db52cf3..d75ca47dd33 100644
--- a/searchsummary/src/vespa/searchsummary/docsummary/keywordextractor.cpp
+++ b/searchsummary/src/vespa/searchsummary/docsummary/keywordextractor.cpp
@@ -195,7 +195,7 @@ KeywordExtractor::ExtractKeywords(vespalib::stringref buf) const
                             phraseterms_was_added = true;
                         }
 
-                        keywords.append(term.c_str(), term.size());
+                        keywords.append(term.data(), term.size());
                     }
                 }
             }
@@ -218,7 +218,7 @@ KeywordExtractor::ExtractKeywords(vespalib::stringref buf) const
                 vespalib::stringref term = si.getTerm();
                 if ( !term.empty() && useful(creator)) {
                     // An actual string to add
-                    keywords.append(term.c_str(), term.size());
+                    keywords.append(term.data(), term.size());
                     keywords.append("\0", 1);
                 }
             }
diff --git a/searchsummary/src/vespa/searchsummary/docsummary/rankfeaturesdfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/rankfeaturesdfw.cpp
index fdc38d8d6f9..9748bdac3b3 100644
--- a/searchsummary/src/vespa/searchsummary/docsummary/rankfeaturesdfw.cpp
+++ b/searchsummary/src/vespa/searchsummary/docsummary/rankfeaturesdfw.cpp
@@ -8,10 +8,10 @@
 namespace search::docsummary {
 
 RankFeaturesDFW::RankFeaturesDFW() :
-    _env(NULL)
+    _env(nullptr)
 { }
 
-RankFeaturesDFW::~RankFeaturesDFW() { }
+RankFeaturesDFW::~RankFeaturesDFW() = default;
 
 void
 RankFeaturesDFW::init(IDocsumEnvironment * env)
@@ -23,15 +23,15 @@ void
 RankFeaturesDFW::insertField(uint32_t docid, GeneralResult *, GetDocsumsState *state,
                              ResType type, vespalib::slime::Inserter &target)
 {
-    if (state->_rankFeatures.get() == NULL) {
+    if (state->_rankFeatures.get() == nullptr) {
         state->_callback.FillRankFeatures(state, _env);
-        if (state->_rankFeatures.get() == NULL) { // still no rank features to write
+        if (state->_rankFeatures.get() == nullptr) { // still no rank features to write
             return;
         }
     }
     const FeatureSet::StringVector & names = state->_rankFeatures->getNames();
     const feature_t * values = state->_rankFeatures->getFeaturesByDocId(docid);
-    if (type == RES_FEATUREDATA && values != NULL) {
+    if (type == RES_FEATUREDATA && values != nullptr) {
         vespalib::slime::Cursor& obj = target.insertObject();
         for (uint32_t i = 0; i < names.size(); ++i) {
             vespalib::Memory name(names[i].c_str(), names[i].size());
@@ -40,14 +40,14 @@ RankFeaturesDFW::insertField(uint32_t docid, GeneralResult *, GetDocsumsState *s
         return;
     }
     vespalib::JSONStringer & json(state->_jsonStringer);
-    if (values != NULL) {
+    if (values != nullptr) {
         json.clear();
         json.beginObject();
         for (uint32_t i = 0; i < names.size(); ++i) {
             featureDump(json, names[i], values[i]);
         }
         json.endObject();
-        vespalib::Memory value(json.toString().c_str(),
+        vespalib::Memory value(json.toString().data(),
                                       json.toString().size());
         if (type == RES_STRING || type == RES_LONG_STRING) {
             target.insertString(value);
diff --git a/searchsummary/src/vespa/searchsummary/docsummary/summaryfeaturesdfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/summaryfeaturesdfw.cpp
index c21d6c40066..a5fb9bd8539 100644
--- a/searchsummary/src/vespa/searchsummary/docsummary/summaryfeaturesdfw.cpp
+++ b/searchsummary/src/vespa/searchsummary/docsummary/summaryfeaturesdfw.cpp
@@ -68,7 +68,7 @@ SummaryFeaturesDFW::insertField(uint32_t docid, GeneralResult *, GetDocsumsState
             json.appendDouble(0.0);
         }
         json.endObject();
-        vespalib::Memory value(json.toString().c_str(), json.toString().size());
+        vespalib::Memory value(json.toString().data(), json.toString().size());
         if (type == RES_STRING || type == RES_LONG_STRING) {
             target.insertString(value);
         }
diff --git a/staging_vespalib/src/tests/memorydatastore/memorydatastore.cpp b/staging_vespalib/src/tests/memorydatastore/memorydatastore.cpp
index f6427f8acd2..7d047e36566 100644
--- a/staging_vespalib/src/tests/memorydatastore/memorydatastore.cpp
+++ b/staging_vespalib/src/tests/memorydatastore/memorydatastore.cpp
@@ -40,20 +40,20 @@ MemoryDataStoreTest::testVariableSizeVector()
     for (size_t i(0); i < 10000; i++) {
         asciistream os;
         os << i;
-        v.push_back(os.str().c_str(), os.str().size());
+        v.push_back(os.str().data(), os.str().size());
     }
     for (size_t i(0); i < v.size(); i++) {
         asciistream os;
         os << i;
         EXPECT_EQUAL(os.str().size(), v[i].size());
-        EXPECT_EQUAL(0, memcmp(os.str().c_str(), v[i].data(), os.str().size()));
+        EXPECT_EQUAL(0, memcmp(os.str().data(), v[i].data(), os.str().size()));
     }
     size_t i(0);
     for (auto it(v.begin()), mt(v.end()); it != mt; it++, i++) {
         asciistream os;
         os << i;
         EXPECT_EQUAL(os.str().size(), it->size());
-        EXPECT_EQUAL(0, memcmp(os.str().c_str(), (*it).data(), os.str().size()));
+        EXPECT_EQUAL(0, memcmp(os.str().data(), (*it).data(), os.str().size()));
     }
     
 }
diff --git a/staging_vespalib/src/tests/stllike/cache_test.cpp b/staging_vespalib/src/tests/stllike/cache_test.cpp
index 142c4065673..6e852644602 100644
--- a/staging_vespalib/src/tests/stllike/cache_test.cpp
+++ b/staging_vespalib/src/tests/stllike/cache_test.cpp
@@ -28,36 +28,10 @@ public:
     }
 };
 
-class Test : public TestApp
-{
-public:
-    int Main() override;
-private:
-    typedef LruParam<uint32_t, string> P;
-    typedef Map<uint32_t, string> B;
-    void testCache();
-    void testCacheSize();
-    void testCacheSizeDeep();
-    void testCacheEntriesHonoured();
-    void testCacheMaxSizeHonoured();
-    void testThatMultipleRemoveOnOverflowIsFine();
-};
-
-int
-Test::Main()
-{
-    TEST_INIT("cache_test");
-    testCache();
-    testCacheSize();
-    testCacheSizeDeep();
-    testCacheEntriesHonoured();
-    testCacheMaxSizeHonoured();
-    testThatMultipleRemoveOnOverflowIsFine();
-    TEST_DONE();
-}
+using P = LruParam<uint32_t, string>;
+using B = Map<uint32_t, string>;
 
-void Test::testCache()
-{
+TEST("testCache") {
     B m;
     cache< CacheParam<P, B> > cache(m, -1);
     // Verfify start conditions.
@@ -74,24 +48,29 @@ void Test::testCache()
     EXPECT_TRUE(cache.size() == 1);
 }
 
-void Test::testCacheSize()
+TEST("testCacheSize")
 {
     B m;
     cache< CacheParam<P, B> > cache(m, -1);
     cache.write(1, "10 bytes string");
     EXPECT_EQUAL(80u, cache.sizeBytes());
+    cache.write(1, "10 bytes string"); // Still the same size
+    EXPECT_EQUAL(80u, cache.sizeBytes());
 }
 
-void Test::testCacheSizeDeep()
+TEST("testCacheSizeDeep")
 {
     B m;
     cache< CacheParam<P, B, zero<uint32_t>, size<string> > > cache(m, -1);
     cache.write(1, "15 bytes string");
     EXPECT_EQUAL(95u, cache.sizeBytes());
+    cache.write(1, "10 bytes s");
+    EXPECT_EQUAL(90u, cache.sizeBytes());
+    cache.write(1, "20 bytes string ssss");
+    EXPECT_EQUAL(100u, cache.sizeBytes());
 }
 
-void Test::testCacheEntriesHonoured()
-{
+TEST("testCacheEntriesHonoured") {
     B m;
     cache< CacheParam<P, B, zero<uint32_t>, size<string> > > cache(m, -1);
     cache.maxElements(1);
@@ -105,8 +84,7 @@ void Test::testCacheEntriesHonoured()
     EXPECT_EQUAL(96u, cache.sizeBytes());
 }
 
-void Test::testCacheMaxSizeHonoured()
-{
+TEST("testCacheMaxSizeHonoured") {
     B m;
     cache< CacheParam<P, B, zero<uint32_t>, size<string> > > cache(m, 200);
     cache.write(1, "15 bytes string");
@@ -123,8 +101,7 @@ void Test::testCacheMaxSizeHonoured()
     EXPECT_EQUAL(291u, cache.sizeBytes());
 }
 
-void Test::testThatMultipleRemoveOnOverflowIsFine()
-{
+TEST("testThatMultipleRemoveOnOverflowIsFine") {
     B m;
     cache< CacheParam<P, B, zero<uint32_t>, size<string> > > cache(m, 2000);
     
@@ -159,5 +136,4 @@ void Test::testThatMultipleRemoveOnOverflowIsFine()
     EXPECT_EQUAL(2924u, cache.sizeBytes());
 }
 
-
-TEST_APPHOOK(Test)
+TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/staging_vespalib/src/vespa/vespalib/objects/fieldbase.h b/staging_vespalib/src/vespa/vespalib/objects/fieldbase.h
index cdb9ae95c72..d10687a36ad 100644
--- a/staging_vespalib/src/vespa/vespalib/objects/fieldbase.h
+++ b/staging_vespalib/src/vespa/vespalib/objects/fieldbase.h
@@ -9,6 +9,7 @@ class IFieldBase
 {
 public:
     virtual ~IFieldBase() { }
+    // Overrides must guarantee that returned reference is zero-terminated.
     virtual stringref getName() const = 0;
 };
 
diff --git a/staging_vespalib/src/vespa/vespalib/objects/identifiable.h b/staging_vespalib/src/vespa/vespalib/objects/identifiable.h
index dfa59e40d7f..4a87dde82e4 100644
--- a/staging_vespalib/src/vespa/vespalib/objects/identifiable.h
+++ b/staging_vespalib/src/vespa/vespalib/objects/identifiable.h
@@ -171,7 +171,7 @@ public:
         virtual void loadClass(unsigned classId) = 0;
         virtual void loadClass(const char * className) = 0;
     };
-    struct RuntimeClass : public IFieldBase {
+    struct RuntimeClass final : public IFieldBase {
     public:
         RuntimeClass(RuntimeInfo * info);
         ~RuntimeClass();
diff --git a/staging_vespalib/src/vespa/vespalib/stllike/cache.h b/staging_vespalib/src/vespa/vespalib/stllike/cache.h
index 3d5ab155877..99b601f7c3c 100644
--- a/staging_vespalib/src/vespa/vespalib/stllike/cache.h
+++ b/staging_vespalib/src/vespa/vespalib/stllike/cache.h
@@ -101,7 +101,7 @@ public:
      * Update the cache and write through to backing store.
      * Object is then put at head of LRU list.
      */
-    void write(const K & key, const V & value);
+    void write(const K & key, V value);
 
     /**
      * Tell if an object with given key exists in the cache.
@@ -148,6 +148,7 @@ private:
     mutable size_t      _race;
     mutable size_t      _insert;
     mutable size_t      _write;
+    mutable size_t      _update;
     mutable size_t      _erase;
     mutable size_t      _invalidate;
     mutable size_t      _lookup;
diff --git a/staging_vespalib/src/vespa/vespalib/stllike/cache.hpp b/staging_vespalib/src/vespa/vespalib/stllike/cache.hpp
index a8c7d16473c..906621d623c 100644
--- a/staging_vespalib/src/vespa/vespalib/stllike/cache.hpp
+++ b/staging_vespalib/src/vespa/vespalib/stllike/cache.hpp
@@ -61,6 +61,7 @@ cache<P>::cache(BackingStore & b, size_t maxBytes) :
     _race(0),
     _insert(0),
     _write(0),
+    _update(0),
     _erase(0),
     _invalidate(0),
     _lookup(0),
@@ -120,16 +121,25 @@ cache<P>::read(const K & key)
 
 template< typename P >
 void
-cache<P>::write(const K & key, const V & value)
+cache<P>::write(const K & key, V value)
 {
+    size_t newSize = calcSize(key, value);
     vespalib::LockGuard storeGuard(getLock(key));
     {
         vespalib::LockGuard guard(_hashLock);
-        (*this)[key] = value;
-        _sizeBytes += calcSize(key, value);
-        _write++;
+        if (Lru::hasKey(key)) {
+            _sizeBytes -= calcSize(key, (*this)[key]);
+            _update++;
+        }
     }
+
     _store.write(key, value);
+    {
+        vespalib::LockGuard guard(_hashLock);
+        (*this)[key] = std::move(value);
+        _sizeBytes += newSize;
+        _write++;
+    }
 }
 
 template< typename P >
diff --git a/staging_vespalib/src/vespa/vespalib/util/growablebytebuffer.cpp b/staging_vespalib/src/vespa/vespalib/util/growablebytebuffer.cpp
index 53875fe7241..22064864b22 100644
--- a/staging_vespalib/src/vespa/vespalib/util/growablebytebuffer.cpp
+++ b/staging_vespalib/src/vespa/vespalib/util/growablebytebuffer.cpp
@@ -72,7 +72,7 @@ void
 GrowableByteBuffer::putString(const vespalib::stringref& v)
 {
     putInt(v.size());
-    putBytes(v.c_str(), v.size());
+    putBytes(v.data(), v.size());
 }
 
 void
diff --git a/staging_vespalib/src/vespa/vespalib/util/jsonwriter.cpp b/staging_vespalib/src/vespa/vespalib/util/jsonwriter.cpp
index ebeda4f1b8b..0ad52f9aac2 100644
--- a/staging_vespalib/src/vespa/vespalib/util/jsonwriter.cpp
+++ b/staging_vespalib/src/vespa/vespalib/util/jsonwriter.cpp
@@ -177,7 +177,7 @@ JSONWriter::appendKey(const vespalib::stringref & str)
 {
     considerComma();
     indent();
-    quote(str.c_str(), str.size());
+    quote(str.data(), str.size());
     (*_os) << ':';
     _comma = false;
     return *this;
@@ -246,7 +246,7 @@ JSONWriter &
 JSONWriter::appendString(const vespalib::stringref & str)
 {
     considerComma();
-    quote(str.c_str(), str.size());
+    quote(str.data(), str.size());
     updateCommaState();
     return *this;
 }
diff --git a/standalone-container/src/main/sh/standalone-container.sh b/standalone-container/src/main/sh/standalone-container.sh
index dc1f5c7b71e..f9eddafbfbb 100755
--- a/standalone-container/src/main/sh/standalone-container.sh
+++ b/standalone-container/src/main/sh/standalone-container.sh
@@ -157,7 +157,7 @@ StartCommand() {
         -XX:+PreserveFramePointer \
         -XX:+HeapDumpOnOutOfMemoryError \
         -XX:HeapDumpPath="$VESPA_HOME/var/crash" \
-        -XX:OnOutOfMemoryError="kill -9 %p" \
+        -XX:+ExitOnOutOfMemoryError \
         -Djava.library.path="$VESPA_HOME/lib64" \
         -Djava.awt.headless=true \
 	-Dsun.rmi.dgc.client.gcInterval=3600000 \
diff --git a/standalone-container/vespa-standalone-container.spec b/standalone-container/vespa-standalone-container.spec
index 05a126e49b3..281516ae552 100644
--- a/standalone-container/vespa-standalone-container.spec
+++ b/standalone-container/vespa-standalone-container.spec
@@ -48,6 +48,7 @@ declare -a modules=(
   container-disc
   container-jersey2
   container-search-and-docproc
+  container-search-gui
   defaults
   docprocs
   jdisc-security-filters
diff --git a/storage/src/tests/bucketdb/bucketinfotest.cpp b/storage/src/tests/bucketdb/bucketinfotest.cpp
index 3eb8d60befd..0298c50866c 100644
--- a/storage/src/tests/bucketdb/bucketinfotest.cpp
+++ b/storage/src/tests/bucketdb/bucketinfotest.cpp
@@ -51,14 +51,14 @@ getBucketInfo(std::string nodeList, std::string order) {
     {
         vespalib::StringTokenizer tokenizer(order, ",");
         for (uint32_t i = 0; i < tokenizer.size(); i++) {
-            ordering.push_back(atoi(tokenizer[i].c_str()));
+            ordering.push_back(atoi(tokenizer[i].data()));
         }
     }
 
     vespalib::StringTokenizer tokenizer(nodeList, ",");
     for (uint32_t i = 0; i < tokenizer.size(); i++) {
         info.addNode(BucketCopy(0,
-                                atoi(tokenizer[i].c_str()),
+                                atoi(tokenizer[i].data()),
                                 api::BucketInfo(1,1,1)),
                      ordering);
     }
diff --git a/storage/src/tests/distributor/bucketdbupdatertest.cpp b/storage/src/tests/distributor/bucketdbupdatertest.cpp
index 559afffc795..56f88b7f98f 100644
--- a/storage/src/tests/distributor/bucketdbupdatertest.cpp
+++ b/storage/src/tests/distributor/bucketdbupdatertest.cpp
@@ -1800,7 +1800,7 @@ parseInputData(const std::string& data,
     for (uint32_t i = 0; i < tokenizer.size(); i++) {
         vespalib::StringTokenizer tok2(tokenizer[i], ":");
 
-        uint16_t node = atoi(tok2[0].c_str());
+        uint16_t node = atoi(tok2[0].data());
 
         state.setNodeReplied(node);
         auto &pendingTransition = state.getPendingBucketSpaceDbTransition(makeBucketSpace());
@@ -1811,19 +1811,19 @@ parseInputData(const std::string& data,
                 vespalib::StringTokenizer tok4(tok3[j], "/");
 
                 pendingTransition.addNodeInfo(
-                        document::BucketId(16, atoi(tok4[0].c_str())),
+                        document::BucketId(16, atoi(tok4[0].data())),
                         BucketCopy(
                                 timestamp,
                                 node,
                                 api::BucketInfo(
-                                        atoi(tok4[1].c_str()),
-                                        atoi(tok4[2].c_str()),
-                                        atoi(tok4[3].c_str()),
-                                        atoi(tok4[2].c_str()),
-                                        atoi(tok4[3].c_str()))));
+                                        atoi(tok4[1].data()),
+                                        atoi(tok4[2].data()),
+                                        atoi(tok4[3].data()),
+                                        atoi(tok4[2].data()),
+                                        atoi(tok4[3].data()))));
             } else {
                 pendingTransition.addNodeInfo(
-                        document::BucketId(16, atoi(tok3[j].c_str())),
+                        document::BucketId(16, atoi(tok3[j].data())),
                         BucketCopy(timestamp,
                                    node,
                                    api::BucketInfo(3, 3, 3, 3, 3)));
diff --git a/storage/src/tests/distributor/distributortest.cpp b/storage/src/tests/distributor/distributortest.cpp
index ce20546dd44..46c756001d9 100644
--- a/storage/src/tests/distributor/distributortest.cpp
+++ b/storage/src/tests/distributor/distributortest.cpp
@@ -176,11 +176,11 @@ private:
                     trusted = true;
                 }
 
-                uint16_t node = atoi(tokenizer2[0].c_str());
+                uint16_t node = atoi(tokenizer2[0].data());
                 if (tokenizer2[1] == "r") {
                     removedNodes.push_back(node);
                 } else {
-                    uint32_t checksum = atoi(tokenizer2[1].c_str());
+                    uint32_t checksum = atoi(tokenizer2[1].data());
                     changedNodes.push_back(
                             BucketCopy(
                                     i + 1,
diff --git a/storage/src/tests/distributor/distributortestutil.cpp b/storage/src/tests/distributor/distributortestutil.cpp
index e43161946fb..d3496d0c9f6 100644
--- a/storage/src/tests/distributor/distributortestutil.cpp
+++ b/storage/src/tests/distributor/distributortestutil.cpp
@@ -187,16 +187,16 @@ void DistributorTestUtil::addNodesToBucketDB(const document::Bucket& bucket, con
         vespalib::StringTokenizer tok2(tokenizer[i], "=");
         vespalib::StringTokenizer tok3(tok2[1], "/");
 
-        api::BucketInfo info(atoi(tok3[0].c_str()),
-                             atoi(tok3.size() > 1 ? tok3[1].c_str() : tok3[0].c_str()),
-                             atoi(tok3.size() > 2 ? tok3[2].c_str() : tok3[0].c_str()));
+        api::BucketInfo info(atoi(tok3[0].data()),
+                             atoi(tok3.size() > 1 ? tok3[1].data() : tok3[0].data()),
+                             atoi(tok3.size() > 2 ? tok3[2].data() : tok3[0].data()));
 
         size_t flagsIdx = 3;
 
         // Meta info override? For simplicity, require both meta count and size
         if (tok3.size() > 4 && (!tok3[3].empty() && isdigit(tok3[3][0]))) {
-            info.setMetaCount(atoi(tok3[3].c_str()));
-            info.setUsedFileSize(atoi(tok3[4].c_str()));
+            info.setMetaCount(atoi(tok3[3].data()));
+            info.setUsedFileSize(atoi(tok3[4].data()));
             flagsIdx = 5;
         }
 
@@ -211,7 +211,7 @@ void DistributorTestUtil::addNodesToBucketDB(const document::Bucket& bucket, con
             info.setReady(false);
         }
 
-        uint16_t idx = atoi(tok2[0].c_str());
+        uint16_t idx = atoi(tok2[0].data());
         BucketCopy node(
                 0,
                 idx,
diff --git a/storage/src/tests/distributor/putoperationtest.cpp b/storage/src/tests/distributor/putoperationtest.cpp
index b43d3cf64ad..5551d0a5010 100644
--- a/storage/src/tests/distributor/putoperationtest.cpp
+++ b/storage/src/tests/distributor/putoperationtest.cpp
@@ -572,8 +572,8 @@ parseBucketInfoString(const std::string& nodeList) {
     BucketInfo entry;
     for (uint32_t i = 0; i < tokenizer.size(); i++) {
         vespalib::StringTokenizer tokenizer2(tokenizer[i], "-");
-        int node = atoi(tokenizer2[0].c_str());
-        int size = atoi(tokenizer2[1].c_str());
+        int node = atoi(tokenizer2[0].data());
+        int size = atoi(tokenizer2[1].data());
         bool trusted = (tokenizer2[2] == "true");
 
         entry.addNode(BucketCopy(0,
diff --git a/storage/src/tests/persistence/common/filestortestfixture.cpp b/storage/src/tests/persistence/common/filestortestfixture.cpp
index c92687f798b..835b8ef1044 100644
--- a/storage/src/tests/persistence/common/filestortestfixture.cpp
+++ b/storage/src/tests/persistence/common/filestortestfixture.cpp
@@ -18,19 +18,17 @@ spi::LoadType FileStorTestFixture::defaultLoadType = spi::LoadType(0, "default")
 const uint32_t FileStorTestFixture::MSG_WAIT_TIME;
 
 void
-FileStorTestFixture::setupDisks(uint32_t diskCount)
+FileStorTestFixture::setupPersistenceThreads(uint32_t threads)
 {
     std::string rootOfRoot = "todo-make-unique-filestorefixture";
-    _config.reset(new vdstestlib::DirConfig(getStandardConfig(true, rootOfRoot)));
-
-    _config2.reset(new vdstestlib::DirConfig(*_config));
-    _config2->getConfig("stor-server").set("root_folder", (rootOfRoot + "-vdsroot.2"));
-    _config2->getConfig("stor-devices").set("root_folder", (rootOfRoot + "-vdsroot.2"));
-    _config2->getConfig("stor-server").set("node_index", "1");
-
-    _smallConfig.reset(new vdstestlib::DirConfig(*_config));
-    _node.reset(new TestServiceLayerApp(DiskCount(diskCount), NodeIndex(1),
-                                        _config->getConfigId()));
+    _config = std::make_unique<vdstestlib::DirConfig>(getStandardConfig(true, rootOfRoot));
+    _config->getConfig("stor-server").set("root_folder", (rootOfRoot + "-vdsroot.2"));
+    _config->getConfig("stor-devices").set("root_folder", (rootOfRoot + "-vdsroot.2"));
+    _config->getConfig("stor-server").set("node_index", "1");
+    _config->getConfig("stor-filestor").set("num_threads", std::to_string(threads));
+
+    _node = std::make_unique<TestServiceLayerApp>(
+            DiskCount(1), NodeIndex(1), _config->getConfigId());
     _testdoctype1 = _node->getTypeRepo()->getDocumentType("testdoctype1");
 }
 
@@ -38,16 +36,15 @@ FileStorTestFixture::setupDisks(uint32_t diskCount)
 void
 FileStorTestFixture::setUp()
 {
-    setupDisks(1);
+    setupPersistenceThreads(1);
     _node->setPersistenceProvider(
-            spi::PersistenceProvider::UP(
-                    new spi::dummy::DummyPersistence(_node->getTypeRepo(), 1)));
+            std::make_unique<spi::dummy::DummyPersistence>(_node->getTypeRepo(), 1));
 }
 
 void
 FileStorTestFixture::tearDown()
 {
-    _node.reset(0);
+    _node.reset();
 }
 
 void
@@ -91,7 +88,7 @@ FileStorTestFixture::TestFileStorComponents::TestFileStorComponents(
 }
 
 api::StorageMessageAddress
-FileStorTestFixture::TestFileStorComponents::makeSelfAddress() const {
+FileStorTestFixture::makeSelfAddress() {
     return api::StorageMessageAddress("storage", lib::NodeType::STORAGE, 0);
 }
 
diff --git a/storage/src/tests/persistence/common/filestortestfixture.h b/storage/src/tests/persistence/common/filestortestfixture.h
index c8158d01224..c46f9de24fc 100644
--- a/storage/src/tests/persistence/common/filestortestfixture.h
+++ b/storage/src/tests/persistence/common/filestortestfixture.h
@@ -19,8 +19,6 @@ public:
 
     std::unique_ptr<TestServiceLayerApp> _node;
     std::unique_ptr<vdstestlib::DirConfig> _config;
-    std::unique_ptr<vdstestlib::DirConfig> _config2;
-    std::unique_ptr<vdstestlib::DirConfig> _smallConfig;
     const document::DocumentType* _testdoctype1;
 
     static const uint32_t MSG_WAIT_TIME = 60 * 1000;
@@ -30,10 +28,12 @@ public:
 
     void setUp() override;
     void tearDown() override;
-    void setupDisks(uint32_t diskCount);
+    void setupPersistenceThreads(uint32_t diskCount);
     void createBucket(const document::BucketId& bid);
     bool bucketExistsInDb(const document::BucketId& bucket) const;
 
+    static api::StorageMessageAddress makeSelfAddress();
+
     api::ReturnCode::Result resultOf(const api::StorageReply& reply) const {
         return reply.getResult().getResult();
     }
@@ -99,8 +99,6 @@ public:
                                const char* testName,
                                const StorageLinkInjector& i = NoOpStorageLinkInjector());
 
-        api::StorageMessageAddress makeSelfAddress() const;
-
         void sendDummyGet(const document::BucketId& bid);
         void sendPut(const document::BucketId& bid,
                      uint32_t docIdx,
diff --git a/storage/src/tests/persistence/filestorage/mergeblockingtest.cpp b/storage/src/tests/persistence/filestorage/mergeblockingtest.cpp
index 50999f5883e..d4cec415937 100644
--- a/storage/src/tests/persistence/filestorage/mergeblockingtest.cpp
+++ b/storage/src/tests/persistence/filestorage/mergeblockingtest.cpp
@@ -16,7 +16,7 @@ class MergeBlockingTest : public FileStorTestFixture
 {
 public:
     void setupDisks() {
-        FileStorTestFixture::setupDisks(1);
+        FileStorTestFixture::setupPersistenceThreads(1);
         _node->setPersistenceProvider(
                 spi::PersistenceProvider::UP(
                         new spi::dummy::DummyPersistence(_node->getTypeRepo(), 1)));
diff --git a/storage/src/tests/persistence/filestorage/operationabortingtest.cpp b/storage/src/tests/persistence/filestorage/operationabortingtest.cpp
index 64ef48b5719..e12f48bcdea 100644
--- a/storage/src/tests/persistence/filestorage/operationabortingtest.cpp
+++ b/storage/src/tests/persistence/filestorage/operationabortingtest.cpp
@@ -86,9 +86,9 @@ public:
     std::unique_ptr<vespalib::Barrier> _queueBarrier;
     std::unique_ptr<vespalib::Barrier> _completionBarrier;
 
-    void setupDisks(uint32_t diskCount, uint32_t queueBarrierThreads) {
-        FileStorTestFixture::setupDisks(diskCount);
-        _dummyProvider.reset(new spi::dummy::DummyPersistence(_node->getTypeRepo(), diskCount));
+    void setupProviderAndBarriers(uint32_t queueBarrierThreads) {
+        FileStorTestFixture::setupPersistenceThreads(1);
+        _dummyProvider.reset(new spi::dummy::DummyPersistence(_node->getTypeRepo(), 1));
         _queueBarrier.reset(new vespalib::Barrier(queueBarrierThreads));
         _completionBarrier.reset(new vespalib::Barrier(2));
         _blockingProvider = new BlockingMockProvider(*_dummyProvider, *_queueBarrier, *_completionBarrier);
@@ -219,7 +219,7 @@ makeAbortCmd(const Container& buckets)
 void
 OperationAbortingTest::testAbortMessageClearsRelevantQueuedOperations()
 {
-    setupDisks(1, 2);
+    setupProviderAndBarriers(2);
     TestFileStorComponents c(*this, "testAbortMessageClearsRelevantQueuedOperations");
     document::BucketId bucket(16, 1);
     createBucket(bucket);
@@ -305,7 +305,7 @@ public:
 void
 OperationAbortingTest::testWaitForCurrentOperationCompletionForAbortedBucket()
 {
-    setupDisks(1, 3);
+    setupProviderAndBarriers(3);
     TestFileStorComponents c(*this, "testWaitForCurrentOperationCompletionForAbortedBucket");
 
     document::BucketId bucket(16, 1);
@@ -386,7 +386,7 @@ OperationAbortingTest::doTestSpecificOperationsNotAborted(const char* testName,
                                                           const std::vector<api::StorageMessage::SP>& msgs,
                                                           bool shouldCreateBucketInitially)
 {
-    setupDisks(1, 2);
+    setupProviderAndBarriers(2);
     TestFileStorComponents c(*this, testName);
     document::BucketId bucket(16, 1);
     document::BucketId blockerBucket(16, 2);    
diff --git a/storage/src/tests/persistence/persistencequeuetest.cpp b/storage/src/tests/persistence/persistencequeuetest.cpp
index f31623eed61..e96ad013923 100644
--- a/storage/src/tests/persistence/persistencequeuetest.cpp
+++ b/storage/src/tests/persistence/persistencequeuetest.cpp
@@ -15,86 +15,190 @@ using document::test::makeDocumentBucket;
 
 namespace storage {
 
-class PersistenceQueueTest : public FileStorTestFixture
-{
+class PersistenceQueueTest : public FileStorTestFixture {
 public:
     void testFetchNextUnlockedMessageIfBucketLocked();
+    void shared_locked_operations_allow_concurrent_bucket_access();
+    void exclusive_locked_operation_not_started_if_shared_op_active();
+    void shared_locked_operation_not_started_if_exclusive_op_active();
+    void exclusive_locked_operation_not_started_if_exclusive_op_active();
+    void operation_batching_not_allowed_across_different_lock_modes();
 
-    std::shared_ptr<api::StorageMessage>
-    createPut(uint64_t bucket, uint64_t docIdx);
+    std::shared_ptr<api::StorageMessage> createPut(uint64_t bucket, uint64_t docIdx);
+    std::shared_ptr<api::StorageMessage> createGet(uint64_t bucket) const;
 
     void setUp() override;
-    void tearDown() override;
 
     CPPUNIT_TEST_SUITE(PersistenceQueueTest);
     CPPUNIT_TEST(testFetchNextUnlockedMessageIfBucketLocked);
+    CPPUNIT_TEST(shared_locked_operations_allow_concurrent_bucket_access);
+    CPPUNIT_TEST(exclusive_locked_operation_not_started_if_shared_op_active);
+    CPPUNIT_TEST(shared_locked_operation_not_started_if_exclusive_op_active);
+    CPPUNIT_TEST(exclusive_locked_operation_not_started_if_exclusive_op_active);
+    CPPUNIT_TEST(operation_batching_not_allowed_across_different_lock_modes);
     CPPUNIT_TEST_SUITE_END();
+
+    struct Fixture {
+        FileStorTestFixture& parent;
+        DummyStorageLink top;
+        std::unique_ptr<DummyStorageLink> dummyManager;
+        ForwardingMessageSender messageSender;
+        documentapi::LoadTypeSet loadTypes;
+        FileStorMetrics metrics;
+        std::unique_ptr<FileStorHandler> filestorHandler;
+        uint32_t stripeId;
+
+        explicit Fixture(FileStorTestFixture& parent);
+        ~Fixture();
+    };
+
+    static constexpr uint16_t _disk = 0;
 };
 
 CPPUNIT_TEST_SUITE_REGISTRATION(PersistenceQueueTest);
 
-void
-PersistenceQueueTest::setUp()
+PersistenceQueueTest::Fixture::Fixture(FileStorTestFixture& parent_)
+    : parent(parent_),
+      top(),
+      dummyManager(std::make_unique<DummyStorageLink>()),
+      messageSender(*dummyManager),
+      loadTypes("raw:"),
+      metrics(loadTypes.getMetricLoadTypes())
 {
-    setupDisks(1);
-    _node->setPersistenceProvider(
-            spi::PersistenceProvider::UP(
-                    new spi::dummy::DummyPersistence(_node->getTypeRepo(), 1)));
+    top.push_back(std::move(dummyManager));
+    top.open();
+
+    metrics.initDiskMetrics(parent._node->getPartitions().size(), loadTypes.getMetricLoadTypes(), 1, 1);
+
+    filestorHandler = std::make_unique<FileStorHandler>(messageSender, metrics, parent._node->getPartitions(),
+                                                        parent._node->getComponentRegister());
+    // getNextMessage will time out if no unlocked buckets are present. Choose a timeout
+    // that is large enough to fail tests with high probability if this is not the case,
+    // and small enough to not slow down testing too much.
+    filestorHandler->setGetNextMessageTimeout(20);
+
+    stripeId = filestorHandler->getNextStripeId(0);
 }
 
-void
-PersistenceQueueTest::tearDown()
-{
-    _node.reset(0);
+PersistenceQueueTest::Fixture::~Fixture() = default;
+
+void PersistenceQueueTest::setUp() {
+    setupPersistenceThreads(1);
+    _node->setPersistenceProvider(std::make_unique<spi::dummy::DummyPersistence>(_node->getTypeRepo(), 1));
 }
 
-std::shared_ptr<api::StorageMessage>
-PersistenceQueueTest::createPut(uint64_t bucket, uint64_t docIdx)
-{
-    std::ostringstream id;
-    id << "id:foo:testdoctype1:n=" << bucket << ":" << docIdx;
-    document::Document::SP doc(
-            _node->getTestDocMan().createDocument("foobar", id.str()));
-    std::shared_ptr<api::PutCommand> cmd(
-            new api::PutCommand(makeDocumentBucket(document::BucketId(16, bucket)), doc, 1234));
-    cmd->setAddress(api::StorageMessageAddress(
-            "storage", lib::NodeType::STORAGE, 0));
+std::shared_ptr<api::StorageMessage> PersistenceQueueTest::createPut(uint64_t bucket, uint64_t docIdx) {
+    std::shared_ptr<document::Document> doc = _node->getTestDocMan().createDocument(
+            "foobar", vespalib::make_string("id:foo:testdoctype1:n=%zu:%zu", bucket, docIdx));
+    auto cmd = std::make_shared<api::PutCommand>(makeDocumentBucket(document::BucketId(16, bucket)), doc, 1234);
+    cmd->setAddress(makeSelfAddress());
     return cmd;
 }
 
-void
-PersistenceQueueTest::testFetchNextUnlockedMessageIfBucketLocked()
-{
-    DummyStorageLink top;
-    DummyStorageLink *dummyManager;
-    top.push_back(std::unique_ptr<StorageLink>(dummyManager = new DummyStorageLink));
-    top.open();
-    ForwardingMessageSender messageSender(*dummyManager);
-
-    documentapi::LoadTypeSet loadTypes("raw:");
-    FileStorMetrics metrics(loadTypes.getMetricLoadTypes());
-    metrics.initDiskMetrics(_node->getPartitions().size(), loadTypes.getMetricLoadTypes(), 1, 1);
-
-    FileStorHandler filestorHandler(messageSender, metrics, _node->getPartitions(), _node->getComponentRegister());
-    uint32_t stripeId = filestorHandler.getNextStripeId(0);
+std::shared_ptr<api::StorageMessage> PersistenceQueueTest::createGet(uint64_t bucket) const {
+    auto cmd = std::make_shared<api::GetCommand>(
+            makeDocumentBucket(document::BucketId(16, bucket)),
+            document::DocumentId(vespalib::make_string("id:foo:testdoctype1:n=%zu:0", bucket)), "[all]");
+    cmd->setAddress(makeSelfAddress());
+    return cmd;
+}
 
+void PersistenceQueueTest::testFetchNextUnlockedMessageIfBucketLocked() {
+    Fixture f(*this);
     // Send 2 puts, 2 to the first bucket, 1 to the second. Calling
     // getNextMessage 2 times should then return a lock on the first bucket,
     // then subsequently on the second, skipping the already locked bucket.
     // Puts all have same pri, so order is well defined.
-    filestorHandler.schedule(createPut(1234, 0), 0);
-    filestorHandler.schedule(createPut(1234, 1), 0);
-    filestorHandler.schedule(createPut(5432, 0), 0);
+    f.filestorHandler->schedule(createPut(1234, 0), _disk);
+    f.filestorHandler->schedule(createPut(1234, 1), _disk);
+    f.filestorHandler->schedule(createPut(5432, 0), _disk);
 
-    auto lock0 = filestorHandler.getNextMessage(0, stripeId);
+    auto lock0 = f.filestorHandler->getNextMessage(_disk, f.stripeId);
     CPPUNIT_ASSERT(lock0.first.get());
     CPPUNIT_ASSERT_EQUAL(document::BucketId(16, 1234),
                          dynamic_cast<api::PutCommand&>(*lock0.second).getBucketId());
 
-    auto lock1 = filestorHandler.getNextMessage(0, stripeId);
+    auto lock1 = f.filestorHandler->getNextMessage(_disk, f.stripeId);
     CPPUNIT_ASSERT(lock1.first.get());
     CPPUNIT_ASSERT_EQUAL(document::BucketId(16, 5432),
                          dynamic_cast<api::PutCommand&>(*lock1.second).getBucketId());
 }
 
+void PersistenceQueueTest::shared_locked_operations_allow_concurrent_bucket_access() {
+    Fixture f(*this);
+
+    f.filestorHandler->schedule(createGet(1234), _disk);
+    f.filestorHandler->schedule(createGet(1234), _disk);
+
+    auto lock0 = f.filestorHandler->getNextMessage(_disk, f.stripeId);
+    CPPUNIT_ASSERT(lock0.first.get());
+    CPPUNIT_ASSERT_EQUAL(api::LockingRequirements::Shared, lock0.first->lockingRequirements());
+
+    // Even though we already have a lock on the bucket, Gets allow shared locking and we
+    // should therefore be able to get another lock.
+    auto lock1 = f.filestorHandler->getNextMessage(_disk, f.stripeId);
+    CPPUNIT_ASSERT(lock1.first.get());
+    CPPUNIT_ASSERT_EQUAL(api::LockingRequirements::Shared, lock1.first->lockingRequirements());
+}
+
+void PersistenceQueueTest::exclusive_locked_operation_not_started_if_shared_op_active() {
+    Fixture f(*this);
+
+    f.filestorHandler->schedule(createGet(1234), _disk);
+    f.filestorHandler->schedule(createPut(1234, 0), _disk);
+
+    auto lock0 = f.filestorHandler->getNextMessage(_disk, f.stripeId);
+    CPPUNIT_ASSERT(lock0.first.get());
+    CPPUNIT_ASSERT_EQUAL(api::LockingRequirements::Shared, lock0.first->lockingRequirements());
+
+    // Expected to time out
+    auto lock1 = f.filestorHandler->getNextMessage(_disk, f.stripeId);
+    CPPUNIT_ASSERT(!lock1.first.get());
+}
+
+void PersistenceQueueTest::shared_locked_operation_not_started_if_exclusive_op_active() {
+    Fixture f(*this);
+
+    f.filestorHandler->schedule(createPut(1234, 0), _disk);
+    f.filestorHandler->schedule(createGet(1234), _disk);
+
+    auto lock0 = f.filestorHandler->getNextMessage(_disk, f.stripeId);
+    CPPUNIT_ASSERT(lock0.first.get());
+    CPPUNIT_ASSERT_EQUAL(api::LockingRequirements::Exclusive, lock0.first->lockingRequirements());
+
+    // Expected to time out
+    auto lock1 = f.filestorHandler->getNextMessage(_disk, f.stripeId);
+    CPPUNIT_ASSERT(!lock1.first.get());
+}
+
+void PersistenceQueueTest::exclusive_locked_operation_not_started_if_exclusive_op_active() {
+    Fixture f(*this);
+
+    f.filestorHandler->schedule(createPut(1234, 0), _disk);
+    f.filestorHandler->schedule(createPut(1234, 0), _disk);
+
+    auto lock0 = f.filestorHandler->getNextMessage(_disk, f.stripeId);
+    CPPUNIT_ASSERT(lock0.first.get());
+    CPPUNIT_ASSERT_EQUAL(api::LockingRequirements::Exclusive, lock0.first->lockingRequirements());
+
+    // Expected to time out
+    auto lock1 = f.filestorHandler->getNextMessage(_disk, f.stripeId);
+    CPPUNIT_ASSERT(!lock1.first.get());
+}
+
+void PersistenceQueueTest::operation_batching_not_allowed_across_different_lock_modes() {
+    Fixture f(*this);
+
+    f.filestorHandler->schedule(createPut(1234, 0), _disk);
+    f.filestorHandler->schedule(createGet(1234), _disk);
+
+    auto lock0 = f.filestorHandler->getNextMessage(_disk, f.stripeId);
+    CPPUNIT_ASSERT(lock0.first);
+    CPPUNIT_ASSERT(lock0.second);
+    CPPUNIT_ASSERT_EQUAL(api::LockingRequirements::Exclusive, lock0.first->lockingRequirements());
+
+    f.filestorHandler->getNextMessage(_disk, f.stripeId, lock0);
+    CPPUNIT_ASSERT(!lock0.second);
+}
+
 } // namespace storage
diff --git a/storage/src/tests/storageserver/fnet_listener_test.cpp b/storage/src/tests/storageserver/fnet_listener_test.cpp
index cc9c424ac28..84051041d25 100644
--- a/storage/src/tests/storageserver/fnet_listener_test.cpp
+++ b/storage/src/tests/storageserver/fnet_listener_test.cpp
@@ -135,7 +135,7 @@ vespalib::string make_compressable_state_string() {
         ss << " ." << i << ".s:d";
     }
     return vespalib::make_string("version:123 distributor:100%s storage:100%s",
-                                 ss.str().c_str(), ss.str().c_str());
+                                 ss.str().data(), ss.str().data());
 }
 
 }
diff --git a/storage/src/tests/visiting/visitortest.cpp b/storage/src/tests/visiting/visitortest.cpp
index 27281d9b95f..4fc577226ca 100644
--- a/storage/src/tests/visiting/visitortest.cpp
+++ b/storage/src/tests/visiting/visitortest.cpp
@@ -62,7 +62,7 @@ private:
     CPPUNIT_TEST(testNormalUsage);
     CPPUNIT_TEST(testFailedCreateIterator);
     CPPUNIT_TEST(testFailedGetIter);
-    CPPUNIT_TEST(testMultipleFailedGetIter);
+    CPPUNIT_TEST(iterators_per_bucket_config_is_ignored_and_hardcoded_to_1);
     CPPUNIT_TEST(testDocumentAPIClientError);
     CPPUNIT_TEST(testNoDocumentAPIResendingForFailedVisitor);
     CPPUNIT_TEST(testIteratorCreatedForFailedVisitor);
@@ -90,7 +90,7 @@ public:
     void testNormalUsage();
     void testFailedCreateIterator();
     void testFailedGetIter();
-    void testMultipleFailedGetIter();
+    void iterators_per_bucket_config_is_ignored_and_hardcoded_to_1();
     void testDocumentAPIClientError();
     void testNoDocumentAPIResendingForFailedVisitor();
     void testIteratorCreatedForFailedVisitor();
@@ -592,36 +592,31 @@ VisitorTest::testFailedGetIter()
     CPPUNIT_ASSERT(waitUntilNoActiveVisitors());
 }
 
-void
-VisitorTest::testMultipleFailedGetIter()
-{
-    initializeTest(TestParams().iteratorsPerBucket(2));
-    std::shared_ptr<api::CreateVisitorCommand> cmd(
-            makeCreateVisitor());
+void VisitorTest::iterators_per_bucket_config_is_ignored_and_hardcoded_to_1() {
+    initializeTest(TestParams().iteratorsPerBucket(20));
+    auto cmd = makeCreateVisitor();
     _top->sendDown(cmd);
     sendCreateIteratorReply();
 
-    std::vector<GetIterCommand::SP> getIterCmds(
-            fetchMultipleCommands<GetIterCommand>(*_bottom, 2));
-
-    sendGetIterReply(*getIterCmds[0],
-                     api::ReturnCode(api::ReturnCode::BUCKET_NOT_FOUND));
-
-    // Wait for an "appropriate" amount of time so that wrongful logic
-    // will send a DestroyIteratorCommand before all pending GetIters
-    // have been replied to.
-    std::this_thread::sleep_for(100ms);
+    auto getIterCmd = fetchSingleCommand<GetIterCommand>(*_bottom);
+    CPPUNIT_ASSERT_EQUAL(spi::IteratorId(1234),
+                         getIterCmd->getIteratorId());
+    sendGetIterReply(*getIterCmd);
 
     CPPUNIT_ASSERT_EQUAL(size_t(0), _bottom->getNumCommands());
 
-    sendGetIterReply(*getIterCmds[1],
-                     api::ReturnCode(api::ReturnCode::BUCKET_DELETED));
+    std::vector<document::Document::SP> docs;
+    std::vector<document::DocumentId> docIds;
+    std::vector<std::string> infoMessages;
+    getMessagesAndReply(_documents.size(), getSession(0), docs, docIds, infoMessages);
+    CPPUNIT_ASSERT_EQUAL(size_t(0), infoMessages.size());
+    CPPUNIT_ASSERT_EQUAL(size_t(0), docIds.size());
 
-    DestroyIteratorCommand::SP destroyIterCmd(
-            fetchSingleCommand<DestroyIteratorCommand>(*_bottom));
+    auto destroyIterCmd = fetchSingleCommand<DestroyIteratorCommand>(*_bottom);
 
-    verifyCreateVisitorReply(api::ReturnCode::BUCKET_DELETED, 0, 0);
+    verifyCreateVisitorReply(api::ReturnCode::OK);
     CPPUNIT_ASSERT(waitUntilNoActiveVisitors());
+    CPPUNIT_ASSERT_EQUAL(0L, getFailedVisitorDestinationReplyCount());
 }
 
 void
diff --git a/storage/src/vespa/storage/common/content_bucket_space.cpp b/storage/src/vespa/storage/common/content_bucket_space.cpp
index 2bae118e364..0827c721100 100644
--- a/storage/src/vespa/storage/common/content_bucket_space.cpp
+++ b/storage/src/vespa/storage/common/content_bucket_space.cpp
@@ -9,7 +9,8 @@ ContentBucketSpace::ContentBucketSpace(document::BucketSpace bucketSpace)
       _bucketDatabase(),
       _lock(),
       _clusterState(),
-      _distribution()
+      _distribution(),
+      _nodeUpInLastNodeStateSeenByProvider(false)
 {
 }
 
@@ -41,4 +42,18 @@ ContentBucketSpace::getDistribution() const
     return _distribution;
 }
 
+bool
+ContentBucketSpace::getNodeUpInLastNodeStateSeenByProvider() const
+{
+    std::lock_guard guard(_lock);
+    return _nodeUpInLastNodeStateSeenByProvider;
+}
+
+void
+ContentBucketSpace::setNodeUpInLastNodeStateSeenByProvider(bool nodeUpInLastNodeStateSeenByProvider)
+{
+    std::lock_guard guard(_lock);
+    _nodeUpInLastNodeStateSeenByProvider = nodeUpInLastNodeStateSeenByProvider;
+}
+
 }
diff --git a/storage/src/vespa/storage/common/content_bucket_space.h b/storage/src/vespa/storage/common/content_bucket_space.h
index 6ccc82bc4fb..81ce6234879 100644
--- a/storage/src/vespa/storage/common/content_bucket_space.h
+++ b/storage/src/vespa/storage/common/content_bucket_space.h
@@ -22,6 +22,7 @@ private:
     mutable std::mutex _lock;
     std::shared_ptr<const lib::ClusterState> _clusterState;
     std::shared_ptr<const lib::Distribution> _distribution;
+    bool _nodeUpInLastNodeStateSeenByProvider;
 
 public:
     using UP = std::unique_ptr<ContentBucketSpace>;
@@ -33,6 +34,8 @@ public:
     std::shared_ptr<const lib::ClusterState> getClusterState() const;
     void setDistribution(std::shared_ptr<const lib::Distribution> distribution);
     std::shared_ptr<const lib::Distribution> getDistribution() const;
+    bool getNodeUpInLastNodeStateSeenByProvider() const;
+    void setNodeUpInLastNodeStateSeenByProvider(bool nodeUpInLastNodeStateSeenByProvider);
 };
 
 }
diff --git a/storage/src/vespa/storage/common/hostreporter/kernelmetrictool.cpp b/storage/src/vespa/storage/common/hostreporter/kernelmetrictool.cpp
index 0ff178c3f0f..c8d73737d7c 100644
--- a/storage/src/vespa/storage/common/hostreporter/kernelmetrictool.cpp
+++ b/storage/src/vespa/storage/common/hostreporter/kernelmetrictool.cpp
@@ -65,8 +65,9 @@ uint32_t getTokenCount(const vespalib::string& line) {
 
 uint64_t toLong(const vespalib::stringref& s, int base) {
     char* endptr;
-    uint64_t result(strtoull(s.c_str(), &endptr, base));
-    if ((s.c_str() + s.size()) != endptr) {
+    // FIXME C++17 range-safe from_chars() instead of strtoull()
+    uint64_t result(strtoull(s.data(), &endptr, base));
+    if ((s.data() + s.size()) != endptr) {
         throw vespalib::IllegalArgumentException("Parsing '" + s + "' as a long.");
     }
     return result;
diff --git a/storage/src/vespa/storage/frameworkimpl/thread/deadlockdetector.cpp b/storage/src/vespa/storage/frameworkimpl/thread/deadlockdetector.cpp
index 7c6a708fd8b..e1b7d9c52ea 100644
--- a/storage/src/vespa/storage/frameworkimpl/thread/deadlockdetector.cpp
+++ b/storage/src/vespa/storage/frameworkimpl/thread/deadlockdetector.cpp
@@ -169,7 +169,7 @@ namespace {
             } else if (state != DeadLockDetector::OK) {
                 vespalib::asciistream ost;
                 ost << "Thread " << id << " has registered tick again.\n";
-                LOGBP(info, "%s", ost.str().c_str());
+                LOGBP(info, "%s", ost.str().data());
                 state = DeadLockDetector::OK;
             }
         }
@@ -200,7 +200,7 @@ DeadLockDetector::handleDeadlock(const framework::MilliSecTime& currentTime,
     if (warnOnly) {
         if (_enableWarning) {
             LOGBT(warning, "deadlockw-" + id, "%s",
-                  error.str().c_str());
+                  error.str().data());
             if (_reportedBucketDBLocksAtState != WARNED) {
                 _reportedBucketDBLocksAtState = WARNED;
                 LOG(info, "Locks in bucket database at deadlock time:"
@@ -212,7 +212,7 @@ DeadLockDetector::handleDeadlock(const framework::MilliSecTime& currentTime,
     } else {
         if (_enableShutdown || _enableWarning) {
             LOGBT(error, "deadlock-" + id, "%s",
-                  error.str().c_str());
+                  error.str().data());
         }
     }
     if (!_enableShutdown) return;
diff --git a/storage/src/vespa/storage/persistence/filestorage/filestorhandler.cpp b/storage/src/vespa/storage/persistence/filestorage/filestorhandler.cpp
index 74baecbf026..0da0fd5ce66 100644
--- a/storage/src/vespa/storage/persistence/filestorage/filestorhandler.cpp
+++ b/storage/src/vespa/storage/persistence/filestorage/filestorhandler.cpp
@@ -71,9 +71,9 @@ FileStorHandler::getNextMessage(uint16_t disk, uint32_t stripeId, LockedMessage&
 }
 
 FileStorHandler::BucketLockInterface::SP
-FileStorHandler::lock(const document::Bucket& bucket, uint16_t disk)
+FileStorHandler::lock(const document::Bucket& bucket, uint16_t disk, api::LockingRequirements lockReq)
 {
-    return _impl->lock(bucket, disk);
+    return _impl->lock(bucket, disk, lockReq);
 }
 
 void
diff --git a/storage/src/vespa/storage/persistence/filestorage/filestorhandler.h b/storage/src/vespa/storage/persistence/filestorage/filestorhandler.h
index b74765b17d2..02c959df2f0 100644
--- a/storage/src/vespa/storage/persistence/filestorage/filestorhandler.h
+++ b/storage/src/vespa/storage/persistence/filestorage/filestorhandler.h
@@ -58,8 +58,9 @@ public:
         typedef std::shared_ptr<BucketLockInterface> SP;
 
         virtual const document::Bucket &getBucket() const = 0;
+        virtual api::LockingRequirements lockingRequirements() const noexcept = 0;
 
-        virtual ~BucketLockInterface() {};
+        virtual ~BucketLockInterface() = default;
     };
 
     typedef std::pair<BucketLockInterface::SP, api::StorageMessage::SP> LockedMessage;
@@ -139,7 +140,7 @@ public:
      *
      *
      */
-    BucketLockInterface::SP lock(const document::Bucket&, uint16_t disk);
+    BucketLockInterface::SP lock(const document::Bucket&, uint16_t disk, api::LockingRequirements lockReq);
 
     /**
      * Called by FileStorThread::onBucketDiskMove() after moving file, in case
diff --git a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp
index a01881b6fbe..f9571228ef9 100644
--- a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp
+++ b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp
@@ -370,16 +370,16 @@ FileStorHandlerImpl::getNextMessage(uint16_t disk, uint32_t stripeId)
 }
 
 std::shared_ptr<FileStorHandler::BucketLockInterface>
-FileStorHandlerImpl::Stripe::lock(const document::Bucket &bucket)
-{
+FileStorHandlerImpl::Stripe::lock(const document::Bucket &bucket, api::LockingRequirements lockReq) {
     vespalib::MonitorGuard guard(_lock);
 
-    while (isLocked(guard, bucket)) {
-        LOG(spam, "Contending for filestor lock for %s", bucket.getBucketId().toString().c_str());
+    while (isLocked(guard, bucket, lockReq)) {
+        LOG(spam, "Contending for filestor lock for %s with %s access",
+            bucket.getBucketId().toString().c_str(), api::to_string(lockReq));
         guard.wait(100);
     }
 
-    auto locker = std::make_shared<BucketLock>(guard, *this, bucket, 255, api::MessageType::INTERNAL_ID, 0);
+    auto locker = std::make_shared<BucketLock>(guard, *this, bucket, 255, api::MessageType::INTERNAL_ID, 0, lockReq);
 
     guard.broadcast();
     return locker;
@@ -388,9 +388,9 @@ FileStorHandlerImpl::Stripe::lock(const document::Bucket &bucket)
 namespace {
     struct MultiLockGuard {
         std::map<uint16_t, vespalib::Monitor*> monitors;
-        std::vector<std::shared_ptr<vespalib::MonitorGuard> > guards;
+        std::vector<std::shared_ptr<vespalib::MonitorGuard>> guards;
 
-        MultiLockGuard() {}
+        MultiLockGuard() = default;
 
         void addLock(vespalib::Monitor& monitor, uint16_t index) {
             monitors[index] = &monitor;
@@ -931,7 +931,7 @@ FileStorHandlerImpl::Stripe::getNextMessage(uint32_t timeout, Disk & disk)
         PriorityIdx& idx(bmi::get<1>(_queue));
         PriorityIdx::iterator iter(idx.begin()), end(idx.end());
 
-        while (iter != end && isLocked(guard, iter->_bucket)) {
+        while (iter != end && isLocked(guard, iter->_bucket, iter->_command->lockingRequirements())) {
             iter++;
         }
         if (iter != end) {
@@ -959,6 +959,13 @@ FileStorHandlerImpl::Stripe::getNextMessage(FileStorHandler::LockedMessage& lck)
     }
 
     api::StorageMessage & m(*range.first->_command);
+    // For now, don't allow batching of operations across lock requirement modes.
+    // We might relax this requirement later once we're 100% sure it can't trigger
+    // any unfortunate edge cases.
+    if (lck.first->lockingRequirements() != m.lockingRequirements()) {
+        lck.second.reset();
+        return lck;
+    }
 
     uint64_t waitTime(range.first->_timer.stop(_metrics->averageQueueWaitingTime[m.getLoadType()]));
 
@@ -992,7 +999,8 @@ FileStorHandlerImpl::Stripe::getMessage(vespalib::MonitorGuard & guard, Priority
 
     if (!messageTimedOutInQueue(*msg, waitTime)) {
         auto locker = std::make_unique<BucketLock>(guard, *this, bucket, msg->getPriority(),
-                                                   msg->getType().getId(), msg->getMsgId());
+                                                   msg->getType().getId(), msg->getMsgId(),
+                                                   msg->lockingRequirements());
         guard.unlock();
         return FileStorHandler::LockedMessage(std::move(locker), std::move(msg));
     } else {
@@ -1090,10 +1098,65 @@ FileStorHandlerImpl::Stripe::flush()
         lockGuard.wait(100);
     }
 }
+
+void FileStorHandlerImpl::Stripe::release(const document::Bucket & bucket,
+                                          api::LockingRequirements reqOfReleasedLock,
+                                          api::StorageMessage::Id lockMsgId) {
+    vespalib::MonitorGuard guard(_lock);
+    auto iter = _lockedBuckets.find(bucket);
+    assert(iter != _lockedBuckets.end());
+    auto& entry = iter->second;
+
+    if (reqOfReleasedLock == api::LockingRequirements::Exclusive) {
+        assert(entry._exclusiveLock);
+        assert(entry._exclusiveLock->msgId == lockMsgId);
+        entry._exclusiveLock.reset();
+    } else {
+        assert(!entry._exclusiveLock);
+        auto shared_iter = entry._sharedLocks.find(lockMsgId);
+        assert(shared_iter != entry._sharedLocks.end());
+        entry._sharedLocks.erase(shared_iter);
+    }
+
+    if (!entry._exclusiveLock && entry._sharedLocks.empty()) {
+        _lockedBuckets.erase(iter); // No more locks held
+    }
+    guard.broadcast();
+}
+
+void FileStorHandlerImpl::Stripe::lock(const vespalib::MonitorGuard &, const document::Bucket & bucket,
+                                       api::LockingRequirements lockReq, const LockEntry & lockEntry) {
+    auto& entry = _lockedBuckets[bucket];
+    assert(!entry._exclusiveLock);
+    if (lockReq == api::LockingRequirements::Exclusive) {
+        assert(entry._sharedLocks.empty());
+        entry._exclusiveLock = lockEntry;
+    } else {
+        // TODO use a hash set with a custom comparator/hasher instead...?
+        auto inserted = entry._sharedLocks.insert(std::make_pair(lockEntry.msgId, lockEntry));
+        (void) inserted;
+        assert(inserted.second);
+    }
+}
+
 bool
-FileStorHandlerImpl::Stripe::isLocked(const vespalib::MonitorGuard &, const document::Bucket& bucket) const noexcept
+FileStorHandlerImpl::Stripe::isLocked(const vespalib::MonitorGuard &, const document::Bucket& bucket,
+                                      api::LockingRequirements lockReq) const noexcept
 {
-    return (bucket.getBucketId().getRawId() != 0) && (_lockedBuckets.find(bucket) != _lockedBuckets.end());
+    if (bucket.getBucketId().getRawId() == 0) {
+        return false;
+    }
+    auto iter = _lockedBuckets.find(bucket);
+    if (iter == _lockedBuckets.end()) {
+        return false;
+    }
+    if (iter->second._exclusiveLock) {
+        return true;
+    }
+    // Shared locks can be taken alongside other shared locks, but exclusive locks
+    // require that no shared locks are currently present.
+    return ((lockReq == api::LockingRequirements::Exclusive)
+            && !iter->second._sharedLocks.empty());
 }
 
 uint32_t
@@ -1114,33 +1177,26 @@ FileStorHandlerImpl::getQueueSize(uint16_t disk) const
 
 FileStorHandlerImpl::BucketLock::BucketLock(const vespalib::MonitorGuard & guard, Stripe& stripe,
                                             const document::Bucket &bucket, uint8_t priority,
-                                            api::MessageType::Id msgType, api::StorageMessage::Id msgId)
+                                            api::MessageType::Id msgType, api::StorageMessage::Id msgId,
+                                            api::LockingRequirements lockReq)
     : _stripe(stripe),
-      _bucket(bucket)
+      _bucket(bucket),
+      _uniqueMsgId(msgId),
+      _lockReq(lockReq)
 {
-    (void) guard;
     if (_bucket.getBucketId().getRawId() != 0) {
-        // Lock the bucket and wait until it is not the current operation for
-        // the disk itself.
-        _stripe.lock(guard, _bucket, Stripe::LockEntry(priority, msgType, msgId));
-        LOG(debug, "Locked bucket %s with priority %u",
-            bucket.getBucketId().toString().c_str(), priority);
-
-        LOG_BUCKET_OPERATION_SET_LOCK_STATE(
-                _bucket.getBucketId(), "acquired filestor lock", false,
-                debug::BucketOperationLogger::State::BUCKET_LOCKED);
+        _stripe.lock(guard, _bucket, lockReq, Stripe::LockEntry(priority, msgType, msgId));
+        LOG(debug, "Locked bucket %s for message %zu with priority %u in mode %s",
+            bucket.getBucketId().toString().c_str(), msgId, priority, api::to_string(lockReq));
     }
 }
 
 
-FileStorHandlerImpl::BucketLock::~BucketLock()
-{
+FileStorHandlerImpl::BucketLock::~BucketLock() {
     if (_bucket.getBucketId().getRawId() != 0) {
-        _stripe.release(_bucket);
-        LOG(debug, "Unlocked bucket %s", _bucket.getBucketId().toString().c_str());
-        LOG_BUCKET_OPERATION_SET_LOCK_STATE(
-                _bucket.getBucketId(), "released filestor lock", true,
-                debug::BucketOperationLogger::State::BUCKET_UNLOCKED);
+        _stripe.release(_bucket, _lockReq, _uniqueMsgId);
+        LOG(debug, "Unlocked bucket %s for message %zu in mode %s",
+            _bucket.getBucketId().toString().c_str(), _uniqueMsgId, api::to_string(_lockReq));
     }
 }
 
@@ -1182,14 +1238,31 @@ FileStorHandlerImpl::Stripe::dumpQueueHtml(std::ostream & os) const
     }
 }
 
+namespace {
+
+void dump_lock_entry(const document::BucketId& bucketId, const FileStorHandlerImpl::Stripe::LockEntry& entry,
+                     api::LockingRequirements lock_mode, uint32_t now_ts, std::ostream& os) {
+    os << api::MessageType::get(entry.msgType).getName() << ":" << entry.msgId << " ("
+       << bucketId << ", " << api::to_string(lock_mode)
+       << " lock) Running for " << (now_ts - entry.timestamp) << " secs<br/>\n";
+}
+
+}
+
 void
 FileStorHandlerImpl::Stripe::dumpActiveHtml(std::ostream & os) const
 {
     uint32_t now = time(nullptr);
     vespalib::MonitorGuard guard(_lock);
     for (const auto & e : _lockedBuckets) {
-        os << api::MessageType::get(e.second.msgType).getName() << ":" << e.second.msgId << " (" << e.first.getBucketId()
-           << ") Running for " << (now - e.second.timestamp) << " secs<br/>\n";
+        if (e.second._exclusiveLock) {
+            dump_lock_entry(e.first.getBucketId(), *e.second._exclusiveLock,
+                            api::LockingRequirements::Exclusive, now, os);
+        }
+        for (const auto& shared : e.second._sharedLocks) {
+            dump_lock_entry(e.first.getBucketId(), shared.second,
+                            api::LockingRequirements::Shared, now, os);
+        }
     }
 }
 
@@ -1238,7 +1311,6 @@ FileStorHandlerImpl::getStatus(std::ostream& out, const framework::HttpUrlPath&
         }
         for (auto & entry : _mergeStates) {
             out << "<b>" << entry.first.toString() << "</b><br>\n";
-            //    << "<p>" << it->second << "</p>\n"; // Gets very spammy with the complete state here..
         }
     }
 }
diff --git a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.h b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.h
index 45ac5ded47f..f9dcca4315b 100644
--- a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.h
+++ b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.h
@@ -30,6 +30,7 @@
 #include <vespa/storage/common/messagesender.h>
 #include <vespa/vespalib/stllike/hash_map.h>
 #include <atomic>
+#include <optional>
 
 namespace storage {
 
@@ -82,13 +83,19 @@ public:
             api::MessageType::Id    msgType;
             api::StorageMessage::Id msgId;
 
-
             LockEntry() : timestamp(0), priority(0), msgType(), msgId(0) { }
 
             LockEntry(uint8_t priority_, api::MessageType::Id msgType_, api::StorageMessage::Id msgId_)
                 : timestamp(time(nullptr)), priority(priority_), msgType(msgType_), msgId(msgId_)
             { }
         };
+
+        struct MultiLockEntry {
+            std::optional<LockEntry> _exclusiveLock;
+            using SharedLocks = vespalib::hash_map<api::StorageMessage::Id, LockEntry>;
+            SharedLocks _sharedLocks;
+        };
+
         Stripe(const FileStorHandlerImpl & owner, MessageSender & messageSender);
         ~Stripe();
         void flush();
@@ -105,19 +112,16 @@ public:
             vespalib::MonitorGuard guard(_lock);
             return _queue.size();
         }
-        void release(const document::Bucket & bucket){
-            vespalib::MonitorGuard guard(_lock);
-            _lockedBuckets.erase(bucket);
-            guard.broadcast();
-        }
+        void release(const document::Bucket & bucket, api::LockingRequirements reqOfReleasedLock,
+                     api::StorageMessage::Id lockMsgId);
 
-        bool isLocked(const vespalib::MonitorGuard &, const document::Bucket&) const noexcept;
+        bool isLocked(const vespalib::MonitorGuard &, const document::Bucket&,
+                      api::LockingRequirements lockReq) const noexcept;
 
-        void lock(const vespalib::MonitorGuard &, const document::Bucket & bucket, const LockEntry & lockEntry) {
-            _lockedBuckets.insert(std::make_pair(bucket, lockEntry));
-        }
+        void lock(const vespalib::MonitorGuard &, const document::Bucket & bucket,
+                  api::LockingRequirements lockReq, const LockEntry & lockEntry);
 
-        std::shared_ptr<FileStorHandler::BucketLockInterface> lock(const document::Bucket & bucket);
+        std::shared_ptr<FileStorHandler::BucketLockInterface> lock(const document::Bucket & bucket, api::LockingRequirements lockReq);
         void failOperations(const document::Bucket & bucket, const api::ReturnCode & code);
 
         FileStorHandler::LockedMessage getNextMessage(uint32_t timeout, Disk & disk);
@@ -131,9 +135,11 @@ public:
         void setMetrics(FileStorStripeMetrics * metrics) { _metrics = metrics; }
     private:
         bool hasActive(vespalib::MonitorGuard & monitor, const AbortBucketOperationsCommand& cmd) const;
+        // Precondition: the bucket used by `iter`s operation is not locked in a way that conflicts
+        // with its locking requirements.
         FileStorHandler::LockedMessage getMessage(vespalib::MonitorGuard & guard, PriorityIdx & idx,
                                                   PriorityIdx::iterator iter);
-        typedef vespalib::hash_map<document::Bucket, LockEntry, document::Bucket::hash> LockedBuckets;
+        using LockedBuckets = vespalib::hash_map<document::Bucket, MultiLockEntry, document::Bucket::hash>;
         const FileStorHandlerImpl  &_owner;
         MessageSender              &_messageSender;
         FileStorStripeMetrics      *_metrics;
@@ -178,8 +184,8 @@ public:
             return _stripes[stripeId].getNextMessage(lck);
         }
         std::shared_ptr<FileStorHandler::BucketLockInterface>
-        lock(const document::Bucket & bucket) {
-            return stripe(bucket).lock(bucket);
+        lock(const document::Bucket & bucket, api::LockingRequirements lockReq) {
+            return stripe(bucket).lock(bucket, lockReq);
         }
         void failOperations(const document::Bucket & bucket, const api::ReturnCode & code) {
             stripe(bucket).failOperations(bucket, code);
@@ -194,7 +200,7 @@ public:
             // Disperse bucket bits by multiplying with the 64-bit FNV-1 prime.
             // This avoids an inherent affinity between the LSB of a bucket's bits
             // and the stripe an operation ends up on.
-            return bucket.getBucketId().getRawId() * 1099511628211ULL;
+            return bucket.getBucketId().getId() * 1099511628211ULL;
         }
         Stripe & stripe(const document::Bucket & bucket) {
             return _stripes[dispersed_bucket_bits(bucket) % _stripes.size()];
@@ -208,15 +214,20 @@ public:
 
     class BucketLock : public FileStorHandler::BucketLockInterface {
     public:
+        // TODO refactor, too many params
         BucketLock(const vespalib::MonitorGuard & guard, Stripe& disk, const document::Bucket &bucket,
-                   uint8_t priority, api::MessageType::Id msgType, api::StorageMessage::Id);
+                   uint8_t priority, api::MessageType::Id msgType, api::StorageMessage::Id,
+                   api::LockingRequirements lockReq);
         ~BucketLock();
 
         const document::Bucket &getBucket() const override { return _bucket; }
+        api::LockingRequirements lockingRequirements() const noexcept override { return _lockReq; }
 
     private:
         Stripe & _stripe;
         document::Bucket _bucket;
+        api::StorageMessage::Id _uniqueMsgId;
+        api::LockingRequirements _lockReq;
     };
 
     FileStorHandlerImpl(uint32_t numStripes, MessageSender&, FileStorMetrics&,
@@ -253,8 +264,8 @@ public:
     uint32_t getNextStripeId(uint32_t disk);
 
     std::shared_ptr<FileStorHandler::BucketLockInterface>
-    lock(const document::Bucket & bucket, uint16_t disk) {
-        return _diskInfo[disk].lock(bucket);
+    lock(const document::Bucket & bucket, uint16_t disk, api::LockingRequirements lockReq) {
+        return _diskInfo[disk].lock(bucket, lockReq);
     }
 
     void addMergeStatus(const document::Bucket&, MergeStatus::SP);
diff --git a/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp b/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp
index 9d4d7223411..d1f0a24178a 100644
--- a/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp
+++ b/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp
@@ -36,7 +36,6 @@ FileStorManager(const config::ConfigUri & configUri, const spi::PartitionStateLi
       _partitions(partitions),
       _providerCore(provider),
       _providerErrorWrapper(_providerCore),
-      _nodeUpInLastNodeStateSeenByProvider(false),
       _providerMetric(new spi::MetricPersistenceProvider(_providerErrorWrapper)),
       _provider(_providerMetric.get()),
       _bucketIdFactory(_component.getBucketIdFactory()),
@@ -432,7 +431,7 @@ FileStorManager::onDeleteBucket(const shared_ptr<api::DeleteBucketCommand>& cmd)
                 << ", but storage bucket database contains "
                 << entry->getBucketInfo().toString();
 
-            LOG(debug, "Rejecting bucket delete: %s", ost.str().c_str());
+            LOG(debug, "Rejecting bucket delete: %s", ost.str().data());
             std::shared_ptr<api::StorageReply> reply = cmd->makeReply();
             static_cast<api::DeleteBucketReply&>(*reply).setBucketInfo(entry->getBucketInfo());
             reply->setResult(api::ReturnCode(api::ReturnCode::REJECTED, ost.str()));
@@ -886,21 +885,23 @@ FileStorManager::updateState()
     auto clusterStateBundle = _component.getStateUpdater().getClusterStateBundle();
     lib::ClusterState::CSP state(clusterStateBundle->getBaselineClusterState());
     lib::Node node(_component.getNodeType(), _component.getIndex());
-    bool nodeUp = state->getNodeState(node).getState().oneOf("uir");
 
     LOG(debug, "FileStorManager received cluster state '%s'", state->toString().c_str());
-    // If edge where we go down
-    if (_nodeUpInLastNodeStateSeenByProvider && !nodeUp) {
-        LOG(debug, "Received cluster state where this node is down; de-activating all buckets in database");
-        Deactivator deactivator;
-        _component.getBucketSpaceRepo().forEachBucket(deactivator, "FileStorManager::updateState");
-    }
     for (const auto &elem : _component.getBucketSpaceRepo()) {
         BucketSpace bucketSpace(elem.first);
-        spi::ClusterState spiState(*elem.second->getClusterState(), _component.getIndex(), *elem.second->getDistribution());
+        ContentBucketSpace &contentBucketSpace = *elem.second;
+        auto derivedClusterState = contentBucketSpace.getClusterState();
+        bool nodeUp = derivedClusterState->getNodeState(node).getState().oneOf("uir");
+        // If edge where we go down
+        if (contentBucketSpace.getNodeUpInLastNodeStateSeenByProvider() && !nodeUp) {
+            LOG(debug, "Received cluster state where this node is down; de-activating all buckets in database for bucket space %s", bucketSpace.toString().c_str());
+            Deactivator deactivator;
+            contentBucketSpace.bucketDatabase().all(deactivator, "FileStorManager::updateState");
+        }
+        contentBucketSpace.setNodeUpInLastNodeStateSeenByProvider(nodeUp);
+        spi::ClusterState spiState(*derivedClusterState, _component.getIndex(), *contentBucketSpace.getDistribution());
         _provider->setClusterState(bucketSpace, spiState);
     }
-    _nodeUpInLastNodeStateSeenByProvider = nodeUp;
 }
 
 void
diff --git a/storage/src/vespa/storage/persistence/filestorage/filestormanager.h b/storage/src/vespa/storage/persistence/filestorage/filestormanager.h
index 0bd2ffa5910..4bf2c1049cf 100644
--- a/storage/src/vespa/storage/persistence/filestorage/filestormanager.h
+++ b/storage/src/vespa/storage/persistence/filestorage/filestormanager.h
@@ -56,7 +56,6 @@ class FileStorManager : public StorageLinkQueued,
     const spi::PartitionStateList& _partitions;
     spi::PersistenceProvider& _providerCore;
     ProviderErrorWrapper _providerErrorWrapper;
-    bool _nodeUpInLastNodeStateSeenByProvider;
     spi::MetricPersistenceProvider::UP _providerMetric;
     spi::PersistenceProvider* _provider;
     
diff --git a/storage/src/vespa/storage/persistence/messages.h b/storage/src/vespa/storage/persistence/messages.h
index ba7f5979569..d0572e7dbf8 100644
--- a/storage/src/vespa/storage/persistence/messages.h
+++ b/storage/src/vespa/storage/persistence/messages.h
@@ -38,6 +38,9 @@ public:
     void setMaxByteSize(uint32_t maxByteSize) { _maxByteSize = maxByteSize; }
     uint32_t getMaxByteSize() const { return _maxByteSize; }
 
+    api::LockingRequirements lockingRequirements() const noexcept override {
+        return api::LockingRequirements::Shared;
+    }
 
     void print(std::ostream& out, bool verbose, const std::string& indent) const override;
 private:
@@ -105,6 +108,9 @@ public:
     spi::ReadConsistency getReadConsistency() const noexcept {
         return _readConsistency;
     }
+    api::LockingRequirements lockingRequirements() const noexcept override {
+        return api::LockingRequirements::Shared;
+    }
 
     std::unique_ptr<api::StorageReply> makeReply() override;
 
diff --git a/storage/src/vespa/storage/persistence/persistenceutil.cpp b/storage/src/vespa/storage/persistence/persistenceutil.cpp
index c2dcb8e2a29..888dc93dd82 100644
--- a/storage/src/vespa/storage/persistence/persistenceutil.cpp
+++ b/storage/src/vespa/storage/persistence/persistenceutil.cpp
@@ -122,9 +122,14 @@ PersistenceUtil::lockAndGetDisk(const document::Bucket &bucket,
     result.disk = getPreferredAvailableDisk(bucket);
 
     while (true) {
+        // This function is only called in a context where we require exclusive
+        // locking (split/join). Refactor if this no longer the case.
         std::shared_ptr<FileStorHandler::BucketLockInterface> lock(
-                _fileStorHandler.lock(bucket, result.disk));
+                _fileStorHandler.lock(bucket, result.disk, api::LockingRequirements::Exclusive));
 
+        // TODO disks are no longer used in practice, can we safely discard this?
+        // Might need it for synchronization purposes if something has taken the
+        // disk lock _and_ the bucket lock...?
         StorBucketDatabase::WrappedEntry entry(getBucketDatabase(bucket.getBucketSpace()).get(
                 bucket.getBucketId(), "join-lockAndGetDisk-1", flags));
         if (entry.exist() && entry->disk != result.disk) {
diff --git a/storage/src/vespa/storage/storageserver/mergethrottler.cpp b/storage/src/vespa/storage/storageserver/mergethrottler.cpp
index 370f1c85241..22af1a73633 100644
--- a/storage/src/vespa/storage/storageserver/mergethrottler.cpp
+++ b/storage/src/vespa/storage/storageserver/mergethrottler.cpp
@@ -787,7 +787,7 @@ MergeThrottler::validateNewMerge(
         oss << mergeCmd.toString() << " sent to node "
             << _component.getIndex()
             << ", which is not in its forwarding chain";
-        LOG(error, "%s", oss.str().c_str());
+        LOG(error, "%s", oss.str().data());
     } else if (mergeCmd.getChain().size() >= nodeSeq.getSortedNodes().size()) {
         // Chain is full but we haven't seen the merge! This means
         // the node has probably gone down with a merge it previously
@@ -795,12 +795,12 @@ MergeThrottler::validateNewMerge(
         oss << mergeCmd.toString()
             << " is not in node's internal state, but has a "
             << "full chain, meaning it cannot be forwarded.";
-        LOG(debug, "%s", oss.str().c_str());
+        LOG(debug, "%s", oss.str().data());
     } else if (nodeSeq.chainContainsIndex(nodeSeq.getThisNodeIndex())) {
         oss << mergeCmd.toString()
             << " is not in node's internal state, but contains "
             << "this node in its non-full chain. This should not happen!";
-        LOG(error, "%s", oss.str().c_str());
+        LOG(error, "%s", oss.str().data());
     } else {
         valid = true;
     }
@@ -1117,7 +1117,7 @@ MergeThrottler::makeAbortReply(api::StorageCommand& cmd,
                                vespalib::stringref reason) const
 {
     LOG(debug, "Aborting message %s with reason '%s'",
-        cmd.toString().c_str(), reason.c_str());
+        cmd.toString().c_str(), reason.data());
     std::unique_ptr<api::StorageReply> reply(cmd.makeReply());
     reply->setResult(api::ReturnCode(api::ReturnCode::ABORTED, reason));
     return std::shared_ptr<api::StorageMessage>(reply.release());
diff --git a/storage/src/vespa/storage/storageserver/service_layer_error_listener.cpp b/storage/src/vespa/storage/storageserver/service_layer_error_listener.cpp
index 41177fe46b8..e26549a3b37 100644
--- a/storage/src/vespa/storage/storageserver/service_layer_error_listener.cpp
+++ b/storage/src/vespa/storage/storageserver/service_layer_error_listener.cpp
@@ -15,21 +15,21 @@ void ServiceLayerErrorListener::on_fatal_error(vespalib::stringref message) {
         LOG(info,
             "Received FATAL_ERROR from persistence provider, "
             "shutting down node: %s",
-            message.c_str());
+            vespalib::string(message).c_str());
         _component.requestShutdown(message); // Thread safe
     } else {
         LOG(debug,
             "Received FATAL_ERROR from persistence provider: %s. "
             "Node has already been instructed to shut down so "
             "not doing anything now.",
-            message.c_str());
+            vespalib::string(message).c_str());
     }
 }
 
 void ServiceLayerErrorListener::on_resource_exhaustion_error(vespalib::stringref message) {
     LOG(debug, "SPI reports resource exhaustion ('%s'). "
                 "Applying back-pressure to merge throttler",
-        message.c_str());
+        vespalib::string(message).c_str());
     _merge_throttler.apply_timed_backpressure(); // Thread safe
 }
 
diff --git a/storage/src/vespa/storage/storageserver/statemanager.cpp b/storage/src/vespa/storage/storageserver/statemanager.cpp
index d6ca8c266d3..77d9299169f 100644
--- a/storage/src/vespa/storage/storageserver/statemanager.cpp
+++ b/storage/src/vespa/storage/storageserver/statemanager.cpp
@@ -375,7 +375,7 @@ considerInsertDerivedTransition(const lib::State &currentBaseline,
             ((currentDerived != currentBaseline) || (newDerived != newBaseline)));
     if (considerDerivedTransition && (transitions.find(bucketSpace) == transitions.end())) {
         transitions[bucketSpace] = vespalib::make_string("%s space: '%s' to '%s'",
-                                                         document::FixedBucketSpaces::to_string(bucketSpace).c_str(),
+                                                         document::FixedBucketSpaces::to_string(bucketSpace).data(),
                                                          currentDerived.getName().c_str(),
                                                          newDerived.getName().c_str());
     }
diff --git a/storage/src/vespa/storage/visiting/stor-visitor.def b/storage/src/vespa/storage/visiting/stor-visitor.def
index 1e80f2993a5..6f16bcb60a2 100644
--- a/storage/src/vespa/storage/visiting/stor-visitor.def
+++ b/storage/src/vespa/storage/visiting/stor-visitor.def
@@ -24,6 +24,7 @@ defaultparalleliterators int default=8
 ## will be 16 requests to persistence layer, but only 8 will be able to execute
 ## at the same time, since only one operation can be executed at the same time
 ## for one bucket)
+## DEPRECATED: ignored by backend, 1 is always used.
 iterators_per_bucket int default=1
 
 ## Default number of maximum client replies pending.
diff --git a/storage/src/vespa/storage/visiting/visitorthread.cpp b/storage/src/vespa/storage/visiting/visitorthread.cpp
index a8f31514eb1..6bf28b08540 100644
--- a/storage/src/vespa/storage/visiting/visitorthread.cpp
+++ b/storage/src/vespa/storage/visiting/visitorthread.cpp
@@ -253,7 +253,7 @@ VisitorThread::run(framework::ThreadHandle& thread)
         } catch (std::exception& e) {
             vespalib::asciistream ost;
             ost << "Failed to handle visitor message:" << e.what();
-            LOG(warning, "Failed handling visitor message: %s", ost.str().c_str());
+            LOG(warning, "Failed handling visitor message: %s", ost.str().data());
             result = ReturnCode(ReturnCode::INTERNAL_FAILURE, ost.str());
             if (entry._message.get() && entry._message->getType() == api::MessageType::VISITOR_CREATE) {
                 _messageSender.closed(entry._visitorId);
@@ -466,7 +466,7 @@ VisitorThread::onCreateVisitor(
         if (visitor.get() == 0) {
             result = ReturnCode(ReturnCode::ILLEGAL_PARAMETERS, errors.str());
             LOG(warning, "CreateVisitor(%s): Failed to create visitor: %s",
-                         cmd->getInstanceId().c_str(), errors.str().c_str());
+                         cmd->getInstanceId().c_str(), errors.str().data());
             break;
         }
             // Set visitor parameters
@@ -510,7 +510,7 @@ VisitorThread::onCreateVisitor(
                 << cmd->getDocumentSelection() << "': " << e.getMessage();
             result = ReturnCode(ReturnCode::ILLEGAL_PARAMETERS, ost.str());
             LOG(warning, "CreateVisitor(%s): %s",
-                         cmd->getInstanceId().c_str(), ost.str().c_str());
+                         cmd->getInstanceId().c_str(), ost.str().data());
             break;
         } catch (document::select::ParsingFailedException& e) {
             vespalib::asciistream ost;
@@ -518,7 +518,7 @@ VisitorThread::onCreateVisitor(
                 << cmd->getDocumentSelection() << "': " << e.getMessage();
             result = ReturnCode(ReturnCode::ILLEGAL_PARAMETERS, ost.str());
             LOG(warning, "CreateVisitor(%s): %s",
-                         cmd->getInstanceId().c_str(), ost.str().c_str());
+                         cmd->getInstanceId().c_str(), ost.str().data());
             break;
         }
         LOG(debug, "CreateVisitor(%s): Successfully created visitor",
@@ -637,7 +637,6 @@ VisitorThread::onInternal(const std::shared_ptr<api::InternalCommand>& cmd)
             _ignoreNonExistingVisitorTimeLimit
                     = config.ignorenonexistingvisitortimelimit;
             _defaultParallelIterators = config.defaultparalleliterators;
-            _iteratorsPerBucket = config.iteratorsPerBucket;
             _defaultPendingMessages = config.defaultpendingmessages;
             _defaultDocBlockSize = config.defaultdocblocksize;
             _visitorMemoryUsageLimit = config.visitorMemoryUsageLimit;
@@ -647,12 +646,6 @@ VisitorThread::onInternal(const std::shared_ptr<api::InternalCommand>& cmd)
                 LOG(config, "Cannot use value of defaultParallelIterators < 1");
                 _defaultParallelIterators = 1;
             }
-            if (_iteratorsPerBucket < 1 && _iteratorsPerBucket > 10) {
-                if (_iteratorsPerBucket < 1) _iteratorsPerBucket = 1;
-                else _iteratorsPerBucket = 10;
-                LOG(config, "Invalid value of iterators per bucket %u using %u",
-                    config.iteratorsPerBucket, _iteratorsPerBucket);
-            }
             if (_defaultPendingMessages < 1) {
                 LOG(config, "Cannot use value of defaultPendingMessages < 1");
                 _defaultPendingMessages = 1;
diff --git a/storageapi/src/vespa/storageapi/message/persistence.h b/storageapi/src/vespa/storageapi/message/persistence.h
index bda1bd0f038..59934154cf5 100644
--- a/storageapi/src/vespa/storageapi/message/persistence.h
+++ b/storageapi/src/vespa/storageapi/message/persistence.h
@@ -24,7 +24,7 @@ class TestAndSetCommand : public BucketInfoCommand {
     TestAndSetCondition _condition;
 public:
     TestAndSetCommand(const MessageType & messageType, const document::Bucket &bucket);
-    ~TestAndSetCommand();
+    ~TestAndSetCommand() override;
 
     void setCondition(const TestAndSetCondition & condition) { _condition = condition; }
     const TestAndSetCondition & getCondition() const { return _condition; }
@@ -49,7 +49,7 @@ class PutCommand : public TestAndSetCommand {
 
 public:
     PutCommand(const document::Bucket &bucket, const DocumentSP&, Timestamp);
-    ~PutCommand();
+    ~PutCommand() override;
 
     void setTimestamp(Timestamp ts) { _timestamp = ts; }
 
@@ -86,7 +86,7 @@ class PutReply : public BucketInfoReply {
 
 public:
     explicit PutReply(const PutCommand& cmd, bool wasFound = true);
-    ~PutReply();
+    ~PutReply() override;
 
     const document::DocumentId& getDocumentId() const { return _docId; }
     bool hasDocument() const { return _document.get(); }
@@ -116,7 +116,7 @@ class UpdateCommand : public TestAndSetCommand {
 public:
     UpdateCommand(const document::Bucket &bucket,
                   const std::shared_ptr<document::DocumentUpdate>&, Timestamp);
-    ~UpdateCommand();
+    ~UpdateCommand() override;
 
     void setTimestamp(Timestamp ts) { _timestamp = ts; }
     void setOldTimestamp(Timestamp ts) { _oldTimestamp = ts; }
@@ -147,7 +147,7 @@ class UpdateReply : public BucketInfoReply {
 
 public:
     UpdateReply(const UpdateCommand& cmd, Timestamp oldTimestamp = 0);
-    ~UpdateReply();
+    ~UpdateReply() override;
 
     void setOldTimestamp(Timestamp ts) { _oldTimestamp = ts; }
 
@@ -189,7 +189,7 @@ class GetCommand : public BucketInfoCommand {
 public:
     GetCommand(const document::Bucket &bucket, const document::DocumentId&,
                const vespalib::stringref & fieldSet, Timestamp before = MAX_TIMESTAMP);
-    ~GetCommand();
+    ~GetCommand() override;
     void setBeforeTimestamp(Timestamp ts) { _beforeTimestamp = ts; }
     const document::DocumentId& getDocumentId() const { return _docId; }
     Timestamp getBeforeTimestamp() const { return _beforeTimestamp; }
@@ -199,6 +199,10 @@ public:
     vespalib::string getSummary() const override;
     void print(std::ostream& out, bool verbose, const std::string& indent) const override;
 
+    api::LockingRequirements lockingRequirements() const noexcept override {
+        return api::LockingRequirements::Shared;
+    }
+
     DECLARE_STORAGECOMMAND(GetCommand, onGet)
 };
 
@@ -219,7 +223,7 @@ public:
     GetReply(const GetCommand& cmd,
              const DocumentSP& doc = DocumentSP(),
              Timestamp lastModified = 0);
-    ~GetReply();
+    ~GetReply() override;
 
     const DocumentSP& getDocument() const { return _doc; }
     const document::DocumentId& getDocumentId() const { return _docId; }
@@ -245,7 +249,7 @@ class RemoveCommand : public TestAndSetCommand {
 
 public:
     RemoveCommand(const document::Bucket &bucket, const document::DocumentId& docId, Timestamp timestamp);
-    ~RemoveCommand();
+    ~RemoveCommand() override;
 
     void setTimestamp(Timestamp ts) { _timestamp = ts; }
     const document::DocumentId& getDocumentId() const override { return _docId; }
@@ -267,7 +271,7 @@ class RemoveReply : public BucketInfoReply {
     Timestamp _oldTimestamp;
 public:
     explicit RemoveReply(const RemoveCommand& cmd, Timestamp oldTimestamp = 0);
-    ~RemoveReply();
+    ~RemoveReply() override;
 
     const document::DocumentId& getDocumentId() const { return _docId; }
     Timestamp getTimestamp() { return _timestamp; };
@@ -289,7 +293,7 @@ class RevertCommand : public BucketInfoCommand {
 public:
     RevertCommand(const document::Bucket &bucket,
                   const std::vector<Timestamp>& revertTokens);
-    ~RevertCommand();
+    ~RevertCommand() override;
     const std::vector<Timestamp>& getRevertTokens() const { return _tokens; }
     void print(std::ostream& out, bool verbose, const std::string& indent) const override;
     DECLARE_STORAGECOMMAND(RevertCommand, onRevert)
@@ -305,7 +309,7 @@ class RevertReply : public BucketInfoReply {
     std::vector<Timestamp> _tokens;
 public:
     explicit RevertReply(const RevertCommand& cmd);
-    ~RevertReply();
+    ~RevertReply() override;
     const std::vector<Timestamp>& getRevertTokens() const { return _tokens; }
     void print(std::ostream& out, bool verbose, const std::string& indent) const override;
     DECLARE_STORAGEREPLY(RevertReply, onRevertReply)
diff --git a/storageapi/src/vespa/storageapi/messageapi/storagemessage.cpp b/storageapi/src/vespa/storageapi/messageapi/storagemessage.cpp
index f970091f695..380d846dd93 100644
--- a/storageapi/src/vespa/storageapi/messageapi/storagemessage.cpp
+++ b/storageapi/src/vespa/storageapi/messageapi/storagemessage.cpp
@@ -302,4 +302,19 @@ StorageMessage::getSummary() const {
     return toString();
 }
 
+const char* to_string(LockingRequirements req) noexcept {
+    switch (req) {
+    case LockingRequirements::Exclusive:
+        return "Exclusive";
+    case LockingRequirements::Shared:
+        return "Shared";
+    }
+    assert(false);
+}
+
+std::ostream& operator<<(std::ostream& os, LockingRequirements req) {
+    os << to_string(req);
+    return os;
+}
+
 }
diff --git a/storageapi/src/vespa/storageapi/messageapi/storagemessage.h b/storageapi/src/vespa/storageapi/messageapi/storagemessage.h
index dadb68c644d..6c561f3af21 100644
--- a/storageapi/src/vespa/storageapi/messageapi/storagemessage.h
+++ b/storageapi/src/vespa/storageapi/messageapi/storagemessage.h
@@ -19,6 +19,7 @@
 #include <vespa/document/bucket/bucket.h>
 #include <vespa/vespalib/util/printable.h>
 #include <map>
+#include <iosfwd>
 
 namespace vespalib {
     class asciistream;
@@ -306,6 +307,20 @@ struct TransportContext {
     virtual ~TransportContext() = 0;
 };
 
+enum class LockingRequirements : uint8_t {
+    // Operations with exclusive locking can only be executed iff no other
+    // exclusive or shared locks are taken for its bucket.
+    Exclusive = 0,
+    // Operations with shared locking can only be executed iff no exclusive
+    // lock is taken for its bucket. Should only be used for read-only operations
+    // that cannot mutate a bucket's state.
+    Shared
+};
+
+const char* to_string(LockingRequirements req) noexcept;
+
+std::ostream& operator<<(std::ostream&, LockingRequirements);
+
 class StorageMessage : public vespalib::Printable
 {
     friend class StorageMessageTest; // Used for testing only
@@ -421,6 +436,10 @@ public:
     virtual document::Bucket getBucket() const { return getDummyBucket(); }
     document::BucketId getBucketId() const { return getBucket().getBucketId(); }
     virtual bool hasSingleBucketId() const { return false; }
+    virtual LockingRequirements lockingRequirements() const noexcept {
+        // Safe default: assume exclusive locking is required.
+        return LockingRequirements::Exclusive;
+    }
 };
 
 }
diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/component/componentregisterimpl.cpp b/storageframework/src/vespa/storageframework/defaultimplementation/component/componentregisterimpl.cpp
index a5611451ee8..ae448664f14 100644
--- a/storageframework/src/vespa/storageframework/defaultimplementation/component/componentregisterimpl.cpp
+++ b/storageframework/src/vespa/storageframework/defaultimplementation/component/componentregisterimpl.cpp
@@ -132,7 +132,7 @@ namespace {
 
         MetricHookWrapper(vespalib::stringref name,
                           MetricUpdateHook& hook)
-            : metrics::UpdateHook(name.c_str()),
+            : metrics::UpdateHook(name.data()), // Expected to point to static name
               _hook(hook)
         {
         }
diff --git a/vagrant/README.md b/vagrant/README.md
index d81e80e8808..d5212692638 100644
--- a/vagrant/README.md
+++ b/vagrant/README.md
@@ -1,6 +1,6 @@
-<!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<!-- Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
 
-# Create C++ dev environment on CentOS using VirtualBox and Vagrant
+# Create C++ / Java dev environment on CentOS using VirtualBox and Vagrant
 
 ## Prerequisites
 * [Install VirtualBox](https://www.virtualbox.org/wiki/Downloads)
@@ -12,16 +12,41 @@
 
     cd <vespa-source>/vagrant
 
-#### 2. Install Vagrant VirtualBox Guest Additions plugin
+#### 2. Choose dev environment
+
+##### a. For a dev environment with plain centos/7 and no GUI:
+
+    export VESPA_VAGRANT_VM_BOX=centos/7
+    export VESPA_VAGRANT_DISABLE_GUI=true
+
+##### b. For a dev environment with GUI and CLion:
+
+Create centos7-desktop box:
+
+* Install packer by following guide at [packer.io](https://www.packer.io/intro/getting-started/install.html)
+
+* Clone boxcutter centos repo and build the box:
+```
+git clone https://github.com/boxcutter/centos.git
+./bin/box build centos7-desktop.json virtualbox
+```
+
+Example exports:
+
+    export VESPA_VAGRANT_VM_BOX="centos7-desktop"
+    export VESPA_VAGRANT_VM_BOX_URL="$HOME/git/boxcutter/centos/box/virtualbox/centos7-desktop-xx.yyyy.z.box"
+
+
+#### 3. Install Vagrant VirtualBox Guest Additions plugin
 This is required for mounting shared folders and get mouse pointer integration and seamless windows in the virtual CentOS desktop.
 
     vagrant plugin install vagrant-vbguest
 
-#### 3. Start and provision the environment
+#### 4. Start and provision the environment
 
     vagrant up
 
-#### 4. Connect to machine via SSH
+#### 5. Connect to machine via SSH
 SSH agent forwarding is enabled to ensure easy interaction with GitHub inside the machine.
 
     vagrant ssh
@@ -31,6 +56,10 @@ This is needed in order to compile and run tests fast on the local file system i
 
     git clone git@github.com:vespa-engine/vespa.git
 
+## Build Java modules
+Please follow the build instructions described [here](../README.md#build-java-modules).
+
+
 ## Build C++ modules
 Please follow the build instructions described [here](../README.md#build-c-modules).
 Skip these steps if doing development with CLion.
diff --git a/vagrant/Vagrantfile b/vagrant/Vagrantfile
index 84836d05e72..f15f45d75a0 100644
--- a/vagrant/Vagrantfile
+++ b/vagrant/Vagrantfile
@@ -1,25 +1,37 @@
 # -*- mode: ruby -*-
 # vi: set ft=ruby :
 
+disable_gui = ENV['VESPA_VAGRANT_DISABLE_GUI']
 
-def validate_vm_env_option(name)
+def get_mandatory_env_value(name)
   opt = ENV[name]
   if opt.nil? or opt.empty?
     raise Vagrant::Errors::VagrantError.new, "Environment variable #{name} must be set to a valid value before running vagrant"
   end
+  return opt
 end
 
-validate_vm_env_option('VESPA_VAGRANT_VM_BOX')
-validate_vm_env_option('VESPA_VAGRANT_VM_BOX_URL')
+def get_env_value(name, fallback)
+  opt = ENV[name]
+  if opt.nil? or opt.empty?
+    return fallback
+  end
+  return opt
+end
 
-vm_box = ENV['VESPA_VAGRANT_VM_BOX']
-vm_box_url = ENV['VESPA_VAGRANT_VM_BOX_URL']
+vm_box = get_mandatory_env_value('VESPA_VAGRANT_VM_BOX')
+vm_memory = get_env_value('VESPA_VAGRANT_VM_MEMORY', "8192")
+vm_cpus = get_env_value('VESPA_VAGRANT_VM_CPUS', 4)
+
+unless disable_gui
+  vm_box_url = get_mandatory_env_value('VESPA_VAGRANT_VM_BOX_URL')
+end
 
 # For a complete reference, please see the online documentation at https://docs.vagrantup.com.
 Vagrant.configure("2") do |config|
 
   config.vm.box = vm_box
-  config.vm.box_url = vm_box_url
+  config.vm.box_url = vm_box_url unless disable_gui
 
   config.ssh.forward_agent = true
 
@@ -27,11 +39,11 @@ Vagrant.configure("2") do |config|
 
   config.vm.provider "virtualbox" do |vb|
     # Display the VirtualBox GUI when booting the machine
-    vb.gui = true
+    vb.gui = true unless disable_gui
     vb.name = "vespa-dev"
 
-    vb.memory = "8192"
-    vb.cpus = 4
+    vb.memory = vm_memory
+    vb.cpus = vm_cpus
   end
   
   # Install required and nice-to-have packages
@@ -52,9 +64,13 @@ Vagrant.configure("2") do |config|
     yum-builddep -y /vagrant/dist/vespa.spec
     echo -e "* soft nproc 409600\n* hard nproc 409600" > /etc/security/limits.d/99-nproc.conf
     echo -e "* soft nofile 262144\n* hard nofile 262144" > /etc/security/limits.d/99-nofile.conf
-    echo -e "fs.inotify.max_user_watches = 524288" > /etc/sysctl.d/clion.conf
-    wget -q -O - https://download.jetbrains.com/cpp/CLion-2017.3.3.tar.gz | tar -C /opt -zx
-    ln -sf /opt/clion-2017.3.3/bin/clion.sh /usr/bin/clion
+
+    unless disable_gui
+      echo -e "fs.inotify.max_user_watches = 524288" > /etc/sysctl.d/clion.conf
+      wget -q -O - https://download.jetbrains.com/cpp/CLion-2018.1.6.tar.gz | tar -C /opt -zx
+      ln -sf /opt/clion-2018.1.6/bin/clion.sh /usr/bin/clion
+    end
+
     yum update -y
     hostname localhost
   SHELL
diff --git a/vdslib/src/tests/distribution/distributiontest.cpp b/vdslib/src/tests/distribution/distributiontest.cpp
index 3090910d2bb..7af48c1904b 100644
--- a/vdslib/src/tests/distribution/distributiontest.cpp
+++ b/vdslib/src/tests/distribution/distributiontest.cpp
@@ -191,7 +191,7 @@ namespace {
             try{
                 std::vector<uint16_t> nvect;
                 distribution.getIdealNodes(nodeType, state, results[i].bucket,
-                                           nvect, upStates.c_str(), redundancy);
+                                           nvect, upStates.data(), redundancy);
                 IdealNodeList nodes;
                 for (uint32_t j=0, m=nvect.size(); j<m; ++j) {
                     nodes.push_back(Node(nodeType, nvect[j]));
diff --git a/vdslib/src/tests/distribution/grouptest.cpp b/vdslib/src/tests/distribution/grouptest.cpp
index c108a1e8a38..71d8b8bac0c 100644
--- a/vdslib/src/tests/distribution/grouptest.cpp
+++ b/vdslib/src/tests/distribution/grouptest.cpp
@@ -30,7 +30,7 @@ namespace {
         vespalib::StringTokenizer st(nodelist, ",");
         std::vector<uint16_t> nodes(st.size());
         for (uint32_t i=0; i<st.size(); ++i) {
-            nodes[i] = atoi(st[i].c_str());
+            nodes[i] = atoi(st[i].data());
         }
         group->setNodes(nodes);
         return group;
diff --git a/vdslib/src/vespa/vdslib/container/parameters.h b/vdslib/src/vespa/vdslib/container/parameters.h
index 4572b2f6f1c..ab65932496a 100644
--- a/vdslib/src/vespa/vdslib/container/parameters.h
+++ b/vdslib/src/vespa/vdslib/container/parameters.h
@@ -68,7 +68,7 @@ public:
     ParametersMap::const_iterator begin() const { return _parameters.begin(); }
     ParametersMap::const_iterator end() const { return _parameters.end(); }
     /// Convenience from earlier use.
-    void set(const KeyT & id, const vespalib::stringref & value) { _parameters[id] = Value(value.c_str(), value.size()); }
+    void set(const KeyT & id, const vespalib::stringref & value) { _parameters[id] = Value(value.data(), value.size()); }
     vespalib::stringref get(const KeyT & id, const vespalib::stringref & def = "") const;
     /**
      * Set the value identified by the id given. This requires the type to be
diff --git a/vdslib/src/vespa/vdslib/distribution/redundancygroupdistribution.cpp b/vdslib/src/vespa/vdslib/distribution/redundancygroupdistribution.cpp
index f620dc15928..99974225e9e 100644
--- a/vdslib/src/vespa/vdslib/distribution/redundancygroupdistribution.cpp
+++ b/vdslib/src/vespa/vdslib/distribution/redundancygroupdistribution.cpp
@@ -30,7 +30,7 @@ namespace {
                 firstAsterisk = i;
                 continue;
             }
-            uint32_t number = atoi(st[i].c_str());
+            uint32_t number = atoi(vespalib::string(st[i]).c_str());
             if (number <= 0 || number >= 256) {
                 throw vespalib::IllegalArgumentException(
                     "Illegal distribution spec \"" + serialized + "\". "
@@ -48,7 +48,7 @@ namespace {
         }
     }
 
-    std::vector<uint16_t> parse(vespalib::stringref& serialized) {
+    std::vector<uint16_t> parse(vespalib::stringref serialized) {
         std::vector<uint16_t> result;
         if (serialized == "") return result;
         vespalib::StringTokenizer st(serialized, "|");
diff --git a/vdslib/src/vespa/vdslib/state/clusterstate.cpp b/vdslib/src/vespa/vdslib/state/clusterstate.cpp
index fbc9943e22d..e50b18a5a82 100644
--- a/vdslib/src/vespa/vdslib/state/clusterstate.cpp
+++ b/vdslib/src/vespa/vdslib/state/clusterstate.cpp
@@ -88,7 +88,7 @@ ClusterState::ClusterState(const vespalib::string& serialized)
         if (key.empty() || ! parse(key, value, nodeData) ) {
             LOG(debug, "Unknown key %s in systemstate. Ignoring it, assuming it's "
                        "a new feature from a newer version than ourself: %s",
-                       key.c_str(), serialized.c_str());
+                       vespalib::string(key).c_str(), serialized.c_str());
         }
     }
     nodeData.addTo(_nodeStates, _nodeCount);
@@ -106,13 +106,13 @@ ClusterState::parse(vespalib::stringref key, vespalib::stringref value, NodeData
         break;
     case 'b':
         if (key == "bits") {
-            _distributionBits = atoi(value.c_str());
+            _distributionBits = atoi(value.data());
             return true;
         }
         break;
     case 'v':
         if (key == "version") {
-            _version = atoi(value.c_str());
+            _version = atoi(value.data());
             return true;
         }
         break;
@@ -145,7 +145,7 @@ ClusterState::parseSorD(vespalib::stringref key, vespalib::stringref value, Node
     if (nodeType == 0) return false;
     if (dot == vespalib::string::npos) { // Entry that set node counts
         uint16_t nodeCount = 0;
-        nodeCount = atoi(value.c_str());
+        nodeCount = atoi(value.data());
 
         if (nodeCount > _nodeCount[*nodeType] ) {
             _nodeCount[*nodeType] = nodeCount;
@@ -155,9 +155,9 @@ ClusterState::parseSorD(vespalib::stringref key, vespalib::stringref value, Node
     vespalib::string::size_type dot2 = key.find('.', dot + 1);
     Node node;
     if (dot2 == vespalib::string::npos) {
-        node = Node(*nodeType, atoi(key.substr(dot + 1).c_str()));
+        node = Node(*nodeType, atoi(key.substr(dot + 1).data()));
     } else {
-        node = Node(*nodeType, atoi(key.substr(dot + 1, dot2 - dot - 1).c_str()));
+        node = Node(*nodeType, atoi(key.substr(dot + 1, dot2 - dot - 1).data()));
     }
 
     if (node.getIndex() >= _nodeCount[*nodeType]) {
diff --git a/vdslib/src/vespa/vdslib/state/clusterstate.h b/vdslib/src/vespa/vdslib/state/clusterstate.h
index 9e8dd0f292a..26c6f1b95ef 100644
--- a/vdslib/src/vespa/vdslib/state/clusterstate.h
+++ b/vdslib/src/vespa/vdslib/state/clusterstate.h
@@ -72,7 +72,9 @@ public:
                              const std::string& indent = "") const;
 
 private:
+    // Preconditions: `key` and `value` MUST point into null-terminated strings.
     bool parse(vespalib::stringref key, vespalib::stringref value, NodeData & nodeData);
+    // Preconditions: `key` and `value` MUST point into null-terminated strings.
     bool parseSorD(vespalib::stringref key, vespalib::stringref value, NodeData & nodeData);
     void removeExtraElements();
     void printStateGroupwise(std::ostream& out, const Group&, bool verbose,
diff --git a/vdslib/src/vespa/vdslib/state/diskstate.cpp b/vdslib/src/vespa/vdslib/state/diskstate.cpp
index bcc380e0d75..c963dacff82 100644
--- a/vdslib/src/vespa/vdslib/state/diskstate.cpp
+++ b/vdslib/src/vespa/vdslib/state/diskstate.cpp
@@ -73,7 +73,7 @@ DiskState::DiskState(const vespalib::stringref & serialized)
         }
         LOG(debug, "Unknown key %s in diskstate. Ignoring it, assuming it's a "
                    "new feature from a newer version than ourself: %s",
-            key.c_str(), serialized.c_str());
+            key.c_str(), vespalib::string(serialized).c_str());
     }
 
 }
diff --git a/vdslib/src/vespa/vdslib/state/nodestate.cpp b/vdslib/src/vespa/vdslib/state/nodestate.cpp
index ff6c8f31c89..d59686dcb1c 100644
--- a/vdslib/src/vespa/vdslib/state/nodestate.cpp
+++ b/vdslib/src/vespa/vdslib/state/nodestate.cpp
@@ -229,7 +229,7 @@ NodeState::NodeState(const vespalib::stringref & serialized, const NodeType* typ
         }
         LOG(debug, "Unknown key %s in nodestate. Ignoring it, assuming it's a "
                    "new feature from a newer version than ourself: %s",
-            key.c_str(), serialized.c_str());
+            key.c_str(), vespalib::string(serialized).c_str());
     }
     diskData.addTo(_diskStates);
     updateAnyDiskDownFlag();
diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/api/AthenzPublicKey.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/api/AthenzPublicKey.java
deleted file mode 100644
index 1c810e3e9c9..00000000000
--- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/api/AthenzPublicKey.java
+++ /dev/null
@@ -1,49 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.athenz.api;
-
-import java.security.PublicKey;
-import java.util.Objects;
-
-/**
- * @author bjorncs
- */
-public class AthenzPublicKey {
-
-    private final PublicKey publicKey;
-    private final String keyId;
-
-    public AthenzPublicKey(PublicKey publicKey, String keyId) {
-        this.publicKey = publicKey;
-        this.keyId = keyId;
-    }
-
-    public PublicKey getPublicKey() {
-        return publicKey;
-    }
-
-    public String getKeyId() {
-        return keyId;
-    }
-
-    @Override
-    public boolean equals(Object o) {
-        if (this == o) return true;
-        if (o == null || getClass() != o.getClass()) return false;
-        AthenzPublicKey that = (AthenzPublicKey) o;
-        return Objects.equals(publicKey, that.publicKey) &&
-                Objects.equals(keyId, that.keyId);
-    }
-
-    @Override
-    public int hashCode() {
-        return Objects.hash(publicKey, keyId);
-    }
-
-    @Override
-    public String toString() {
-        return "AthenzPublicKey{" +
-                "publicKey=" + publicKey +
-                ", keyId='" + keyId + '\'' +
-                '}';
-    }
-}
diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/api/ZToken.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/api/ZToken.java
index 36c06132532..1691704afd6 100644
--- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/api/ZToken.java
+++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/api/ZToken.java
@@ -6,7 +6,6 @@ import com.yahoo.vespa.athenz.utils.AthenzIdentities;
 
 import java.util.List;
 import java.util.Objects;
-import java.util.stream.Collectors;
 
 import static java.util.stream.Collectors.toList;
 
@@ -31,6 +30,10 @@ public class ZToken {
         return AthenzIdentities.from(token.getPrincipal());
     }
 
+    public AthenzDomain getDomain() {
+        return new AthenzDomain(token.getDomain());
+    }
+
     public List<AthenzRole> getRoles() {
         String domain = token.getDomain();
         return token.getRoles().stream()
diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/SignedIdentityDocumentEntity.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/SignedIdentityDocumentEntity.java
index 3d4872549d6..6f773d51df3 100644
--- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/SignedIdentityDocumentEntity.java
+++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/bindings/SignedIdentityDocumentEntity.java
@@ -54,7 +54,7 @@ public class SignedIdentityDocumentEntity {
                                         @JsonProperty("ip-addresses") Set<String> ipAddresses,
                                         @JsonProperty("identity-type") String identityType) {
         this.rawIdentityDocument = rawIdentityDocument;
-        this.identityDocument = parseIdentityDocument(rawIdentityDocument);
+        this.identityDocument = rawIdentityDocument != null ? parseIdentityDocument(rawIdentityDocument) : null;
         this.signature = signature;
         this.signingKeyVersion = signingKeyVersion;
         this.providerUniqueId = providerUniqueId;
diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/tls/KeyUtils.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/tls/KeyUtils.java
index 563cae80da2..c2be1a40893 100644
--- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/tls/KeyUtils.java
+++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/tls/KeyUtils.java
@@ -2,6 +2,8 @@
 package com.yahoo.vespa.athenz.tls;
 
 import com.yahoo.athenz.auth.util.Crypto;
+import org.bouncycastle.asn1.ASN1Encodable;
+import org.bouncycastle.asn1.ASN1Primitive;
 import org.bouncycastle.asn1.pkcs.PrivateKeyInfo;
 import org.bouncycastle.openssl.PEMKeyPair;
 import org.bouncycastle.openssl.PEMParser;
@@ -70,11 +72,21 @@ public class KeyUtils {
 
     public static String toPem(PrivateKey privateKey) {
         try (StringWriter stringWriter = new StringWriter(); JcaPEMWriter pemWriter = new JcaPEMWriter(stringWriter)) {
-            pemWriter.writeObject(new PemObject("PRIVATE KEY", privateKey.getEncoded()));
+            // Note: Encoding using PKCS#1 as this is to be read by tools only supporting PKCS#1
+            pemWriter.writeObject(new PemObject("RSA PRIVATE KEY", getPkcs1Bytes(privateKey)));
             pemWriter.flush();
             return stringWriter.toString();
         } catch (IOException e) {
             throw new UncheckedIOException(e);
         }
     }
+
+    private static byte[] getPkcs1Bytes(PrivateKey privateKey) throws IOException{
+
+        byte[] privBytes = privateKey.getEncoded();
+        PrivateKeyInfo pkInfo = PrivateKeyInfo.getInstance(privBytes);
+        ASN1Encodable encodable = pkInfo.parsePrivateKey();
+        ASN1Primitive primitive = encodable.toASN1Primitive();
+        return primitive.getEncoded();
+    }
 }
diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/utils/SiaUtils.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/utils/SiaUtils.java
index 55e9103b040..05459e5488b 100644
--- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/utils/SiaUtils.java
+++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/utils/SiaUtils.java
@@ -7,13 +7,18 @@ import com.yahoo.vespa.athenz.tls.X509CertificateUtils;
 
 import java.io.IOException;
 import java.io.UncheckedIOException;
+import java.nio.file.DirectoryStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.nio.file.StandardCopyOption;
 import java.security.PrivateKey;
 import java.security.cert.X509Certificate;
+import java.util.List;
 import java.util.Optional;
+import java.util.stream.StreamSupport;
+
+import static java.util.stream.Collectors.toList;
 
 /**
  * Misc utility methods for SIA provided credentials
@@ -105,6 +110,25 @@ public class SiaUtils {
         }
     }
 
+    public static List<AthenzService> findSiaServices() {
+        return findSiaServices(DEFAULT_SIA_DIRECTORY);
+    }
+
+    public static List<AthenzService> findSiaServices(Path root) {
+        String keyFileSuffix = ".key.pem";
+        Path keysDirectory = root.resolve("keys");
+        try (DirectoryStream<Path> directoryStream = Files.newDirectoryStream(keysDirectory)) {
+            return StreamSupport.stream(directoryStream.spliterator(), false)
+                    .map(path -> path.getFileName().toString())
+                    .filter(fileName -> fileName.endsWith(keyFileSuffix))
+                    .map(fileName -> fileName.substring(0, fileName.length() - keyFileSuffix.length()))
+                    .map(AthenzService::new)
+                    .collect(toList());
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+    }
+
     private static Path toTempFile(Path file) {
         return Paths.get(file.toAbsolutePath().toString() + ".tmp");
     }
diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/utils/ntoken/AthenzConfTruststore.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/utils/ntoken/AthenzConfTruststore.java
new file mode 100644
index 00000000000..4cb3470635e
--- /dev/null
+++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/utils/ntoken/AthenzConfTruststore.java
@@ -0,0 +1,57 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.athenz.utils.ntoken;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.yahoo.athenz.auth.util.Crypto;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.file.Path;
+import java.security.PublicKey;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+
+/**
+ * A {@link AthenzTruststore} that is backed by athenz.conf
+ *
+ * @author bjorncs
+ */
+public class AthenzConfTruststore implements AthenzTruststore {
+
+    private final Map<String, PublicKey> zmsPublicKeys;
+    private final Map<String, PublicKey> ztsPublicKeys;
+
+    public AthenzConfTruststore(Path athenzConfFile) {
+        try {
+            JsonNode root = new ObjectMapper().readTree(athenzConfFile.toFile());
+            this.zmsPublicKeys = loadPublicKeys((ArrayNode) root.get("zmsPublicKeys"));
+            this.ztsPublicKeys = loadPublicKeys((ArrayNode) root.get("ztsPublicKeys"));
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+    }
+
+    private static Map<String, PublicKey> loadPublicKeys(ArrayNode keysArray) {
+        Map<String, PublicKey> publicKeys = new HashMap<>();
+        for (JsonNode keyEntry : keysArray) {
+            String keyId = keyEntry.get("id").textValue();
+            String encodedPublicKey = keyEntry.get("key").textValue();
+            PublicKey publicKey = Crypto.loadPublicKey(Crypto.ybase64DecodeString(encodedPublicKey));
+            publicKeys.put(keyId, publicKey);
+        }
+        return publicKeys;
+    }
+
+    @Override
+    public Optional<PublicKey> getZmsPublicKey(String keyId) {
+        return Optional.ofNullable(zmsPublicKeys.get(keyId));
+    }
+
+    @Override
+    public Optional<PublicKey> getZtsPublicKey(String keyId) {
+        return Optional.ofNullable(ztsPublicKeys.get(keyId));
+    }
+}
diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/utils/ntoken/AthenzTruststore.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/utils/ntoken/AthenzTruststore.java
new file mode 100644
index 00000000000..83afa288cf0
--- /dev/null
+++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/utils/ntoken/AthenzTruststore.java
@@ -0,0 +1,15 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.athenz.utils.ntoken;
+
+import java.security.PublicKey;
+import java.util.Optional;
+
+/**
+ * A truststore that contains all ZMS and ZTS public keys
+ *
+ * @author bjorncs
+ */
+public interface AthenzTruststore {
+   Optional<PublicKey> getZmsPublicKey(String keyId);
+   Optional<PublicKey> getZtsPublicKey(String keyId);
+}
diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/utils/ntoken/NTokenValidator.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/utils/ntoken/NTokenValidator.java
new file mode 100644
index 00000000000..f4ec0b168d7
--- /dev/null
+++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/utils/ntoken/NTokenValidator.java
@@ -0,0 +1,72 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.athenz.utils.ntoken;
+
+import com.yahoo.athenz.auth.token.PrincipalToken;
+import com.yahoo.log.LogLevel;
+import com.yahoo.vespa.athenz.api.AthenzDomain;
+import com.yahoo.vespa.athenz.api.AthenzPrincipal;
+import com.yahoo.vespa.athenz.api.NToken;
+import com.yahoo.vespa.athenz.utils.AthenzIdentities;
+
+import java.nio.file.Path;
+import java.security.PublicKey;
+import java.time.Duration;
+import java.util.logging.Logger;
+
+/**
+ * Validates the content of an NToken:
+ *   1) Verifies that the token is signed by Athenz
+ *   2) Verifies that the token is not expired
+ *
+ * @author bjorncs
+ */
+public class NTokenValidator {
+    // Max allowed skew in token timestamp (only for creation, not expiry timestamp)
+    private static final long ALLOWED_TIMESTAMP_OFFSET = Duration.ofMinutes(5).getSeconds();
+
+    private static final Logger log = Logger.getLogger(NTokenValidator.class.getName());
+    private final AthenzTruststore truststore;
+
+
+    public NTokenValidator(AthenzTruststore truststore) {
+        this.truststore = truststore;
+    }
+
+    public NTokenValidator(Path athenzConfFile) {
+        this(new AthenzConfTruststore(athenzConfFile));
+    }
+
+    public AthenzPrincipal validate(NToken token) throws InvalidTokenException {
+        PrincipalToken principalToken = new PrincipalToken(token.getRawToken());
+        String keyId = principalToken.getKeyId();
+        String keyService = principalToken.getKeyService();
+        PublicKey zmsPublicKey = (keyService == null || keyService.equals("zms") ? truststore.getZmsPublicKey(keyId) : truststore.getZtsPublicKey(keyId))
+                .orElseThrow(() -> {
+                    String message = "NToken has an unknown keyId: " + keyId;
+                    log.log(LogLevel.WARNING, message);
+                    return new InvalidTokenException(message);
+                });
+        validateSignatureAndExpiration(principalToken, zmsPublicKey);
+        return new AthenzPrincipal(
+                AthenzIdentities.from(
+                        new AthenzDomain(principalToken.getDomain()),
+                        principalToken.getName()),
+                token);
+    }
+
+    private static void validateSignatureAndExpiration(PrincipalToken token, PublicKey zmsPublicKey) throws InvalidTokenException {
+        StringBuilder errorMessageBuilder = new StringBuilder();
+        if (!token.validate(zmsPublicKey, (int) ALLOWED_TIMESTAMP_OFFSET, true, errorMessageBuilder)) {
+            String message = "NToken is expired or has invalid signature: " + errorMessageBuilder.toString();
+            log.log(LogLevel.WARNING, message);
+            throw new InvalidTokenException(message);
+        }
+    }
+
+    public static class InvalidTokenException extends RuntimeException {
+        public InvalidTokenException(String message) {
+            super(message);
+        }
+    }
+
+}
diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/utils/ntoken/package-info.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/utils/ntoken/package-info.java
new file mode 100644
index 00000000000..8760c02d27d
--- /dev/null
+++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/utils/ntoken/package-info.java
@@ -0,0 +1,8 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * @author bjorncs
+ */
+@ExportPackage
+package com.yahoo.vespa.athenz.utils.ntoken;
+
+import com.yahoo.osgi.annotation.ExportPackage;
\ No newline at end of file
diff --git a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/tls/KeyUtilsTest.java b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/tls/KeyUtilsTest.java
index fca4353d400..fbdc6f1e3bd 100644
--- a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/tls/KeyUtilsTest.java
+++ b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/tls/KeyUtilsTest.java
@@ -27,8 +27,8 @@ public class KeyUtilsTest {
     public void can_serialize_deserialize_pem() {
         KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.RSA);
         String pem = KeyUtils.toPem(keyPair.getPrivate());
-        assertThat(pem, containsString("BEGIN PRIVATE KEY"));
-        assertThat(pem, containsString("END PRIVATE KEY"));
+        assertThat(pem, containsString("BEGIN RSA PRIVATE KEY"));
+        assertThat(pem, containsString("END RSA PRIVATE KEY"));
         PrivateKey deserializedKey = KeyUtils.fromPemEncodedPrivateKey(pem);
         assertEquals(keyPair.getPrivate(), deserializedKey);
     }
diff --git a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/utils/SiaUtilsTest.java b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/utils/SiaUtilsTest.java
new file mode 100644
index 00000000000..0282373cdaf
--- /dev/null
+++ b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/utils/SiaUtilsTest.java
@@ -0,0 +1,40 @@
+package com.yahoo.vespa.athenz.utils;
+
+import com.yahoo.vespa.athenz.api.AthenzService;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.List;
+
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.CoreMatchers.hasItem;
+import static org.junit.Assert.assertThat;
+
+/**
+ * @author bjorncs
+ */
+public class SiaUtilsTest {
+
+    @Rule
+    public TemporaryFolder tempDirectory = new TemporaryFolder();
+
+    @Test
+    public void it_finds_all_identity_names_from_files_in_sia_keys_directory() throws IOException {
+        Path siaRoot = tempDirectory.getRoot().toPath();
+        Files.createDirectory(siaRoot.resolve("keys"));
+        AthenzService fooService = new AthenzService("my.domain.foo");
+        Files.createFile(SiaUtils.getPrivateKeyFile(siaRoot, fooService));
+        AthenzService barService = new AthenzService("my.domain.bar");
+        Files.createFile(SiaUtils.getPrivateKeyFile(siaRoot, barService));
+
+        List<AthenzService> siaIdentities = SiaUtils.findSiaServices(siaRoot);
+        assertThat(siaIdentities.size(), equalTo(2));
+        assertThat(siaIdentities, hasItem(fooService));
+        assertThat(siaIdentities, hasItem(barService));
+    }
+
+}
\ No newline at end of file
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/athenz/filter/NTokenValidatorTest.java b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/utils/ntoken/NTokenValidatorTest.java
index 510c806383c..22f97ca8b60 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/athenz/filter/NTokenValidatorTest.java
+++ b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/utils/ntoken/NTokenValidatorTest.java
@@ -1,23 +1,24 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.controller.athenz.filter;
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.athenz.utils.ntoken;
 
 import com.yahoo.athenz.auth.token.PrincipalToken;
 import com.yahoo.vespa.athenz.api.AthenzIdentity;
 import com.yahoo.vespa.athenz.api.AthenzPrincipal;
 import com.yahoo.vespa.athenz.api.AthenzUser;
 import com.yahoo.vespa.athenz.api.NToken;
-import com.yahoo.vespa.hosted.controller.api.integration.athenz.InvalidTokenException;
-import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZmsKeystore;
+import com.yahoo.vespa.athenz.tls.KeyAlgorithm;
+import com.yahoo.vespa.athenz.tls.KeyUtils;
+import com.yahoo.vespa.athenz.utils.ntoken.NTokenValidator.InvalidTokenException;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.ExpectedException;
 
 import java.security.KeyPair;
 import java.security.PrivateKey;
+import java.security.PublicKey;
 import java.time.Instant;
 import java.util.Optional;
 
-import static com.yahoo.vespa.athenz.utils.AthenzIdentities.ZMS_ATHENZ_SERVICE;
 import static org.junit.Assert.assertEquals;
 
 /**
@@ -25,8 +26,8 @@ import static org.junit.Assert.assertEquals;
  */
 public class NTokenValidatorTest {
 
-    private static final KeyPair TRUSTED_KEY = AthenzTestUtils.generateRsaKeypair();
-    private static final KeyPair UNKNOWN_KEY = AthenzTestUtils.generateRsaKeypair();
+    private static final KeyPair TRUSTED_KEY = KeyUtils.generateKeypair(KeyAlgorithm.RSA);
+    private static final KeyPair UNKNOWN_KEY = KeyUtils.generateKeypair(KeyAlgorithm.RSA);
     private static final AthenzIdentity IDENTITY = AthenzUser.fromUserId("myuser");
 
     @Rule
@@ -34,7 +35,7 @@ public class NTokenValidatorTest {
 
     @Test
     public void valid_token_is_accepted() throws InvalidTokenException {
-        NTokenValidator validator = new NTokenValidator(createKeystore());
+        NTokenValidator validator = new NTokenValidator(createTruststore());
         NToken token = createNToken(IDENTITY, Instant.now(), TRUSTED_KEY.getPrivate(), "0");
         AthenzPrincipal principal = validator.validate(token);
         assertEquals("user.myuser", principal.getIdentity().getFullName());
@@ -42,7 +43,7 @@ public class NTokenValidatorTest {
 
     @Test
     public void invalid_signature_is_not_accepted() throws InvalidTokenException {
-        NTokenValidator validator = new NTokenValidator(createKeystore());
+        NTokenValidator validator = new NTokenValidator(createTruststore());
         NToken token = createNToken(IDENTITY, Instant.now(), UNKNOWN_KEY.getPrivate(), "0");
         exceptionRule.expect(InvalidTokenException.class);
         exceptionRule.expectMessage("NToken is expired or has invalid signature");
@@ -51,7 +52,7 @@ public class NTokenValidatorTest {
 
     @Test
     public void expired_token_is_not_accepted() throws InvalidTokenException {
-        NTokenValidator validator = new NTokenValidator(createKeystore());
+        NTokenValidator validator = new NTokenValidator(createTruststore());
         NToken token = createNToken(IDENTITY, Instant.ofEpochMilli(1234) /*long time ago*/, TRUSTED_KEY.getPrivate(), "0");
         exceptionRule.expect(InvalidTokenException.class);
         exceptionRule.expectMessage("NToken is expired or has invalid signature");
@@ -60,28 +61,25 @@ public class NTokenValidatorTest {
 
     @Test
     public void unknown_keyId_is_not_accepted() throws InvalidTokenException {
-        NTokenValidator validator = new NTokenValidator(createKeystore());
+        NTokenValidator validator = new NTokenValidator(createTruststore());
         NToken token = createNToken(IDENTITY, Instant.now(), TRUSTED_KEY.getPrivate(), "unknown-key-id");
         exceptionRule.expect(InvalidTokenException.class);
         exceptionRule.expectMessage("NToken has an unknown keyId");
         validator.validate(token);
     }
 
-    @Test
-    public void failing_to_find_key_should_throw_exception() throws InvalidTokenException {
-        ZmsKeystore keystore = (athensService, keyId) -> { throw new  RuntimeException(); };
-        NTokenValidator validator = new NTokenValidator(keystore);
-        NToken token = createNToken(IDENTITY, Instant.now(), TRUSTED_KEY.getPrivate(), "0");
-        exceptionRule.expect(InvalidTokenException.class);
-        exceptionRule.expectMessage("Failed to retrieve public key");
-        validator.validate(token);
-    }
+    private static AthenzTruststore createTruststore() {
+        return new AthenzTruststore() {
+            @Override
+            public Optional<PublicKey> getZmsPublicKey(String keyId) {
+                return keyId.equals("0") ? Optional.of(TRUSTED_KEY.getPublic()) : Optional.empty();
+            }
 
-    private static ZmsKeystore createKeystore() {
-        return (athensService, keyId) ->
-            athensService.equals(ZMS_ATHENZ_SERVICE) && keyId.equals("0")
-                    ? Optional.of(TRUSTED_KEY.getPublic())
-                    : Optional.empty();
+            @Override
+            public Optional<PublicKey> getZtsPublicKey(String keyId) {
+                return Optional.empty();
+            }
+        };
     }
 
     private static NToken createNToken(AthenzIdentity identity, Instant issueTime, PrivateKey privateKey, String keyId) {
@@ -90,6 +88,7 @@ public class NTokenValidatorTest {
                 .salt("1234")
                 .host("host")
                 .ip("1.2.3.4")
+                .keyService("zms")
                 .issueTime(issueTime.getEpochSecond())
                 .expirationWindow(1000)
                 .build();
diff --git a/vespa-documentgen-plugin/pom.xml b/vespa-documentgen-plugin/pom.xml
index 33f2bd9904b..6ce6fef7fce 100644
--- a/vespa-documentgen-plugin/pom.xml
+++ b/vespa-documentgen-plugin/pom.xml
@@ -14,11 +14,18 @@
   <name>Vespa DocumentGen Plugin</name>
   <dependencies>
     <dependency>
+      <!-- TODO: Explicitly list deps instead - container-dev is for bundle development. -->
       <groupId>com.yahoo.vespa</groupId>
       <artifactId>container-dev</artifactId>
       <version>${project.version}</version>
     </dependency>
     <dependency>
+      <!-- TODO: Excluded from container-dev. Remove when deps are explicitly listed.  -->
+      <groupId>com.yahoo.vespa</groupId>
+      <artifactId>linguistics</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+    <dependency>
       <groupId>com.yahoo.vespa</groupId>
       <artifactId>config-model-api</artifactId>
       <version>${project.version}</version>
diff --git a/vespa-http-client/pom.xml b/vespa-http-client/pom.xml
index 39e633cdb70..6355f7a56a5 100644
--- a/vespa-http-client/pom.xml
+++ b/vespa-http-client/pom.xml
@@ -26,7 +26,6 @@
     <dependency>
       <groupId>org.apache.httpcomponents</groupId>
       <artifactId>httpclient</artifactId>
-      <version>4.4.1</version>
     </dependency>
     <dependency>
       <groupId>org.apache.commons</groupId>
@@ -34,11 +33,6 @@
      <version>3.4</version>
     </dependency>
     <dependency>
-      <groupId>org.apache.httpcomponents</groupId>
-      <artifactId>httpcore</artifactId>
-      <version>4.4.1</version>
-    </dependency>
-    <dependency>
       <groupId>junit</groupId>
       <artifactId>junit</artifactId>
       <scope>test</scope>
diff --git a/vespabase/src/common-env.sh b/vespabase/src/common-env.sh
index 8dfcf8d2c4c..018630b0622 100755
--- a/vespabase/src/common-env.sh
+++ b/vespabase/src/common-env.sh
@@ -107,6 +107,13 @@ populate_environment () {
     fi
 }
 
+add_valgrind_suppressions_file() {
+    if [ -f "$1" ]
+    then
+	VESPA_VALGRIND_SUPPREESSIONS_OPT="$VESPA_VALGRIND_SUPPREESSIONS_OPT --suppressions=$1"
+    fi
+}
+
 populate_environment
 
 PATH=$VESPA_HOME/bin64:$VESPA_HOME/bin:/usr/local/bin:/usr/X11R6/bin:/sbin:/bin:/usr/sbin:/usr/bin
@@ -119,13 +126,17 @@ if [ "$JAVA_HOME" ] && [ -f "${JAVA_HOME}/bin/java" ]; then
     PATH="${PATH}:${JAVA_HOME}/bin"
 fi
 
+VESPA_VALGRIND_SUPPREESSIONS_OPT=""
+add_valgrind_suppressions_file ${VESPA_HOME}/etc/vespa/valgrind-suppressions.txt
+add_valgrind_suppressions_file ${VESPA_HOME}/etc/vespa/suppressions.txt
+
 consider_fallback VESPA_VALGRIND_OPT "--num-callers=32 \
 --run-libc-freeres=yes \
 --track-origins=yes \
 --freelist-vol=1000000000 \
 --leak-check=full \
 --show-reachable=yes \
---suppressions=${VESPA_HOME}/etc/vespa/suppressions.txt"
+${VESPA_VALGRIND_SUPPREESSIONS_OPT}"
 
 consider_fallback VESPA_USE_HUGEPAGES_LIST  $(get_var "hugepages_list")
 consider_fallback VESPA_USE_HUGEPAGES_LIST  "all"
diff --git a/vespalib/src/tests/stllike/string_test.cpp b/vespalib/src/tests/stllike/string_test.cpp
index 2973ffd1ef1..96c095667ee 100644
--- a/vespalib/src/tests/stllike/string_test.cpp
+++ b/vespalib/src/tests/stllike/string_test.cpp
@@ -265,10 +265,10 @@ TEST("testString") {
     // Test std::string conversion of empty string
     stringref sref;
     std::string stdString(sref);
-    EXPECT_TRUE(strcmp("", sref.c_str()) == 0);
+    EXPECT_TRUE(strcmp("", sref.data()) == 0);
     stdString = "abc";
     stringref sref2(stdString);
-    EXPECT_TRUE(stdString.c_str() == sref2.c_str());
+    EXPECT_TRUE(stdString.c_str() == sref2.data());
     EXPECT_TRUE(stdString == sref2);
     EXPECT_TRUE(sref2 == stdString);
     {
diff --git a/vespalib/src/vespa/vespalib/component/version.cpp b/vespalib/src/vespa/vespalib/component/version.cpp
index af38a675de8..3aa8e134e36 100644
--- a/vespalib/src/vespa/vespalib/component/version.cpp
+++ b/vespalib/src/vespa/vespalib/component/version.cpp
@@ -62,10 +62,11 @@ Version::verifySanity()
     }
 }
 
+// Precondition: input.empty() == false
 static int parseInteger(const stringref & input) __attribute__((noinline));
 static int parseInteger(const stringref & input)
 {
-    const char *s = input.c_str();
+    const char *s = input.data();
     unsigned char firstDigit = s[0];
     if (!isdigit(firstDigit))
         throw IllegalArgumentException("integer must start with a digit");
diff --git a/vespalib/src/vespa/vespalib/data/databuffer.cpp b/vespalib/src/vespa/vespalib/data/databuffer.cpp
index 5558a371836..9b04724b601 100644
--- a/vespalib/src/vespa/vespalib/data/databuffer.cpp
+++ b/vespalib/src/vespa/vespalib/data/databuffer.cpp
@@ -11,11 +11,11 @@ size_t padbefore(size_t alignment, const char *buf) {
 
 DataBuffer::DataBuffer(size_t len, size_t alignment, const Alloc & initial)
     : _alignment(alignment),
-      _externalBuf(NULL),
-      _bufstart(NULL),
-      _bufend(NULL),
-      _datapt(NULL),
-      _freept(NULL),
+      _externalBuf(nullptr),
+      _bufstart(nullptr),
+      _bufend(nullptr),
+      _datapt(nullptr),
+      _freept(nullptr),
       _buffer(initial.create(0))
 {
     assert(_alignment > 0);
@@ -29,7 +29,7 @@ DataBuffer::DataBuffer(size_t len, size_t alignment, const Alloc & initial)
         _datapt = _bufstart + padbefore(alignment, _bufstart);
         _freept = _datapt;
         _bufend = _bufstart + bufsize;
-        assert(_bufstart != NULL);
+        assert(_bufstart != nullptr);
     }
 }
 
@@ -68,8 +68,8 @@ DataBuffer::shrink(size_t newsize)
     if (getBufSize() <= newsize || getDataLen() > newsize) {
         return false;
     }
-    char *newbuf = NULL;
-    char *newdata = NULL;
+    char *newbuf = nullptr;
+    char *newdata = nullptr;
     newsize += (_alignment - 1);
     Alloc newBuf(_buffer.create(newsize));
     if (newsize != 0) {
diff --git a/vespalib/src/vespa/vespalib/data/databuffer.h b/vespalib/src/vespa/vespalib/data/databuffer.h
index 28524f373b2..a520ecd58bd 100644
--- a/vespalib/src/vespa/vespalib/data/databuffer.h
+++ b/vespalib/src/vespa/vespalib/data/databuffer.h
@@ -44,6 +44,8 @@ public:
     typedef std::unique_ptr<DataBuffer> UP;
     DataBuffer(const DataBuffer &) = delete;
     DataBuffer &operator=(const DataBuffer &) = delete;
+    DataBuffer(DataBuffer &&) = default;
+    DataBuffer &operator=(DataBuffer &&) = default;
 
     /**
      * Construct a databuffer.
@@ -61,19 +63,19 @@ public:
      * @param buf pointer to preallocated memory
      * @param len length of preallocated memory
      **/
-    DataBuffer(char *buf, size_t len) :
+    DataBuffer(void *buf, size_t len) :
         _alignment(1),
-        _externalBuf(buf),
-        _bufstart(buf),
-        _bufend(buf + len),
+        _externalBuf(static_cast<char *>(buf)),
+        _bufstart(_externalBuf),
+        _bufend(_externalBuf + len),
         _datapt(_bufstart),
         _freept(_bufstart),
         _buffer(Alloc::alloc(0))
     { }
 
-    DataBuffer(const char *buf, size_t len) :
+    DataBuffer(const void *buf, size_t len) :
         _alignment(1),
-        _externalBuf(const_cast<char *>(buf)),
+        _externalBuf(static_cast<char *>(const_cast<void *>(buf))),
         _bufstart(_externalBuf),
         _bufend(_bufstart + len),
         _datapt(_bufstart),
diff --git a/vespalib/src/vespa/vespalib/io/fileutil.cpp b/vespalib/src/vespa/vespalib/io/fileutil.cpp
index 389218cea35..5ab5fb99a0d 100644
--- a/vespalib/src/vespa/vespalib/io/fileutil.cpp
+++ b/vespalib/src/vespa/vespalib/io/fileutil.cpp
@@ -19,7 +19,7 @@ namespace vespalib {
 namespace {
 
     FileInfo::UP
-    processStat(struct stat& filestats, bool result, const stringref & path) {
+    processStat(struct stat& filestats, bool result, stringref path) {
         FileInfo::UP resval;
         if (result) {
             resval.reset(new FileInfo);
@@ -36,7 +36,7 @@ namespace {
         }
         LOG(debug, "stat(%s): Existed? %s, Plain file? %s, Directory? %s, "
                    "Size: %" PRIu64,
-            path.c_str(),
+            string(path).c_str(),
             resval.get() ? "true" : "false",
             resval.get() && resval->_plainfile ? "true" : "false",
             resval.get() && resval->_directory ? "true" : "false",
@@ -69,7 +69,7 @@ operator<<(std::ostream& out, const FileInfo& info)
     return out;
 }
 
-File::File(const stringref & filename)
+File::File(stringref filename)
     : _fd(-1),
       _flags(0),
       _filename(filename),
@@ -79,7 +79,7 @@ File::File(const stringref & filename)
 {
 }
 
-File::File(int fileDescriptor, const stringref & filename)
+File::File(int fileDescriptor, stringref filename)
     : _fd(fileDescriptor),
       _flags(0),
       _filename(filename),
@@ -128,7 +128,7 @@ File::operator=(File& f)
 }
 
 void
-File::setFilename(const stringref & filename)
+File::setFilename(stringref filename)
 {
     if (_filename == filename) return;
     if (_close && _fd != -1) close();
@@ -139,14 +139,14 @@ File::setFilename(const stringref & filename)
 }
 
 namespace {
-    int openAndCreateDirsIfMissing(const stringref & filename, int flags,
+    int openAndCreateDirsIfMissing(const string & filename, int flags,
                                    bool createDirsIfMissing)
     {
         int fd = ::open(filename.c_str(), flags, 0644);
         if (fd < 0 && errno == ENOENT && ((flags & O_CREAT) != 0)
             && createDirsIfMissing)
         {
-            string::size_type pos = filename.rfind('/');
+            auto pos = filename.rfind('/');
             if (pos != string::npos) {
                 string path(filename.substr(0, pos));
                 mkdir(path);
@@ -381,7 +381,7 @@ File::readAll() const
 }
 
 vespalib::string
-File::readAll(const vespalib::stringref & path)
+File::readAll(vespalib::stringref path)
 {
     File file(path);
     file.open(File::READONLY);
@@ -408,6 +408,15 @@ File::sync()
     }
 }
 
+void
+File::sync(vespalib::stringref path)
+{
+    File file(path);
+    file.open(READONLY);
+    file.sync();
+    file.close();
+}
+
 bool
 File::close()
 {
@@ -452,14 +461,14 @@ getCurrentDirectory()
 }
 
 bool
-mkdir(const stringref & directory, bool recursive)
+mkdir(const string & directory, bool recursive)
 {
     if (::mkdir(directory.c_str(), 0777) == 0) {
         LOG(debug, "mkdir(%s): Created directory", directory.c_str());
         return true;
     }
     if (recursive && errno == ENOENT) {
-        string::size_type slashpos = directory.rfind('/');
+        auto slashpos = directory.rfind('/');
         if (slashpos != string::npos) {
             /* Recursively make superdirs.*/
             string superdir = directory.substr(0, slashpos);
@@ -499,8 +508,8 @@ mkdir(const stringref & directory, bool recursive)
 }
 
 void
-symlink(const stringref & oldPath,
-        const stringref & newPath)
+symlink(const string & oldPath,
+        const string & newPath)
 {
     if (::symlink(oldPath.c_str(), newPath.c_str())) {
         asciistream ss;
@@ -513,7 +522,7 @@ symlink(const stringref & oldPath,
 }
 
 string
-readLink(const stringref & path)
+readLink(const string & path)
 {
     char buf[256];
     ssize_t bytes(::readlink(path.c_str(), buf, sizeof(buf)));
@@ -528,7 +537,7 @@ readLink(const stringref & path)
 }
 
 void
-chdir(const stringref & directory)
+chdir(const string & directory)
 {
     if (::chdir(directory.c_str()) != 0) {
         asciistream ost;
@@ -537,12 +546,11 @@ chdir(const stringref & directory)
         throw IoException(ost.str(), IoException::getErrorType(errno),
                           VESPA_STRLOC);
     }
-    LOG(debug, "chdir(%s): Working directory changed.",
-        directory.c_str());
+    LOG(debug, "chdir(%s): Working directory changed.", directory.c_str());
 }
 
 bool
-rmdir(const stringref & directory, bool recursive)
+rmdir(const string & directory, bool recursive)
 {
     string dirname(directory);
     if (!dirname.empty() && *dirname.rbegin() == '/') {
@@ -595,26 +603,26 @@ rmdir(const stringref & directory, bool recursive)
 }
 
 FileInfo::UP
-stat(const stringref & path)
+stat(const string & path)
 {
     struct ::stat filestats;
     return processStat(filestats, ::stat(path.c_str(), &filestats) == 0, path);
 }
 
 FileInfo::UP
-lstat(const stringref & path)
+lstat(const string & path)
 {
     struct ::stat filestats;
     return processStat(filestats, ::lstat(path.c_str(), &filestats) == 0, path);
 }
 
 bool
-fileExists(const vespalib::stringref & path) {
+fileExists(const string & path) {
     return (stat(path).get() != 0);
 }
 
 bool
-unlink(const stringref & filename)
+unlink(const string & filename)
 {
     if (::unlink(filename.c_str()) != 0) {
         if (errno == ENOENT) {
@@ -631,7 +639,7 @@ unlink(const stringref & filename)
 }
 
 bool
-rename(const stringref & frompath, const stringref & topath,
+rename(const string & frompath, const string & topath,
        bool copyDeleteBetweenFilesystems, bool createTargetDirectoryIfMissing)
 {
     LOG(spam, "rename(%s, %s): Renaming file%s.",
@@ -696,7 +704,7 @@ namespace {
 }
 
 void
-copy(const stringref & frompath, const stringref & topath,
+copy(const string & frompath, const string & topath,
      bool createTargetDirectoryIfMissing, bool useDirectIO)
 {
         // Get aligned buffer, so it works with direct IO
@@ -730,7 +738,7 @@ copy(const stringref & frompath, const stringref & topath,
 }
 
 DirectoryList
-listDirectory(const stringref & path)
+listDirectory(const string & path)
 {
     DIR* dir = ::opendir(path.c_str());
     struct dirent* entry;
@@ -762,7 +770,7 @@ getAlignedBuffer(size_t size)
         return MallocAutoPtr(ptr);
 }
 
-string dirname(const stringref name)
+string dirname(stringref name)
 {
     size_t found = name.rfind('/');
     if (found == string::npos) {
@@ -776,7 +784,7 @@ string dirname(const stringref name)
 
 namespace {
 
-void addStat(asciistream &os, const stringref name)
+void addStat(asciistream &os, const string & name)
 {
     struct ::stat filestat;
     memset(&filestat, '\0', sizeof(filestat));
@@ -801,7 +809,7 @@ void addStat(asciistream &os, const stringref name)
 }
 
 string
-getOpenErrorString(const int osError, const stringref filename)
+getOpenErrorString(const int osError, stringref filename)
 {
     asciistream os;
     string dirName(dirname(filename));
@@ -809,12 +817,12 @@ getOpenErrorString(const int osError, const stringref filename)
         getErrorString(osError) << "\") fileStat";
     addStat(os, filename);
     os << " dirStat";
-    addStat(os, dirName.c_str());
+    addStat(os, dirName);
     return os.str();
 }
 
 bool
-isDirectory(const vespalib::stringref & path) {
+isDirectory(const string & path) {
     FileInfo::UP info(stat(path));
     return (info.get() && info->_directory);
 }
diff --git a/vespalib/src/vespa/vespalib/io/fileutil.h b/vespalib/src/vespa/vespalib/io/fileutil.h
index ece37ab0108..1b493a9ebf1 100644
--- a/vespalib/src/vespa/vespalib/io/fileutil.h
+++ b/vespalib/src/vespa/vespalib/io/fileutil.h
@@ -83,10 +83,10 @@ public:
     enum Flag { READONLY = 1, CREATE = 2, DIRECTIO = 4, TRUNC = 8 };
 
     /** Create a file instance, without opening the file. */
-    File(const vespalib::stringref & filename);
+    File(vespalib::stringref filename);
 
     /** Create a file instance of an already open file. */
-    File(int fileDescriptor, const vespalib::stringref & filename);
+    File(int fileDescriptor, vespalib::stringref filename);
 
     /** Copying a file instance, moves any open file descriptor. */
     File(File& f);
@@ -99,7 +99,7 @@ public:
      * Make this instance point at another file.
      * Closes the old file it it was open.
      */
-    void setFilename(const vespalib::stringref & filename);
+    void setFilename(vespalib::stringref filename);
 
     const vespalib::string& getFilename() const { return _filename; }
 
@@ -188,7 +188,17 @@ public:
      * @throw   IoException If we failed to read from file.
      * @return  The content of the file.
      */
-    static vespalib::string readAll(const vespalib::stringref & path);
+    static vespalib::string readAll(vespalib::stringref path);
+
+    /**
+     * Sync file or directory.
+     *
+     * This is a convenience function for the member functions open() and
+     * sync(), see there for more details.
+     *
+     * @throw IoException If we failed to sync the file.
+     */
+    static void sync(vespalib::stringref path);
 
     virtual void sync();
     virtual bool close();
@@ -273,7 +283,7 @@ extern vespalib::string getCurrentDirectory();
  *
  * @return True if it did not exist, false if it did.
  */
-extern bool mkdir(const vespalib::stringref & directory, bool recursive = true);
+extern bool mkdir(const vespalib::string & directory, bool recursive = true);
 
 /**
  * Change working directory.
@@ -281,7 +291,7 @@ extern bool mkdir(const vespalib::stringref & directory, bool recursive = true);
  * @param directory   The directory to change to.
  * @throw IoException If we failed to change to the new working directory.
  */
-extern void chdir(const vespalib::stringref & directory);
+extern void chdir(const vespalib::string & directory);
 
 /**
  * Remove a directory.
@@ -293,7 +303,7 @@ extern void chdir(const vespalib::stringref & directory);
  *
  * @return True if directory existed, false if not.
  */
-extern bool rmdir(const vespalib::stringref & directory, bool recursive = false);
+extern bool rmdir(const vespalib::string & directory, bool recursive = false);
 
 /**
  * Stat a file.
@@ -302,7 +312,7 @@ extern bool rmdir(const vespalib::stringref & directory, bool recursive = false)
  * @return A file info object if everything went well, a null pointer if the
  *         file was not found.
  */
-extern FileInfo::UP stat(const vespalib::stringref & path);
+extern FileInfo::UP stat(const vespalib::string & path);
 
 /**
  * Stat a file. Give info on symlink rather than on file pointed to.
@@ -311,14 +321,14 @@ extern FileInfo::UP stat(const vespalib::stringref & path);
  * @return A file info object if everything went well, a null pointer if the
  *         file was not found.
  */
-extern FileInfo::UP lstat(const vespalib::stringref & path);
+extern FileInfo::UP lstat(const vespalib::string & path);
 
 /**
  * Check if a file exists or not. See also pathExists.
  *
  * @throw IoException If we failed to stat the file.
  */
-extern bool fileExists(const vespalib::stringref & path);
+extern bool fileExists(const vespalib::string & path);
 
 /**
  * Check if a path exists, i.e. whether it's a symbolic link, regular file,
@@ -328,7 +338,7 @@ extern bool fileExists(const vespalib::stringref & path);
  * This function returns true, while fileExists returns true only if the path
  * the symbolic link points to exists.
  */
-extern inline bool pathExists(const vespalib::stringref & path) {
+extern inline bool pathExists(const vespalib::string & path) {
     return (lstat(path).get() != 0);
 }
 
@@ -336,7 +346,7 @@ extern inline bool pathExists(const vespalib::stringref & path) {
  * Get the filesize of the given file. Ignoring if it exists or not.
  * (None-existing files will be reported to have size zero)
  */
-extern inline off_t getFileSize(const vespalib::stringref & path) {
+extern inline off_t getFileSize(const vespalib::string & path) {
     FileInfo::UP info(stat(path));
     return (info.get() == 0 ? 0 : info->_size);
 }
@@ -347,7 +357,7 @@ extern inline off_t getFileSize(const vespalib::stringref & path) {
  * @return True if it is a plain file, false if it don't exist or isn't.
  * @throw IoException If we failed to stat the file.
  */
-extern inline bool isPlainFile(const vespalib::stringref & path) {
+extern inline bool isPlainFile(const vespalib::string & path) {
     FileInfo::UP info(stat(path));
     return (info.get() && info->_plainfile);
 }
@@ -358,7 +368,7 @@ extern inline bool isPlainFile(const vespalib::stringref & path) {
  * @return True if it is a directory, false if it don't exist or isn't.
  * @throw IoException If we failed to stat the file.
  */
-extern bool isDirectory(const vespalib::stringref & path);
+extern bool isDirectory(const vespalib::string & path);
 
 /**
  * Check whether a path is a symlink.
@@ -366,7 +376,7 @@ extern bool isDirectory(const vespalib::stringref & path);
  * @return True if path exists and is a symbolic link.
  * @throw IoException If there's an unexpected stat failure.
  */
-extern inline bool isSymLink(const vespalib::stringref & path) {
+extern inline bool isSymLink(const vespalib::string & path) {
     FileInfo::UP info(lstat(path));
     return (info.get() && info->_symlink);
 }
@@ -384,8 +394,8 @@ extern inline bool isSymLink(const vespalib::stringref & path) {
  * @param newPath Relative link to be created. See above note for semantics.
  * @throw IoException if we fail to create the symlink.
  */
-extern void symlink(const vespalib::stringref & oldPath,
-                    const vespalib::stringref & newPath);
+extern void symlink(const vespalib::string & oldPath,
+                    const vespalib::string & newPath);
 
 /**
  * Read and return the contents of symbolic link at the given path.
@@ -394,7 +404,7 @@ extern void symlink(const vespalib::stringref & oldPath,
  * @return Contents of symbolic link.
  * @throw IoException if we cannot read the link.
  */
-extern vespalib::string readLink(const vespalib::stringref & path);
+extern vespalib::string readLink(const vespalib::string & path);
 
 /**
  * Remove the given file.
@@ -403,7 +413,7 @@ extern vespalib::string readLink(const vespalib::stringref & path);
  * @return True if file was removed, false if it did not exist.
  * @throw IoException If we failed to unlink the file.
  */
-extern bool unlink(const vespalib::stringref & filename);
+extern bool unlink(const vespalib::string & filename);
 
 /**
  * Rename the file at frompath to topath.
@@ -421,16 +431,16 @@ extern bool unlink(const vespalib::stringref & filename);
  * @throw IoException If we failed to rename the file.
  * @return True if file was renamed, false if frompath did not exist.
  */
-extern bool rename(const vespalib::stringref & frompath,
-                   const vespalib::stringref & topath,
+extern bool rename(const vespalib::string & frompath,
+                   const vespalib::string & topath,
                    bool copyDeleteBetweenFilesystems = true,
                    bool createTargetDirectoryIfMissing = false);
 
 /**
  * Copies a file to a destination using Direct IO.
  */
-extern void copy(const vespalib::stringref & frompath,
-                 const vespalib::stringref & topath,
+extern void copy(const vespalib::string & frompath,
+                 const vespalib::string & topath,
                  bool createTargetDirectoryIfMissing = false,
                  bool useDirectIO = true);
 
@@ -438,11 +448,11 @@ extern void copy(const vespalib::stringref & frompath,
  * List the contents of the given directory.
  */
 typedef std::vector<vespalib::string> DirectoryList;
-extern DirectoryList listDirectory(const vespalib::stringref & path);
+extern DirectoryList listDirectory(const vespalib::string & path);
 
 extern MallocAutoPtr getAlignedBuffer(size_t size);
 
-string dirname(const stringref name);
-string getOpenErrorString(const int osError, const stringref name);
+string dirname(stringref name);
+string getOpenErrorString(const int osError, stringref name);
 
 } // vespalib
diff --git a/vespalib/src/vespa/vespalib/objects/nbostream.h b/vespalib/src/vespa/vespalib/objects/nbostream.h
index b51fff1b7cc..c3127e06133 100644
--- a/vespalib/src/vespa/vespalib/objects/nbostream.h
+++ b/vespalib/src/vespa/vespalib/objects/nbostream.h
@@ -71,7 +71,7 @@ public:
         return *this;
      }
     nbostream & operator << (const char * v) { uint32_t sz(strlen(v)); (*this) << sz; write(v, sz); return *this; }
-    nbostream & operator << (const vespalib::stringref & v) { uint32_t sz(v.size()); (*this) << sz; write(v.c_str(), sz); return *this; }
+    nbostream & operator << (const vespalib::stringref & v) { uint32_t sz(v.size()); (*this) << sz; write(v.data(), sz); return *this; }
     nbostream & operator << (const vespalib::string & v) { uint32_t sz(v.size()); (*this) << sz; write(v.c_str(), sz); return *this; }
     nbostream & operator >> (vespalib::string & v) {
         uint32_t sz; (*this) >> sz;
diff --git a/vespalib/src/vespa/vespalib/stllike/asciistream.cpp b/vespalib/src/vespa/vespalib/stllike/asciistream.cpp
index 1be24175ede..7e8570b3d61 100644
--- a/vespalib/src/vespa/vespalib/stllike/asciistream.cpp
+++ b/vespalib/src/vespa/vespalib/stllike/asciistream.cpp
@@ -520,7 +520,7 @@ void asciistream::write(const void * buf, size_t len)
     if (_rPos > 0 && _rPos == length()) {
         clear();
     }
-    if (_rbuf.c_str() != _wbuf.c_str()) {
+    if (_rbuf.data() != _wbuf.data()) {
         if (_wbuf.empty()) {
             _wbuf = _rbuf; // Read only to RW
         } else {
@@ -557,7 +557,7 @@ string asciistream::getline(char delim)
 
 asciistream asciistream::createFromFile(const stringref & fileName)
 {
-    FastOS_File file(fileName.c_str());
+    FastOS_File file(vespalib::string(fileName).c_str());
     asciistream is;
     if (file.OpenReadOnly()) {
         ssize_t sz = file.getSize();
@@ -578,7 +578,7 @@ asciistream asciistream::createFromFile(const stringref & fileName)
 
 asciistream asciistream::createFromDevice(const stringref & fileName)
 {
-    FastOS_File file(fileName.c_str());
+    FastOS_File file(vespalib::string(fileName).c_str());
     asciistream is;
     if (file.OpenReadOnly()) {
         char buf[8192];
diff --git a/vespalib/src/vespa/vespalib/stllike/asciistream.h b/vespalib/src/vespa/vespalib/stllike/asciistream.h
index 1e2b4bc1823..9dd73706d0a 100644
--- a/vespalib/src/vespa/vespalib/stllike/asciistream.h
+++ b/vespalib/src/vespa/vespalib/stllike/asciistream.h
@@ -42,9 +42,9 @@ public:
     asciistream & operator << (char v)                { doFill(1); write(&v, 1); return *this; }
     asciistream & operator << (unsigned char v)       { doFill(1); write(&v, 1); return *this; }
     asciistream & operator << (const char * v)        { if (v != nullptr) { size_t n(strlen(v)); doFill(n); write(v, n); } return *this; }
-    asciistream & operator << (const string & v)      { doFill(v.size()); write(v.c_str(), v.size()); return *this; }
-    asciistream & operator << (const stringref & v)   { doFill(v.size()); write(v.c_str(), v.size()); return *this; }
-    asciistream & operator << (const std::string & v) { doFill(v.size()); write(v.c_str(), v.size()); return *this; }
+    asciistream & operator << (const string & v)      { doFill(v.size()); write(v.data(), v.size()); return *this; }
+    asciistream & operator << (const stringref & v)   { doFill(v.size()); write(v.data(), v.size()); return *this; }
+    asciistream & operator << (const std::string & v) { doFill(v.size()); write(v.data(), v.size()); return *this; }
     asciistream & operator << (int16_t v)    { return *this << static_cast<int64_t>(v); }
     asciistream & operator << (uint16_t v)   { return *this << static_cast<uint64_t>(v); }
     asciistream & operator << (int32_t v)    { return *this << static_cast<int64_t>(v); }
@@ -74,7 +74,7 @@ public:
     asciistream & operator >> (float & v);
     asciistream & operator >> (double & v);
     stringref str() const { return stringref(c_str(), size()); }
-    const char * c_str() const { return _rbuf.c_str() + _rPos; }
+    const char * c_str() const { return _rbuf.data() + _rPos; }
     size_t        size() const { return length() - _rPos; }
     bool         empty() const { return size() == 0; }
     bool           eof() const { return empty(); }
diff --git a/vespalib/src/vespa/vespalib/stllike/hash_fun.h b/vespalib/src/vespa/vespalib/stllike/hash_fun.h
index 8b14d5ce67c..7d7be666136 100644
--- a/vespalib/src/vespa/vespalib/stllike/hash_fun.h
+++ b/vespalib/src/vespa/vespalib/stllike/hash_fun.h
@@ -67,7 +67,7 @@ template<> struct hash<const char *> {
 };
 
 template<> struct hash<vespalib::stringref> {
-    size_t operator() (const vespalib::stringref & arg) const { return hashValue(arg.c_str(), arg.size()); }
+    size_t operator() (const vespalib::stringref & arg) const { return hashValue(arg.data(), arg.size()); }
 };
 
 template<> struct hash<vespalib::string> {
diff --git a/vespalib/src/vespa/vespalib/stllike/string.cpp b/vespalib/src/vespa/vespalib/stllike/string.cpp
index 081d45936c7..47d424e93aa 100644
--- a/vespalib/src/vespa/vespalib/stllike/string.cpp
+++ b/vespalib/src/vespa/vespalib/stllike/string.cpp
@@ -47,7 +47,7 @@ stringref::find(const stringref & s, size_type start) const {
 
 std::ostream & operator << (std::ostream & os, const stringref & v)
 {
-    return os.write(v.c_str(), v.size());
+    return os.write(v.data(), v.size());
 }
 
 template<uint32_t SS>
diff --git a/vespalib/src/vespa/vespalib/stllike/string.h b/vespalib/src/vespa/vespalib/stllike/string.h
index 3db36f5fd41..98ed0929a9b 100644
--- a/vespalib/src/vespa/vespalib/stllike/string.h
+++ b/vespalib/src/vespa/vespalib/stllike/string.h
@@ -31,24 +31,21 @@ public:
      * return a pointer to the data held, or NULL.
      * Note that the data may not be zero terminated, and a default
      * constructed stringref will give a NULL pointer back.  If you
-     * need to make sure c_str() gives a valid zero-terminated string
+     * need to make sure data() gives a valid zero-terminated string
      * you should make a vespalib::string from the stringref.
      **/
-    const char * c_str() const { return _s; }
-
-    /** return a pointer to the data held, or NULL. See c_str(). */
     const char * data() const { return _s; }
 
     size_type      size() const { return _sz; }
     size_type    length() const { return size(); }
     bool          empty() const { return _sz == 0; }
-    const char *  begin() const { return c_str(); }
+    const char *  begin() const { return data(); }
     const char *    end() const { return begin() + size(); }
     const char * rbegin() const { return end() - 1; }
     const char *   rend() const { return begin() - 1; }
     stringref substr(size_type start, size_type sz=npos) const {
         if (start < size()) {
-            return stringref(c_str() + start, std::min(sz, size()-start));
+            return stringref(data() + start, std::min(sz, size()-start));
         }
         return stringref();
     }
@@ -119,7 +116,7 @@ public:
      *     was found, or npos if the substring could not be located
      */
     size_type rfind(const char * s, size_type e=npos) const;
-    int compare(const stringref & s) const { return compare(s.c_str(), s.size()); }
+    int compare(const stringref & s) const { return compare(s.data(), s.size()); }
     int compare(const char *s, size_type sz) const {
         int diff(memcmp(_s, s, std::min(sz, size())));
         return (diff != 0) ? diff : (size() - sz);
@@ -127,23 +124,23 @@ public:
     const char & operator [] (size_t i) const { return _s[i]; }
     operator std::string () const { return std::string(_s, _sz); }
     bool operator  <        (const char * s) const { return compare(s, strlen(s)) < 0; }
-    bool operator  < (const std::string & s) const { return compare(s.c_str(), s.size()) < 0; }
-    bool operator  <   (const stringref & s) const { return compare(s.c_str(), s.size()) < 0; }
+    bool operator  < (const std::string & s) const { return compare(s.data(), s.size()) < 0; }
+    bool operator  <   (const stringref & s) const { return compare(s.data(), s.size()) < 0; }
     bool operator <=        (const char * s) const { return compare(s, strlen(s)) <= 0; }
-    bool operator <= (const std::string & s) const { return compare(s.c_str(), s.size()) <= 0; }
-    bool operator <=   (const stringref & s) const { return compare(s.c_str(), s.size()) <= 0; }
+    bool operator <= (const std::string & s) const { return compare(s.data(), s.size()) <= 0; }
+    bool operator <=   (const stringref & s) const { return compare(s.data(), s.size()) <= 0; }
     bool operator !=        (const char * s) const { return compare(s, strlen(s)) != 0; }
-    bool operator != (const std::string & s) const { return compare(s.c_str(), s.size()) != 0; }
-    bool operator !=   (const stringref & s) const { return compare(s.c_str(), s.size()) != 0; }
+    bool operator != (const std::string & s) const { return compare(s.data(), s.size()) != 0; }
+    bool operator !=   (const stringref & s) const { return compare(s.data(), s.size()) != 0; }
     bool operator ==        (const char * s) const { return compare(s, strlen(s)) == 0; }
-    bool operator == (const std::string & s) const { return compare(s.c_str(), s.size()) == 0; }
-    bool operator ==   (const stringref & s) const { return compare(s.c_str(), s.size()) == 0; }
+    bool operator == (const std::string & s) const { return compare(s.data(), s.size()) == 0; }
+    bool operator ==   (const stringref & s) const { return compare(s.data(), s.size()) == 0; }
     bool operator >=        (const char * s) const { return compare(s, strlen(s)) >= 0; }
-    bool operator >= (const std::string & s) const { return compare(s.c_str(), s.size()) >= 0; }
-    bool operator >=   (const stringref & s) const { return compare(s.c_str(), s.size()) >= 0; }
+    bool operator >= (const std::string & s) const { return compare(s.data(), s.size()) >= 0; }
+    bool operator >=   (const stringref & s) const { return compare(s.data(), s.size()) >= 0; }
     bool operator  >        (const char * s) const { return compare(s, strlen(s)) > 0; }
-    bool operator  > (const std::string & s) const { return compare(s.c_str(), s.size()) > 0; }
-    bool operator  >   (const stringref & s) const { return compare(s.c_str(), s.size()) > 0; }
+    bool operator  > (const std::string & s) const { return compare(s.data(), s.size()) > 0; }
+    bool operator  >   (const stringref & s) const { return compare(s.data(), s.size()) > 0; }
 private:
     const char *_s;
     size_type   _sz;
@@ -178,13 +175,13 @@ public:
     small_string() : _buf(_stack), _sz(0), _bufferSize(StackSize) { _stack[0] = '\0'; }
     small_string(const char * s) : _buf(_stack), _sz(s ? strlen(s) : 0) { init(s); }
     small_string(const void * s, size_type sz) : _buf(_stack), _sz(sz) { init(s); }
-    small_string(const stringref & s) : _buf(_stack), _sz(s.size()) { init(s.c_str()); }
-    small_string(const std::string & s) : _buf(_stack), _sz(s.size()) { init(s.c_str()); }
-    small_string(const small_string & rhs) noexcept : _buf(_stack), _sz(rhs.size()) { init(rhs.c_str()); }
+    small_string(const stringref & s) : _buf(_stack), _sz(s.size()) { init(s.data()); }
+    small_string(const std::string & s) : _buf(_stack), _sz(s.size()) { init(s.data()); }
+    small_string(const small_string & rhs) noexcept : _buf(_stack), _sz(rhs.size()) { init(rhs.data()); }
     small_string(const small_string & rhs, size_type pos, size_type sz=npos) noexcept
         : _buf(_stack), _sz(std::min(sz, rhs.size()-pos))
     {
-        init(rhs.c_str()+pos);
+        init(rhs.data()+pos);
     }
     small_string(size_type sz, char c)
         : _buf(_stack), _sz(0), _bufferSize(StackSize)
@@ -204,10 +201,10 @@ public:
         }
     }
     small_string& operator= (const small_string &rhs) {
-        return assign(rhs.c_str(), rhs.size());
+        return assign(rhs.data(), rhs.size());
     }
     small_string & operator= (const stringref &rhs) {
-        return assign(rhs.c_str(), rhs.size());
+        return assign(rhs.data(), rhs.size());
     }
     small_string& operator= (const char *s) {
         return assign(s);
@@ -321,18 +318,18 @@ public:
     small_string & assign(const char * s) { return assign(s, strlen(s)); }
     small_string & assign(const void * s, size_type sz);
     small_string & assign(const stringref &s, size_type pos, size_type sz) {
-        return assign(s.c_str() + pos, sz);
+        return assign(s.data() + pos, sz);
     }
     small_string & assign(const stringref &rhs) {
-        if (c_str() != rhs.c_str()) assign(rhs.c_str(), rhs.size());
+        if (data() != rhs.data()) assign(rhs.data(), rhs.size());
         return *this;
     }
     small_string & push_back(char c)              { return append(&c, 1); }
     small_string & append(char c)                 { return append(&c, 1); }
     small_string & append(const char * s)         { return append(s, strlen(s)); }
-    small_string & append(const stringref & s)    { return append(s.c_str(), s.size()); }
-    small_string & append(const std::string & s)  { return append(s.c_str(), s.size()); }
-    small_string & append(const small_string & s) { return append(s.c_str(), s.size()); }
+    small_string & append(const stringref & s)    { return append(s.data(), s.size()); }
+    small_string & append(const std::string & s)  { return append(s.data(), s.size()); }
+    small_string & append(const small_string & s) { return append(s.data(), s.size()); }
     small_string & append(const void * s, size_type sz);
     small_string & operator += (char c)                 { return append(c); }
     small_string & operator += (const char * s)         { return append(s); }
@@ -358,7 +355,7 @@ public:
     }
 
     small_string & insert(iterator p, const_iterator f, const_iterator l) { return insert(p-c_str(), f, l-f); }
-    small_string & insert(size_type start, const stringref & v) { return insert(start, v.c_str(), v.size()); }
+    small_string & insert(size_type start, const stringref & v) { return insert(start, v.data(), v.size()); }
     small_string & insert(size_type start, const void * v, size_type sz);
 
     /**
@@ -447,29 +444,29 @@ public:
     */
 
     bool operator  <         (const char * s) const { return compare(s, strlen(s)) < 0; }
-    bool operator  <  (const std::string & s) const { return compare(s.c_str(), s.size()) < 0; }
-    bool operator  < (const small_string & s) const { return compare(s.c_str(), s.size()) < 0; }
-    bool operator  <    (const stringref & s) const { return compare(s.c_str(), s.size()) < 0; }
+    bool operator  <  (const std::string & s) const { return compare(s.data(), s.size()) < 0; }
+    bool operator  < (const small_string & s) const { return compare(s.data(), s.size()) < 0; }
+    bool operator  <    (const stringref & s) const { return compare(s.data(), s.size()) < 0; }
     bool operator <=         (const char * s) const { return compare(s, strlen(s)) <= 0; }
-    bool operator <=  (const std::string & s) const { return compare(s.c_str(), s.size()) <= 0; }
-    bool operator <= (const small_string & s) const { return compare(s.c_str(), s.size()) <= 0; }
-    bool operator <=    (const stringref & s) const { return compare(s.c_str(), s.size()) <= 0; }
+    bool operator <=  (const std::string & s) const { return compare(s.data(), s.size()) <= 0; }
+    bool operator <= (const small_string & s) const { return compare(s.data(), s.size()) <= 0; }
+    bool operator <=    (const stringref & s) const { return compare(s.data(), s.size()) <= 0; }
     bool operator ==         (const char * s) const { return compare(s, strlen(s)) == 0; }
-    bool operator ==  (const std::string & s) const { return compare(s.c_str(), s.size()) == 0; }
-    bool operator == (const small_string & s) const { return compare(s.c_str(), s.size()) == 0; }
-    bool operator ==    (const stringref & s) const { return compare(s.c_str(), s.size()) == 0; }
+    bool operator ==  (const std::string & s) const { return compare(s.data(), s.size()) == 0; }
+    bool operator == (const small_string & s) const { return compare(s.data(), s.size()) == 0; }
+    bool operator ==    (const stringref & s) const { return compare(s.data(), s.size()) == 0; }
     bool operator !=         (const char * s) const { return compare(s, strlen(s)) != 0; }
-    bool operator !=  (const std::string & s) const { return compare(s.c_str(), s.size()) != 0; }
-    bool operator != (const small_string & s) const { return compare(s.c_str(), s.size()) != 0; }
-    bool operator !=    (const stringref & s) const { return compare(s.c_str(), s.size()) != 0; }
+    bool operator !=  (const std::string & s) const { return compare(s.data(), s.size()) != 0; }
+    bool operator != (const small_string & s) const { return compare(s.data(), s.size()) != 0; }
+    bool operator !=    (const stringref & s) const { return compare(s.data(), s.size()) != 0; }
     bool operator >=         (const char * s) const { return compare(s, strlen(s)) >= 0; }
-    bool operator >=  (const std::string & s) const { return compare(s.c_str(), s.size()) >= 0; }
-    bool operator >= (const small_string & s) const { return compare(s.c_str(), s.size()) >= 0; }
-    bool operator >=    (const stringref & s) const { return compare(s.c_str(), s.size()) >= 0; }
+    bool operator >=  (const std::string & s) const { return compare(s.data(), s.size()) >= 0; }
+    bool operator >= (const small_string & s) const { return compare(s.data(), s.size()) >= 0; }
+    bool operator >=    (const stringref & s) const { return compare(s.data(), s.size()) >= 0; }
     bool operator  >         (const char * s) const { return compare(s, strlen(s)) > 0; }
-    bool operator  >  (const std::string & s) const { return compare(s.c_str(), s.size()) > 0; }
-    bool operator  > (const small_string & s) const { return compare(s.c_str(), s.size()) > 0; }
-    bool operator  >    (const stringref & s) const { return compare(s.c_str(), s.size()) > 0; }
+    bool operator  >  (const std::string & s) const { return compare(s.data(), s.size()) > 0; }
+    bool operator  > (const small_string & s) const { return compare(s.data(), s.size()) > 0; }
+    bool operator  >    (const stringref & s) const { return compare(s.data(), s.size()) > 0; }
 
     template<typename T> bool operator != (const T& s) const { return ! operator == (s); }
 
diff --git a/vespalib/src/vespa/vespalib/util/box.h b/vespalib/src/vespa/vespalib/util/box.h
index 5fb81cc9d28..efa5d9653f1 100644
--- a/vespalib/src/vespa/vespalib/util/box.h
+++ b/vespalib/src/vespa/vespalib/util/box.h
@@ -44,11 +44,16 @@ Box<T> make_box(const T &t1, const T &t2, const T &t3, const T &t4) {
 }
 
 template <typename T>
-Box<T> make_box(const T &t1, const T &t2, const T &t3, const T &t4,
-                const T &t5)
+Box<T> make_box(const T &t1, const T &t2, const T &t3, const T &t4, const T &t5)
 {
     return Box<T>().add(t1).add(t2).add(t3).add(t4).add(t5);
 }
 
+template <typename T>
+Box<T> make_box(const T &t1, const T &t2, const T &t3, const T &t4, const T &t5, const T &t6)
+{
+    return Box<T>().add(t1).add(t2).add(t3).add(t4).add(t5).add(t6);
+}
+
 } // namespace vespalib
 
diff --git a/vespalib/src/vespa/vespalib/util/exceptions.cpp b/vespalib/src/vespa/vespalib/util/exceptions.cpp
index 3db697e3b68..fc64aa79ca4 100644
--- a/vespalib/src/vespa/vespalib/util/exceptions.cpp
+++ b/vespalib/src/vespa/vespalib/util/exceptions.cpp
@@ -74,7 +74,8 @@ PortListenException::make_message(int port, const vespalib::stringref &protocol,
                                   const vespalib::stringref &msg)
 {
     return make_string("failed to listen on port %d with protocol %s%s%s",
-                       port, protocol.c_str(), msg.empty() ? "" : ": ", msg.c_str());
+                       port, vespalib::string(protocol).c_str(), msg.empty() ? "" : ": ",
+                       vespalib::string(msg).c_str());
 }
 
 PortListenException::PortListenException(int port, const vespalib::stringref &protocol,
diff --git a/vespalib/src/vespa/vespalib/util/regexp.cpp b/vespalib/src/vespa/vespalib/util/regexp.cpp
index b376b451b6d..e1aa4e9b189 100644
--- a/vespalib/src/vespa/vespalib/util/regexp.cpp
+++ b/vespalib/src/vespa/vespalib/util/regexp.cpp
@@ -32,13 +32,13 @@ Regexp::compile(const vespalib::stringref & re, Flags flags)
     preg->fastmap = static_cast<char *>(malloc(256));
     preg->buffer = NULL;
     preg->allocated = 0;
-    const char * error = re_compile_pattern(re.c_str(), re.size(), preg);
+    const char * error = re_compile_pattern(re.data(), re.size(), preg);
     if (error != 0) {
-        LOG(warning, "invalid regexp '%s': %s", re.c_str(), error);
+        LOG(warning, "invalid regexp '%s': %s", vespalib::string(re).c_str(), error);
         return false;
     }
     if (re_compile_fastmap(preg) != 0) {
-        LOG(warning, "re_compile_fastmap failed for regexp '%s'", re.c_str());
+        LOG(warning, "re_compile_fastmap failed for regexp '%s'", vespalib::string(re).c_str());
         return false;
     }
     return true;
@@ -57,7 +57,7 @@ Regexp::match(const vespalib::stringref & s) const
 {
     if ( ! valid() ) { return false; }
     regex_t *preg = const_cast<regex_t *>(static_cast<const regex_t *>(_data));
-    int pos(re_search(preg, s.c_str(), s.size(), 0, s.size(), NULL));
+    int pos(re_search(preg, s.data(), s.size(), 0, s.size(), NULL));
     if (pos < -1) {
         throw IllegalArgumentException(make_string("re_search failed with code(%d)", pos));
     }
@@ -70,13 +70,13 @@ vespalib::string Regexp::replace(const vespalib::stringref & s, const vespalib::
     regex_t *preg = const_cast<regex_t *>(static_cast<const regex_t *>(_data));
     vespalib::string modified;
     int prev(0);
-    for(int pos(re_search(preg, s.c_str(), s.size(), 0, s.size(), NULL));
+    for(int pos(re_search(preg, s.data(), s.size(), 0, s.size(), NULL));
         pos >=0;
-        pos = re_search(preg, s.c_str()+prev, s.size()-prev, 0, s.size()-prev, NULL))
+        pos = re_search(preg, s.data()+prev, s.size()-prev, 0, s.size()-prev, NULL))
     {
         modified += s.substr(prev, pos);
         modified += replacement;
-        int count = re_match(preg, s.c_str()+prev, s.size()-prev, pos, NULL);
+        int count = re_match(preg, s.data()+prev, s.size()-prev, pos, NULL);
         prev += pos + count;
     }
     modified += s.substr(prev);
diff --git a/vespalib/src/vespa/vespalib/util/rendezvous.h b/vespalib/src/vespa/vespalib/util/rendezvous.h
index 383793ce2a3..5370968c408 100644
--- a/vespalib/src/vespa/vespalib/util/rendezvous.h
+++ b/vespalib/src/vespa/vespalib/util/rendezvous.h
@@ -71,7 +71,7 @@ public:
      * @param n the size of this Rendezvous
      **/
     Rendezvous(size_t n);
-    virtual ~Rendezvous() {}
+    virtual ~Rendezvous();
 
     /**
      * Called by individual threads to synchronize execution and share
diff --git a/vespalib/src/vespa/vespalib/util/rendezvous.hpp b/vespalib/src/vespa/vespalib/util/rendezvous.hpp
index 423e33c6080..07112055de7 100644
--- a/vespalib/src/vespa/vespalib/util/rendezvous.hpp
+++ b/vespalib/src/vespa/vespalib/util/rendezvous.hpp
@@ -19,6 +19,9 @@ Rendezvous<IN, OUT>::Rendezvous(size_t n)
 }
 
 template <typename IN, typename OUT>
+Rendezvous<IN, OUT>::~Rendezvous() = default;
+
+template <typename IN, typename OUT>
 OUT
 Rendezvous<IN, OUT>::rendezvous(const IN &input)
 {
diff --git a/vsm/src/vespa/vsm/common/document.cpp b/vsm/src/vespa/vsm/common/document.cpp
index dc820cd99b3..7e4713e672c 100644
--- a/vsm/src/vespa/vsm/common/document.cpp
+++ b/vsm/src/vespa/vsm/common/document.cpp
@@ -12,10 +12,10 @@ namespace vsm
 
 vespalib::asciistream & operator << (vespalib::asciistream & os, const FieldRef & f)
 {
-  const char *s = f.c_str();
+  const char *s = f.data();
   os << f.size();
   if (s) {
-    os << s;
+      os << s; // Better hope it's null terminated!
   }
   os << " : ";
   return os;
diff --git a/vsm/src/vespa/vsm/searcher/fieldsearcher.cpp b/vsm/src/vespa/vsm/searcher/fieldsearcher.cpp
index 78531f41cf8..2ba5fea3153 100644
--- a/vsm/src/vespa/vsm/searcher/fieldsearcher.cpp
+++ b/vsm/src/vespa/vsm/searcher/fieldsearcher.cpp
@@ -115,7 +115,7 @@ void FieldSearcher::prepare(QueryTermList & qtl, const SharedSearcherBuf & UNUSE
 size_t FieldSearcher::countWords(const FieldRef & f)
 {
     size_t words = 0;
-    const char * n = f.c_str();
+    const char * n = f.data();
     const char * e = n + f.size();
     for( ; n < e; ++n) {
         for (; isspace(*n) && (n<e); ++n);
diff --git a/vsm/src/vespa/vsm/searcher/futf8strchrfieldsearcher.cpp b/vsm/src/vespa/vsm/searcher/futf8strchrfieldsearcher.cpp
index b388507aed5..b26b4bd5133 100644
--- a/vsm/src/vespa/vsm/searcher/futf8strchrfieldsearcher.cpp
+++ b/vsm/src/vespa/vsm/searcher/futf8strchrfieldsearcher.cpp
@@ -222,7 +222,7 @@ size_t FUTF8StrChrFieldSearcher::matchTerm(const FieldRef & f, QueryTerm & qt)
 {
   _folded.reserve(f.size()+16*3);  //Enable fulle xmm0 store
   size_t unalignedStart(0);
-  bool ascii7Bit = lfoldua(f.c_str(), f.size(), &_folded[0], unalignedStart);
+  bool ascii7Bit = lfoldua(f.data(), f.size(), &_folded[0], unalignedStart);
   if (ascii7Bit) {
     char * folded = &_folded[unalignedStart];
     /// Add the pattern 00 01 00 to avoid multiple eof tests of falling off the edge.
@@ -240,7 +240,7 @@ size_t FUTF8StrChrFieldSearcher::matchTerms(const FieldRef & f, const size_t min
 {
   _folded.reserve(f.size()+16*3);  //Enable fulle xmm0 store
   size_t unalignedStart(0);
-  bool ascii7Bit = lfoldua(f.c_str(), f.size(), &_folded[0], unalignedStart);
+  bool ascii7Bit = lfoldua(f.data(), f.size(), &_folded[0], unalignedStart);
   if (ascii7Bit) {
     char * folded = &_folded[unalignedStart];
     /// Add the pattern 00 01 00 to avoid multiple eof tests of falling off the edge.
diff --git a/vsm/src/vespa/vsm/searcher/strchrfieldsearcher.cpp b/vsm/src/vespa/vsm/searcher/strchrfieldsearcher.cpp
index 93bfa76081f..1be1326807e 100644
--- a/vsm/src/vespa/vsm/searcher/strchrfieldsearcher.cpp
+++ b/vsm/src/vespa/vsm/searcher/strchrfieldsearcher.cpp
@@ -16,7 +16,7 @@ void StrChrFieldSearcher::onValue(const document::FieldValue & fv)
 {
     const document::LiteralFieldValueB & sfv = static_cast<const document::LiteralFieldValueB &>(fv);
     vespalib::stringref val = sfv.getValueRef();
-    FieldRef fr(val.c_str(), std::min(maxFieldLength(), val.size()));
+    FieldRef fr(val.data(), std::min(maxFieldLength(), val.size()));
     matchDoc(fr);
 }
 
diff --git a/vsm/src/vespa/vsm/searcher/utf8strchrfieldsearcher.cpp b/vsm/src/vespa/vsm/searcher/utf8strchrfieldsearcher.cpp
index 82821dfeaff..b54ed2c583d 100644
--- a/vsm/src/vespa/vsm/searcher/utf8strchrfieldsearcher.cpp
+++ b/vsm/src/vespa/vsm/searcher/utf8strchrfieldsearcher.cpp
@@ -14,7 +14,7 @@ UTF8StrChrFieldSearcher::matchTerms(const FieldRef & f, const size_t mintsz)
 {
     (void) mintsz;
     termcount_t words(0);
-    const byte * n = reinterpret_cast<const byte *> (f.c_str());
+    const byte * n = reinterpret_cast<const byte *> (f.data());
     const byte * e = n + f.size();
     if (f.size() >= _buf->size()) {
         _buf->reserve(f.size() + 1);
diff --git a/vsm/src/vespa/vsm/searcher/utf8stringfieldsearcherbase.cpp b/vsm/src/vespa/vsm/searcher/utf8stringfieldsearcherbase.cpp
index 0ce002e6765..872cfebfd70 100644
--- a/vsm/src/vespa/vsm/searcher/utf8stringfieldsearcherbase.cpp
+++ b/vsm/src/vespa/vsm/searcher/utf8stringfieldsearcherbase.cpp
@@ -104,7 +104,7 @@ size_t
 UTF8StringFieldSearcherBase::matchTermRegular(const FieldRef & f, QueryTerm & qt)
 {
     termcount_t words(0);
-    const byte * n = reinterpret_cast<const byte *> (f.c_str());
+    const byte * n = reinterpret_cast<const byte *> (f.data());
     // __builtin_prefetch(n, 0, 0);
     const cmptype_t * term;
     termsize_t tsz = qt.term(term);
@@ -134,7 +134,7 @@ UTF8StringFieldSearcherBase::matchTermRegular(const FieldRef & f, QueryTerm & qt
 size_t
 UTF8StringFieldSearcherBase::matchTermExact(const FieldRef & f, QueryTerm & qt)
 {
-    const byte * n = reinterpret_cast<const byte *> (f.c_str());
+    const byte * n = reinterpret_cast<const byte *> (f.data());
     const cmptype_t * term;
     termsize_t tsz = qt.term(term);
     const cmptype_t * eterm = term+tsz;
@@ -161,7 +161,7 @@ size_t
 UTF8StringFieldSearcherBase::matchTermSubstring(const FieldRef & f, QueryTerm & qt)
 {
     if (qt.termLen() == 0) { return 0; }
-    const byte * n = reinterpret_cast<const byte *> (f.c_str());
+    const byte * n = reinterpret_cast<const byte *> (f.data());
     const cmptype_t * term;
     termsize_t tsz = qt.term(term);
     if ( f.size() >= _buf->size()) {
@@ -195,7 +195,7 @@ size_t
 UTF8StringFieldSearcherBase::matchTermSuffix(const FieldRef & f, QueryTerm & qt)
 {
     termcount_t words = 0;
-    const byte * srcbuf = reinterpret_cast<const byte *> (f.c_str());
+    const byte * srcbuf = reinterpret_cast<const byte *> (f.data());
     const byte * srcend = srcbuf + f.size();
     const cmptype_t * term;
     termsize_t tsz = qt.term(term);
diff --git a/vsm/src/vespa/vsm/searcher/utf8substringsearcher.cpp b/vsm/src/vespa/vsm/searcher/utf8substringsearcher.cpp
index 94fb14b6217..4b8c6e31927 100644
--- a/vsm/src/vespa/vsm/searcher/utf8substringsearcher.cpp
+++ b/vsm/src/vespa/vsm/searcher/utf8substringsearcher.cpp
@@ -13,7 +13,7 @@ IMPLEMENT_DUPLICATE(UTF8SubStringFieldSearcher);
 size_t
 UTF8SubStringFieldSearcher::matchTerms(const FieldRef & f, const size_t mintsz)
 {
-    const byte * n = reinterpret_cast<const byte *> (f.c_str());
+    const byte * n = reinterpret_cast<const byte *> (f.data());
     if ( f.size() >= _buf->size()) {
         _buf->reserve(f.size() + 1);
     }
diff --git a/vsm/src/vespa/vsm/searcher/utf8substringsnippetmodifier.cpp b/vsm/src/vespa/vsm/searcher/utf8substringsnippetmodifier.cpp
index b229b7a5ebd..eee88b34ea6 100644
--- a/vsm/src/vespa/vsm/searcher/utf8substringsnippetmodifier.cpp
+++ b/vsm/src/vespa/vsm/searcher/utf8substringsnippetmodifier.cpp
@@ -14,8 +14,8 @@ size_t
 UTF8SubstringSnippetModifier::matchTerms(const FieldRef & f, const size_t mintsz)
 {
     _modified->reset();
-    _readPtr = f.c_str();
-    const byte * src = reinterpret_cast<const byte *> (f.c_str());
+    _readPtr = f.data();
+    const byte * src = reinterpret_cast<const byte *> (f.data());
     // resize ucs4 buffer
     if (f.size() >= _buf->size()) {
         _buf->resize(f.size() + 1);
@@ -46,8 +46,8 @@ UTF8SubstringSnippetModifier::matchTerms(const FieldRef & f, const size_t mintsz
             const cmptype_t * dtmp = ditr;
             for (; (titr < tend) && (*titr == *dtmp); ++titr, ++dtmp);
             if (titr == tend) {
-                const char * mbegin = f.c_str() + (*_offsets)[ditr - dbegin];
-                const char * mend = f.c_str() + ((dtmp < dend) ? ((*_offsets)[dtmp - dbegin]) : f.size());
+                const char * mbegin = f.data() + (*_offsets)[ditr - dbegin];
+                const char * mend = f.data() + ((dtmp < dend) ? ((*_offsets)[dtmp - dbegin]) : f.size());
                 if (_readPtr <= mbegin) {
                     // We will only copy from the field ref once.
                     // If we have overlapping matches only the first one will be considered.
@@ -61,9 +61,9 @@ UTF8SubstringSnippetModifier::matchTerms(const FieldRef & f, const size_t mintsz
             for(; (ditr < drend) && ! Fast_UnicodeUtil::IsWordChar(*ditr) ; ++ditr );
         }
     }
-    assert(_readPtr <= (f.c_str() + f.size()));
+    assert(_readPtr <= (f.data() + f.size()));
     // copy remaining
-    size_t toCopy = f.size() - (_readPtr - f.c_str());
+    size_t toCopy = f.size() - (_readPtr - f.data());
     copyToModified(toCopy);
 
     return words + 1; // we must also count the last word
diff --git a/vsm/src/vespa/vsm/searcher/utf8suffixstringfieldsearcher.cpp b/vsm/src/vespa/vsm/searcher/utf8suffixstringfieldsearcher.cpp
index f469fa2ea73..13074937185 100644
--- a/vsm/src/vespa/vsm/searcher/utf8suffixstringfieldsearcher.cpp
+++ b/vsm/src/vespa/vsm/searcher/utf8suffixstringfieldsearcher.cpp
@@ -14,7 +14,7 @@ UTF8SuffixStringFieldSearcher::matchTerms(const FieldRef & f, const size_t mints
 {
     (void) mintsz;
     termcount_t words = 0;
-    const byte * srcbuf = reinterpret_cast<const byte *> (f.c_str());
+    const byte * srcbuf = reinterpret_cast<const byte *> (f.data());
     const byte * srcend = srcbuf + f.size();
     if (f.size() >= _buf->size()) {
         _buf->reserve(f.size() + 1);
diff --git a/vsm/src/vespa/vsm/vsm/docsumfilter.cpp b/vsm/src/vespa/vsm/vsm/docsumfilter.cpp
index e6af5fb6477..034c3c57bde 100644
--- a/vsm/src/vespa/vsm/vsm/docsumfilter.cpp
+++ b/vsm/src/vespa/vsm/vsm/docsumfilter.cpp
@@ -74,7 +74,7 @@ public:
         if (fv.getClass().inherits(document::LiteralFieldValueB::classId)) {
             const document::LiteralFieldValueB & lfv = static_cast<const document::LiteralFieldValueB &>(fv);
             vespalib::stringref s = lfv.getValueRef();
-            addToPacker(s.c_str(), s.size());
+            addToPacker(s.data(), s.size());
         } else {
             vespalib::string s = fv.toString();
             addToPacker(s.c_str(), s.size());
diff --git a/vsm/src/vespa/vsm/vsm/flattendocsumwriter.cpp b/vsm/src/vespa/vsm/vsm/flattendocsumwriter.cpp
index bf690e50719..080723e1dbd 100644
--- a/vsm/src/vespa/vsm/vsm/flattendocsumwriter.cpp
+++ b/vsm/src/vespa/vsm/vsm/flattendocsumwriter.cpp
@@ -21,13 +21,13 @@ FlattenDocsumWriter::onPrimitive(uint32_t, const Content & c)
     if (fv.getClass().inherits(document::LiteralFieldValueB::classId)) {
         const document::LiteralFieldValueB & lfv = static_cast<const document::LiteralFieldValueB &>(fv);
         vespalib::stringref value = lfv.getValueRef();
-        _output.put(value.c_str(), value.size());
+        _output.put(value.data(), value.size());
     } else if (fv.getClass().inherits(document::NumericFieldValueBase::classId)) {
         vespalib::string value = fv.getAsString();
-        _output.put(value.c_str(), value.size());
+        _output.put(value.data(), value.size());
     } else {
         vespalib::string value = fv.toString();
-        _output.put(value.c_str(), value.size());
+        _output.put(value.data(), value.size());
     }
     _useSeparator = true;
 }
diff --git a/zkfacade/pom.xml b/zkfacade/pom.xml
index 0aa689d896d..3a4e9232d83 100644
--- a/zkfacade/pom.xml
+++ b/zkfacade/pom.xml
@@ -53,7 +53,6 @@
     <dependency>
       <groupId>com.google.guava</groupId>
       <artifactId>guava</artifactId>
-      <version>13.0.1</version>
       <scope>provided</scope>
     </dependency>
     <dependency>
diff --git a/zkfacade/src/main/java/com/yahoo/vespa/zookeeper/ZooKeeperServer.java b/zkfacade/src/main/java/com/yahoo/vespa/zookeeper/ZooKeeperServer.java
index 352635ac920..405afcd3c39 100644
--- a/zkfacade/src/main/java/com/yahoo/vespa/zookeeper/ZooKeeperServer.java
+++ b/zkfacade/src/main/java/com/yahoo/vespa/zookeeper/ZooKeeperServer.java
@@ -9,6 +9,7 @@ import com.yahoo.component.AbstractComponent;
 import com.yahoo.log.LogLevel;
 import static com.yahoo.vespa.defaults.Defaults.getDefaults;
 
+import java.io.File;
 import java.io.FileWriter;
 import java.io.IOException;
 import java.util.List;
@@ -61,9 +62,10 @@ public class ZooKeeperServer extends AbstractComponent implements Runnable {
     public static ImmutableSet<String> getAllowedClientHostnames() { return allowedClientHostnames; }
     
     private void writeConfigToDisk(ZookeeperServerConfig config) {
-       String cfg = transformConfigToString(config);
-       try (FileWriter writer = new FileWriter(getDefaults().underVespaHome(config.zooKeeperConfigFile()))) {
-           writer.write(cfg);
+       String configFilePath = getDefaults().underVespaHome(config.zooKeeperConfigFile());
+       new File(configFilePath).getParentFile().mkdirs();
+       try (FileWriter writer = new FileWriter(configFilePath)) {
+           writer.write(transformConfigToString(config));
            writeMyIdFile(config);
        } catch (IOException e) {
            throw new RuntimeException("Error writing zookeeper config", e);