Merge branch 'master' into hakonhall/remove-use-bucket-space-metric-feature-flag

2622 files changed, 55949 insertions, 28508 deletions

diff --git a/CMakeLists.txt b/CMakeLists.txt
index a4ef1bdba25..906a00ad843 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -5,7 +5,11 @@
 cmake_minimum_required(VERSION 3.5 FATAL_ERROR)
 
 include(functions.cmake)
-list(APPEND CMAKE_MODULE_PATH "$ENV{HOME}/share/cmake/Modules" "/opt/vespa-deps/share/cmake/Modules")
+list(APPEND CMAKE_MODULE_PATH
+     "$ENV{HOME}/share/cmake/Modules"
+     "/opt/vespa-deps/share/cmake/Modules"
+     "${CMAKE_CURRENT_SOURCE_DIR}/cmake"
+)
 include(default_build_settings.cmake)
 vespa_detect_build_platform()
 message("-- Vespa build platform is ${VESPA_OS_DISTRO} ${VESPA_OS_DISTRO_VERSION}")
@@ -13,6 +17,9 @@ vespa_use_default_cxx_compiler()
 vespa_use_default_java_home()
 
 project(vespa CXX C)
+vespa_use_default_vespa_unprivileged()
+vespa_use_default_cmake_install_prefix()
+vespa_use_default_vespa_user()
 vespa_use_default_build_settings()
 
 # allows import of project in CLion on OSX
diff --git a/README.md b/README.md
index 09a3f9a7124..194d24885f1 100644
--- a/README.md
+++ b/README.md
@@ -63,17 +63,13 @@ You do not need to build Vespa to use it, but if you want to contribute you need
 This section explains how to build and test Vespa. To understand where to make changes, see [Code-map.md](Code-map.md).
 Some suggested improvements with pointers to code are in [TODO.md](TODO.md).
 
-### Set up the build environment
+### Development environment
 
-C++ and Java building is supported on CentOS 7. The Java source can also be built on any platform having Java 11 and Maven installed. 
-We recommend using the following environment: [Create C++ / Java dev environment on CentOS using VirtualBox and Vagrant](vagrant/README.md).
-You can also setup CentOS 7 natively and install the following build dependencies:
-
-    sudo yum-config-manager --add-repo https://copr.fedorainfracloud.org/coprs/g/vespa/vespa/repo/epel-7/group_vespa-vespa-epel-7.repo
-    sudo yum -y install epel-release centos-release-scl yum-utils
-    sudo yum -y install ccache \
-        rpm-build
-    yum-builddep -y <vespa-source>/dist/vespa.spec
+C++ and Java building is supported on CentOS 7.
+The Java source can also be built on any platform having Java 11 and Maven installed.
+Use the following guide to setup a complete development environment using Docker
+for building Vespa, running unit tests and running system tests:
+[Vespa development on CentOS 7](https://github.com/vespa-engine/docker-image-dev#vespa-development-on-centos-7).
 
 ### Build Java modules
 
@@ -82,20 +78,13 @@ You can also setup CentOS 7 natively and install the following build dependencie
     bash bootstrap.sh java
     mvn -T <num-threads> install
 
-### Build C++ modules
+Use this if you only need to build the Java modules, otherwise follow the complete development guide above.
 
-Replace `<build-dir>` with the name of the directory in which you'd like to build Vespa.
-Replace `<source-dir>` with the directory in which you've cloned/unpacked the source tree.
+### Build RPM packages
 
-    bash bootstrap-cpp.sh <source-dir> <build-dir>
-    cd <build-dir>
-    make -j <num-threads>
-    ctest3 -j <num-threads>
+See [Building Vespa RPM](docker/README.md#building-vespa-rpm) for details.
 
-### Create RPM packages
 
-    sh dist.sh VERSION && rpmbuild -ba ~/rpmbuild/SPECS/vespa-VERSION.spec
-    
 ## License
 
 Code licensed under the Apache 2.0 license. See [LICENSE](LICENSE) for terms.
diff --git a/application-model/src/main/java/com/yahoo/vespa/applicationmodel/ApplicationInstance.java b/application-model/src/main/java/com/yahoo/vespa/applicationmodel/ApplicationInstance.java
index 3363ddb040f..1d230d6cb47 100644
--- a/application-model/src/main/java/com/yahoo/vespa/applicationmodel/ApplicationInstance.java
+++ b/application-model/src/main/java/com/yahoo/vespa/applicationmodel/ApplicationInstance.java
@@ -11,24 +11,28 @@ import java.util.Set;
  */
 public class ApplicationInstance {
 
-    private final TenantId tenantId;
-    private final ApplicationInstanceId applicationInstanceId;
+    private final ApplicationInstanceReference reference;
     private final Set<ServiceCluster> serviceClusters;
 
-    public ApplicationInstance(TenantId tenantId, ApplicationInstanceId applicationInstanceId, Set<ServiceCluster> serviceClusters) {
-        this.tenantId = tenantId;
-        this.applicationInstanceId = applicationInstanceId;
+    public ApplicationInstance(TenantId tenantId,
+                               ApplicationInstanceId applicationInstanceId,
+                               Set<ServiceCluster> serviceClusters) {
+        this(new ApplicationInstanceReference(tenantId, applicationInstanceId), serviceClusters);
+    }
+
+    public ApplicationInstance(ApplicationInstanceReference reference, Set<ServiceCluster> serviceClusters) {
+        this.reference = reference;
         this.serviceClusters = serviceClusters;
     }
 
     @JsonProperty("tenantId")
     public TenantId tenantId() {
-        return tenantId;
+        return reference.tenantId();
     }
 
     @JsonProperty("applicationInstanceId")
     public ApplicationInstanceId applicationInstanceId() {
-        return applicationInstanceId;
+        return reference.applicationInstanceId();
     }
 
     @JsonProperty("serviceClusters")
@@ -38,7 +42,7 @@ public class ApplicationInstance {
 
     @JsonProperty("reference")
     public ApplicationInstanceReference reference() {
-        return new ApplicationInstanceReference(tenantId, applicationInstanceId);
+        return reference;
     }
 
     @Override
@@ -46,21 +50,19 @@ public class ApplicationInstance {
         if (this == o) return true;
         if (o == null || getClass() != o.getClass()) return false;
         ApplicationInstance that = (ApplicationInstance) o;
-        return Objects.equals(tenantId, that.tenantId) &&
-                Objects.equals(applicationInstanceId, that.applicationInstanceId) &&
+        return Objects.equals(reference, that.reference) &&
                 Objects.equals(serviceClusters, that.serviceClusters);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(tenantId, applicationInstanceId, serviceClusters);
+        return Objects.hash(reference, serviceClusters);
     }
 
     @Override
     public String toString() {
         return "ApplicationInstance{" +
-                "tenantId=" + tenantId +
-                ", applicationInstanceId=" + applicationInstanceId +
+                "reference=" + reference +
                 ", serviceClusters=" + serviceClusters +
                 '}';
     }
diff --git a/application/pom.xml b/application/pom.xml
index 7bcda0ce386..cd27b53f557 100644
--- a/application/pom.xml
+++ b/application/pom.xml
@@ -119,9 +119,13 @@
     <dependency>
       <groupId>org.apache.httpcomponents</groupId>
       <artifactId>httpclient</artifactId>
-      <scope>test</scope>
     </dependency>
     <dependency>
+      <groupId>org.apache.commons</groupId>
+      <artifactId>commons-math3</artifactId>
+    </dependency>
+
+    <dependency>
       <groupId>com.yahoo.vespa</groupId>
       <artifactId>testutil</artifactId>
       <version>${project.version}</version>
diff --git a/application/src/main/java/com/yahoo/application/Application.java b/application/src/main/java/com/yahoo/application/Application.java
index 5f9b1f51863..87e82a3fdeb 100644
--- a/application/src/main/java/com/yahoo/application/Application.java
+++ b/application/src/main/java/com/yahoo/application/Application.java
@@ -2,6 +2,7 @@
 package com.yahoo.application;
 
 import ai.vespa.rankingexpression.importer.configmodelview.MlModelImporter;
+import ai.vespa.rankingexpression.importer.lightgbm.LightGBMImporter;
 import ai.vespa.rankingexpression.importer.onnx.OnnxImporter;
 import ai.vespa.rankingexpression.importer.tensorflow.TensorFlowImporter;
 import ai.vespa.rankingexpression.importer.vespa.VespaImporter;
@@ -117,11 +118,13 @@ public final class Application implements AutoCloseable {
             List<MlModelImporter> modelImporters = List.of(new VespaImporter(),
                                                            new TensorFlowImporter(),
                                                            new OnnxImporter(),
+                                                           new LightGBMImporter(),
                                                            new XGBoostImporter());
             DeployState deployState = new DeployState.Builder()
                     .applicationPackage(FilesApplicationPackage.fromFile(path.toFile(), true))
                     .modelImporters(modelImporters)
                     .deployLogger((level, s) -> { })
+                    .accessLoggingEnabledByDefault(false)
                     .build();
             return new VespaModel(new NullConfigModelRegistry(), deployState);
         } catch (IOException | SAXException e) {
@@ -251,13 +254,13 @@ public final class Application implements AutoCloseable {
          * @throws java.io.IOException e.g.if file not found
          */
         public Builder documentType(String name, String searchDefinition) throws IOException {
-            Path path = nestedResource(ApplicationPackage.SEARCH_DEFINITIONS_DIR, name, ApplicationPackage.SD_NAME_SUFFIX);
+            Path path = nestedResource(ApplicationPackage.SCHEMAS_DIR, name, ApplicationPackage.SD_NAME_SUFFIX);
             createFile(path, searchDefinition);
             return this;
         }
 
         public Builder expressionInclude(String name, String searchDefinition) throws IOException {
-            Path path = nestedResource(ApplicationPackage.SEARCH_DEFINITIONS_DIR, name, ApplicationPackage.RANKEXPRESSION_NAME_SUFFIX);
+            Path path = nestedResource(ApplicationPackage.SCHEMAS_DIR, name, ApplicationPackage.RANKEXPRESSION_NAME_SUFFIX);
             createFile(path, searchDefinition);
             return this;
         }
@@ -268,7 +271,7 @@ public final class Application implements AutoCloseable {
          * @throws java.io.IOException e.g.if file not found
          */
         public Builder rankExpression(String name, String rankExpressionContent) throws IOException {
-            Path path = nestedResource(ApplicationPackage.SEARCH_DEFINITIONS_DIR, name, ApplicationPackage.RANKEXPRESSION_NAME_SUFFIX);
+            Path path = nestedResource(ApplicationPackage.SCHEMAS_DIR, name, ApplicationPackage.RANKEXPRESSION_NAME_SUFFIX);
             createFile(path, rankExpressionContent);
             return this;
         }
@@ -584,6 +587,7 @@ public final class Application implements AutoCloseable {
                     xml.println("</search>");
                 }
 
+                xml.println("<accesslog type=\"disabled\" />");
                 xml.println("</container>");
             }
 
diff --git a/application/src/main/java/com/yahoo/application/ApplicationBuilder.java b/application/src/main/java/com/yahoo/application/ApplicationBuilder.java
index 9d7cf2c8673..ac64ae188e5 100644
--- a/application/src/main/java/com/yahoo/application/ApplicationBuilder.java
+++ b/application/src/main/java/com/yahoo/application/ApplicationBuilder.java
@@ -37,13 +37,13 @@ public class ApplicationBuilder {
     }
 
     public ApplicationBuilder documentType(String name, String searchDefinition) throws IOException {
-        Path path = nestedResource(ApplicationPackage.SEARCH_DEFINITIONS_DIR, name, ApplicationPackage.SD_NAME_SUFFIX);
+        Path path = nestedResource(ApplicationPackage.SCHEMAS_DIR, name, ApplicationPackage.SD_NAME_SUFFIX);
         createFile(path, searchDefinition);
         return this;
     }
 
     public ApplicationBuilder rankExpression(String name, String rankExpressionContent) throws IOException {
-        Path path = nestedResource(ApplicationPackage.SEARCH_DEFINITIONS_DIR, name, ApplicationPackage.RANKEXPRESSION_NAME_SUFFIX);
+        Path path = nestedResource(ApplicationPackage.SCHEMAS_DIR, name, ApplicationPackage.RANKEXPRESSION_NAME_SUFFIX);
         createFile(path, rankExpressionContent);
         return this;
     }
diff --git a/application/src/test/app-packages/model-evaluation/models/lightgbm/regression.json b/application/src/test/app-packages/model-evaluation/models/lightgbm/regression.json
new file mode 100644
index 00000000000..cf0488ecd8b
--- /dev/null
+++ b/application/src/test/app-packages/model-evaluation/models/lightgbm/regression.json
@@ -0,0 +1,275 @@
+{
+  "name": "tree",
+  "version": "v3",
+  "num_class": 1,
+  "num_tree_per_iteration": 1,
+  "label_index": 0,
+  "max_feature_idx": 3,
+  "average_output": false,
+  "objective": "regression",
+  "feature_names": [
+    "numerical_1",
+    "numerical_2",
+    "categorical_1",
+    "categorical_2"
+  ],
+  "monotone_constraints": [],
+  "tree_info": [
+    {
+      "tree_index": 0,
+      "num_leaves": 3,
+      "num_cat": 1,
+      "shrinkage": 1,
+      "tree_structure": {
+        "split_index": 0,
+        "split_feature": 1,
+        "split_gain": 68.5353012084961,
+        "threshold": 0.46643291586559305,
+        "decision_type": "<=",
+        "default_left": true,
+        "missing_type": "NaN",
+        "internal_value": 0,
+        "internal_weight": 0,
+        "internal_count": 1000,
+        "left_child": {
+          "leaf_index": 0,
+          "leaf_value": 2.1594397038037663,
+          "leaf_weight": 469,
+          "leaf_count": 469
+        },
+        "right_child": {
+          "split_index": 1,
+          "split_feature": 3,
+          "split_gain": 41.27640151977539,
+          "threshold": "2||3||4",
+          "decision_type": "==",
+          "default_left": false,
+          "missing_type": "NaN",
+          "internal_value": 0.246035,
+          "internal_weight": 531,
+          "internal_count": 531,
+          "left_child": {
+            "leaf_index": 1,
+            "leaf_value": 2.235297305276056,
+            "leaf_weight": 302,
+            "leaf_count": 302
+          },
+          "right_child": {
+            "leaf_index": 2,
+            "leaf_value": 2.1792953471546546,
+            "leaf_weight": 229,
+            "leaf_count": 229
+          }
+        }
+      }
+    },
+    {
+      "tree_index": 1,
+      "num_leaves": 3,
+      "num_cat": 1,
+      "shrinkage": 0.1,
+      "tree_structure": {
+        "split_index": 0,
+        "split_feature": 2,
+        "split_gain": 64.22250366210938,
+        "threshold": "3||4",
+        "decision_type": "==",
+        "default_left": false,
+        "missing_type": "NaN",
+        "internal_value": 0,
+        "internal_weight": 0,
+        "internal_count": 1000,
+        "left_child": {
+          "leaf_index": 0,
+          "leaf_value": 0.03070842919354316,
+          "leaf_weight": 399,
+          "leaf_count": 399
+        },
+        "right_child": {
+          "split_index": 1,
+          "split_feature": 0,
+          "split_gain": 36.74250030517578,
+          "threshold": 0.5102250691730842,
+          "decision_type": "<=",
+          "default_left": true,
+          "missing_type": "NaN",
+          "internal_value": -0.204906,
+          "internal_weight": 601,
+          "internal_count": 601,
+          "left_child": {
+            "leaf_index": 1,
+            "leaf_value": -0.04439151147520909,
+            "leaf_weight": 315,
+            "leaf_count": 315
+          },
+          "right_child": {
+            "leaf_index": 2,
+            "leaf_value": 0.005117411709368601,
+            "leaf_weight": 286,
+            "leaf_count": 286
+          }
+        }
+      }
+    },
+    {
+      "tree_index": 2,
+      "num_leaves": 3,
+      "num_cat": 0,
+      "shrinkage": 0.1,
+      "tree_structure": {
+        "split_index": 0,
+        "split_feature": 1,
+        "split_gain": 57.1327018737793,
+        "threshold": 0.668665477622446,
+        "decision_type": "<=",
+        "default_left": true,
+        "missing_type": "NaN",
+        "internal_value": 0,
+        "internal_weight": 0,
+        "internal_count": 1000,
+        "left_child": {
+          "split_index": 1,
+          "split_feature": 1,
+          "split_gain": 40.859100341796875,
+          "threshold": 0.008118820676863816,
+          "decision_type": "<=",
+          "default_left": true,
+          "missing_type": "NaN",
+          "internal_value": -0.162926,
+          "internal_weight": 681,
+          "internal_count": 681,
+          "left_child": {
+            "leaf_index": 0,
+            "leaf_value": -0.15361238490967524,
+            "leaf_weight": 21,
+            "leaf_count": 21
+          },
+          "right_child": {
+            "leaf_index": 2,
+            "leaf_value": -0.01192330846157292,
+            "leaf_weight": 660,
+            "leaf_count": 660
+          }
+        },
+        "right_child": {
+          "leaf_index": 1,
+          "leaf_value": 0.03499044894987518,
+          "leaf_weight": 319,
+          "leaf_count": 319
+        }
+      }
+    },
+    {
+      "tree_index": 3,
+      "num_leaves": 3,
+      "num_cat": 1,
+      "shrinkage": 0.1,
+      "tree_structure": {
+        "split_index": 0,
+        "split_feature": 0,
+        "split_gain": 54.77090072631836,
+        "threshold": 0.5201391072644542,
+        "decision_type": "<=",
+        "default_left": true,
+        "missing_type": "NaN",
+        "internal_value": 0,
+        "internal_weight": 0,
+        "internal_count": 1000,
+        "left_child": {
+          "leaf_index": 0,
+          "leaf_value": -0.02141000620783247,
+          "leaf_weight": 543,
+          "leaf_count": 543
+        },
+        "right_child": {
+          "split_index": 1,
+          "split_feature": 2,
+          "split_gain": 27.200700759887695,
+          "threshold": "0||1",
+          "decision_type": "==",
+          "default_left": false,
+          "missing_type": "NaN",
+          "internal_value": 0.255704,
+          "internal_weight": 457,
+          "internal_count": 457,
+          "left_child": {
+            "leaf_index": 1,
+            "leaf_value": -0.004121485787596721,
+            "leaf_weight": 191,
+            "leaf_count": 191
+          },
+          "right_child": {
+            "leaf_index": 2,
+            "leaf_value": 0.04534090904886873,
+            "leaf_weight": 266,
+            "leaf_count": 266
+          }
+        }
+      }
+    },
+    {
+      "tree_index": 4,
+      "num_leaves": 3,
+      "num_cat": 1,
+      "shrinkage": 0.1,
+      "tree_structure": {
+        "split_index": 0,
+        "split_feature": 3,
+        "split_gain": 51.84349822998047,
+        "threshold": "2||3||4",
+        "decision_type": "==",
+        "default_left": false,
+        "missing_type": "NaN",
+        "internal_value": 0,
+        "internal_weight": 0,
+        "internal_count": 1000,
+        "left_child": {
+          "split_index": 1,
+          "split_feature": 1,
+          "split_gain": 39.352699279785156,
+          "threshold": 0.27283279016959255,
+          "decision_type": "<=",
+          "default_left": true,
+          "missing_type": "NaN",
+          "internal_value": 0.188414,
+          "internal_weight": 593,
+          "internal_count": 593,
+          "left_child": {
+            "leaf_index": 0,
+            "leaf_value": -0.01924803254356527,
+            "leaf_weight": 184,
+            "leaf_count": 184
+          },
+          "right_child": {
+            "leaf_index": 2,
+            "leaf_value": 0.03643772842347651,
+            "leaf_weight": 409,
+            "leaf_count": 409
+          }
+        },
+        "right_child": {
+          "leaf_index": 1,
+          "leaf_value": -0.02701711918923075,
+          "leaf_weight": 407,
+          "leaf_count": 407
+        }
+      }
+    }
+  ],
+  "pandas_categorical": [
+    [
+      "a",
+      "b",
+      "c",
+      "d",
+      "e"
+    ],
+    [
+      "i",
+      "j",
+      "k",
+      "l",
+      "m"
+    ]
+  ]
+}
\ No newline at end of file
diff --git a/application/src/test/app-packages/searcher-app/services.xml b/application/src/test/app-packages/searcher-app/services.xml
index 0fd37554c3b..5b9f4265b5a 100644
--- a/application/src/test/app-packages/searcher-app/services.xml
+++ b/application/src/test/app-packages/searcher-app/services.xml
@@ -12,4 +12,5 @@
 
     </chain>
   </search>
+  <accesslog type="disabled" />
 </container>
diff --git a/application/src/test/app-packages/withcontent/services.xml b/application/src/test/app-packages/withcontent/services.xml
index 251f591b6b8..4338579b49a 100644
--- a/application/src/test/app-packages/withcontent/services.xml
+++ b/application/src/test/app-packages/withcontent/services.xml
@@ -22,7 +22,7 @@
           </chain>
       </document-processing>
       -->
-
+    <accesslog type="disabled" />
   </container>
 
   <content version="1.0" id="foo">
diff --git a/application/src/test/java/com/yahoo/application/ApplicationBuilderTest.java b/application/src/test/java/com/yahoo/application/ApplicationBuilderTest.java
index a4f566e2a71..a0c2ec9983b 100644
--- a/application/src/test/java/com/yahoo/application/ApplicationBuilderTest.java
+++ b/application/src/test/java/com/yahoo/application/ApplicationBuilderTest.java
@@ -32,9 +32,10 @@ public class ApplicationBuilderTest {
     @Test
     public void query_profile_can_be_added() throws Exception {
         withApplicationBuilder(builder -> {
-            builder.queryProfile("MyProfile", "<query-profile id=\"MyProfile\">" + //
-            "<field name=\"message\">Hello world!</field>" + //
-            "</query-profile>");
+            builder.queryProfile("MyProfile",
+                                 "<query-profile id=\"MyProfile\">" +
+                                 "<field name=\"message\">Hello world!</field>" +
+                                 "</query-profile>");
 
             assertTrue(Files.exists(builder.getPath().resolve("search/query-profiles/MyProfile.xml")));
         });
@@ -44,7 +45,7 @@ public class ApplicationBuilderTest {
     public void rank_expression_can_be_added() throws Exception {
         withApplicationBuilder(builder -> {
             builder.rankExpression("myExpression", "content");
-            assertTrue(Files.exists(builder.getPath().resolve("searchdefinitions/myExpression.expression")));
+            assertTrue(Files.exists(builder.getPath().resolve("schemas/myExpression.expression")));
         });
     }
 
diff --git a/application/src/test/java/com/yahoo/application/ApplicationTest.java b/application/src/test/java/com/yahoo/application/ApplicationTest.java
index f2af5537490..29ef0c7c4cf 100644
--- a/application/src/test/java/com/yahoo/application/ApplicationTest.java
+++ b/application/src/test/java/com/yahoo/application/ApplicationTest.java
@@ -61,7 +61,7 @@ public class ApplicationTest {
         try (Application application =
                      Application.fromApplicationPackage(new File("src/test/app-packages/withcontent"), Networking.disable)) {
             Result result = application.getJDisc("default").search().process(new ComponentSpecification("default"),
-                                                                                 new Query("?query=substring:foobar&tracelevel=3"));
+                                                                                 new Query("?query=substring:foobar&timeout=20000"));
             assertEquals("AND substring:fo substring:oo substring:ob substring:ba substring:ar",
                          result.hits().get("hasQuery").getQuery().getModel().getQueryTree().toString());
         }
@@ -183,6 +183,7 @@ public class ApplicationTest {
     }
 
     @Test
+    // TODO: Creates access log
     public void renderer() throws Exception {
         try (
                 ApplicationFacade app = new ApplicationFacade(Application.fromBuilder(new Application.Builder().container("default", new Application.Builder.Container()
@@ -203,7 +204,7 @@ public class ApplicationTest {
                 ApplicationFacade app = new ApplicationFacade(Application.fromBuilder(new Application.Builder().container("default", new Application.Builder.Container()
                         .searcher(MockSearcher.class))))
         ) {
-            Result result = app.search(new Query("?query=foo"));
+            Result result = app.search(new Query("?query=foo&timeout=20000"));
             assertEquals(1, result.hits().size());
         }
     }
@@ -214,7 +215,7 @@ public class ApplicationTest {
                 ApplicationFacade app = new ApplicationFacade(Application.fromBuilder(new Application.Builder().container("default", new Application.Builder.Container()
                         .searcher("foo", MockSearcher.class))))
         ) {
-            Result result = app.search("foo", new Query("?query=foo"));
+            Result result = app.search("foo", new Query("?query=foo&timeout=20000"));
             assertEquals(1, result.hits().size());
         }
     }
@@ -375,7 +376,8 @@ public class ApplicationTest {
     private static String servicesXmlWithServer(int port) {
         return "<container version='1.0'>" +
                 "  <http> <server port='" + port +"' id='foo'/> </http>" +
-                "</container>";
+               "  <accesslog type=\"disabled\" />" +
+               "</container>";
     }
 
     @Test
@@ -392,7 +394,8 @@ public class ApplicationTest {
                 "      <access-control domain='foo' />" +
                 "    </filtering>" +
                 "  </http>" +
-                "</container>";
+               "  <accesslog type=\"disabled\" />" +
+               "</container>";
     }
 
 }
diff --git a/application/src/test/java/com/yahoo/application/container/ContainerDocprocTest.java b/application/src/test/java/com/yahoo/application/container/ContainerDocprocTest.java
index 529f6a60c0d..81f5a681f1a 100644
--- a/application/src/test/java/com/yahoo/application/container/ContainerDocprocTest.java
+++ b/application/src/test/java/com/yahoo/application/container/ContainerDocprocTest.java
@@ -44,6 +44,7 @@ public class ContainerDocprocTest {
         xml +=
                 "    </chain>\n" +
                 "  </document-processing>\n" +
+                "  <accesslog type=\"disabled\" />" +
                 "</container>\n";
         return xml;
     }
diff --git a/application/src/test/java/com/yahoo/application/container/ContainerModelEvaluationTest.java b/application/src/test/java/com/yahoo/application/container/ContainerModelEvaluationTest.java
index 79510375414..3d7eed1e729 100644
--- a/application/src/test/java/com/yahoo/application/container/ContainerModelEvaluationTest.java
+++ b/application/src/test/java/com/yahoo/application/container/ContainerModelEvaluationTest.java
@@ -45,7 +45,7 @@ public class ContainerModelEvaluationTest {
     }
     private void assertLoadedModels(JDisc jdisc) {
         {
-            String expected = "{\"xgboost_xgboost_2_2\":\"http://localhost/model-evaluation/v1/xgboost_xgboost_2_2\",\"onnx_mnist_softmax\":\"http://localhost/model-evaluation/v1/onnx_mnist_softmax\",\"tensorflow_mnist_softmax_saved\":\"http://localhost/model-evaluation/v1/tensorflow_mnist_softmax_saved\",\"tensorflow_mnist_saved\":\"http://localhost/model-evaluation/v1/tensorflow_mnist_saved\",\"vespa_example\":\"http://localhost/model-evaluation/v1/vespa_example\"}";
+            String expected = "{\"xgboost_xgboost_2_2\":\"http://localhost/model-evaluation/v1/xgboost_xgboost_2_2\",\"onnx_mnist_softmax\":\"http://localhost/model-evaluation/v1/onnx_mnist_softmax\",\"tensorflow_mnist_softmax_saved\":\"http://localhost/model-evaluation/v1/tensorflow_mnist_softmax_saved\",\"tensorflow_mnist_saved\":\"http://localhost/model-evaluation/v1/tensorflow_mnist_saved\",\"vespa_example\":\"http://localhost/model-evaluation/v1/vespa_example\",\"lightgbm_regression\":\"http://localhost/model-evaluation/v1/lightgbm_regression\"}";
             assertResponse("http://localhost/model-evaluation/v1", expected, jdisc);
         }
 
@@ -55,6 +55,11 @@ public class ContainerModelEvaluationTest {
         }
 
         {
+            String expected = "{\"cells\":[{\"address\":{},\"value\":1.9130086820218188}]}";
+            assertResponse("http://localhost/model-evaluation/v1/lightgbm_regression/eval", expected, jdisc);
+        }
+
+        {
             // Note: The specific response value here has not been verified
             String expected = "{\"cells\":[{\"address\":{\"d0\":\"0\",\"d1\":\"0\"},\"value\":-0.5066885003407351},{\"address\":{\"d0\":\"0\",\"d1\":\"1\"},\"value\":0.3912837743150205},{\"address\":{\"d0\":\"0\",\"d1\":\"2\"},\"value\":-0.12401806321703948},{\"address\":{\"d0\":\"0\",\"d1\":\"3\"},\"value\":-0.7019029168606575},{\"address\":{\"d0\":\"0\",\"d1\":\"4\"},\"value\":0.13120114146441697},{\"address\":{\"d0\":\"0\",\"d1\":\"5\"},\"value\":0.6611923203384626},{\"address\":{\"d0\":\"0\",\"d1\":\"6\"},\"value\":-0.22365810810026446},{\"address\":{\"d0\":\"0\",\"d1\":\"7\"},\"value\":-0.0740018307465809},{\"address\":{\"d0\":\"0\",\"d1\":\"8\"},\"value\":0.056492490256153896},{\"address\":{\"d0\":\"0\",\"d1\":\"9\"},\"value\":-0.18422015072393733}]}";
             assertResponse("http://localhost/model-evaluation/v1/tensorflow_mnist_saved/serving_default.y/eval?input=" + inputTensor(), expected, jdisc);
diff --git a/application/src/test/java/com/yahoo/application/container/ContainerProcessingTest.java b/application/src/test/java/com/yahoo/application/container/ContainerProcessingTest.java
index 93ca09ac5fc..22c307c0a0c 100644
--- a/application/src/test/java/com/yahoo/application/container/ContainerProcessingTest.java
+++ b/application/src/test/java/com/yahoo/application/container/ContainerProcessingTest.java
@@ -31,6 +31,7 @@ public class ContainerProcessingTest {
         xml +=
                 "    </chain>\n" +
                 "  </processing>\n" +
+                "  <accesslog type=\"disabled\" />" +
                 "</container>\n";
         return xml;
     }
diff --git a/application/src/test/java/com/yahoo/application/container/ContainerRequestTest.java b/application/src/test/java/com/yahoo/application/container/ContainerRequestTest.java
index 8f3e7693bc5..a66547e6ed9 100644
--- a/application/src/test/java/com/yahoo/application/container/ContainerRequestTest.java
+++ b/application/src/test/java/com/yahoo/application/container/ContainerRequestTest.java
@@ -33,6 +33,7 @@ public class ContainerRequestTest {
                binding +
                "</binding>\n" +
                "  </handler>\n" +
+               "  <accesslog type=\"disabled\" />" +
                "</container>";
     }
 
diff --git a/application/src/test/java/com/yahoo/application/container/ContainerSearchTest.java b/application/src/test/java/com/yahoo/application/container/ContainerSearchTest.java
index d133b71b8da..793ee9ab094 100644
--- a/application/src/test/java/com/yahoo/application/container/ContainerSearchTest.java
+++ b/application/src/test/java/com/yahoo/application/container/ContainerSearchTest.java
@@ -8,6 +8,8 @@ import com.yahoo.search.Query;
 import com.yahoo.search.Result;
 import org.junit.Test;
 
+import java.nio.charset.StandardCharsets;
+
 import static org.hamcrest.CoreMatchers.containsString;
 import static org.hamcrest.CoreMatchers.is;
 import static org.junit.Assert.assertThat;
@@ -25,7 +27,7 @@ public class ContainerSearchTest {
         try (JDisc container = containerWithSearch(searcherId)) {
             byte[] rendered = container.search().processAndRender(ComponentSpecification.fromString("mychain"),
                     ComponentSpecification.fromString("XmlRenderer"), new Query(""));
-            String renderedAsString = new String(rendered, "utf-8");
+            String renderedAsString = new String(rendered, StandardCharsets.UTF_8);
             assertThat(renderedAsString, containsString(searcherId));
         }
     }
@@ -44,11 +46,12 @@ public class ContainerSearchTest {
 
     public JDisc containerWithSearch(String searcherId) {
         return JDisc.fromServicesXml("<container version=\"1.0\">" + //
-                "<search>" + //
-                "<chain id=\"mychain\">" + //
-                "<searcher id=\"" + searcherId + "\"/>" + //
-                "</chain>" + //
-                "</search>" + //
+                "  <search>" + //
+                "    <chain id=\"mychain\">" + //
+                "      <searcher id=\"" + searcherId + "\"/>" + //
+                "    </chain>" + //
+                "  </search>" + //
+                "  <accesslog type=\"disabled\" />" + //
                 "</container>", Networking.disable);
     }
 
diff --git a/application/src/test/java/com/yahoo/application/container/ContainerTest.java b/application/src/test/java/com/yahoo/application/container/ContainerTest.java
index c284087243d..63347c2475a 100644
--- a/application/src/test/java/com/yahoo/application/container/ContainerTest.java
+++ b/application/src/test/java/com/yahoo/application/container/ContainerTest.java
@@ -164,6 +164,7 @@ public class ContainerTest {
                         "<http>\n" + //
                         "<server id=\"main\" port=\"9999\" />\n" + //
                         "</http>\n" + //
+                        "<accesslog type=\"disabled\" />" +
                         "</container>";
         return JDisc.fromServicesXml(xml, Networking.disable);
     }
diff --git a/application/src/test/java/com/yahoo/application/container/jersey/JerseyTest.java b/application/src/test/java/com/yahoo/application/container/jersey/JerseyTest.java
index 0eb308f124c..caddabff8ed 100644
--- a/application/src/test/java/com/yahoo/application/container/jersey/JerseyTest.java
+++ b/application/src/test/java/com/yahoo/application/container/jersey/JerseyTest.java
@@ -66,11 +66,10 @@ public class JerseyTest {
 
     @Test
     public void jersey_resources_in_provided_dependencies_can_be_invoked_from_application() throws Exception {
-        BundleClasspathMapping providedDependency = new BundleClasspathMapping(bundleSymbolicName,
-                Arrays.asList(testClassesDirectory));
+        BundleClasspathMapping providedDependency =
+                new BundleClasspathMapping(bundleSymbolicName, List.of(testClassesDirectory));
 
-        save(new ProjectBundleClassPaths(new BundleClasspathMapping("main", emptyList()),
-                Arrays.asList(providedDependency)));
+        save(new ProjectBundleClassPaths(new BundleClasspathMapping("main", emptyList()), List.of(providedDependency)));
         with_jersey_resources(emptyList(), httpGetter -> assertResourcesResponds(classPathResources, httpGetter));
     }
 
@@ -113,16 +112,16 @@ public class JerseyTest {
     }
 
     private interface ThrowingConsumer<T> {
-        public void accept(T arg) throws Exception;
+        void accept(T arg) throws Exception;
     }
 
     private interface HttpGetter {
-        public HttpResponse get(String path) throws Exception;
+        HttpResponse get(String path) throws Exception;
     }
 
     @SuppressWarnings("try") // jdisc unreferenced inside try
     private void with_jersey_resources(List<String> packagesToScan, ThrowingConsumer<HttpGetter> f) throws Exception {
-        StringBuffer packageElements = new StringBuffer();
+        StringBuilder packageElements = new StringBuilder();
         for (String p : packagesToScan) {
             packageElements.append("<package>");
             packageElements.append(p);
@@ -140,6 +139,7 @@ public class JerseyTest {
                         "<http>" + //
                         "<server id=\"mainServer\" port=\"0\" />" + //
                         "</http>" + //
+                        "<accesslog type=\"disabled\" />" +
                         "</container>" + //
                         "</services>", //
                 Networking.enable)) {
@@ -177,8 +177,8 @@ public class JerseyTest {
     }
 
     public void saveMainBundleClassPathMappings(String classPathElement) throws Exception {
-        BundleClasspathMapping mainBundleClassPathMappings = new BundleClasspathMapping(bundleSymbolicName,
-                Arrays.asList(classPathElement));
+        BundleClasspathMapping mainBundleClassPathMappings =
+                new BundleClasspathMapping(bundleSymbolicName, List.of(classPathElement));
         save(new ProjectBundleClassPaths(mainBundleClassPathMappings, emptyList()));
     }
 
diff --git a/athenz-identity-provider-service/pom.xml b/athenz-identity-provider-service/pom.xml
index 212c66a3431..f15ebe87b03 100644
--- a/athenz-identity-provider-service/pom.xml
+++ b/athenz-identity-provider-service/pom.xml
@@ -133,12 +133,18 @@
             <version>${project.version}</version>
             <scope>test</scope>
         </dependency>
-      <dependency>
-        <groupId>com.yahoo.vespa</groupId>
-        <artifactId>testutil</artifactId>
-        <version>${project.version}</version>
-        <scope>test</scope>
-      </dependency>
+        <dependency>
+            <groupId>com.yahoo.vespa</groupId>
+            <artifactId>testutil</artifactId>
+            <version>${project.version}</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.yahoo.vespa</groupId>
+            <artifactId>flags</artifactId>
+            <version>${project.version}</version>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
 
     <build>
diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidator.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidator.java
index 89e7e340641..3a926922f2b 100644
--- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidator.java
+++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidator.java
@@ -85,11 +85,11 @@ public class InstanceValidator {
             return false;
         }
 
-        log.log(LogLevel.INFO, () -> String.format("Validating instance %s.", providerUniqueId));
+        log.log(LogLevel.DEBUG, () -> String.format("Validating instance %s.", providerUniqueId));
 
         PublicKey publicKey = keyProvider.getPublicKey(signedIdentityDocument.signingKeyVersion());
         if (signer.hasValidSignature(signedIdentityDocument, publicKey)) {
-            log.log(LogLevel.INFO, () -> String.format("Instance %s is valid.", providerUniqueId));
+            log.log(LogLevel.DEBUG, () -> String.format("Instance %s is valid.", providerUniqueId));
             return true;
         }
         log.log(LogLevel.ERROR, () -> String.format("Instance %s has invalid signature.", providerUniqueId));
@@ -100,7 +100,7 @@ public class InstanceValidator {
     //      We'll have to perform some validation on the instance id and other fields of the attribute map.
     //      Separate between tenant and node certificate as well.
     public boolean isValidRefresh(InstanceConfirmation confirmation) {
-        log.log(LogLevel.INFO, () -> String.format("Accepting refresh for instance with identity '%s', provider '%s', instanceId '%s'.",
+        log.log(LogLevel.DEBUG, () -> String.format("Accepting refresh for instance with identity '%s', provider '%s', instanceId '%s'.",
                                                    new AthenzService(confirmation.domain, confirmation.service).getFullName(),
                                                    confirmation.provider,
                                                    confirmation.attributes.get(SAN_DNS_ATTRNAME)));
diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/restapi/CertificateAuthorityApiHandler.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/restapi/CertificateAuthorityApiHandler.java
index a1984557c31..466a9783bb3 100644
--- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/restapi/CertificateAuthorityApiHandler.java
+++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/restapi/CertificateAuthorityApiHandler.java
@@ -17,7 +17,7 @@ import com.yahoo.security.X509CertificateUtils;
 import com.yahoo.slime.Slime;
 import com.yahoo.vespa.athenz.api.AthenzService;
 import com.yahoo.vespa.athenz.identityprovider.api.EntityBindingsMapper;
-import com.yahoo.vespa.config.SlimeUtils;
+import com.yahoo.slime.SlimeUtils;
 import com.yahoo.vespa.hosted.athenz.instanceproviderservice.config.AthenzProviderServiceConfig;
 import com.yahoo.vespa.hosted.athenz.instanceproviderservice.instanceconfirmation.InstanceConfirmation;
 import com.yahoo.vespa.hosted.athenz.instanceproviderservice.instanceconfirmation.InstanceValidator;
diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/restapi/InstanceSerializer.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/restapi/InstanceSerializer.java
index d164555dc3e..0ac833b7aee 100644
--- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/restapi/InstanceSerializer.java
+++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/restapi/InstanceSerializer.java
@@ -13,7 +13,7 @@ import com.yahoo.vespa.athenz.api.AthenzService;
 import com.yahoo.vespa.athenz.identityprovider.api.IdentityType;
 import com.yahoo.vespa.athenz.identityprovider.api.SignedIdentityDocument;
 import com.yahoo.vespa.athenz.identityprovider.api.VespaUniqueInstanceId;
-import com.yahoo.vespa.config.SlimeUtils;
+import com.yahoo.slime.SlimeUtils;
 import com.yahoo.vespa.hosted.ca.instance.InstanceIdentity;
 import com.yahoo.vespa.hosted.ca.instance.InstanceRefresh;
 import com.yahoo.vespa.hosted.ca.instance.InstanceRegistration;
diff --git a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidatorTest.java b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidatorTest.java
index 2208b470ed8..fdf2bbfccff 100644
--- a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidatorTest.java
+++ b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidatorTest.java
@@ -201,7 +201,8 @@ public class InstanceValidatorTest {
                                                                   ApplicationInfo::getApplicationId,
                                                                   Function.identity()
                                                                    )
-                                                          ));
+                                                          ),
+                                               true);
 
         SuperModelProvider superModelProvider = mock(SuperModelProvider.class);
         when(superModelProvider.getSuperModel()).thenReturn(superModel);
diff --git a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/CertificateAuthorityApiTest.java b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/CertificateAuthorityApiTest.java
index c63ed3fceba..343a9feeed6 100644
--- a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/CertificateAuthorityApiTest.java
+++ b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/CertificateAuthorityApiTest.java
@@ -12,7 +12,7 @@ import com.yahoo.text.StringUtilities;
 import com.yahoo.vespa.athenz.api.AthenzPrincipal;
 import com.yahoo.vespa.athenz.api.AthenzService;
 import com.yahoo.vespa.athenz.client.zts.DefaultZtsClient;
-import com.yahoo.vespa.config.SlimeUtils;
+import com.yahoo.slime.SlimeUtils;
 import com.yahoo.vespa.hosted.ca.CertificateTester;
 import org.apache.http.client.ResponseHandler;
 import org.apache.http.client.methods.HttpUriRequest;
diff --git a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/ContainerTester.java b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/ContainerTester.java
index 0eda6bd946b..6783763b210 100644
--- a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/ContainerTester.java
+++ b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/ContainerTester.java
@@ -54,6 +54,7 @@ public class ContainerTester {
 
     private static String servicesXml() {
         return "<container version='1.0'>\n" +
+               "  <accesslog type=\"disabled\"/>\n" +
                "  <config name=\"container.handler.threadpool\">\n" +
                "    <maxthreads>10</maxthreads>\n" +
                "  </config>\n" +
diff --git a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/InstanceSerializerTest.java b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/InstanceSerializerTest.java
index f04e155aa2c..99d1b064a34 100644
--- a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/InstanceSerializerTest.java
+++ b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/ca/restapi/InstanceSerializerTest.java
@@ -10,7 +10,7 @@ import com.yahoo.vespa.athenz.identityprovider.api.EntityBindingsMapper;
 import com.yahoo.vespa.athenz.identityprovider.api.IdentityType;
 import com.yahoo.vespa.athenz.identityprovider.api.SignedIdentityDocument;
 import com.yahoo.vespa.athenz.identityprovider.api.VespaUniqueInstanceId;
-import com.yahoo.vespa.config.SlimeUtils;
+import com.yahoo.slime.SlimeUtils;
 import com.yahoo.vespa.hosted.ca.CertificateTester;
 import com.yahoo.vespa.hosted.ca.instance.InstanceIdentity;
 import com.yahoo.vespa.hosted.ca.instance.InstanceRefresh;
diff --git a/bootstrap-cmake.sh b/bootstrap-cmake.sh
index 5e3e2a42ed5..dbe27f5b8bf 100755
--- a/bootstrap-cmake.sh
+++ b/bootstrap-cmake.sh
@@ -39,7 +39,8 @@ else
 fi
 
 if [ -z "$VESPA_LLVM_VERSION" ]; then
-    VESPA_LLVM_VERSION=5.0
+    VESPA_LLVM_VERSION_PATH=7.0
+    VESPA_LLVM_VERSION=7
 fi
 
 if $UNPRIVILEGED; then
@@ -54,8 +55,8 @@ cmake3 \
     -DCMAKE_INSTALL_PREFIX=${VESPA_INSTALL_PREFIX} \
     -DJAVA_HOME=${JAVA_HOME:-/usr/lib/jvm/java-openjdk} \
     -DCMAKE_PREFIX_PATH="/opt/vespa-deps" \
-    -DEXTRA_LINK_DIRECTORY="/opt/vespa-deps/lib64;/usr/lib64/llvm$VESPA_LLVM_VERSION/lib" \
-    -DEXTRA_INCLUDE_DIRECTORY="/opt/vespa-deps/include;/usr/include/llvm$VESPA_LLVM_VERSION" \
+    -DEXTRA_LINK_DIRECTORY="/opt/vespa-deps/lib64;/usr/lib64/llvm$VESPA_LLVM_VERSION_PATH/lib" \
+    -DEXTRA_INCLUDE_DIRECTORY="/opt/vespa-deps/include;/usr/include/llvm$VESPA_LLVM_VERSION_PATH;/usr/include/openblas" \
     -DCMAKE_INSTALL_RPATH="${VESPA_INSTALL_PREFIX}/lib64;/opt/vespa-deps/lib64;/usr/lib/jvm/java-1.8.0/jre/lib/amd64/server;/usr/lib64/llvm$VESPA_LLVM_VERSION/lib" \
     ${UNPRIVILEGED_ARGS} \
     ${EXTRA_CMAKE_ARGS} \
diff --git a/bootstrap-cpp.sh b/bootstrap-cpp.sh
index 5fe4723ea90..5753d4cfcfd 100755
--- a/bootstrap-cpp.sh
+++ b/bootstrap-cpp.sh
@@ -48,7 +48,7 @@ mkdir -p "${BUILD_DIR}" || {
 BUILD_DIR=$(realpath "${BUILD_DIR}")
 
 # Build it
-source /opt/rh/devtoolset-8/enable || true
+source /opt/rh/devtoolset-9/enable || true
 cd "${SOURCE_DIR}"
 bash ./bootstrap.sh full
 cd "${BUILD_DIR}"
diff --git a/build_settings.cmake b/build_settings.cmake
index 679eed558d5..2edf48103cf 100644
--- a/build_settings.cmake
+++ b/build_settings.cmake
@@ -1,7 +1,15 @@
 # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 # @author Vegard Sjonfjell
 
-include(vtag.cmake)
+include(${CMAKE_CURRENT_LIST_DIR}/vtag.cmake)
+
+if (VESPA_USE_SANITIZER)
+    if (VESPA_USE_SANITIZER STREQUAL "address" OR VESPA_USE_SANITIZER STREQUAL "thread")
+        message("-- Instrumenting code using ${VESPA_USE_SANITIZER} sanitizer")
+    else()
+        message(FATAL_ERROR "Unsupported sanitizer option '${VESPA_USE_SANITIZER}'. Supported: 'address' or 'thread'")
+    endif()
+endif()
 
 # Build options
 # Whether to build unit tests as part of the 'all' target
@@ -17,7 +25,13 @@ set(RUN_BENCHMARKS FALSE CACHE BOOL "If TRUE, benchmarks are run together with t
 set(AUTORUN_UNIT_TESTS FALSE CACHE BOOL "If TRUE, tests will be run immediately after linking the test executable")
 
 # Warnings
-set(C_WARN_OPTS "-Winline -Wuninitialized -Werror -Wall -W -Wchar-subscripts -Wcomment -Wformat -Wparentheses -Wreturn-type -Wswitch -Wtrigraphs -Wunused -Wshadow -Wpointer-arith -Wcast-qual -Wcast-align -Wwrite-strings")
+set(C_WARN_OPTS "-Wuninitialized -Werror -Wall -W -Wchar-subscripts -Wcomment -Wformat -Wparentheses -Wreturn-type -Wswitch -Wtrigraphs -Wunused -Wshadow -Wpointer-arith -Wcast-qual -Wcast-align -Wwrite-strings")
+if (VESPA_USE_SANITIZER)
+    # Instrumenting code changes binary size, which triggers inlining warnings that
+    # don't happen during normal, non-instrumented compilation.
+else()
+    set(C_WARN_OPTS "-Winline ${C_WARN_OPTS}")
+endif()
 
 # Warnings that are specific to C++ compilation
 # Note: this is not a union of C_WARN_OPTS, since CMAKE_CXX_FLAGS already includes CMAKE_C_FLAGS, which in turn includes C_WARN_OPTS transitively
@@ -46,7 +60,11 @@ else()
 endif()
 
 # C and C++ compiler flags
-set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -O3 -fno-omit-frame-pointer ${C_WARN_OPTS} -fPIC ${VESPA_CXX_ABI_FLAGS} -DBOOST_DISABLE_ASSERTS ${VESPA_CPU_ARCH_FLAGS} ${EXTRA_C_FLAGS}")
+set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -O3 -fno-omit-frame-pointer ${C_WARN_OPTS} -fPIC ${VESPA_CXX_ABI_FLAGS} -DXXH_INLINE_ALL -DBOOST_DISABLE_ASSERTS ${VESPA_CPU_ARCH_FLAGS} ${EXTRA_C_FLAGS}")
+# AddressSanitizer/ThreadSanitizer work for both GCC and Clang
+if (VESPA_USE_SANITIZER)
+    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsanitize=${VESPA_USE_SANITIZER}")
+endif()
 if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "AppleClang")
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CMAKE_C_FLAGS} ${CXX_SPECIFIC_WARN_OPTS} -std=c++1z -fdiagnostics-color=auto ${EXTRA_CXX_FLAGS}")
 else()
diff --git a/bundle-plugin-test/integration-test/pom.xml b/bundle-plugin-test/integration-test/pom.xml
index 42d3b0e4d62..e65cf077d62 100644
--- a/bundle-plugin-test/integration-test/pom.xml
+++ b/bundle-plugin-test/integration-test/pom.xml
@@ -62,7 +62,6 @@
           <systemPropertyVariables>
             <test.bundle.path>${project.build.directory}/dependency</test.bundle.path>
           </systemPropertyVariables>
-          <trimStackTrace>false</trimStackTrace>
         </configuration>
       </plugin>
       <plugin>
diff --git a/client/pom.xml b/client/pom.xml
index 3dee66f31ec..6f281fddf8c 100644
--- a/client/pom.xml
+++ b/client/pom.xml
@@ -63,6 +63,21 @@
           </execution>
         </executions>
       </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-shade-plugin</artifactId>
+        <configuration>
+          <finalName>${project.artifactId}-jar-with-dependencies</finalName>
+        </configuration>
+        <executions>
+          <execution>
+            <phase>package</phase>
+            <goals>
+              <goal>shade</goal>
+            </goals>
+          </execution>
+        </executions>
+      </plugin>
     </plugins>
   </build>
 </project>
diff --git a/client/src/main/java/ai/vespa/client/dsl/EndQuery.java b/client/src/main/java/ai/vespa/client/dsl/EndQuery.java
index 4facf6ce0fb..2af1e0bb49d 100644
--- a/client/src/main/java/ai/vespa/client/dsl/EndQuery.java
+++ b/client/src/main/java/ai/vespa/client/dsl/EndQuery.java
@@ -177,7 +177,7 @@ public class EndQuery {
         } else if (others.isEmpty()) {
             sb.append("order by ").append(orderStr);
         } else {
-            sb.append("order by ").append(orderStr).append(", ").append(others);
+            sb.append("order by ").append(orderStr).append(" ").append(others);
         }
 
         if (groupQueryStr != null) {
diff --git a/client/src/test/groovy/ai/vespa/client/dsl/QTest.groovy b/client/src/test/groovy/ai/vespa/client/dsl/QTest.groovy
index e363a9bcc13..19c87d6aecd 100644
--- a/client/src/test/groovy/ai/vespa/client/dsl/QTest.groovy
+++ b/client/src/test/groovy/ai/vespa/client/dsl/QTest.groovy
@@ -59,7 +59,7 @@ class QTest extends Specification {
             .build()
 
         expect:
-        q == """yql=select * from sd1 where f1 contains "v1" and f2 contains "v2" or f3 contains "v3" and !(f4 contains "v4") order by f1 desc, f2 asc, limit 2 offset 1 timeout 3;&paramk1=paramv1"""
+        q == """yql=select * from sd1 where f1 contains "v1" and f2 contains "v2" or f3 contains "v3" and !(f4 contains "v4") order by f1 desc, f2 asc limit 2 offset 1 timeout 3;&paramk1=paramv1"""
     }
 
     def "matches"() {
diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ZooKeeperTestServer.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ZooKeeperTestServer.java
index 34c8fafa702..73b4163d542 100644
--- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ZooKeeperTestServer.java
+++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ZooKeeperTestServer.java
@@ -8,6 +8,7 @@ import org.apache.zookeeper.server.ZooKeeperServer;
 import java.io.File;
 import java.io.IOException;
 import java.net.InetSocketAddress;
+import java.time.Duration;
 
 /**
  * This class sets up a zookeeper server, such that we can test fleetcontroller zookeeper parts without stubbing in the client.
@@ -15,7 +16,7 @@ import java.net.InetSocketAddress;
 public class ZooKeeperTestServer {
     private File zooKeeperDir;
     private ZooKeeperServer server;
-    private static final int tickTime = 100;
+    private static final Duration tickTime = Duration.ofMillis(2000);
     private NIOServerCnxnFactory factory;
     private static final String DIR_PREFIX = "test_fltctrl_zk";
     private static final String DIR_POSTFIX = "sdir";
@@ -31,7 +32,7 @@ public class ZooKeeperTestServer {
             throw new IllegalStateException("Failed to create directory " + zooKeeperDir);
         }
         zooKeeperDir.deleteOnExit();
-        server = new ZooKeeperServer(zooKeeperDir, zooKeeperDir, tickTime);
+        server = new ZooKeeperServer(zooKeeperDir, zooKeeperDir, (int)tickTime.toMillis());
         final int maxcc = 10000; // max number of connections from the same client
         factory = new NIOServerCnxnFactory();
         factory.configure(new InetSocketAddress(port), maxcc); // Use any port
diff --git a/cmake/FindRE2.cmake b/cmake/FindRE2.cmake
new file mode 100644
index 00000000000..af1ff799bd7
--- /dev/null
+++ b/cmake/FindRE2.cmake
@@ -0,0 +1,19 @@
+# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+# There is no bundled FindRE2, so we supply our own minimal version to find
+# the system RE2 library and header files.
+
+find_path(RE2_INCLUDE_DIR
+    NAMES re2/re2.h
+)
+
+find_library(RE2_LIBRARIES
+    NAMES re2
+)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(RE2
+    FOUND_VAR RE2_FOUND
+    REQUIRED_VARS RE2_LIBRARIES RE2_INCLUDE_DIR
+)
+
diff --git a/component/abi-spec.json b/component/abi-spec.json
index a6b6a558db6..5665075f1f2 100644
--- a/component/abi-spec.json
+++ b/component/abi-spec.json
@@ -73,6 +73,7 @@
       "public static com.yahoo.component.ComponentId fromString(java.lang.String)",
       "public java.lang.String toFileName()",
       "public static com.yahoo.component.ComponentId fromFileName(java.lang.String)",
+      "public static void resetGlobalCountersForTests()",
       "public bridge synthetic int compareTo(java.lang.Object)"
     ],
     "fields": []
@@ -128,6 +129,7 @@
       "public int getMicro()",
       "public java.lang.String getQualifier()",
       "public java.lang.String toString()",
+      "public com.yahoo.text.Utf8Array toUtf8()",
       "public int hashCode()",
       "public boolean isEmpty()",
       "public boolean equals(java.lang.Object)",
diff --git a/component/src/main/java/com/yahoo/component/ComponentId.java b/component/src/main/java/com/yahoo/component/ComponentId.java
index 05c710e3fc1..4613be09543 100644
--- a/component/src/main/java/com/yahoo/component/ComponentId.java
+++ b/component/src/main/java/com/yahoo/component/ComponentId.java
@@ -32,15 +32,15 @@ public final class ComponentId implements Comparable<ComponentId> {
         private int count = 0;
         public int getAndIncrement() { return count++; }
     }
-    private static ThreadLocal<Counter> gid = new ThreadLocal<Counter>() {
+    private static ThreadLocal<Counter> threadLocalUniqueId = new ThreadLocal<Counter>() {
         @Override protected Counter initialValue() {
             return new Counter();
         }
     };
-    private static AtomicInteger uniqueTid = new AtomicInteger(0);
-    private static ThreadLocal<String> tid = new ThreadLocal<String>() {
+    private static AtomicInteger threadIdCounter = new AtomicInteger(0);
+    private static ThreadLocal<String> threadId = new ThreadLocal<String>() {
         @Override protected String initialValue() {
-            return new String("_"+uniqueTid.getAndIncrement()+"_");
+            return new String("_" + threadIdCounter.getAndIncrement() + "_");
         }
     };
 
@@ -58,7 +58,7 @@ public final class ComponentId implements Comparable<ComponentId> {
     }
 
     private String createAnonymousName(String name) {
-        return new StringBuilder(name).append(tid.get()).append(gid.get().getAndIncrement()).toString();
+        return new StringBuilder(name).append(threadId.get()).append(threadLocalUniqueId.get().getAndIncrement()).toString();
     }
 
     public ComponentId(String name, Version version, ComponentId namespace) {
@@ -148,10 +148,7 @@ public final class ComponentId implements Comparable<ComponentId> {
         return spec.compareTo(other.spec);
     }
 
-    /**
-     * Creates a componentId that is unique for this run-time instance
-     */
-    // TODO: Check if we really need this. -JB
+    /** Creates a componentId that is unique for this run-time instance */
     public static ComponentId createAnonymousComponentId(String baseName) {
         return new ComponentId(baseName, null, null, true);
     }
@@ -196,27 +193,27 @@ public final class ComponentId implements Comparable<ComponentId> {
      * Creates an id from a file <b>first</b> name string encoded in the standard translation (see {@link #toFileName}).
      * <b>Note</b> that any file last name, like e.g ".xml" must be stripped off before handoff to this method.
      */
-    public static ComponentId fromFileName(final String fileName) {
+    public static ComponentId fromFileName(String fileName) {
         // Initial assumptions
-        String id=fileName;
-        Version version =null;
-        ComponentId namespace=null;
+        String id = fileName;
+        Version version = null;
+        ComponentId namespace = null;
 
         // Split out namespace, if any
-        int at=id.indexOf("@");
-        if (at>0) {
-            String newId=id.substring(0,at);
-            namespace=ComponentId.fromString(id.substring(at+1));
-            id=newId;
+        int at = id.indexOf("@");
+        if (at > 0) {
+            String newId = id.substring(0, at);
+            namespace = ComponentId.fromString(id.substring(at + 1));
+            id = newId;
         }
 
         // Split out version, if any
-        int dash=id.lastIndexOf("-");
-        if (dash>0) {
-            String newId=id.substring(0,dash);
+        int dash = id.lastIndexOf("-");
+        if (dash > 0) {
+            String newId = id.substring(0, dash);
             try {
-                version=new Version(id.substring(dash+1));
-                id=newId;
+                version = new Version(id.substring(dash + 1));
+                id = newId;
             }
             catch (IllegalArgumentException e) {
                 // don't interpret the text following the dash as a version
@@ -229,4 +226,10 @@ public final class ComponentId implements Comparable<ComponentId> {
         return new ComponentId(id,version,namespace);
     }
 
+    /** WARNING: For testing only: Resets counters creating anonymous component ids for this thread. */
+    public static void resetGlobalCountersForTests() {
+        threadId.set("_0_");
+        threadLocalUniqueId.set(new Counter());
+    }
+
 }
diff --git a/component/src/main/java/com/yahoo/component/Version.java b/component/src/main/java/com/yahoo/component/Version.java
index 51ff6ad5dbd..d611e1a6271 100644
--- a/component/src/main/java/com/yahoo/component/Version.java
+++ b/component/src/main/java/com/yahoo/component/Version.java
@@ -3,6 +3,7 @@ package com.yahoo.component;
 
 import com.yahoo.text.Utf8;
 import com.yahoo.text.Utf8Array;
+import com.yahoo.text.Utf8String;
 
 import java.nio.ByteBuffer;
 
@@ -28,18 +29,18 @@ import java.nio.ByteBuffer;
  */
 public final class Version implements Comparable<Version> {
 
-    private int major = 0;
-    private int minor = 0;
-    private int micro = 0;
-    private String qualifier = "";
-    private String stringValue;
+    private final int major;
+    private final int minor;
+    private final int micro;
+    private final String qualifier;
+    private final Utf8Array utf8;
 
     /** The empty version */
     public static final Version emptyVersion = new Version();
 
     /** Creates an empty version */
     public Version() {
-        this(0, 0, 0, null);
+        this(0, 0, 0, "");
     }
 
     /**
@@ -50,7 +51,7 @@ public final class Version implements Comparable<Version> {
      *         negative.
      */
     public Version(int major) {
-        this(major, 0, 0, null);
+        this(major, 0, 0, "");
     }
 
     /**
@@ -62,7 +63,7 @@ public final class Version implements Comparable<Version> {
      *         negative.
      */
     public Version(int major, int minor) {
-        this(major, minor, 0, null);
+        this(major, minor, 0, "");
     }
 
     /**
@@ -75,7 +76,7 @@ public final class Version implements Comparable<Version> {
      *         negative.
      */
     public Version(int major, int minor, int micro) {
-        this(major, minor, micro, null);
+        this(major, minor, micro, "");
     }
 
     /**
@@ -93,8 +94,8 @@ public final class Version implements Comparable<Version> {
         this.major = major;
         this.minor = minor;
         this.micro = micro;
-        if (qualifier != null) this.qualifier = qualifier;
-        stringValue = toStringValue();
+        this.qualifier = (qualifier != null) ? qualifier : "";
+        utf8 = new Utf8String(toString());
         verify();
     }
 
@@ -120,19 +121,19 @@ public final class Version implements Comparable<Version> {
     public Version(String versionString) {
         if (! "".equals(versionString)) {
             String[] components=versionString.split("\\."); // Split on dot
-
-            if (components.length > 0)
-                major = Integer.parseInt(components[0]);
-            if (components.length > 1)
-                minor = Integer.parseInt(components[1]);
-            if (components.length > 2)
-                micro = Integer.parseInt(components[2]);
-            if (components.length > 3)
-                qualifier = components[3];
+            major = (components.length > 0) ? Integer.parseInt(components[0]) : 0;
+            minor =  (components.length > 1) ? Integer.parseInt(components[1]) : 0;
+            micro = (components.length > 2) ? Integer.parseInt(components[2]) : 0;
+            qualifier = (components.length > 3) ? components[3] : "";
             if (components.length > 4)
                 throw new IllegalArgumentException("Too many components in '" + versionString + "'");
+        } else {
+            major = 0;
+            minor = 0;
+            micro = 0;
+            qualifier = "";
         }
-        stringValue = toStringValue();
+        utf8 = new Utf8String(versionString);
         verify();
     }
 
@@ -178,16 +179,21 @@ public final class Version implements Comparable<Version> {
                 minor = readInt(bb);
                 if (bb.remaining() > 0) {
                     micro = readInt(bb);
-                    if (bb.remaining() > 0) {
-                        qualifier = Utf8.toString(bb);
-                    }
+                    qualifier = (bb.remaining() > 0) ? Utf8.toString(bb) : "";
+                } else {
+                    micro = 0;
+                    qualifier = "";
                 }
+            } else {
+                minor = 0;
+                micro = 0;
+                qualifier = "";
             }
         } else {
             throw new IllegalArgumentException("Empty version specification");
         }
+        utf8 = versionString;
 
-        stringValue = versionString.toString();
         verify();
     }
 
@@ -270,10 +276,14 @@ public final class Version implements Comparable<Version> {
      * The string representation of a Version specified here is a part of the API and will never change.
      */
     @Override
-    public String toString() { return stringValue; }
+    public String toString() { return toStringValue(); }
+
+    public Utf8Array toUtf8() {
+        return utf8;
+    }
 
     @Override
-    public int hashCode() { return stringValue.hashCode(); }
+    public int hashCode() { return major*3 + minor*5 + micro*7 + qualifier.hashCode()*11; }
 
     /** Returns whether this equals the empty version */
     public boolean isEmpty() { return this.equals(emptyVersion); }
diff --git a/config-application-package/src/main/java/com/yahoo/config/application/OverrideProcessor.java b/config-application-package/src/main/java/com/yahoo/config/application/OverrideProcessor.java
index 15b775a4543..23273f9f1bf 100644
--- a/config-application-package/src/main/java/com/yahoo/config/application/OverrideProcessor.java
+++ b/config-application-package/src/main/java/com/yahoo/config/application/OverrideProcessor.java
@@ -47,11 +47,6 @@ class OverrideProcessor implements PreProcessor {
     private static final String ENVIRONMENT_ATTRIBUTE = "environment";
     private static final String REGION_ATTRIBUTE = "region";
 
-    // TODO: Remove after September 2019
-    public OverrideProcessor(Environment environment, RegionName region) {
-        this(InstanceName.from("default"), environment, region);
-    }
-
     public OverrideProcessor(InstanceName instance, Environment environment, RegionName region) {
         this.instance = instance;
         this.environment = environment;
diff --git a/config-application-package/src/main/java/com/yahoo/config/application/XmlPreProcessor.java b/config-application-package/src/main/java/com/yahoo/config/application/XmlPreProcessor.java
index 73f97cf516f..c49a799f2b7 100644
--- a/config-application-package/src/main/java/com/yahoo/config/application/XmlPreProcessor.java
+++ b/config-application-package/src/main/java/com/yahoo/config/application/XmlPreProcessor.java
@@ -38,20 +38,10 @@ public class XmlPreProcessor {
     private final RegionName region;
     private final List<PreProcessor> chain;
 
-    // TODO: Remove after September 2019
-    public XmlPreProcessor(File applicationDir, File xmlInput, Environment environment, RegionName region) throws IOException {
-        this(applicationDir, new FileReader(xmlInput), InstanceName.from("default"), environment, region);
-    }
-
     public XmlPreProcessor(File applicationDir, File xmlInput, InstanceName instance, Environment environment, RegionName region) throws IOException {
         this(applicationDir, new FileReader(xmlInput), instance, environment, region);
     }
 
-    // TODO: Remove after September 2019
-    public XmlPreProcessor(File applicationDir, Reader xmlInput, Environment environment, RegionName region) throws IOException {
-        this(applicationDir, xmlInput, InstanceName.from("default"), environment, region);
-    }
-
     public XmlPreProcessor(File applicationDir, Reader xmlInput, InstanceName instance, Environment environment, RegionName region) {
         this.applicationDir = applicationDir;
         this.xmlInput = xmlInput;
diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/AppSubDirs.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/AppSubDirs.java
index 32054b1ad9a..2b7aa5ab6e1 100644
--- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/AppSubDirs.java
+++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/AppSubDirs.java
@@ -59,8 +59,7 @@ public class AppSubDirs {
         return configDefs.first;
     }
 
-    public File searchdefinitions() {
-        return searchdefinitions.first;
-    }
+    @Deprecated // Remove after March 2020
+    public File searchdefinitions() { return searchdefinitions.first; }
 
 }
diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/ApplicationPackageXmlFilesValidator.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/ApplicationPackageXmlFilesValidator.java
index 74ade9d8e14..b0c4f74f9f6 100644
--- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/ApplicationPackageXmlFilesValidator.java
+++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/ApplicationPackageXmlFilesValidator.java
@@ -35,20 +35,11 @@ public class ApplicationPackageXmlFilesValidator {
         return new ApplicationPackageXmlFilesValidator(new AppSubDirs(appDir), vespaVersion);
     }
 
-    @SuppressWarnings("deprecation")
     public void checkApplication() throws IOException {
         validate(validators.servicesXmlValidator(), servicesFileName());
         validateOptional(validators.hostsXmlValidator(), FilesApplicationPackage.HOSTS);
         validateOptional(validators.deploymentXmlValidator(), FilesApplicationPackage.DEPLOYMENT_FILE.getName());
         validateOptional(validators.validationOverridesXmlValidator(), FilesApplicationPackage.VALIDATION_OVERRIDES.getName());
-
-        if (appDirs.searchdefinitions().exists()) {
-            if (FilesApplicationPackage.getSearchDefinitionFiles(appDirs.root()).isEmpty()) {
-                throw new IllegalArgumentException("Application package in " + appDirs.root() +
-                        " must contain at least one search definition (.sd) file when directory searchdefinitions/ exists.");
-            }
-        }
-
         validateRouting(appDirs.routingtables);
     }
 
@@ -71,7 +62,6 @@ public class ApplicationPackageXmlFilesValidator {
         validator.validate(appDirs.file(filename));
     }
 
-    @SuppressWarnings("deprecation")
     private String servicesFileName() {
         String servicesFile = FilesApplicationPackage.SERVICES;
         if ( ! appDirs.file(servicesFile).exists()) {
diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/DeployData.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/DeployData.java
index 344035745f8..82da1e2c6ea 100644
--- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/DeployData.java
+++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/DeployData.java
@@ -2,9 +2,6 @@
 package com.yahoo.config.model.application.provider;
 
 import com.yahoo.config.provision.ApplicationId;
-import com.yahoo.config.provision.ApplicationName;
-import com.yahoo.config.provision.InstanceName;
-import com.yahoo.config.provision.TenantName;
 
 /**
  * Data generated or computed during deployment
@@ -31,22 +28,6 @@ public class DeployData {
     private final long generation;
     private final long currentlyActiveGeneration;
 
-    // TODO: Remove after September 2019
-    public DeployData(String deployedByUser,
-                      String deployedFromDir,
-                      String applicationName,
-                      Long deployTimestamp,
-                      boolean internalRedeploy,
-                      Long generation,
-                      long currentlyActiveGeneration) {
-        this(deployedByUser,
-             deployedFromDir,
-             ApplicationId.from(TenantName.defaultName(), ApplicationName.from(applicationName), InstanceName.from("default")),
-             deployTimestamp,
-             internalRedeploy,
-             generation, currentlyActiveGeneration);
-    }
-
     public DeployData(String deployedByUser,
                       String deployedFromDir,
                       ApplicationId applicationId,
@@ -77,7 +58,4 @@ public class DeployData {
 
     public ApplicationId getApplicationId() { return applicationId; }
 
-    // TODO: remove after September 2019
-    public String getApplicationName() { return applicationId.application().toString(); }
-
 }
diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java
index 8c3ccdc4c0b..e93246f49e7 100644
--- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java
+++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java
@@ -49,6 +49,7 @@ import java.security.MessageDigest;
 import java.util.*;
 import java.util.jar.JarFile;
 import java.util.logging.Logger;
+import java.util.stream.Stream;
 
 import static com.yahoo.text.Lowercase.toLowerCase;
 
@@ -336,77 +337,73 @@ public class FilesApplicationPackage implements ApplicationPackage {
     public static Map<String, String> allSdsFromDocprocBundlesAndClasspath(File appDir) throws IOException {
         File dpChains = new File(appDir, ApplicationPackage.COMPONENT_DIR);
         if (!dpChains.exists() || !dpChains.isDirectory()) return Collections.emptyMap();
-        List<String> usedNames = new ArrayList<>();
-        Map<String, String> ret = new LinkedHashMap<>();
+        Set<String> usedNames = new HashSet<>();
+        Map<String, String> schemas = new LinkedHashMap<>();
 
         // try classpath first
-        allSdsOnClassPath(usedNames, ret);
+        allSdsOnClassPath(usedNames, schemas);
 
-        for (File bundle : dpChains.listFiles(new FilenameFilter() {
-            @Override
-            public boolean accept(File dir, String name) {
-                return name.endsWith(".jar");
-            }})) {
+        for (File bundle : dpChains.listFiles((File dir, String name) -> name.endsWith(".jar"))) {
             for(Map.Entry<String, String> entry : ApplicationPackage.getBundleSdFiles("", new JarFile(bundle)).entrySet()) {
                 String sdName = entry.getKey();
                 if (usedNames.contains(sdName)) {
-                    throw new IllegalArgumentException("The search definition name '"+sdName+"' used in bundle '"+
-                                                       bundle.getName()+"' is already used in classpath or previous bundle.");
+                    throw new IllegalArgumentException("The search definition name '" + sdName + "' used in bundle '"+
+                                                       bundle.getName()+ "' is already used in classpath or previous bundle.");
                 }
                 usedNames.add(sdName);
                 String sdPayload = entry.getValue();
-                ret.put(sdName, sdPayload);
+                schemas.put(sdName, sdPayload);
             }
         }
-        return ret;
+        return schemas;
     }
 
-	private static void allSdsOnClassPath(List<String> usedNames, Map<String, String> ret) throws IOException {
-		Enumeration<java.net.URL> resources = FilesApplicationPackage.class.getClassLoader().getResources(ApplicationPackage.SEARCH_DEFINITIONS_DIR.getRelative());
-
-        while(resources.hasMoreElements()) {
-            URL resource = resources.nextElement();
-
-            String protocol = resource.getProtocol();
+	private static void allSdsOnClassPath(Set<String> usedNames, Map<String, String> schemas) {
+        ClassLoader cl = FilesApplicationPackage.class.getClassLoader();
+        Stream<URL> resources = Stream.concat(cl.resources(ApplicationPackage.SEARCH_DEFINITIONS_DIR.getRelative()),
+                                              cl.resources(ApplicationPackage.SCHEMAS_DIR.getRelative()));
+        resources.forEach(resource -> addSchemaFrom(resource, schemas, usedNames));
+    }
 
-            if ("file".equals(protocol)) {
-                File file;
-                try {
-                    file = new File(resource.toURI());
-                } catch (URISyntaxException e) {
-                    continue;
-                }
-                // only interested in directories
-                if (file.isDirectory()) {
-                    List<File> sdFiles = getSearchDefinitionFiles(file);
-                    for (File sdFile : sdFiles) {
-                        String sdName = sdFile.getName();
+    private static void addSchemaFrom(URL resource, Map<String, String> schemas, Set<String> usedNames) {
+        try {
+            switch (resource.getProtocol()) {
+                case "file":
+                    File file = new File(resource.toURI());
+                    if (file.isDirectory()) {
+                        List<File> sdFiles = getSearchDefinitionFiles(file);
+                        for (File sdFile : sdFiles) {
+                            String sdName = sdFile.getName();
+                            if (usedNames.contains(sdName)) {
+                                throw new IllegalArgumentException("The search definition name '" + sdName +
+                                                                   "' found in classpath already used earlier in classpath.");
+                            }
+                            usedNames.add(sdName);
+                            String contents = IOUtils.readAll(new FileReader(sdFile));
+                            schemas.put(sdFile.getName(), contents);
+                        }
+                    }
+                    break;
+                case "jar":
+                    JarURLConnection jarConnection = (JarURLConnection) resource.openConnection();
+                    JarFile jarFile = jarConnection.getJarFile();
+                    for (Map.Entry<String, String> entry : ApplicationPackage.getBundleSdFiles("", jarFile).entrySet()) {
+                        String sdName = entry.getKey();
                         if (usedNames.contains(sdName)) {
-                            throw new IllegalArgumentException("The search definition name '"+sdName+
-                                                               "' found in classpath already used earlier in classpath.");
+                            throw new IllegalArgumentException("The search definitions name '" + sdName +
+                                                               "' used in bundle '" + jarFile.getName() + "' " +
+                                                               "is already used in classpath or previous bundle.");
                         }
                         usedNames.add(sdName);
-                        String contents = IOUtils.readAll(new FileReader(sdFile));
-                        ret.put(sdFile.getName(), contents);
-                    }
-                }
-            }
-            else if ("jar".equals(protocol)) {
-                JarURLConnection jarConnection = (JarURLConnection) resource.openConnection();
-                JarFile jarFile = jarConnection.getJarFile();
-                for(Map.Entry<String, String> entry : ApplicationPackage.getBundleSdFiles("", jarFile).entrySet()) {
-                    String sdName = entry.getKey();
-                    if (usedNames.contains(sdName)) {
-                        throw new IllegalArgumentException("The search definitions name '"+sdName+
-                                                           "' used in bundle '"+jarFile.getName()+"' " +
-                                                           "is already used in classpath or previous bundle.");
+                        String sdPayload = entry.getValue();
+                        schemas.put(sdName, sdPayload);
                     }
-                    usedNames.add(sdName);
-                    String sdPayload = entry.getValue();
-                    ret.put(sdName, sdPayload);
-                }
+                    break;
             }
         }
+        catch (IOException | URISyntaxException e) {
+            throw new IllegalArgumentException("Could not read schema from '" + resource + "'", e);
+        }
 	}
 
     /**
@@ -462,11 +459,7 @@ public class FilesApplicationPackage implements ApplicationPackage {
         if (! configDefsDir.isDirectory()) return;
 
         log.log(LogLevel.DEBUG, "Getting all config definitions from '" + configDefsDir + "'");
-        for (File def : configDefsDir.listFiles(
-                new FilenameFilter() { @Override public boolean accept(File dir, String name) { // TODO: Fix
-                    return name.matches(".*\\.def");}})) {
-
-            log.log(LogLevel.DEBUG, "Processing config definition '" + def + "'");
+        for (File def : configDefsDir.listFiles((File dir, String name) -> name.matches(".*\\.def"))) {
             String[] nv = def.getName().split("\\.def");
             ConfigDefinitionKey key;
             try {
@@ -521,30 +514,25 @@ public class FilesApplicationPackage implements ApplicationPackage {
         }
     }
 
-    //Only intended for DeployProcessor, others should use the member version
     static List<File> getSearchDefinitionFiles(File appDir) {
-        //The dot is escaped later in this method:
-        assert (ApplicationPackage.SD_NAME_SUFFIX.charAt(0) == '.');
+        List<File> schemaFiles = new ArrayList<>();
+
+        File sdDir = new File(appDir, ApplicationPackage.SEARCH_DEFINITIONS_DIR.getRelative());
+        if (sdDir.isDirectory())
+            schemaFiles.addAll(Arrays.asList(sdDir.listFiles((dir, name) -> name.matches(".*\\" + ApplicationPackage.SD_NAME_SUFFIX))));
 
-        List<File> ret = new ArrayList<>();
-        File sdDir;
+        sdDir = new File(appDir, ApplicationPackage.SCHEMAS_DIR.getRelative());
+        if (sdDir.isDirectory())
+            schemaFiles.addAll(Arrays.asList(sdDir.listFiles((dir, name) -> name.matches(".*\\" + ApplicationPackage.SD_NAME_SUFFIX))));
 
-        sdDir = new File(appDir, ApplicationPackage.SEARCH_DEFINITIONS_DIR.getRelative());
-        if (!sdDir.isDirectory()) {
-            return ret;
-        }
-        ret.addAll(Arrays.asList(
-                sdDir.listFiles(
-                        new FilenameFilter() { @Override public boolean accept(File dir, String name) {
-                            return name.matches(".*\\" + ApplicationPackage.SD_NAME_SUFFIX);}})));
-        return ret;
+        return schemaFiles;
     }
 
     public List<File> getSearchDefinitionFiles() {
         return getSearchDefinitionFiles(appDir);
     }
 
-    //Only for use by deploy processor
+    // Only for use by deploy processor
     public static List<Component> getComponents(File appDir) {
         List<Component> components = new ArrayList<>();
         for (Bundle bundle : Bundle.getBundles(new File(appDir, ApplicationPackage.COMPONENT_DIR))) {
@@ -622,6 +610,7 @@ public class FilesApplicationPackage implements ApplicationPackage {
         public Bundle getBundle() {
             return bundle;
         }
+
     } // class Component
 
     /**
@@ -643,12 +632,16 @@ public class FilesApplicationPackage implements ApplicationPackage {
     }
 
     private File expressionFileNameToFile(String name) {
-        File expressionFile = new File(name);
-        if (expressionFile.isAbsolute()) {
+        if (new File(name).isAbsolute())
             throw new IllegalArgumentException("Absolute path to ranking expression file is not allowed: " + name);
+
+        File sdDir = new File(appDir, ApplicationPackage.SCHEMAS_DIR.getRelative());
+        File expressionFile = new File(sdDir, name);
+        if ( ! expressionFile.exists()) {
+            sdDir = new File(appDir, ApplicationPackage.SEARCH_DEFINITIONS_DIR.getRelative());
+            expressionFile = new File(sdDir, name);
         }
-        File sdDir = new File(appDir, ApplicationPackage.SEARCH_DEFINITIONS_DIR.getRelative());
-        return new File(sdDir, name);
+        return expressionFile;
     }
 
     @Override
@@ -700,9 +693,8 @@ public class FilesApplicationPackage implements ApplicationPackage {
                                                                                          ! name.equals(HOSTS) &&
                                                                                          ! name.equals(CONFIG_DEFINITIONS_DIR));
         preprocessXML(new File(preprocessedDir, SERVICES), getServicesFile(), zone);
-        if (getHostsFile().exists()) {
+        if (getHostsFile().exists())
             preprocessXML(new File(preprocessedDir, HOSTS), getHostsFile(), zone);
-        }
         FilesApplicationPackage preprocessed = FilesApplicationPackage.fromFile(preprocessedDir, includeSourceFiles);
         preprocessed.copyUserDefsIntoApplication();
         return preprocessed;
@@ -742,11 +734,12 @@ public class FilesApplicationPackage implements ApplicationPackage {
 
     /**
      * Adds the given path to the digest, or does nothing if path is neither file nor dir
+     *
      * @param path path to add to message digest
      * @param suffix only files with this suffix are considered
      * @param digest the {link @MessageDigest} to add the file paths to
      * @param recursive whether to recursively find children in the paths
-     * @param fullPathNames Whether to include the full paths in checksum or only the names
+     * @param fullPathNames whether to include the full paths in checksum or only the names
      * @throws java.io.IOException if adding path to digest fails when reading files from path
      */
     private static void addPathToDigest(File path, String suffix, MessageDigest digest, boolean recursive, boolean fullPathNames) throws IOException {
@@ -773,16 +766,17 @@ public class FilesApplicationPackage implements ApplicationPackage {
     }
 
     private static final int MD5_BUFFER_SIZE = 65536;
+
     private static void addToDigest(InputStream is, MessageDigest digest) throws IOException {
-        if (is==null) return;
+        if (is == null) return;
         byte[] buffer = new byte[MD5_BUFFER_SIZE];
         int i;
         do {
-            i=is.read(buffer);
+            i = is.read(buffer);
             if (i > 0) {
                 digest.update(buffer, 0, i);
             }
-        } while(i!=-1);
+        } while(i != -1);
     }
 
     /**
diff --git a/config-lib/src/main/java/com/yahoo/config/FileReference.java b/config-lib/src/main/java/com/yahoo/config/FileReference.java
index 3b95c2fbd4c..ee99ebfa2b7 100755
--- a/config-lib/src/main/java/com/yahoo/config/FileReference.java
+++ b/config-lib/src/main/java/com/yahoo/config/FileReference.java
@@ -28,14 +28,16 @@ public final class FileReference {
     }
 
     @Override
-    public int hashCode() {
-        return value.hashCode();
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        FileReference that = (FileReference) o;
+        return value.equals(that.value);
     }
 
     @Override
-    public boolean equals(Object other) {
-        return other instanceof FileReference &&
-                value.equals(((FileReference)other).value);
+    public int hashCode() {
+        return Objects.hash(value);
     }
 
     @Override
diff --git a/config-model-api/abi-spec.json b/config-model-api/abi-spec.json
index 0f5a5e6271d..1b5a8924fea 100644
--- a/config-model-api/abi-spec.json
+++ b/config-model-api/abi-spec.json
@@ -71,17 +71,13 @@
       "public"
     ],
     "methods": [
-      "public void <init>(java.io.File, java.lang.String, java.lang.String, java.lang.Long, boolean, java.lang.String, java.lang.Long, long)",
-      "public void <init>(java.lang.String, java.lang.String, java.lang.Long, boolean, java.lang.String, java.lang.String, java.lang.Long, long)",
       "public void <init>(java.lang.String, java.lang.String, java.lang.Long, boolean, com.yahoo.config.provision.ApplicationId, java.lang.String, java.lang.Long, long)",
-      "public java.lang.String getApplicationName()",
       "public java.lang.String getDeployedByUser()",
       "public java.lang.String getDeployPath()",
       "public com.yahoo.config.provision.ApplicationId getApplicationId()",
       "public java.lang.Long getDeployTimestamp()",
       "public java.lang.Long getGeneration()",
       "public boolean isInternalRedeploy()",
-      "public java.lang.String getCheckSum()",
       "public java.lang.String getChecksum()",
       "public long getPreviousActiveGeneration()",
       "public java.lang.String toString()",
@@ -137,6 +133,7 @@
     "fields": [
       "public static final java.lang.String HOSTS",
       "public static final java.lang.String SERVICES",
+      "public static final com.yahoo.path.Path SCHEMAS_DIR",
       "public static final com.yahoo.path.Path SEARCH_DEFINITIONS_DIR",
       "public static final java.lang.String COMPONENT_DIR",
       "public static final java.lang.String SEARCHCHAINS_DIR",
@@ -605,511 +602,5 @@
       "public static final com.yahoo.config.application.api.ValidationOverrides empty",
       "public static final com.yahoo.config.application.api.ValidationOverrides all"
     ]
-  },
-  "com.yahoo.config.model.api.ApplicationInfo": {
-    "superClass": "java.lang.Object",
-    "interfaces": [],
-    "attributes": [
-      "public"
-    ],
-    "methods": [
-      "public void <init>(com.yahoo.config.provision.ApplicationId, long, com.yahoo.config.model.api.Model)",
-      "public com.yahoo.config.provision.ApplicationId getApplicationId()",
-      "public long getGeneration()",
-      "public com.yahoo.config.model.api.Model getModel()",
-      "public java.lang.String toString()"
-    ],
-    "fields": []
-  },
-  "com.yahoo.config.model.api.ConfigChangeAction$Type": {
-    "superClass": "java.lang.Enum",
-    "interfaces": [],
-    "attributes": [
-      "public",
-      "final",
-      "enum"
-    ],
-    "methods": [
-      "public static com.yahoo.config.model.api.ConfigChangeAction$Type[] values()",
-      "public static com.yahoo.config.model.api.ConfigChangeAction$Type valueOf(java.lang.String)",
-      "public java.lang.String toString()"
-    ],
-    "fields": [
-      "public static final enum com.yahoo.config.model.api.ConfigChangeAction$Type RESTART",
-      "public static final enum com.yahoo.config.model.api.ConfigChangeAction$Type REFEED"
-    ]
-  },
-  "com.yahoo.config.model.api.ConfigChangeAction": {
-    "superClass": "java.lang.Object",
-    "interfaces": [],
-    "attributes": [
-      "public",
-      "interface",
-      "abstract"
-    ],
-    "methods": [
-      "public abstract com.yahoo.config.model.api.ConfigChangeAction$Type getType()",
-      "public abstract java.lang.String getMessage()",
-      "public abstract java.util.List getServices()",
-      "public abstract boolean allowed()"
-    ],
-    "fields": []
-  },
-  "com.yahoo.config.model.api.ConfigChangeRefeedAction": {
-    "superClass": "java.lang.Object",
-    "interfaces": [
-      "com.yahoo.config.model.api.ConfigChangeAction"
-    ],
-    "attributes": [
-      "public",
-      "interface",
-      "abstract"
-    ],
-    "methods": [
-      "public com.yahoo.config.model.api.ConfigChangeAction$Type getType()",
-      "public java.lang.String name()",
-      "public abstract java.lang.String getDocumentType()"
-    ],
-    "fields": []
-  },
-  "com.yahoo.config.model.api.ConfigChangeRestartAction": {
-    "superClass": "java.lang.Object",
-    "interfaces": [
-      "com.yahoo.config.model.api.ConfigChangeAction"
-    ],
-    "attributes": [
-      "public",
-      "interface",
-      "abstract"
-    ],
-    "methods": [
-      "public com.yahoo.config.model.api.ConfigChangeAction$Type getType()",
-      "public boolean allowed()"
-    ],
-    "fields": []
-  },
-  "com.yahoo.config.model.api.ConfigDefinitionRepo": {
-    "superClass": "java.lang.Object",
-    "interfaces": [],
-    "attributes": [
-      "public",
-      "interface",
-      "abstract"
-    ],
-    "methods": [
-      "public abstract java.util.Map getConfigDefinitions()",
-      "public abstract com.yahoo.vespa.config.buildergen.ConfigDefinition get(com.yahoo.vespa.config.ConfigDefinitionKey)"
-    ],
-    "fields": []
-  },
-  "com.yahoo.config.model.api.ConfigDefinitionStore": {
-    "superClass": "java.lang.Object",
-    "interfaces": [],
-    "attributes": [
-      "public",
-      "interface",
-      "abstract"
-    ],
-    "methods": [
-      "public abstract com.yahoo.vespa.config.ConfigDefinition getConfigDefinition(com.yahoo.vespa.config.ConfigDefinitionKey)"
-    ],
-    "fields": []
-  },
-  "com.yahoo.config.model.api.ConfigModelPlugin": {
-    "superClass": "java.lang.Object",
-    "interfaces": [],
-    "attributes": [
-      "public",
-      "interface",
-      "abstract"
-    ],
-    "methods": [],
-    "fields": []
-  },
-  "com.yahoo.config.model.api.ConfigServerSpec": {
-    "superClass": "java.lang.Object",
-    "interfaces": [],
-    "attributes": [
-      "public",
-      "interface",
-      "abstract"
-    ],
-    "methods": [
-      "public abstract java.lang.String getHostName()",
-      "public abstract int getConfigServerPort()",
-      "public int getHttpPort()",
-      "public abstract int getZooKeeperPort()"
-    ],
-    "fields": []
-  },
-  "com.yahoo.config.model.api.ContainerEndpoint": {
-    "superClass": "java.lang.Object",
-    "interfaces": [],
-    "attributes": [
-      "public"
-    ],
-    "methods": [
-      "public void <init>(java.lang.String, java.util.List)",
-      "public java.lang.String clusterId()",
-      "public java.util.List names()",
-      "public boolean equals(java.lang.Object)",
-      "public int hashCode()",
-      "public java.lang.String toString()"
-    ],
-    "fields": []
-  },
-  "com.yahoo.config.model.api.EndpointCertificateMetadata": {
-    "superClass": "java.lang.Object",
-    "interfaces": [],
-    "attributes": [
-      "public"
-    ],
-    "methods": [
-      "public void <init>(java.lang.String, java.lang.String, int)",
-      "public java.lang.String keyName()",
-      "public java.lang.String certName()",
-      "public int version()",
-      "public java.lang.String toString()"
-    ],
-    "fields": []
-  },
-  "com.yahoo.config.model.api.EndpointCertificateSecrets": {
-    "superClass": "java.lang.Object",
-    "interfaces": [],
-    "attributes": [
-      "public"
-    ],
-    "methods": [
-      "public void <init>(java.lang.String, java.lang.String)",
-      "public java.lang.String certificate()",
-      "public java.lang.String key()",
-      "public boolean isMissing()"
-    ],
-    "fields": [
-      "public static final com.yahoo.config.model.api.EndpointCertificateSecrets MISSING"
-    ]
-  },
-  "com.yahoo.config.model.api.FileDistribution": {
-    "superClass": "java.lang.Object",
-    "interfaces": [],
-    "attributes": [
-      "public",
-      "interface",
-      "abstract"
-    ],
-    "methods": [
-      "public abstract void startDownload(java.lang.String, int, java.util.Set)",
-      "public abstract java.io.File getFileReferencesDir()"
-    ],
-    "fields": []
-  },
-  "com.yahoo.config.model.api.HostInfo": {
-    "superClass": "java.lang.Object",
-    "interfaces": [],
-    "attributes": [
-      "public"
-    ],
-    "methods": [
-      "public void <init>(java.lang.String, java.util.Collection)",
-      "public java.lang.String getHostname()",
-      "public java.util.Collection getServices()",
-      "public boolean equals(java.lang.Object)",
-      "public int hashCode()"
-    ],
-    "fields": []
-  },
-  "com.yahoo.config.model.api.HostProvisioner": {
-    "superClass": "java.lang.Object",
-    "interfaces": [],
-    "attributes": [
-      "public",
-      "interface",
-      "abstract"
-    ],
-    "methods": [
-      "public abstract com.yahoo.config.provision.HostSpec allocateHost(java.lang.String)",
-      "public abstract java.util.List prepare(com.yahoo.config.provision.ClusterSpec, com.yahoo.config.provision.Capacity, int, com.yahoo.config.provision.ProvisionLogger)"
-    ],
-    "fields": []
-  },
-  "com.yahoo.config.model.api.Model": {
-    "superClass": "java.lang.Object",
-    "interfaces": [],
-    "attributes": [
-      "public",
-      "interface",
-      "abstract"
-    ],
-    "methods": [
-      "public abstract com.yahoo.vespa.config.ConfigPayload getConfig(com.yahoo.vespa.config.ConfigKey, com.yahoo.vespa.config.buildergen.ConfigDefinition)",
-      "public abstract java.util.Set allConfigsProduced()",
-      "public abstract java.util.Collection getHosts()",
-      "public abstract java.util.Set allConfigIds()",
-      "public abstract void distributeFiles(com.yahoo.config.model.api.FileDistribution)",
-      "public abstract java.util.Set fileReferences()",
-      "public abstract com.yahoo.config.provision.AllocatedHosts allocatedHosts()",
-      "public boolean allowModelVersionMismatch(java.time.Instant)",
-      "public boolean skipOldConfigModels(java.time.Instant)",
-      "public com.yahoo.component.Version version()"
-    ],
-    "fields": []
-  },
-  "com.yahoo.config.model.api.ModelContext$Properties": {
-    "superClass": "java.lang.Object",
-    "interfaces": [],
-    "attributes": [
-      "public",
-      "interface",
-      "abstract"
-    ],
-    "methods": [
-      "public abstract boolean multitenant()",
-      "public abstract com.yahoo.config.provision.ApplicationId applicationId()",
-      "public abstract java.util.List configServerSpecs()",
-      "public abstract com.yahoo.config.provision.HostName loadBalancerName()",
-      "public abstract java.net.URI ztsUrl()",
-      "public abstract java.lang.String athenzDnsSuffix()",
-      "public abstract boolean hostedVespa()",
-      "public abstract com.yahoo.config.provision.Zone zone()",
-      "public abstract java.util.Set endpoints()",
-      "public abstract boolean isBootstrap()",
-      "public abstract boolean isFirstTimeDeployment()",
-      "public boolean useDedicatedNodeForLogserver()",
-      "public abstract boolean useAdaptiveDispatch()",
-      "public java.util.Optional tlsSecrets()",
-      "public java.util.Optional endpointCertificateSecrets()",
-      "public abstract double defaultTermwiseLimit()",
-      "public abstract boolean useBucketSpaceMetric()"
-    ],
-    "fields": []
-  },
-  "com.yahoo.config.model.api.ModelContext": {
-    "superClass": "java.lang.Object",
-    "interfaces": [],
-    "attributes": [
-      "public",
-      "interface",
-      "abstract"
-    ],
-    "methods": [
-      "public abstract com.yahoo.config.application.api.ApplicationPackage applicationPackage()",
-      "public abstract java.util.Optional previousModel()",
-      "public abstract java.util.Optional permanentApplicationPackage()",
-      "public abstract java.util.Optional hostProvisioner()",
-      "public abstract com.yahoo.config.application.api.DeployLogger deployLogger()",
-      "public abstract com.yahoo.config.model.api.ConfigDefinitionRepo configDefinitionRepo()",
-      "public abstract com.yahoo.config.application.api.FileRegistry getFileRegistry()",
-      "public abstract com.yahoo.config.model.api.ModelContext$Properties properties()",
-      "public java.util.Optional appDir()",
-      "public abstract com.yahoo.component.Version modelVespaVersion()",
-      "public abstract com.yahoo.component.Version wantedNodeVespaVersion()"
-    ],
-    "fields": []
-  },
-  "com.yahoo.config.model.api.ModelCreateResult": {
-    "superClass": "java.lang.Object",
-    "interfaces": [],
-    "attributes": [
-      "public"
-    ],
-    "methods": [
-      "public void <init>(com.yahoo.config.model.api.Model, java.util.List)",
-      "public com.yahoo.config.model.api.Model getModel()",
-      "public java.util.List getConfigChangeActions()"
-    ],
-    "fields": []
-  },
-  "com.yahoo.config.model.api.ModelFactory": {
-    "superClass": "java.lang.Object",
-    "interfaces": [],
-    "attributes": [
-      "public",
-      "interface",
-      "abstract"
-    ],
-    "methods": [
-      "public abstract com.yahoo.component.Version version()",
-      "public abstract com.yahoo.config.model.api.Model createModel(com.yahoo.config.model.api.ModelContext)",
-      "public abstract com.yahoo.config.model.api.ModelCreateResult createAndValidateModel(com.yahoo.config.model.api.ModelContext, com.yahoo.config.model.api.ValidationParameters)"
-    ],
-    "fields": []
-  },
-  "com.yahoo.config.model.api.ModelState": {
-    "superClass": "java.lang.Object",
-    "interfaces": [],
-    "attributes": [
-      "public",
-      "interface",
-      "abstract"
-    ],
-    "methods": [
-      "public abstract com.yahoo.config.model.api.Model getModel()"
-    ],
-    "fields": []
-  },
-  "com.yahoo.config.model.api.PortInfo": {
-    "superClass": "java.lang.Object",
-    "interfaces": [],
-    "attributes": [
-      "public"
-    ],
-    "methods": [
-      "public void <init>(int, java.util.Collection)",
-      "public int getPort()",
-      "public java.util.Collection getTags()",
-      "public boolean equals(java.lang.Object)",
-      "public int hashCode()"
-    ],
-    "fields": []
-  },
-  "com.yahoo.config.model.api.ServiceInfo": {
-    "superClass": "java.lang.Object",
-    "interfaces": [],
-    "attributes": [
-      "public"
-    ],
-    "methods": [
-      "public void <init>(java.lang.String, java.lang.String, java.util.Collection, java.util.Map, java.lang.String, java.lang.String)",
-      "public java.lang.String getServiceName()",
-      "public java.lang.String getConfigId()",
-      "public java.lang.String getServiceType()",
-      "public java.util.Optional getProperty(java.lang.String)",
-      "public java.util.Collection getPorts()",
-      "public java.lang.String getHostName()",
-      "public boolean equals(java.lang.Object)",
-      "public int hashCode()"
-    ],
-    "fields": []
-  },
-  "com.yahoo.config.model.api.SuperModel": {
-    "superClass": "java.lang.Object",
-    "interfaces": [],
-    "attributes": [
-      "public"
-    ],
-    "methods": [
-      "public void <init>()",
-      "public void <init>(java.util.Map)",
-      "public java.util.Map getModelsPerTenant()",
-      "public java.util.Map getModels()",
-      "public java.util.List getAllApplicationInfos()",
-      "public java.util.Optional getApplicationInfo(com.yahoo.config.provision.ApplicationId)",
-      "public com.yahoo.config.model.api.SuperModel cloneAndSetApplication(com.yahoo.config.model.api.ApplicationInfo)",
-      "public com.yahoo.config.model.api.SuperModel cloneAndRemoveApplication(com.yahoo.config.provision.ApplicationId)"
-    ],
-    "fields": []
-  },
-  "com.yahoo.config.model.api.SuperModelListener": {
-    "superClass": "java.lang.Object",
-    "interfaces": [],
-    "attributes": [
-      "public",
-      "interface",
-      "abstract"
-    ],
-    "methods": [
-      "public abstract void applicationActivated(com.yahoo.config.model.api.SuperModel, com.yahoo.config.model.api.ApplicationInfo)",
-      "public abstract void applicationRemoved(com.yahoo.config.model.api.SuperModel, com.yahoo.config.provision.ApplicationId)"
-    ],
-    "fields": []
-  },
-  "com.yahoo.config.model.api.SuperModelProvider": {
-    "superClass": "java.lang.Object",
-    "interfaces": [],
-    "attributes": [
-      "public",
-      "interface",
-      "abstract"
-    ],
-    "methods": [
-      "public abstract void registerListener(com.yahoo.config.model.api.SuperModelListener)",
-      "public abstract com.yahoo.config.model.api.SuperModel getSuperModel()"
-    ],
-    "fields": []
-  },
-  "com.yahoo.config.model.api.TlsSecrets": {
-    "superClass": "java.lang.Object",
-    "interfaces": [],
-    "attributes": [
-      "public"
-    ],
-    "methods": [
-      "public void <init>(java.lang.String, java.lang.String)",
-      "public void <init>(com.yahoo.config.model.api.EndpointCertificateSecrets)",
-      "public java.lang.String certificate()",
-      "public java.lang.String key()",
-      "public boolean isMissing()"
-    ],
-    "fields": [
-      "public static final com.yahoo.config.model.api.TlsSecrets MISSING"
-    ]
-  },
-  "com.yahoo.config.model.api.ValidationParameters$CheckRouting": {
-    "superClass": "java.lang.Enum",
-    "interfaces": [],
-    "attributes": [
-      "public",
-      "final",
-      "enum"
-    ],
-    "methods": [
-      "public static com.yahoo.config.model.api.ValidationParameters$CheckRouting[] values()",
-      "public static com.yahoo.config.model.api.ValidationParameters$CheckRouting valueOf(java.lang.String)"
-    ],
-    "fields": [
-      "public static final enum com.yahoo.config.model.api.ValidationParameters$CheckRouting TRUE",
-      "public static final enum com.yahoo.config.model.api.ValidationParameters$CheckRouting FALSE"
-    ]
-  },
-  "com.yahoo.config.model.api.ValidationParameters$FailOnIncompatibleChange": {
-    "superClass": "java.lang.Enum",
-    "interfaces": [],
-    "attributes": [
-      "public",
-      "final",
-      "enum"
-    ],
-    "methods": [
-      "public static com.yahoo.config.model.api.ValidationParameters$FailOnIncompatibleChange[] values()",
-      "public static com.yahoo.config.model.api.ValidationParameters$FailOnIncompatibleChange valueOf(java.lang.String)"
-    ],
-    "fields": [
-      "public static final enum com.yahoo.config.model.api.ValidationParameters$FailOnIncompatibleChange TRUE",
-      "public static final enum com.yahoo.config.model.api.ValidationParameters$FailOnIncompatibleChange FALSE"
-    ]
-  },
-  "com.yahoo.config.model.api.ValidationParameters$IgnoreValidationErrors": {
-    "superClass": "java.lang.Enum",
-    "interfaces": [],
-    "attributes": [
-      "public",
-      "final",
-      "enum"
-    ],
-    "methods": [
-      "public static com.yahoo.config.model.api.ValidationParameters$IgnoreValidationErrors[] values()",
-      "public static com.yahoo.config.model.api.ValidationParameters$IgnoreValidationErrors valueOf(java.lang.String)"
-    ],
-    "fields": [
-      "public static final enum com.yahoo.config.model.api.ValidationParameters$IgnoreValidationErrors TRUE",
-      "public static final enum com.yahoo.config.model.api.ValidationParameters$IgnoreValidationErrors FALSE"
-    ]
-  },
-  "com.yahoo.config.model.api.ValidationParameters": {
-    "superClass": "java.lang.Object",
-    "interfaces": [],
-    "attributes": [
-      "public"
-    ],
-    "methods": [
-      "public void <init>()",
-      "public void <init>(com.yahoo.config.model.api.ValidationParameters$IgnoreValidationErrors)",
-      "public void <init>(com.yahoo.config.model.api.ValidationParameters$CheckRouting)",
-      "public void <init>(com.yahoo.config.model.api.ValidationParameters$IgnoreValidationErrors, com.yahoo.config.model.api.ValidationParameters$FailOnIncompatibleChange, com.yahoo.config.model.api.ValidationParameters$CheckRouting)",
-      "public boolean ignoreValidationErrors()",
-      "public boolean failOnIncompatibleChanges()",
-      "public boolean checkRouting()"
-    ],
-    "fields": []
   }
 }
\ No newline at end of file
diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationMetaData.java b/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationMetaData.java
index c4dee70cd86..480d12b6700 100644
--- a/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationMetaData.java
+++ b/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationMetaData.java
@@ -2,14 +2,16 @@
 package com.yahoo.config.application.api;
 
 import com.yahoo.config.provision.ApplicationId;
-import com.yahoo.config.provision.ApplicationName;
-import com.yahoo.config.provision.InstanceName;
-import com.yahoo.config.provision.TenantName;
-import com.yahoo.slime.*;
-
+import com.yahoo.slime.Cursor;
+import com.yahoo.slime.Inspector;
+import com.yahoo.slime.JsonDecoder;
+import com.yahoo.slime.JsonFormat;
+import com.yahoo.slime.Slime;
 import com.yahoo.text.Utf8;
 
-import java.io.*;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
 
 /**
  * Metadata about an application package.
@@ -27,32 +29,6 @@ public class ApplicationMetaData {
     private final long generation;
     private final long previousActiveGeneration;
 
-    // TODO: Remove after September 2019
-    public ApplicationMetaData(File appDir,
-                               String deployedByUser,
-                               String deployedFromDir,
-                               Long deployTimestamp,
-                               boolean internalRedeploy,
-                               String checksum,
-                               Long generation,
-                               long previousActiveGeneration) {
-        this(deployedByUser, deployedFromDir, deployTimestamp, internalRedeploy,
-             appDir.getName(), checksum, generation, previousActiveGeneration);
-    }
-
-    // TODO: Remove after September 2019
-    public ApplicationMetaData(String deployedByUser, String deployedFromDir, Long deployTimestamp, boolean internalRedeploy,
-                               String applicationName, String checksum, Long generation, long previousActiveGeneration) {
-        this(deployedByUser,
-             deployedFromDir,
-             deployTimestamp,
-             internalRedeploy,
-             ApplicationId.from(TenantName.defaultName(), ApplicationName.from(applicationName), InstanceName.from("default")),
-             checksum,
-             generation,
-             previousActiveGeneration);
-    }
-
     public ApplicationMetaData(String deployedByUser, String deployedFromDir, Long deployTimestamp, boolean internalRedeploy,
                                ApplicationId applicationId, String checksum, Long generation, long previousActiveGeneration) {
         this.deployedByUser = deployedByUser;
@@ -66,15 +42,6 @@ public class ApplicationMetaData {
     }
 
     /**
-     * Gets the name of the application (name of the directory from which application was deployed.
-     * Will return null if a problem occurred while getting metadata
-     *
-     * @return application name
-     */
-    // TODO: Remove after September 2019
-    public String getApplicationName() { return applicationId.application().toString(); }
-
-    /**
      * Gets the user who deployed the application.
      * Will return null if a problem occurred while getting metadata
      *
@@ -117,10 +84,6 @@ public class ApplicationMetaData {
     public boolean isInternalRedeploy() { return internalRedeploy; }
 
     /** Returns an md5 hash of the contents of the application package */
-    // TODO: Remove after September 2019
-    public String getCheckSum() { return checksum; }
-
-    /** Returns an md5 hash of the contents of the application package */
     public String getChecksum() { return checksum; }
 
     /** Returns the previously active generation at the point when this application was created. */
@@ -140,18 +103,11 @@ public class ApplicationMetaData {
             Inspector deploy = root.field("deploy");
             Inspector app = root.field("application");
 
-            // TODO: Simplify to just ApplicationId.fromSerializedForm(app.field("id").asString()) after September 2019
-            ApplicationId applicationId = app.field("id").valid() ?
-                                          ApplicationId.fromSerializedForm(app.field("id").asString()) :
-                                          ApplicationId.from(TenantName.defaultName(),
-                                                             ApplicationName.from(app.field("name").asString()),
-                                                             InstanceName.from("default"));
-
             return new ApplicationMetaData(deploy.field("user").asString(),
                                            deploy.field("from").asString(),
                                            deploy.field("timestamp").asLong(),
                                            booleanField("internalRedeploy", false, deploy),
-                                           applicationId,
+                                           ApplicationId.fromSerializedForm(app.field("id").asString()),
                                            app.field("checksum").asString(),
                                            app.field("generation").asLong(),
                                            app.field("previousActiveGeneration").asLong());
@@ -170,7 +126,6 @@ public class ApplicationMetaData {
         deploy.setBool("internalRedeploy", internalRedeploy);
         Cursor app = meta.setObject("application");
         app.setString("id", applicationId.serializedForm());
-        app.setString("name", applicationId.application().value()); // TODO: Remove after September 2019
         app.setString("checksum", checksum);
         app.setLong("generation", generation);
         app.setLong("previousActiveGeneration", previousActiveGeneration);
@@ -188,7 +143,7 @@ public class ApplicationMetaData {
         ByteArrayOutputStream baos = new ByteArrayOutputStream();
         try {
             new JsonFormat(false).encode(baos, slime);
-            return baos.toString("UTF-8");
+            return baos.toString(StandardCharsets.UTF_8);
         } catch (IOException e) {
             throw new RuntimeException("Unable to encode metadata", e);
         }
diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java b/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java
index db3d391d19b..174f9bb54d7 100644
--- a/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java
+++ b/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java
@@ -36,8 +36,6 @@ import java.util.jar.JarFile;
  * The class hides detail as to whether the source is local files or ZooKeeper
  * data in config server.
  *
- * Anyone wanting to access application data should use this interface.
- *
  * @author Vegard Havdal
  */
 public interface ApplicationPackage {
@@ -47,7 +45,8 @@ public interface ApplicationPackage {
     String HOSTS = "hosts.xml";
     String SERVICES = "services.xml";
 
-    Path SEARCH_DEFINITIONS_DIR = Path.fromString("searchdefinitions");
+    Path SCHEMAS_DIR = Path.fromString("schemas");
+    Path SEARCH_DEFINITIONS_DIR = Path.fromString("searchdefinitions"); // Legacy addition to schemas
     String COMPONENT_DIR = "components";
     String SEARCHCHAINS_DIR = "search/chains";
     String DOCPROCCHAINS_DIR = "docproc/chains";
@@ -168,7 +167,7 @@ public interface ApplicationPackage {
     }
 
     /**
-     * Returns inforamtion about a file
+     * Returns information about a file
      *
      * @param relativePath the relative path of the file within this application package.
      * @return information abut the file, returned whether or not the file exists
@@ -206,23 +205,26 @@ public interface ApplicationPackage {
     Reader getRankingExpression(String name);
 
     /**
-     * Returns the name-payload pairs of any sd files under path/searchdefinitions/ in the given jar bundle
-     * @param bundle The jar file, which will be closed afterwards by this method.
-     * @param path For example 'complex/'
+     * Returns the name-payload pairs of any sd files under path/schemas and path/searchdefinitions/
+     * in the given jar bundle.
+     *
+     * @param bundle the jar file, which will be closed afterwards by this method
+     * @param path for example 'complex/'
      * @return map of the SD payloads
      * @throws IOException if it is reading sd files fails
      */
     static Map<String, String> getBundleSdFiles(String path, JarFile bundle) throws IOException {
-        Map<String,String> ret = new LinkedHashMap<>();
+        Map<String, String> schemas = new LinkedHashMap<>();
         for (Enumeration<JarEntry> e = bundle.entries(); e.hasMoreElements();) {
-            JarEntry je=e.nextElement();
-            if (je.getName().startsWith(path+SEARCH_DEFINITIONS_DIR+"/") && je.getName().endsWith(SD_NAME_SUFFIX)) {
-                String contents = IOUtils.readAll(new InputStreamReader(bundle.getInputStream(je)));
-                ret.put(getFileName(je), contents);
+            JarEntry entry = e.nextElement();
+            if ((entry.getName().startsWith(path + SCHEMAS_DIR + "/") || entry.getName().startsWith(path + SEARCH_DEFINITIONS_DIR + "/"))
+                && entry.getName().endsWith(SD_NAME_SUFFIX)) {
+                String contents = IOUtils.readAll(new InputStreamReader(bundle.getInputStream(entry)));
+                schemas.put(getFileName(entry), contents);
             }
         }
         bundle.close();
-        return ret;
+        return schemas;
     }
 
     /**
diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ConfigServerSpec.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ConfigServerSpec.java
index e63e4ce3af6..96e76e46cda 100644
--- a/config-model-api/src/main/java/com/yahoo/config/model/api/ConfigServerSpec.java
+++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ConfigServerSpec.java
@@ -10,8 +10,6 @@ public interface ConfigServerSpec {
 
     String getHostName();
     int getConfigServerPort();
-    // TODO: Remove when latest model version in use is 7.47
-    default int getHttpPort() { return getConfigServerPort() + 1; }
     int getZooKeeperPort();
 
 }
diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/HostProvisioner.java b/config-model-api/src/main/java/com/yahoo/config/model/api/HostProvisioner.java
index bf58000dd36..4edf3c455d0 100644
--- a/config-model-api/src/main/java/com/yahoo/config/model/api/HostProvisioner.java
+++ b/config-model-api/src/main/java/com/yahoo/config/model/api/HostProvisioner.java
@@ -18,15 +18,17 @@ public interface HostProvisioner {
     // TODO: Remove
     HostSpec allocateHost(String alias);
 
+    @Deprecated // TODO: Remove after April 2020
+    List<HostSpec> prepare(ClusterSpec cluster, Capacity capacity, int groups, ProvisionLogger logger);
+
     /**
      * Prepares allocation of a set of hosts with a given type, common id and the amount.
      *
      * @param  cluster the cluster to allocate nodes to
      * @param  capacity the capacity describing the capacity requested
-     * @param  groups the number of groups to divide the nodes into
      * @param  logger a logger to which messages to the deployer may be written
      * @return the specification of the allocated hosts
      */
-    List<HostSpec> prepare(ClusterSpec cluster, Capacity capacity, int groups, ProvisionLogger logger);
+    List<HostSpec> prepare(ClusterSpec cluster, Capacity capacity, ProvisionLogger logger);
 
 }
diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java
index 843bce6de7c..8ebb449ceda 100644
--- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java
+++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java
@@ -6,6 +6,7 @@ import com.yahoo.config.application.api.ApplicationPackage;
 import com.yahoo.config.application.api.DeployLogger;
 import com.yahoo.config.application.api.FileRegistry;
 import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.AthenzDomain;
 import com.yahoo.config.provision.HostName;
 import com.yahoo.config.provision.Zone;
 
@@ -26,12 +27,16 @@ public interface ModelContext {
     Optional<Model> previousModel();
     Optional<ApplicationPackage> permanentApplicationPackage();
     Optional<HostProvisioner> hostProvisioner();
+    Provisioned provisioned();
     DeployLogger deployLogger();
     ConfigDefinitionRepo configDefinitionRepo();
     FileRegistry getFileRegistry();
     Properties properties();
     default Optional<File> appDir() { return Optional.empty();}
 
+    /** The Docker image repo we want to use for images for this deployment (optional, will use default if empty) */
+    default Optional<String> wantedDockerImageRepository() { return Optional.empty(); }
+
     /** The Vespa version this model is built for */
     Version modelVespaVersion();
 
@@ -51,15 +56,40 @@ public interface ModelContext {
         Set<ContainerEndpoint> endpoints();
         boolean isBootstrap();
         boolean isFirstTimeDeployment();
-        // TODO: Remove when Vespa 7.112 is the oldest config model in use
+
+        // TODO: Only needed for LbServicesProducerTest
         default boolean useDedicatedNodeForLogserver() { return true; }
+
+        // TODO Revisit in May or June 2020
         boolean useAdaptiveDispatch();
-        // TODO: Remove temporary default implementations
+
+        // TODO: Remove after April 2020
         default Optional<TlsSecrets> tlsSecrets() { return Optional.empty(); }
+
         default Optional<EndpointCertificateSecrets> endpointCertificateSecrets() { return Optional.empty(); }
+
+        // TODO Revisit in May or June 2020
         double defaultTermwiseLimit();
+
+        // TODO Revisit in May or June 2020
+        double defaultSoftStartSeconds();
+
+        // TODO Revisit in May or June 2020
+        double defaultTopKProbability();
+
         // TODO: Remove once there are no Vespa versions below 7.170
         boolean useBucketSpaceMetric();
+
+        default boolean useNewAthenzFilter() { return true; } // TODO bjorncs: Remove after end of April
+
+        // TODO: Remove after April 2020
+        default boolean usePhraseSegmenting() { return false; }
+
+        default String proxyProtocol() { return "https-only"; }
+        default Optional<AthenzDomain> athenzDomain() { return Optional.empty(); }
+
+        // TODO(mpolden): Remove after May 2020
+        default boolean useDedicatedNodesWhenUnspecified() { return true; }
     }
 
 }
diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/Provisioned.java b/config-model-api/src/main/java/com/yahoo/config/model/api/Provisioned.java
new file mode 100644
index 00000000000..e4201836e57
--- /dev/null
+++ b/config-model-api/src/main/java/com/yahoo/config/model/api/Provisioned.java
@@ -0,0 +1,28 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.config.model.api;
+
+import com.yahoo.config.provision.Capacity;
+import com.yahoo.config.provision.ClusterSpec;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * A recording of the capacity requests issued during a model build.
+ * Requests are only recorded here if provision requests are issued to the node repo.
+ *
+ * @author bratseth
+ */
+public class Provisioned {
+
+    private final Map<ClusterSpec.Id, Capacity> provisioned = new HashMap<>();
+
+    public void add(ClusterSpec.Id id, Capacity capacity) {
+        provisioned.put(id, capacity);
+    }
+
+    /** Returns an unmodifiable map of all the provision requests recorded during build of the model this belongs to */
+    public Map<ClusterSpec.Id, Capacity> all() { return Collections.unmodifiableMap(provisioned); }
+
+}
diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/SuperModel.java b/config-model-api/src/main/java/com/yahoo/config/model/api/SuperModel.java
index 1735e08c930..50f1ea2336d 100644
--- a/config-model-api/src/main/java/com/yahoo/config/model/api/SuperModel.java
+++ b/config-model-api/src/main/java/com/yahoo/config/model/api/SuperModel.java
@@ -20,13 +20,15 @@ import java.util.Set;
 public class SuperModel {
 
     private final Map<ApplicationId, ApplicationInfo> models;
+    private final boolean complete;
 
     public SuperModel() {
-        this.models = Collections.emptyMap();
+        this(Collections.emptyMap(), false);
     }
 
-    public SuperModel(Map<ApplicationId, ApplicationInfo> models) {
+    public SuperModel(Map<ApplicationId, ApplicationInfo> models, boolean complete) {
         this.models = models;
+        this.complete = complete;
     }
 
     public Map<TenantName, Set<ApplicationInfo>> getModelsPerTenant() {
@@ -45,6 +47,8 @@ public class SuperModel {
         return ImmutableMap.copyOf(models);
     }
 
+    public boolean isComplete() { return complete; }
+
     public List<ApplicationInfo> getAllApplicationInfos() {
         return new ArrayList<>(models.values());
     }
@@ -57,17 +61,19 @@ public class SuperModel {
     public SuperModel cloneAndSetApplication(ApplicationInfo application) {
         Map<ApplicationId, ApplicationInfo> newModels = cloneModels(models);
         newModels.put(application.getApplicationId(), application);
-
-        return new SuperModel(newModels);
+        return new SuperModel(newModels, complete);
     }
 
     public SuperModel cloneAndRemoveApplication(ApplicationId applicationId) {
         Map<ApplicationId, ApplicationInfo> newModels = cloneModels(models);
         newModels.remove(applicationId);
-
-        return new SuperModel(newModels);
+        return new SuperModel(newModels, complete);
     }
 
+    public SuperModel cloneAsComplete() { return new SuperModel(models, true); }
+
+    public Set<ApplicationId> getApplicationIds() { return models.keySet(); }
+
     private static Map<ApplicationId, ApplicationInfo> cloneModels(Map<ApplicationId, ApplicationInfo> models) {
         return new LinkedHashMap<>(models);
     }
diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/SuperModelListener.java b/config-model-api/src/main/java/com/yahoo/config/model/api/SuperModelListener.java
index 497c38af908..930c3c94907 100644
--- a/config-model-api/src/main/java/com/yahoo/config/model/api/SuperModelListener.java
+++ b/config-model-api/src/main/java/com/yahoo/config/model/api/SuperModelListener.java
@@ -7,6 +7,7 @@ import com.yahoo.config.provision.ApplicationId;
  * Interface for those wanting to be notified about changes to the SuperModel.
  */
 public interface SuperModelListener {
+
     /**
      * Application has been activated: Either deployed the first time,
      * internally redeployed, or externally triggered redeploy.
@@ -17,4 +18,12 @@ public interface SuperModelListener {
      * Application has been removed.
      */
     void applicationRemoved(SuperModel superModel, ApplicationId id);
+
+    /**
+     * Invoked once all applications that were supposed to be deployed on bootstrap
+     * have been activated (and the respective {@link #applicationActivated(SuperModel, ApplicationInfo)
+     * applicationActivated} have been invoked). The SuperModel is then said to be "complete".
+     */
+    void notifyOfCompleteness(SuperModel superModel);
+
 }
diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/package-info.java b/config-model-api/src/main/java/com/yahoo/config/model/api/package-info.java
index a3478026520..689e2524dde 100644
--- a/config-model-api/src/main/java/com/yahoo/config/model/api/package-info.java
+++ b/config-model-api/src/main/java/com/yahoo/config/model/api/package-info.java
@@ -1,6 +1,5 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 @ExportPackage
-@PublicApi // Not really "public", only annotated as such to enable the ABI checker plugin
 package com.yahoo.config.model.api;
 
 import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java b/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java
index 5bf103d1836..32d903786dc 100644
--- a/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java
+++ b/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java
@@ -1002,14 +1002,14 @@ public class DeploymentSpecTest {
     public void notificationsDefault() {
         StringReader r = new StringReader(
                 "<deployment version='1.0'>" +
-                "   <notifications when=\"failing-commit\">" +
-                "      <email role=\"author\"/>" +
+                "   <notifications>" +
+                "      <email role=\"author\" when=\"failing\"/>" +
                 "      <email address=\"mary@dev\"/>" +
                 "   </notifications>" +
                 "   <instance id='instance1'>" +
                 "      <notifications when=\"failing\">" +
                 "         <email role=\"author\"/>" +
-                "         <email address=\"john@operator\"/>" +
+                "         <email address=\"john@operator\" when=\"failing-commit\"/>" +
                 "      </notifications>" +
                 "   </instance>" +
                 "   <instance id='instance2'>" +
@@ -1020,9 +1020,13 @@ public class DeploymentSpecTest {
         DeploymentSpec spec = DeploymentSpec.fromXml(r);
         DeploymentInstanceSpec instance1 = spec.requireInstance("instance1");
         assertEquals(Set.of(author), instance1.notifications().emailRolesFor(failing));
-        assertEquals(Set.of("john@operator"), instance1.notifications().emailAddressesFor(failing));
+        assertEquals(Set.of(), instance1.notifications().emailAddressesFor(failing));
+        assertEquals(Set.of(author), instance1.notifications().emailRolesFor(failingCommit));
+        assertEquals(Set.of("john@operator"), instance1.notifications().emailAddressesFor(failingCommit));
 
         DeploymentInstanceSpec instance2 = spec.requireInstance("instance2");
+        assertEquals(Set.of(author), instance2.notifications().emailRolesFor(failing));
+        assertEquals(Set.of(), instance2.notifications().emailAddressesFor(failing));
         assertEquals(Set.of(author), instance2.notifications().emailRolesFor(failingCommit));
         assertEquals(Set.of("mary@dev"), instance2.notifications().emailAddressesFor(failingCommit));
     }
diff --git a/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecWithoutInstanceTest.java b/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecWithoutInstanceTest.java
index 6a815a467f5..4a7ef7b43f7 100644
--- a/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecWithoutInstanceTest.java
+++ b/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecWithoutInstanceTest.java
@@ -318,6 +318,7 @@ public class DeploymentSpecWithoutInstanceTest {
 
     @Test
     public void testEmpty() {
+        assertEquals(DeploymentSpec.empty, DeploymentSpec.fromXml("<deployment version='1.0'>\n</deployment>"));
         assertEquals(0, DeploymentSpec.empty.steps().size());
         assertTrue(DeploymentSpec.empty.athenzDomain().isEmpty());
         assertTrue(DeploymentSpec.empty.athenzService().isEmpty());
diff --git a/config-model/pom.xml b/config-model/pom.xml
index 25b733985f5..33f6657561c 100644
--- a/config-model/pom.xml
+++ b/config-model/pom.xml
@@ -105,6 +105,10 @@
           <groupId>org.apache.httpcomponents</groupId>
           <artifactId>httpclient</artifactId>
         </exclusion>
+        <exclusion>
+          <groupId>org.apache.velocity</groupId>
+          <artifactId>velocity</artifactId>
+        </exclusion>
       </exclusions>
     </dependency>
     <dependency>
diff --git a/config-model/src/main/java/com/yahoo/config/model/builder/xml/XmlHelper.java b/config-model/src/main/java/com/yahoo/config/model/builder/xml/XmlHelper.java
index a9677d4b34c..4cd0c1815dd 100644
--- a/config-model/src/main/java/com/yahoo/config/model/builder/xml/XmlHelper.java
+++ b/config-model/src/main/java/com/yahoo/config/model/builder/xml/XmlHelper.java
@@ -133,6 +133,11 @@ public final class XmlHelper {
         return Optional.ofNullable(element.getAttribute(name)).filter(s -> !s.isEmpty());
     }
 
+    public static Optional<Element> getOptionalChild(Element parent, String childName) {
+        return Optional.ofNullable(XML.getChild(parent, childName));
+
+    }
+
     public static Optional<String> getOptionalChildValue(Element parent, String childName) {
         Element child = XML.getChild(parent, childName);
         if (child == null) return Optional.empty();
diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/ConfigDefinitionStore.java b/config-model/src/main/java/com/yahoo/config/model/deploy/ConfigDefinitionStore.java
index 140cb3001a0..b9ad830090c 100644
--- a/config-model/src/main/java/com/yahoo/config/model/deploy/ConfigDefinitionStore.java
+++ b/config-model/src/main/java/com/yahoo/config/model/deploy/ConfigDefinitionStore.java
@@ -8,7 +8,6 @@ import java.util.Optional;
 
 /**
  * @author Ulf Lilleengen
- * @since 5.1
  */
 public interface ConfigDefinitionStore {
 
diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java
index 7c9e930bb4f..3fb7ba6bc3a 100644
--- a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java
+++ b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java
@@ -12,10 +12,11 @@ import com.yahoo.config.application.api.UnparsedConfigDefinition;
 import com.yahoo.config.application.api.ValidationOverrides;
 import com.yahoo.config.model.api.ConfigDefinitionRepo;
 import com.yahoo.config.model.api.ContainerEndpoint;
+import com.yahoo.config.model.api.EndpointCertificateSecrets;
 import com.yahoo.config.model.api.HostProvisioner;
 import com.yahoo.config.model.api.Model;
 import com.yahoo.config.model.api.ModelContext;
-import com.yahoo.config.model.api.EndpointCertificateSecrets;
+import com.yahoo.config.model.api.Provisioned;
 import com.yahoo.config.model.api.ValidationParameters;
 import com.yahoo.config.model.application.provider.BaseDeployLogger;
 import com.yahoo.config.model.application.provider.MockFileRegistry;
@@ -36,9 +37,8 @@ import com.yahoo.vespa.model.container.search.QueryProfiles;
 import com.yahoo.vespa.model.container.search.QueryProfilesBuilder;
 import com.yahoo.vespa.model.container.search.SemanticRuleBuilder;
 import com.yahoo.vespa.model.container.search.SemanticRules;
-import com.yahoo.vespa.model.search.SearchDefinition;
+import com.yahoo.vespa.model.search.NamedSchema;
 
-import java.io.File;
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.Reader;
@@ -62,11 +62,12 @@ public class DeployState implements ConfigDefinitionStore {
     private final DeployLogger logger;
     private final FileRegistry fileRegistry;
     private final DocumentModel documentModel;
-    private final List<SearchDefinition> searchDefinitions;
+    private final List<NamedSchema> schemas;
     private final ApplicationPackage applicationPackage;
     private final Optional<ConfigDefinitionRepo> configDefinitionRepo;
     private final Optional<ApplicationPackage> permanentApplicationPackage;
     private final Optional<Model> previousModel;
+    private final boolean accessLoggingEnabledByDefault;
     private final ModelContext.Properties properties;
     private final Version vespaVersion;
     private final Set<ContainerEndpoint> endpoints;
@@ -76,8 +77,10 @@ public class DeployState implements ConfigDefinitionStore {
     private final ImportedMlModels importedModels;
     private final ValidationOverrides validationOverrides;
     private final Version wantedNodeVespaVersion;
+    private final Optional<String> wantedDockerImageRepo;
     private final Instant now;
     private final HostProvisioner provisioner;
+    private final Provisioned provisioned;
 
     public static DeployState createTestState() {
         return new Builder().build();
@@ -97,18 +100,21 @@ public class DeployState implements ConfigDefinitionStore {
                         FileRegistry fileRegistry,
                         DeployLogger deployLogger,
                         Optional<HostProvisioner> hostProvisioner,
+                        Provisioned provisioned,
                         ModelContext.Properties properties,
                         Version vespaVersion,
                         Optional<ApplicationPackage> permanentApplicationPackage,
                         Optional<ConfigDefinitionRepo> configDefinitionRepo,
-                        java.util.Optional<Model> previousModel,
+                        Optional<Model> previousModel,
                         Set<ContainerEndpoint> endpoints,
                         Collection<MlModelImporter> modelImporters,
                         Zone zone,
                         QueryProfiles queryProfiles,
                         SemanticRules semanticRules,
                         Instant now,
-                        Version wantedNodeVespaVersion) {
+                        Version wantedNodeVespaVersion,
+                        boolean accessLoggingEnabledByDefault,
+                        Optional<String> wantedDockerImageRepo) {
         this.logger = deployLogger;
         this.fileRegistry = fileRegistry;
         this.rankProfileRegistry = rankProfileRegistry;
@@ -116,8 +122,10 @@ public class DeployState implements ConfigDefinitionStore {
         this.properties = properties;
         this.vespaVersion = vespaVersion;
         this.previousModel = previousModel;
+        this.accessLoggingEnabledByDefault = accessLoggingEnabledByDefault;
         this.provisioner = hostProvisioner.orElse(getDefaultModelHostProvisioner(applicationPackage));
-        this.searchDefinitions = searchDocumentModel.getSearchDefinitions();
+        this.provisioned = provisioned;
+        this.schemas = searchDocumentModel.getSchemas();
         this.documentModel = searchDocumentModel.getDocumentModel();
         this.permanentApplicationPackage = permanentApplicationPackage;
         this.configDefinitionRepo = configDefinitionRepo;
@@ -137,6 +145,7 @@ public class DeployState implements ConfigDefinitionStore {
 
         this.wantedNodeVespaVersion = wantedNodeVespaVersion;
         this.now = now;
+        this.wantedDockerImageRepo = wantedDockerImageRepo;
     }
 
     public static HostProvisioner getDefaultModelHostProvisioner(ApplicationPackage applicationPackage) {
@@ -147,6 +156,8 @@ public class DeployState implements ConfigDefinitionStore {
         }
     }
 
+    public Provisioned provisioned() { return provisioned; }
+
     /** Get the global rank profile registry for this application. */
     public final RankProfileRegistry rankProfileRegistry() { return rankProfileRegistry; }
 
@@ -157,9 +168,7 @@ public class DeployState implements ConfigDefinitionStore {
     public final Optional<ConfigDefinition> getConfigDefinition(ConfigDefinitionKey defKey) {
         if (existingConfigDefs == null) {
             existingConfigDefs = new LinkedHashMap<>();
-            if (configDefinitionRepo.isPresent()) {
-                existingConfigDefs.putAll(createLazyMapping(configDefinitionRepo.get()));
-            }
+            configDefinitionRepo.ifPresent(definitionRepo -> existingConfigDefs.putAll(createLazyMapping(definitionRepo)));
             existingConfigDefs.putAll(applicationPackage.getAllExistingConfigDefs());
         }
         if ( ! existingConfigDefs.containsKey(defKey)) return Optional.empty();
@@ -205,8 +214,8 @@ public class DeployState implements ConfigDefinitionStore {
         return applicationPackage;
     }
 
-    public List<SearchDefinition> getSearchDefinitions() {
-        return searchDefinitions;
+    public List<NamedSchema> getSchemas() {
+        return schemas;
     }
 
     public DocumentModel getDocumentModel() {
@@ -217,6 +226,10 @@ public class DeployState implements ConfigDefinitionStore {
         return logger;
     }
 
+    public boolean getAccessLoggingEnabledByDefault() {
+        return accessLoggingEnabledByDefault;
+    }
+
     public FileRegistry getFileRegistry() {
         return fileRegistry;
     }
@@ -253,6 +266,8 @@ public class DeployState implements ConfigDefinitionStore {
 
     public Version getWantedNodeVespaVersion() { return wantedNodeVespaVersion; }
 
+    public Optional<String> getWantedDockerImageRepo() { return wantedDockerImageRepo; }
+
     public Instant now() { return now; }
 
     public Optional<EndpointCertificateSecrets> endpointCertificateSecrets() { return properties.endpointCertificateSecrets(); }
@@ -279,6 +294,7 @@ public class DeployState implements ConfigDefinitionStore {
         private FileRegistry fileRegistry = new MockFileRegistry();
         private DeployLogger logger = new BaseDeployLogger();
         private Optional<HostProvisioner> hostProvisioner = Optional.empty();
+        private Provisioned provisioned = new Provisioned();
         private Optional<ApplicationPackage> permanentApplicationPackage = Optional.empty();
         private ModelContext.Properties properties = new TestProperties();
         private Version version = new Version(1, 0, 0);
@@ -289,6 +305,8 @@ public class DeployState implements ConfigDefinitionStore {
         private Zone zone = Zone.defaultZone();
         private Instant now = Instant.now();
         private Version wantedNodeVespaVersion = Vtag.currentVersion;
+        private boolean accessLoggingEnabledByDefault = true;
+        private Optional<String> wantedDockerImageRepo = Optional.empty();
 
         public Builder applicationPackage(ApplicationPackage applicationPackage) {
             this.applicationPackage = applicationPackage;
@@ -310,6 +328,11 @@ public class DeployState implements ConfigDefinitionStore {
             return this;
         }
 
+        public Builder provisioned(Provisioned provisioned) {
+            this.provisioned = provisioned;
+            return this;
+        }
+
         public Builder permanentApplicationPackage(Optional<ApplicationPackage> permanentApplicationPackage) {
             this.permanentApplicationPackage = permanentApplicationPackage;
             return this;
@@ -360,6 +383,20 @@ public class DeployState implements ConfigDefinitionStore {
             return this;
         }
 
+        public Builder wantedDockerImageRepo(Optional<String> dockerImageRepo) {
+            this.wantedDockerImageRepo = dockerImageRepo;
+            return this;
+        }
+
+        /**
+         * Whether access logging is enabled for an application without an accesslog element in services.xml.
+         * True by default.
+         */
+        public Builder accessLoggingEnabledByDefault(boolean accessLoggingEnabledByDefault) {
+            this.accessLoggingEnabledByDefault = accessLoggingEnabledByDefault;
+            return this;
+        }
+
         public DeployState build() {
             return build(new ValidationParameters());
         }
@@ -375,6 +412,7 @@ public class DeployState implements ConfigDefinitionStore {
                                    fileRegistry,
                                    logger,
                                    hostProvisioner,
+                                   provisioned,
                                    properties,
                                    version,
                                    permanentApplicationPackage,
@@ -386,7 +424,9 @@ public class DeployState implements ConfigDefinitionStore {
                                    queryProfiles,
                                    semanticRules,
                                    now,
-                                   wantedNodeVespaVersion);
+                                   wantedNodeVespaVersion,
+                                   accessLoggingEnabledByDefault,
+                                   wantedDockerImageRepo);
         }
 
         private SearchDocumentModel createSearchDocumentModel(RankProfileRegistry rankProfileRegistry,
@@ -399,20 +439,18 @@ public class DeployState implements ConfigDefinitionStore {
             for (NamedReader reader : readers) {
                 try {
                     String readerName = reader.getName();
-                    String searchName = builder.importReader(reader, readerName, logger);
+                    String topLevelName = builder.importReader(reader, readerName, logger);
                     String sdName = stripSuffix(readerName, ApplicationPackage.SD_NAME_SUFFIX);
-                    names.put(searchName, sdName);
-                    if ( ! sdName.equals(searchName)) {
-                        throw new IllegalArgumentException("Search definition file name ('" + sdName + "') and name of " +
-                                                           "search element ('" + searchName +
+                    names.put(topLevelName, sdName);
+                    if ( ! sdName.equals(topLevelName)) {
+                        throw new IllegalArgumentException("Schema definition file name ('" + sdName + "') and name of " +
+                                                           "top level element ('" + topLevelName +
                                                            "') are not equal for file '" + readerName + "'");
                     }
                 } catch (ParseException e) {
-                    throw new IllegalArgumentException("Could not parse search definition file '" +
-                                                       getSearchDefinitionRelativePath(reader.getName()) + "': " + e.getMessage(), e);
+                    throw new IllegalArgumentException("Could not parse sd file '" + reader.getName() + "'", e);
                 } catch (IOException e) {
-                    throw new IllegalArgumentException("Could not read search definition file '" +
-                                                       getSearchDefinitionRelativePath(reader.getName()) + "': " + e.getMessage(), e);
+                    throw new IllegalArgumentException("Could not read sd file '" + reader.getName() + "'", e);
                 } finally {
                     closeIgnoreException(reader.getReader());
                 }
@@ -421,10 +459,6 @@ public class DeployState implements ConfigDefinitionStore {
             return SearchDocumentModel.fromBuilderAndNames(builder, names);
         }
 
-        private String getSearchDefinitionRelativePath(String name) {
-            return ApplicationPackage.SEARCH_DEFINITIONS_DIR + File.separator + name;
-        }
-
         private static String stripSuffix(String nodeName, String postfix) {
             assert (nodeName.endsWith(postfix));
             return nodeName.substring(0, nodeName.length() - postfix.length());
diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/SearchDocumentModel.java b/config-model/src/main/java/com/yahoo/config/model/deploy/SearchDocumentModel.java
index cdd4f6f8e8a..9b9729dddb3 100644
--- a/config-model/src/main/java/com/yahoo/config/model/deploy/SearchDocumentModel.java
+++ b/config-model/src/main/java/com/yahoo/config/model/deploy/SearchDocumentModel.java
@@ -3,7 +3,7 @@ package com.yahoo.config.model.deploy;
 
 import com.yahoo.searchdefinition.SearchBuilder;
 import com.yahoo.vespa.documentmodel.DocumentModel;
-import com.yahoo.vespa.model.search.SearchDefinition;
+import com.yahoo.vespa.model.search.NamedSchema;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -13,16 +13,15 @@ import java.util.Map;
  * Internal helper class to retrieve document model and search definitions.
  *
  * @author Ulf Lilleengen
- * @since 5.1
  */
 public class SearchDocumentModel {
 
     private final DocumentModel documentModel;
-    private final List<SearchDefinition> searchDefinitions;
+    private final List<NamedSchema> schemas;
 
-    public SearchDocumentModel(DocumentModel documentModel, List<SearchDefinition> searchDefinitions) {
+    public SearchDocumentModel(DocumentModel documentModel, List<NamedSchema> schemas) {
         this.documentModel = documentModel;
-        this.searchDefinitions = searchDefinitions;
+        this.schemas = schemas;
 
     }
 
@@ -30,22 +29,22 @@ public class SearchDocumentModel {
         return documentModel;
     }
 
-    public List<SearchDefinition> getSearchDefinitions() {
-        return searchDefinitions;
+    public List<NamedSchema> getSchemas() {
+        return schemas;
     }
 
     public static SearchDocumentModel fromBuilderAndNames(SearchBuilder builder, Map<String, String> names) {
-        List<SearchDefinition> ret = new ArrayList<>();
+        List<NamedSchema> ret = new ArrayList<>();
         for (com.yahoo.searchdefinition.Search search : builder.getSearchList()) {
-            ret.add(new SearchDefinition(names.get(search.getName()), search));
+            ret.add(new NamedSchema(names.get(search.getName()), search));
         }
         return new SearchDocumentModel(builder.getModel(), ret);
     }
 
     public static SearchDocumentModel fromBuilder(SearchBuilder builder) {
-        List<SearchDefinition> ret = new ArrayList<>();
+        List<NamedSchema> ret = new ArrayList<>();
         for (com.yahoo.searchdefinition.Search search : builder.getSearchList()) {
-            ret.add(new SearchDefinition(search.getName(), search));
+            ret.add(new NamedSchema(search.getName(), search));
         }
         return new SearchDocumentModel(builder.getModel(), ret);
     }
diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java
index 9f4d1b09f91..99225beba4f 100644
--- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java
+++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java
@@ -4,10 +4,11 @@ package com.yahoo.config.model.deploy;
 import com.google.common.collect.ImmutableList;
 import com.yahoo.config.model.api.ConfigServerSpec;
 import com.yahoo.config.model.api.ContainerEndpoint;
-import com.yahoo.config.model.api.ModelContext;
 import com.yahoo.config.model.api.EndpointCertificateSecrets;
+import com.yahoo.config.model.api.ModelContext;
 import com.yahoo.config.model.api.TlsSecrets;
 import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.AthenzDomain;
 import com.yahoo.config.provision.HostName;
 import com.yahoo.config.provision.Zone;
 
@@ -39,9 +40,11 @@ public class TestProperties implements ModelContext.Properties {
     private boolean isFirstTimeDeployment = false;
     private boolean useDedicatedNodeForLogserver = false;
     private boolean useAdaptiveDispatch = false;
+    private double topKProbability = 1.0;
     private double defaultTermwiseLimit = 1.0;
+    private double softStartSeconds = 0.0;
     private Optional<EndpointCertificateSecrets> endpointCertificateSecrets = Optional.empty();
-
+    private AthenzDomain athenzDomain;
 
     @Override public boolean multitenant() { return multitenant; }
     @Override public ApplicationId applicationId() { return applicationId; }
@@ -60,13 +63,30 @@ public class TestProperties implements ModelContext.Properties {
     @Override public Optional<EndpointCertificateSecrets> endpointCertificateSecrets() { return endpointCertificateSecrets; }
     @Override public Optional<TlsSecrets> tlsSecrets() { return endpointCertificateSecrets.map(TlsSecrets::new); }
     @Override public double defaultTermwiseLimit() { return defaultTermwiseLimit; }
+
+    @Override
+    public double defaultSoftStartSeconds() {
+        return softStartSeconds;
+    }
+
+    @Override public double defaultTopKProbability() { return topKProbability; }
     @Override public boolean useBucketSpaceMetric() { return true; }
+    @Override public Optional<AthenzDomain> athenzDomain() { return Optional.ofNullable(athenzDomain); }
 
     public TestProperties setDefaultTermwiseLimit(double limit) {
         defaultTermwiseLimit = limit;
         return this;
     }
 
+    public TestProperties setTopKProbability(double probability) {
+        topKProbability = probability;
+        return this;
+    }
+    public TestProperties setSoftStartSeconds(double softStartSeconds) {
+        this.softStartSeconds = softStartSeconds;
+        return this;
+    }
+
     public TestProperties setApplicationId(ApplicationId applicationId) {
         this.applicationId = applicationId;
         return this;
@@ -102,6 +122,16 @@ public class TestProperties implements ModelContext.Properties {
         return this;
     }
 
+    public TestProperties setZone(Zone zone) {
+        this.zone = zone;
+        return this;
+    }
+
+    public TestProperties setAthenzDomain(AthenzDomain domain) {
+        this.athenzDomain = domain;
+        return this;
+    }
+
     public static class Spec implements ConfigServerSpec {
 
         private final String hostName;
diff --git a/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java b/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java
index 48c21f370f4..cce5a7850a0 100644
--- a/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java
+++ b/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java
@@ -7,7 +7,6 @@ import com.yahoo.config.model.ApplicationConfigProducerRoot;
 import com.yahoo.config.model.deploy.DeployState;
 import com.yahoo.config.subscription.ConfigInstanceUtil;
 import com.yahoo.log.LogLevel;
-import com.yahoo.text.Utf8;
 import com.yahoo.vespa.config.ConfigDefinitionKey;
 import com.yahoo.vespa.config.ConfigPayload;
 import com.yahoo.vespa.config.ConfigPayloadBuilder;
@@ -21,14 +20,8 @@ import com.yahoo.vespa.model.admin.Admin;
 import com.yahoo.vespa.model.admin.monitoring.Monitoring;
 import com.yahoo.vespa.model.utils.FreezableMap;
 
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
 import java.io.PrintStream;
 import java.io.Serializable;
-import java.lang.reflect.InvocationTargetException;
-import java.lang.reflect.Method;
-import java.lang.reflect.Type;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.LinkedHashMap;
@@ -291,60 +284,6 @@ public abstract class AbstractConfigProducer<CHILD extends AbstractConfigProduce
 
     public AbstractConfigProducer getParent() { return parent; }
 
-    /**
-     * Writes files that need to be written. The files will usually only be
-     * written when the Vespa model is generated through the deploy-application
-     * script.
-     *
-     * TODO: Make sure all implemented ConfigProducers call createConfig()
-     * instead of getConfig() when implementing this method.
-     */
-    public void writeFiles(File directory) throws java.io.IOException {
-        if (!directory.isDirectory() && !directory.mkdirs()) {
-            throw new java.io.IOException("Cannot create directory: "+ directory);
-        }
-        for (Method m : getClass().getMethods()) {
-            try {
-                ConfigInstance.Builder builder = getBuilderIfIsGetConfig(m);
-                if (builder!=null) {
-                    writeBuilder(directory, m, builder);
-                }
-            } catch (Exception e) {
-                throw new RuntimeException(e);
-            }
-        }
-    }
-
-    private void writeBuilder(File directory, Method m,
-            ConfigInstance.Builder builder) throws IllegalAccessException,
-            InvocationTargetException, InstantiationException,
-            NoSuchMethodException, IOException {
-        m.invoke(this, builder);
-        Class<?> configInstClass = builder.getClass().getEnclosingClass();
-        ConfigInstance inst = (ConfigInstance) configInstClass.getConstructor(builder.getClass()).newInstance(builder);
-        List<String> payloadList = ConfigInstance.serialize(inst);
-        File outfn = new File(directory, ConfigInstance.getDefName(inst.getClass()) + ".MODEL.cfg");
-        FileOutputStream out = new FileOutputStream(outfn);
-        for (String s : payloadList) {
-            out.write(Utf8.toBytes(s));
-            out.write('\n');
-        }
-    }
-
-    /**
-     * New Builder instance if m is getConfig(SomeConfig.Builder), or null
-     */
-    private ConfigInstance.Builder getBuilderIfIsGetConfig(Method m) throws ReflectiveOperationException {
-        if (!"getConfig".equals(m.getName())) return null;
-        Type[] params = m.getParameterTypes();
-        if (params.length!=1) return null;
-        Type param = params[0];
-        if (!(param instanceof Class)) return null;
-        Class<?> paramClass = (Class<?>) param;
-        if  (!(ConfigInstance.Builder.class.isAssignableFrom(paramClass))) return null;
-        return (ConfigInstance.Builder) paramClass.getDeclaredConstructor().newInstance();
-    }
-
     public void dump(PrintStream out) {
         for (ConfigProducer c : getChildren().values()) {
             out.println("id: " + c.getConfigId());
diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/HostsXmlProvisioner.java b/config-model/src/main/java/com/yahoo/config/model/provision/HostsXmlProvisioner.java
index f909f3864da..201b69c1aae 100644
--- a/config-model/src/main/java/com/yahoo/config/model/provision/HostsXmlProvisioner.java
+++ b/config-model/src/main/java/com/yahoo/config/model/provision/HostsXmlProvisioner.java
@@ -45,10 +45,16 @@ public class HostsXmlProvisioner implements HostProvisioner {
     }
 
     @Override
+    @Deprecated // TODO: Remove after April 2020
     public List<HostSpec> prepare(ClusterSpec cluster, Capacity quantity, int groups, ProvisionLogger logger) {
         throw new UnsupportedOperationException("Prepare on an XML host provisioner is not supported");
     }
 
+    @Override
+    public List<HostSpec> prepare(ClusterSpec cluster, Capacity quantity, ProvisionLogger logger) {
+        throw new UnsupportedOperationException("Prepare on an XML host provisioner is not supported");
+    }
+
     private HostSpec host2HostSpec(Host host) {
         return new HostSpec(host.hostname(), host.aliases());
     }
diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java b/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java
index bfbe2eaddb3..8706bb44ded 100644
--- a/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java
+++ b/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java
@@ -4,8 +4,10 @@ package com.yahoo.config.model.provision;
 import com.yahoo.collections.ListMap;
 import com.yahoo.collections.Pair;
 import com.yahoo.config.model.api.HostProvisioner;
+import com.yahoo.config.model.api.Provisioned;
 import com.yahoo.config.provision.Capacity;
 import com.yahoo.config.provision.ClusterMembership;
+import com.yahoo.config.provision.ClusterResources;
 import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.config.provision.Flavor;
 import com.yahoo.config.provision.HostSpec;
@@ -49,7 +51,6 @@ public class InMemoryProvisioner implements HostProvisioner {
 
     /** Free hosts of each resource size */
     private final ListMap<NodeResources, Host> freeNodes = new ListMap<>();
-    private final Map<String, HostSpec> legacyMapping = new LinkedHashMap<>();
     private final Map<ClusterSpec, List<HostSpec>> allocations = new LinkedHashMap<>();
 
     /** Indexes must be unique across all groups in a cluster */
@@ -58,6 +59,10 @@ public class InMemoryProvisioner implements HostProvisioner {
     /** Use this index as start index for all clusters */
     private final int startIndexForClusters;
 
+    private final boolean useMaxResources;
+
+    private Provisioned provisioned = new Provisioned();
+
     /** Creates this with a number of nodes with resources 1, 3, 9, 1 */
     public InMemoryProvisioner(int nodeCount) {
         this(nodeCount, defaultResources);
@@ -65,27 +70,31 @@ public class InMemoryProvisioner implements HostProvisioner {
 
     /** Creates this with a number of nodes with given resources */
     public InMemoryProvisioner(int nodeCount, NodeResources resources) {
-        this(Map.of(resources, createHostInstances(nodeCount)), true, 0);
+        this(Map.of(resources, createHostInstances(nodeCount)), true, false, 0);
     }
 
     /** Creates this with a set of host names of the flavor 'default' */
     public InMemoryProvisioner(boolean failOnOutOfCapacity, String... hosts) {
-        this(Map.of(defaultResources, toHostInstances(hosts)), failOnOutOfCapacity, 0);
+        this(Map.of(defaultResources, toHostInstances(hosts)), failOnOutOfCapacity, false, 0);
     }
 
     /** Creates this with a set of hosts of the flavor 'default' */
     public InMemoryProvisioner(Hosts hosts, boolean failOnOutOfCapacity, String ... retiredHostNames) {
-        this(Map.of(defaultResources, hosts.asCollection()), failOnOutOfCapacity, 0, retiredHostNames);
+        this(Map.of(defaultResources, hosts.asCollection()), failOnOutOfCapacity, false, 0, retiredHostNames);
     }
 
     /** Creates this with a set of hosts of the flavor 'default' */
     public InMemoryProvisioner(Hosts hosts, boolean failOnOutOfCapacity, int startIndexForClusters, String ... retiredHostNames) {
-        this(Map.of(defaultResources, hosts.asCollection()), failOnOutOfCapacity, startIndexForClusters, retiredHostNames);
+        this(Map.of(defaultResources, hosts.asCollection()), failOnOutOfCapacity, false, startIndexForClusters, retiredHostNames);
     }
 
-    public InMemoryProvisioner(Map<NodeResources, Collection<Host>> hosts, boolean failOnOutOfCapacity,
-                               int startIndexForClusters, String ... retiredHostNames) {
+    public InMemoryProvisioner(Map<NodeResources, Collection<Host>> hosts,
+                               boolean failOnOutOfCapacity,
+                               boolean useMaxResources,
+                               int startIndexForClusters,
+                               String ... retiredHostNames) {
         this.failOnOutOfCapacity = failOnOutOfCapacity;
+        this.useMaxResources = useMaxResources;
         for (Map.Entry<NodeResources, Collection<Host>> hostsWithResources : hosts.entrySet())
             for (Host host : hostsWithResources.getValue())
                 freeNodes.put(hostsWithResources.getKey(), host);
@@ -106,44 +115,52 @@ public class InMemoryProvisioner implements HostProvisioner {
 
     @Override
     public HostSpec allocateHost(String alias) {
-        if (legacyMapping.containsKey(alias)) return legacyMapping.get(alias);
         List<Host> defaultHosts = freeNodes.get(defaultResources);
         if (defaultHosts.isEmpty()) throw new IllegalArgumentException("No more hosts with default resources available");
         Host newHost = freeNodes.removeValue(defaultResources, 0);
-        HostSpec hostSpec = new HostSpec(newHost.hostname(), newHost.aliases(), newHost.flavor(), Optional.empty(), newHost.version());
-        legacyMapping.put(alias, hostSpec);
-        return hostSpec;
+        // Note: Always returns HostSpec with empty dockerImageRepo, which is OK since this method is never used when docker image repo is set
+        return new HostSpec(newHost.hostname(), newHost.aliases(), newHost.flavor(), Optional.empty(), newHost.version(), Optional.empty());
     }
 
     @Override
+    @Deprecated // TODO: Remove after April 2020
     public List<HostSpec> prepare(ClusterSpec cluster, Capacity requestedCapacity, int groups, ProvisionLogger logger) {
-        if (cluster.group().isPresent() && groups > 1)
+        return prepare(cluster, requestedCapacity.withGroups(groups), logger);
+    }
+
+    @Override
+    public List<HostSpec> prepare(ClusterSpec cluster, Capacity requested, ProvisionLogger logger) {
+        provisioned.add(cluster.id(), requested);
+        if (useMaxResources)
+            return prepare(cluster, requested.maxResources(), requested.isRequired(), requested.canFail());
+        else
+            return prepare(cluster, requested.minResources(), requested.isRequired(), requested.canFail());
+    }
+
+    public List<HostSpec> prepare(ClusterSpec cluster, ClusterResources requested, boolean required, boolean canFail) {
+        if (cluster.group().isPresent() && requested.groups() > 1)
             throw new IllegalArgumentException("Cannot both be specifying a group and ask for groups to be created");
-        if (requestedCapacity.nodeCount() % groups != 0)
-            throw new IllegalArgumentException("Requested " + requestedCapacity.nodeCount() + " nodes in " +
-                                               groups + " groups, but the node count is not divisible into this number of groups");
 
-        int capacity = failOnOutOfCapacity || requestedCapacity.isRequired() 
-                       ? requestedCapacity.nodeCount() 
-                       : Math.min(requestedCapacity.nodeCount(), freeNodes.get(defaultResources).size() + totalAllocatedTo(cluster));
-        if (groups > capacity)
-            groups = capacity;
+        int capacity = failOnOutOfCapacity || required
+                       ? requested.nodes()
+                       : Math.min(requested.nodes(), freeNodes.get(defaultResources).size() + totalAllocatedTo(cluster));
+        int groups = requested.groups() > capacity ? capacity : requested.groups();
 
         List<HostSpec> allocation = new ArrayList<>();
         if (groups == 1) {
             allocation.addAll(allocateHostGroup(cluster.with(Optional.of(ClusterSpec.Group.from(0))),
-                                                requestedCapacity.nodeResources(),
+                                                requested.nodeResources(),
                                                 capacity,
                                                 startIndexForClusters,
-                                                requestedCapacity.canFail()));
+                                                canFail));
         }
         else {
             for (int i = 0; i < groups; i++) {
                 allocation.addAll(allocateHostGroup(cluster.with(Optional.of(ClusterSpec.Group.from(i))),
-                                                    requestedCapacity.nodeResources(),
+                                                    requested.nodeResources(),
                                                     capacity / groups,
                                                     allocation.size(),
-                                                    requestedCapacity.canFail()));
+                                                    canFail));
             }
         }
         for (ListIterator<HostSpec> i = allocation.listIterator(); i.hasNext(); ) {
@@ -154,15 +171,24 @@ public class InMemoryProvisioner implements HostProvisioner {
         return allocation;
     }
 
+    /** Create a new provisioned instance to record provision requests to this and returns it */
+    public Provisioned startProvisionedRecording() {
+        provisioned = new Provisioned();
+        return provisioned;
+    }
+
     private HostSpec retire(HostSpec host) {
         return new HostSpec(host.hostname(),
                             host.aliases(),
                             host.flavor(),
                             Optional.of(host.membership().get().retire()),
-                            host.version());
+                            host.version(),
+                            Optional.empty(),
+                            Optional.empty(),
+                            host.dockerImageRepo());
     }
 
-    private List<HostSpec> allocateHostGroup(ClusterSpec clusterGroup, Optional<NodeResources> requestedResources,
+    private List<HostSpec> allocateHostGroup(ClusterSpec clusterGroup, NodeResources requestedResources,
                                              int nodesInGroup, int startIndex, boolean canFail) {
         List<HostSpec> allocation = allocations.getOrDefault(clusterGroup, new ArrayList<>());
         allocations.put(clusterGroup, allocation);
@@ -170,8 +196,8 @@ public class InMemoryProvisioner implements HostProvisioner {
         // Check if the current allocations are compatible with the new request
         for (int i = allocation.size() - 1; i >= 0; i--) {
             Optional<NodeResources> currentResources = allocation.get(0).flavor().map(Flavor::resources);
-            if (currentResources.isEmpty() || requestedResources.isEmpty()) continue;
-            if (!currentResources.get().compatibleWith(requestedResources.get())) {
+            if (currentResources.isEmpty() || requestedResources == NodeResources.unspecified) continue;
+            if (!currentResources.get().compatibleWith(requestedResources)) {
                 HostSpec removed = allocation.remove(i);
                 freeNodes.put(currentResources.get(), new Host(removed.hostname())); // Return the node back to free pool
             }
@@ -182,7 +208,7 @@ public class InMemoryProvisioner implements HostProvisioner {
             // Find the smallest host that can fit the requested requested
             Optional<NodeResources> hostResources = freeNodes.keySet().stream()
                     .sorted(new MemoryDiskCpu())
-                    .filter(resources -> requestedResources.isEmpty() || resources.satisfies(requestedResources.get()))
+                    .filter(resources -> requestedResources == NodeResources.unspecified || resources.satisfies(requestedResources))
                     .findFirst();
             if (hostResources.isEmpty()) {
                 if (canFail)
@@ -195,8 +221,9 @@ public class InMemoryProvisioner implements HostProvisioner {
             if (freeNodes.get(hostResources.get()).isEmpty()) freeNodes.removeAll(hostResources.get());
             ClusterMembership membership = ClusterMembership.from(clusterGroup, nextIndex++);
             allocation.add(new HostSpec(newHost.hostname(), newHost.aliases(),
-                    hostResources.map(Flavor::new), Optional.of(membership),
-                    newHost.version(), Optional.empty(), requestedResources));
+                                        hostResources.map(Flavor::new), Optional.of(membership),
+                                        newHost.version(), Optional.empty(),
+                                        requestedResources == NodeResources.unspecified ? Optional.empty() : Optional.of(requestedResources)));
         }
         nextIndexInCluster.put(new Pair<>(clusterGroup.type(), clusterGroup.id()), nextIndex);
 
diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/SingleNodeProvisioner.java b/config-model/src/main/java/com/yahoo/config/model/provision/SingleNodeProvisioner.java
index 180a16f3c8f..8945223447f 100644
--- a/config-model/src/main/java/com/yahoo/config/model/provision/SingleNodeProvisioner.java
+++ b/config-model/src/main/java/com/yahoo/config/model/provision/SingleNodeProvisioner.java
@@ -30,6 +30,7 @@ public class SingleNodeProvisioner implements HostProvisioner {
         host = new Host(HostName.getLocalhost());
         this.hostSpec = new HostSpec(host.hostname(), host.aliases());
     }
+
     public SingleNodeProvisioner(Flavor flavor) {
         host = new Host(HostName.getLocalhost());
         this.hostSpec = new HostSpec(host.hostname(), host.aliases(), flavor);
@@ -41,7 +42,14 @@ public class SingleNodeProvisioner implements HostProvisioner {
     }
 
     @Override
-    public List<HostSpec> prepare(ClusterSpec cluster, Capacity capacity, int groups, ProvisionLogger logger) { // TODO: This should fail if capacity requested is more than 1
+    @Deprecated // TODO: Remove after April 2020
+    public List<HostSpec> prepare(ClusterSpec cluster, Capacity capacity, int groups, ProvisionLogger logger) {
+        return prepare(cluster, capacity.withGroups(groups), logger);
+    }
+
+    @Override
+    public List<HostSpec> prepare(ClusterSpec cluster, Capacity capacity, ProvisionLogger logger) {
+        // TODO: This should fail if capacity requested is more than 1
         List<HostSpec> hosts = new ArrayList<>();
         hosts.add(new HostSpec(host.hostname(), host.aliases(), ClusterMembership.from(cluster, counter++)));
         return hosts;
diff --git a/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java
index eb61bda83a6..10649df88e1 100644
--- a/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java
+++ b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java
@@ -55,23 +55,23 @@ public class MockApplicationPackage implements ApplicationPackage {
     private final File root;
     private final String hostsS;
     private final String servicesS;
-    private final List<String> searchDefinitions;
-    private final String searchDefinitionDir;
+    private final List<String> schemas;
+    private final String schemaDir;
     private final Optional<String> deploymentSpec;
     private final Optional<String> validationOverrides;
     private final boolean failOnValidateXml;
     private final QueryProfileRegistry queryProfileRegistry;
     private final ApplicationMetaData applicationMetaData;
 
-    protected MockApplicationPackage(File root, String hosts, String services, List<String> searchDefinitions,
-                                     String searchDefinitionDir,
+    protected MockApplicationPackage(File root, String hosts, String services, List<String> schemas,
+                                     String schemaDir,
                                      String deploymentSpec, String validationOverrides, boolean failOnValidateXml,
                                      String queryProfile, String queryProfileType) {
         this.root = root;
         this.hostsS = hosts;
         this.servicesS = services;
-        this.searchDefinitions = searchDefinitions;
-        this.searchDefinitionDir = searchDefinitionDir;
+        this.schemas = schemas;
+        this.schemaDir = schemaDir;
         this.deploymentSpec = Optional.ofNullable(deploymentSpec);
         this.validationOverrides = Optional.ofNullable(validationOverrides);
         this.failOnValidateXml = failOnValidateXml;
@@ -108,7 +108,7 @@ public class MockApplicationPackage implements ApplicationPackage {
 
     @Override
     public Reader getHosts() {
-        if (hostsS==null) return null;
+        if (hostsS == null) return null;
         return new StringReader(hostsS);
     }
 
@@ -118,7 +118,7 @@ public class MockApplicationPackage implements ApplicationPackage {
         SearchBuilder searchBuilder = new SearchBuilder(this,
                                                         new RankProfileRegistry(),
                                                         queryProfileRegistry);
-        for (String sd : searchDefinitions) {
+        for (String sd : schemas) {
             try  {
                 String name = searchBuilder.importString(sd);
                 readers.add(new NamedReader(name + ApplicationPackage.SD_NAME_SUFFIX, new StringReader(sd)));
@@ -184,7 +184,7 @@ public class MockApplicationPackage implements ApplicationPackage {
 
     @Override
     public Reader getRankingExpression(String name) {
-        File expressionFile = new File(searchDefinitionDir, name);
+        File expressionFile = new File(schemaDir, name);
         try {
             return IOUtils.createReader(expressionFile, "utf-8");
         }
@@ -200,9 +200,9 @@ public class MockApplicationPackage implements ApplicationPackage {
 
     public static ApplicationPackage fromSearchDefinitionDirectory(String dir) {
         return new MockApplicationPackage.Builder()
-                .withEmptyHosts()
-                .withEmptyServices()
-                .withSearchDefinitionDir(dir).build();
+                       .withEmptyHosts()
+                       .withEmptyServices()
+                       .withSchemaDir(dir).build();
     }
 
     public static class Builder {
@@ -210,8 +210,8 @@ public class MockApplicationPackage implements ApplicationPackage {
         private File root = new File("nonexisting");
         private String hosts = null;
         private String services = null;
-        private List<String> searchDefinitions = Collections.emptyList();
-        private String searchDefinitionDir = null;
+        private List<String> schemas = Collections.emptyList();
+        private String schemaDir = null;
         private String deploymentSpec = null;
         private String validationOverrides = null;
         private boolean failOnValidateXml = false;
@@ -245,17 +245,17 @@ public class MockApplicationPackage implements ApplicationPackage {
         }
 
         public Builder withSearchDefinition(String searchDefinition) {
-            this.searchDefinitions = Collections.singletonList(searchDefinition);
+            this.schemas = Collections.singletonList(searchDefinition);
             return this;
         }
 
-        public Builder withSearchDefinitions(List<String> searchDefinition) {
-            this.searchDefinitions = Collections.unmodifiableList(searchDefinition);
+        public Builder withSchemas(List<String> searchDefinition) {
+            this.schemas = Collections.unmodifiableList(searchDefinition);
             return this;
         }
 
-        public Builder withSearchDefinitionDir(String searchDefinitionDir) {
-            this.searchDefinitionDir = searchDefinitionDir;
+        public Builder withSchemaDir(String schemaDir) {
+            this.schemaDir = schemaDir;
             return this;
         }
 
@@ -285,7 +285,7 @@ public class MockApplicationPackage implements ApplicationPackage {
         }
 
         public ApplicationPackage build() {
-                return new MockApplicationPackage(root, hosts, services, searchDefinitions, searchDefinitionDir,
+                return new MockApplicationPackage(root, hosts, services, schemas, schemaDir,
                                                   deploymentSpec, validationOverrides, failOnValidateXml,
                                                   queryProfile, queryProfileType);
         }
diff --git a/config-model/src/main/java/com/yahoo/config/model/test/MockRoot.java b/config-model/src/main/java/com/yahoo/config/model/test/MockRoot.java
index 13f271ebe9d..87d6554f691 100644
--- a/config-model/src/main/java/com/yahoo/config/model/test/MockRoot.java
+++ b/config-model/src/main/java/com/yahoo/config/model/test/MockRoot.java
@@ -26,6 +26,7 @@ import java.io.IOException;
 import java.io.StringReader;
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.List;
 import java.util.Set;
 
 
@@ -66,7 +67,7 @@ public class MockRoot extends AbstractConfigProducerRoot {
         super(rootConfigId);
         hostSystem = new HostSystem(this, "hostsystem", deployState.getProvisioner(), deployState.getDeployLogger());
         this.deployState = deployState;
-        fileDistributor = new FileDistributor(deployState.getFileRegistry(), null);
+        fileDistributor = new FileDistributor(deployState.getFileRegistry(), List.of(), deployState.isHosted());
     }
 
     public FileDistributionConfigProducer getFileDistributionConfigProducer() {
diff --git a/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java b/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java
index fc42864f1d0..41a30c4553d 100644
--- a/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java
+++ b/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java
@@ -69,25 +69,37 @@ public final class NewDocumentType extends StructuredDataType implements DataTyp
     private final StructDataType body;
     private final Set<FieldSet> fieldSets = new LinkedHashSet<>();
     private final Set<Name> documentReferences;
+    // Imported fields are virtual and therefore exist outside of the SD's document field definition
+    // block itself. But for features like imported fields in a non-search context (e.g. GC selections)
+    // it is necessary to know that certain identifiers refer to imported fields instead of being unknown
+    // document fields. To achieve this, we track the names of imported fields as part of the document
+    // config itself.
+    private final Set<String> importedFieldNames;
 
     public NewDocumentType(Name name) {
         this(name, emptySet());
     }
 
-    public NewDocumentType(Name name, Set<Name> documentReferences) {
+    public NewDocumentType(Name name, Set<Name> documentReferences, Set<String> importedFieldNames) {
         this(
                 name,
                 new StructDataType(name.getName() + ".header"),
                 new StructDataType(name.getName() + ".body"),
                 new FieldSets(),
-                documentReferences);
+                documentReferences,
+                importedFieldNames);
+    }
+
+    public NewDocumentType(Name name, Set<Name> documentReferences) {
+        this(name, documentReferences, emptySet());
     }
 
     public NewDocumentType(Name name,
                            StructDataType header,
                            StructDataType body,
                            FieldSets fs,
-                           Set<Name> documentReferences) {
+                           Set<Name> documentReferences,
+                           Set<String> importedFieldNames) {
         super(name.getName());
         this.name = name;
         this.header = header;
@@ -102,6 +114,7 @@ public final class NewDocumentType extends StructuredDataType implements DataTyp
             }
         }
         this.documentReferences = documentReferences;
+        this.importedFieldNames = importedFieldNames;
     }
 
     public Name getFullName() {
@@ -389,4 +402,8 @@ public final class NewDocumentType extends StructuredDataType implements DataTyp
         return documentReferences;
     }
 
+    public Set<String> getImportedFieldNames() {
+        return importedFieldNames;
+    }
+
 }
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java
index f0b1b427531..d3a78321106 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java
@@ -18,6 +18,7 @@ import com.yahoo.documentmodel.VespaDocumentType;
 import com.yahoo.searchdefinition.document.Attribute;
 import com.yahoo.searchdefinition.document.SDDocumentType;
 import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.searchdefinition.document.TemporaryImportedFields;
 import com.yahoo.searchdefinition.document.annotation.SDAnnotationType;
 import com.yahoo.searchdefinition.document.annotation.TemporaryAnnotationReferenceDataType;
 import com.yahoo.vespa.documentmodel.DocumentModel;
@@ -27,6 +28,7 @@ import com.yahoo.vespa.documentmodel.SearchField;
 
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.LinkedList;
@@ -338,7 +340,8 @@ public class DocumentModelBuilder {
                                                  sdoc.getDocumentType().contentStruct(),
                                                  sdoc.getDocumentType().getBodyType(),
                                                  sdoc.getFieldSets(),
-                                                 convertDocumentReferencesToNames(sdoc.getDocumentReferences()));
+                                                 convertDocumentReferencesToNames(sdoc.getDocumentReferences()),
+                                                 convertTemporaryImportedFieldsToNames(sdoc.getTemporaryImportedFields()));
         for (SDDocumentType n : sdoc.getInheritedTypes()) {
             NewDocumentType.Name name = new NewDocumentType.Name(n.getName());
                 NewDocumentType inherited =  model.getDocumentManager().getDocumentType(name);
@@ -404,6 +407,13 @@ public class DocumentModelBuilder {
                 .collect(toSet());
     }
 
+    private static Set<String> convertTemporaryImportedFieldsToNames(TemporaryImportedFields importedFields) {
+        if (importedFields == null) {
+            return emptySet();
+        }
+        return Collections.unmodifiableSet(importedFields.fields().keySet());
+    }
+
     private static void extractDataTypesFromFields(NewDocumentType dt, Collection<Field> fields) {
         for (Field f : fields) {
             DataType type = f.getDataType();
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentReferenceResolver.java b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentReferenceResolver.java
index 14f8a0a9d37..0d0da71bd0f 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentReferenceResolver.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentReferenceResolver.java
@@ -25,8 +25,8 @@ public class DocumentReferenceResolver {
 
     private final Map<String, Search> searchMapping;
 
-    public DocumentReferenceResolver(List<Search> searchDefinitions) {
-        this.searchMapping = createDocumentNameToSearchMapping(searchDefinitions);
+    public DocumentReferenceResolver(List<Search> schemas) {
+        this.searchMapping = createDocumentNameToSearchMapping(schemas);
     }
 
     public void resolveReferences(SDDocumentType documentType) {
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplierForStructs.java b/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplierForStructs.java
index 9ff749a994c..ade8ae21870 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplierForStructs.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplierForStructs.java
@@ -41,7 +41,7 @@ public class FieldOperationApplierForStructs extends FieldOperationApplier {
                 }
                 if (structUsedByField.getName().equals(structType.getName())) {
                     //this field is using this type!!
-                    field.populateWithStructFields(sdoc, field.getName(), field.getDataType(), field.isHeader(), 0);
+                    field.populateWithStructFields(sdoc, field.getName(), field.getDataType(), 0);
                     field.populateWithStructMatching(sdoc, field.getName(), field.getDataType(), field.getMatching());
                 }
             }
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/ImportedFieldsEnumerator.java b/config-model/src/main/java/com/yahoo/searchdefinition/ImportedFieldsEnumerator.java
new file mode 100644
index 00000000000..25cdd1e08cd
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/ImportedFieldsEnumerator.java
@@ -0,0 +1,31 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition;
+
+import com.yahoo.searchdefinition.document.SDDocumentType;
+
+import java.util.List;
+
+/**
+ * Enumerates and emplaces a set of all imported fields into a SDDocumentType from
+ * its corresponding Search instance.
+ */
+public class ImportedFieldsEnumerator {
+
+    private final List<Search> schemas;
+
+    public ImportedFieldsEnumerator(List<Search> schemas) {
+        this.schemas = schemas;
+    }
+
+    public void enumerateImportedFields(SDDocumentType documentType) {
+        var search = this.schemas.stream()
+                                 .filter(s -> s.getDocument() != null)
+                                 .filter(s -> s.getDocument().getName().equals(documentType.getName()))
+                                 .findFirst();
+        if (search.isEmpty()) {
+            return; // No imported fields present.
+        }
+        search.get().temporaryImportedFields().ifPresent(documentType::setTemporaryImportedFields);
+    }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/Index.java b/config-model/src/main/java/com/yahoo/searchdefinition/Index.java
index e46db1d1b5f..aba6cf9a233 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/Index.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/Index.java
@@ -2,6 +2,7 @@
 package com.yahoo.searchdefinition;
 
 import com.yahoo.searchdefinition.document.BooleanIndexDefinition;
+import com.yahoo.searchdefinition.document.HnswIndexParams;
 import com.yahoo.searchdefinition.document.RankType;
 import com.yahoo.searchdefinition.document.Stemming;
 
@@ -9,6 +10,9 @@ import java.io.Serializable;
 import java.util.Collections;
 import java.util.Iterator;
 import java.util.LinkedHashSet;
+import java.util.Locale;
+import java.util.Objects;
+import java.util.Optional;
 import java.util.Set;
 
 /**
@@ -20,6 +24,8 @@ import java.util.Set;
  */
 public class Index implements Cloneable, Serializable {
 
+    public static enum DistanceMetric { EUCLIDEAN, ANGULAR, GEODEGREES }
+
     public enum Type {
 
         VESPA("vespa");
@@ -57,6 +63,10 @@ public class Index implements Cloneable, Serializable {
     /** The boolean index definition, if set */
     private BooleanIndexDefinition boolIndex;
 
+    private Optional<HnswIndexParams> hnswIndexParams = Optional.empty();
+
+    private Optional<DistanceMetric> distanceMetric = Optional.empty();
+
     /** Whether the posting lists of this index field should have interleaved features (num occs, field length) in document id stream. */
     private boolean interleavedFeatures = false;
 
@@ -115,20 +125,26 @@ public class Index implements Cloneable, Serializable {
     }
 
     @Override
-    public int hashCode() {
-            return name.hashCode() + ( prefix ? 17 : 0 );
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        Index index = (Index) o;
+        return prefix == index.prefix &&
+                normalized == index.normalized &&
+                interleavedFeatures == index.interleavedFeatures &&
+                Objects.equals(name, index.name) &&
+                rankType == index.rankType &&
+                Objects.equals(aliases, index.aliases) &&
+                stemming == index.stemming &&
+                type == index.type &&
+                Objects.equals(boolIndex, index.boolIndex) &&
+                Objects.equals(distanceMetric, index.distanceMetric) &&
+                Objects.equals(hnswIndexParams, index.hnswIndexParams);
     }
 
     @Override
-    public boolean equals(Object object) {
-            if ( ! (object instanceof Index)) return false;
-
-            Index other=(Index)object;
-            return
-                this.name.equals(other.name) &&
-                this.prefix==other.prefix &&
-                this.stemming==other.stemming &&
-                this.normalized==other.normalized;
+    public int hashCode() {
+        return Objects.hash(name, rankType, prefix, aliases, stemming, normalized, type, boolIndex, distanceMetric, hnswIndexParams, interleavedFeatures);
     }
 
     public String toString() {
@@ -176,6 +192,24 @@ public class Index implements Cloneable, Serializable {
         boolIndex = def;
     }
 
+    public Optional<DistanceMetric> getDistanceMetric() {
+        return distanceMetric;
+    }
+
+    public void setDistanceMetric(String value) {
+        String upper = value.toUpperCase(Locale.ENGLISH);
+        DistanceMetric dm = DistanceMetric.valueOf(upper);
+        distanceMetric = Optional.of(dm);
+    }
+
+    public Optional<HnswIndexParams> getHnswIndexParams() {
+        return hnswIndexParams;
+    }
+
+    public void setHnswIndexParams(HnswIndexParams params) {
+        hnswIndexParams = Optional.of(params);
+    }
+
     public void setInterleavedFeatures(boolean value) {
         interleavedFeatures = value;
     }
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java
index fa196cd3bbf..23eb814de81 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java
@@ -788,8 +788,7 @@ public class RankProfile implements Cloneable {
                     type = existingType.dimensionwiseGeneralizationWith(type).orElseThrow( () ->
                         new IllegalArgumentException(queryProfileType + " contains query feature " + feature.get() +
                                                      " with type " + field.getType().asTensorType() +
-                                                     ", but this is already defined " +
-                                                     "in another query profile with type " +
+                                                     ", but this is already defined in another query profile with type " +
                                                      context.getType(feature.get())));
                 context.setType(feature.get(), type);
             }
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/Search.java b/config-model/src/main/java/com/yahoo/searchdefinition/Search.java
index f90a7e4f6cd..0ab8a2308a4 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/Search.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/Search.java
@@ -288,7 +288,7 @@ public class Search implements ImmutableSearch {
     /**
      * Adds an extra field of this search definition not contained in a document
      *
-     * @param field to add to the searchdefinitions list of external fields.
+     * @param field to add to the schemas list of external fields
      */
     public void addExtraField(SDField field) {
         if (fields.containsKey(field.getName())) {
@@ -383,7 +383,7 @@ public class Search implements ImmutableSearch {
      * Consolidates a set of index settings for the same index into one
      *
      * @param indices The list of indexes to consolidate.
-     * @return The consolidated index
+     * @return the consolidated index
      */
     private Index consolidateIndices(List<Index> indices) {
         Index first = indices.get(0);
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java b/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java
index 949539ff99f..eb68e6af203 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java
@@ -230,20 +230,22 @@ public class SearchBuilder {
                 sdocs.add(search.getDocument());
             }
         }
-        SDDocumentTypeOrderer orderer = new SDDocumentTypeOrderer(sdocs, deployLogger);
+        var orderer = new SDDocumentTypeOrderer(sdocs, deployLogger);
         orderer.process();
         for (SDDocumentType sdoc : orderer.getOrdered()) {
             new FieldOperationApplierForStructs().process(sdoc);
             new FieldOperationApplier().process(sdoc);
         }
 
-        DocumentReferenceResolver resolver = new DocumentReferenceResolver(searchList);
+        var resolver = new DocumentReferenceResolver(searchList);
         sdocs.forEach(resolver::resolveReferences);
+        var importedFieldsEnumerator = new ImportedFieldsEnumerator(searchList);
+        sdocs.forEach(importedFieldsEnumerator::enumerateImportedFields);
 
         if (validate)
             new DocumentGraphValidator().validateDocumentGraph(sdocs);
 
-        DocumentModelBuilder builder = new DocumentModelBuilder(model);
+        var builder = new DocumentModelBuilder(model);
         for (Search search : new SearchOrderer().order(searchList)) {
             new FieldOperationApplierForSearch().process(search); // TODO: Why is this not in the regular list?
             process(search, deployLogger, new QueryProfiles(queryProfileRegistry, deployLogger), validate);
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java
index 76dff404568..8b5f7658475 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java
@@ -240,6 +240,17 @@ public class AttributeFields extends Derived implements AttributesConfig.Produce
             aaB.tensortype(attribute.tensorType().get().toString());
         }
         aaB.imported(imported);
+        var dma = attribute.distanceMetric();
+        if (attribute.hnswIndexParams().isPresent()) {
+            var ib = new AttributesConfig.Attribute.Index.Builder();
+            var params = attribute.hnswIndexParams().get();
+            ib.hnsw.enabled(true);
+            ib.hnsw.maxlinkspernode(params.maxLinksPerNode());
+            ib.hnsw.neighborstoexploreatinsert(params.neighborsToExploreAtInsert());
+            var dm = AttributesConfig.Attribute.Index.Hnsw.Distancemetric.Enum.valueOf(dma.toString());
+            ib.hnsw.distancemetric(dm);
+            aaB.index(ib);
+        }
         return aaB;
     }
 
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java
index 245913d7822..fc8710fa1a1 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java
@@ -15,9 +15,11 @@ import com.yahoo.search.query.profile.QueryProfileRegistry;
 import com.yahoo.searchdefinition.RankProfileRegistry;
 import com.yahoo.searchdefinition.Search;
 import com.yahoo.searchdefinition.derived.validation.Validation;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
 
 import java.io.IOException;
 import java.io.Writer;
+import java.util.logging.Level;
 
 /**
  * A set of all derived configuration of a search definition. Use this as a facade to individual configurations when
@@ -39,12 +41,13 @@ public class DerivedConfiguration {
     private VsmSummary streamingSummary;
     private IndexSchema indexSchema;
     private ImportedFields importedFields;
+    private QueryProfileRegistry queryProfiles;
 
     /**
      * Creates a complete derived configuration from a search definition.
      * Only used in tests.
      *
-     * @param search The search to derive a configuration from. Derived objects will be snapshots, but this argument is
+     * @param search the search to derive a configuration from. Derived objects will be snapshots, but this argument is
      *               live. Which means that this object will be inconsistent when the given search definition is later
      *               modified.
      * @param rankProfileRegistry a {@link com.yahoo.searchdefinition.RankProfileRegistry}
@@ -56,11 +59,11 @@ public class DerivedConfiguration {
     /**
      * Creates a complete derived configuration snapshot from a search definition.
      *
-     * @param search             The search to derive a configuration from. Derived objects will be snapshots, but this
+     * @param search             the search to derive a configuration from. Derived objects will be snapshots, but this
      *                           argument is live. Which means that this object will be inconsistent when the given
      *                           search definition is later modified.
      * @param deployLogger       a {@link DeployLogger} for logging when doing operations on this
-     * @param deployProperties   Properties set on deploy.
+     * @param deployProperties   properties set on deploy
      * @param rankProfileRegistry a {@link com.yahoo.searchdefinition.RankProfileRegistry}
      * @param queryProfiles      the query profiles of this application
      */
@@ -72,6 +75,7 @@ public class DerivedConfiguration {
                                 ImportedMlModels importedModels) {
         Validator.ensureNotNull("Search definition", search);
         this.search = search;
+        this.queryProfiles = queryProfiles;
         if ( ! search.isDocumentsOnly()) {
             streamingFields = new VsmFields(search);
             streamingSummary = new VsmSummary(search);
@@ -120,6 +124,10 @@ public class DerivedConfiguration {
         exportCfg(new DocumenttypesConfig(documentTypesCfg), toDirectory + "/" + "documenttypes.cfg");
     }
 
+    public static void exportQueryProfiles(QueryProfileRegistry queryProfileRegistry, String toDirectory) throws IOException {
+        exportCfg(new QueryProfiles(queryProfileRegistry, (level, message) -> {}).getConfig(), toDirectory + "/" + "query-profiles.cfg");
+    }
+
     private static void exportCfg(ConfigInstance instance, String fileName) throws IOException {
         Writer writer = null;
         try {
@@ -186,4 +194,7 @@ public class DerivedConfiguration {
     public ImportedFields getImportedFields() {
         return importedFields;
     }
+
+    public QueryProfileRegistry getQueryProfiles() { return queryProfiles; }
+
 }
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexInfo.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexInfo.java
index 02b2a383318..da25680ca47 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexInfo.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexInfo.java
@@ -1,16 +1,25 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.searchdefinition.derived;
 
-import com.yahoo.document.*;
+import com.yahoo.document.CollectionDataType;
+import com.yahoo.document.DataType;
+import com.yahoo.document.NumericDataType;
+import com.yahoo.document.PositionDataType;
 import com.yahoo.searchdefinition.Index;
 import com.yahoo.searchdefinition.Search;
-import com.yahoo.searchdefinition.document.*;
+import com.yahoo.searchdefinition.document.Attribute;
+import com.yahoo.searchdefinition.document.BooleanIndexDefinition;
+import com.yahoo.searchdefinition.document.FieldSet;
+import com.yahoo.searchdefinition.document.ImmutableSDField;
+import com.yahoo.searchdefinition.document.Matching;
+import com.yahoo.searchdefinition.document.Stemming;
 import com.yahoo.searchdefinition.processing.ExactMatch;
 import com.yahoo.searchdefinition.processing.NGramMatch;
 import com.yahoo.vespa.documentmodel.SummaryField;
 import com.yahoo.search.config.IndexInfoConfig;
 
 import java.util.Map;
+import java.util.Optional;
 import java.util.Set;
 
 /**
@@ -34,8 +43,10 @@ public class IndexInfo extends Derived implements IndexInfoConfig.Producer {
     private static final String CMD_PLAIN_TOKENS = "plain-tokens";
     private static final String CMD_MULTIVALUE = "multivalue";
     private static final String CMD_FAST_SEARCH = "fast-search";
+    private static final String CMD_PREDICATE = "predicate";
     private static final String CMD_PREDICATE_BOUNDS = "predicate-bounds";
     private static final String CMD_NUMERICAL = "numerical";
+    private static final String CMD_PHRASE_SEGMENTING = "phrase-segmenting";
     private Set<IndexCommand> commands = new java.util.LinkedHashSet<>();
     private Map<String, String> aliases = new java.util.LinkedHashMap<>();
     private Map<String, FieldSet> fieldSets;
@@ -90,6 +101,7 @@ public class IndexInfo extends Derived implements IndexInfoConfig.Producer {
 
     protected void derive(ImmutableSDField field, Search search, boolean inPosition) {
         if (field.getDataType().equals(DataType.PREDICATE)) {
+            addIndexCommand(field, CMD_PREDICATE);
             Index index = field.getIndex(field.getName());
             if (index != null) {
                 BooleanIndexDefinition options = index.getBooleanIndexDefiniton();
@@ -286,21 +298,14 @@ public class IndexInfo extends Derived implements IndexInfoConfig.Producer {
 
     // TODO: Move this to the FieldSetSettings processor (and rename it) as that already has to look at this.
     private void addFieldSetCommands(IndexInfoConfig.Indexinfo.Builder iiB, FieldSet fieldSet) {
-        // Explicit query commands on the field set, overrides everything.
-        if (!fieldSet.queryCommands().isEmpty()) {
-            for (String qc : fieldSet.queryCommands()) {
-                iiB.command(
-                        new IndexInfoConfig.Indexinfo.Command.Builder()
-                        .indexname(fieldSet.getName())
-                        .command(qc));
-            }
-            return;
-        }        
+        for (String qc : fieldSet.queryCommands())
+            iiB.command(new IndexInfoConfig.Indexinfo.Command.Builder().indexname(fieldSet.getName()).command(qc));
         boolean anyIndexing = false;
         boolean anyAttributing = false;
         boolean anyLowerCasing = false;
         boolean anyStemming = false;
         boolean anyNormalizing = false;
+        String phraseSegmentingCommand = null;
         String stemmingCommand = null;
         Matching fieldSetMatching = fieldSet.getMatching(); // null if no explicit matching
         // First a pass over the fields to read some params to decide field settings implicitly:
@@ -321,8 +326,13 @@ public class IndexInfo extends Derived implements IndexInfoConfig.Producer {
             if (field.getNormalizing().doRemoveAccents()) {
                 anyNormalizing = true;
             }
-            if (fieldSetMatching == null && field.getMatching().getType() != Matching.defaultType)
+            if (fieldSetMatching == null && field.getMatching().getType() != Matching.defaultType) {
                 fieldSetMatching = field.getMatching();
+            }
+            Optional<String> explicitPhraseSegmentingCommand = field.getQueryCommands().stream().filter(c -> c.startsWith(CMD_PHRASE_SEGMENTING)).findFirst();
+            if (explicitPhraseSegmentingCommand.isPresent()) {
+                phraseSegmentingCommand = explicitPhraseSegmentingCommand.get();
+            }
         }
         if (anyIndexing && anyAttributing && fieldSet.getMatching() == null) {
             // We have both attributes and indexes and no explicit match setting ->
@@ -365,6 +375,11 @@ public class IndexInfo extends Derived implements IndexInfoConfig.Producer {
                             new IndexInfoConfig.Indexinfo.Command.Builder()
                                 .indexname(fieldSet.getName())
                                 .command(CMD_NORMALIZE));
+                if (phraseSegmentingCommand != null)
+                    iiB.command(
+                            new IndexInfoConfig.Indexinfo.Command.Builder()
+                                    .indexname(fieldSet.getName())
+                                    .command(phraseSegmentingCommand));
             }
         } else {
             // Assume only attribute fields
@@ -400,9 +415,7 @@ public class IndexInfo extends Derived implements IndexInfoConfig.Producer {
             } else if (fieldSetMatching.getType().equals(Matching.Type.TEXT)) {
                 
             }
-            
         }
-       
     }
 
     private boolean hasMultiValueField(FieldSet fieldSet) {
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexSchema.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexSchema.java
index 60b8ee78c7b..39a67a69575 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexSchema.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexSchema.java
@@ -5,6 +5,7 @@ import com.yahoo.document.ArrayDataType;
 import com.yahoo.document.DataType;
 import com.yahoo.document.Field;
 import com.yahoo.document.StructuredDataType;
+import com.yahoo.document.TensorDataType;
 import com.yahoo.document.WeightedSetDataType;
 import com.yahoo.searchdefinition.Search;
 import com.yahoo.searchdefinition.document.BooleanIndexDefinition;
@@ -20,7 +21,7 @@ import java.util.List;
 import java.util.Map;
 
 /**
- * Deriver of indexschema config containing information of all index fields with name and data type.
+ * Deriver of indexschema config containing information of all text index fields with name and data type.
  *
  * @author geirst
  */
@@ -44,9 +45,14 @@ public class IndexSchema extends Derived implements IndexschemaConfig.Producer {
         super.derive(search);
     }
 
+    private boolean isTensorField(ImmutableSDField field) {
+        return field.getDataType() instanceof TensorDataType;
+    }
+
     private void deriveIndexFields(ImmutableSDField field, Search search) {
-        if (!field.doesIndexing() &&
-            !field.isIndexStructureField())
+        // Note: Indexes for tensor fields are NOT part of the index schema for text fields.
+        if ((!field.doesIndexing() && !field.isIndexStructureField()) ||
+                isTensorField(field))
         {
             return;
         }
@@ -138,11 +144,10 @@ public class IndexSchema extends Derived implements IndexschemaConfig.Producer {
             return Collections.singletonList(field);
         }
         if (fieldType instanceof ArrayDataType) {
-            boolean header = field.isHeader();
             List<Field> ret = new LinkedList<>();
-            Field innerField = new Field(field.getName(), ((ArrayDataType)fieldType).getNestedType(), header);
+            Field innerField = new Field(field.getName(), ((ArrayDataType)fieldType).getNestedType());
             for (Field flatField : flattenField(innerField)) {
-                ret.add(new Field(flatField.getName(), DataType.getArray(flatField.getDataType()), header));
+                ret.add(new Field(flatField.getName(), DataType.getArray(flatField.getDataType())));
             }
             return ret;
         }
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java
index fbcaf2a3a80..1661a80f238 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java
@@ -24,6 +24,7 @@ import com.yahoo.document.datatypes.Float16FieldValue;
 import com.yahoo.document.datatypes.StringFieldValue;
 import com.yahoo.document.datatypes.TensorFieldValue;
 import com.yahoo.tensor.TensorType;
+import static com.yahoo.searchdefinition.Index.DistanceMetric;
 
 import java.io.Serializable;
 import java.util.LinkedHashSet;
@@ -66,6 +67,10 @@ public final class Attribute implements Cloneable, Serializable {
     /** This is set if the type of this is REFERENCE */
     private final Optional<StructuredDataType> referenceDocumentType;
 
+    private Optional<DistanceMetric> distanceMetric = Optional.empty();
+
+    private Optional<HnswIndexParams> hnswIndexParams = Optional.empty();
+
     private boolean isPosition = false;
     private final Sorting sorting = new Sorting();
 
@@ -195,6 +200,12 @@ public final class Attribute implements Cloneable, Serializable {
     public Optional<TensorType> tensorType() { return tensorType; }
     public Optional<StructuredDataType> referenceDocumentType() { return referenceDocumentType; }
 
+    public static final DistanceMetric DEFAULT_DISTANCE_METRIC = DistanceMetric.EUCLIDEAN;
+    public DistanceMetric distanceMetric() {
+        return distanceMetric.orElse(DEFAULT_DISTANCE_METRIC);
+    }
+    public Optional<HnswIndexParams> hnswIndexParams() { return hnswIndexParams; }
+
     public Sorting getSorting() { return sorting; }
 
     public void setRemoveIfZero(boolean remove)                  { this.removeIfZero = remove; }
@@ -217,6 +228,8 @@ public final class Attribute implements Cloneable, Serializable {
     public void setUpperBound(long upperBound)                   { this.upperBound = upperBound; }
     public void setDensePostingListThreshold(double threshold)   { this.densePostingListThreshold = threshold; }
     public void setTensorType(TensorType tensorType)             { this.tensorType = Optional.of(tensorType); }
+    public void setDistanceMetric(Optional<DistanceMetric> dm)   { this.distanceMetric = dm; }
+    public void setHnswIndexParams(HnswIndexParams params)       { this.hnswIndexParams = Optional.of(params); }
 
     public String         getName()                     { return name; }
     public Type           getType()                     { return type; }
@@ -335,7 +348,7 @@ public final class Attribute implements Cloneable, Serializable {
     public int hashCode() {
         return Objects.hash(
                 name, type, collectionType, sorting, isPrefetch(), fastAccess, removeIfZero, createIfNonExistent,
-                isPosition, huge, enableBitVectors, enableOnlyBitVector, tensorType, referenceDocumentType);
+                isPosition, huge, enableBitVectors, enableOnlyBitVector, tensorType, referenceDocumentType, hnswIndexParams);
     }
 
     @Override
@@ -349,8 +362,8 @@ public final class Attribute implements Cloneable, Serializable {
 
     /** Returns whether these attributes describes the same entity, even if they have different names */
     public boolean isCompatible(Attribute other) {
-        if ( ! this.type.equals(other.type)) return false;
-        if ( ! this.collectionType.equals(other.collectionType)) return false;
+        if (! this.type.equals(other.type)) return false;
+        if (! this.collectionType.equals(other.collectionType)) return false;
         if (this.isPrefetch() != other.isPrefetch()) return false;
         if (this.removeIfZero != other.removeIfZero) return false;
         if (this.createIfNonExistent != other.createIfNonExistent) return false;
@@ -359,9 +372,11 @@ public final class Attribute implements Cloneable, Serializable {
         // if (this.noSearch != other.noSearch) return false; No backend consequences so compatible for now
         if (this.fastSearch != other.fastSearch) return false;
         if (this.huge != other.huge) return false;
-        if ( ! this.sorting.equals(other.sorting)) return false;
-        if (!this.tensorType.equals(other.tensorType)) return false;
-        if (!this.referenceDocumentType.equals(other.referenceDocumentType)) return false;
+        if (! this.sorting.equals(other.sorting)) return false;
+        if (! Objects.equals(tensorType, other.tensorType)) return false;
+        if (! Objects.equals(referenceDocumentType, other.referenceDocumentType)) return false;
+        if (! Objects.equals(distanceMetric, other.distanceMetric)) return false;
+        if (! Objects.equals(hnswIndexParams, other.hnswIndexParams)) return false;
 
         return true;
     }
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/FieldSet.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/FieldSet.java
index 55dedc4a1d7..22424286ef9 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/document/FieldSet.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/FieldSet.java
@@ -25,10 +25,7 @@ public class FieldSet {
     public FieldSet addFieldName(String field) { fieldNames.add(field); return this; }
     public Set<String> getFieldNames() { return fieldNames; }
     public Set<ImmutableSDField> fields() { return fields; }
-
-    public Set<String> queryCommands() {
-        return queryCommands;
-    }
+    public Set<String> queryCommands() { return queryCommands; }
 
     public void setMatching(Matching matching) {
         this.matching = matching;
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/HnswIndexParams.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/HnswIndexParams.java
new file mode 100644
index 00000000000..2f084d3e513
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/HnswIndexParams.java
@@ -0,0 +1,64 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.document;
+
+import java.util.Locale;
+import java.util.Optional;
+
+/**
+ * Configuration parameters for a hnsw index used together with a 1-dimensional indexed tensor for approximate nearest neighbor search.
+ *
+ * @author geirst
+ */
+public class HnswIndexParams {
+
+    public static final int DEFAULT_MAX_LINKS_PER_NODE = 16;
+    public static final int DEFAULT_NEIGHBORS_TO_EXPLORE_AT_INSERT = 200;
+
+    private final Optional<Integer> maxLinksPerNode;
+    private final Optional<Integer> neighborsToExploreAtInsert;
+
+    public static class Builder {
+        private Optional<Integer> maxLinksPerNode = Optional.empty();
+        private Optional<Integer> neighborsToExploreAtInsert = Optional.empty();
+
+        public void setMaxLinksPerNode(int value) {
+            maxLinksPerNode = Optional.of(value);
+        }
+        public void setNeighborsToExploreAtInsert(int value) {
+            neighborsToExploreAtInsert = Optional.of(value);
+        }
+        public HnswIndexParams build() {
+            return new HnswIndexParams(maxLinksPerNode, neighborsToExploreAtInsert);
+        }
+    }
+
+    public HnswIndexParams() {
+        this.maxLinksPerNode = Optional.empty();
+        this.neighborsToExploreAtInsert = Optional.empty();
+    }
+
+    public HnswIndexParams(Optional<Integer> maxLinksPerNode,
+                           Optional<Integer> neighborsToExploreAtInsert) {
+        this.maxLinksPerNode = maxLinksPerNode;
+        this.neighborsToExploreAtInsert = neighborsToExploreAtInsert;
+    }
+
+    /**
+     * Creates a new instance where values from the given parameter instance are used where they are present,
+     * otherwise we use values from this.
+     */
+    public HnswIndexParams overrideFrom(Optional<HnswIndexParams> other) {
+        if (! other.isPresent()) return this;
+        HnswIndexParams rhs = other.get();
+        return new HnswIndexParams(rhs.maxLinksPerNode.or(() ->  maxLinksPerNode),
+                rhs.neighborsToExploreAtInsert.or(() ->  neighborsToExploreAtInsert));
+    }
+
+    public int maxLinksPerNode() {
+        return maxLinksPerNode.orElse(DEFAULT_MAX_LINKS_PER_NODE);
+    }
+
+    public int neighborsToExploreAtInsert() {
+        return neighborsToExploreAtInsert.orElse(DEFAULT_NEIGHBORS_TO_EXPLORE_AT_INSERT);
+    }
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java
index 414a605c621..b3f98cc6f26 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java
@@ -47,6 +47,7 @@ public class SDDocumentType implements Cloneable, Serializable {
     private FieldSets fieldSets;
     // Document references
     private Optional<DocumentReferences> documentReferences = Optional.empty();
+    private TemporaryImportedFields temporaryImportedFields;
 
     static {
         VESPA_DOCUMENT = new SDDocumentType(VespaDocumentType.INSTANCE.getFullName().getName());
@@ -58,6 +59,7 @@ public class SDDocumentType implements Cloneable, Serializable {
         type.docType = docType.clone();
         type.inheritedTypes.putAll(inheritedTypes);
         type.structType = structType;
+        // TODO this isn't complete; should it be..?!
         return type;
     }
 
@@ -334,4 +336,11 @@ public class SDDocumentType implements Cloneable, Serializable {
         this.documentReferences = Optional.of(documentReferences);
     }
 
+    public TemporaryImportedFields getTemporaryImportedFields() {
+        return temporaryImportedFields;
+    }
+
+    public void setTemporaryImportedFields(TemporaryImportedFields temporaryImportedFields) {
+        this.temporaryImportedFields = temporaryImportedFields;
+    }
 }
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java
index c657d29033a..30ce142d503 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java
@@ -38,9 +38,8 @@ import java.util.TreeMap;
 
 /**
  * The field class represents a document field. It is used in
- * the Document class to get and set fields. Each SDField has
- * a name, a numeric ID, a data type, and a boolean that says whether it's
- * a header field. The numeric ID is used when the fields are stored
+ * the Document class to get and set fields. Each SDField has a name, a numeric ID,
+ * a data type. The numeric ID is used when the fields are stored
  * in serialized form.
  *
  * @author bratseth
@@ -92,7 +91,7 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer,
     private NormalizeLevel normalizing = new NormalizeLevel();
 
     /** Extra query commands of this field */
-    private List<String> queryCommands=new java.util.ArrayList<>(0);
+    private List<String> queryCommands = new java.util.ArrayList<>(0);
 
     /** Summary fields defined in this field */
     private Map<String, SummaryField> summaryFields = new java.util.LinkedHashMap<>(0);
@@ -120,15 +119,14 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer,
      * Creates a new field. This method is only used to create reserved fields
      * @param name The name of the field
      * @param dataType The datatype of the field
-     * @param isHeader Whether this is a "header" field or a "content" field (true = "header").
     */
-    protected SDField(SDDocumentType repo, String name, int id, DataType dataType, boolean isHeader, boolean populate) {
-        super(name, id, dataType, isHeader);
-        populate(populate, repo, name, dataType, isHeader);
+    protected SDField(SDDocumentType repo, String name, int id, DataType dataType, boolean populate) {
+        super(name, id, dataType);
+        populate(populate, repo, name, dataType);
     }
 
-    public SDField(SDDocumentType repo, String name, int id, DataType dataType, boolean isHeader) {
-        this(repo, name, id, dataType, isHeader, true);
+    public SDField(SDDocumentType repo, String name, int id, DataType dataType) {
+        this(repo, name, id, dataType, true);
     }
 
     /**
@@ -136,41 +134,35 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer,
 
        @param name The name of the field
        @param dataType The datatype of the field
-       @param isHeader Whether this is a "header" field or a "content" field
-       (true = "header").
     */
-    public SDField(SDDocumentType repo, String name, DataType dataType, boolean isHeader, boolean populate) {
-        super(name,dataType,isHeader);
-        populate(populate, repo, name, dataType, isHeader);
+    public SDField(SDDocumentType repo, String name, DataType dataType, boolean populate) {
+        super(name,dataType);
+        populate(populate, repo, name, dataType);
     }
 
-    private void populate(boolean populate, SDDocumentType repo, String name, DataType dataType, boolean isHeader) {
-        populate(populate,repo, name, dataType, isHeader, null, 0);
+    private void populate(boolean populate, SDDocumentType repo, String name, DataType dataType) {
+        populate(populate,repo, name, dataType, null, 0);
     }
 
-    private void populate(boolean populate, SDDocumentType repo, String name, DataType dataType, boolean isHeader, Matching fieldMatching,  int recursion) {
+    private void populate(boolean populate, SDDocumentType repo, String name, DataType dataType, Matching fieldMatching,  int recursion) {
         if (populate || (dataType instanceof MapDataType)) {
-            populateWithStructFields(repo, name, dataType, isHeader, recursion);
+            populateWithStructFields(repo, name, dataType, recursion);
             populateWithStructMatching(repo, name, dataType, fieldMatching);
         }
     }
 
-    public SDField(String name, DataType dataType, boolean isHeader) {
-        this(null, name, dataType, isHeader, true);
-    }
 
     /**
      * Creates a new field.
      *
      * @param name The name of the field
      * @param dataType The datatype of the field
-     * @param isHeader Whether this is a "header" field or a "content" field (true = "header").
      * @param owner the owning document (used to check for id collisions)
      */
-    protected SDField(SDDocumentType repo, String name, DataType dataType, boolean isHeader, SDDocumentType owner, boolean populate) {
-        super(name, dataType, isHeader, owner == null ? null : owner.getDocumentType());
+    protected SDField(SDDocumentType repo, String name, DataType dataType, SDDocumentType owner, boolean populate) {
+        super(name, dataType, owner == null ? null : owner.getDocumentType());
         this.ownerDocType=owner;
-        populate(populate, repo, name, dataType, isHeader);
+        populate(populate, repo, name, dataType);
     }
 
     /**
@@ -178,27 +170,25 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer,
      *
      * @param name The name of the field
      * @param dataType The datatype of the field
-     * @param isHeader Whether this is a "header" field or a "content" field (true = "header").
      * @param owner The owning document (used to check for id collisions)
      * @param fieldMatching The matching object to set for the field
      */
-    protected SDField(SDDocumentType repo, String name, DataType dataType, boolean isHeader, SDDocumentType owner,
+    protected SDField(SDDocumentType repo, String name, DataType dataType, SDDocumentType owner,
                       Matching fieldMatching, boolean populate, int recursion) {
-        super(name, dataType, isHeader, owner == null ? null : owner.getDocumentType());
+        super(name, dataType, owner == null ? null : owner.getDocumentType());
         this.ownerDocType=owner;
         if (fieldMatching != null)
             this.setMatching(fieldMatching);
-        populate(populate, repo, name, dataType, isHeader, fieldMatching, recursion);
+        populate(populate, repo, name, dataType, fieldMatching, recursion);
     }
 
     /**
-     * Constructor for <b>header</b> fields
      *
      * @param name The name of the field
      * @param dataType The datatype of the field
      */
     public SDField(SDDocumentType repo,  String name, DataType dataType) {
-        this(repo, name,dataType,true, true);
+        this(repo, name,dataType, true);
     }
     public SDField(String name, DataType dataType) {
         this(null, name,dataType);
@@ -277,7 +267,7 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer,
         }
     }
 
-    public void populateWithStructFields(SDDocumentType sdoc, String name, DataType dataType, boolean isHeader, int recursion) {
+    public void populateWithStructFields(SDDocumentType sdoc, String name, DataType dataType, int recursion) {
         DataType dt = getFirstStructOrMapRecursive();
         if (dt == null) {
             return;
@@ -286,11 +276,11 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer,
             MapDataType mdt = (MapDataType) dataType;
 
             SDField keyField = new SDField(sdoc, name.concat(".key"), mdt.getKeyType(),
-                                           isHeader, getOwnerDocType(), new Matching(), true, recursion + 1);
+                                           getOwnerDocType(), new Matching(), true, recursion + 1);
             structFields.put("key", keyField);
 
             SDField valueField = new SDField(sdoc, name.concat(".value"), mdt.getValueType(),
-                                             isHeader, getOwnerDocType(), new Matching(), true, recursion + 1);
+                                             getOwnerDocType(), new Matching(), true, recursion + 1);
             structFields.put("value", valueField);
         } else {
             if (recursion >= 10) {
@@ -306,7 +296,7 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer,
                 }
                 for (Field field : subType.fieldSet()) {
                     SDField subField = new SDField(sdoc, name.concat(".").concat(field.getName()), field.getDataType(),
-                                                   isHeader, subType, new Matching(), true, recursion + 1);
+                                                   subType, new Matching(), true, recursion + 1);
                     structFields.put(field.getName(), subField);
                 }
             }
@@ -759,20 +749,11 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer,
         return queryCommands.contains(name);
     }
 
-    /**
-     * A list of query commands
-     *
-     * @return a list of strings with query commands.
-     */
+    /** Returns a list of query commands */
     @Override
-    public List<String> getQueryCommands() {
-        return queryCommands;
-    }
+    public List<String> getQueryCommands() { return queryCommands; }
 
-    /**
-     * The document that this field was declared in, or null
-     *
-     */
+    /** Returns the document that this field was declared in, or null */
     private SDDocumentType getOwnerDocType() {
         return ownerDocType;
     }
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/TemporarySDField.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/TemporarySDField.java
index 886bf777d3a..04d11792379 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/document/TemporarySDField.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/TemporarySDField.java
@@ -8,12 +8,12 @@ import com.yahoo.document.DataType;
  */
 public class TemporarySDField extends SDField {
 
-    public TemporarySDField(String name, DataType dataType, boolean isHeader, SDDocumentType owner) {
-        super(owner, name, dataType, isHeader, owner, false);
+    public TemporarySDField(String name, DataType dataType, SDDocumentType owner) {
+        super(owner, name, dataType, owner, false);
     }
 
     public TemporarySDField(String name, DataType dataType) {
-        super(null, name, dataType, true, false);
+        super(null, name, dataType, false);
     }
 
 }
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ExpressionTransforms.java b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ExpressionTransforms.java
index 6fdf448a39b..a6707ec7ac0 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ExpressionTransforms.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ExpressionTransforms.java
@@ -27,6 +27,7 @@ public class ExpressionTransforms {
                 ImmutableList.of(new TensorFlowFeatureConverter(),
                                  new OnnxFeatureConverter(),
                                  new XgboostFeatureConverter(),
+                                 new LightGBMFeatureConverter(),
                                  new ConstantDereferencer(),
                                  new ConstantTensorTransformer(),
                                  new FunctionInliner(),
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/LightGBMFeatureConverter.java b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/LightGBMFeatureConverter.java
new file mode 100644
index 00000000000..5bde627dc0a
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/LightGBMFeatureConverter.java
@@ -0,0 +1,59 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.expressiontransforms;
+
+import com.yahoo.path.Path;
+import com.yahoo.searchlib.rankingexpression.rule.Arguments;
+import com.yahoo.searchlib.rankingexpression.rule.CompositeNode;
+import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode;
+import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode;
+import com.yahoo.searchlib.rankingexpression.transform.ExpressionTransformer;
+import com.yahoo.vespa.model.ml.ConvertedModel;
+import com.yahoo.vespa.model.ml.FeatureArguments;
+
+import java.io.UncheckedIOException;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Replaces instances of the lightgbm(model-path) pseudofeature with the
+ * native Vespa ranking expression implementing the same computation.
+ *
+ * @author lesters
+ */
+public class LightGBMFeatureConverter extends ExpressionTransformer<RankProfileTransformContext> {
+
+    /** A cache of imported models indexed by model path. This avoids importing the same model multiple times. */
+    private final Map<Path, ConvertedModel> convertedLightGBMModels = new HashMap<>();
+
+    @Override
+    public ExpressionNode transform(ExpressionNode node, RankProfileTransformContext context) {
+        if (node instanceof ReferenceNode)
+            return transformFeature((ReferenceNode) node, context);
+        else if (node instanceof CompositeNode)
+            return super.transformChildren((CompositeNode) node, context);
+        else
+            return node;
+    }
+
+    private ExpressionNode transformFeature(ReferenceNode feature, RankProfileTransformContext context) {
+        if ( ! feature.getName().equals("lightgbm")) return feature;
+
+        try {
+            FeatureArguments arguments = asFeatureArguments(feature.getArguments());
+            ConvertedModel convertedModel =
+                    convertedLightGBMModels.computeIfAbsent(arguments.path(),
+                                                           path -> ConvertedModel.fromSourceOrStore(path, true, context));
+            return convertedModel.expression(arguments, context);
+        } catch (IllegalArgumentException | UncheckedIOException e) {
+            throw new IllegalArgumentException("Could not use LightGBM model from " + feature, e);
+        }
+    }
+
+    private FeatureArguments asFeatureArguments(Arguments arguments) {
+        if (arguments.size() != 1)
+            throw new IllegalArgumentException("A lightgbm node must take a single argument pointing to " +
+                                               "the LightGBM model file under [application]/models");
+        return new FeatureArguments(arguments);
+    }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/FieldOperationContainer.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/FieldOperationContainer.java
index 09e8b3ccdfa..ab9148992e0 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/FieldOperationContainer.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/FieldOperationContainer.java
@@ -11,7 +11,7 @@ public interface FieldOperationContainer {
     /** Adds an operation */
     void addOperation(FieldOperation op);
 
-    /** Apply all operations. Operations must be sorted in thjeir natural order before applying each operation. */
+    /** Apply all operations. Operations must be sorted in their natural order before applying each operation. */
     void applyOperations(SDField field);
 
     String getName();
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexOperation.java
index 39f543c7db3..0c1f443dee3 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexOperation.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexOperation.java
@@ -4,6 +4,7 @@ package com.yahoo.searchdefinition.fieldoperation;
 import com.yahoo.searchdefinition.Index;
 import com.yahoo.searchdefinition.Index.Type;
 import com.yahoo.searchdefinition.document.BooleanIndexDefinition;
+import com.yahoo.searchdefinition.document.HnswIndexParams;
 import com.yahoo.searchdefinition.document.SDField;
 import com.yahoo.searchdefinition.document.Stemming;
 
@@ -31,6 +32,9 @@ public class IndexOperation implements FieldOperation {
     private OptionalDouble densePostingListThreshold = OptionalDouble.empty();
     private Optional<Boolean> enableBm25 = Optional.empty();
 
+    private Optional<String> distanceMetric = Optional.empty();
+    private Optional<HnswIndexParams.Builder> hnswIndexParams = Optional.empty();
+
     public String getIndexName() {
         return indexName;
     }
@@ -91,6 +95,12 @@ public class IndexOperation implements FieldOperation {
         if (enableBm25.isPresent()) {
             index.setInterleavedFeatures(enableBm25.get());
         }
+        if (distanceMetric.isPresent()) {
+            index.setDistanceMetric(distanceMetric.get());
+        }
+        if (hnswIndexParams.isPresent()) {
+            index.setHnswIndexParams(hnswIndexParams.get().build());
+        }
     }
 
     public Type getType() {
@@ -116,8 +126,17 @@ public class IndexOperation implements FieldOperation {
     public void setDensePostingListThreshold(double densePostingListThreshold) {
         this.densePostingListThreshold = OptionalDouble.of(densePostingListThreshold);
     }
+
     public void setEnableBm25(boolean value) {
         enableBm25 = Optional.of(value);
     }
 
+    public void setDistanceMetric(String value) {
+        this.distanceMetric = Optional.of(value);
+    }
+
+    public void setHnswIndexParams(HnswIndexParams.Builder params) {
+        this.hnswIndexParams = Optional.of(params);
+    }
+
 }
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexingRewriteOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexingRewriteOperation.java
index a0d47d7fa81..0a29fae04bf 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexingRewriteOperation.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexingRewriteOperation.java
@@ -4,9 +4,10 @@ package com.yahoo.searchdefinition.fieldoperation;
 import com.yahoo.searchdefinition.document.SDField;
 
 /**
- * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
+ * @author Einar M R Rosenvinge
  */
 public class IndexingRewriteOperation implements FieldOperation {
-    public void apply(SDField field) {
-    }
+
+    public void apply(SDField field) { }
+
 }
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java
index d1eb18c4916..0ffd13927b4 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java
@@ -71,7 +71,7 @@ public class AddExtraFieldsToDocument extends Processor {
         if (docField == null) {
             ImmutableSDField existingField = search.getField(field.getName());
             if (existingField == null) {
-                SDField newField = new SDField(document, field.getName(), field.getDataType(), field.isHeader(), true);
+                SDField newField = new SDField(document, field.getName(), field.getDataType(), true);
                 newField.setIsExtraField(true);
                 document.addField(newField);
             } else if (!existingField.isImportedField()) {
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolver.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolver.java
index d6c334ee80b..6d3de23238d 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolver.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolver.java
@@ -64,7 +64,7 @@ public class ImportedFieldsResolver extends Processor {
     }
 
     private void resolveImportedPositionField(TemporaryImportedField importedField, DocumentReference reference,
-                                                   ImmutableSDField targetField, boolean validate) {
+                                              ImmutableSDField targetField, boolean validate) {
         TemporaryImportedField importedZCurveField = new TemporaryImportedField(PositionDataType.getZCurveFieldName(importedField.fieldName()),
                 reference.referenceField().getName(), PositionDataType.getZCurveFieldName(targetField.getName()));
         ImmutableSDField targetZCurveField = getTargetField(importedZCurveField, reference);
@@ -175,7 +175,7 @@ public class ImportedFieldsResolver extends Processor {
     }
 
     private void validateTargetField(TemporaryImportedField importedField,
-                                                 ImmutableSDField targetField, DocumentReference reference) {
+                                     ImmutableSDField targetField, DocumentReference reference) {
         if (!targetField.doesAttributing()) {
             fail(importedField, targetFieldAsString(targetField.getName(), reference) +
                     ": Is not an attribute field. Only attribute fields supported");
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolver.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolver.java
index 9a9e9bbba5f..89b8889b4ae 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolver.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolver.java
@@ -15,7 +15,10 @@ import com.yahoo.tensor.TensorType;
 import com.yahoo.tensor.evaluation.TypeContext;
 import com.yahoo.vespa.model.container.search.QueryProfiles;
 
+import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Map;
+import java.util.Set;
 import java.util.logging.Level;
 
 /**
@@ -45,9 +48,10 @@ public class RankingExpressionTypeResolver extends Processor {
     public void process(boolean validate, boolean documentsOnly) {
         if (documentsOnly) return;
 
+        Set<Reference> warnedAbout = new HashSet<>();
         for (RankProfile profile : rankProfileRegistry.rankProfilesOf(search)) {
             try {
-                resolveTypesIn(profile, validate);
+                resolveTypesIn(profile, validate, warnedAbout);
             }
             catch (IllegalArgumentException e) {
                 throw new IllegalArgumentException("In " + (search != null ? search + ", " : "") + profile, e);
@@ -60,7 +64,7 @@ public class RankingExpressionTypeResolver extends Processor {
      *
      * @throws IllegalArgumentException if validate is true and the given rank profile does not produce valid types
      */
-    private void resolveTypesIn(RankProfile profile, boolean validate) {
+    private void resolveTypesIn(RankProfile profile, boolean validate, Set<Reference> warnedAbout) {
         MapEvaluationTypeContext context = profile.typeContext(queryProfiles);
         for (Map.Entry<String, RankProfile.RankingExpressionFunction> function : profile.getFunctions().entrySet()) {
             ExpressionFunction expressionFunction = function.getValue().function();
@@ -83,10 +87,14 @@ public class RankingExpressionTypeResolver extends Processor {
             profile.getSummaryFeatures().forEach(f -> resolveType(f, "summary feature " + f, context));
             ensureValidDouble(profile.getFirstPhaseRanking(), "first-phase expression", context);
             ensureValidDouble(profile.getSecondPhaseRanking(), "second-phase expression", context);
-            if ( context.tensorsAreUsed() && ! context.queryFeaturesNotDeclared().isEmpty()) {
-                deployLogger.log(Level.WARNING, "The following query features are not declared in query profile " +
+            if ( context.tensorsAreUsed() &&
+                 ! context.queryFeaturesNotDeclared().isEmpty() &&
+                 ! warnedAbout.containsAll(context.queryFeaturesNotDeclared())) {
+                deployLogger.log(Level.WARNING, "The following query features used in '" + profile.getName() +
+                                                "' are not declared in query profile " +
                                                 "types and will be interpreted as scalars, not tensors: " +
                                                 context.queryFeaturesNotDeclared());
+                warnedAbout.addAll(context.queryFeaturesNotDeclared());
             }
         }
     }
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java
index 8e54d7c00d6..c97ee2bd935 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java
@@ -6,7 +6,8 @@ import com.yahoo.document.CollectionDataType;
 import com.yahoo.document.TensorDataType;
 import com.yahoo.searchdefinition.RankProfileRegistry;
 import com.yahoo.searchdefinition.Search;
-import com.yahoo.searchdefinition.document.Attribute;
+import com.yahoo.searchdefinition.document.HnswIndexParams;
+import com.yahoo.searchdefinition.document.ImmutableSDField;
 import com.yahoo.searchdefinition.document.SDField;
 import com.yahoo.vespa.model.container.search.QueryProfiles;
 
@@ -23,34 +24,71 @@ public class TensorFieldProcessor extends Processor {
 
     @Override
     public void process(boolean validate, boolean documentsOnly) {
-        if ( ! validate) return;
-
-        for (SDField field : search.allConcreteFields()) {
+        for (var field : search.allConcreteFields()) {
             if ( field.getDataType() instanceof TensorDataType ) {
-                validateIndexingScripsForTensorField(field);
-                validateAttributeSettingForTensorField(field);
+                if (validate) {
+                    validateIndexingScripsForTensorField(field);
+                    validateAttributeSettingForTensorField(field);
+                }
+                processIndexSettingsForTensorField(field, validate);
             }
             else if (field.getDataType() instanceof CollectionDataType){
-                validateDataTypeForCollectionField(field);
+                if (validate) {
+                    validateDataTypeForCollectionField(field);
+                }
             }
         }
     }
 
     private void validateIndexingScripsForTensorField(SDField field) {
-        if (field.doesIndexing()) {
-            fail(search, field, "A field of type 'tensor' cannot be specified as an 'index' field.");
+        if (field.doesIndexing() && !isTensorTypeThatSupportsHnswIndex(field)) {
+            fail(search, field, "A tensor of type '" + tensorTypeToString(field) + "' does not support having an 'index'. " +
+                    "Currently, only tensors with 1 indexed dimension supports that.");
+        }
+    }
+
+    private boolean isTensorTypeThatSupportsHnswIndex(ImmutableSDField field) {
+        var type = ((TensorDataType)field.getDataType()).getTensorType();
+        // Tensors with 1 indexed dimension supports a hnsw index (used for approximate nearest neighbor search).
+        if ((type.dimensions().size() == 1) &&
+                type.dimensions().get(0).isIndexed()) {
+            return true;
         }
+        return false;
+    }
+
+    private String tensorTypeToString(ImmutableSDField field) {
+        return ((TensorDataType)field.getDataType()).getTensorType().toString();
     }
 
     private void validateAttributeSettingForTensorField(SDField field) {
         if (field.doesAttributing()) {
-            Attribute attribute = field.getAttributes().get(field.getName());
+            var attribute = field.getAttributes().get(field.getName());
             if (attribute != null && attribute.isFastSearch()) {
                 fail(search, field, "An attribute of type 'tensor' cannot be 'fast-search'.");
             }
         }
     }
 
+    private void processIndexSettingsForTensorField(SDField field, boolean validate) {
+        if (!field.doesIndexing()) {
+            return;
+        }
+        if (isTensorTypeThatSupportsHnswIndex(field)) {
+            if (validate && !field.doesAttributing()) {
+                fail(search, field, "A tensor that has an index must also be an attribute.");
+            }
+            var index = field.getIndex(field.getName());
+            // TODO: Calculate default params based on tensor dimension size
+            var params = new HnswIndexParams();
+            if (index != null) {
+                params = params.overrideFrom(index.getHnswIndexParams());
+                field.getAttribute().setDistanceMetric(index.getDistanceMetric());
+            }
+            field.getAttribute().setHnswIndexParams(params);
+        }
+    }
+
     private void validateDataTypeForCollectionField(SDField field) {
         if (((CollectionDataType)field.getDataType()).getNestedType() instanceof TensorDataType)
             fail(search, field, "A field with collection type of tensor is not supported. Use simple type 'tensor' instead.");
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/UriHack.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/UriHack.java
index d0a0bbfb748..d6398bc348c 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/UriHack.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/UriHack.java
@@ -61,7 +61,7 @@ public class UriHack extends Processor {
             String partName = uriName + "." + suffix;
             // I wonder if this is explicit in qrs or implicit in backend?
             // search.addFieldSetItem(uriName, partName);
-            SDField partField = new SDField(partName, generatedType, true);
+            SDField partField = new SDField(partName, generatedType);
             partField.setIndexStructureField(uriField.doesIndexing());
             partField.setRankType(uriField.getRankType());
             partField.setStemming(Stemming.NONE);
diff --git a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java
index 02d500931d7..e0d015cc8b2 100644
--- a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java
+++ b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java
@@ -62,6 +62,7 @@ public class DocumentManager {
             }
         }
     }
+
     private void buildConfig(Collection<AnnotationType> types, DocumentmanagerConfig.Builder builder) {
         for (AnnotationType type : types) {
             DocumentmanagerConfig.Annotationtype.Builder atb = new DocumentmanagerConfig.Annotationtype.Builder();
@@ -110,6 +111,7 @@ public class DocumentManager {
                 doc.inherits(new Datatype.Documenttype.Inherits.Builder().name(inherited.getName()));
             }
             buildConfig(dt.getFieldSets(), doc);
+            buildImportedFieldsConfig(dt.getImportedFieldNames(), doc);
         } else if (type instanceof TemporaryStructuredDataType) {
             //Ignored
         } else if (type instanceof StructDataType) {
@@ -164,4 +166,12 @@ public class DocumentManager {
         doc.fieldsets(fs.getName(), new Datatype.Documenttype.Fieldsets.Builder().fields(fs.getFieldNames()));
     }
 
+    private void buildImportedFieldsConfig(Collection<String> fieldNames, Datatype.Documenttype.Builder builder) {
+        for (String fieldName : fieldNames) {
+            var ib = new DocumentmanagerConfig.Datatype.Documenttype.Importedfield.Builder();
+            ib.name(fieldName);
+            builder.importedfield(ib);
+        }
+    }
+
 }
diff --git a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java
index d4228b52746..e6bf826dccc 100644
--- a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java
+++ b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java
@@ -58,6 +58,7 @@ public class DocumentTypes {
             buildConfig(annotation, atb);
         }
         buildConfig(documentType.getFieldSets(), db);
+        buildImportedFieldsConfig(documentType.getImportedFieldNames(), db);
         builder.documenttype(db);
     }
 
@@ -120,6 +121,14 @@ public class DocumentTypes {
         }
     }
 
+    private void buildImportedFieldsConfig(Collection<String> fieldNames, DocumenttypesConfig.Documenttype.Builder  builder) {
+        for (String fieldName : fieldNames) {
+            var ib = new DocumenttypesConfig.Documenttype.Importedfield.Builder();
+            ib.name(fieldName);
+            builder.importedfield(ib);
+        }
+    }
+
     private void buildConfig(StructDataType type,
                              DocumenttypesConfig.Documenttype.Datatype.Builder dataTypeBuilder,
                              DocumenttypesConfig.Documenttype.Builder documentBuilder,
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java b/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java
index 72c475fb091..75b33e184d9 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java
@@ -461,6 +461,7 @@ public abstract class AbstractService extends AbstractConfigProducer<AbstractCon
     public FileReference sendFile(String relativePath) {
         return getRoot().getFileDistributor().sendFileToHost(relativePath, getHost());
     }
+
     public FileReference sendUri(String uri) {
         return getRoot().getFileDistributor().sendUriToHost(uri, getHost());
     }
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/ConfigProducer.java b/config-model/src/main/java/com/yahoo/vespa/model/ConfigProducer.java
index bf86bc4a453..48cd378a9a6 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/ConfigProducer.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/ConfigProducer.java
@@ -1,16 +1,14 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.model;
 
-import java.io.File;
-import java.io.IOException;
-import java.io.PrintStream;
-import java.util.List;
-import java.util.Map;
-
 import com.yahoo.config.ConfigInstance;
 import com.yahoo.config.ConfigInstance.Builder;
 import com.yahoo.config.model.producer.UserConfigRepo;
 
+import java.io.PrintStream;
+import java.util.List;
+import java.util.Map;
+
 /**
  * Interface that should be implemented by all config producing modules
  * in the vespa model.
@@ -35,17 +33,6 @@ public interface ConfigProducer extends com.yahoo.config.ConfigInstance.Producer
     List<Service> getDescendantServices();
 
     /**
-     * Writes files that need to be written.  The files will usually
-     * only be written when the Vespa model is generated through the
-     * deploy-application script.
-     * This is primarily intended for debugging.
-     * 
-     * @param directory directory to write files to
-     * @throws java.io.IOException if writing fails
-     */
-    void writeFiles(File directory) throws IOException;
-
-    /**
      * Dump the three of config producers to the specified stream.
      * 
      * @param out The stream to print to, e.g. System.out
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/Host.java b/config-model/src/main/java/com/yahoo/vespa/model/Host.java
index a952d63526b..db469c2091e 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/Host.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/Host.java
@@ -1,12 +1,11 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.model;
 
-import java.io.File;
-import java.util.Objects;
-
 import com.yahoo.cloud.config.SentinelConfig;
 import com.yahoo.config.model.producer.AbstractConfigProducer;
 
+import java.util.Objects;
+
 /**
  * A physical host, running a set of services.
  * The identity of a host is its hostname. Hosts are comparable on their host name.
@@ -67,10 +66,6 @@ public final class Host extends AbstractConfigProducer<AbstractConfigProducer<?>
     }
 
     @Override
-    public void writeFiles(File directory) {
-    }
-
-    @Override
     public void getConfig(SentinelConfig.Builder builder) {
         // TODO (MAJOR_RELEASE): This shouldn't really be here, but we need to make sure users can upgrade if we change sentinel to use hosts/<hostname>/sentinel instead of hosts/<hostname>
         // as config id. We should probably wait for a major release
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java b/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java
index eda562bea5a..557a61ec211 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java
@@ -11,7 +11,6 @@ import com.yahoo.config.provision.HostSpec;
 import com.yahoo.config.provision.ProvisionLogger;
 
 import java.net.UnknownHostException;
-import java.util.Collections;
 import java.util.LinkedHashMap;
 import java.util.LinkedHashSet;
 import java.util.List;
@@ -34,13 +33,11 @@ public class HostSystem extends AbstractConfigProducer<Host> {
 
     private static Logger log = Logger.getLogger(HostSystem.class.getName());
 
-    private Map<String,String> hostnames = new LinkedHashMap<>();
-
     private final Map<String, HostResource> hostname2host = new LinkedHashMap<>();
     private final HostProvisioner provisioner;
     private final DeployLogger deployLogger;
 
-    public HostSystem(AbstractConfigProducer parent, String name, HostProvisioner provisioner, DeployLogger deployLogger) {
+    public HostSystem(AbstractConfigProducer<?> parent, String name, HostProvisioner provisioner, DeployLogger deployLogger) {
         super(parent, name);
         this.provisioner = provisioner;
         this.deployLogger = deployLogger;
@@ -49,7 +46,8 @@ public class HostSystem extends AbstractConfigProducer<Host> {
     void checkName(String hostname) {
         // Give a warning if the host does not exist
         try {
-            Object address = java.net.InetAddress.getByName(hostname);
+            @SuppressWarnings("unused")
+            Object ignore = java.net.InetAddress.getByName(hostname);
         } catch (UnknownHostException e) {
             deployLogger.log(Level.WARNING, "Unable to lookup IP address of host: " + hostname);
         }
@@ -78,36 +76,10 @@ public class HostSystem extends AbstractConfigProducer<Host> {
         return hostname2host.get(name);
     }
 
-    /**
-     * Returns the canonical name of a given host. This will cache names for faster lookup.
-     *
-     * @param hostname the hostname to retrieve the canonical hostname for.
-     * @return The canonical hostname, or null if unable to resolve.
-     * @throws UnknownHostException if the hostname cannot be resolved
-     */
-    public String getCanonicalHostname(String hostname) throws UnknownHostException {
-        if ( ! hostnames.containsKey(hostname)) {
-            hostnames.put(hostname, lookupCanonicalHostname(hostname));
-        }
-        return hostnames.get(hostname);
-    }
-
-    /**
-     * Static helper method that looks up the canonical name of a given host.
-     *
-     * @param hostname the hostname to retrieve the canonical hostname for.
-     * @return The canonical hostname, or null if unable to resolve.
-     * @throws UnknownHostException if the hostname cannot be resolved
-     */
-    // public - This is used by amenders outside this repo
-    public static String lookupCanonicalHostname(String hostname) throws UnknownHostException {
-        return java.net.InetAddress.getByName(hostname).getCanonicalHostName();
-    }
-
     @Override
     public String toString() {
         return "hosts [" + hostname2host.values().stream()
-                                                 .map(host -> host.getHostname())
+                                                 .map(HostResource::getHostname)
                                                  .collect(Collectors.joining(", ")) +
                "]";
     }
@@ -139,8 +111,8 @@ public class HostSystem extends AbstractConfigProducer<Host> {
         }
     }
 
-    public Map<HostResource, ClusterMembership> allocateHosts(ClusterSpec cluster, Capacity capacity, int groups, DeployLogger logger) {
-        List<HostSpec> allocatedHosts = provisioner.prepare(cluster, capacity, groups, new ProvisionDeployLogger(logger));
+    public Map<HostResource, ClusterMembership> allocateHosts(ClusterSpec cluster, Capacity capacity, DeployLogger logger) {
+        List<HostSpec> allocatedHosts = provisioner.prepare(cluster, capacity, new ProvisionDeployLogger(logger));
         // TODO: Even if HostResource owns a set of memberships, we need to return a map because the caller needs the current membership.
         Map<HostResource, ClusterMembership> retAllocatedHosts = new LinkedHashMap<>();
         for (HostSpec spec : allocatedHosts) {
@@ -169,7 +141,7 @@ public class HostSystem extends AbstractConfigProducer<Host> {
     }
 
     Set<HostSpec> getHostSpecs() {
-        return getHosts().stream().map(host -> host.spec()).collect(Collectors.toCollection(LinkedHashSet::new));
+        return getHosts().stream().map(HostResource::spec).collect(Collectors.toCollection(LinkedHashSet::new));
     }
 
     /** A provision logger which forwards to a deploy logger */
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java
index 2cfd2e23d08..6f2123f248c 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java
@@ -23,11 +23,13 @@ import com.yahoo.config.model.api.FileDistribution;
 import com.yahoo.config.model.api.HostInfo;
 import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels;
 import com.yahoo.config.model.api.Model;
+import com.yahoo.config.model.api.Provisioned;
 import com.yahoo.config.model.deploy.DeployState;
 import com.yahoo.config.model.producer.AbstractConfigProducer;
 import com.yahoo.config.model.producer.AbstractConfigProducerRoot;
 import com.yahoo.config.model.producer.UserConfigRepo;
 import com.yahoo.config.provision.AllocatedHosts;
+import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.log.LogLevel;
 import com.yahoo.searchdefinition.RankProfile;
 import com.yahoo.searchdefinition.RankProfileRegistry;
@@ -61,7 +63,6 @@ import com.yahoo.vespa.model.utils.internal.ReflectionUtil;
 import com.yahoo.yolean.Exceptions;
 import org.xml.sax.SAXException;
 
-import java.io.File;
 import java.io.IOException;
 import java.io.Serializable;
 import java.lang.reflect.Constructor;
@@ -123,6 +124,8 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri
 
     private final FileDistributor fileDistributor;
 
+    private final Provisioned provisioned;
+
     /** Creates a Vespa Model from internal model types only */
     public VespaModel(ApplicationPackage app) throws IOException, SAXException {
         this(app, new NullConfigModelRegistry());
@@ -163,6 +166,7 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri
         configModelRegistry = new VespaConfigModelRegistry(configModelRegistry);
         VespaModelBuilder builder = new VespaDomBuilder();
         this.applicationPackage = deployState.getApplicationPackage();
+        this.provisioned = deployState.provisioned();
         root = builder.getRoot(VespaModel.ROOT_CONFIGID, deployState, this);
 
         createGlobalRankProfiles(deployState.getDeployLogger(), deployState.getImportedModels(),
@@ -205,7 +209,8 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri
 
     /** Creates a mutable model with no services instantiated */
     public static VespaModel createIncomplete(DeployState deployState) throws IOException, SAXException {
-        return new VespaModel(new NullConfigModelRegistry(), deployState, false, new FileDistributor(deployState.getFileRegistry(), null));
+        return new VespaModel(new NullConfigModelRegistry(), deployState, false,
+                              new FileDistributor(deployState.getFileRegistry(), List.of(), deployState.isHosted()));
     }
 
     private void validateWrapExceptions() {
@@ -564,23 +569,6 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri
         id2producer.put(configId, descendant);
     }
 
-    /**
-     * Writes MODEL.cfg files for all config producers.
-     *
-     * @param baseDirectory dir to write files to
-     */
-    public void writeFiles(File baseDirectory) throws IOException {
-        super.writeFiles(baseDirectory);
-        for (ConfigProducer cp : id2producer.values()) {
-            try {
-                File destination = new File(baseDirectory, cp.getConfigId().replace("/", File.separator));
-                cp.writeFiles(destination);
-            } catch (IOException e) {
-                throw new IOException(cp.getConfigId() + ": " + e.getMessage());
-            }
-        }
-    }
-
     public Clients getClients() {
         return configModelRepo.getClients();
     }
@@ -629,11 +617,23 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri
         return Collections.unmodifiableMap(id2producer);
     }
 
-    /**
-     * Returns this root's model repository
-     */
+    /** Returns this root's model repository */
     public ConfigModelRepo configModelRepo() {
         return configModelRepo;
     }
 
+    /** If provisioning through the node repo, returns the provision requests issued during build of this */
+    public Provisioned provisioned() { return provisioned; }
+
+    /** Returns the id of all clusters in this */
+    public Set<ClusterSpec.Id> allClusters() {
+        return hostSystem().getHosts().stream()
+                                      .map(HostResource::spec)
+                                      .filter(spec -> spec.membership().isPresent())
+                                      .map(spec -> spec.membership().get().cluster().id())
+                                      .collect(Collectors.toSet());
+    }
+
+
+
 }
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java b/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java
index 54807697da4..631f4dab1a7 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java
@@ -146,11 +146,13 @@ public class VespaModelFactory implements ModelFactory {
             .properties(modelContext.properties())
             .vespaVersion(version())
             .modelHostProvisioner(createHostProvisioner(modelContext))
+            .provisioned(modelContext.provisioned())
             .endpoints(modelContext.properties().endpoints())
             .modelImporters(modelImporters)
             .zone(zone)
             .now(clock.instant())
-            .wantedNodeVespaVersion(modelContext.wantedNodeVespaVersion());
+            .wantedNodeVespaVersion(modelContext.wantedNodeVespaVersion())
+            .wantedDockerImageRepo(modelContext.wantedDockerImageRepository());
         modelContext.previousModel().ifPresent(builder::previousModel);
         return builder.build(validationParameters);
     }
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainerCluster.java
index 5714d41ef67..24e88d7ef7d 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainerCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainerCluster.java
@@ -4,13 +4,14 @@ package com.yahoo.vespa.model.admin;
 import com.yahoo.config.model.deploy.DeployState;
 import com.yahoo.config.model.producer.AbstractConfigProducer;
 import com.yahoo.container.handler.ThreadpoolConfig;
+import com.yahoo.search.config.QrStartConfig;
 import com.yahoo.vespa.model.container.ContainerCluster;
 import com.yahoo.vespa.model.container.component.Handler;
 
 /**
  * @author hmusum
  */
-public class LogserverContainerCluster extends ContainerCluster<LogserverContainer> implements ThreadpoolConfig.Producer {
+public class LogserverContainerCluster extends ContainerCluster<LogserverContainer> {
 
     public LogserverContainerCluster(AbstractConfigProducer<?> parent, String name, DeployState deployState) {
         super(parent, name, name, deployState);
@@ -27,6 +28,12 @@ public class LogserverContainerCluster extends ContainerCluster<LogserverContain
         builder.maxthreads(10);
     }
 
+    @Override
+    public void getConfig(QrStartConfig.Builder builder) {
+        super.getConfig(builder);
+        builder.jvm.heapsize(384);
+    }
+
     protected boolean messageBusEnabled() { return false; }
 
     private void addLogHandler() {
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainerCluster.java
index 2de4e5f5950..41d9df414ea 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainerCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainerCluster.java
@@ -11,8 +11,7 @@ import com.yahoo.vespa.model.container.ContainerCluster;
  *
  * @author gjoranv
  */
-public class ClusterControllerContainerCluster extends ContainerCluster<ClusterControllerContainer> implements
-        ThreadpoolConfig.Producer
+public class ClusterControllerContainerCluster extends ContainerCluster<ClusterControllerContainer>
 {
     public ClusterControllerContainerCluster(AbstractConfigProducer<?> parent, String subId, String name, DeployState deployState) {
         super(parent, subId, name, deployState);
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java
index fd924eb2a0f..fccacc3210d 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java
@@ -2,6 +2,8 @@
 
 package com.yahoo.vespa.model.admin.metricsproxy;
 
+import ai.vespa.metricsproxy.http.metrics.MetricsV2Handler;
+import ai.vespa.metricsproxy.http.metrics.NodeInfoConfig;
 import ai.vespa.metricsproxy.metric.dimensions.NodeDimensions;
 import ai.vespa.metricsproxy.metric.dimensions.NodeDimensionsConfig;
 import ai.vespa.metricsproxy.metric.dimensions.PublicDimensions;
@@ -20,6 +22,7 @@ import java.util.Map;
 
 import static com.yahoo.config.model.api.container.ContainerServiceType.METRICS_PROXY_CONTAINER;
 import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerCluster.METRICS_PROXY_BUNDLE_NAME;
+import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerCluster.createMetricsHandler;
 
 /**
  * Container running a metrics proxy.
@@ -28,9 +31,11 @@ import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerClus
  */
 public class MetricsProxyContainer extends Container implements
         NodeDimensionsConfig.Producer,
+        NodeInfoConfig.Producer,
         RpcConnectorConfig.Producer,
         VespaServicesConfig.Producer
 {
+    public static final int BASEPORT = 19092;
 
     final boolean isHostedVespa;
 
@@ -46,6 +51,7 @@ public class MetricsProxyContainer extends Container implements
         addMetricsProxyComponent(NodeDimensions.class);
         addMetricsProxyComponent(RpcConnector.class);
         addMetricsProxyComponent(VespaServices.class);
+        addHandler(createMetricsHandler(MetricsV2Handler.class, MetricsV2Handler.V2_PATH));
     }
 
     @Override
@@ -53,8 +59,6 @@ public class MetricsProxyContainer extends Container implements
         return METRICS_PROXY_CONTAINER;
     }
 
-    static public int BASEPORT = 19092;
-
     @Override
     public int getWantedPort() {
         return BASEPORT;
@@ -121,7 +125,21 @@ public class MetricsProxyContainer extends Container implements
         }
     }
 
-    private void  addMetricsProxyComponent(Class<?> componentClass) {
+    @Override
+    public void getConfig(NodeInfoConfig.Builder builder) {
+        builder.role(getNodeRole())
+                .hostname(getHostName());
+    }
+
+    private String getNodeRole() {
+        String hostConfigId = getHost().getConfigId();
+        if (! isHostedVespa) return hostConfigId;
+        return getHostResource().spec().membership()
+                    .map(ClusterMembership::stringValue)
+                    .orElse(hostConfigId);
+    }
+
+    private void addMetricsProxyComponent(Class<?> componentClass) {
         addSimpleComponent(componentClass.getName(), null, METRICS_PROXY_BUNDLE_NAME);
     }
 
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java
index fcc6c3279de..20f2bfe6636 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java
@@ -20,6 +20,9 @@ import ai.vespa.metricsproxy.metric.dimensions.PublicDimensions;
 import ai.vespa.metricsproxy.rpc.RpcServer;
 import ai.vespa.metricsproxy.service.ConfigSentinelClient;
 import ai.vespa.metricsproxy.service.SystemPollerProvider;
+import ai.vespa.metricsproxy.telegraf.Telegraf;
+import ai.vespa.metricsproxy.telegraf.TelegrafConfig;
+import ai.vespa.metricsproxy.telegraf.TelegrafRegistry;
 import com.yahoo.config.model.deploy.DeployState;
 import com.yahoo.config.model.producer.AbstractConfigProducer;
 import com.yahoo.config.model.producer.AbstractConfigProducerRoot;
@@ -67,7 +70,7 @@ public class MetricsProxyContainerCluster extends ContainerCluster<MetricsProxyC
         ApplicationDimensionsConfig.Producer,
         ConsumersConfig.Producer,
         MonitoringConfig.Producer,
-        ThreadpoolConfig.Producer,
+        TelegrafConfig.Producer,
         MetricsNodesConfig.Producer
 {
     public static final Logger log = Logger.getLogger(MetricsProxyContainerCluster.class.getName());
@@ -116,14 +119,30 @@ public class MetricsProxyContainerCluster extends ContainerCluster<MetricsProxyC
 
         addHttpHandler(ApplicationMetricsHandler.class, ApplicationMetricsHandler.V1_PATH);
         addMetricsProxyComponent(ApplicationMetricsRetriever.class);
+
+        addTelegrafComponents();
     }
 
     private void addHttpHandler(Class<? extends ThreadedHttpRequestHandler> clazz, String bindingPath) {
+        Handler<AbstractConfigProducer<?>> metricsHandler = createMetricsHandler(clazz, bindingPath);
+        addComponent(metricsHandler);
+    }
+
+    static Handler<AbstractConfigProducer<?>> createMetricsHandler(Class<? extends ThreadedHttpRequestHandler> clazz, String bindingPath) {
         Handler<AbstractConfigProducer<?>> metricsHandler = new Handler<>(
                 new ComponentModel(clazz.getName(), null, METRICS_PROXY_BUNDLE_NAME, null));
         metricsHandler.addServerBindings("http://*" + bindingPath,
                                          "http://*" + bindingPath + "/*");
-        addComponent(metricsHandler);
+        return metricsHandler;
+    }
+
+    private void addTelegrafComponents() {
+        getAdmin().ifPresent(admin -> {
+            if (admin.getUserMetrics().usesExternalMetricSystems()) {
+                addMetricsProxyComponent(Telegraf.class);
+                addMetricsProxyComponent(TelegrafRegistry.class);
+            }
+        });
     }
 
     @Override
@@ -156,6 +175,32 @@ public class MetricsProxyContainerCluster extends ContainerCluster<MetricsProxyC
     }
 
     @Override
+    public void getConfig(TelegrafConfig.Builder builder) {
+        builder.isHostedVespa(isHostedVespa());
+
+        var userConsumers = getUserMetricsConsumers();
+        for (var consumer : userConsumers.values()) {
+            for (var cloudWatch : consumer.cloudWatches()) {
+                var cloudWatchBuilder  = new TelegrafConfig.CloudWatch.Builder();
+                cloudWatchBuilder
+                        .region(cloudWatch.region())
+                        .namespace(cloudWatch.namespace())
+                        .consumer(cloudWatch.consumer());
+
+                cloudWatch.hostedAuth().ifPresent(hostedAuth -> cloudWatchBuilder
+                        .accessKeyName(hostedAuth.accessKeyName)
+                        .secretKeyName(hostedAuth.secretKeyName));
+
+                cloudWatch.sharedCredentials().ifPresent(sharedCredentials -> {
+                                                             cloudWatchBuilder.file(sharedCredentials.file);
+                                                             sharedCredentials.profile.ifPresent(cloudWatchBuilder::profile);
+                                                         });
+                builder.cloudWatch(cloudWatchBuilder);
+            }
+        }
+    }
+
+    @Override
     public void getConfig(ThreadpoolConfig.Builder builder) {
         builder.maxthreads(10);
     }
@@ -198,7 +243,7 @@ public class MetricsProxyContainerCluster extends ContainerCluster<MetricsProxyC
                 Optional.of(monitoring.getInterval()) : Optional.empty();
     }
 
-    private void  addMetricsProxyComponent(Class<?> componentClass) {
+    private void addMetricsProxyComponent(Class<?> componentClass) {
         addSimpleComponent(componentClass.getName(), null, METRICS_PROXY_BUNDLE_NAME);
     }
 
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/CloudWatch.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/CloudWatch.java
new file mode 100644
index 00000000000..5351c3fb3a7
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/CloudWatch.java
@@ -0,0 +1,59 @@
+package com.yahoo.vespa.model.admin.monitoring;
+
+import java.util.Optional;
+
+/**
+ * Helper object for CloudWatch configuration.
+ *
+ * @author gjoranv
+ */
+public class CloudWatch {
+    private final String region;
+    private final String namespace;
+    private final MetricsConsumer consumer;
+
+    private HostedAuth hostedAuth;
+    private SharedCredentials sharedCredentials;
+
+    public CloudWatch(String region, String namespace, MetricsConsumer consumer) {
+        this.region = region;
+        this.namespace = namespace;
+        this.consumer = consumer;
+    }
+
+    public String region() { return region; }
+    public String namespace() { return namespace; }
+    public String consumer() { return consumer.getId(); }
+
+    public Optional<HostedAuth> hostedAuth() {return Optional.ofNullable(hostedAuth); }
+    public Optional<SharedCredentials> sharedCredentials() {return Optional.ofNullable(sharedCredentials); }
+
+    public void setHostedAuth(String accessKeyName, String secretKeyName) {
+        hostedAuth = new HostedAuth(accessKeyName, secretKeyName);
+    }
+
+    public void setSharedCredentials(String file, Optional<String> profile) {
+        sharedCredentials = new SharedCredentials(file, profile);
+    }
+
+    public static class HostedAuth {
+        public final String accessKeyName;
+        public final String secretKeyName;
+
+        HostedAuth(String accessKeyName, String secretKeyName) {
+            this.accessKeyName = accessKeyName;
+            this.secretKeyName = secretKeyName;
+        }
+    }
+
+    public static class SharedCredentials {
+        public final String file;
+        public final Optional<String> profile;
+
+        SharedCredentials(String file, Optional<String> profile) {
+            this.file = file;
+            this.profile = profile;
+        }
+    }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/DefaultPublicMetrics.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/DefaultPublicMetrics.java
index 19a62f47e39..c80cebe3d5b 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/DefaultPublicMetrics.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/DefaultPublicMetrics.java
@@ -20,10 +20,12 @@ import static java.util.Collections.singleton;
  */
 public class DefaultPublicMetrics {
 
+    public static final String DEFAULT_METRIC_SET_ID = "default";
+
     public static MetricSet defaultPublicMetricSet = createMetricSet();
 
     private static MetricSet createMetricSet() {
-        return new MetricSet("public",
+        return new MetricSet(DEFAULT_METRIC_SET_ID,
                              getAllMetrics(),
                              singleton(defaultVespaMetricSet));
     }
@@ -84,6 +86,7 @@ public class DefaultPublicMetrics {
 
         metrics.add(new Metric("content.proton.documentdb.matching.docs_matched.rate"));
         metrics.add(new Metric("content.proton.documentdb.matching.docs_reranked.rate"));
+        metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_setup_time.average"));
         metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_latency.average"));
         metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.rerank_time.average"));
 
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/MetricsConsumer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/MetricsConsumer.java
index 529ed6ecf67..a8fbcf50b02 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/MetricsConsumer.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/MetricsConsumer.java
@@ -2,9 +2,13 @@
 package com.yahoo.vespa.model.admin.monitoring;
 
 import javax.annotation.concurrent.Immutable;
+import java.util.ArrayList;
+import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 
+import static java.util.Collections.unmodifiableList;
+
 /**
  * Represents an arbitrary metric consumer
  *
@@ -13,12 +17,15 @@ import java.util.Objects;
  */
 @Immutable
 public class MetricsConsumer {
+
     private final String id;
     private final MetricSet metricSet;
 
+    private final List<CloudWatch> cloudWatches = new ArrayList<>();
+
     /**
-     * @param id The consumer
-     * @param metricSet  The metrics for this consumer
+     * @param id the consumer
+     * @param metricSet the metrics for this consumer
      */
     public MetricsConsumer(String id, MetricSet metricSet) {
         this.id = Objects.requireNonNull(id, "A consumer must have a non-null id.");;
@@ -38,4 +45,12 @@ public class MetricsConsumer {
         return metricSet.getMetrics();
     }
 
+    public void addCloudWatch(CloudWatch cloudWatch) {
+        cloudWatches.add(cloudWatch);
+    }
+
+    public List<CloudWatch> cloudWatches() {
+        return unmodifiableList(cloudWatches);
+    }
+
 }
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/SystemMetrics.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/SystemMetrics.java
index 58b77ee1297..c05cad89852 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/SystemMetrics.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/SystemMetrics.java
@@ -9,6 +9,7 @@ import java.util.Set;
  * @author gjoranv
  */
 public class SystemMetrics {
+
     public static final String CPU_UTIL = "cpu.util";
     public static final String CPU_SYS_UTIL = "cpu.sys.util";
     public static final String CPU_THROTTLED_TIME = "cpu.throttled_time.rate";
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java
index 57d938f8a71..141794256b8 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java
@@ -72,6 +72,9 @@ public class VespaMetricSet {
         metrics.add(new Metric("vds.server.network.tls-connections-broken"));
         metrics.add(new Metric("vds.server.network.failed-tls-config-reloads"));
 
+        // C++ Fnet metrics
+        metrics.add(new Metric("vds.server.fnet.num-connections"));
+
         return metrics;
     }
 
@@ -126,6 +129,8 @@ public class VespaMetricSet {
         metrics.add(new Metric("serverActiveThreads.count"));
         metrics.add(new Metric("serverActiveThreads.last"));
 
+        metrics.add(new Metric("jdisc.thread_pool.unhandled_exceptions.rate"));
+
         metrics.add(new Metric("httpapi_latency.max"));
         metrics.add(new Metric("httpapi_latency.sum"));
         metrics.add(new Metric("httpapi_latency.count"));
@@ -179,6 +184,15 @@ public class VespaMetricSet {
         metrics.add(new Metric("jdisc.http.request.content_size.sum"));
         metrics.add(new Metric("jdisc.http.request.content_size.count"));
         metrics.add(new Metric("jdisc.http.request.content_size.average")); // TODO: Remove in Vespa 8
+
+        metrics.add(new Metric("jdisc.http.ssl.handshake.failure.missing_client_cert.rate"));
+        metrics.add(new Metric("jdisc.http.ssl.handshake.failure.expired_client_cert.rate"));
+        metrics.add(new Metric("jdisc.http.ssl.handshake.failure.invalid_client_cert.rate"));
+        metrics.add(new Metric("jdisc.http.ssl.handshake.failure.incompatible_protocols.rate"));
+        metrics.add(new Metric("jdisc.http.ssl.handshake.failure.incompatible_ciphers.rate"));
+        metrics.add(new Metric("jdisc.http.ssl.handshake.failure.unknown.rate"));
+
+        metrics.add(new Metric("jdisc.http.handler.unhandled_exceptions.rate"));
         return metrics;
     }
 
@@ -288,6 +302,14 @@ public class VespaMetricSet {
         return metrics;
     }
 
+    private static void addSearchNodeExecutorMetrics(Set<Metric> metrics, String prefix) {
+        metrics.add(new Metric(prefix + ".queuesize.max"));
+        metrics.add(new Metric(prefix + ".queuesize.sum"));
+        metrics.add(new Metric(prefix + ".queuesize.count"));
+        metrics.add(new Metric(prefix + ".maxpending.last")); // TODO: Remove in Vespa 8
+        metrics.add(new Metric(prefix + ".accepted.rate"));
+    }
+
     private static Set<Metric> getSearchNodeMetrics() {
         Set<Metric> metrics = new LinkedHashSet<>();
 
@@ -332,18 +354,12 @@ public class VespaMetricSet {
         metrics.add(new Metric("content.proton.search_protocol.docsum.requested_documents.count"));        
         
         // Executors shared between all document dbs
-        metrics.add(new Metric("content.proton.executor.proton.maxpending.last"));
-        metrics.add(new Metric("content.proton.executor.proton.accepted.rate"));
-        metrics.add(new Metric("content.proton.executor.flush.maxpending.last"));
-        metrics.add(new Metric("content.proton.executor.flush.accepted.rate"));
-        metrics.add(new Metric("content.proton.executor.match.maxpending.last"));
-        metrics.add(new Metric("content.proton.executor.match.accepted.rate"));
-        metrics.add(new Metric("content.proton.executor.docsum.maxpending.last"));
-        metrics.add(new Metric("content.proton.executor.docsum.accepted.rate"));
-        metrics.add(new Metric("content.proton.executor.shared.maxpending.last"));
-        metrics.add(new Metric("content.proton.executor.shared.accepted.rate"));
-        metrics.add(new Metric("content.proton.executor.warmup.maxpending.last"));
-        metrics.add(new Metric("content.proton.executor.warmup.accepted.rate"));
+        addSearchNodeExecutorMetrics(metrics, "content.proton.executor.proton");
+        addSearchNodeExecutorMetrics(metrics, "content.proton.executor.flush");
+        addSearchNodeExecutorMetrics(metrics, "content.proton.executor.match");
+        addSearchNodeExecutorMetrics(metrics, "content.proton.executor.docsum");
+        addSearchNodeExecutorMetrics(metrics, "content.proton.executor.shared");
+        addSearchNodeExecutorMetrics(metrics, "content.proton.executor.warmup");
 
         // jobs
         metrics.add(new Metric("content.proton.documentdb.job.total.average"));
@@ -357,18 +373,12 @@ public class VespaMetricSet {
         metrics.add(new Metric("content.proton.documentdb.job.removed_documents_prune.average"));
 
         // Threading service (per document db)
-        metrics.add(new Metric("content.proton.documentdb.threading_service.master.maxpending.last"));
-        metrics.add(new Metric("content.proton.documentdb.threading_service.master.accepted.rate"));
-        metrics.add(new Metric("content.proton.documentdb.threading_service.index.maxpending.last"));
-        metrics.add(new Metric("content.proton.documentdb.threading_service.index.accepted.rate"));
-        metrics.add(new Metric("content.proton.documentdb.threading_service.summary.maxpending.last"));
-        metrics.add(new Metric("content.proton.documentdb.threading_service.summary.accepted.rate"));
-        metrics.add(new Metric("content.proton.documentdb.threading_service.index_field_inverter.maxpending.last"));
-        metrics.add(new Metric("content.proton.documentdb.threading_service.index_field_inverter.accepted.rate"));
-        metrics.add(new Metric("content.proton.documentdb.threading_service.index_field_writer.maxpending.last"));
-        metrics.add(new Metric("content.proton.documentdb.threading_service.index_field_writer.accepted.rate"));
-        metrics.add(new Metric("content.proton.documentdb.threading_service.attribute_field_writer.maxpending.last"));
-        metrics.add(new Metric("content.proton.documentdb.threading_service.attribute_field_writer.accepted.rate"));
+        addSearchNodeExecutorMetrics(metrics, "content.proton.documentdb.threading_service.master");
+        addSearchNodeExecutorMetrics(metrics, "content.proton.documentdb.threading_service.index");
+        addSearchNodeExecutorMetrics(metrics, "content.proton.documentdb.threading_service.summary");
+        addSearchNodeExecutorMetrics(metrics, "content.proton.documentdb.threading_service.index_field_inverter");
+        addSearchNodeExecutorMetrics(metrics, "content.proton.documentdb.threading_service.index_field_writer");
+        addSearchNodeExecutorMetrics(metrics, "content.proton.documentdb.threading_service.attribute_field_writer");
 
         // lid space
         metrics.add(new Metric("content.proton.documentdb.ready.lid_space.lid_bloat_factor.average"));
@@ -453,10 +463,13 @@ public class VespaMetricSet {
         metrics.add(new Metric("content.proton.documentdb.matching.query_latency.sum"));
         metrics.add(new Metric("content.proton.documentdb.matching.query_latency.count"));
         metrics.add(new Metric("content.proton.documentdb.matching.query_latency.average")); // TODO: Remove in Vespa 8
-        metrics.add(new Metric("content.proton.documentdb.matching.query_collateral_time.max"));
-        metrics.add(new Metric("content.proton.documentdb.matching.query_collateral_time.sum"));
-        metrics.add(new Metric("content.proton.documentdb.matching.query_collateral_time.count"));
+        metrics.add(new Metric("content.proton.documentdb.matching.query_collateral_time.max")); // TODO: Remove in Vespa 8
+        metrics.add(new Metric("content.proton.documentdb.matching.query_collateral_time.sum")); // TODO: Remove in Vespa 8
+        metrics.add(new Metric("content.proton.documentdb.matching.query_collateral_time.count")); // TODO: Remove in Vespa 8
         metrics.add(new Metric("content.proton.documentdb.matching.query_collateral_time.average")); // TODO: Remove in Vespa 8
+        metrics.add(new Metric("content.proton.documentdb.matching.query_setup_time.max"));
+        metrics.add(new Metric("content.proton.documentdb.matching.query_setup_time.sum"));
+        metrics.add(new Metric("content.proton.documentdb.matching.query_setup_time.count"));
         metrics.add(new Metric("content.proton.documentdb.matching.docs_matched.rate"));
         metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.queries.rate"));
         metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.soft_doomed_queries.rate"));
@@ -468,10 +481,13 @@ public class VespaMetricSet {
         metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_latency.sum"));
         metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_latency.count"));
         metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_latency.average")); // TODO: Remove in Vespa 8
-        metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_collateral_time.max"));
-        metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_collateral_time.sum"));
-        metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_collateral_time.count"));
+        metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_collateral_time.max")); // TODO: Remove in Vespa 8
+        metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_collateral_time.sum")); // TODO: Remove in Vespa 8
+        metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_collateral_time.count")); // TODO: Remove in Vespa 8
         metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_collateral_time.average")); // TODO: Remove in Vespa 8
+        metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_setup_time.max"));
+        metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_setup_time.sum"));
+        metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.query_setup_time.count"));
         metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.rerank_time.max"));
         metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.rerank_time.sum"));
         metrics.add(new Metric("content.proton.documentdb.matching.rank_profile.rerank_time.count"));
@@ -513,13 +529,24 @@ public class VespaMetricSet {
         metrics.add(new Metric("vds.filestor.alldisks.averagequeuewait.sum.sum"));
         metrics.add(new Metric("vds.filestor.alldisks.averagequeuewait.sum.count"));
         metrics.add(new Metric("vds.filestor.alldisks.averagequeuewait.sum.average")); // TODO: Remove in Vespa 8
+        metrics.add(new Metric("vds.filestor.alldisks.allthreads.mergemetadatareadlatency.max"));
+        metrics.add(new Metric("vds.filestor.alldisks.allthreads.mergemetadatareadlatency.sum"));
+        metrics.add(new Metric("vds.filestor.alldisks.allthreads.mergemetadatareadlatency.count"));
+        metrics.add(new Metric("vds.filestor.alldisks.allthreads.mergedatareadlatency.max"));
+        metrics.add(new Metric("vds.filestor.alldisks.allthreads.mergedatareadlatency.sum"));
+        metrics.add(new Metric("vds.filestor.alldisks.allthreads.mergedatareadlatency.count"));
+        metrics.add(new Metric("vds.filestor.alldisks.allthreads.mergedatawritelatency.max"));
+        metrics.add(new Metric("vds.filestor.alldisks.allthreads.mergedatawritelatency.sum"));
+        metrics.add(new Metric("vds.filestor.alldisks.allthreads.mergedatawritelatency.count"));
 
         metrics.add(new Metric("vds.visitor.allthreads.queuesize.count.max"));
         metrics.add(new Metric("vds.visitor.allthreads.queuesize.count.sum"));
         metrics.add(new Metric("vds.visitor.allthreads.queuesize.count.count"));
         metrics.add(new Metric("vds.visitor.allthreads.queuesize.count.average")); // TODO: Remove in Vespa 8
-        metrics.add(new Metric("vds.visitor.allthreads.completed.sum.average"));
+        metrics.add(new Metric("vds.visitor.allthreads.completed.sum.average")); // TODO: Remove in Vespa 8
+        metrics.add(new Metric("vds.visitor.allthreads.completed.sum.rate"));
         metrics.add(new Metric("vds.visitor.allthreads.created.sum.rate"));
+        metrics.add(new Metric("vds.visitor.allthreads.failed.sum.rate"));
         metrics.add(new Metric("vds.visitor.allthreads.averagemessagesendtime.sum.max"));
         metrics.add(new Metric("vds.visitor.allthreads.averagemessagesendtime.sum.sum"));
         metrics.add(new Metric("vds.visitor.allthreads.averagemessagesendtime.sum.count"));
@@ -528,19 +555,27 @@ public class VespaMetricSet {
         metrics.add(new Metric("vds.visitor.allthreads.averageprocessingtime.sum.sum"));
         metrics.add(new Metric("vds.visitor.allthreads.averageprocessingtime.sum.count"));
         metrics.add(new Metric("vds.visitor.allthreads.averageprocessingtime.sum.average")); // TODO: Remove in Vespa 8
-
+        
+        metrics.add(new Metric("vds.filestor.alldisks.allthreads.put.sum.count.rate"));
+        metrics.add(new Metric("vds.filestor.alldisks.allthreads.put.sum.failed.rate"));
         metrics.add(new Metric("vds.filestor.alldisks.allthreads.put.sum.latency.max"));
         metrics.add(new Metric("vds.filestor.alldisks.allthreads.put.sum.latency.sum"));
         metrics.add(new Metric("vds.filestor.alldisks.allthreads.put.sum.latency.count"));
         metrics.add(new Metric("vds.filestor.alldisks.allthreads.put.sum.latency.average")); // TODO: Remove in Vespa 8
+        metrics.add(new Metric("vds.filestor.alldisks.allthreads.remove.sum.count.rate"));
+        metrics.add(new Metric("vds.filestor.alldisks.allthreads.remove.sum.failed.rate"));
         metrics.add(new Metric("vds.filestor.alldisks.allthreads.remove.sum.latency.max"));
         metrics.add(new Metric("vds.filestor.alldisks.allthreads.remove.sum.latency.sum"));
         metrics.add(new Metric("vds.filestor.alldisks.allthreads.remove.sum.latency.count"));
         metrics.add(new Metric("vds.filestor.alldisks.allthreads.remove.sum.latency.average")); // TODO: Remove in Vespa 8
+        metrics.add(new Metric("vds.filestor.alldisks.allthreads.get.sum.count.rate"));
+        metrics.add(new Metric("vds.filestor.alldisks.allthreads.get.sum.failed.rate"));
         metrics.add(new Metric("vds.filestor.alldisks.allthreads.get.sum.latency.max"));
         metrics.add(new Metric("vds.filestor.alldisks.allthreads.get.sum.latency.sum"));
         metrics.add(new Metric("vds.filestor.alldisks.allthreads.get.sum.latency.count"));
         metrics.add(new Metric("vds.filestor.alldisks.allthreads.get.sum.latency.average")); // TODO: Remove in Vespa 8
+        metrics.add(new Metric("vds.filestor.alldisks.allthreads.update.sum.count.rate"));
+        metrics.add(new Metric("vds.filestor.alldisks.allthreads.update.sum.failed.rate"));
         metrics.add(new Metric("vds.filestor.alldisks.allthreads.update.sum.latency.max"));
         metrics.add(new Metric("vds.filestor.alldisks.allthreads.update.sum.latency.sum"));
         metrics.add(new Metric("vds.filestor.alldisks.allthreads.update.sum.latency.count"));
@@ -560,13 +595,13 @@ public class VespaMetricSet {
         metrics.add(new Metric("vds.filestor.alldisks.allthreads.splitbuckets.count.rate"));
         metrics.add(new Metric("vds.filestor.alldisks.allthreads.joinbuckets.count.rate"));
         metrics.add(new Metric("vds.filestor.alldisks.allthreads.deletebuckets.count.rate"));
+        metrics.add(new Metric("vds.filestor.alldisks.allthreads.deletebuckets.failed.rate"));        
         metrics.add(new Metric("vds.filestor.alldisks.allthreads.deletebuckets.latency.max"));
         metrics.add(new Metric("vds.filestor.alldisks.allthreads.deletebuckets.latency.sum"));
         metrics.add(new Metric("vds.filestor.alldisks.allthreads.deletebuckets.latency.count"));
         metrics.add(new Metric("vds.filestor.alldisks.allthreads.deletebuckets.latency.average")); // TODO: Remove in Vespa 8
         metrics.add(new Metric("vds.filestor.alldisks.allthreads.setbucketstates.count.rate"));
 
-
         //Distributor
         metrics.add(new Metric("vds.idealstate.buckets_rechecking.average"));
         metrics.add(new Metric("vds.idealstate.idealstate_diff.average"));
@@ -589,6 +624,8 @@ public class VespaMetricSet {
         metrics.add(new Metric("vds.idealstate.garbage_collection.done_ok.rate"));
         metrics.add(new Metric("vds.idealstate.garbage_collection.done_failed.rate"));
         metrics.add(new Metric("vds.idealstate.garbage_collection.pending.average"));
+        metrics.add(new Metric("vds.idealstate.garbage_collection.documents_removed.count"));
+        metrics.add(new Metric("vds.idealstate.garbage_collection.documents_removed.rate"));
 
         metrics.add(new Metric("vds.distributor.puts.sum.latency.max"));
         metrics.add(new Metric("vds.distributor.puts.sum.latency.sum"));
@@ -596,18 +633,24 @@ public class VespaMetricSet {
         metrics.add(new Metric("vds.distributor.puts.sum.latency.average")); // TODO: Remove in Vespa 8
         metrics.add(new Metric("vds.distributor.puts.sum.ok.rate"));
         metrics.add(new Metric("vds.distributor.puts.sum.failures.total.rate"));
+        metrics.add(new Metric("vds.distributor.puts.sum.failures.notfound.rate"));
+        metrics.add(new Metric("vds.distributor.puts.sum.failures.test_and_set_failed.rate"));
         metrics.add(new Metric("vds.distributor.removes.sum.latency.max"));
         metrics.add(new Metric("vds.distributor.removes.sum.latency.sum"));
         metrics.add(new Metric("vds.distributor.removes.sum.latency.count"));
         metrics.add(new Metric("vds.distributor.removes.sum.latency.average")); // TODO: Remove in Vespa 8
         metrics.add(new Metric("vds.distributor.removes.sum.ok.rate"));
         metrics.add(new Metric("vds.distributor.removes.sum.failures.total.rate"));
+        metrics.add(new Metric("vds.distributor.removes.sum.failures.notfound.rate"));
+        metrics.add(new Metric("vds.distributor.removes.sum.failures.test_and_set_failed.rate"));
         metrics.add(new Metric("vds.distributor.updates.sum.latency.max"));
         metrics.add(new Metric("vds.distributor.updates.sum.latency.sum"));
         metrics.add(new Metric("vds.distributor.updates.sum.latency.count"));
         metrics.add(new Metric("vds.distributor.updates.sum.latency.average")); // TODO: Remove in Vespa 8
         metrics.add(new Metric("vds.distributor.updates.sum.ok.rate"));
         metrics.add(new Metric("vds.distributor.updates.sum.failures.total.rate"));
+        metrics.add(new Metric("vds.distributor.updates.sum.failures.notfound.rate"));
+        metrics.add(new Metric("vds.distributor.updates.sum.failures.test_and_set_failed.rate"));
         metrics.add(new Metric("vds.distributor.updates.sum.diverging_timestamp_updates.rate"));
         metrics.add(new Metric("vds.distributor.removelocations.sum.ok.rate"));
         metrics.add(new Metric("vds.distributor.removelocations.sum.failures.total.rate"));
@@ -617,6 +660,7 @@ public class VespaMetricSet {
         metrics.add(new Metric("vds.distributor.gets.sum.latency.average")); // TODO: Remove in Vespa 8
         metrics.add(new Metric("vds.distributor.gets.sum.ok.rate"));
         metrics.add(new Metric("vds.distributor.gets.sum.failures.total.rate"));
+        metrics.add(new Metric("vds.distributor.gets.sum.failures.notfound.rate"));
         metrics.add(new Metric("vds.distributor.visitor.sum.latency.max"));
         metrics.add(new Metric("vds.distributor.visitor.sum.latency.sum"));
         metrics.add(new Metric("vds.distributor.visitor.sum.latency.count"));
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/Metrics.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/Metrics.java
index 9b0d9dbfadc..1f81f16a80b 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/Metrics.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/Metrics.java
@@ -28,4 +28,13 @@ public class Metrics {
         return consumers.keySet().stream()
                 .anyMatch(existing -> existing.equalsIgnoreCase(id));
     }
+
+    /**
+     * Returns true if any of the consumers have specified external metric systems.
+     */
+    public boolean usesExternalMetricSystems() {
+        return consumers.values().stream()
+                .anyMatch(consumer -> ! consumer.cloudWatches().isEmpty());
+    }
+
 }
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/CloudWatchBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/CloudWatchBuilder.java
new file mode 100644
index 00000000000..314ef9cc5a5
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/CloudWatchBuilder.java
@@ -0,0 +1,40 @@
+package com.yahoo.vespa.model.admin.monitoring.builder.xml;
+
+import com.yahoo.vespa.model.admin.monitoring.CloudWatch;
+import com.yahoo.vespa.model.admin.monitoring.MetricsConsumer;
+import org.w3c.dom.Element;
+
+import static com.yahoo.config.model.builder.xml.XmlHelper.getOptionalAttribute;
+import static com.yahoo.config.model.builder.xml.XmlHelper.getOptionalChild;
+
+/**
+ * @author gjoranv
+ */
+public class CloudWatchBuilder {
+
+    private static final String REGION_ATTRIBUTE = "region";
+    private static final String NAMESPACE_ATTRIBUTE = "namespace";
+    private static final String CREDENTIALS_ELEMENT = "credentials";
+    private static final String ACCESS_KEY_ATTRIBUTE = "access-key-name";
+    private static final String SECRET_KEY_ATTRIBUTE = "secret-key-name";
+    private static final String SHARED_CREDENTIALS_ELEMENT = "shared-credentials";
+    private static final String PROFILE_ATTRIBUTE = "profile";
+    private static final String FILE_ATTRIBUTE = "file";
+
+    public static CloudWatch buildCloudWatch(Element cloudwatchElement, MetricsConsumer consumer) {
+        CloudWatch cloudWatch = new CloudWatch(cloudwatchElement.getAttribute(REGION_ATTRIBUTE),
+                                               cloudwatchElement.getAttribute(NAMESPACE_ATTRIBUTE),
+                                               consumer);
+
+        getOptionalChild(cloudwatchElement, CREDENTIALS_ELEMENT)
+                .ifPresent(elem -> cloudWatch.setHostedAuth(elem.getAttribute(ACCESS_KEY_ATTRIBUTE),
+                                                            elem.getAttribute(SECRET_KEY_ATTRIBUTE)));
+
+        getOptionalChild(cloudwatchElement, SHARED_CREDENTIALS_ELEMENT)
+                .ifPresent(elem -> cloudWatch.setSharedCredentials(elem.getAttribute(FILE_ATTRIBUTE),
+                                                                   getOptionalAttribute(elem, PROFILE_ATTRIBUTE)));
+
+        return cloudWatch;
+    }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/MetricsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/MetricsBuilder.java
index b13fa4917e4..b686288868f 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/MetricsBuilder.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/MetricsBuilder.java
@@ -42,7 +42,11 @@ public class MetricsBuilder {
             throwIfIllegalConsumerId(metrics, consumerId);
 
             MetricSet metricSet = buildMetricSet(consumerId, consumerElement);
-            metrics.addConsumer(new MetricsConsumer(consumerId, metricSet));
+            var consumer = new MetricsConsumer(consumerId, metricSet);
+            for (Element cloudwatchElement : XML.getChildren(consumerElement, "cloudwatch")) {
+                consumer.addCloudWatch(CloudWatchBuilder.buildCloudWatch(cloudwatchElement, consumer));
+            }
+            metrics.addConsumer(consumer);
         }
         return metrics;
     }
@@ -58,11 +62,11 @@ public class MetricsBuilder {
 
     private MetricSet buildMetricSet(String consumerId, Element consumerElement) {
         List<Metric> metrics = XML.getChildren(consumerElement, "metric").stream()
-                .map(metricElement -> metricFromElement(metricElement))
+                .map(MetricsBuilder::metricFromElement)
                 .collect(Collectors.toCollection(LinkedList::new));
 
         List<MetricSet> metricSets = XML.getChildren(consumerElement, "metric-set").stream()
-                .map(metricSetElement -> availableMetricSets.get(metricSetElement.getAttribute(ID_ATTRIBUTE)))
+                .map(metricSetElement -> getMetricSetOrThrow(metricSetElement.getAttribute(ID_ATTRIBUTE)))
                 .collect(Collectors.toCollection(LinkedList::new));
 
         metricSets.add(defaultVespaMetricSet);
@@ -75,6 +79,11 @@ public class MetricsBuilder {
         return "user-metrics-" + consumerName;
     }
 
+    private MetricSet getMetricSetOrThrow(String id) {
+        if (! availableMetricSets.containsKey(id)) throw new IllegalArgumentException("No such metric-set: " + id);
+        return availableMetricSets.get(id);
+    }
+
     private void throwIfIllegalConsumerId(Metrics metrics, String consumerId) {
         if (consumerId.equalsIgnoreCase(VESPA_CONSUMER_ID) && applicationType != ApplicationType.HOSTED_INFRASTRUCTURE)
             throw new IllegalArgumentException("'Vespa' is not allowed as metrics consumer id (case is ignored.)");
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/AwsAccessControlValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/AwsAccessControlValidator.java
new file mode 100644
index 00000000000..631ab0e2640
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/AwsAccessControlValidator.java
@@ -0,0 +1,48 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.application.validation;
+
+import com.yahoo.config.application.api.ValidationId;
+import com.yahoo.config.model.deploy.DeployState;
+import com.yahoo.config.provision.CloudName;
+import com.yahoo.vespa.model.VespaModel;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import static com.yahoo.collections.CollectionUtil.mkString;
+import static com.yahoo.vespa.model.application.validation.first.AccessControlOnFirstDeploymentValidator.needsAccessControlValidation;
+import static com.yahoo.vespa.model.container.http.AccessControl.hasHandlerThatNeedsProtection;
+
+/**
+ * @author gjoranv
+ */
+public class AwsAccessControlValidator extends Validator {
+
+    // NOTE: must be the same as the name in the declaration of the AWS cloud in hosted.
+    static final String AWS_CLOUD_NAME = "aws";
+
+    @Override
+    public void validate(VespaModel model, DeployState deployState) {
+
+        if (! needsAccessControlValidation(model, deployState)) return;
+        if(! deployState.zone().cloud().equals(CloudName.from(AWS_CLOUD_NAME))) return;
+
+        List<String> offendingClusters = new ArrayList<>();
+        for (var cluster : model.getContainerClusters().values()) {
+            var http = cluster.getHttp();
+            if (http == null
+                    || ! http.getAccessControl().isPresent()
+                    || ! http.getAccessControl().get().writeEnabled
+                    || ! http.getAccessControl().get().readEnabled)
+
+                if (hasHandlerThatNeedsProtection(cluster) || ! cluster.getAllServlets().isEmpty())
+                    offendingClusters.add(cluster.getName());
+        }
+        if (! offendingClusters.isEmpty())
+            deployState.validationOverrides()
+                    .invalid(ValidationId.accessControl,
+                             "Access-control must be enabled for read/write operations to container clusters in AWS production zones: " +
+                                     mkString(offendingClusters, "[", ", ", "]"), deployState.now());
+    }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/CloudWatchValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/CloudWatchValidator.java
new file mode 100644
index 00000000000..462ac39fa84
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/CloudWatchValidator.java
@@ -0,0 +1,37 @@
+package com.yahoo.vespa.model.application.validation;
+
+import com.yahoo.config.model.ConfigModelContext;
+import com.yahoo.config.model.deploy.DeployState;
+import com.yahoo.vespa.model.VespaModel;
+import com.yahoo.vespa.model.admin.monitoring.MetricsConsumer;
+
+import java.util.List;
+
+import static java.util.stream.Collectors.toList;
+
+/**
+ * @author gjoranv
+ */
+public class CloudWatchValidator extends Validator {
+
+    @Override
+    public void validate(VespaModel model, DeployState deployState) {
+        if (!deployState.isHosted()) return;
+        if (deployState.zone().system().isPublic()) return;
+        if (model.getAdmin().getApplicationType() != ConfigModelContext.ApplicationType.DEFAULT) return;
+
+        var offendingConsumers = model.getAdmin().getUserMetrics().getConsumers().values().stream()
+                .filter(consumer -> !consumer.cloudWatches().isEmpty())
+                .collect(toList());
+
+        if (! offendingConsumers.isEmpty()) {
+            throw new IllegalArgumentException("CloudWatch cannot be set up for non-public hosted Vespa and must " +
+                                                       "be removed for consumers: " + consumerIds(offendingConsumers));
+        }
+    }
+
+    private List<String> consumerIds(List<MetricsConsumer> offendingConsumers) {
+        return offendingConsumers.stream().map(MetricsConsumer::getId).collect(toList());
+    }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexAttributeFieldsValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexAttributeFieldsValidator.java
index f9762ce58fa..43c1a88b0a1 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexAttributeFieldsValidator.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexAttributeFieldsValidator.java
@@ -33,7 +33,7 @@ public class ComplexAttributeFieldsValidator extends Validator {
                 continue;
             }
             SearchCluster searchCluster = (SearchCluster) cluster;
-            for (AbstractSearchCluster.SearchDefinitionSpec spec : searchCluster.getLocalSDS()) {
+            for (AbstractSearchCluster.SchemaSpec spec : searchCluster.getLocalSDS()) {
                 validateComplexFields(searchCluster.getClusterName(), spec.getSearchDefinition().getSearch());
             }
         }
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/DeploymentSpecValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/DeploymentSpecValidator.java
index ac38336a405..d5bea2a2959 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/DeploymentSpecValidator.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/DeploymentSpecValidator.java
@@ -4,6 +4,7 @@ package com.yahoo.vespa.model.application.validation;
 import com.yahoo.config.application.api.DeploymentInstanceSpec;
 import com.yahoo.config.application.api.DeploymentSpec;
 import com.yahoo.config.model.deploy.DeployState;
+import com.yahoo.config.provision.InstanceName;
 import com.yahoo.vespa.model.VespaModel;
 import com.yahoo.vespa.model.container.ContainerModel;
 
@@ -12,8 +13,7 @@ import java.util.List;
 import java.util.Optional;
 
 /**
- * Validates that deployment spec (deployment.xml) has valid values (for now
- * only global-service-id is validated)
+ * Validate deployment spec (deployment.xml).
  *
  * @author hmusum
  * @author bratseth
@@ -30,11 +30,20 @@ public class DeploymentSpecValidator extends Validator {
         List<ContainerModel> containers = model.getRoot().configModelRepo().getModels(ContainerModel.class);
         for (DeploymentInstanceSpec instance : deploymentSpec.instances()) {
             instance.globalServiceId().ifPresent(globalServiceId -> {
-                if ( containers.stream().noneMatch(container -> container.getCluster().getName().equals(globalServiceId)))
-                    throw new IllegalArgumentException("The global-service-id in " + instance + ", '" + globalServiceId +
-                                                       "' specified in deployment.xml does not match any container cluster id");
+                requireClusterId(containers, instance.name(), "Attribute 'globalServiceId'", globalServiceId);
+            });
+            instance.endpoints().forEach(endpoint -> {
+                requireClusterId(containers, instance.name(), "Endpoint '" + endpoint.endpointId() + "'",
+                                 endpoint.containerId());
             });
         }
     }
 
+    private static void requireClusterId(List<ContainerModel> containers, InstanceName instanceName, String context,
+                                         String id) {
+        if (containers.stream().noneMatch(container -> container.getCluster().getName().equals(id)))
+            throw new IllegalArgumentException(context + " in instance " + instanceName + ": '" + id +
+                                               "' specified in deployment.xml does not match any container cluster ID");
+    }
+
 }
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/EndpointCertificateSecretsValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/EndpointCertificateSecretsValidator.java
index f00ad0f0dbb..4b8bbd4ff08 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/EndpointCertificateSecretsValidator.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/EndpointCertificateSecretsValidator.java
@@ -12,7 +12,7 @@ public class EndpointCertificateSecretsValidator extends Validator {
     @Override
     public void validate(VespaModel model, DeployState deployState) {
         if (deployState.endpointCertificateSecrets().isPresent() && deployState.endpointCertificateSecrets().get() == EndpointCertificateSecrets.MISSING) {
-            throw new CertificateNotReadyException("TLS enabled, but could not retrieve certificate yet");
+            throw new CertificateNotReadyException("TLS enabled, but could not yet retrieve certificate for application " + deployState.getProperties().applicationId().serializedForm());
         }
     }
 }
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java
index 7f8ff6edd85..b6f7ab4ff62 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java
@@ -33,24 +33,29 @@ import java.util.logging.Logger;
 
 /**
  * Validate rank setup for all search clusters (rank-profiles, index-schema, attributes configs), validating done
- * by running through the binary 'vespa-verify-ranksetup'
+ * by running the binary 'vespa-verify-ranksetup-bin'
  *
  * @author vegardh
  */
 public class RankSetupValidator extends Validator {
 
     private static final Logger log = Logger.getLogger(RankSetupValidator.class.getName());
-    private final boolean force;
+    private static final String binaryName = "vespa-verify-ranksetup-bin ";
 
-    public RankSetupValidator(boolean force) {
-        this.force = force;
+    private final boolean ignoreValidationErrors;
+
+    public RankSetupValidator(boolean ignoreValidationErrors) {
+        this.ignoreValidationErrors = ignoreValidationErrors;
     }
 
     @Override
     public void validate(VespaModel model, DeployState deployState) {
         File cfgDir = null;
         try {
-            cfgDir = Files.createTempDirectory("deploy_ranksetup").toFile();
+            cfgDir = Files.createTempDirectory("verify-ranksetup." +
+                                               deployState.getProperties().applicationId().toFullString() +
+                                               ".")
+                    .toFile();
 
             for (AbstractSearchCluster cluster : model.getSearchClusters()) {
                 // Skipping rank expression checking for streaming clusters, not implemented yet
@@ -100,29 +105,29 @@ public class RankSetupValidator extends Validator {
         IOUtils.recursiveDeleteDir(dir);
     }
 
-    private void writeConfigs(String dir, AbstractConfigProducer producer) throws IOException {
+    private void writeConfigs(String dir, AbstractConfigProducer<?> producer) throws IOException {
             RankProfilesConfig.Builder rpcb = new RankProfilesConfig.Builder();
-            RankProfilesConfig.Producer.class.cast(producer).getConfig(rpcb);
+            ((RankProfilesConfig.Producer) producer).getConfig(rpcb);
             RankProfilesConfig rpc = new RankProfilesConfig(rpcb);
             writeConfig(dir, RankProfilesConfig.getDefName() + ".cfg", rpc);
 
             IndexschemaConfig.Builder iscb = new IndexschemaConfig.Builder();
-            IndexschemaConfig.Producer.class.cast(producer).getConfig(iscb);
+            ((IndexschemaConfig.Producer) producer).getConfig(iscb);
             IndexschemaConfig isc = new IndexschemaConfig(iscb);
             writeConfig(dir, IndexschemaConfig.getDefName() + ".cfg", isc);
 
             AttributesConfig.Builder acb = new AttributesConfig.Builder();
-            AttributesConfig.Producer.class.cast(producer).getConfig(acb);
+            ((AttributesConfig.Producer) producer).getConfig(acb);
             AttributesConfig ac = new AttributesConfig(acb);
             writeConfig(dir, AttributesConfig.getDefName() + ".cfg", ac);
 
             RankingConstantsConfig.Builder rccb = new RankingConstantsConfig.Builder();
-            RankingConstantsConfig.Producer.class.cast(producer).getConfig(rccb);
+            ((RankingConstantsConfig.Producer) producer).getConfig(rccb);
             RankingConstantsConfig rcc = new RankingConstantsConfig(rccb);
             writeConfig(dir, RankingConstantsConfig.getDefName() + ".cfg", rcc);
 
             ImportedFieldsConfig.Builder ifcb = new ImportedFieldsConfig.Builder();
-            ImportedFieldsConfig.Producer.class.cast(producer).getConfig(ifcb);
+            ((ImportedFieldsConfig.Producer) producer).getConfig(ifcb);
             ImportedFieldsConfig ifc = new ImportedFieldsConfig(ifcb);
             writeConfig(dir, ImportedFieldsConfig.getDefName() + ".cfg", ifc);
     }
@@ -132,8 +137,8 @@ public class RankSetupValidator extends Validator {
     }
 
     private boolean execValidate(String configId, SearchCluster sc, String sdName, DeployLogger deployLogger) {
-        String job = "vespa-verify-ranksetup-bin " + configId;
-        ProcessExecuter executer = new ProcessExecuter();
+        String job = String.format("%s %s", binaryName, configId);
+        ProcessExecuter executer = new ProcessExecuter(true);
         try {
             Pair<Integer, String> ret = executer.exec(job);
             if (ret.getFirst() != 0) {
@@ -147,27 +152,28 @@ public class RankSetupValidator extends Validator {
     }
 
     private void validateWarn(Exception e, DeployLogger deployLogger) {
-        String msg = "Unable to execute 'vespa-verify-ranksetup', validation of rank expressions will only take place when you start Vespa: " +
+        String msg = "Unable to execute '"+ binaryName + "', validation of rank expressions will only take place when you start Vespa: " +
                 Exceptions.toMessageString(e);
         deployLogger.log(LogLevel.WARNING, msg);
     }
 
     private void validateFail(String output, SearchCluster sc, String sdName, DeployLogger deployLogger) {
-        String errMsg = "For search cluster '" + sc.getClusterName() + "', search definition '" + sdName + "': error in rank setup. Details:\n";
+        StringBuilder errMsg = new StringBuilder("For search cluster '").append(sc.getClusterName()).append("', ")
+                .append("search definition '").append(sdName).append("': error in rank setup. Details:\n");
         for (String line : output.split("\n")) {
             // Remove debug lines from start script
             if (line.startsWith("debug\t")) continue;
             try {
-                LogMessage logmsg = LogMessage.parseNativeFormat(line);
-                errMsg = errMsg + logmsg.getLevel() + ": " + logmsg.getPayload() + "\n";
+                LogMessage logMessage = LogMessage.parseNativeFormat(line);
+                errMsg.append(logMessage.getLevel()).append(": ").append(logMessage.getPayload()).append("\n");
             } catch (InvalidLogFormatException e) {
-                errMsg = errMsg + line + "\n";
+                errMsg.append(line).append("\n");
             }
         }
-        if (force) {
-            deployLogger.log(LogLevel.WARNING, errMsg + "(Continuing because of force.)");
+        if (ignoreValidationErrors) {
+            deployLogger.log(LogLevel.WARNING, errMsg.append("(Continuing since ignoreValidationErrors flag is set.)").toString());
         } else {
-            throw new IllegalArgumentException(errMsg);
+            throw new IllegalArgumentException(errMsg.toString());
         }
     }
 
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankingConstantsValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankingConstantsValidator.java
index 907418ea9f0..9568ea5c27c 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankingConstantsValidator.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankingConstantsValidator.java
@@ -9,7 +9,7 @@ import com.yahoo.path.Path;
 import com.yahoo.searchdefinition.RankingConstant;
 import com.yahoo.vespa.model.VespaModel;
 import com.yahoo.vespa.model.application.validation.ConstantTensorJsonValidator.InvalidConstantTensor;
-import com.yahoo.vespa.model.search.SearchDefinition;
+import com.yahoo.vespa.model.search.NamedSchema;
 
 import java.io.FileNotFoundException;
 
@@ -47,7 +47,7 @@ public class RankingConstantsValidator extends Validator {
         ApplicationPackage applicationPackage = deployState.getApplicationPackage();
         ExceptionMessageCollector exceptionMessageCollector = new ExceptionMessageCollector("Invalid constant tensor file(s):");
 
-        for (SearchDefinition sd : deployState.getSearchDefinitions()) {
+        for (NamedSchema sd : deployState.getSchemas()) {
             for (RankingConstant rc : sd.getSearch().rankingConstants().asMap().values()) {
                 try {
                     validateRankingConstant(rc, applicationPackage);
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/SearchDataTypeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/SearchDataTypeValidator.java
index 85ba75639eb..031ce0dbdd4 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/SearchDataTypeValidator.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/SearchDataTypeValidator.java
@@ -15,7 +15,7 @@ import com.yahoo.searchdefinition.document.SDDocumentType;
 import com.yahoo.searchdefinition.document.SDField;
 import com.yahoo.vespa.model.VespaModel;
 import com.yahoo.vespa.model.search.AbstractSearchCluster;
-import com.yahoo.vespa.model.search.SearchDefinition;
+import com.yahoo.vespa.model.search.NamedSchema;
 
 import java.util.List;
 
@@ -34,7 +34,7 @@ public class SearchDataTypeValidator extends Validator {
             if (cluster.isStreaming()) {
                 continue;
             }
-            for (AbstractSearchCluster.SearchDefinitionSpec spec : cluster.getLocalSDS()) {
+            for (AbstractSearchCluster.SchemaSpec spec : cluster.getLocalSDS()) {
                 SDDocumentType docType = spec.getSearchDefinition().getSearch().getDocument();
                 if (docType == null) {
                     continue;
@@ -44,7 +44,7 @@ public class SearchDataTypeValidator extends Validator {
         }
     }
 
-    private void validateDocument(AbstractSearchCluster cluster, SearchDefinition def, SDDocumentType doc) {
+    private void validateDocument(AbstractSearchCluster cluster, NamedSchema def, SDDocumentType doc) {
         for (SDDocumentType child : doc.getTypes()) {
             validateDocument(cluster, def, child);
         }
@@ -84,7 +84,7 @@ public class SearchDataTypeValidator extends Validator {
         }
     }
 
-    private void disallowIndexingOfMaps(AbstractSearchCluster cluster, SearchDefinition def, Field field) {
+    private void disallowIndexingOfMaps(AbstractSearchCluster cluster, NamedSchema def, Field field) {
         DataType fieldType = field.getDataType();
         if ((fieldType instanceof MapDataType) && (((SDField) field).doesIndexing())) {
             throw new IllegalArgumentException("Field type '" + fieldType.getName() + "' cannot be indexed for search " +
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java
index 1e4a45428b8..22dd0289390 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java
@@ -59,6 +59,8 @@ public class Validation {
         new SecretStoreValidator().validate(model, deployState);
         new EndpointCertificateSecretsValidator().validate(model, deployState);
         new AccessControlFilterValidator().validate(model, deployState);
+        new CloudWatchValidator().validate(model, deployState);
+        new AwsAccessControlValidator().validate(model, deployState);
 
         List<ConfigChangeAction> result = Collections.emptyList();
         if (deployState.getProperties().isFirstTimeDeployment()) {
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ClusterSizeReductionValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ClusterSizeReductionValidator.java
index d14fe91a53b..162f6798462 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ClusterSizeReductionValidator.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ClusterSizeReductionValidator.java
@@ -2,6 +2,8 @@
 package com.yahoo.vespa.model.application.validation.change;
 
 import com.yahoo.config.model.api.ConfigChangeAction;
+import com.yahoo.config.provision.Capacity;
+import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.vespa.model.VespaModel;
 import com.yahoo.config.application.api.ValidationId;
 import com.yahoo.config.application.api.ValidationOverrides;
@@ -21,35 +23,32 @@ public class ClusterSizeReductionValidator implements ChangeValidator {
 
     @Override
     public List<ConfigChangeAction> validate(VespaModel current, VespaModel next, ValidationOverrides overrides, Instant now) {
-        for (ContainerCluster currentCluster : current.getContainerClusters().values()) {
-            ContainerCluster nextCluster = next.getContainerClusters().get(currentCluster.getName());
-            if (nextCluster == null) continue;
-            validate(currentCluster.getContainers().size(),
-                     nextCluster.getContainers().size(),
-                     currentCluster.getName(),
+        for (var clusterId : current.allClusters()) {
+            Capacity currentCapacity = current.provisioned().all().get(clusterId);
+            Capacity nextCapacity = next.provisioned().all().get(clusterId);
+            if (currentCapacity == null || nextCapacity == null) continue;
+            validate(currentCapacity,
+                     nextCapacity,
+                     clusterId,
                      overrides,
                      now);
         }
-
-        for (ContentCluster currentCluster : current.getContentClusters().values()) {
-            ContentCluster nextCluster = next.getContentClusters().get(currentCluster.getName());
-            if (nextCluster == null) continue;
-            validate(currentCluster.getSearch().getSearchNodes().size(),
-                     nextCluster.getSearch().getSearchNodes().size(),
-                     currentCluster.getName(),
-                     overrides,
-                     now);
-        }
-
         return Collections.emptyList();
     }
 
-    private void validate(int currentSize, int nextSize, String clusterName, ValidationOverrides overrides, Instant now) {
+    private void validate(Capacity current,
+                          Capacity next,
+                          ClusterSpec.Id clusterId,
+                          ValidationOverrides overrides,
+                          Instant now) {
+        int currentSize = current.minResources().nodes();
+        int nextSize = next.minResources().nodes();
         // don't allow more than 50% reduction, but always allow to reduce size with 1
         if ( nextSize < ((double)currentSize) * 0.5 && nextSize != currentSize - 1)
             overrides.invalid(ValidationId.clusterSizeReduction,
-                              "Size reduction in '" + clusterName + "' is too large. Current size: " + currentSize +
-                              ", new size: " + nextSize + ". New size must be at least 50% of the current size",
+                              "Size reduction in '" + clusterId.value() + "' is too large: " +
+                              "New min size must be at least 50% of the current min size. " +
+                              "Current size: " + currentSize + ", new size: " + nextSize,
                               now);
     }
 
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidator.java
index 8fdcf249bbc..5343a322382 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidator.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidator.java
@@ -5,6 +5,7 @@ import com.yahoo.collections.Pair;
 import com.yahoo.config.application.api.ValidationId;
 import com.yahoo.config.application.api.ValidationOverrides;
 import com.yahoo.config.model.api.ConfigChangeAction;
+import com.yahoo.config.provision.Capacity;
 import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.config.provision.NodeResources;
 import com.yahoo.vespa.model.HostResource;
@@ -15,6 +16,7 @@ import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Optional;
+import java.util.Set;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
@@ -27,35 +29,43 @@ public class ResourcesReductionValidator implements ChangeValidator {
 
     @Override
     public List<ConfigChangeAction> validate(VespaModel current, VespaModel next, ValidationOverrides overrides, Instant now) {
-        var currentRequestedResourcesByClusterId = getRequestedResourcesByClusterId(current);
-        var nextRequestedResourcesByClusterId = getRequestedResourcesByClusterId(next);
-
-        for (var clusterTypeAndId : currentRequestedResourcesByClusterId.keySet()) {
-            if (!nextRequestedResourcesByClusterId.containsKey(clusterTypeAndId)) continue;
-            validate(currentRequestedResourcesByClusterId.get(clusterTypeAndId),
-                     nextRequestedResourcesByClusterId.get(clusterTypeAndId),
-                     clusterTypeAndId.getSecond(),
-                     overrides,
-                     now);
+        for (var clusterId : current.allClusters()) {
+            Capacity currentCapacity = current.provisioned().all().get(clusterId);
+            Capacity nextCapacity = next.provisioned().all().get(clusterId);
+            if (currentCapacity == null || nextCapacity == null) continue;
+            validate(currentCapacity, nextCapacity, clusterId, overrides, now);
         }
 
         return List.of();
     }
 
-    private void validate(NodeResources currentResources, NodeResources nextResources, ClusterSpec.Id clusterId,
-                          ValidationOverrides overrides, Instant now) {
+    private void validate(Capacity current,
+                          Capacity next,
+                          ClusterSpec.Id clusterId,
+                          ValidationOverrides overrides,
+                          Instant now) {
+        if (current.minResources().nodeResources() == NodeResources.unspecified) return;
+        if (next.minResources().nodeResources() == NodeResources.unspecified) return;
+
         List<String> illegalChanges = Stream.of(
-                validateResource("vCPU", currentResources.vcpu(), nextResources.vcpu()),
-                validateResource("memory GB", currentResources.memoryGb(), nextResources.memoryGb()),
-                validateResource("disk GB", currentResources.diskGb(), nextResources.diskGb()))
+                validateResource("vCPU",
+                                 current.minResources().nodeResources().vcpu(),
+                                 next.minResources().nodeResources().vcpu()),
+                validateResource("memory GB",
+                                 current.minResources().nodeResources().memoryGb(),
+                                 next.minResources().nodeResources().memoryGb()),
+                validateResource("disk GB",
+                                 current.minResources().nodeResources().diskGb(),
+                                 next.minResources().nodeResources().diskGb()))
                 .flatMap(Optional::stream)
                 .collect(Collectors.toList());
         if (illegalChanges.isEmpty()) return;
 
         overrides.invalid(ValidationId.resourcesReduction,
-                "Resource reduction in '" + clusterId.value() + "' is too large. " +
-                String.join(" ", illegalChanges) + " New resources must be at least 50% of the current resources",
-                now);
+                          "Resource reduction in '" + clusterId.value() + "' is too large. " +
+                          String.join(" ", illegalChanges) +
+                          " New min resources must be at least 50% of the current min resources",
+                          now);
     }
 
     private static Optional<String> validateResource(String resourceName, double currentValue, double nextValue) {
@@ -64,15 +74,4 @@ public class ResourcesReductionValidator implements ChangeValidator {
         return Optional.of(String.format(Locale.ENGLISH ,"Current %s: %.2f, new: %.2f.", resourceName, currentValue, nextValue));
     }
 
-    private static Map<Pair<ClusterSpec.Type, ClusterSpec.Id>, NodeResources> getRequestedResourcesByClusterId(VespaModel vespaModel) {
-        return vespaModel.hostSystem().getHosts().stream()
-                .map(HostResource::spec)
-                .filter(spec -> spec.membership().isPresent() && spec.requestedResources().isPresent())
-                .filter(spec -> !spec.membership().get().retired())
-                .collect(Collectors.toMap(
-                        spec -> new Pair<>(spec.membership().get().cluster().type(), spec.membership().get().cluster().id()),
-                        spec -> spec.requestedResources().get(),
-                        (e1, e2) -> e1));
-    }
-
 }
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/first/AccessControlOnFirstDeploymentValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/first/AccessControlOnFirstDeploymentValidator.java
index 97153e42ee5..df5fe15ca82 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/first/AccessControlOnFirstDeploymentValidator.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/first/AccessControlOnFirstDeploymentValidator.java
@@ -7,16 +7,16 @@ import com.yahoo.config.model.deploy.DeployState;
 import com.yahoo.config.provision.InstanceName;
 import com.yahoo.vespa.model.VespaModel;
 import com.yahoo.vespa.model.application.validation.Validator;
+import com.yahoo.vespa.model.container.ApplicationContainerCluster;
 import com.yahoo.vespa.model.container.Container;
 import com.yahoo.vespa.model.container.ContainerCluster;
-import com.yahoo.vespa.model.container.ApplicationContainerCluster;
-import com.yahoo.vespa.model.container.component.Handler;
 
 import java.util.ArrayList;
 import java.util.List;
 
 import static com.yahoo.collections.CollectionUtil.mkString;
-import static com.yahoo.vespa.model.container.http.AccessControl.isBuiltinGetOnly;
+import static com.yahoo.config.provision.InstanceName.defaultName;
+import static com.yahoo.vespa.model.container.http.AccessControl.hasHandlerThatNeedsProtection;
 
 /**
  * Validates that hosted applications in prod zones have write protection enabled.
@@ -28,10 +28,7 @@ public class AccessControlOnFirstDeploymentValidator extends Validator {
     @Override
     public void validate(VespaModel model, DeployState deployState) {
 
-        if (! deployState.isHosted()) return;
-        if (! deployState.zone().environment().isProduction()) return;
-        if (deployState.zone().system().isPublic()) return;
-        if (model.getAdmin().getApplicationType() != ApplicationType.DEFAULT) return;
+        if (! needsAccessControlValidation(model, deployState)) return;
 
         List<String> offendingClusters = new ArrayList<>();
         for (ContainerCluster<? extends Container> c : model.getContainerClusters().values()) {
@@ -44,23 +41,19 @@ public class AccessControlOnFirstDeploymentValidator extends Validator {
                 if (hasHandlerThatNeedsProtection(cluster) || ! cluster.getAllServlets().isEmpty())
                     offendingClusters.add(cluster.getName());
         }
-        if (! offendingClusters.isEmpty()
-            && deployState.getApplicationPackage().getApplicationId().instance().equals(InstanceName.defaultName()))
+        if (! offendingClusters.isEmpty())
             deployState.validationOverrides().invalid(ValidationId.accessControl,
                                                       "Access-control must be enabled for write operations to container clusters in production zones: " +
-                                                              mkString(offendingClusters, "[", ", ", "]."), deployState.now());
-    }
-
-    private boolean hasHandlerThatNeedsProtection(ApplicationContainerCluster cluster) {
-        return cluster.getHandlers().stream().anyMatch(this::handlerNeedsProtection);
+                                                              mkString(offendingClusters, "[", ", ", "]"), deployState.now());
     }
 
-    private boolean handlerNeedsProtection(Handler<?> handler) {
-        return ! isBuiltinGetOnly(handler) && hasNonMbusBinding(handler);
-    }
+    public static boolean needsAccessControlValidation(VespaModel model, DeployState deployState) {
+        if (! deployState.isHosted()) return false;
+        if (! deployState.zone().environment().isProduction()) return false;
+        if (deployState.zone().system().isPublic()) return false;
+        if (! deployState.getApplicationPackage().getApplicationId().instance().equals(defaultName())) return false;
+        if (model.getAdmin().getApplicationType() != ApplicationType.DEFAULT) return false;
 
-    private boolean hasNonMbusBinding(Handler<?> handler) {
-        return handler.getServerBindings().stream().anyMatch(binding -> ! binding.startsWith("mbus"));
+        return true;
     }
-
 }
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java
index a146e2cd7e7..d2f06da992c 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java
@@ -71,7 +71,7 @@ public abstract class DomAdminBuilderBase extends VespaDomBuilder.DomConfigProdu
         Monitoring monitoring = getMonitoring(XML.getChild(adminElement,"monitoring"), deployState.isHosted());
         Metrics metrics = new MetricsBuilder(applicationType, predefinedMetricSets)
                 .buildMetrics(XML.getChild(adminElement, "metrics"));
-        FileDistributionConfigProducer fileDistributionConfigProducer = getFileDistributionConfigProducer(parent);
+        FileDistributionConfigProducer fileDistributionConfigProducer = getFileDistributionConfigProducer(parent, deployState.isHosted());
 
         Admin admin = new Admin(parent, monitoring, metrics, multitenant, fileDistributionConfigProducer, deployState.isHosted());
         admin.setApplicationType(applicationType);
@@ -81,8 +81,8 @@ public abstract class DomAdminBuilderBase extends VespaDomBuilder.DomConfigProdu
         return admin;
     }
 
-    private FileDistributionConfigProducer getFileDistributionConfigProducer(AbstractConfigProducer parent) {
-        return new FileDistributionConfigProducer(parent, fileRegistry, configServerSpecs);
+    private FileDistributionConfigProducer getFileDistributionConfigProducer(AbstractConfigProducer parent, boolean isHosted) {
+        return new FileDistributionConfigProducer(parent, fileRegistry, configServerSpecs, isHosted);
     }
 
     protected abstract void doBuildAdmin(DeployState deployState, Admin admin, Element adminE);
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java
index 61d0cd7cd1e..804a5442608 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java
@@ -65,12 +65,12 @@ public class DomAdminV4Builder extends DomAdminBuilderBase {
             createSlobroks(deployLogger, admin, allocateHosts(admin.hostSystem(), "slobroks", nodesSpecification));
         }
         else {
-            createSlobroks(deployLogger, admin, pickContainerHostsForSlobrok(nodesSpecification.count(), 2));
+            createSlobroks(deployLogger, admin, pickContainerHostsForSlobrok(nodesSpecification.minResources().nodes(), 2));
         }
     }
 
     private void assignLogserver(DeployState deployState, NodesSpecification nodesSpecification, Admin admin) {
-        if (nodesSpecification.count() > 1) throw new IllegalArgumentException("You can only request a single log server");
+        if (nodesSpecification.minResources().nodes() > 1) throw new IllegalArgumentException("You can only request a single log server");
         if (deployState.getProperties().applicationId().instance().isTester()) return; // No logserver is needed on tester applications
         if (nodesSpecification.isDedicated()) {
             Collection<HostResource> hosts = allocateHosts(admin.hostSystem(), "logserver", nodesSpecification);
@@ -79,7 +79,7 @@ public class DomAdminV4Builder extends DomAdminBuilderBase {
             Logserver logserver = createLogserver(deployState.getDeployLogger(), admin, hosts);
             createContainerOnLogserverHost(deployState, admin, logserver.getHostResource());
         } else if (containerModels.iterator().hasNext()) {
-            List<HostResource> hosts = sortedContainerHostsFrom(containerModels.iterator().next(), nodesSpecification.count(), false);
+            List<HostResource> hosts = sortedContainerHostsFrom(containerModels.iterator().next(), nodesSpecification.minResources().nodes(), false);
             if (hosts.isEmpty()) return; // No log server can be created (and none is needed)
 
             createLogserver(deployState.getDeployLogger(), admin, hosts);
@@ -91,8 +91,8 @@ public class DomAdminV4Builder extends DomAdminBuilderBase {
     private NodesSpecification createNodesSpecificationForLogserver() {
         DeployState deployState = context.getDeployState();
         if (deployState.getProperties().useDedicatedNodeForLogserver() &&
-                context.getApplicationType() == ConfigModelContext.ApplicationType.DEFAULT &&
-                deployState.isHosted())
+            context.getApplicationType() == ConfigModelContext.ApplicationType.DEFAULT &&
+            deployState.isHosted())
             return NodesSpecification.dedicated(1, context);
         else
             return NodesSpecification.nonDedicated(1, context);
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomClientProviderBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomClientProviderBuilder.java
index f8f1f88e339..11fab0ada29 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomClientProviderBuilder.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomClientProviderBuilder.java
@@ -4,6 +4,7 @@ package com.yahoo.vespa.model.builder.xml.dom;
 import com.yahoo.config.model.deploy.DeployState;
 import com.yahoo.text.XML;
 import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.vespa.model.container.ApplicationContainerCluster;
 import com.yahoo.vespa.model.container.component.Component;
 import com.yahoo.vespa.model.container.component.Handler;
 import org.w3c.dom.Element;
@@ -14,9 +15,13 @@ import org.w3c.dom.Element;
  */
 public class DomClientProviderBuilder extends DomHandlerBuilder {
 
+    public DomClientProviderBuilder(ApplicationContainerCluster cluster) {
+        super(cluster);
+    }
+
     @Override
-    protected Handler doBuild(DeployState deployState, AbstractConfigProducer ancestor, Element clientElement) {
-        Handler<? super Component<?, ?>> client = getHandler(clientElement);
+    protected Handler doBuild(DeployState deployState, AbstractConfigProducer parent, Element clientElement) {
+        Handler<? super Component<?, ?>> client = createHandler(clientElement);
 
         for (Element binding : XML.getChildren(clientElement, "binding"))
             client.addClientBindings(XML.getValue(binding));
@@ -24,7 +29,7 @@ public class DomClientProviderBuilder extends DomHandlerBuilder {
         for (Element serverBinding : XML.getChildren(clientElement, "serverBinding"))
             client.addServerBindings(XML.getValue(serverBinding));
 
-        DomComponentBuilder.addChildren(deployState, ancestor, clientElement, client);
+        DomComponentBuilder.addChildren(deployState, parent, clientElement, client);
 
         return client;
     }
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomHandlerBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomHandlerBuilder.java
index 558c4428d15..ac6d089cf24 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomHandlerBuilder.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomHandlerBuilder.java
@@ -1,38 +1,86 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.model.builder.xml.dom;
 
+import com.yahoo.config.application.api.DeployLogger;
 import com.yahoo.config.model.deploy.DeployState;
+import com.yahoo.config.model.producer.AbstractConfigProducer;
 import com.yahoo.container.bundle.BundleInstantiationSpecification;
 import com.yahoo.osgi.provider.model.ComponentModel;
 import com.yahoo.text.XML;
-import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.vespa.model.container.ApplicationContainerCluster;
 import com.yahoo.vespa.model.container.component.Component;
 import com.yahoo.vespa.model.container.component.Handler;
 import com.yahoo.vespa.model.container.xml.BundleInstantiationSpecificationBuilder;
 import org.w3c.dom.Element;
 
+import java.util.Set;
+
+import static com.yahoo.vespa.model.container.ApplicationContainerCluster.METRICS_V2_HANDLER_BINDING_1;
+import static com.yahoo.vespa.model.container.ApplicationContainerCluster.METRICS_V2_HANDLER_BINDING_2;
+import static com.yahoo.vespa.model.container.ContainerCluster.STATE_HANDLER_BINDING_1;
+import static com.yahoo.vespa.model.container.ContainerCluster.STATE_HANDLER_BINDING_2;
+import static com.yahoo.vespa.model.container.ContainerCluster.VIP_HANDLER_BINDING;
+import static java.util.logging.Level.INFO;
+
 /**
  * @author gjoranv
  */
 public class DomHandlerBuilder extends VespaDomBuilder.DomConfigProducerBuilder<Handler> {
 
+    private static final Set<String> reservedBindings = Set.of(METRICS_V2_HANDLER_BINDING_1,
+                                                               METRICS_V2_HANDLER_BINDING_2,
+                                                               STATE_HANDLER_BINDING_1,
+                                                               STATE_HANDLER_BINDING_2,
+                                                               VIP_HANDLER_BINDING);
+    private final ApplicationContainerCluster cluster;
+
+    public DomHandlerBuilder(ApplicationContainerCluster cluster) {
+        this.cluster = cluster;
+    }
+
     @Override
-    protected Handler doBuild(DeployState deployState, AbstractConfigProducer ancestor, Element handlerElement) {
-        Handler<? super Component<?, ?>> handler = getHandler(handlerElement);
+    protected Handler doBuild(DeployState deployState, AbstractConfigProducer parent, Element handlerElement) {
+        Handler<? super Component<?, ?>> handler = createHandler(handlerElement);
 
         for (Element binding : XML.getChildren(handlerElement, "binding"))
-            handler.addServerBindings(XML.getValue(binding));
+            addServerBinding(handler, XML.getValue(binding), deployState.getDeployLogger());
 
         for (Element clientBinding : XML.getChildren(handlerElement, "clientBinding"))
             handler.addClientBindings(XML.getValue(clientBinding));
 
-        DomComponentBuilder.addChildren(deployState, ancestor, handlerElement, handler);
+        DomComponentBuilder.addChildren(deployState, parent, handlerElement, handler);
 
         return handler;
     }
 
-    protected Handler<? super Component<?, ?>> getHandler(Element handlerElement) {
+    Handler<? super Component<?, ?>> createHandler(Element handlerElement) {
         BundleInstantiationSpecification bundleSpec = BundleInstantiationSpecificationBuilder.build(handlerElement);
         return new Handler<>(new ComponentModel(bundleSpec));
     }
+
+    private void addServerBinding(Handler<? super Component<?, ?>> handler, String binding, DeployLogger log) {
+        throwIfBindingIsReserved(binding, handler);
+        handler.addServerBindings(binding);
+        removeExistingServerBinding(binding, handler, log);
+    }
+
+    private void throwIfBindingIsReserved(String binding, Handler<?> newHandler) {
+        for (var reserved : reservedBindings) {
+            if (binding.equals(reserved)) {
+                throw new IllegalArgumentException("Binding '" + binding + "' is a reserved Vespa binding and " +
+                                                           "cannot be used by handler: " + newHandler.getComponentId());
+            }
+        }
+    }
+
+    private void removeExistingServerBinding(String binding, Handler<?> newHandler, DeployLogger log) {
+        for (var handler : cluster.getHandlers()) {
+            if (handler.getServerBindings().contains(binding)) {
+                handler.removeServerBinding(binding);
+                log.log(INFO, "Binding '" + binding + "' was already in use by handler '" +
+                        handler.getComponentId() + "', but will now be taken over by handler: " + newHandler.getComponentId());
+            }
+        }
+    }
+
 }
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/ModelElement.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/ModelElement.java
index d34a11abdf4..80c95ad6b59 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/ModelElement.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/ModelElement.java
@@ -172,7 +172,12 @@ public class ModelElement {
 
     /** Returns the content of the attribute with the given name, or null if none */
     public String stringAttribute(String name) {
-        if ( ! xml.hasAttribute(name)) return null;
+        return stringAttribute(name, null);
+    }
+
+    /** Returns the content of the attribute with the given name, or the default value if none */
+    public String stringAttribute(String name, String defaultValue) {
+        if ( ! xml.hasAttribute(name)) return defaultValue;
         return xml.getAttribute(name);
     }
 
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java
index 2c88f965f1f..ad6eebe1ca5 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java
@@ -1,11 +1,13 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.model.builder.xml.dom;
 
+import com.yahoo.collections.Pair;
 import com.yahoo.component.Version;
 import com.yahoo.config.application.api.DeployLogger;
 import com.yahoo.config.model.ConfigModelContext;
 import com.yahoo.config.provision.Capacity;
 import com.yahoo.config.provision.ClusterMembership;
+import com.yahoo.config.provision.ClusterResources;
 import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.config.provision.NodeResources;
 import com.yahoo.text.XML;
@@ -15,9 +17,11 @@ import com.yahoo.vespa.model.container.xml.ContainerModelBuilder;
 import org.w3c.dom.Element;
 import org.w3c.dom.Node;
 
+import java.util.List;
 import java.util.Map;
 import java.util.Optional;
-import java.util.Set;
+import java.util.function.Function;
+import java.util.regex.Pattern;
 
 /**
  * A common utility class to represent a requirement for nodes during model building.
@@ -27,11 +31,9 @@ import java.util.Set;
  */
 public class NodesSpecification {
 
-    private final boolean dedicated;
-
-    private final int count;
+    private final ClusterResources min, max;
 
-    private final int groups;
+    private final boolean dedicated;
 
     /** The Vespa version we want the nodes to run */
     private Version version;
@@ -43,50 +45,76 @@ public class NodesSpecification {
     private final boolean required;
 
     private final boolean canFail;
-    
-    private final boolean exclusive;
-
-    /** Whether this requires running container and content processes co-located on the same node. */
-    private final boolean combined;
 
-    /** The resources each node should have, or empty to use the default */
-    private final Optional<NodeResources> resources;
-
-    /** The identifier of the custom docker image layer to use (not supported yet) */
-    private final Optional<String> dockerImage;
+    private final boolean exclusive;
 
-    private NodesSpecification(boolean dedicated, int count, int groups, Version version,
-                               boolean required, boolean canFail, boolean exclusive, boolean combined,
-                               Optional<NodeResources> resources, Optional<String> dockerImage) {
+    /** The repo part of a docker image (without tag), optional */
+    private final Optional<String> dockerImageRepo;
+
+    /** The ID of the cluster referencing this node specification, if any */
+    private final Optional<String> combinedId;
+
+    private NodesSpecification(ClusterResources min,
+                               ClusterResources max,
+                               boolean dedicated, Version version,
+                               boolean required, boolean canFail, boolean exclusive,
+                               Optional<String> dockerImageRepo,
+                               Optional<String> combinedId) {
+        if (max.smallerThan(min))
+            throw new IllegalArgumentException("Min resources must be larger or equal to max resources, but " +
+                                               max + " is smaller than " + min);
+
+        // Non-scaled resources must be equal
+        if ( ! min.nodeResources().justNonNumbers().equals(max.nodeResources().justNonNumbers()))
+            throw new IllegalArgumentException("Min and max resources must have the same non-numeric settings, but " +
+                                               "min is " + min + " and max " + max);
+        if (min.nodeResources().bandwidthGbps() != max.nodeResources().bandwidthGbps())
+            throw new IllegalArgumentException("Min and max resources must have the same bandwith, but " +
+                                               "min is " + min + " and max " + max);
+
+        this.min = min;
+        this.max = max;
         this.dedicated = dedicated;
-        this.count = count;
-        this.groups = groups;
         this.version = version;
         this.required = required;
         this.canFail = canFail;
         this.exclusive = exclusive;
-        this.resources = resources;
-        this.dockerImage = dockerImage;
-        this.combined = combined;
+        this.dockerImageRepo = dockerImageRepo;
+        this.combinedId = combinedId;
     }
 
-    private NodesSpecification(boolean dedicated, boolean canFail, boolean combined, Version version, ModelElement nodesElement) {
-        this(dedicated,
-             nodesElement.integerAttribute("count", 1),
-             nodesElement.integerAttribute("groups", 1),
-             version,
-             nodesElement.booleanAttribute("required", false),
-             canFail,
-             nodesElement.booleanAttribute("exclusive", false),
-             combined,
-             getResources(nodesElement),
-             Optional.ofNullable(nodesElement.stringAttribute("docker-image")));
+    private static NodesSpecification create(boolean dedicated, boolean canFail, Version version,
+                                             ModelElement nodesElement, Optional<String> dockerImageRepo) {
+        var resolvedElement = resolveElement(nodesElement);
+        var combinedId = findCombinedId(nodesElement, resolvedElement);
+        var resources = toResources(resolvedElement);
+        return new NodesSpecification(resources.getFirst(),
+                                      resources.getSecond(),
+                                      dedicated,
+                                      version,
+                                      resolvedElement.booleanAttribute("required", false),
+                                      canFail,
+                                      resolvedElement.booleanAttribute("exclusive", false),
+                                      dockerImageToUse(resolvedElement, dockerImageRepo),
+                                      combinedId);
     }
 
-    private static NodesSpecification create(boolean dedicated, boolean canFail, Version version, ModelElement nodesElement) {
-        var resolvedElement = resolveElement(nodesElement);
-        boolean combined = resolvedElement != nodesElement || isReferencedByOtherElement(nodesElement);
-        return new NodesSpecification(dedicated, canFail, combined, version, resolvedElement);
+    private static Pair<ClusterResources, ClusterResources> toResources(ModelElement nodesElement) {
+        Pair<Integer, Integer> nodes =  toRange(nodesElement.stringAttribute("count"),  1, Integer::parseInt);
+        Pair<Integer, Integer> groups = toRange(nodesElement.stringAttribute("groups"), 1, Integer::parseInt);
+        var min = new ClusterResources(nodes.getFirst(),  groups.getFirst(),  nodeResources(nodesElement).getFirst());
+        var max = new ClusterResources(nodes.getSecond(), groups.getSecond(), nodeResources(nodesElement).getSecond());
+        return new Pair<>(min, max);
+    }
+
+    /** Returns the ID of the cluster referencing this node specification, if any */
+    private static Optional<String> findCombinedId(ModelElement nodesElement, ModelElement resolvedElement) {
+        if (resolvedElement != nodesElement) {
+            // Specification for a container cluster referencing nodes in a content cluster
+            return containerIdOf(nodesElement);
+        }
+        // Specification for a content cluster that is referenced by a container cluster
+        return containerIdReferencing(nodesElement);
     }
 
     /** Returns a requirement for dedicated nodes taken from the given <code>nodes</code> element */
@@ -94,7 +122,8 @@ public class NodesSpecification {
         return create(true,
                       ! context.getDeployState().getProperties().isBootstrap(),
                       context.getDeployState().getWantedNodeVespaVersion(),
-                      nodesElement);
+                      nodesElement,
+                      context.getDeployState().getWantedDockerImageRepo());
     }
 
     /**
@@ -110,7 +139,7 @@ public class NodesSpecification {
     }
 
     /**
-     * Returns a requirement for undedicated or dedicated nodes taken from the <code>nodes</code> element
+     * Returns a requirement for non-dedicated or dedicated nodes taken from the <code>nodes</code> element
      * contained in the given parent element, or empty if the parent element is null, or the nodes elements
      * is not present.
      */
@@ -121,37 +150,42 @@ public class NodesSpecification {
         if (nodesElement == null) return Optional.empty();
         return Optional.of(create(nodesElement.booleanAttribute("dedicated", false),
                                   ! context.getDeployState().getProperties().isBootstrap(),
-                                  context.getDeployState().getWantedNodeVespaVersion(), nodesElement));
+                                  context.getDeployState().getWantedNodeVespaVersion(),
+                                  nodesElement,
+                                  context.getDeployState().getWantedDockerImageRepo()));
     }
 
-    /** Returns a requirement from <code>count</code> nondedicated nodes in one group */
+    /**
+     * Returns a requirement from <code>count</code> non-dedicated nodes in one group
+     */
     public static NodesSpecification nonDedicated(int count, ConfigModelContext context) {
-        return new NodesSpecification(false,
-                                      count,
-                                      1,
+        return new NodesSpecification(new ClusterResources(count, 1, NodeResources.unspecified),
+                                      new ClusterResources(count, 1, NodeResources.unspecified),
+                                      false,
                                       context.getDeployState().getWantedNodeVespaVersion(),
                                       false,
                                       ! context.getDeployState().getProperties().isBootstrap(),
                                       false,
-                                      false,
-                                      Optional.empty(),
+                                      context.getDeployState().getWantedDockerImageRepo(),
                                       Optional.empty());
     }
 
     /** Returns a requirement from <code>count</code> dedicated nodes in one group */
     public static NodesSpecification dedicated(int count, ConfigModelContext context) {
-        return new NodesSpecification(true,
-                                      count,
-                                      1,
+        return new NodesSpecification(new ClusterResources(count, 1, NodeResources.unspecified),
+                                      new ClusterResources(count, 1, NodeResources.unspecified),
+                                      true,
                                       context.getDeployState().getWantedNodeVespaVersion(),
                                       false,
                                       ! context.getDeployState().getProperties().isBootstrap(),
                                       false,
-                                      false,
-                                      Optional.empty(),
+                                      context.getDeployState().getWantedDockerImageRepo(),
                                       Optional.empty());
     }
 
+    public ClusterResources minResources() { return min; }
+    public ClusterResources maxResources() { return max; }
+
     /**
      * Returns whether this requires dedicated nodes.
      * Otherwise the model encountering this request should reuse nodes requested for other purposes whenever possible.
@@ -165,42 +199,48 @@ public class NodesSpecification {
      */
     public boolean isExclusive() { return exclusive; }
 
-    /** Returns the number of nodes required */
-    public int count() { return count; }
-
-    /** Returns the number of host groups this specifies. Default is 1 */
-    public int groups() { return groups; }
-
     public Map<HostResource, ClusterMembership> provision(HostSystem hostSystem,
                                                           ClusterSpec.Type clusterType,
                                                           ClusterSpec.Id clusterId,
                                                           DeployLogger logger) {
-        if (combined)
+        if (combinedId.isPresent())
             clusterType = ClusterSpec.Type.combined;
-        ClusterSpec cluster = ClusterSpec.request(clusterType, clusterId, version, exclusive);
-        return hostSystem.allocateHosts(cluster, Capacity.fromCount(count, resources, required, canFail), groups, logger);
+        ClusterSpec cluster = ClusterSpec.request(clusterType, clusterId)
+                .vespaVersion(version)
+                .exclusive(exclusive)
+                .combinedId(combinedId.map(ClusterSpec.Id::from))
+                .dockerImageRepo(dockerImageRepo)
+                .build();
+        return hostSystem.allocateHosts(cluster, Capacity.from(min, max, required, canFail), logger);
     }
 
-    private static Optional<NodeResources> getResources(ModelElement nodesElement) {
+    private static Pair<NodeResources, NodeResources> nodeResources(ModelElement nodesElement) {
         ModelElement resources = nodesElement.child("resources");
         if (resources != null) {
-            return Optional.of(new NodeResources(resources.requiredDoubleAttribute("vcpu"),
-                                                 parseGbAmount(resources.requiredStringAttribute("memory"), "B"),
-                                                 parseGbAmount(resources.requiredStringAttribute("disk"), "B"),
-                                                 Optional.ofNullable(resources.stringAttribute("bandwidth"))
-                                                         .map(b -> parseGbAmount(b, "BPS"))
-                                                         .orElse(0.3),
-                                                 parseOptionalDiskSpeed(resources.stringAttribute("disk-speed")),
-                                                 parseOptionalStorageType(resources.stringAttribute("storage-type"))));
+            return nodeResourcesFromResorcesElement(resources);
         }
         else if (nodesElement.stringAttribute("flavor") != null) { // legacy fallback
-            return Optional.of(NodeResources.fromLegacyName(nodesElement.stringAttribute("flavor")));
+            var flavorResources = NodeResources.fromLegacyName(nodesElement.stringAttribute("flavor"));
+            return new Pair<>(flavorResources, flavorResources);
         }
-        else { // Get the default
-            return Optional.empty();
+        else {
+            return new Pair<>(NodeResources.unspecified, NodeResources.unspecified);
         }
     }
 
+    private static Pair<NodeResources, NodeResources> nodeResourcesFromResorcesElement(ModelElement element) {
+        Pair<Double, Double> vcpu       = toRange(element.requiredStringAttribute("vcpu"),   .0, Double::parseDouble);
+        Pair<Double, Double> memory     = toRange(element.requiredStringAttribute("memory"), .0, s -> parseGbAmount(s, "B"));
+        Pair<Double, Double> disk       = toRange(element.requiredStringAttribute("disk"),   .0, s -> parseGbAmount(s, "B"));
+        Pair<Double, Double> bandwith   = toRange(element.stringAttribute("bandwith"),       .3, s -> parseGbAmount(s, "BPS"));
+        NodeResources.DiskSpeed   diskSpeed   = parseOptionalDiskSpeed(element.stringAttribute("disk-speed"));
+        NodeResources.StorageType storageType = parseOptionalStorageType(element.stringAttribute("storage-type"));
+
+        var min = new NodeResources(vcpu.getFirst(),  memory.getFirst(),  disk.getFirst(),  bandwith.getFirst(),  diskSpeed, storageType);
+        var max = new NodeResources(vcpu.getSecond(), memory.getSecond(), disk.getSecond(), bandwith.getSecond(), diskSpeed, storageType);
+        return new Pair<>(min, max);
+    }
+
     private static double parseGbAmount(String byteAmount, String unit) {
         byteAmount = byteAmount.strip();
         byteAmount = byteAmount.toUpperCase();
@@ -280,24 +320,35 @@ public class NodesSpecification {
         return new ModelElement(referencedNodesElement);
     }
 
-    /** Returns whether the given nodesElement is referenced by any other nodes element */
-    private static boolean isReferencedByOtherElement(ModelElement nodesElement) {
+    /** Returns the ID of the parent container element of nodesElement, if any  */
+    private static Optional<String> containerIdOf(ModelElement nodesElement) {
+        var element = nodesElement.getXml();
+        for (var containerTag : List.of("container", "jdisc")) {
+            var container = findParentByTag(containerTag, element);
+            if (container.isEmpty()) continue;
+            return container.map(el -> el.getAttribute("id"));
+        }
+        return Optional.empty();
+    }
+
+    /** Returns the ID of the container element referencing nodesElement, if any */
+    private static Optional<String> containerIdReferencing(ModelElement nodesElement) {
         var element = nodesElement.getXml();
         var services = findParentByTag("services", element);
-        if (services.isEmpty()) return false;
+        if (services.isEmpty()) return Optional.empty();
 
         var content = findParentByTag("content", element);
-        if (content.isEmpty()) return false;
+        if (content.isEmpty()) return Optional.empty();
         var contentClusterId = content.get().getAttribute("id");
-        if (contentClusterId.isEmpty()) return false;
+        if (contentClusterId.isEmpty()) return Optional.empty();
         for (var rootChild : XML.getChildren(services.get())) {
             if ( ! ContainerModelBuilder.isContainerTag(rootChild)) continue;
             var nodes = XML.getChild(rootChild, "nodes");
             if (nodes == null) continue;
             if (!contentClusterId.equals(nodes.getAttribute("of"))) continue;
-            return true;
+            return Optional.of(rootChild.getAttribute("id"));
         }
-        return false;
+        return Optional.empty();
     }
 
     private static Optional<Element> findChildById(Element parent, String id) {
@@ -319,11 +370,33 @@ public class NodesSpecification {
         return new IllegalArgumentException("referenced service '" + referenceId + "' is not defined");
     }
 
+    private static Optional<String> dockerImageToUse(ModelElement nodesElement, Optional<String> dockerImage) {
+        String dockerImageFromElement = nodesElement.stringAttribute("docker-image");
+        return dockerImageFromElement == null ? dockerImage : Optional.of(dockerImageFromElement);
+    }
+
+    /** Parses a value ("value") or value range ("[min-value, max-value]") */
+    private static <T> Pair<T, T> toRange(String s, T defaultValue, Function<String, T> valueParser) {
+        try {
+            if (s == null) return new Pair<>(defaultValue, defaultValue);
+            s = s.trim();
+            if (s.startsWith("[") && s.endsWith("]")) {
+                String[] numbers = s.substring(1, s.length() - 1).split(",");
+                if (numbers.length != 2) throw new IllegalArgumentException();
+                return new Pair<>(valueParser.apply(numbers[0].trim()), valueParser.apply(numbers[1].trim()));
+            } else {
+                return new Pair<>(valueParser.apply(s), valueParser.apply(s));
+            }
+        }
+        catch (IllegalArgumentException e) {
+            throw new IllegalArgumentException("Expected a number or range on the form [min, max], but got '" + s + "'", e);
+        }
+    }
+
     @Override
     public String toString() {
-        return "specification of " + count + (dedicated ? " dedicated " : " ") + "nodes" +
-               (resources.isPresent() ? " with resources " + resources.get() : "") +
-               (groups > 1 ? " in " + groups + " groups" : "");
+        return "specification of " + (dedicated ? "dedicated " : "") +
+               (min.equals(max) ? min : "min " + min + " max " + max);
     }
 
 }
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java
index 97b78e1b9b1..c9caca1831f 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java
@@ -141,8 +141,7 @@ public class VespaDomBuilder extends VespaModelBuilder {
         }
 
         private void initializeService(AbstractService t, DeployState deployState,
-                                       HostSystem hostSystem, Element producerSpec)
-        {
+                                       HostSystem hostSystem, Element producerSpec) {
             initializeProducer(t, deployState, producerSpec);
             if (producerSpec != null) {
                 if (producerSpec.hasAttribute(JVM_OPTIONS)) {
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java
index efd00528d54..63e6af03c44 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java
@@ -1,6 +1,7 @@
 // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.model.container;
 
+import ai.vespa.metricsproxy.http.application.ApplicationMetricsHandler;
 import com.yahoo.component.ComponentId;
 import com.yahoo.component.ComponentSpecification;
 import com.yahoo.config.FileReference;
@@ -9,6 +10,9 @@ import com.yahoo.config.model.deploy.DeployState;
 import com.yahoo.config.model.producer.AbstractConfigProducer;
 import com.yahoo.container.BundlesConfig;
 import com.yahoo.container.bundle.BundleInstantiationSpecification;
+import com.yahoo.container.handler.ThreadpoolConfig;
+import com.yahoo.container.handler.metrics.MetricsProxyApiConfig;
+import com.yahoo.container.handler.metrics.MetricsV2Handler;
 import com.yahoo.container.jdisc.ContainerMbusConfig;
 import com.yahoo.container.jdisc.messagebus.MbusServerProvider;
 import com.yahoo.jdisc.http.ServletPathsConfig;
@@ -17,8 +21,10 @@ import com.yahoo.search.config.QrStartConfig;
 import com.yahoo.vespa.config.search.RankProfilesConfig;
 import com.yahoo.vespa.config.search.core.RankingConstantsConfig;
 import com.yahoo.vespa.defaults.Defaults;
+import com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainer;
 import com.yahoo.vespa.model.container.component.Component;
 import com.yahoo.vespa.model.container.component.ConfigProducerGroup;
+import com.yahoo.vespa.model.container.component.Handler;
 import com.yahoo.vespa.model.container.component.Servlet;
 import com.yahoo.vespa.model.container.jersey.Jersey2Servlet;
 import com.yahoo.vespa.model.container.jersey.RestApi;
@@ -45,7 +51,12 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat
         RankProfilesConfig.Producer,
         RankingConstantsConfig.Producer,
         ServletPathsConfig.Producer,
-        ContainerMbusConfig.Producer {
+        ContainerMbusConfig.Producer,
+        MetricsProxyApiConfig.Producer {
+
+    public static final String METRICS_V2_HANDLER_CLASS = MetricsV2Handler.class.getName();
+    public static final String METRICS_V2_HANDLER_BINDING_1 = "http://*" + MetricsV2Handler.V2_PATH;
+    public static final String METRICS_V2_HANDLER_BINDING_2 = METRICS_V2_HANDLER_BINDING_1 + "/*";
 
     private final Set<FileReference> applicationBundles = new LinkedHashSet<>();
 
@@ -58,6 +69,7 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat
 
     private MbusParams mbusParams;
     private boolean messageBusEnabled = true;
+    private final double softStartSeconds;
 
     private Integer memoryPercentage = null;
 
@@ -72,7 +84,9 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat
         addSimpleComponent("com.yahoo.container.jdisc.DeprecatedSecretStoreProvider");
         addSimpleComponent("com.yahoo.container.jdisc.CertificateStoreProvider");
         addSimpleComponent("com.yahoo.container.jdisc.AthenzIdentityProviderProvider");
+        addMetricsV2Handler();
         addTestrunnerComponentsIfTester(deployState);
+        softStartSeconds = deployState.getProperties().defaultSoftStartSeconds();
     }
 
     @Override
@@ -99,6 +113,13 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat
         }
     }
 
+    public void addMetricsV2Handler() {
+        Handler<AbstractConfigProducer<?>> handler = new Handler<>(
+                new ComponentModel(METRICS_V2_HANDLER_CLASS, null, null, null));
+        handler.addServerBindings(METRICS_V2_HANDLER_BINDING_1, METRICS_V2_HANDLER_BINDING_2);
+        addComponent(handler);
+    }
+
     private void addTestrunnerComponentsIfTester(DeployState deployState) {
         if (deployState.isHosted() && deployState.getProperties().applicationId().instance().isTester())
             addPlatformBundle(Paths.get(Defaults.getDefaults().underVespaHome("lib/jars/vespa-testrunner-components-jar-with-dependencies.jar")));
@@ -188,10 +209,17 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat
     }
 
     @Override
+    public void getConfig(MetricsProxyApiConfig.Builder builder) {
+        builder.metricsPort(MetricsProxyContainer.BASEPORT)
+                .metricsApiPath(ApplicationMetricsHandler.VALUES_PATH);
+    }
+
+    @Override
     public void getConfig(QrStartConfig.Builder builder) {
         super.getConfig(builder);
         builder.jvm.verbosegc(true)
                 .availableProcessors(0)
+                .compressedClassSpaceSize(0)  //TODO Reduce, next step is 512m
                 .minHeapsize(1536)
                 .heapsize(1536);
         if (getMemoryPercentage().isPresent()) {
@@ -223,6 +251,11 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat
                         null))));
     }
 
+    @Override
+    public void getConfig(ThreadpoolConfig.Builder builder) {
+        builder.softStartSeconds(softStartSeconds);
+    }
+
     public static class MbusParams {
         // the amount of the maxpendingbytes to process concurrently, typically 0.2 (20%)
         final Double maxConcurrentFactor;
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java
index 7e2d6680827..31c8724d634 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java
@@ -140,7 +140,7 @@ public abstract class Container extends AbstractService implements
         if (http == null) {
             return defaultHttpServer;
         } else {
-            return http.getHttpServer();
+            return http.getHttpServer().orElse(null);
         }
     }
 
@@ -228,10 +228,10 @@ public abstract class Container extends AbstractService implements
             // XXX unused - remove:
             from.allocatePort("http/1");
             portsMeta.on(offset++).tag("http").tag("external");
-        } else if (getHttp().getHttpServer() == null) {
+        } else if (getHttp().getHttpServer().isEmpty()) {
             // no http server ports
         } else {
-            for (ConnectorFactory connectorFactory : getHttp().getHttpServer().getConnectorFactories()) {
+            for (ConnectorFactory connectorFactory : getHttp().getHttpServer().get().getConnectorFactories()) {
                 int port = getPort(connectorFactory);
                 String name = "http/" + connectorFactory.getName();
                 from.requirePort(port, name);
@@ -280,7 +280,7 @@ public abstract class Container extends AbstractService implements
         final Http http = getHttp();
         if (http != null) {
             // TODO: allow the user to specify health port manually
-            if (http.getHttpServer() == null) {
+            if (http.getHttpServer().isEmpty()) {
                 return -1;
             } else {
                 return getRelativePort(0);
@@ -303,7 +303,7 @@ public abstract class Container extends AbstractService implements
                         .slobrokId(serviceSlobrokId())).
                 filedistributor(filedistributorConfig());
         if (clusterName != null) {
-            builder.discriminator(clusterName+"."+name);
+            builder.discriminator(clusterName + "." + name);
         } else {
             builder.discriminator(name);
         }
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java
index 6fa446bf365..e2b1f97a6eb 100755
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java
@@ -19,6 +19,7 @@ import com.yahoo.container.bundle.BundleInstantiationSpecification;
 import com.yahoo.container.core.ApplicationMetadataConfig;
 import com.yahoo.container.core.document.ContainerDocumentConfig;
 import com.yahoo.container.handler.ThreadPoolProvider;
+import com.yahoo.container.handler.ThreadpoolConfig;
 import com.yahoo.container.jdisc.JdiscBindingsConfig;
 import com.yahoo.container.jdisc.config.HealthMonitorConfig;
 import com.yahoo.container.jdisc.state.StateHandler;
@@ -99,7 +100,9 @@ public abstract class ContainerCluster<CONTAINER extends Container>
         DocprocConfig.Producer,
         ClusterInfoConfig.Producer,
         RoutingProviderConfig.Producer,
-        ConfigserverConfig.Producer {
+        ConfigserverConfig.Producer,
+        ThreadpoolConfig.Producer
+{
 
     /**
      * URI prefix used for internal, usually programmatic, APIs. URIs using this
@@ -111,15 +114,20 @@ public abstract class ContainerCluster<CONTAINER extends Container>
 
     public static final String APPLICATION_STATUS_HANDLER_CLASS = "com.yahoo.container.handler.observability.ApplicationStatusHandler";
     public static final String BINDINGS_OVERVIEW_HANDLER_CLASS = BindingsOverviewHandler.class.getName();
-    public static final String STATE_HANDLER_CLASS = "com.yahoo.container.jdisc.state.StateHandler";
     public static final String LOG_HANDLER_CLASS = com.yahoo.container.handler.LogHandler.class.getName();
     public static final String DEFAULT_LINGUISTICS_PROVIDER = "com.yahoo.language.provider.DefaultLinguisticsProvider";
     public static final String CMS = "-XX:+UseConcMarkSweepGC -XX:MaxTenuringThreshold=15 -XX:NewRatio=1";
     public static final String G1GC = "-XX:+UseG1GC -XX:MaxTenuringThreshold=15";
 
+    public static final String STATE_HANDLER_CLASS = "com.yahoo.container.jdisc.state.StateHandler";
+    public static final String STATE_HANDLER_BINDING_1 = "http://*" + StateHandler.STATE_API_ROOT;
+    public static final String STATE_HANDLER_BINDING_2 = STATE_HANDLER_BINDING_1 + "/*";
+
     public static final String ROOT_HANDLER_PATH = "/";
     public static final String ROOT_HANDLER_BINDING = "http://*" + ROOT_HANDLER_PATH;
 
+    public static final String VIP_HANDLER_BINDING = "http://*/status.html";
+
     private final String name;
 
     protected List<CONTAINER> containers = new ArrayList<>();
@@ -200,15 +208,11 @@ public abstract class ContainerCluster<CONTAINER extends Container>
     public void addMetricStateHandler() {
         Handler<AbstractConfigProducer<?>> stateHandler = new Handler<>(
                 new ComponentModel(STATE_HANDLER_CLASS, null, null, null));
-        stateHandler.addServerBindings("http://*" + StateHandler.STATE_API_ROOT,
-                                       "http://*" + StateHandler.STATE_API_ROOT + "/*");
+        stateHandler.addServerBindings(STATE_HANDLER_BINDING_1, STATE_HANDLER_BINDING_2);
         addComponent(stateHandler);
     }
 
     public void addDefaultRootHandler() {
-        if (hasHandlerWithBinding(ROOT_HANDLER_BINDING))
-            return;
-
         Handler<AbstractConfigProducer<?>> handler = new Handler<>(
                 new ComponentModel(BundleInstantiationSpecification.getFromStrings(
                         BINDINGS_OVERVIEW_HANDLER_CLASS, null, null), null));  // null bundle, as the handler is in container-disc
@@ -216,15 +220,6 @@ public abstract class ContainerCluster<CONTAINER extends Container>
         addComponent(handler);
     }
 
-    private boolean hasHandlerWithBinding(String binding) {
-        Collection<Handler<?>> handlers = getHandlers();
-        for (Handler handler : handlers) {
-            if (handler.getServerBindings().contains(binding))
-                return true;
-        }
-        return false;
-    }
-
     public void addApplicationStatusHandler() {
         Handler<AbstractConfigProducer<?>> statusHandler = new Handler<>(
                 new ComponentModel(BundleInstantiationSpecification.getInternalHandlerSpecificationFromStrings(
@@ -235,7 +230,7 @@ public abstract class ContainerCluster<CONTAINER extends Container>
 
     public void addVipHandler() {
         Handler<?> vipHandler = Handler.fromClassName(FileStatusHandlerComponent.CLASS);
-        vipHandler.addServerBindings("http://*/status.html");
+        vipHandler.addServerBindings(VIP_HANDLER_BINDING);
         addComponent(vipHandler);
     }
 
@@ -484,6 +479,7 @@ public abstract class ContainerCluster<CONTAINER extends Container>
         builder.jvm
                 .verbosegc(false)
                 .availableProcessors(2)
+                .compressedClassSpaceSize(32)
                 .minHeapsize(32)
                 .heapsize(512)
                 .heapSizeAsPercentageOfPhysicalMemory(0)
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/ConfigProducerGroup.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/ConfigProducerGroup.java
index c671749cff0..a466dabe984 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/ConfigProducerGroup.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/ConfigProducerGroup.java
@@ -4,7 +4,12 @@ package com.yahoo.vespa.model.container.component;
 import com.yahoo.component.ComponentId;
 import com.yahoo.config.model.producer.AbstractConfigProducer;
 
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
 
 /**
  * A group of config producers that have a component id.
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/FileStatusHandlerComponent.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/FileStatusHandlerComponent.java
index ee61b34987a..3d9a1b2e665 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/FileStatusHandlerComponent.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/FileStatusHandlerComponent.java
@@ -10,6 +10,7 @@ import com.yahoo.osgi.provider.model.ComponentModel;
  * @author Tony Vaagenes
  */
 public class FileStatusHandlerComponent extends Handler implements VipStatusConfig.Producer {
+
     public static final String CLASS = "com.yahoo.container.handler.VipStatusHandler";
 
     private final String fileName;
@@ -26,4 +27,5 @@ public class FileStatusHandlerComponent extends Handler implements VipStatusConf
         builder.accessdisk(true).
                 statusfile(fileName);
     }
+
 }
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/Handler.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/Handler.java
index e07c3216850..82484e07773 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/Handler.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/Handler.java
@@ -8,7 +8,9 @@ import com.yahoo.config.model.producer.AbstractConfigProducer;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.LinkedHashSet;
 import java.util.List;
+import java.util.Set;
 
 /**
  * Models a jdisc RequestHandler (including ClientProvider).
@@ -21,7 +23,7 @@ import java.util.List;
  */
 public class Handler<CHILD extends AbstractConfigProducer<?>> extends Component<CHILD, ComponentModel> {
 
-    private List<String> serverBindings = new ArrayList<>();
+    private Set<String> serverBindings = new LinkedHashSet<>();
     private List<String> clientBindings = new ArrayList<>();
 
     public Handler(ComponentModel model) {
@@ -40,12 +42,16 @@ public class Handler<CHILD extends AbstractConfigProducer<?>> extends Component<
         serverBindings.addAll(Arrays.asList(bindings));
     }
 
+    public void removeServerBinding(String binding) {
+        serverBindings.remove(binding);
+    }
+
     public void addClientBindings(String... bindings) {
         clientBindings.addAll(Arrays.asList(bindings));
     }
 
-    public final List<String> getServerBindings() {
-        return Collections.unmodifiableList(serverBindings);
+    public final Set<String> getServerBindings() {
+        return Collections.unmodifiableSet(serverBindings);
     }
 
     public final List<String> getClientBindings() {
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java
index f3758def2b1..470b82496a3 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java
@@ -9,6 +9,7 @@ import com.yahoo.config.provision.RegionName;
 import com.yahoo.config.provision.SystemName;
 import com.yahoo.config.provision.Zone;
 import com.yahoo.container.StatisticsConfig;
+import com.yahoo.container.core.VipStatusConfig;
 import com.yahoo.container.jdisc.config.HealthMonitorConfig;
 import com.yahoo.net.HostName;
 import com.yahoo.vespa.defaults.Defaults;
@@ -29,7 +30,8 @@ public class ConfigserverCluster extends AbstractConfigProducer
         ZookeeperServerConfig.Producer,
         ConfigserverConfig.Producer,
         StatisticsConfig.Producer,
-        HealthMonitorConfig.Producer {
+        HealthMonitorConfig.Producer,
+        VipStatusConfig.Producer {
     private final CloudConfigOptions options;
     private ContainerCluster containerCluster;
 
@@ -116,8 +118,8 @@ public class ConfigserverCluster extends AbstractConfigProducer
         }
 
         builder.serverId(HostName.getLocalhost());
-        if (!containerCluster.getHttp().getHttpServer().getConnectorFactories().isEmpty()) {
-            builder.httpport(containerCluster.getHttp().getHttpServer().getConnectorFactories().get(0).getListenPort());
+        if (!containerCluster.getHttp().getHttpServer().get().getConnectorFactories().isEmpty()) {
+            builder.httpport(containerCluster.getHttp().getHttpServer().get().getConnectorFactories().get(0).getListenPort());
         }
         if (options.useVespaVersionInRequest().isPresent()) {
             builder.useVespaVersionInRequest(options.useVespaVersionInRequest().get());
@@ -178,4 +180,8 @@ public class ConfigserverCluster extends AbstractConfigProducer
         builder.snapshot_interval(60.0);
     }
 
+    @Override
+    public void getConfig(VipStatusConfig.Builder builder) {
+        builder.initiallyInRotation(false);
+    }
 }
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/AccessControl.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/AccessControl.java
index d3ba2718d71..dd48e65c340 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/AccessControl.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/AccessControl.java
@@ -1,10 +1,10 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.model.container.http;
 
-import com.google.common.collect.ImmutableList;
 import com.yahoo.component.ComponentId;
 import com.yahoo.component.ComponentSpecification;
 import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.vespa.model.container.ApplicationContainerCluster;
 import com.yahoo.vespa.model.container.ContainerCluster;
 import com.yahoo.vespa.model.container.component.FileStatusHandlerComponent;
 import com.yahoo.vespa.model.container.component.Handler;
@@ -15,7 +15,6 @@ import java.util.Collection;
 import java.util.Collections;
 import java.util.LinkedHashSet;
 import java.util.List;
-import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
@@ -24,23 +23,23 @@ import java.util.stream.Stream;
  * Helper class for http access control.
  *
  * @author gjoranv
+ * @author bjorncs
  */
 public final class AccessControl {
 
     public static final ComponentId ACCESS_CONTROL_CHAIN_ID = ComponentId.fromString("access-control-chain");
 
-    private static final List<String> UNPROTECTED_HANDLERS = ImmutableList.of(
+    public static final List<String> UNPROTECTED_HANDLERS = List.of(
             FileStatusHandlerComponent.CLASS,
             ContainerCluster.APPLICATION_STATUS_HANDLER_CLASS,
             ContainerCluster.BINDINGS_OVERVIEW_HANDLER_CLASS,
             ContainerCluster.STATE_HANDLER_CLASS,
-            ContainerCluster.LOG_HANDLER_CLASS
+            ContainerCluster.LOG_HANDLER_CLASS,
+            ApplicationContainerCluster.METRICS_V2_HANDLER_CLASS
     );
 
     public static final class Builder {
         private String domain;
-        private String applicationId;
-        private Optional<String> vespaDomain = Optional.empty();
         private boolean readEnabled = false;
         private boolean writeEnabled = true;
         private final Set<String> excludeBindings = new LinkedHashSet<>();
@@ -48,9 +47,8 @@ public final class AccessControl {
         private Collection<Servlet> servlets = Collections.emptyList();
         private final DeployLogger logger;
 
-        public Builder(String domain, String applicationId, DeployLogger logger) {
+        public Builder(String domain, DeployLogger logger) {
             this.domain = domain;
-            this.applicationId = applicationId;
             this.logger = logger;
         }
 
@@ -69,52 +67,37 @@ public final class AccessControl {
             return this;
         }
 
-        public Builder vespaDomain(String vespaDomain) {
-            this.vespaDomain = Optional.ofNullable(vespaDomain);
-            return this;
-        }
-
-        public Builder setHandlers(Collection<Handler<?>> handlers) {
-            this.handlers = handlers;
-            return this;
-        }
-
-        public Builder setServlets(Collection<Servlet> servlets) {
-            this.servlets = servlets;
+        public Builder setHandlers(ApplicationContainerCluster cluster) {
+            this.handlers = cluster.getHandlers();
+            this.servlets = cluster.getAllServlets();
             return this;
         }
 
         public AccessControl build() {
-            return new AccessControl(domain, applicationId, writeEnabled, readEnabled,
-                                     excludeBindings, vespaDomain, servlets, handlers, logger);
+            return new AccessControl(domain, writeEnabled, readEnabled,
+                                     excludeBindings, servlets, handlers, logger);
         }
     }
 
     public final String domain;
-    public final String applicationId;
     public final boolean readEnabled;
     public final boolean writeEnabled;
-    public final Optional<String> vespaDomain;
     private final Set<String> excludedBindings;
     private final Collection<Handler<?>> handlers;
     private final Collection<Servlet> servlets;
     private final DeployLogger logger;
 
     private AccessControl(String domain,
-                          String applicationId,
                           boolean writeEnabled,
                           boolean readEnabled,
                           Set<String> excludedBindings,
-                          Optional<String> vespaDomain,
                           Collection<Servlet> servlets,
                           Collection<Handler<?>> handlers,
                           DeployLogger logger) {
         this.domain = domain;
-        this.applicationId = applicationId;
         this.readEnabled = readEnabled;
         this.writeEnabled = writeEnabled;
         this.excludedBindings = Collections.unmodifiableSet(excludedBindings);
-        this.vespaDomain = vespaDomain;
         this.handlers = handlers;
         this.servlets = servlets;
         this.logger = logger;
@@ -125,6 +108,10 @@ public final class AccessControl {
                 .collect(Collectors.toCollection(ArrayList::new));
     }
 
+    public static boolean hasHandlerThatNeedsProtection(ApplicationContainerCluster cluster) {
+        return cluster.getHandlers().stream().anyMatch(AccessControl::handlerNeedsProtection);
+    }
+
     private Stream<Binding> getHandlerBindings() {
         return handlers.stream()
                         .filter(this::shouldHandlerBeProtected)
@@ -144,7 +131,7 @@ public final class AccessControl {
                 && handler.getServerBindings().stream().noneMatch(excludedBindings::contains);
     }
 
-    public static boolean isBuiltinGetOnly(Handler<?> handler) {
+    private static boolean isBuiltinGetOnly(Handler<?> handler) {
         return UNPROTECTED_HANDLERS.contains(handler.getClassId().getName());
     }
 
@@ -159,4 +146,13 @@ public final class AccessControl {
     private static Stream<String> servletBindings(Servlet servlet) {
         return Stream.of("http://*/").map(protocol -> protocol + servlet.bindingPath);
     }
+
+    private static boolean handlerNeedsProtection(Handler<?> handler) {
+        return ! isBuiltinGetOnly(handler) && hasNonMbusBinding(handler);
+    }
+
+    private static boolean hasNonMbusBinding(Handler<?> handler) {
+        return handler.getServerBindings().stream().anyMatch(binding -> ! binding.startsWith("mbus"));
+    }
+
 }
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/Http.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/Http.java
index 400ddf80cf9..0fcf7b2d06c 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/Http.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/Http.java
@@ -8,46 +8,39 @@ import com.yahoo.vespa.model.container.component.chain.Chain;
 import com.yahoo.vespa.model.container.component.chain.ChainedComponent;
 
 import java.util.Collection;
-import java.util.Collections;
 import java.util.List;
 import java.util.Optional;
+import java.util.concurrent.CopyOnWriteArrayList;
 
 /**
  * Represents the http servers and filters of a container cluster.
  *
  * @author Tony Vaagenes
+ * @author bjorncs
  */
 public class Http extends AbstractConfigProducer<AbstractConfigProducer<?>> implements ServerConfig.Producer {
 
-    private FilterChains filterChains;
-    private JettyHttpServer httpServer;
-    private List<Binding> bindings;
-    private final Optional<AccessControl> accessControl;
+    private final FilterChains filterChains;
+    private final List<Binding> bindings = new CopyOnWriteArrayList<>();
+    private volatile JettyHttpServer httpServer;
+    private volatile AccessControl accessControl;
 
-    public Http(List<Binding> bindings) {
-        this(bindings, null);
+    public Http(FilterChains chains) {
+        super("http");
+        this.filterChains = chains;
     }
 
-    public Http(List<Binding> bindings, AccessControl accessControl) {
-        super( "http");
-        this.bindings = Collections.unmodifiableList(bindings);
-        this.accessControl = Optional.ofNullable(accessControl);
-    }
-
-    public void setFilterChains(FilterChains filterChains) {
-        this.filterChains = filterChains;
-    }
-
-    public void setBindings(List<Binding> bindings) {
-        this.bindings = Collections.unmodifiableList(bindings);
+    public void setAccessControl(AccessControl accessControl) {
+            if (this.accessControl != null) throw new IllegalStateException("Access control already assigned");
+            this.accessControl = accessControl;
     }
 
     public FilterChains getFilterChains() {
         return filterChains;
     }
 
-    public JettyHttpServer getHttpServer() {
-        return httpServer;
+    public Optional<JettyHttpServer> getHttpServer() {
+        return Optional.ofNullable(httpServer);
     }
 
     public void setHttpServer(JettyHttpServer newServer) {
@@ -76,25 +69,21 @@ public class Http extends AbstractConfigProducer<AbstractConfigProducer<?>> impl
     }
 
     public Optional<AccessControl> getAccessControl() {
-        return accessControl;
+        return Optional.ofNullable(accessControl);
     }
 
     @Override
     public void getConfig(ServerConfig.Builder builder) {
         for (Binding binding : bindings) {
             builder.filter(new ServerConfig.Filter.Builder()
-                                   .id(binding.filterId().stringValue())
-                                   .binding(binding.binding()));
+                    .id(binding.filterId().stringValue())
+                    .binding(binding.binding()));
         }
     }
 
     @Override
     public void validate() {
-        validate(bindings);
-    }
-
-    void validate(Collection<Binding> bindings) {
-        if (bindings.isEmpty()) return;
+        if (((Collection<Binding>) bindings).isEmpty()) return;
 
         if (filterChains == null)
             throw new IllegalArgumentException("Null FilterChains are not allowed when there are filter bindings");
@@ -107,5 +96,4 @@ public class Http extends AbstractConfigProducer<AbstractConfigProducer<?>> impl
                 throw new RuntimeException("Can't find filter " + binding.filterId() + " for binding " + binding.binding());
         }
     }
-
 }
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/ConfiguredFilebasedSslProvider.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/ConfiguredFilebasedSslProvider.java
index 4f84a01ff94..4a331718985 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/ConfiguredFilebasedSslProvider.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/ConfiguredFilebasedSslProvider.java
@@ -8,6 +8,7 @@ import com.yahoo.jdisc.http.ssl.impl.ConfiguredSslContextFactoryProvider;
 import com.yahoo.osgi.provider.model.ComponentModel;
 import com.yahoo.vespa.model.container.component.SimpleComponent;
 
+import java.util.List;
 import java.util.Optional;
 
 import static com.yahoo.component.ComponentSpecification.fromString;
@@ -16,6 +17,7 @@ import static com.yahoo.component.ComponentSpecification.fromString;
  * Configure SSL using file references
  *
  * @author mortent
+ * @author bjorncs
  */
 public class ConfiguredFilebasedSslProvider extends SimpleComponent implements ConnectorConfig.Producer {
     public static final String COMPONENT_ID_PREFIX = "configured-ssl-provider@";
@@ -26,8 +28,16 @@ public class ConfiguredFilebasedSslProvider extends SimpleComponent implements C
     private final String certificatePath;
     private final String caCertificatePath;
     private final ConnectorConfig.Ssl.ClientAuth.Enum clientAuthentication;
+    private final List<String> cipherSuites;
+    private final List<String> protocolVersions;
 
-    public ConfiguredFilebasedSslProvider(String servername, String privateKeyPath, String certificatePath, String caCertificatePath, String clientAuthentication) {
+    public ConfiguredFilebasedSslProvider(String servername,
+                                          String privateKeyPath,
+                                          String certificatePath,
+                                          String caCertificatePath,
+                                          String clientAuthentication,
+                                          List<String> cipherSuites,
+                                          List<String> protocolVersions) {
         super(new ComponentModel(
                 new BundleInstantiationSpecification(new ComponentId(COMPONENT_ID_PREFIX+servername),
                                                      fromString(COMPONENT_CLASS),
@@ -36,15 +46,21 @@ public class ConfiguredFilebasedSslProvider extends SimpleComponent implements C
         this.certificatePath = certificatePath;
         this.caCertificatePath = caCertificatePath;
         this.clientAuthentication = mapToConfigEnum(clientAuthentication);
+        this.cipherSuites = cipherSuites;
+        this.protocolVersions = protocolVersions;
     }
 
     @Override
     public void getConfig(ConnectorConfig.Builder builder) {
-        builder.ssl.enabled(true);
-        builder.ssl.privateKeyFile(privateKeyPath);
-        builder.ssl.certificateFile(certificatePath);
-        builder.ssl.caCertificateFile(Optional.ofNullable(caCertificatePath).orElse(""));
-        builder.ssl.clientAuth(clientAuthentication);
+        builder.ssl(
+                new ConnectorConfig.Ssl.Builder()
+                        .enabled(true)
+                        .privateKeyFile(privateKeyPath)
+                        .certificateFile(certificatePath)
+                        .caCertificateFile(Optional.ofNullable(caCertificatePath).orElse(""))
+                        .clientAuth(clientAuthentication)
+                        .enabledCipherSuites(cipherSuites)
+                        .enabledProtocols(protocolVersions));
     }
 
     public SimpleComponent getComponent() {
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java
index 12db3b87243..fb8e9dffbbb 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java
@@ -7,6 +7,7 @@ import com.yahoo.jdisc.http.ConnectorConfig.Ssl.ClientAuth;
 import com.yahoo.vespa.model.container.component.SimpleComponent;
 import com.yahoo.vespa.model.container.http.ConnectorFactory;
 
+import java.time.Duration;
 import java.util.List;
 
 /**
@@ -17,42 +18,49 @@ import java.util.List;
 public class HostedSslConnectorFactory extends ConnectorFactory {
 
     private static final List<String> INSECURE_WHITELISTED_PATHS = List.of("/status.html");
+    private static final String DEFAULT_HOSTED_TRUSTSTORE = "/opt/yahoo/share/ssl/certs/athenz_certificate_bundle.pem";
 
     private final boolean enforceClientAuth;
+    private final String proxyProtocol;
 
     /**
-     * Create connector factory that uses a certificate provided by the config-model / configserver.
+     * Create connector factory that uses a certificate provided by the config-model / configserver and default hosted Vespa truststore.
      */
-    public static HostedSslConnectorFactory withProvidedCertificate(String serverName, EndpointCertificateSecrets endpointCertificateSecrets) {
-        return new HostedSslConnectorFactory(createConfiguredDirectSslProvider(serverName, endpointCertificateSecrets, /*tlsCaCertificates*/null), false);
+    // TODO Enforce client authentication
+    public static HostedSslConnectorFactory withProvidedCertificate(String proxyProtocol, String serverName, EndpointCertificateSecrets endpointCertificateSecrets) {
+        return new HostedSslConnectorFactory(proxyProtocol,
+                createConfiguredDirectSslProvider(serverName, endpointCertificateSecrets, DEFAULT_HOSTED_TRUSTSTORE, /*tlsCaCertificates*/null), false);
     }
 
     /**
      * Create connector factory that uses a certificate provided by the config-model / configserver and a truststore configured by the application.
      */
-    public static HostedSslConnectorFactory withProvidedCertificateAndTruststore(String serverName, EndpointCertificateSecrets endpointCertificateSecrets, String tlsCaCertificates) {
-        return new HostedSslConnectorFactory(createConfiguredDirectSslProvider(serverName, endpointCertificateSecrets, tlsCaCertificates), true);
+    public static HostedSslConnectorFactory withProvidedCertificateAndTruststore(
+            String proxyProtocol, String serverName, EndpointCertificateSecrets endpointCertificateSecrets, String tlsCaCertificates) {
+        return new HostedSslConnectorFactory(proxyProtocol,
+                createConfiguredDirectSslProvider(serverName, endpointCertificateSecrets, /*tlsCaCertificatesPath*/null, tlsCaCertificates), true);
     }
 
     /**
      * Create connector factory that uses the default certificate and truststore provided by Vespa (through Vespa-global TLS configuration).
      */
-    public static HostedSslConnectorFactory withDefaultCertificateAndTruststore(String serverName) {
-        return new HostedSslConnectorFactory(new DefaultSslProvider(serverName), true);
+    public static HostedSslConnectorFactory withDefaultCertificateAndTruststore(String proxyProtocol, String serverName) {
+        return new HostedSslConnectorFactory(proxyProtocol, new DefaultSslProvider(serverName), true);
     }
 
-    private HostedSslConnectorFactory(SimpleComponent sslProviderComponent, boolean enforceClientAuth) {
+    private HostedSslConnectorFactory(String proxyProtocol, SimpleComponent sslProviderComponent, boolean enforceClientAuth) {
         super("tls4443", 4443, sslProviderComponent);
+        this.proxyProtocol = proxyProtocol;
         this.enforceClientAuth = enforceClientAuth;
     }
 
     private static ConfiguredDirectSslProvider createConfiguredDirectSslProvider(
-            String serverName, EndpointCertificateSecrets endpointCertificateSecrets, String tlsCaCertificates) {
+            String serverName, EndpointCertificateSecrets endpointCertificateSecrets, String tlsCaCertificatesPath, String tlsCaCertificates) {
         return new ConfiguredDirectSslProvider(
                 serverName,
                 endpointCertificateSecrets.key(),
                 endpointCertificateSecrets.certificate(),
-                /*caCertificatePath*/null,
+                tlsCaCertificatesPath,
                 tlsCaCertificates,
                 ClientAuth.Enum.WANT_AUTH);
     }
@@ -60,9 +68,27 @@ public class HostedSslConnectorFactory extends ConnectorFactory {
     @Override
     public void getConfig(ConnectorConfig.Builder connectorBuilder) {
         super.getConfig(connectorBuilder);
-        connectorBuilder.tlsClientAuthEnforcer(new ConnectorConfig.TlsClientAuthEnforcer.Builder()
-                                                       .pathWhitelist(INSECURE_WHITELISTED_PATHS)
-                                                       .enable(enforceClientAuth));
+        connectorBuilder
+                .tlsClientAuthEnforcer(new ConnectorConfig.TlsClientAuthEnforcer.Builder()
+                                               .pathWhitelist(INSECURE_WHITELISTED_PATHS)
+                                               .enable(enforceClientAuth))
+                .proxyProtocol(configureProxyProtocol())
+                .idleTimeout(Duration.ofMinutes(3).toSeconds())
+                .maxConnectionLife(Duration.ofMinutes(10).toSeconds());
+    }
+
+    private ConnectorConfig.ProxyProtocol.Builder configureProxyProtocol() {
+        ConnectorConfig.ProxyProtocol.Builder proxyProtocolBuilder = new ConnectorConfig.ProxyProtocol.Builder();
+        switch (proxyProtocol) {
+            case "https-only":
+                return proxyProtocolBuilder.enabled(false).mixedMode(false);
+            case "https+proxy-protocol":
+                return proxyProtocolBuilder.enabled(true).mixedMode(true);
+            case "proxy-protocol-only":
+                return proxyProtocolBuilder.enabled(true).mixedMode(false);
+            default:
+                throw new IllegalArgumentException("Unknown proxy-protocol settings: " + proxyProtocol);
+        }
     }
 
 }
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/HttpBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/HttpBuilder.java
index b37caf22216..bfde9b9add1 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/HttpBuilder.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/HttpBuilder.java
@@ -6,19 +6,18 @@ import com.yahoo.config.application.api.DeployLogger;
 import com.yahoo.config.model.builder.xml.XmlHelper;
 import com.yahoo.config.model.deploy.DeployState;
 import com.yahoo.config.model.producer.AbstractConfigProducer;
-import com.yahoo.config.provision.ApplicationId;
-import com.yahoo.config.provision.ApplicationName;
+import com.yahoo.config.provision.AthenzDomain;
 import com.yahoo.text.XML;
 import com.yahoo.vespa.defaults.Defaults;
 import com.yahoo.vespa.model.builder.xml.dom.ModelElement;
 import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder;
-import com.yahoo.vespa.model.container.Container;
 import com.yahoo.vespa.model.container.ApplicationContainerCluster;
+import com.yahoo.vespa.model.container.Container;
 import com.yahoo.vespa.model.container.component.chain.Chain;
 import com.yahoo.vespa.model.container.http.AccessControl;
+import com.yahoo.vespa.model.container.http.Binding;
 import com.yahoo.vespa.model.container.http.FilterChains;
 import com.yahoo.vespa.model.container.http.Http;
-import com.yahoo.vespa.model.container.http.Binding;
 import org.w3c.dom.Element;
 
 import java.util.ArrayList;
@@ -55,24 +54,18 @@ public class HttpBuilder extends VespaDomBuilder.DomConfigProducerBuilder<Http>
             filterChains = new FilterChainsBuilder().newChainsInstance(ancestor);
         }
 
-        Http http = new Http(bindings, accessControl);
-        http.setFilterChains(filterChains);
-
-        buildHttpServers(deployState, ancestor, http, spec);
-
+        Http http = new Http(filterChains);
+        http.getBindings().addAll(bindings);
+        http.setAccessControl(accessControl);
+        http.setHttpServer(new JettyHttpServerBuilder().build(deployState, ancestor, spec));
         return http;
     }
 
     private AccessControl buildAccessControl(DeployState deployState, AbstractConfigProducer ancestor, Element accessControlElem) {
-        String application = XmlHelper.getOptionalChildValue(accessControlElem, "application")
-                .orElse(getDeployedApplicationId(deployState, ancestor).value());
-
-        AccessControl.Builder builder = new AccessControl.Builder(accessControlElem.getAttribute("domain"), application, deployState.getDeployLogger());
+        AthenzDomain domain = getAccessControlDomain(deployState, accessControlElem);
+        AccessControl.Builder builder = new AccessControl.Builder(domain.value(), deployState.getDeployLogger());
 
-        getContainerCluster(ancestor).ifPresent(cluster -> {
-            builder.setHandlers(cluster.getHandlers());
-            builder.setServlets(cluster.getAllServlets());
-        });
+        getContainerCluster(ancestor).ifPresent(builder::setHandlers);
 
         XmlHelper.getOptionalAttribute(accessControlElem, "read").ifPresent(
                 readAttr -> builder.readEnabled(Boolean.valueOf(readAttr)));
@@ -85,17 +78,29 @@ public class HttpBuilder extends VespaDomBuilder.DomConfigProducerBuilder<Http>
                     .map(XML::getValue)
                     .forEach(builder::excludeBinding);
         }
-        XmlHelper.getOptionalChildValue(accessControlElem, "vespa-domain").ifPresent(builder::vespaDomain);
         return builder.build();
     }
 
-    /**
-     * Returns the id of the deployed application, or the default value if not explicitly set (self-hosted).
-     */
-    private static ApplicationName getDeployedApplicationId(DeployState deployState, AbstractConfigProducer ancestor) {
-        return getContainerCluster(ancestor)
-                .map(cluster -> deployState.getProperties().applicationId().application())
-                .orElse(ApplicationId.defaultId().application());
+    // TODO Fail if domain is not provided through deploy properties
+    private static AthenzDomain getAccessControlDomain(DeployState deployState, Element accessControlElem) {
+        AthenzDomain tenantDomain = deployState.getProperties().athenzDomain().orElse(null);
+        AthenzDomain explicitDomain = XmlHelper.getOptionalAttribute(accessControlElem, "domain")
+                .map(AthenzDomain::from)
+                .orElse(null);
+        if (tenantDomain == null) {
+            if (explicitDomain == null) {
+                throw new IllegalStateException("No Athenz domain provided for 'access-control'");
+            }
+            deployState.getDeployLogger().log(Level.WARNING, "Athenz tenant is not provided by deploy call. This will soon be handled as failure.");
+        }
+        if (explicitDomain != null) {
+            if (tenantDomain != null && !explicitDomain.equals(tenantDomain)) {
+                throw new IllegalArgumentException(
+                        String.format("Domain in access-control ('%s') does not match tenant domain ('%s')", explicitDomain.value(), tenantDomain.value()));
+            }
+            deployState.getDeployLogger().log(Level.WARNING, "Domain in 'access-control' is deprecated and will be removed soon");
+        }
+        return tenantDomain != null ? tenantDomain : explicitDomain;
     }
 
     private static Optional<ApplicationContainerCluster> getContainerCluster(AbstractConfigProducer configProducer) {
@@ -125,10 +130,6 @@ public class HttpBuilder extends VespaDomBuilder.DomConfigProducerBuilder<Http>
         return result;
     }
 
-    private void buildHttpServers(DeployState deployState, AbstractConfigProducer ancestor, Http http, Element spec) {
-        http.setHttpServer(new JettyHttpServerBuilder().build(deployState, ancestor, spec));
-    }
-
     static int readPort(ModelElement spec, boolean isHosted, DeployLogger logger) {
         Integer port = spec.integerAttribute("port");
         if (port == null)
@@ -139,13 +140,9 @@ public class HttpBuilder extends VespaDomBuilder.DomConfigProducerBuilder<Http>
 
         int legalPortInHostedVespa = Container.BASEPORT;
         if (isHosted && port != legalPortInHostedVespa && ! spec.booleanAttribute("required", false)) {
-            // TODO: After January 2020:
-            // - Set required='true' for the http server on port 4443 in the tester services.xml in InternalStepRunner
-            // - Enable 2 currently ignored tests in this module
-            // - throw IllegalArgumentException here instead of warning
-            logger.log(Level.WARNING, "Illegal port " + port + " in http server '" +
-                                      spec.stringAttribute("id") + "'" +
-                                      ": Port must be set to " + legalPortInHostedVespa);
+            throw new IllegalArgumentException("Illegal port " + port + " in http server '" +
+                                               spec.stringAttribute("id") + "'" +
+                                               ": Port must be set to " + legalPortInHostedVespa);
         }
         return port;
     }
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyConnectorBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyConnectorBuilder.java
index db831a1ec2f..562026ab4dd 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyConnectorBuilder.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyConnectorBuilder.java
@@ -9,13 +9,17 @@ import com.yahoo.vespa.model.builder.xml.dom.ModelElement;
 import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder;
 import com.yahoo.vespa.model.container.component.SimpleComponent;
 import com.yahoo.vespa.model.container.http.ConnectorFactory;
-import com.yahoo.vespa.model.container.http.ssl.CustomSslProvider;
 import com.yahoo.vespa.model.container.http.ssl.ConfiguredFilebasedSslProvider;
+import com.yahoo.vespa.model.container.http.ssl.CustomSslProvider;
 import com.yahoo.vespa.model.container.http.ssl.DefaultSslProvider;
 import org.w3c.dom.Element;
 
+import java.util.Arrays;
+import java.util.List;
 import java.util.Optional;
 
+import static java.util.stream.Collectors.toList;
+
 /**
  * @author Einar M R Rosenvinge
  * @author mortent
@@ -40,12 +44,16 @@ public class JettyConnectorBuilder extends VespaDomBuilder.DomConfigProducerBuil
             String certificateFile = XML.getValue(XML.getChild(sslConfigurator, "certificate-file"));
             Optional<String> caCertificateFile = XmlHelper.getOptionalChildValue(sslConfigurator, "ca-certificates-file");
             Optional<String> clientAuthentication = XmlHelper.getOptionalChildValue(sslConfigurator, "client-authentication");
+            List<String> cipherSuites = extractOptionalCommaSeparatedList(sslConfigurator, "cipher-suites");
+            List<String> protocols = extractOptionalCommaSeparatedList(sslConfigurator, "protocols");
             return new ConfiguredFilebasedSslProvider(
                     serverName,
                     privateKeyFile,
                     certificateFile,
                     caCertificateFile.orElse(null),
-                    clientAuthentication.orElse(null));
+                    clientAuthentication.orElse(null),
+                    cipherSuites,
+                    protocols);
         } else if (sslProviderConfigurator != null) {
             String className = sslProviderConfigurator.getAttribute("class");
             String bundle = sslProviderConfigurator.getAttribute("bundle");
@@ -55,4 +63,13 @@ public class JettyConnectorBuilder extends VespaDomBuilder.DomConfigProducerBuil
         }
     }
 
+    private static List<String> extractOptionalCommaSeparatedList(Element sslElement, String listElementName) {
+        return XmlHelper.getOptionalChildValue(sslElement, listElementName)
+                .map(element ->
+                             Arrays.stream(element.split(","))
+                                     .filter(listEntry -> !listEntry.isBlank())
+                                     .map(String::trim)
+                                     .collect(toList()))
+                .orElse(List.of());
+    }
 }
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java
index e19d81e7fb2..4c4b1ca7f82 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java
@@ -58,7 +58,8 @@ public class ContainerSearch extends ContainerSubsystem<SearchChains>
     private void initializeDispatchers(Collection<AbstractSearchCluster> searchClusters) {
         for (AbstractSearchCluster searchCluster : searchClusters) {
             if ( ! ( searchCluster instanceof IndexedSearchCluster)) continue;
-            owningCluster.addComponent(new DispatcherComponent((IndexedSearchCluster)searchCluster));
+            var dispatcher = new DispatcherComponent((IndexedSearchCluster)searchCluster);
+            owningCluster.addComponent(dispatcher);
         }
     }
 
@@ -130,7 +131,7 @@ public class ContainerSearch extends ContainerSubsystem<SearchChains>
     	    AbstractSearchCluster sys = findClusterWithId(searchClusters, i);
     		QrSearchersConfig.Searchcluster.Builder scB = new QrSearchersConfig.Searchcluster.Builder().
     				name(sys.getClusterName());
-    		for (AbstractSearchCluster.SearchDefinitionSpec spec : sys.getLocalSDS()) {
+    		for (AbstractSearchCluster.SchemaSpec spec : sys.getLocalSDS()) {
     			scB.searchdef(spec.getSearchDefinition().getName());
     		}
     		scB.rankprofiles(new QrSearchersConfig.Searchcluster.Rankprofiles.Builder().configid(sys.getConfigId()));
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/DispatcherComponent.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/DispatcherComponent.java
index 704188e80e8..284aa3b46c0 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/DispatcherComponent.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/DispatcherComponent.java
@@ -1,6 +1,7 @@
 // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.model.container.search;
 
+import com.yahoo.config.model.producer.AbstractConfigProducer;
 import com.yahoo.osgi.provider.model.ComponentModel;
 import com.yahoo.vespa.config.search.DispatchConfig;
 import com.yahoo.vespa.model.container.component.Component;
@@ -13,7 +14,7 @@ import com.yahoo.vespa.model.search.IndexedSearchCluster;
  *
  * @author bratseth
  */
-public class DispatcherComponent extends Component<DispatcherComponent, ComponentModel>
+public class DispatcherComponent extends Component<AbstractConfigProducer<?>, ComponentModel>
         implements DispatchConfig.Producer {
 
     private final IndexedSearchCluster indexedSearchCluster;
@@ -21,14 +22,17 @@ public class DispatcherComponent extends Component<DispatcherComponent, Componen
     public DispatcherComponent(IndexedSearchCluster indexedSearchCluster) {
         super(toComponentModel(indexedSearchCluster));
         this.indexedSearchCluster = indexedSearchCluster;
+        String clusterName = indexedSearchCluster.getClusterName();
+        var rpcResoucePool = new RpcResourcePoolComponent(clusterName);
+        inject(rpcResoucePool);
+        addComponent(rpcResoucePool);
     }
 
     private static ComponentModel toComponentModel(IndexedSearchCluster indexedSearchCluster) {
         String dispatcherComponentId = "dispatcher." + indexedSearchCluster.getClusterName(); // used by ClusterSearcher
         return new ComponentModel(dispatcherComponentId,
-                                  "com.yahoo.search.dispatch.Dispatcher",
-                                  BundleMapper.searchAndDocprocBundle,
-                                  null);
+                                  com.yahoo.search.dispatch.Dispatcher.class.getName(),
+                                  BundleMapper.searchAndDocprocBundle);
     }
 
     @Override
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/QueryProfiles.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/QueryProfiles.java
index 0abb0803405..0a9618e7b08 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/QueryProfiles.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/QueryProfiles.java
@@ -232,8 +232,7 @@ public class QueryProfiles implements Serializable, QueryProfilesConfig.Producer
         return propB;
     }
 
-    private QueryProfilesConfig.Queryprofile.Queryprofilevariant.Property.Builder createVariantPropertyFieldConfig(
-            String fullName, Object value) {
+    private QueryProfilesConfig.Queryprofile.Queryprofilevariant.Property.Builder createVariantPropertyFieldConfig(String fullName, Object value) {
         QueryProfilesConfig.Queryprofile.Queryprofilevariant.Property.Builder propB = new QueryProfilesConfig.Queryprofile.Queryprofilevariant.Property.Builder();
         if (value instanceof SubstituteString)
             value=value.toString(); // Send only types understood by configBuilder downwards
@@ -251,7 +250,7 @@ public class QueryProfiles implements Serializable, QueryProfilesConfig.Producer
             qtB.matchaspath(true);
         for (QueryProfileType inherited : profileType.inherited())
             qtB.inherit(inherited.getId().stringValue());
-        List<FieldDescription> fields=new ArrayList<>(profileType.declaredFields().values());
+        List<FieldDescription> fields = new ArrayList<>(profileType.declaredFields().values());
         Collections.sort(fields);
         for (FieldDescription field : fields)
             qtB.field(createConfig(field));
@@ -260,22 +259,20 @@ public class QueryProfiles implements Serializable, QueryProfilesConfig.Producer
 
     private QueryProfilesConfig.Queryprofiletype.Field.Builder createConfig(FieldDescription field) {
         QueryProfilesConfig.Queryprofiletype.Field.Builder fB = new QueryProfilesConfig.Queryprofiletype.Field.Builder();
-        fB.
-        name(field.getName()).
-        type(field.getType().stringValue());
+        fB.name(field.getName()).type(field.getType().stringValue());
         if ( ! field.isOverridable())
             fB.overridable(false);
         if (field.isMandatory())
             fB.mandatory(true);
-        String aliases=toSpaceSeparatedString(field.getAliases());
-        if (!aliases.isEmpty())
+        String aliases = toSpaceSeparatedString(field.getAliases());
+        if ( ! aliases.isEmpty())
             fB.alias(aliases);
         return fB;
     }
 
     public String toSpaceSeparatedString(List<String> list) {
-        StringBuilder b=new StringBuilder();
-        for (Iterator<String> i=list.iterator(); i.hasNext(); ) {
+        StringBuilder b = new StringBuilder();
+        for (Iterator<String> i = list.iterator(); i.hasNext(); ) {
             b.append(i.next());
             if (i.hasNext())
                 b.append(" ");
@@ -290,10 +287,7 @@ public class QueryProfiles implements Serializable, QueryProfilesConfig.Producer
         }
     }
 
-    /**
-     * The config produced by this
-     * @return query profiles config
-     */
+    /** Returns the config produced by this */
     public QueryProfilesConfig getConfig() {
         QueryProfilesConfig.Builder qB = new QueryProfilesConfig.Builder();
         getConfig(qB);
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/RpcResourcePoolComponent.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/RpcResourcePoolComponent.java
new file mode 100644
index 00000000000..2689c2ce71b
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/RpcResourcePoolComponent.java
@@ -0,0 +1,18 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.container.search;
+
+import com.yahoo.osgi.provider.model.ComponentModel;
+import com.yahoo.vespa.model.container.component.Component;
+import com.yahoo.vespa.model.container.xml.BundleMapper;
+
+public class RpcResourcePoolComponent extends Component<RpcResourcePoolComponent, ComponentModel> {
+
+    public RpcResourcePoolComponent(String clusterName) {
+        super(toComponentModel(clusterName));
+    }
+
+    private static ComponentModel toComponentModel(String clusterName) {
+        String componentId = "rpcresourcepool." + clusterName;
+        return new ComponentModel(componentId, com.yahoo.search.dispatch.rpc.RpcResourcePool.class.getName(), BundleMapper.searchAndDocprocBundle);
+    }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/LocalProvider.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/LocalProvider.java
index e05b2d27e09..4ecc666a9f2 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/LocalProvider.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/LocalProvider.java
@@ -118,7 +118,7 @@ public class LocalProvider extends Provider implements
     public List<String> getDocumentTypes() {
         List<String> documentTypes = new ArrayList<>();
 
-        for (AbstractSearchCluster.SearchDefinitionSpec spec : searchCluster.getLocalSDS()) {
+        for (AbstractSearchCluster.SchemaSpec spec : searchCluster.getLocalSDS()) {
             documentTypes.add(spec.getSearchDefinition().getSearch().getDocument().getName());
         }
 
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilder.java
index 2828fcf09d0..19d1b6546a6 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilder.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilder.java
@@ -9,6 +9,8 @@ import org.w3c.dom.Element;
 import java.util.Arrays;
 import java.util.List;
 
+import static com.yahoo.vespa.model.container.xml.ContainerModelBuilder.SEARCH_HANDLER_CLASS;
+
 /**
  * This object builds a bundle instantiation spec from an XML element.
  *
@@ -36,7 +38,7 @@ public class BundleInstantiationSpecificationBuilder {
 
     private static void validate(BundleInstantiationSpecification instSpec) {
         List<String> forbiddenClasses = Arrays.asList(
-                "com.yahoo.search.handler.SearchHandler",
+                SEARCH_HANDLER_CLASS,
                 "com.yahoo.processing.handler.ProcessingHandler");
 
         for (String forbiddenClass: forbiddenClasses) {
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java
index aef2697a5dd..4bd9f5fa8b0 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java
@@ -19,12 +19,15 @@ import com.yahoo.config.model.builder.xml.ConfigModelBuilder;
 import com.yahoo.config.model.builder.xml.ConfigModelId;
 import com.yahoo.config.model.deploy.DeployState;
 import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.config.provision.AthenzDomain;
 import com.yahoo.config.provision.AthenzService;
 import com.yahoo.config.provision.Capacity;
 import com.yahoo.config.provision.ClusterMembership;
+import com.yahoo.config.provision.ClusterResources;
 import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.config.provision.Environment;
 import com.yahoo.config.provision.HostName;
+import com.yahoo.config.provision.NodeResources;
 import com.yahoo.config.provision.NodeType;
 import com.yahoo.config.provision.SystemName;
 import com.yahoo.config.provision.Zone;
@@ -57,9 +60,11 @@ import com.yahoo.vespa.model.container.SecretStore;
 import com.yahoo.vespa.model.container.component.Component;
 import com.yahoo.vespa.model.container.component.FileStatusHandlerComponent;
 import com.yahoo.vespa.model.container.component.Handler;
+import com.yahoo.vespa.model.container.component.chain.Chain;
 import com.yahoo.vespa.model.container.component.chain.ProcessingHandler;
 import com.yahoo.vespa.model.container.docproc.ContainerDocproc;
 import com.yahoo.vespa.model.container.docproc.DocprocChains;
+import com.yahoo.vespa.model.container.http.AccessControl;
 import com.yahoo.vespa.model.container.http.ConnectorFactory;
 import com.yahoo.vespa.model.container.http.FilterChains;
 import com.yahoo.vespa.model.container.http.Http;
@@ -88,6 +93,7 @@ import java.util.function.Consumer;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
+import static com.yahoo.vespa.model.container.http.AccessControl.ACCESS_CONTROL_CHAIN_ID;
 import static java.util.logging.Level.WARNING;
 
 /**
@@ -106,6 +112,9 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> {
     private static final String DEPRECATED_CONTAINER_TAG = "jdisc";
     private static final String ENVIRONMENT_VARIABLES_ELEMENT = "environment-variables";
 
+    static final String SEARCH_HANDLER_CLASS = com.yahoo.search.handler.SearchHandler.class.getName();
+    static final String SEARCH_HANDLER_BINDING = "http://*/search/*";
+
     public enum Networking { disable, enable }
 
     private ApplicationPackage app;
@@ -175,7 +184,6 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> {
         DocumentFactoryBuilder.buildDocumentFactories(cluster, spec);
         addConfiguredComponents(deployState, cluster, spec);
         addSecretStore(cluster, spec);
-        addHandlers(deployState, cluster, spec);
 
         addRestApis(deployState, spec, cluster);
         addServlets(deployState, spec, cluster);
@@ -188,8 +196,9 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> {
 
         cluster.addDefaultHandlersExceptStatus();
         addStatusHandlers(cluster, context.getDeployState().isHosted());
+        addUserHandlers(deployState, cluster, spec);
 
-        addHttp(deployState, spec, cluster, context.getApplicationType(), deployState.getProperties().applicationId().instance().isTester());
+        addHttp(deployState, spec, cluster, context);
 
         addAccessLogs(deployState, cluster, spec);
         addRoutingAliases(cluster, spec, deployState.zone().environment());
@@ -287,7 +296,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> {
 
     private void addClientProviders(DeployState deployState, Element spec, ApplicationContainerCluster cluster) {
         for (Element clientSpec: XML.getChildren(spec, "client")) {
-            cluster.addComponent(new DomClientProviderBuilder().build(deployState, cluster, clientSpec));
+            cluster.addComponent(new DomClientProviderBuilder(cluster).build(deployState, cluster, clientSpec));
         }
     }
 
@@ -302,7 +311,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> {
             AccessLogBuilder.buildIfNotDisabled(deployState, cluster, accessLog).ifPresent(cluster::addComponent);
         }
 
-        if (accessLogElements.isEmpty() && cluster.getSearch() != null)
+        if (accessLogElements.isEmpty() && deployState.getAccessLoggingEnabledByDefault())
             cluster.addDefaultSearchAccessLog();
     }
 
@@ -311,21 +320,23 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> {
     }
 
 
-    private void addHttp(DeployState deployState, Element spec, ApplicationContainerCluster cluster, ApplicationType applicationType, boolean isTesterApplication) {
+    private void addHttp(DeployState deployState, Element spec, ApplicationContainerCluster cluster, ConfigModelContext context) {
         Element httpElement = XML.getChild(spec, "http");
         if (httpElement != null) {
             cluster.setHttp(buildHttp(deployState, cluster, httpElement));
         }
-        if (deployState.isHosted() && applicationType == ApplicationType.DEFAULT && !isTesterApplication) {
+        if (isHostedTenantApplication(context)) {
+            addHostedImplicitHttpIfNotPresent(cluster);
+            addHostedImplicitAccessControlIfNotPresent(deployState, cluster);
             addAdditionalHostedConnector(deployState, cluster);
         }
     }
 
     private void addAdditionalHostedConnector(DeployState deployState, ApplicationContainerCluster cluster) {
-        addImplicitHttpIfNotPresent(cluster);
-        JettyHttpServer server = cluster.getHttp().getHttpServer();
+        JettyHttpServer server = cluster.getHttp().getHttpServer().get();
         String serverName = server.getComponentId().getName();
 
+        String proxyProtocol = deployState.getProperties().proxyProtocol();
         // If the deployment contains certificate/private key reference, setup TLS port
         if (deployState.endpointCertificateSecrets().isPresent()) {
             boolean authorizeClient = deployState.zone().system().isPublic();
@@ -334,27 +345,47 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> {
             }
             EndpointCertificateSecrets endpointCertificateSecrets = deployState.endpointCertificateSecrets().get();
             HostedSslConnectorFactory connectorFactory = authorizeClient
-                    ? HostedSslConnectorFactory.withProvidedCertificateAndTruststore(serverName, endpointCertificateSecrets, deployState.tlsClientAuthority().get())
-                    : HostedSslConnectorFactory.withProvidedCertificate(serverName, endpointCertificateSecrets);
+                    ? HostedSslConnectorFactory.withProvidedCertificateAndTruststore(proxyProtocol, serverName, endpointCertificateSecrets, deployState.tlsClientAuthority().get())
+                    : HostedSslConnectorFactory.withProvidedCertificate(proxyProtocol, serverName, endpointCertificateSecrets);
             server.addConnector(connectorFactory);
         } else {
-            server.addConnector(HostedSslConnectorFactory.withDefaultCertificateAndTruststore(serverName));
+            server.addConnector(HostedSslConnectorFactory.withDefaultCertificateAndTruststore(proxyProtocol, serverName));
         }
     }
 
-    private static void addImplicitHttpIfNotPresent(ApplicationContainerCluster cluster) {
+    private static boolean isHostedTenantApplication(ConfigModelContext context) {
+        var deployState = context.getDeployState();
+        boolean isTesterApplication = deployState.getProperties().applicationId().instance().isTester();
+        return deployState.isHosted() && context.getApplicationType() == ApplicationType.DEFAULT && !isTesterApplication;
+    }
+
+    private static void addHostedImplicitHttpIfNotPresent(ApplicationContainerCluster cluster) {
         if(cluster.getHttp() == null) {
-            Http http = new Http(Collections.emptyList());
-            http.setFilterChains(new FilterChains(cluster));
-            cluster.setHttp(http);
+            cluster.setHttp(new Http(new FilterChains(cluster)));
         }
-        if(cluster.getHttp().getHttpServer() == null) {
+        if(cluster.getHttp().getHttpServer().isEmpty()) {
             JettyHttpServer defaultHttpServer = new JettyHttpServer(new ComponentId("DefaultHttpServer"));
             cluster.getHttp().setHttpServer(defaultHttpServer);
             defaultHttpServer.addConnector(new ConnectorFactory("SearchServer", Defaults.getDefaults().vespaWebServicePort()));
         }
     }
 
+    private void addHostedImplicitAccessControlIfNotPresent(DeployState deployState, ApplicationContainerCluster cluster) {
+        Http http = cluster.getHttp();
+        if (http.getAccessControl().isPresent()) return; // access control added explicitly
+        AthenzDomain tenantDomain = deployState.getProperties().athenzDomain().orElse(null);
+        if (tenantDomain == null) return; // tenant domain not present, cannot add access control. this should eventually be a failure.
+        AccessControl accessControl =
+                new AccessControl.Builder(tenantDomain.value(), deployState.getDeployLogger())
+                        .setHandlers(cluster)
+                        .readEnabled(false)
+                        .writeEnabled(false)
+                        .build();
+        http.getFilterChains().add(new Chain<>(FilterChains.emptyChainSpec(ACCESS_CONTROL_CHAIN_ID)));
+        http.setAccessControl(accessControl);
+        http.getBindings().addAll(accessControl.getBindings());
+    }
+
     private Http buildHttp(DeployState deployState, ApplicationContainerCluster cluster, Element httpElement) {
         Http http = new HttpBuilder().build(deployState, cluster, httpElement);
 
@@ -441,10 +472,10 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> {
         containerSearch.setPageTemplates(PageTemplates.create(applicationPackage));
     }
 
-    private void addHandlers(DeployState deployState, ApplicationContainerCluster cluster, Element spec) {
+    private void addUserHandlers(DeployState deployState, ApplicationContainerCluster cluster, Element spec) {
         for (Element component: XML.getChildren(spec, "handler")) {
             cluster.addComponent(
-                    new DomHandlerBuilder().build(deployState, cluster, component));
+                    new DomHandlerBuilder(cluster).build(deployState, cluster, component));
         }
     }
 
@@ -532,14 +563,17 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> {
         cluster.setJvmGCOptions(buildJvmGCOptions(context.getDeployState().zone(), jvmGCOptions, context.getDeployState().isHosted()));
     }
 
+    /**
+     * Add nodes to cluster according to the given containerElement.
+     *
+     * Note: DO NOT change allocation behaviour to allow version X and Y of the config-model to allocate a different set
+     * of nodes. Such changes must be guarded by a common condition (e.g. feature flag) so the behaviour can be changed
+     * simultaneously for all active config models.
+     */
     private void addNodesFromXml(ApplicationContainerCluster cluster, Element containerElement, ConfigModelContext context) {
         Element nodesElement = XML.getChild(containerElement, "nodes");
-        if (nodesElement == null) { // default single node on localhost
-            ApplicationContainer node = new ApplicationContainer(cluster, "container.0", 0, cluster.isHostedVespa());
-            HostResource host = allocateSingleNodeHost(cluster, log, containerElement, context);
-            node.setHostResource(host);
-            node.initService(context.getDeployLogger());
-            cluster.addContainers(Collections.singleton(node));
+        if (nodesElement == null) {
+            cluster.addContainers(allocateWithoutNodesTag(cluster, context));
         } else {
             List<ApplicationContainer> nodes = createNodes(cluster, nodesElement, context);
 
@@ -615,30 +649,33 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> {
                                                " must be an integer percentage ending by the '%' sign");
         }
     }
-    
-    /** Creates a single host when there is no nodes tag */
-    private HostResource allocateSingleNodeHost(ApplicationContainerCluster cluster, DeployLogger logger, Element containerElement, ConfigModelContext context) {
+
+    /** Allocate a container cluster without a nodes tag */
+    private List<ApplicationContainer> allocateWithoutNodesTag(ApplicationContainerCluster cluster, ConfigModelContext context) {
         DeployState deployState = context.getDeployState();
         HostSystem hostSystem = cluster.hostSystem();
         if (deployState.isHosted()) {
-            Optional<HostResource> singleContentHost = getHostResourceFromContentClusters(cluster, containerElement, context);
-            if (singleContentHost.isPresent()) { // there is a content cluster; put the container on its first node 
-                return singleContentHost.get();
-            }
-            else { // request 1 node
-                ClusterSpec clusterSpec = ClusterSpec.request(ClusterSpec.Type.container,
-                                                              ClusterSpec.Id.from(cluster.getName()),
-                                                              deployState.getWantedNodeVespaVersion(),
-                                                              false);
-                Capacity capacity = Capacity.fromCount(1,
-                                                       Optional.empty(),
-                                                       false,
-                                                       ! deployState.getProperties().isBootstrap());
-                return hostSystem.allocateHosts(clusterSpec, capacity, 1, logger).keySet().iterator().next();
-            }
-        } else {
-            return hostSystem.getHost(Container.SINGLENODE_CONTAINER_SERVICESPEC);
+            // request just enough nodes to satisfy environment capacity requirement
+            ClusterSpec clusterSpec = ClusterSpec.request(ClusterSpec.Type.container,
+                                                          ClusterSpec.Id.from(cluster.getName()))
+                                                 .vespaVersion(deployState.getWantedNodeVespaVersion())
+                                                 .dockerImageRepo(deployState.getWantedDockerImageRepo())
+                                                 .build();
+            int nodeCount = deployState.zone().environment().isProduction() ? 2 : 1;
+            Capacity capacity = Capacity.from(new ClusterResources(nodeCount, 1, NodeResources.unspecified),
+                                              false,
+                                              !deployState.getProperties().isBootstrap());
+            var hosts = hostSystem.allocateHosts(clusterSpec, capacity, log);
+            return createNodesFromHosts(log, hosts, cluster);
         }
+        return singleHostContainerCluster(cluster, hostSystem.getHost(Container.SINGLENODE_CONTAINER_SERVICESPEC), context);
+    }
+
+    private List<ApplicationContainer> singleHostContainerCluster(ApplicationContainerCluster cluster, HostResource host, ConfigModelContext context) {
+        ApplicationContainer node = new ApplicationContainer(cluster, "container.0", 0, cluster.isHostedVespa());
+        node.setHostResource(host);
+        node.initService(context.getDeployLogger());
+        return List.of(node);
     }
 
     private List<ApplicationContainer> createNodesFromNodeCount(ApplicationContainerCluster cluster, Element nodesElement, ConfigModelContext context) {
@@ -652,13 +689,13 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> {
 
     private List<ApplicationContainer> createNodesFromNodeType(ApplicationContainerCluster cluster, Element nodesElement, ConfigModelContext context) {
         NodeType type = NodeType.valueOf(nodesElement.getAttribute("type"));
-        ClusterSpec clusterSpec = ClusterSpec.request(ClusterSpec.Type.container, 
-                                                      ClusterSpec.Id.from(cluster.getName()), 
-                                                      context.getDeployState().getWantedNodeVespaVersion(),
-                                                      false);
+        ClusterSpec clusterSpec = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from(cluster.getName()))
+                .vespaVersion(context.getDeployState().getWantedNodeVespaVersion())
+                .dockerImageRepo(context.getDeployState().getWantedDockerImageRepo())
+                .build();
         Map<HostResource, ClusterMembership> hosts = 
                 cluster.getRoot().hostSystem().allocateHosts(clusterSpec,
-                                                             Capacity.fromRequiredNodeType(type), 1, log);
+                                                             Capacity.fromRequiredNodeType(type), log);
         return createNodesFromHosts(context.getDeployLogger(), hosts, cluster);
     }
     
@@ -766,7 +803,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> {
 
         ProcessingHandler<SearchChains> searchHandler = new ProcessingHandler<>(cluster.getSearch().getChains(),
                                                                                 "com.yahoo.search.handler.SearchHandler");
-        String[] defaultBindings = {"http://*/search/*"};
+        String[] defaultBindings = {SEARCH_HANDLER_BINDING};
         for (String binding: serverBindings(searchElement, defaultBindings)) {
             searchHandler.addServerBindings(binding);
         }
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java
index 75161859068..fcaba66ef69 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java
@@ -16,8 +16,8 @@ import com.yahoo.vespa.model.search.AbstractSearchCluster;
 import com.yahoo.vespa.model.search.IndexedSearchCluster;
 import com.yahoo.vespa.model.search.NodeSpec;
 import com.yahoo.vespa.model.search.SearchCluster;
-import com.yahoo.vespa.model.search.SearchDefinition;
-import com.yahoo.vespa.model.search.SearchDefinitionXMLHandler;
+import com.yahoo.vespa.model.search.NamedSchema;
+import com.yahoo.vespa.model.search.SchemaDefinitionXMLHandler;
 import com.yahoo.vespa.model.search.SearchNode;
 import com.yahoo.vespa.model.search.StreamingSearchCluster;
 import com.yahoo.vespa.model.search.TransactionLogServer;
@@ -52,6 +52,7 @@ public class ContentSearchCluster extends AbstractConfigProducer implements Prot
     private final String clusterName;
     private final Map<String, NewDocumentType> documentDefinitions;
     private final Set<NewDocumentType> globallyDistributedDocuments;
+    private Double visibilityDelay = 0.0;
 
     /** The search nodes of this if it does not have an indexed cluster */
     private List<SearchNode> nonIndexed = new ArrayList<>();
@@ -135,21 +136,21 @@ public class ContentSearchCluster extends AbstractConfigProducer implements Prot
 
         private void buildIndexedSearchCluster(DeployState deployState, ModelElement clusterElem,
                                                String clusterName, ContentSearchCluster search) {
-            List<ModelElement> indexedDefs = getIndexedSearchDefinitions(clusterElem);
+            List<ModelElement> indexedDefs = getIndexedSchemas(clusterElem);
             if (!indexedDefs.isEmpty()) {
                 IndexedSearchCluster isc = new IndexedSearchCluster(search, clusterName, 0, deployState);
                 isc.setRoutingSelector(clusterElem.childAsString("documents.selection"));
 
                 Double visibilityDelay = clusterElem.childAsDouble("engine.proton.visibility-delay");
                 if (visibilityDelay != null) {
-                    isc.setVisibilityDelay(visibilityDelay);
+                    search.setVisibilityDelay(visibilityDelay);
                 }
 
                 search.addSearchCluster(deployState, isc, getQueryTimeout(clusterElem), indexedDefs);
             }
         }
 
-        private List<ModelElement> getIndexedSearchDefinitions(ModelElement clusterElem) {
+        private List<ModelElement> getIndexedSchemas(ModelElement clusterElem) {
             List<ModelElement> indexedDefs = new ArrayList<>();
             ModelElement docElem = clusterElem.child("documents");
             if (docElem == null) {
@@ -179,29 +180,36 @@ public class ContentSearchCluster extends AbstractConfigProducer implements Prot
         this.flushOnShutdown = flushOnShutdown;
     }
 
+    public void setVisibilityDelay(double delay) {
+        this.visibilityDelay=delay;
+        if (hasIndexedCluster()) {
+            indexedCluster.setVisibilityDelay(delay);
+        }
+    }
+
     private void addSearchCluster(DeployState deployState, SearchCluster cluster, Double queryTimeout, List<ModelElement> documentDefs) {
-        addSearchDefinitions(deployState, documentDefs, cluster);
+        addSchemas(deployState, documentDefs, cluster);
 
         if (queryTimeout != null) {
             cluster.setQueryTimeout(queryTimeout);
         }
         cluster.defaultDocumentsConfig();
-        cluster.deriveSearchDefinitions(deployState);
+        cluster.deriveSchemas(deployState);
         addCluster(cluster);
     }
 
-    private void addSearchDefinitions(DeployState deployState, List<ModelElement> searchDefs, AbstractSearchCluster sc) {
+    private void addSchemas(DeployState deployState, List<ModelElement> searchDefs, AbstractSearchCluster sc) {
         for (ModelElement e : searchDefs) {
-            SearchDefinitionXMLHandler searchDefinitionXMLHandler = new SearchDefinitionXMLHandler(e);
-            SearchDefinition searchDefinition =
-                    searchDefinitionXMLHandler.getResponsibleSearchDefinition(deployState.getSearchDefinitions());
+            SchemaDefinitionXMLHandler schemaDefinitionXMLHandler = new SchemaDefinitionXMLHandler(e);
+            NamedSchema searchDefinition =
+                    schemaDefinitionXMLHandler.getResponsibleSearchDefinition(deployState.getSchemas());
             if (searchDefinition == null)
                 throw new RuntimeException("Search definition parsing error or file does not exist: '" +
-                        searchDefinitionXMLHandler.getName() + "'");
+                                           schemaDefinitionXMLHandler.getName() + "'");
 
             // TODO: remove explicit building of user configs when the complete content model is built using builders.
-            sc.getLocalSDS().add(new AbstractSearchCluster.SearchDefinitionSpec(searchDefinition,
-                    UserConfigBuilder.build(e.getXml(), deployState, deployState.getDeployLogger())));
+            sc.getLocalSDS().add(new AbstractSearchCluster.SchemaSpec(searchDefinition,
+                                                                      UserConfigBuilder.build(e.getXml(), deployState, deployState.getDeployLogger())));
             //need to get the document names from this sdfile
             sc.addDocumentNames(searchDefinition);
         }
@@ -307,7 +315,6 @@ public class ContentSearchCluster extends AbstractConfigProducer implements Prot
 
     @Override
     public void getConfig(ProtonConfig.Builder builder) {
-        double visibilityDelay = hasIndexedCluster() ? getIndexed().getVisibilityDelay() : 0.0;
         builder.feeding.concurrency(0.40); // As if specified 0.8 in services.xml
         boolean hasAnyNonIndexedCluster = false;
         for (NewDocumentType type : TopologicalDocumentTypeSorter.sort(documentDefinitions.values())) {
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/DispatchTuning.java b/config-model/src/main/java/com/yahoo/vespa/model/content/DispatchTuning.java
index 0d15207b6ce..0f9eb5341ab 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/content/DispatchTuning.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/DispatchTuning.java
@@ -11,18 +11,25 @@ public class DispatchTuning {
 
     public static final DispatchTuning empty = new DispatchTuning.Builder().build();
 
-    public enum DispatchPolicy { ROUNDROBIN, ADAPTIVE};
+    public enum DispatchPolicy { ROUNDROBIN, ADAPTIVE}
 
     private final Integer maxHitsPerPartition;
     private DispatchPolicy dispatchPolicy;
     private final Double minGroupCoverage;
     private final Double minActiveDocsCoverage;
 
+    public Double getTopkProbability() {
+        return topkProbability;
+    }
+
+    private final Double topkProbability;
+
     private DispatchTuning(Builder builder) {
         maxHitsPerPartition = builder.maxHitsPerPartition;
         dispatchPolicy = builder.dispatchPolicy;
         minGroupCoverage = builder.minGroupCoverage;
         minActiveDocsCoverage = builder.minActiveDocsCoverage;
+        topkProbability = builder.topKProbability;
     }
 
     /** Returns the max number of hits to fetch from each partition, or null to fetch all */
@@ -46,6 +53,7 @@ public class DispatchTuning {
         private DispatchPolicy dispatchPolicy;
         private Double minGroupCoverage;
         private Double minActiveDocsCoverage;
+        private Double topKProbability;
 
         public DispatchTuning build() {
             return new DispatchTuning(this);
@@ -55,6 +63,10 @@ public class DispatchTuning {
             this.maxHitsPerPartition = maxHitsPerPartition;
             return this;
         }
+        public Builder setTopKProbability(Double topKProbability) {
+            this.topKProbability = topKProbability;
+            return this;
+        }
         public Builder setDispatchPolicy(String policy) {
             if (policy != null)
                 dispatchPolicy = toDispatchPolicy(policy);
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java b/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java
index adfc703f747..f0c374b398f 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java
@@ -376,6 +376,10 @@ public class StorageGroup {
          *                            specified using a group count attribute.
          * <li>Neither element is present: Create a single node.
          * </ul>
+         *
+         * Note: DO NOT change allocation behaviour to allow version X and Y of the config-model to allocate a different
+         * set of nodes. Such changes must be guarded by a common condition (e.g. feature flag) so the behaviour can be
+         * changed simultaneously for all active config models.
          */
         private GroupBuilder collectGroup(boolean isHosted, Optional<ModelElement> groupElement, Optional<ModelElement> nodesElement, String name, String index) {
             StorageGroup group = new StorageGroup(
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java
index 9b17412b83a..6dd3e619ec2 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java
@@ -136,10 +136,7 @@ public class ContentCluster extends AbstractConfigProducer implements
             c.rootGroup = new StorageGroup.Builder(contentElement, context).buildRootGroup(deployState, redundancyBuilder, c);
             validateThatGroupSiblingsAreUnique(c.clusterName, c.rootGroup);
             c.search.handleRedundancy(c.redundancy);
-
-            IndexedSearchCluster index = c.search.getIndexed();
-            if (index != null)
-                setupIndexedCluster(index, contentElement, deployState.getDeployLogger());
+            setupSearchCluster(c.search, contentElement, deployState.getDeployLogger());
 
             if (c.search.hasIndexedCluster() && !(c.persistenceFactory instanceof ProtonEngine.Factory) )
                 throw new RuntimeException("Indexed search requires proton as engine");
@@ -166,17 +163,25 @@ public class ContentCluster extends AbstractConfigProducer implements
             return c;
         }
 
-        private void setupIndexedCluster(IndexedSearchCluster index, ModelElement element, DeployLogger logger) {
+        private void setupSearchCluster(ContentSearchCluster csc, ModelElement element, DeployLogger logger) {
             ContentSearch search = DomContentSearchBuilder.build(element);
+            Double visibilityDelay = search.getVisibilityDelay();
+            if (visibilityDelay != null) {
+                csc.setVisibilityDelay(visibilityDelay);
+            }
+            if (csc.hasIndexedCluster()) {
+                setupIndexedCluster(csc.getIndexed(), search, element, logger);
+            }
+
+
+        }
+        private void setupIndexedCluster(IndexedSearchCluster index, ContentSearch search, ModelElement element, DeployLogger logger) {
             Double queryTimeout = search.getQueryTimeout();
             if (queryTimeout != null) {
                 Preconditions.checkState(index.getQueryTimeout() == null,
-                                         "In " + index + ": You may not specify query-timeout in both proton and content.");
+                        "In " + index + ": You may not specify query-timeout in both proton and content.");
                 index.setQueryTimeout(queryTimeout);
             }
-            Double visibilityDelay = search.getVisibilityDelay();
-            if (visibilityDelay != null)
-                index.setVisibilityDelay(visibilityDelay);
             index.setSearchCoverage(DomSearchCoverageBuilder.build(element));
             index.setDispatchSpec(DomDispatchBuilder.build(element));
 
@@ -281,7 +286,7 @@ public class ContentCluster extends AbstractConfigProducer implements
                                       .orElse(NodesSpecification.nonDedicated(3, context));
                 Collection<HostResource> hosts = nodesSpecification.isDedicated() ?
                                                  getControllerHosts(nodesSpecification, admin, clusterName, context) :
-                                                 drawControllerHosts(nodesSpecification.count(), rootGroup, containers);
+                                                 drawControllerHosts(nodesSpecification.minResources().nodes(), rootGroup, containers);
                 clusterControllers = createClusterControllers(new ClusterControllerCluster(contentCluster, "standalone"),
                                                               hosts,
                                                               clusterName,
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomTuningDispatchBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomTuningDispatchBuilder.java
index b53d66632a8..d599a1a1aca 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomTuningDispatchBuilder.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomTuningDispatchBuilder.java
@@ -23,6 +23,7 @@ public class DomTuningDispatchBuilder {
             return builder.build();
         }
         builder.setMaxHitsPerPartition(dispatchElement.childAsInteger("max-hits-per-partition"));
+        builder.setTopKProbability(dispatchElement.childAsDouble("top-k-probability"));
         builder.setDispatchPolicy(dispatchElement.childAsString("dispatch-policy"));
         builder.setMinGroupCoverage(dispatchElement.childAsDouble("min-group-coverage"));
         builder.setMinActiveDocsCoverage(dispatchElement.childAsDouble("min-active-docs-coverage"));
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributionConfigProducer.java b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributionConfigProducer.java
index 9662540e8df..2c6808b5773 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributionConfigProducer.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributionConfigProducer.java
@@ -20,8 +20,9 @@ public class FileDistributionConfigProducer extends AbstractConfigProducer {
     private final Map<Host, FileDistributionConfigProvider> fileDistributionConfigProviders = new IdentityHashMap<>();
     private final FileDistributor fileDistributor;
 
-    public FileDistributionConfigProducer(AbstractConfigProducer ancestor, FileRegistry fileRegistry, List<ConfigServerSpec> configServerSpec) {
-        this(ancestor, new FileDistributor(fileRegistry, configServerSpec));
+    public FileDistributionConfigProducer(AbstractConfigProducer ancestor, FileRegistry fileRegistry,
+                                          List<ConfigServerSpec> configServerSpec, boolean isHosted) {
+        this(ancestor, new FileDistributor(fileRegistry, configServerSpec, isHosted));
     }
 
     private FileDistributionConfigProducer(AbstractConfigProducer parent, FileDistributor fileDistributor) {
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java
index 576b009c846..5ccf86f9ba8 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java
@@ -11,7 +11,10 @@ import com.yahoo.vespa.model.Host;
 import java.util.*;
 
 /**
- * Responsible for directing distribution of files to hosts.
+ * Sends RPC requests to hosts (tenant hosts and config servers) to start download of files. This is used during prepare
+ * of an application. Services themselves will also request files, the work done in this class is done so that hosts can
+ * start downloading files before services gets new config that needs these files. This also tries to make sure that
+ * all config servers (not just the one where the application was deployed) have the files available.
  *
  * @author Tony Vaagenes
  */
@@ -19,61 +22,42 @@ public class FileDistributor {
 
     private final FileRegistry fileRegistry;
     private final List<ConfigServerSpec> configServerSpecs;
+    private final boolean isHosted;
 
-    /** A map from files to the hosts to which that file should be distributed */
+    /** A map from file reference to the hosts to which that file reference should be distributed */
     private final Map<FileReference, Set<Host>> filesToHosts = new LinkedHashMap<>();
 
+    public FileDistributor(FileRegistry fileRegistry, List<ConfigServerSpec> configServerSpecs, boolean isHosted) {
+        this.fileRegistry = fileRegistry;
+        this.configServerSpecs = configServerSpecs;
+        this.isHosted = isHosted;
+    }
+
     /**
      * Adds the given file to the associated application packages' registry of file and marks the file
-     * for distribution to the given hosts.
+     * for distribution to the given host.
      * <b>Note: This class receives ownership of the given collection.</b>
      *
      * @return the reference to the file, created by the application package
      */
-    public FileReference sendFileToHosts(String relativePath, Collection<Host> hosts) {
-        FileReference reference = fileRegistry.addFile(relativePath);
-        addToFilesToDistribute(reference, hosts);
-
-        return reference;
+    public FileReference sendFileToHost(String relativePath, Host host) {
+        return addFileReference(fileRegistry.addFile(relativePath), host);
     }
 
     /**
      * Adds the given file to the associated application packages' registry of file and marks the file
-     * for distribution to the given hosts.
+     * for distribution to the given host.
      * <b>Note: This class receives ownership of the given collection.</b>
      *
      * @return the reference to the file, created by the application package
      */
-    public FileReference sendUriToHosts(String uri, Collection<Host> hosts) {
-        FileReference reference = fileRegistry.addUri(uri);
-        if (reference != null) {
-            addToFilesToDistribute(reference, hosts);
-        }
-
-        return reference;
-    }
-
-    /** Same as sendFileToHost(relativePath,Collections.singletonList(host) */
-    public FileReference sendFileToHost(String relativePath, Host host) {
-        return sendFileToHosts(relativePath, Arrays.asList(host));
-    }
-
     public FileReference sendUriToHost(String uri, Host host) {
-        return sendUriToHosts(uri, Arrays.asList(host));
-    }
-
-    private void addToFilesToDistribute(FileReference reference, Collection<Host> hosts) {
-        Set<Host> oldHosts = getHosts(reference);
-        oldHosts.addAll(hosts);
-    }
-
-    private Set<Host> getHosts(FileReference reference) {
-        return filesToHosts.computeIfAbsent(reference, k -> new HashSet<>());
+        return addFileReference(fileRegistry.addUri(uri), host);
     }
 
-    public FileDistributor(FileRegistry fileRegistry, List<ConfigServerSpec> configServerSpecs) {
-        this.fileRegistry = fileRegistry;
-        this.configServerSpecs = configServerSpecs;
+    private FileReference addFileReference(FileReference reference, Host host) {
+        filesToHosts.computeIfAbsent(reference, k -> new HashSet<>()).add(host);
+        return reference;
     }
 
     /** Returns the files which has been marked for distribution to the given host */
@@ -107,16 +91,20 @@ public class FileDistributor {
     // should only be called during deploy
     public void sendDeployedFiles(FileDistribution dbHandler) {
         String fileSourceHost = fileSourceHost();
-        for (Host host : getTargetHosts()) {
-            if ( ! host.getHostname().equals(fileSourceHost)) {
-                dbHandler.startDownload(host.getHostname(), ConfigProxy.BASEPORT, filesToSendToHost(host));
-            }
-        }
+
         // Ask other config servers to download, for redundancy
-        if (configServerSpecs != null)
-            configServerSpecs.stream()
-                    .filter(configServerSpec -> !configServerSpec.getHostName().equals(fileSourceHost))
-                    .forEach(spec -> dbHandler.startDownload(spec.getHostName(), spec.getConfigServerPort(), allFilesToSend()));
+        configServerSpecs.stream()
+                .filter(spec -> !spec.getHostName().equals(fileSourceHost))
+                .forEach(spec -> dbHandler.startDownload(spec.getHostName(), spec.getConfigServerPort(), allFilesToSend()));
+
+        // Skip starting download for application hosts when on hosted, since this is just a hint and requests for files
+        // will fail until the application is activated (this call is done when preparing an application deployment)
+        // due to authorization of RPC requests on config servers only considering files belonging to active applications
+        if (isHosted) return;
+
+        getTargetHosts().stream()
+                .filter(host -> ! host.getHostname().equals(fileSourceHost))
+                .forEach(host -> dbHandler.startDownload(host.getHostname(), ConfigProxy.BASEPORT, filesToSendToHost(host)));
     }
 
 }
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/AbstractSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/AbstractSearchCluster.java
index 8a88e720bed..fe6c6c52e2d 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/search/AbstractSearchCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/search/AbstractSearchCluster.java
@@ -29,7 +29,7 @@ public abstract class AbstractSearchCluster extends AbstractConfigProducer
     protected int index;
     private Double visibilityDelay = 0.0;
     private List<String> documentNames = new ArrayList<>();
-    private List<SearchDefinitionSpec> localSDS = new LinkedList<>();
+    private List<SchemaSpec> localSDS = new LinkedList<>();
 
     public AbstractSearchCluster(AbstractConfigProducer parent, String clusterName, int index) {
         super(parent, "cluster." + clusterName);
@@ -38,11 +38,11 @@ public abstract class AbstractSearchCluster extends AbstractConfigProducer
     }
 
     public void prepareToDistributeFiles(List<SearchNode> backends) {
-        for (SearchDefinitionSpec sds : localSDS)
+        for (SchemaSpec sds : localSDS)
             sds.getSearchDefinition().getSearch().rankingConstants().sendTo(backends);
     }
 
-    public void addDocumentNames(SearchDefinition searchDefinition) {
+    public void addDocumentNames(NamedSchema searchDefinition) {
         String dName = searchDefinition.getSearch().getDocument().getDocumentName().getName();
         documentNames.add(dName);
     }
@@ -50,7 +50,7 @@ public abstract class AbstractSearchCluster extends AbstractConfigProducer
     /** Returns a List with document names used in this search cluster */
     public List<String> getDocumentNames() { return documentNames; }
 
-    public List<SearchDefinitionSpec> getLocalSDS() {
+    public List<SchemaSpec> getLocalSDS() {
         return localSDS;
     }
 
@@ -107,18 +107,17 @@ public abstract class AbstractSearchCluster extends AbstractConfigProducer
         }
     }
 
-    public static final class SearchDefinitionSpec {
+    public static final class SchemaSpec {
 
-        private final SearchDefinition searchDefinition;
+        private final NamedSchema searchDefinition;
         private final UserConfigRepo userConfigRepo;
 
-        public SearchDefinitionSpec(SearchDefinition searchDefinition,
-                                    UserConfigRepo userConfigRepo) {
+        public SchemaSpec(NamedSchema searchDefinition, UserConfigRepo userConfigRepo) {
             this.searchDefinition = searchDefinition;
             this.userConfigRepo = userConfigRepo;
         }
 
-        public SearchDefinition getSearchDefinition() {
+        public NamedSchema getSearchDefinition() {
             return searchDefinition;
         }
 
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java
index 1b4c03a2182..56adc227df4 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java
@@ -53,6 +53,7 @@ public class IndexedSearchCluster extends SearchCluster
     private final DispatchGroup rootDispatch;
     private DispatchSpec dispatchSpec;
     private final boolean useAdaptiveDispatch;
+    private final double defaultTopKProbability;
     private List<SearchNode> searchNodes = new ArrayList<>();
 
     /**
@@ -70,6 +71,7 @@ public class IndexedSearchCluster extends SearchCluster
         unionCfg = new UnionConfiguration(this, documentDbs);
         rootDispatch =  new DispatchGroup(this);
         useAdaptiveDispatch = deployState.getProperties().useAdaptiveDispatch();
+        defaultTopKProbability = deployState.getProperties().defaultTopKProbability();
     }
 
     @Override
@@ -154,8 +156,7 @@ public class IndexedSearchCluster extends SearchCluster
 
     private void fillDocumentDBConfig(DocumentDatabase sdoc, ProtonConfig.Documentdb.Builder ddbB) {
         ddbB.inputdoctypename(sdoc.getInputDocType())
-            .configid(sdoc.getConfigId())
-            .visibilitydelay(getVisibilityDelay());
+            .configid(sdoc.getConfigId());
     }
 
     @Override
@@ -196,13 +197,15 @@ public class IndexedSearchCluster extends SearchCluster
             routingSelector = sb.toString();
         }
     }
+
     @Override
-    protected void deriveAllSearchDefinitions(List<SearchDefinitionSpec> localSearches, DeployState deployState) {
-        for (SearchDefinitionSpec spec : localSearches) {
+    protected void deriveAllSchemas(List<SchemaSpec> localSearches, DeployState deployState) {
+        for (SchemaSpec spec : localSearches) {
             com.yahoo.searchdefinition.Search search = spec.getSearchDefinition().getSearch();
             if ( ! (search instanceof DocumentOnlySearch)) {
                 DocumentDatabase db = new DocumentDatabase(this, search.getName(),
-                                                           new DerivedConfiguration(search, deployState.getDeployLogger(),
+                                                           new DerivedConfiguration(search,
+                                                                                    deployState.getDeployLogger(),
                                                                                     deployState.getProperties(),
                                                                                     deployState.rankProfileRegistry(),
                                                                                     deployState.getQueryProfiles().getRegistry(),
@@ -306,7 +309,11 @@ public class IndexedSearchCluster extends SearchCluster
         }
         if (useAdaptiveDispatch)
             builder.distributionPolicy(DistributionPolicy.ADAPTIVE);
-
+        if (tuning.dispatch.getTopkProbability() != null) {
+            builder.topKProbability(tuning.dispatch.getTopkProbability());
+        } else {
+            builder.topKProbability(defaultTopKProbability);
+        }
         if (tuning.dispatch.getMinActiveDocsCoverage() != null)
             builder.minActivedocsPercentage(tuning.dispatch.getMinActiveDocsCoverage());
         if (tuning.dispatch.getMinGroupCoverage() != null)
@@ -334,6 +341,7 @@ public class IndexedSearchCluster extends SearchCluster
             if (searchCoverage.getMaxWaitAfterCoverageFactor() != null)
                 builder.maxWaitAfterCoverageFactor(searchCoverage.getMaxWaitAfterCoverageFactor());
         }
+        builder.warmuptime(5.0);
     }
 
     @Override
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchDefinition.java b/config-model/src/main/java/com/yahoo/vespa/model/search/NamedSchema.java
index 860f89792e2..ba81073709e 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchDefinition.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/search/NamedSchema.java
@@ -8,7 +8,8 @@ import java.util.Collection;
 /**
  * @author Tony Vaagenes
  */
-public class SearchDefinition {
+// TODO: This class is quite pointless
+public class NamedSchema {
 
     private final Search search;
     private final String name;
@@ -23,15 +24,15 @@ public class SearchDefinition {
         return name;
     }
 
-    public SearchDefinition(String name, Search search) {
+    public NamedSchema(String name, Search search) {
         this.name = name;
         this.search = search;
     }
 
     //Find search definition from a collection with the name specified
-    public static SearchDefinition findByName(final String searchDefinitionName, Collection<SearchDefinition> searchDefinitions) {
-        for (SearchDefinition candidate : searchDefinitions) {
-            if (candidate.getName().equals(searchDefinitionName) )
+    public static NamedSchema findByName(String schemaName, Collection<NamedSchema> schemas) {
+        for (NamedSchema candidate : schemas) {
+            if (candidate.getName().equals(schemaName) )
                 return candidate;
         }
 
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/NodeFlavorTuning.java b/config-model/src/main/java/com/yahoo/vespa/model/search/NodeFlavorTuning.java
index c337c77d2a2..5d8b0b688d8 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/search/NodeFlavorTuning.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/search/NodeFlavorTuning.java
@@ -30,6 +30,7 @@ public class NodeFlavorTuning implements ProtonConfig.Producer {
     public void getConfig(ProtonConfig.Builder builder) {
         setHwInfo(builder);
         tuneDiskWriteSpeed(builder);
+        tuneRequestThreads(builder);
         tuneDocumentStoreMaxFileSize(builder.summary.log);
         tuneFlushStrategyMemoryLimits(builder.flush.memory);
         tuneFlushStrategyTlsSize(builder.flush.memory);
@@ -105,6 +106,12 @@ public class NodeFlavorTuning implements ProtonConfig.Producer {
         }
     }
 
+    private void tuneRequestThreads(ProtonConfig.Builder builder) {
+        int numCores = (int)Math.ceil(nodeFlavor.getMinCpuCores());
+        builder.numsearcherthreads(numCores);
+        builder.numsummarythreads(numCores);
+    }
+
     private void tuneWriteFilter(ProtonConfig.Writefilter.Builder builder) {
         // "Reserve" 1GB of memory for other processes running on the content node (config-proxy, cluster-controller, metrics-proxy)
         double reservedMemoryGb = 1;
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchDefinitionXMLHandler.java b/config-model/src/main/java/com/yahoo/vespa/model/search/SchemaDefinitionXMLHandler.java
index 1054253e3f0..b505b5e681c 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchDefinitionXMLHandler.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/search/SchemaDefinitionXMLHandler.java
@@ -7,15 +7,15 @@ import java.io.Serializable;
 import java.util.List;
 
 /**
- * Represents a single searchdefinition file.
+ * Represents a single schema file.
  *
  * @author arnej27959
  */
-public class SearchDefinitionXMLHandler implements Serializable {
+public class SchemaDefinitionXMLHandler implements Serializable {
 
     private String sdName;
 
-    public SearchDefinitionXMLHandler(ModelElement elem) {
+    public SchemaDefinitionXMLHandler(ModelElement elem) {
         sdName = elem.stringAttribute("name");
         if (sdName == null) {
             sdName = elem.stringAttribute("type");
@@ -24,8 +24,8 @@ public class SearchDefinitionXMLHandler implements Serializable {
 
     public String getName() { return sdName; }
 
-    public SearchDefinition getResponsibleSearchDefinition(List<SearchDefinition> searchDefinitions) {
-        return SearchDefinition.findByName( getName(), searchDefinitions );
+    public NamedSchema getResponsibleSearchDefinition(List<NamedSchema> schemas) {
+        return NamedSchema.findByName(getName(), schemas );
     }
 
 }
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java
index 60b3cb6987c..0139e949c7a 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java
@@ -35,22 +35,14 @@ public abstract class SearchCluster extends AbstractSearchCluster
         super(parent, clusterName, index);
     }
 
-    public void writeFiles(File directory) throws java.io.IOException {
-        if (!directory.isDirectory() && !directory.mkdirs()) {
-            throw new java.io.IOException("Cannot create directory: "+ directory);
-        }
-        writeSdFiles(directory);
-        super.writeFiles(directory);
-    }
-
    /**
      * Must be called after cluster is built, to derive SD configs
      * Derives the search definitions from the application package..
      * Also stores the document names contained in the search
      * definitions.
      */
-    public void deriveSearchDefinitions(DeployState deployState) {
-        deriveAllSearchDefinitions(getLocalSDS(), deployState);
+    public void deriveSchemas(DeployState deployState) {
+        deriveAllSchemas(getLocalSDS(), deployState);
     }
 
     @Override
@@ -140,7 +132,7 @@ public abstract class SearchCluster extends AbstractSearchCluster
         return false;
     }
 
-    protected abstract void deriveAllSearchDefinitions(List<SearchDefinitionSpec> localSearches, DeployState deployState);
+    protected abstract void deriveAllSchemas(List<SchemaSpec> localSearches, DeployState deployState);
 
     public abstract void defaultDocumentsConfig();
     public abstract DerivedConfiguration getSdConfig();
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java
index d668adea116..28e7b3eb37a 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java
@@ -91,7 +91,7 @@ public class StreamingSearchCluster extends SearchCluster implements
     }
 
     @Override
-    protected void deriveAllSearchDefinitions(List<SearchDefinitionSpec> local, DeployState deployState) {
+    protected void deriveAllSchemas(List<SchemaSpec> local, DeployState deployState) {
         if (local.size() == 1) {
             deriveSingleSearchDefinition(local.get(0).getSearchDefinition().getSearch(), deployState);
         } else if (local.size() > 1){
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/Tuning.java b/config-model/src/main/java/com/yahoo/vespa/model/search/Tuning.java
index f5a91297e9e..f93bf6fc872 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/search/Tuning.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/search/Tuning.java
@@ -241,7 +241,12 @@ public class Tuning extends AbstractConfigProducer implements ProtonConfig.Produ
                     public Integer level = null;
 
                     public void getConfig(ProtonConfig.Summary.Cache.Compression.Builder compression) {
-                        if (type != null)  compression.type(ProtonConfig.Summary.Cache.Compression.Type.Enum.valueOf(type.name));
+                        if (type != null) compression.type(ProtonConfig.Summary.Cache.Compression.Type.Enum.valueOf(type.name));
+                        if (level != null) compression.level(level);
+                    }
+
+                    public void getConfig(ProtonConfig.Summary.Log.Compact.Compression.Builder compression) {
+                        if (type != null) compression.type(ProtonConfig.Summary.Log.Compact.Compression.Type.Enum.valueOf(type.name));
                         if (level != null) compression.level(level);
                     }
 
@@ -281,6 +286,12 @@ public class Tuning extends AbstractConfigProducer implements ProtonConfig.Produ
                         }
                     }
 
+                    public void getConfig(ProtonConfig.Summary.Log.Compact.Builder compact) {
+                        if (compression != null) {
+                            compression.getConfig(compact.compression);
+                        }
+                    }
+
                     public void getConfig(ProtonConfig.Summary.Log.Chunk.Builder chunk) {
                         if (outputInt) {
                             if (maxSize!=null) chunk.maxbytes(maxSize.intValue());
@@ -288,7 +299,7 @@ public class Tuning extends AbstractConfigProducer implements ProtonConfig.Produ
                             throw new IllegalStateException("Fix this, chunk does not have long types");
                         }
                         if (compression != null) {
-                           compression.getConfig(chunk.compression);
+                            compression.getConfig(chunk.compression);
                         }
                     }
                 }
@@ -303,6 +314,7 @@ public class Tuning extends AbstractConfigProducer implements ProtonConfig.Produ
                         if (minFileSizeFactor!=null) log.minfilesizefactor(minFileSizeFactor);
                         if (chunk != null) {
                             chunk.getConfig(log.chunk);
+                            chunk.getConfig(log.compact);
                         }
                     }
                 }
diff --git a/config-model/src/main/javacc/SDParser.jj b/config-model/src/main/javacc/SDParser.jj
index 52665ff56a9..3560cf2cd84 100644
--- a/config-model/src/main/javacc/SDParser.jj
+++ b/config-model/src/main/javacc/SDParser.jj
@@ -197,6 +197,7 @@ TOKEN :
   < NL: "\n" >
 | < ANNOTATION: "annotation" >
 | < ANNOTATIONREFERENCE: "annotationreference" >
+| < SCHEMA: "schema" >
 | < SEARCH: "search" >
 | < DIVERSITY: "diversity" >
 | < MIN_GROUPS: "min-groups" >
@@ -332,6 +333,10 @@ TOKEN :
 | < UPPERBOUND: "upper-bound" >
 | < DENSEPOSTINGLISTTHRESHOLD: "dense-posting-list-threshold" >
 | < ENABLE_BM25: "enable-bm25" >
+| < HNSW: "hnsw" >
+| < MAXLINKSPERNODE: "max-links-per-node" >
+| < DISTANCEMETRIC: "distance-metric" >
+| < NEIGHBORSTOEXPLOREATINSERT: "neighbors-to-explore-at-insert" >
 | < SUMMARYFEATURES_SL: "summary-features" (" ")* ":" (~["}","\n"])* ("\n")? >
 | < SUMMARYFEATURES_ML: "summary-features" (<SEARCHLIB_SKIP>)? "{" (~["}"])* "}" >
 | < RANKFEATURES_SL: "rank-features" (" ")* ":" (~["}","\n"])* ("\n")? >
@@ -400,38 +405,38 @@ Search search(DocumentTypeManager docMan, String dir) :
     Search search;
 }
 {
-    (<NL>)* (search = rootSearch(dir) | search = rootDocument(dir))
+    (<NL>)* (search = rootSchema(dir) | search = rootDocument(dir))
     { return search; }
 }
 
 /**
- * This rule consumes a proper search block. This and rootDocument() are the only rules that should ever consume
+ * This rule consumes a proper schema block. This and rootDocument() are the only rules that should ever consume
  * trailing newline tokens.
  *
- * @param dir The directory containing the file being parsed.
- * @return The search definition object.
+ * @param dir the directory containing the file being parsed.
+ * @return the schema definition object.
  */
-Search rootSearch(String dir) :
+Search rootSchema(String dir) :
 {
     String name;
     Search search;
 }
 {
-    ( <SEARCH> name = identifier() { search = new Search(name, app);
+    ( ( <SCHEMA> | <SEARCH> ) name = identifier() { search = new Search(name, app);
                                      rankProfileRegistry.add(new DefaultRankProfile(search, rankProfileRegistry));
                                      rankProfileRegistry.add(new UnrankedRankProfile(search, rankProfileRegistry));}
-      lbrace() (rootSearchItem(search) (<NL>)*)* <RBRACE> (<NL>)* <EOF>)
+      lbrace() (rootSchemaItem(search) (<NL>)*)* <RBRACE> (<NL>)* <EOF>)
     { return search; }
 }
 
 /**
- * Consumes an element of a search block. This and rootSearch() are the only rules that should ever consume
+ * Consumes an element of a schema block. This and rootSearch() are the only rules that should ever consume
  * trailing newline tokens.
  *
  * @param search The search object to modify.
  * @return Null.
  */
-Object rootSearchItem(Search search) : { }
+Object rootSchemaItem(Search search) : { }
 {
     ( document(search)
       | documentSummary(search)
@@ -449,10 +454,10 @@ Object rootSearchItem(Search search) : { }
 }
 
 /**
- * Consumes a search definition that contains only documents to be used for inheritance, etc.
+ * Consumes a schema definition that contains only documents to be used for inheritance, etc.
  *
- * @param dir The directory containing the file being parsed.
- * @return The search definition object.
+ * @param dir the directory containing the file being parsed.
+ * @return the schema definition object.
  */
 Search rootDocument(String dir) :
 {
@@ -478,7 +483,7 @@ Object rootDocumentItem(Search search) : { }
 /**
  * Consumes a use-document statement. This currently does nothing.
  *
- * @param search The search object to modify.
+ * @param search the search object to modify.
  */
 void useDocument(Search search) : { }
 {
@@ -488,7 +493,7 @@ void useDocument(Search search) : { }
 /**
  * Consumes a document element. The name defaults to the search's name, but may be set.
  *
- * @param search The search object to add content to.
+ * @param search the search object to add content to.
  */
 void document(Search search) :
 {
@@ -507,7 +512,7 @@ void document(Search search) :
 /**
  * Consumes a document element, explicitly named
  *
- * @param search The search object to add content to.
+ * @param search the search object to add content to.
  */
 void namedDocument(Search search) :
 {
@@ -637,7 +642,7 @@ void field(SDDocumentType document, Search search) :
         if (name != null && com.yahoo.searchdefinition.Search.isReservedName(name.toLowerCase())) {
             throw new IllegalArgumentException("Reserved name '" + name + "' can not be used as a field name.");
         }
-        field = new TemporarySDField(name, type, true, document);
+        field = new TemporarySDField(name, type, document);
     }
     lbrace() (fieldBody(field, search, document) (<NL>)*)* <RBRACE>
     {
@@ -651,27 +656,35 @@ void field(SDDocumentType document, Search search) :
 
 void fieldSet(Search search) :
 {
-  String name;
-}
-{
-  <FIELDSET> name = identifier() lbrace()
-    (fieldSetItem(name, search)(<NL>)*)+
-  <RBRACE>
-}
-
-void fieldSetItem(String setName, Search search) :
-{
+  String setName;
   String field;
   String queryCommand;
-  SDField matchSettings = new SDField(setName, DataType.STRING); // match etc for fieldset represented as SDField or ease of parsing
+  List queryCommands = new ArrayList();
+  FieldOperationContainer matchSetting;
+  List matchSettings = new ArrayList();
 }
 {
-  ( <FIELDS><COLON> field=identifier() { search.fieldSets().addUserFieldSetItem(setName, field); }
-    ( <COMMA> field=identifier() { search.fieldSets().addUserFieldSetItem(setName, field); } )* )
-  |
-    ( <QUERYCOMMAND> <COLON> (queryCommand = identifierWithDash() | queryCommand = quotedString())) { search.fieldSets().userFieldSets().get(setName).queryCommands().add(queryCommand);}
-  |
-    ( match(matchSettings) ) { matchSettings.applyOperations(); search.fieldSets().userFieldSets().get(setName).setMatching(matchSettings.getMatching());}
+  <FIELDSET> setName = identifier() lbrace()
+    ((
+      ( <FIELDS><COLON> field = identifier() { search.fieldSets().addUserFieldSetItem(setName, field); }
+      ( <COMMA> field = identifier() { search.fieldSets().addUserFieldSetItem(setName, field); } )* )
+    |
+      ( <QUERYCOMMAND> <COLON> (queryCommand = identifierWithDash() | queryCommand = quotedString())) { queryCommands.add(queryCommand); }
+    |
+      ( matchSetting = match(new SDField(setName, DataType.STRING)) ) { matchSettings.add(matchSetting); }
+    )(<NL>)*)+
+  <RBRACE>
+  {
+     // Apply settings after parsing since all user field items must be set first
+
+     for (Object command : queryCommands)
+         search.fieldSets().userFieldSets().get(setName).queryCommands().add((String)command);
+
+     for (Object setting : matchSettings) {
+         ((SDField)setting).applyOperations();
+         search.fieldSets().userFieldSets().get(setName).setMatching(((SDField)setting).getMatching());
+     }
+  }
 }
 
 /**
@@ -923,7 +936,7 @@ void structFieldDefinition(SDDocumentType struct) :
         if (name != null && com.yahoo.searchdefinition.Search.isReservedName(name.toLowerCase())) {
             throw new IllegalArgumentException("Reserved name '" + name + "' can not be used as a field name.");
         }
-        field = new TemporarySDField(name, type, true, struct);
+        field = new TemporarySDField(name, type, struct);
         struct.addField(field);
     }
     lbrace() (id(field,struct) (<NL>)*)? (match(field) (<NL>)*)* <RBRACE> {
@@ -1531,7 +1544,7 @@ void queryCommand(FieldOperationContainer container) :
     QueryCommandOperation field = new QueryCommandOperation();
 }
 {
-    <QUERYCOMMAND> <COLON> command = identifierWithDash()
+    <QUERYCOMMAND> <COLON> ( command = identifierWithDash() | command = quotedString() )
     {
         field.addQueryCommand(command);
         container.addOperation(field);
@@ -1551,11 +1564,11 @@ void alias(FieldOperationContainer container) :
     }
 }
 
-Object match(FieldOperationContainer field) : { }
+FieldOperationContainer match(FieldOperationContainer field) : { }
 {
     <MATCH> ( (<COLON> matchType(field))
               | (lbrace() (matchItem(field) (<NL>)*)* <RBRACE>) )
-    { return null; }
+    { return field; }
 }
 
 /**
@@ -1803,10 +1816,35 @@ Object indexBody(IndexOperation index) :
       | <UPPERBOUND> <COLON> num = consumeLong()                       { index.setUpperBound(num); }
       | <DENSEPOSTINGLISTTHRESHOLD> <COLON> threshold = consumeFloat() { index.setDensePostingListThreshold(threshold); }
       | <ENABLE_BM25>                                                  { index.setEnableBm25(true); }
+      | <DISTANCEMETRIC> <COLON> str = identifierWithDash()            { index.setDistanceMetric(str); }
+      | hnswIndex(index)                                               { }
     )
     { return null; }
 }
 
+void hnswIndex(IndexOperation index) :
+{
+    HnswIndexParams.Builder params = new HnswIndexParams.Builder();
+}
+{
+    ( LOOKAHEAD(<HNSW> lbrace())
+      <HNSW> ( (lbrace() (hnswIndexBody(params) (<NL>)*)* <RBRACE>) ) |
+      <HNSW> )
+    {
+        index.setHnswIndexParams(params);
+    }
+}
+
+void hnswIndexBody(HnswIndexParams.Builder params) :
+{
+    int num;
+    String str;
+}
+{
+    ( <MAXLINKSPERNODE> <COLON> num = integer() { params.setMaxLinksPerNode(num); }
+      | <NEIGHBORSTOEXPLOREATINSERT> <COLON> num = integer() { params.setNeighborsToExploreAtInsert(num); } )
+}
+
 /**
  * Consumes a constant block of a search element.
  *
@@ -2583,6 +2621,7 @@ String identifier() : { }
       | <REFERENCE>
       | <REMOVEIFZERO>
       | <RERANKCOUNT>
+      | <SCHEMA>
       | <SEARCH>
       | <SECONDARY>
       | <SECONDPHASE>
diff --git a/config-model/src/main/perl/vespa-deploy b/config-model/src/main/perl/vespa-deploy
index 59a84f5b0c0..a128e4a8d4c 100755
--- a/config-model/src/main/perl/vespa-deploy
+++ b/config-model/src/main/perl/vespa-deploy
@@ -154,7 +154,7 @@ my $command = shift;
 $command ||= "help";
 
 # The '--insecure' parameter is sadly required as it is not possible to disable or alter hostname verification with curl
-my $curl_command = $VESPA_HOME . '/libexec/vespa/vespa-curl-wrapper --insecure -A vespa-deploy --silent --show-error --connect-timeout 30 --max-time 1200';
+my $curl_command = $VESPA_HOME . '/libexec/vespa/vespa-curl-wrapper -A vespa-deploy --silent --show-error --connect-timeout 30 --max-time 1200';
 
 my $CURL_PUT = $curl_command . ' --write-out \%{http_code} --request PUT';
 my $CURL_GET = $curl_command . ' --request GET';
diff --git a/config-model/src/main/resources/schema/admin.rnc b/config-model/src/main/resources/schema/admin.rnc
index 7a3e2916f94..1eac11a879c 100644
--- a/config-model/src/main/resources/schema/admin.rnc
+++ b/config-model/src/main/resources/schema/admin.rnc
@@ -82,10 +82,27 @@ Metrics = element metrics {
         element metric {
             attribute id { xsd:Name } &
             attribute display-name { xsd:Name }?
-        }*
+        }* &
+        Cloudwatch?
     }+
 }
 
+Cloudwatch = element cloudwatch {
+    attribute region { xsd:Name } &
+    attribute namespace { xsd:string { pattern = "[\w_\-/#:\.]+" } } &
+    (
+      element credentials {
+            attribute access-key-name { xsd:Name } &
+            attribute secret-key-name { xsd:Name }
+      }
+      |
+      element shared-credentials {
+          attribute file { string } &
+          attribute profile { xsd:Name }?
+      }
+    )?
+}
+
 ClusterControllers = element cluster-controllers {
     attribute standalone-zookeeper { xsd:string  }? &
     element cluster-controller {
diff --git a/config-model/src/main/resources/schema/common.rnc b/config-model/src/main/resources/schema/common.rnc
index e3ad942e7b3..878faabfec1 100644
--- a/config-model/src/main/resources/schema/common.rnc
+++ b/config-model/src/main/resources/schema/common.rnc
@@ -17,14 +17,14 @@ anyElement = element * {
 JavaId = xsd:string { pattern = "([a-zA-Z_$][a-zA-Z\d_$]*\.)*[a-zA-Z_$][a-zA-Z\d_$]*" }
 
 Nodes = element nodes {
-    attribute count { xsd:positiveInteger } &
+    attribute count { xsd:positiveInteger | xsd:string } &
     attribute flavor { xsd:string }? &
     attribute docker-image { xsd:string }? &
     Resources?
 }
 
 Resources = element resources {
-    attribute vcpu { xsd:double { minExclusive = "0.0" } } &
+    attribute vcpu { xsd:double { minExclusive = "0.0" } | xsd:string } &
     attribute memory { xsd:string } &
     attribute disk { xsd:string } &
     attribute disk-speed { xsd:string }? &
@@ -32,12 +32,13 @@ Resources = element resources {
 }
 
 OptionalDedicatedNodes = element nodes {
-    attribute count { xsd:positiveInteger } &
+    attribute count { xsd:positiveInteger | xsd:string } &
     attribute flavor { xsd:string }? &
     attribute required { xsd:boolean }? &
     attribute docker-image { xsd:string }? &
     attribute dedicated { xsd:boolean }? &
-    attribute exclusive { xsd:boolean }?
+    attribute exclusive { xsd:boolean }? &
+    Resources?
 }
 
 GenericConfig = element config {
diff --git a/config-model/src/main/resources/schema/container.rnc b/config-model/src/main/resources/schema/container.rnc
index 3f0e1f626ac..034de1113c2 100644
--- a/config-model/src/main/resources/schema/container.rnc
+++ b/config-model/src/main/resources/schema/container.rnc
@@ -23,11 +23,10 @@ Server = element server {
 }
 
 AccessControl = element access-control {
-    attribute domain { xsd:NCName } &
-    attribute read { string "true" | string "false" }? &
-    attribute write { string "true" | string "false" }? &
-    element vespa-domain { xsd:NCName }? &
-    element application { xsd:NCName }? &
+    attribute domain { xsd:NCName }? &  # TODO Vespa 8 Remove
+    attribute read { string "true" | string "false" }? & # TODO Vespa 8 Remove
+    attribute write { string "true" | string "false" }? & # TODO Vespa 8 Remove
+    element vespa-domain { xsd:NCName }? & # TODO Remove after end of March 2020
     element exclude {
         Binding+
     }?
diff --git a/config-model/src/main/resources/schema/containercluster.rnc b/config-model/src/main/resources/schema/containercluster.rnc
index 6e4346d96ee..3c8b60fb84b 100644
--- a/config-model/src/main/resources/schema/containercluster.rnc
+++ b/config-model/src/main/resources/schema/containercluster.rnc
@@ -96,7 +96,9 @@ Ssl = element ssl {
     element private-key-file { string } &
     element certificate-file { string } &
     element ca-certificates-file { string }? &
-    element client-authentication { string "disabled" | string "want" | string "need" }?
+    element client-authentication { string "disabled" | string "want" | string "need" }? &
+    element cipher-suites { string }? &
+    element protocols { string }?
 }
 
 SslProvider = element ssl-provider {
@@ -237,7 +239,7 @@ NodesOfContainerCluster = element nodes {
       attribute type { xsd:string }
       |
       (
-        attribute count { xsd:positiveInteger } &
+        attribute count { xsd:positiveInteger | xsd:string  } &
         attribute flavor { xsd:string }? &
         attribute required { xsd:boolean }? &
         attribute exclusive { xsd:boolean }? &
diff --git a/config-model/src/main/resources/schema/content.rnc b/config-model/src/main/resources/schema/content.rnc
index 8b3868c132e..481d82ebb4b 100644
--- a/config-model/src/main/resources/schema/content.rnc
+++ b/config-model/src/main/resources/schema/content.rnc
@@ -57,11 +57,14 @@ PersistenceThread = element thread {
 
 ## Declare which storage threads each disk should have.
 PersistenceThreads = element persistence-threads {
+  ## The number of threads to create
+  attribute count { xsd:integer }? &
+  ## All of the below settings are deprecated.
   ## Operations with priority worse than this can be blocked
   attribute highest-priority-to-block { xsd:string } ? &
   ## Operations with priority better than this can block others
   attribute lowest-priority-to-block-others { xsd:string } ? &
-  Thread+
+  Thread*
 }
 
 MinNodeRatioPerGroup = element min-node-ratio-per-group {
@@ -82,6 +85,7 @@ DispatchTuning = element dispatch {
     element dispatch-policy { string "round-robin" | string "adaptive" | string "random" }? &
     element min-group-coverage { xsd:double }? &
     element min-active-docs-coverage { xsd:double }? &
+    element top-k-probability { xsd:double }? &
     element use-local-node { string "true" | string "false" }?
 }
 
@@ -218,11 +222,11 @@ ContentNodes = element nodes {
     attribute vespamalloc-debug-stacktrace { xsd:string }? &
     (
       (
-        attribute count { xsd:positiveInteger } &
+        attribute count { xsd:positiveInteger | xsd:string } &
         attribute flavor { xsd:string }? &
         attribute required { xsd:boolean }? &
         attribute docker-image { xsd:string }? &
-        attribute groups { xsd:positiveInteger }?
+        attribute groups { xsd:positiveInteger | xsd:string }?
       )
       |
       ContentNode +
@@ -263,12 +267,12 @@ Group = element group {
         |
         (
             element nodes {
-                attribute count { xsd:positiveInteger } &
+                attribute count { xsd:positiveInteger | xsd:string  } &
                 attribute flavor { xsd:string }? &
                 attribute required { xsd:boolean }? &
                 attribute exclusive { xsd:boolean }? &
                 attribute docker-image { xsd:string }? &
-                attribute groups { xsd:positiveInteger }?
+                attribute groups { xsd:positiveInteger | xsd:string  }?
             }
         )
         |
diff --git a/config-model/src/test/cfg/admin/sdconfigs/pan-rtx.cfg b/config-model/src/test/cfg/admin/sdconfigs/pan-rtx.cfg
deleted file mode 100644
index 6e18bdf64e7..00000000000
--- a/config-model/src/test/cfg/admin/sdconfigs/pan-rtx.cfg
+++ /dev/null
@@ -1,3 +0,0 @@
-namespace=config
-foo bar
-baz []678
diff --git a/config-model/src/test/cfg/application/ml_models/models/lightgbm_regression.json b/config-model/src/test/cfg/application/ml_models/models/lightgbm_regression.json
new file mode 100644
index 00000000000..cf0488ecd8b
--- /dev/null
+++ b/config-model/src/test/cfg/application/ml_models/models/lightgbm_regression.json
@@ -0,0 +1,275 @@
+{
+  "name": "tree",
+  "version": "v3",
+  "num_class": 1,
+  "num_tree_per_iteration": 1,
+  "label_index": 0,
+  "max_feature_idx": 3,
+  "average_output": false,
+  "objective": "regression",
+  "feature_names": [
+    "numerical_1",
+    "numerical_2",
+    "categorical_1",
+    "categorical_2"
+  ],
+  "monotone_constraints": [],
+  "tree_info": [
+    {
+      "tree_index": 0,
+      "num_leaves": 3,
+      "num_cat": 1,
+      "shrinkage": 1,
+      "tree_structure": {
+        "split_index": 0,
+        "split_feature": 1,
+        "split_gain": 68.5353012084961,
+        "threshold": 0.46643291586559305,
+        "decision_type": "<=",
+        "default_left": true,
+        "missing_type": "NaN",
+        "internal_value": 0,
+        "internal_weight": 0,
+        "internal_count": 1000,
+        "left_child": {
+          "leaf_index": 0,
+          "leaf_value": 2.1594397038037663,
+          "leaf_weight": 469,
+          "leaf_count": 469
+        },
+        "right_child": {
+          "split_index": 1,
+          "split_feature": 3,
+          "split_gain": 41.27640151977539,
+          "threshold": "2||3||4",
+          "decision_type": "==",
+          "default_left": false,
+          "missing_type": "NaN",
+          "internal_value": 0.246035,
+          "internal_weight": 531,
+          "internal_count": 531,
+          "left_child": {
+            "leaf_index": 1,
+            "leaf_value": 2.235297305276056,
+            "leaf_weight": 302,
+            "leaf_count": 302
+          },
+          "right_child": {
+            "leaf_index": 2,
+            "leaf_value": 2.1792953471546546,
+            "leaf_weight": 229,
+            "leaf_count": 229
+          }
+        }
+      }
+    },
+    {
+      "tree_index": 1,
+      "num_leaves": 3,
+      "num_cat": 1,
+      "shrinkage": 0.1,
+      "tree_structure": {
+        "split_index": 0,
+        "split_feature": 2,
+        "split_gain": 64.22250366210938,
+        "threshold": "3||4",
+        "decision_type": "==",
+        "default_left": false,
+        "missing_type": "NaN",
+        "internal_value": 0,
+        "internal_weight": 0,
+        "internal_count": 1000,
+        "left_child": {
+          "leaf_index": 0,
+          "leaf_value": 0.03070842919354316,
+          "leaf_weight": 399,
+          "leaf_count": 399
+        },
+        "right_child": {
+          "split_index": 1,
+          "split_feature": 0,
+          "split_gain": 36.74250030517578,
+          "threshold": 0.5102250691730842,
+          "decision_type": "<=",
+          "default_left": true,
+          "missing_type": "NaN",
+          "internal_value": -0.204906,
+          "internal_weight": 601,
+          "internal_count": 601,
+          "left_child": {
+            "leaf_index": 1,
+            "leaf_value": -0.04439151147520909,
+            "leaf_weight": 315,
+            "leaf_count": 315
+          },
+          "right_child": {
+            "leaf_index": 2,
+            "leaf_value": 0.005117411709368601,
+            "leaf_weight": 286,
+            "leaf_count": 286
+          }
+        }
+      }
+    },
+    {
+      "tree_index": 2,
+      "num_leaves": 3,
+      "num_cat": 0,
+      "shrinkage": 0.1,
+      "tree_structure": {
+        "split_index": 0,
+        "split_feature": 1,
+        "split_gain": 57.1327018737793,
+        "threshold": 0.668665477622446,
+        "decision_type": "<=",
+        "default_left": true,
+        "missing_type": "NaN",
+        "internal_value": 0,
+        "internal_weight": 0,
+        "internal_count": 1000,
+        "left_child": {
+          "split_index": 1,
+          "split_feature": 1,
+          "split_gain": 40.859100341796875,
+          "threshold": 0.008118820676863816,
+          "decision_type": "<=",
+          "default_left": true,
+          "missing_type": "NaN",
+          "internal_value": -0.162926,
+          "internal_weight": 681,
+          "internal_count": 681,
+          "left_child": {
+            "leaf_index": 0,
+            "leaf_value": -0.15361238490967524,
+            "leaf_weight": 21,
+            "leaf_count": 21
+          },
+          "right_child": {
+            "leaf_index": 2,
+            "leaf_value": -0.01192330846157292,
+            "leaf_weight": 660,
+            "leaf_count": 660
+          }
+        },
+        "right_child": {
+          "leaf_index": 1,
+          "leaf_value": 0.03499044894987518,
+          "leaf_weight": 319,
+          "leaf_count": 319
+        }
+      }
+    },
+    {
+      "tree_index": 3,
+      "num_leaves": 3,
+      "num_cat": 1,
+      "shrinkage": 0.1,
+      "tree_structure": {
+        "split_index": 0,
+        "split_feature": 0,
+        "split_gain": 54.77090072631836,
+        "threshold": 0.5201391072644542,
+        "decision_type": "<=",
+        "default_left": true,
+        "missing_type": "NaN",
+        "internal_value": 0,
+        "internal_weight": 0,
+        "internal_count": 1000,
+        "left_child": {
+          "leaf_index": 0,
+          "leaf_value": -0.02141000620783247,
+          "leaf_weight": 543,
+          "leaf_count": 543
+        },
+        "right_child": {
+          "split_index": 1,
+          "split_feature": 2,
+          "split_gain": 27.200700759887695,
+          "threshold": "0||1",
+          "decision_type": "==",
+          "default_left": false,
+          "missing_type": "NaN",
+          "internal_value": 0.255704,
+          "internal_weight": 457,
+          "internal_count": 457,
+          "left_child": {
+            "leaf_index": 1,
+            "leaf_value": -0.004121485787596721,
+            "leaf_weight": 191,
+            "leaf_count": 191
+          },
+          "right_child": {
+            "leaf_index": 2,
+            "leaf_value": 0.04534090904886873,
+            "leaf_weight": 266,
+            "leaf_count": 266
+          }
+        }
+      }
+    },
+    {
+      "tree_index": 4,
+      "num_leaves": 3,
+      "num_cat": 1,
+      "shrinkage": 0.1,
+      "tree_structure": {
+        "split_index": 0,
+        "split_feature": 3,
+        "split_gain": 51.84349822998047,
+        "threshold": "2||3||4",
+        "decision_type": "==",
+        "default_left": false,
+        "missing_type": "NaN",
+        "internal_value": 0,
+        "internal_weight": 0,
+        "internal_count": 1000,
+        "left_child": {
+          "split_index": 1,
+          "split_feature": 1,
+          "split_gain": 39.352699279785156,
+          "threshold": 0.27283279016959255,
+          "decision_type": "<=",
+          "default_left": true,
+          "missing_type": "NaN",
+          "internal_value": 0.188414,
+          "internal_weight": 593,
+          "internal_count": 593,
+          "left_child": {
+            "leaf_index": 0,
+            "leaf_value": -0.01924803254356527,
+            "leaf_weight": 184,
+            "leaf_count": 184
+          },
+          "right_child": {
+            "leaf_index": 2,
+            "leaf_value": 0.03643772842347651,
+            "leaf_weight": 409,
+            "leaf_count": 409
+          }
+        },
+        "right_child": {
+          "leaf_index": 1,
+          "leaf_value": -0.02701711918923075,
+          "leaf_weight": 407,
+          "leaf_count": 407
+        }
+      }
+    }
+  ],
+  "pandas_categorical": [
+    [
+      "a",
+      "b",
+      "c",
+      "d",
+      "e"
+    ],
+    [
+      "i",
+      "j",
+      "k",
+      "l",
+      "m"
+    ]
+  ]
+}
\ No newline at end of file
diff --git a/config-model/src/test/cfg/application/ml_models/searchdefinitions/test.sd b/config-model/src/test/cfg/application/ml_models/searchdefinitions/test.sd
index ab5e42f983d..247df8a0241 100644
--- a/config-model/src/test/cfg/application/ml_models/searchdefinitions/test.sd
+++ b/config-model/src/test/cfg/application/ml_models/searchdefinitions/test.sd
@@ -33,8 +33,12 @@ search test {
             expression: xgboost("xgboost_2_2")
         }
 
+        function my_lightgbm() {
+            expression: lightgbm("lightgbm_regression")
+        }
+
         first-phase {
-            expression: mnist_tensorflow + mnist_softmax_tensorflow + mnist_softmax_onnx + my_xgboost
+            expression: mnist_tensorflow + mnist_softmax_tensorflow + mnist_softmax_onnx + my_xgboost + my_lightgbm
         }
 
     }
diff --git a/config-model/src/test/cfg/application/ml_serving/models/lightgbm_regression.json b/config-model/src/test/cfg/application/ml_serving/models/lightgbm_regression.json
new file mode 100644
index 00000000000..cf0488ecd8b
--- /dev/null
+++ b/config-model/src/test/cfg/application/ml_serving/models/lightgbm_regression.json
@@ -0,0 +1,275 @@
+{
+  "name": "tree",
+  "version": "v3",
+  "num_class": 1,
+  "num_tree_per_iteration": 1,
+  "label_index": 0,
+  "max_feature_idx": 3,
+  "average_output": false,
+  "objective": "regression",
+  "feature_names": [
+    "numerical_1",
+    "numerical_2",
+    "categorical_1",
+    "categorical_2"
+  ],
+  "monotone_constraints": [],
+  "tree_info": [
+    {
+      "tree_index": 0,
+      "num_leaves": 3,
+      "num_cat": 1,
+      "shrinkage": 1,
+      "tree_structure": {
+        "split_index": 0,
+        "split_feature": 1,
+        "split_gain": 68.5353012084961,
+        "threshold": 0.46643291586559305,
+        "decision_type": "<=",
+        "default_left": true,
+        "missing_type": "NaN",
+        "internal_value": 0,
+        "internal_weight": 0,
+        "internal_count": 1000,
+        "left_child": {
+          "leaf_index": 0,
+          "leaf_value": 2.1594397038037663,
+          "leaf_weight": 469,
+          "leaf_count": 469
+        },
+        "right_child": {
+          "split_index": 1,
+          "split_feature": 3,
+          "split_gain": 41.27640151977539,
+          "threshold": "2||3||4",
+          "decision_type": "==",
+          "default_left": false,
+          "missing_type": "NaN",
+          "internal_value": 0.246035,
+          "internal_weight": 531,
+          "internal_count": 531,
+          "left_child": {
+            "leaf_index": 1,
+            "leaf_value": 2.235297305276056,
+            "leaf_weight": 302,
+            "leaf_count": 302
+          },
+          "right_child": {
+            "leaf_index": 2,
+            "leaf_value": 2.1792953471546546,
+            "leaf_weight": 229,
+            "leaf_count": 229
+          }
+        }
+      }
+    },
+    {
+      "tree_index": 1,
+      "num_leaves": 3,
+      "num_cat": 1,
+      "shrinkage": 0.1,
+      "tree_structure": {
+        "split_index": 0,
+        "split_feature": 2,
+        "split_gain": 64.22250366210938,
+        "threshold": "3||4",
+        "decision_type": "==",
+        "default_left": false,
+        "missing_type": "NaN",
+        "internal_value": 0,
+        "internal_weight": 0,
+        "internal_count": 1000,
+        "left_child": {
+          "leaf_index": 0,
+          "leaf_value": 0.03070842919354316,
+          "leaf_weight": 399,
+          "leaf_count": 399
+        },
+        "right_child": {
+          "split_index": 1,
+          "split_feature": 0,
+          "split_gain": 36.74250030517578,
+          "threshold": 0.5102250691730842,
+          "decision_type": "<=",
+          "default_left": true,
+          "missing_type": "NaN",
+          "internal_value": -0.204906,
+          "internal_weight": 601,
+          "internal_count": 601,
+          "left_child": {
+            "leaf_index": 1,
+            "leaf_value": -0.04439151147520909,
+            "leaf_weight": 315,
+            "leaf_count": 315
+          },
+          "right_child": {
+            "leaf_index": 2,
+            "leaf_value": 0.005117411709368601,
+            "leaf_weight": 286,
+            "leaf_count": 286
+          }
+        }
+      }
+    },
+    {
+      "tree_index": 2,
+      "num_leaves": 3,
+      "num_cat": 0,
+      "shrinkage": 0.1,
+      "tree_structure": {
+        "split_index": 0,
+        "split_feature": 1,
+        "split_gain": 57.1327018737793,
+        "threshold": 0.668665477622446,
+        "decision_type": "<=",
+        "default_left": true,
+        "missing_type": "NaN",
+        "internal_value": 0,
+        "internal_weight": 0,
+        "internal_count": 1000,
+        "left_child": {
+          "split_index": 1,
+          "split_feature": 1,
+          "split_gain": 40.859100341796875,
+          "threshold": 0.008118820676863816,
+          "decision_type": "<=",
+          "default_left": true,
+          "missing_type": "NaN",
+          "internal_value": -0.162926,
+          "internal_weight": 681,
+          "internal_count": 681,
+          "left_child": {
+            "leaf_index": 0,
+            "leaf_value": -0.15361238490967524,
+            "leaf_weight": 21,
+            "leaf_count": 21
+          },
+          "right_child": {
+            "leaf_index": 2,
+            "leaf_value": -0.01192330846157292,
+            "leaf_weight": 660,
+            "leaf_count": 660
+          }
+        },
+        "right_child": {
+          "leaf_index": 1,
+          "leaf_value": 0.03499044894987518,
+          "leaf_weight": 319,
+          "leaf_count": 319
+        }
+      }
+    },
+    {
+      "tree_index": 3,
+      "num_leaves": 3,
+      "num_cat": 1,
+      "shrinkage": 0.1,
+      "tree_structure": {
+        "split_index": 0,
+        "split_feature": 0,
+        "split_gain": 54.77090072631836,
+        "threshold": 0.5201391072644542,
+        "decision_type": "<=",
+        "default_left": true,
+        "missing_type": "NaN",
+        "internal_value": 0,
+        "internal_weight": 0,
+        "internal_count": 1000,
+        "left_child": {
+          "leaf_index": 0,
+          "leaf_value": -0.02141000620783247,
+          "leaf_weight": 543,
+          "leaf_count": 543
+        },
+        "right_child": {
+          "split_index": 1,
+          "split_feature": 2,
+          "split_gain": 27.200700759887695,
+          "threshold": "0||1",
+          "decision_type": "==",
+          "default_left": false,
+          "missing_type": "NaN",
+          "internal_value": 0.255704,
+          "internal_weight": 457,
+          "internal_count": 457,
+          "left_child": {
+            "leaf_index": 1,
+            "leaf_value": -0.004121485787596721,
+            "leaf_weight": 191,
+            "leaf_count": 191
+          },
+          "right_child": {
+            "leaf_index": 2,
+            "leaf_value": 0.04534090904886873,
+            "leaf_weight": 266,
+            "leaf_count": 266
+          }
+        }
+      }
+    },
+    {
+      "tree_index": 4,
+      "num_leaves": 3,
+      "num_cat": 1,
+      "shrinkage": 0.1,
+      "tree_structure": {
+        "split_index": 0,
+        "split_feature": 3,
+        "split_gain": 51.84349822998047,
+        "threshold": "2||3||4",
+        "decision_type": "==",
+        "default_left": false,
+        "missing_type": "NaN",
+        "internal_value": 0,
+        "internal_weight": 0,
+        "internal_count": 1000,
+        "left_child": {
+          "split_index": 1,
+          "split_feature": 1,
+          "split_gain": 39.352699279785156,
+          "threshold": 0.27283279016959255,
+          "decision_type": "<=",
+          "default_left": true,
+          "missing_type": "NaN",
+          "internal_value": 0.188414,
+          "internal_weight": 593,
+          "internal_count": 593,
+          "left_child": {
+            "leaf_index": 0,
+            "leaf_value": -0.01924803254356527,
+            "leaf_weight": 184,
+            "leaf_count": 184
+          },
+          "right_child": {
+            "leaf_index": 2,
+            "leaf_value": 0.03643772842347651,
+            "leaf_weight": 409,
+            "leaf_count": 409
+          }
+        },
+        "right_child": {
+          "leaf_index": 1,
+          "leaf_value": -0.02701711918923075,
+          "leaf_weight": 407,
+          "leaf_count": 407
+        }
+      }
+    }
+  ],
+  "pandas_categorical": [
+    [
+      "a",
+      "b",
+      "c",
+      "d",
+      "e"
+    ],
+    [
+      "i",
+      "j",
+      "k",
+      "l",
+      "m"
+    ]
+  ]
+}
\ No newline at end of file
diff --git a/config-model/src/test/cfg/search/compare/complex/search/cluster.music/c0/r0/translogserver.MODEL.cfg b/config-model/src/test/cfg/search/compare/complex/search/cluster.music/c0/r0/translogserver.MODEL.cfg
deleted file mode 100644
index c5b1dfef610..00000000000
--- a/config-model/src/test/cfg/search/compare/complex/search/cluster.music/c0/r0/translogserver.MODEL.cfg
+++ /dev/null
@@ -1,7 +0,0 @@
-listenport 19125
-filesizemax 50000000
-servername "tls"
-basedir "tls"
-usefsync false
-maxthreads 4
-crcmethod xxh64
diff --git a/config-model/src/test/cfg/search/compare/complex/search/cluster.music/c0/r1/translogserver.MODEL.cfg b/config-model/src/test/cfg/search/compare/complex/search/cluster.music/c0/r1/translogserver.MODEL.cfg
deleted file mode 100644
index 7aed9bdc244..00000000000
--- a/config-model/src/test/cfg/search/compare/complex/search/cluster.music/c0/r1/translogserver.MODEL.cfg
+++ /dev/null
@@ -1,7 +0,0 @@
-listenport 19131
-filesizemax 50000000
-servername "tls"
-basedir "tls"
-usefsync false
-maxthreads 4
-crcmethod xxh64
diff --git a/config-model/src/test/cfg/search/compare/complex/search/cluster.music/c1/r0/translogserver.MODEL.cfg b/config-model/src/test/cfg/search/compare/complex/search/cluster.music/c1/r0/translogserver.MODEL.cfg
deleted file mode 100644
index 49e5f59b9be..00000000000
--- a/config-model/src/test/cfg/search/compare/complex/search/cluster.music/c1/r0/translogserver.MODEL.cfg
+++ /dev/null
@@ -1,7 +0,0 @@
-listenport 19137
-filesizemax 50000000
-servername "tls"
-basedir "tls"
-usefsync false
-maxthreads 4
-crcmethod xxh64
diff --git a/config-model/src/test/cfg/search/compare/complex/search/cluster.music/c1/r1/translogserver.MODEL.cfg b/config-model/src/test/cfg/search/compare/complex/search/cluster.music/c1/r1/translogserver.MODEL.cfg
deleted file mode 100644
index 8d5d4fdde7f..00000000000
--- a/config-model/src/test/cfg/search/compare/complex/search/cluster.music/c1/r1/translogserver.MODEL.cfg
+++ /dev/null
@@ -1,7 +0,0 @@
-listenport 19143
-filesizemax 50000000
-servername "tls"
-basedir "tls"
-usefsync false
-maxthreads 4
-crcmethod xxh64
diff --git a/config-model/src/test/cfg/search/compare/complex/search/cluster.music/rtx/0/pan-rtx.MODEL.cfg b/config-model/src/test/cfg/search/compare/complex/search/cluster.music/rtx/0/pan-rtx.MODEL.cfg
deleted file mode 100644
index be7d8e44c16..00000000000
--- a/config-model/src/test/cfg/search/compare/complex/search/cluster.music/rtx/0/pan-rtx.MODEL.cfg
+++ /dev/null
@@ -1,6 +0,0 @@
-port 19110
-slobrok.name "search/cluster.music/rtx/0/clustercontroller"
-slobrok.config search/cluster.music/rtx
-servicemonitor.autodisable false
-servicemonitor.autoenable false
-servicemonitor.timeout 120
diff --git a/config-model/src/test/cfg/search/compare/complex/search/cluster.music/rtx/1/pan-rtx.MODEL.cfg b/config-model/src/test/cfg/search/compare/complex/search/cluster.music/rtx/1/pan-rtx.MODEL.cfg
deleted file mode 100644
index f39dc6adc03..00000000000
--- a/config-model/src/test/cfg/search/compare/complex/search/cluster.music/rtx/1/pan-rtx.MODEL.cfg
+++ /dev/null
@@ -1,6 +0,0 @@
-port 19112
-slobrok.name "search/cluster.music/rtx/1/clustercontroller"
-slobrok.config search/cluster.music/rtx
-servicemonitor.autodisable false
-servicemonitor.autoenable false
-servicemonitor.timeout 120
diff --git a/config-model/src/test/cfg/search/compare/complex/search/cluster.rt/c0/r0/translogserver.MODEL.cfg b/config-model/src/test/cfg/search/compare/complex/search/cluster.rt/c0/r0/translogserver.MODEL.cfg
deleted file mode 100644
index 5c9f46bf8ce..00000000000
--- a/config-model/src/test/cfg/search/compare/complex/search/cluster.rt/c0/r0/translogserver.MODEL.cfg
+++ /dev/null
@@ -1,7 +0,0 @@
-listenport 19156
-filesizemax 50000000
-servername "tls"
-basedir "tls"
-usefsync false
-maxthreads 4
-crcmethod xxh64
diff --git a/config-model/src/test/cfg/search/compare/complex/search/cluster.rt/c0/r1/translogserver.MODEL.cfg b/config-model/src/test/cfg/search/compare/complex/search/cluster.rt/c0/r1/translogserver.MODEL.cfg
deleted file mode 100644
index 69f91eab48c..00000000000
--- a/config-model/src/test/cfg/search/compare/complex/search/cluster.rt/c0/r1/translogserver.MODEL.cfg
+++ /dev/null
@@ -1,7 +0,0 @@
-listenport 19105
-filesizemax 50000000
-servername "tls"
-basedir "tls"
-usefsync false
-maxthreads 4
-crcmethod xxh64
diff --git a/config-model/src/test/cfg/search/compare/complex/search/cluster.rt/c1/r0/translogserver.MODEL.cfg b/config-model/src/test/cfg/search/compare/complex/search/cluster.rt/c1/r0/translogserver.MODEL.cfg
deleted file mode 100644
index eebb1cb6b40..00000000000
--- a/config-model/src/test/cfg/search/compare/complex/search/cluster.rt/c1/r0/translogserver.MODEL.cfg
+++ /dev/null
@@ -1,7 +0,0 @@
-listenport 19111
-filesizemax 50000000
-servername "tls"
-basedir "tls"
-usefsync false
-maxthreads 4
-crcmethod xxh64
diff --git a/config-model/src/test/cfg/search/compare/complex/search/cluster.rt/c1/r1/translogserver.MODEL.cfg b/config-model/src/test/cfg/search/compare/complex/search/cluster.rt/c1/r1/translogserver.MODEL.cfg
deleted file mode 100644
index 82e9aafc5a8..00000000000
--- a/config-model/src/test/cfg/search/compare/complex/search/cluster.rt/c1/r1/translogserver.MODEL.cfg
+++ /dev/null
@@ -1,7 +0,0 @@
-listenport 19162
-filesizemax 50000000
-servername "tls"
-basedir "tls"
-usefsync false
-maxthreads 4
-crcmethod xxh64
diff --git a/config-model/src/test/cfg/search/compare/complex/search/cluster.rt/rtx/0/pan-rtx.MODEL.cfg b/config-model/src/test/cfg/search/compare/complex/search/cluster.rt/rtx/0/pan-rtx.MODEL.cfg
deleted file mode 100644
index aa557a9ae04..00000000000
--- a/config-model/src/test/cfg/search/compare/complex/search/cluster.rt/rtx/0/pan-rtx.MODEL.cfg
+++ /dev/null
@@ -1,6 +0,0 @@
-port 19144
-slobrok.name "search/cluster.rt/rtx/0/clustercontroller"
-slobrok.config search/cluster.rt/rtx
-servicemonitor.autodisable true
-servicemonitor.autoenable false
-servicemonitor.timeout 120
diff --git a/config-model/src/test/cfg/search/compare/complex/search/cluster.rt/rtx/1/pan-rtx.MODEL.cfg b/config-model/src/test/cfg/search/compare/complex/search/cluster.rt/rtx/1/pan-rtx.MODEL.cfg
deleted file mode 100644
index 1407c3ff209..00000000000
--- a/config-model/src/test/cfg/search/compare/complex/search/cluster.rt/rtx/1/pan-rtx.MODEL.cfg
+++ /dev/null
@@ -1,6 +0,0 @@
-port 19146
-slobrok.name "search/cluster.rt/rtx/1/clustercontroller"
-slobrok.config search/cluster.rt/rtx
-servicemonitor.autodisable true
-servicemonitor.autoenable false
-servicemonitor.timeout 120
diff --git a/config-model/src/test/cfg/search/compare/optionals/search/cluster.music/c0/r0/translogserver.MODEL.cfg b/config-model/src/test/cfg/search/compare/optionals/search/cluster.music/c0/r0/translogserver.MODEL.cfg
deleted file mode 100644
index c5b1dfef610..00000000000
--- a/config-model/src/test/cfg/search/compare/optionals/search/cluster.music/c0/r0/translogserver.MODEL.cfg
+++ /dev/null
@@ -1,7 +0,0 @@
-listenport 19125
-filesizemax 50000000
-servername "tls"
-basedir "tls"
-usefsync false
-maxthreads 4
-crcmethod xxh64
diff --git a/config-model/src/test/cfg/search/compare/optionals/search/cluster.music/rtx/0/pan-rtx.MODEL.cfg b/config-model/src/test/cfg/search/compare/optionals/search/cluster.music/rtx/0/pan-rtx.MODEL.cfg
deleted file mode 100644
index d0888e9be96..00000000000
--- a/config-model/src/test/cfg/search/compare/optionals/search/cluster.music/rtx/0/pan-rtx.MODEL.cfg
+++ /dev/null
@@ -1,6 +0,0 @@
-port 19115
-slobrok.name "search/cluster.music/rtx/0/clustercontroller"
-slobrok.config search/cluster.music/rtx
-servicemonitor.autodisable false
-servicemonitor.autoenable false
-servicemonitor.timeout 120
diff --git a/config-model/src/test/cfg/search/compare/simple/search/cluster.music/c0/r0/translogserver.MODEL.cfg b/config-model/src/test/cfg/search/compare/simple/search/cluster.music/c0/r0/translogserver.MODEL.cfg
deleted file mode 100644
index 53dcc3f9686..00000000000
--- a/config-model/src/test/cfg/search/compare/simple/search/cluster.music/c0/r0/translogserver.MODEL.cfg
+++ /dev/null
@@ -1,7 +0,0 @@
-listenport 19118
-filesizemax 50000000
-servername "tls"
-basedir "tls"
-usefsync false
-maxthreads 4
-crcmethod xxh64
diff --git a/config-model/src/test/cfg/search/compare/simple/search/cluster.music/rtx/0/pan-rtx.MODEL.cfg b/config-model/src/test/cfg/search/compare/simple/search/cluster.music/rtx/0/pan-rtx.MODEL.cfg
deleted file mode 100644
index 474b8c68cdb..00000000000
--- a/config-model/src/test/cfg/search/compare/simple/search/cluster.music/rtx/0/pan-rtx.MODEL.cfg
+++ /dev/null
@@ -1,6 +0,0 @@
-port 19108
-slobrok.name "search/cluster.music/rtx/0/clustercontroller"
-slobrok.config search/cluster.music/rtx
-servicemonitor.autodisable false
-servicemonitor.autoenable false
-servicemonitor.timeout 120
diff --git a/config-model/src/test/cfg/search/compare/twoFeedTargetClusters/search/cluster.music1/c0/r0/translogserver.MODEL.cfg b/config-model/src/test/cfg/search/compare/twoFeedTargetClusters/search/cluster.music1/c0/r0/translogserver.MODEL.cfg
deleted file mode 100644
index 6dabdfc6af7..00000000000
--- a/config-model/src/test/cfg/search/compare/twoFeedTargetClusters/search/cluster.music1/c0/r0/translogserver.MODEL.cfg
+++ /dev/null
@@ -1,7 +0,0 @@
-listenport 19115
-filesizemax 50000000
-servername "tls"
-basedir "tls"
-usefsync false
-maxthreads 4
-crcmethod xxh64
diff --git a/config-model/src/test/cfg/search/compare/twoFeedTargetClusters/search/cluster.music1/rtx/0/pan-rtx.MODEL.cfg b/config-model/src/test/cfg/search/compare/twoFeedTargetClusters/search/cluster.music1/rtx/0/pan-rtx.MODEL.cfg
deleted file mode 100644
index 3c1b537236a..00000000000
--- a/config-model/src/test/cfg/search/compare/twoFeedTargetClusters/search/cluster.music1/rtx/0/pan-rtx.MODEL.cfg
+++ /dev/null
@@ -1,6 +0,0 @@
-port 19105
-slobrok.name "search/cluster.music1/rtx/0/clustercontroller"
-slobrok.config search/cluster.music1/rtx
-servicemonitor.autodisable false
-servicemonitor.autoenable false
-servicemonitor.timeout 120
diff --git a/config-model/src/test/cfg/search/compare/twoFeedTargetClusters/search/cluster.music2/c0/r0/translogserver.MODEL.cfg b/config-model/src/test/cfg/search/compare/twoFeedTargetClusters/search/cluster.music2/c0/r0/translogserver.MODEL.cfg
deleted file mode 100644
index b6d62fc678b..00000000000
--- a/config-model/src/test/cfg/search/compare/twoFeedTargetClusters/search/cluster.music2/c0/r0/translogserver.MODEL.cfg
+++ /dev/null
@@ -1,7 +0,0 @@
-listenport 19126
-filesizemax 50000000
-servername "tls"
-basedir "tls"
-usefsync false
-maxthreads 4
-crcmethod xxh64
diff --git a/config-model/src/test/cfg/search/compare/twoFeedTargetClusters/search/cluster.music2/c0/r1/translogserver.MODEL.cfg b/config-model/src/test/cfg/search/compare/twoFeedTargetClusters/search/cluster.music2/c0/r1/translogserver.MODEL.cfg
deleted file mode 100644
index 887eaa6b634..00000000000
--- a/config-model/src/test/cfg/search/compare/twoFeedTargetClusters/search/cluster.music2/c0/r1/translogserver.MODEL.cfg
+++ /dev/null
@@ -1,7 +0,0 @@
-listenport 19132
-filesizemax 50000000
-servername "tls"
-basedir "tls"
-usefsync false
-maxthreads 4
-crcmethod xxh64
diff --git a/config-model/src/test/cfg/search/compare/twoFeedTargetClusters/search/cluster.music2/rtx/0/pan-rtx.MODEL.cfg b/config-model/src/test/cfg/search/compare/twoFeedTargetClusters/search/cluster.music2/rtx/0/pan-rtx.MODEL.cfg
deleted file mode 100644
index c00fcc456d6..00000000000
--- a/config-model/src/test/cfg/search/compare/twoFeedTargetClusters/search/cluster.music2/rtx/0/pan-rtx.MODEL.cfg
+++ /dev/null
@@ -1,6 +0,0 @@
-port 19116
-slobrok.name "search/cluster.music2/rtx/0/clustercontroller"
-slobrok.config search/cluster.music2/rtx
-servicemonitor.autodisable false
-servicemonitor.autoenable false
-servicemonitor.timeout 120
diff --git a/config-model/src/test/configmodel/types/references/documentmanager_multiple_imported_fields.cfg b/config-model/src/test/configmodel/types/references/documentmanager_multiple_imported_fields.cfg
new file mode 100644
index 00000000000..7b50176625d
--- /dev/null
+++ b/config-model/src/test/configmodel/types/references/documentmanager_multiple_imported_fields.cfg
@@ -0,0 +1,104 @@
+enablecompression false
+datatype[0].id 1381038251
+datatype[0].structtype[0].name "position"
+datatype[0].structtype[0].version 0
+datatype[0].structtype[0].compresstype NONE
+datatype[0].structtype[0].compresslevel 0
+datatype[0].structtype[0].compressthreshold 95
+datatype[0].structtype[0].compressminsize 800
+datatype[0].structtype[0].field[0].name "x"
+datatype[0].structtype[0].field[0].datatype 0
+datatype[0].structtype[0].field[0].detailedtype ""
+datatype[0].structtype[0].field[1].name "y"
+datatype[0].structtype[0].field[1].datatype 0
+datatype[0].structtype[0].field[1].detailedtype ""
+datatype[1].id 595216861
+datatype[1].referencetype[0].target_type_id -1318255918
+datatype[2].id 542332920
+datatype[2].referencetype[0].target_type_id 443162583
+datatype[3].id 959075962
+datatype[3].structtype[0].name "ad.header"
+datatype[3].structtype[0].version 0
+datatype[3].structtype[0].compresstype NONE
+datatype[3].structtype[0].compresslevel 0
+datatype[3].structtype[0].compressthreshold 95
+datatype[3].structtype[0].compressminsize 800
+datatype[3].structtype[0].field[0].name "campaign_ref"
+datatype[3].structtype[0].field[0].datatype 595216861
+datatype[3].structtype[0].field[0].detailedtype ""
+datatype[3].structtype[0].field[1].name "person_ref"
+datatype[3].structtype[0].field[1].datatype 542332920
+datatype[3].structtype[0].field[1].detailedtype ""
+datatype[4].id -255288561
+datatype[4].structtype[0].name "ad.body"
+datatype[4].structtype[0].version 0
+datatype[4].structtype[0].compresstype NONE
+datatype[4].structtype[0].compresslevel 0
+datatype[4].structtype[0].compressthreshold 95
+datatype[4].structtype[0].compressminsize 800
+datatype[5].id 2987301
+datatype[5].documenttype[0].name "ad"
+datatype[5].documenttype[0].version 0
+datatype[5].documenttype[0].inherits[0].name "document"
+datatype[5].documenttype[0].inherits[0].version 0
+datatype[5].documenttype[0].headerstruct 959075962
+datatype[5].documenttype[0].bodystruct -255288561
+datatype[5].documenttype[0].fieldsets{[document]}.fields[0] "campaign_ref"
+datatype[5].documenttype[0].fieldsets{[document]}.fields[1] "person_ref"
+datatype[5].documenttype[0].importedfield[0].name "my_cool_field"
+datatype[5].documenttype[0].importedfield[1].name "my_swag_field"
+datatype[5].documenttype[0].importedfield[2].name "my_name"
+datatype[6].id -2041471955
+datatype[6].structtype[0].name "campaign.header"
+datatype[6].structtype[0].version 0
+datatype[6].structtype[0].compresstype NONE
+datatype[6].structtype[0].compresslevel 0
+datatype[6].structtype[0].compressthreshold 95
+datatype[6].structtype[0].compressminsize 800
+datatype[6].structtype[0].field[0].name "cool_field"
+datatype[6].structtype[0].field[0].datatype 2
+datatype[6].structtype[0].field[0].detailedtype ""
+datatype[6].structtype[0].field[1].name "swag_field"
+datatype[6].structtype[0].field[1].datatype 4
+datatype[6].structtype[0].field[1].detailedtype ""
+datatype[7].id 1448849794
+datatype[7].structtype[0].name "campaign.body"
+datatype[7].structtype[0].version 0
+datatype[7].structtype[0].compresstype NONE
+datatype[7].structtype[0].compresslevel 0
+datatype[7].structtype[0].compressthreshold 95
+datatype[7].structtype[0].compressminsize 800
+datatype[8].id -1318255918
+datatype[8].documenttype[0].name "campaign"
+datatype[8].documenttype[0].version 0
+datatype[8].documenttype[0].inherits[0].name "document"
+datatype[8].documenttype[0].inherits[0].version 0
+datatype[8].documenttype[0].headerstruct -2041471955
+datatype[8].documenttype[0].bodystruct 1448849794
+datatype[8].documenttype[0].fieldsets{[document]}.fields[0] "cool_field"
+datatype[8].documenttype[0].fieldsets{[document]}.fields[1] "swag_field"
+datatype[9].id 3129224
+datatype[9].structtype[0].name "person.header"
+datatype[9].structtype[0].version 0
+datatype[9].structtype[0].compresstype NONE
+datatype[9].structtype[0].compresslevel 0
+datatype[9].structtype[0].compressthreshold 95
+datatype[9].structtype[0].compressminsize 800
+datatype[9].structtype[0].field[0].name "name"
+datatype[9].structtype[0].field[0].datatype 2
+datatype[9].structtype[0].field[0].detailedtype ""
+datatype[10].id -2003767395
+datatype[10].structtype[0].name "person.body"
+datatype[10].structtype[0].version 0
+datatype[10].structtype[0].compresstype NONE
+datatype[10].structtype[0].compresslevel 0
+datatype[10].structtype[0].compressthreshold 95
+datatype[10].structtype[0].compressminsize 800
+datatype[11].id 443162583
+datatype[11].documenttype[0].name "person"
+datatype[11].documenttype[0].version 0
+datatype[11].documenttype[0].inherits[0].name "document"
+datatype[11].documenttype[0].inherits[0].version 0
+datatype[11].documenttype[0].headerstruct 3129224
+datatype[11].documenttype[0].bodystruct -2003767395
+datatype[11].documenttype[0].fieldsets{[document]}.fields[0] "name"
\ No newline at end of file
diff --git a/config-model/src/test/configmodel/types/references/documenttypes_multiple_imported_fields.cfg b/config-model/src/test/configmodel/types/references/documenttypes_multiple_imported_fields.cfg
new file mode 100644
index 00000000000..7859703ffe0
--- /dev/null
+++ b/config-model/src/test/configmodel/types/references/documenttypes_multiple_imported_fields.cfg
@@ -0,0 +1,141 @@
+enablecompression false
+documenttype[0].id 2987301
+documenttype[0].name "ad"
+documenttype[0].version 0
+documenttype[0].headerstruct 959075962
+documenttype[0].bodystruct -255288561
+documenttype[0].inherits[0].id 8
+documenttype[0].datatype[0].id 959075962
+documenttype[0].datatype[0].type STRUCT
+documenttype[0].datatype[0].array.element.id 0
+documenttype[0].datatype[0].map.key.id 0
+documenttype[0].datatype[0].map.value.id 0
+documenttype[0].datatype[0].wset.key.id 0
+documenttype[0].datatype[0].wset.createifnonexistent false
+documenttype[0].datatype[0].wset.removeifzero false
+documenttype[0].datatype[0].annotationref.annotation.id 0
+documenttype[0].datatype[0].sstruct.name "ad.header"
+documenttype[0].datatype[0].sstruct.version 0
+documenttype[0].datatype[0].sstruct.compression.type NONE
+documenttype[0].datatype[0].sstruct.compression.level 0
+documenttype[0].datatype[0].sstruct.compression.threshold 95
+documenttype[0].datatype[0].sstruct.compression.minsize 200
+documenttype[0].datatype[0].sstruct.field[0].name "campaign_ref"
+documenttype[0].datatype[0].sstruct.field[0].id 23963250
+documenttype[0].datatype[0].sstruct.field[0].datatype 595216861
+documenttype[0].datatype[0].sstruct.field[0].detailedtype ""
+documenttype[0].datatype[0].sstruct.field[1].name "person_ref"
+documenttype[0].datatype[0].sstruct.field[1].id 100779805
+documenttype[0].datatype[0].sstruct.field[1].datatype 542332920
+documenttype[0].datatype[0].sstruct.field[1].detailedtype ""
+documenttype[0].datatype[1].id -255288561
+documenttype[0].datatype[1].type STRUCT
+documenttype[0].datatype[1].array.element.id 0
+documenttype[0].datatype[1].map.key.id 0
+documenttype[0].datatype[1].map.value.id 0
+documenttype[0].datatype[1].wset.key.id 0
+documenttype[0].datatype[1].wset.createifnonexistent false
+documenttype[0].datatype[1].wset.removeifzero false
+documenttype[0].datatype[1].annotationref.annotation.id 0
+documenttype[0].datatype[1].sstruct.name "ad.body"
+documenttype[0].datatype[1].sstruct.version 0
+documenttype[0].datatype[1].sstruct.compression.type NONE
+documenttype[0].datatype[1].sstruct.compression.level 0
+documenttype[0].datatype[1].sstruct.compression.threshold 95
+documenttype[0].datatype[1].sstruct.compression.minsize 200
+documenttype[0].fieldsets{[document]}.fields[0] "campaign_ref"
+documenttype[0].fieldsets{[document]}.fields[1] "person_ref"
+documenttype[0].referencetype[0].id 595216861
+documenttype[0].referencetype[0].target_type_id -1318255918
+documenttype[0].referencetype[1].id 542332920
+documenttype[0].referencetype[1].target_type_id 443162583
+documenttype[0].importedfield[0].name "my_cool_field"
+documenttype[0].importedfield[1].name "my_swag_field"
+documenttype[0].importedfield[2].name "my_name"
+documenttype[1].id -1318255918
+documenttype[1].name "campaign"
+documenttype[1].version 0
+documenttype[1].headerstruct -2041471955
+documenttype[1].bodystruct 1448849794
+documenttype[1].inherits[0].id 8
+documenttype[1].datatype[0].id -2041471955
+documenttype[1].datatype[0].type STRUCT
+documenttype[1].datatype[0].array.element.id 0
+documenttype[1].datatype[0].map.key.id 0
+documenttype[1].datatype[0].map.value.id 0
+documenttype[1].datatype[0].wset.key.id 0
+documenttype[1].datatype[0].wset.createifnonexistent false
+documenttype[1].datatype[0].wset.removeifzero false
+documenttype[1].datatype[0].annotationref.annotation.id 0
+documenttype[1].datatype[0].sstruct.name "campaign.header"
+documenttype[1].datatype[0].sstruct.version 0
+documenttype[1].datatype[0].sstruct.compression.type NONE
+documenttype[1].datatype[0].sstruct.compression.level 0
+documenttype[1].datatype[0].sstruct.compression.threshold 95
+documenttype[1].datatype[0].sstruct.compression.minsize 200
+documenttype[1].datatype[0].sstruct.field[0].name "cool_field"
+documenttype[1].datatype[0].sstruct.field[0].id 1588702436
+documenttype[1].datatype[0].sstruct.field[0].datatype 2
+documenttype[1].datatype[0].sstruct.field[0].detailedtype ""
+documenttype[1].datatype[0].sstruct.field[1].name "swag_field"
+documenttype[1].datatype[0].sstruct.field[1].id 1691224741
+documenttype[1].datatype[0].sstruct.field[1].datatype 4
+documenttype[1].datatype[0].sstruct.field[1].detailedtype ""
+documenttype[1].datatype[1].id 1448849794
+documenttype[1].datatype[1].type STRUCT
+documenttype[1].datatype[1].array.element.id 0
+documenttype[1].datatype[1].map.key.id 0
+documenttype[1].datatype[1].map.value.id 0
+documenttype[1].datatype[1].wset.key.id 0
+documenttype[1].datatype[1].wset.createifnonexistent false
+documenttype[1].datatype[1].wset.removeifzero false
+documenttype[1].datatype[1].annotationref.annotation.id 0
+documenttype[1].datatype[1].sstruct.name "campaign.body"
+documenttype[1].datatype[1].sstruct.version 0
+documenttype[1].datatype[1].sstruct.compression.type NONE
+documenttype[1].datatype[1].sstruct.compression.level 0
+documenttype[1].datatype[1].sstruct.compression.threshold 95
+documenttype[1].datatype[1].sstruct.compression.minsize 200
+documenttype[1].fieldsets{[document]}.fields[0] "cool_field"
+documenttype[1].fieldsets{[document]}.fields[1] "swag_field"
+documenttype[2].id 443162583
+documenttype[2].name "person"
+documenttype[2].version 0
+documenttype[2].headerstruct 3129224
+documenttype[2].bodystruct -2003767395
+documenttype[2].inherits[0].id 8
+documenttype[2].datatype[0].id 3129224
+documenttype[2].datatype[0].type STRUCT
+documenttype[2].datatype[0].array.element.id 0
+documenttype[2].datatype[0].map.key.id 0
+documenttype[2].datatype[0].map.value.id 0
+documenttype[2].datatype[0].wset.key.id 0
+documenttype[2].datatype[0].wset.createifnonexistent false
+documenttype[2].datatype[0].wset.removeifzero false
+documenttype[2].datatype[0].annotationref.annotation.id 0
+documenttype[2].datatype[0].sstruct.name "person.header"
+documenttype[2].datatype[0].sstruct.version 0
+documenttype[2].datatype[0].sstruct.compression.type NONE
+documenttype[2].datatype[0].sstruct.compression.level 0
+documenttype[2].datatype[0].sstruct.compression.threshold 95
+documenttype[2].datatype[0].sstruct.compression.minsize 200
+documenttype[2].datatype[0].sstruct.field[0].name "name"
+documenttype[2].datatype[0].sstruct.field[0].id 1160796772
+documenttype[2].datatype[0].sstruct.field[0].datatype 2
+documenttype[2].datatype[0].sstruct.field[0].detailedtype ""
+documenttype[2].datatype[1].id -2003767395
+documenttype[2].datatype[1].type STRUCT
+documenttype[2].datatype[1].array.element.id 0
+documenttype[2].datatype[1].map.key.id 0
+documenttype[2].datatype[1].map.value.id 0
+documenttype[2].datatype[1].wset.key.id 0
+documenttype[2].datatype[1].wset.createifnonexistent false
+documenttype[2].datatype[1].wset.removeifzero false
+documenttype[2].datatype[1].annotationref.annotation.id 0
+documenttype[2].datatype[1].sstruct.name "person.body"
+documenttype[2].datatype[1].sstruct.version 0
+documenttype[2].datatype[1].sstruct.compression.type NONE
+documenttype[2].datatype[1].sstruct.compression.level 0
+documenttype[2].datatype[1].sstruct.compression.threshold 95
+documenttype[2].datatype[1].sstruct.compression.minsize 200
+documenttype[2].fieldsets{[document]}.fields[0] "name"
\ No newline at end of file
diff --git a/config-model/src/test/derived/advanced/attributes.cfg b/config-model/src/test/derived/advanced/attributes.cfg
index 97f480745cf..0a76e44c4ac 100644
--- a/config-model/src/test/derived/advanced/attributes.cfg
+++ b/config-model/src/test/derived/advanced/attributes.cfg
@@ -19,3 +19,7 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
diff --git a/config-model/src/test/derived/array_of_struct_attribute/attributes.cfg b/config-model/src/test/derived/array_of_struct_attribute/attributes.cfg
index 56fd15f9f5d..006400c09d4 100644
--- a/config-model/src/test/derived/array_of_struct_attribute/attributes.cfg
+++ b/config-model/src/test/derived/array_of_struct_attribute/attributes.cfg
@@ -19,6 +19,10 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
 attribute[].name "elem_array.weight"
 attribute[].datatype INT32
 attribute[].collectiontype ARRAY
@@ -40,3 +44,7 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
diff --git a/config-model/src/test/derived/attributeprefetch/attributes.cfg b/config-model/src/test/derived/attributeprefetch/attributes.cfg
index 022bdbd31a4..cd91c15700b 100644
--- a/config-model/src/test/derived/attributeprefetch/attributes.cfg
+++ b/config-model/src/test/derived/attributeprefetch/attributes.cfg
@@ -19,6 +19,10 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
 attribute[].name "multibyte"
 attribute[].datatype INT8
 attribute[].collectiontype ARRAY
@@ -40,6 +44,10 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
 attribute[].name "wsbyte"
 attribute[].datatype INT8
 attribute[].collectiontype WEIGHTEDSET
@@ -61,6 +69,10 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
 attribute[].name "singleint"
 attribute[].datatype INT32
 attribute[].collectiontype SINGLE
@@ -82,6 +94,10 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
 attribute[].name "multiint"
 attribute[].datatype INT32
 attribute[].collectiontype ARRAY
@@ -103,6 +119,10 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
 attribute[].name "wsint"
 attribute[].datatype INT32
 attribute[].collectiontype WEIGHTEDSET
@@ -124,6 +144,10 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
 attribute[].name "singlelong"
 attribute[].datatype INT64
 attribute[].collectiontype SINGLE
@@ -145,6 +169,10 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
 attribute[].name "multilong"
 attribute[].datatype INT64
 attribute[].collectiontype ARRAY
@@ -166,6 +194,10 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
 attribute[].name "wslong"
 attribute[].datatype INT64
 attribute[].collectiontype WEIGHTEDSET
@@ -187,6 +219,10 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
 attribute[].name "singlefloat"
 attribute[].datatype FLOAT
 attribute[].collectiontype SINGLE
@@ -208,6 +244,10 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
 attribute[].name "multifloat"
 attribute[].datatype FLOAT
 attribute[].collectiontype ARRAY
@@ -229,6 +269,10 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
 attribute[].name "wsfloat"
 attribute[].datatype FLOAT
 attribute[].collectiontype WEIGHTEDSET
@@ -250,6 +294,10 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
 attribute[].name "singledouble"
 attribute[].datatype DOUBLE
 attribute[].collectiontype SINGLE
@@ -271,6 +319,10 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
 attribute[].name "multidouble"
 attribute[].datatype DOUBLE
 attribute[].collectiontype ARRAY
@@ -292,6 +344,10 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
 attribute[].name "wsdouble"
 attribute[].datatype DOUBLE
 attribute[].collectiontype WEIGHTEDSET
@@ -313,6 +369,10 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
 attribute[].name "singlestring"
 attribute[].datatype STRING
 attribute[].collectiontype SINGLE
@@ -334,6 +394,10 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
 attribute[].name "multistring"
 attribute[].datatype STRING
 attribute[].collectiontype ARRAY
@@ -355,6 +419,10 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
 attribute[].name "wsstring"
 attribute[].datatype STRING
 attribute[].collectiontype WEIGHTEDSET
@@ -376,3 +444,7 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
diff --git a/config-model/src/test/derived/attributes/attributes.cfg b/config-model/src/test/derived/attributes/attributes.cfg
index 7a21001a9ed..29e4c209ef2 100644
--- a/config-model/src/test/derived/attributes/attributes.cfg
+++ b/config-model/src/test/derived/attributes/attributes.cfg
@@ -19,6 +19,10 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
 attribute[].name "a2"
 attribute[].datatype STRING
 attribute[].collectiontype SINGLE
@@ -40,6 +44,10 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
 attribute[].name "a3"
 attribute[].datatype STRING
 attribute[].collectiontype SINGLE
@@ -61,6 +69,10 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
 attribute[].name "a5"
 attribute[].datatype STRING
 attribute[].collectiontype SINGLE
@@ -82,6 +94,10 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
 attribute[].name "a6"
 attribute[].datatype STRING
 attribute[].collectiontype SINGLE
@@ -103,6 +119,10 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
 attribute[].name "b1"
 attribute[].datatype STRING
 attribute[].collectiontype SINGLE
@@ -124,6 +144,10 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
 attribute[].name "b2"
 attribute[].datatype STRING
 attribute[].collectiontype SINGLE
@@ -145,6 +169,10 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
 attribute[].name "b3"
 attribute[].datatype STRING
 attribute[].collectiontype SINGLE
@@ -166,6 +194,10 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
 attribute[].name "b4"
 attribute[].datatype INT32
 attribute[].collectiontype SINGLE
@@ -187,6 +219,10 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
 attribute[].name "b5"
 attribute[].datatype INT32
 attribute[].collectiontype SINGLE
@@ -208,6 +244,10 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
 attribute[].name "b6"
 attribute[].datatype INT64
 attribute[].collectiontype ARRAY
@@ -229,6 +269,10 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
 attribute[].name "b7"
 attribute[].datatype DOUBLE
 attribute[].collectiontype WEIGHTEDSET
@@ -250,6 +294,10 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
 attribute[].name "a9"
 attribute[].datatype INT32
 attribute[].collectiontype SINGLE
@@ -271,6 +319,10 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
 attribute[].name "a10"
 attribute[].datatype INT32
 attribute[].collectiontype ARRAY
@@ -292,6 +344,10 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
 attribute[].name "a11"
 attribute[].datatype INT32
 attribute[].collectiontype SINGLE
@@ -313,6 +369,10 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
 attribute[].name "a12"
 attribute[].datatype INT32
 attribute[].collectiontype SINGLE
@@ -334,6 +394,10 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
 attribute[].name "a7_arr"
 attribute[].datatype STRING
 attribute[].collectiontype ARRAY
@@ -355,6 +419,10 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
 attribute[].name "a8_arr"
 attribute[].datatype STRING
 attribute[].collectiontype ARRAY
@@ -376,3 +444,7 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
diff --git a/config-model/src/test/derived/complex/attributes.cfg b/config-model/src/test/derived/complex/attributes.cfg
index dda746da5b4..5606e5ea730 100644
--- a/config-model/src/test/derived/complex/attributes.cfg
+++ b/config-model/src/test/derived/complex/attributes.cfg
@@ -19,6 +19,10 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
 attribute[].name "fleeting"
 attribute[].datatype FLOAT
 attribute[].collectiontype ARRAY
@@ -40,6 +44,10 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
 attribute[].name "fleeting2"
 attribute[].datatype FLOAT
 attribute[].collectiontype SINGLE
@@ -61,6 +69,10 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
 attribute[].name "foundat"
 attribute[].datatype INT64
 attribute[].collectiontype SINGLE
@@ -82,6 +94,10 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
 attribute[].name "collapseby"
 attribute[].datatype INT32
 attribute[].collectiontype SINGLE
@@ -103,6 +119,10 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
 attribute[].name "ts"
 attribute[].datatype INT64
 attribute[].collectiontype SINGLE
@@ -124,6 +144,10 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
 attribute[].name "combineda"
 attribute[].datatype INT32
 attribute[].collectiontype SINGLE
@@ -145,6 +169,10 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
 attribute[].name "year_arr"
 attribute[].datatype INT32
 attribute[].collectiontype ARRAY
@@ -166,6 +194,10 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
 attribute[].name "year_sub"
 attribute[].datatype INT32
 attribute[].collectiontype SINGLE
@@ -187,3 +219,7 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
diff --git a/config-model/src/test/derived/hnsw_index/attributes.cfg b/config-model/src/test/derived/hnsw_index/attributes.cfg
new file mode 100644
index 00000000000..b61fd7e2ee5
--- /dev/null
+++ b/config-model/src/test/derived/hnsw_index/attributes.cfg
@@ -0,0 +1,25 @@
+attribute[].name "t1"
+attribute[].datatype TENSOR
+attribute[].collectiontype SINGLE
+attribute[].removeifzero false
+attribute[].createifnonexistent false
+attribute[].fastsearch false
+attribute[].huge false
+attribute[].ismutable false
+attribute[].sortascending true
+attribute[].sortfunction UCA
+attribute[].sortstrength PRIMARY
+attribute[].sortlocale ""
+attribute[].enablebitvectors false
+attribute[].enableonlybitvector false
+attribute[].fastaccess false
+attribute[].arity 8
+attribute[].lowerbound -9223372036854775808
+attribute[].upperbound 9223372036854775807
+attribute[].densepostinglistthreshold 0.4
+attribute[].tensortype "tensor(x[128])"
+attribute[].imported false
+attribute[].index.hnsw.enabled true
+attribute[].index.hnsw.maxlinkspernode 32
+attribute[].index.hnsw.distancemetric ANGULAR
+attribute[].index.hnsw.neighborstoexploreatinsert 300
diff --git a/config-model/src/test/derived/hnsw_index/ilscripts.cfg b/config-model/src/test/derived/hnsw_index/ilscripts.cfg
new file mode 100644
index 00000000000..e9fc265ca67
--- /dev/null
+++ b/config-model/src/test/derived/hnsw_index/ilscripts.cfg
@@ -0,0 +1,5 @@
+maxtermoccurrences 100
+fieldmatchmaxlength 1000000
+ilscript[].doctype "test"
+ilscript[].docfield[] "t1"
+ilscript[].content[] "clear_state | guard { input t1 | attribute t1 | index t1; }"
diff --git a/config-model/src/test/derived/hnsw_index/test.sd b/config-model/src/test/derived/hnsw_index/test.sd
new file mode 100644
index 00000000000..207ed764a87
--- /dev/null
+++ b/config-model/src/test/derived/hnsw_index/test.sd
@@ -0,0 +1,14 @@
+search test {
+  document test {
+    field t1 type tensor(x[128]) {
+      indexing: attribute | index
+      index {
+        distance-metric: angular
+        hnsw {
+          max-links-per-node: 32
+          neighbors-to-explore-at-insert: 300
+        }
+      }
+    }
+  }
+}
diff --git a/config-model/src/test/derived/imported_position_field/attributes.cfg b/config-model/src/test/derived/imported_position_field/attributes.cfg
index db2280a7846..8f68068c8e7 100644
--- a/config-model/src/test/derived/imported_position_field/attributes.cfg
+++ b/config-model/src/test/derived/imported_position_field/attributes.cfg
@@ -1,42 +1,50 @@
-attribute[0].name "parent_ref"
-attribute[0].datatype REFERENCE
-attribute[0].collectiontype SINGLE
-attribute[0].removeifzero false
-attribute[0].createifnonexistent false
-attribute[0].fastsearch false
-attribute[0].huge false
-attribute[0].ismutable false
-attribute[0].sortascending true
-attribute[0].sortfunction UCA
-attribute[0].sortstrength PRIMARY
-attribute[0].sortlocale ""
-attribute[0].enablebitvectors false
-attribute[0].enableonlybitvector false
-attribute[0].fastaccess false
-attribute[0].arity 8
-attribute[0].lowerbound -9223372036854775808
-attribute[0].upperbound 9223372036854775807
-attribute[0].densepostinglistthreshold 0.4
-attribute[0].tensortype ""
-attribute[0].imported false
-attribute[1].name "my_pos_zcurve"
-attribute[1].datatype INT64
-attribute[1].collectiontype SINGLE
-attribute[1].removeifzero false
-attribute[1].createifnonexistent false
-attribute[1].fastsearch true
-attribute[1].huge false
-attribute[1].ismutable false
-attribute[1].sortascending true
-attribute[1].sortfunction UCA
-attribute[1].sortstrength PRIMARY
-attribute[1].sortlocale ""
-attribute[1].enablebitvectors false
-attribute[1].enableonlybitvector false
-attribute[1].fastaccess false
-attribute[1].arity 8
-attribute[1].lowerbound -9223372036854775808
-attribute[1].upperbound 9223372036854775807
-attribute[1].densepostinglistthreshold 0.4
-attribute[1].tensortype ""
-attribute[1].imported true
\ No newline at end of file
+attribute[].name "parent_ref"
+attribute[].datatype REFERENCE
+attribute[].collectiontype SINGLE
+attribute[].removeifzero false
+attribute[].createifnonexistent false
+attribute[].fastsearch false
+attribute[].huge false
+attribute[].ismutable false
+attribute[].sortascending true
+attribute[].sortfunction UCA
+attribute[].sortstrength PRIMARY
+attribute[].sortlocale ""
+attribute[].enablebitvectors false
+attribute[].enableonlybitvector false
+attribute[].fastaccess false
+attribute[].arity 8
+attribute[].lowerbound -9223372036854775808
+attribute[].upperbound 9223372036854775807
+attribute[].densepostinglistthreshold 0.4
+attribute[].tensortype ""
+attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
+attribute[].name "my_pos_zcurve"
+attribute[].datatype INT64
+attribute[].collectiontype SINGLE
+attribute[].removeifzero false
+attribute[].createifnonexistent false
+attribute[].fastsearch true
+attribute[].huge false
+attribute[].ismutable false
+attribute[].sortascending true
+attribute[].sortfunction UCA
+attribute[].sortstrength PRIMARY
+attribute[].sortlocale ""
+attribute[].enablebitvectors false
+attribute[].enableonlybitvector false
+attribute[].fastaccess false
+attribute[].arity 8
+attribute[].lowerbound -9223372036854775808
+attribute[].upperbound 9223372036854775807
+attribute[].densepostinglistthreshold 0.4
+attribute[].tensortype ""
+attribute[].imported true
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
diff --git a/config-model/src/test/derived/imported_struct_fields/attributes.cfg b/config-model/src/test/derived/imported_struct_fields/attributes.cfg
index ce6ff5e54ae..eb87c6cf53e 100644
--- a/config-model/src/test/derived/imported_struct_fields/attributes.cfg
+++ b/config-model/src/test/derived/imported_struct_fields/attributes.cfg
@@ -1,168 +1,200 @@
-attribute[0].name "parent_ref"
-attribute[0].datatype REFERENCE
-attribute[0].collectiontype SINGLE
-attribute[0].removeifzero false
-attribute[0].createifnonexistent false
-attribute[0].fastsearch false
-attribute[0].huge false
-attribute[0].ismutable false
-attribute[0].sortascending true
-attribute[0].sortfunction UCA
-attribute[0].sortstrength PRIMARY
-attribute[0].sortlocale ""
-attribute[0].enablebitvectors false
-attribute[0].enableonlybitvector false
-attribute[0].fastaccess false
-attribute[0].arity 8
-attribute[0].lowerbound -9223372036854775808
-attribute[0].upperbound 9223372036854775807
-attribute[0].densepostinglistthreshold 0.4
-attribute[0].tensortype ""
-attribute[0].imported false
-attribute[1].name "my_elem_array.name"
-attribute[1].datatype STRING
-attribute[1].collectiontype SINGLE
-attribute[1].removeifzero false
-attribute[1].createifnonexistent false
-attribute[1].fastsearch true
-attribute[1].huge false
-attribute[1].ismutable false
-attribute[1].sortascending true
-attribute[1].sortfunction UCA
-attribute[1].sortstrength PRIMARY
-attribute[1].sortlocale ""
-attribute[1].enablebitvectors false
-attribute[1].enableonlybitvector false
-attribute[1].fastaccess false
-attribute[1].arity 8
-attribute[1].lowerbound -9223372036854775808
-attribute[1].upperbound 9223372036854775807
-attribute[1].densepostinglistthreshold 0.4
-attribute[1].tensortype ""
-attribute[1].imported true
-attribute[2].name "my_elem_array.weight"
-attribute[2].datatype INT32
-attribute[2].collectiontype SINGLE
-attribute[2].removeifzero false
-attribute[2].createifnonexistent false
-attribute[2].fastsearch false
-attribute[2].huge false
-attribute[2].ismutable false
-attribute[2].sortascending true
-attribute[2].sortfunction UCA
-attribute[2].sortstrength PRIMARY
-attribute[2].sortlocale ""
-attribute[2].enablebitvectors false
-attribute[2].enableonlybitvector false
-attribute[2].fastaccess false
-attribute[2].arity 8
-attribute[2].lowerbound -9223372036854775808
-attribute[2].upperbound 9223372036854775807
-attribute[2].densepostinglistthreshold 0.4
-attribute[2].tensortype ""
-attribute[2].imported true
-attribute[3].name "my_elem_map.key"
-attribute[3].datatype STRING
-attribute[3].collectiontype SINGLE
-attribute[3].removeifzero false
-attribute[3].createifnonexistent false
-attribute[3].fastsearch true
-attribute[3].huge false
-attribute[3].ismutable false
-attribute[3].sortascending true
-attribute[3].sortfunction UCA
-attribute[3].sortstrength PRIMARY
-attribute[3].sortlocale ""
-attribute[3].enablebitvectors false
-attribute[3].enableonlybitvector false
-attribute[3].fastaccess false
-attribute[3].arity 8
-attribute[3].lowerbound -9223372036854775808
-attribute[3].upperbound 9223372036854775807
-attribute[3].densepostinglistthreshold 0.4
-attribute[3].tensortype ""
-attribute[3].imported true
-attribute[4].name "my_elem_map.value.name"
-attribute[4].datatype STRING
-attribute[4].collectiontype SINGLE
-attribute[4].removeifzero false
-attribute[4].createifnonexistent false
-attribute[4].fastsearch true
-attribute[4].huge false
-attribute[4].ismutable false
-attribute[4].sortascending true
-attribute[4].sortfunction UCA
-attribute[4].sortstrength PRIMARY
-attribute[4].sortlocale ""
-attribute[4].enablebitvectors false
-attribute[4].enableonlybitvector false
-attribute[4].fastaccess false
-attribute[4].arity 8
-attribute[4].lowerbound -9223372036854775808
-attribute[4].upperbound 9223372036854775807
-attribute[4].densepostinglistthreshold 0.4
-attribute[4].tensortype ""
-attribute[4].imported true
-attribute[5].name "my_elem_map.value.weight"
-attribute[5].datatype INT32
-attribute[5].collectiontype SINGLE
-attribute[5].removeifzero false
-attribute[5].createifnonexistent false
-attribute[5].fastsearch false
-attribute[5].huge false
-attribute[5].ismutable false
-attribute[5].sortascending true
-attribute[5].sortfunction UCA
-attribute[5].sortstrength PRIMARY
-attribute[5].sortlocale ""
-attribute[5].enablebitvectors false
-attribute[5].enableonlybitvector false
-attribute[5].fastaccess false
-attribute[5].arity 8
-attribute[5].lowerbound -9223372036854775808
-attribute[5].upperbound 9223372036854775807
-attribute[5].densepostinglistthreshold 0.4
-attribute[5].tensortype ""
-attribute[5].imported true
-attribute[6].name "my_str_int_map.key"
-attribute[6].datatype STRING
-attribute[6].collectiontype SINGLE
-attribute[6].removeifzero false
-attribute[6].createifnonexistent false
-attribute[6].fastsearch true
-attribute[6].huge false
-attribute[6].ismutable false
-attribute[6].sortascending true
-attribute[6].sortfunction UCA
-attribute[6].sortstrength PRIMARY
-attribute[6].sortlocale ""
-attribute[6].enablebitvectors false
-attribute[6].enableonlybitvector false
-attribute[6].fastaccess false
-attribute[6].arity 8
-attribute[6].lowerbound -9223372036854775808
-attribute[6].upperbound 9223372036854775807
-attribute[6].densepostinglistthreshold 0.4
-attribute[6].tensortype ""
-attribute[6].imported true
-attribute[7].name "my_str_int_map.value"
-attribute[7].datatype INT32
-attribute[7].collectiontype SINGLE
-attribute[7].removeifzero false
-attribute[7].createifnonexistent false
-attribute[7].fastsearch false
-attribute[7].huge false
-attribute[7].ismutable false
-attribute[7].sortascending true
-attribute[7].sortfunction UCA
-attribute[7].sortstrength PRIMARY
-attribute[7].sortlocale ""
-attribute[7].enablebitvectors false
-attribute[7].enableonlybitvector false
-attribute[7].fastaccess false
-attribute[7].arity 8
-attribute[7].lowerbound -9223372036854775808
-attribute[7].upperbound 9223372036854775807
-attribute[7].densepostinglistthreshold 0.4
-attribute[7].tensortype ""
-attribute[7].imported true
\ No newline at end of file
+attribute[].name "parent_ref"
+attribute[].datatype REFERENCE
+attribute[].collectiontype SINGLE
+attribute[].removeifzero false
+attribute[].createifnonexistent false
+attribute[].fastsearch false
+attribute[].huge false
+attribute[].ismutable false
+attribute[].sortascending true
+attribute[].sortfunction UCA
+attribute[].sortstrength PRIMARY
+attribute[].sortlocale ""
+attribute[].enablebitvectors false
+attribute[].enableonlybitvector false
+attribute[].fastaccess false
+attribute[].arity 8
+attribute[].lowerbound -9223372036854775808
+attribute[].upperbound 9223372036854775807
+attribute[].densepostinglistthreshold 0.4
+attribute[].tensortype ""
+attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
+attribute[].name "my_elem_array.name"
+attribute[].datatype STRING
+attribute[].collectiontype SINGLE
+attribute[].removeifzero false
+attribute[].createifnonexistent false
+attribute[].fastsearch true
+attribute[].huge false
+attribute[].ismutable false
+attribute[].sortascending true
+attribute[].sortfunction UCA
+attribute[].sortstrength PRIMARY
+attribute[].sortlocale ""
+attribute[].enablebitvectors false
+attribute[].enableonlybitvector false
+attribute[].fastaccess false
+attribute[].arity 8
+attribute[].lowerbound -9223372036854775808
+attribute[].upperbound 9223372036854775807
+attribute[].densepostinglistthreshold 0.4
+attribute[].tensortype ""
+attribute[].imported true
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
+attribute[].name "my_elem_array.weight"
+attribute[].datatype INT32
+attribute[].collectiontype SINGLE
+attribute[].removeifzero false
+attribute[].createifnonexistent false
+attribute[].fastsearch false
+attribute[].huge false
+attribute[].ismutable false
+attribute[].sortascending true
+attribute[].sortfunction UCA
+attribute[].sortstrength PRIMARY
+attribute[].sortlocale ""
+attribute[].enablebitvectors false
+attribute[].enableonlybitvector false
+attribute[].fastaccess false
+attribute[].arity 8
+attribute[].lowerbound -9223372036854775808
+attribute[].upperbound 9223372036854775807
+attribute[].densepostinglistthreshold 0.4
+attribute[].tensortype ""
+attribute[].imported true
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
+attribute[].name "my_elem_map.key"
+attribute[].datatype STRING
+attribute[].collectiontype SINGLE
+attribute[].removeifzero false
+attribute[].createifnonexistent false
+attribute[].fastsearch true
+attribute[].huge false
+attribute[].ismutable false
+attribute[].sortascending true
+attribute[].sortfunction UCA
+attribute[].sortstrength PRIMARY
+attribute[].sortlocale ""
+attribute[].enablebitvectors false
+attribute[].enableonlybitvector false
+attribute[].fastaccess false
+attribute[].arity 8
+attribute[].lowerbound -9223372036854775808
+attribute[].upperbound 9223372036854775807
+attribute[].densepostinglistthreshold 0.4
+attribute[].tensortype ""
+attribute[].imported true
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
+attribute[].name "my_elem_map.value.name"
+attribute[].datatype STRING
+attribute[].collectiontype SINGLE
+attribute[].removeifzero false
+attribute[].createifnonexistent false
+attribute[].fastsearch true
+attribute[].huge false
+attribute[].ismutable false
+attribute[].sortascending true
+attribute[].sortfunction UCA
+attribute[].sortstrength PRIMARY
+attribute[].sortlocale ""
+attribute[].enablebitvectors false
+attribute[].enableonlybitvector false
+attribute[].fastaccess false
+attribute[].arity 8
+attribute[].lowerbound -9223372036854775808
+attribute[].upperbound 9223372036854775807
+attribute[].densepostinglistthreshold 0.4
+attribute[].tensortype ""
+attribute[].imported true
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
+attribute[].name "my_elem_map.value.weight"
+attribute[].datatype INT32
+attribute[].collectiontype SINGLE
+attribute[].removeifzero false
+attribute[].createifnonexistent false
+attribute[].fastsearch false
+attribute[].huge false
+attribute[].ismutable false
+attribute[].sortascending true
+attribute[].sortfunction UCA
+attribute[].sortstrength PRIMARY
+attribute[].sortlocale ""
+attribute[].enablebitvectors false
+attribute[].enableonlybitvector false
+attribute[].fastaccess false
+attribute[].arity 8
+attribute[].lowerbound -9223372036854775808
+attribute[].upperbound 9223372036854775807
+attribute[].densepostinglistthreshold 0.4
+attribute[].tensortype ""
+attribute[].imported true
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
+attribute[].name "my_str_int_map.key"
+attribute[].datatype STRING
+attribute[].collectiontype SINGLE
+attribute[].removeifzero false
+attribute[].createifnonexistent false
+attribute[].fastsearch true
+attribute[].huge false
+attribute[].ismutable false
+attribute[].sortascending true
+attribute[].sortfunction UCA
+attribute[].sortstrength PRIMARY
+attribute[].sortlocale ""
+attribute[].enablebitvectors false
+attribute[].enableonlybitvector false
+attribute[].fastaccess false
+attribute[].arity 8
+attribute[].lowerbound -9223372036854775808
+attribute[].upperbound 9223372036854775807
+attribute[].densepostinglistthreshold 0.4
+attribute[].tensortype ""
+attribute[].imported true
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
+attribute[].name "my_str_int_map.value"
+attribute[].datatype INT32
+attribute[].collectiontype SINGLE
+attribute[].removeifzero false
+attribute[].createifnonexistent false
+attribute[].fastsearch false
+attribute[].huge false
+attribute[].ismutable false
+attribute[].sortascending true
+attribute[].sortfunction UCA
+attribute[].sortstrength PRIMARY
+attribute[].sortlocale ""
+attribute[].enablebitvectors false
+attribute[].enableonlybitvector false
+attribute[].fastaccess false
+attribute[].arity 8
+attribute[].lowerbound -9223372036854775808
+attribute[].upperbound 9223372036854775807
+attribute[].densepostinglistthreshold 0.4
+attribute[].tensortype ""
+attribute[].imported true
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
diff --git a/config-model/src/test/derived/importedfields/attributes.cfg b/config-model/src/test/derived/importedfields/attributes.cfg
index 15b1771a2e8..d3a51523e05 100644
--- a/config-model/src/test/derived/importedfields/attributes.cfg
+++ b/config-model/src/test/derived/importedfields/attributes.cfg
@@ -19,6 +19,10 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
 attribute[].name "b_ref"
 attribute[].datatype REFERENCE
 attribute[].collectiontype SINGLE
@@ -40,6 +44,10 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
 attribute[].name "b_ref_with_summary"
 attribute[].datatype REFERENCE
 attribute[].collectiontype SINGLE
@@ -61,6 +69,10 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
 attribute[].name "my_int_field"
 attribute[].datatype INT32
 attribute[].collectiontype SINGLE
@@ -82,6 +94,10 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported true
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
 attribute[].name "my_string_field"
 attribute[].datatype STRING
 attribute[].collectiontype SINGLE
@@ -103,6 +119,10 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported true
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
 attribute[].name "my_int_array_field"
 attribute[].datatype INT32
 attribute[].collectiontype ARRAY
@@ -124,6 +144,10 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported true
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
 attribute[].name "my_int_wset_field"
 attribute[].datatype INT32
 attribute[].collectiontype WEIGHTEDSET
@@ -145,6 +169,10 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported true
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
 attribute[].name "my_ancient_int_field"
 attribute[].datatype INT32
 attribute[].collectiontype SINGLE
@@ -166,3 +194,7 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported true
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
diff --git a/config-model/src/test/derived/indexschema/index-info.cfg b/config-model/src/test/derived/indexschema/index-info.cfg
index a83ec45c5e9..388b212689a 100644
--- a/config-model/src/test/derived/indexschema/index-info.cfg
+++ b/config-model/src/test/derived/indexschema/index-info.cfg
@@ -44,6 +44,8 @@ indexinfo[].command[].command "normalize"
 indexinfo[].command[].indexname "sd"
 indexinfo[].command[].command "plain-tokens"
 indexinfo[].command[].indexname "sd"
+indexinfo[].command[].command "phrase-segmenting false"
+indexinfo[].command[].indexname "sd"
 indexinfo[].command[].command "literal-boost"
 indexinfo[].command[].indexname "pos.x"
 indexinfo[].command[].command "index"
@@ -307,6 +309,16 @@ indexinfo[].command[].indexname "exactexplicit"
 indexinfo[].command[].command "exact ARNOLD"
 indexinfo[].command[].indexname "exactexplicit"
 indexinfo[].command[].command "dynteaser"
+indexinfo[].command[].indexname "exactexplicit"
+indexinfo[].command[].command "lowercase"
+indexinfo[].command[].indexname "exactexplicit"
+indexinfo[].command[].command "index"
+indexinfo[].command[].indexname "exactexplicit"
+indexinfo[].command[].command "plain-tokens"
+indexinfo[].command[].indexname "exactexplicit"
+indexinfo[].command[].command "stem:BEST"
+indexinfo[].command[].indexname "exactexplicit"
+indexinfo[].command[].command "normalize"
 indexinfo[].command[].indexname "exactexplicit2"
 indexinfo[].command[].command "lowercase"
 indexinfo[].command[].indexname "exactexplicit2"
@@ -322,6 +334,8 @@ indexinfo[].command[].command "stem:BEST"
 indexinfo[].command[].indexname "gram"
 indexinfo[].command[].command "normalize"
 indexinfo[].command[].indexname "gram"
+indexinfo[].command[].command "phrase-segmenting false"
+indexinfo[].command[].indexname "gram"
 indexinfo[].command[].command "ngram 2"
 indexinfo[].command[].indexname "nostem1"
 indexinfo[].command[].command "lowercase"
diff --git a/config-model/src/test/derived/indexschema/indexschema.sd b/config-model/src/test/derived/indexschema/indexschema.sd
index 49f0f7dfca6..01f552a8c32 100644
--- a/config-model/src/test/derived/indexschema/indexschema.sd
+++ b/config-model/src/test/derived/indexschema/indexschema.sd
@@ -24,6 +24,7 @@ search indexschema {
     field sd type string {
       indexing: index
       rank:literal
+      query-command: "phrase-segmenting false"
     }
     field pos type position {
       indexing: attribute
@@ -130,17 +131,17 @@ search indexschema {
   }
 
   fieldset exactexplicit {
-    fields:sa, sb
     query-command: "exact ARNOLD"
+    fields:sa, sb
     query-command: dynteaser
   }
   
   fieldset exactexplicit2 {
-    fields:sc, sd
     match {
       exact
       exact-terminator: "Arnold"
     }
+    fields:sc, sd
   }
 
   fieldset gram {
diff --git a/config-model/src/test/derived/inheritance/attributes.cfg b/config-model/src/test/derived/inheritance/attributes.cfg
index 4a081edbf54..397d8878792 100644
--- a/config-model/src/test/derived/inheritance/attributes.cfg
+++ b/config-model/src/test/derived/inheritance/attributes.cfg
@@ -19,6 +19,10 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
 attribute[].name "overridden"
 attribute[].datatype INT32
 attribute[].collectiontype SINGLE
@@ -40,6 +44,10 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
 attribute[].name "onlymother"
 attribute[].datatype STRING
 attribute[].collectiontype SINGLE
@@ -61,3 +69,7 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
diff --git a/config-model/src/test/derived/inheritfromparent/attributes.cfg b/config-model/src/test/derived/inheritfromparent/attributes.cfg
index 13f59d4925f..3b30af10a8f 100644
--- a/config-model/src/test/derived/inheritfromparent/attributes.cfg
+++ b/config-model/src/test/derived/inheritfromparent/attributes.cfg
@@ -19,3 +19,7 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
diff --git a/config-model/src/test/derived/map_attribute/attributes.cfg b/config-model/src/test/derived/map_attribute/attributes.cfg
index 8901acf63d1..4de78c52efd 100644
--- a/config-model/src/test/derived/map_attribute/attributes.cfg
+++ b/config-model/src/test/derived/map_attribute/attributes.cfg
@@ -19,6 +19,10 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
 attribute[].name "str_map.value"
 attribute[].datatype STRING
 attribute[].collectiontype ARRAY
@@ -40,6 +44,10 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
 attribute[].name "int_map.key"
 attribute[].datatype INT32
 attribute[].collectiontype ARRAY
@@ -61,3 +69,7 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
diff --git a/config-model/src/test/derived/map_of_struct_attribute/attributes.cfg b/config-model/src/test/derived/map_of_struct_attribute/attributes.cfg
index 665edcdf45d..72e60b857a2 100644
--- a/config-model/src/test/derived/map_of_struct_attribute/attributes.cfg
+++ b/config-model/src/test/derived/map_of_struct_attribute/attributes.cfg
@@ -19,6 +19,10 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
 attribute[].name "str_elem_map.value.name"
 attribute[].datatype STRING
 attribute[].collectiontype ARRAY
@@ -40,6 +44,10 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
 attribute[].name "str_elem_map.value.weight"
 attribute[].datatype INT32
 attribute[].collectiontype ARRAY
@@ -61,6 +69,10 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
 attribute[].name "int_elem_map.key"
 attribute[].datatype INT32
 attribute[].collectiontype ARRAY
@@ -82,6 +94,10 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
 attribute[].name "int_elem_map.value.name"
 attribute[].datatype STRING
 attribute[].collectiontype ARRAY
@@ -103,3 +119,7 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
diff --git a/config-model/src/test/derived/music/attributes.cfg b/config-model/src/test/derived/music/attributes.cfg
index 7f9592dafc8..4d19dc69b33 100644
--- a/config-model/src/test/derived/music/attributes.cfg
+++ b/config-model/src/test/derived/music/attributes.cfg
@@ -19,6 +19,10 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
 attribute[].name "pto"
 attribute[].datatype INT32
 attribute[].collectiontype SINGLE
@@ -40,6 +44,10 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
 attribute[].name "mid"
 attribute[].datatype INT32
 attribute[].collectiontype SINGLE
@@ -61,6 +69,10 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
 attribute[].name "weight"
 attribute[].datatype FLOAT
 attribute[].collectiontype SINGLE
@@ -82,6 +94,10 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
 attribute[].name "bgnpfrom"
 attribute[].datatype FLOAT
 attribute[].collectiontype SINGLE
@@ -103,6 +119,10 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
 attribute[].name "newestedition"
 attribute[].datatype INT32
 attribute[].collectiontype SINGLE
@@ -124,6 +144,10 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
 attribute[].name "year"
 attribute[].datatype INT32
 attribute[].collectiontype SINGLE
@@ -145,6 +169,10 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
 attribute[].name "did"
 attribute[].datatype INT32
 attribute[].collectiontype SINGLE
@@ -166,6 +194,10 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
 attribute[].name "cbid"
 attribute[].datatype INT32
 attribute[].collectiontype SINGLE
@@ -187,6 +219,10 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
 attribute[].name "hiphopvalue_arr"
 attribute[].datatype STRING
 attribute[].collectiontype ARRAY
@@ -208,6 +244,10 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
 attribute[].name "metalvalue_arr"
 attribute[].datatype STRING
 attribute[].collectiontype ARRAY
@@ -229,3 +269,7 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
diff --git a/config-model/src/test/derived/nearestneighbor/query-profiles/default.xml b/config-model/src/test/derived/nearestneighbor/query-profiles/default.xml
new file mode 100644
index 00000000000..b8140f34617
--- /dev/null
+++ b/config-model/src/test/derived/nearestneighbor/query-profiles/default.xml
@@ -0,0 +1 @@
+<query-profile id="default" type="root" />
diff --git a/config-model/src/test/derived/nearestneighbor/query-profiles/types/root.xml b/config-model/src/test/derived/nearestneighbor/query-profiles/types/root.xml
new file mode 100644
index 00000000000..895e0663181
--- /dev/null
+++ b/config-model/src/test/derived/nearestneighbor/query-profiles/types/root.xml
@@ -0,0 +1,3 @@
+<query-profile-type id="root" inherits="native">
+  <field name="ranking.features.query(q_vec)" type="tensor&lt;float&gt;(x[5])" />
+</query-profile-type>
diff --git a/config-model/src/test/derived/nearestneighbor/test.sd b/config-model/src/test/derived/nearestneighbor/test.sd
new file mode 100644
index 00000000000..ab5f6d85448
--- /dev/null
+++ b/config-model/src/test/derived/nearestneighbor/test.sd
@@ -0,0 +1,27 @@
+search test {
+  document test {
+    field id type int {
+      indexing: attribute | summary
+    }
+    field vec type tensor<float>(x[5]) {
+      indexing: attribute | summary
+    }
+    field vec_hnsw type tensor<float>(x[5]) {
+      indexing: attribute | index | summary
+      index {
+        hnsw {
+          max-links-per-node: 16
+          neighbors-to-explore-at-insert: 200
+        }
+      }
+    }
+  }
+  rank-profile default {
+    first-phase {
+      expression: 10000 - itemRawScore(nns)
+    }
+  }
+  document-summary minimal {
+    summary id type int {}
+  }
+}
diff --git a/config-model/src/test/derived/neuralnet/query-profiles.cfg b/config-model/src/test/derived/neuralnet/query-profiles.cfg
new file mode 100644
index 00000000000..817640a89fd
--- /dev/null
+++ b/config-model/src/test/derived/neuralnet/query-profiles.cfg
@@ -0,0 +1,54 @@
+queryprofile[].id "default"
+queryprofile[].type "DefaultQueryProfileType"
+queryprofiletype[].id "DefaultQueryProfileType"
+queryprofiletype[].strict false
+queryprofiletype[].matchaspath false
+queryprofiletype[].inherit[] "native"
+queryprofiletype[].field[].name "ranking"
+queryprofiletype[].field[].type "query-profile:ranking_0_0"
+queryprofiletype[].field[].overridable true
+queryprofiletype[].field[].mandatory false
+queryprofiletype[].field[].alias ""
+queryprofiletype[].id "ranking_0_0"
+queryprofiletype[].strict false
+queryprofiletype[].matchaspath false
+queryprofiletype[].inherit[] "ranking"
+queryprofiletype[].field[].name "features"
+queryprofiletype[].field[].type "query-profile:features_0_1"
+queryprofiletype[].field[].overridable true
+queryprofiletype[].field[].mandatory false
+queryprofiletype[].field[].alias "rankfeature"
+queryprofiletype[].id "features_0_1"
+queryprofiletype[].strict false
+queryprofiletype[].matchaspath false
+queryprofiletype[].field[].name "query(W_0)"
+queryprofiletype[].field[].type "tensor(hidden[9],x[9])"
+queryprofiletype[].field[].overridable true
+queryprofiletype[].field[].mandatory false
+queryprofiletype[].field[].alias ""
+queryprofiletype[].field[].name "query(W_1)"
+queryprofiletype[].field[].type "tensor(hidden[9],out[9])"
+queryprofiletype[].field[].overridable true
+queryprofiletype[].field[].mandatory false
+queryprofiletype[].field[].alias ""
+queryprofiletype[].field[].name "query(W_out)"
+queryprofiletype[].field[].type "tensor(out[9])"
+queryprofiletype[].field[].overridable true
+queryprofiletype[].field[].mandatory false
+queryprofiletype[].field[].alias ""
+queryprofiletype[].field[].name "query(b_0)"
+queryprofiletype[].field[].type "tensor(hidden[9])"
+queryprofiletype[].field[].overridable true
+queryprofiletype[].field[].mandatory false
+queryprofiletype[].field[].alias ""
+queryprofiletype[].field[].name "query(b_1)"
+queryprofiletype[].field[].type "tensor(out[9])"
+queryprofiletype[].field[].overridable true
+queryprofiletype[].field[].mandatory false
+queryprofiletype[].field[].alias ""
+queryprofiletype[].field[].name "query(b_out)"
+queryprofiletype[].field[].type "tensor(out[1])"
+queryprofiletype[].field[].overridable true
+queryprofiletype[].field[].mandatory false
+queryprofiletype[].field[].alias ""
+enableGroupingSessionCache true
diff --git a/config-model/src/test/derived/neuralnet/query-profiles/types/DefaultQueryProfileType.xml b/config-model/src/test/derived/neuralnet/query-profiles/types/DefaultQueryProfileType.xml
index e74152638fb..42336098a9a 100644
--- a/config-model/src/test/derived/neuralnet/query-profiles/types/DefaultQueryProfileType.xml
+++ b/config-model/src/test/derived/neuralnet/query-profiles/types/DefaultQueryProfileType.xml
@@ -1,5 +1,5 @@
 <!-- Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
-<query-profile-type id="DefaultQueryProfileType">
+<query-profile-type id="DefaultQueryProfileType" inherits="native">
     <field name="ranking.features.query(W_0)" type="tensor(x[9],hidden[9])" />
     <field name="ranking.features.query(b_0)" type="tensor(hidden[9])" />
     <field name="ranking.features.query(W_1)" type="tensor(hidden[9],out[9])" />
diff --git a/config-model/src/test/derived/newrank/attributes.cfg b/config-model/src/test/derived/newrank/attributes.cfg
index b33c2fbdf9b..2aed2288773 100644
--- a/config-model/src/test/derived/newrank/attributes.cfg
+++ b/config-model/src/test/derived/newrank/attributes.cfg
@@ -19,6 +19,10 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
 attribute[].name "pto"
 attribute[].datatype INT32
 attribute[].collectiontype SINGLE
@@ -40,6 +44,10 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
 attribute[].name "mid"
 attribute[].datatype INT32
 attribute[].collectiontype SINGLE
@@ -61,6 +69,10 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
 attribute[].name "weight"
 attribute[].datatype FLOAT
 attribute[].collectiontype SINGLE
@@ -82,6 +94,10 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
 attribute[].name "bgnpfrom"
 attribute[].datatype FLOAT
 attribute[].collectiontype SINGLE
@@ -103,6 +119,10 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
 attribute[].name "newestedition"
 attribute[].datatype INT32
 attribute[].collectiontype SINGLE
@@ -124,6 +144,10 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
 attribute[].name "year"
 attribute[].datatype INT32
 attribute[].collectiontype SINGLE
@@ -145,6 +169,10 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
 attribute[].name "did"
 attribute[].datatype INT32
 attribute[].collectiontype SINGLE
@@ -166,6 +194,10 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
 attribute[].name "scorekey"
 attribute[].datatype INT32
 attribute[].collectiontype SINGLE
@@ -187,6 +219,10 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
 attribute[].name "cbid"
 attribute[].datatype INT32
 attribute[].collectiontype SINGLE
@@ -208,3 +244,7 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
diff --git a/config-model/src/test/derived/predicate_attribute/attributes.cfg b/config-model/src/test/derived/predicate_attribute/attributes.cfg
index 47e07e2a524..3a9daf7af94 100644
--- a/config-model/src/test/derived/predicate_attribute/attributes.cfg
+++ b/config-model/src/test/derived/predicate_attribute/attributes.cfg
@@ -19,3 +19,7 @@ attribute[].upperbound 200
 attribute[].densepostinglistthreshold 0.2
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
diff --git a/config-model/src/test/derived/predicate_attribute/index-info.cfg b/config-model/src/test/derived/predicate_attribute/index-info.cfg
index 3d9f57dd84b..4ebac65e1f5 100644
--- a/config-model/src/test/derived/predicate_attribute/index-info.cfg
+++ b/config-model/src/test/derived/predicate_attribute/index-info.cfg
@@ -4,6 +4,8 @@ indexinfo[].command[].command "index"
 indexinfo[].command[].indexname "sddocname"
 indexinfo[].command[].command "word"
 indexinfo[].command[].indexname "some_predicate_field"
+indexinfo[].command[].command "predicate"
+indexinfo[].command[].indexname "some_predicate_field"
 indexinfo[].command[].command "predicate-bounds [3..200]"
 indexinfo[].command[].indexname "some_predicate_field"
 indexinfo[].command[].command "index"
diff --git a/config-model/src/test/derived/prefixexactattribute/attributes.cfg b/config-model/src/test/derived/prefixexactattribute/attributes.cfg
index d7922a0de69..0a8cbd82186 100644
--- a/config-model/src/test/derived/prefixexactattribute/attributes.cfg
+++ b/config-model/src/test/derived/prefixexactattribute/attributes.cfg
@@ -19,6 +19,10 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
 attribute[].name "attributefield2"
 attribute[].datatype STRING
 attribute[].collectiontype SINGLE
@@ -40,3 +44,7 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
diff --git a/config-model/src/test/derived/reference_fields/attributes.cfg b/config-model/src/test/derived/reference_fields/attributes.cfg
index 12dbf896edc..e83b187a0cc 100644
--- a/config-model/src/test/derived/reference_fields/attributes.cfg
+++ b/config-model/src/test/derived/reference_fields/attributes.cfg
@@ -19,6 +19,10 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
 attribute[].name "other_ref"
 attribute[].datatype REFERENCE
 attribute[].collectiontype SINGLE
@@ -40,6 +44,10 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
 attribute[].name "yet_another_ref"
 attribute[].datatype REFERENCE
 attribute[].collectiontype SINGLE
@@ -61,3 +69,7 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
diff --git a/config-model/src/test/derived/sorting/attributes.cfg b/config-model/src/test/derived/sorting/attributes.cfg
index e88dfde03bb..83c310ca5ca 100644
--- a/config-model/src/test/derived/sorting/attributes.cfg
+++ b/config-model/src/test/derived/sorting/attributes.cfg
@@ -19,6 +19,10 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
 attribute[].name "syntaxcheck2"
 attribute[].datatype STRING
 attribute[].collectiontype SINGLE
@@ -40,6 +44,10 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
 attribute[].name "infieldonly"
 attribute[].datatype STRING
 attribute[].collectiontype SINGLE
@@ -61,3 +69,7 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
diff --git a/config-model/src/test/derived/tensor/attributes.cfg b/config-model/src/test/derived/tensor/attributes.cfg
index 4634e120a3a..780f47ee3d5 100644
--- a/config-model/src/test/derived/tensor/attributes.cfg
+++ b/config-model/src/test/derived/tensor/attributes.cfg
@@ -19,6 +19,10 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype "tensor<float>(x[2],y[1])"
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
 attribute[].name "f3"
 attribute[].datatype TENSOR
 attribute[].collectiontype SINGLE
@@ -40,6 +44,10 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype "tensor(x{})"
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
 attribute[].name "f4"
 attribute[].datatype TENSOR
 attribute[].collectiontype SINGLE
@@ -61,6 +69,10 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype "tensor(x[10],y[10])"
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
 attribute[].name "f5"
 attribute[].datatype TENSOR
 attribute[].collectiontype SINGLE
@@ -82,6 +94,10 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype "tensor<float>(x[10])"
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
 attribute[].name "f6"
 attribute[].datatype FLOAT
 attribute[].collectiontype SINGLE
@@ -103,3 +119,7 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
diff --git a/config-model/src/test/derived/types/attributes.cfg b/config-model/src/test/derived/types/attributes.cfg
index e6ffc37e871..82535324864 100644
--- a/config-model/src/test/derived/types/attributes.cfg
+++ b/config-model/src/test/derived/types/attributes.cfg
@@ -19,6 +19,10 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
 attribute[].name "along"
 attribute[].datatype INT64
 attribute[].collectiontype SINGLE
@@ -40,6 +44,10 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
 attribute[].name "abool"
 attribute[].datatype BOOL
 attribute[].collectiontype SINGLE
@@ -61,6 +69,10 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
 attribute[].name "ashortfloat"
 attribute[].datatype FLOAT16
 attribute[].collectiontype SINGLE
@@ -82,6 +94,10 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
 attribute[].name "arrayfield"
 attribute[].datatype INT32
 attribute[].collectiontype ARRAY
@@ -103,6 +119,10 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
 attribute[].name "setfield"
 attribute[].datatype STRING
 attribute[].collectiontype WEIGHTEDSET
@@ -124,6 +144,10 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
 attribute[].name "setfield2"
 attribute[].datatype STRING
 attribute[].collectiontype WEIGHTEDSET
@@ -145,6 +169,10 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
 attribute[].name "setfield3"
 attribute[].datatype STRING
 attribute[].collectiontype WEIGHTEDSET
@@ -166,6 +194,10 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
 attribute[].name "setfield4"
 attribute[].datatype STRING
 attribute[].collectiontype WEIGHTEDSET
@@ -187,6 +219,10 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
 attribute[].name "tagfield"
 attribute[].datatype STRING
 attribute[].collectiontype WEIGHTEDSET
@@ -208,6 +244,10 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
 attribute[].name "juletre"
 attribute[].datatype INT64
 attribute[].collectiontype SINGLE
@@ -229,6 +269,10 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
 attribute[].name "album1"
 attribute[].datatype STRING
 attribute[].collectiontype WEIGHTEDSET
@@ -250,6 +294,10 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
 attribute[].name "other"
 attribute[].datatype INT64
 attribute[].collectiontype SINGLE
@@ -271,3 +319,7 @@ attribute[].upperbound 9223372036854775807
 attribute[].densepostinglistthreshold 0.4
 attribute[].tensortype ""
 attribute[].imported false
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.distancemetric EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert 200
diff --git a/config-model/src/test/examples/simple.sd b/config-model/src/test/examples/simple.sd
index e61b64dc0ed..a8c801b1421 100644
--- a/config-model/src/test/examples/simple.sd
+++ b/config-model/src/test/examples/simple.sd
@@ -3,7 +3,7 @@
 # You can get a reasonable configuration by only configuring
 # a document
 # ...this has become less and less simple over time actually
-search simple {
+schema simple {
 
   document simple {
 
diff --git a/config-model/src/test/integration/lightgbm/models/regression.json b/config-model/src/test/integration/lightgbm/models/regression.json
new file mode 100644
index 00000000000..cf0488ecd8b
--- /dev/null
+++ b/config-model/src/test/integration/lightgbm/models/regression.json
@@ -0,0 +1,275 @@
+{
+  "name": "tree",
+  "version": "v3",
+  "num_class": 1,
+  "num_tree_per_iteration": 1,
+  "label_index": 0,
+  "max_feature_idx": 3,
+  "average_output": false,
+  "objective": "regression",
+  "feature_names": [
+    "numerical_1",
+    "numerical_2",
+    "categorical_1",
+    "categorical_2"
+  ],
+  "monotone_constraints": [],
+  "tree_info": [
+    {
+      "tree_index": 0,
+      "num_leaves": 3,
+      "num_cat": 1,
+      "shrinkage": 1,
+      "tree_structure": {
+        "split_index": 0,
+        "split_feature": 1,
+        "split_gain": 68.5353012084961,
+        "threshold": 0.46643291586559305,
+        "decision_type": "<=",
+        "default_left": true,
+        "missing_type": "NaN",
+        "internal_value": 0,
+        "internal_weight": 0,
+        "internal_count": 1000,
+        "left_child": {
+          "leaf_index": 0,
+          "leaf_value": 2.1594397038037663,
+          "leaf_weight": 469,
+          "leaf_count": 469
+        },
+        "right_child": {
+          "split_index": 1,
+          "split_feature": 3,
+          "split_gain": 41.27640151977539,
+          "threshold": "2||3||4",
+          "decision_type": "==",
+          "default_left": false,
+          "missing_type": "NaN",
+          "internal_value": 0.246035,
+          "internal_weight": 531,
+          "internal_count": 531,
+          "left_child": {
+            "leaf_index": 1,
+            "leaf_value": 2.235297305276056,
+            "leaf_weight": 302,
+            "leaf_count": 302
+          },
+          "right_child": {
+            "leaf_index": 2,
+            "leaf_value": 2.1792953471546546,
+            "leaf_weight": 229,
+            "leaf_count": 229
+          }
+        }
+      }
+    },
+    {
+      "tree_index": 1,
+      "num_leaves": 3,
+      "num_cat": 1,
+      "shrinkage": 0.1,
+      "tree_structure": {
+        "split_index": 0,
+        "split_feature": 2,
+        "split_gain": 64.22250366210938,
+        "threshold": "3||4",
+        "decision_type": "==",
+        "default_left": false,
+        "missing_type": "NaN",
+        "internal_value": 0,
+        "internal_weight": 0,
+        "internal_count": 1000,
+        "left_child": {
+          "leaf_index": 0,
+          "leaf_value": 0.03070842919354316,
+          "leaf_weight": 399,
+          "leaf_count": 399
+        },
+        "right_child": {
+          "split_index": 1,
+          "split_feature": 0,
+          "split_gain": 36.74250030517578,
+          "threshold": 0.5102250691730842,
+          "decision_type": "<=",
+          "default_left": true,
+          "missing_type": "NaN",
+          "internal_value": -0.204906,
+          "internal_weight": 601,
+          "internal_count": 601,
+          "left_child": {
+            "leaf_index": 1,
+            "leaf_value": -0.04439151147520909,
+            "leaf_weight": 315,
+            "leaf_count": 315
+          },
+          "right_child": {
+            "leaf_index": 2,
+            "leaf_value": 0.005117411709368601,
+            "leaf_weight": 286,
+            "leaf_count": 286
+          }
+        }
+      }
+    },
+    {
+      "tree_index": 2,
+      "num_leaves": 3,
+      "num_cat": 0,
+      "shrinkage": 0.1,
+      "tree_structure": {
+        "split_index": 0,
+        "split_feature": 1,
+        "split_gain": 57.1327018737793,
+        "threshold": 0.668665477622446,
+        "decision_type": "<=",
+        "default_left": true,
+        "missing_type": "NaN",
+        "internal_value": 0,
+        "internal_weight": 0,
+        "internal_count": 1000,
+        "left_child": {
+          "split_index": 1,
+          "split_feature": 1,
+          "split_gain": 40.859100341796875,
+          "threshold": 0.008118820676863816,
+          "decision_type": "<=",
+          "default_left": true,
+          "missing_type": "NaN",
+          "internal_value": -0.162926,
+          "internal_weight": 681,
+          "internal_count": 681,
+          "left_child": {
+            "leaf_index": 0,
+            "leaf_value": -0.15361238490967524,
+            "leaf_weight": 21,
+            "leaf_count": 21
+          },
+          "right_child": {
+            "leaf_index": 2,
+            "leaf_value": -0.01192330846157292,
+            "leaf_weight": 660,
+            "leaf_count": 660
+          }
+        },
+        "right_child": {
+          "leaf_index": 1,
+          "leaf_value": 0.03499044894987518,
+          "leaf_weight": 319,
+          "leaf_count": 319
+        }
+      }
+    },
+    {
+      "tree_index": 3,
+      "num_leaves": 3,
+      "num_cat": 1,
+      "shrinkage": 0.1,
+      "tree_structure": {
+        "split_index": 0,
+        "split_feature": 0,
+        "split_gain": 54.77090072631836,
+        "threshold": 0.5201391072644542,
+        "decision_type": "<=",
+        "default_left": true,
+        "missing_type": "NaN",
+        "internal_value": 0,
+        "internal_weight": 0,
+        "internal_count": 1000,
+        "left_child": {
+          "leaf_index": 0,
+          "leaf_value": -0.02141000620783247,
+          "leaf_weight": 543,
+          "leaf_count": 543
+        },
+        "right_child": {
+          "split_index": 1,
+          "split_feature": 2,
+          "split_gain": 27.200700759887695,
+          "threshold": "0||1",
+          "decision_type": "==",
+          "default_left": false,
+          "missing_type": "NaN",
+          "internal_value": 0.255704,
+          "internal_weight": 457,
+          "internal_count": 457,
+          "left_child": {
+            "leaf_index": 1,
+            "leaf_value": -0.004121485787596721,
+            "leaf_weight": 191,
+            "leaf_count": 191
+          },
+          "right_child": {
+            "leaf_index": 2,
+            "leaf_value": 0.04534090904886873,
+            "leaf_weight": 266,
+            "leaf_count": 266
+          }
+        }
+      }
+    },
+    {
+      "tree_index": 4,
+      "num_leaves": 3,
+      "num_cat": 1,
+      "shrinkage": 0.1,
+      "tree_structure": {
+        "split_index": 0,
+        "split_feature": 3,
+        "split_gain": 51.84349822998047,
+        "threshold": "2||3||4",
+        "decision_type": "==",
+        "default_left": false,
+        "missing_type": "NaN",
+        "internal_value": 0,
+        "internal_weight": 0,
+        "internal_count": 1000,
+        "left_child": {
+          "split_index": 1,
+          "split_feature": 1,
+          "split_gain": 39.352699279785156,
+          "threshold": 0.27283279016959255,
+          "decision_type": "<=",
+          "default_left": true,
+          "missing_type": "NaN",
+          "internal_value": 0.188414,
+          "internal_weight": 593,
+          "internal_count": 593,
+          "left_child": {
+            "leaf_index": 0,
+            "leaf_value": -0.01924803254356527,
+            "leaf_weight": 184,
+            "leaf_count": 184
+          },
+          "right_child": {
+            "leaf_index": 2,
+            "leaf_value": 0.03643772842347651,
+            "leaf_weight": 409,
+            "leaf_count": 409
+          }
+        },
+        "right_child": {
+          "leaf_index": 1,
+          "leaf_value": -0.02701711918923075,
+          "leaf_weight": 407,
+          "leaf_count": 407
+        }
+      }
+    }
+  ],
+  "pandas_categorical": [
+    [
+      "a",
+      "b",
+      "c",
+      "d",
+      "e"
+    ],
+    [
+      "i",
+      "j",
+      "k",
+      "l",
+      "m"
+    ]
+  ]
+}
\ No newline at end of file
diff --git a/config-model/src/test/java/com/yahoo/config/model/ApplicationDeployTest.java b/config-model/src/test/java/com/yahoo/config/model/ApplicationDeployTest.java
index cb1577417b4..fe82f2406f2 100644
--- a/config-model/src/test/java/com/yahoo/config/model/ApplicationDeployTest.java
+++ b/config-model/src/test/java/com/yahoo/config/model/ApplicationDeployTest.java
@@ -20,7 +20,7 @@ import com.yahoo.searchdefinition.DocumentOnlySearch;
 import com.yahoo.vespa.config.ConfigDefinition;
 import com.yahoo.vespa.config.ConfigDefinitionKey;
 import com.yahoo.vespa.model.VespaModel;
-import com.yahoo.vespa.model.search.SearchDefinition;
+import com.yahoo.vespa.model.search.NamedSchema;
 import org.junit.After;
 import org.junit.Rule;
 import org.junit.Test;
@@ -31,7 +31,6 @@ import java.io.File;
 import java.io.IOException;
 import java.io.Reader;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -60,9 +59,9 @@ public class ApplicationDeployTest {
     public void testVespaModel() throws SAXException, IOException {
         ApplicationPackageTester tester = ApplicationPackageTester.create(TESTDIR + "app1");
         VespaModel model = new VespaModel(tester.app());
-        List<SearchDefinition> searchDefinitions = tester.getSearchDefinitions();
-        assertEquals(searchDefinitions.size(), 5);
-        for (SearchDefinition searchDefinition : searchDefinitions) {
+        List<NamedSchema> schemas = tester.getSchemas();
+        assertEquals(schemas.size(), 5);
+        for (NamedSchema searchDefinition : schemas) {
             Search s = searchDefinition.getSearch();
             switch (s.getName()) {
                 case "music":
@@ -72,21 +71,18 @@ public class ApplicationDeployTest {
                     break;
                 case "product":
                     assertTrue(s instanceof DocumentOnlySearch);
-                    assertEquals(s.getDocument().getField("title").getDataType(), DataType.STRING);
+                    assertEquals(DataType.STRING, s.getDocument().getField("title").getDataType());
                     break;
                 default:
                     fail();
             }
         }
-        File[] truth = new File[]{new File(TESTSDDIR + "laptop.sd"),
-                new File(TESTSDDIR + "music.sd"),
-                new File(TESTSDDIR + "pc.sd"),
-                new File(TESTSDDIR + "product.sd"),
-                new File(TESTSDDIR + "sock.sd")};
-        Arrays.sort(truth);
-        List<File> appSdFiles = tester.app().getSearchDefinitionFiles();
-        Collections.sort(appSdFiles);
-        assertEquals(appSdFiles, Arrays.asList(truth));
+        assertEquals(Set.of(new File(TESTSDDIR + "laptop.sd"),
+                            new File(TESTSDDIR + "music.sd"),
+                            new File(TESTSDDIR + "pc.sd"),
+                            new File(TESTSDDIR + "product.sd"),
+                            new File(TESTSDDIR + "sock.sd")),
+                     new HashSet<>(tester.app().getSearchDefinitionFiles()));
 
         List<FilesApplicationPackage.Component> components = tester.app().getComponents();
         assertEquals(1, components.size());
@@ -103,7 +99,7 @@ public class ApplicationDeployTest {
 
         // Check that getFilename works
         ArrayList<String> sdFileNames = new ArrayList<>();
-        for (SearchDefinition sd : searchDefinitions)
+        for (NamedSchema sd : schemas)
             sdFileNames.add(sd.getFilename());
         Collections.sort(sdFileNames);
         assertEquals("laptop.sd", sdFileNames.get(0));
@@ -190,11 +186,11 @@ public class ApplicationDeployTest {
         File tmpDir = tmpFolder.getRoot();
         IOUtils.copyDirectory(new File(TESTDIR, "app1"), tmpDir);
         ApplicationPackageTester tester = ApplicationPackageTester.create(tmpDir.getAbsolutePath());
-        assertEquals(5, tester.getSearchDefinitions().size());
-        File sdDir = new File(tmpDir, "searchdefinitions");
+        assertEquals(5, tester.getSchemas().size());
+        File sdDir = new File(tmpDir, "schemas");
         File sd = new File(sdDir, "testfoo.sd");
         IOUtils.writeFile(sd, "search testfoo { document testfoo { field bar type string { } } }", false);
-        assertEquals(6, tester.getSearchDefinitions().size());
+        assertEquals(6, tester.getSchemas().size());
     }
 
     @Test
@@ -300,6 +296,16 @@ public class ApplicationDeployTest {
 
     @Test
     public void testGetJarEntryName() {
+        JarEntry e = new JarEntry("/schemas/foo.sd");
+        assertEquals(ApplicationPackage.getFileName(e), "foo.sd");
+        e = new JarEntry("bar");
+        assertEquals(ApplicationPackage.getFileName(e), "bar");
+        e = new JarEntry("");
+        assertEquals(ApplicationPackage.getFileName(e), "");
+    }
+
+    @Test
+    public void testGetJarEntryNameForLegacyPath() {
         JarEntry e = new JarEntry("/searchdefinitions/foo.sd");
         assertEquals(ApplicationPackage.getFileName(e), "foo.sd");
         e = new JarEntry("bar");
diff --git a/config-model/src/test/java/com/yahoo/config/model/ApplicationPackageTester.java b/config-model/src/test/java/com/yahoo/config/model/ApplicationPackageTester.java
index 87b6efa83d6..8e7d5aadb36 100644
--- a/config-model/src/test/java/com/yahoo/config/model/ApplicationPackageTester.java
+++ b/config-model/src/test/java/com/yahoo/config/model/ApplicationPackageTester.java
@@ -5,7 +5,7 @@ import com.yahoo.component.Version;
 import com.yahoo.config.model.application.provider.ApplicationPackageXmlFilesValidator;
 import com.yahoo.config.model.application.provider.FilesApplicationPackage;
 import com.yahoo.config.model.deploy.DeployState;
-import com.yahoo.vespa.model.search.SearchDefinition;
+import com.yahoo.vespa.model.search.NamedSchema;
 
 import java.io.File;
 import java.io.IOException;
@@ -39,8 +39,8 @@ public class ApplicationPackageTester {
 
     public FilesApplicationPackage app() { return applicationPackage; }
 
-    public List<SearchDefinition> getSearchDefinitions() {
-        return new DeployState.Builder().applicationPackage(app()).build().getSearchDefinitions();
+    public List<NamedSchema> getSchemas() {
+        return new DeployState.Builder().applicationPackage(app()).build().getSchemas();
     }
 
     public static ApplicationPackageTester create(String applicationPackageDir) {
diff --git a/config-model/src/test/java/com/yahoo/config/model/MockModelContext.java b/config-model/src/test/java/com/yahoo/config/model/MockModelContext.java
index 38e438e4d3a..f8ab3cc54c8 100644
--- a/config-model/src/test/java/com/yahoo/config/model/MockModelContext.java
+++ b/config-model/src/test/java/com/yahoo/config/model/MockModelContext.java
@@ -9,6 +9,7 @@ import com.yahoo.config.model.api.ConfigDefinitionRepo;
 import com.yahoo.config.model.api.HostProvisioner;
 import com.yahoo.config.model.api.Model;
 import com.yahoo.config.model.api.ModelContext;
+import com.yahoo.config.model.api.Provisioned;
 import com.yahoo.config.model.application.provider.BaseDeployLogger;
 import com.yahoo.config.model.application.provider.MockFileRegistry;
 import com.yahoo.config.model.application.provider.StaticConfigDefinitionRepo;
@@ -53,6 +54,9 @@ public class MockModelContext implements ModelContext {
     }
 
     @Override
+    public Provisioned provisioned() { return new Provisioned(); }
+
+    @Override
     public DeployLogger deployLogger() {
         return new BaseDeployLogger();
     }
diff --git a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java
index 7b4b650295c..7208d8c5fc1 100644
--- a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java
+++ b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java
@@ -264,7 +264,7 @@ public class ModelProvisioningTest {
             assertEquals("Heap size is lowered with combined clusters",
                          17, physicalMemoryPercentage(model.getContainerClusters().get("container1")));
             assertProvisioned(0, ClusterSpec.Id.from("container1"), ClusterSpec.Type.container, model);
-            assertProvisioned(2, ClusterSpec.Id.from("content1"), ClusterSpec.Type.combined, model);
+            assertProvisioned(2, ClusterSpec.Id.from("content1"), ClusterSpec.Id.from("container1"), ClusterSpec.Type.combined, model);
         }
     }
 
@@ -1205,6 +1205,62 @@ public class ModelProvisioningTest {
     }
 
     @Test
+    public void testRequestingRangesMin() {
+        String services =
+                "<?xml version='1.0' encoding='utf-8' ?>" +
+                "<services>" +
+                "   <container version='1.0' id='container'>" +
+                "      <nodes count='[4, 6]'>" +
+                "         <resources vcpu='[11.5, 13.5]' memory='[10Gb, 100Gb]' disk='[30Gb, 1Tb]'/>" +
+                "      </nodes>" +
+                "   </container>" +
+                "   <content version='1.0' id='foo'>" +
+                "      <documents>" +
+                "        <document type='type1' mode='index'/>" +
+                "      </documents>" +
+                "      <nodes count='[6, 20]' groups='[3,4]'>" +
+                "         <resources vcpu='8' memory='200Gb' disk='1Pb'/>" +
+                "      </nodes>" +
+                "   </content>" +
+                "</services>";
+
+        int totalHosts = 10;
+        VespaModelTester tester = new VespaModelTester();
+        tester.addHosts(new NodeResources(11.5, 10, 30, 0.3), 6);
+        tester.addHosts(new NodeResources(85, 200, 1000_000_000, 0.3), 20);
+        VespaModel model = tester.createModel(services, true);
+        assertEquals(totalHosts, model.getRoot().hostSystem().getHosts().size());
+    }
+
+    @Test
+    public void testRequestingRangesMax() {
+        String services =
+                "<?xml version='1.0' encoding='utf-8' ?>" +
+                "<services>" +
+                "   <container version='1.0' id='container'>" +
+                "      <nodes count='[4, 6]'>" +
+                "         <resources vcpu='[11.5, 13.5]' memory='[10Gb, 100Gb]' disk='[30Gb, 1Tb]'/>" +
+                "      </nodes>" +
+                "   </container>" +
+                "   <content version='1.0' id='foo'>" +
+                "      <documents>" +
+                "        <document type='type1' mode='index'/>" +
+                "      </documents>" +
+                "      <nodes count='[6, 20]' groups='[3,4]'>" +
+                "         <resources vcpu='8' memory='200Gb' disk='1Pb'/>" +
+                "      </nodes>" +
+                "   </content>" +
+                "</services>";
+
+        int totalHosts = 26;
+        VespaModelTester tester = new VespaModelTester();
+        tester.addHosts(new NodeResources(13.5, 100, 1000, 0.3), 6);
+        tester.addHosts(new NodeResources(85, 200, 1000_000_000, 0.3), 20);
+        VespaModel model = tester.createModel(services, true, true);
+        assertEquals(totalHosts, model.getRoot().hostSystem().getHosts().size());
+    }
+
+    @Test
     public void testContainerOnly() {
         String services =
                 "<?xml version='1.0' encoding='utf-8' ?>\n" +
@@ -1308,14 +1364,14 @@ public class ModelProvisioningTest {
                 "  </http>" +
                 "</container>";
         VespaModelTester tester = new VespaModelTester();
-        tester.addHosts(1);
+        tester.addHosts(2);
         VespaModel model = tester.createModel(services, true);
-        assertEquals(1, model.getHosts().size());
+        assertEquals(2, model.getHosts().size());
         assertEquals(1, model.getContainerClusters().size());
+        assertEquals(2, model.getContainerClusters().get("foo").getContainers().size());
     }
 
     @Test
-    @Ignore // TODO: Enable when turning the port check on
     public void testThatStandaloneSyntaxOnHostedVespaRequiresDefaultPort() {
         try {
             String services =
@@ -1375,7 +1431,7 @@ public class ModelProvisioningTest {
     }
 
     @Test
-    public void testNoNodeTagMeans1Node() {
+    public void testNoNodeTagMeansTwoNodes() {
         String services =
                 "<?xml version='1.0' encoding='utf-8' ?>\n" +
                 "<services>" +
@@ -1390,16 +1446,16 @@ public class ModelProvisioningTest {
                 "  </content>" +
                 "</services>";
         VespaModelTester tester = new VespaModelTester();
-        tester.addHosts(1);
+        tester.addHosts(3);
         VespaModel model = tester.createModel(services, true);
-        assertEquals(1, model.getRoot().hostSystem().getHosts().size());
-        assertEquals(1, model.getAdmin().getSlobroks().size());
-        assertEquals(1, model.getContainerClusters().get("foo").getContainers().size());
+        assertEquals(3, model.getRoot().hostSystem().getHosts().size());
+        assertEquals(2, model.getAdmin().getSlobroks().size());
+        assertEquals(2, model.getContainerClusters().get("foo").getContainers().size());
         assertEquals(1, model.getContentClusters().get("bar").getRootGroup().countNodes());
     }
 
     @Test
-    public void testNoNodeTagMeans1NodeNoContent() {
+    public void testNoNodeTagMeansTwoNodesNoContent() {
         String services =
                 "<?xml version='1.0' encoding='utf-8' ?>\n" +
                 "<services>" +
@@ -1409,11 +1465,11 @@ public class ModelProvisioningTest {
                 "  </container>" +
                 "</services>";
         VespaModelTester tester = new VespaModelTester();
-        tester.addHosts(1);
+        tester.addHosts(2);
         VespaModel model = tester.createModel(services, true);
-        assertEquals(1, model.getRoot().hostSystem().getHosts().size());
-        assertEquals(1, model.getAdmin().getSlobroks().size());
-        assertEquals(1, model.getContainerClusters().get("foo").getContainers().size());
+        assertEquals(2, model.getRoot().hostSystem().getHosts().size());
+        assertEquals(2, model.getAdmin().getSlobroks().size());
+        assertEquals(2, model.getContainerClusters().get("foo").getContainers().size());
     }
 
     @Test
@@ -1806,12 +1862,17 @@ public class ModelProvisioningTest {
         assertTrue(logdConfig.logserver().use());
     }
 
-    private static void assertProvisioned(int nodeCount, ClusterSpec.Id id, ClusterSpec.Type type, VespaModel model) {
-        assertEquals("Nodes in cluster " + id + " with type " + type, nodeCount,
+    private static void assertProvisioned(int nodeCount, ClusterSpec.Id id, ClusterSpec.Id combinedId,
+                                          ClusterSpec.Type type, VespaModel model) {
+        assertEquals("Nodes in cluster " + id + " with type " + type + (combinedId != null ? ", combinedId " + combinedId : ""), nodeCount,
                      model.hostSystem().getHosts().stream()
                           .map(h -> h.spec().membership().get().cluster())
-                          .filter(spec -> spec.id().equals(id) && spec.type().equals(type))
+                          .filter(spec -> spec.id().equals(id) && spec.type().equals(type) && spec.combinedId().equals(Optional.ofNullable(combinedId)))
                           .count());
     }
 
+    private static void assertProvisioned(int nodeCount, ClusterSpec.Id id, ClusterSpec.Type type, VespaModel model) {
+        assertProvisioned(nodeCount, id, null, type, model);
+    }
+
 }
diff --git a/config-model/src/test/java/com/yahoo/document/test/SDDocumentTypeTestCase.java b/config-model/src/test/java/com/yahoo/document/test/SDDocumentTypeTestCase.java
index 94602d5201a..127c121197b 100644
--- a/config-model/src/test/java/com/yahoo/document/test/SDDocumentTypeTestCase.java
+++ b/config-model/src/test/java/com/yahoo/document/test/SDDocumentTypeTestCase.java
@@ -4,7 +4,7 @@ package com.yahoo.document.test;
 import com.yahoo.document.DataType;
 import com.yahoo.document.DataTypeName;
 import com.yahoo.documentmodel.VespaDocumentType;
-import com.yahoo.searchdefinition.SearchDefinitionTestCase;
+import com.yahoo.searchdefinition.SchemaTestCase;
 import com.yahoo.searchdefinition.document.SDDocumentType;
 import com.yahoo.searchdefinition.document.SDField;
 import org.junit.Test;
@@ -17,7 +17,7 @@ import static org.junit.Assert.*;
  * @author Thomas Gundersen
  * @author bratseth
  */
-public class SDDocumentTypeTestCase extends SearchDefinitionTestCase {
+public class SDDocumentTypeTestCase extends SchemaTestCase {
 
     // Verify that we can register and retrieve fields.
     @Test
diff --git a/config-model/src/test/java/com/yahoo/document/test/SDFieldTestCase.java b/config-model/src/test/java/com/yahoo/document/test/SDFieldTestCase.java
index 7dcbd92655b..b3109c3c2e4 100644
--- a/config-model/src/test/java/com/yahoo/document/test/SDFieldTestCase.java
+++ b/config-model/src/test/java/com/yahoo/document/test/SDFieldTestCase.java
@@ -2,7 +2,7 @@
 package com.yahoo.document.test;
 
 import com.yahoo.document.DataType;
-import com.yahoo.searchdefinition.SearchDefinitionTestCase;
+import com.yahoo.searchdefinition.SchemaTestCase;
 import com.yahoo.searchdefinition.document.SDDocumentType;
 import org.junit.Test;
 
@@ -11,7 +11,7 @@ import static org.junit.Assert.fail;
 /**
  * @author Thomas Gundersen
  */
-public class SDFieldTestCase extends SearchDefinitionTestCase {
+public class SDFieldTestCase extends SchemaTestCase {
 
     @Test
     public void testIdSettingConflict() {
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/ArraysTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/ArraysTestCase.java
index 846166ae93c..6a40778c9c4 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/ArraysTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/ArraysTestCase.java
@@ -18,7 +18,7 @@ import static org.junit.Assert.assertTrue;
  *
  * @author bratseth
  */
-public class ArraysTestCase extends SearchDefinitionTestCase {
+public class ArraysTestCase extends SchemaTestCase {
 
     @Test
     public void testArrayImporting() throws IOException, ParseException {
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/ArraysWeightedSetsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/ArraysWeightedSetsTestCase.java
index cfd02c22b89..af1e061b7fa 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/ArraysWeightedSetsTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/ArraysWeightedSetsTestCase.java
@@ -16,7 +16,7 @@ import static org.junit.Assert.assertTrue;
  *
  * @author Einar M R Rosenvinge
  */
-public class ArraysWeightedSetsTestCase extends SearchDefinitionTestCase {
+public class ArraysWeightedSetsTestCase extends SchemaTestCase {
     @Test
     public void testArrayWeightedSetsImporting() throws java.io.IOException, com.yahoo.searchdefinition.parser.ParseException {
         Search search = SearchBuilder.buildFromFile("src/test/examples/arraysweightedsets.sd");
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/AttributeSettingsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/AttributeSettingsTestCase.java
index 084cbcfdfc0..83cad4cf266 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/AttributeSettingsTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/AttributeSettingsTestCase.java
@@ -25,7 +25,7 @@ import static org.junit.Assert.*;
  *
  * @author  bratseth
  */
-public class AttributeSettingsTestCase extends SearchDefinitionTestCase {
+public class AttributeSettingsTestCase extends SchemaTestCase {
 
     @Rule
     public final ExpectedException exceptionRule = ExpectedException.none();
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/AttributeUtils.java b/config-model/src/test/java/com/yahoo/searchdefinition/AttributeUtils.java
new file mode 100644
index 00000000000..2c13427760f
--- /dev/null
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/AttributeUtils.java
@@ -0,0 +1,15 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition;
+
+import com.yahoo.searchdefinition.document.SDField;
+
+/**
+ * Convenience class for tests that need to set attribute properties on fields.
+ */
+public class AttributeUtils {
+
+    public static void addAttributeAspect(SDField field) {
+        field.parseIndexingScript("{ attribute }");
+    }
+
+}
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/CommentTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/CommentTestCase.java
index 8ccb1ed969a..3bb464c5fa5 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/CommentTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/CommentTestCase.java
@@ -14,7 +14,7 @@ import static org.junit.Assert.assertEquals;
  *
  * @author bratseth
  */
-public class CommentTestCase extends SearchDefinitionTestCase {
+public class CommentTestCase extends SchemaTestCase {
 
     @Test
     public void testComments() throws IOException, ParseException {
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/DocumentReferenceResolverTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/DocumentReferenceResolverTest.java
index 8378ec811a5..e46208c770d 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/DocumentReferenceResolverTest.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/DocumentReferenceResolverTest.java
@@ -36,7 +36,7 @@ public class DocumentReferenceResolverTest {
         // Create foo document with document reference to bar and add another field
         SDField fooRefToBarField = new SDField
                 ("bar_ref", ReferenceDataType.createWithInferredId(barDocument.getDocumentType()));
-        addAttributeAspect(fooRefToBarField);
+        AttributeUtils.addAttributeAspect(fooRefToBarField);
         SDField irrelevantField = new SDField("irrelevant_stuff", DataType.INT);
         Search fooSearch = new Search();
         SDDocumentType fooDocument = new SDDocumentType("foo", fooSearch);
@@ -59,7 +59,7 @@ public class DocumentReferenceResolverTest {
         // Create foo document with document reference to non-existing document bar
         SDField fooRefToBarField = new SDField(
                 "bar_ref", ReferenceDataType.createWithInferredId(TemporaryStructuredDataType.create("bar")));
-        addAttributeAspect(fooRefToBarField);
+        AttributeUtils.addAttributeAspect(fooRefToBarField);
         Search fooSearch = new Search();
         SDDocumentType fooDocument = new SDDocumentType("foo", fooSearch);
         fooDocument.addField(fooRefToBarField);
@@ -95,8 +95,4 @@ public class DocumentReferenceResolverTest {
         resolver.resolveReferences(fooDocument);
     }
 
-    private static void addAttributeAspect(SDField fooRefToBarField) {
-        fooRefToBarField.parseIndexingScript("{ attribute }");
-    }
-
 }
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/FieldOfTypeDocumentTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/FieldOfTypeDocumentTestCase.java
index be3bae05c5b..1c7b3e19663 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/FieldOfTypeDocumentTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/FieldOfTypeDocumentTestCase.java
@@ -19,7 +19,7 @@ import static org.junit.Assert.assertSame;
 /**
  * @author Einar M R Rosenvinge
  */
-public class FieldOfTypeDocumentTestCase extends SearchDefinitionTestCase {
+public class FieldOfTypeDocumentTestCase extends SchemaTestCase {
 
     @Test
     public void testDocument() throws IOException {
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/ImportedFieldsEnumeratorTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/ImportedFieldsEnumeratorTest.java
new file mode 100644
index 00000000000..fcbb89b5c42
--- /dev/null
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/ImportedFieldsEnumeratorTest.java
@@ -0,0 +1,66 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition;
+
+import com.yahoo.document.DataType;
+import com.yahoo.document.ReferenceDataType;
+import com.yahoo.searchdefinition.document.SDDocumentType;
+import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.searchdefinition.document.TemporaryImportedField;
+import org.junit.Test;
+
+import java.util.HashSet;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
+public class ImportedFieldsEnumeratorTest {
+
+    @Test
+    public void imported_fields_are_enumerated_and_copied_from_correct_search_instance() {
+        Search parentSearch = new Search();
+        SDDocumentType parentDocument = new SDDocumentType("parent", parentSearch);
+        var parentField = new SDField("their_field", DataType.INT);
+        AttributeUtils.addAttributeAspect(parentField);
+        parentDocument.addField(parentField);
+        parentSearch.addDocument(parentDocument);
+
+        Search fooSearch = new Search();
+        SDField fooRefToParent = new SDField(
+                "foo_ref", ReferenceDataType.createWithInferredId(parentDocument.getDocumentType()));
+        AttributeUtils.addAttributeAspect(fooRefToParent);
+        var fooImports = fooSearch.temporaryImportedFields().get();
+        fooImports.add(new TemporaryImportedField("my_first_import", "foo_ref", "their_field"));
+        fooImports.add(new TemporaryImportedField("my_second_import", "foo_ref", "their_field"));
+        SDDocumentType fooDocument = new SDDocumentType("foo", fooSearch);
+        fooSearch.addDocument(fooDocument);
+
+        Search barSearch = new Search();
+        SDField barRefToParent = new SDField(
+                "bar_ref", ReferenceDataType.createWithInferredId(parentDocument.getDocumentType()));
+        AttributeUtils.addAttributeAspect(barRefToParent);
+        var barImports = barSearch.temporaryImportedFields().get();
+        barImports.add(new TemporaryImportedField("my_cool_import", "my_ref", "their_field"));
+        SDDocumentType barDocument = new SDDocumentType("bar", barSearch);
+        barSearch.addDocument(barDocument);
+
+        var enumerator = new ImportedFieldsEnumerator(List.of(parentSearch, fooSearch, barSearch));
+
+        enumerator.enumerateImportedFields(parentDocument);
+        assertImportedFieldsAre(parentDocument, List.of()); // No imported fields in parent
+
+        enumerator.enumerateImportedFields(fooDocument);
+        assertImportedFieldsAre(fooDocument, List.of("my_first_import", "my_second_import"));
+
+        enumerator.enumerateImportedFields(barDocument);
+        assertImportedFieldsAre(barDocument, List.of("my_cool_import"));
+    }
+
+    private void assertImportedFieldsAre(SDDocumentType documentType, List<String> expectedNames) {
+        assertNotNull(documentType.getTemporaryImportedFields());
+        var actualNames = documentType.getTemporaryImportedFields().fields().keySet();
+        var expectedNameSet = new HashSet<>(expectedNames);
+        assertEquals(expectedNameSet, actualNames);
+    }
+
+}
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectRankingExpressionFileRefTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectRankingExpressionFileRefTestCase.java
index 519828497fe..4453f327bb4 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectRankingExpressionFileRefTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectRankingExpressionFileRefTestCase.java
@@ -18,7 +18,7 @@ import static org.junit.Assert.fail;
 /**
  * @author bratseth
  */
-public class IncorrectRankingExpressionFileRefTestCase extends SearchDefinitionTestCase {
+public class IncorrectRankingExpressionFileRefTestCase extends SchemaTestCase {
 
     @Test
     public void testIncorrectRef() throws IOException, ParseException {
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectSummaryTypesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectSummaryTypesTestCase.java
index c145c0e5634..91ab5e2b5df 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectSummaryTypesTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectSummaryTypesTestCase.java
@@ -11,7 +11,7 @@ import static org.junit.Assert.fail;
  *
  * @author bratseth
  */
-public class IncorrectSummaryTypesTestCase extends SearchDefinitionTestCase {
+public class IncorrectSummaryTypesTestCase extends SchemaTestCase {
     @Test
     public void testImportingIncorrect() throws ParseException {
         try {
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/IndexSettingsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/IndexSettingsTestCase.java
index 2cfb542d06b..f992d5ee0ba 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/IndexSettingsTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/IndexSettingsTestCase.java
@@ -15,7 +15,7 @@ import static org.junit.Assert.assertEquals;
  *
  * @author bratseth
  */
-public class IndexSettingsTestCase extends SearchDefinitionTestCase {
+public class IndexSettingsTestCase extends SchemaTestCase {
 
     @Test
     public void testStemmingSettings() throws IOException, ParseException {
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/IndexingParsingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/IndexingParsingTestCase.java
index 21ba3a5e80a..70119ad42f9 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/IndexingParsingTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/IndexingParsingTestCase.java
@@ -11,7 +11,7 @@ import static org.junit.Assert.assertNotNull;
  *
  * @author frodelu
  */
-public class IndexingParsingTestCase extends SearchDefinitionTestCase {
+public class IndexingParsingTestCase extends SchemaTestCase {
 
     @Test
     public void requireThatIndexingExpressionsCanBeParsed() throws Exception {
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/MultipleSummariesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/MultipleSummariesTestCase.java
index df9f5778614..5721dbf06e8 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/MultipleSummariesTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/MultipleSummariesTestCase.java
@@ -11,7 +11,7 @@ import java.io.IOException;
  *
  * @author bratseth
  */
-public class MultipleSummariesTestCase extends SearchDefinitionTestCase {
+public class MultipleSummariesTestCase extends SchemaTestCase {
     @Test
     public void testArrayImporting() throws IOException, ParseException {
         SearchBuilder.buildFromFile("src/test/examples/multiplesummaries.sd");
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/NameFieldCheckTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/NameFieldCheckTestCase.java
index d2360453976..47b6905c677 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/NameFieldCheckTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/NameFieldCheckTestCase.java
@@ -14,7 +14,7 @@ import static org.junit.Assert.fail;
  *
  * @author Lars Christian Jensen
  */
-public class NameFieldCheckTestCase extends SearchDefinitionTestCase {
+public class NameFieldCheckTestCase extends SchemaTestCase {
 
     @Test
     public void testNameField() {
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/OutsideTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/OutsideTestCase.java
index 5ac37bf0a3a..64527e7f7a5 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/OutsideTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/OutsideTestCase.java
@@ -13,7 +13,7 @@ import static org.junit.Assert.assertTrue;
  *
  * @author  bratseth
  */
-public class OutsideTestCase extends SearchDefinitionTestCase {
+public class OutsideTestCase extends SchemaTestCase {
 
     @Test
     public void testOutsideIndex() throws IOException, ParseException {
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileTestCase.java
index 5ba508e3ef3..e2f2c1fd407 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileTestCase.java
@@ -34,7 +34,7 @@ import static org.junit.Assert.assertTrue;
  *
  * @author bratseth
  */
-public class RankProfileTestCase extends SearchDefinitionTestCase {
+public class RankProfileTestCase extends SchemaTestCase {
 
     @Test
     public void testRankProfileInheritance() {
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankPropertiesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankPropertiesTestCase.java
index f4666f7fb3b..3fe2861de0c 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/RankPropertiesTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankPropertiesTestCase.java
@@ -13,7 +13,7 @@ import static org.junit.Assert.assertEquals;
 /**
  * @author bratseth
  */
-public class RankPropertiesTestCase extends SearchDefinitionTestCase {
+public class RankPropertiesTestCase extends SchemaTestCase {
 
     @Test
     public void testRankPropertyInheritance() throws ParseException {
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionConstantsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionConstantsTestCase.java
index 51508414205..d84d967a184 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionConstantsTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionConstantsTestCase.java
@@ -17,7 +17,7 @@ import static org.junit.Assert.*;
 /**
  * @author bratseth
  */
-public class RankingExpressionConstantsTestCase extends SearchDefinitionTestCase {
+public class RankingExpressionConstantsTestCase extends SchemaTestCase {
 
     @Test
     public void testConstants() throws ParseException {
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionInliningTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionInliningTestCase.java
index 58e62353e5c..e0679eb5175 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionInliningTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionInliningTestCase.java
@@ -17,7 +17,7 @@ import static org.junit.Assert.assertTrue;
 /**
  * @author bratseth
  */
-public class RankingExpressionInliningTestCase extends SearchDefinitionTestCase {
+public class RankingExpressionInliningTestCase extends SchemaTestCase {
 
     @Test
     public void testFunctionInliningPreserveArithmeticOrdering() throws ParseException {
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java
index 3d842be129f..5c1134f928c 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java
@@ -19,7 +19,7 @@ import static org.junit.Assert.assertEquals;
 /**
  * @author lesters
  */
-public class RankingExpressionShadowingTestCase extends SearchDefinitionTestCase {
+public class RankingExpressionShadowingTestCase extends SchemaTestCase {
 
     @Test
     public void testBasicFunctionShadowing() throws ParseException {
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionValidationTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionValidationTestCase.java
index c5027af2a0c..c1fe5e42dfa 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionValidationTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionValidationTestCase.java
@@ -15,7 +15,7 @@ import static org.junit.Assert.fail;
 /**
  * @author bratseth
  */
-public class RankingExpressionValidationTestCase extends SearchDefinitionTestCase {
+public class RankingExpressionValidationTestCase extends SchemaTestCase {
 
     @Test
     public void testInvalidExpressionProducesException() throws ParseException {
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/ReservedWordsAsFieldNamesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/ReservedWordsAsFieldNamesTestCase.java
index 5a5fc1cc312..1a939d71937 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/ReservedWordsAsFieldNamesTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/ReservedWordsAsFieldNamesTestCase.java
@@ -11,7 +11,7 @@ import static org.junit.Assert.assertNotNull;
 /**
  * @author bratseth
  */
-public class ReservedWordsAsFieldNamesTestCase extends SearchDefinitionTestCase {
+public class ReservedWordsAsFieldNamesTestCase extends SchemaTestCase {
 
     @Test
     public void testIt() throws IOException, ParseException {
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/SearchDefinitionsParsingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/SchemaParsingTestCase.java
index fd4bb393c49..0ae39b7f8b6 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/SearchDefinitionsParsingTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/SchemaParsingTestCase.java
@@ -15,7 +15,7 @@ import static org.junit.Assert.*;
  *
  * @author hmusum
  */
-public class SearchDefinitionsParsingTestCase extends SearchDefinitionTestCase {
+public class SchemaParsingTestCase extends SchemaTestCase {
 
     @Test
     public void requireThatIndexingExpressionsCanBeParsed() throws Exception {
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/SearchDefinitionTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/SchemaTestCase.java
index ba6da8792fa..7f3ea7d14bc 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/SearchDefinitionTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/SchemaTestCase.java
@@ -10,7 +10,7 @@ import java.io.IOException;
 import static helpers.CompareConfigTestHelper.assertSerializedConfigEquals;
 import static helpers.CompareConfigTestHelper.assertSerializedConfigFileEquals;
 
-public abstract class SearchDefinitionTestCase {
+public abstract class SchemaTestCase {
 
     protected static void assertConfigFile(String filename, String cfg) throws IOException {
         assertSerializedConfigFileEquals(filename, cfg);
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/SearchImporterTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/SearchImporterTestCase.java
index 66ff1877994..018703153ac 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/SearchImporterTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/SearchImporterTestCase.java
@@ -3,7 +3,6 @@ package com.yahoo.searchdefinition;
 
 import com.yahoo.config.model.application.provider.BaseDeployLogger;
 import com.yahoo.document.DataType;
-import com.yahoo.document.Document;
 import com.yahoo.search.query.profile.QueryProfileRegistry;
 import com.yahoo.searchdefinition.document.Attribute;
 import com.yahoo.searchdefinition.document.RankType;
@@ -31,7 +30,7 @@ import static org.junit.Assert.fail;
  *
  * @author bratseth
  */
-public class SearchImporterTestCase extends SearchDefinitionTestCase {
+public class SearchImporterTestCase extends SchemaTestCase {
 
     @Test
     @SuppressWarnings("deprecation")
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/StemmingSettingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/StemmingSettingTestCase.java
index 9b27d338ced..e5b8ec85d75 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/StemmingSettingTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/StemmingSettingTestCase.java
@@ -16,7 +16,7 @@ import static org.junit.Assert.assertNull;
  *
  * @author bratseth
  */
-public class StemmingSettingTestCase extends SearchDefinitionTestCase {
+public class StemmingSettingTestCase extends SchemaTestCase {
 
     @Test
     public void testStemmingSettings() throws IOException, ParseException {
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/StructTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/StructTestCase.java
index 001ad64e2da..77df5b391dc 100755
--- a/config-model/src/test/java/com/yahoo/searchdefinition/StructTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/StructTestCase.java
@@ -14,7 +14,7 @@ import static org.junit.Assert.fail;
  *
  * @author bratseth
  */
-public class StructTestCase extends SearchDefinitionTestCase {
+public class StructTestCase extends SchemaTestCase {
 
     @Test
     public void testStruct() throws IOException {
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AbstractExportingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AbstractExportingTestCase.java
index d67df3a5239..a345cabe909 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AbstractExportingTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AbstractExportingTestCase.java
@@ -1,11 +1,13 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.searchdefinition.derived;
 
+import com.yahoo.config.model.application.provider.BaseDeployLogger;
+import com.yahoo.config.model.deploy.TestProperties;
 import com.yahoo.document.DocumenttypesConfig;
 import com.yahoo.document.config.DocumentmanagerConfig;
 import com.yahoo.searchdefinition.Search;
 import com.yahoo.searchdefinition.SearchBuilder;
-import com.yahoo.searchdefinition.SearchDefinitionTestCase;
+import com.yahoo.searchdefinition.SchemaTestCase;
 import com.yahoo.searchdefinition.parser.ParseException;
 import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels;
 import com.yahoo.vespa.configmodel.producers.DocumentManager;
@@ -19,22 +21,27 @@ import java.io.IOException;
  *
  * @author bratseth
  */
-public abstract class AbstractExportingTestCase extends SearchDefinitionTestCase {
+public abstract class AbstractExportingTestCase extends SchemaTestCase {
 
     private static final String tempDir = "temp/";
     private static final String searchDefRoot = "src/test/derived/";
 
-    private DerivedConfiguration derive(String dirName, String searchDefinitionName) throws IOException, ParseException {
+    private DerivedConfiguration derive(String dirName, String searchDefinitionName, TestProperties properties) throws IOException, ParseException {
         File toDir = new File(tempDir + dirName);
         toDir.mkdirs();
         deleteContent(toDir);
 
         SearchBuilder builder = SearchBuilder.createFromDirectory(searchDefRoot + dirName + "/");
-        return derive(dirName, searchDefinitionName, builder);
+        return derive(dirName, searchDefinitionName, properties, builder);
     }
 
-    private DerivedConfiguration derive(String dirName, String searchDefinitionName, SearchBuilder builder) throws IOException {
+    private DerivedConfiguration derive(String dirName,
+                                        String searchDefinitionName,
+                                        TestProperties properties,
+                                        SearchBuilder builder) throws IOException {
         DerivedConfiguration config = new DerivedConfiguration(builder.getSearch(searchDefinitionName),
+                                                               new BaseDeployLogger(),
+                                                               properties,
                                                                builder.getRankProfileRegistry(),
                                                                builder.getQueryProfileRegistry(),
                                                                new ImportedMlModels());
@@ -53,6 +60,7 @@ public abstract class AbstractExportingTestCase extends SearchDefinitionTestCase
         String path = exportConfig(name, config);
         DerivedConfiguration.exportDocuments(new DocumentManager().produce(builder.getModel(), new DocumentmanagerConfig.Builder()), path);
         DerivedConfiguration.exportDocuments(new DocumentTypes().produce(builder.getModel(), new DocumenttypesConfig.Builder()), path);
+        DerivedConfiguration.exportQueryProfiles(builder.getQueryProfileRegistry(), path);
         return config;
     }
 
@@ -76,7 +84,11 @@ public abstract class AbstractExportingTestCase extends SearchDefinitionTestCase
     }
 
     protected DerivedConfiguration assertCorrectDeriving(String dirName, String searchDefinitionName) throws IOException, ParseException {
-        DerivedConfiguration derived = derive(dirName, searchDefinitionName);
+        return assertCorrectDeriving(dirName, searchDefinitionName, new TestProperties());
+    }
+
+    protected DerivedConfiguration assertCorrectDeriving(String dirName, String searchDefinitionName, TestProperties properties) throws IOException, ParseException {
+        DerivedConfiguration derived = derive(dirName, searchDefinitionName, properties);
         assertCorrectConfigFiles(dirName);
         return derived;
     }
@@ -87,7 +99,7 @@ public abstract class AbstractExportingTestCase extends SearchDefinitionTestCase
      */
     protected DerivedConfiguration assertCorrectDeriving(SearchBuilder builder, String dirName) throws IOException {
         builder.build();
-        DerivedConfiguration derived = derive(dirName, null, builder);
+        DerivedConfiguration derived = derive(dirName, null, new TestProperties(), builder);
         assertCorrectConfigFiles(dirName);
         return derived;
     }
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributeListTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributeListTestCase.java
index 79ec3027c20..80a92a5b5ec 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributeListTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributeListTestCase.java
@@ -3,7 +3,7 @@ package com.yahoo.searchdefinition.derived;
 
 import com.yahoo.searchdefinition.Search;
 import com.yahoo.searchdefinition.SearchBuilder;
-import com.yahoo.searchdefinition.SearchDefinitionTestCase;
+import com.yahoo.searchdefinition.SchemaTestCase;
 import com.yahoo.searchdefinition.document.Attribute;
 import com.yahoo.searchdefinition.parser.ParseException;
 import org.junit.Test;
@@ -20,7 +20,7 @@ import static org.junit.Assert.assertFalse;
  *
  * @author bratseth
  */
-public class AttributeListTestCase extends SearchDefinitionTestCase {
+public class AttributeListTestCase extends SchemaTestCase {
 
     @Test
     public void testDeriving() throws IOException, ParseException {
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/CasingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/CasingTestCase.java
index b47a268d95a..07762fc6937 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/CasingTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/CasingTestCase.java
@@ -3,7 +3,7 @@ package com.yahoo.searchdefinition.derived;
 
 import com.yahoo.searchdefinition.Search;
 import com.yahoo.searchdefinition.SearchBuilder;
-import com.yahoo.searchdefinition.SearchDefinitionTestCase;
+import com.yahoo.searchdefinition.SchemaTestCase;
 import com.yahoo.searchdefinition.parser.ParseException;
 import org.junit.Test;
 
@@ -16,7 +16,7 @@ import static org.junit.Assert.assertEquals;
  *
  * @author vegardh
  */
-public class CasingTestCase extends SearchDefinitionTestCase {
+public class CasingTestCase extends SchemaTestCase {
 
     @Test
     public void testCasing() throws IOException, ParseException {
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/DeriverTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/DeriverTestCase.java
index 1209da6d64b..8b09a4efd57 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/DeriverTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/DeriverTestCase.java
@@ -4,7 +4,7 @@ package com.yahoo.searchdefinition.derived;
 import com.yahoo.document.DataType;
 import com.yahoo.document.DocumentTypeManager;
 import com.yahoo.document.config.DocumentmanagerConfig;
-import com.yahoo.searchdefinition.SearchDefinitionTestCase;
+import com.yahoo.searchdefinition.SchemaTestCase;
 import org.junit.Test;
 
 import java.util.List;
@@ -16,7 +16,7 @@ import static org.junit.Assert.assertEquals;
  *
  * @author bratseth
  */
-public class DeriverTestCase extends SearchDefinitionTestCase {
+public class DeriverTestCase extends SchemaTestCase {
 
     @Test
     public void testDeriveDocManager() {
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/EmptyRankProfileTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/EmptyRankProfileTestCase.java
index aaac1631722..47862a2611b 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/EmptyRankProfileTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/EmptyRankProfileTestCase.java
@@ -6,7 +6,7 @@ import com.yahoo.search.query.profile.QueryProfileRegistry;
 import com.yahoo.searchdefinition.RankProfileRegistry;
 import com.yahoo.searchdefinition.Search;
 import com.yahoo.searchdefinition.SearchBuilder;
-import com.yahoo.searchdefinition.SearchDefinitionTestCase;
+import com.yahoo.searchdefinition.SchemaTestCase;
 import com.yahoo.searchdefinition.document.SDDocumentType;
 import com.yahoo.searchdefinition.document.SDField;
 import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels;
@@ -17,7 +17,7 @@ import org.junit.Test;
  *
  * @author bratseth
  */
-public class EmptyRankProfileTestCase extends SearchDefinitionTestCase {
+public class EmptyRankProfileTestCase extends SchemaTestCase {
 
     @Test
     public void testDeriving() {
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java
index 61065cd4bcc..3c55aa808b5 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java
@@ -1,6 +1,7 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.searchdefinition.derived;
 
+import com.yahoo.config.model.deploy.TestProperties;
 import com.yahoo.searchdefinition.SearchBuilder;
 import com.yahoo.searchdefinition.parser.ParseException;
 import org.junit.Test;
@@ -150,4 +151,9 @@ public class ExportingTestCase extends AbstractExportingTestCase {
         assertCorrectConfigFiles("tensor2");
     }
 
+    @Test
+    public void testHnswIndex() throws IOException, ParseException {
+        assertCorrectDeriving("hnsw_index");
+    }
+
 }
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/NativeRankTypeDefinitionsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/NativeRankTypeDefinitionsTestCase.java
index c8ba5168e1c..69c247b94d4 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/NativeRankTypeDefinitionsTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/NativeRankTypeDefinitionsTestCase.java
@@ -1,7 +1,7 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.searchdefinition.derived;
 
-import com.yahoo.searchdefinition.SearchDefinitionTestCase;
+import com.yahoo.searchdefinition.SchemaTestCase;
 import com.yahoo.searchdefinition.document.RankType;
 import org.junit.Test;
 
@@ -14,7 +14,7 @@ import static org.junit.Assert.*;
  *
  * @author geirst
  */
-public class NativeRankTypeDefinitionsTestCase extends SearchDefinitionTestCase {
+public class NativeRankTypeDefinitionsTestCase extends SchemaTestCase {
     @Test
     public void testTables() {
         assertEquals(NativeTable.Type.FIRST_OCCURRENCE.getName(), "firstOccurrenceTable");
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/NearestNeighborTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/NearestNeighborTestCase.java
new file mode 100644
index 00000000000..9f57b22fd58
--- /dev/null
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/NearestNeighborTestCase.java
@@ -0,0 +1,40 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.derived;
+
+import com.yahoo.component.ComponentId;
+import com.yahoo.prelude.query.QueryException;
+import com.yahoo.search.Query;
+import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry;
+import com.yahoo.search.query.profile.config.QueryProfileConfigurer;
+import com.yahoo.searchdefinition.parser.ParseException;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+import org.junit.Test;
+
+import java.io.IOException;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+public class NearestNeighborTestCase extends AbstractExportingTestCase {
+
+    @Test
+    public void testNearestNeighbor() throws IOException, ParseException {
+        try {
+            ComponentId.resetGlobalCountersForTests();
+            DerivedConfiguration c = assertCorrectDeriving("nearestneighbor");
+
+            CompiledQueryProfileRegistry queryProfiles =
+                    QueryProfileConfigurer.createFromConfig(new QueryProfiles(c.getQueryProfiles(), (level, message) -> {}).getConfig()).compile();
+            Query q = new Query("?ranking.features.query(q_vec)=[1,2,3,4,5,6]", // length is 6, not 5
+                                queryProfiles.getComponent("default"));
+            fail("This should fail when q_vec is parsed as a tensor");
+        } catch (QueryException e) {
+            // success
+            assertEquals("Invalid request parameter", e.getMessage());
+        } catch (RuntimeException e) {
+            e.printStackTrace();
+            throw e;
+        }
+    }
+
+}
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/NeuralNetTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/NeuralNetTestCase.java
index b299c7fa299..a6171901d2d 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/NeuralNetTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/NeuralNetTestCase.java
@@ -1,16 +1,34 @@
 // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.searchdefinition.derived;
 
+import com.yahoo.search.Query;
+import com.yahoo.search.query.profile.QueryProfileRegistry;
+import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry;
+import com.yahoo.search.query.profile.config.QueryProfileConfigurer;
 import com.yahoo.searchdefinition.parser.ParseException;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
 import org.junit.Test;
 
 import java.io.IOException;
 
+import com.yahoo.component.ComponentId;
+
+import static org.junit.Assert.assertEquals;
+
 public class NeuralNetTestCase extends AbstractExportingTestCase {
 
     @Test
     public void testNeuralNet() throws IOException, ParseException {
-        assertCorrectDeriving("neuralnet");
+        ComponentId.resetGlobalCountersForTests();
+        DerivedConfiguration c = assertCorrectDeriving("neuralnet");
+
+        // Verify that query profiles end up correct when passed through the same intermediate forms as a full system
+        CompiledQueryProfileRegistry queryProfiles =
+                QueryProfileConfigurer.createFromConfig(new QueryProfiles(c.getQueryProfiles(), (level, message) -> {}).getConfig()).compile();
+        Query q = new Query("?test=foo&ranking.features.query(b_1)=[1,2,3,4,5,6,7,8,9]",
+                            queryProfiles.getComponent("default"));
+        assertEquals("tensor(out[9]):[1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0]",
+                     q.properties().get("ranking.features.query(b_1)").toString());
     }
 
 }
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SearchOrdererTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SearchOrdererTestCase.java
index b770024ebf1..0c677456a87 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SearchOrdererTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SearchOrdererTestCase.java
@@ -6,7 +6,7 @@ import com.yahoo.document.TemporaryStructuredDataType;
 import com.yahoo.searchdefinition.DocumentReference;
 import com.yahoo.searchdefinition.DocumentReferences;
 import com.yahoo.searchdefinition.Search;
-import com.yahoo.searchdefinition.SearchDefinitionTestCase;
+import com.yahoo.searchdefinition.SchemaTestCase;
 import com.yahoo.searchdefinition.document.SDDocumentType;
 import com.yahoo.searchdefinition.document.SDField;
 import com.yahoo.searchdefinition.document.TemporarySDField;
@@ -26,47 +26,47 @@ import static org.junit.Assert.assertEquals;
  * @author bratseth
  * @author bjorncs
  */
-public class SearchOrdererTestCase extends SearchDefinitionTestCase {
+public class SearchOrdererTestCase extends SchemaTestCase {
 
-    private static Map<String, Search> createSearchDefinitions() {
-        Map<String, Search> searchDefinitions = new HashMap<>();
+    private static Map<String, Search> createSchemas() {
+        Map<String, Search> schemas = new HashMap<>();
 
-        Search grandParent = createSearchDefinition("grandParent", searchDefinitions);
+        Search grandParent = createSchema("grandParent", schemas);
 
-        Search mother = createSearchDefinition("mother", searchDefinitions);
+        Search mother = createSchema("mother", schemas);
         inherit(mother, grandParent);
 
-        Search father = createSearchDefinition("father", searchDefinitions);
+        Search father = createSchema("father", schemas);
         inherit(father, grandParent);
         createDocumentReference(father, mother, "wife_ref");
 
-        Search daugther = createSearchDefinition("daughter", searchDefinitions);
+        Search daugther = createSchema("daughter", schemas);
         inherit(daugther, father);
         inherit(daugther, mother);
 
-        Search son = createSearchDefinition("son", searchDefinitions);
+        Search son = createSchema("son", schemas);
         inherit(son, father);
         inherit(son, mother);
 
-        Search product = createSearchDefinition("product", searchDefinitions);
+        Search product = createSchema("product", schemas);
 
-        Search pc = createSearchDefinition("pc", searchDefinitions);
+        Search pc = createSchema("pc", schemas);
         inherit(pc, product);
-        Search pcAccessory = createSearchDefinition("accessory-pc", searchDefinitions);
+        Search pcAccessory = createSchema("accessory-pc", schemas);
         inherit(pcAccessory, product);
         createDocumentReference(pcAccessory, pc, "pc_ref");
 
-        createSearchDefinition("alone", searchDefinitions);
+        createSchema("alone", schemas);
 
-        return searchDefinitions;
+        return schemas;
     }
 
-    private static Search createSearchDefinition(String name, Map<String, Search> searchDefinitions) {
+    private static Search createSchema(String name, Map<String, Search> schemas) {
         Search search = new Search(name, null);
         SDDocumentType document = new SDDocumentType(name);
         document.setDocumentReferences(new DocumentReferences(emptyMap()));
         search.addDocument(document);
-        searchDefinitions.put(search.getName(), search);
+        schemas.put(search.getName(), search);
         return search;
     }
 
@@ -75,13 +75,13 @@ public class SearchOrdererTestCase extends SearchDefinitionTestCase {
     }
 
     private static void assertOrder(List<String> expectedSearchOrder, List<String> inputNames) {
-        Map<String, Search> searchDefinitions = createSearchDefinitions();
-        List<Search> inputSearchDefinitions = inputNames.stream()
-                .map(searchDefinitions::get)
+        Map<String, Search> schemas = createSchemas();
+        List<Search> inputSchemas = inputNames.stream()
+                .map(schemas::get)
                 .map(Objects::requireNonNull)
                 .collect(toList());
         List<String> actualSearchOrder = new SearchOrderer()
-                .order(inputSearchDefinitions)
+                .order(inputSchemas)
                 .stream()
                 .map(Search::getName)
                 .collect(toList());
@@ -104,31 +104,37 @@ public class SearchOrdererTestCase extends SearchDefinitionTestCase {
         assertOrder(Arrays.asList("alone", "grandParent", "mother", "father", "daughter", "product", "pc", "son"),
                     Arrays.asList("grandParent", "mother", "father", "daughter", "son", "product", "pc", "alone"));
     }
+
     @Test
     public void testOneLevelReordering() {
         assertOrder(Arrays.asList("alone", "grandParent", "mother", "father", "daughter", "product", "pc", "son"),
                     Arrays.asList("grandParent", "daughter", "son", "mother", "father", "pc", "product", "alone"));
     }
+
     @Test
     public void testMultiLevelReordering() {
         assertOrder(Arrays.asList("alone", "grandParent", "mother", "father", "daughter", "product", "pc", "son"),
                     Arrays.asList("daughter", "son", "mother", "father", "grandParent", "pc", "product", "alone"));
     }
+
     @Test
     public void testAloneIsKeptInPlaceWithMultiLevelReordering() {
         assertOrder(Arrays.asList("alone", "grandParent", "mother", "father", "daughter", "product", "pc", "son"),
                     Arrays.asList("alone", "daughter", "son", "mother", "father", "grandParent", "pc", "product"));
     }
+
     @Test
     public void testPartialMultiLevelReordering() {
         assertOrder(Arrays.asList("alone", "grandParent", "mother", "father", "daughter", "product", "pc", "son"),
                     Arrays.asList("daughter", "grandParent", "mother", "son", "father", "product", "pc", "alone"));
     }
+
     @Test
     public void testMultilevelReorderingAccrossHierarchies() {
         assertOrder(Arrays.asList("alone", "grandParent", "mother", "father", "daughter", "product", "pc", "son"),
                     Arrays.asList("daughter", "pc", "son", "mother", "grandParent", "father", "product", "alone"));
     }
+
     @Test
     public void referees_are_ordered_before_referrer() {
         assertOrder(Arrays.asList("alone", "grandParent", "mother", "father", "daughter", "product", "pc", "accessory-pc", "son"),
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryMapTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryMapTestCase.java
index f85f9994e04..07d7405b1db 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryMapTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryMapTestCase.java
@@ -25,7 +25,7 @@ import static org.junit.Assert.assertTrue;
  *
  * @author  bratseth
  */
-public class SummaryMapTestCase extends SearchDefinitionTestCase {
+public class SummaryMapTestCase extends SchemaTestCase {
     @Test
     public void testDeriving() throws IOException, ParseException {
         Search search = SearchBuilder.buildFromFile("src/test/examples/simple.sd");
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryTestCase.java
index b82620b1cf5..afbc9f52f6b 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryTestCase.java
@@ -4,7 +4,7 @@ package com.yahoo.searchdefinition.derived;
 import com.yahoo.config.model.application.provider.BaseDeployLogger;
 import com.yahoo.searchdefinition.Search;
 import com.yahoo.searchdefinition.SearchBuilder;
-import com.yahoo.searchdefinition.SearchDefinitionTestCase;
+import com.yahoo.searchdefinition.SchemaTestCase;
 import com.yahoo.searchdefinition.parser.ParseException;
 import org.junit.Test;
 
@@ -20,7 +20,7 @@ import static org.junit.Assert.assertNull;
  *
  * @author bratseth
  */
-public class SummaryTestCase extends SearchDefinitionTestCase {
+public class SummaryTestCase extends SchemaTestCase {
 
     @Test
     public void testDeriving() throws IOException, ParseException {
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/TypeConversionTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/TypeConversionTestCase.java
index 24575df8c91..c03e915aa8b 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/TypeConversionTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/TypeConversionTestCase.java
@@ -6,7 +6,7 @@ import com.yahoo.document.DataType;
 import com.yahoo.search.query.profile.QueryProfileRegistry;
 import com.yahoo.searchdefinition.RankProfileRegistry;
 import com.yahoo.searchdefinition.Search;
-import com.yahoo.searchdefinition.SearchDefinitionTestCase;
+import com.yahoo.searchdefinition.SchemaTestCase;
 import com.yahoo.searchdefinition.document.SDDocumentType;
 import com.yahoo.searchdefinition.document.SDField;
 import com.yahoo.searchdefinition.processing.Processing;
@@ -20,7 +20,7 @@ import static org.junit.Assert.assertFalse;
  *
  * @author bratseth
  */
-public class TypeConversionTestCase extends SearchDefinitionTestCase {
+public class TypeConversionTestCase extends SchemaTestCase {
 
     /** Tests that exact-string stuff is not spilled over to the default index */
     @Test
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/document/HnswIndexParamsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/document/HnswIndexParamsTestCase.java
new file mode 100644
index 00000000000..e3dcc925e5e
--- /dev/null
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/document/HnswIndexParamsTestCase.java
@@ -0,0 +1,45 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+package com.yahoo.searchdefinition.document;
+
+import java.util.Optional;
+import org.junit.Test;
+
+import static org.hamcrest.core.Is.is;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
+
+public class HnswIndexParamsTestCase {
+
+    @Test
+    public void override_from() throws Exception {
+        var empty = new HnswIndexParams();
+        var builder = new HnswIndexParams.Builder();
+        builder.setMaxLinksPerNode(7);
+        var one = builder.build();
+        builder.setNeighborsToExploreAtInsert(42);
+        var three = builder.build();
+        builder.setMaxLinksPerNode(17);
+        builder.setNeighborsToExploreAtInsert(500);
+        var four = builder.build();
+
+        assertThat(empty.maxLinksPerNode(), is(16));
+        assertThat(empty.neighborsToExploreAtInsert(), is(200));
+
+        assertThat(one.maxLinksPerNode(), is(7));
+        assertThat(three.neighborsToExploreAtInsert(), is(42));
+
+        assertThat(four.maxLinksPerNode(), is(17));
+        assertThat(four.neighborsToExploreAtInsert(), is(500));
+
+        var five = four.overrideFrom(Optional.of(empty));
+        assertThat(five.maxLinksPerNode(), is(17));
+        assertThat(five.neighborsToExploreAtInsert(), is(500));
+
+        var six = four.overrideFrom(Optional.of(one));
+        assertThat(six.maxLinksPerNode(), is(7));
+        assertThat(six.neighborsToExploreAtInsert(), is(500));
+    }
+
+}
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/AttributesExactMatchTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/AttributesExactMatchTestCase.java
index 7236ccbc117..35ce4dff730 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/AttributesExactMatchTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/AttributesExactMatchTestCase.java
@@ -3,7 +3,7 @@ package com.yahoo.searchdefinition.processing;
 
 import com.yahoo.searchdefinition.Search;
 import com.yahoo.searchdefinition.SearchBuilder;
-import com.yahoo.searchdefinition.SearchDefinitionTestCase;
+import com.yahoo.searchdefinition.SchemaTestCase;
 import com.yahoo.searchdefinition.document.Matching;
 import com.yahoo.searchdefinition.parser.ParseException;
 import org.junit.Test;
@@ -17,7 +17,7 @@ import static org.junit.Assert.assertFalse;
  * @author vegardh
  *
  */
-public class AttributesExactMatchTestCase extends SearchDefinitionTestCase {
+public class AttributesExactMatchTestCase extends SchemaTestCase {
     @Test
     public void testAttributesExactMatch() throws IOException, ParseException {
         Search search = SearchBuilder.buildFromFile("src/test/examples/attributesexactmatch.sd");
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/BoldingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/BoldingTestCase.java
index ac3ba1d98d9..9a4357c5d65 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/BoldingTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/BoldingTestCase.java
@@ -5,7 +5,7 @@ import com.yahoo.config.model.application.provider.BaseDeployLogger;
 import com.yahoo.searchdefinition.RankProfileRegistry;
 import com.yahoo.searchdefinition.Search;
 import com.yahoo.searchdefinition.SearchBuilder;
-import com.yahoo.searchdefinition.SearchDefinitionTestCase;
+import com.yahoo.searchdefinition.SchemaTestCase;
 import com.yahoo.searchdefinition.parser.ParseException;
 import com.yahoo.vespa.model.container.search.QueryProfiles;
 import org.junit.Test;
@@ -18,7 +18,7 @@ import static org.junit.Assert.fail;
 /**
  * @author Mathias Mølster Lidal
  */
-public class BoldingTestCase extends SearchDefinitionTestCase {
+public class BoldingTestCase extends SchemaTestCase {
 
     @Test
     public void testBoldingNonString() throws IOException, ParseException {
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSearchFieldsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSearchFieldsTestCase.java
index 620cee49ac4..809ccdb3a3a 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSearchFieldsTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSearchFieldsTestCase.java
@@ -6,7 +6,7 @@ import com.yahoo.config.model.deploy.TestProperties;
 import com.yahoo.search.query.profile.QueryProfileRegistry;
 import com.yahoo.searchdefinition.Search;
 import com.yahoo.searchdefinition.SearchBuilder;
-import com.yahoo.searchdefinition.SearchDefinitionTestCase;
+import com.yahoo.searchdefinition.SchemaTestCase;
 import com.yahoo.searchdefinition.derived.DerivedConfiguration;
 import com.yahoo.searchdefinition.document.SDDocumentType;
 import com.yahoo.searchdefinition.parser.ParseException;
@@ -18,7 +18,7 @@ import java.io.IOException;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
 
-public class ImplicitSearchFieldsTestCase extends SearchDefinitionTestCase {
+public class ImplicitSearchFieldsTestCase extends SchemaTestCase {
 
     @Test
     public void testRequireThatExtraFieldsAreIncluded() throws IOException, ParseException {
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitStructTypesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitStructTypesTestCase.java
index f2d81414b5a..c9ea57c5b9b 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitStructTypesTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitStructTypesTestCase.java
@@ -4,7 +4,7 @@ package com.yahoo.searchdefinition.processing;
 import com.yahoo.document.*;
 import com.yahoo.searchdefinition.Search;
 import com.yahoo.searchdefinition.SearchBuilder;
-import com.yahoo.searchdefinition.SearchDefinitionTestCase;
+import com.yahoo.searchdefinition.SchemaTestCase;
 import com.yahoo.searchdefinition.document.SDDocumentType;
 import com.yahoo.searchdefinition.document.SDField;
 import com.yahoo.searchdefinition.parser.ParseException;
@@ -13,7 +13,7 @@ import org.junit.Test;
 import java.io.IOException;
 
 import static org.junit.Assert.*;
-public class ImplicitStructTypesTestCase extends SearchDefinitionTestCase {
+public class ImplicitStructTypesTestCase extends SchemaTestCase {
     @Test
     public void testRequireThatImplicitStructsAreCreated() throws IOException, ParseException {
         Search search = SearchBuilder.buildFromFile("src/test/examples/nextgen/toggleon.sd");
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSummaryFieldsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSummaryFieldsTestCase.java
index 7acbf67772a..ae00e4f3079 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSummaryFieldsTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSummaryFieldsTestCase.java
@@ -3,7 +3,7 @@ package com.yahoo.searchdefinition.processing;
 
 import com.yahoo.searchdefinition.Search;
 import com.yahoo.searchdefinition.SearchBuilder;
-import com.yahoo.searchdefinition.SearchDefinitionTestCase;
+import com.yahoo.searchdefinition.SchemaTestCase;
 import com.yahoo.searchdefinition.parser.ParseException;
 import com.yahoo.vespa.documentmodel.DocumentSummary;
 import org.junit.Test;
@@ -13,7 +13,7 @@ import java.io.IOException;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
 
-public class ImplicitSummaryFieldsTestCase extends SearchDefinitionTestCase {
+public class ImplicitSummaryFieldsTestCase extends SchemaTestCase {
 
     @Test
     public void testRequireThatImplicitFieldsAreCreated() throws IOException, ParseException {
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingScriptRewriterTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingScriptRewriterTestCase.java
index d313c2391fd..7863c544b60 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingScriptRewriterTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingScriptRewriterTestCase.java
@@ -7,7 +7,7 @@ import com.yahoo.searchdefinition.Index;
 import com.yahoo.searchdefinition.RankProfileRegistry;
 import com.yahoo.searchdefinition.Search;
 import com.yahoo.searchdefinition.SearchBuilder;
-import com.yahoo.searchdefinition.SearchDefinitionTestCase;
+import com.yahoo.searchdefinition.SchemaTestCase;
 import com.yahoo.searchdefinition.document.BooleanIndexDefinition;
 import com.yahoo.searchdefinition.document.SDDocumentType;
 import com.yahoo.searchdefinition.document.SDField;
@@ -28,7 +28,7 @@ import static org.junit.Assert.assertEquals;
 /**
  * @author Simon Thoresen Hult
  */
-public class IndexingScriptRewriterTestCase extends SearchDefinitionTestCase {
+public class IndexingScriptRewriterTestCase extends SchemaTestCase {
 
     @Test
     public void testSetLanguageRewriting() {
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/IntegerIndex2AttributeTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/IntegerIndex2AttributeTestCase.java
index cac50354dc2..fcf1c39f5b4 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/IntegerIndex2AttributeTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/IntegerIndex2AttributeTestCase.java
@@ -5,7 +5,7 @@ import com.yahoo.config.model.application.provider.BaseDeployLogger;
 import com.yahoo.searchdefinition.RankProfileRegistry;
 import com.yahoo.searchdefinition.Search;
 import com.yahoo.searchdefinition.SearchBuilder;
-import com.yahoo.searchdefinition.SearchDefinitionTestCase;
+import com.yahoo.searchdefinition.SchemaTestCase;
 import com.yahoo.searchdefinition.document.SDField;
 import com.yahoo.searchdefinition.parser.ParseException;
 import com.yahoo.vespa.model.container.search.QueryProfiles;
@@ -20,7 +20,7 @@ import static org.junit.Assert.assertTrue;
 /**
  * @author baldersheim
  */
-public class IntegerIndex2AttributeTestCase extends SearchDefinitionTestCase {
+public class IntegerIndex2AttributeTestCase extends SchemaTestCase {
 
     @Test
     public void testIntegerIndex2Attribute() throws IOException, ParseException {
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/NGramTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/NGramTestCase.java
index 385d1df90ad..c792d3bf40b 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/NGramTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/NGramTestCase.java
@@ -3,7 +3,7 @@ package com.yahoo.searchdefinition.processing;
 
 import com.yahoo.searchdefinition.Search;
 import com.yahoo.searchdefinition.SearchBuilder;
-import com.yahoo.searchdefinition.SearchDefinitionTestCase;
+import com.yahoo.searchdefinition.SchemaTestCase;
 import com.yahoo.searchdefinition.document.Matching;
 import com.yahoo.searchdefinition.document.SDField;
 import com.yahoo.searchdefinition.document.Stemming;
@@ -21,7 +21,7 @@ import static org.junit.Assert.fail;
 /**
  * @author bratseth
  */
-public class NGramTestCase extends SearchDefinitionTestCase {
+public class NGramTestCase extends SchemaTestCase {
 
     @Test
     public void testNGram() throws IOException, ParseException {
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankModifierTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankModifierTestCase.java
index 0d6334a5223..4ab56f809c9 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankModifierTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankModifierTestCase.java
@@ -3,7 +3,7 @@ package com.yahoo.searchdefinition.processing;
 
 import com.yahoo.searchdefinition.Search;
 import com.yahoo.searchdefinition.SearchBuilder;
-import com.yahoo.searchdefinition.SearchDefinitionTestCase;
+import com.yahoo.searchdefinition.SchemaTestCase;
 import com.yahoo.searchdefinition.parser.ParseException;
 import org.junit.Test;
 
@@ -14,7 +14,7 @@ import java.io.IOException;
  * @author vegardh
  *
  */
-public class RankModifierTestCase extends SearchDefinitionTestCase {
+public class RankModifierTestCase extends SchemaTestCase {
     @Test
     public void testLiteral() throws IOException, ParseException {
         Search search = SearchBuilder.buildFromFile("src/test/examples/rankmodifier/literal.sd");
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java
index 08dd5148b29..0cd6674751e 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java
@@ -15,6 +15,7 @@ import com.yahoo.searchdefinition.parser.ParseException;
 import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels;
 import ai.vespa.rankingexpression.importer.onnx.OnnxImporter;
 import ai.vespa.rankingexpression.importer.tensorflow.TensorFlowImporter;
+import ai.vespa.rankingexpression.importer.lightgbm.LightGBMImporter;
 import ai.vespa.rankingexpression.importer.xgboost.XGBoostImporter;
 
 import java.util.HashMap;
@@ -33,6 +34,7 @@ class RankProfileSearchFixture {
 
     private final ImmutableList<MlModelImporter> importers = ImmutableList.of(new TensorFlowImporter(),
                                                                               new OnnxImporter(),
+                                                                              new LightGBMImporter(),
                                                                               new XGBoostImporter());
     private RankProfileRegistry rankProfileRegistry = new RankProfileRegistry();
     private final QueryProfileRegistry queryProfileRegistry;
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankPropertyVariablesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankPropertyVariablesTestCase.java
index d740884d3e5..502fc4472bc 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankPropertyVariablesTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankPropertyVariablesTestCase.java
@@ -7,7 +7,7 @@ import com.yahoo.searchdefinition.RankProfile.RankProperty;
 import com.yahoo.searchdefinition.RankProfileRegistry;
 import com.yahoo.searchdefinition.Search;
 import com.yahoo.searchdefinition.SearchBuilder;
-import com.yahoo.searchdefinition.SearchDefinitionTestCase;
+import com.yahoo.searchdefinition.SchemaTestCase;
 import com.yahoo.searchdefinition.parser.ParseException;
 import org.junit.Test;
 
@@ -16,7 +16,7 @@ import java.util.List;
 
 import static org.junit.Assert.fail;
 
-public class RankPropertyVariablesTestCase extends SearchDefinitionTestCase {
+public class RankPropertyVariablesTestCase extends SchemaTestCase {
 
     @Test
     public void testRankPropVariables() throws IOException, ParseException {
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolverTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolverTestCase.java
index 9e9ef2589be..a306e0f2c90 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolverTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolverTestCase.java
@@ -275,7 +275,7 @@ public class RankingExpressionTypeResolverTestCase {
         builder.build(true, logger);
         String message = logger.findMessage("The following query features");
         assertNotNull(message);
-        assertEquals("WARNING: The following query features are not declared in query profile types and " +
+        assertEquals("WARNING: The following query features used in 'my_rank_profile' are not declared in query profile types and " +
                      "will be interpreted as scalars, not tensors: [query(bar), query(baz), query(foo)]",
                      message);
     }
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithLightGBMTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithLightGBMTestCase.java
new file mode 100644
index 00000000000..79d19371f1c
--- /dev/null
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithLightGBMTestCase.java
@@ -0,0 +1,88 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.searchdefinition.processing;
+
+import com.yahoo.config.application.api.ApplicationPackage;
+import com.yahoo.io.IOUtils;
+import com.yahoo.path.Path;
+import com.yahoo.searchdefinition.parser.ParseException;
+import org.junit.After;
+import org.junit.Test;
+
+import java.io.IOException;
+
+/**
+ * @author lesters
+ */
+public class RankingExpressionWithLightGBMTestCase {
+
+    private final Path applicationDir = Path.fromString("src/test/integration/lightgbm/");
+
+    private final static String lightGBMExpression =
+            "if (!(numerical_2 >= 0.46643291586559305), 2.1594397038037663, if (categorical_2 in [\"k\", \"l\", \"m\"], 2.235297305276056, 2.1792953471546546)) + if (categorical_1 in [\"d\", \"e\"], 0.03070842919354316, if (!(numerical_1 >= 0.5102250691730842), -0.04439151147520909, 0.005117411709368601)) + if (!(numerical_2 >= 0.668665477622446), if (!(numerical_2 >= 0.008118820676863816), -0.15361238490967524, -0.01192330846157292), 0.03499044894987518) + if (!(numerical_1 >= 0.5201391072644542), -0.02141000620783247, if (categorical_1 in [\"a\", \"b\"], -0.004121485787596721, 0.04534090904886873)) + if (categorical_2 in [\"k\", \"l\", \"m\"], if (!(numerical_2 >= 0.27283279016959255), -0.01924803254356527, 0.03643772842347651), -0.02701711918923075)";
+
+    @After
+    public void removeGeneratedModelFiles() {
+        IOUtils.recursiveDeleteDir(applicationDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile());
+    }
+
+    @Test
+    public void testLightGBMReference() {
+        RankProfileSearchFixture search = fixtureWith("lightgbm('regression.json')");
+        search.assertFirstPhaseExpression(lightGBMExpression, "my_profile");
+    }
+
+    @Test
+    public void testNestedLightGBMReference() {
+        RankProfileSearchFixture search = fixtureWith("5 + sum(lightgbm('regression.json'))");
+        search.assertFirstPhaseExpression("5 + reduce(" + lightGBMExpression + ", sum)", "my_profile");
+    }
+
+    @Test
+    public void testImportingFromStoredExpressions() throws IOException {
+        RankProfileSearchFixture search = fixtureWith("lightgbm('regression.json')");
+        search.assertFirstPhaseExpression(lightGBMExpression, "my_profile");
+
+        // At this point the expression is stored - copy application to another location which do not have a models dir
+        Path storedApplicationDirectory = applicationDir.getParentPath().append("copy");
+        try {
+            storedApplicationDirectory.toFile().mkdirs();
+            IOUtils.copyDirectory(applicationDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile(),
+                                  storedApplicationDirectory.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile());
+            RankingExpressionWithTensorFlowTestCase.StoringApplicationPackage storedApplication = new RankingExpressionWithTensorFlowTestCase.StoringApplicationPackage(storedApplicationDirectory);
+            RankProfileSearchFixture searchFromStored = fixtureWith("lightgbm('regression.json')");
+            searchFromStored.assertFirstPhaseExpression(lightGBMExpression, "my_profile");
+        }
+        finally {
+            IOUtils.recursiveDeleteDir(storedApplicationDirectory.toFile());
+        }
+    }
+
+    private RankProfileSearchFixture fixtureWith(String firstPhaseExpression) {
+        return fixtureWith(firstPhaseExpression, null, null,
+                new RankingExpressionWithTensorFlowTestCase.StoringApplicationPackage(applicationDir));
+    }
+
+    private RankProfileSearchFixture fixtureWith(String firstPhaseExpression,
+                                                 String constant,
+                                                 String field,
+                                                 RankingExpressionWithTensorFlowTestCase.StoringApplicationPackage application) {
+        try {
+            RankProfileSearchFixture fixture = new RankProfileSearchFixture(
+                    application,
+                    application.getQueryProfiles(),
+                    "  rank-profile my_profile {\n" +
+                            "    first-phase {\n" +
+                            "      expression: " + firstPhaseExpression +
+                            "    }\n" +
+                            "  }",
+                    constant,
+                    field);
+            fixture.compileRankProfile("my_profile", applicationDir.append("models"));
+            return fixture;
+        } catch (ParseException e) {
+            throw new IllegalArgumentException(e);
+        }
+    }
+
+}
+
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java
index 96fa59a77cc..b3eda9b7e13 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java
@@ -19,7 +19,7 @@ import java.util.Map;
 
 import static org.junit.Assert.assertEquals;
 
-public class RankingExpressionsTestCase extends SearchDefinitionTestCase {
+public class RankingExpressionsTestCase extends SchemaTestCase {
 
     @Test
     public void testFunctions() throws IOException, ParseException {
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSourceTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSourceTestCase.java
index dbcfc8c202d..ca8744a07bb 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSourceTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSourceTestCase.java
@@ -6,7 +6,7 @@ import com.yahoo.config.model.application.provider.BaseDeployLogger;
 import com.yahoo.searchdefinition.RankProfileRegistry;
 import com.yahoo.searchdefinition.Search;
 import com.yahoo.searchdefinition.SearchBuilder;
-import com.yahoo.searchdefinition.SearchDefinitionTestCase;
+import com.yahoo.searchdefinition.SchemaTestCase;
 import com.yahoo.searchdefinition.parser.ParseException;
 import com.yahoo.vespa.model.container.search.QueryProfiles;
 import org.junit.Test;
@@ -16,7 +16,7 @@ import java.io.IOException;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.fail;
 
-public class SummaryFieldsMustHaveValidSourceTestCase extends SearchDefinitionTestCase {
+public class SummaryFieldsMustHaveValidSourceTestCase extends SchemaTestCase {
 
     @Test
     public void requireThatInvalidSourceIsCaught() throws IOException, ParseException {
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorFieldTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorFieldTestCase.java
index b6569357495..b9702c6c4f7 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorFieldTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorFieldTestCase.java
@@ -1,11 +1,16 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.searchdefinition.processing;
 
-import com.yahoo.searchdefinition.SearchBuilder;
+import com.yahoo.config.model.test.TestUtil;
 import com.yahoo.searchdefinition.parser.ParseException;
 import org.junit.Test;
 
+
+import static com.yahoo.searchdefinition.SearchBuilder.createFromString;
+import static com.yahoo.config.model.test.TestUtil.joinLines;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
 /**
@@ -16,7 +21,7 @@ public class TensorFieldTestCase {
     @Test
     public void requireThatTensorFieldCannotBeOfCollectionType() throws ParseException {
         try {
-            SearchBuilder.createFromString(getSd("field f1 type array<tensor(x{})> {}"));
+            createFromString(getSd("field f1 type array<tensor(x{})> {}"));
             fail("Expected exception");
         }
         catch (IllegalArgumentException e) {
@@ -28,11 +33,12 @@ public class TensorFieldTestCase {
     @Test
     public void requireThatTensorFieldCannotBeIndexField() throws ParseException {
         try {
-            SearchBuilder.createFromString(getSd("field f1 type tensor(x{}) { indexing: index }"));
+            createFromString(getSd("field f1 type tensor(x{}) { indexing: index }"));
             fail("Expected exception");
         }
         catch (IllegalArgumentException e) {
-            assertEquals("For search 'test', field 'f1': A field of type 'tensor' cannot be specified as an 'index' field.",
+            assertEquals("For search 'test', field 'f1': A tensor of type 'tensor(x{})' does not support having an 'index'. " +
+                            "Currently, only tensors with 1 indexed dimension supports that.",
                          e.getMessage());
         }
     }
@@ -40,7 +46,7 @@ public class TensorFieldTestCase {
     @Test
     public void requireThatTensorAttributeCannotBeFastSearch() throws ParseException {
         try {
-            SearchBuilder.createFromString(getSd("field f1 type tensor(x{}) { indexing: attribute \n attribute: fast-search }"));
+            createFromString(getSd("field f1 type tensor(x{}) { indexing: attribute \n attribute: fast-search }"));
             fail("Expected exception");
         }
         catch (IllegalArgumentException e) {
@@ -51,7 +57,7 @@ public class TensorFieldTestCase {
     @Test
     public void requireThatIllegalTensorTypeSpecThrowsException() throws ParseException {
         try {
-            SearchBuilder.createFromString(getSd("field f1 type tensor(invalid) { indexing: attribute }"));
+            createFromString(getSd("field f1 type tensor(invalid) { indexing: attribute }"));
             fail("Expected exception");
         }
         catch (IllegalArgumentException e) {
@@ -59,8 +65,67 @@ public class TensorFieldTestCase {
         }
     }
 
+    @Test
+    public void hnsw_index_is_default_turned_off() throws ParseException {
+        var attr = createFromString(getSd("field t1 type tensor(x[64]) { indexing: attribute }"))
+                .getSearch().getAttribute("t1");
+        assertFalse(attr.hnswIndexParams().isPresent());
+    }
+
+    @Test
+    public void hnsw_index_gets_default_parameters_if_not_specified() throws ParseException {
+        assertHnswIndexParams("", 16, 200);
+        assertHnswIndexParams("index: hnsw", 16, 200);
+    }
+
+    @Test
+    public void hnsw_index_parameters_can_be_specified() throws ParseException {
+        assertHnswIndexParams("index { hnsw { max-links-per-node: 32 } }", 32, 200);
+        assertHnswIndexParams("index { hnsw { neighbors-to-explore-at-insert: 300 } }", 16, 300);
+        assertHnswIndexParams(joinLines("index {",
+                "  hnsw {",
+                "    max-links-per-node: 32",
+                "    neighbors-to-explore-at-insert: 300",
+                "  }",
+                "}"),
+                32, 300);
+    }
+
+    @Test
+    public void tensor_with_hnsw_index_must_be_an_attribute() throws ParseException {
+        try {
+            createFromString(getSd("field t1 type tensor(x[64]) { indexing: index }"));
+            fail("Expected exception");
+        }
+        catch (IllegalArgumentException e) {
+            assertEquals("For search 'test', field 't1': A tensor that has an index must also be an attribute.", e.getMessage());
+        }
+    }
+
     private static String getSd(String field) {
-        return "search test {\n document test {\n" + field + "}\n}\n";
+        return joinLines("search test {",
+                "  document test {",
+                "    " + field,
+                "  }",
+                "}");
+    }
+
+    private void assertHnswIndexParams(String indexSpec, int maxLinksPerNode, int neighborsToExploreAtInsert) throws ParseException {
+        var sd = getSdWithIndexSpec(indexSpec);
+        System.out.println(sd);
+        var search = createFromString(sd).getSearch();
+        var attr = search.getAttribute("t1");
+        var params = attr.hnswIndexParams();
+        assertTrue(params.isPresent());
+        assertEquals(maxLinksPerNode, params.get().maxLinksPerNode());
+        assertEquals(neighborsToExploreAtInsert, params.get().neighborsToExploreAtInsert());
+    }
+
+    private String getSdWithIndexSpec(String indexSpec) {
+        return getSd(joinLines("field t1 type tensor(x[64]) {",
+                "  indexing: attribute | index",
+                "  " + indexSpec,
+                "}"));
     }
 
     private void assertStartsWith(String prefix, String string) {
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java
index f90320ad686..8308b638497 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java
@@ -13,7 +13,7 @@ import com.yahoo.searchdefinition.RankProfile;
 import com.yahoo.searchdefinition.RankProfileRegistry;
 import com.yahoo.searchdefinition.Search;
 import com.yahoo.searchdefinition.SearchBuilder;
-import com.yahoo.searchdefinition.SearchDefinitionTestCase;
+import com.yahoo.searchdefinition.SchemaTestCase;
 import com.yahoo.searchdefinition.derived.AttributeFields;
 import com.yahoo.searchdefinition.derived.RawRankProfile;
 import com.yahoo.searchdefinition.parser.ParseException;
@@ -25,7 +25,7 @@ import java.util.List;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.fail;
 
-public class TensorTransformTestCase extends SearchDefinitionTestCase {
+public class TensorTransformTestCase extends SchemaTestCase {
 
     @Test
     public void requireThatNormalMaxAndMinAreNotReplaced() throws ParseException {
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/WeightedSetSummaryToTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/WeightedSetSummaryToTestCase.java
index ef6bc57223d..957b5c55889 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/WeightedSetSummaryToTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/WeightedSetSummaryToTestCase.java
@@ -3,7 +3,7 @@ package com.yahoo.searchdefinition.processing;
 
 import com.yahoo.searchdefinition.Search;
 import com.yahoo.searchdefinition.SearchBuilder;
-import com.yahoo.searchdefinition.SearchDefinitionTestCase;
+import com.yahoo.searchdefinition.SchemaTestCase;
 import com.yahoo.searchdefinition.parser.ParseException;
 import org.junit.Test;
 
@@ -12,7 +12,7 @@ import java.io.IOException;
 import static org.junit.Assert.assertNotNull;
 
 /** @author bratseth */
-public class WeightedSetSummaryToTestCase extends SearchDefinitionTestCase {
+public class WeightedSetSummaryToTestCase extends SchemaTestCase {
 
     @Test
     public void testRequireThatImplicitFieldsAreCreated() throws IOException, ParseException {
diff --git a/config-model/src/test/java/com/yahoo/vespa/documentmodel/AbstractReferenceFieldTestCase.java b/config-model/src/test/java/com/yahoo/vespa/documentmodel/AbstractReferenceFieldTestCase.java
new file mode 100644
index 00000000000..9144ad411b2
--- /dev/null
+++ b/config-model/src/test/java/com/yahoo/vespa/documentmodel/AbstractReferenceFieldTestCase.java
@@ -0,0 +1,35 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.documentmodel;
+
+import com.yahoo.document.DocumenttypesConfig;
+import com.yahoo.document.config.DocumentmanagerConfig;
+import com.yahoo.searchdefinition.SchemaTestCase;
+import com.yahoo.vespa.configmodel.producers.DocumentManager;
+import com.yahoo.vespa.configmodel.producers.DocumentTypes;
+
+import java.io.IOException;
+
+/**
+ * Utility functions for testing generated configs for reference/imported fields.
+ */
+public abstract class AbstractReferenceFieldTestCase extends SchemaTestCase {
+
+    private static String TEST_FOLDER = "src/test/configmodel/types/references/";
+
+    protected void assertDocumentConfigs(DocumentModel model,
+                                         String cfgFileSpec) throws IOException {
+        assertDocumentmanagerCfg(model, "documentmanager_" + cfgFileSpec + ".cfg");
+        assertDocumenttypesCfg(model , "documenttypes_" + cfgFileSpec + ".cfg");
+    }
+
+    protected void assertDocumentmanagerCfg(DocumentModel model, String documentmanagerCfgFile) throws IOException {
+        DocumentmanagerConfig.Builder documentmanagerCfg = new DocumentManager().produce(model, new DocumentmanagerConfig.Builder());
+        assertConfigFile(TEST_FOLDER + documentmanagerCfgFile, new DocumentmanagerConfig(documentmanagerCfg).toString());
+    }
+
+    protected void assertDocumenttypesCfg(DocumentModel model, String documenttypesCfgFile) throws IOException {
+        DocumenttypesConfig.Builder documenttypesCfg = new DocumentTypes().produce(model, new DocumenttypesConfig.Builder());
+        assertConfigFile(TEST_FOLDER + documenttypesCfgFile, new DocumenttypesConfig(documenttypesCfg).toString());
+    }
+
+}
diff --git a/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderImportedFieldsTestCase.java b/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderImportedFieldsTestCase.java
new file mode 100644
index 00000000000..599ae77a456
--- /dev/null
+++ b/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderImportedFieldsTestCase.java
@@ -0,0 +1,55 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.documentmodel;
+
+import com.yahoo.searchdefinition.SearchBuilder;
+import com.yahoo.searchdefinition.parser.ParseException;
+import org.junit.Test;
+
+import java.io.IOException;
+
+import static com.yahoo.config.model.test.TestUtil.joinLines;
+
+public class DocumentModelBuilderImportedFieldsTestCase extends AbstractReferenceFieldTestCase {
+
+    @Test
+    public void imported_fields_are_included_in_generated_document_configs() throws ParseException, IOException {
+        assertDocumentConfigs(new TestDocumentModelBuilder().addCampaign().addPerson().build(joinLines(
+                "search ad {",
+                "  document ad {",
+                "    field campaign_ref type reference<campaign> { indexing: attribute }",
+                "    field person_ref type reference<person> { indexing: attribute }",
+                "  }",
+                "  import field campaign_ref.cool_field as my_cool_field {}",
+                "  import field campaign_ref.swag_field as my_swag_field {}",
+                "  import field person_ref.name as my_name {}",
+                "}")),
+                "multiple_imported_fields");
+    }
+
+    private static class TestDocumentModelBuilder {
+        private final SearchBuilder builder = new SearchBuilder();
+        public TestDocumentModelBuilder addCampaign() throws ParseException {
+            builder.importString(joinLines("search campaign {",
+                    "  document campaign {",
+                    "    field cool_field type string { indexing: attribute }",
+                    "    field swag_field type long { indexing: attribute }",
+                    "  }",
+                    "}"));
+            return this;
+        }
+        public TestDocumentModelBuilder addPerson() throws ParseException {
+            builder.importString(joinLines("search person {",
+                    "  document person {",
+                    "    field name type string { indexing: attribute }",
+                    "  }",
+                    "}"));
+            return this;
+        }
+        public DocumentModel build(String adSdContent) throws ParseException {
+            builder.importString(adSdContent);
+            builder.build();
+            return builder.getModel();
+        }
+    }
+
+}
diff --git a/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderReferenceTypeTestCase.java b/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderReferenceTypeTestCase.java
index e9a7a6ed33e..55980ee5fea 100644
--- a/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderReferenceTypeTestCase.java
+++ b/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderReferenceTypeTestCase.java
@@ -1,15 +1,10 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.documentmodel;
 
-import com.yahoo.document.DocumenttypesConfig;
 import com.yahoo.document.ReferenceDataType;
-import com.yahoo.document.config.DocumentmanagerConfig;
 import com.yahoo.documentmodel.NewDocumentType;
 import com.yahoo.searchdefinition.SearchBuilder;
-import com.yahoo.searchdefinition.SearchDefinitionTestCase;
 import com.yahoo.searchdefinition.parser.ParseException;
-import com.yahoo.vespa.configmodel.producers.DocumentManager;
-import com.yahoo.vespa.configmodel.producers.DocumentTypes;
 import org.junit.Test;
 
 import java.io.IOException;
@@ -20,7 +15,7 @@ import static org.junit.Assert.assertEquals;
 /**
  * @author geirst
  */
-public class DocumentModelBuilderReferenceTypeTestCase extends SearchDefinitionTestCase {
+public class DocumentModelBuilderReferenceTypeTestCase extends AbstractReferenceFieldTestCase {
 
     @Test
     public void reference_fields_can_reference_other_document_types() throws ParseException, IOException {
@@ -60,24 +55,6 @@ public class DocumentModelBuilderReferenceTypeTestCase extends SearchDefinitionT
         assertEquals(campaignRefType.getTargetType(), campaignType);
     }
 
-    private static String TEST_FOLDER = "src/test/configmodel/types/references/";
-
-    private void assertDocumentConfigs(DocumentModel model,
-                                       String cfgFileSpec) throws IOException {
-        assertDocumentmanagerCfg(model, "documentmanager_" + cfgFileSpec + ".cfg");
-        assertDocumenttypesCfg(model , "documenttypes_" + cfgFileSpec + ".cfg");
-    }
-
-    private void assertDocumentmanagerCfg(DocumentModel model, String documentmanagerCfgFile) throws IOException {
-        DocumentmanagerConfig.Builder documentmanagerCfg = new DocumentManager().produce(model, new DocumentmanagerConfig.Builder());
-        assertConfigFile(TEST_FOLDER + documentmanagerCfgFile, new DocumentmanagerConfig(documentmanagerCfg).toString());
-    }
-
-    private void assertDocumenttypesCfg(DocumentModel model, String documenttypesCfgFile) throws IOException {
-        DocumenttypesConfig.Builder documenttypesCfg = new DocumentTypes().produce(model, new DocumenttypesConfig.Builder());
-        assertConfigFile(TEST_FOLDER + documenttypesCfgFile, new DocumenttypesConfig(documenttypesCfg).toString());
-    }
-
     private static class TestDocumentModelBuilder {
         private final SearchBuilder builder = new SearchBuilder();
         public TestDocumentModelBuilder addCampaign() throws ParseException {
diff --git a/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderTestCase.java b/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderTestCase.java
index f5034d0530b..91152648b10 100644
--- a/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderTestCase.java
+++ b/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderTestCase.java
@@ -4,7 +4,7 @@ package com.yahoo.vespa.documentmodel;
 import com.yahoo.document.DocumenttypesConfig;
 import com.yahoo.document.config.DocumentmanagerConfig;
 import com.yahoo.searchdefinition.SearchBuilder;
-import com.yahoo.searchdefinition.SearchDefinitionTestCase;
+import com.yahoo.searchdefinition.SchemaTestCase;
 import com.yahoo.searchdefinition.parser.ParseException;
 import com.yahoo.vespa.configmodel.producers.DocumentManager;
 import com.yahoo.vespa.configmodel.producers.DocumentTypes;
@@ -13,7 +13,7 @@ import java.io.IOException;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
 
-public class DocumentModelBuilderTestCase extends SearchDefinitionTestCase {
+public class DocumentModelBuilderTestCase extends SchemaTestCase {
 
     @Test
     public void testDocumentManagerSimple()  throws IOException, ParseException {
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/HostResourceTest.java b/config-model/src/test/java/com/yahoo/vespa/model/HostResourceTest.java
index 234841f2b6c..6aea0593f8a 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/HostResourceTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/HostResourceTest.java
@@ -1,7 +1,6 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.model;
 
-import com.yahoo.component.Version;
 import com.yahoo.config.model.producer.AbstractConfigProducer;
 import com.yahoo.config.model.test.MockRoot;
 import com.yahoo.config.provision.ClusterMembership;
@@ -9,17 +8,12 @@ import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.config.provision.HostSpec;
 import org.junit.Test;
 
-import java.util.Arrays;
 import java.util.Optional;
 
-import static com.yahoo.config.provision.ClusterSpec.Type.admin;
 import static com.yahoo.config.provision.ClusterSpec.Type.container;
-import static com.yahoo.config.provision.ClusterSpec.Type.content;
 import static org.hamcrest.Matchers.endsWith;
 import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertThat;
-import static org.junit.Assert.assertTrue;
 
 /**
  * @author gjoranv
@@ -54,7 +48,7 @@ public class HostResourceTest {
     }
 
     private static ClusterSpec clusterSpec(ClusterSpec.Type type, String id) {
-        return ClusterSpec.from(type, ClusterSpec.Id.from(id), ClusterSpec.Group.from(0), Version.fromString("6.42"), false);
+        return ClusterSpec.specification(type, ClusterSpec.Id.from(id)).group(ClusterSpec.Group.from(0)).vespaVersion("6.42").build();
     }
 
     private static HostResource hostResourceWithMemberships(ClusterMembership membership) {
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/VespaModelFactoryTest.java b/config-model/src/test/java/com/yahoo/vespa/model/VespaModelFactoryTest.java
index 90ec1779f39..cf1ae637cf9 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/VespaModelFactoryTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/VespaModelFactoryTest.java
@@ -1,7 +1,6 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.model;
 
-import com.yahoo.component.Version;
 import com.yahoo.config.application.api.ApplicationPackage;
 import com.yahoo.config.model.MockModelContext;
 import com.yahoo.config.model.NullConfigModelRegistry;
@@ -23,7 +22,6 @@ import org.junit.Before;
 import org.junit.Test;
 
 import java.util.ArrayList;
-import java.util.Collections;
 import java.util.List;
 import java.util.Optional;
 
@@ -105,23 +103,23 @@ public class VespaModelFactoryTest {
             @Override
             public HostSpec allocateHost(String alias) {
                 return new HostSpec(hostName,
-                                    Collections.emptyList(),
-                                    ClusterMembership.from(ClusterSpec.from(ClusterSpec.Type.admin,
-                                                                            new ClusterSpec.Id(routingClusterName),
-                                                                            ClusterSpec.Group.from(0),
-                                                                            Version.fromString("6.42"), false),
+                                    List.of(),
+                                    ClusterMembership.from(ClusterSpec.request(ClusterSpec.Type.admin, new ClusterSpec.Id(routingClusterName)).vespaVersion("6.42").build(),
                                                            0));
             }
 
             @Override
+            @Deprecated // TODO: Remove after April 2020
             public List<HostSpec> prepare(ClusterSpec cluster, Capacity capacity, int groups, ProvisionLogger logger) {
-                return Collections.singletonList(new HostSpec(hostName,
-                                                              Collections.emptyList(),
-                                                              ClusterMembership.from(ClusterSpec.from(ClusterSpec.Type.container,
-                                                                                                      new ClusterSpec.Id(routingClusterName),
-                                                                                                      ClusterSpec.Group.from(0),
-                                                                                                      Version.fromString("6.42"), false),
-                                                                                     0)));
+                return prepare(cluster, capacity.withGroups(groups), logger);
+            }
+
+            @Override
+            public List<HostSpec> prepare(ClusterSpec cluster, Capacity capacity, ProvisionLogger logger) {
+                return List.of(new HostSpec(hostName,
+                                            List.of(),
+                                            ClusterMembership.from(ClusterSpec.request(ClusterSpec.Type.container, new ClusterSpec.Id(routingClusterName)).vespaVersion("6.42").build(),
+                                                                   0)));
             }
         };
 
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/ClusterControllerTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/ClusterControllerTestCase.java
index cd67e432b8f..4b23a7e2e71 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/admin/ClusterControllerTestCase.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/ClusterControllerTestCase.java
@@ -52,7 +52,7 @@ public class ClusterControllerTestCase extends DomBuilderTest {
 
     @Before
     public void setup() {
-        sds = ApplicationPackageUtils.generateSearchDefinitions("type1", "type2");
+        sds = ApplicationPackageUtils.generateSchemas("type1", "type2");
     }
 
     @Test
@@ -457,7 +457,7 @@ public class ClusterControllerTestCase extends DomBuilderTest {
     private VespaModel createVespaModel(String servicesXml, boolean isHosted) throws IOException, SAXException {
         ApplicationPackage applicationPackage = new MockApplicationPackage.Builder()
                 .withServices(servicesXml)
-                .withSearchDefinitions(sds)
+                .withSchemas(sds)
                 .build();
         // Need to create VespaModel to make deploy properties have effect
         DeployLogger logger = new DeployLoggerStub();
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsConsumersTest.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsConsumersTest.java
new file mode 100644
index 00000000000..12a10a7e354
--- /dev/null
+++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsConsumersTest.java
@@ -0,0 +1,244 @@
+package com.yahoo.vespa.model.admin.metricsproxy;
+
+import ai.vespa.metricsproxy.core.ConsumersConfig;
+import com.yahoo.vespa.model.VespaModel;
+import com.yahoo.vespa.model.admin.monitoring.Metric;
+import com.yahoo.vespa.model.admin.monitoring.MetricSet;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.TestMode.hosted;
+import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.TestMode.self_hosted;
+import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.checkMetric;
+import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.consumersConfigFromModel;
+import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.consumersConfigFromXml;
+import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getCustomConsumer;
+import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getModel;
+import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.servicesWithAdminOnly;
+import static com.yahoo.vespa.model.admin.monitoring.DefaultPublicConsumer.DEFAULT_PUBLIC_CONSUMER_ID;
+import static com.yahoo.vespa.model.admin.monitoring.DefaultPublicMetrics.defaultPublicMetricSet;
+import static com.yahoo.vespa.model.admin.monitoring.DefaultVespaMetrics.defaultVespaMetricSet;
+import static com.yahoo.vespa.model.admin.monitoring.NetworkMetrics.networkMetricSet;
+import static com.yahoo.vespa.model.admin.monitoring.SystemMetrics.systemMetricSet;
+import static com.yahoo.vespa.model.admin.monitoring.VespaMetricSet.vespaMetricSet;
+import static com.yahoo.vespa.model.admin.monitoring.VespaMetricsConsumer.VESPA_CONSUMER_ID;
+import static java.util.Collections.singleton;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Tests for {@link MetricsProxyContainerCluster} related to metrics consumers.
+ *
+ * @author gjoranv
+ */
+public class MetricsConsumersTest {
+
+    private static int numPublicDefaultMetrics = defaultPublicMetricSet.getMetrics().size();
+    private static int numDefaultVespaMetrics = defaultVespaMetricSet.getMetrics().size();
+    private static int numVespaMetrics = vespaMetricSet.getMetrics().size();
+    private static int numSystemMetrics = systemMetricSet.getMetrics().size();
+    private static int numNetworkMetrics = networkMetricSet.getMetrics().size();
+    private static int numMetricsForVespaConsumer = numVespaMetrics + numSystemMetrics + numNetworkMetrics;
+
+    @Rule
+    public ExpectedException thrown = ExpectedException.none();
+
+    @Test
+    public void default_public_consumer_is_set_up_for_self_hosted() {
+        ConsumersConfig config = consumersConfigFromXml(servicesWithAdminOnly(), self_hosted);
+        assertEquals(2, config.consumer().size());
+        assertEquals(config.consumer(1).name(), DEFAULT_PUBLIC_CONSUMER_ID);
+
+        int numMetricsForPublicDefaultConsumer = defaultPublicMetricSet.getMetrics().size() + numSystemMetrics;
+        assertEquals(numMetricsForPublicDefaultConsumer, config.consumer(1).metric().size());
+    }
+
+    @Test
+    public void vespa_consumer_and_default_public_consumer_is_set_up_for_hosted() {
+        ConsumersConfig config = consumersConfigFromXml(servicesWithAdminOnly(), hosted);
+        assertEquals(2, config.consumer().size());
+        assertEquals(config.consumer(0).name(), VESPA_CONSUMER_ID);
+        assertEquals(config.consumer(1).name(), DEFAULT_PUBLIC_CONSUMER_ID);
+    }
+
+    @Test
+    public void vespa_consumer_is_always_present_and_has_all_vespa_metrics_and_all_system_metrics() {
+        ConsumersConfig config = consumersConfigFromXml(servicesWithAdminOnly(), self_hosted);
+        assertEquals(config.consumer(0).name(), VESPA_CONSUMER_ID);
+        assertEquals(numMetricsForVespaConsumer, config.consumer(0).metric().size());
+    }
+
+    @Test
+    public void vespa_consumer_can_be_amended_via_admin_object() {
+        VespaModel model = getModel(servicesWithAdminOnly(), self_hosted);
+        var additionalMetric = new Metric("additional-metric");
+        model.getAdmin().setAdditionalDefaultMetrics(new MetricSet("amender-metrics", singleton(additionalMetric)));
+
+        ConsumersConfig config = consumersConfigFromModel(model);
+        assertEquals(numMetricsForVespaConsumer + 1, config.consumer(0).metric().size());
+
+        ConsumersConfig.Consumer vespaConsumer = config.consumer(0);
+        assertTrue("Did not contain additional metric", checkMetric(vespaConsumer, additionalMetric));
+    }
+
+    @Test
+    public void vespa_is_a_reserved_consumer_id() {
+        assertReservedConsumerId("Vespa");
+    }
+
+    @Test
+    public void default_is_a_reserved_consumer_id() {
+        assertReservedConsumerId("default");
+    }
+
+    private void assertReservedConsumerId(String consumerId) {
+        String services = String.join("\n",
+                                      "<services>",
+                                      "    <admin version='2.0'>",
+                                      "        <adminserver hostalias='node1'/>",
+                                      "        <metrics>",
+                                      "            <consumer id='"  + consumerId + "'/>",
+                                      "        </metrics>",
+                                      "    </admin>",
+                                      "</services>"
+        );
+        thrown.expect(IllegalArgumentException.class);
+        thrown.expectMessage("'" + consumerId + "' is not allowed as metrics consumer id");
+        consumersConfigFromXml(services, self_hosted);
+    }
+
+    @Test
+    public void vespa_consumer_id_is_allowed_for_hosted_infrastructure_applications() {
+        String services = String.join("\n",
+                                      "<services application-type='hosted-infrastructure'>",
+                                      "    <admin version='4.0'>",
+                                      "        <adminserver hostalias='node1'/>",
+                                      "        <metrics>",
+                                      "            <consumer id='Vespa'>",
+                                      "                <metric id='custom.metric1'/>",
+                                      "            </consumer>",
+                                      "        </metrics>",
+                                      "    </admin>",
+                                      "</services>"
+        );
+        VespaModel hostedModel = getModel(services, hosted);
+        ConsumersConfig config = consumersConfigFromModel(hostedModel);
+        assertEquals(2, config.consumer().size());
+
+        // All default metrics are retained
+        ConsumersConfig.Consumer vespaConsumer = config.consumer(0);
+        assertEquals(numMetricsForVespaConsumer + 1, vespaConsumer.metric().size());
+
+        Metric customMetric1 = new Metric("custom.metric1");
+        assertTrue("Did not contain metric: " + customMetric1, checkMetric(vespaConsumer, customMetric1));
+    }
+
+    @Test
+    public void consumer_id_is_case_insensitive() {
+        String services = String.join("\n",
+                                      "<services>",
+                                      "    <admin version='2.0'>",
+                                      "        <adminserver hostalias='node1'/>",
+                                      "        <metrics>",
+                                      "            <consumer id='A'/>",
+                                      "            <consumer id='a'/>",
+                                      "        </metrics>",
+                                      "    </admin>",
+                                      "</services>"
+        );
+        thrown.expect(IllegalArgumentException.class);
+        thrown.expectMessage("'a' is used as id for two metrics consumers");
+        consumersConfigFromXml(services, self_hosted);
+    }
+
+    @Test
+    public void non_existent_metric_set_causes_exception() {
+        String services = String.join("\n",
+                                      "<services>",
+                                      "    <admin version='2.0'>",
+                                      "        <adminserver hostalias='node1'/>",
+                                      "        <metrics>",
+                                      "            <consumer id='consumer-with-non-existent-default-set'>",
+                                      "                <metric-set id='non-existent'/>",
+                                      "            </consumer>",
+                                      "        </metrics>",
+                                      "    </admin>",
+                                      "</services>"
+        );
+        thrown.expect(IllegalArgumentException.class);
+        thrown.expectMessage("No such metric-set: non-existent");
+        consumersConfigFromXml(services, self_hosted);
+    }
+
+    @Test
+    public void consumer_with_no_metric_set_has_its_own_metrics_plus_system_metrics_plus_default_vespa_metrics() {
+        String services = String.join("\n",
+                                      "<services>",
+                                      "    <admin version='2.0'>",
+                                      "        <adminserver hostalias='node1'/>",
+                                      "        <metrics>",
+                                      "            <consumer id='consumer-with-metrics-only'>",
+                                      "                <metric id='custom.metric1'/>",
+                                      "                <metric id='custom.metric2'/>",
+                                      "            </consumer>",
+                                      "        </metrics>",
+                                      "    </admin>",
+                                      "</services>"
+        );
+        ConsumersConfig.Consumer consumer = getCustomConsumer(services);
+
+        assertEquals(numSystemMetrics + numDefaultVespaMetrics + 2, consumer.metric().size());
+
+        Metric customMetric1 = new Metric("custom.metric1");
+        Metric customMetric2 = new Metric("custom.metric2");
+        assertTrue("Did not contain metric: " + customMetric1, checkMetric(consumer, customMetric1));
+        assertTrue("Did not contain metric: " + customMetric2, checkMetric(consumer, customMetric2));
+    }
+
+    @Test
+    public void consumer_with_default_metric_set_has_all_its_metrics_plus_all_system_metrics_plus_its_own() {
+        String services = String.join("\n",
+                                      "<services>",
+                                      "    <admin version='2.0'>",
+                                      "        <adminserver hostalias='node1'/>",
+                                      "        <metrics>",
+                                      "            <consumer id='consumer-with-public-default-set'>",
+                                      "                <metric-set id='default'/>",
+                                      "                <metric id='custom.metric'/>",
+                                      "            </consumer>",
+                                      "        </metrics>",
+                                      "    </admin>",
+                                      "</services>"
+        );
+        ConsumersConfig.Consumer consumer = getCustomConsumer(services);
+
+        assertEquals(numPublicDefaultMetrics + numSystemMetrics + 1, consumer.metric().size());
+
+        Metric customMetric = new Metric("custom.metric");
+        assertTrue("Did not contain metric: " + customMetric, checkMetric(consumer, customMetric));
+    }
+
+    @Test
+    public void consumer_with_vespa_metric_set_has_all_vespa_metrics_plus_all_system_metrics_plus_its_own() {
+        String services = String.join("\n",
+                                      "<services>",
+                                      "    <admin version='2.0'>",
+                                      "        <adminserver hostalias='node1'/>",
+                                      "        <metrics>",
+                                      "            <consumer id='consumer-with-vespa-set'>",
+                                      "                <metric-set id='vespa'/>",
+                                      "                <metric id='my.extra.metric'/>",
+                                      "            </consumer>",
+                                      "        </metrics>",
+                                      "    </admin>",
+                                      "</services>"
+        );
+        ConsumersConfig.Consumer consumer = getCustomConsumer(services);
+        assertEquals(numVespaMetrics + numSystemMetrics + 1, consumer.metric().size());
+
+        Metric customMetric = new Metric("my.extra.metric");
+        assertTrue("Did not contain metric: " + customMetric, checkMetric(consumer, customMetric));
+    }
+
+}
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java
index de40e557265..bed77bd5c77 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java
@@ -5,10 +5,9 @@
 
 package com.yahoo.vespa.model.admin.metricsproxy;
 
-import ai.vespa.metricsproxy.core.ConsumersConfig;
-import ai.vespa.metricsproxy.http.metrics.MetricsV1Handler;
 import ai.vespa.metricsproxy.http.application.ApplicationMetricsHandler;
 import ai.vespa.metricsproxy.http.application.MetricsNodesConfig;
+import ai.vespa.metricsproxy.http.metrics.MetricsV1Handler;
 import ai.vespa.metricsproxy.http.prometheus.PrometheusHandler;
 import ai.vespa.metricsproxy.http.yamas.YamasHandler;
 import ai.vespa.metricsproxy.metric.dimensions.ApplicationDimensionsConfig;
@@ -21,13 +20,9 @@ import com.yahoo.container.core.ApplicationMetadataConfig;
 import com.yahoo.search.config.QrStartConfig;
 import com.yahoo.vespa.model.VespaModel;
 import com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerCluster.AppDimensionNames;
-import com.yahoo.vespa.model.admin.monitoring.Metric;
-import com.yahoo.vespa.model.admin.monitoring.MetricSet;
 import com.yahoo.vespa.model.container.component.Component;
 import com.yahoo.vespa.model.container.component.Handler;
-import org.junit.Rule;
 import org.junit.Test;
-import org.junit.rules.ExpectedException;
 
 import java.util.Collection;
 
@@ -39,26 +34,17 @@ import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.M
 import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.MY_TENANT;
 import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.TestMode.hosted;
 import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.TestMode.self_hosted;
-import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.checkMetric;
-import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.consumersConfigFromModel;
-import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.consumersConfigFromXml;
 import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getApplicationDimensionsConfig;
-import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getCustomConsumer;
 import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getMetricsNodesConfig;
 import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getModel;
 import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getQrStartConfig;
-import static com.yahoo.vespa.model.admin.monitoring.DefaultPublicConsumer.DEFAULT_PUBLIC_CONSUMER_ID;
-import static com.yahoo.vespa.model.admin.monitoring.DefaultPublicMetrics.defaultPublicMetricSet;
-import static com.yahoo.vespa.model.admin.monitoring.DefaultVespaMetrics.defaultVespaMetricSet;
-import static com.yahoo.vespa.model.admin.monitoring.NetworkMetrics.networkMetricSet;
-import static com.yahoo.vespa.model.admin.monitoring.SystemMetrics.systemMetricSet;
-import static com.yahoo.vespa.model.admin.monitoring.VespaMetricSet.vespaMetricSet;
-import static com.yahoo.vespa.model.admin.monitoring.VespaMetricsConsumer.VESPA_CONSUMER_ID;
-import static java.util.Collections.singleton;
+import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.servicesWithAdminOnly;
 import static java.util.stream.Collectors.toList;
 import static org.hamcrest.CoreMatchers.endsWith;
 import static org.hamcrest.CoreMatchers.hasItem;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotSame;
 import static org.junit.Assert.assertThat;
 import static org.junit.Assert.assertTrue;
 
@@ -67,15 +53,6 @@ import static org.junit.Assert.assertTrue;
  */
 public class MetricsProxyContainerClusterTest {
 
-    private static int numDefaultVespaMetrics = defaultVespaMetricSet.getMetrics().size();
-    private static int numVespaMetrics = vespaMetricSet.getMetrics().size();
-    private static int numSystemMetrics = systemMetricSet.getMetrics().size();
-    private static int numNetworkMetrics = networkMetricSet.getMetrics().size();
-    private static int numMetricsForVespaConsumer = numVespaMetrics + numSystemMetrics + numNetworkMetrics;
-
-    @Rule
-    public ExpectedException thrown = ExpectedException.none();
-
     @Test
     public void metrics_proxy_bundle_is_included_in_bundles_config() {
         VespaModel model = getModel(servicesWithAdminOnly(), self_hosted);
@@ -104,10 +81,11 @@ public class MetricsProxyContainerClusterTest {
         assertEquals(512, qrStartConfig.jvm().heapsize());
         assertEquals(0, qrStartConfig.jvm().heapSizeAsPercentageOfPhysicalMemory());
         assertEquals(2, qrStartConfig.jvm().availableProcessors());
-        assertEquals(false, qrStartConfig.jvm().verbosegc());
+        assertFalse(qrStartConfig.jvm().verbosegc());
         assertEquals("-XX:+UseG1GC -XX:MaxTenuringThreshold=15", qrStartConfig.jvm().gcopts());
         assertEquals(512, qrStartConfig.jvm().stacksize());
         assertEquals(0, qrStartConfig.jvm().directMemorySizeCache());
+        assertEquals(32, qrStartConfig.jvm().compressedClassSpaceSize());
         assertEquals(75, qrStartConfig.jvm().baseMaxDirectMemorySize());
     }
 
@@ -130,161 +108,6 @@ public class MetricsProxyContainerClusterTest {
     }
 
     @Test
-    public void default_public_consumer_is_set_up_for_self_hosted() {
-        ConsumersConfig config = consumersConfigFromXml(servicesWithAdminOnly(), self_hosted);
-        assertEquals(2, config.consumer().size());
-        assertEquals(config.consumer(1).name(), DEFAULT_PUBLIC_CONSUMER_ID);
-
-        int numMetricsForPublicDefaultConsumer = defaultPublicMetricSet.getMetrics().size() + numSystemMetrics;
-        assertEquals(numMetricsForPublicDefaultConsumer, config.consumer(1).metric().size());
-    }
-
-    @Test
-    public void vespa_consumer_and_default_public_consumer_is_set_up_for_hosted() {
-        ConsumersConfig config = consumersConfigFromXml(servicesWithAdminOnly(), hosted);
-        assertEquals(2, config.consumer().size());
-        assertEquals(config.consumer(0).name(), VESPA_CONSUMER_ID);
-        assertEquals(config.consumer(1).name(), DEFAULT_PUBLIC_CONSUMER_ID);
-    }
-
-    @Test
-    public void vespa_consumer_is_always_present_and_has_all_vespa_metrics_and_all_system_metrics() {
-        ConsumersConfig config = consumersConfigFromXml(servicesWithAdminOnly(), self_hosted);
-        assertEquals(config.consumer(0).name(), VESPA_CONSUMER_ID);
-        assertEquals(numMetricsForVespaConsumer, config.consumer(0).metric().size());
-    }
-
-    @Test
-    public void vespa_consumer_can_be_amended_via_admin_object() {
-        VespaModel model = getModel(servicesWithAdminOnly(), self_hosted);
-        var additionalMetric = new Metric("additional-metric");
-        model.getAdmin().setAdditionalDefaultMetrics(new MetricSet("amender-metrics", singleton(additionalMetric)));
-
-        ConsumersConfig config = consumersConfigFromModel(model);
-        assertEquals(numMetricsForVespaConsumer + 1, config.consumer(0).metric().size());
-
-        ConsumersConfig.Consumer vespaConsumer = config.consumer(0);
-        assertTrue("Did not contain additional metric", checkMetric(vespaConsumer, additionalMetric));
-    }
-
-    @Test
-    public void vespa_is_a_reserved_consumer_id() {
-        assertReservedConsumerId("Vespa");
-    }
-
-    @Test
-    public void default_is_a_reserved_consumer_id() {
-        assertReservedConsumerId("default");
-    }
-
-    private void assertReservedConsumerId(String consumerId) {
-        String services = String.join("\n",
-                                      "<services>",
-                                      "    <admin version='2.0'>",
-                                      "        <adminserver hostalias='node1'/>",
-                                      "        <metrics>",
-                                      "            <consumer id='"  + consumerId + "'/>",
-                                      "        </metrics>",
-                                      "    </admin>",
-                                      "</services>"
-        );
-        thrown.expect(IllegalArgumentException.class);
-        thrown.expectMessage("'" + consumerId + "' is not allowed as metrics consumer id");
-        consumersConfigFromXml(services, self_hosted);
-    }
-
-    @Test
-    public void vespa_consumer_id_is_allowed_for_hosted_infrastructure_applications() {
-        String services = String.join("\n",
-                "<services application-type='hosted-infrastructure'>",
-                "    <admin version='4.0'>",
-                "        <adminserver hostalias='node1'/>",
-                "        <metrics>",
-                "            <consumer id='Vespa'>",
-                "                <metric id='custom.metric1'/>",
-                "            </consumer>",
-                "        </metrics>",
-                "    </admin>",
-                "</services>"
-        );
-        VespaModel hostedModel = getModel(services, hosted);
-        ConsumersConfig config = consumersConfigFromModel(hostedModel);
-        assertEquals(2, config.consumer().size());
-
-        // All default metrics are retained
-        ConsumersConfig.Consumer vespaConsumer = config.consumer(0);
-        assertEquals(numMetricsForVespaConsumer + 1, vespaConsumer.metric().size());
-
-        Metric customMetric1 = new Metric("custom.metric1");
-        assertTrue("Did not contain metric: " + customMetric1, checkMetric(vespaConsumer, customMetric1));
-    }
-
-    @Test
-    public void consumer_id_is_case_insensitive() {
-        String services = String.join("\n",
-                "<services>",
-                "    <admin version='2.0'>",
-                "        <adminserver hostalias='node1'/>",
-                "        <metrics>",
-                "            <consumer id='A'/>",
-                "            <consumer id='a'/>",
-                "        </metrics>",
-                "    </admin>",
-                "</services>"
-        );
-        thrown.expect(IllegalArgumentException.class);
-        thrown.expectMessage("'a' is used as id for two metrics consumers");
-        consumersConfigFromXml(services, self_hosted);
-    }
-
-    @Test
-    public void consumer_with_no_metric_set_has_its_own_metrics_plus_system_metrics_plus_default_vespa_metrics() {
-        String services = String.join("\n",
-                "<services>",
-                "    <admin version='2.0'>",
-                "        <adminserver hostalias='node1'/>",
-                "        <metrics>",
-                "            <consumer id='consumer-with-metrics-only'>",
-                "                <metric id='custom.metric1'/>",
-                "                <metric id='custom.metric2'/>",
-                "            </consumer>",
-                "        </metrics>",
-                "    </admin>",
-                "</services>"
-        );
-        ConsumersConfig.Consumer consumer = getCustomConsumer(services);
-
-        assertEquals(numSystemMetrics + numDefaultVespaMetrics + 2, consumer.metric().size());
-
-        Metric customMetric1 = new Metric("custom.metric1");
-        Metric customMetric2 = new Metric("custom.metric2");
-        assertTrue("Did not contain metric: " + customMetric1, checkMetric(consumer, customMetric1));
-        assertTrue("Did not contain metric: " + customMetric2, checkMetric(consumer, customMetric2));
-    }
-
-    @Test
-    public void consumer_with_vespa_metric_set_has_all_vespa_metrics_plus_all_system_metrics_plus_its_own() {
-        String services = String.join("\n",
-                "<services>",
-                "    <admin version='2.0'>",
-                "        <adminserver hostalias='node1'/>",
-                "        <metrics>",
-                "            <consumer id='consumer-with-vespa-set'>",
-                "                <metric-set id='vespa'/>",
-                "                <metric id='my.extra.metric'/>",
-                "            </consumer>",
-                "        </metrics>",
-                "    </admin>",
-                "</services>"
-        );
-        ConsumersConfig.Consumer consumer = getCustomConsumer(services);
-        assertEquals(numVespaMetrics + numSystemMetrics + 1, consumer.metric().size());
-
-        Metric customMetric = new Metric("my.extra.metric");
-        assertTrue("Did not contain metric: " + customMetric, checkMetric(consumer, customMetric));
-    }
-
-    @Test
     public void hosted_application_propagates_application_dimensions() {
         VespaModel hostedModel = getModel(servicesWithAdminOnly(), hosted);
         ApplicationDimensionsConfig config = getApplicationDimensionsConfig(hostedModel);
@@ -314,16 +137,6 @@ public class MetricsProxyContainerClusterTest {
         assertEquals(MetricsV1Handler.VALUES_PATH, node.metricsPath());
     }
 
-    private static String servicesWithAdminOnly() {
-        return String.join("\n",
-                           "<services>",
-                           "    <admin version='4.0'>",
-                           "        <adminserver hostalias='node1'/>",
-                           "    </admin>",
-                           "</services>"
-        );
-    }
-
     private static String servicesWithTwoNodes() {
         return String.join("\n",
                            "<services>",
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java
index 621cebd6246..eddad6fce89 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java
@@ -1,6 +1,7 @@
 // Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.model.admin.metricsproxy;
 
+import ai.vespa.metricsproxy.http.metrics.NodeInfoConfig;
 import ai.vespa.metricsproxy.metric.dimensions.NodeDimensionsConfig;
 import ai.vespa.metricsproxy.metric.dimensions.PublicDimensions;
 import ai.vespa.metricsproxy.rpc.RpcConnectorConfig;
@@ -10,10 +11,10 @@ import com.yahoo.vespa.model.test.VespaModelTester;
 import org.junit.Test;
 
 import static com.yahoo.config.model.api.container.ContainerServiceType.METRICS_PROXY_CONTAINER;
-import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.CLUSTER_CONFIG_ID;
 import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.CONTAINER_CONFIG_ID;
 import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.TestMode.hosted;
 import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.TestMode.self_hosted;
+import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.containerConfigId;
 import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getModel;
 
 import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getNodeDimensionsConfig;
@@ -90,13 +91,28 @@ public class MetricsProxyContainerTest {
         String services = servicesWithContent();
         VespaModel hostedModel = getModel(services, hosted);
         assertEquals(1, hostedModel.getHosts().size());
-        String configId = CLUSTER_CONFIG_ID + "/" + hostedModel.getHosts().iterator().next().getHostname();
+        String configId = containerConfigId(hostedModel, hosted);
         NodeDimensionsConfig config = getNodeDimensionsConfig(hostedModel, configId);
 
         assertEquals("content", config.dimensions(PublicDimensions.INTERNAL_CLUSTER_TYPE));
         assertEquals("my-content", config.dimensions(PublicDimensions.INTERNAL_CLUSTER_ID));
     }
 
+    @Test
+    public void metrics_v2_handler_is_set_up_with_node_info_config() {
+        String services = servicesWithContent();
+        VespaModel hostedModel = getModel(services, hosted);
+
+        var container = (MetricsProxyContainer)hostedModel.id2producer().get(containerConfigId(hostedModel, hosted));
+        var handlers = container.getHandlers().getComponents();
+
+        assertEquals(1, handlers.size());
+        var metricsV2Handler = handlers.iterator().next();
+
+        NodeInfoConfig config = hostedModel.getConfig(NodeInfoConfig.class, metricsV2Handler.getConfigId());
+        assertTrue(config.role().startsWith("content/my-content/0/"));
+        assertTrue(config.hostname().startsWith("node-1-3-9-"));
+    }
 
     @Test
     public void vespa_services_config_has_all_services() {
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyModelTester.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyModelTester.java
index f3140aafdaf..8ecb13d7ae5 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyModelTester.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyModelTester.java
@@ -49,12 +49,22 @@ class MetricsProxyModelTester {
         return tester.createModel(servicesXml, true);
     }
 
-    static String configId(VespaModel model, MetricsProxyModelTester.TestMode mode) {
+    static String containerConfigId(VespaModel model, MetricsProxyModelTester.TestMode mode) {
         return (mode == hosted)
                 ? CLUSTER_CONFIG_ID + "/" + model.getHosts().iterator().next().getHostname()
                 : CONTAINER_CONFIG_ID;
     }
 
+    static String servicesWithAdminOnly() {
+        return String.join("\n",
+                           "<services>",
+                           "    <admin version='4.0'>",
+                           "        <adminserver hostalias='node1'/>",
+                           "    </admin>",
+                           "</services>"
+        );
+    }
+
     static boolean checkMetric(ConsumersConfig.Consumer consumer, Metric metric) {
         for (ConsumersConfig.Consumer.Metric m : consumer.metric()) {
             if (metric.name.equals(m.name()) && metric.outputName.equals(m.outputname()))
@@ -77,32 +87,32 @@ class MetricsProxyModelTester {
     }
 
     static ConsumersConfig consumersConfigFromModel(VespaModel model) {
-        return new ConsumersConfig((ConsumersConfig.Builder) model.getConfig(new ConsumersConfig.Builder(), CLUSTER_CONFIG_ID));
+        return model.getConfig(ConsumersConfig.class, CLUSTER_CONFIG_ID);
     }
 
     static MetricsNodesConfig getMetricsNodesConfig(VespaModel model) {
-        return new MetricsNodesConfig((MetricsNodesConfig.Builder) model.getConfig(new MetricsNodesConfig.Builder(), CLUSTER_CONFIG_ID));
+        return model.getConfig(MetricsNodesConfig.class, CLUSTER_CONFIG_ID);
     }
 
     static ApplicationDimensionsConfig getApplicationDimensionsConfig(VespaModel model) {
-        return new ApplicationDimensionsConfig((ApplicationDimensionsConfig.Builder) model.getConfig(new ApplicationDimensionsConfig.Builder(), CLUSTER_CONFIG_ID));
+        return model.getConfig(ApplicationDimensionsConfig.class, CLUSTER_CONFIG_ID);
     }
 
     static QrStartConfig getQrStartConfig(VespaModel model) {
-        return new QrStartConfig((QrStartConfig.Builder) model.getConfig(new QrStartConfig.Builder(), CLUSTER_CONFIG_ID));
+        return model.getConfig(QrStartConfig.class, CLUSTER_CONFIG_ID);
     }
 
     static NodeDimensionsConfig getNodeDimensionsConfig(VespaModel model, String configId) {
-        return new NodeDimensionsConfig((NodeDimensionsConfig.Builder) model.getConfig(new NodeDimensionsConfig.Builder(), configId));
+        return model.getConfig(NodeDimensionsConfig.class, configId);
     }
 
     static VespaServicesConfig getVespaServicesConfig(String servicesXml) {
         VespaModel model = getModel(servicesXml, self_hosted);
-        return new VespaServicesConfig((VespaServicesConfig.Builder) model.getConfig(new VespaServicesConfig.Builder(), CONTAINER_CONFIG_ID));
+        return model.getConfig(VespaServicesConfig.class, CONTAINER_CONFIG_ID);
     }
 
     static RpcConnectorConfig getRpcConnectorConfig(VespaModel model) {
-        return new RpcConnectorConfig((RpcConnectorConfig.Builder) model.getConfig(new RpcConnectorConfig.Builder(), CONTAINER_CONFIG_ID));
+        return model.getConfig(RpcConnectorConfig.class, CONTAINER_CONFIG_ID);
     }
 
 }
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/TelegrafTest.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/TelegrafTest.java
new file mode 100644
index 00000000000..9a904592744
--- /dev/null
+++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/TelegrafTest.java
@@ -0,0 +1,151 @@
+package com.yahoo.vespa.model.admin.metricsproxy;
+
+import ai.vespa.metricsproxy.telegraf.Telegraf;
+import ai.vespa.metricsproxy.telegraf.TelegrafConfig;
+import ai.vespa.metricsproxy.telegraf.TelegrafRegistry;
+import com.yahoo.component.ComponentId;
+import com.yahoo.vespa.model.VespaModel;
+import org.junit.Test;
+
+import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.CLUSTER_CONFIG_ID;
+import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.TestMode.hosted;
+import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.TestMode.self_hosted;
+import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.getModel;
+import static org.hamcrest.CoreMatchers.hasItem;
+import static org.hamcrest.CoreMatchers.not;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * @author gjoranv
+ */
+public class TelegrafTest {
+
+    @Test
+    public void telegraf_components_are_set_up_when_cloudwatch_is_configured() {
+        String services = servicesWithCloudwatch();
+        VespaModel hostedModel = getModel(services, hosted);
+
+        var clusterComponents = hostedModel.getAdmin().getMetricsProxyCluster().getComponentsMap();
+        assertThat(clusterComponents.keySet(), hasItem(ComponentId.fromString(Telegraf.class.getName())));
+        assertThat(clusterComponents.keySet(), hasItem(ComponentId.fromString(TelegrafRegistry.class.getName())));
+    }
+
+    @Test
+    public void telegraf_components_are_not_set_up_when_no_external_systems_are_added_in_services() {
+        String services = String.join("\n",
+                                      "<services>",
+                                      "    <admin version='2.0'>",
+                                      "        <adminserver hostalias='node1'/>",
+                                      "        <metrics>",
+                                      "            <consumer id='foo' />",
+                                      "        </metrics>",
+                                      "    </admin>",
+                                      "</services>");
+        VespaModel hostedModel = getModel(services, hosted);
+
+        var clusterComponents = hostedModel.getAdmin().getMetricsProxyCluster().getComponentsMap();
+        assertThat(clusterComponents.keySet(), not(hasItem(ComponentId.fromString(Telegraf.class.getName()))));
+        assertThat(clusterComponents.keySet(), not(hasItem(ComponentId.fromString(TelegrafRegistry.class.getName()))));
+    }
+
+    @Test
+    public void telegraf_config_is_generated_for_cloudwatch_in_services() {
+        String services = servicesWithCloudwatch();
+        VespaModel hostedModel = getModel(services, hosted);
+        TelegrafConfig config = hostedModel.getConfig(TelegrafConfig.class, CLUSTER_CONFIG_ID);
+        assertTrue(config.isHostedVespa());
+
+        var cloudWatch0 = config.cloudWatch(0);
+        assertEquals("cloudwatch-consumer", cloudWatch0.consumer());
+        assertEquals("us-east-1", cloudWatch0.region());
+        assertEquals("my-namespace", cloudWatch0.namespace());
+        assertEquals("my-access-key", cloudWatch0.accessKeyName());
+        assertEquals("my-secret-key", cloudWatch0.secretKeyName());
+        assertEquals("default", cloudWatch0.profile());
+    }
+
+    private String servicesWithCloudwatch() {
+        return String.join("\n",
+                           "<services>",
+                           "    <admin version='2.0'>",
+                           "        <adminserver hostalias='node1'/>",
+                           "        <metrics>",
+                           "            <consumer id='cloudwatch-consumer'>",
+                           "                <metric id='my-metric'/>",
+                           "                <cloudwatch region='us-east-1' namespace='my-namespace' >",
+                           "                    <credentials access-key-name='my-access-key' ",
+                           "                                 secret-key-name='my-secret-key' />",
+                           "                </cloudwatch>",
+                           "            </consumer>",
+                           "        </metrics>",
+                           "    </admin>",
+                           "</services>"
+        );
+    }
+
+    @Test
+    public void multiple_cloudwatches_are_allowed_for_the_same_consumer() {
+        String services = String.join("\n",
+                                      "<services>",
+                                      "    <admin version='2.0'>",
+                                      "        <adminserver hostalias='node1'/>",
+                                      "        <metrics>",
+                                      "            <consumer id='cloudwatch-consumer'>",
+                                      "                <metric id='my-metric'/>",
+                                      "                <cloudwatch region='us-east-1' namespace='namespace-1' >",
+                                      "                    <credentials access-key-name='access-key-1' ",
+                                      "                                 secret-key-name='secret-key-1' />",
+                                      "                </cloudwatch>",
+                                      "                <cloudwatch region='us-east-1' namespace='namespace-2' >",
+                                      "                    <shared-credentials profile='profile-2' />",
+                                      "                </cloudwatch>",
+                                      "            </consumer>",
+                                      "        </metrics>",
+                                      "    </admin>",
+                                      "</services>"
+        );
+        VespaModel hostedModel = getModel(services, hosted);
+        TelegrafConfig config = hostedModel.getConfig(TelegrafConfig.class, CLUSTER_CONFIG_ID);
+
+        var cloudWatch0 = config.cloudWatch(0);
+        assertEquals("cloudwatch-consumer", cloudWatch0.consumer());
+        assertEquals("us-east-1", cloudWatch0.region());
+        assertEquals("namespace-1", cloudWatch0.namespace());
+        assertEquals("access-key-1", cloudWatch0.accessKeyName());
+        assertEquals("secret-key-1", cloudWatch0.secretKeyName());
+        assertEquals("default", cloudWatch0.profile());
+
+        var cloudWatch1 = config.cloudWatch(1);
+        assertEquals("cloudwatch-consumer", cloudWatch1.consumer());
+        assertEquals("us-east-1", cloudWatch1.region());
+        assertEquals("namespace-2", cloudWatch1.namespace());
+        assertEquals("", cloudWatch1.accessKeyName());
+        assertEquals("", cloudWatch1.secretKeyName());
+        assertEquals("profile-2", cloudWatch1.profile());
+    }
+
+    @Test
+    public void profile_named_default_is_used_when_no_profile_is_given_in_shared_credentials() {
+        String services = String.join("\n",
+                                      "<services>",
+                                      "    <admin version='2.0'>",
+                                      "        <adminserver hostalias='node1'/>",
+                                      "        <metrics>",
+                                      "            <consumer id='cloudwatch-consumer'>",
+                                      "                <metric id='my-metric'/>",
+                                      "                <cloudwatch region='us-east-1' namespace='foo' >",
+                                      "                    <shared-credentials file='/path/to/file' />",
+                                      "                </cloudwatch>",
+                                      "            </consumer>",
+                                      "        </metrics>",
+                                      "    </admin>",
+                                      "</services>"
+        );
+        VespaModel model = getModel(services, self_hosted);
+        TelegrafConfig config = model.getConfig(TelegrafConfig.class, CLUSTER_CONFIG_ID);
+        assertEquals("default", config.cloudWatch(0).profile());
+    }
+
+}
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/AccessControlValidatorTestBase.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/AccessControlValidatorTestBase.java
new file mode 100644
index 00000000000..58b65199729
--- /dev/null
+++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/AccessControlValidatorTestBase.java
@@ -0,0 +1,174 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.application.validation;
+
+import com.yahoo.config.application.api.ApplicationPackage;
+import com.yahoo.config.model.NullConfigModelRegistry;
+import com.yahoo.config.model.deploy.DeployState;
+import com.yahoo.config.model.deploy.TestProperties;
+import com.yahoo.config.model.test.MockApplicationPackage;
+import com.yahoo.config.provision.Zone;
+import com.yahoo.vespa.model.VespaModel;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.xml.sax.SAXException;
+
+import java.io.IOException;
+import java.time.Instant;
+import java.time.LocalDate;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
+
+import static com.yahoo.config.model.test.TestUtil.joinLines;
+import static com.yahoo.config.provision.Environment.prod;
+import static org.hamcrest.CoreMatchers.containsString;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * @author gjoranv
+ */
+public abstract class AccessControlValidatorTestBase {
+
+    protected Validator validator;
+    protected Zone zone;
+
+    @Rule
+    public final ExpectedException exceptionRule = ExpectedException.none();
+
+    private static String servicesXml(boolean addHandler, boolean protection) {
+        return joinLines("<services version='1.0'>",
+                         "  <container id='default' version='1.0'>",
+                         addHandler ? httpHandlerXml : "",
+                         "    <http>",
+                         "      <filtering>",
+                         "        <access-control domain='foo' read='" + protection + "' write='" + protection + "' />",
+                         "      </filtering>",
+                         "    </http>",
+                         "  </container>",
+                         "</services>");
+    }
+
+    private static final String httpHandlerXml =
+            joinLines("    <handler id='foo'>",
+                      "      <binding>http://foo/bar</binding>",
+                      "    </handler>");
+
+    @Test
+    public void cluster_with_protection_passes_validation() throws IOException, SAXException {
+        DeployState deployState = deployState(servicesXml(true, true));
+        VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState);
+
+        validator.validate(model, deployState);
+    }
+
+    @Test
+    public void cluster_with_no_handlers_passes_validation_without_protection() throws IOException, SAXException{
+        DeployState deployState = deployState(servicesXml(false, false));
+        VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState);
+
+        validator.validate(model, deployState);
+    }
+
+    @Test
+    public void cluster_without_custom_components_passes_validation_without_protection() throws IOException, SAXException{
+        String servicesXml = joinLines("<services version='1.0'>",
+                                       "  <container id='default' version='1.0' />",
+                                       "</services>");
+        DeployState deployState = deployState(servicesXml);
+        VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState);
+
+        validator.validate(model, deployState);
+    }
+
+    @Test
+    public void cluster_with_handler_fails_validation_without_protection() throws IOException, SAXException{
+        exceptionRule.expect(IllegalArgumentException.class);
+        exceptionRule.expectMessage(containsString("Access-control must be enabled"));
+        exceptionRule.expectMessage(containsString("production zones: [default]"));
+
+        DeployState deployState = deployState(servicesXml(true, false));
+        VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState);
+
+        validator.validate(model, deployState);
+    }
+
+    @Test
+    public void no_http_element_has_same_effect_as_no_write_protection() throws IOException, SAXException{
+        exceptionRule.expect(IllegalArgumentException.class);
+        exceptionRule.expectMessage(containsString("Access-control must be enabled"));
+        exceptionRule.expectMessage(containsString("production zones: [default]"));
+
+        String servicesXml = joinLines("<services version='1.0'>",
+                                       "  <container id='default' version='1.0'>",
+                                       httpHandlerXml,
+                                       "  </container>",
+                                       "</services>");
+        DeployState deployState = deployState(servicesXml);
+        VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState);
+
+        validator.validate(model, deployState);
+    }
+
+    @Test
+    public void cluster_with_mbus_handler_passes_validation_without_write_protection() throws IOException, SAXException{
+        String servicesXml = joinLines("<services version='1.0'>",
+                                       "  <container id='default' version='1.0'>",
+                                       "    <handler id='foo'>",
+                                       "      <binding>mbus://*/foo</binding>",
+                                       "    </handler>",
+                                       "  </container>",
+                                       "</services>");
+        DeployState deployState = deployState(servicesXml);
+        VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState);
+
+        validator.validate(model, deployState);
+    }
+
+    @Test
+    public void write_protection_is_not_required_for_non_default_application_type() throws IOException, SAXException{
+        String servicesXml = joinLines("<services version='1.0' application-type='hosted-infrastructure'>",
+                                       "  <container id='default' version='1.0'>",
+                                       httpHandlerXml,
+                                       "  </container>",
+                                       "</services>");
+        DeployState deployState = deployState(servicesXml);
+        VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState);
+
+        validator.validate(model, deployState);
+    }
+
+    @Test
+    public void write_protection_is_not_required_with_validation_override() throws IOException, SAXException{
+        DeployState deployState = deployState(servicesXml(true, false),
+                                              "<validation-overrides><allow until='2000-01-30'>access-control</allow></validation-overrides>",
+                                              LocalDate.parse("2000-01-01", DateTimeFormatter.ISO_DATE).atStartOfDay().atZone(ZoneOffset.UTC).toInstant());
+        VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState);
+
+        validator.validate(model, deployState);
+    }
+
+    private DeployState deployState(String servicesXml) {
+        return deployState(servicesXml, "<validation-overrides></validation-overrides>", Instant.now());
+    }
+
+    private DeployState deployState(String servicesXml, String validationOverrides, Instant now) {
+        ApplicationPackage app = new MockApplicationPackage.Builder()
+                .withServices(servicesXml)
+                .withValidationOverrides(validationOverrides)
+                .build();
+
+        DeployState.Builder builder = new DeployState.Builder()
+                .applicationPackage(app)
+                .zone(zone)
+                .properties(new TestProperties().setHostedVespa(true))
+                .now(now);
+        final DeployState deployState = builder.build();
+
+        assertTrue("Test must emulate a hosted deployment.", deployState.isHosted());
+        assertEquals("Test must emulate a prod environment.", prod, deployState.zone().environment());
+
+        return deployState;
+    }
+
+}
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/AwsAccessControlValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/AwsAccessControlValidatorTest.java
new file mode 100644
index 00000000000..4eb69ec405a
--- /dev/null
+++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/AwsAccessControlValidatorTest.java
@@ -0,0 +1,24 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.application.validation;
+
+import com.yahoo.config.provision.CloudName;
+import com.yahoo.config.provision.Environment;
+import com.yahoo.config.provision.RegionName;
+import com.yahoo.config.provision.SystemName;
+import com.yahoo.config.provision.Zone;
+import org.junit.Before;
+
+import static com.yahoo.vespa.model.application.validation.AwsAccessControlValidator.AWS_CLOUD_NAME;
+
+/**
+ * @author gjoranv
+ */
+public class AwsAccessControlValidatorTest extends AccessControlValidatorTestBase {
+
+    @Before
+    public void setup() {
+        validator = new AwsAccessControlValidator();
+        zone = new Zone(CloudName.from(AWS_CLOUD_NAME), SystemName.main, Environment.prod, RegionName.from("foo"));
+    }
+
+}
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/CloudWatchValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/CloudWatchValidatorTest.java
new file mode 100644
index 00000000000..40b8223479d
--- /dev/null
+++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/CloudWatchValidatorTest.java
@@ -0,0 +1,101 @@
+package com.yahoo.vespa.model.application.validation;
+
+import com.yahoo.config.application.api.ApplicationPackage;
+import com.yahoo.config.model.NullConfigModelRegistry;
+import com.yahoo.config.model.deploy.DeployState;
+import com.yahoo.config.model.deploy.TestProperties;
+import com.yahoo.config.model.test.MockApplicationPackage;
+import com.yahoo.config.provision.Environment;
+import com.yahoo.config.provision.RegionName;
+import com.yahoo.config.provision.SystemName;
+import com.yahoo.config.provision.Zone;
+import com.yahoo.vespa.model.VespaModel;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.xml.sax.SAXException;
+
+import java.io.IOException;
+
+import static com.yahoo.config.provision.Environment.prod;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+
+/**
+ * @author gjoranv
+ */
+public class CloudWatchValidatorTest {
+
+    @Rule
+    public final ExpectedException exceptionRule = ExpectedException.none();
+
+    @Test
+    public void cloudwatch_in_public_zones_passes_validation() throws IOException, SAXException {
+        DeployState deployState = deployState(servicesWithCloudwatch(), true, true);
+        VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState);
+
+        new CloudWatchValidator().validate(model, deployState);
+    }
+
+    @Test
+    public void cloudwatch_passes_validation_for_self_hosted_vespa() throws IOException, SAXException {
+        DeployState deployState = deployState(servicesWithCloudwatch(), false, false);
+        VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState);
+
+        new CloudWatchValidator().validate(model, deployState);
+    }
+
+    @Test
+    public void cloudwatch_in_non_public_zones_fails_validation() throws IOException, SAXException {
+        exceptionRule.expect(IllegalArgumentException.class);
+        exceptionRule.expectMessage(
+                "CloudWatch cannot be set up for non-public hosted Vespa and must be removed for consumers: [cloudwatch-consumer]");
+
+        DeployState deployState = deployState(servicesWithCloudwatch(), true, false);
+        VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState);
+
+        new CloudWatchValidator().validate(model, deployState);
+    }
+
+    private static DeployState deployState(String servicesXml, boolean isHosted, boolean isPublic) {
+        ApplicationPackage app = new MockApplicationPackage.Builder()
+                .withServices(servicesXml)
+                .build();
+
+        DeployState.Builder builder = new DeployState.Builder()
+                .applicationPackage(app)
+                .properties(new TestProperties().setHostedVespa(isHosted));
+        if (isHosted) {
+            var system = isPublic ? SystemName.Public : SystemName.main;
+            builder.zone(new Zone(system, Environment.prod, RegionName.from("foo")));
+        }
+        final DeployState deployState = builder.build();
+
+        if (isHosted) {
+            assertTrue("Test must emulate a hosted deployment.", deployState.isHosted());
+            assertEquals("Test must emulate a prod environment.", prod, deployState.zone().environment());
+        }
+        return deployState;
+    }
+
+    private String servicesWithCloudwatch() {
+        return String.join("\n",
+                           "<services>",
+                           "    <admin version='2.0'>",
+                           "        <adminserver hostalias='node1'/>",
+                           "        <metrics>",
+                           "            <consumer id='cloudwatch-consumer'>",
+                           "                <metric id='my-metric'/>",
+                           "                <cloudwatch region='us-east-1' namespace='my-namespace' >",
+                           "                    <credentials access-key-name='my-access-key' ",
+                           "                                 secret-key-name='my-secret-key' />",
+                           "                </cloudwatch>",
+                           "            </consumer>",
+                           "        </metrics>",
+                           "    </admin>",
+                           "</services>"
+        );
+    }
+
+}
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/DeploymentSpecValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/DeploymentSpecValidatorTest.java
index c6d56455d44..e0366b62933 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/DeploymentSpecValidatorTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/DeploymentSpecValidatorTest.java
@@ -1,7 +1,6 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.model.application.validation;
 
-import com.yahoo.config.application.api.ApplicationPackage;
 import com.yahoo.config.model.NullConfigModelRegistry;
 import com.yahoo.config.model.deploy.DeployState;
 import com.yahoo.config.model.test.MockApplicationPackage;
@@ -11,8 +10,7 @@ import org.xml.sax.SAXException;
 
 import java.io.IOException;
 
-import static org.hamcrest.CoreMatchers.containsString;
-import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.fail;
 
 /**
@@ -21,48 +19,71 @@ import static org.junit.Assert.fail;
 public class DeploymentSpecValidatorTest {
 
     @Test
-    public void testDeploymentWithNonExistentGlobalId() throws IOException, SAXException {
-        final String simpleHosts = "<?xml version=\"1.0\" encoding=\"utf-8\" ?>" +
-                "<hosts>  " +
-                "<host name=\"localhost\">" +
-                "<alias>node0</alias>" +
-                "</host>" +
-                "</hosts>";
-
-        final String services = "<services version='1.0'>" +
-                "  <admin  version='2.0'>" +
-                "    <adminserver hostalias='node0' />" +
-                "  </admin>" +
-                "  <container id='default' version='1.0'>" +
-                "    <search/>" +
-                "      <nodes>" +
-                "        <node hostalias='node0'/>" +
-                "     </nodes>" +
-                "   </container>" +
-                "</services>";
-
-        final String deploymentSpec = "<?xml version='1.0' encoding='UTF-8'?>" +
+    public void testDeploymentWithNonExistentGlobalId() {
+        var deploymentXml = "<?xml version='1.0' encoding='UTF-8'?>" +
                 "<deployment version='1.0'>" +
                 "  <test />" +
                 "  <prod global-service-id='non-existing'>" +
                 "    <region active='true'>us-east</region>" +
                 "  </prod>" +
                 "</deployment>";
+        assertValidationError("Attribute 'globalServiceId' in instance default: 'non-existing' specified in " +
+                              "deployment.xml does not match any container cluster ID", deploymentXml);
+    }
 
-        ApplicationPackage app = new MockApplicationPackage.Builder()
+    @Test
+    public void testEndpointNonExistentContainerId() {
+        var deploymentXml = "<?xml version='1.0' encoding='UTF-8'?>" +
+                            "<deployment version='1.0'>" +
+                            "  <test />" +
+                            "  <prod>" +
+                            "    <region active='true'>us-east</region>" +
+                            "  </prod>" +
+                            "  <endpoints>" +
+                            "    <endpoint container-id='non-existing'/>" +
+                            "  </endpoints>" +
+                            "</deployment>";
+        assertValidationError("Endpoint 'default' in instance default: 'non-existing' specified in " +
+                              "deployment.xml does not match any container cluster ID", deploymentXml);
+    }
+
+    private static void assertValidationError(String message, String deploymentXml) {
+        var simpleHosts = "<?xml version=\"1.0\" encoding=\"utf-8\" ?>" +
+                          "<hosts>  " +
+                          "<host name=\"localhost\">" +
+                          "<alias>node0</alias>" +
+                          "</host>" +
+                          "</hosts>";
+
+        var services = "<services version='1.0'>" +
+                       "  <admin  version='2.0'>" +
+                       "    <adminserver hostalias='node0' />" +
+                       "  </admin>" +
+                       "  <container id='default' version='1.0'>" +
+                       "    <search/>" +
+                       "      <nodes>" +
+                       "        <node hostalias='node0'/>" +
+                       "     </nodes>" +
+                       "   </container>" +
+                       "</services>";
+
+        var app = new MockApplicationPackage.Builder()
                 .withHosts(simpleHosts)
                 .withServices(services)
-                .withDeploymentSpec(deploymentSpec)
+                .withDeploymentSpec(deploymentXml)
                 .build();
-        DeployState.Builder builder = new DeployState.Builder().applicationPackage(app);
+        var builder = new DeployState.Builder().applicationPackage(app);
         try {
-            final DeployState deployState = builder.build();
+            var deployState = builder.build();
             VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState);
             new DeploymentSpecValidator().validate(model, deployState);
             fail("Did not get expected exception");
         } catch (IllegalArgumentException e) {
-            assertThat(e.getMessage(), containsString("specified in deployment.xml does not match any container cluster id"));
+            assertEquals(message, e.getMessage());
+        } catch (SAXException|IOException e) {
+            throw new RuntimeException(e);
         }
     }
 
 }
+
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/EndpointCertificateSecretsValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/EndpointCertificateSecretsValidatorTest.java
index 21df39ebde8..318a0630c4d 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/EndpointCertificateSecretsValidatorTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/EndpointCertificateSecretsValidatorTest.java
@@ -47,7 +47,7 @@ public class EndpointCertificateSecretsValidatorTest {
         VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState);
 
         exceptionRule.expect(CertificateNotReadyException.class);
-        exceptionRule.expectMessage("TLS enabled, but could not retrieve certificate yet");
+        exceptionRule.expectMessage("TLS enabled, but could not yet retrieve certificate for application default:default:default");
 
         new EndpointCertificateSecretsValidator().validate(model, deployState);
     }
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ValidationTester.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ValidationTester.java
index f7b6d7b25e1..915b3c01e1b 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ValidationTester.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ValidationTester.java
@@ -6,6 +6,7 @@ import com.yahoo.collections.Pair;
 import com.yahoo.config.application.api.ApplicationPackage;
 import com.yahoo.config.model.api.ConfigChangeAction;
 import com.yahoo.config.model.api.HostProvisioner;
+import com.yahoo.config.model.api.Provisioned;
 import com.yahoo.config.model.deploy.DeployState;
 import com.yahoo.config.model.deploy.TestProperties;
 import com.yahoo.config.model.provision.InMemoryProvisioner;
@@ -32,7 +33,7 @@ import static com.yahoo.config.model.test.MockApplicationPackage.MUSIC_SEARCHDEF
  */
 public class ValidationTester {
 
-    private final HostProvisioner hostProvisioner;
+    private final InMemoryProvisioner hostProvisioner;
 
     /** Creates a validation tester with 1 node available */
     public ValidationTester() {
@@ -45,7 +46,7 @@ public class ValidationTester {
     }
 
     /** Creates a validation tester with a given host provisioner */
-    public ValidationTester(HostProvisioner hostProvisioner) {
+    public ValidationTester(InMemoryProvisioner hostProvisioner) {
         this.hostProvisioner = hostProvisioner;
     }
 
@@ -63,9 +64,10 @@ public class ValidationTester {
                                                              Environment environment,
                                                              String validationOverrides) {
         Instant now = LocalDate.parse("2000-01-01", DateTimeFormatter.ISO_DATE).atStartOfDay().atZone(ZoneOffset.UTC).toInstant();
+        Provisioned provisioned = hostProvisioner.startProvisionedRecording();
         ApplicationPackage newApp = new MockApplicationPackage.Builder()
                 .withServices(services)
-                .withSearchDefinitions(ImmutableList.of(MUSIC_SEARCHDEFINITION, BOOK_SEARCHDEFINITION))
+                .withSchemas(ImmutableList.of(MUSIC_SEARCHDEFINITION, BOOK_SEARCHDEFINITION))
                 .withValidationOverrides(validationOverrides)
                 .build();
         VespaModelCreatorWithMockPkg newModelCreator = new VespaModelCreatorWithMockPkg(newApp);
@@ -77,6 +79,7 @@ public class ValidationTester {
                                                              .applicationPackage(newApp)
                                                              .properties(new TestProperties().setHostedVespa(true))
                                                              .modelHostProvisioner(hostProvisioner)
+                                                             .provisioned(provisioned)
                                                              .now(now);
         if (previousModel != null)
             deployStateBuilder.previousModel(previousModel);
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ClusterSizeReductionValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ClusterSizeReductionValidatorTest.java
index 895aa4f6a36..87684aca174 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ClusterSizeReductionValidatorTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ClusterSizeReductionValidatorTest.java
@@ -27,7 +27,9 @@ public class ClusterSizeReductionValidatorTest {
             fail("Expected exception due to cluster size reduction");
         }
         catch (IllegalArgumentException expected) {
-            assertEquals("cluster-size-reduction: Size reduction in 'default' is too large. Current size: 30, new size: 14. New size must be at least 50% of the current size. " +
+            assertEquals("cluster-size-reduction: Size reduction in 'default' is too large: " +
+                         "New min size must be at least 50% of the current min size. " +
+                         "Current size: 30, new size: 14. " +
                          ValidationOverrides.toAllowMessage(ValidationId.clusterSizeReduction),
                          Exceptions.toMessageString(expected));
         }
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ConfigValueChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ConfigValueChangeValidatorTest.java
index bb209e58e24..81b7f870a10 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ConfigValueChangeValidatorTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ConfigValueChangeValidatorTest.java
@@ -201,7 +201,7 @@ public class ConfigValueChangeValidatorTest {
                         "       </engine>\n" +
                         "   </content>\n" +
                         "</services>",
-                createSearchDefinitions(docTypes)
+                createSchemas(docTypes)
         ).create();
     }
 
@@ -213,7 +213,7 @@ public class ConfigValueChangeValidatorTest {
                 "</documents>";
     }
 
-    private static List<String> createSearchDefinitions(List<String> docTypes) {
+    private static List<String> createSchemas(List<String> docTypes) {
         return docTypes.stream()
                 .map(type -> "search " + type + " { document " + type + " { } }")
                 .collect(Collectors.toList());
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/IndexedSearchClusterChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/IndexedSearchClusterChangeValidatorTest.java
index 43ad1bc0e8a..80127ac6854 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/IndexedSearchClusterChangeValidatorTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/IndexedSearchClusterChangeValidatorTest.java
@@ -7,7 +7,7 @@ import com.yahoo.vespa.model.VespaModel;
 import com.yahoo.config.application.api.ValidationOverrides;
 import com.yahoo.vespa.model.content.utils.ApplicationPackageBuilder;
 import com.yahoo.vespa.model.content.utils.ContentClusterBuilder;
-import com.yahoo.vespa.model.content.utils.SearchDefinitionBuilder;
+import com.yahoo.vespa.model.content.utils.SchemaBuilder;
 import org.junit.Test;
 
 import java.time.Instant;
@@ -41,7 +41,7 @@ public class IndexedSearchClusterChangeValidatorTest {
         public static VespaModel newOneDocModel(String sdContent) {
             return new ApplicationPackageBuilder().
                     addCluster(new ContentClusterBuilder().name("foo").docTypes("d1")).
-                    addSearchDefinition(new SearchDefinitionBuilder().
+                                                                                              addSchemas(new SchemaBuilder().
                             name("d1").content(sdContent).build()).buildCreator().create();
         }
 
@@ -52,9 +52,9 @@ public class IndexedSearchClusterChangeValidatorTest {
         public static VespaModel newTwoDocModel(String d1Content, String d2Content) {
             return new ApplicationPackageBuilder().
                     addCluster(new ContentClusterBuilder().name("foo").docTypes("d1", "d2")).
-                    addSearchDefinition(new SearchDefinitionBuilder().
+                                                                                                    addSchemas(new SchemaBuilder().
                             name("d1").content(d1Content).build()).
-                    addSearchDefinition(new SearchDefinitionBuilder().
+                                                                          addSchemas(new SchemaBuilder().
                             name("d2").content(d2Content).build()).
                     buildCreator().create();
         }
@@ -67,9 +67,9 @@ public class IndexedSearchClusterChangeValidatorTest {
             return new ApplicationPackageBuilder().
                     addCluster(new ContentClusterBuilder().name("foo").docTypes("d1")).
                     addCluster(new ContentClusterBuilder().name("bar").docTypes("d2")).
-                    addSearchDefinition(new SearchDefinitionBuilder().
+                                                                                              addSchemas(new SchemaBuilder().
                             name("d1").content(d1Content).build()).
-                    addSearchDefinition(new SearchDefinitionBuilder().
+                                                                          addSchemas(new SchemaBuilder().
                             name("d2").content(d2Content).build()).
                     buildCreator().create();
         }
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidatorTest.java
index 1322a9061ed..9a363789798 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidatorTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidatorTest.java
@@ -30,7 +30,7 @@ public class ResourcesReductionValidatorTest {
             fail("Expected exception due to resources reduction");
         } catch (IllegalArgumentException expected) {
             assertEquals("resources-reduction: Resource reduction in 'default' is too large. " +
-                            "Current memory GB: 64.00, new: 16.00. New resources must be at least 50% of the current resources. " +
+                            "Current memory GB: 64.00, new: 16.00. New min resources must be at least 50% of the current min resources. " +
                             ValidationOverrides.toAllowMessage(ValidationId.resourcesReduction),
                     Exceptions.toMessageString(expected));
         }
@@ -45,7 +45,7 @@ public class ResourcesReductionValidatorTest {
         } catch (IllegalArgumentException expected) {
             assertEquals("resources-reduction: Resource reduction in 'default' is too large. " +
                             "Current vCPU: 8.00, new: 3.00. Current memory GB: 64.00, new: 16.00. Current disk GB: 800.00, new: 200.00. " +
-                            "New resources must be at least 50% of the current resources. " +
+                            "New min resources must be at least 50% of the current min resources. " +
                             ValidationOverrides.toAllowMessage(ValidationId.resourcesReduction),
                     Exceptions.toMessageString(expected));
         }
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidatorTest.java
index c5fee3efa99..8edbc964bfb 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidatorTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidatorTest.java
@@ -8,7 +8,7 @@ import com.yahoo.config.application.api.ValidationOverrides;
 import com.yahoo.vespa.model.content.utils.ApplicationPackageBuilder;
 import com.yahoo.vespa.model.content.utils.ContentClusterBuilder;
 import com.yahoo.vespa.model.content.utils.DocType;
-import com.yahoo.vespa.model.content.utils.SearchDefinitionBuilder;
+import com.yahoo.vespa.model.content.utils.SchemaBuilder;
 import org.junit.Test;
 
 import java.time.Instant;
@@ -40,7 +40,7 @@ public class StreamingSearchClusterChangeValidatorTest {
         public static VespaModel createOneDocModel(String sdContent) {
             return new ApplicationPackageBuilder()
                     .addCluster(new ContentClusterBuilder().name("foo").docTypes(Arrays.asList(DocType.streaming("d1"))))
-                    .addSearchDefinition(new SearchDefinitionBuilder().name("d1").content(sdContent).build())
+                    .addSchemas(new SchemaBuilder().name("d1").content(sdContent).build())
                     .buildCreator().create();
         }
 
@@ -51,8 +51,8 @@ public class StreamingSearchClusterChangeValidatorTest {
         public static VespaModel createTwoDocModel(String d1Content, String d2Content) {
             return new ApplicationPackageBuilder()
                     .addCluster(new ContentClusterBuilder().name("foo").docTypes(Arrays.asList(DocType.streaming("d1"), DocType.streaming("d2"))))
-                    .addSearchDefinition(new SearchDefinitionBuilder().name("d1").content(d1Content).build())
-                    .addSearchDefinition(new SearchDefinitionBuilder().name("d2").content(d2Content).build())
+                    .addSchemas(new SchemaBuilder().name("d1").content(d1Content).build())
+                    .addSchemas(new SchemaBuilder().name("d2").content(d2Content).build())
                     .buildCreator().create();
         }
 
@@ -64,8 +64,8 @@ public class StreamingSearchClusterChangeValidatorTest {
             return new ApplicationPackageBuilder()
                     .addCluster(new ContentClusterBuilder().name("foo").docTypes(Arrays.asList(DocType.streaming("d1"))))
                     .addCluster(new ContentClusterBuilder().name("bar").docTypes(Arrays.asList(DocType.streaming("d2"))))
-                    .addSearchDefinition(new SearchDefinitionBuilder().name("d1").content(d1Content).build())
-                    .addSearchDefinition(new SearchDefinitionBuilder().name("d2").content(d2Content).build())
+                    .addSchemas(new SchemaBuilder().name("d1").content(d1Content).build())
+                    .addSchemas(new SchemaBuilder().name("d2").content(d2Content).build())
                     .buildCreator().create();
         }
 
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/ContentClusterFixture.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/ContentClusterFixture.java
index 3827da08679..20ff9afd530 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/ContentClusterFixture.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/ContentClusterFixture.java
@@ -6,7 +6,7 @@ import com.yahoo.vespa.model.application.validation.change.VespaConfigChangeActi
 import com.yahoo.vespa.model.content.cluster.ContentCluster;
 import com.yahoo.vespa.model.content.utils.ContentClusterBuilder;
 import com.yahoo.vespa.model.content.utils.ContentClusterUtils;
-import com.yahoo.vespa.model.content.utils.SearchDefinitionBuilder;
+import com.yahoo.vespa.model.content.utils.SchemaBuilder;
 import com.yahoo.vespa.model.search.DocumentDatabase;
 
 import java.util.Arrays;
@@ -34,7 +34,7 @@ public abstract class ContentClusterFixture {
     private static ContentCluster createCluster(String sdContent) throws Exception {
         return new ContentClusterBuilder().build(
                 ContentClusterUtils.createMockRoot(
-                        Arrays.asList(new SearchDefinitionBuilder().content(sdContent).build())));
+                        Arrays.asList(new SchemaBuilder().content(sdContent).build())));
     }
 
     protected DocumentDatabase currentDb() {
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidatorTest.java
index 4064e53dfb7..3dfcef70aba 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidatorTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidatorTest.java
@@ -203,6 +203,7 @@ public class DocumentTypeChangeValidatorTest {
                 headerfields,
                 new StructDataType("bodyfields"),
                 new FieldSets(),
+                Collections.emptySet(),
                 Collections.emptySet());
     }
 
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/first/AccessControlOnFirstDeploymentValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/first/AccessControlOnFirstDeploymentValidatorTest.java
index c8f1224693b..d1a78a548d9 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/first/AccessControlOnFirstDeploymentValidatorTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/first/AccessControlOnFirstDeploymentValidatorTest.java
@@ -1,172 +1,21 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.model.application.validation.first;
 
-import com.yahoo.config.application.api.ApplicationPackage;
-import com.yahoo.config.model.NullConfigModelRegistry;
-import com.yahoo.config.model.deploy.DeployState;
-import com.yahoo.config.model.deploy.TestProperties;
-import com.yahoo.config.model.test.MockApplicationPackage;
 import com.yahoo.config.provision.Environment;
 import com.yahoo.config.provision.RegionName;
 import com.yahoo.config.provision.Zone;
-import com.yahoo.vespa.model.VespaModel;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.ExpectedException;
-import org.xml.sax.SAXException;
-
-import java.io.IOException;
-import java.time.Instant;
-import java.time.LocalDate;
-import java.time.ZoneOffset;
-import java.time.format.DateTimeFormatter;
-
-import static com.yahoo.config.model.test.TestUtil.joinLines;
-import static com.yahoo.config.provision.Environment.prod;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
+import com.yahoo.vespa.model.application.validation.AccessControlValidatorTestBase;
+import org.junit.Before;
 
 /**
  * @author gjoranv
  */
-public class AccessControlOnFirstDeploymentValidatorTest {
-
-    @Rule
-    public final ExpectedException exceptionRule = ExpectedException.none();
-
-    private static String servicesXml(boolean addHandler, boolean writeProtection) {
-        return joinLines("<services version='1.0'>",
-                         "  <container id='default' version='1.0'>",
-                         addHandler ? httpHandlerXml : "",
-                         "    <http>",
-                         "      <filtering>",
-                         "        <access-control domain='foo' write='" + writeProtection + "' />",
-                         "      </filtering>",
-                         "    </http>",
-                         "  </container>",
-                         "</services>");
-    }
-
-    private static final String httpHandlerXml =
-            joinLines("    <handler id='foo'>",
-                      "      <binding>http://foo/bar</binding>",
-                      "    </handler>");
-
-    @Test
-    public void cluster_with_write_protection_passes_validation() throws IOException, SAXException{
-        DeployState deployState = deployState(servicesXml(true, true));
-        VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState);
-
-        new AccessControlOnFirstDeploymentValidator().validate(model, deployState);
-    }
-
-    @Test
-    public void cluster_with_no_handlers_passes_validation_without_write_protection() throws IOException, SAXException{
-        DeployState deployState = deployState(servicesXml(false, false));
-        VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState);
-
-        new AccessControlOnFirstDeploymentValidator().validate(model, deployState);
-    }
-
-    @Test
-    public void cluster_without_custom_components_passes_validation_without_write_protection() throws IOException, SAXException{
-        String servicesXml = joinLines("<services version='1.0'>",
-                                       "  <container id='default' version='1.0' />",
-                                       "</services>");
-        DeployState deployState = deployState(servicesXml);
-        VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState);
-
-        new AccessControlOnFirstDeploymentValidator().validate(model, deployState);
-    }
-
-    @Test
-    public void cluster_with_handler_fails_validation_without_write_protection() throws IOException, SAXException{
-        exceptionRule.expect(IllegalArgumentException.class);
-        exceptionRule.expectMessage(
-                "Access-control must be enabled for write operations to container clusters in production zones: [default]");
-
-        DeployState deployState = deployState(servicesXml(true, false));
-        VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState);
-
-        new AccessControlOnFirstDeploymentValidator().validate(model, deployState);
-    }
-
-    @Test
-    public void no_http_element_has_same_effect_as_no_write_protection() throws IOException, SAXException{
-        exceptionRule.expect(IllegalArgumentException.class);
-        exceptionRule.expectMessage(
-                "Access-control must be enabled for write operations to container clusters in production zones: [default]");
-
-        String servicesXml = joinLines("<services version='1.0'>",
-                                       "  <container id='default' version='1.0'>",
-                                       httpHandlerXml,
-                                       "  </container>",
-                                       "</services>");
-        DeployState deployState = deployState(servicesXml);
-        VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState);
-
-        new AccessControlOnFirstDeploymentValidator().validate(model, deployState);
-    }
-
-    @Test
-    public void cluster_with_mbus_handler_passes_validation_without_write_protection() throws IOException, SAXException{
-        String servicesXml = joinLines("<services version='1.0'>",
-                                       "  <container id='default' version='1.0'>",
-                                       "    <handler id='foo'>",
-                                       "      <binding>mbus://*/foo</binding>",
-                                       "    </handler>",
-                                       "  </container>",
-                                       "</services>");
-        DeployState deployState = deployState(servicesXml);
-        VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState);
-
-        new AccessControlOnFirstDeploymentValidator().validate(model, deployState);
-    }
-
-    @Test
-    public void write_protection_is_not_required_for_non_default_application_type() throws IOException, SAXException{
-        String servicesXml = joinLines("<services version='1.0' application-type='hosted-infrastructure'>",
-                                       "  <container id='default' version='1.0'>",
-                                       httpHandlerXml,
-                                       "  </container>",
-                                       "</services>");
-        DeployState deployState = deployState(servicesXml);
-        VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState);
-
-        new AccessControlOnFirstDeploymentValidator().validate(model, deployState);
-    }
-
-    @Test
-    public void write_protection_is_not_required_with_validation_override() throws IOException, SAXException{
-        DeployState deployState = deployState(servicesXml(true, false),
-                                              "<validation-overrides><allow until='2000-01-30'>access-control</allow></validation-overrides>",
-                                              LocalDate.parse("2000-01-01", DateTimeFormatter.ISO_DATE).atStartOfDay().atZone(ZoneOffset.UTC).toInstant());
-        VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState);
-
-        new AccessControlOnFirstDeploymentValidator().validate(model, deployState);
-    }
-
-    private static DeployState deployState(String servicesXml) {
-        return deployState(servicesXml, "<validation-overrides></validation-overrides>", Instant.now());
-    }
-
-    private static DeployState deployState(String servicesXml, String validationOverrides, Instant now) {
-        ApplicationPackage app = new MockApplicationPackage.Builder()
-                .withServices(servicesXml)
-                .withValidationOverrides(validationOverrides)
-                .build();
-
-        DeployState.Builder builder = new DeployState.Builder()
-                .applicationPackage(app)
-                .zone(new Zone(Environment.prod, RegionName.from("foo")) )
-                .properties(new TestProperties().setHostedVespa(true))
-                .now(now);
-        final DeployState deployState = builder.build();
-
-        assertTrue("Test must emulate a hosted deployment.", deployState.isHosted());
-        assertEquals("Test must emulate a prod environment.", prod, deployState.zone().environment());
+public class AccessControlOnFirstDeploymentValidatorTest extends AccessControlValidatorTestBase {
 
-        return deployState;
+    @Before
+    public void setup() {
+        validator = new AccessControlOnFirstDeploymentValidator();
+        zone = new Zone(Environment.prod, RegionName.from("foo"));
     }
 
 }
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java
index 31a2a6496d7..b4fe4175707 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java
@@ -236,8 +236,8 @@ public class ContentBuilderTest extends DomBuilderTest {
     }
 
     @Test
-    public void requireThatContentStreamingHandlesMultipleSearchDefinitions() {
-        final String musicClusterId = "music-cluster-id";
+    public void requireThatContentStreamingHandlesMultipleSchemas() {
+        String musicClusterId = "music-cluster-id";
 
         ContentCluster cluster = createContentWithBooksToo(
                 "<content version='1.0' id='" + musicClusterId + "'>" +
@@ -825,8 +825,8 @@ public class ContentBuilderTest extends DomBuilderTest {
         VespaModel m = new VespaModelCreatorWithMockPkg(new MockApplicationPackage.Builder()
                                                                 .withHosts(getHosts())
                                                                 .withServices(combined)
-                                                                .withSearchDefinitions(Arrays.asList(MockApplicationPackage.MUSIC_SEARCHDEFINITION,
-                                                                                                     MockApplicationPackage.BOOK_SEARCHDEFINITION))
+                                                                .withSchemas(Arrays.asList(MockApplicationPackage.MUSIC_SEARCHDEFINITION,
+                                                                                           MockApplicationPackage.BOOK_SEARCHDEFINITION))
                                                                 .build())
                 .create();
 
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomSearchTuningBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomSearchTuningBuilderTest.java
index 060fff5bf66..73dd1ca3f3b 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomSearchTuningBuilderTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomSearchTuningBuilderTest.java
@@ -4,7 +4,6 @@ package com.yahoo.vespa.model.builder.xml.dom;
 import com.yahoo.collections.CollectionUtil;
 import com.yahoo.vespa.config.search.core.ProtonConfig;
 import com.yahoo.config.model.builder.xml.test.DomBuilderTest;
-import com.yahoo.vespa.model.content.DispatchTuning;
 import com.yahoo.vespa.model.search.Tuning;
 import org.junit.Test;
 import org.w3c.dom.Element;
@@ -228,6 +227,8 @@ public class DomSearchTuningBuilderTest extends DomBuilderTest {
         assertEquals(cfg.summary().log().chunk().maxbytes(), 256);
         assertEquals(cfg.summary().log().chunk().compression().type(), ProtonConfig.Summary.Log.Chunk.Compression.Type.LZ4);
         assertEquals(cfg.summary().log().chunk().compression().level(), 5);
+        assertEquals(cfg.summary().log().compact().compression().type(), ProtonConfig.Summary.Log.Compact.Compression.Type.LZ4);
+        assertEquals(cfg.summary().log().compact().compression().level(), 5);
     }
 
     @Test
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java
index d1c5e344c24..33cf0635349 100755
--- a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java
@@ -97,6 +97,7 @@ public class ContainerClusterTest {
         cluster.getConfig(qsB);
         QrStartConfig qsC= new QrStartConfig(qsB);
         assertEquals(expectedMemoryPercentage, qsC.jvm().heapSizeAsPercentageOfPhysicalMemory());
+        assertEquals(0, qsC.jvm().compressedClassSpaceSize());
     }
 
     @Test
@@ -156,6 +157,7 @@ public class ContainerClusterTest {
         QrStartConfig qrStartConfig = new QrStartConfig(qrBuilder);
         assertEquals(32, qrStartConfig.jvm().minHeapsize());
         assertEquals(512, qrStartConfig.jvm().heapsize());
+        assertEquals(32, qrStartConfig.jvm().compressedClassSpaceSize());
         assertEquals(0, qrStartConfig.jvm().heapSizeAsPercentageOfPhysicalMemory());
 
         ThreadpoolConfig.Builder tpBuilder = new ThreadpoolConfig.Builder();
@@ -200,6 +202,34 @@ public class ContainerClusterTest {
     }
 
     @Test
+    public void requireThatSoftStartSecondsCanBeControlledByProperties() {
+        DeployState state = new DeployState.Builder().properties(new TestProperties().setSoftStartSeconds(300.0))
+                .build();
+        MockRoot root = new MockRoot("foo", state);
+        ApplicationContainerCluster cluster = createContainerCluster(root, false);
+        addContainer(root.deployLogger(), cluster, "c1", "host-c1");
+
+        ThreadpoolConfig.Builder tpBuilder = new ThreadpoolConfig.Builder();
+        cluster.getConfig(tpBuilder);
+        ThreadpoolConfig threadpoolConfig = new ThreadpoolConfig(tpBuilder);
+        assertEquals(500, threadpoolConfig.maxthreads());
+        assertEquals(300.0, threadpoolConfig.softStartSeconds(), 0.0);
+    }
+
+    @Test
+    public void requireThatDefaultThreadPoolConfigIsSane() {
+        MockRoot root = new MockRoot("foo");
+        ApplicationContainerCluster cluster = createContainerCluster(root, false);
+        addContainer(root.deployLogger(), cluster, "c1", "host-c1");
+
+        ThreadpoolConfig.Builder tpBuilder = new ThreadpoolConfig.Builder();
+        cluster.getConfig(tpBuilder);
+        ThreadpoolConfig threadpoolConfig = new ThreadpoolConfig(tpBuilder);
+        assertEquals(500, threadpoolConfig.maxthreads());
+        assertEquals(0.0, threadpoolConfig.softStartSeconds(), 0.0);
+    }
+
+    @Test
     public void requireThatRoutingProviderIsDisabledForNonHosted() {
         DeployState state = new DeployState.Builder().properties(new TestProperties().setHostedVespa(false)).build();
         MockRoot root = new MockRoot("foo", state);
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessControlTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessControlTest.java
index 7cd26b2f1e6..28e23ce3222 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessControlTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessControlTest.java
@@ -3,8 +3,13 @@ package com.yahoo.vespa.model.container.xml;
 
 import com.google.common.collect.ImmutableSet;
 import com.yahoo.collections.CollectionUtil;
+import com.yahoo.component.ComponentId;
 import com.yahoo.config.model.builder.xml.test.DomBuilderTest;
+import com.yahoo.config.model.deploy.DeployState;
+import com.yahoo.config.model.deploy.TestProperties;
+import com.yahoo.config.provision.AthenzDomain;
 import com.yahoo.container.jdisc.state.StateHandler;
+import com.yahoo.vespa.model.container.ApplicationContainer;
 import com.yahoo.vespa.model.container.ContainerCluster;
 import com.yahoo.vespa.model.container.http.AccessControl;
 import com.yahoo.vespa.model.container.http.Http;
@@ -16,14 +21,19 @@ import org.w3c.dom.Element;
 
 import java.util.Collection;
 import java.util.HashSet;
+import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
 
 import static com.yahoo.config.model.test.TestUtil.joinLines;
+import static com.yahoo.vespa.defaults.Defaults.getDefaults;
+import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.Matchers.equalTo;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotEquals;
 import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertThat;
 import static org.junit.Assert.assertTrue;
 
 /**
@@ -64,10 +74,7 @@ public class AccessControlTest extends ContainerModelBuilderTestBase {
         Element clusterElem = DomBuilderTest.parse(
                 "  <http>",
                 "    <filtering>",
-                "      <access-control domain='my-domain'>",
-                "        <application>my-app</application>",
-                "        <vespa-domain>custom-vespa-domain</vespa-domain>",
-                "      </access-control>",
+                "      <access-control domain='my-domain'/>",
                 "    </filtering>",
                 "  </http>");
 
@@ -76,8 +83,6 @@ public class AccessControlTest extends ContainerModelBuilderTestBase {
         AccessControl accessControl = http.getAccessControl().get();
 
         assertEquals("Wrong domain.", "my-domain", accessControl.domain);
-        assertEquals("Wrong application.", "my-app", accessControl.applicationId);
-        assertEquals("Wrong vespa-domain.", "custom-vespa-domain", accessControl.vespaDomain.get());
     }
 
     @Test
@@ -234,6 +239,58 @@ public class AccessControlTest extends ContainerModelBuilderTestBase {
 
     }
 
+
+    @Test
+    public void access_control_is_implicitly_added_for_hosted_apps() {
+        Element clusterElem = DomBuilderTest.parse(
+                "<container version='1.0'>",
+                nodesXml,
+                "</container>" );
+        AthenzDomain tenantDomain = AthenzDomain.from("my-tenant-domain");
+        DeployState state = new DeployState.Builder().properties(
+                new TestProperties()
+                        .setAthenzDomain(tenantDomain)
+                        .setHostedVespa(true))
+                .build();
+        createModel(root, state, null, clusterElem);
+        Optional<AccessControl> maybeAccessControl =
+                ((ApplicationContainer) root.getProducer("container/container.0")).getHttp().getAccessControl();
+        assertThat(maybeAccessControl.isPresent(), is(true));
+        AccessControl accessControl = maybeAccessControl.get();
+        assertThat(accessControl.writeEnabled, is(false));
+        assertThat(accessControl.readEnabled, is(false));
+        assertThat(accessControl.domain, equalTo(tenantDomain.value()));
+    }
+
+    @Test
+    public void access_control_is_implicitly_added_for_hosted_apps_with_existing_http_element() {
+        Element clusterElem = DomBuilderTest.parse(
+                "<container version='1.0'>",
+                "  <http>",
+                "    <server port='" + getDefaults().vespaWebServicePort() + "' id='main' />",
+                "    <filtering>",
+                "      <filter id='outer' />",
+                "      <request-chain id='myChain'>",
+                "        <filter id='inner' />",
+                "      </request-chain>",
+                "    </filtering>",
+                "  </http>",
+                nodesXml,
+                "</container>" );
+        AthenzDomain tenantDomain = AthenzDomain.from("my-tenant-domain");
+        DeployState state = new DeployState.Builder().properties(
+                new TestProperties()
+                        .setAthenzDomain(tenantDomain)
+                        .setHostedVespa(true))
+                .build();
+        createModel(root, state, null, clusterElem);
+        Http http = ((ApplicationContainer) root.getProducer("container/container.0")).getHttp();
+        assertThat(http.getAccessControl().isPresent(), is(true));
+        assertThat(http.getFilterChains().hasChain(AccessControl.ACCESS_CONTROL_CHAIN_ID), is(true));
+        assertThat(http.getFilterChains().hasChain(ComponentId.fromString("myChain")), is(true));
+    }
+
+
     private String httpWithExcludedBinding(String excludedBinding) {
         return joinLines(
                 "  <http>",
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessLogTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessLogTest.java
index 4ea10c9a1c1..00ab175f496 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessLogTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessLogTest.java
@@ -18,30 +18,22 @@ import static org.junit.Assert.assertEquals;
 
 /**
  * @author gjoranv
- * @since 5.5
  */
 public class AccessLogTest extends ContainerModelBuilderTestBase {
 
     @Test
-    public void default_access_log_is_only_added_when_search_is_present() {
+    public void default_access_log_is_added_by_default() {
         Element cluster1Elem = DomBuilderTest.parse(
                 "<container id='cluster1' version='1.0'>",
-                "<search />",
-                nodesXml,
-                "</container>");
-        Element cluster2Elem = DomBuilderTest.parse(
-                "<container id='cluster2' version='1.0'>",
                 "  <nodes>",
                 "    <node hostalias='mockhost' baseport='1234' />",
                 "  </nodes>",
                 "</container>" );
 
-        createModel(root, cluster1Elem, cluster2Elem);
+        createModel(root, cluster1Elem);
 
         assertNotNull(getJsonAccessLog("cluster1"));
-        assertNull(   getJsonAccessLog("cluster2"));
         assertNull(getVespaAccessLog("cluster1"));
-        assertNull(getVespaAccessLog("cluster2"));
     }
 
     @Test
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java
index 1bbc4ea2684..dcd1c46e52f 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java
@@ -25,6 +25,7 @@ import com.yahoo.container.QrConfig;
 import com.yahoo.container.core.ChainsConfig;
 import com.yahoo.container.core.VipStatusConfig;
 import com.yahoo.container.handler.VipStatusHandler;
+import com.yahoo.container.handler.metrics.MetricsV2Handler;
 import com.yahoo.container.handler.observability.ApplicationStatusHandler;
 import com.yahoo.container.jdisc.JdiscBindingsConfig;
 import com.yahoo.container.servlet.ServletConfigConfig;
@@ -39,7 +40,6 @@ import com.yahoo.vespa.defaults.Defaults;
 import com.yahoo.vespa.model.AbstractService;
 import com.yahoo.vespa.model.VespaModel;
 import com.yahoo.vespa.model.container.ApplicationContainer;
-import com.yahoo.vespa.model.container.Container;
 import com.yahoo.vespa.model.container.ContainerCluster;
 import com.yahoo.vespa.model.container.SecretStore;
 import com.yahoo.vespa.model.container.component.Component;
@@ -47,7 +47,6 @@ import com.yahoo.vespa.model.container.http.ConnectorFactory;
 import com.yahoo.vespa.model.content.utils.ContentClusterUtils;
 import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithFilePkg;
 import org.hamcrest.Matchers;
-import org.junit.Ignore;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.TemporaryFolder;
@@ -66,12 +65,17 @@ import java.util.stream.Collectors;
 import static com.yahoo.config.model.test.TestUtil.joinLines;
 import static com.yahoo.test.LinePatternMatcher.containsLineWithPattern;
 import static com.yahoo.vespa.defaults.Defaults.getDefaults;
+import static com.yahoo.vespa.model.container.ContainerCluster.ROOT_HANDLER_BINDING;
+import static com.yahoo.vespa.model.container.ContainerCluster.STATE_HANDLER_BINDING_1;
 import static org.hamcrest.CoreMatchers.is;
 import static org.hamcrest.CoreMatchers.not;
+import static org.hamcrest.CoreMatchers.notNullValue;
 import static org.hamcrest.CoreMatchers.nullValue;
 import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasItem;
+import static org.hamcrest.Matchers.isEmptyString;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotNull;
@@ -149,7 +153,6 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase {
     }
 
     @Test
-    @Ignore // TODO: Enable when turning the port check on
     public void fail_if_http_port_is_not_default_in_hosted_vespa() throws Exception {
         try {
             String servicesXml =
@@ -224,26 +227,49 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase {
         assertThat(defaultRootHandler.serverBindings(), contains("http://*/"));
 
         JdiscBindingsConfig.Handlers applicationStatusHandler = config.handlers(ApplicationStatusHandler.class.getName());
-        assertThat(applicationStatusHandler.serverBindings(),
-                   contains("http://*/ApplicationStatus"));
+        assertThat(applicationStatusHandler.serverBindings(), contains("http://*/ApplicationStatus"));
 
         JdiscBindingsConfig.Handlers fileRequestHandler = config.handlers(VipStatusHandler.class.getName());
-        assertThat(fileRequestHandler.serverBindings(),
-                   contains("http://*/status.html"));
+        assertThat(fileRequestHandler.serverBindings(), contains("http://*/status.html"));
+
+        JdiscBindingsConfig.Handlers metricsV2Handler = config.handlers(MetricsV2Handler.class.getName());
+        assertThat(metricsV2Handler.serverBindings(), contains("http://*/metrics/v2", "http://*/metrics/v2/*"));
     }
 
     @Test
-    public void default_root_handler_is_disabled_when_user_adds_a_handler_with_same_binding() {
+    public void default_root_handler_binding_can_be_stolen_by_user_configured_handler() {
         Element clusterElem = DomBuilderTest.parse(
                 "<container id='default' version='1.0'>" +
                         "  <handler id='userRootHandler'>" +
-                        "    <binding>" + ContainerCluster.ROOT_HANDLER_BINDING + "</binding>" +
+                        "    <binding>" + ROOT_HANDLER_BINDING + "</binding>" +
                         "  </handler>" +
                         "</container>");
         createModel(root, clusterElem);
 
+        // The handler is still set up.
         ComponentsConfig.Components userRootHandler = getComponent(componentsConfig(), BindingsOverviewHandler.class.getName());
-        assertThat(userRootHandler, nullValue());
+        assertThat(userRootHandler, notNullValue());
+
+        // .. but it has no bindings
+        var discBindingsConfig = root.getConfig(JdiscBindingsConfig.class, "default");
+        assertThat(discBindingsConfig.handlers(BindingsOverviewHandler.class.getName()), is(nullValue()));
+    }
+
+    @Test
+    public void reserved_binding_cannot_be_stolen_by_user_configured_handler() {
+        Element clusterElem = DomBuilderTest.parse(
+                "<container id='default' version='1.0'>" +
+                        "  <handler id='userHandler'>" +
+                        "    <binding>" + STATE_HANDLER_BINDING_1 + "</binding>" +
+                        "  </handler>" +
+                        "</container>");
+        try {
+            createModel(root, clusterElem);
+            fail("Expected exception when stealing a reserved binding.");
+        } catch (IllegalArgumentException e) {
+            assertThat(e.getMessage(), is("Binding 'http://*/state/v1' is a reserved Vespa binding " +
+                                                  "and cannot be used by handler: userHandler"));
+        }
     }
 
     @Test
@@ -598,7 +624,7 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase {
                         .setMultitenant(true)
                         .setHostedVespa(true))
                 .build());
-        assertEquals(1, model.hostSystem().getHosts().size());
+        assertEquals(2, model.hostSystem().getHosts().size());
     }
 
     @Test(expected = IllegalArgumentException.class)
@@ -783,7 +809,7 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase {
         ApplicationContainer container = (ApplicationContainer)root.getProducer("container/container.0");
 
         // Verify that there are two connectors
-        List<ConnectorFactory> connectorFactories = container.getHttp().getHttpServer().getConnectorFactories();
+        List<ConnectorFactory> connectorFactories = container.getHttp().getHttpServer().get().getConnectorFactories();
         assertEquals(2, connectorFactories.size());
         List<Integer> ports = connectorFactories.stream()
                 .map(ConnectorFactory::getListenPort)
@@ -801,6 +827,10 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase {
         assertEquals("CERT", connectorConfig.ssl().certificate());
         assertEquals("KEY", connectorConfig.ssl().privateKey());
         assertEquals(4443, connectorConfig.listenPort());
+
+        assertThat("Connector must use Athenz truststore in a non-public system.",
+                   connectorConfig.ssl().caCertificateFile(), equalTo("/opt/yahoo/share/ssl/certs/athenz_certificate_bundle.pem"));
+        assertThat(connectorConfig.ssl().caCertificate(), isEmptyString());
     }
 
 
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/DocprocBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/DocprocBuilderTest.java
index 9ecd33f4273..eda90b03147 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/DocprocBuilderTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/DocprocBuilderTest.java
@@ -28,7 +28,11 @@ import java.util.Map;
 
 import static org.hamcrest.CoreMatchers.is;
 import static org.hamcrest.core.IsNull.notNullValue;
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertThat;
+
 
 /**
  * @author einarmr
@@ -204,24 +208,25 @@ public class DocprocBuilderTest extends DomBuilderTest {
 
     @Test
     public void testBundlesConfig() {
-        assertThat(bundlesConfig.bundle().size(), is(0));
+        assertTrue(bundlesConfig.bundle().isEmpty());
     }
 
     @Test
     public void testSchemaMappingConfig() {
-        assertThat(schemamappingConfig.fieldmapping().size(), is(0));
+        assertTrue(schemamappingConfig.fieldmapping().isEmpty());
     }
 
     @Test
     public void testQrStartConfig() {
         QrStartConfig.Jvm jvm = qrStartConfig.jvm();
-        assertThat(jvm.server(), is(true));
-        assertThat(jvm.verbosegc(), is(true));
-        assertThat(jvm.gcopts(), is("-XX:+UseG1GC -XX:MaxTenuringThreshold=15"));
-        assertThat(jvm.minHeapsize(), is(1536));
-        assertThat(jvm.heapsize(), is(1536));
-        assertThat(jvm.stacksize(), is(512));
-        assertThat(qrStartConfig.ulimitv(), is(""));
+        assertTrue(jvm.server());
+        assertTrue(jvm.verbosegc());
+        assertEquals("-XX:+UseG1GC -XX:MaxTenuringThreshold=15", jvm.gcopts());
+        assertEquals(1536, jvm.minHeapsize());
+        assertEquals(1536, jvm.heapsize());
+        assertEquals(512, jvm.stacksize());
+        assertTrue(qrStartConfig.ulimitv().isEmpty());
+        assertEquals(0, jvm.compressedClassSpaceSize());
     }
 
 }
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JettyContainerModelBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JettyContainerModelBuilderTest.java
index 68f507c810d..0f9bd506310 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JettyContainerModelBuilderTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JettyContainerModelBuilderTest.java
@@ -152,6 +152,14 @@ public class JettyContainerModelBuilderTest extends ContainerModelBuilderTestBas
                 "                <client-authentication>need</client-authentication>",
                 "            </ssl>",
                 "        </server>",
+                "        <server port='9003' id='with-ciphers-and-protocols'>",
+                "            <ssl>",
+                "                <private-key-file>/foo/key</private-key-file>",
+                "                <certificate-file>/foo/cert</certificate-file>",
+                "                <cipher-suites>TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384</cipher-suites>",
+                "                <protocols>TLSv1.3</protocols>",
+                "            </ssl>",
+                "        </server>",
                 "    </http>",
                 nodesXml,
                 "",
@@ -179,6 +187,13 @@ public class JettyContainerModelBuilderTest extends ContainerModelBuilderTestBas
         assertThat(needClientAuth.ssl().caCertificateFile(), is(equalTo("")));
         assertThat(needClientAuth.ssl().clientAuth(), is(equalTo(ConnectorConfig.Ssl.ClientAuth.Enum.NEED_AUTH)));
 
+        ConnectorConfig withCiphersAndProtocols = root.getConfig(ConnectorConfig.class, "default/http/jdisc-jetty/with-ciphers-and-protocols/configured-ssl-provider@with-ciphers-and-protocols");
+        assertTrue(withCiphersAndProtocols.ssl().enabled());
+        assertThat(withCiphersAndProtocols.ssl().privateKeyFile(), is(equalTo("/foo/key")));
+        assertThat(withCiphersAndProtocols.ssl().certificateFile(), is(equalTo("/foo/cert")));
+        assertThat(withCiphersAndProtocols.ssl().enabledCipherSuites(), is(equalTo(List.of("TLS_AES_128_GCM_SHA256", "TLS_AES_256_GCM_SHA384"))));
+        assertThat(withCiphersAndProtocols.ssl().enabledProtocols(), is(equalTo(List.of("TLSv1.3"))));
+
         ContainerCluster cluster = (ContainerCluster) root.getChildren().get("default");
         List<ConnectorFactory> connectorFactories = cluster.getChildrenByTypeRecursive(ConnectorFactory.class);
         connectorFactories.forEach(connectorFactory -> assertChildComponentExists(connectorFactory, ConfiguredFilebasedSslProvider.COMPONENT_CLASS));
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/SearchBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/SearchBuilderTest.java
index 1f0b0188681..47f37a75fd0 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/SearchBuilderTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/SearchBuilderTest.java
@@ -16,8 +16,11 @@ import org.w3c.dom.Element;
 
 import static com.yahoo.config.model.api.container.ContainerServiceType.QRSERVER;
 import static com.yahoo.test.Matchers.hasItemWithMethod;
+import static com.yahoo.vespa.model.container.xml.ContainerModelBuilder.SEARCH_HANDLER_BINDING;
+import static com.yahoo.vespa.model.container.xml.ContainerModelBuilder.SEARCH_HANDLER_CLASS;
 import static org.hamcrest.CoreMatchers.is;
 import static org.hamcrest.CoreMatchers.not;
+import static org.hamcrest.CoreMatchers.nullValue;
 import static org.hamcrest.Matchers.containsString;
 import static org.junit.Assert.*;
 
@@ -54,8 +57,6 @@ public class SearchBuilderTest extends ContainerModelBuilderTestBase {
         if (guiHandler == null) fail();
     }
 
-
-
     @Test
     public void search_handler_bindings_can_be_overridden() {
         Element clusterElem = DomBuilderTest.parse(
@@ -91,6 +92,25 @@ public class SearchBuilderTest extends ContainerModelBuilderTestBase {
         assertThat(discBindingsConfig, not(containsString("/search/*")));
     }
 
+    @Test
+    public void search_handler_binding_can_be_stolen_by_user_configured_handler() {
+        var myHandler = "replaces_search_handler";
+        Element clusterElem = DomBuilderTest.parse(
+                "<container id='default' version='1.0'>",
+                "  <search />",
+                "  <handler id='" + myHandler + "'>",
+                "    <binding>" + SEARCH_HANDLER_BINDING + "</binding>",
+                "  </handler>",
+                nodesXml,
+                "</container>");
+
+        createModel(root, clusterElem);
+
+        var discBindingsConfig = root.getConfig(JdiscBindingsConfig.class, "default");
+        assertThat(discBindingsConfig.handlers(myHandler).serverBindings(0), is(SEARCH_HANDLER_BINDING));
+        assertThat(discBindingsConfig.handlers(SEARCH_HANDLER_CLASS), is(nullValue()));
+    }
+
     // TODO: remove test when all containers are named 'container'
     @Test
     public void cluster_with_only_search_gets_qrserver_as_service_name() {
@@ -190,7 +210,7 @@ public class SearchBuilderTest extends ContainerModelBuilderTestBase {
 
 
     private VespaModel getVespaModelWithMusic(String hosts, String services) {
-        return new VespaModelCreatorWithMockPkg(hosts, services, ApplicationPackageUtils.generateSearchDefinitions("music")).create();
+        return new VespaModelCreatorWithMockPkg(hosts, services, ApplicationPackageUtils.generateSchemas("music")).create();
     }
 
     private String hostsXml() {
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java
index 305bcc1d7d5..4d5df7c1965 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java
@@ -26,7 +26,7 @@ import com.yahoo.vespa.model.content.cluster.ContentCluster;
 import com.yahoo.vespa.model.content.engines.ProtonEngine;
 import com.yahoo.vespa.model.content.utils.ContentClusterBuilder;
 import com.yahoo.vespa.model.content.utils.ContentClusterUtils;
-import com.yahoo.vespa.model.content.utils.SearchDefinitionBuilder;
+import com.yahoo.vespa.model.content.utils.SchemaBuilder;
 import com.yahoo.vespa.model.routing.DocumentProtocol;
 import com.yahoo.vespa.model.routing.Routing;
 import com.yahoo.vespa.model.test.utils.ApplicationPackageUtils;
@@ -218,7 +218,7 @@ public class ContentClusterTest extends ContentBaseTest {
             "\n" +
             "</services>";
 
-        List<String> sds = ApplicationPackageUtils.generateSearchDefinitions("type1", "type2");
+        List<String> sds = ApplicationPackageUtils.generateSchemas("type1", "type2");
         VespaModel model = (new VespaModelCreatorWithMockPkg(null, xml, sds)).create();
         assertEquals(2, model.getContentClusters().get("bar").getDocumentDefinitions().size());
         ContainerCluster cluster = model.getAdmin().getClusterControllers();
@@ -259,7 +259,7 @@ public class ContentClusterTest extends ContentBaseTest {
                         " </services>";
 
         DeployState.Builder deployStateBuilder = new DeployState.Builder().properties(properties);
-        List<String> sds = ApplicationPackageUtils.generateSearchDefinitions("type1");
+        List<String> sds = ApplicationPackageUtils.generateSchemas("type1");
         return (new VespaModelCreatorWithMockPkg(null, services, sds)).create(deployStateBuilder);
     }
     @Test
@@ -301,7 +301,7 @@ public class ContentClusterTest extends ContentBaseTest {
             "\n" +
             "</services>";
 
-        List<String> sds = ApplicationPackageUtils.generateSearchDefinitions("type1", "type2");
+        List<String> sds = ApplicationPackageUtils.generateSchemas("type1", "type2");
         VespaModel model = new VespaModelCreatorWithMockPkg(getHosts(), xml, sds).create();
 
         assertTrue(model.getContentClusters().get("bar").getPersistence() instanceof ProtonEngine.Factory);
@@ -340,7 +340,7 @@ public class ContentClusterTest extends ContentBaseTest {
             "   </content>\n" +
             "</services>\n";
 
-        List<String> sds = ApplicationPackageUtils.generateSearchDefinitions("type1", "type2");
+        List<String> sds = ApplicationPackageUtils.generateSchemas("type1", "type2");
         try{
             new VespaModelCreatorWithMockPkg(getHosts(), xml, sds).create();
             assertTrue("Deploying without redundancy should fail", false);
@@ -697,7 +697,7 @@ public class ContentClusterTest extends ContentBaseTest {
             "</services>";
 
 
-        List<String> sds = ApplicationPackageUtils.generateSearchDefinitions("type1", "type2");
+        List<String> sds = ApplicationPackageUtils.generateSchemas("type1", "type2");
         VespaModel model = new VespaModelCreatorWithMockPkg(getHosts(), xml, sds).create();
 
         {
@@ -818,7 +818,7 @@ public class ContentClusterTest extends ContentBaseTest {
               "  </group>" +
               "</content>";
 
-        List<String> sds = ApplicationPackageUtils.generateSearchDefinitions("true");
+        List<String> sds = ApplicationPackageUtils.generateSchemas("true");
         new VespaModelCreatorWithMockPkg(null, xml, sds).create();
     }
 
@@ -865,7 +865,7 @@ public class ContentClusterTest extends ContentBaseTest {
                         "  </group>" +
                         "</content>" +
                         "</services>";
-        List<String> sds = ApplicationPackageUtils.generateSearchDefinitions("bunnies", "hares", "rabbits");
+        List<String> sds = ApplicationPackageUtils.generateSchemas("bunnies", "hares", "rabbits");
         return new VespaModelCreatorWithMockPkg(getHosts(), xml, sds).create();
     }
 
@@ -912,8 +912,8 @@ public class ContentClusterTest extends ContentBaseTest {
         DeployState.Builder deployStateBuilder = new DeployState.Builder()
                 .zone(zone)
                 .properties(new TestProperties().setHostedVespa(true));
-        List<String> searchDefinitions = SearchDefinitionBuilder.createSearchDefinitions("test");
-        MockRoot root = ContentClusterUtils.createMockRoot(searchDefinitions, deployStateBuilder);
+        List<String> schemas = SchemaBuilder.createSchemas("test");
+        MockRoot root = ContentClusterUtils.createMockRoot(schemas, deployStateBuilder);
         ContentCluster cluster = ContentClusterUtils.createCluster(clusterXml, root);
         root.freezeModelTopology();
         cluster.validate();
@@ -933,6 +933,17 @@ public class ContentClusterTest extends ContentBaseTest {
         assertEquals(distributionBits, storDistributormanagerConfig.minsplitcount());
     }
 
+    private void verifyTopKProbabilityPropertiesControl(double topKProbability) {
+        VespaModel model = createEnd2EndOneNode(new TestProperties().setTopKProbability(topKProbability));
+
+        ContentCluster cc = model.getContentClusters().get("storage");
+        DispatchConfig.Builder builder = new DispatchConfig.Builder();
+        cc.getSearch().getConfig(builder);
+
+        DispatchConfig cfg = new DispatchConfig(builder);
+        assertEquals(topKProbability, cfg.topKProbability(), 0.0);
+    }
+
     private void verifyRoundRobinPropertiesControl(boolean useAdaptiveDispatch) {
         VespaModel model = createEnd2EndOneNode(new TestProperties().setUseAdaptiveDispatch(useAdaptiveDispatch));
 
@@ -946,7 +957,6 @@ public class ContentClusterTest extends ContentBaseTest {
         } else {
             assertEquals(DispatchConfig.DistributionPolicy.ROUNDROBIN, cfg.distributionPolicy());
         }
-
     }
 
     @Test
@@ -955,5 +965,12 @@ public class ContentClusterTest extends ContentBaseTest {
         verifyRoundRobinPropertiesControl(true);
     }
 
+    @Test
+    public void default_topKprobability_controlled_by_properties() {
+        verifyTopKProbabilityPropertiesControl(1.0);
+        verifyTopKProbabilityPropertiesControl(0.999);
+        verifyTopKProbabilityPropertiesControl(0.77);
+    }
+
 
 }
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java
index 98fa179b219..3415044b088 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java
@@ -8,7 +8,7 @@ import com.yahoo.vespa.config.search.core.ProtonConfig;
 import com.yahoo.vespa.model.content.cluster.ContentCluster;
 import com.yahoo.vespa.model.content.utils.ContentClusterBuilder;
 import com.yahoo.vespa.model.content.utils.DocType;
-import com.yahoo.vespa.model.content.utils.SearchDefinitionBuilder;
+import com.yahoo.vespa.model.content.utils.SchemaBuilder;
 import org.junit.Test;
 
 import java.util.ArrayList;
@@ -17,7 +17,7 @@ import java.util.List;
 
 import static com.yahoo.config.model.test.TestUtil.joinLines;
 import static com.yahoo.vespa.model.content.utils.ContentClusterUtils.createCluster;
-import static com.yahoo.vespa.model.content.utils.SearchDefinitionBuilder.createSearchDefinitions;
+import static com.yahoo.vespa.model.content.utils.SchemaBuilder.createSchemas;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotNull;
@@ -38,7 +38,7 @@ public class ContentSearchClusterTest {
 
     private static ContentCluster createClusterWithTwoDocumentType() throws Exception {
         return createCluster(new ContentClusterBuilder().docTypes("foo", "bar").getXml(),
-                createSearchDefinitions("foo", "bar"));
+                             createSchemas("foo", "bar"));
     }
 
     private static ContentCluster createClusterWithGlobalType() throws Exception {
@@ -55,7 +55,7 @@ public class ContentSearchClusterTest {
                 "<node distribution-key='1' hostalias='mockhost'/>",
                 "</group>"));
         String clusterXml = builder.getXml();
-        return createCluster(clusterXml, createSearchDefinitions(docTypes));
+        return createCluster(clusterXml, createSchemas(docTypes));
     }
 
     private static ContentClusterBuilder createClusterBuilderWithGlobalType() {
@@ -127,18 +127,19 @@ public class ContentSearchClusterTest {
     }
 
     private static ContentCluster createClusterWithThreeDocumentTypes() throws Exception {
-        List<String> searchDefinitions = new ArrayList<>();
-        searchDefinitions.add(new SearchDefinitionBuilder().name("a")
-                .content(joinLines("field ref_to_b type reference<b> { indexing: attribute }",
-                                   "field ref_to_c type reference<c> { indexing: attribute }")).build());
-        searchDefinitions.add(new SearchDefinitionBuilder().name("b")
-                .content("field ref_to_c type reference<c> { indexing: attribute }").build());
-        searchDefinitions.add(new SearchDefinitionBuilder().name("c").build());
-        return createCluster(new ContentClusterBuilder().docTypes(Arrays.asList(
-                DocType.index("a"),
-                DocType.indexGlobal("b"),
-                DocType.indexGlobal("c"))).getXml(),
-                searchDefinitions);
+        List<String> schemas = new ArrayList<>();
+        schemas.add(new SchemaBuilder().name("a")
+                                       .content(joinLines("field ref_to_b type reference<b> { indexing: attribute }",
+                                                          "field ref_to_c type reference<c> { indexing: attribute }"))
+                                       .build());
+        schemas.add(new SchemaBuilder().name("b")
+                                       .content("field ref_to_c type reference<c> { indexing: attribute }")
+                                       .build());
+        schemas.add(new SchemaBuilder().name("c").build());
+        return createCluster(new ContentClusterBuilder().docTypes(List.of(DocType.index("a"),
+                                                                          DocType.indexGlobal("b"),
+                                                                          DocType.indexGlobal("c"))).getXml(),
+                             schemas);
     }
 
     private static BucketspacesConfig getBucketspacesConfig(ContentCluster cluster) {
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/DispatchTuningTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/DispatchTuningTest.java
index f708d7673e2..8a46aaaa230 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/content/DispatchTuningTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/content/DispatchTuningTest.java
@@ -19,11 +19,13 @@ public class DispatchTuningTest {
                 .setDispatchPolicy("round-robin")
                 .setMinGroupCoverage(7.5)
                 .setMinActiveDocsCoverage(12.5)
+                .setTopKProbability(18.3)
                 .build();
         assertEquals(69, dispatch.getMaxHitsPerPartition().intValue());
         assertEquals(7.5, dispatch.getMinGroupCoverage().doubleValue(), 0.0);
         assertEquals(12.5, dispatch.getMinActiveDocsCoverage().doubleValue(), 0.0);
         assertTrue(DispatchTuning.DispatchPolicy.ROUNDROBIN == dispatch.getDispatchPolicy());
+        assertEquals(18.3, dispatch.getTopkProbability(), 0.0);
     }
     @Test
     public void requireThatRandomDispatchWork() {
@@ -52,6 +54,7 @@ public class DispatchTuningTest {
         assertNull(dispatch.getDispatchPolicy());
         assertNull(dispatch.getMinActiveDocsCoverage());
         assertNull(dispatch.getMinGroupCoverage());
+        assertNull(dispatch.getTopkProbability());
     }
 
 }
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/DistributorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/DistributorTest.java
index f36ef6c3ba3..365dc74274d 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/content/DistributorTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/content/DistributorTest.java
@@ -24,7 +24,7 @@ public class DistributorTest {
 
     ContentCluster parseCluster(String xml) {
         try {
-            List<String> searchDefs = ApplicationPackageUtils.generateSearchDefinitions("music", "movies", "bunnies");
+            List<String> searchDefs = ApplicationPackageUtils.generateSchemas("music", "movies", "bunnies");
             MockRoot root = ContentClusterUtils.createMockRoot(searchDefs);
             return ContentClusterUtils.createCluster(xml, root);
         } catch (Exception e) {
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/GenericConfigTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/GenericConfigTest.java
index 992edf6b1bb..51d0afc1f93 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/content/GenericConfigTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/content/GenericConfigTest.java
@@ -50,7 +50,7 @@ public class GenericConfigTest {
 
     @Before
     public void getVespaModel() {
-        model = (new VespaModelCreatorWithMockPkg(ContentBaseTest.getHosts(), servicesXml(), ApplicationPackageUtils.generateSearchDefinitions("type1"))).create();
+        model = (new VespaModelCreatorWithMockPkg(ContentBaseTest.getHosts(), servicesXml(), ApplicationPackageUtils.generateSchemas("type1"))).create();
     }
 
     @Test
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedTest.java
index ecf8f100288..504c3d9ba9c 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedTest.java
@@ -112,12 +112,12 @@ public class IndexedTest extends ContentBaseTest {
     }
 
     private VespaModelCreatorWithMockPkg getIndexedVespaModelCreator() {
-        List<String> sds = ApplicationPackageUtils.generateSearchDefinitions("type1", "type2", "type3");
+        List<String> sds = ApplicationPackageUtils.generateSchemas("type1", "type2", "type3");
         return new VespaModelCreatorWithMockPkg(getHosts(), createProtonIndexedVespaServices(Arrays.asList("type1", "type2", "type3")), sds);
     }
 
     private VespaModel getStreamingVespaModel() {
-        List<String> sds = ApplicationPackageUtils.generateSearchDefinitions("type1");
+        List<String> sds = ApplicationPackageUtils.generateSchemas("type1");
         return new VespaModelCreatorWithMockPkg(getHosts(), createProtonStreamingVespaServices(Arrays.asList("type1")), sds).create();
     }
 
@@ -229,7 +229,7 @@ public class IndexedTest extends ContentBaseTest {
                 "  </content>\n" +
                 "  </services>";
 
-        List<String> sds = ApplicationPackageUtils.generateSearchDefinitions("docstorebench");
+        List<String> sds = ApplicationPackageUtils.generateSchemas("docstorebench");
         VespaModel model = new VespaModelCreatorWithMockPkg(getHosts(), services, sds).create();
         ProtonConfig.Builder pb = new ProtonConfig.Builder();
         model.getConfig(pb, "docstore/search/cluster.docstore/0");
@@ -252,7 +252,7 @@ public class IndexedTest extends ContentBaseTest {
                 "  </content>" +
                 "</services>";
 
-        List<String> sds = ApplicationPackageUtils.generateSearchDefinitions("index_me", "store_me");
+        List<String> sds = ApplicationPackageUtils.generateSchemas("index_me", "store_me");
         VespaModel model = new VespaModelCreatorWithMockPkg(getHosts(), services, sds).create();
         ProtonConfig.Builder pb = new ProtonConfig.Builder();
         model.getConfig(pb, "docstore/search/cluster.docstore/0");
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/IndexingAndDocprocRoutingTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/IndexingAndDocprocRoutingTest.java
index 55d070d7247..177b86c953e 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/content/IndexingAndDocprocRoutingTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/content/IndexingAndDocprocRoutingTest.java
@@ -164,7 +164,7 @@ public class IndexingAndDocprocRoutingTest extends ContentBaseTest {
                         "  </container>\n" +
                         "</services>\n";
 
-        List<String> sds = ApplicationPackageUtils.generateSearchDefinitions("music", "title", "artist");
+        List<String> sds = ApplicationPackageUtils.generateSchemas("music", "title", "artist");
         VespaModel model = new VespaModelCreatorWithMockPkg(getHosts(),
                 services, sds).create();
         assertIndexing(model, new DocprocClusterSpec("dokprok"));
@@ -448,7 +448,7 @@ public class IndexingAndDocprocRoutingTest extends ContentBaseTest {
     }
 
     private VespaModel getIndexedSearchVespaModel(String xml) {
-        List<String> sds = ApplicationPackageUtils.generateSearchDefinitions("music", "album", "artist");
+        List<String> sds = ApplicationPackageUtils.generateSchemas("music", "album", "artist");
         return new VespaModelCreatorWithMockPkg(getHosts(), xml, sds).create();
     }
 
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java
index 21c384dfc69..883d8b89765 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java
@@ -112,11 +112,7 @@ public class StorageClusterTest {
                 "<cluster id=\"bees\">\n" +
                 "    <documents/>" +
                 "    <tuning>\n" +
-                "        <persistence-threads>\n" +
-                "            <thread lowest-priority=\"VERY_LOW\" count=\"2\"/>\n" +
-                "            <thread lowest-priority=\"VERY_HIGH\" count=\"1\"/>\n" +
-                "            <thread count=\"1\"/>\n" +
-                "        </persistence-threads>\n" +
+                "        <persistence-threads count=\"7\"/>\n" +
                 "    </tuning>\n" +
                 "  <group>" +
                 "     <node distribution-key=\"0\" hostalias=\"mockhost\"/>" +
@@ -130,6 +126,44 @@ public class StorageClusterTest {
             stc.getConfig(builder);
             StorFilestorConfig config = new StorFilestorConfig(builder);
 
+            assertEquals(7, config.num_threads());
+            assertEquals(false, config.enable_multibit_split_optimalization());
+        }
+        {
+            assertEquals(1, stc.getChildren().size());
+            StorageNode sn = stc.getChildren().values().iterator().next();
+            StorFilestorConfig.Builder builder = new StorFilestorConfig.Builder();
+            sn.getConfig(builder);
+            StorFilestorConfig config = new StorFilestorConfig(builder);
+            assertEquals(7, config.num_threads());
+        }
+    }
+
+    @Test
+    public void testPersistenceThreadsOld() throws Exception {
+
+        StorageCluster stc = parse(
+                "<cluster id=\"bees\">\n" +
+                        "    <documents/>" +
+                        "    <tuning>\n" +
+                        "        <persistence-threads>\n" +
+                        "            <thread lowest-priority=\"VERY_LOW\" count=\"2\"/>\n" +
+                        "            <thread lowest-priority=\"VERY_HIGH\" count=\"1\"/>\n" +
+                        "            <thread count=\"1\"/>\n" +
+                        "        </persistence-threads>\n" +
+                        "    </tuning>\n" +
+                        "  <group>" +
+                        "     <node distribution-key=\"0\" hostalias=\"mockhost\"/>" +
+                        "  </group>" +
+                        "</cluster>",
+                new Flavor(new FlavorsConfig.Flavor.Builder().name("test-flavor").minCpuCores(9).build())
+        );
+
+        {
+            StorFilestorConfig.Builder builder = new StorFilestorConfig.Builder();
+            stc.getConfig(builder);
+            StorFilestorConfig config = new StorFilestorConfig(builder);
+
             assertEquals(4, config.num_threads());
             assertEquals(false, config.enable_multibit_split_optimalization());
         }
@@ -161,7 +195,7 @@ public class StorageClusterTest {
             StorFilestorConfig.Builder builder = new StorFilestorConfig.Builder();
             stc.getConfig(builder);
             StorFilestorConfig config = new StorFilestorConfig(builder);
-            assertEquals(6, config.num_threads());
+            assertEquals(8, config.num_threads());
         }
         {
             assertEquals(1, stc.getChildren().size());
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageContentTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageContentTest.java
index c0ddd49069d..e099476ebb6 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageContentTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageContentTest.java
@@ -49,7 +49,7 @@ public class StorageContentTest extends ContentBaseTest {
     }
 
     private VespaModel getStorageVespaModel(String cluster1docs, String cluster2docs) {
-        List<String> sds = ApplicationPackageUtils.generateSearchDefinitions("type1", "type2", "type3");
+        List<String> sds = ApplicationPackageUtils.generateSchemas("type1", "type2", "type3");
         return new VespaModelCreatorWithMockPkg(getHosts(), createStorageVespaServices(cluster1docs, cluster2docs), sds).create();
     }
 
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java
index 62221c206fd..852844fe451 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java
@@ -176,7 +176,7 @@ public class ClusterTest {
                         "    </tuning>",
                         "  </content>",
                         "</services>"))
-                .withSearchDefinitions(ApplicationPackageUtils.generateSearchDefinition("my_document"))
+                .withSchemas(ApplicationPackageUtils.generateSearchDefinition("my_document"))
                 .build();
         List<Content> contents = new TestDriver().buildModel(app).getConfigModels(Content.class);
         assertEquals(1, contents.size());
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/DomDispatchTuningBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/DomDispatchTuningBuilderTest.java
index 7fa27f74d74..abfb03e41dd 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/DomDispatchTuningBuilderTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/DomDispatchTuningBuilderTest.java
@@ -47,6 +47,7 @@ public class DomDispatchTuningBuilderTest {
         assertNull(dispatch.getMinGroupCoverage());
         assertNull(dispatch.getMinActiveDocsCoverage());
         assertNull(dispatch.getDispatchPolicy());
+        assertNull(dispatch.getTopkProbability());
     }
 
     @Test
@@ -58,12 +59,14 @@ public class DomDispatchTuningBuilderTest {
                 "      <max-hits-per-partition>69</max-hits-per-partition>" +
                 "      <min-group-coverage>7.5</min-group-coverage>" +
                 "      <min-active-docs-coverage>12.5</min-active-docs-coverage>" +
+                "      <top-k-probability>0.999</top-k-probability>" +
                 "    </dispatch>" +
                 "  </tuning>" +
                 "</content>");
         assertEquals(69, dispatch.getMaxHitsPerPartition().intValue());
         assertEquals(7.5, dispatch.getMinGroupCoverage().doubleValue(), 0.0);
         assertEquals(12.5, dispatch.getMinActiveDocsCoverage().doubleValue(), 0.0);
+        assertEquals(0.999, dispatch.getTopkProbability().doubleValue(), 0.0);
     }
     @Test
     public void requireThatTuningDispatchPolicyRoundRobin() throws Exception {
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ApplicationPackageBuilder.java b/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ApplicationPackageBuilder.java
index 1b7f3e9b14d..5fc213fad1d 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ApplicationPackageBuilder.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ApplicationPackageBuilder.java
@@ -14,7 +14,7 @@ import java.util.List;
 public class ApplicationPackageBuilder {
 
     private List<ContentClusterBuilder> contentClusters = new ArrayList<>();
-    private List<String> searchDefinitions = new ArrayList<>();
+    private List<String> schemas = new ArrayList<>();
 
     public ApplicationPackageBuilder() {
     }
@@ -24,13 +24,13 @@ public class ApplicationPackageBuilder {
         return this;
     }
 
-    public ApplicationPackageBuilder addSearchDefinition(String searchDefinition) {
-        searchDefinitions.add(searchDefinition);
+    public ApplicationPackageBuilder addSchemas(String schemas) {
+        this.schemas.add(schemas);
         return this;
     }
 
     public VespaModelCreatorWithMockPkg buildCreator() {
-        return new VespaModelCreatorWithMockPkg(null, getServices(), searchDefinitions);
+        return new VespaModelCreatorWithMockPkg(null, getServices(), schemas);
     }
 
     private String getServices() {
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterUtils.java b/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterUtils.java
index 62d2bc51830..b1550d90f35 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterUtils.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterUtils.java
@@ -30,46 +30,46 @@ import java.util.Optional;
 public class ContentClusterUtils {
 
     public static MockRoot createMockRoot(String[] hosts) {
-        return createMockRoot(hosts, SearchDefinitionBuilder.createSearchDefinitions("test"));
+        return createMockRoot(hosts, SchemaBuilder.createSchemas("test"));
     }
 
-    private static MockRoot createMockRoot(HostProvisioner provisioner, List<String> searchDefinitions) {
-        return createMockRoot(provisioner, searchDefinitions, new DeployState.Builder());
+    private static MockRoot createMockRoot(HostProvisioner provisioner, List<String> schemas) {
+        return createMockRoot(provisioner, schemas, new DeployState.Builder());
     }
 
-    private static MockRoot createMockRoot(HostProvisioner provisioner, List<String> searchDefinitions, DeployState.Builder deployStateBuilder) {
-        ApplicationPackage applicationPackage = new MockApplicationPackage.Builder().withSearchDefinitions(searchDefinitions).build();
+    private static MockRoot createMockRoot(HostProvisioner provisioner, List<String> schemas, DeployState.Builder deployStateBuilder) {
+        ApplicationPackage applicationPackage = new MockApplicationPackage.Builder().withSchemas(schemas).build();
         DeployState deployState = deployStateBuilder.applicationPackage(applicationPackage)
                           .modelHostProvisioner(provisioner)
                           .build();
         return new MockRoot("", deployState);
     }
 
-    public static MockRoot createMockRoot(String[] hosts, List<String> searchDefinitions) {
-        return createMockRoot(new InMemoryProvisioner(true, hosts), searchDefinitions);
+    public static MockRoot createMockRoot(String[] hosts, List<String> schemas) {
+        return createMockRoot(new InMemoryProvisioner(true, hosts), schemas);
     }
 
-    public static MockRoot createMockRoot(List<String> searchDefinitions) {
-        return createMockRoot(new SingleNodeProvisioner(), searchDefinitions);
+    public static MockRoot createMockRoot(List<String> schemas) {
+        return createMockRoot(new SingleNodeProvisioner(), schemas);
     }
 
-    public static MockRoot createMockRoot(List<String> searchDefinitions, DeployState.Builder deployStateBuilder) {
-        return createMockRoot(new SingleNodeProvisioner(), searchDefinitions, deployStateBuilder);
+    public static MockRoot createMockRoot(List<String> schemas, DeployState.Builder deployStateBuilder) {
+        return createMockRoot(new SingleNodeProvisioner(), schemas, deployStateBuilder);
     }
 
     public static ContentCluster createCluster(String clusterXml, MockRoot root) {
         Document doc = XML.getDocument(clusterXml);
         Admin admin = new Admin(root, new DefaultMonitoring("vespa", 60), new Metrics(), false,
-                                new FileDistributionConfigProducer(root, new MockFileRegistry(), null),
+                                new FileDistributionConfigProducer(root, new MockFileRegistry(), List.of(), false),
                                 root.getDeployState().isHosted());
         ConfigModelContext context = ConfigModelContext.create(null, root.getDeployState(),
-                                                    null,null, root, null);
+                                                               null,null, root, null);
 
         return new ContentCluster.Builder(admin).build(Collections.emptyList(), context, doc.getDocumentElement());
     }
 
-    public static ContentCluster createCluster(String clusterXml, List<String> searchDefinitions) throws Exception {
-        MockRoot root = createMockRoot(searchDefinitions);
+    public static ContentCluster createCluster(String clusterXml, List<String> schemas) throws Exception {
+        MockRoot root = createMockRoot(schemas);
         ContentCluster cluster = createCluster(clusterXml, root);
         root.freezeModelTopology();
         cluster.validate();
@@ -77,7 +77,7 @@ public class ContentClusterUtils {
     }
 
     public static ContentCluster createCluster(String clusterXml) throws Exception {
-        return createCluster(clusterXml, SearchDefinitionBuilder.createSearchDefinitions("test"));
+        return createCluster(clusterXml, SchemaBuilder.createSchemas("test"));
     }
 
     public static String createClusterXml(String groupXml, int redundancy, int searchableCopies) {
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/utils/SearchDefinitionBuilder.java b/config-model/src/test/java/com/yahoo/vespa/model/content/utils/SchemaBuilder.java
index c622bcfaec5..c18dac17064 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/content/utils/SearchDefinitionBuilder.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/content/utils/SchemaBuilder.java
@@ -12,20 +12,20 @@ import static com.yahoo.config.model.test.TestUtil.joinLines;
  *
  * @author geirst
  */
-public class SearchDefinitionBuilder {
+public class SchemaBuilder {
 
     private String name = "test";
     private String content = "";
 
-    public SearchDefinitionBuilder() {
+    public SchemaBuilder() {
     }
 
-    public SearchDefinitionBuilder name(String name) {
+    public SchemaBuilder name(String name) {
         this.name = name;
         return this;
     }
 
-    public SearchDefinitionBuilder content(String content) {
+    public SchemaBuilder content(String content) {
         this.content = content;
         return this;
     }
@@ -38,10 +38,10 @@ public class SearchDefinitionBuilder {
                 "}");
     }
 
-    public static List<String> createSearchDefinitions(String ... docTypes) {
+    public static List<String> createSchemas(String ... docTypes) {
         return Arrays.asList(docTypes)
                 .stream()
-                .map(type -> new SearchDefinitionBuilder().name(type).build())
+                .map(type -> new SchemaBuilder().name(type).build())
                 .collect(Collectors.toList());
     }
 
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/filedistribution/FileDistributorTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/filedistribution/FileDistributorTestCase.java
index 131a5344116..55672c5aa16 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/filedistribution/FileDistributorTestCase.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/filedistribution/FileDistributorTestCase.java
@@ -10,31 +10,34 @@ import org.junit.Test;
 import java.io.File;
 import java.util.Arrays;
 import java.util.HashSet;
+import java.util.List;
 import java.util.Set;
 
 import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertNotNull;
 
 /**
  * @author bratseth
  */
 public class FileDistributorTestCase {
+
     @Test
     public void fileDistributor() {
         MockHosts hosts = new MockHosts();
 
-        FileDistributor fileDistributor = new FileDistributor(new MockFileRegistry(), null);
+        FileDistributor fileDistributor = new FileDistributor(new MockFileRegistry(), List.of(), false);
 
         String file1 = "component/path1";
         String file2 = "component/path2";
-        FileReference ref1 = fileDistributor.sendFileToHosts(file1, Arrays.asList(hosts.host1, hosts.host2));
-        FileReference ref2 = fileDistributor.sendFileToHosts(file2, Arrays.asList(hosts.host3));
+        FileReference ref1 = fileDistributor.sendFileToHost(file1, hosts.host1);
+        fileDistributor.sendFileToHost(file1, hosts.host2); // same file reference as above
+        FileReference ref2 = fileDistributor.sendFileToHost(file2, hosts.host3);
 
         assertEquals(new HashSet<>(Arrays.asList(hosts.host1, hosts.host2, hosts.host3)),
                 fileDistributor.getTargetHosts());
 
-        assertTrue( ref1 != null );
-        assertTrue( ref2 != null );
+        assertNotNull(ref1);
+        assertNotNull(ref2);
 
         MockFileDistribution dbHandler = new MockFileDistribution();
         fileDistributor.sendDeployedFiles(dbHandler);
@@ -54,4 +57,5 @@ public class FileDistributorTestCase {
             return null;
         }
     }
+
 }
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/ml/ImportedModelTester.java b/config-model/src/test/java/com/yahoo/vespa/model/ml/ImportedModelTester.java
index ce36ecc4a1c..7a3b76db7f8 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/ml/ImportedModelTester.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/ml/ImportedModelTester.java
@@ -1,7 +1,6 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.model.ml;
 
-import ai.vespa.rankingexpression.importer.vespa.VespaImporter;
 import com.google.common.collect.ImmutableList;
 import com.yahoo.config.model.ApplicationPackageTester;
 import ai.vespa.rankingexpression.importer.configmodelview.MlModelImporter;
@@ -10,8 +9,10 @@ import com.yahoo.io.GrowableByteBuffer;
 import com.yahoo.io.IOUtils;
 import com.yahoo.path.Path;
 import com.yahoo.searchdefinition.RankingConstant;
+import ai.vespa.rankingexpression.importer.lightgbm.LightGBMImporter;
 import ai.vespa.rankingexpression.importer.onnx.OnnxImporter;
 import ai.vespa.rankingexpression.importer.tensorflow.TensorFlowImporter;
+import ai.vespa.rankingexpression.importer.vespa.VespaImporter;
 import ai.vespa.rankingexpression.importer.xgboost.XGBoostImporter;
 import com.yahoo.tensor.Tensor;
 import com.yahoo.tensor.serialization.TypedBinaryFormat;
@@ -35,6 +36,7 @@ public class ImportedModelTester {
 
     private final ImmutableList<MlModelImporter> importers = ImmutableList.of(new TensorFlowImporter(),
                                                                               new OnnxImporter(),
+                                                                              new LightGBMImporter(),
                                                                               new XGBoostImporter(),
                                                                               new VespaImporter());
 
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/ml/MlModelsTest.java b/config-model/src/test/java/com/yahoo/vespa/model/ml/MlModelsTest.java
index c5c475360a3..ced7243adf5 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/ml/MlModelsTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/ml/MlModelsTest.java
@@ -45,7 +45,7 @@ public class MlModelsTest {
 
     private void verify(VespaModel model) {
         assertEquals("Global models are created (although not used directly here",
-                     4, model.rankProfileList().getRankProfiles().size());
+                     5, model.rankProfileList().getRankProfiles().size());
 
         RankProfilesConfig.Builder builder = new RankProfilesConfig.Builder();
         model.getSearchClusters().get(0).getConfig(builder);
@@ -71,8 +71,9 @@ public class MlModelsTest {
             "rankingExpression(mnist_softmax_tensorflow).rankingScript: join(reduce(join(rename(rankingExpression(Placeholder), (d0, d1), (d0, d2)), constant(mnist_softmax_saved_layer_Variable_read), f(a,b)(a * b)), sum, d2), constant(mnist_softmax_saved_layer_Variable_1_read), f(a,b)(a + b))\n" +
             "rankingExpression(mnist_softmax_onnx).rankingScript: join(reduce(join(rename(rankingExpression(Placeholder), (d0, d1), (d0, d2)), constant(mnist_softmax_Variable), f(a,b)(a * b)), sum, d2), constant(mnist_softmax_Variable_1), f(a,b)(a + b))\n" +
             "rankingExpression(my_xgboost).rankingScript: if (f29 < -0.1234567, if (!(f56 >= -0.242398), 1.71218, -1.70044), if (f109 < 0.8723473, -1.94071, 1.85965)) + if (!(f60 >= -0.482947), if (f29 < -4.2387498, 0.784718, -0.96853), -6.23624)\n" +
+            "rankingExpression(my_lightgbm).rankingScript: if (!(numerical_2 >= 0.46643291586559305), 2.1594397038037663, if (categorical_2 in [\"k\", \"l\", \"m\"], 2.235297305276056, 2.1792953471546546)) + if (categorical_1 in [\"d\", \"e\"], 0.03070842919354316, if (!(numerical_1 >= 0.5102250691730842), -0.04439151147520909, 0.005117411709368601)) + if (!(numerical_2 >= 0.668665477622446), if (!(numerical_2 >= 0.008118820676863816), -0.15361238490967524, -0.01192330846157292), 0.03499044894987518) + if (!(numerical_1 >= 0.5201391072644542), -0.02141000620783247, if (categorical_1 in [\"a\", \"b\"], -0.004121485787596721, 0.04534090904886873)) + if (categorical_2 in [\"k\", \"l\", \"m\"], if (!(numerical_2 >= 0.27283279016959255), -0.01924803254356527, 0.03643772842347651), -0.02701711918923075)\n" +
             "vespa.rank.firstphase: rankingExpression(firstphase)\n" +
-            "rankingExpression(firstphase).rankingScript: rankingExpression(mnist_tensorflow) + rankingExpression(mnist_softmax_tensorflow) + rankingExpression(mnist_softmax_onnx) + rankingExpression(my_xgboost)\n" +
+            "rankingExpression(firstphase).rankingScript: rankingExpression(mnist_tensorflow) + rankingExpression(mnist_softmax_tensorflow) + rankingExpression(mnist_softmax_onnx) + rankingExpression(my_xgboost) + rankingExpression(my_lightgbm)\n" +
             "vespa.type.attribute.argument: tensor<float>(d0[],d1[784])\n";
 
 }
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/ml/ModelEvaluationTest.java b/config-model/src/test/java/com/yahoo/vespa/model/ml/ModelEvaluationTest.java
index 3d4ac7f2eeb..2d3ddc33afb 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/ml/ModelEvaluationTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/ml/ModelEvaluationTest.java
@@ -96,9 +96,10 @@ public class ModelEvaluationTest {
         cluster.getConfig(cb);
         RankingConstantsConfig constantsConfig = new RankingConstantsConfig(cb);
 
-        assertEquals(4, config.rankprofile().size());
+        assertEquals(5, config.rankprofile().size());
         Set<String> modelNames = config.rankprofile().stream().map(v -> v.name()).collect(Collectors.toSet());
         assertTrue(modelNames.contains("xgboost_2_2"));
+        assertTrue(modelNames.contains("lightgbm_regression"));
         assertTrue(modelNames.contains("mnist_saved"));
         assertTrue(modelNames.contains("mnist_softmax"));
         assertTrue(modelNames.contains("mnist_softmax_saved"));
@@ -112,13 +113,18 @@ public class ModelEvaluationTest {
         ModelsEvaluator evaluator = new ModelsEvaluator(new ToleratingMissingConstantFilesRankProfilesConfigImporter(MockFileAcquirer.returnFile(null))
                                                                 .importFrom(config, constantsConfig));
 
-        assertEquals(4, evaluator.models().size());
+        assertEquals(5, evaluator.models().size());
 
         Model xgboost = evaluator.models().get("xgboost_2_2");
         assertNotNull(xgboost);
         assertNotNull(xgboost.evaluatorOf());
         assertNotNull(xgboost.evaluatorOf("xgboost_2_2"));
 
+        Model lightgbm = evaluator.models().get("lightgbm_regression");
+        assertNotNull(lightgbm);
+        assertNotNull(lightgbm.evaluatorOf());
+        assertNotNull(lightgbm.evaluatorOf("lightgbm_regression"));
+
         Model tensorflow_mnist = evaluator.models().get("mnist_saved");
         assertNotNull(tensorflow_mnist);
         assertEquals(1, tensorflow_mnist.functions().size());
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/NodeFlavorTuningTest.java b/config-model/src/test/java/com/yahoo/vespa/model/search/NodeFlavorTuningTest.java
index a4b414ba0da..78c6d2eacc9 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/search/NodeFlavorTuningTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/search/NodeFlavorTuningTest.java
@@ -69,6 +69,13 @@ public class NodeFlavorTuningTest {
     }
 
     @Test
+    public void require_that_num_search_threads_and_summary_threads_follow_cores() {
+        ProtonConfig cfg = configFromNumCoresSetting(4.5);
+        assertEquals(5, cfg.numsearcherthreads());
+        assertEquals(5, cfg.numsummarythreads());
+    }
+
+    @Test
     public void require_that_fast_disk_is_reflected_in_proton_config() {
         ProtonConfig cfg = configFromDiskSetting(true);
         assertEquals(200, cfg.hwinfo().disk().writespeed(), delta);
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java
index 177e741937d..97417f5a522 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java
@@ -90,7 +90,7 @@ public class DocumentDatabaseTestCase {
     private void assertSingleSD(String mode) {
         final List<String> sds = Arrays.asList("type1");
         VespaModel model = new VespaModelCreatorWithMockPkg(vespaHosts, createVespaServices(sds, mode),
-                ApplicationPackageUtils.generateSearchDefinitions(sds)).create();
+                ApplicationPackageUtils.generateSchemas(sds)).create();
         IndexedSearchCluster indexedSearchCluster = (IndexedSearchCluster)model.getSearchClusters().get(0);
         ContentSearchCluster contentSearchCluster = model.getContentClusters().get("test").getSearch();
         assertEquals(1, indexedSearchCluster.getDocumentDbs().size());
@@ -111,7 +111,7 @@ public class DocumentDatabaseTestCase {
             sds.add(nameAndMode.getType());
         }
         return new VespaModelCreatorWithMockPkg(vespaHosts, createVespaServicesXml(nameAndModes, xmlTuning),
-                ApplicationPackageUtils.generateSearchDefinitions(sds)).create();
+                ApplicationPackageUtils.generateSchemas(sds)).create();
     }
 
     @Test
@@ -211,10 +211,10 @@ public class DocumentDatabaseTestCase {
     }
 
     @Test
-    public void requireThatWeCanHaveMultipleSearchDefinitions() {
-        final List<String> sds = Arrays.asList("type1", "type2", "type3");
+    public void testMultipleSchemas() {
+        List<String> sds = List.of("type1", "type2", "type3");
         VespaModel model = new VespaModelCreatorWithMockPkg(vespaHosts, createVespaServices(sds, "index"),
-                ApplicationPackageUtils.generateSearchDefinitions(sds)).create();
+                                                            ApplicationPackageUtils.generateSchemas(sds)).create();
         IndexedSearchCluster indexedSearchCluster = (IndexedSearchCluster)model.getSearchClusters().get(0);
         ContentSearchCluster contentSearchCluster = model.getContentClusters().get("test").getSearch();
         String type1Id = "test/search/cluster.test/type1";
@@ -264,7 +264,7 @@ public class DocumentDatabaseTestCase {
     public void requireThatRelevantConfigIsAvailableForClusterSearcher() {
         final List<String> sds = Arrays.asList("type1", "type2");
         VespaModel model = new VespaModelCreatorWithMockPkg(vespaHosts, createVespaServices(sds, "index"),
-                ApplicationPackageUtils.generateSearchDefinitions(sds)).create();
+                ApplicationPackageUtils.generateSchemas(sds)).create();
         String searcherId = "container/searchchains/chain/test/component/com.yahoo.prelude.cluster.ClusterSearcher";
 
         { // documentdb-info config
@@ -325,7 +325,7 @@ public class DocumentDatabaseTestCase {
     private void assertDocumentDBConfigAvailableForStreaming(String mode) {
         final List<String> sds = Arrays.asList("type");
         VespaModel model = new VespaModelCreatorWithMockPkg(vespaHosts,  createVespaServices(sds, mode),
-                ApplicationPackageUtils.generateSearchDefinitions(sds)).create();
+                ApplicationPackageUtils.generateSchemas(sds)).create();
 
         DocumentdbInfoConfig dcfg = model.getConfig(DocumentdbInfoConfig.class, "test/search/cluster.test.type");
         assertEquals(1, dcfg.documentdb().size());
@@ -343,7 +343,7 @@ public class DocumentDatabaseTestCase {
                                                          List<String> documentDBConfigIds,
                                                          Map<String, List<String>> expectedAttributesMap) {
         VespaModel model = new VespaModelCreatorWithMockPkg(vespaHosts,  createVespaServices(sds, mode),
-                ApplicationPackageUtils.generateSearchDefinitions(sds)).create();
+                ApplicationPackageUtils.generateSchemas(sds)).create();
         ContentSearchCluster contentSearchCluster = model.getContentClusters().get("test").getSearch();
 
         ProtonConfig proton = getProtonCfg(contentSearchCluster);
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchClusterTest.java
index 1c4e005cb67..ea4b3db5ebb 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchClusterTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchClusterTest.java
@@ -18,12 +18,15 @@ import com.yahoo.vespa.model.VespaModel;
 import com.yahoo.vespa.model.container.ContainerCluster;
 import com.yahoo.vespa.model.container.component.Component;
 import com.yahoo.vespa.model.search.AbstractSearchCluster;
-import com.yahoo.vespa.model.search.SearchCluster;
 import com.yahoo.vespa.model.test.utils.ApplicationPackageUtils;
 import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithMockPkg;
 import org.junit.Test;
 
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
 
 /**
  * Unit tests for SearchCluster. Please use this instead of SearchModelTestCase if possible and
@@ -124,7 +127,7 @@ public class SearchClusterTest {
                 "  </content>" +
                 "</services>";
 
-        VespaModel model = new VespaModelCreatorWithMockPkg(vespaHosts, services, ApplicationPackageUtils.generateSearchDefinitions("music")).create();
+        VespaModel model = new VespaModelCreatorWithMockPkg(vespaHosts, services, ApplicationPackageUtils.generateSchemas("music")).create();
 
         ContainerCluster containerCluster1 = (ContainerCluster)model.getConfigProducer("j1").get();
         assertFalse(containerCluster1.getSearch().getChains().localProviders().isEmpty());
@@ -160,23 +163,30 @@ public class SearchClusterTest {
         AbstractSearchCluster searchCluster2 = model.getSearchClusters().get(xbulkIndex);
         assertEquals("xbulk", searchCluster2.getClusterName());
 
-        Component<?,?> normalDispatcher = (Component<?, ?>)containerCluster1.getComponentsMap().get(new ComponentId("dispatcher.normal"));
-        assertNotNull(normalDispatcher);
-        assertEquals("dispatcher.normal", normalDispatcher.getComponentId().stringValue());
-        assertEquals("com.yahoo.search.dispatch.Dispatcher", normalDispatcher.getClassId().stringValue());
-        assertEquals("j1/component/dispatcher.normal", normalDispatcher.getConfigId());
-        DispatchConfig.Builder normalDispatchConfigBuilder = new DispatchConfig.Builder();
-        model.getConfig(normalDispatchConfigBuilder, "j1/component/dispatcher.normal");
-        assertEquals("node2host", normalDispatchConfigBuilder.build().node(0).host());
-
-        Component<?,?> xbulkDispatcher = (Component<?, ?>)containerCluster1.getComponentsMap().get(new ComponentId("dispatcher.xbulk"));
-        assertNotNull(xbulkDispatcher);
-        assertEquals("dispatcher.xbulk", xbulkDispatcher.getComponentId().stringValue());
-        assertEquals("com.yahoo.search.dispatch.Dispatcher", xbulkDispatcher.getClassId().stringValue());
-        assertEquals("j1/component/dispatcher.xbulk", xbulkDispatcher.getConfigId());
-        DispatchConfig.Builder xbulkDispatchConfigBuilder = new DispatchConfig.Builder();
-        model.getConfig(xbulkDispatchConfigBuilder, "j1/component/dispatcher.xbulk");
-        assertEquals("node0host", xbulkDispatchConfigBuilder.build().node(0).host());
+        verifyDispatch(model, containerCluster1, "normal", "node2host");
+        verifyDispatch(model, containerCluster1, "xbulk", "node0host");
+    }
+
+    private void verifyDispatch(VespaModel model, ContainerCluster containerCluster, String cluster, String host) {
+        Component<?,?> dispatcher = (Component<?, ?>)containerCluster.getComponentsMap().get(new ComponentId("dispatcher." + cluster));
+        assertNotNull(dispatcher);
+        assertEquals("dispatcher." + cluster, dispatcher.getComponentId().stringValue());
+        assertEquals("com.yahoo.search.dispatch.Dispatcher", dispatcher.getClassId().stringValue());
+        assertEquals("j1/component/dispatcher." + cluster, dispatcher.getConfigId());
+        DispatchConfig.Builder dispatchConfigBuilder = new DispatchConfig.Builder();
+        model.getConfig(dispatchConfigBuilder, dispatcher.getConfigId());
+        assertEquals(host, dispatchConfigBuilder.build().node(0).host());
+
+        assertTrue(dispatcher.getInjectedComponentIds().contains("rpcresourcepool." + cluster));
+
+        Component<?,?> rpcResourcePool = (Component<?, ?>)dispatcher.getChildren().get("rpcresourcepool." + cluster);
+        assertNotNull(rpcResourcePool);
+        assertEquals("rpcresourcepool." + cluster, rpcResourcePool.getComponentId().stringValue());
+        assertEquals("com.yahoo.search.dispatch.rpc.RpcResourcePool", rpcResourcePool.getClassId().stringValue());
+        assertEquals("j1/component/dispatcher." + cluster + "/rpcresourcepool." + cluster, rpcResourcePool.getConfigId());
+        dispatchConfigBuilder = new DispatchConfig.Builder();
+        model.getConfig(dispatchConfigBuilder, rpcResourcePool.getConfigId());
+        assertEquals(host, dispatchConfigBuilder.build().node(0).host());
     }
 
 }
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java
index b6180ab78b9..cdfd9fab194 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java
@@ -104,33 +104,41 @@ public class VespaModelTester {
 
     /** Creates a model which uses 0 as start index */
     public VespaModel createModel(String services, boolean failOnOutOfCapacity, String ... retiredHostNames) {
-        return createModel(Zone.defaultZone(), services, failOnOutOfCapacity, 0, retiredHostNames);
+        return createModel(Zone.defaultZone(), services, failOnOutOfCapacity, false, 0, retiredHostNames);
+    }
+
+    /** Creates a model which uses 0 as start index */
+    public VespaModel createModel(String services, boolean failOnOutOfCapacity, boolean useMaxResources, String ... retiredHostNames) {
+        return createModel(Zone.defaultZone(), services, failOnOutOfCapacity, useMaxResources, 0, retiredHostNames);
     }
 
     /** Creates a model which uses 0 as start index */
     public VespaModel createModel(String services, boolean failOnOutOfCapacity, int startIndexForClusters, String ... retiredHostNames) {
-        return createModel(Zone.defaultZone(), services, failOnOutOfCapacity, startIndexForClusters, retiredHostNames);
+        return createModel(Zone.defaultZone(), services, failOnOutOfCapacity, false, startIndexForClusters, retiredHostNames);
     }
 
     /** Creates a model which uses 0 as start index */
     public VespaModel createModel(Zone zone, String services, boolean failOnOutOfCapacity, String ... retiredHostNames) {
-        return createModel(zone, services, failOnOutOfCapacity, 0, retiredHostNames);
+        return createModel(zone, services, failOnOutOfCapacity, false, 0, retiredHostNames);
     }
 
     /**
      * Creates a model using the hosts already added to this
      *
      * @param services the services xml string
+     * @param useMaxResources false to use the minmal resources (when given a range), true to use max
      * @param failOnOutOfCapacity whether we should get an exception when not enough hosts of the requested flavor
      *        is available or if we should just silently receive a smaller allocation
      * @return the resulting model
      */
-    public VespaModel createModel(Zone zone, String services, boolean failOnOutOfCapacity, int startIndexForClusters, String ... retiredHostNames) {
+    public VespaModel createModel(Zone zone, String services, boolean failOnOutOfCapacity, boolean useMaxResources,
+                                  int startIndexForClusters, String ... retiredHostNames) {
         VespaModelCreatorWithMockPkg modelCreatorWithMockPkg = new VespaModelCreatorWithMockPkg(null, services, ApplicationPackageUtils.generateSearchDefinition("type1"));
         ApplicationPackage appPkg = modelCreatorWithMockPkg.appPkg;
 
-        HostProvisioner provisioner = hosted ? 
-                                      new InMemoryProvisioner(hostsByResources, failOnOutOfCapacity, startIndexForClusters, retiredHostNames) :
+        HostProvisioner provisioner = hosted ?
+                                      new InMemoryProvisioner(hostsByResources, failOnOutOfCapacity, useMaxResources,
+                                                              startIndexForClusters, retiredHostNames) :
                                       new SingleNodeProvisioner();
 
         TestProperties properties = new TestProperties()
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/utils/ApplicationPackageUtils.java b/config-model/src/test/java/com/yahoo/vespa/model/test/utils/ApplicationPackageUtils.java
index ee6fc60ba46..df62a3bff07 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/test/utils/ApplicationPackageUtils.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/test/utils/ApplicationPackageUtils.java
@@ -44,14 +44,14 @@ public class ApplicationPackageUtils {
     }
 
     public static List<String> generateSearchDefinition(String name) {
-        return generateSearchDefinitions(name);
+        return generateSchemas(name);
     }
 
-    public static List<String> generateSearchDefinitions(String ... sdNames) {
-        return generateSearchDefinitions(Arrays.asList(sdNames));
+    public static List<String> generateSchemas(String ... sdNames) {
+        return generateSchemas(Arrays.asList(sdNames));
     }
 
-    public static List<String> generateSearchDefinitions(List<String> sdNames) {
+    public static List<String> generateSchemas(List<String> sdNames) {
         List<String> sds = new ArrayList<>();
         int i = 0;
         for (String sdName : sdNames) {
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/utils/VespaModelCreatorWithMockPkg.java b/config-model/src/test/java/com/yahoo/vespa/model/test/utils/VespaModelCreatorWithMockPkg.java
index 814ec008285..70ce588bec1 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/test/utils/VespaModelCreatorWithMockPkg.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/test/utils/VespaModelCreatorWithMockPkg.java
@@ -31,8 +31,8 @@ public class VespaModelCreatorWithMockPkg {
         this(new MockApplicationPackage.Builder().withHosts(hosts).withServices(services).build());
     }
 
-    public VespaModelCreatorWithMockPkg(String hosts, String services, List<String> searchDefinitions) {
-        this(new MockApplicationPackage.Builder().withHosts(hosts).withServices(services).withSearchDefinitions(searchDefinitions).build());
+    public VespaModelCreatorWithMockPkg(String hosts, String services, List<String> schemas) {
+        this(new MockApplicationPackage.Builder().withHosts(hosts).withServices(services).withSchemas(schemas).build());
     }
 
     public VespaModelCreatorWithMockPkg(ApplicationPackage appPkg) {
diff --git a/config-model/src/test/schema-test-files/services-hosted.xml b/config-model/src/test/schema-test-files/services-hosted.xml
index 07839239c81..71a07926240 100644
--- a/config-model/src/test/schema-test-files/services-hosted.xml
+++ b/config-model/src/test/schema-test-files/services-hosted.xml
@@ -7,7 +7,7 @@
   </admin>
 
   <container id="container1" version="1.0">
-    <nodes count="5" required="true">
+    <nodes count="[5,7]" required="true">
       <resources vcpu="1.2" memory="10Gb" disk="0.3 TB"/>
     </nodes>
   </container>
@@ -27,4 +27,11 @@
     </nodes>
   </content>
 
+  <content id="ml" version="1.0">
+    <redundancy>2</redundancy>
+    <nodes count="[10,20]" flavor="large" groups="[1,3]">
+      <resources vcpu="[3.0, 4]" memory="[32000.0Mb, 33Gb]" disk="[300 Gb, 1Tb]"/>
+    </nodes>
+  </content>
+
 </services>
diff --git a/config-model/src/test/schema-test-files/services.xml b/config-model/src/test/schema-test-files/services.xml
index 2bbd98f72ac..e7ea2683e3f 100644
--- a/config-model/src/test/schema-test-files/services.xml
+++ b/config-model/src/test/schema-test-files/services.xml
@@ -15,15 +15,29 @@
       <slobrok hostalias="rtc-1" />
     </slobroks>
     <metrics>
-      <consumer id="my-consumer">
+
+      <consumer id="cloudwatch-hosted">
         <metric-set id="my-set" />
         <metric id="my-metric"/>
         <metric id="my-metric2" display-name="my-metric3"/>
         <metric display-name="my-metric4" id="my-metric4.avg"/>
+        <cloudwatch region="us-east1" namespace="my-namespace">
+          <credentials access-key-name="my-access-key" secret-key-name="my-secret-key" />
+        </cloudwatch>
       </consumer>
-      <consumer id="my-consumer2">
-        <metric-set id="my-set2" />
+
+      <consumer id="cloudwatch-self-hosted-with-default-auth">
+        <metric-set id="public" />
+        <cloudwatch region="us-east1" namespace="namespace_legal.chars:/#1" />
       </consumer>
+
+      <consumer id="cloudwatch-self-hosted-with-profile">
+        <metric id="my-custom-metric" />
+        <cloudwatch region="us-east1" namespace="another-namespace">
+          <shared-credentials file="/user/.aws/credentials" profile="profile-in-credentials-file" />
+        </cloudwatch>
+      </consumer>
+
     </metrics>
     <logforwarding>
       <splunk deployment-server="foo:8989" client-name="foobar" splunk-home="/opt/splunk" phone-home-interval="900"/>
@@ -76,7 +90,6 @@
           <exclude>
             <binding>http//*/foo/*</binding>
           </exclude>
-          <application>my-app</application>
           <vespa-domain>vespa.vespa.cd</vespa-domain>
         </access-control>
 
@@ -119,6 +132,13 @@
           <certificate-file>/foo/cert</certificate-file>
           <ca-certificates-file>/foo/cacerts</ca-certificates-file>
           <client-authentication>want</client-authentication>
+          <cipher-suites>
+            TLS_AES_128_GCM_SHA256,
+            TLS_AES_256_GCM_SHA384,
+            TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
+            TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384
+          </cipher-suites>
+          <protocols>TLSv1.2,TLSv1.3</protocols>
         </ssl>
       </server>
       <server port="4083" id="sslProvider">
diff --git a/config-provisioning/abi-spec.json b/config-provisioning/abi-spec.json
index 45f8171436d..9e26dfeeb6e 100644
--- a/config-provisioning/abi-spec.json
+++ b/config-provisioning/abi-spec.json
@@ -1,848 +1 @@
-{
-  "com.yahoo.config.provision.AllocatedHosts": {
-    "superClass": "java.lang.Object",
-    "interfaces": [],
-    "attributes": [
-      "public"
-    ],
-    "methods": [
-      "public static com.yahoo.config.provision.AllocatedHosts withHosts(java.util.Set)",
-      "public java.util.Set getHosts()",
-      "public boolean equals(java.lang.Object)",
-      "public int hashCode()",
-      "public java.lang.String toString()"
-    ],
-    "fields": []
-  },
-  "com.yahoo.config.provision.ApplicationId$Builder": {
-    "superClass": "java.lang.Object",
-    "interfaces": [],
-    "attributes": [
-      "public"
-    ],
-    "methods": [
-      "public void <init>()",
-      "public com.yahoo.config.provision.ApplicationId$Builder tenant(com.yahoo.config.provision.TenantName)",
-      "public com.yahoo.config.provision.ApplicationId$Builder tenant(java.lang.String)",
-      "public com.yahoo.config.provision.ApplicationId$Builder applicationName(com.yahoo.config.provision.ApplicationName)",
-      "public com.yahoo.config.provision.ApplicationId$Builder applicationName(java.lang.String)",
-      "public com.yahoo.config.provision.ApplicationId$Builder instanceName(com.yahoo.config.provision.InstanceName)",
-      "public com.yahoo.config.provision.ApplicationId$Builder instanceName(java.lang.String)",
-      "public com.yahoo.config.provision.ApplicationId build()"
-    ],
-    "fields": []
-  },
-  "com.yahoo.config.provision.ApplicationId": {
-    "superClass": "java.lang.Object",
-    "interfaces": [
-      "java.lang.Comparable"
-    ],
-    "attributes": [
-      "public",
-      "final"
-    ],
-    "methods": [
-      "public void <init>(com.yahoo.cloud.config.ApplicationIdConfig)",
-      "public static com.yahoo.config.provision.ApplicationId from(com.yahoo.config.provision.TenantName, com.yahoo.config.provision.ApplicationName, com.yahoo.config.provision.InstanceName)",
-      "public static com.yahoo.config.provision.ApplicationId from(java.lang.String, java.lang.String, java.lang.String)",
-      "public static com.yahoo.config.provision.ApplicationId fromSerializedForm(java.lang.String)",
-      "public int hashCode()",
-      "public boolean equals(java.lang.Object)",
-      "public java.lang.String serializedForm()",
-      "public java.lang.String toShortString()",
-      "public java.lang.String toFullString()",
-      "public java.lang.String toString()",
-      "public com.yahoo.config.provision.TenantName tenant()",
-      "public com.yahoo.config.provision.ApplicationName application()",
-      "public com.yahoo.config.provision.InstanceName instance()",
-      "public int compareTo(com.yahoo.config.provision.ApplicationId)",
-      "public static com.yahoo.config.provision.ApplicationId defaultId()",
-      "public static com.yahoo.config.provision.ApplicationId global()",
-      "public bridge synthetic int compareTo(java.lang.Object)"
-    ],
-    "fields": []
-  },
-  "com.yahoo.config.provision.ApplicationLockException": {
-    "superClass": "java.lang.RuntimeException",
-    "interfaces": [],
-    "attributes": [
-      "public"
-    ],
-    "methods": [
-      "public void <init>(java.lang.Exception)"
-    ],
-    "fields": []
-  },
-  "com.yahoo.config.provision.ApplicationName": {
-    "superClass": "java.lang.Object",
-    "interfaces": [
-      "java.lang.Comparable"
-    ],
-    "attributes": [
-      "public"
-    ],
-    "methods": [
-      "public int hashCode()",
-      "public boolean equals(java.lang.Object)",
-      "public java.lang.String toString()",
-      "public static com.yahoo.config.provision.ApplicationName from(java.lang.String)",
-      "public static com.yahoo.config.provision.ApplicationName defaultName()",
-      "public boolean isDefault()",
-      "public java.lang.String value()",
-      "public int compareTo(com.yahoo.config.provision.ApplicationName)",
-      "public bridge synthetic int compareTo(java.lang.Object)"
-    ],
-    "fields": []
-  },
-  "com.yahoo.config.provision.AthenzDomain": {
-    "superClass": "java.lang.Object",
-    "interfaces": [],
-    "attributes": [
-      "public"
-    ],
-    "methods": [
-      "public static com.yahoo.config.provision.AthenzDomain from(java.lang.String)",
-      "public java.lang.String value()",
-      "public java.lang.String toString()",
-      "public boolean equals(java.lang.Object)",
-      "public int hashCode()"
-    ],
-    "fields": []
-  },
-  "com.yahoo.config.provision.AthenzService": {
-    "superClass": "java.lang.Object",
-    "interfaces": [],
-    "attributes": [
-      "public"
-    ],
-    "methods": [
-      "public java.lang.String value()",
-      "public static com.yahoo.config.provision.AthenzService from(java.lang.String)",
-      "public java.lang.String toString()",
-      "public boolean equals(java.lang.Object)",
-      "public int hashCode()"
-    ],
-    "fields": []
-  },
-  "com.yahoo.config.provision.Capacity": {
-    "superClass": "java.lang.Object",
-    "interfaces": [],
-    "attributes": [
-      "public",
-      "final"
-    ],
-    "methods": [
-      "public int nodeCount()",
-      "public java.util.Optional flavor()",
-      "public java.util.Optional nodeResources()",
-      "public boolean isRequired()",
-      "public boolean canFail()",
-      "public com.yahoo.config.provision.NodeType type()",
-      "public java.lang.String toString()",
-      "public static com.yahoo.config.provision.Capacity fromNodeCount(int)",
-      "public static com.yahoo.config.provision.Capacity fromCount(int, com.yahoo.config.provision.NodeResources)",
-      "public static com.yahoo.config.provision.Capacity fromCount(int, com.yahoo.config.provision.NodeResources, boolean, boolean)",
-      "public static com.yahoo.config.provision.Capacity fromCount(int, java.util.Optional, boolean, boolean)",
-      "public static com.yahoo.config.provision.Capacity fromRequiredNodeType(com.yahoo.config.provision.NodeType)"
-    ],
-    "fields": []
-  },
-  "com.yahoo.config.provision.CertificateNotReadyException": {
-    "superClass": "com.yahoo.config.provision.TransientException",
-    "interfaces": [],
-    "attributes": [
-      "public"
-    ],
-    "methods": [
-      "public void <init>(java.lang.String)"
-    ],
-    "fields": []
-  },
-  "com.yahoo.config.provision.CloudName": {
-    "superClass": "java.lang.Object",
-    "interfaces": [
-      "java.lang.Comparable"
-    ],
-    "attributes": [
-      "public"
-    ],
-    "methods": [
-      "public java.lang.String value()",
-      "public boolean isDefault()",
-      "public static com.yahoo.config.provision.CloudName defaultName()",
-      "public static com.yahoo.config.provision.CloudName from(java.lang.String)",
-      "public boolean equals(java.lang.Object)",
-      "public int hashCode()",
-      "public java.lang.String toString()",
-      "public int compareTo(com.yahoo.config.provision.CloudName)",
-      "public bridge synthetic int compareTo(java.lang.Object)"
-    ],
-    "fields": []
-  },
-  "com.yahoo.config.provision.ClusterMembership": {
-    "superClass": "java.lang.Object",
-    "interfaces": [],
-    "attributes": [
-      "public"
-    ],
-    "methods": [
-      "protected void <init>()",
-      "protected java.lang.String toStringValue()",
-      "public com.yahoo.config.provision.ClusterSpec cluster()",
-      "public int index()",
-      "public boolean retired()",
-      "public com.yahoo.config.provision.ClusterMembership retire()",
-      "public com.yahoo.config.provision.ClusterMembership unretire()",
-      "public com.yahoo.config.provision.ClusterMembership with(com.yahoo.config.provision.ClusterSpec)",
-      "public java.lang.String stringValue()",
-      "public int hashCode()",
-      "public boolean equals(java.lang.Object)",
-      "public java.lang.String toString()",
-      "public static com.yahoo.config.provision.ClusterMembership from(java.lang.String, com.yahoo.component.Version)",
-      "public static com.yahoo.config.provision.ClusterMembership from(com.yahoo.config.provision.ClusterSpec, int)",
-      "public static com.yahoo.config.provision.ClusterMembership retiredFrom(com.yahoo.config.provision.ClusterSpec, int)"
-    ],
-    "fields": []
-  },
-  "com.yahoo.config.provision.ClusterSpec$Group": {
-    "superClass": "java.lang.Object",
-    "interfaces": [],
-    "attributes": [
-      "public",
-      "final"
-    ],
-    "methods": [
-      "public static com.yahoo.config.provision.ClusterSpec$Group from(int)",
-      "public int index()",
-      "public java.lang.String toString()",
-      "public boolean equals(java.lang.Object)",
-      "public int hashCode()"
-    ],
-    "fields": []
-  },
-  "com.yahoo.config.provision.ClusterSpec$Id": {
-    "superClass": "java.lang.Object",
-    "interfaces": [],
-    "attributes": [
-      "public",
-      "final"
-    ],
-    "methods": [
-      "public void <init>(java.lang.String)",
-      "public static com.yahoo.config.provision.ClusterSpec$Id from(java.lang.String)",
-      "public java.lang.String value()",
-      "public java.lang.String toString()",
-      "public boolean equals(java.lang.Object)",
-      "public int hashCode()"
-    ],
-    "fields": []
-  },
-  "com.yahoo.config.provision.ClusterSpec$Type": {
-    "superClass": "java.lang.Enum",
-    "interfaces": [],
-    "attributes": [
-      "public",
-      "final",
-      "enum"
-    ],
-    "methods": [
-      "public static com.yahoo.config.provision.ClusterSpec$Type[] values()",
-      "public static com.yahoo.config.provision.ClusterSpec$Type valueOf(java.lang.String)",
-      "public boolean isContent()",
-      "public boolean isContainer()",
-      "public static com.yahoo.config.provision.ClusterSpec$Type from(java.lang.String)"
-    ],
-    "fields": [
-      "public static final enum com.yahoo.config.provision.ClusterSpec$Type admin",
-      "public static final enum com.yahoo.config.provision.ClusterSpec$Type container",
-      "public static final enum com.yahoo.config.provision.ClusterSpec$Type content",
-      "public static final enum com.yahoo.config.provision.ClusterSpec$Type combined"
-    ]
-  },
-  "com.yahoo.config.provision.ClusterSpec": {
-    "superClass": "java.lang.Object",
-    "interfaces": [],
-    "attributes": [
-      "public",
-      "final"
-    ],
-    "methods": [
-      "public com.yahoo.config.provision.ClusterSpec$Type type()",
-      "public com.yahoo.config.provision.ClusterSpec$Id id()",
-      "public com.yahoo.component.Version vespaVersion()",
-      "public java.util.Optional group()",
-      "public boolean isExclusive()",
-      "public com.yahoo.config.provision.ClusterSpec with(java.util.Optional)",
-      "public com.yahoo.config.provision.ClusterSpec exclusive(boolean)",
-      "public static com.yahoo.config.provision.ClusterSpec request(com.yahoo.config.provision.ClusterSpec$Type, com.yahoo.config.provision.ClusterSpec$Id, com.yahoo.component.Version, boolean)",
-      "public static com.yahoo.config.provision.ClusterSpec from(com.yahoo.config.provision.ClusterSpec$Type, com.yahoo.config.provision.ClusterSpec$Id, com.yahoo.config.provision.ClusterSpec$Group, com.yahoo.component.Version, boolean)",
-      "public java.lang.String toString()",
-      "public int hashCode()",
-      "public boolean equals(java.lang.Object)",
-      "public boolean satisfies(com.yahoo.config.provision.ClusterSpec)"
-    ],
-    "fields": []
-  },
-  "com.yahoo.config.provision.Deployer": {
-    "superClass": "java.lang.Object",
-    "interfaces": [],
-    "attributes": [
-      "public",
-      "interface",
-      "abstract"
-    ],
-    "methods": [
-      "public java.util.Optional deployFromLocalActive(com.yahoo.config.provision.ApplicationId)",
-      "public abstract java.util.Optional deployFromLocalActive(com.yahoo.config.provision.ApplicationId, boolean)",
-      "public java.util.Optional deployFromLocalActive(com.yahoo.config.provision.ApplicationId, java.time.Duration)",
-      "public abstract java.util.Optional deployFromLocalActive(com.yahoo.config.provision.ApplicationId, java.time.Duration, boolean)",
-      "public abstract java.util.Optional lastDeployTime(com.yahoo.config.provision.ApplicationId)"
-    ],
-    "fields": []
-  },
-  "com.yahoo.config.provision.Deployment": {
-    "superClass": "java.lang.Object",
-    "interfaces": [],
-    "attributes": [
-      "public",
-      "interface",
-      "abstract"
-    ],
-    "methods": [
-      "public abstract void prepare()",
-      "public abstract void activate()",
-      "public abstract void restart(com.yahoo.config.provision.HostFilter)"
-    ],
-    "fields": []
-  },
-  "com.yahoo.config.provision.DockerImage": {
-    "superClass": "java.lang.Object",
-    "interfaces": [],
-    "attributes": [
-      "public"
-    ],
-    "methods": [
-      "public java.lang.String repository()",
-      "public java.util.Optional tag()",
-      "public com.yahoo.component.Version tagAsVersion()",
-      "public com.yahoo.config.provision.DockerImage withTag(com.yahoo.component.Version)",
-      "public java.lang.String asString()",
-      "public java.lang.String toString()",
-      "public boolean equals(java.lang.Object)",
-      "public int hashCode()",
-      "public static com.yahoo.config.provision.DockerImage fromString(java.lang.String)"
-    ],
-    "fields": [
-      "public static final com.yahoo.config.provision.DockerImage EMPTY"
-    ]
-  },
-  "com.yahoo.config.provision.Environment": {
-    "superClass": "java.lang.Enum",
-    "interfaces": [],
-    "attributes": [
-      "public",
-      "final",
-      "enum"
-    ],
-    "methods": [
-      "public static com.yahoo.config.provision.Environment[] values()",
-      "public static com.yahoo.config.provision.Environment valueOf(java.lang.String)",
-      "public boolean isManuallyDeployed()",
-      "public boolean isTest()",
-      "public boolean isProduction()",
-      "public boolean isMultiRegion()",
-      "public static com.yahoo.config.provision.Environment defaultEnvironment()",
-      "public static com.yahoo.config.provision.Environment from(java.lang.String)",
-      "public java.lang.String value()"
-    ],
-    "fields": [
-      "public static final enum com.yahoo.config.provision.Environment prod",
-      "public static final enum com.yahoo.config.provision.Environment staging",
-      "public static final enum com.yahoo.config.provision.Environment test",
-      "public static final enum com.yahoo.config.provision.Environment dev",
-      "public static final enum com.yahoo.config.provision.Environment perf"
-    ]
-  },
-  "com.yahoo.config.provision.Flavor$Type": {
-    "superClass": "java.lang.Enum",
-    "interfaces": [],
-    "attributes": [
-      "public",
-      "final",
-      "enum"
-    ],
-    "methods": [
-      "public static com.yahoo.config.provision.Flavor$Type[] values()",
-      "public static com.yahoo.config.provision.Flavor$Type valueOf(java.lang.String)"
-    ],
-    "fields": [
-      "public static final enum com.yahoo.config.provision.Flavor$Type undefined",
-      "public static final enum com.yahoo.config.provision.Flavor$Type BARE_METAL",
-      "public static final enum com.yahoo.config.provision.Flavor$Type VIRTUAL_MACHINE",
-      "public static final enum com.yahoo.config.provision.Flavor$Type DOCKER_CONTAINER"
-    ]
-  },
-  "com.yahoo.config.provision.Flavor": {
-    "superClass": "java.lang.Object",
-    "interfaces": [],
-    "attributes": [
-      "public"
-    ],
-    "methods": [
-      "public void <init>(com.yahoo.config.provisioning.FlavorsConfig$Flavor)",
-      "public void <init>(com.yahoo.config.provision.NodeResources)",
-      "public com.yahoo.config.provision.Flavor with(com.yahoo.config.provision.host.FlavorOverrides)",
-      "public com.yahoo.config.provision.Flavor with(com.yahoo.config.provision.NodeResources)",
-      "public java.lang.String name()",
-      "public int cost()",
-      "public boolean isConfigured()",
-      "public com.yahoo.config.provision.NodeResources resources()",
-      "public java.util.Optional flavorOverrides()",
-      "public double getMinMainMemoryAvailableGb()",
-      "public double getMinDiskAvailableGb()",
-      "public boolean hasFastDisk()",
-      "public double getBandwidthGbps()",
-      "public double getMinCpuCores()",
-      "public com.yahoo.config.provision.Flavor$Type getType()",
-      "public boolean isDocker()",
-      "public int hashCode()",
-      "public boolean equals(java.lang.Object)",
-      "public java.lang.String toString()"
-    ],
-    "fields": []
-  },
-  "com.yahoo.config.provision.HostFilter": {
-    "superClass": "java.lang.Object",
-    "interfaces": [],
-    "attributes": [
-      "public"
-    ],
-    "methods": [
-      "public boolean matches(java.lang.String, java.lang.String, java.util.Optional)",
-      "public static com.yahoo.config.provision.HostFilter all()",
-      "public static com.yahoo.config.provision.HostFilter hostname(java.lang.String)",
-      "public static com.yahoo.config.provision.HostFilter flavor(java.lang.String)",
-      "public static com.yahoo.config.provision.HostFilter clusterType(com.yahoo.config.provision.ClusterSpec$Type)",
-      "public static com.yahoo.config.provision.HostFilter clusterId(com.yahoo.config.provision.ClusterSpec$Id)",
-      "public static com.yahoo.config.provision.HostFilter from(java.util.Collection, java.util.Collection, java.util.Collection, java.util.Collection)",
-      "public static com.yahoo.config.provision.HostFilter from(java.lang.String, java.lang.String, java.lang.String, java.lang.String)"
-    ],
-    "fields": []
-  },
-  "com.yahoo.config.provision.HostLivenessTracker": {
-    "superClass": "java.lang.Object",
-    "interfaces": [],
-    "attributes": [
-      "public",
-      "interface",
-      "abstract"
-    ],
-    "methods": [
-      "public abstract void receivedRequestFrom(java.lang.String)",
-      "public abstract java.util.Optional lastRequestFrom(java.lang.String)"
-    ],
-    "fields": []
-  },
-  "com.yahoo.config.provision.HostName": {
-    "superClass": "java.lang.Object",
-    "interfaces": [
-      "java.lang.Comparable"
-    ],
-    "attributes": [
-      "public"
-    ],
-    "methods": [
-      "public java.lang.String value()",
-      "public static com.yahoo.config.provision.HostName from(java.lang.String)",
-      "public int hashCode()",
-      "public boolean equals(java.lang.Object)",
-      "public java.lang.String toString()",
-      "public int compareTo(com.yahoo.config.provision.HostName)",
-      "public bridge synthetic int compareTo(java.lang.Object)"
-    ],
-    "fields": []
-  },
-  "com.yahoo.config.provision.HostSpec": {
-    "superClass": "java.lang.Object",
-    "interfaces": [
-      "java.lang.Comparable"
-    ],
-    "attributes": [
-      "public"
-    ],
-    "methods": [
-      "public void <init>(java.lang.String, java.util.Optional)",
-      "public void <init>(java.lang.String, com.yahoo.config.provision.ClusterMembership, com.yahoo.config.provision.Flavor, java.util.Optional)",
-      "public void <init>(java.lang.String, java.util.List)",
-      "public void <init>(java.lang.String, java.util.List, com.yahoo.config.provision.Flavor)",
-      "public void <init>(java.lang.String, java.util.List, com.yahoo.config.provision.ClusterMembership)",
-      "public void <init>(java.lang.String, java.util.List, java.util.Optional, java.util.Optional)",
-      "public void <init>(java.lang.String, java.util.List, java.util.Optional, java.util.Optional, java.util.Optional)",
-      "public void <init>(java.lang.String, java.util.List, java.util.Optional, java.util.Optional, java.util.Optional, java.util.Optional)",
-      "public void <init>(java.lang.String, java.util.List, java.util.Optional, java.util.Optional, java.util.Optional, java.util.Optional, java.util.Optional)",
-      "public java.lang.String hostname()",
-      "public java.util.List aliases()",
-      "public java.util.Optional flavor()",
-      "public java.util.Optional version()",
-      "public java.util.Optional membership()",
-      "public java.util.Optional networkPorts()",
-      "public java.util.Optional requestedResources()",
-      "public com.yahoo.config.provision.HostSpec withPorts(java.util.Optional)",
-      "public java.lang.String toString()",
-      "public boolean equals(java.lang.Object)",
-      "public int hashCode()",
-      "public int compareTo(com.yahoo.config.provision.HostSpec)",
-      "public bridge synthetic int compareTo(java.lang.Object)"
-    ],
-    "fields": []
-  },
-  "com.yahoo.config.provision.InfraDeployer": {
-    "superClass": "java.lang.Object",
-    "interfaces": [],
-    "attributes": [
-      "public",
-      "interface",
-      "abstract"
-    ],
-    "methods": [
-      "public abstract java.util.Optional getDeployment(com.yahoo.config.provision.ApplicationId)",
-      "public abstract java.util.Map getSupportedInfraDeployments()"
-    ],
-    "fields": []
-  },
-  "com.yahoo.config.provision.InstanceName": {
-    "superClass": "java.lang.Object",
-    "interfaces": [
-      "java.lang.Comparable"
-    ],
-    "attributes": [
-      "public"
-    ],
-    "methods": [
-      "public int hashCode()",
-      "public boolean equals(java.lang.Object)",
-      "public java.lang.String toString()",
-      "public static com.yahoo.config.provision.InstanceName from(java.lang.String)",
-      "public static com.yahoo.config.provision.InstanceName defaultName()",
-      "public boolean isDefault()",
-      "public boolean isTester()",
-      "public java.lang.String value()",
-      "public int compareTo(com.yahoo.config.provision.InstanceName)",
-      "public bridge synthetic int compareTo(java.lang.Object)"
-    ],
-    "fields": []
-  },
-  "com.yahoo.config.provision.NetworkPorts$Allocation": {
-    "superClass": "java.lang.Object",
-    "interfaces": [],
-    "attributes": [
-      "public"
-    ],
-    "methods": [
-      "public void <init>(int, java.lang.String, java.lang.String, java.lang.String)",
-      "public java.lang.String key()",
-      "public java.lang.String toString()",
-      "public int hashCode()",
-      "public boolean equals(java.lang.Object)"
-    ],
-    "fields": [
-      "public final int port",
-      "public final java.lang.String serviceType",
-      "public final java.lang.String configId",
-      "public final java.lang.String portSuffix"
-    ]
-  },
-  "com.yahoo.config.provision.NetworkPorts": {
-    "superClass": "java.lang.Object",
-    "interfaces": [],
-    "attributes": [
-      "public"
-    ],
-    "methods": [
-      "public void <init>(java.util.Collection)",
-      "public java.util.Collection allocations()",
-      "public int size()",
-      "public int hashCode()",
-      "public boolean equals(java.lang.Object)"
-    ],
-    "fields": []
-  },
-  "com.yahoo.config.provision.NodeFlavors": {
-    "superClass": "java.lang.Object",
-    "interfaces": [],
-    "attributes": [
-      "public"
-    ],
-    "methods": [
-      "public void <init>(com.yahoo.config.provisioning.FlavorsConfig)",
-      "public java.util.List getFlavors()",
-      "public java.util.Optional getFlavor(java.lang.String)",
-      "public com.yahoo.config.provision.Flavor getFlavorOrThrow(java.lang.String)",
-      "public boolean exists(java.lang.String)"
-    ],
-    "fields": []
-  },
-  "com.yahoo.config.provision.NodeResources$DiskSpeed": {
-    "superClass": "java.lang.Enum",
-    "interfaces": [],
-    "attributes": [
-      "public",
-      "final",
-      "enum"
-    ],
-    "methods": [
-      "public static com.yahoo.config.provision.NodeResources$DiskSpeed[] values()",
-      "public static com.yahoo.config.provision.NodeResources$DiskSpeed valueOf(java.lang.String)",
-      "public static int compare(com.yahoo.config.provision.NodeResources$DiskSpeed, com.yahoo.config.provision.NodeResources$DiskSpeed)",
-      "public boolean compatibleWith(com.yahoo.config.provision.NodeResources$DiskSpeed)",
-      "public boolean isDefault()",
-      "public static com.yahoo.config.provision.NodeResources$DiskSpeed getDefault()"
-    ],
-    "fields": [
-      "public static final enum com.yahoo.config.provision.NodeResources$DiskSpeed fast",
-      "public static final enum com.yahoo.config.provision.NodeResources$DiskSpeed slow",
-      "public static final enum com.yahoo.config.provision.NodeResources$DiskSpeed any"
-    ]
-  },
-  "com.yahoo.config.provision.NodeResources$StorageType": {
-    "superClass": "java.lang.Enum",
-    "interfaces": [],
-    "attributes": [
-      "public",
-      "final",
-      "enum"
-    ],
-    "methods": [
-      "public static com.yahoo.config.provision.NodeResources$StorageType[] values()",
-      "public static com.yahoo.config.provision.NodeResources$StorageType valueOf(java.lang.String)",
-      "public static int compare(com.yahoo.config.provision.NodeResources$StorageType, com.yahoo.config.provision.NodeResources$StorageType)",
-      "public boolean compatibleWith(com.yahoo.config.provision.NodeResources$StorageType)",
-      "public boolean isDefault()",
-      "public static com.yahoo.config.provision.NodeResources$StorageType getDefault()"
-    ],
-    "fields": [
-      "public static final enum com.yahoo.config.provision.NodeResources$StorageType remote",
-      "public static final enum com.yahoo.config.provision.NodeResources$StorageType local",
-      "public static final enum com.yahoo.config.provision.NodeResources$StorageType any"
-    ]
-  },
-  "com.yahoo.config.provision.NodeResources": {
-    "superClass": "java.lang.Object",
-    "interfaces": [],
-    "attributes": [
-      "public"
-    ],
-    "methods": [
-      "public void <init>(double, double, double, double)",
-      "public void <init>(double, double, double, double, com.yahoo.config.provision.NodeResources$DiskSpeed)",
-      "public void <init>(double, double, double, double, com.yahoo.config.provision.NodeResources$DiskSpeed, com.yahoo.config.provision.NodeResources$StorageType)",
-      "public double vcpu()",
-      "public double memoryGb()",
-      "public double diskGb()",
-      "public double bandwidthGbps()",
-      "public com.yahoo.config.provision.NodeResources$DiskSpeed diskSpeed()",
-      "public com.yahoo.config.provision.NodeResources$StorageType storageType()",
-      "public com.yahoo.config.provision.NodeResources withVcpu(double)",
-      "public com.yahoo.config.provision.NodeResources withMemoryGb(double)",
-      "public com.yahoo.config.provision.NodeResources withDiskGb(double)",
-      "public com.yahoo.config.provision.NodeResources withBandwidthGbps(double)",
-      "public com.yahoo.config.provision.NodeResources withDiskSpeed(com.yahoo.config.provision.NodeResources$DiskSpeed)",
-      "public com.yahoo.config.provision.NodeResources with(com.yahoo.config.provision.NodeResources$DiskSpeed)",
-      "public com.yahoo.config.provision.NodeResources with(com.yahoo.config.provision.NodeResources$StorageType)",
-      "public com.yahoo.config.provision.NodeResources justNumbers()",
-      "public com.yahoo.config.provision.NodeResources subtract(com.yahoo.config.provision.NodeResources)",
-      "public com.yahoo.config.provision.NodeResources add(com.yahoo.config.provision.NodeResources)",
-      "public boolean equals(java.lang.Object)",
-      "public int hashCode()",
-      "public java.lang.String toString()",
-      "public boolean satisfies(com.yahoo.config.provision.NodeResources)",
-      "public boolean compatibleWith(com.yahoo.config.provision.NodeResources)",
-      "public static com.yahoo.config.provision.NodeResources fromLegacyName(java.lang.String)"
-    ],
-    "fields": []
-  },
-  "com.yahoo.config.provision.NodeType": {
-    "superClass": "java.lang.Enum",
-    "interfaces": [],
-    "attributes": [
-      "public",
-      "final",
-      "enum"
-    ],
-    "methods": [
-      "public static com.yahoo.config.provision.NodeType[] values()",
-      "public static com.yahoo.config.provision.NodeType valueOf(java.lang.String)",
-      "public boolean isDockerHost()",
-      "public java.lang.String description()",
-      "public com.yahoo.config.provision.NodeType childNodeType()",
-      "public java.util.List childNodeTypes()",
-      "public boolean canRun(com.yahoo.config.provision.NodeType)"
-    ],
-    "fields": [
-      "public static final enum com.yahoo.config.provision.NodeType tenant",
-      "public static final enum com.yahoo.config.provision.NodeType host",
-      "public static final enum com.yahoo.config.provision.NodeType proxy",
-      "public static final enum com.yahoo.config.provision.NodeType proxyhost",
-      "public static final enum com.yahoo.config.provision.NodeType config",
-      "public static final enum com.yahoo.config.provision.NodeType confighost",
-      "public static final enum com.yahoo.config.provision.NodeType controller",
-      "public static final enum com.yahoo.config.provision.NodeType controllerhost",
-      "public static final enum com.yahoo.config.provision.NodeType devhost"
-    ]
-  },
-  "com.yahoo.config.provision.OutOfCapacityException": {
-    "superClass": "java.lang.RuntimeException",
-    "interfaces": [],
-    "attributes": [
-      "public"
-    ],
-    "methods": [
-      "public void <init>(java.lang.String)"
-    ],
-    "fields": []
-  },
-  "com.yahoo.config.provision.ParentHostUnavailableException": {
-    "superClass": "com.yahoo.config.provision.TransientException",
-    "interfaces": [],
-    "attributes": [
-      "public"
-    ],
-    "methods": [
-      "public void <init>(java.lang.String)"
-    ],
-    "fields": []
-  },
-  "com.yahoo.config.provision.ProvisionLogger": {
-    "superClass": "java.lang.Object",
-    "interfaces": [],
-    "attributes": [
-      "public",
-      "interface",
-      "abstract"
-    ],
-    "methods": [
-      "public abstract void log(java.util.logging.Level, java.lang.String)"
-    ],
-    "fields": []
-  },
-  "com.yahoo.config.provision.Provisioner": {
-    "superClass": "java.lang.Object",
-    "interfaces": [],
-    "attributes": [
-      "public",
-      "interface",
-      "abstract"
-    ],
-    "methods": [
-      "public abstract java.util.List prepare(com.yahoo.config.provision.ApplicationId, com.yahoo.config.provision.ClusterSpec, com.yahoo.config.provision.Capacity, int, com.yahoo.config.provision.ProvisionLogger)",
-      "public abstract void activate(com.yahoo.transaction.NestedTransaction, com.yahoo.config.provision.ApplicationId, java.util.Collection)",
-      "public abstract void remove(com.yahoo.transaction.NestedTransaction, com.yahoo.config.provision.ApplicationId)",
-      "public abstract void restart(com.yahoo.config.provision.ApplicationId, com.yahoo.config.provision.HostFilter)"
-    ],
-    "fields": []
-  },
-  "com.yahoo.config.provision.RegionName": {
-    "superClass": "java.lang.Object",
-    "interfaces": [
-      "java.lang.Comparable"
-    ],
-    "attributes": [
-      "public"
-    ],
-    "methods": [
-      "public int hashCode()",
-      "public boolean equals(java.lang.Object)",
-      "public java.lang.String toString()",
-      "public static com.yahoo.config.provision.RegionName from(java.lang.String)",
-      "public static com.yahoo.config.provision.RegionName defaultName()",
-      "public boolean isDefault()",
-      "public java.lang.String value()",
-      "public int compareTo(com.yahoo.config.provision.RegionName)",
-      "public bridge synthetic int compareTo(java.lang.Object)"
-    ],
-    "fields": []
-  },
-  "com.yahoo.config.provision.SystemName": {
-    "superClass": "java.lang.Enum",
-    "interfaces": [],
-    "attributes": [
-      "public",
-      "final",
-      "enum"
-    ],
-    "methods": [
-      "public static com.yahoo.config.provision.SystemName[] values()",
-      "public static com.yahoo.config.provision.SystemName valueOf(java.lang.String)",
-      "public static com.yahoo.config.provision.SystemName defaultSystem()",
-      "public static com.yahoo.config.provision.SystemName from(java.lang.String)",
-      "public java.lang.String value()",
-      "public boolean isPublic()",
-      "public boolean isCd()",
-      "public static java.util.Set all()",
-      "public static java.util.Set allOf(java.util.function.Predicate)"
-    ],
-    "fields": [
-      "public static final enum com.yahoo.config.provision.SystemName cd",
-      "public static final enum com.yahoo.config.provision.SystemName main",
-      "public static final enum com.yahoo.config.provision.SystemName Public",
-      "public static final enum com.yahoo.config.provision.SystemName PublicCd",
-      "public static final enum com.yahoo.config.provision.SystemName dev"
-    ]
-  },
-  "com.yahoo.config.provision.TenantName": {
-    "superClass": "java.lang.Object",
-    "interfaces": [
-      "java.lang.Comparable"
-    ],
-    "attributes": [
-      "public"
-    ],
-    "methods": [
-      "public java.lang.String value()",
-      "public static com.yahoo.config.provision.TenantName from(java.lang.String)",
-      "public int hashCode()",
-      "public boolean equals(java.lang.Object)",
-      "public java.lang.String toString()",
-      "public static com.yahoo.config.provision.TenantName defaultName()",
-      "public int compareTo(com.yahoo.config.provision.TenantName)",
-      "public bridge synthetic int compareTo(java.lang.Object)"
-    ],
-    "fields": []
-  },
-  "com.yahoo.config.provision.TransientException": {
-    "superClass": "java.lang.RuntimeException",
-    "interfaces": [],
-    "attributes": [
-      "public",
-      "abstract"
-    ],
-    "methods": [
-      "public void <init>(java.lang.String)",
-      "public void <init>(java.lang.String, java.lang.Throwable)"
-    ],
-    "fields": []
-  },
-  "com.yahoo.config.provision.Zone": {
-    "superClass": "java.lang.Object",
-    "interfaces": [],
-    "attributes": [
-      "public"
-    ],
-    "methods": [
-      "public void <init>(com.yahoo.cloud.config.ConfigserverConfig, com.yahoo.config.provision.NodeFlavors)",
-      "public void <init>(com.yahoo.config.provision.Environment, com.yahoo.config.provision.RegionName)",
-      "public void <init>(com.yahoo.config.provision.SystemName, com.yahoo.config.provision.Environment, com.yahoo.config.provision.RegionName)",
-      "public void <init>(com.yahoo.config.provision.CloudName, com.yahoo.config.provision.SystemName, com.yahoo.config.provision.Environment, com.yahoo.config.provision.RegionName)",
-      "public com.yahoo.config.provision.CloudName cloud()",
-      "public com.yahoo.config.provision.SystemName system()",
-      "public com.yahoo.config.provision.Environment environment()",
-      "public com.yahoo.config.provision.RegionName region()",
-      "public java.util.Optional nodeFlavors()",
-      "public static com.yahoo.config.provision.Zone defaultZone()",
-      "public java.lang.String toString()",
-      "public boolean equals(java.lang.Object)",
-      "public int hashCode()"
-    ],
-    "fields": []
-  }
-}
\ No newline at end of file
+{}
\ No newline at end of file
diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/AthenzDomain.java b/config-provisioning/src/main/java/com/yahoo/config/provision/AthenzDomain.java
index 193a02ccf26..7b2a33a17f5 100644
--- a/config-provisioning/src/main/java/com/yahoo/config/provision/AthenzDomain.java
+++ b/config-provisioning/src/main/java/com/yahoo/config/provision/AthenzDomain.java
@@ -1,17 +1,29 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.config.provision;
 
+import java.util.regex.Pattern;
+
 /**
  * @author mortent
  */
 public class AthenzDomain {
 
+    private static final Pattern PATTERN = Pattern.compile("[a-zA-Z0-9_][a-zA-Z0-9_\\-.]*[a-zA-Z0-9_]");
+
     private final String name;
 
     private AthenzDomain(String name) {
+        // TODO bjorncs: Temporarily disable name validation
+        // validateName(name);
         this.name = name;
     }
 
+    private static void validateName(String name) {
+        if (!PATTERN.matcher(name).matches()) {
+            throw new IllegalArgumentException("Not a valid domain name: '" + name + "'");
+        }
+    }
+
     public static AthenzDomain from(String value) {
         return new AthenzDomain(value);
     }
diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/Capacity.java b/config-provisioning/src/main/java/com/yahoo/config/provision/Capacity.java
index 59d6ec8feb8..09ee25fb437 100644
--- a/config-provisioning/src/main/java/com/yahoo/config/provision/Capacity.java
+++ b/config-provisioning/src/main/java/com/yahoo/config/provision/Capacity.java
@@ -11,26 +11,46 @@ import java.util.Optional;
  */
 public final class Capacity {
 
-    private final int nodeCount;
+    /** Resources should stay between these values, inclusive */
+    private final ClusterResources min, max;
 
     private final boolean required;
 
     private final boolean canFail;
 
-    private final Optional<NodeResources> nodeResources;
-
     private final NodeType type;
 
-    private Capacity(int nodeCount, Optional<NodeResources> nodeResources, boolean required, boolean canFail, NodeType type) {
-        this.nodeCount = nodeCount;
+    private Capacity(ClusterResources min, ClusterResources max, boolean required, boolean canFail, NodeType type) {
+        validate(min);
+        validate(max);
+        if (max.smallerThan(min))
+            throw new IllegalArgumentException("The max capacity must be larger than the min capacity, but got min " +
+                                               min + " and max " + max);
+        this.min = min;
+        this.max = max;
         this.required = required;
         this.canFail = canFail;
-        this.nodeResources = nodeResources;
         this.type = type;
     }
 
+    private static void validate(ClusterResources resources) {
+        if (resources.nodes() == 0 && resources.groups() == 0) return; // unspecified
+        if (resources.nodes() % resources.groups() != 0)
+            throw new IllegalArgumentException("The number of nodes (" + resources.nodes() +
+                                               ") must be divisible by the number of groups (" + resources.groups() + ")");
+    }
+
     /** Returns the number of nodes requested */
-    public int nodeCount() { return nodeCount; }
+    @Deprecated // TODO: Remove after April 2020
+    public int nodeCount() { return min.nodes(); }
+
+    /** Returns the number of nodes requested (across all groups), or 0 if not specified */
+    @Deprecated // TODO: Remove after April 2020
+    public int nodes() { return min.nodes(); }
+
+    /** Returns the number of groups requested, or 0 if not specified */
+    @Deprecated // TODO: Remove after April 2020
+    public int groups() { return min.groups(); }
 
     /**
      * The node flavor requested, or empty if no legacy flavor name has been used.
@@ -38,14 +58,21 @@ public final class Capacity {
      *
      * @deprecated use nodeResources instead
      */
-    @Deprecated
+    @Deprecated // TODO: Remove after March 2020
     public Optional<String> flavor() {
         if (nodeResources().isEmpty()) return Optional.empty();
-        return nodeResources.map(n -> n.toString());
+        return Optional.of(min.nodeResources().toString());
     }
 
     /** Returns the resources requested for each node, or empty to leave this decision to provisioning */
-    public Optional<NodeResources> nodeResources() { return nodeResources; }
+    @Deprecated // TODO: Remove after March 2020
+    public Optional<NodeResources> nodeResources() {
+        if (min.nodeResources() == NodeResources.unspecified) return Optional.empty();
+        return Optional.of(min.nodeResources());
+    }
+
+    public ClusterResources minResources() { return min; }
+    public ClusterResources maxResources() { return max; }
 
     /** Returns whether the requested number of nodes must be met exactly for a request for this to succeed */
     public boolean isRequired() { return required; }
@@ -64,32 +91,53 @@ public final class Capacity {
      */
     public NodeType type() { return type; }
 
+    public Capacity withGroups(int groups) {
+        return new Capacity(min.withGroups(groups), max.withGroups(groups), required, canFail, type);
+    }
+
     @Override
     public String toString() {
-        return nodeCount + " nodes " + (nodeResources.isPresent() ? nodeResources.get() : "with default resources" );
+        return (required ? "required " : "") +
+               (min.equals(max) ? min : "between " + min + " and " + max);
+    }
+
+    /** Create a non-required, failable capacity request */
+    public static Capacity from(ClusterResources resources) {
+        return from(resources, false, true);
     }
 
-    /** Creates this from a desired node count: The request may be satisfied with a smaller number of nodes. */
-    public static Capacity fromNodeCount(int capacity) {
-        return fromCount(capacity, Optional.empty(), false, true);
+    public static Capacity from(ClusterResources resources, boolean required, boolean canFail) {
+        return from(resources, required, canFail, NodeType.tenant);
+    }
+
+    public static Capacity from(ClusterResources min, ClusterResources max, boolean required, boolean canFail) {
+        return new Capacity(min, max, required, canFail, NodeType.tenant);
     }
 
     /** Create a non-required, failable capacity request */
-    public static Capacity fromCount(int nodeCount, NodeResources resources) {
-        return fromCount(nodeCount, resources, false, true);
+    @Deprecated // TODO: Remove after April 2020
+    public static Capacity fromCount(int nodes, NodeResources resources) {
+        return fromCount(nodes, resources, false, true);
     }
 
-    public static Capacity fromCount(int nodeCount, NodeResources resources, boolean required, boolean canFail) {
-        return new Capacity(nodeCount, Optional.of(resources), required, canFail, NodeType.tenant);
+    @Deprecated // TODO: Remove after April 2020
+    public static Capacity fromCount(int nodes, NodeResources resources, boolean required, boolean canFail) {
+        return fromCount(nodes, Optional.of(resources), required, canFail);
     }
 
-    public static Capacity fromCount(int nodeCount, Optional<NodeResources> resources, boolean required, boolean canFail) {
-        return new Capacity(nodeCount, resources, required, canFail, NodeType.tenant);
+    @Deprecated // TODO: Remove after April 2020
+    public static Capacity fromCount(int nodes, Optional<NodeResources> resources, boolean required, boolean canFail) {
+        return from(new ClusterResources(nodes, 0, resources.orElse(NodeResources.unspecified)),
+                    required, canFail, NodeType.tenant);
     }
 
     /** Creates this from a node type */
     public static Capacity fromRequiredNodeType(NodeType type) {
-        return new Capacity(0,  Optional.empty(), true, false, type);
+        return from(new ClusterResources(0, 0, NodeResources.unspecified), true, false, type);
+    }
+
+    private static Capacity from(ClusterResources resources, boolean required, boolean canFail, NodeType type) {
+        return new Capacity(resources, resources, required, canFail, type);
     }
 
 }
diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterMembership.java b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterMembership.java
index f041823bf04..178bbac9e64 100644
--- a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterMembership.java
+++ b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterMembership.java
@@ -3,9 +3,11 @@ package com.yahoo.config.provision;
 
 import com.yahoo.component.Version;
 
+import java.util.Optional;
+
 /**
  * A node's membership in a cluster. This is a value object.
- * The format is "clusterType/clusterId/groupId/index[/exclusive][/retired]"
+ * The format is "clusterType/clusterId/groupId/index[/exclusive][/retired][/combinedId]"
  *
  * @author bratseth
  */
@@ -18,25 +20,32 @@ public class ClusterMembership {
 
     protected ClusterMembership() {}
 
-    private ClusterMembership(String stringValue, Version vespaVersion) {
+    private ClusterMembership(String stringValue, Version vespaVersion, Optional<String> dockerImageRepo) {
         String[] components = stringValue.split("/");
         if (components.length < 4)
             throw new RuntimeException("Could not parse '" + stringValue + "' to a cluster membership. " +
-                                       "Expected 'clusterType/clusterId/groupId/index[/retired][/exclusive]'");
+                                       "Expected 'clusterType/clusterId/groupId/index[/retired][/exclusive][/combinedId]'");
 
         boolean exclusive = false;
+        var combinedId = Optional.<String>empty();
         if (components.length > 4) {
             for (int i = 4; i < components.length; i++) {
                 String component = components[i];
                 switch (component) {
                     case "exclusive": exclusive = true; break;
                     case "retired": retired = true; break;
+                    default: combinedId = Optional.of(component); break;
                 }
             }
         }
 
-        this.cluster = ClusterSpec.from(ClusterSpec.Type.valueOf(components[0]), ClusterSpec.Id.from(components[1]),
-                                        ClusterSpec.Group.from(Integer.valueOf(components[2])), vespaVersion, exclusive);
+        this.cluster = ClusterSpec.specification(ClusterSpec.Type.valueOf(components[0]), ClusterSpec.Id.from(components[1]))
+                .group(ClusterSpec.Group.from(Integer.parseInt(components[2])))
+                .vespaVersion(vespaVersion)
+                .exclusive(exclusive)
+                .combinedId(combinedId.map(ClusterSpec.Id::from))
+                .dockerImageRepo(dockerImageRepo)
+                .build();
         this.index = Integer.parseInt(components[3]);
         this.stringValue = toStringValue();
     }
@@ -54,7 +63,8 @@ public class ClusterMembership {
                (cluster.group().isPresent() ? "/" + cluster.group().get().index() : "") +
                "/" + index +
                ( cluster.isExclusive() ? "/exclusive" : "") +
-               ( retired ? "/retired" : "");
+               ( retired ? "/retired" : "") +
+               ( cluster.combinedId().isPresent() ? "/" + cluster.combinedId().get().value() : "");
 
     }
 
@@ -101,7 +111,11 @@ public class ClusterMembership {
     public String toString() { return stringValue(); }
 
     public static ClusterMembership from(String stringValue, Version vespaVersion) {
-        return new ClusterMembership(stringValue, vespaVersion);
+        return new ClusterMembership(stringValue, vespaVersion, Optional.empty());
+    }
+
+    public static ClusterMembership from(String stringValue, Version vespaVersion, Optional<String> dockerImageRepo) {
+        return new ClusterMembership(stringValue, vespaVersion, dockerImageRepo);
     }
 
     public static ClusterMembership from(ClusterSpec cluster, int index) {
diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterResources.java b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterResources.java
new file mode 100644
index 00000000000..11ae0845fb0
--- /dev/null
+++ b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterResources.java
@@ -0,0 +1,77 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.config.provision;
+
+import java.util.Objects;
+
+/**
+ * The resources of a cluster
+ *
+ * @author bratseth
+ */
+public class ClusterResources {
+
+    /** The node count in the cluster */
+    private final int nodes;
+
+    /** The number of node groups in the cluster */
+    private final int groups;
+
+    /** The resources of each node in the cluster */
+    private final NodeResources nodeResources;
+
+    public ClusterResources(int nodes, int groups, NodeResources nodeResources) {
+        this.nodes = nodes;
+        this.groups = groups;
+        this.nodeResources = Objects.requireNonNull(nodeResources);
+    }
+
+    /** Returns the total number of allocated nodes (over all groups) */
+    public int nodes() { return nodes; }
+    public int groups() { return groups; }
+    public NodeResources nodeResources() { return nodeResources; }
+
+    public ClusterResources with(NodeResources resources) { return new ClusterResources(nodes, groups, resources); }
+    public ClusterResources withGroups(int groups) { return new ClusterResources(nodes, groups, nodeResources); }
+
+    /** Returns true if this is smaller than the given resources in any dimension */
+    public boolean smallerThan(ClusterResources other) {
+        if (this.nodes < other.nodes) return true;
+        if (this.groups < other.groups) return true;
+        if ( ! this.nodeResources.justNumbers().satisfies(other.nodeResources.justNumbers())) return true;
+        return false;
+    }
+
+    /** Returns true if this is within the given limits (inclusive) and is compatible with them */
+    public boolean isWithin(ClusterResources min, ClusterResources max) {
+        if (this.smallerThan(min)) return false;
+        if (max.smallerThan(this)) return false;
+        if ( ! this.nodeResources.justNonNumbers().compatibleWith(min.nodeResources.justNonNumbers())) return false;
+        if ( ! this.nodeResources.justNonNumbers().compatibleWith(max.nodeResources.justNonNumbers())) return false;
+        return true;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (o == this) return true;
+        if ( ! (o instanceof ClusterResources)) return false;
+
+        ClusterResources other = (ClusterResources)o;
+        if (other.nodes != this.nodes) return false;
+        if (other.groups != this.groups) return false;
+        if ( ! other.nodeResources.equals(this.nodeResources)) return false;
+        return true;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(nodes, groups, nodeResources);
+    }
+
+    @Override
+    public String toString() {
+        return nodes + " nodes" +
+               (groups > 1 ? " (in " + groups + " groups)" : "") +
+               " with " + nodeResources;
+    }
+
+}
diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java
index 16369d82f9f..f7aacbc757b 100644
--- a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java
+++ b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java
@@ -21,13 +21,22 @@ public final class ClusterSpec {
     private final Optional<Group> groupId;
     private final Version vespaVersion;
     private boolean exclusive;
+    private final Optional<Id> combinedId;
+    private final Optional<String> dockerImageRepo;
 
-    private ClusterSpec(Type type, Id id, Optional<Group> groupId, Version vespaVersion, boolean exclusive) {
+    private ClusterSpec(Type type, Id id, Optional<Group> groupId, Version vespaVersion, boolean exclusive,
+                        Optional<Id> combinedId, Optional<String> dockerImageRepo) {
         this.type = type;
         this.id = id;
         this.groupId = groupId;
         this.vespaVersion = vespaVersion;
         this.exclusive = exclusive;
+        // TODO(mpolden): Require combinedId to always be present for type combined after April 2020
+        if (type != Type.combined && combinedId.isPresent()) {
+            throw new IllegalArgumentException("combinedId must be empty for cluster of type " + type);
+        }
+        this.combinedId = combinedId;
+        this.dockerImageRepo = dockerImageRepo;
     }
 
     /** Returns the cluster type */
@@ -36,12 +45,23 @@ public final class ClusterSpec {
     /** Returns the cluster id */
     public Id id() { return id; }
 
+    /** Returns the docker image repository part of a docker image we want this cluster to run */
+    public Optional<String> dockerImageRepo() { return dockerImageRepo; }
+
+    /** Returns the docker image (repository + vespa version) we want this cluster to run */
+    public Optional<String> dockerImage() { return dockerImageRepo.map(repo -> repo + ":" + vespaVersion.toFullString()); }
+
     /** Returns the version of Vespa that we want this cluster to run */
     public Version vespaVersion() { return vespaVersion; }
 
     /** Returns the group within the cluster this specifies, or empty to specify the whole cluster */
     public Optional<Group> group() { return groupId; }
 
+    /** Returns the ID of the container cluster that is combined with this. This is only present for combined clusters */
+    public Optional<Id> combinedId() {
+        return combinedId;
+    }
+
     /**
      * Returns whether the physical hosts running the nodes of this application can
      * also run nodes of other applications. Using exclusive nodes for containers increases security
@@ -50,24 +70,85 @@ public final class ClusterSpec {
     public boolean isExclusive() { return exclusive; }
 
     public ClusterSpec with(Optional<Group> newGroup) {
-        return new ClusterSpec(type, id, newGroup, vespaVersion, exclusive);
+        return new ClusterSpec(type, id, newGroup, vespaVersion, exclusive, combinedId, dockerImageRepo);
     }
 
     public ClusterSpec exclusive(boolean exclusive) {
-        return new ClusterSpec(type, id, groupId, vespaVersion, exclusive);
+        return new ClusterSpec(type, id, groupId, vespaVersion, exclusive, combinedId, dockerImageRepo);
+    }
+
+    /** Creates a ClusterSpec when requesting a cluster */
+    public static Builder request(Type type, Id id) {
+        return new Builder(type, id, false);
     }
 
-    public static ClusterSpec request(Type type, Id id, Version vespaVersion, boolean exclusive) {
-        return new ClusterSpec(type, id, Optional.empty(), vespaVersion, exclusive);
+    /** Creates a ClusterSpec for an existing cluster, group id and vespa version needs to be set */
+    public static Builder specification(Type type, Id id) {
+        return new Builder(type, id, true);
     }
 
-    public static ClusterSpec from(Type type, Id id, Group groupId, Version vespaVersion, boolean exclusive) {
-        return new ClusterSpec(type, id, Optional.of(groupId), vespaVersion, exclusive);
+    public static class Builder {
+
+        private final Type type;
+        private final Id id;
+        private final boolean specification;
+
+        private Optional<Group> groupId = Optional.empty();
+        private Optional<String> dockerImageRepo = Optional.empty();
+        private Version vespaVersion;
+        private boolean exclusive = false;
+        private Optional<Id> combinedId = Optional.empty();
+
+        Builder(Type type, Id id, boolean specification) {
+            this.type = type;
+            this.id = id;
+            this.specification = specification;
+        }
+
+        public ClusterSpec build() {
+            if (specification) {
+                if (groupId.isEmpty()) throw new IllegalArgumentException("groupIs is required to be set when creating a ClusterSpec with specification()");
+                if (vespaVersion == null) throw new IllegalArgumentException("vespaVersion is required to be set when creating a ClusterSpec with specification()");
+            } else
+                if (groupId.isPresent()) throw new IllegalArgumentException("groupId is not allowed to be set when creating a ClusterSpec with request()");
+            return new ClusterSpec(type, id, groupId, vespaVersion, exclusive, combinedId, dockerImageRepo);
+        }
+
+        public Builder group(Group groupId) {
+            this.groupId = Optional.ofNullable(groupId);
+            return this;
+        }
+
+        public Builder vespaVersion(Version vespaVersion) {
+            this.vespaVersion = vespaVersion;
+            return this;
+        }
+
+        public Builder vespaVersion(String vespaVersion) {
+            this.vespaVersion = Version.fromString(vespaVersion);
+            return this;
+        }
+
+        public Builder exclusive(boolean exclusive) {
+            this.exclusive = exclusive;
+            return this;
+        }
+
+        public Builder combinedId(Optional<Id> combinedId) {
+            this.combinedId = combinedId;
+            return this;
+        }
+
+        public Builder dockerImageRepo(Optional<String> dockerImageRepo) {
+            this.dockerImageRepo = dockerImageRepo;
+            return this;
+        }
+
     }
 
     @Override
     public String toString() {
-        return type + " " + id + " " + groupId.map(group -> group + " ").orElse("") + vespaVersion;
+        return type + " " + id + " " + groupId.map(group -> group + " ").orElse("") + vespaVersion + (dockerImageRepo.map(repo -> " " + repo).orElse(""));
     }
 
     @Override
@@ -82,6 +163,7 @@ public final class ClusterSpec {
         if ( ! other.id.equals(this.id)) return false;
         if ( ! other.groupId.equals(this.groupId)) return false;
         if ( ! other.vespaVersion.equals(this.vespaVersion)) return false;
+        if ( ! other.dockerImageRepo.orElse("").equals(this.dockerImageRepo.orElse(""))) return false;
         return true;
     }
 
@@ -90,8 +172,10 @@ public final class ClusterSpec {
      * are ignored.
      */
     public boolean satisfies(ClusterSpec other) {
-        return other.id.equals(this.id) &&
-               other.type.equals(this.type);
+        if (!other.id.equals(this.id)) return false; // ID mismatch
+        if (other.type.isContent() || this.type.isContent()) // Allow seamless transition between content and combined
+            return other.type.isContent() == this.type.isContent();
+        return other.type.equals(this.type);
     }
 
     /** A cluster type */
@@ -158,7 +242,6 @@ public final class ClusterSpec {
     }
 
     /** Identifier of a group within a cluster */
-    @SuppressWarnings("deprecation")
     public static final class Group {
 
         private final int index;
diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/Flavor.java b/config-provisioning/src/main/java/com/yahoo/config/provision/Flavor.java
index 2711406c216..8d767e9f4ad 100644
--- a/config-provisioning/src/main/java/com/yahoo/config/provision/Flavor.java
+++ b/config-provisioning/src/main/java/com/yahoo/config/provision/Flavor.java
@@ -49,6 +49,11 @@ public class Flavor {
         this(resources.toString(), resources, Optional.empty(), Type.DOCKER_CONTAINER, false, 0, resources.vcpu());
     }
 
+    /** Creates a *host* flavor for testing */
+    public Flavor(String name, NodeResources resources) {
+        this(name, resources, Optional.empty(), Flavor.Type.VIRTUAL_MACHINE, true, 0, resources.vcpu());
+    }
+
     private Flavor(String name,
                    NodeResources resources,
                    Optional<FlavorOverrides> flavorOverrides,
@@ -102,9 +107,7 @@ public class Flavor {
 
     public NodeResources resources() { return resources; }
 
-    public Optional<FlavorOverrides> flavorOverrides() {
-        return flavorOverrides;
-    }
+    public Optional<FlavorOverrides> flavorOverrides() { return flavorOverrides; }
 
     public double getMinMainMemoryAvailableGb() { return resources.memoryGb(); }
 
diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/HostName.java b/config-provisioning/src/main/java/com/yahoo/config/provision/HostName.java
index 510122c2342..25c42884295 100644
--- a/config-provisioning/src/main/java/com/yahoo/config/provision/HostName.java
+++ b/config-provisioning/src/main/java/com/yahoo/config/provision/HostName.java
@@ -4,7 +4,7 @@ package com.yahoo.config.provision;
 import java.util.Objects;
 
 /**
- * Represents a host name
+ * A host name
  *
  * @author mortent
  */
@@ -18,12 +18,7 @@ public class HostName implements Comparable<HostName> {
 
     public String value() { return name; }
 
-    /**
-     * Create a {@link HostName} with a given name.
-     *
-     * @param name Name
-     * @return instance of {@link HostName}.
-     */
+    /** Create a {@link HostName} with a given name */
     public static HostName from(String name) {
         return new HostName(name);
     }
diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/HostSpec.java b/config-provisioning/src/main/java/com/yahoo/config/provision/HostSpec.java
index 63725d9a535..2a5d27a0fe7 100644
--- a/config-provisioning/src/main/java/com/yahoo/config/provision/HostSpec.java
+++ b/config-provisioning/src/main/java/com/yahoo/config/provision/HostSpec.java
@@ -1,6 +1,8 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.config.provision;
 
+import com.yahoo.component.Version;
+
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Objects;
@@ -25,7 +27,9 @@ public class HostSpec implements Comparable<HostSpec> {
 
     private final Optional<Flavor> flavor;
 
-    private final Optional<com.yahoo.component.Version> version;
+    private final Optional<Version> version;
+
+    private final Optional<DockerImage> dockerImageRepo;
 
     private final Optional<NetworkPorts> networkPorts;
 
@@ -35,7 +39,7 @@ public class HostSpec implements Comparable<HostSpec> {
         this(hostname, new ArrayList<>(), Optional.empty(), membership);
     }
 
-    public HostSpec(String hostname, ClusterMembership membership, Flavor flavor, Optional<com.yahoo.component.Version> version) {
+    public HostSpec(String hostname, ClusterMembership membership, Flavor flavor, Optional<Version> version) {
         this(hostname, new ArrayList<>(), Optional.of(flavor), Optional.of(membership), version);
     }
 
@@ -56,27 +60,35 @@ public class HostSpec implements Comparable<HostSpec> {
     }
 
     public HostSpec(String hostname, List<String> aliases, Optional<Flavor> flavor,
-                    Optional<ClusterMembership> membership, Optional<com.yahoo.component.Version> version) {
+                    Optional<ClusterMembership> membership, Optional<Version> version) {
         this(hostname, aliases, flavor, membership, version, Optional.empty());
     }
 
     public HostSpec(String hostname, List<String> aliases, Optional<Flavor> flavor,
-                    Optional<ClusterMembership> membership, Optional<com.yahoo.component.Version> version,
+                    Optional<ClusterMembership> membership, Optional<Version> version,
                     Optional<NetworkPorts> networkPorts) {
         this(hostname, aliases, flavor, membership, version, networkPorts, Optional.empty());
     }
 
     public HostSpec(String hostname, List<String> aliases, Optional<Flavor> flavor,
-                    Optional<ClusterMembership> membership, Optional<com.yahoo.component.Version> version,
+                    Optional<ClusterMembership> membership, Optional<Version> version,
                     Optional<NetworkPorts> networkPorts, Optional<NodeResources> requestedResources) {
+        this(hostname, aliases, flavor, membership, version, networkPorts, requestedResources, Optional.empty());
+    }
+
+    public HostSpec(String hostname, List<String> aliases, Optional<Flavor> flavor,
+                    Optional<ClusterMembership> membership, Optional<Version> version,
+                    Optional<NetworkPorts> networkPorts, Optional<NodeResources> requestedResources,
+                    Optional<DockerImage> dockerImageRepo) {
         if (hostname == null || hostname.isEmpty()) throw new IllegalArgumentException("Hostname must be specified");
         this.hostname = hostname;
         this.aliases = List.copyOf(aliases);
         this.flavor = flavor;
         this.membership = membership;
-        this.version = Objects.requireNonNull(version, "Version cannot be null but can be empty");;
-        this.networkPorts = Objects.requireNonNull(networkPorts, "Network ports cannot be null but can be empty");;
+        this.version = Objects.requireNonNull(version, "Version cannot be null but can be empty");
+        this.networkPorts = Objects.requireNonNull(networkPorts, "Network ports cannot be null but can be empty");
         this.requestedResources = Objects.requireNonNull(requestedResources, "RequestedResources cannot be null");
+        this.dockerImageRepo = Objects.requireNonNull(dockerImageRepo, "Docker image repo cannot be null but can be empty");
     }
 
     /** Returns the name identifying this host */
@@ -99,8 +111,10 @@ public class HostSpec implements Comparable<HostSpec> {
     /** Returns the requested resources leading to this host being provisioned, or empty if not known */
     public Optional<NodeResources> requestedResources() { return requestedResources; }
 
+    public Optional<DockerImage> dockerImageRepo() { return dockerImageRepo; }
+
     public HostSpec withPorts(Optional<NetworkPorts> ports) {
-        return new HostSpec(hostname, aliases, flavor, membership, version, ports, requestedResources);
+        return new HostSpec(hostname, aliases, flavor, membership, version, ports, requestedResources, dockerImageRepo);
     }
 
     @Override
diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/InfraDeployer.java b/config-provisioning/src/main/java/com/yahoo/config/provision/InfraDeployer.java
index 6fbabfd0c95..363732ee8a7 100644
--- a/config-provisioning/src/main/java/com/yahoo/config/provision/InfraDeployer.java
+++ b/config-provisioning/src/main/java/com/yahoo/config/provision/InfraDeployer.java
@@ -1,7 +1,6 @@
 // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.config.provision;
 
-import java.util.Map;
 import java.util.Optional;
 
 /**
@@ -17,6 +16,6 @@ public interface InfraDeployer {
      */
     Optional<Deployment> getDeployment(ApplicationId application);
 
-    /** Returns deployments by application id for the supported infrastructure applications in this zone */
-    Map<ApplicationId, Deployment> getSupportedInfraDeployments();
+    /** Deploys all supported infrastructure applications in this zone. */
+    void activateAllSupportedInfraApplications(boolean propagateException);
 }
diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeFlavors.java b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeFlavors.java
index a9f031cae70..eb462c86f4f 100644
--- a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeFlavors.java
+++ b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeFlavors.java
@@ -14,7 +14,7 @@ import java.util.Optional;
 import java.util.stream.Collectors;
 
 /**
- * All the flavors *configured* in this zone (i.e this should be called HostFlavors).
+ * All the flavors configured in this zone (i.e this should be called HostFlavors).
  *
  * @author bratseth
  */
diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java
index ce3dd579d2f..05b604b263f 100644
--- a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java
+++ b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java
@@ -10,6 +10,8 @@ import java.util.Objects;
  */
 public class NodeResources {
 
+    public static final NodeResources unspecified = new NodeResources(0, 0, 0, 0);
+
     public enum DiskSpeed {
 
         fast, // Has/requires SSD disk or similar speed
@@ -112,31 +114,32 @@ public class NodeResources {
     public StorageType storageType() { return storageType; }
 
     public NodeResources withVcpu(double vcpu) {
+        if (vcpu == this.vcpu) return this;
         return new NodeResources(vcpu, memoryGb, diskGb, bandwidthGbps, diskSpeed, storageType);
     }
 
     public NodeResources withMemoryGb(double memoryGb) {
+        if (memoryGb == this.memoryGb) return this;
         return new NodeResources(vcpu, memoryGb, diskGb, bandwidthGbps, diskSpeed, storageType);
     }
 
     public NodeResources withDiskGb(double diskGb) {
+        if (diskGb == this.diskGb) return this;
         return new NodeResources(vcpu, memoryGb, diskGb, bandwidthGbps, diskSpeed, storageType);
     }
 
     public NodeResources withBandwidthGbps(double bandwidthGbps) {
+        if (bandwidthGbps == this.bandwidthGbps) return this;
         return new NodeResources(vcpu, memoryGb, diskGb, bandwidthGbps, diskSpeed, storageType);
     }
 
-    // TODO: Remove after November 2019
-    public NodeResources withDiskSpeed(DiskSpeed speed) {
-        return new NodeResources(vcpu, memoryGb, diskGb, bandwidthGbps, speed, storageType);
-    }
-
-    public NodeResources with(DiskSpeed speed) {
-        return new NodeResources(vcpu, memoryGb, diskGb, bandwidthGbps, speed, storageType);
+    public NodeResources with(DiskSpeed diskSpeed) {
+        if (diskSpeed == this.diskSpeed) return this;
+        return new NodeResources(vcpu, memoryGb, diskGb, bandwidthGbps, diskSpeed, storageType);
     }
 
     public NodeResources with(StorageType storageType) {
+        if (storageType == this.storageType) return this;
         return new NodeResources(vcpu, memoryGb, diskGb, bandwidthGbps, diskSpeed, storageType);
     }
 
@@ -145,6 +148,11 @@ public class NodeResources {
         return with(NodeResources.DiskSpeed.any).with(StorageType.any);
     }
 
+    /** Returns this with all numbers set to 0 */
+    public NodeResources justNonNumbers() {
+        return withVcpu(0).withMemoryGb(0).withDiskGb(0).withBandwidthGbps(0);
+    }
+
     public NodeResources subtract(NodeResources other) {
         if ( ! this.isInterchangeableWith(other))
             throw new IllegalArgumentException(this + " and " + other + " are not interchangeable");
@@ -196,10 +204,11 @@ public class NodeResources {
 
     @Override
     public String toString() {
-        return "[vcpu: " + vcpu + ", memory: " + memoryGb + " Gb, disk " + diskGb + " Gb" +
-               (bandwidthGbps > 0 ? ", bandwidth: " + bandwidthGbps + " Gbps" : "") +
-               ( ! diskSpeed.isDefault() ? ", disk speed: " + diskSpeed : "") +
-               ( ! storageType.isDefault() ? ", storage type: " + storageType : "") + "]";
+        return String.format("[vcpu: %1$.1f, memory: %2$.1f Gb, disk %3$.1f Gb" +
+                            (bandwidthGbps > 0 ? ", bandwidth: %4$.1f Gbps" : "") +
+                            ( ! diskSpeed.isDefault() ? ", disk speed: " + diskSpeed : "") +
+                            ( ! storageType.isDefault() ? ", storage type: " + storageType : "") + "]",
+                            vcpu, memoryGb, diskGb, bandwidthGbps);
     }
 
     /** Returns true if all the resources of this are the same or larger than the given resources */
diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/Provisioner.java b/config-provisioning/src/main/java/com/yahoo/config/provision/Provisioner.java
index 6be1d49ebd3..e308d631442 100644
--- a/config-provisioning/src/main/java/com/yahoo/config/provision/Provisioner.java
+++ b/config-provisioning/src/main/java/com/yahoo/config/provision/Provisioner.java
@@ -19,10 +19,12 @@ public interface Provisioner {
      * @param applicationId the application requesting hosts
      * @param cluster the specification of the cluster to allocate nodes for
      * @param capacity the capacity requested
-     * @param groups the number of node groups to divide the requested capacity into
      * @param logger a logger which receives messages which are returned to the requestor
      * @return the specification of the hosts allocated
      */
+    List<HostSpec> prepare(ApplicationId applicationId, ClusterSpec cluster, Capacity capacity, ProvisionLogger logger);
+
+    @Deprecated // TODO: Remove after April 2020
     List<HostSpec> prepare(ApplicationId applicationId, ClusterSpec cluster, Capacity capacity, int groups, ProvisionLogger logger);
 
     /**
@@ -40,7 +42,6 @@ public interface Provisioner {
      * @param transaction Transaction with operations to commit together with any operations done within the provisioner.
      * @param application the application to remove
      */
-    @SuppressWarnings("deprecation")
     void remove(NestedTransaction transaction, ApplicationId application);
 
     /**
diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/package-info.java b/config-provisioning/src/main/java/com/yahoo/config/provision/package-info.java
index 208640033fc..c85c00d8ebe 100644
--- a/config-provisioning/src/main/java/com/yahoo/config/provision/package-info.java
+++ b/config-provisioning/src/main/java/com/yahoo/config/provision/package-info.java
@@ -1,6 +1,5 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 @ExportPackage
-@PublicApi
 package com.yahoo.config.provision;
 
 import com.yahoo.api.annotations.PublicApi;
diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializer.java b/config-provisioning/src/main/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializer.java
index 413e277655a..56c5544f6d7 100644
--- a/config-provisioning/src/main/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializer.java
+++ b/config-provisioning/src/main/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializer.java
@@ -3,16 +3,16 @@ package com.yahoo.config.provision.serialization;
 
 import com.yahoo.config.provision.AllocatedHosts;
 import com.yahoo.config.provision.ClusterMembership;
+import com.yahoo.config.provision.DockerImage;
 import com.yahoo.config.provision.Flavor;
 import com.yahoo.config.provision.HostSpec;
-import com.yahoo.config.provision.NetworkPorts;
 import com.yahoo.config.provision.NodeFlavors;
 import com.yahoo.config.provision.NodeResources;
 import com.yahoo.slime.ArrayTraverser;
 import com.yahoo.slime.Cursor;
 import com.yahoo.slime.Inspector;
 import com.yahoo.slime.Slime;
-import com.yahoo.vespa.config.SlimeUtils;
+import com.yahoo.slime.SlimeUtils;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -60,6 +60,9 @@ public class AllocatedHostsSerializer {
     /** Wanted version */
     private static final String hostSpecVespaVersionKey = "vespaVersion";
 
+    /** Wanted docker image repo */
+    private static final String hostSpecDockerImageRepoKey = "dockerImageRepo";
+
     /** Current version */
     private static final String hostSpecCurrentVespaVersionKey = "currentVespaVersion";
     private static final String hostSpecNetworkPortsKey = "ports";
@@ -83,6 +86,9 @@ public class AllocatedHostsSerializer {
         host.membership().ifPresent(membership -> {
             object.setString(hostSpecMembershipKey, membership.stringValue());
             object.setString(hostSpecVespaVersionKey, membership.cluster().vespaVersion().toFullString());
+            membership.cluster().dockerImageRepo().ifPresent(repo -> {
+                object.setString(hostSpecDockerImageRepoKey, repo);
+            });
         });
         host.flavor().ifPresent(flavor -> toSlime(flavor, object));
         host.requestedResources().ifPresent(resources -> toSlime(resources, object.setObject(requestedResourcesKey)));
@@ -120,7 +126,9 @@ public class AllocatedHostsSerializer {
     public static AllocatedHosts fromSlime(Inspector inspector, Optional<NodeFlavors> nodeFlavors) {
         Inspector array = inspector.field(mappingKey);
         Set<HostSpec> hosts = new LinkedHashSet<>();
-        array.traverse((ArrayTraverser)(i, host) -> hosts.add(hostFromSlime(host.field(hostSpecKey), nodeFlavors)));
+        array.traverse((ArrayTraverser)(i, host) -> {
+            hosts.add(hostFromSlime(host.field(hostSpecKey), nodeFlavors));
+        });
         return AllocatedHosts.withHosts(hosts);
     }
 
@@ -131,7 +139,8 @@ public class AllocatedHostsSerializer {
                             object.field(hostSpecMembershipKey).valid() ? Optional.of(membershipFromSlime(object)) : Optional.empty(),
                             optionalString(object.field(hostSpecCurrentVespaVersionKey)).map(com.yahoo.component.Version::new),
                             NetworkPortsSerializer.fromSlime(object.field(hostSpecNetworkPortsKey)),
-                            nodeResourcesFromSlime(object.field(requestedResourcesKey)));
+                            nodeResourcesFromSlime(object.field(requestedResourcesKey)),
+                            optionalDockerImage(object.field(hostSpecDockerImageRepoKey)));
     }
 
     private static List<String> aliasesFromSlime(Inspector object) {
@@ -197,11 +206,20 @@ public class AllocatedHostsSerializer {
 
     private static ClusterMembership membershipFromSlime(Inspector object) {
         return ClusterMembership.from(object.field(hostSpecMembershipKey).asString(),
-                                      com.yahoo.component.Version.fromString(object.field(hostSpecVespaVersionKey).asString()));
+                                      com.yahoo.component.Version.fromString(object.field(hostSpecVespaVersionKey).asString()),
+                                      object.field(hostSpecDockerImageRepoKey).valid()
+                                              ? Optional.of(object.field(hostSpecDockerImageRepoKey).asString())
+                                              : Optional.empty());
     }
 
     private static Optional<String> optionalString(Inspector inspector) {
         if ( ! inspector.valid()) return Optional.empty();
         return Optional.of(inspector.asString());
     }
+
+    private static Optional<DockerImage> optionalDockerImage(Inspector inspector) {
+        if ( ! inspector.valid()) return Optional.empty();
+        return Optional.of(DockerImage.fromString(inspector.asString()));
+    }
+
 }
diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/zone/RoutingMethod.java b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/RoutingMethod.java
new file mode 100644
index 00000000000..5c3d9d97bdd
--- /dev/null
+++ b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/RoutingMethod.java
@@ -0,0 +1,30 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.config.provision.zone;
+
+/**
+ * The routing methods supported by a zone.
+ *
+ * @author mpolden
+ */
+public enum RoutingMethod {
+
+    /** Routing happens through shared routing layer */
+    shared,
+
+    /** Routing happens through a dedicated layer 4 load balancer */
+    exclusive,
+
+    /** Routing happens through a shared layer 4 load balancer */
+    sharedLayer4;
+
+    /** Returns whether this method routes requests directly to the Vespa container cluster */
+    public boolean isDirect() {
+        return this == exclusive || this == sharedLayer4;
+    }
+
+    /** Returns whether this method routes requests through a shared routing layer */
+    public boolean isShared() {
+        return this == shared || this == sharedLayer4;
+    }
+
+}
diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneFilter.java b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneFilter.java
index 46efe7a440d..5ef23c80eac 100644
--- a/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneFilter.java
+++ b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneFilter.java
@@ -1,4 +1,4 @@
-// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.config.provision.zone;
 
 import com.yahoo.config.provision.CloudName;
@@ -22,6 +22,9 @@ public interface ZoneFilter {
     /** Zones which support direct routing through exclusive load balancers. */
     ZoneList directlyRouted();
 
+    /** Zones where traffic is routed using given method */
+    ZoneList routingMethod(RoutingMethod method);
+
     /** Zones where config servers are up and running. */
     ZoneList reachable();
 
diff --git a/config-provisioning/src/test/java/com/yahoo/config/provision/CapacityTest.java b/config-provisioning/src/test/java/com/yahoo/config/provision/CapacityTest.java
new file mode 100644
index 00000000000..326ed7317f6
--- /dev/null
+++ b/config-provisioning/src/test/java/com/yahoo/config/provision/CapacityTest.java
@@ -0,0 +1,48 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.config.provision;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+/**
+ * @author bratseth
+ */
+public class CapacityTest {
+
+    @Test
+    public void testCapacityValidation() {
+        // Equal min and max is allowed
+        Capacity.from(new ClusterResources(4, 2, new NodeResources(1,2,3,4)),
+                      new ClusterResources(4, 2, new NodeResources(1,2,3,4)),
+                      false, true);
+        assertValidationFailure(new ClusterResources(4, 2, new NodeResources(1,2,3,4)),
+                                new ClusterResources(2, 2, new NodeResources(1,2,3,4)));
+        assertValidationFailure(new ClusterResources(4, 4, new NodeResources(1,2,3,4)),
+                                new ClusterResources(4, 2, new NodeResources(1,2,3,4)));
+        assertValidationFailure(new ClusterResources(4, 2, new NodeResources(2,2,3,4)),
+                                new ClusterResources(4, 2, new NodeResources(1,2,3,4)));
+        assertValidationFailure(new ClusterResources(4, 2, new NodeResources(1,3,3,4)),
+                                new ClusterResources(4, 2, new NodeResources(1,2,3,4)));
+        assertValidationFailure(new ClusterResources(4, 2, new NodeResources(1,2,4,4)),
+                                new ClusterResources(4, 2, new NodeResources(1,2,3,4)));
+        assertValidationFailure(new ClusterResources(4, 2, new NodeResources(1,2,3,5)),
+                                new ClusterResources(4, 2, new NodeResources(1,2,3,4)));
+        // It's enough than one dimension is smaller also when the others are larger
+        assertValidationFailure(new ClusterResources(4, 2, new NodeResources(1,2,3,4)),
+                                new ClusterResources(8, 4, new NodeResources(2,1,6,8)));
+    }
+
+    private void assertValidationFailure(ClusterResources min, ClusterResources max) {
+        try {
+            Capacity.from(min, max, false, true);
+            fail("Expected exception with min " + min + " and max " + max);
+        }
+        catch (IllegalArgumentException e) {
+            assertEquals("The max capacity must be larger than the min capacity, but got min " + min + " and max " + max,
+                         e.getMessage());
+        }
+    }
+
+}
diff --git a/config-provisioning/src/test/java/com/yahoo/config/provision/ClusterMembershipTest.java b/config-provisioning/src/test/java/com/yahoo/config/provision/ClusterMembershipTest.java
index 3a36afcfdce..c38d67ddc81 100644
--- a/config-provisioning/src/test/java/com/yahoo/config/provision/ClusterMembershipTest.java
+++ b/config-provisioning/src/test/java/com/yahoo/config/provision/ClusterMembershipTest.java
@@ -1,10 +1,11 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.config.provision;
 
-import com.yahoo.component.Version;
 import com.yahoo.component.Vtag;
 import org.junit.Test;
 
+import java.util.Optional;
+
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
@@ -16,59 +17,80 @@ public class ClusterMembershipTest {
 
     @Test
     public void testContainerServiceInstance() {
-        ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("id1"), Version.fromString("6.42"), false);
+        ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("id1")).vespaVersion("6.42").build();
         assertContainerService(ClusterMembership.from(cluster, 3));
     }
 
     @Test
-    public void testContainerInstanceWithOptionalParts() {
+    public void testSerializationWithOptionalParts() {
         {
-            ClusterMembership instance = ClusterMembership.from("container/id1/4/37/exclusive/retired", Vtag.currentVersion);
+            ClusterMembership instance = ClusterMembership.from("container/id1/4/37/exclusive/retired", Vtag.currentVersion, Optional.empty());
+            ClusterMembership serialized = ClusterMembership.from(instance.stringValue(), Vtag.currentVersion, Optional.empty());
+            assertEquals(instance, serialized);
             assertTrue(instance.retired());
             assertTrue(instance.cluster().isExclusive());
+            assertFalse(instance.cluster().combinedId().isPresent());
+            assertTrue(instance.cluster().dockerImageRepo().isEmpty());
+        }
+        {
+            ClusterMembership instance = ClusterMembership.from("container/id1/4/37/exclusive", Vtag.currentVersion, Optional.empty());
+            ClusterMembership serialized = ClusterMembership.from(instance.stringValue(), Vtag.currentVersion, Optional.empty());
+            assertEquals(instance, serialized);
+            assertFalse(instance.retired());
+            assertTrue(instance.cluster().isExclusive());
+            assertFalse(instance.cluster().combinedId().isPresent());
+            assertTrue(instance.cluster().dockerImageRepo().isEmpty());
         }
-
         {
-            ClusterMembership instance = ClusterMembership.from("container/id1/4/37/exclusive", Vtag.currentVersion);
+            Optional<String> dockerImageRepo = Optional.of("docker.foo.com:4443/vespa/bar");
+            ClusterMembership instance = ClusterMembership.from("combined/id1/4/37/exclusive/containerId1", Vtag.currentVersion, dockerImageRepo);
+            ClusterMembership serialized = ClusterMembership.from(instance.stringValue(), Vtag.currentVersion, dockerImageRepo);
+            assertEquals(instance, serialized);
             assertFalse(instance.retired());
             assertTrue(instance.cluster().isExclusive());
+            assertEquals(ClusterSpec.Id.from("containerId1"), instance.cluster().combinedId().get());
+            assertEquals(dockerImageRepo.get(), instance.cluster().dockerImageRepo().get());
         }
     }
 
     @Test
     public void testServiceInstance() {
-        ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("id1"), Version.fromString("6.42"), false);
+        ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("id1")).vespaVersion("6.42").build();
         assertContentService(ClusterMembership.from(cluster, 37));
     }
 
     @Test
     public void testServiceInstanceWithGroup() {
-        ClusterSpec cluster = ClusterSpec.from(ClusterSpec.Type.content, ClusterSpec.Id.from("id1"),
-                                               ClusterSpec.Group.from(4), Version.fromString("6.42"), false);
+        ClusterSpec cluster = ClusterSpec.specification(ClusterSpec.Type.content, ClusterSpec.Id.from("id1"))
+                .group(ClusterSpec.Group.from(4))
+                .vespaVersion("6.42")
+                .build();
         assertContentServiceWithGroup(ClusterMembership.from(cluster, 37));
     }
 
     @Test
     public void testServiceInstanceWithGroupFromString() {
-        assertContentServiceWithGroup(ClusterMembership.from("content/id1/4/37", Vtag.currentVersion));
+        assertContentServiceWithGroup(ClusterMembership.from("content/id1/4/37", Vtag.currentVersion, Optional.empty()));
     }
 
     @Test
     public void testServiceInstanceWithRetire() {
-        ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("id1"), Version.fromString("6.42"), false);
+        ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("id1")).vespaVersion("6.42").build();
         assertContentServiceWithRetire(ClusterMembership.retiredFrom(cluster, 37));
     }
 
     @Test
     public void testServiceInstanceWithGroupAndRetire() {
-        ClusterSpec cluster = ClusterSpec.from(ClusterSpec.Type.content, ClusterSpec.Id.from("id1"),
-                                               ClusterSpec.Group.from(4), Version.fromString("6.42"), false);
+        ClusterSpec cluster = ClusterSpec.specification(ClusterSpec.Type.content, ClusterSpec.Id.from("id1"))
+                .group(ClusterSpec.Group.from(4))
+                .vespaVersion("6.42")
+                .build();
         assertContentServiceWithGroupAndRetire(ClusterMembership.retiredFrom(cluster, 37));
     }
 
     @Test
     public void testServiceInstanceWithGroupAndRetireFromString() {
-        assertContentServiceWithGroupAndRetire(ClusterMembership.from("content/id1/4/37/retired", Vtag.currentVersion));
+        assertContentServiceWithGroupAndRetire(ClusterMembership.from("content/id1/4/37/retired", Vtag.currentVersion, Optional.empty()));
     }
 
     private void assertContainerService(ClusterMembership instance) {
diff --git a/config-provisioning/src/test/java/com/yahoo/config/provision/ClusterSpecTest.java b/config-provisioning/src/test/java/com/yahoo/config/provision/ClusterSpecTest.java
index dc198e5bb0d..40ed7500269 100644
--- a/config-provisioning/src/test/java/com/yahoo/config/provision/ClusterSpecTest.java
+++ b/config-provisioning/src/test/java/com/yahoo/config/provision/ClusterSpecTest.java
@@ -7,6 +7,7 @@ import org.junit.Test;
 
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 
 import static org.junit.Assert.assertEquals;
 
@@ -40,7 +41,7 @@ public class ClusterSpecTest {
                 List.of(spec(ClusterSpec.Type.admin, "id1"), spec(ClusterSpec.Type.container, "id1")), false,
                 List.of(spec(ClusterSpec.Type.admin, "id1"), spec(ClusterSpec.Type.content, "id1")), false,
                 List.of(spec(ClusterSpec.Type.combined, "id1"), spec(ClusterSpec.Type.container, "id1")), false,
-                List.of(spec(ClusterSpec.Type.combined, "id1"), spec(ClusterSpec.Type.content, "id1")), false,
+                List.of(spec(ClusterSpec.Type.combined, "id1"), spec(ClusterSpec.Type.content, "id1")), true,
                 List.of(spec(ClusterSpec.Type.content, "id1"), spec(ClusterSpec.Type.content, "id1")), true
         );
         tests.forEach((specs, satisfies) -> {
@@ -52,7 +53,10 @@ public class ClusterSpecTest {
     }
 
     private static ClusterSpec spec(ClusterSpec.Type type, String id) {
-        return ClusterSpec.from(type, ClusterSpec.Id.from(id), ClusterSpec.Group.from(1), Version.emptyVersion, false);
+        return ClusterSpec.specification(type, ClusterSpec.Id.from(id))
+                .group(ClusterSpec.Group.from(1))
+                .vespaVersion(Version.emptyVersion)
+                .build();
     }
 
 }
diff --git a/config-provisioning/src/test/java/com/yahoo/config/provision/HostFilterTest.java b/config-provisioning/src/test/java/com/yahoo/config/provision/HostFilterTest.java
index e1d6f061446..66ca1170487 100644
--- a/config-provisioning/src/test/java/com/yahoo/config/provision/HostFilterTest.java
+++ b/config-provisioning/src/test/java/com/yahoo/config/provision/HostFilterTest.java
@@ -71,7 +71,7 @@ public class HostFilterTest {
     }
 
     private Optional<ClusterMembership> membership(String membershipString) {
-        return Optional.of(ClusterMembership.from(membershipString, Vtag.currentVersion));
+        return Optional.of(ClusterMembership.from(membershipString, Vtag.currentVersion, Optional.empty()));
     }
 
 }
diff --git a/config-provisioning/src/test/java/com/yahoo/config/provision/NodeResourcesTest.java b/config-provisioning/src/test/java/com/yahoo/config/provision/NodeResourcesTest.java
new file mode 100644
index 00000000000..21e1afeed17
--- /dev/null
+++ b/config-provisioning/src/test/java/com/yahoo/config/provision/NodeResourcesTest.java
@@ -0,0 +1,21 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.config.provision;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author bratseth
+ */
+public class NodeResourcesTest {
+
+    @Test
+    public void testToString() {
+        assertEquals("[vcpu: 1.0, memory: 10.0 Gb, disk 100.0 Gb]",
+                     new NodeResources(1., 10., 100., 0).toString());
+        assertEquals("[vcpu: 0.3, memory: 3.3 Gb, disk 33.3 Gb, bandwidth: 0.3 Gbps]",
+                     new NodeResources(1/3., 10/3., 100/3., 0.3).toString());
+    }
+
+}
diff --git a/config-provisioning/src/test/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializerTest.java b/config-provisioning/src/test/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializerTest.java
index 193caf3edba..f2163608050 100644
--- a/config-provisioning/src/test/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializerTest.java
+++ b/config-provisioning/src/test/java/com/yahoo/config/provision/serialization/AllocatedHostsSerializerTest.java
@@ -4,6 +4,7 @@ package com.yahoo.config.provision.serialization;
 import com.yahoo.component.Version;
 import com.yahoo.config.provision.AllocatedHosts;
 import com.yahoo.config.provision.ClusterMembership;
+import com.yahoo.config.provision.DockerImage;
 import com.yahoo.config.provision.Flavor;
 import com.yahoo.config.provision.HostSpec;
 import com.yahoo.config.provision.NetworkPorts;
@@ -37,7 +38,12 @@ public class AllocatedHostsSerializerTest {
         hosts.add(new HostSpec("with-aliases",
                                List.of("alias1", "alias2")));
         hosts.add(new HostSpec("allocated",
-                               Optional.of(ClusterMembership.from("container/test/0/0", Version.fromString("6.73.1")))));
+                               List.of(),
+                               Optional.empty(),
+                               Optional.of(ClusterMembership.from("container/test/0/0", Version.fromString("6.73.1"),
+                                                                  Optional.of("docker.foo.com:4443/vespa/bar"))),
+                               Optional.empty(), Optional.empty(), Optional.empty(),
+                               Optional.of(DockerImage.fromString("docker.foo.com:4443/vespa/bar"))));
         hosts.add(new HostSpec("flavor-from-resources-1",
                                Collections.emptyList(), new Flavor(new NodeResources(0.5, 3.1, 4, 1))));
         hosts.add(new HostSpec("flavor-from-resources-2",
@@ -72,6 +78,7 @@ public class AllocatedHostsSerializerTest {
             assertEquals(expectedHost.networkPorts(), deserializedHost.networkPorts());
             assertEquals(expectedHost.aliases(), deserializedHost.aliases());
             assertEquals(expectedHost.requestedResources(), deserializedHost.requestedResources());
+            assertEquals(expectedHost.dockerImageRepo(), deserializedHost.dockerImageRepo());
         }
     }
 
diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServer.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServer.java
index 9d33824e0be..0a36f06a1b9 100644
--- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServer.java
+++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServer.java
@@ -13,22 +13,18 @@ import com.yahoo.jrt.Supervisor;
 import com.yahoo.jrt.Target;
 import com.yahoo.jrt.TargetWatcher;
 import com.yahoo.log.LogLevel;
-import com.yahoo.vespa.config.ErrorCode;
 import com.yahoo.vespa.config.JRTMethods;
 import com.yahoo.vespa.config.RawConfig;
-import com.yahoo.vespa.config.protocol.JRTConfigRequestFactory;
 import com.yahoo.vespa.config.protocol.JRTServerConfigRequest;
 import com.yahoo.vespa.config.protocol.JRTServerConfigRequestV3;
 
 import java.util.Arrays;
 import java.util.Iterator;
-import java.util.Set;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.TimeUnit;
 import java.util.logging.Logger;
 
-
 /**
  * An RPC server that handles config and file distribution requests.
  *
@@ -132,12 +128,8 @@ public class ConfigProxyRpcServer implements Runnable, TargetWatcher, RpcServer
     private void getConfigV3(Request req) {
         dispatchRpcRequest(req, () -> {
             JRTServerConfigRequest request = JRTServerConfigRequestV3.createFromRequest(req);
-            if (isProtocolVersionSupported(request)) {
-                req.target().addWatcher(this);
-                getConfigImpl(request);
-                return;
-            }
-            req.returnRequest();
+            req.target().addWatcher(this);
+            getConfigImpl(request);
         });
     }
 
@@ -162,9 +154,9 @@ public class ConfigProxyRpcServer implements Runnable, TargetWatcher, RpcServer
         dispatchRpcRequest(req, () -> {
             StringBuilder sb = new StringBuilder();
             sb.append("\nDelayed responses queue size: ");
-            sb.append(proxyServer.delayedResponses.size());
+            sb.append(proxyServer.delayedResponses().size());
             sb.append("\nContents: ");
-            for (DelayedResponse delayed : proxyServer.delayedResponses.responses()) {
+            for (DelayedResponse delayed : proxyServer.delayedResponses().responses()) {
                 sb.append(delayed.getRequest().toString()).append("\n");
             }
 
@@ -271,19 +263,6 @@ public class ConfigProxyRpcServer implements Runnable, TargetWatcher, RpcServer
         return String.format("%s/%08X", request.methodName(), request.hashCode());
     }
 
-    private boolean isProtocolVersionSupported(JRTServerConfigRequest request) {
-        Set<Long> supportedProtocolVersions = JRTConfigRequestFactory.supportedProtocolVersions();
-        if (supportedProtocolVersions.contains(request.getProtocolVersion())) {
-            return true;
-        } else {
-            String message = "Illegal protocol version " + request.getProtocolVersion() +
-                    " in request " + request.getShortDescription() + ", only protocol versions " + supportedProtocolVersions + " are supported";
-            log.log(LogLevel.ERROR, message);
-            request.addErrorResponse(ErrorCode.ILLEGAL_PROTOCOL_VERSION, message);
-        }
-        return false;
-    }
-
     /**
      * Handles all versions of "getConfig" requests.
      *
@@ -357,7 +336,7 @@ public class ConfigProxyRpcServer implements Runnable, TargetWatcher, RpcServer
     @Override
     public void notifyTargetInvalid(Target target) {
         log.log(LogLevel.DEBUG, () -> "Target invalid " + target);
-        for (Iterator<DelayedResponse> it = proxyServer.delayedResponses.responses().iterator(); it.hasNext(); ) {
+        for (Iterator<DelayedResponse> it = proxyServer.delayedResponses().responses().iterator(); it.hasNext(); ) {
             DelayedResponse delayed = it.next();
             JRTServerConfigRequest request = delayed.getRequest();
             if (request.getRequest().target().equals(target)) {
diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigSourceClient.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigSourceClient.java
index 2f8a1b463e3..9fb78e7e812 100644
--- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigSourceClient.java
+++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigSourceClient.java
@@ -26,4 +26,6 @@ interface ConfigSourceClient {
 
     void updateSubscribers(RawConfig config);
 
+    DelayedResponses delayedResponses();
+
 }
diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/MemoryCacheConfigClient.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/MemoryCacheConfigClient.java
index f9f5c475723..51446882025 100644
--- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/MemoryCacheConfigClient.java
+++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/MemoryCacheConfigClient.java
@@ -18,6 +18,7 @@ class MemoryCacheConfigClient implements ConfigSourceClient {
 
     private final static Logger log = Logger.getLogger(MemoryCacheConfigClient.class.getName());
     private final MemoryCache cache;
+    private final DelayedResponses delayedResponses = new DelayedResponses();
 
     MemoryCacheConfigClient(MemoryCache cache) {
         this.cache = cache;
@@ -61,4 +62,9 @@ class MemoryCacheConfigClient implements ConfigSourceClient {
     @Override
     public void updateSubscribers(RawConfig config) {}
 
+    @Override
+    public DelayedResponses delayedResponses() {
+        return delayedResponses;
+    }
+
 }
diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ProxyServer.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ProxyServer.java
index d77206aee81..545b962f6ff 100644
--- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ProxyServer.java
+++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ProxyServer.java
@@ -1,7 +1,6 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.config.proxy;
 
-import com.yahoo.concurrent.DaemonThreadFactory;
 import com.yahoo.config.subscription.ConfigSourceSet;
 import com.yahoo.jrt.Spec;
 import com.yahoo.jrt.Supervisor;
@@ -10,20 +9,15 @@ import com.yahoo.log.LogLevel;
 import com.yahoo.log.LogSetup;
 import com.yahoo.log.event.Event;
 import com.yahoo.vespa.config.RawConfig;
-import com.yahoo.vespa.config.TimingValues;
 import com.yahoo.vespa.config.protocol.JRTServerConfigRequest;
 import com.yahoo.vespa.config.proxy.filedistribution.FileDistributionAndUrlDownload;
 import com.yahoo.yolean.system.CatchSignals;
 
 import java.util.List;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.ScheduledFuture;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.logging.Logger;
 
 import static com.yahoo.vespa.config.proxy.Mode.ModeName.DEFAULT;
-import static java.util.concurrent.TimeUnit.SECONDS;
 
 /**
  * A proxy server that handles RPC config requests. The proxy can run in two modes:
@@ -40,59 +34,34 @@ public class ProxyServer implements Runnable {
 
     private final static Logger log = Logger.getLogger(ProxyServer.class.getName());
     private final AtomicBoolean signalCaught = new AtomicBoolean(false);
-
-    // Scheduled executor that periodically checks for requests that have timed out and response should be returned to clients
-    private final ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1, new DaemonThreadFactory());
     private final Supervisor supervisor = new Supervisor(new Transport(JRT_TRANSPORT_THREADS));
-    private ScheduledFuture<?> delayedResponseScheduler;
 
     private final ConfigProxyRpcServer rpcServer;
-    final DelayedResponses delayedResponses;
     private ConfigSourceSet configSource;
 
     private volatile ConfigSourceClient configClient;
 
-    private final TimingValues timingValues;
     private final MemoryCache memoryCache;
-    private static final double timingValuesRatio = 0.8;
-    private final static TimingValues defaultTimingValues;
     private final FileDistributionAndUrlDownload fileDistributionAndUrlDownload;
 
     private volatile Mode mode = new Mode(DEFAULT);
 
-    static {
-        // Proxy should time out before clients upon subscription.
-        TimingValues tv = new TimingValues();
-        tv.setUnconfiguredDelay((long)(tv.getUnconfiguredDelay()* timingValuesRatio)).
-                setConfiguredErrorDelay((long)(tv.getConfiguredErrorDelay()* timingValuesRatio)).
-                setSubscribeTimeout((long)(tv.getSubscribeTimeout()* timingValuesRatio)).
-                setConfiguredErrorTimeout(-1);  // Never cache errors
-        defaultTimingValues = tv;
-    }
-
-    ProxyServer(Spec spec, ConfigSourceSet source, TimingValues timingValues,
-                MemoryCache memoryCache, ConfigSourceClient configClient) {
-        this.delayedResponses = new DelayedResponses();
+    ProxyServer(Spec spec, ConfigSourceSet source, MemoryCache memoryCache, ConfigSourceClient configClient) {
         this.configSource = source;
         log.log(LogLevel.DEBUG, "Using config source '" + source);
-        this.timingValues = timingValues;
         this.memoryCache = memoryCache;
         this.rpcServer = createRpcServer(spec);
-        this.configClient = createClient(rpcServer, delayedResponses, source, timingValues, memoryCache, configClient);
+        this.configClient = (configClient == null) ? createRpcClient(rpcServer, source, memoryCache) : configClient;
         this.fileDistributionAndUrlDownload = new FileDistributionAndUrlDownload(supervisor, source);
     }
 
+    @Override
     public void run() {
         if (rpcServer != null) {
             Thread t = new Thread(rpcServer);
             t.setName("RpcServer");
             t.start();
         }
-        // Wait for 5 seconds initially, then run every second
-        delayedResponseScheduler = scheduler.scheduleAtFixedRate(new DelayedResponseHandler(delayedResponses,
-                                                                                            memoryCache,
-                                                                                            rpcServer),
-                                                                 5, 1, SECONDS);
     }
 
     RawConfig resolveConfig(JRTServerConfigRequest req) {
@@ -124,7 +93,7 @@ public class ProxyServer implements Runnable {
                 break;
             case DEFAULT:
                 flush();
-                configClient = createRpcClient();
+                configClient = createRpcClient(rpcServer, configSource, memoryCache);
                 this.mode = new Mode(modeName);
                 break;
             default:
@@ -133,20 +102,12 @@ public class ProxyServer implements Runnable {
         log.log(LogLevel.INFO, "Switched from '" + oldMode.name().toLowerCase() + "' mode to '" + getMode().name().toLowerCase() + "' mode");
     }
 
-    private ConfigSourceClient createClient(RpcServer rpcServer, DelayedResponses delayedResponses,
-                                            ConfigSourceSet source, TimingValues timingValues,
-                                            MemoryCache memoryCache, ConfigSourceClient client) {
-        return (client == null)
-                ? new RpcConfigSourceClient(rpcServer, source, memoryCache, timingValues, delayedResponses)
-                : client;
-    }
-
     private ConfigProxyRpcServer createRpcServer(Spec spec) {
         return  (spec == null) ? null : new ConfigProxyRpcServer(this, supervisor, spec); // TODO: Try to avoid first argument being 'this'
     }
 
-    private RpcConfigSourceClient createRpcClient() {
-        return new RpcConfigSourceClient(rpcServer, configSource, memoryCache, timingValues, delayedResponses);
+    private static RpcConfigSourceClient createRpcClient(RpcServer rpcServer, ConfigSourceSet source, MemoryCache memoryCache) {
+        return new RpcConfigSourceClient(rpcServer, source, memoryCache);
     }
 
     private void setupSignalHandler() {
@@ -181,8 +142,7 @@ public class ProxyServer implements Runnable {
         Event.started("configproxy");
 
         ConfigSourceSet configSources = new ConfigSourceSet(properties.configSources);
-        ProxyServer proxyServer = new ProxyServer(new Spec(null, port), configSources,
-                                                  defaultTimingValues(), new MemoryCache(), null);
+        ProxyServer proxyServer = new ProxyServer(new Spec(null, port), configSources, new MemoryCache(), null);
         // catch termination and interrupt signal
         proxyServer.setupSignalHandler();
         Thread proxyserverThread = new Thread(proxyServer);
@@ -204,14 +164,6 @@ public class ProxyServer implements Runnable {
         }
     }
 
-    static TimingValues defaultTimingValues() {
-        return defaultTimingValues;
-    }
-
-    TimingValues getTimingValues() {
-        return timingValues;
-    }
-
     // Cancels all config instances and flushes the cache. When this method returns,
     // the cache will not be updated again before someone calls getConfig().
     private synchronized void flush() {
@@ -222,7 +174,7 @@ public class ProxyServer implements Runnable {
     void stop() {
         Event.stopping("configproxy", "shutdown");
         if (rpcServer != null) rpcServer.shutdown();
-        if (delayedResponseScheduler != null) delayedResponseScheduler.cancel(true);
+        if (configClient != null) configClient.cancel();
         flush();
         fileDistributionAndUrlDownload.close();
     }
@@ -242,7 +194,11 @@ public class ProxyServer implements Runnable {
     void updateSourceConnections(List<String> sources) {
         configSource = new ConfigSourceSet(sources);
         flush();
-        configClient = createRpcClient();
+        configClient = createRpcClient(rpcServer, configSource, memoryCache);
+    }
+
+    DelayedResponses delayedResponses() {
+        return configClient.delayedResponses();
     }
 
 }
diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClient.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClient.java
index d809a3c97ed..2a33e8c6928 100644
--- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClient.java
+++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClient.java
@@ -11,18 +11,23 @@ import com.yahoo.jrt.Supervisor;
 import com.yahoo.jrt.Target;
 import com.yahoo.jrt.Transport;
 import com.yahoo.log.LogLevel;
-import com.yahoo.vespa.config.*;
+import com.yahoo.vespa.config.ConfigCacheKey;
+import com.yahoo.vespa.config.RawConfig;
+import com.yahoo.vespa.config.TimingValues;
 import com.yahoo.vespa.config.protocol.JRTServerConfigRequest;
 
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
-import java.util.Map;
 import java.util.concurrent.DelayQueue;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
 import java.util.logging.Logger;
 
+import static java.util.concurrent.TimeUnit.SECONDS;
+
 /**
  * An Rpc client to a config source
  *
@@ -31,6 +36,8 @@ import java.util.logging.Logger;
 class RpcConfigSourceClient implements ConfigSourceClient {
 
     private final static Logger log = Logger.getLogger(RpcConfigSourceClient.class.getName());
+    private static final double timingValuesRatio = 0.8;
+
     private final Supervisor supervisor = new Supervisor(new Transport());
 
     private final RpcServer rpcServer;
@@ -39,38 +46,37 @@ class RpcConfigSourceClient implements ConfigSourceClient {
     private final Object activeSubscribersLock = new Object();
     private final MemoryCache memoryCache;
     private final DelayedResponses delayedResponses;
-    private final TimingValues timingValues;
-
+    private final static TimingValues timingValues;
     private final ExecutorService exec;
-    private final Map<ConfigSourceSet, JRTConfigRequester> requesterPool;
-
+    private final JRTConfigRequester requester;
+    // Scheduled executor that periodically checks for requests that have timed out and response should be returned to clients
+    private final ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1, new DaemonThreadFactory());
+    private ScheduledFuture<?> delayedResponseScheduler;
+
+    static {
+        // Proxy should time out before clients upon subscription.
+        TimingValues tv = new TimingValues();
+        tv.setUnconfiguredDelay((long)(tv.getUnconfiguredDelay()* timingValuesRatio)).
+                setConfiguredErrorDelay((long)(tv.getConfiguredErrorDelay()* timingValuesRatio)).
+                setSubscribeTimeout((long)(tv.getSubscribeTimeout()* timingValuesRatio)).
+                setConfiguredErrorTimeout(-1);  // Never cache errors
+        timingValues = tv;
+    }
 
-    RpcConfigSourceClient(RpcServer rpcServer,
-                          ConfigSourceSet configSourceSet,
-                          MemoryCache memoryCache,
-                          TimingValues timingValues,
-                          DelayedResponses delayedResponses) {
+    RpcConfigSourceClient(RpcServer rpcServer, ConfigSourceSet configSourceSet, MemoryCache memoryCache) {
         this.rpcServer = rpcServer;
         this.configSourceSet = configSourceSet;
         this.memoryCache = memoryCache;
-        this.delayedResponses = delayedResponses;
-        this.timingValues = timingValues;
+        this.delayedResponses = new DelayedResponses();
         checkConfigSources();
         exec = Executors.newCachedThreadPool(new DaemonThreadFactory("subscriber-"));
-        requesterPool = createRequesterPool(configSourceSet, timingValues);
-    }
-
-    /**
-     * Creates a requester (connection) pool of one entry, to be used each time this {@link RpcConfigSourceClient} is used
-     * @param ccs a {@link ConfigSourceSet}
-     * @param timingValues a {@link TimingValues}
-     * @return requester map
-     */
-    private Map<ConfigSourceSet, JRTConfigRequester> createRequesterPool(ConfigSourceSet ccs, TimingValues timingValues) {
-        Map<ConfigSourceSet, JRTConfigRequester> ret = new HashMap<>();
-        if (ccs.getSources().isEmpty()) return ret; // unit test, just skip creating any requester
-        ret.put(ccs, JRTConfigRequester.get(new JRTConnectionPool(ccs), timingValues));
-        return ret;
+        requester = JRTConfigRequester.create(configSourceSet, timingValues);
+        // Wait for 5 seconds initially, then run every second
+        delayedResponseScheduler = scheduler.scheduleAtFixedRate(
+                new DelayedResponseHandler(delayedResponses, memoryCache, rpcServer),
+                5,
+                1,
+                SECONDS);
     }
 
     /**
@@ -152,8 +158,8 @@ class RpcConfigSourceClient implements ConfigSourceClient {
                 log.log(LogLevel.DEBUG, () -> "Already a subscriber running for: " + configCacheKey);
             } else {
                 log.log(LogLevel.DEBUG, () -> "Could not find good config in cache, creating subscriber for: " + configCacheKey);
-                UpstreamConfigSubscriber subscriber = new UpstreamConfigSubscriber(input, this, configSourceSet,
-                                                                                   timingValues, requesterPool, memoryCache);
+                UpstreamConfigSubscriber subscriber =
+                        new UpstreamConfigSubscriber(input, this, configSourceSet, timingValues, requester, memoryCache);
                 try {
                     subscriber.subscribe();
                     activeSubscribers.put(configCacheKey, subscriber);
@@ -169,6 +175,8 @@ class RpcConfigSourceClient implements ConfigSourceClient {
     @Override
     public void cancel() {
         shutdownSourceConnections();
+        delayedResponseScheduler.cancel(true);
+        scheduler.shutdown();
     }
 
     /**
@@ -183,25 +191,18 @@ class RpcConfigSourceClient implements ConfigSourceClient {
             activeSubscribers.clear();
         }
         exec.shutdown();
-        for (JRTConfigRequester requester : requesterPool.values()) {
-            requester.close();
-        }
+        requester.close();
     }
 
     @Override
     public String getActiveSourceConnection() {
-        if (requesterPool.get(configSourceSet) != null) {
-            return requesterPool.get(configSourceSet).getConnectionPool().getCurrent().getAddress();
-        } else {
-            return "";
-        }
+        return requester.getConnectionPool().getCurrent().getAddress();
     }
 
     @Override
     public List<String> getSourceConnections() {
         ArrayList<String> ret = new ArrayList<>();
-        final JRTConfigRequester jrtConfigRequester = requesterPool.get(configSourceSet);
-        if (jrtConfigRequester != null) {
+        if (configSourceSet != null) {
             ret.addAll(configSourceSet.getSources());
         }
         return ret;
@@ -244,4 +245,9 @@ class RpcConfigSourceClient implements ConfigSourceClient {
         log.log(LogLevel.DEBUG, () -> "Finished updating config for " + config.getKey() + "," + config.getGeneration());
     }
 
+    @Override
+    public DelayedResponses delayedResponses() {
+        return delayedResponses;
+    }
+
 }
diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/UpstreamConfigSubscriber.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/UpstreamConfigSubscriber.java
index f8df16cb3d2..d8a8c5ce941 100644
--- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/UpstreamConfigSubscriber.java
+++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/UpstreamConfigSubscriber.java
@@ -1,16 +1,15 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.config.proxy;
 
-import com.yahoo.config.subscription.ConfigSource;
 import com.yahoo.config.subscription.ConfigSourceSet;
 import com.yahoo.config.subscription.impl.GenericConfigHandle;
 import com.yahoo.config.subscription.impl.GenericConfigSubscriber;
 import com.yahoo.config.subscription.impl.JRTConfigRequester;
 import com.yahoo.log.LogLevel;
 import com.yahoo.vespa.config.ConfigKey;
-import com.yahoo.yolean.Exceptions;
 import com.yahoo.vespa.config.RawConfig;
 import com.yahoo.vespa.config.TimingValues;
+import com.yahoo.yolean.Exceptions;
 
 import java.util.Map;
 import java.util.logging.Logger;
@@ -24,26 +23,26 @@ public class UpstreamConfigSubscriber implements Subscriber {
 
     private final RawConfig config;
     private final ConfigSourceClient configSourceClient;
-    private final ConfigSource configSourceSet;
+    private final ConfigSourceSet configSourceSet;
     private final TimingValues timingValues;
-    private final Map<ConfigSourceSet, JRTConfigRequester> requesterPool;
+    private final JRTConfigRequester requester;
     private final MemoryCache memoryCache;
     private GenericConfigSubscriber subscriber;
     private GenericConfigHandle handle;
 
-    UpstreamConfigSubscriber(RawConfig config, ConfigSourceClient configSourceClient, ConfigSource configSourceSet,
-                             TimingValues timingValues, Map<ConfigSourceSet, JRTConfigRequester> requesterPool,
+    UpstreamConfigSubscriber(RawConfig config, ConfigSourceClient configSourceClient, ConfigSourceSet configSourceSet,
+                             TimingValues timingValues, JRTConfigRequester requester,
                              MemoryCache memoryCache) {
         this.config = config;
         this.configSourceClient = configSourceClient;
         this.configSourceSet = configSourceSet;
         this.timingValues = timingValues;
-        this.requesterPool = requesterPool;
+        this.requester = requester;
         this.memoryCache = memoryCache;
     }
 
     void subscribe() {
-        subscriber = new GenericConfigSubscriber(requesterPool);
+        subscriber = new GenericConfigSubscriber(Map.of(configSourceSet, requester));
         ConfigKey<?> key = config.getKey();
         handle = subscriber.subscribe(new ConfigKey<>(key.getName(), key.getConfigId(), key.getNamespace()),
                                       config.getDefContent(), configSourceSet, timingValues);
diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDistributionRpcServer.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDistributionRpcServer.java
index c6f33a29410..7db9761d86a 100644
--- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDistributionRpcServer.java
+++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDistributionRpcServer.java
@@ -35,7 +35,7 @@ class FileDistributionRpcServer {
 
     private final Supervisor supervisor;
     private final FileDownloader downloader;
-    private final ExecutorService rpcDownloadExecutor = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors(),
+    private final ExecutorService rpcDownloadExecutor = Executors.newFixedThreadPool(Math.max(8, Runtime.getRuntime().availableProcessors()),
                                                                                      new DaemonThreadFactory("Rpc executor"));
 
     FileDistributionRpcServer(Supervisor supervisor, FileDownloader downloader) {
diff --git a/config-proxy/src/main/sh/vespa-config-ctl.sh b/config-proxy/src/main/sh/vespa-config-ctl.sh
index a17f0abed92..91e4cd39bd2 100755
--- a/config-proxy/src/main/sh/vespa-config-ctl.sh
+++ b/config-proxy/src/main/sh/vespa-config-ctl.sh
@@ -113,15 +113,19 @@ case $1 in
         if [ "$userargs" == "" ]; then
             userargs=$services__jvmargs_configproxy
         fi
-        jvmopts="-Xms32M -Xmx256M -XX:ThreadStackSize=256 -XX:MaxJavaStackTraceDepth=1000000"
+        jvmopts="-Xms32M -Xmx256M -XX:CompressedClassSpaceSize=32m -XX:MaxDirectMemorySize=32m -XX:ThreadStackSize=256 -XX:MaxJavaStackTraceDepth=1000"
 
         VESPA_SERVICE_NAME=configproxy
         export VESPA_SERVICE_NAME
+        start_seconds=$SECONDS
         echo "Starting config proxy using $configsources as config source(s)"
         vespa-runserver -r 10 -s configproxy -p $P_CONFIG_PROXY -- \
             java ${jvmopts} \
                  -XX:+ExitOnOutOfMemoryError $(getJavaOptionsIPV46) \
-                 -Dproxyconfigsources="${configsources}" ${userargs} \
+                 -Dproxyconfigsources="${configsources}" \
+		 -Djava.io.tmpdir=${VESPA_HOME}/tmp \
+		 ${userargs} \
+                 -XX:ActiveProcessorCount=2 \
                  -cp $cp com.yahoo.vespa.config.proxy.ProxyServer 19090
 
         echo "Waiting for config proxy to start"
@@ -130,15 +134,35 @@ case $1 in
             sleep 0.1
             if [ -f $P_CONFIG_PROXY ] && kill -0 `cat $P_CONFIG_PROXY` && vespa-ping-configproxy -s $hname 2>/dev/null
             then
-                echo "config proxy started (runserver pid `cat $P_CONFIG_PROXY`)"
+                startup_seconds=$(( SECONDS - start_seconds ))
+                echo "config proxy started after ${startup_seconds}s (runserver pid `cat $P_CONFIG_PROXY`)"
                 fail=false
                 break
             fi
         done
         if $fail ; then
-            echo "Failed to start config proxy!" 1>&2
-            echo "look for reason in vespa.log, last part follows..."
-            tail -n 15 $LOGFILE | vespa-logfmt -
+            startup_seconds=$(( SECONDS - start_seconds ))
+            echo "Config proxy failed to start in ${startup_seconds}S!" 1>&2
+
+            if ! [ -f $P_CONFIG_PROXY ]
+            then
+                echo "pid file $P_CONFIG_PROXY was not created" 1>&2
+            elif ! kill -0 `cat $P_CONFIG_PROXY`
+            then
+                echo "config proxy process `cat $P_CONFIG_PROXY` has terminated" 1>&2
+            elif ! vespa-ping-configproxy -s $hname
+            then
+                echo "failed to ping config proxy $hname" 1>&2
+                # TODO: Dump stack trace for debugging, remove after April 2020
+                kill -3 `pgrep -f -n configproxy`
+                sleep 2
+                kill -3 `pgrep -f -n configproxy`
+                sleep 2
+                kill -3 `pgrep -f -n configproxy`
+            fi
+
+            echo "look for reason in vespa.log, last part follows..." 1>&2
+            tail -n 15 $LOGFILE | vespa-logfmt -  1>&2
             exit 1
         fi
 
diff --git a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServerTest.java b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServerTest.java
index dc1c995fbb5..29bd38ea891 100644
--- a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServerTest.java
+++ b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServerTest.java
@@ -264,7 +264,7 @@ public class ConfigProxyRpcServerTest {
     }
 
     private static ProxyServer createTestServer(ConfigSourceSet source) {
-        return new ProxyServer(null, source, ProxyServer.defaultTimingValues(), new MemoryCache(), null);
+        return new ProxyServer(null, source, new MemoryCache(), null);
     }
 
     private static class TestServer implements AutoCloseable {
diff --git a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/MockConfigSourceClient.java b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/MockConfigSourceClient.java
index 963c922d5b5..06e55eef4fa 100644
--- a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/MockConfigSourceClient.java
+++ b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/MockConfigSourceClient.java
@@ -16,6 +16,7 @@ import java.util.List;
 public class MockConfigSourceClient implements ConfigSourceClient{
     private final MockConfigSource configSource;
     private final MemoryCache memoryCache;
+    private final DelayedResponses delayedResponses = new DelayedResponses();
 
     MockConfigSourceClient(MockConfigSource configSource, MemoryCache memoryCache) {
         this.configSource = configSource;
@@ -53,7 +54,9 @@ public class MockConfigSourceClient implements ConfigSourceClient{
     }
 
     @Override
-    public void updateSubscribers(RawConfig config) {
+    public void updateSubscribers(RawConfig config) { }
+
+    @Override
+    public DelayedResponses delayedResponses() { return delayedResponses; }
 
-    }
 }
diff --git a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ProxyServerTest.java b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ProxyServerTest.java
index 712567774f1..f52598b3ee5 100644
--- a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ProxyServerTest.java
+++ b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/ProxyServerTest.java
@@ -14,7 +14,11 @@ import org.junit.rules.TemporaryFolder;
 import java.util.Optional;
 
 import static org.hamcrest.CoreMatchers.is;
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
 
 /**
  * @author hmusum
@@ -23,7 +27,7 @@ public class ProxyServerTest {
 
     private final MemoryCache memoryCache = new MemoryCache();
     private final MockConfigSource source = new MockConfigSource();
-    private MockConfigSourceClient client = new MockConfigSourceClient(source, memoryCache);
+    private final MockConfigSourceClient client = new MockConfigSourceClient(source, memoryCache);
     private ProxyServer proxy;
 
     static final RawConfig fooConfig = ConfigTester.fooConfig;
@@ -54,7 +58,6 @@ public class ProxyServerTest {
     public void basic() {
         assertTrue(proxy.getMode().isDefault());
         assertThat(proxy.getMemoryCache().size(), is(0));
-        assertThat(proxy.getTimingValues(), is(ProxyServer.defaultTimingValues()));
 
         ConfigTester tester = new ConfigTester();
         final MemoryCache memoryCache = proxy.getMemoryCache();
@@ -222,7 +225,7 @@ public class ProxyServerTest {
     private static ProxyServer createTestServer(ConfigSourceSet source,
                                                 ConfigSourceClient configSourceClient,
                                                 MemoryCache memoryCache) {
-        return new ProxyServer(null, source, ProxyServer.defaultTimingValues(), memoryCache, configSourceClient);
+        return new ProxyServer(null, source, memoryCache, configSourceClient);
     }
 
     static RawConfig createConfigWithNextConfigGeneration(RawConfig config, int errorCode) {
diff --git a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClientTest.java b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClientTest.java
index 35f1dd8fcd8..8510b23bbd2 100644
--- a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClientTest.java
+++ b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClientTest.java
@@ -18,7 +18,6 @@ import static org.junit.Assert.assertEquals;
 public class RpcConfigSourceClientTest {
 
     private MockRpcServer rpcServer;
-    private DelayedResponses delayedResponses;
     private RpcConfigSourceClient rpcConfigSourceClient;
 
     @Rule
@@ -28,10 +27,7 @@ public class RpcConfigSourceClientTest {
     @Before
     public void setup() {
         rpcServer = new MockRpcServer();
-        delayedResponses = new DelayedResponses();
-        rpcConfigSourceClient =
-                new RpcConfigSourceClient(rpcServer, new MockConfigSource(),
-                                          new MemoryCache(), ProxyServer.defaultTimingValues(), delayedResponses);
+        rpcConfigSourceClient = new RpcConfigSourceClient(rpcServer, new MockConfigSource(), new MemoryCache());
     }
 
     @Test
@@ -98,7 +94,7 @@ public class RpcConfigSourceClientTest {
     }
 
     private void simulateClientRequestingConfig(RawConfig config) {
-        delayedResponses.add(new DelayedResponse(JRTServerConfigRequestV3.createFromRequest(JRTConfigRequestFactory.createFromRaw(config, -10L).getRequest())));
+        rpcConfigSourceClient.delayedResponses().add(new DelayedResponse(JRTServerConfigRequestV3.createFromRequest(JRTConfigRequestFactory.createFromRaw(config, -10L).getRequest())));
     }
 
     private void configUpdatedSendResponse(RawConfig config) {
diff --git a/config/pom.xml b/config/pom.xml
index d4f6fd1ebf5..ad036c442d1 100755
--- a/config/pom.xml
+++ b/config/pom.xml
@@ -66,10 +66,6 @@
       <groupId>com.fasterxml.jackson.core</groupId>
       <artifactId>jackson-core</artifactId>
     </dependency>
-    <dependency>
-      <groupId>net.jpountz.lz4</groupId>
-      <artifactId>lz4</artifactId>
-    </dependency>
 
     <!-- test scope -->
     <dependency>
diff --git a/config/src/apps/vespa-ping-configproxy/pingproxy.cpp b/config/src/apps/vespa-ping-configproxy/pingproxy.cpp
index 084ad3d8d1c..208f9312ada 100644
--- a/config/src/apps/vespa-ping-configproxy/pingproxy.cpp
+++ b/config/src/apps/vespa-ping-configproxy/pingproxy.cpp
@@ -146,6 +146,7 @@ PingProxy::Main()
             retval = 1;
         }
     }
+    req->SubRef();
     finiRPC();
     return retval;
 }
diff --git a/config/src/main/java/com/yahoo/config/subscription/CfgConfigPayloadBuilder.java b/config/src/main/java/com/yahoo/config/subscription/CfgConfigPayloadBuilder.java
index 80c55c7b558..474c9f7a4db 100644
--- a/config/src/main/java/com/yahoo/config/subscription/CfgConfigPayloadBuilder.java
+++ b/config/src/main/java/com/yahoo/config/subscription/CfgConfigPayloadBuilder.java
@@ -8,7 +8,8 @@ import com.yahoo.log.LogLevel;
 import com.yahoo.vespa.config.ConfigPayload;
 import com.yahoo.vespa.config.ConfigPayloadBuilder;
 
-import java.util.*;
+import java.util.ArrayList;
+import java.util.List;
 
 /**
  * Deserializes config payload (cfg format) to a ConfigPayload.
@@ -33,15 +34,10 @@ public class CfgConfigPayloadBuilder {
         int lineNum = 1;
         ConfigPayloadBuilder payloadBuilder = new ConfigPayloadBuilder();
         for (String line : lines) {
-            if (log.isLoggable(LogLevel.SPAM)) {
-               log.log(LogLevel.SPAM, "line " + lineNum + ": '" + line + "'");
-            }
             parseLine(line, lineNum, payloadBuilder);
             lineNum++;
         }
-        if (log.isLoggable(LogLevel.DEBUG)) {
-            log.log(LogLevel.DEBUG, "payload=" + payloadBuilder.toString());
-        }
+        log.log(LogLevel.SPAM, () -> "payload=" + payloadBuilder.toString());
         return payloadBuilder;
     }
 
@@ -52,16 +48,13 @@ public class CfgConfigPayloadBuilder {
         String field = fieldAndValue.getFirst();
         String value = fieldAndValue.getSecond();
         if (field==null || value==null) {
-            log.log(LogLevel.DEBUG, "Got field without value in line " + lineNum + ": " + line + ", skipping");
+            log.log(LogLevel.DEBUG, () -> "Got field without value in line " + lineNum + ": " + line + ", skipping");
             return;
         }
         field=field.trim();
         value=value.trim();
         validateField(field, trimmedLine, lineNum);
         validateValue(value, trimmedLine, lineNum);
-        if (log.isLoggable(LogLevel.DEBUG)) {
-            log.log(LogLevel.DEBUG, "field=" + field + ",value=" + value);
-        }
         List<String> fields = parseFieldList(field);
         ConfigPayloadBuilder currentBuilder = payloadBuilder;
         for (int fieldNum = 0; fieldNum < fields.size(); fieldNum++) {
diff --git a/config/src/main/java/com/yahoo/config/subscription/ConfigInstanceSerializer.java b/config/src/main/java/com/yahoo/config/subscription/ConfigInstanceSerializer.java
index 1069f1cdd53..7c042656bda 100644
--- a/config/src/main/java/com/yahoo/config/subscription/ConfigInstanceSerializer.java
+++ b/config/src/main/java/com/yahoo/config/subscription/ConfigInstanceSerializer.java
@@ -9,7 +9,6 @@ import com.yahoo.slime.Slime;
  * Implements a config instance serializer, serializing a config instance to a slime object.
  *
  * @author Ulf Lilleengen
- * @since 5.1.14
  */
 public class ConfigInstanceSerializer implements Serializer {
     private final Slime slime;
diff --git a/config/src/main/java/com/yahoo/config/subscription/ConfigInstanceUtil.java b/config/src/main/java/com/yahoo/config/subscription/ConfigInstanceUtil.java
index fb2a3acbfdc..9710ee607eb 100644
--- a/config/src/main/java/com/yahoo/config/subscription/ConfigInstanceUtil.java
+++ b/config/src/main/java/com/yahoo/config/subscription/ConfigInstanceUtil.java
@@ -38,9 +38,7 @@ public class ConfigInstanceUtil {
         }
     }
 
-    public static <T extends ConfigInstance> T getNewInstance(Class<T> type,
-                                                              String configId,
-                                                              ConfigPayload payload) {
+    public static <T extends ConfigInstance> T getNewInstance(Class<T> type, String configId, ConfigPayload payload) {
         T instance;
         try {
             ConfigTransformer<?> transformer = new ConfigTransformer<>(type);
diff --git a/config/src/main/java/com/yahoo/config/subscription/ConfigInterruptedException.java b/config/src/main/java/com/yahoo/config/subscription/ConfigInterruptedException.java
index 1eaada5ae9d..121dba1e555 100644
--- a/config/src/main/java/com/yahoo/config/subscription/ConfigInterruptedException.java
+++ b/config/src/main/java/com/yahoo/config/subscription/ConfigInterruptedException.java
@@ -4,7 +4,6 @@ package com.yahoo.config.subscription;
 /**
  * This exception is thrown when any blocking call within the Config API is interrupted.
  * @author Ulf Lilleengen
- * @since 5.1
  */
 @SuppressWarnings("serial")
 public class ConfigInterruptedException extends RuntimeException {
diff --git a/config/src/main/java/com/yahoo/config/subscription/ConfigSet.java b/config/src/main/java/com/yahoo/config/subscription/ConfigSet.java
index 57f4ad863f2..c3ceb81d3db 100644
--- a/config/src/main/java/com/yahoo/config/subscription/ConfigSet.java
+++ b/config/src/main/java/com/yahoo/config/subscription/ConfigSet.java
@@ -10,7 +10,7 @@ import com.yahoo.vespa.config.ConfigKey;
 /**
  * Config source as a programmatically built set of {@link com.yahoo.config.ConfigInstance}s
  *
- * @author vegardh
+ * @author Vegard Havdal
  */
 public class ConfigSet implements ConfigSource {
     private final Map<ConfigKey<?>, ConfigInstance.Builder> configs = new ConcurrentHashMap<>();
diff --git a/config/src/main/java/com/yahoo/config/subscription/ConfigSource.java b/config/src/main/java/com/yahoo/config/subscription/ConfigSource.java
index d0eda9d27cc..0aac060abf6 100644
--- a/config/src/main/java/com/yahoo/config/subscription/ConfigSource.java
+++ b/config/src/main/java/com/yahoo/config/subscription/ConfigSource.java
@@ -4,7 +4,7 @@ package com.yahoo.config.subscription;
 /**
  * A type of source of config
  *
- * @author vegardh
+ * @author Vegard Havdal
  */
 public interface ConfigSource {
 
diff --git a/config/src/main/java/com/yahoo/config/subscription/ConfigSourceSet.java b/config/src/main/java/com/yahoo/config/subscription/ConfigSourceSet.java
index c799186435c..7472439d6a4 100755
--- a/config/src/main/java/com/yahoo/config/subscription/ConfigSourceSet.java
+++ b/config/src/main/java/com/yahoo/config/subscription/ConfigSourceSet.java
@@ -3,7 +3,11 @@ package com.yahoo.config.subscription;
 
 import com.yahoo.log.LogLevel;
 
-import java.util.*;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Set;
 import java.util.logging.Logger;
 
 
diff --git a/config/src/main/java/com/yahoo/config/subscription/ConfigSubscriber.java b/config/src/main/java/com/yahoo/config/subscription/ConfigSubscriber.java
index ae6f14ad59e..3891d710fa3 100644
--- a/config/src/main/java/com/yahoo/config/subscription/ConfigSubscriber.java
+++ b/config/src/main/java/com/yahoo/config/subscription/ConfigSubscriber.java
@@ -25,7 +25,7 @@ import static java.util.stream.Collectors.toList;
  * {@link #subscribe(Class, String)} on the configs needed, call {@link #nextConfig(long)} and get the config from the
  * {@link ConfigHandle} which {@link #subscribe(Class, String)} returned.
  *
- * @author vegardh
+ * @author Vegard Havdal
  */
 public class ConfigSubscriber implements AutoCloseable {
 
@@ -319,13 +319,14 @@ public class ConfigSubscriber implements AutoCloseable {
     @Override
     public void close() {
         synchronized (monitor) {
+            if (state == State.CLOSED) return;
             state = State.CLOSED;
         }
         for (ConfigHandle<? extends ConfigInstance> h : subscriptionHandles) {
             h.subscription().close();
         }
         closeRequesters();
-        log.log(LogLevel.DEBUG, "Config subscriber has been closed.");
+        log.log(LogLevel.DEBUG, () -> "Config subscriber has been closed.");
     }
 
     /**
diff --git a/config/src/main/java/com/yahoo/config/subscription/DirSource.java b/config/src/main/java/com/yahoo/config/subscription/DirSource.java
index d90a0493838..6e12a38faa3 100644
--- a/config/src/main/java/com/yahoo/config/subscription/DirSource.java
+++ b/config/src/main/java/com/yahoo/config/subscription/DirSource.java
@@ -5,8 +5,7 @@ import java.io.File;
 
 /**
  * Source specifying config from a local directory
- * @author vegardh
- * @since 5.1
+ * @author Vegard Havdal
  *
  */
 public class DirSource implements ConfigSource {
diff --git a/config/src/main/java/com/yahoo/config/subscription/FileSource.java b/config/src/main/java/com/yahoo/config/subscription/FileSource.java
index 27129853eae..9b6fb6442e8 100644
--- a/config/src/main/java/com/yahoo/config/subscription/FileSource.java
+++ b/config/src/main/java/com/yahoo/config/subscription/FileSource.java
@@ -6,7 +6,7 @@ import java.io.File;
 /**
  * Source specifying config from one local file
  *
- * @author vegardh
+ * @author Vegard Havdal
  */
 public class FileSource implements ConfigSource {
 
diff --git a/config/src/main/java/com/yahoo/config/subscription/JarSource.java b/config/src/main/java/com/yahoo/config/subscription/JarSource.java
index 6275befe18c..a098dec7f21 100644
--- a/config/src/main/java/com/yahoo/config/subscription/JarSource.java
+++ b/config/src/main/java/com/yahoo/config/subscription/JarSource.java
@@ -5,8 +5,7 @@ import java.util.jar.JarFile;
 
 /**
  * Source specifying config as a jar file entry
- * @author vegardh
- * @since 5.1
+ * @author Vegard Havdal
  *
  */
 public class JarSource implements ConfigSource {
diff --git a/config/src/main/java/com/yahoo/config/subscription/RawSource.java b/config/src/main/java/com/yahoo/config/subscription/RawSource.java
index d68b503eb74..099a6bdc334 100644
--- a/config/src/main/java/com/yahoo/config/subscription/RawSource.java
+++ b/config/src/main/java/com/yahoo/config/subscription/RawSource.java
@@ -4,7 +4,7 @@ package com.yahoo.config.subscription;
 /**
  * Source specifying raw config, where payload is given programmatically
  *
- * @author vegardh
+ * @author Vegard Havdal
  */
 public class RawSource implements ConfigSource {
 
diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/ConfigSetSubscription.java b/config/src/main/java/com/yahoo/config/subscription/impl/ConfigSetSubscription.java
index aa80cc75ef0..d3a1bfb50a9 100644
--- a/config/src/main/java/com/yahoo/config/subscription/impl/ConfigSetSubscription.java
+++ b/config/src/main/java/com/yahoo/config/subscription/impl/ConfigSetSubscription.java
@@ -11,8 +11,7 @@ import java.lang.reflect.Constructor;
 
 /**
  * Subscription on a programmatically built set of configs
- * @author vegardh
- * @since 5.1
+ * @author Vegard Havdal
  */
 public class ConfigSetSubscription<T extends ConfigInstance> extends ConfigSubscription<T> {
 
diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/ConfigSubscription.java b/config/src/main/java/com/yahoo/config/subscription/impl/ConfigSubscription.java
index c02301a0c17..3bf6093e872 100644
--- a/config/src/main/java/com/yahoo/config/subscription/impl/ConfigSubscription.java
+++ b/config/src/main/java/com/yahoo/config/subscription/impl/ConfigSubscription.java
@@ -21,7 +21,7 @@ import com.yahoo.vespa.config.protocol.DefContent;
 /**
  * Represents one active subscription to one config
  *
- * @author vegardh
+ * @author Vegard Havdal
  */
 public abstract class ConfigSubscription<T extends ConfigInstance> {
 
diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/FileConfigSubscription.java b/config/src/main/java/com/yahoo/config/subscription/impl/FileConfigSubscription.java
index bcee06cd667..60c920d1dfa 100644
--- a/config/src/main/java/com/yahoo/config/subscription/impl/FileConfigSubscription.java
+++ b/config/src/main/java/com/yahoo/config/subscription/impl/FileConfigSubscription.java
@@ -18,7 +18,7 @@ import com.yahoo.log.LogLevel;
 /**
  * Subscription used when config id is file:...
  *
- * @author vegardh
+ * @author Vegard Havdal
  */
 public class FileConfigSubscription<T extends ConfigInstance> extends ConfigSubscription<T> {
 
diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/GenericConfigHandle.java b/config/src/main/java/com/yahoo/config/subscription/impl/GenericConfigHandle.java
index bf247dffb7c..a40da85c88b 100644
--- a/config/src/main/java/com/yahoo/config/subscription/impl/GenericConfigHandle.java
+++ b/config/src/main/java/com/yahoo/config/subscription/impl/GenericConfigHandle.java
@@ -7,7 +7,7 @@ import com.yahoo.vespa.config.RawConfig;
 /**
  * A config handle which does not use the config class, but payload instead. Used in config proxy.
  *
- * @author vegardh
+ * @author Vegard Havdal
  */
 @SuppressWarnings({"rawtypes", "unchecked"})
 public class GenericConfigHandle extends ConfigHandle {
diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/GenericConfigSubscriber.java b/config/src/main/java/com/yahoo/config/subscription/impl/GenericConfigSubscriber.java
index 82d4708f53b..8c0a8f27555 100644
--- a/config/src/main/java/com/yahoo/config/subscription/impl/GenericConfigSubscriber.java
+++ b/config/src/main/java/com/yahoo/config/subscription/impl/GenericConfigSubscriber.java
@@ -16,7 +16,7 @@ import com.yahoo.vespa.config.TimingValues;
 /**
  * A subscriber that can subscribe without the class. Used by configproxy.
  *
- * @author vegardh
+ * @author Vegard Havdal
  */
 public class GenericConfigSubscriber extends ConfigSubscriber {
 
@@ -24,7 +24,7 @@ public class GenericConfigSubscriber extends ConfigSubscriber {
      * Constructs a new subscriber using the given pool of requesters (JRTConfigRequester holds 1 connection which in
      * turn is subject to failover across the elems in the source set.)
      * The behaviour is undefined if the map key is different from the source set the requester was built with.
-     * See also {@link JRTConfigRequester#get(com.yahoo.vespa.config.ConnectionPool, com.yahoo.vespa.config.TimingValues)}
+     * See also {@link JRTConfigRequester#JRTConfigRequester(com.yahoo.vespa.config.ConnectionPool, com.yahoo.vespa.config.TimingValues)}
      *
      * @param requesters a map from config source set to config requester
      */
@@ -32,10 +32,6 @@ public class GenericConfigSubscriber extends ConfigSubscriber {
         this.requesters = requesters;
     }
 
-    public GenericConfigSubscriber() {
-        super();
-    }
-
     /**
      * Subscribes to config without using the class. For internal use in config proxy.
      *
diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/GenericJRTConfigSubscription.java b/config/src/main/java/com/yahoo/config/subscription/impl/GenericJRTConfigSubscription.java
index c1f9ce02650..8ce3449fba5 100644
--- a/config/src/main/java/com/yahoo/config/subscription/impl/GenericJRTConfigSubscription.java
+++ b/config/src/main/java/com/yahoo/config/subscription/impl/GenericJRTConfigSubscription.java
@@ -16,7 +16,7 @@ import com.yahoo.vespa.config.protocol.JRTClientConfigRequest;
  * A JRT subscription which does not use the config class, but {@link com.yahoo.vespa.config.RawConfig} instead.
  * Used by config proxy.
  *
- * @author vegardh
+ * @author Vegard Havdal
  *
  */
 public class GenericJRTConfigSubscription extends JRTConfigSubscription<RawConfig> {
@@ -33,9 +33,7 @@ public class GenericJRTConfigSubscription extends JRTConfigSubscription<RawConfi
     @Override
     protected void setNewConfig(JRTClientConfigRequest jrtReq) {
         setConfig(jrtReq.getNewGeneration(), jrtReq.responseIsInternalRedeploy(), RawConfig.createFromResponseParameters(jrtReq) );
-        if (log.isLoggable(LogLevel.DEBUG)) {
-            log.log(LogLevel.DEBUG, "in setNewConfig, config=" + this.getConfigState().getConfig());
-        }
+        log.log(LogLevel.DEBUG, () -> "in setNewConfig, config=" + this.getConfigState().getConfig());
     }
 
     // This method is overridden because config needs to have its generation
diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigRequester.java b/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigRequester.java
index 06c772ac456..49c5dcd343c 100644
--- a/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigRequester.java
+++ b/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigRequester.java
@@ -20,7 +20,6 @@ import com.yahoo.yolean.Exceptions;
 import java.time.Duration;
 import java.time.Instant;
 import java.util.concurrent.ScheduledThreadPoolExecutor;
-import java.util.concurrent.ThreadFactory;
 import java.util.concurrent.TimeUnit;
 import java.util.logging.Level;
 import java.util.logging.Logger;
@@ -37,39 +36,44 @@ public class JRTConfigRequester implements RequestWaiter {
 
     private static final Logger log = Logger.getLogger(JRTConfigRequester.class.getName());
     public static final ConfigSourceSet defaultSourceSet = ConfigSourceSet.createDefault();
+    private static final JRTManagedConnectionPools managedPool = new JRTManagedConnectionPools();
     private static final int TRACELEVEL = 6;
     private final TimingValues timingValues;
     private int fatalFailures = 0; // independent of transientFailures
     private int transientFailures = 0;  // independent of fatalFailures
-    private final ScheduledThreadPoolExecutor scheduler = new ScheduledThreadPoolExecutor(1, new JRTSourceThreadFactory());
+    private final ScheduledThreadPoolExecutor scheduler;
     private Instant suspendWarningLogged = Instant.MIN;
     private Instant noApplicationWarningLogged = Instant.MIN;
     private static final Duration delayBetweenWarnings = Duration.ofSeconds(60);
     private final ConnectionPool connectionPool;
+    private final ConfigSourceSet configSourceSet;
     static final float randomFraction = 0.2f;
     /* Time to be added to server timeout to create client timeout. This is the time allowed for the server to respond after serverTimeout has elapsed. */
     private static final Double additionalTimeForClientTimeout = 10.0;
 
     /**
      * Returns a new requester
-     * @param connectionPool The connectionPool to use
-     * @param timingValues The timing values
-     * @return new requester object
-     */
-    public static JRTConfigRequester get(ConnectionPool connectionPool, TimingValues timingValues) {
-        return new JRTConfigRequester(connectionPool, timingValues);
-    }
-
-    /**
-     * New requester
-     *  @param connectionPool the connectionPool this requester should use
-     * @param timingValues timeouts and delays used when sending JRT config requests
+     *
+     * @param connectionPool the connectionPool this requester should use
+     * @param timingValues   timeouts and delays used when sending JRT config requests
      */
-    JRTConfigRequester(ConnectionPool connectionPool, TimingValues timingValues) {
+    JRTConfigRequester(ConfigSourceSet configSourceSet, ScheduledThreadPoolExecutor scheduler,
+                       ConnectionPool connectionPool, TimingValues timingValues) {
+        this.configSourceSet = configSourceSet;
+        this.scheduler = scheduler;
         this.connectionPool = connectionPool;
         this.timingValues = timingValues;
     }
 
+    /** Only for testing */
+    public JRTConfigRequester(ConnectionPool connectionPool, TimingValues timingValues) {
+        this(null, new ScheduledThreadPoolExecutor(1), connectionPool, timingValues);
+    }
+
+    public static JRTConfigRequester create(ConfigSourceSet sourceSet, TimingValues timingValues) {
+        return managedPool.acquire(sourceSet, timingValues);
+    }
+
     /**
      * Requests the config for the {@link com.yahoo.config.ConfigInstance} on the given {@link ConfigSubscription}
      *
@@ -86,11 +90,9 @@ public class JRTConfigRequester implements RequestWaiter {
         if ( ! req.validateParameters()) throw new ConfigurationRuntimeException("Error in parameters for config request: " + req);
 
         double jrtClientTimeout = getClientTimeout(req);
-        if (log.isLoggable(LogLevel.DEBUG)) {
-            log.log(LogLevel.DEBUG, "Requesting config for " + sub + " on connection " + connection
-                    + " with client timeout " + jrtClientTimeout +
-                    (log.isLoggable(LogLevel.SPAM) ? (",defcontent=" + req.getDefContent().asString()) : ""));
-        }
+        log.log(LogLevel.DEBUG, () -> "Requesting config for " + sub + " on connection " + connection
+                                      + " with client timeout " + jrtClientTimeout +
+                                      (log.isLoggable(LogLevel.SPAM) ? (",defcontent=" + req.getDefContent().asString()) : ""));
         connection.invokeAsync(req.getRequest(), jrtClientTimeout, this);
     }
 
@@ -120,7 +122,7 @@ public class JRTConfigRequester implements RequestWaiter {
         if (sub.getState() == ConfigSubscription.State.CLOSED) return; // Avoid error messages etc. after closing
         Trace trace = jrtReq.getResponseTrace();
         trace.trace(TRACELEVEL, "JRTConfigRequester.doHandle()");
-        log.log(LogLevel.SPAM, trace::toString);
+        log.log(LogLevel.SPAM, () -> trace.toString());
         if (validResponse) {
             handleOKRequest(jrtReq, sub, connection);
         } else {
@@ -157,7 +159,7 @@ public class JRTConfigRequester implements RequestWaiter {
             // The subscription object has an "old" config, which is all we have to offer back now
             log.log(LogLevel.INFO, "Failure of config subscription, clients will keep existing config until resolved: " + sub);
         }
-        final ErrorType errorType = ErrorType.getErrorType(jrtReq.errorCode());
+        ErrorType errorType = ErrorType.getErrorType(jrtReq.errorCode());
         connectionPool.setError(connection, jrtReq.errorCode());
         long delay = calculateFailedRequestDelay(errorType, transientFailures, fatalFailures, timingValues, configured);
         if (errorType == ErrorType.TRANSIENT) {
@@ -233,7 +235,8 @@ public class JRTConfigRequester implements RequestWaiter {
         suspendWarningLogged = Instant.MIN;
         noApplicationWarningLogged = Instant.MIN;
         connection.setSuccess();
-        sub.setLastCallBackOKTS(System.currentTimeMillis());
+        sub.setLastCallBackOKTS(Instant.now());
+        log.log(LogLevel.DEBUG, () -> "OK response received in handleOkRequest: " + jrtReq);
         if (jrtReq.hasUpdatedGeneration()) {
             // We only want this latest generation to be in the queue, we do not preserve history in this system
             sub.getReqQueue().clear();
@@ -257,7 +260,7 @@ public class JRTConfigRequester implements RequestWaiter {
     private void scheduleNextRequest(JRTClientConfigRequest jrtReq, JRTConfigSubscription<?> sub, long delay, long timeout) {
         long delayBeforeSendingRequest = (delay < 0) ? 0 : delay;
         JRTClientConfigRequest jrtReqNew = jrtReq.nextRequest(timeout);
-        log.log(LogLevel.DEBUG, timingValues::toString);
+        log.log(LogLevel.SPAM, () -> timingValues.toString());
         log.log(LogLevel.DEBUG, () -> "Scheduling new request " + delayBeforeSendingRequest + " millis from now for " + jrtReqNew.getConfigKey());
         scheduler.schedule(new GetConfigTask(jrtReqNew, sub), delayBeforeSendingRequest, TimeUnit.MILLISECONDS);
     }
@@ -283,18 +286,8 @@ public class JRTConfigRequester implements RequestWaiter {
         // Fake that we have logged to avoid printing warnings after this
         suspendWarningLogged = Instant.now();
         noApplicationWarningLogged = Instant.now();
-
-        connectionPool.close();
-        scheduler.shutdown();
-    }
-
-    private static class JRTSourceThreadFactory implements ThreadFactory {
-        @Override
-        public Thread newThread(Runnable runnable) {
-            Thread t = new Thread(runnable, String.format("jrt-config-requester-%d", System.currentTimeMillis()));
-            // We want a daemon thread to avoid hanging threads in case something goes wrong in the config system
-            t.setDaemon(true);
-            return t;
+        if (configSourceSet != null) {
+            managedPool.release(configSourceSet);
         }
     }
 
diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigSubscription.java b/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigSubscription.java
index d9366c28b9b..a94a135f9d8 100644
--- a/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigSubscription.java
+++ b/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigSubscription.java
@@ -1,6 +1,8 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.config.subscription.impl;
 
+import java.time.Duration;
+import java.time.Instant;
 import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.TimeUnit;
 
@@ -26,10 +28,10 @@ import com.yahoo.vespa.config.protocol.Payload;
 public class JRTConfigSubscription<T extends ConfigInstance> extends ConfigSubscription<T> {
 
     private JRTConfigRequester requester;
-    private TimingValues timingValues;
+    private final TimingValues timingValues;
 
-    // Last time we got an OK JRT callback for this
-    private long lastOK = 0;
+    // Last time we got an OK JRT callback
+    private Instant lastOK = Instant.MIN;
 
     /**
      * The queue containing either nothing or the one (newest) request that has got callback from JRT,
@@ -40,9 +42,9 @@ public class JRTConfigSubscription<T extends ConfigInstance> extends ConfigSubsc
 
     public JRTConfigSubscription(ConfigKey<T> key, ConfigSubscriber subscriber, ConfigSource source, TimingValues timingValues) {
         super(key, subscriber);
-        this.timingValues=timingValues;
+        this.timingValues = timingValues;
         if (source instanceof ConfigSourceSet) {
-            this.sources=(ConfigSourceSet) source;
+            this.sources = (ConfigSourceSet) source;
         }
     }
 
@@ -53,7 +55,7 @@ public class JRTConfigSubscription<T extends ConfigInstance> extends ConfigSubsc
         ConfigState<T> configState = getConfigState();
         boolean gotNew = configState.isGenerationChanged() || configState.isConfigChanged() || hasException();
         // Return that now, if there's nothing in queue, so that ConfigSubscriber can move on to other subscriptions to check
-        if (getReqQueue().peek()==null && gotNew) {
+        if (getReqQueue().peek() == null && gotNew) {
             return true;
         }
         // Otherwise poll the queue for another generation or timeout
@@ -88,6 +90,7 @@ public class JRTConfigSubscription<T extends ConfigInstance> extends ConfigSubsc
             // timed out, we know nothing new.
             return false;
         }
+        log.log(LogLevel.DEBUG, () -> "Polled queue and found config " + jrtReq);
         if (jrtReq.hasUpdatedGeneration()) {
             setInternalRedeploy(jrtReq.responseIsInternalRedeploy());
             if (jrtReq.hasUpdatedConfig()) {
@@ -135,11 +138,11 @@ public class JRTConfigSubscription<T extends ConfigInstance> extends ConfigSubsc
 
     @Override
     public boolean subscribe(long timeout) {
-        lastOK=System.currentTimeMillis();
+        lastOK = Instant.now();
         requester = getRequester();
         requester.request(this);
         JRTClientConfigRequest req = reqQueue.peek();
-        while (req == null && (System.currentTimeMillis() - lastOK <= timeout)) {
+        while (req == null && (Instant.now().isBefore(lastOK.plus(Duration.ofMillis(timeout))))) {
             try {
                 Thread.sleep(10);
             } catch (InterruptedException e) {
@@ -152,8 +155,8 @@ public class JRTConfigSubscription<T extends ConfigInstance> extends ConfigSubsc
 
     private JRTConfigRequester getRequester() {
         JRTConfigRequester requester = subscriber.requesters().get(sources);
-        if (requester==null) {
-            requester = new JRTConfigRequester(new JRTConnectionPool(sources), timingValues);
+        if (requester == null) {
+            requester = JRTConfigRequester.create(sources, timingValues);
             subscriber.requesters().put(sources, requester);
         }
         return requester;
@@ -164,8 +167,9 @@ public class JRTConfigSubscription<T extends ConfigInstance> extends ConfigSubsc
     public void close() {
         super.close();
         reqQueue = new LinkedBlockingQueue<>() {
+            @SuppressWarnings("NullableProblems")
             @Override
-            public void put(JRTClientConfigRequest e) throws InterruptedException {
+            public void put(JRTClientConfigRequest e) {
                 // When closed, throw away all requests that callbacks try to put
             }
         };
@@ -191,7 +195,7 @@ public class JRTConfigSubscription<T extends ConfigInstance> extends ConfigSubsc
         log.log(LogLevel.DEBUG, "reload() is without effect on a JRTConfigSubscription.");
     }
 
-    void setLastCallBackOKTS(long lastCallBackOKTS) {
+    void setLastCallBackOKTS(Instant lastCallBackOKTS) {
         this.lastOK = lastCallBackOKTS;
     }
 
diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/JRTManagedConnectionPools.java b/config/src/main/java/com/yahoo/config/subscription/impl/JRTManagedConnectionPools.java
new file mode 100644
index 00000000000..32d2d962e4d
--- /dev/null
+++ b/config/src/main/java/com/yahoo/config/subscription/impl/JRTManagedConnectionPools.java
@@ -0,0 +1,66 @@
+package com.yahoo.config.subscription.impl;
+
+import com.yahoo.config.subscription.ConfigSourceSet;
+import com.yahoo.vespa.config.JRTConnectionPool;
+import com.yahoo.vespa.config.TimingValues;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.TimeUnit;
+
+public class JRTManagedConnectionPools {
+    private static class JRTSourceThreadFactory implements ThreadFactory {
+        @Override
+        public Thread newThread(Runnable runnable) {
+            Thread t = new Thread(runnable, String.format("jrt-config-requester-%d", System.currentTimeMillis()));
+            // We want a daemon thread to avoid hanging threads in case something goes wrong in the config system
+            t.setDaemon(true);
+            return t;
+        }
+    }
+    private static class CountedPool {
+        long count;
+        final JRTConnectionPool pool;
+        final ScheduledThreadPoolExecutor scheduler = new ScheduledThreadPoolExecutor(1, new JRTSourceThreadFactory());
+        CountedPool(JRTConnectionPool requester) {
+            this.pool = requester;
+            count = 0;
+        }
+    }
+
+    private final Map<ConfigSourceSet, CountedPool> pools = new HashMap<>();
+
+    public JRTConfigRequester acquire(ConfigSourceSet sourceSet, TimingValues timingValues) {
+        CountedPool countedPool;
+        synchronized (pools) {
+            countedPool = pools.get(sourceSet);
+            if (countedPool == null) {
+                countedPool = new CountedPool(new JRTConnectionPool(sourceSet));
+                pools.put(sourceSet, countedPool);
+            }
+            countedPool.count++;
+        }
+        return new JRTConfigRequester(sourceSet, countedPool.scheduler, countedPool.pool, timingValues);
+    }
+
+    public synchronized void release(ConfigSourceSet sourceSet) {
+        CountedPool countedPool;
+        synchronized (pools) {
+            countedPool = pools.get(sourceSet);
+            if (countedPool != null)
+                countedPool.count--;
+            if (countedPool == null || countedPool.count > 0) return;
+            pools.remove(sourceSet);
+        }
+
+        countedPool.pool.close();
+        countedPool.scheduler.shutdown();
+        try {
+            countedPool.scheduler.awaitTermination(30, TimeUnit.SECONDS);
+        } catch (InterruptedException e) {
+            throw new RuntimeException("Failed shutting down scheduler:", e);
+        }
+    }
+}
diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/JarConfigSubscription.java b/config/src/main/java/com/yahoo/config/subscription/impl/JarConfigSubscription.java
index a96499482c5..05da9a72837 100644
--- a/config/src/main/java/com/yahoo/config/subscription/impl/JarConfigSubscription.java
+++ b/config/src/main/java/com/yahoo/config/subscription/impl/JarConfigSubscription.java
@@ -21,7 +21,7 @@ import com.yahoo.vespa.config.ConfigPayload;
 /**
  * Subscription to use when config id is jar:.../foo.jar[!/pathInJar/]
  *
- * @author vegardh
+ * @author Vegard Havdal
  * @author gjoranv
  */
 public class JarConfigSubscription<T extends ConfigInstance> extends ConfigSubscription<T> {
@@ -33,8 +33,8 @@ public class JarConfigSubscription<T extends ConfigInstance> extends ConfigSubsc
     // jar:configs/app.jar!/configs/
     JarConfigSubscription(ConfigKey<T> key, ConfigSubscriber subscriber, String jarName, String path) {
         super(key, subscriber);
-        this.jarName=jarName;
-        this.path=path;
+        this.jarName = jarName;
+        this.path = path;
     }
 
     @Override
@@ -43,17 +43,18 @@ public class JarConfigSubscription<T extends ConfigInstance> extends ConfigSubsc
             // Not supporting changing the payload for jar
             return true;
         }
-        if (zipEntry==null) {
+        if (zipEntry == null) {
             // First time polled
-            JarFile jarFile = null;
+            JarFile jarFile;
             try {
                 jarFile = new JarFile(jarName);
             } catch (IOException e) {
                 throw new IllegalArgumentException(e);
             }
             zipEntry = getEntry(jarFile, path);
-            if (zipEntry==null) throw new IllegalArgumentException("Config '" + key.getName() + "' not found in '" + jarName + "!/" + path + "'.");
-            T config = null;
+            if (zipEntry == null)
+                throw new IllegalArgumentException("Config '" + key.getName() + "' not found in '" + jarName + "!/" + path + "'.");
+            T config;
             try {
                 ConfigPayload payload = new CfgConfigPayloadBuilder().deserialize(Arrays.asList(IOUtils.readAll(new InputStreamReader(jarFile.getInputStream(zipEntry), StandardCharsets.UTF_8)).split("\n")));
                 config = payload.toInstance(configClass, key.getConfigId());
@@ -78,6 +79,7 @@ public class JarConfigSubscription<T extends ConfigInstance> extends ConfigSubsc
         }
         return false;
     }
+
     /**
      * Returns the entry corresponding to the ConfigInstance's defName/Version in the given directory in
      * the given JarFile.
diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/MockConnection.java b/config/src/main/java/com/yahoo/config/subscription/impl/MockConnection.java
index 9ad8f5c6ba2..58eed7f9e78 100644
--- a/config/src/main/java/com/yahoo/config/subscription/impl/MockConnection.java
+++ b/config/src/main/java/com/yahoo/config/subscription/impl/MockConnection.java
@@ -15,7 +15,6 @@ import com.yahoo.vespa.config.util.ConfigUtils;
  * For unit testing
  *
  * @author hmusum
- * @since 5.1.11
  */
 public class MockConnection implements ConnectionPool, com.yahoo.vespa.config.Connection {
 
diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/RawConfigSubscription.java b/config/src/main/java/com/yahoo/config/subscription/impl/RawConfigSubscription.java
index bc546cfdd1f..68ff6bb0135 100644
--- a/config/src/main/java/com/yahoo/config/subscription/impl/RawConfigSubscription.java
+++ b/config/src/main/java/com/yahoo/config/subscription/impl/RawConfigSubscription.java
@@ -12,10 +12,10 @@ import com.yahoo.vespa.config.ConfigPayload;
 
 /**
  * Subscription used when config id is raw:...
- *
+ * <p>
  * Config is the actual text given after the config id, with newlines
  *
- * @author vegardh
+ * @author Vegard Havdal
  */
 public class RawConfigSubscription<T extends ConfigInstance> extends ConfigSubscription<T> {
 
@@ -24,7 +24,7 @@ public class RawConfigSubscription<T extends ConfigInstance> extends ConfigSubsc
 
     RawConfigSubscription(ConfigKey<T> key, ConfigSubscriber subscriber, String pl) {
         super(key, subscriber);
-        this.inputPayload=pl;
+        this.inputPayload = pl;
     }
 
     @Override
@@ -32,7 +32,7 @@ public class RawConfigSubscription<T extends ConfigInstance> extends ConfigSubsc
         if (checkReloaded()) {
             return true;
         }
-        if (payload==null) {
+        if (payload == null) {
             payload = inputPayload;
             ConfigPayload configPayload = new CfgConfigPayloadBuilder().deserialize(Arrays.asList(payload.split("\n")));
             setConfig(0L, false, configPayload.toInstance(configClass, key.getConfigId()));
diff --git a/config/src/main/java/com/yahoo/vespa/config/ConfigKey.java b/config/src/main/java/com/yahoo/vespa/config/ConfigKey.java
index 55cb05ec230..6fcdc19cf97 100755
--- a/config/src/main/java/com/yahoo/vespa/config/ConfigKey.java
+++ b/config/src/main/java/com/yahoo/vespa/config/ConfigKey.java
@@ -78,7 +78,7 @@ public class ConfigKey<CONFIGCLASS extends ConfigInstance> implements Comparable
         if (!(o instanceof ConfigKey)) {
             return false;
         }
-        ConfigKey<?> key = (ConfigKey) o;
+        ConfigKey<?> key = (ConfigKey<?>) o;
         return (name.equals(key.name) &&
                 configId.equals(key.configId) &&
                 namespace.equals(key.namespace));
diff --git a/config/src/main/java/com/yahoo/vespa/config/ConfigPayload.java b/config/src/main/java/com/yahoo/vespa/config/ConfigPayload.java
index 4f02df04b9b..f4858843574 100644
--- a/config/src/main/java/com/yahoo/vespa/config/ConfigPayload.java
+++ b/config/src/main/java/com/yahoo/vespa/config/ConfigPayload.java
@@ -20,7 +20,6 @@ import java.io.OutputStream;
  * A class that holds a representation of a config payload.
  *
  * @author Ulf Lilleengen
- * @since 5.1.6
  */
 public class ConfigPayload {
     private final Slime slime;
diff --git a/config/src/main/java/com/yahoo/vespa/config/ConfigPayloadApplier.java b/config/src/main/java/com/yahoo/vespa/config/ConfigPayloadApplier.java
index 65106f158fc..134352736b6 100644
--- a/config/src/main/java/com/yahoo/vespa/config/ConfigPayloadApplier.java
+++ b/config/src/main/java/com/yahoo/vespa/config/ConfigPayloadApplier.java
@@ -33,7 +33,6 @@ import java.util.logging.Logger;
  * TODO: This can be refactored a lot, since many of the reflection methods are duplicated
  *
  * @author Ulf Lilleengen, hmusum, Tony Vaagenes
- * @since 5.1.6
  */
 public class ConfigPayloadApplier<T extends ConfigInstance.Builder> {
     private final static Logger log = Logger.getLogger(ConfigPayloadApplier.class.getPackage().getName());
@@ -480,15 +479,11 @@ public class ConfigPayloadApplier<T extends ConfigInstance.Builder> {
     }
 
     private void debug(String message) {
-        if (log.isLoggable(LogLevel.DEBUG)) {
-            log.log(LogLevel.DEBUG, message);
-        }
+        log.log(LogLevel.DEBUG, () -> message);
     }
 
     private void trace(String message) {
-        if (log.isLoggable(LogLevel.SPAM)) {
-            log.log(LogLevel.SPAM, message);
-        }
+        log.log(LogLevel.SPAM, () -> message);
     }
 
     private void printStack() {
diff --git a/config/src/main/java/com/yahoo/vespa/config/ConfigPayloadBuilder.java b/config/src/main/java/com/yahoo/vespa/config/ConfigPayloadBuilder.java
index bb974ddae42..2e470ec55e4 100644
--- a/config/src/main/java/com/yahoo/vespa/config/ConfigPayloadBuilder.java
+++ b/config/src/main/java/com/yahoo/vespa/config/ConfigPayloadBuilder.java
@@ -12,7 +12,6 @@ import java.util.*;
  *
  * TODO: Add toString
  * @author Ulf Lilleengen
- * @since 5.1
  */
 public class ConfigPayloadBuilder {
 
diff --git a/config/src/main/java/com/yahoo/vespa/config/ConfigTransformer.java b/config/src/main/java/com/yahoo/vespa/config/ConfigTransformer.java
index 163e010cdd6..6d2ae3ef13e 100644
--- a/config/src/main/java/com/yahoo/vespa/config/ConfigTransformer.java
+++ b/config/src/main/java/com/yahoo/vespa/config/ConfigTransformer.java
@@ -12,7 +12,6 @@ import static com.yahoo.vespa.config.ConfigPayloadApplier.IdentityPathAcquirer;
  * A utility class that can be used to transform config from one format to another.
  *
  * @author Ulf Lilleengen, hmusum, Tony Vaagenes
- * @since 5.1.6
  */
 public class ConfigTransformer<T extends ConfigInstance> {
     /**
diff --git a/config/src/main/java/com/yahoo/vespa/config/DefaultValueApplier.java b/config/src/main/java/com/yahoo/vespa/config/DefaultValueApplier.java
index f34b66e17a8..5bbc985cce0 100644
--- a/config/src/main/java/com/yahoo/vespa/config/DefaultValueApplier.java
+++ b/config/src/main/java/com/yahoo/vespa/config/DefaultValueApplier.java
@@ -11,7 +11,6 @@ import com.yahoo.slime.*;
  * TODO: Support giving correct type of default values
  *
  * @author Ulf Lilleengen
- * @since 5.1
  */
 public class DefaultValueApplier {
 
@@ -31,21 +30,11 @@ public class DefaultValueApplier {
     }
 
     private void applyDefaultsToMap(final Cursor cursor, final InnerCNode def) {
-        cursor.traverse(new ObjectTraverser() {
-            @Override
-            public void field(String name, Inspector inspector) {
-                applyDefaultsToObject(cursor.field(name), def);
-            }
-        });
+        cursor.traverse((ObjectTraverser) (name, inspector) -> applyDefaultsToObject(cursor.field(name), def));
     }
 
     private void applyDefaultsToArray(final Cursor cursor, final InnerCNode def) {
-        cursor.traverse(new ArrayTraverser() {
-            @Override
-            public void entry(int idx, Inspector inspector) {
-                applyDefaultsToObject(cursor.entry(idx), def);
-            }
-        });
+        cursor.traverse((ArrayTraverser) (idx, inspector) -> applyDefaultsToObject(cursor.entry(idx), def));
     }
 
     private void applyDefaultsToObject(Cursor cursor, InnerCNode def) {
diff --git a/config/src/main/java/com/yahoo/vespa/config/GenerationCounter.java b/config/src/main/java/com/yahoo/vespa/config/GenerationCounter.java
index c151121f0e9..3da09d07016 100644
--- a/config/src/main/java/com/yahoo/vespa/config/GenerationCounter.java
+++ b/config/src/main/java/com/yahoo/vespa/config/GenerationCounter.java
@@ -5,7 +5,6 @@ package com.yahoo.vespa.config;
  * Interface for counters.
  *
  * @author Ulf Lilleengen
- * @since 5.9
  */
 public interface GenerationCounter {
     /**
diff --git a/config/src/main/java/com/yahoo/vespa/config/GenericConfig.java b/config/src/main/java/com/yahoo/vespa/config/GenericConfig.java
index 2fbbda2aa38..2f351cc2bd4 100644
--- a/config/src/main/java/com/yahoo/vespa/config/GenericConfig.java
+++ b/config/src/main/java/com/yahoo/vespa/config/GenericConfig.java
@@ -6,7 +6,6 @@ import com.yahoo.config.ConfigInstance;
 
 /**
  *
- /**
  * A generic config with an internal generic builder that mimics a real config builder in order to support builders
  * when we don't have the schema.
  *
diff --git a/config/src/main/java/com/yahoo/vespa/config/JRTConnectionPool.java b/config/src/main/java/com/yahoo/vespa/config/JRTConnectionPool.java
index efeaacf225b..326c1287468 100644
--- a/config/src/main/java/com/yahoo/vespa/config/JRTConnectionPool.java
+++ b/config/src/main/java/com/yahoo/vespa/config/JRTConnectionPool.java
@@ -74,18 +74,14 @@ public class JRTConnectionPool implements ConnectionPool {
     public synchronized JRTConnection setNewCurrentConnection() {
         List<JRTConnection> sources = getSources();
         currentConnection = sources.get(ThreadLocalRandom.current().nextInt(0, sources.size()));
-        if (log.isLoggable(LogLevel.DEBUG)) {
-            log.log(LogLevel.DEBUG, "Choosing new connection: " + currentConnection);
-        }
+        log.log(LogLevel.DEBUG, () -> "Choosing new connection: " + currentConnection);
         return currentConnection;
     }
 
     List<JRTConnection> getSources() {
-        List<JRTConnection> ret = new ArrayList<>();
+        List<JRTConnection> ret;
         synchronized (connections) {
-            for (JRTConnection source : connections.values()) {
-                ret.add(source);
-            }
+            ret = new ArrayList<>(connections.values());
         }
         return ret;
     }
@@ -116,19 +112,6 @@ public class JRTConnectionPool implements ConnectionPool {
         return this;
     }
 
-    public String getAllSourceAddresses() {
-        StringBuilder sb = new StringBuilder();
-        synchronized (connections) {
-            for (JRTConnection conn : connections.values()) {
-                sb.append(conn.getAddress());
-                sb.append(",");
-            }
-        }
-        // Remove trailing ","
-        sb.deleteCharAt(sb.length() - 1);
-        return sb.toString();
-    }
-
     public String toString() {
         StringBuilder sb = new StringBuilder();
         synchronized (connections) {
diff --git a/config/src/main/java/com/yahoo/vespa/config/LZ4PayloadCompressor.java b/config/src/main/java/com/yahoo/vespa/config/LZ4PayloadCompressor.java
index 7a27902c89c..ba2caaf3e91 100644
--- a/config/src/main/java/com/yahoo/vespa/config/LZ4PayloadCompressor.java
+++ b/config/src/main/java/com/yahoo/vespa/config/LZ4PayloadCompressor.java
@@ -1,9 +1,9 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.config;
 
+import com.yahoo.compress.CompressionType;
+import com.yahoo.compress.Compressor;
 import com.yahoo.vespa.config.util.ConfigUtils;
-import net.jpountz.lz4.LZ4Compressor;
-import net.jpountz.lz4.LZ4Factory;
 
 /**
  * Wrapper for LZ4 compression that selects compression level based on properties.
@@ -12,9 +12,8 @@ import net.jpountz.lz4.LZ4Factory;
  */
 public class LZ4PayloadCompressor {
 
-    private static final LZ4Factory lz4Factory = LZ4Factory.safeInstance();
     private static final String VESPA_CONFIG_PROTOCOL_COMPRESSION_LEVEL = "VESPA_CONFIG_PROTOCOL_COMPRESSION_LEVEL";
-    private static final int compressionLevel = getCompressionLevel();
+    private static final Compressor compressor = new Compressor(CompressionType.LZ4, getCompressionLevel());
 
     private static int getCompressionLevel() {
         return Integer.parseInt(ConfigUtils.getEnvValue("0",
@@ -24,17 +23,11 @@ public class LZ4PayloadCompressor {
     }
 
     public byte[] compress(byte[] input) {
-        return getCompressor().compress(input);
+        return compressor.compressUnconditionally(input);
     }
 
-    public void decompress(byte[] input, byte[] outputbuffer) {
-        if (input.length > 0) {
-            lz4Factory.safeDecompressor().decompress(input, outputbuffer);
-        }
-    }
-
-    private LZ4Compressor getCompressor() {
-        return (compressionLevel < 7) ? lz4Factory.fastCompressor() : lz4Factory.highCompressor();
+    public byte [] decompress(byte[] input, int uncompressedLen) {
+        return compressor.decompressUnconditionally(input, 0, uncompressedLen);
     }
 
 }
diff --git a/config/src/main/java/com/yahoo/vespa/config/TimingValues.java b/config/src/main/java/com/yahoo/vespa/config/TimingValues.java
index 113d561bf87..5d5967e56c4 100644
--- a/config/src/main/java/com/yahoo/vespa/config/TimingValues.java
+++ b/config/src/main/java/com/yahoo/vespa/config/TimingValues.java
@@ -12,12 +12,11 @@ public class TimingValues {
     public static final long defaultNextConfigTimeout = 1000;
     // See getters below for an explanation of how these values are used and interpreted
     // All time values in milliseconds.
-    private long successTimeout = 600000;
-    private long errorTimeout = 20000;
-    private long initialTimeout = 15000;
+    private final long successTimeout;
+    private final long errorTimeout;
+    private final long initialTimeout;
     private long subscribeTimeout = 55000;
     private long configuredErrorTimeout = -1;  // Don't ever timeout (and do not use error response) when we are already configured
-    private long nextConfigTimeout = defaultNextConfigTimeout;
 
     private long fixedDelay = 5000;
     private long unconfiguredDelay = 1000;
@@ -26,6 +25,9 @@ public class TimingValues {
     private final Random rand;
 
     public TimingValues() {
+        successTimeout = 600000;
+        errorTimeout = 20000;
+        initialTimeout = 15000;
         this.rand = new Random(System.currentTimeMillis());
     }
 
@@ -38,7 +40,6 @@ public class TimingValues {
                         long configuredErrorDelay,
                         long fixedDelay,
                         int maxDelayMultiplier) {
-
         this.successTimeout = successTimeout;
         this.errorTimeout = errorTimeout;
         this.initialTimeout = initialTimeout;
@@ -51,15 +52,14 @@ public class TimingValues {
     }
 
     private TimingValues(long successTimeout,
-                        long errorTimeout,
-                        long initialTimeout,
-                        long subscribeTimeout,
-                        long unconfiguredDelay,
-                        long configuredErrorDelay,
-                        long fixedDelay,
-                        int maxDelayMultiplier,
-                        Random rand) {
-
+                         long errorTimeout,
+                         long initialTimeout,
+                         long subscribeTimeout,
+                         long unconfiguredDelay,
+                         long configuredErrorDelay,
+                         long fixedDelay,
+                         int maxDelayMultiplier,
+                         Random rand) {
         this.successTimeout = successTimeout;
         this.errorTimeout = errorTimeout;
         this.initialTimeout = initialTimeout;
@@ -71,18 +71,6 @@ public class TimingValues {
         this.rand = rand;
     }
 
-    public TimingValues(TimingValues tv) {
-        this(tv.successTimeout,
-                tv.errorTimeout,
-                tv.initialTimeout,
-                tv.subscribeTimeout,
-                tv.unconfiguredDelay,
-                tv.configuredErrorDelay,
-                tv.fixedDelay,
-                tv.maxDelayMultiplier,
-                tv.getRandom());
-    }
-
     public TimingValues(TimingValues tv, Random random) {
         this(tv.successTimeout,
                 tv.errorTimeout,
@@ -114,20 +102,6 @@ public class TimingValues {
     }
 
     /**
-     * Returns initial timeout to use as server timeout when a config is requested for the first time.
-     *
-     * @return timeout in milliseconds.
-     */
-    public long getInitialTimeout() {
-        return initialTimeout;
-    }
-
-    public TimingValues setInitialTimeout(long t) {
-        initialTimeout = t;
-        return this;
-    }
-
-    /**
      * Returns timeout to use as server timeout when subscribing for the first time.
      *
      * @return timeout in milliseconds.
@@ -141,38 +115,12 @@ public class TimingValues {
         return this;
     }
 
-    /**
-     * Returns the time to retry getting config from the remote sources, until the next error response will
-     * be set as config. Counted from the last ok request was received. A negative value means that
-     * we will always retry getting config and never set an error response as config.
-     *
-     * @return timeout in milliseconds.
-     */
-    public long getConfiguredErrorTimeout() {
-        return configuredErrorTimeout;
-    }
-
     public TimingValues setConfiguredErrorTimeout(long t) {
         configuredErrorTimeout = t;
         return this;
     }
 
     /**
-     * Returns timeout used when calling {@link com.yahoo.config.subscription.ConfigSubscriber#nextConfig()} or
-     * {@link com.yahoo.config.subscription.ConfigSubscriber#nextGeneration()}
-     *
-     * @return timeout in milliseconds.
-     */
-    public long getNextConfigTimeout() {
-        return nextConfigTimeout;
-    }
-
-    public TimingValues setNextConfigTimeout(long t) {
-        nextConfigTimeout = t;
-        return this;
-    }
-
-    /**
      * Returns time to wait until next attempt to get config after a failed request when the client has not
      * gotten a successful response to a config subscription (i.e, the client has not been configured).
      * A negative value means that there will never be a next attempt. If a negative value is set, the
@@ -215,12 +163,6 @@ public class TimingValues {
         return maxDelayMultiplier;
     }
 
-
-    public TimingValues setSuccessTimeout(long successTimeout) {
-        this.successTimeout = successTimeout;
-        return this;
-    }
-
     /**
      * Returns fixed delay that is used when retrying getting config no matter if it was a success or an error
      * and independent of number of retries.
@@ -242,10 +184,6 @@ public class TimingValues {
         return Math.round(val - (val * fraction) + (rand.nextFloat() * 2L * val * fraction));
     }
 
-    Random getRandom() {
-        return rand;
-    }
-
     @Override
     public String toString() {
         return "TimingValues [successTimeout=" + successTimeout
diff --git a/config/src/main/java/com/yahoo/vespa/config/UrlDownloader.java b/config/src/main/java/com/yahoo/vespa/config/UrlDownloader.java
index b7d9f9e14ca..863e95c5625 100644
--- a/config/src/main/java/com/yahoo/vespa/config/UrlDownloader.java
+++ b/config/src/main/java/com/yahoo/vespa/config/UrlDownloader.java
@@ -47,7 +47,7 @@ public class UrlDownloader {
                 Request request = new Request("frt.rpc.ping");
                 target.invokeSync(request, 5.0);
                 if (! request.isError()) {
-                    log.log(LogLevel.DEBUG, "Successfully connected to '" + spec + "', this = " + System.identityHashCode(this));
+                    log.log(LogLevel.DEBUG, () -> "Successfully connected to '" + spec + "', this = " + System.identityHashCode(this));
                     return;
                 } else {
                     target.close();
@@ -78,7 +78,7 @@ public class UrlDownloader {
             request.parameters().add(new StringValue(urlReference.value()));
 
             double rpcTimeout = Math.min(timeLeft, 60 * 60.0);
-            log.log(LogLevel.DEBUG, "InvokeSync waitFor " + urlReference + " with " + rpcTimeout + " seconds timeout");
+            log.log(LogLevel.DEBUG, () -> "InvokeSync waitFor " + urlReference + " with " + rpcTimeout + " seconds timeout");
             target.invokeSync(request, rpcTimeout);
 
             if (request.checkReturnTypes("s")) {
diff --git a/config/src/main/java/com/yahoo/vespa/config/benchmark/LoadTester.java b/config/src/main/java/com/yahoo/vespa/config/benchmark/LoadTester.java
index dcaa71dda25..f20371d203c 100644
--- a/config/src/main/java/com/yahoo/vespa/config/benchmark/LoadTester.java
+++ b/config/src/main/java/com/yahoo/vespa/config/benchmark/LoadTester.java
@@ -47,7 +47,6 @@ public class LoadTester {
     protected Supervisor supervisor = new Supervisor(transport);
     private List<ConfigKey<?>> configs = new ArrayList<>();
     private Map<ConfigDefinitionKey, Tuple2<String, String[]>> defs = new HashMap<>();
-    private long protocolVersion = Long.parseLong(JRTConfigRequestFactory.getProtocolVersion());
     private CompressionType compressionType = JRTConfigRequestFactory.getCompressionType();
 
     /**
@@ -261,13 +260,9 @@ public class LoadTester {
         private JRTClientConfigRequest getRequest(ConfigKey<?> reqKey, String[] defContent) {
             if (defContent == null) defContent = new String[0];
             final long serverTimeout = 1000;
-            if (protocolVersion == 3) {
-                return JRTClientConfigRequestV3.createWithParams(reqKey, DefContent.fromList(Arrays.asList(defContent)),
-                                                                 "unknown", "", 0, serverTimeout, Trace.createDummy(),
-                                                                 compressionType, Optional.empty());
-            } else {
-                throw new RuntimeException("Unsupported protocol version" + protocolVersion);
-            }
+            return JRTClientConfigRequestV3.createWithParams(reqKey, DefContent.fromList(Arrays.asList(defContent)),
+                                                             "unknown", "", 0, serverTimeout, Trace.createDummy(),
+                                                             compressionType, Optional.empty());
         }
 
         private Target connect(Spec spec) {
diff --git a/config/src/main/java/com/yahoo/vespa/config/buildergen/CompilationTask.java b/config/src/main/java/com/yahoo/vespa/config/buildergen/CompilationTask.java
index 7ac03ea6151..05125f8b08b 100644
--- a/config/src/main/java/com/yahoo/vespa/config/buildergen/CompilationTask.java
+++ b/config/src/main/java/com/yahoo/vespa/config/buildergen/CompilationTask.java
@@ -11,7 +11,6 @@ import javax.tools.JavaFileObject;
  * TODO: Assumes that diagnostics is the same as given to the task, not ideal.
  *
  * @author Ulf Lilleengen
- * @since 5.2
  */
 class CompilationTask {
     private final JavaCompiler.CompilationTask task;
diff --git a/config/src/main/java/com/yahoo/vespa/config/buildergen/CompiledBuilder.java b/config/src/main/java/com/yahoo/vespa/config/buildergen/CompiledBuilder.java
index 80909ab653d..fa5cf427aad 100644
--- a/config/src/main/java/com/yahoo/vespa/config/buildergen/CompiledBuilder.java
+++ b/config/src/main/java/com/yahoo/vespa/config/buildergen/CompiledBuilder.java
@@ -7,7 +7,6 @@ import com.yahoo.config.ConfigInstance;
  * Represents a builder that can be instantiated.
  *
  * @author Ulf Lilleengen
- * @since 5.2
  */
 public interface CompiledBuilder {
     <BUILDER extends ConfigInstance.Builder> BUILDER newInstance();
diff --git a/config/src/main/java/com/yahoo/vespa/config/buildergen/StringSourceObject.java b/config/src/main/java/com/yahoo/vespa/config/buildergen/StringSourceObject.java
index 1939a007059..c67d2121844 100644
--- a/config/src/main/java/com/yahoo/vespa/config/buildergen/StringSourceObject.java
+++ b/config/src/main/java/com/yahoo/vespa/config/buildergen/StringSourceObject.java
@@ -8,7 +8,6 @@ import java.net.URI;
  * Represents an in memory source object that can be compiled.
  *
  * @author Ulf Lilleengen
- * @since 5.2
  */
 class StringSourceObject extends SimpleJavaFileObject {
     private final String code;
diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/CompressionInfo.java b/config/src/main/java/com/yahoo/vespa/config/protocol/CompressionInfo.java
index ba9fa601c81..bc6f97a3e00 100644
--- a/config/src/main/java/com/yahoo/vespa/config/protocol/CompressionInfo.java
+++ b/config/src/main/java/com/yahoo/vespa/config/protocol/CompressionInfo.java
@@ -10,7 +10,6 @@ import java.io.IOException;
  * Contains info relevant for compression and decompression.
  *
  * @author Ulf Lilleengen
- * @since 5.19
  */
 public class CompressionInfo {
     private static final String COMPRESSION_TYPE = "compressionType";
diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/CompressionType.java b/config/src/main/java/com/yahoo/vespa/config/protocol/CompressionType.java
index c6548f63cd7..5fbd61cde3d 100644
--- a/config/src/main/java/com/yahoo/vespa/config/protocol/CompressionType.java
+++ b/config/src/main/java/com/yahoo/vespa/config/protocol/CompressionType.java
@@ -3,7 +3,6 @@ package com.yahoo.vespa.config.protocol;
 
 /**
  * @author Ulf Lilleengen
- * @since 5.18
  */
 public enum CompressionType {
     UNCOMPRESSED, LZ4;
diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/ConfigResponse.java b/config/src/main/java/com/yahoo/vespa/config/protocol/ConfigResponse.java
index c07be8337fe..bb0ee2bb935 100644
--- a/config/src/main/java/com/yahoo/vespa/config/protocol/ConfigResponse.java
+++ b/config/src/main/java/com/yahoo/vespa/config/protocol/ConfigResponse.java
@@ -5,7 +5,6 @@ import com.yahoo.text.Utf8Array;
 
 import java.io.IOException;
 import java.io.OutputStream;
-import java.util.List;
 
 /**
  * A config response encapsulates the payload and some meta information. This makes it possible to
@@ -19,8 +18,6 @@ public interface ConfigResponse {
 
     Utf8Array getPayload();
 
-    List<String> getLegacyPayload();
-
     long getGeneration();
 
     boolean isInternalRedeploy();
diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/DefContent.java b/config/src/main/java/com/yahoo/vespa/config/protocol/DefContent.java
index ea2cf1a1bd8..3aa1898fcd1 100644
--- a/config/src/main/java/com/yahoo/vespa/config/protocol/DefContent.java
+++ b/config/src/main/java/com/yahoo/vespa/config/protocol/DefContent.java
@@ -12,7 +12,6 @@ import java.util.List;
 
 /**
 * @author Ulf Lilleengen
-* @since 5.3
 */
 public class DefContent {
     private final List<String> data;
@@ -35,12 +34,7 @@ public class DefContent {
 
     static DefContent fromSlime(Inspector data) {
         final List<String> lst = new ArrayList<>();
-        data.traverse(new ArrayTraverser() {
-            @Override
-            public void entry(int idx, Inspector inspector) {
-                lst.add(inspector.asString());
-            }
-        });
+        data.traverse((ArrayTraverser) (idx, inspector) -> lst.add(inspector.asString()));
         return new DefContent(lst);
     }
 
diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/JRTClientConfigRequest.java b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTClientConfigRequest.java
index 25fecba425c..ab47fec0641 100644
--- a/config/src/main/java/com/yahoo/vespa/config/protocol/JRTClientConfigRequest.java
+++ b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTClientConfigRequest.java
@@ -64,14 +64,6 @@ public interface JRTClientConfigRequest extends JRTConfigRequest {
     String getNewConfigMd5();
 
     /**
-     * Test whether or not the payload is contained in this response or not.
-     * Should return false for error responses as well.
-     *
-     * @return true if empty, false if not.
-     */
-    boolean containsPayload();
-
-    /**
      * Test whether or not the response contains an updated config or not.
      * False if no response has been returned.
      *
diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/JRTClientConfigRequestV3.java b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTClientConfigRequestV3.java
index 1c842a4d1b0..12e5968ab83 100644
--- a/config/src/main/java/com/yahoo/vespa/config/protocol/JRTClientConfigRequestV3.java
+++ b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTClientConfigRequestV3.java
@@ -5,13 +5,20 @@ import com.yahoo.config.ConfigInstance;
 import com.yahoo.config.subscription.impl.ConfigSubscription;
 import com.yahoo.config.subscription.impl.JRTConfigSubscription;
 import com.yahoo.jrt.Request;
+import com.yahoo.jrt.StringValue;
+import com.yahoo.slime.JsonFormat;
+import com.yahoo.slime.Slime;
+import com.yahoo.text.Utf8;
 import com.yahoo.text.Utf8Array;
 import com.yahoo.vespa.config.ConfigKey;
 import com.yahoo.vespa.config.JRTMethods;
 import com.yahoo.vespa.config.RawConfig;
 import com.yahoo.vespa.config.util.ConfigUtils;
 
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
 import java.util.Optional;
+import java.util.logging.Logger;
 
 /**
  * Represents version 3 config request for config clients. Provides methods for inspecting request and response
@@ -21,7 +28,12 @@ import java.util.Optional;
  *
  * @author Ulf Lilleengen
  */
-public class JRTClientConfigRequestV3 extends SlimeClientConfigRequest {
+public class JRTClientConfigRequestV3 implements JRTClientConfigRequest {
+
+    protected static final Logger log = Logger.getLogger(JRTClientConfigRequestV3.class.getName());
+    protected final SlimeRequestData requestData;
+    protected final Request request;
+    private final SlimeResponseData responseData;
 
     protected JRTClientConfigRequestV3(ConfigKey<?> key,
                                        String hostname,
@@ -32,15 +44,38 @@ public class JRTClientConfigRequestV3 extends SlimeClientConfigRequest {
                                        Trace trace,
                                        CompressionType compressionType,
                                        Optional<VespaVersion> vespaVersion) {
-        super(key, hostname, defSchema, configMd5, generation, timeout, trace, compressionType, vespaVersion);
+        Slime data = SlimeRequestData.encodeRequest(key,
+                                                    hostname,
+                                                    defSchema,
+                                                    configMd5,
+                                                    generation,
+                                                    timeout,
+                                                    trace,
+                                                    getProtocolVersion(),
+                                                    compressionType,
+                                                    vespaVersion);
+        Request jrtReq = new Request(getJRTMethodName());
+        jrtReq.parameters().add(new StringValue(encodeAsUtf8String(data)));
+
+        this.requestData = new SlimeRequestData(jrtReq, data);
+        this.responseData = new SlimeResponseData(jrtReq);
+        this.request = jrtReq;
+    }
+
+    protected static String encodeAsUtf8String(Slime data) {
+        ByteArrayOutputStream baos = new NoCopyByteArrayOutputStream();
+        try {
+            new JsonFormat(true /* compact format */).encode(baos, data);
+        } catch (IOException e) {
+            throw new RuntimeException("Unable to encode config request", e);
+        }
+        return Utf8.toString(baos.toByteArray());
     }
 
-    @Override
     protected String getJRTMethodName() {
         return JRTMethods.configV3getConfigMethodName;
     }
 
-    @Override
     protected boolean checkReturnTypes(Request request) {
         return JRTMethods.checkV3ReturnTypes(request);
     }
@@ -74,22 +109,19 @@ public class JRTClientConfigRequestV3 extends SlimeClientConfigRequest {
                                                                                   Trace trace,
                                                                                   CompressionType compressionType,
                                                                                   Optional<VespaVersion> vespaVersion) {
-        String hostname = ConfigUtils.getCanonicalHostName();
-        ConfigKey<T> key = sub.getKey();
         ConfigSubscription.ConfigState<T> configState = sub.getConfigState();
-        T i = configState.getConfig();
-        return createWithParams(key,
-                sub.getDefContent(),
-                hostname,
-                i != null ? i.getConfigMd5() : "",
-                configState.getGeneration() != null ? configState.getGeneration() : 0L,
-                sub.timingValues().getSubscribeTimeout(),
-                trace,
-                compressionType,
-                vespaVersion);
+        T config = configState.getConfig();
+        return createWithParams(sub.getKey(),
+                                sub.getDefContent(),
+                                ConfigUtils.getCanonicalHostName(),
+                                config != null ? config.getConfigMd5() : "",
+                                configState.getGeneration(),
+                                sub.timingValues().getSubscribeTimeout(),
+                                trace,
+                                compressionType,
+                                vespaVersion);
     }
 
-
     public static JRTClientConfigRequest createFromRaw(RawConfig config,
                                                        long serverTimeout,
                                                        Trace trace,
@@ -107,7 +139,6 @@ public class JRTClientConfigRequestV3 extends SlimeClientConfigRequest {
                 vespaVersion);
     }
 
-
     public static JRTClientConfigRequest createWithParams(ConfigKey<?> reqKey,
                                                           DefContent defContent,
                                                           String hostname,
@@ -133,4 +164,142 @@ public class JRTClientConfigRequestV3 extends SlimeClientConfigRequest {
         return requestData.getVespaVersion();
     }
 
+    public ConfigKey<?> getConfigKey() {
+        return requestData.getConfigKey();
+    }
+
+    @Override
+    public String toString() {
+        StringBuilder sb = new StringBuilder();
+        sb.append("request='").append(getConfigKey())
+                .append(",").append(getClientHostName())
+                .append(",").append(getRequestConfigMd5())
+                .append(",").append(getRequestGeneration())
+                .append(",").append(getTimeout())
+                .append(",").append(getVespaVersion().map(VespaVersion::toString).orElse(""))
+                .append("'\n");
+        sb.append("response='").append(getNewConfigMd5())
+                .append(",").append(getNewGeneration())
+                .append(",").append(responseIsInternalRedeploy())
+                .append("'\n");
+        return sb.toString();
+    }
+
+    @Override
+    public String getClientHostName() {
+        return requestData.getClientHostName();
+    }
+
+    @Override
+    public Request getRequest() {
+        return request;
+    }
+
+    @Override
+    public int errorCode() {
+        return request.errorCode();
+    }
+
+    @Override
+    public String errorMessage() {
+        return request.errorMessage();
+    }
+
+    @Override
+    public String getShortDescription() {
+        return toString();
+    }
+
+    @Override
+    public boolean hasUpdatedGeneration() {
+        long prevGen = getRequestGeneration();
+        long newGen = getNewGeneration();
+        return ConfigUtils.isGenerationNewer(newGen, prevGen);
+    }
+
+    @Override
+    public long getTimeout() {
+        return requestData.getTimeout();
+    }
+
+    protected String newConfMd5() {
+        String newMd5 = getNewConfigMd5();
+        if ("".equals(newMd5)) return getRequestConfigMd5();
+        return newMd5;
+    }
+
+    protected long newGen() {
+        long newGen = getNewGeneration();
+        if (newGen == 0) return getRequestGeneration();
+        return newGen;
+    }
+
+    @Override
+    public DefContent getDefContent() {
+        return requestData.getSchema();
+    }
+
+    @Override
+    public boolean isError() {
+        return request.isError();
+    }
+
+    @Override
+    public boolean hasUpdatedConfig() {
+        String respMd5 = getNewConfigMd5();
+        return !respMd5.equals("") && !getRequestConfigMd5().equals(respMd5);
+    }
+
+    @Override
+    public Trace getResponseTrace() {
+        return responseData.getResponseTrace();
+    }
+
+    @Override
+    public String getRequestConfigMd5() {
+        return requestData.getRequestConfigMd5();
+    }
+
+    @Override
+    public boolean validateResponse() {
+        if (request.isError()) {
+            return false;
+        } else if (request.returnValues().size() == 0) {
+            return false;
+        } else if (!checkReturnTypes(request)) {
+            log.warning("Invalid return types for config response: " + errorMessage());
+            return false;
+        }
+        return true;
+    }
+
+    @Override
+    public boolean validateParameters() {
+        int errorCode = RequestValidation.validateRequest(this);
+        return (errorCode == 0);
+    }
+
+    @Override
+    public String getNewConfigMd5() {
+        return responseData.getResponseConfigMd5();
+    }
+
+    @Override
+    public long getNewGeneration() {
+        return responseData.getResponseConfigGeneration();
+    }
+
+    @Override
+    public boolean responseIsInternalRedeploy() {
+        return responseData.getResponseInternalRedeployment();
+    }
+
+    @Override
+    public long getRequestGeneration() {
+        return requestData.getRequestGeneration();
+    }
+
+    protected SlimeResponseData getResponseData() {
+        return responseData;
+    }
 }
diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/JRTConfigRequestFactory.java b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTConfigRequestFactory.java
index 9dec28b13e0..b6277f750dc 100644
--- a/config/src/main/java/com/yahoo/vespa/config/protocol/JRTConfigRequestFactory.java
+++ b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTConfigRequestFactory.java
@@ -6,20 +6,17 @@ import com.yahoo.config.subscription.impl.JRTConfigSubscription;
 import com.yahoo.vespa.config.RawConfig;
 import com.yahoo.vespa.config.util.ConfigUtils;
 
-import java.util.*;
+import java.util.Optional;
 
 /**
  * To hide JRT implementations.
  *
  * @author Ulf Lilleengen
- * @since 5.3
  */
 public class JRTConfigRequestFactory {
 
-    public static final String VESPA_CONFIG_PROTOCOL_VERSION = "VESPA_CONFIG_PROTOCOL_VERSION"; // Unused, but should be used if we add a new version
     private static final CompressionType compressionType = getCompressionType();
     private static final String VESPA_CONFIG_PROTOCOL_COMPRESSION = "VESPA_CONFIG_PROTOCOL_COMPRESSION";
-    public static final String VESPA_VERSION = "VESPA_VERSION";
 
     public static <T extends ConfigInstance> JRTClientConfigRequest createFromSub(JRTConfigSubscription<T> sub) {
         // TODO: Get trace from caller
@@ -31,18 +28,6 @@ public class JRTConfigRequestFactory {
         return JRTClientConfigRequestV3.createFromRaw(config, serverTimeout, Trace.createNew(), compressionType, getVespaVersion());
     }
 
-    public static String getProtocolVersion() {
-        return "3";
-    }
-
-    static String getProtocolVersion(String env, String alternateEnv, String property) {
-        return ConfigUtils.getEnvValue("3", env, alternateEnv, property);
-    }
-
-    public static Set<Long> supportedProtocolVersions() {
-        return Collections.singleton(3L);
-    }
-
     public static CompressionType getCompressionType() {
         return getCompressionType(System.getenv(VESPA_CONFIG_PROTOCOL_COMPRESSION),
                 System.getenv("services__config_protocol_compression"),
@@ -54,10 +39,6 @@ public class JRTConfigRequestFactory {
     }
 
     static Optional<VespaVersion> getVespaVersion() {
-        final String envValue = ConfigUtils.getEnvValue("", System.getenv(VESPA_VERSION), System.getProperty(VESPA_VERSION));
-        if (envValue != null && !envValue.isEmpty()) {
-            return Optional.of(VespaVersion.fromString(envValue));
-        }
         return Optional.of(getCompiledVespaVersion());
     }
 
diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/JRTServerConfigRequestV3.java b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTServerConfigRequestV3.java
index f70ebf39a28..3609ba04424 100644
--- a/config/src/main/java/com/yahoo/vespa/config/protocol/JRTServerConfigRequestV3.java
+++ b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTServerConfigRequestV3.java
@@ -1,17 +1,24 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.config.protocol;
 
+import com.fasterxml.jackson.core.JsonFactory;
 import com.fasterxml.jackson.core.JsonGenerator;
 import com.yahoo.jrt.DataValue;
 import com.yahoo.jrt.Request;
-import com.yahoo.log.LogLevel;
+import com.yahoo.jrt.StringValue;
+import com.yahoo.jrt.Value;
+import com.yahoo.text.Utf8Array;
+import com.yahoo.vespa.config.ConfigKey;
+import com.yahoo.vespa.config.ErrorCode;
 import com.yahoo.vespa.config.util.ConfigUtils;
 
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
+import java.util.Optional;
+import java.util.logging.Logger;
 
 /**
- * The V3 config protocol implemented on the server side. The V3 protocol uses 2 fields JRT
+ * The V3 config protocol implemented on the server side. The V3 protocol uses 2 fields:
  *
  * * A metadata field containing json data describing config generation, md5 and compression info
  * * A data field containing compressed or uncompressed json config payload. This field can be empty if the payload
@@ -22,14 +29,41 @@ import java.io.IOException;
  *
  * @author Ulf Lilleengen
  */
-// TODO: Merge with parent
-public class JRTServerConfigRequestV3 extends SlimeServerConfigRequest {
+public class JRTServerConfigRequestV3 implements JRTServerConfigRequest {
 
+    protected static final Logger log = Logger.getLogger(JRTServerConfigRequestV3.class.getName());
+    private static final JsonFactory jsonFactory = new JsonFactory();
+    protected final Request request;
+    private final SlimeRequestData requestData;
     /** Response field */
     private boolean internalRedeploy = false;
+    // Response values
+    private boolean isDelayed = false;
+    private Trace requestTrace = null;
 
     protected JRTServerConfigRequestV3(Request request) {
-        super(request);
+        this.requestData = new SlimeRequestData(request);
+        this.request = request;
+    }
+
+    protected static JsonGenerator createJsonGenerator(ByteArrayOutputStream byteArrayOutputStream) throws IOException {
+        return jsonFactory.createGenerator(byteArrayOutputStream);
+    }
+
+    protected static Value createResponseValue(ByteArrayOutputStream byteArrayOutputStream) {
+        return new StringValue(new Utf8Array(byteArrayOutputStream.toByteArray()));
+    }
+
+    protected static void setResponseField(JsonGenerator jsonGenerator, String fieldName, String value) throws IOException {
+        jsonGenerator.writeStringField(fieldName, value);
+    }
+
+    protected static void setResponseField(JsonGenerator jsonGenerator, String fieldName, long value) throws IOException {
+        jsonGenerator.writeNumberField(fieldName, value);
+    }
+
+    protected static void setResponseField(JsonGenerator jsonGenerator, String fieldName, boolean value) throws IOException {
+        jsonGenerator.writeBooleanField(fieldName, value);
     }
 
     @Override
@@ -57,9 +91,7 @@ public class JRTServerConfigRequestV3 extends SlimeServerConfigRequest {
             }
             compressionInfo.serialize(jsonGenerator);
             jsonGenerator.writeEndObject();
-            if (log.isLoggable(LogLevel.SPAM)) {
-                log.log(LogLevel.SPAM, getConfigKey() + ": response dataXXXXX" + payload.withCompression(CompressionType.UNCOMPRESSED) + "XXXXX");
-            }
+
             jsonGenerator.writeEndObject();
             jsonGenerator.close();
         } catch (IOException e) {
@@ -85,4 +117,145 @@ public class JRTServerConfigRequestV3 extends SlimeServerConfigRequest {
         return new JRTServerConfigRequestV3(req);
     }
 
+    @Override
+    public ConfigKey<?> getConfigKey() {
+        return requestData.getConfigKey();
+    }
+
+    @Override
+    public DefContent getDefContent() {
+        return getSchema();
+    }
+
+    @Override
+    public boolean noCache() {
+        return false;
+    }
+
+    @Override
+    public String toString() {
+        StringBuilder sb = new StringBuilder();
+        sb.append("request='").append(getConfigKey())
+                .append(",").append(getClientHostName())
+                .append(",").append(getRequestConfigMd5())
+                .append(",").append(getRequestGeneration())
+                .append(",").append(getTimeout()).append("'\n");
+        return sb.toString();
+    }
+
+    @Override
+    public Payload payloadFromResponse(ConfigResponse response) {
+        return Payload.from(response.getPayload(), response.getCompressionInfo());
+    }
+
+    private DefContent getSchema() {
+        return requestData.getSchema();
+    }
+
+    @Override
+    public String getClientHostName() {
+        return requestData.getClientHostName();
+    }
+
+    public Trace getRequestTrace() {
+        if (requestTrace == null) {
+            requestTrace = requestData.getRequestTrace();
+        }
+        return requestTrace;
+    }
+
+    @Override
+    public Request getRequest() {
+        return request;
+    }
+
+    @Override
+    public boolean validateParameters() {
+        int errorCode = RequestValidation.validateRequest(this);
+        if (errorCode != 0) {
+            addErrorResponse(errorCode);
+        }
+        return (errorCode == 0);
+    }
+
+    @Override
+    public String getRequestConfigMd5() {
+        return requestData.getRequestConfigMd5();
+    }
+
+    private void addErrorResponse(int errorCode) {
+        addErrorResponse(errorCode, ErrorCode.getName(errorCode));
+    }
+
+    @Override
+    public void setDelayedResponse(boolean delayedResponse) {
+        this.isDelayed = delayedResponse;
+    }
+
+    @Override
+    public void addErrorResponse(int errorCode, String name) {
+        ByteArrayOutputStream byteArrayOutputStream = new NoCopyByteArrayOutputStream();
+        try {
+            JsonGenerator jsonWriter = jsonFactory.createGenerator(byteArrayOutputStream);
+            jsonWriter.writeStartObject();
+            addCommonReturnValues(jsonWriter);
+            jsonWriter.writeEndObject();
+            jsonWriter.close();
+        } catch (IOException e) {
+            throw new IllegalArgumentException("Could not add error response for " + this);
+        }
+        request.setError(errorCode, name);
+        request.returnValues().add(createResponseValue(byteArrayOutputStream));
+    }
+
+    protected void addCommonReturnValues(JsonGenerator jsonGenerator) throws IOException {
+        ConfigKey<?> key = requestData.getConfigKey();
+        setResponseField(jsonGenerator, SlimeResponseData.RESPONSE_VERSION, getProtocolVersion());
+        setResponseField(jsonGenerator, SlimeResponseData.RESPONSE_DEF_NAME, key.getName());
+        setResponseField(jsonGenerator, SlimeResponseData.RESPONSE_DEF_NAMESPACE, key.getNamespace());
+        setResponseField(jsonGenerator, SlimeResponseData.RESPONSE_DEF_MD5, key.getMd5());
+        setResponseField(jsonGenerator, SlimeResponseData.RESPONSE_CONFIGID, key.getConfigId());
+        setResponseField(jsonGenerator, SlimeResponseData.RESPONSE_CLIENT_HOSTNAME, requestData.getClientHostName());
+        jsonGenerator.writeFieldName(SlimeResponseData.RESPONSE_TRACE);
+        jsonGenerator.writeRawValue(getRequestTrace().toString(true));
+    }
+
+    @Override
+    public long getRequestGeneration() {
+        return requestData.getRequestGeneration();
+    }
+
+    @Override
+    public boolean isDelayedResponse() {
+        return isDelayed;
+    }
+
+    @Override
+    public int errorCode() {
+        return request.errorCode();
+    }
+
+    @Override
+    public String errorMessage() {
+        return request.errorMessage();
+    }
+
+    @Override
+    public String getShortDescription() {
+        return toString();
+    }
+
+    protected CompressionType getCompressionType() {
+        return requestData.getCompressionType();
+    }
+
+    @Override
+    public long getTimeout() {
+        return requestData.getTimeout();
+    }
+
+    @Override
+    public Optional<VespaVersion> getVespaVersion() {
+        return requestData.getVespaVersion();
+    }
 }
diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/NoCopyByteArrayOutputStream.java b/config/src/main/java/com/yahoo/vespa/config/protocol/NoCopyByteArrayOutputStream.java
index 79746127bcd..61a742ade3b 100644
--- a/config/src/main/java/com/yahoo/vespa/config/protocol/NoCopyByteArrayOutputStream.java
+++ b/config/src/main/java/com/yahoo/vespa/config/protocol/NoCopyByteArrayOutputStream.java
@@ -7,7 +7,6 @@ import java.io.ByteArrayOutputStream;
  * Subclass of {@link java.io.ByteArrayOutputStream} that gives effective {@link #toByteArray()} method.
  *
  * @author Ulf Lilleengen
- * @since 5.19
  */
 class NoCopyByteArrayOutputStream extends ByteArrayOutputStream {
     public NoCopyByteArrayOutputStream() {
diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/Payload.java b/config/src/main/java/com/yahoo/vespa/config/protocol/Payload.java
index 88e8b21347f..13f9602f70c 100644
--- a/config/src/main/java/com/yahoo/vespa/config/protocol/Payload.java
+++ b/config/src/main/java/com/yahoo/vespa/config/protocol/Payload.java
@@ -60,8 +60,7 @@ public class Payload {
     public Payload withCompression(CompressionType requestedCompression) {
         CompressionType responseCompression = compressionInfo.getCompressionType();
         if (requestedCompression == CompressionType.UNCOMPRESSED && responseCompression == CompressionType.LZ4) {
-            byte[] buffer = new byte[compressionInfo.getUncompressedSize()];
-            compressor.decompress(data.getBytes(), buffer);
+            byte[] buffer = compressor.decompress(data.getBytes(), compressionInfo.getUncompressedSize());
             Utf8Array data = new Utf8Array(buffer);
             CompressionInfo info = CompressionInfo.create(CompressionType.UNCOMPRESSED, compressionInfo.getUncompressedSize());
             return Payload.from(data, info);
diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/RequestValidation.java b/config/src/main/java/com/yahoo/vespa/config/protocol/RequestValidation.java
index fed9a96504d..8c7c7f2703e 100644
--- a/config/src/main/java/com/yahoo/vespa/config/protocol/RequestValidation.java
+++ b/config/src/main/java/com/yahoo/vespa/config/protocol/RequestValidation.java
@@ -14,7 +14,6 @@ import java.util.regex.Pattern;
  * Static utility methods for verifying common request properties.
  *
  * @author Ulf Lilleengen
- * @since 5.3
  */
 public class RequestValidation {
     private static final Logger log = Logger.getLogger(RequestValidation.class.getName());
diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeClientConfigRequest.java b/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeClientConfigRequest.java
deleted file mode 100644
index 3ccf71ec519..00000000000
--- a/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeClientConfigRequest.java
+++ /dev/null
@@ -1,221 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.config.protocol;
-
-import com.yahoo.jrt.Request;
-import com.yahoo.jrt.StringValue;
-import com.yahoo.slime.JsonFormat;
-import com.yahoo.slime.Slime;
-import com.yahoo.text.Utf8;
-import com.yahoo.vespa.config.ConfigKey;
-import com.yahoo.vespa.config.util.ConfigUtils;
-
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.util.Optional;
-import java.util.logging.Logger;
-
-/**
- * Base class for new generation of config requests based on {@link Slime}. Allows for some customization of
- * payload encoding and decoding, as well as adding extra request/response fields.
- *
- * @author Ulf Lilleengen
- */
-public abstract class SlimeClientConfigRequest implements JRTClientConfigRequest {
-
-    protected static final Logger log = Logger.getLogger(SlimeClientConfigRequest.class.getName());
-
-    protected final SlimeRequestData requestData;
-    private final SlimeResponseData responseData;
-
-    protected final Request request;
-
-    protected SlimeClientConfigRequest(ConfigKey<?> key,
-                                       String hostname,
-                                       DefContent defSchema,
-                                       String configMd5,
-                                       long generation,
-                                       long timeout,
-                                       Trace trace,
-                                       CompressionType compressionType,
-                                       Optional<VespaVersion> vespaVersion) {
-        Slime data = SlimeRequestData.encodeRequest(key,
-                hostname,
-                defSchema,
-                configMd5,
-                generation,
-                timeout,
-                trace,
-                getProtocolVersion(),
-                compressionType,
-                vespaVersion);
-        Request jrtReq = new Request(getJRTMethodName());
-        jrtReq.parameters().add(new StringValue(encodeAsUtf8String(data, true)));
-
-        this.requestData = new SlimeRequestData(jrtReq, data);
-        this.responseData = new SlimeResponseData(jrtReq);
-        this.request = jrtReq;
-    }
-
-    protected abstract String getJRTMethodName();
-
-    protected static String encodeAsUtf8String(Slime data, boolean compact) {
-        ByteArrayOutputStream baos = new NoCopyByteArrayOutputStream();
-        try {
-            new JsonFormat(compact).encode(baos, data);
-        } catch (IOException e) {
-            throw new RuntimeException("Unable to encode config request", e);
-        }
-        return Utf8.toString(baos.toByteArray());
-    }
-
-    public ConfigKey<?> getConfigKey() {
-        return requestData.getConfigKey();
-    }
-
-    @Override
-    public String toString() {
-        StringBuilder sb = new StringBuilder();
-        sb.append("request='").append(getConfigKey())
-                .append(",").append(getClientHostName())
-                .append(",").append(getRequestConfigMd5())
-                .append(",").append(getRequestGeneration())
-                .append(",").append(getTimeout())
-                .append(",").append(getVespaVersion().map(VespaVersion::toString).orElse(""))
-                .append("'\n");
-        sb.append("response='").append(getNewConfigMd5())
-                .append(",").append(getNewGeneration())
-                .append(",").append(responseIsInternalRedeploy())
-                .append("'\n");
-        return sb.toString();
-    }
-
-    @Override
-    public String getClientHostName() {
-        return requestData.getClientHostName();
-    }
-
-    @Override
-    public Request getRequest() {
-        return request;
-    }
-
-    @Override
-    public int errorCode() {
-        return request.errorCode();
-    }
-
-    @Override
-    public String errorMessage() {
-        return request.errorMessage();
-    }
-
-    @Override
-    public String getShortDescription() {
-        return toString();
-    }
-
-    @Override
-    public boolean hasUpdatedGeneration() {
-        long prevGen = getRequestGeneration();
-        long newGen = getNewGeneration();
-        return ConfigUtils.isGenerationNewer(newGen, prevGen);
-    }
-
-    @Override
-    public long getTimeout() {
-        return requestData.getTimeout();
-    }
-
-    protected String newConfMd5() {
-        String newMd5 = getNewConfigMd5();
-        if ("".equals(newMd5)) return getRequestConfigMd5();
-        return newMd5;
-    }
-
-    protected long newGen() {
-        long newGen = getNewGeneration();
-        if (newGen==0) return getRequestGeneration();
-        return newGen;
-    }
-
-    @Override
-    public DefContent getDefContent() {
-        return requestData.getSchema();
-    }
-
-    @Override
-    public boolean isError() {
-        return request.isError();
-    }
-
-    @Override
-    public boolean containsPayload() {
-        return false;
-    }
-
-    @Override
-    public boolean hasUpdatedConfig() {
-        String respMd5 = getNewConfigMd5();
-        return !respMd5.equals("") && !getRequestConfigMd5().equals(respMd5);
-    }
-
-    @Override
-    public Trace getResponseTrace() {
-        return responseData.getResponseTrace();
-    }
-
-    @Override
-    public String getRequestConfigMd5() {
-        return requestData.getRequestConfigMd5();
-    }
-
-    @Override
-    public boolean validateResponse() {
-        if (request.isError()) {
-            return false;
-        } else if (request.returnValues().size() == 0) {
-            return false;
-        } else if (!checkReturnTypes(request)) {
-            log.warning("Invalid return types for config response: " + errorMessage());
-            return false;
-        }
-        return true;
-    }
-
-    @Override
-    public boolean validateParameters() {
-        int errorCode = RequestValidation.validateRequest(this);
-        return (errorCode == 0);
-    }
-
-    protected abstract boolean checkReturnTypes(Request request);
-
-    @Override
-    public String getNewConfigMd5() {
-        return responseData.getResponseConfigMd5();
-    }
-
-    @Override
-    public long getNewGeneration() {
-        return responseData.getResponseConfigGeneration();
-    }
-
-    @Override
-    public boolean responseIsInternalRedeploy() {
-        return responseData.getResponseInternalRedeployment();
-    }
-
-    @Override
-    public long getRequestGeneration() {
-        return requestData.getRequestGeneration();
-    }
-
-    protected SlimeResponseData getResponseData() {
-        return responseData;
-    }
-
-    public Optional<VespaVersion> getVespaVersion() {
-        return requestData.getVespaVersion();
-    }
-
-}
diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeConfigResponse.java b/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeConfigResponse.java
index 327acab53d3..ff0b7f964bf 100644
--- a/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeConfigResponse.java
+++ b/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeConfigResponse.java
@@ -1,17 +1,11 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.config.protocol;
 
-import com.yahoo.config.codegen.InnerCNode;
 import com.yahoo.text.Utf8Array;
-import com.yahoo.vespa.config.ConfigFileFormat;
 import com.yahoo.vespa.config.ConfigPayload;
 
-import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.OutputStream;
-import java.nio.charset.StandardCharsets;
-import java.util.Arrays;
-import java.util.List;
 
 /**
  * Class for serializing config responses based on {@link com.yahoo.slime.Slime} implementing the {@link ConfigResponse} interface.
@@ -22,27 +16,24 @@ public class SlimeConfigResponse implements ConfigResponse {
 
     private final Utf8Array payload;
     private final CompressionInfo compressionInfo;
-    private final InnerCNode targetDef;
     private final long generation;
     private final boolean internalRedeploy;
     private final String configMd5;
 
-    public static SlimeConfigResponse fromConfigPayload(ConfigPayload payload, InnerCNode targetDef, long generation,
+    public static SlimeConfigResponse fromConfigPayload(ConfigPayload payload, long generation,
                                                         boolean internalRedeploy, String configMd5) {
         Utf8Array data = payload.toUtf8Array(true);
-        return new SlimeConfigResponse(data, targetDef, generation, internalRedeploy,
+        return new SlimeConfigResponse(data, generation, internalRedeploy,
                                        configMd5,
                                        CompressionInfo.create(CompressionType.UNCOMPRESSED, data.getByteLength()));
     }
 
     public SlimeConfigResponse(Utf8Array payload,
-                               InnerCNode targetDef,
                                long generation,
                                boolean internalRedeploy,
                                String configMd5,
                                CompressionInfo compressionInfo) {
         this.payload = payload;
-        this.targetDef = targetDef;
         this.generation = generation;
         this.internalRedeploy = internalRedeploy;
         this.configMd5 = configMd5;
@@ -55,19 +46,6 @@ public class SlimeConfigResponse implements ConfigResponse {
     }
 
     @Override
-    public List<String> getLegacyPayload() {
-        ByteArrayOutputStream baos = new ByteArrayOutputStream();
-        ConfigFileFormat format = new ConfigFileFormat(targetDef);
-        Payload v1payload = Payload.from(payload, compressionInfo).withCompression(CompressionType.UNCOMPRESSED);
-        try {
-            ConfigPayload.fromUtf8Array(v1payload.getData()).serialize(baos, format);
-            return Arrays.asList(baos.toString(StandardCharsets.UTF_8).split("\\n"));
-        } catch (IOException e) {
-            throw new RuntimeException(e);
-        }
-    }
-
-    @Override
     public long getGeneration() {
         return generation;
     }
diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeServerConfigRequest.java b/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeServerConfigRequest.java
deleted file mode 100644
index 34d6f90cbcb..00000000000
--- a/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeServerConfigRequest.java
+++ /dev/null
@@ -1,204 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.config.protocol;
-
-import com.fasterxml.jackson.core.JsonFactory;
-import com.fasterxml.jackson.core.JsonGenerator;
-import com.yahoo.jrt.*;
-import com.yahoo.slime.*;
-import com.yahoo.text.Utf8Array;
-import com.yahoo.vespa.config.*;
-import com.yahoo.vespa.config.ErrorCode;
-
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.util.Optional;
-import java.util.logging.Logger;
-
-/**
- * Base class for new generation of config requests based on {@link Slime}. Allows for some customization of
- * payload encoding and decoding, as well as adding extra request/response fields. Used by both V2 and V3
- * config protocol.
- *
- * @author Ulf Lilleengen
- */
-abstract class SlimeServerConfigRequest implements JRTServerConfigRequest {
-
-    protected static final Logger log = Logger.getLogger(SlimeServerConfigRequest.class.getName());
-
-    private static final JsonFactory jsonFactory = new JsonFactory();
-
-    private final SlimeRequestData requestData;
-
-    // Response values
-    private boolean isDelayed = false;
-    private Trace requestTrace = null;
-    protected final Request request;
-
-    protected SlimeServerConfigRequest(Request request) {
-        this.requestData = new SlimeRequestData(request);
-        this.request = request;
-    }
-
-    protected static JsonGenerator createJsonGenerator(ByteArrayOutputStream byteArrayOutputStream) throws IOException {
-        return jsonFactory.createGenerator(byteArrayOutputStream);
-    }
-
-    @Override
-    public ConfigKey<?> getConfigKey() {
-        return requestData.getConfigKey();
-    }
-
-    @Override
-    public DefContent getDefContent() {
-        return getSchema();
-    }
-
-    @Override
-    public boolean noCache() {
-        return false;
-    }
-
-    @Override
-    public String toString() {
-        StringBuilder sb = new StringBuilder();
-        sb.append("request='").append(getConfigKey())
-                .append(",").append(getClientHostName())
-                .append(",").append(getRequestConfigMd5())
-                .append(",").append(getRequestGeneration())
-                .append(",").append(getTimeout()).append("'\n");
-        return sb.toString();
-    }
-
-    @Override
-    public Payload payloadFromResponse(ConfigResponse response) {
-        return Payload.from(response.getPayload(), response.getCompressionInfo());
-    }
-
-    private DefContent getSchema() {
-        return requestData.getSchema();
-    }
-
-    @Override
-    public String getClientHostName() {
-        return requestData.getClientHostName();
-    }
-
-    public Trace getRequestTrace() {
-        if (requestTrace == null) {
-            requestTrace = requestData.getRequestTrace();
-        }
-        return requestTrace;
-    }
-
-    @Override
-    public Request getRequest() {
-        return request;
-    }
-
-    @Override
-    public boolean validateParameters() {
-        int errorCode = RequestValidation.validateRequest(this);
-        if (errorCode != 0) {
-            addErrorResponse(errorCode);
-        }
-        return (errorCode == 0);
-    }
-
-    @Override
-    public String getRequestConfigMd5() {
-        return requestData.getRequestConfigMd5();
-    }
-
-    private void addErrorResponse(int errorCode) {
-        addErrorResponse(errorCode, ErrorCode.getName(errorCode));
-    }
-
-    @Override
-    public void setDelayedResponse(boolean delayedResponse) {
-        this.isDelayed = delayedResponse;
-    }
-
-    @Override
-    public void addErrorResponse(int errorCode, String name) {
-        ByteArrayOutputStream byteArrayOutputStream = new NoCopyByteArrayOutputStream();
-        try {
-            JsonGenerator jsonWriter = jsonFactory.createGenerator(byteArrayOutputStream);
-            jsonWriter.writeStartObject();
-            addCommonReturnValues(jsonWriter);
-            jsonWriter.writeEndObject();
-            jsonWriter.close();
-        } catch (IOException e) {
-            throw new IllegalArgumentException("Could not add error response for " + this);
-        }
-        request.setError(errorCode, name);
-        request.returnValues().add(createResponseValue(byteArrayOutputStream));
-    }
-
-    protected static Value createResponseValue(ByteArrayOutputStream byteArrayOutputStream) {
-        return new StringValue(new Utf8Array(byteArrayOutputStream.toByteArray()));
-    }
-
-    protected void addCommonReturnValues(JsonGenerator jsonGenerator) throws IOException {
-        ConfigKey<?> key = requestData.getConfigKey();
-        setResponseField(jsonGenerator, SlimeResponseData.RESPONSE_VERSION, getProtocolVersion());
-        setResponseField(jsonGenerator, SlimeResponseData.RESPONSE_DEF_NAME, key.getName());
-        setResponseField(jsonGenerator, SlimeResponseData.RESPONSE_DEF_NAMESPACE, key.getNamespace());
-        setResponseField(jsonGenerator, SlimeResponseData.RESPONSE_DEF_MD5, key.getMd5());
-        setResponseField(jsonGenerator, SlimeResponseData.RESPONSE_CONFIGID, key.getConfigId());
-        setResponseField(jsonGenerator, SlimeResponseData.RESPONSE_CLIENT_HOSTNAME, requestData.getClientHostName());
-        jsonGenerator.writeFieldName(SlimeResponseData.RESPONSE_TRACE);
-        jsonGenerator.writeRawValue(getRequestTrace().toString(true));
-    }
-
-    protected static void setResponseField(JsonGenerator jsonGenerator, String fieldName, String value) throws IOException {
-        jsonGenerator.writeStringField(fieldName, value);
-    }
-
-    protected static void setResponseField(JsonGenerator jsonGenerator, String fieldName, long value) throws IOException {
-        jsonGenerator.writeNumberField(fieldName, value);
-    }
-
-    protected static void setResponseField(JsonGenerator jsonGenerator, String fieldName, boolean value) throws IOException {
-        jsonGenerator.writeBooleanField(fieldName, value);
-    }
-
-    @Override
-    public long getRequestGeneration() {
-        return requestData.getRequestGeneration();
-    }
-
-    @Override
-    public boolean isDelayedResponse() {
-        return isDelayed;
-    }
-
-    @Override
-    public int errorCode() {
-        return request.errorCode();
-    }
-
-    @Override
-    public String errorMessage() {
-        return request.errorMessage();
-    }
-
-    @Override
-    public String getShortDescription() {
-        return toString();
-    }
-
-    protected CompressionType getCompressionType() {
-        return requestData.getCompressionType();
-    }
-
-    @Override
-    public long getTimeout() {
-        return requestData.getTimeout();
-    }
-
-    @Override
-    public Optional<VespaVersion> getVespaVersion() {
-        return requestData.getVespaVersion();
-    }
-
-}
diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeTraceDeserializer.java b/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeTraceDeserializer.java
index f4754f4b0e1..998d2c8bcf5 100644
--- a/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeTraceDeserializer.java
+++ b/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeTraceDeserializer.java
@@ -9,7 +9,6 @@ import com.yahoo.yolean.trace.TraceNode;
  * Deserializing from a {@link Inspector} (slime) representation to a {@link TraceNode}
  *
  * @author Ulf Lilleengen
- * @since 5.5
  */
 public class SlimeTraceDeserializer {
     private final Inspector entry;
diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeTraceSerializer.java b/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeTraceSerializer.java
index 62fa6f8a882..5f5f057a26b 100644
--- a/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeTraceSerializer.java
+++ b/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeTraceSerializer.java
@@ -12,7 +12,6 @@ import java.util.Stack;
  * Serialize a {@link TraceNode} to {@link com.yahoo.slime.Slime}.
  *
  * @author Ulf Lilleengen
- * @since 5.5
  */
 public class SlimeTraceSerializer extends TraceVisitor {
     static final String TIMESTAMP = "timestamp";
@@ -30,7 +29,6 @@ public class SlimeTraceSerializer extends TraceVisitor {
         current.setLong(TIMESTAMP, node.timestamp());
         encodePayload(current, node.payload());
         addChildrenCursors(current, node);
-
     }
 
     private void encodePayload(Cursor current, Object payload) {
diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/Trace.java b/config/src/main/java/com/yahoo/vespa/config/protocol/Trace.java
index 17740c9f40d..fbeafdc3f6f 100644
--- a/config/src/main/java/com/yahoo/vespa/config/protocol/Trace.java
+++ b/config/src/main/java/com/yahoo/vespa/config/protocol/Trace.java
@@ -13,7 +13,6 @@ import java.time.Clock;
  * A trace utility that can serialize/deserialize to/from {@link Slime}
  *
  * @author Ulf Lilleengen
- * @since 5.3
  */
 public class Trace {
     private static final String TRACE_TRACELOG = "traceLog";
diff --git a/config/src/main/java/com/yahoo/vespa/config/util/ConfigUtils.java b/config/src/main/java/com/yahoo/vespa/config/util/ConfigUtils.java
index d7653572773..8f856ff4771 100644
--- a/config/src/main/java/com/yahoo/vespa/config/util/ConfigUtils.java
+++ b/config/src/main/java/com/yahoo/vespa/config/util/ConfigUtils.java
@@ -33,6 +33,7 @@ import java.util.regex.Pattern;
  * Utilities for mangling config text, finding md5sums, finding name and namespace in .def files etc.
  */
 public class ConfigUtils {
+
     /* Patterns used for finding ranges in config definitions */
     private static final Pattern intPattern = Pattern.compile(".*int.*range.*");
     private static final Pattern doublePattern = Pattern.compile(".*double.*range.*");
@@ -88,6 +89,7 @@ public class ConfigUtils {
 
     /**
      * Replaces sequences of spaces with 1 space, unless inside quotes. Public for testing;
+     *
      * @param str String to strip spaces from
      * @return String with spaces stripped
      */
@@ -103,15 +105,15 @@ public class ConfigUtils {
                 }
                 if (!inSpaceSequence) {
                     // start of space sequence
-                    inSpaceSequence=true;
+                    inSpaceSequence = true;
                     ret.append(" ");
                 }
             } else {
                 if (inSpaceSequence) {
-                    inSpaceSequence=false;
+                    inSpaceSequence = false;
                 }
-                if (c=='\"') {
-                    inQuotes=!inQuotes;
+                if (c == '\"') {
+                    inQuotes = !inQuotes;
                 }
                 ret.append(c);
             }
@@ -121,7 +123,7 @@ public class ConfigUtils {
 
     /**
      * Computes Md5 hash of a list of strings with the contents of a def-file.
-     *
+     * <p>
      * Each string is normalized according to the
      * rules of Vespa config definition files before they are used:
      * <ol>
@@ -132,12 +134,12 @@ public class ConfigUtils {
      * <li>Remove 'version=&lt;version-number&gt;'</li>
      * </ol>
      *
-     * @param lines  A list of lines constituting a def-file
+     * @param lines A list of lines constituting a def-file
      * @return the Md5 hash of the list, with lowercase letters
      */
     public static String getDefMd5(List<String> lines) {
         List<String> linesCopy = new ArrayList<>(lines);
-        for (Iterator<String> it=linesCopy.iterator(); it.hasNext(); ) {
+        for (Iterator<String> it = linesCopy.iterator(); it.hasNext(); ) {
             String line = it.next().trim();
             if (! line.startsWith("#") && ! line.equals("")) {
                 if (line.startsWith("version")) {
@@ -169,7 +171,7 @@ public class ConfigUtils {
             }
             if (line.length() > 0) {
                 line = stripSpaces(line);
-                m  = spaceBeforeCommaPatter.matcher(line);
+                m = spaceBeforeCommaPatter.matcher(line);
                 line = m.replaceAll(",");   // Remove space before comma (for enums)
                 sb.append(line).append("\n");
             }
@@ -188,7 +190,7 @@ public class ConfigUtils {
     public static String getDefNamespace(Reader in) {
         List<String> defLines = getDefLines(in);
         String defPackage = getDefKeyword(defLines, "package");
-        if (! defPackage.isEmpty()) return defPackage;
+        if (!defPackage.isEmpty()) return defPackage;
         return getDefKeyword(defLines, "namespace");
     }
 
@@ -285,7 +287,7 @@ public class ConfigUtils {
      */
     public static ConfigDefinitionKey createConfigDefinitionKeyFromDefFile(File file) throws IOException {
         String[] fileName = file.getName().split("\\.");
-        assert(fileName.length >= 2);
+        assert (fileName.length >= 2);
         String name = fileName[fileName.length - 2];
         byte[] content = IOUtils.readFileBytes(file);
 
@@ -295,7 +297,7 @@ public class ConfigUtils {
     /**
      * Creates a ConfigDefinitionKey from a name and the content of a config definition
      *
-     * @param name the name of the config definition
+     * @param name    the name of the config definition
      * @param content content of a config definition
      * @return a ConfigDefinitionKey
      */
@@ -306,6 +308,7 @@ public class ConfigUtils {
 
     /**
      * Escapes a config value according to the cfg format.
+     *
      * @param input the string to escape
      * @return the escaped string
      */
@@ -350,10 +353,10 @@ public class ConfigUtils {
      * Loop through values and return the first one that is set and non-empty.
      *
      * @param defaultValue The default value to use if no environment variables are set.
-     * @param envVars one or more environment variable strings
+     * @param envVars      one or more environment variable strings
      * @return a String with the value of the environment variable
      */
-    public static String getEnvValue(String defaultValue, String ... envVars) {
+    public static String getEnvValue(String defaultValue, String... envVars) {
         String value = null;
         for (String envVar : envVars) {
             if (value == null || value.isEmpty()) {
diff --git a/config/src/test/java/com/yahoo/config/subscription/BasicTest.java b/config/src/test/java/com/yahoo/config/subscription/BasicTest.java
index 3b8b7db6487..5b145d40b7f 100644
--- a/config/src/test/java/com/yahoo/config/subscription/BasicTest.java
+++ b/config/src/test/java/com/yahoo/config/subscription/BasicTest.java
@@ -1,13 +1,13 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.config.subscription;
 
-import static org.junit.Assert.*;
-import static org.hamcrest.CoreMatchers.is;
 
 import com.yahoo.foo.AppConfig;
 
 import org.junit.Test;
 
+import static org.junit.Assert.assertEquals;
+
 
 public class BasicTest {
 
@@ -17,7 +17,8 @@ public class BasicTest {
         ConfigHandle<AppConfig> h = s.subscribe(AppConfig.class, "raw:times 0");
         s.nextConfig(0);
         AppConfig c = h.getConfig();
-        assertThat(c.times(), is(0));
+        assertEquals(0, c.times());
+        s.close();
     }
 
     @Test
@@ -26,6 +27,7 @@ public class BasicTest {
         ConfigHandle<AppConfig> h = s.subscribe(AppConfig.class, "raw:times 2");
         s.nextGeneration(0);
         AppConfig c = h.getConfig();
-        assertThat(c.times(), is(2));
+        assertEquals(2, c.times());
+        s.close();
     }
 }
diff --git a/config/src/test/java/com/yahoo/config/subscription/CfgConfigPayloadBuilderTest.java b/config/src/test/java/com/yahoo/config/subscription/CfgConfigPayloadBuilderTest.java
index d4280c6300f..c8693227920 100644
--- a/config/src/test/java/com/yahoo/config/subscription/CfgConfigPayloadBuilderTest.java
+++ b/config/src/test/java/com/yahoo/config/subscription/CfgConfigPayloadBuilderTest.java
@@ -18,7 +18,6 @@ import static org.junit.Assert.assertEquals;
 /**
  * @author hmusum
  * @author Vegard Sjonfjell
- * @since 5.1
  */
 public class CfgConfigPayloadBuilderTest {
 
@@ -135,7 +134,7 @@ public class CfgConfigPayloadBuilderTest {
                 "                'boolVal': 'true'",
                 "            },",
                 "            {",
-                "                'stringVal': 'blue a=\\\'escaped\\\'',",
+                "                'stringVal': 'blue a=\\'escaped\\'',",
                 "                'boolVal': 'false'",
                 "            }",
                 "        ],",
@@ -182,7 +181,7 @@ public class CfgConfigPayloadBuilderTest {
         assertDeserializedConfigEqualsJson("a b=\"escaped\"",
                 inputJson(
                         "{",
-                        " 'a': 'b=\\\'escaped\\\''",
+                        " 'a': 'b=\\'escaped\\''",
                         "}"
                 )
         );
@@ -305,7 +304,7 @@ public class CfgConfigPayloadBuilderTest {
     }
 
     private static void assertDeserializedConfigEqualsJson(String serializedConfig, String expectedJson) {
-        assertDeserializedConfigEqualsJson(Arrays.asList(serializedConfig), expectedJson);
+        assertDeserializedConfigEqualsJson(List.of(serializedConfig), expectedJson);
     }
 
     private static void assertDeserializedConfigEqualsJson(List<String> inputConfig, String expectedJson) {
diff --git a/config/src/test/java/com/yahoo/config/subscription/ConfigInstancePayloadTest.java b/config/src/test/java/com/yahoo/config/subscription/ConfigInstancePayloadTest.java
index 2d0a40bcdc4..49fd25b6476 100644
--- a/config/src/test/java/com/yahoo/config/subscription/ConfigInstancePayloadTest.java
+++ b/config/src/test/java/com/yahoo/config/subscription/ConfigInstancePayloadTest.java
@@ -22,7 +22,6 @@ import static org.junit.Assert.fail;
 
 /**
  * @author gjoranv
- * @since 5.1.6
  */
 public class ConfigInstancePayloadTest {
 
diff --git a/config/src/test/java/com/yahoo/config/subscription/ConfigInstanceSerializationTest.java b/config/src/test/java/com/yahoo/config/subscription/ConfigInstanceSerializationTest.java
index 8d9b2d96b04..95ffb31fe7d 100644
--- a/config/src/test/java/com/yahoo/config/subscription/ConfigInstanceSerializationTest.java
+++ b/config/src/test/java/com/yahoo/config/subscription/ConfigInstanceSerializationTest.java
@@ -3,7 +3,6 @@ package com.yahoo.config.subscription;
 
 import com.yahoo.config.ConfigInstance;
 import com.yahoo.foo.FunctionTestConfig;
-import com.yahoo.config.codegen.DefLine;
 import com.yahoo.vespa.config.ConfigPayload;
 import org.junit.Test;
 
@@ -18,14 +17,6 @@ import static org.junit.Assert.assertThat;
  */
 public class ConfigInstanceSerializationTest {
 
-    private DefLine.Type stringType = new DefLine.Type("string");
-    private DefLine.Type intType = new DefLine.Type("int");
-    private DefLine.Type longType = new DefLine.Type("long");
-    private DefLine.Type boolType = new DefLine.Type("bool");
-    private DefLine.Type doubleType = new DefLine.Type("double");
-    private DefLine.Type fileType = new DefLine.Type("file");
-    private DefLine.Type refType = new DefLine.Type("reference");
-
     @Test
     public void require_symmetrical_serialization_and_deserialization_with_builder() {
         FunctionTestConfig config = ConfigInstancePayloadTest.createVariableAccessConfigWithBuilder();
@@ -40,218 +31,4 @@ public class ConfigInstanceSerializationTest {
         assertThat(ConfigInstance.serialize(config), is(ConfigInstance.serialize(config2)));
     }
 
-/** Looks like everything in the commented block tests unused api's.. remove?
-
-    @Test
-    public void testSerializeAgainstConfigDefinitionAllowNothing() {
-        FunctionTestConfig config = ConfigInstancePayloadTest.createVariableAccessConfigWithBuilder();
-        InnerCNode def = new InnerCNode("function-test");
-        List<String> payload = Validator.serialize(config, def);
-        Assert.assertEquals(payload.size(), 0);
-    }
-
-    @Test
-    public void testSerializeAgainstConfigDefinitionAllLeaves() {
-        FunctionTestConfig config = ConfigInstancePayloadTest.createVariableAccessConfigWithBuilder();
-        InnerCNode def = new InnerCNode("function-test");
-        def.children().put("bool_val", LeafCNode.newInstance(boolType, def, "bool_val"));
-        def.children().put("bool_with_def", LeafCNode.newInstance(boolType, def, "bool_with_def"));
-        def.children().put("int_val", LeafCNode.newInstance(intType, def, "int_val"));
-        def.children().put("int_with_def", LeafCNode.newInstance(intType, def, "int_with_def"));
-        def.children().put("long_val", LeafCNode.newInstance(longType, def, "long_val"));
-        def.children().put("long_with_def", LeafCNode.newInstance(longType, def, "long_with_def"));
-        def.children().put("double_val", LeafCNode.newInstance(doubleType, def, "double_val"));
-        def.children().put("double_with_def", LeafCNode.newInstance(doubleType, def, "double_with_def"));
-        def.children().put("string_val", LeafCNode.newInstance(stringType, def, "string_val"));
-        def.children().put("stringwithdef", LeafCNode.newInstance(stringType, def, "stringwithdef"));
-        def.children().put("enum_val", LeafCNode.newInstance(enumType(new String[]{"FOO", "BAR", "FOOBAR"}), def, "enum_val"));
-        def.children().put("enumwithdef", LeafCNode.newInstance(enumType(new String[]{"FOO2", "BAR2", "FOOBAR2"}), def, "enumwithdef", "BAR2"));
-        def.children().put("refval", LeafCNode.newInstance(refType, def, "refval"));
-        def.children().put("refwithdef", LeafCNode.newInstance(refType, def, "refwithdef"));
-        def.children().put("fileVal", LeafCNode.newInstance(fileType, def, "fileVal"));
-
-        List<String> payload = Validator.serialize(config, def);
-        String plString = payload.toString();
-        Assert.assertTrue(plString.matches(".*bool_val false.*"));
-        Assert.assertTrue(plString.matches(".*bool_with_def true.*"));
-        Assert.assertTrue(plString.matches(".*int_val 5.*"));
-        Assert.assertTrue(plString.matches(".*int_with_def -14.*"));
-        Assert.assertTrue(plString.matches(".*long_val 12345678901.*"));
-        Assert.assertTrue(plString.matches(".*long_with_def -9876543210.*"));
-        Assert.assertTrue(plString.matches(".*double_val 41\\.23.*"));
-        Assert.assertTrue(plString.matches(".*double_with_def -12.*"));
-        Assert.assertTrue(plString.matches(".*string_val \"foo\".*"));
-        Assert.assertTrue(plString.matches(".*stringwithdef \"bar and foo\".*"));
-        Assert.assertTrue(plString.matches(".*enum_val FOOBAR.*"));
-        Assert.assertTrue(plString.matches(".*enumwithdef BAR2.*"));
-        Assert.assertTrue(plString.matches(".*refval \\:parent\\:.*"));
-        Assert.assertTrue(plString.matches(".*refwithdef \\:parent\\:.*"));
-        Assert.assertTrue(plString.matches(".*fileVal \"etc\".*"));
-    }
-
-    @Test
-    public void testSerializeAgainstConfigDefinitionSomeLeaves() {
-        FunctionTestConfig config = ConfigInstancePayloadTest.createVariableAccessConfigWithBuilder();
-        InnerCNode def = new InnerCNode("function-test");
-        def.children().put("stringwithdef", LeafCNode.newInstance(stringType, def, "stringwithdef"));
-        def.children().put("long_with_def", LeafCNode.newInstance(longType, def, "long_with_def"));
-        // But not double_with_def etc, and no structs/arrays
-        List<String> payload = Validator.serialize(config, def);
-        String plString = payload.toString();
-        Assert.assertTrue(plString.matches(".*long_with_def \\-9876543210.*"));
-        Assert.assertTrue(plString.matches(".*stringwithdef \"bar and foo\".*"));
-        Assert.assertFalse(plString.matches(".*double_with_def.*"));
-        Assert.assertFalse(plString.matches(".*fileVal \"etc\".*"));
-        Assert.assertFalse(plString.matches(".*basicStruct.*"));
-    }
-
-    @Test
-    public void testSerializationAgainstConfigDefinitionAddedValsInDef() {
-        FunctionTestConfig config = ConfigInstancePayloadTest.createVariableAccessConfigWithBuilder();
-        InnerCNode def = new InnerCNode("function-test");
-        def.children().put("stringwithdef", LeafCNode.newInstance(stringType, def, "stringwithdef"));
-        def.children().put("someotherstring", LeafCNode.newInstance(stringType, def, "someotherstring", "some other"));
-        def.children().put("long_with_def", LeafCNode.newInstance(longType, def, "long_with_def"));
-        def.children().put("some_other_long", LeafCNode.newInstance(longType, def, "some_other_long", "88"));
-        def.children().put("some_other_enum", LeafCNode.newInstance(enumType(new String[]{"hey", "ho", "lets", "go"}), def, "some_other_enum", "lets"));
-        def.children().put("int_val_nofdef", LeafCNode.newInstance(intType, def, "int_val_nodef", null));
-
-        // But not double_with_def etc, and no structs/arrays
-        List<String> payload = Validator.serialize(config, def);
-        String plString = payload.toString();
-        Assert.assertTrue(plString.matches(".*long_with_def \\-9876543210.*"));
-        Assert.assertTrue(plString.matches(".*stringwithdef \"bar and foo\".*"));
-        Assert.assertTrue(plString.matches(".*.someotherstring \"some other\".*"));
-        Assert.assertTrue(plString.matches(".*some_other_long 88.*"));
-        Assert.assertTrue(plString.matches(".*some_other_enum lets.*"));
-        Assert.assertFalse(plString.matches(".*double_with_def.*"));
-        Assert.assertFalse(plString.matches(".*fileVal \"etc\".*"));
-        Assert.assertFalse(plString.matches(".*basicStruct.*"));
-        Assert.assertFalse(plString.matches(".*int_val_nodef.*"));
-    }
-
-    @Test
-    public void testSerializeAgainstConfigDefinitionMismatchAllWays() {
-        FunctionTestConfig config = ConfigInstancePayloadTest.createVariableAccessConfigWithBuilder();
-
-        // Create all sorts of mismatches in the def schema used to serialize
-        InnerCNode def = new InnerCNode("function-test");
-        def.children().put("long_with_def", LeafCNode.newInstance(longType, def, "long_with_def"));
-        def.children().put("stringwithdef", LeafCNode.newInstance(intType, def, "stringwithdef"));
-        def.children().put("basicStruct", LeafCNode.newInstance(intType, def, "basicStruct"));
-        InnerCNode doubleValWrong = new InnerCNode("double_val");
-        doubleValWrong.children().put("foo", LeafCNode.newInstance(intType, def, "foo"));
-        def.children().put("double_val", doubleValWrong);
-        InnerCNode myArray = new InnerCNode("myarray");
-        myArray.children().put("intval", LeafCNode.newInstance(stringType, myArray, "foo"));
-        InnerCNode myStruct = new InnerCNode("myStruct");
-        myStruct.children().put("a", LeafCNode.newInstance(stringType, myStruct, "foo"));
-        myArray.children().put("myStruct", myStruct);
-        def.children().put("myarray", myArray);
-
-        List<String> payload = Validator.serialize(config, def);
-        String plString = payload.toString();
-        Assert.assertTrue(plString.matches(".*long_with_def.*"));
-        Assert.assertFalse(plString.matches(".*stringwithdef.*"));
-        Assert.assertFalse(plString.matches(".*basicStruct.*"));
-        Assert.assertFalse(plString.matches(".*double_val.*"));
-        Assert.assertFalse(plString.matches(".*intval.*"));
-        Assert.assertFalse(plString.matches(".*\\.a.*"));
-    }
-
-    @Test
-    public void testSerializeAgainstConfigDefinitionComplex() {
-        FunctionTestConfig config = ConfigInstancePayloadTest.createVariableAccessConfigWithBuilder();
-
-        // Build a pretty complex def programatically
-        InnerCNode def = new InnerCNode("function-test");
-        def.children().put("stringwithdef", LeafCNode.newInstance(stringType, def, "stringwithdef"));
-        def.children().put("someUnknownStringNoDefault", LeafCNode.newInstance(stringType, def, "someUnknownStringNoDefault"));
-        InnerCNode basicStruct = new InnerCNode("basicStruct");
-        basicStruct.children().put("foo", LeafCNode.newInstance(stringType, def, "foo")); // but not bar
-        InnerCNode rootStruct = new InnerCNode("rootStruct");
-        InnerCNode inner1 = new InnerCNode("inner1");
-        InnerCNode someUnknwonStruct = new InnerCNode("someUnknownStruct");
-        InnerCNode someUnknownInner = new InnerCNode("someUnknownInner");
-        InnerCNode innerArr = new InnerCNode("innerArr");
-        rootStruct.children().put("inner1", inner1);
-        rootStruct.children().put("someUnknownStruct", someUnknwonStruct);
-        rootStruct.children().put("someUnknownInner", someUnknownInner);
-        rootStruct.children().put("innerArr", innerArr);
-        InnerCNode myarray = new InnerCNode("myarray");
-        InnerCNode unknownInner = new InnerCNode("unknownInner");
-        def.children().put("basicStruct", basicStruct);
-        def.children().put("rootStruct", rootStruct);
-        def.children().put("myarray", myarray);
-        def.children().put("unknownInner", unknownInner);
-        inner1.children().put("index", LeafCNode.newInstance(intType, inner1, "index"));
-        inner1.children().put("someUnknownInt", LeafCNode.newInstance(intType, inner1, "someUnknownInt", "-98"));
-        inner1.children().put("someUnknownIntNoDefault", LeafCNode.newInstance(intType, inner1, "someUnknownIntNoDefault"));
-        inner1.children().put("someUnknownEnum", LeafCNode.newInstance(enumType(new String[]{"goo", "go", "gorilla"}), inner1, "someUnknownEnum", "go"));
-        inner1.children().put("someUnknownEnumNoDefault", LeafCNode.newInstance(enumType(new String[]{"foo", "bar", "baz"}), inner1, "someUnknownEnumNoDefault"));
-        someUnknwonStruct.children().put("anint", LeafCNode.newInstance(intType, someUnknwonStruct, "anint", "3"));// But no instances of this in config
-        someUnknownInner.children().put("along", LeafCNode.newInstance(longType, someUnknownInner, "along", "234"));// No instance in config
-        innerArr.children().put("boolVal", LeafCNode.newInstance(boolType, innerArr, "boolVal"));
-        innerArr.children().put("someUnknownDouble", LeafCNode.newInstance(doubleType, innerArr, "someUnknownDouble", "-675.789"));
-        innerArr.children().put("someUnknownDoubleNoDefault", LeafCNode.newInstance(doubleType, innerArr, "someUnknownDoubleNoDefault"));
-        myarray.children().put("fileVal", LeafCNode.newInstance(fileType, myarray, "fileVal"));
-        myarray.children().put("stringval", new InnerCNode("stringval[]"));
-        // TODO make sure default for file is not allowed
-        //myarray.children().put("someUnknownFile", LeafCNode.newInstance(fileType, myarray, "someUnknownFile", "opt/"));
-        unknownInner.children().put("aDouble", LeafCNode.newInstance(doubleType, unknownInner, "aDouble", "1234"));
-        def.children().put("longarr", new InnerCNode("longarr[]"));
-        def.children().put("boolarr", new InnerCNode("boolarr[]"));
-        def.children().put("doublearr", new InnerCNode("doublearr[]"));
-        def.children().put("stringarr", new InnerCNode("stringarr[]"));
-        def.children().put("fileArr", new InnerCNode("fileArr[]"));
-        def.children().put("refarr", new InnerCNode("refarr[]"));
-        def.children().put("enumarr", new InnerCNode("enumarr[]"));
-        List<String> payload = Validator.serialize(config, def);
-        String plString = payload.toString();
-        Assert.assertFalse(plString.matches(".*long_with_def \\-9876543210.*"));
-        Assert.assertFalse(plString.matches(".*someUnknownStringNoDefault.*"));
-        Assert.assertTrue(plString.matches(".*stringwithdef \"bar and foo\".*"));
-        Assert.assertFalse(plString.matches(".*double_with_def.*"));
-        Assert.assertFalse(plString.matches(".*fileVal etc.*"));
-        Assert.assertTrue(plString.matches(".*basicStruct\\.foo \"basicFoo\".*"));
-        Assert.assertFalse(plString.matches(".*basicStruct\\.bar.*"));
-        Assert.assertFalse(plString.matches(".*rootStruct\\.inner0.*"));
-        Assert.assertFalse(plString.matches(".*unknownInner.*"));
-        Assert.assertFalse(plString.matches(".*rootStruct\\.someUnknownStruct.*"));
-        Assert.assertFalse(plString.matches(".*rootStruct\\.someUnknownInner.*"));
-        Assert.assertFalse(plString.matches(".*rootStruct\\.inner1\\.name.*"));
-        Assert.assertTrue(plString.matches(".*rootStruct\\.inner1\\.index 12.*"));
-        Assert.assertTrue(plString.matches(".*rootStruct\\.inner1\\.someUnknownInt -98.*"));
-        Assert.assertTrue(plString.matches(".*rootStruct\\.inner1\\.someUnknownEnum go.*"));
-        Assert.assertTrue(plString.matches(".*rootStruct\\.innerArr\\[0\\]\\.boolVal true.*"));
-        Assert.assertFalse(plString.matches(".*someUnknownEnumNoDefault.*"));
-        Assert.assertFalse(plString.matches(".*someUnknownDoubleNoDefault.*"));
-        Assert.assertFalse(plString.matches(".*someUnknownIntNoDefault.*"));
-        Assert.assertTrue(plString.matches(".*rootStruct\\.innerArr\\[0\\]\\.someUnknownDouble -675.789.*"));
-        Assert.assertFalse(plString.matches(".*rootStruct\\.innerArr\\[0\\]\\.stringVal*"));
-        Assert.assertFalse(plString.matches(".*myarray\\[0\\].intval.*"));
-        Assert.assertTrue(plString.matches(".*myarray\\[0\\].fileVal \"file0\".*"));
-        //assertTrue(plString.matches(".*myarray\\[0\\].someUnknownFile \"opt/\".*"));
-        Assert.assertTrue(plString.matches(".*myarray\\[0\\].stringval\\[0\\] \"baah\".*"));
-        Assert.assertTrue(plString.matches(".*myarray\\[0\\].stringval\\[1\\] \"yikes\".*"));
-        Assert.assertTrue(plString.matches(".*myarray\\[1\\].fileVal \"file1\".*"));
-        Assert.assertFalse(plString.matches(".*myarray\\[1\\].enumVal.*"));
-        Assert.assertFalse(plString.matches(".*myarray\\[1\\].refVal.*"));
-        Assert.assertTrue(plString.matches(".*boolarr\\[0\\] false.*"));
-        Assert.assertTrue(plString.matches(".*longarr\\[0\\] 9223372036854775807.*"));
-        Assert.assertTrue(plString.matches(".*longarr\\[1\\] -9223372036854775808.*"));
-        Assert.assertTrue(plString.matches(".*doublearr\\[0\\] 2344\\.0.*"));
-        Assert.assertTrue(plString.matches(".*doublearr\\[1\\] 123\\.0.*"));
-        Assert.assertTrue(plString.matches(".*stringarr\\[0\\] \"bar\".*"));
-        Assert.assertTrue(plString.matches(".*enumarr\\[0\\] VALUES.*"));
-        Assert.assertTrue(plString.matches(".*refarr\\[0\\] \\:parent\\:.*"));
-        Assert.assertTrue(plString.matches(".*refarr\\[1\\] \\:parent.*"));
-        Assert.assertTrue(plString.matches(".*refarr\\[2\\] parent\\:.*"));
-        Assert.assertTrue(plString.matches(".*fileArr\\[0\\] \"bin\".*"));
-    }
-
-    private DefLine.Type enumType(String[] strings) {
-        return new DefLine.Type("enum").setEnumArray(strings);
-    }
-**/
 }
diff --git a/config/src/test/java/com/yahoo/config/subscription/ConfigInstanceSerializerTest.java b/config/src/test/java/com/yahoo/config/subscription/ConfigInstanceSerializerTest.java
index ee8682efe3c..7c08d0175a1 100644
--- a/config/src/test/java/com/yahoo/config/subscription/ConfigInstanceSerializerTest.java
+++ b/config/src/test/java/com/yahoo/config/subscription/ConfigInstanceSerializerTest.java
@@ -21,9 +21,9 @@ import static org.junit.Assert.fail;
 /**
  * @author Ulf Lilleengen
  * @author Vegard Sjonfjell
- * @since 5.1
  */
 public class ConfigInstanceSerializerTest {
+
     @Test
     public void test_that_leaf_types_are_serialized_to_json_types() {
         SimpletypesConfig.Builder builder = new SimpletypesConfig.Builder();
@@ -225,4 +225,5 @@ public class ConfigInstanceSerializerTest {
 
         assertJsonEquals(baos.toString(), expectedJson);
     }
+
 }
diff --git a/config/src/test/java/com/yahoo/config/subscription/ConfigInstanceTest.java b/config/src/test/java/com/yahoo/config/subscription/ConfigInstanceTest.java
index 1da53e4c3b9..4da0c3f51e0 100644
--- a/config/src/test/java/com/yahoo/config/subscription/ConfigInstanceTest.java
+++ b/config/src/test/java/com/yahoo/config/subscription/ConfigInstanceTest.java
@@ -12,7 +12,7 @@ import static org.junit.Assert.fail;
 /**
  * Tests different aspects of the ConfigInstance class and its underlying Nodes.
  *
- * @author <a href="gv@yahoo-inc.com">G. Voldengen</a>
+ * @author gjoranv
  */
 public class ConfigInstanceTest {
     private ConfigSourceSet sourceSet = new ConfigSourceSet("config-instance-test");
diff --git a/config/src/test/java/com/yahoo/config/subscription/ConfigInstanceUtilTest.java b/config/src/test/java/com/yahoo/config/subscription/ConfigInstanceUtilTest.java
index 078be819d33..3bdaee09eaf 100644
--- a/config/src/test/java/com/yahoo/config/subscription/ConfigInstanceUtilTest.java
+++ b/config/src/test/java/com/yahoo/config/subscription/ConfigInstanceUtilTest.java
@@ -22,7 +22,6 @@ import static com.yahoo.foo.FunctionTestConfig.*;
 
 /**
  * @author Ulf Lilleengen
- * @since 5.1
  */
 public class ConfigInstanceUtilTest {
 
diff --git a/config/src/test/java/com/yahoo/config/subscription/ConfigInterruptedExceptionTest.java b/config/src/test/java/com/yahoo/config/subscription/ConfigInterruptedExceptionTest.java
index 35f4f14b4f4..a80da63afc9 100644
--- a/config/src/test/java/com/yahoo/config/subscription/ConfigInterruptedExceptionTest.java
+++ b/config/src/test/java/com/yahoo/config/subscription/ConfigInterruptedExceptionTest.java
@@ -8,9 +8,9 @@ import static org.junit.Assert.assertThat;
 
 /**
  * @author Ulf Lilleengen
- * @since 5.1
  */
 public class ConfigInterruptedExceptionTest {
+
     @Test
     public void require_that_throwable_is_preserved() {
         ConfigInterruptedException e = new ConfigInterruptedException(new RuntimeException("foo"));
diff --git a/config/src/test/java/com/yahoo/config/subscription/ConfigSetSubscriptionTest.java b/config/src/test/java/com/yahoo/config/subscription/ConfigSetSubscriptionTest.java
index 21cdfbe7d30..db30e7b7389 100644
--- a/config/src/test/java/com/yahoo/config/subscription/ConfigSetSubscriptionTest.java
+++ b/config/src/test/java/com/yahoo/config/subscription/ConfigSetSubscriptionTest.java
@@ -128,6 +128,7 @@ public class ConfigSetSubscriptionTest {
         assertEquals(hA0.getConfig().times(), 8800);
         assertEquals(hA1.getConfig().times(), 890);
         assertEquals(hS.getConfig().stringVal(), "new StringVal");
+        subscriber.close();
     }
 
     @Test
diff --git a/config/src/test/java/com/yahoo/config/subscription/ConfigSetTest.java b/config/src/test/java/com/yahoo/config/subscription/ConfigSetTest.java
index a120f19c17b..48a51bbc02c 100644
--- a/config/src/test/java/com/yahoo/config/subscription/ConfigSetTest.java
+++ b/config/src/test/java/com/yahoo/config/subscription/ConfigSetTest.java
@@ -10,9 +10,9 @@ import static org.junit.Assert.assertTrue;
 
 /**
  * @author Ulf Lilleengen
- * @since 5.1
  */
 public class ConfigSetTest {
+
     @Test
     public void testToString() {
         ConfigSet set = new ConfigSet();
diff --git a/config/src/test/java/com/yahoo/config/subscription/ConfigSourceSetTest.java b/config/src/test/java/com/yahoo/config/subscription/ConfigSourceSetTest.java
index 38d4a6a4571..0879c330f45 100755
--- a/config/src/test/java/com/yahoo/config/subscription/ConfigSourceSetTest.java
+++ b/config/src/test/java/com/yahoo/config/subscription/ConfigSourceSetTest.java
@@ -13,6 +13,7 @@ import static org.junit.Assert.*;
  * @author <a href="gv@yahoo-inc.com">G. Voldengen</a>
  */
 public class ConfigSourceSetTest {
+
     @Test
     public void testEquals() {
         assertEquals(new ConfigSourceSet(), new ConfigSourceSet());
diff --git a/config/src/test/java/com/yahoo/config/subscription/ConfigSubscriptionTest.java b/config/src/test/java/com/yahoo/config/subscription/ConfigSubscriptionTest.java
index 933a9fd130a..c8d4c081fc9 100644
--- a/config/src/test/java/com/yahoo/config/subscription/ConfigSubscriptionTest.java
+++ b/config/src/test/java/com/yahoo/config/subscription/ConfigSubscriptionTest.java
@@ -60,6 +60,7 @@ public class ConfigSubscriptionTest {
 
         assertEquals(c1, c1);
         assertNotEquals(c1, c2);
+        sub.close();
     }
 
     @Test
@@ -70,6 +71,7 @@ public class ConfigSubscriptionTest {
         sub.nextConfig();
         assertTrue(handle.getConfig().boolval());
         //assertTrue(sub.getSource() instanceof RawSource);
+        sub.close();
     }
 
     // Test that subscription is closed and subscriptionHandles is empty if we get an exception
diff --git a/config/src/test/java/com/yahoo/config/subscription/ConfigURITest.java b/config/src/test/java/com/yahoo/config/subscription/ConfigURITest.java
index 43e3cf658f3..555434837c8 100644
--- a/config/src/test/java/com/yahoo/config/subscription/ConfigURITest.java
+++ b/config/src/test/java/com/yahoo/config/subscription/ConfigURITest.java
@@ -12,9 +12,9 @@ import static org.junit.Assert.assertTrue;
 
 /**
  * @author Ulf Lilleengen
- * @since 5.1
  */
 public class ConfigURITest {
+
     @Test
     public void testDefaultUri() {
         ConfigURI uri = ConfigURI.createFromId("foo");
diff --git a/config/src/test/java/com/yahoo/config/subscription/DefaultConfigTest.java b/config/src/test/java/com/yahoo/config/subscription/DefaultConfigTest.java
index 85346264837..1bcf09a4028 100644
--- a/config/src/test/java/com/yahoo/config/subscription/DefaultConfigTest.java
+++ b/config/src/test/java/com/yahoo/config/subscription/DefaultConfigTest.java
@@ -24,6 +24,7 @@ import static org.junit.Assert.assertTrue;
  * @author hmusum
  */
 public class DefaultConfigTest {
+
     static final String CONFIG_ID = "raw:" +
             "nondefaultstring ####-------missing--------\n" +
             "defaultstring \"thedefault\"\n" +
diff --git a/config/src/test/java/com/yahoo/config/subscription/GenericConfigSubscriberTest.java b/config/src/test/java/com/yahoo/config/subscription/GenericConfigSubscriberTest.java
index cb25de89bfb..9c83f2f3c9a 100644
--- a/config/src/test/java/com/yahoo/config/subscription/GenericConfigSubscriberTest.java
+++ b/config/src/test/java/com/yahoo/config/subscription/GenericConfigSubscriberTest.java
@@ -1,7 +1,10 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.config.subscription;
 
-import java.util.*;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
 
 import com.yahoo.config.subscription.impl.GenericConfigHandle;
 import com.yahoo.config.subscription.impl.GenericConfigSubscriber;
@@ -14,14 +17,16 @@ import com.yahoo.vespa.config.protocol.CompressionType;
 import org.junit.Test;
 
 import static org.hamcrest.CoreMatchers.is;
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
 
 /**
  *
  * Test cases for the "generic" (class-less) subscription mechanism.
  *
  * @author Ulf Lilleengen
- * @since 5.1
  */
 public class GenericConfigSubscriberTest {
 
@@ -29,9 +34,9 @@ public class GenericConfigSubscriberTest {
     public void testSubscribeGeneric() {
         Map<ConfigSourceSet, JRTConfigRequester> requesters = new HashMap<>();
         ConfigSourceSet sourceSet = new ConfigSourceSet("blabla");
-        requesters.put(sourceSet, JRTConfigRequester.get(new MockConnection(), JRTConfigRequesterTest.getTestTimingValues()));
+        requesters.put(sourceSet, new JRTConfigRequester(new MockConnection(), JRTConfigRequesterTest.getTestTimingValues()));
         GenericConfigSubscriber sub = new GenericConfigSubscriber(requesters);
-        final List<String> defContent = Arrays.asList("myVal int");
+        final List<String> defContent = List.of("myVal int");
         GenericConfigHandle handle = sub.subscribe(new ConfigKey<>("simpletypes", "id", "config"), defContent, sourceSet, JRTConfigRequesterTest.getTestTimingValues());
         assertTrue(sub.nextConfig());
         assertTrue(handle.isChanged());
@@ -44,32 +49,38 @@ public class GenericConfigSubscriberTest {
     public void testGenericRequesterPooling() {
         ConfigSourceSet source1 = new ConfigSourceSet("tcp/foo:78");
         ConfigSourceSet source2 = new ConfigSourceSet("tcp/bar:79");
-        JRTConfigRequester req1 = JRTConfigRequester.get(new JRTConnectionPool(source1), JRTConfigRequesterTest.getTestTimingValues());
-        JRTConfigRequester req2 = JRTConfigRequester.get(new JRTConnectionPool(source2), JRTConfigRequesterTest.getTestTimingValues());
+        JRTConfigRequester req1 = JRTConfigRequester.create(source1, JRTConfigRequesterTest.getTestTimingValues());
+        JRTConfigRequester req2 = JRTConfigRequester.create(source2, JRTConfigRequesterTest.getTestTimingValues());
         Map<ConfigSourceSet, JRTConfigRequester> requesters = new LinkedHashMap<>();
         requesters.put(source1, req1);
         requesters.put(source2, req2);
         GenericConfigSubscriber sub = new GenericConfigSubscriber(requesters);
         assertEquals(sub.requesters().get(source1).getConnectionPool().getCurrent().getAddress(), "tcp/foo:78");
         assertEquals(sub.requesters().get(source2).getConnectionPool().getCurrent().getAddress(), "tcp/bar:79");
-
+        for (JRTConfigRequester requester : requesters.values()) {
+            requester.close();
+        }
     }
 
     @Test(expected=UnsupportedOperationException.class)
     public void testOverriddenSubscribeInvalid1() {
-        GenericConfigSubscriber sub = new GenericConfigSubscriber();
-        sub.subscribe(null, null);
+        createSubscriber().subscribe(null, null);
     }
 
     @Test(expected=UnsupportedOperationException.class)
     public void testOverriddenSubscribeInvalid2() {
-        GenericConfigSubscriber sub = new GenericConfigSubscriber();
-        sub.subscribe(null, null, 0L);
+        createSubscriber().subscribe(null, null, 0L);
     }
 
     @Test(expected=UnsupportedOperationException.class)
     public void testOverriddenSubscribeInvalid3() {
-        GenericConfigSubscriber sub = new GenericConfigSubscriber();
-        sub.subscribe(null, null, "");
+        createSubscriber().subscribe(null, null, "");
+    }
+
+    private GenericConfigSubscriber createSubscriber() {
+        return new GenericConfigSubscriber(Map.of(
+                new ConfigSourceSet("blabla"),
+                new JRTConfigRequester(new MockConnection(), JRTConfigRequesterTest.getTestTimingValues())));
     }
+
 }
diff --git a/config/src/test/java/com/yahoo/config/subscription/NamespaceTest.java b/config/src/test/java/com/yahoo/config/subscription/NamespaceTest.java
index 5e60c273ee4..430de894629 100644
--- a/config/src/test/java/com/yahoo/config/subscription/NamespaceTest.java
+++ b/config/src/test/java/com/yahoo/config/subscription/NamespaceTest.java
@@ -11,6 +11,7 @@ import static org.junit.Assert.assertThat;
  * @author gjoranv
  */
 public class NamespaceTest {
+
     @Test
     public void verifyConfigClassWithExplicitNamespace() {
         NamespaceConfig config = new ConfigGetter<>(NamespaceConfig.class).getConfig("raw: a 0\n");
diff --git a/config/src/test/java/com/yahoo/config/subscription/UnicodeTest.java b/config/src/test/java/com/yahoo/config/subscription/UnicodeTest.java
index 6512b5ea29e..f1bd164c874 100644
--- a/config/src/test/java/com/yahoo/config/subscription/UnicodeTest.java
+++ b/config/src/test/java/com/yahoo/config/subscription/UnicodeTest.java
@@ -13,6 +13,7 @@ import static org.junit.Assert.assertEquals;
  * @author Harald Musum
  */
 public class UnicodeTest {
+
     /**
      * Reads a config from a file which is exactly like one returned from
      * the config server given only default values for this config.
diff --git a/config/src/test/java/com/yahoo/config/subscription/impl/FileConfigSubscriptionTest.java b/config/src/test/java/com/yahoo/config/subscription/impl/FileConfigSubscriptionTest.java
index 2f550fc8e1e..71e7a9f08d7 100644
--- a/config/src/test/java/com/yahoo/config/subscription/impl/FileConfigSubscriptionTest.java
+++ b/config/src/test/java/com/yahoo/config/subscription/impl/FileConfigSubscriptionTest.java
@@ -25,7 +25,6 @@ import static org.junit.Assert.assertTrue;
 
 /**
  * @author Ulf Lilleengen
- * @since 5.1.7
  */
 public class FileConfigSubscriptionTest {
     private File TEST_TYPES_FILE;
diff --git a/config/src/test/java/com/yahoo/config/subscription/impl/JRTConfigRequesterTest.java b/config/src/test/java/com/yahoo/config/subscription/impl/JRTConfigRequesterTest.java
index 757dd99f43b..4211345dff7 100644
--- a/config/src/test/java/com/yahoo/config/subscription/impl/JRTConfigRequesterTest.java
+++ b/config/src/test/java/com/yahoo/config/subscription/impl/JRTConfigRequesterTest.java
@@ -1,10 +1,12 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.config.subscription.impl;
 
+import com.yahoo.config.subscription.ConfigSourceSet;
 import com.yahoo.foo.SimpletypesConfig;
 import com.yahoo.config.subscription.ConfigSubscriber;
 import com.yahoo.jrt.Request;
 import com.yahoo.vespa.config.ConfigKey;
+import com.yahoo.vespa.config.ConnectionPool;
 import com.yahoo.vespa.config.ErrorCode;
 import com.yahoo.vespa.config.ErrorType;
 import com.yahoo.vespa.config.TimingValues;
@@ -17,6 +19,8 @@ import java.util.Random;
 
 import static org.hamcrest.CoreMatchers.is;
 import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNotSame;
+import static org.junit.Assert.assertSame;
 import static org.junit.Assert.assertThat;
 import static org.junit.Assert.assertTrue;
 
@@ -349,4 +353,23 @@ public class JRTConfigRequesterTest {
         }
     }
 
+    @Test
+    public void testManagedPool() {
+        ConfigSourceSet sourceSet = ConfigSourceSet.createDefault();
+        TimingValues timingValues = new TimingValues();
+        JRTConfigRequester requester1 = JRTConfigRequester.create(sourceSet, timingValues);
+        JRTConfigRequester requester2 = JRTConfigRequester.create(sourceSet, timingValues);
+        assertNotSame(requester1, requester2);
+        assertSame(requester1.getConnectionPool(), requester2.getConnectionPool());
+        ConnectionPool firstPool = requester1.getConnectionPool();
+        requester1.close();
+        requester2.close();
+        requester1 = JRTConfigRequester.create(sourceSet, timingValues);
+        assertNotSame(firstPool, requester1.getConnectionPool());
+        requester2 = JRTConfigRequester.create(new ConfigSourceSet("test-managed-pool-2"), timingValues);
+        assertNotSame(requester1.getConnectionPool(), requester2.getConnectionPool());
+        requester1.close();
+        requester2.close();
+    }
+
 }
diff --git a/config/src/test/java/com/yahoo/vespa/config/ConfigBuilderMergeTest.java b/config/src/test/java/com/yahoo/vespa/config/ConfigBuilderMergeTest.java
index c77985c91d8..137d2894164 100644
--- a/config/src/test/java/com/yahoo/vespa/config/ConfigBuilderMergeTest.java
+++ b/config/src/test/java/com/yahoo/vespa/config/ConfigBuilderMergeTest.java
@@ -18,7 +18,6 @@ import static org.junit.Assert.assertThat;
  * SEO keywords: test override() on builders. overrideTest, testOverride
  *
  * @author Ulf Lilleengen
- * @since 5.1
  */
 public class ConfigBuilderMergeTest {
 
diff --git a/config/src/test/java/com/yahoo/vespa/config/ConfigCacheKeyTest.java b/config/src/test/java/com/yahoo/vespa/config/ConfigCacheKeyTest.java
index bb65fdaa153..fce61cc802c 100755
--- a/config/src/test/java/com/yahoo/vespa/config/ConfigCacheKeyTest.java
+++ b/config/src/test/java/com/yahoo/vespa/config/ConfigCacheKeyTest.java
@@ -12,6 +12,7 @@ import static org.junit.Assert.*;
  * @author hmusum
  */
 public class ConfigCacheKeyTest {
+
     @Test
     public void testConfigCacheKey() {
         final String defMd5 = "md5";
diff --git a/config/src/test/java/com/yahoo/vespa/config/ConfigDefinitionBuilderTest.java b/config/src/test/java/com/yahoo/vespa/config/ConfigDefinitionBuilderTest.java
index dba73223097..810b9f58829 100644
--- a/config/src/test/java/com/yahoo/vespa/config/ConfigDefinitionBuilderTest.java
+++ b/config/src/test/java/com/yahoo/vespa/config/ConfigDefinitionBuilderTest.java
@@ -12,7 +12,6 @@ import java.io.IOException;
 import static org.hamcrest.CoreMatchers.is;
 import static org.junit.Assert.*;
 
-
 /**
  * Unit tests for ConfigDefinitionBuilder.
  *
diff --git a/config/src/test/java/com/yahoo/vespa/config/ConfigFileFormatterTest.java b/config/src/test/java/com/yahoo/vespa/config/ConfigFileFormatterTest.java
index 3cc030d944b..5198759d3e2 100644
--- a/config/src/test/java/com/yahoo/vespa/config/ConfigFileFormatterTest.java
+++ b/config/src/test/java/com/yahoo/vespa/config/ConfigFileFormatterTest.java
@@ -24,7 +24,6 @@ import static org.junit.Assert.assertThat;
 
 /**
  * @author Ulf Lilleengen
- * @since 5.1
  */
 public class ConfigFileFormatterTest {
 
diff --git a/config/src/test/java/com/yahoo/vespa/config/ConfigPayloadBuilderTest.java b/config/src/test/java/com/yahoo/vespa/config/ConfigPayloadBuilderTest.java
index eb0c47e3b0a..c13d3ec9b9b 100644
--- a/config/src/test/java/com/yahoo/vespa/config/ConfigPayloadBuilderTest.java
+++ b/config/src/test/java/com/yahoo/vespa/config/ConfigPayloadBuilderTest.java
@@ -17,7 +17,6 @@ import static org.junit.Assert.assertThat;
 
 /**
  * @author Ulf Lilleengen
- * @since 5.1
  */
 public class ConfigPayloadBuilderTest {
 
diff --git a/config/src/test/java/com/yahoo/vespa/config/ConfigPayloadTest.java b/config/src/test/java/com/yahoo/vespa/config/ConfigPayloadTest.java
index f1b0adc03e7..b3db6e2ab43 100644
--- a/config/src/test/java/com/yahoo/vespa/config/ConfigPayloadTest.java
+++ b/config/src/test/java/com/yahoo/vespa/config/ConfigPayloadTest.java
@@ -24,8 +24,7 @@ import static org.junit.Assert.assertThat;
 import static org.junit.Assert.assertTrue;
 
 /**
- * @author Ulf Lilleengen                                               3
- * @since 5.1
+ * @author Ulf Lilleengen
  */
 public class ConfigPayloadTest {
 
diff --git a/config/src/test/java/com/yahoo/vespa/config/DefaultValueApplierTest.java b/config/src/test/java/com/yahoo/vespa/config/DefaultValueApplierTest.java
index e67489b4030..6ab3059f1c0 100644
--- a/config/src/test/java/com/yahoo/vespa/config/DefaultValueApplierTest.java
+++ b/config/src/test/java/com/yahoo/vespa/config/DefaultValueApplierTest.java
@@ -16,9 +16,9 @@ import static org.junit.Assert.assertTrue;
 
 /**
  * @author Ulf Lilleengen
- * @since 5.1
  */
 public class DefaultValueApplierTest {
+
     public Slime apply(Slime slime, String ... extraFields) {
         StringBuilder defBuilder = new StringBuilder();
         defBuilder.append("namespace=test").append("\n");
diff --git a/config/src/test/java/com/yahoo/vespa/config/ErrorCodeTest.java b/config/src/test/java/com/yahoo/vespa/config/ErrorCodeTest.java
index 8e6cc1fc6a0..64210eaa4b8 100644
--- a/config/src/test/java/com/yahoo/vespa/config/ErrorCodeTest.java
+++ b/config/src/test/java/com/yahoo/vespa/config/ErrorCodeTest.java
@@ -8,9 +8,9 @@ import static org.junit.Assert.assertThat;
 
 /**
  * @author hmusum
- * @since 5.1.9
  */
 public class ErrorCodeTest {
+
     @Test
     public void basic() {
         assertThat(ErrorCode.getName(ErrorCode.INTERNAL_ERROR), is("INTERNAL_ERROR"));
diff --git a/config/src/test/java/com/yahoo/vespa/config/ErrorTypeTest.java b/config/src/test/java/com/yahoo/vespa/config/ErrorTypeTest.java
index 450d7e8d6d8..a0b2c1185a7 100644
--- a/config/src/test/java/com/yahoo/vespa/config/ErrorTypeTest.java
+++ b/config/src/test/java/com/yahoo/vespa/config/ErrorTypeTest.java
@@ -8,7 +8,6 @@ import static org.junit.Assert.assertThat;
 
 /**
  * @author Ulf Lilleengen
- * @since 5.1
  */
 public class ErrorTypeTest {
 
@@ -32,4 +31,5 @@ public class ErrorTypeTest {
         assertThat(ErrorType.getErrorType(ErrorCode.ILLEGAL_SUB_FLAG), is(ErrorType.FATAL));
         assertThat(ErrorType.getErrorType(0xdeadc0de), is(ErrorType.FATAL));
     }
+
 }
diff --git a/config/src/test/java/com/yahoo/vespa/config/GenericConfigBuilderTest.java b/config/src/test/java/com/yahoo/vespa/config/GenericConfigBuilderTest.java
index ee7845086c2..2c60ff95fee 100644
--- a/config/src/test/java/com/yahoo/vespa/config/GenericConfigBuilderTest.java
+++ b/config/src/test/java/com/yahoo/vespa/config/GenericConfigBuilderTest.java
@@ -13,9 +13,9 @@ import static org.junit.Assert.assertThat;
 
 /**
  * @author Ulf Lilleengen
- * @since 5.1
  */
 public class GenericConfigBuilderTest {
+
     @Test
     public void require_that_builder_can_be_overridden() throws IOException {
         ConfigPayloadBuilder ba = new ConfigPayloadBuilder();
diff --git a/config/src/test/java/com/yahoo/vespa/config/JRTConnectionPoolTest.java b/config/src/test/java/com/yahoo/vespa/config/JRTConnectionPoolTest.java
index 8a403b45003..cc46301e869 100644
--- a/config/src/test/java/com/yahoo/vespa/config/JRTConnectionPoolTest.java
+++ b/config/src/test/java/com/yahoo/vespa/config/JRTConnectionPoolTest.java
@@ -70,7 +70,7 @@ public class JRTConnectionPoolTest {
 
     // Tests that the number of times each connection is used is close to equal
     private void assertConnectionDistributionIsFair(Map<String, Integer> connectionsUsedPerHost) {
-        double devianceDueToRandomSourceSelection = 0.13;
+        double devianceDueToRandomSourceSelection = 0.14;
         final int size = 1000;
         int minHostCount = (int) (size/2 * (1 - devianceDueToRandomSourceSelection));
         int maxHostCount = (int) (size/2 * (1 + devianceDueToRandomSourceSelection));
diff --git a/config/src/test/java/com/yahoo/vespa/config/LZ4CompressionTest.java b/config/src/test/java/com/yahoo/vespa/config/LZ4CompressionTest.java
deleted file mode 100644
index a3125c73bea..00000000000
--- a/config/src/test/java/com/yahoo/vespa/config/LZ4CompressionTest.java
+++ /dev/null
@@ -1,74 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.config;
-
-import net.jpountz.lz4.LZ4Compressor;
-import net.jpountz.lz4.LZ4Factory;
-import net.jpountz.lz4.LZ4SafeDecompressor;
-import org.junit.Ignore;
-import org.junit.Test;
-
-import java.io.IOException;
-import java.nio.file.FileSystems;
-import java.nio.file.Files;
-
-/**
- * To run this test, place a payload in src/test/ca.json. The file is not checked in because it is huge.
- *
- * @author Ulf Lilleengen
- * @since 5.12
- */
-public class LZ4CompressionTest {
-    private static LZ4Factory factory = LZ4Factory.safeInstance();
-
-    @Test
-    @Ignore
-    public void testCompression() throws IOException {
-        byte[] data = getInput();
-        System.out.println("High compressor");
-        for (int i = 0; i < 10; i++) {
-            timeCompressor(factory.highCompressor(), data);
-        }
-        System.out.println("Fast compressor");
-        for (int i = 0; i < 10; i++) {
-            timeCompressor(factory.fastCompressor(), data);
-        }
-    }
-
-    private byte[] getInput() throws IOException {
-        byte[] data = Files.readAllBytes(FileSystems.getDefault().getPath("src/test/ca.json"));
-        System.out.println("Input size: " + data.length);
-        return data;
-    }
-
-    private void timeCompressor(LZ4Compressor lz4Compressor, byte[] data) {
-        long start = System.currentTimeMillis();
-        byte[] compressed = lz4Compressor.compress(data);
-        long end = System.currentTimeMillis();
-        System.out.println("Compression took " + (end - start) + " millis, and size of data is " + compressed.length + " bytes");
-    }
-
-    @Test
-    @Ignore
-    public void testDecompression() throws IOException {
-        byte[] data = getInput();
-        byte[] outputbuffer = new byte[data.length];
-        byte[] hcCompressedData = factory.highCompressor().compress(data);
-        System.out.println("High compressor");
-        for (int i = 0; i < 10; i++) {
-            timeDecompressor(hcCompressedData, factory.safeDecompressor(), outputbuffer);
-        }
-        byte[] fastCompressedData = factory.fastCompressor().compress(data);
-        System.out.println("Fast compressor");
-        for (int i = 0; i < 10; i++) {
-            timeDecompressor(fastCompressedData, factory.safeDecompressor(), outputbuffer);
-        }
-    }
-
-    private void timeDecompressor(byte[] compressedData, LZ4SafeDecompressor decompressor, byte[] outputbuffer) {
-        long start = System.currentTimeMillis();
-        decompressor.decompress(compressedData, outputbuffer);
-        long end = System.currentTimeMillis();
-        System.out.println("Decompression took " + (end - start) + " millis");
-    }
-
-}
diff --git a/config/src/test/java/com/yahoo/vespa/config/LZ4PayloadCompressorTest.java b/config/src/test/java/com/yahoo/vespa/config/LZ4PayloadCompressorTest.java
index 06bbaad4271..903b6f3b489 100644
--- a/config/src/test/java/com/yahoo/vespa/config/LZ4PayloadCompressorTest.java
+++ b/config/src/test/java/com/yahoo/vespa/config/LZ4PayloadCompressorTest.java
@@ -9,9 +9,9 @@ import static org.junit.Assert.assertThat;
 
 /**
  * @author Ulf Lilleengen
- * @since 5.19
  */
 public class LZ4PayloadCompressorTest {
+
     @Test
     public void testCompression() {
         assertCompression("hei hallo der");
@@ -23,8 +23,7 @@ public class LZ4PayloadCompressorTest {
         LZ4PayloadCompressor compressor = new LZ4PayloadCompressor();
         byte[] data = Utf8.toBytes(input);
         byte[] compressed = compressor.compress(data);
-        byte[] output = new byte[data.length];
-        compressor.decompress(compressed, output);
+        byte[] output = compressor.decompress(compressed, data.length);
         assertThat(data, is(output));
     }
 }
diff --git a/config/src/test/java/com/yahoo/vespa/config/RawConfigTest.java b/config/src/test/java/com/yahoo/vespa/config/RawConfigTest.java
index a564fea8b2e..3f6c54ea46e 100644
--- a/config/src/test/java/com/yahoo/vespa/config/RawConfigTest.java
+++ b/config/src/test/java/com/yahoo/vespa/config/RawConfigTest.java
@@ -113,11 +113,11 @@ public class RawConfigTest {
         assertThat(config.getDefMd5(), is(defMd5));
         config = new RawConfig(key, null, payload, configMd5, generation, false, null, Optional.empty());
         assertNull(config.getDefMd5());
-        config = new RawConfig(key, null, payload, configMd5, generation, false, Arrays.asList(""), Optional.empty());
+        config = new RawConfig(key, null, payload, configMd5, generation, false,List.of(""), Optional.empty());
         assertThat(config.getDefMd5(), is(defMd5ForEmptyDefContent));
         config = new RawConfig(key, "", payload, configMd5, generation, false, null, Optional.empty());
         assertThat(config.getDefMd5(), is(""));
-        config = new RawConfig(key, "", payload, configMd5, generation, false, Arrays.asList(""), Optional.empty());
+        config = new RawConfig(key, "", payload, configMd5, generation, false, List.of(""), Optional.empty());
         assertThat(config.getDefMd5(), is(defMd5ForEmptyDefContent));
     }
 
diff --git a/config/src/test/java/com/yahoo/vespa/config/TimingValuesTest.java b/config/src/test/java/com/yahoo/vespa/config/TimingValuesTest.java
deleted file mode 100644
index 3b97c360c4b..00000000000
--- a/config/src/test/java/com/yahoo/vespa/config/TimingValuesTest.java
+++ /dev/null
@@ -1,24 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.config;
-
-import org.junit.Test;
-
-import static org.hamcrest.CoreMatchers.is;
-import static org.hamcrest.CoreMatchers.not;
-import static org.junit.Assert.assertThat;
-
-/**
- * Note: Most of the functionality is tested implicitly by other tests
- *
- * @author hmusum
- */
-public class TimingValuesTest {
-    @Test
-    public void basic() {
-        TimingValues tv = new TimingValues();
-        TimingValues tv2 = new TimingValues(1L, 1L, 1L, 1L, 1L, 1L, 1L, 1);
-        assertThat(tv.getRandom(), is(not(tv2.getRandom())));
-        TimingValues copy = new TimingValues(tv2);
-        assertThat(copy.toString(), is(tv2.toString()));  // No equals method, just using toString to compare
-    }
-}
diff --git a/config/src/test/java/com/yahoo/vespa/config/protocol/ConfigResponseTest.java b/config/src/test/java/com/yahoo/vespa/config/protocol/ConfigResponseTest.java
index 91adc544d88..a56c7ef2daa 100644
--- a/config/src/test/java/com/yahoo/vespa/config/protocol/ConfigResponseTest.java
+++ b/config/src/test/java/com/yahoo/vespa/config/protocol/ConfigResponseTest.java
@@ -2,9 +2,6 @@
 package com.yahoo.vespa.config.protocol;
 
 import com.yahoo.foo.SimpletypesConfig;
-import com.yahoo.config.codegen.DefParser;
-import com.yahoo.config.codegen.InnerCNode;
-import com.yahoo.text.StringUtilities;
 import com.yahoo.text.Utf8Array;
 import com.yahoo.vespa.config.ConfigPayload;
 import com.yahoo.vespa.config.LZ4PayloadCompressor;
@@ -12,10 +9,10 @@ import org.junit.Test;
 
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
-import java.io.StringReader;
-import java.util.List;
+import java.nio.charset.StandardCharsets;
 
 import static org.hamcrest.core.Is.is;
+import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertThat;
 
@@ -27,17 +24,16 @@ public class ConfigResponseTest {
     @Test
     public void require_that_slime_response_is_initialized() throws IOException {
         ConfigPayload configPayload = ConfigPayload.fromInstance(new SimpletypesConfig(new SimpletypesConfig.Builder()));
-        DefParser dParser = new DefParser(SimpletypesConfig.getDefName(), new StringReader(StringUtilities.implode(SimpletypesConfig.CONFIG_DEF_SCHEMA, "\n")));
-        InnerCNode targetDef = dParser.getTree();
-        ConfigResponse response = SlimeConfigResponse.fromConfigPayload(configPayload, targetDef, 3, false, "mymd5");
-        List<String> payload = response.getLegacyPayload();
+        ConfigResponse response = SlimeConfigResponse.fromConfigPayload(configPayload, 3, false, "mymd5");
+        ByteArrayOutputStream baos = new ByteArrayOutputStream();
+        response.serialize(baos, CompressionType.UNCOMPRESSED);
+        String payload = baos.toString(StandardCharsets.UTF_8);
         assertNotNull(payload);
-        assertThat(payload.size(), is(6));
-        assertThat(payload.get(0), is("boolval false"));
+        assertEquals("{\"boolval\":false,\"doubleval\":0.0,\"enumval\":\"VAL1\",\"intval\":0,\"longval\":0,\"stringval\":\"s\"}", payload);
         assertThat(response.getGeneration(), is(3L));
         assertThat(response.getConfigMd5(), is("mymd5"));
 
-        ByteArrayOutputStream baos = new ByteArrayOutputStream();
+        baos = new ByteArrayOutputStream();
         response.serialize(baos, CompressionType.UNCOMPRESSED);
         assertThat(baos.toString(), is("{\"boolval\":false,\"doubleval\":0.0,\"enumval\":\"VAL1\",\"intval\":0,\"longval\":0,\"stringval\":\"s\"}"));
     }
@@ -45,19 +41,15 @@ public class ConfigResponseTest {
     @Test
     public void require_that_slime_response_decompresses_on_serialize() throws IOException {
         ConfigPayload configPayload = ConfigPayload.fromInstance(new SimpletypesConfig(new SimpletypesConfig.Builder()));
-        DefParser dParser = new DefParser(SimpletypesConfig.getDefName(), new StringReader(StringUtilities.implode(SimpletypesConfig.CONFIG_DEF_SCHEMA, "\n")));
-        InnerCNode targetDef = dParser.getTree();
         Utf8Array data = configPayload.toUtf8Array(true);
         Utf8Array bytes = new Utf8Array(new LZ4PayloadCompressor().compress(data.getBytes()));
-        ConfigResponse response = new SlimeConfigResponse(bytes, targetDef, 3, false, "mymd5", CompressionInfo.create(CompressionType.LZ4, data.getByteLength()));
-        List<String> payload = response.getLegacyPayload();
+        ConfigResponse response = new SlimeConfigResponse(bytes, 3, false, "mymd5", CompressionInfo.create(CompressionType.LZ4, data.getByteLength()));
+        ByteArrayOutputStream baos = new ByteArrayOutputStream();
+        response.serialize(baos, CompressionType.UNCOMPRESSED);
+        String payload = baos.toString(StandardCharsets.UTF_8);
         assertNotNull(payload);
-        assertThat(payload.size(), is(6));
-        assertThat(payload.get(0), is("boolval false"));
-        assertThat(response.getGeneration(), is(3L));
-        assertThat(response.getConfigMd5(), is("mymd5"));
 
-        ByteArrayOutputStream baos = new ByteArrayOutputStream();
+        baos = new ByteArrayOutputStream();
         response.serialize(baos, CompressionType.UNCOMPRESSED);
         assertThat(baos.toString(), is("{\"boolval\":false,\"doubleval\":0.0,\"enumval\":\"VAL1\",\"intval\":0,\"longval\":0,\"stringval\":\"s\"}"));
     }
diff --git a/config/src/test/java/com/yahoo/vespa/config/protocol/JRTConfigRequestBase.java b/config/src/test/java/com/yahoo/vespa/config/protocol/JRTConfigRequestBase.java
deleted file mode 100644
index 75ba1392fd1..00000000000
--- a/config/src/test/java/com/yahoo/vespa/config/protocol/JRTConfigRequestBase.java
+++ /dev/null
@@ -1,288 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.config.protocol;
-
-import com.yahoo.foo.SimpletypesConfig;
-import com.yahoo.config.subscription.ConfigSet;
-import com.yahoo.config.subscription.ConfigSourceSet;
-import com.yahoo.config.subscription.ConfigSubscriber;
-import com.yahoo.config.subscription.impl.GenericConfigSubscriber;
-import com.yahoo.config.subscription.impl.JRTConfigRequester;
-import com.yahoo.config.subscription.impl.JRTConfigSubscription;
-import com.yahoo.config.subscription.impl.MockConnection;
-import com.yahoo.jrt.Request;
-import com.yahoo.slime.Inspector;
-import com.yahoo.slime.JsonDecoder;
-import com.yahoo.slime.Slime;
-import com.yahoo.test.ManualClock;
-import com.yahoo.text.Utf8;
-import com.yahoo.vespa.config.ConfigKey;
-import com.yahoo.vespa.config.ConfigPayload;
-import com.yahoo.vespa.config.ErrorCode;
-import com.yahoo.vespa.config.RawConfig;
-import com.yahoo.vespa.config.TimingValues;
-import com.yahoo.vespa.config.util.ConfigUtils;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.io.IOException;
-import java.util.Collections;
-import java.util.Optional;
-
-import static org.hamcrest.CoreMatchers.is;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertThat;
-import static org.junit.Assert.assertTrue;
-
-/**
- * @author Ulf Lilleengen
- * @since 5.3
- */
-public abstract class JRTConfigRequestBase {
-
-    protected String defName = "mydef";
-    protected String defNamespace = "my.name.space";
-    protected String hostname = "myhost";
-    protected String configId = "config/id";
-    protected String defMd5 = "595f44fec1e92a71d3e9e77456ba80d1";
-    protected long currentGeneration = 3;
-    protected final Optional<VespaVersion> vespaVersion = Optional.of(VespaVersion.fromString("5.38.24"));
-    protected long timeout = 5000;
-    protected Trace trace ;
-    protected String configMd5 = ConfigUtils.getMd5(createPayload().getData());
-    protected JRTClientConfigRequest clientReq;
-    protected JRTServerConfigRequest serverReq;
-
-    @Before
-    public void setupRequest() throws IOException {
-        clientReq = createReq();
-        serverReq = createReq(clientReq.getRequest());
-        assertTrue(serverReq.validateParameters());
-    }
-
-    private JRTClientConfigRequest createReq() throws IOException {
-        trace = Trace.createNew(3, new ManualClock());
-        trace.trace(1, "hei");
-        return createReq(defName, defNamespace, defMd5, hostname, configId, configMd5, currentGeneration, timeout, trace);
-    }
-
-    private JRTClientConfigRequest createReq(Payload payload) throws IOException {
-        trace = Trace.createNew(3, new ManualClock());
-        trace.trace(1, "hei");
-        return createReq(defName, defNamespace, defMd5, hostname, configId, ConfigUtils.getMd5(payload.getData()), currentGeneration, timeout, trace);
-    }
-
-    protected abstract JRTClientConfigRequest createReq(String defName, String defNamespace, String defMd5,
-                                                        String hostname, String configId, String configMd5,
-                                                        long currentGeneration, long timeout, Trace trace);
-    protected abstract JRTServerConfigRequest createReq(Request request);
-    protected abstract JRTClientConfigRequest createReq(JRTConfigSubscription<SimpletypesConfig> sub, Trace aNew);
-    protected abstract JRTClientConfigRequest createFromRaw(RawConfig rawConfig, long serverTimeout, Trace aNew);
-
-    protected void request_is_parsed_base() {
-        String [] expectedContent = new String[]{
-                "namespace=my.name.space",
-                "myfield string"
-        };
-        System.out.println(serverReq.toString());
-        assertThat(serverReq.getConfigKey().getName(), is(defName));
-        assertThat(serverReq.getConfigKey().getNamespace(), is(defNamespace));
-        assertThat(serverReq.getConfigKey().getMd5(), is(defMd5));
-        assertThat(serverReq.getConfigKey().getConfigId(), is(configId));
-        assertThat(serverReq.getDefContent().asStringArray(), is(expectedContent));
-        assertFalse(serverReq.noCache());
-        assertTrue(serverReq.getRequestTrace().toString().contains("hi"));
-        assertThat(serverReq.getRequestConfigMd5(), is(configMd5));
-        assertThat(serverReq.getRequestGeneration(), is(currentGeneration));
-    }
-
-    @Test
-    public void delay_mechanisms_functions() {
-        assertFalse(serverReq.isDelayedResponse());
-        serverReq.setDelayedResponse(true);
-        assertTrue(serverReq.isDelayedResponse());
-        serverReq.setDelayedResponse(false);
-        assertFalse(serverReq.isDelayedResponse());
-    }
-
-    public JRTServerConfigRequest next_request_is_correct_base() {
-        String [] expectedContent = new String[]{
-                "namespace=my.name.space",
-                "myfield string"
-        };
-        JRTServerConfigRequest next = createReq(clientReq.nextRequest(6).getRequest());
-        assertThat(next.getConfigKey().getName(), is(defName));
-        assertThat(next.getConfigKey().getNamespace(), is(defNamespace));
-        assertThat(next.getConfigKey().getMd5(), is(defMd5));
-        assertThat(next.getConfigKey().getConfigId(), is(configId));
-        assertThat(next.getDefContent().asStringArray(), is(expectedContent));
-        assertFalse(next.noCache());
-        assertThat(next.getTimeout(), is(6L));
-        assertThat(next.getTimeout(), is(6L));
-        return next;
-    }
-
-
-    @Test
-    public void next_request_when_error_is_correct() {
-        serverReq.addOkResponse(createPayload(), 999999, false, "newmd5");
-        serverReq.addErrorResponse(ErrorCode.OUTDATED_CONFIG, "error message");
-        System.out.println(serverReq);
-        JRTClientConfigRequest next = clientReq.nextRequest(6);
-        System.out.println(next);
-        // Should use config md5 and generation from the request, not the response
-        // when there are errors
-        assertThat(next.getRequestConfigMd5(), is(clientReq.getRequestConfigMd5()));
-        assertThat(next.getRequestGeneration(), is(clientReq.getRequestGeneration()));
-    }
-
-    @Test
-    public void ok_response_is_added() {
-        Payload payload = createPayload("vale");
-        String md5 = ConfigUtils.getMd5(payload.getData());
-        long generation = 4L;
-        serverReq.addOkResponse(payload, generation, false, md5);
-        assertTrue(clientReq.validateResponse());
-        assertThat(clientReq.getNewPayload().withCompression(CompressionType.UNCOMPRESSED).getData().toString(), is(payload.getData().toString()));
-        assertThat(clientReq.getNewGeneration(), is(4L));
-        assertThat(clientReq.getNewConfigMd5(), is(md5));
-        assertTrue(clientReq.hasUpdatedConfig());
-        assertTrue(clientReq.hasUpdatedGeneration());
-    }
-
-    @Test
-    public void error_response_adds_common_elements() {
-        serverReq.addErrorResponse(ErrorCode.APPLICATION_NOT_LOADED, ErrorCode.getName(ErrorCode.APPLICATION_NOT_LOADED));
-        assertThat(serverReq.getRequest().returnValues().size(), is(1));
-        Slime data = new JsonDecoder().decode(new Slime(), Utf8.toBytes(serverReq.getRequest().returnValues().get(0).asString()));
-        Inspector response = data.get();
-        assertThat(response.field(SlimeResponseData.RESPONSE_DEF_NAME).asString(), is(defName));
-        assertThat(response.field(SlimeResponseData.RESPONSE_DEF_NAMESPACE).asString(), is(defNamespace));
-        assertThat(response.field(SlimeResponseData.RESPONSE_DEF_MD5).asString(), is(defMd5));
-        assertThat(response.field(SlimeResponseData.RESPONSE_CONFIGID).asString(), is(configId));
-        assertThat(response.field(SlimeResponseData.RESPONSE_CLIENT_HOSTNAME).asString(), is(hostname));
-        Trace t = Trace.fromSlime(response.field(SlimeResponseData.RESPONSE_TRACE));
-        assertThat(t.toString(), is(trace.toString()));
-    }
-
-    @Test
-    public void generation_only_is_updated() {
-        Payload payload = createPayload();
-        serverReq.addOkResponse(payload, 4L, false, ConfigUtils.getMd5(payload.getData()));
-        boolean value = clientReq.validateResponse();
-        assertTrue(clientReq.errorMessage(), value);
-        assertFalse(clientReq.hasUpdatedConfig());
-        assertTrue(clientReq.hasUpdatedGeneration());
-    }
-
-    protected static Payload createPayload() {
-        return createPayload("bar");
-    }
-
-    private static Payload createPayload(String value) {
-        Slime slime = new Slime();
-        slime.setObject().setString("myfield", value);
-        return Payload.from(new ConfigPayload(slime));
-    }
-
-    @Test
-    public void nothing_is_updated() {
-        Payload payload = createPayload();
-        serverReq.addOkResponse(payload, currentGeneration, false, configMd5);
-        assertTrue(clientReq.validateResponse());
-        assertFalse(clientReq.hasUpdatedConfig());
-        assertFalse(clientReq.hasUpdatedGeneration());
-    }
-
-    @Test
-    public void payload_is_empty() throws IOException {
-        Payload payload = Payload.from(ConfigPayload.empty());
-        clientReq = createReq(payload);
-        serverReq = createReq(clientReq.getRequest());
-        serverReq.addOkResponse(payload, currentGeneration, false, ConfigUtils.getMd5(payload.getData()));
-        boolean val = clientReq.validateResponse();
-        assertTrue(clientReq.errorMessage(), val);
-        assertFalse(clientReq.hasUpdatedConfig());
-        assertFalse(clientReq.hasUpdatedGeneration());
-    }
-
-    @Test
-    public void request_interface_is_implemented() {
-        JRTClientConfigRequest request = clientReq;
-        assertFalse(request.containsPayload());
-        assertFalse(request.isError());
-        assertThat(request.errorCode(), is(clientReq.getRequest().errorCode()));
-        assertThat(request.errorMessage(), is(clientReq.getRequest().errorMessage()));
-        assertNotNull(request.getRequest());
-        assertFalse(request.validateResponse());
-        //assertNull(request.getNewPayload().getData());
-        assertThat(request.getTimeout(), is(timeout));
-        assertFalse(request.hasUpdatedConfig());
-        assertFalse(request.hasUpdatedGeneration());
-    }
-
-    @Test
-    public void created_from_subscription() {
-        ConfigSubscriber subscriber = new ConfigSubscriber();
-        JRTConfigSubscription<SimpletypesConfig> sub = new JRTConfigSubscription<>(new ConfigKey<>(SimpletypesConfig.class, configId), subscriber, new ConfigSet(), new TimingValues());
-        JRTClientConfigRequest request = createReq(sub, Trace.createNew(9));
-        assertThat(request.getConfigKey().getName(), is(SimpletypesConfig.CONFIG_DEF_NAME));
-        JRTServerConfigRequest serverRequest = createReq(request.getRequest());
-        assertTrue(serverRequest.validateParameters());
-    }
-
-    @Test
-    public void created_from_existing_subscription() {
-        System.setProperty("VESPA_CONFIG_PROTOCOL_VERSION", getProtocolVersion());
-        MockConnection connection = new MockConnection(new MockConnection.AbstractResponseHandler() {
-            @Override
-            protected void createResponse() {
-                JRTServerConfigRequest serverRequest = createReq(request);
-                serverRequest.addOkResponse(createPayload(), currentGeneration, false, configMd5);
-            }
-        });
-
-        ConfigSourceSet src = new ConfigSourceSet();
-        ConfigSubscriber subscriber = new GenericConfigSubscriber(Collections.singletonMap(src, JRTConfigRequester.get(connection, new TimingValues())));
-        JRTConfigSubscription<SimpletypesConfig> sub = new JRTConfigSubscription<>(new ConfigKey<>(SimpletypesConfig.class, configId), subscriber, src, new TimingValues());
-        sub.subscribe(120_0000);
-        assertTrue(sub.nextConfig(120_0000));
-        sub.close();
-        JRTClientConfigRequest nextReq = createReq(sub, Trace.createNew());
-        SimpletypesConfig config = sub.getConfigState().getConfig();
-        assertThat(nextReq.getRequestConfigMd5(), is(config.getConfigMd5()));
-        assertThat(nextReq.getRequestGeneration(), is(currentGeneration));
-        System.setProperty("VESPA_CONFIG_PROTOCOL_VERSION", "");
-    }
-
-    protected abstract String getProtocolVersion();
-
-    @Test
-    public void created_from_raw() throws IOException {
-        RawConfig rawConfig = new RawConfig(new ConfigKey<>(defName, configId, defNamespace), defMd5);
-        long serverTimeout = 100000L;
-        JRTClientConfigRequest request = createFromRaw(rawConfig, serverTimeout, Trace.createNew(9));
-        assertThat(request.getConfigKey().getName(), is(defName));
-        JRTServerConfigRequest serverRequest = createReq(request.getRequest());
-        assertTrue(serverRequest.validateParameters());
-        assertThat(serverRequest.getTimeout(), is(serverTimeout));
-        assertThat(serverRequest.getDefContent().asList(), is(rawConfig.getDefContent()));
-    }
-
-
-    @Test
-    public void parameters_are_validated() throws IOException {
-        assertTrue(serverReq.validateParameters());
-        assertValidationFail(createReq("35#$#!$@#", defNamespace, defMd5, hostname, configId, configMd5, currentGeneration, timeout, trace));
-        assertValidationFail(createReq(defName, "abcd.o#$*(!&$", defMd5, hostname, configId, configMd5, currentGeneration, timeout, trace));
-        assertValidationFail(createReq(defName, defNamespace, "34", hostname, configId, "34", currentGeneration, timeout, trace));
-        assertValidationFail(createReq(defName, defNamespace, defMd5, hostname, configId, "34", currentGeneration, timeout, trace));
-        assertValidationFail(createReq(defName, defNamespace, defMd5, hostname, configId, configMd5, -34, timeout, trace));
-        assertValidationFail(createReq(defName, defNamespace, defMd5, hostname, configId, configMd5, currentGeneration, -23, trace));
-        assertValidationFail(createReq(defName, defNamespace, defMd5, "", configId, configMd5, currentGeneration, timeout, trace));
-    }
-
-    private void assertValidationFail(JRTClientConfigRequest req) {
-        assertFalse(createReq(req.getRequest()).validateParameters());
-    }
-}
diff --git a/config/src/test/java/com/yahoo/vespa/config/protocol/JRTConfigRequestFactoryTest.java b/config/src/test/java/com/yahoo/vespa/config/protocol/JRTConfigRequestFactoryTest.java
index 35332176774..04f3a7abe29 100644
--- a/config/src/test/java/com/yahoo/vespa/config/protocol/JRTConfigRequestFactoryTest.java
+++ b/config/src/test/java/com/yahoo/vespa/config/protocol/JRTConfigRequestFactoryTest.java
@@ -20,41 +20,6 @@ public class JRTConfigRequestFactoryTest {
     private static VespaVersion defaultVespaVersion = JRTConfigRequestFactory.getCompiledVespaVersion();
 
     @Test
-    public void testGetProtocolVersion() {
-        assertThat(JRTConfigRequestFactory.getProtocolVersion("", "", ""), is("3"));
-
-        assertThat(JRTConfigRequestFactory.getProtocolVersion("1", "", ""), is("1"));
-        assertThat(JRTConfigRequestFactory.getProtocolVersion("", "1", ""), is("1"));
-        assertThat(JRTConfigRequestFactory.getProtocolVersion("", "", "1"), is("1"));
-        assertThat(JRTConfigRequestFactory.getProtocolVersion("1", "1", ""), is("1"));
-        assertThat(JRTConfigRequestFactory.getProtocolVersion("1", "", "1"), is("1"));
-        assertThat(JRTConfigRequestFactory.getProtocolVersion("", "1", "1"), is("1"));
-        assertThat(JRTConfigRequestFactory.getProtocolVersion("1", "1", "1"), is("1"));
-
-        assertThat(JRTConfigRequestFactory.getProtocolVersion("2", "", ""), is("2"));
-        assertThat(JRTConfigRequestFactory.getProtocolVersion("", "2", ""), is("2"));
-        assertThat(JRTConfigRequestFactory.getProtocolVersion("", "", "2"), is("2"));
-        assertThat(JRTConfigRequestFactory.getProtocolVersion("2", "2", ""), is("2"));
-        assertThat(JRTConfigRequestFactory.getProtocolVersion("2", "", "2"), is("2"));
-        assertThat(JRTConfigRequestFactory.getProtocolVersion("", "2", "2"), is("2"));
-        assertThat(JRTConfigRequestFactory.getProtocolVersion("2", "2", "2"), is("2"));
-
-        assertThat(JRTConfigRequestFactory.getProtocolVersion("1", "2", ""), is("1"));
-        assertThat(JRTConfigRequestFactory.getProtocolVersion("1", "", "2"), is("1"));
-        assertThat(JRTConfigRequestFactory.getProtocolVersion("", "1", "2"), is("1"));
-        assertThat(JRTConfigRequestFactory.getProtocolVersion("2", "1", ""), is("2"));
-        assertThat(JRTConfigRequestFactory.getProtocolVersion("2", "", "1"), is("2"));
-        assertThat(JRTConfigRequestFactory.getProtocolVersion("", "2", "1"), is("2"));
-
-        assertThat(JRTConfigRequestFactory.getProtocolVersion("1", "2", "2"), is("1"));
-        assertThat(JRTConfigRequestFactory.getProtocolVersion("1", "1", "2"), is("1"));
-        assertThat(JRTConfigRequestFactory.getProtocolVersion("1", "2", "1"), is("1"));
-        assertThat(JRTConfigRequestFactory.getProtocolVersion("2", "1", "1"), is("2"));
-        assertThat(JRTConfigRequestFactory.getProtocolVersion("2", "1", "2"), is("2"));
-        assertThat(JRTConfigRequestFactory.getProtocolVersion("2", "2", "1"), is("2"));
-    }
-
-    @Test
     public void testCompressionType() {
         assertThat(JRTConfigRequestFactory.getCompressionType("", "", ""), is(CompressionType.LZ4));
 
@@ -102,19 +67,8 @@ public class JRTConfigRequestFactoryTest {
         JRTConfigSubscription<FunctionTestConfig> sub = new JRTConfigSubscription<>(
                 new ConfigKey<>(clazz, configId), subscriber, new ConfigSet(), new TimingValues());
 
-        // Default vespa version
         JRTClientConfigRequest request = JRTConfigRequestFactory.createFromSub(sub);
-        assertThat(request.getProtocolVersion(), is(3L));
         assertThat(request.getVespaVersion().get(), is(defaultVespaVersion));
-
-        // Create with vespa version set
-        String version = "5.37.38";
-        System.setProperty(JRTConfigRequestFactory.VESPA_VERSION, version);
-        request = JRTConfigRequestFactory.createFromSub(sub);
-        assertThat(request.getProtocolVersion(), is(3L));
-        assertThat(request.getVespaVersion().get(), is(VespaVersion.fromString(version)));
-
-        System.clearProperty(JRTConfigRequestFactory.VESPA_VERSION);
     }
 
     @Test
@@ -123,19 +77,8 @@ public class JRTConfigRequestFactoryTest {
         final String configId = "foo";
         RawConfig config = new RawConfig(new ConfigKey<>(clazz, configId), "595f44fec1e92a71d3e9e77456ba80d1");
 
-        // Default vespa version
         JRTClientConfigRequest request = JRTConfigRequestFactory.createFromRaw(config, 1000);
-        assertThat(request.getProtocolVersion(), is(3L));
         assertThat(request.getVespaVersion().get(), is(defaultVespaVersion));
-
-        // Create with vespa version set
-        String version = "5.37.38";
-        System.setProperty(JRTConfigRequestFactory.VESPA_VERSION, version);
-        request = JRTConfigRequestFactory.createFromRaw(config, 1000);
-        assertThat(request.getProtocolVersion(), is(3L));
-        assertThat(request.getVespaVersion().get(), is(VespaVersion.fromString(version)));
-
-        System.clearProperty(JRTConfigRequestFactory.VESPA_VERSION);
     }
 
 }
diff --git a/config/src/test/java/com/yahoo/vespa/config/protocol/JRTConfigRequestV3Test.java b/config/src/test/java/com/yahoo/vespa/config/protocol/JRTConfigRequestV3Test.java
index d50a92efc1a..4d1ba2f8793 100644
--- a/config/src/test/java/com/yahoo/vespa/config/protocol/JRTConfigRequestV3Test.java
+++ b/config/src/test/java/com/yahoo/vespa/config/protocol/JRTConfigRequestV3Test.java
@@ -1,59 +1,58 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.config.protocol;
 
+import com.yahoo.config.subscription.ConfigSet;
+import com.yahoo.config.subscription.ConfigSourceSet;
+import com.yahoo.config.subscription.ConfigSubscriber;
+import com.yahoo.config.subscription.impl.GenericConfigSubscriber;
+import com.yahoo.config.subscription.impl.JRTConfigRequester;
+import com.yahoo.config.subscription.impl.MockConnection;
 import com.yahoo.foo.SimpletypesConfig;
 import com.yahoo.config.subscription.impl.JRTConfigSubscription;
 import com.yahoo.jrt.Request;
+import com.yahoo.slime.Inspector;
+import com.yahoo.slime.JsonDecoder;
+import com.yahoo.slime.Slime;
+import com.yahoo.test.ManualClock;
+import com.yahoo.text.Utf8;
 import com.yahoo.vespa.config.*;
 import com.yahoo.vespa.config.util.ConfigUtils;
+import org.junit.Before;
 import org.junit.Test;
 
 import java.util.Arrays;
+import java.util.Collections;
+import java.util.Optional;
 
 import static org.hamcrest.CoreMatchers.is;
 import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertThat;
 import static org.junit.Assert.assertTrue;
 
 /**
  * @author Ulf Lilleengen
- * @since 5.19
  */
-public class JRTConfigRequestV3Test extends JRTConfigRequestBase {
+public class JRTConfigRequestV3Test {
 
-    @Override
-    protected JRTClientConfigRequest createReq(String defName, String defNamespace, String defMd5,
-                                               String hostname, String configId, String configMd5,
-                                               long currentGeneration, long timeout, Trace trace) {
-        return JRTClientConfigRequestV3.createWithParams(ConfigKey.createFull(defName, configId, defNamespace, defMd5),
-                DefContent.fromList(Arrays.asList("namespace=my.name.space", "myfield string")),
-                hostname,
-                configMd5,
-                currentGeneration,
-                timeout,
-                trace,
-                CompressionType.LZ4,
-                vespaVersion);
-    }
-
-    @Override
-    protected JRTServerConfigRequest createReq(Request request) {
-        return JRTServerConfigRequestV3.createFromRequest(request);
-    }
-
-    @Override
-    protected JRTClientConfigRequest createReq(JRTConfigSubscription<SimpletypesConfig> sub, Trace aNew) {
-        return JRTClientConfigRequestV3.createFromSub(sub, aNew, CompressionType.LZ4, vespaVersion);
-    }
+    private final Optional<VespaVersion> vespaVersion = Optional.of(VespaVersion.fromString("5.38.24"));
+    private String defName = "mydef";
+    private String defNamespace = "my.name.space";
+    private String hostname = "myhost";
+    private String configId = "config/id";
+    private String defMd5 = "595f44fec1e92a71d3e9e77456ba80d1";
+    private long currentGeneration = 3;
+    private long timeout = 5000;
+    private Trace trace ;
+    private String configMd5 = ConfigUtils.getMd5(createPayload().getData());
+    private JRTClientConfigRequest clientReq;
+    private JRTServerConfigRequest serverReq;
 
-    @Override
-    protected JRTClientConfigRequest createFromRaw(RawConfig rawConfig, long serverTimeout, Trace aNew) {
-        return JRTClientConfigRequestV3.createFromRaw(rawConfig, serverTimeout, aNew, CompressionType.LZ4, vespaVersion);
-    }
-
-    @Override
-    protected String getProtocolVersion() {
-        return "3";
+    @Before
+    public void setupRequest() {
+        clientReq = createReq();
+        serverReq = createReq(clientReq.getRequest());
+        assertTrue(serverReq.validateParameters());
     }
 
     @Test
@@ -71,11 +70,250 @@ public class JRTConfigRequestV3Test extends JRTConfigRequestBase {
     @Test
     public void emptypayload() {
         ConfigPayload payload = ConfigPayload.empty();
-        SlimeConfigResponse response = SlimeConfigResponse.fromConfigPayload(payload, null, 0, false, ConfigUtils.getMd5(payload));
+        SlimeConfigResponse response = SlimeConfigResponse.fromConfigPayload(payload, 0, false, ConfigUtils.getMd5(payload));
         serverReq.addOkResponse(serverReq.payloadFromResponse(response), response.getGeneration(), false, response.getConfigMd5());
         assertTrue(clientReq.validateResponse());
         assertTrue(clientReq.hasUpdatedGeneration());
         assertThat(clientReq.getNewPayload().withCompression(CompressionType.UNCOMPRESSED).getData().toString(), is("{}"));
         assertFalse(clientReq.responseIsInternalRedeploy());
     }
+
+    @Test
+    public void delay_mechanisms_function() {
+        assertFalse(serverReq.isDelayedResponse());
+        serverReq.setDelayedResponse(true);
+        assertTrue(serverReq.isDelayedResponse());
+        serverReq.setDelayedResponse(false);
+        assertFalse(serverReq.isDelayedResponse());
+    }
+
+    @Test
+    public void next_request_when_error_is_correct() {
+        serverReq.addOkResponse(createPayload(), 999999, false, "newmd5");
+        serverReq.addErrorResponse(ErrorCode.OUTDATED_CONFIG, "error message");
+        System.out.println(serverReq);
+        JRTClientConfigRequest next = clientReq.nextRequest(6);
+        System.out.println(next);
+        // Should use config md5 and generation from the request, not the response
+        // when there are errors
+        assertThat(next.getRequestConfigMd5(), is(clientReq.getRequestConfigMd5()));
+        assertThat(next.getRequestGeneration(), is(clientReq.getRequestGeneration()));
+    }
+
+    @Test
+    public void ok_response_is_added() {
+        Payload payload = createPayload("vale");
+        String md5 = ConfigUtils.getMd5(payload.getData());
+        long generation = 4L;
+        serverReq.addOkResponse(payload, generation, false, md5);
+        assertTrue(clientReq.validateResponse());
+        assertThat(clientReq.getNewPayload().withCompression(CompressionType.UNCOMPRESSED).getData().toString(), is(payload.getData().toString()));
+        assertThat(clientReq.getNewGeneration(), is(4L));
+        assertThat(clientReq.getNewConfigMd5(), is(md5));
+        assertTrue(clientReq.hasUpdatedConfig());
+        assertTrue(clientReq.hasUpdatedGeneration());
+    }
+
+    @Test
+    public void error_response_adds_common_elements() {
+        serverReq.addErrorResponse(ErrorCode.APPLICATION_NOT_LOADED, ErrorCode.getName(ErrorCode.APPLICATION_NOT_LOADED));
+        assertThat(serverReq.getRequest().returnValues().size(), is(1));
+        Slime data = new JsonDecoder().decode(new Slime(), Utf8.toBytes(serverReq.getRequest().returnValues().get(0).asString()));
+        Inspector response = data.get();
+        assertThat(response.field(SlimeResponseData.RESPONSE_DEF_NAME).asString(), is(defName));
+        assertThat(response.field(SlimeResponseData.RESPONSE_DEF_NAMESPACE).asString(), is(defNamespace));
+        assertThat(response.field(SlimeResponseData.RESPONSE_DEF_MD5).asString(), is(defMd5));
+        assertThat(response.field(SlimeResponseData.RESPONSE_CONFIGID).asString(), is(configId));
+        assertThat(response.field(SlimeResponseData.RESPONSE_CLIENT_HOSTNAME).asString(), is(hostname));
+        Trace t = Trace.fromSlime(response.field(SlimeResponseData.RESPONSE_TRACE));
+        assertThat(t.toString(), is(trace.toString()));
+    }
+
+    @Test
+    public void generation_only_is_updated() {
+        Payload payload = createPayload();
+        serverReq.addOkResponse(payload, 4L, false, ConfigUtils.getMd5(payload.getData()));
+        boolean value = clientReq.validateResponse();
+        assertTrue(clientReq.errorMessage(), value);
+        assertFalse(clientReq.hasUpdatedConfig());
+        assertTrue(clientReq.hasUpdatedGeneration());
+    }
+
+    @Test
+    public void nothing_is_updated() {
+        Payload payload = createPayload();
+        serverReq.addOkResponse(payload, currentGeneration, false, configMd5);
+        assertTrue(clientReq.validateResponse());
+        assertFalse(clientReq.hasUpdatedConfig());
+        assertFalse(clientReq.hasUpdatedGeneration());
+    }
+
+    @Test
+    public void payload_is_empty() {
+        Payload payload = Payload.from(ConfigPayload.empty());
+        clientReq = createReq(payload);
+        serverReq = createReq(clientReq.getRequest());
+        serverReq.addOkResponse(payload, currentGeneration, false, ConfigUtils.getMd5(payload.getData()));
+        boolean val = clientReq.validateResponse();
+        assertTrue(clientReq.errorMessage(), val);
+        assertFalse(clientReq.hasUpdatedConfig());
+        assertFalse(clientReq.hasUpdatedGeneration());
+    }
+
+    @Test
+    public void request_interface_is_implemented() {
+        JRTClientConfigRequest request = clientReq;
+        assertFalse(request.isError());
+        assertThat(request.errorCode(), is(clientReq.getRequest().errorCode()));
+        assertThat(request.errorMessage(), is(clientReq.getRequest().errorMessage()));
+        assertNotNull(request.getRequest());
+        assertFalse(request.validateResponse());
+        //assertNull(request.getNewPayload().getData());
+        assertThat(request.getTimeout(), is(timeout));
+        assertFalse(request.hasUpdatedConfig());
+        assertFalse(request.hasUpdatedGeneration());
+    }
+
+    @Test
+    public void created_from_subscription() {
+        ConfigSubscriber subscriber = new ConfigSubscriber();
+        JRTConfigSubscription<SimpletypesConfig> sub = new JRTConfigSubscription<>(new ConfigKey<>(SimpletypesConfig.class, configId), subscriber, new ConfigSet(), new TimingValues());
+        JRTClientConfigRequest request = createReq(sub, Trace.createNew(9));
+        assertThat(request.getConfigKey().getName(), is(SimpletypesConfig.CONFIG_DEF_NAME));
+        JRTServerConfigRequest serverRequest = createReq(request.getRequest());
+        assertTrue(serverRequest.validateParameters());
+    }
+
+    @Test
+    public void created_from_existing_subscription() {
+        MockConnection connection = new MockConnection(new MockConnection.AbstractResponseHandler() {
+            @Override
+            public void createResponse() {
+                JRTServerConfigRequest serverRequest = createReq(request);
+                serverRequest.addOkResponse(createPayload(), currentGeneration, false, configMd5);
+            }
+        });
+
+        ConfigSourceSet src = new ConfigSourceSet();
+        ConfigSubscriber subscriber = new GenericConfigSubscriber(Collections.singletonMap(src, new JRTConfigRequester(connection, new TimingValues())));
+        JRTConfigSubscription<SimpletypesConfig> sub = new JRTConfigSubscription<>(new ConfigKey<>(SimpletypesConfig.class, configId), subscriber, src, new TimingValues());
+        sub.subscribe(120_0000);
+        assertTrue(sub.nextConfig(120_0000));
+        sub.close();
+        JRTClientConfigRequest nextReq = createReq(sub, Trace.createNew());
+        SimpletypesConfig config = sub.getConfigState().getConfig();
+        assertThat(nextReq.getRequestConfigMd5(), is(config.getConfigMd5()));
+        assertThat(nextReq.getRequestGeneration(), is(currentGeneration));
+    }
+
+    @Test
+    public void created_from_raw() {
+        RawConfig rawConfig = new RawConfig(new ConfigKey<>(defName, configId, defNamespace), defMd5);
+        long serverTimeout = 100000L;
+        JRTClientConfigRequest request = createFromRaw(rawConfig, serverTimeout, Trace.createNew(9));
+        assertThat(request.getConfigKey().getName(), is(defName));
+        JRTServerConfigRequest serverRequest = createReq(request.getRequest());
+        assertTrue(serverRequest.validateParameters());
+        assertThat(serverRequest.getTimeout(), is(serverTimeout));
+        assertThat(serverRequest.getDefContent().asList(), is(rawConfig.getDefContent()));
+    }
+
+    @Test
+    public void parameters_are_validated() {
+        assertTrue(serverReq.validateParameters());
+        assertValidationFail(createReq("35#$#!$@#", defNamespace, defMd5, hostname, configId, configMd5, currentGeneration, timeout, trace));
+        assertValidationFail(createReq(defName, "abcd.o#$*(!&$", defMd5, hostname, configId, configMd5, currentGeneration, timeout, trace));
+        assertValidationFail(createReq(defName, defNamespace, "34", hostname, configId, "34", currentGeneration, timeout, trace));
+        assertValidationFail(createReq(defName, defNamespace, defMd5, hostname, configId, "34", currentGeneration, timeout, trace));
+        assertValidationFail(createReq(defName, defNamespace, defMd5, hostname, configId, configMd5, -34, timeout, trace));
+        assertValidationFail(createReq(defName, defNamespace, defMd5, hostname, configId, configMd5, currentGeneration, -23, trace));
+        assertValidationFail(createReq(defName, defNamespace, defMd5, "", configId, configMd5, currentGeneration, timeout, trace));
+    }
+
+    private void assertValidationFail(JRTClientConfigRequest req) {
+        assertFalse(createReq(req.getRequest()).validateParameters());
+    }
+
+    private static Payload createPayload() {
+        return createPayload("bar");
+    }
+
+    private static Payload createPayload(String value) {
+        Slime slime = new Slime();
+        slime.setObject().setString("myfield", value);
+        return Payload.from(new ConfigPayload(slime));
+    }
+
+    private JRTClientConfigRequest createReq(String defName, String defNamespace, String defMd5,
+                                             String hostname, String configId, String configMd5,
+                                             long currentGeneration, long timeout, Trace trace) {
+        return JRTClientConfigRequestV3.createWithParams(ConfigKey.createFull(defName, configId, defNamespace, defMd5),
+                                                         DefContent.fromList(Arrays.asList("namespace=my.name.space", "myfield string")),
+                                                         hostname,
+                                                         configMd5,
+                                                         currentGeneration,
+                                                         timeout,
+                                                         trace,
+                                                         CompressionType.LZ4,
+                                                         vespaVersion);
+    }
+
+    private JRTServerConfigRequest createReq(Request request) {
+        return JRTServerConfigRequestV3.createFromRequest(request);
+    }
+
+    private JRTClientConfigRequest createReq(JRTConfigSubscription<SimpletypesConfig> sub, Trace aNew) {
+        return JRTClientConfigRequestV3.createFromSub(sub, aNew, CompressionType.LZ4, vespaVersion);
+    }
+
+    private JRTClientConfigRequest createFromRaw(RawConfig rawConfig, long serverTimeout, Trace aNew) {
+        return JRTClientConfigRequestV3.createFromRaw(rawConfig, serverTimeout, aNew, CompressionType.LZ4, vespaVersion);
+    }
+
+    private JRTClientConfigRequest createReq() {
+        trace = Trace.createNew(3, new ManualClock());
+        trace.trace(1, "hei");
+        return createReq(defName, defNamespace, defMd5, hostname, configId, configMd5, currentGeneration, timeout, trace);
+    }
+
+    private JRTClientConfigRequest createReq(Payload payload) {
+        trace = Trace.createNew(3, new ManualClock());
+        trace.trace(1, "hei");
+        return createReq(defName, defNamespace, defMd5, hostname, configId, ConfigUtils.getMd5(payload.getData()), currentGeneration, timeout, trace);
+    }
+
+    private void request_is_parsed_base() {
+        String [] expectedContent = new String[]{
+                "namespace=my.name.space",
+                "myfield string"
+        };
+        System.out.println(serverReq.toString());
+        assertThat(serverReq.getConfigKey().getName(), is(defName));
+        assertThat(serverReq.getConfigKey().getNamespace(), is(defNamespace));
+        assertThat(serverReq.getConfigKey().getMd5(), is(defMd5));
+        assertThat(serverReq.getConfigKey().getConfigId(), is(configId));
+        assertThat(serverReq.getDefContent().asStringArray(), is(expectedContent));
+        assertFalse(serverReq.noCache());
+        assertTrue(serverReq.getRequestTrace().toString().contains("hi"));
+        assertThat(serverReq.getRequestConfigMd5(), is(configMd5));
+        assertThat(serverReq.getRequestGeneration(), is(currentGeneration));
+    }
+
+    private JRTServerConfigRequest next_request_is_correct_base() {
+        String [] expectedContent = new String[]{
+                "namespace=my.name.space",
+                "myfield string"
+        };
+        JRTServerConfigRequest next = createReq(clientReq.nextRequest(6).getRequest());
+        assertThat(next.getConfigKey().getName(), is(defName));
+        assertThat(next.getConfigKey().getNamespace(), is(defNamespace));
+        assertThat(next.getConfigKey().getMd5(), is(defMd5));
+        assertThat(next.getConfigKey().getConfigId(), is(configId));
+        assertThat(next.getDefContent().asStringArray(), is(expectedContent));
+        assertFalse(next.noCache());
+        assertThat(next.getTimeout(), is(6L));
+        assertThat(next.getTimeout(), is(6L));
+        return next;
+    }
+
 }
diff --git a/config/src/test/java/com/yahoo/vespa/config/protocol/PayloadTest.java b/config/src/test/java/com/yahoo/vespa/config/protocol/PayloadTest.java
index e5fc5190ad1..c52245ecb35 100644
--- a/config/src/test/java/com/yahoo/vespa/config/protocol/PayloadTest.java
+++ b/config/src/test/java/com/yahoo/vespa/config/protocol/PayloadTest.java
@@ -15,7 +15,6 @@ import static org.junit.Assert.assertThat;
 
 /**
  * @author Ulf Lilleengen
- * @since 5.21
  */
 public class PayloadTest {
 
diff --git a/config/src/test/java/com/yahoo/vespa/config/protocol/SlimeTraceSerializerTest.java b/config/src/test/java/com/yahoo/vespa/config/protocol/SlimeTraceSerializerTest.java
index f2d9edc7c35..22baab85c1c 100644
--- a/config/src/test/java/com/yahoo/vespa/config/protocol/SlimeTraceSerializerTest.java
+++ b/config/src/test/java/com/yahoo/vespa/config/protocol/SlimeTraceSerializerTest.java
@@ -21,9 +21,9 @@ import static org.junit.Assert.assertTrue;
 
 /**
  * @author Ulf Lilleengen
- * @since 5.5
  */
 public class SlimeTraceSerializerTest {
+
     @Test
     public void test_serializer() throws IOException {
         TraceNode root = new TraceNode(null, 1);
diff --git a/config/src/test/java/com/yahoo/vespa/config/protocol/TraceTest.java b/config/src/test/java/com/yahoo/vespa/config/protocol/TraceTest.java
index 596983eebb8..4b191d85121 100644
--- a/config/src/test/java/com/yahoo/vespa/config/protocol/TraceTest.java
+++ b/config/src/test/java/com/yahoo/vespa/config/protocol/TraceTest.java
@@ -9,7 +9,6 @@ import static org.junit.Assert.assertTrue;
 
 /**
  * @author Ulf Lilleengen
- * @since 5.3
  */
 public class TraceTest {
 
@@ -58,4 +57,5 @@ public class TraceTest {
         assertTrue(trace2Str.contains("barbaz"));
         assertTrue(trace2Str.contains("quux"));
     }
+
 }
diff --git a/config/src/tests/trace/trace.cpp b/config/src/tests/trace/trace.cpp
index 41c874eb1d4..9a355f39ecc 100644
--- a/config/src/tests/trace/trace.cpp
+++ b/config/src/tests/trace/trace.cpp
@@ -2,6 +2,7 @@
 
 #include <vespa/vespalib/testkit/test_kit.h>
 #include <vespa/config/common/trace.h>
+#include <vespa/vespalib/data/slime/slime.h>
 
 
 using namespace config;
diff --git a/config/src/vespa/config/common/configdefinition.cpp b/config/src/vespa/config/common/configdefinition.cpp
index 861fc224867..92af068cff5 100644
--- a/config/src/vespa/config/common/configdefinition.cpp
+++ b/config/src/vespa/config/common/configdefinition.cpp
@@ -1,6 +1,7 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 #include "configdefinition.h"
 #include <vespa/vespalib/stllike/asciistream.h>
+#include <vespa/vespalib/data/slime/slime.h>
 
 using namespace vespalib;
 using namespace vespalib::slime;
diff --git a/config/src/vespa/config/common/configdefinition.h b/config/src/vespa/config/common/configdefinition.h
index 3d86e5d2567..8067bf0aea8 100644
--- a/config/src/vespa/config/common/configdefinition.h
+++ b/config/src/vespa/config/common/configdefinition.h
@@ -3,8 +3,11 @@
 
 #include <vespa/vespalib/stllike/string.h>
 #include <vector>
-#include <vespa/vespalib/data/slime/slime.h>
 
+namespace vespalib::slime {
+    struct Cursor;
+    struct Inspector;
+}
 namespace config {
 
 /**
diff --git a/config/src/vespa/config/common/trace.cpp b/config/src/vespa/config/common/trace.cpp
index d1bb154eda9..76310d08c7d 100644
--- a/config/src/vespa/config/common/trace.cpp
+++ b/config/src/vespa/config/common/trace.cpp
@@ -2,6 +2,7 @@
 #include "trace.h"
 #include <vespa/vespalib/trace/slime_trace_serializer.h>
 #include <vespa/vespalib/trace/slime_trace_deserializer.h>
+#include <vespa/vespalib/data/slime/slime.h>
 
 using namespace vespalib;
 using namespace vespalib::slime;
diff --git a/config/src/vespa/config/common/trace.h b/config/src/vespa/config/common/trace.h
index 772cdb6f31e..9abd4cc53a4 100644
--- a/config/src/vespa/config/common/trace.h
+++ b/config/src/vespa/config/common/trace.h
@@ -1,11 +1,14 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 #pragma once
 
-#include <vespa/vespalib/data/slime/slime.h>
 #include <vespa/vespalib/trace/tracenode.h>
 #include <vespa/vespalib/stllike/string.h>
-#include <memory>
+#include <vespa/vespalib/data/memory.h>
 
+namespace vespalib::slime {
+    struct Cursor;
+    struct Inspector;
+}
 namespace config {
 
 /**
diff --git a/config/src/vespa/config/configgen/value_converter.cpp b/config/src/vespa/config/configgen/value_converter.cpp
index 1f78e78cc32..06086939d58 100644
--- a/config/src/vespa/config/configgen/value_converter.cpp
+++ b/config/src/vespa/config/configgen/value_converter.cpp
@@ -7,9 +7,7 @@
 using namespace vespalib;
 using namespace vespalib::slime;
 
-namespace config {
-
-namespace internal {
+namespace config::internal {
 
 template<>
 int32_t convertValue(const ::vespalib::slime::Inspector & __inspector) {
@@ -64,4 +62,3 @@ requireValid(const vespalib::string & __fieldName, const ::vespalib::slime::Insp
 
 }
 
-}
diff --git a/config/src/vespa/config/configgen/value_converter.h b/config/src/vespa/config/configgen/value_converter.h
index d2d04169c1d..6ad67bf6f86 100644
--- a/config/src/vespa/config/configgen/value_converter.h
+++ b/config/src/vespa/config/configgen/value_converter.h
@@ -5,9 +5,7 @@
 #include <vespa/vespalib/stllike/string.h>
 #include <vespa/vespalib/data/slime/inspector.h>
 
-namespace config {
-
-namespace internal {
+namespace config::internal {
 
 void requireValid(const vespalib::string & __fieldName, const ::vespalib::slime::Inspector & __inspector);
 
@@ -43,7 +41,4 @@ struct ValueConverter {
     }
 };
 
-} // namespace internal
-
-} // namespace config
-
+}
diff --git a/configd/src/apps/sentinel/sentinel.cpp b/configd/src/apps/sentinel/sentinel.cpp
index b062d7526b6..9f179e96393 100644
--- a/configd/src/apps/sentinel/sentinel.cpp
+++ b/configd/src/apps/sentinel/sentinel.cpp
@@ -1,12 +1,14 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
+#include "config-handler.h"
 #include <vespa/config/common/exceptions.h>
-#include <unistd.h>
-#include <sys/time.h>
 #include <vespa/vespalib/util/signalhandler.h>
 #include <vespa/vespalib/util/exceptions.h>
 #include <vespa/defaults.h>
-#include "config-handler.h"
+#include <chrono>
+#include <string>
+#include <unistd.h>
+#include <sys/time.h>
 
 #include <vespa/log/log.h>
 LOG_SETUP("config-sentinel");
@@ -31,7 +33,7 @@ main(int argc, char **argv)
         exit(EXIT_FAILURE);
     }
 
-    const char *configId = strdup(optarg);
+    std::string configId(optarg);
 
     const char *rootDir = getenv("ROOT");
     if (!rootDir) {
diff --git a/configdefinitions/src/vespa/attributes.def b/configdefinitions/src/vespa/attributes.def
index f9db9eb2f0d..604ddd40930 100644
--- a/configdefinitions/src/vespa/attributes.def
+++ b/configdefinitions/src/vespa/attributes.def
@@ -30,3 +30,9 @@ attribute[].densepostinglistthreshold   double default=0.40
 attribute[].tensortype         string default=""
 # Whether this is an imported attribute (from parent document db) or not.
 attribute[].imported           bool default=false
+
+# Configuration parameters for a hnsw index used together with a 1-dimensional indexed tensor for approximate nearest neighbor search.
+attribute[].index.hnsw.enabled bool default=false
+attribute[].index.hnsw.maxlinkspernode int default=16
+attribute[].index.hnsw.distancemetric enum { EUCLIDEAN, ANGULAR, GEODEGREES } default=EUCLIDEAN
+attribute[].index.hnsw.neighborstoexploreatinsert int default=200
diff --git a/configdefinitions/src/vespa/dispatch.def b/configdefinitions/src/vespa/dispatch.def
index 77f6dffd714..0776e648ad7 100644
--- a/configdefinitions/src/vespa/dispatch.def
+++ b/configdefinitions/src/vespa/dispatch.def
@@ -23,6 +23,15 @@ distributionPolicy enum { ROUNDROBIN, ADAPTIVE } default=ROUNDROBIN
 ## don't use it if you don't (really) mean it.
 maxHitsPerNode int default=2147483647
 
+## Probability for getting the K best hits (topK).
+## A value of 1.0 will ask all N partitions for K hits.
+## Any value between <0, 1> will use a Student T with 30 degrees freedom and compute a value Q that
+## will give you the globally K best hits according to this formula with the desired probability.
+## q = k/n + qT (p',30) x √(k × (1/n) × (1 − 1/n))
+## With a probability of 0.999 and K=200 and N=10 will give a Q of 38, meaning that you only need to fetch 19% compared to
+## default setting of 1.0. This is a significant optimisation with with very little loss in presicion.
+topKProbability double default=1.0
+
 # Is multi-level dispatch configured for this cluster
 # Deprecated, will go away soon, NOOP
 useMultilevelDispatch bool default=false
@@ -48,6 +57,9 @@ numJrtTransportThreads int default=8
 # Number of JRT connections per backend node
 numJrtConnectionsPerNode int default=8
 
+# Number of seconds to spend warming up code to prevent JIT cold start issues.
+warmuptime double default=0.1
+
 # The unique key of a search node
 node[].key int
 
diff --git a/configdefinitions/src/vespa/lb-services.def b/configdefinitions/src/vespa/lb-services.def
index 8d5e7015947..cc496a99c20 100644
--- a/configdefinitions/src/vespa/lb-services.def
+++ b/configdefinitions/src/vespa/lb-services.def
@@ -4,10 +4,12 @@
 
 namespace=cloud.config
 
+# Enable proxy-protocol for nginx upstreams
+nginxUpstreamProxyProtocol                                      bool default=false
+
 # Active rotation given as flag 'active' for a prod region in deployment.xml
 # Default true for now (since code in config-model to set it is not ready yet), should have no default value
 tenants{}.applications{}.activeRotation                         bool default=true
-tenants{}.applications{}.use4443Upstream                        bool default=false
 
 tenants{}.applications{}.hosts{}.hostname                       string default="(unknownhostname)"
 tenants{}.applications{}.hosts{}.services{}.type                string default="(noservicetype)"
diff --git a/configdefinitions/src/vespa/stor-filestor.def b/configdefinitions/src/vespa/stor-filestor.def
index c02a9018064..7816ee74fde 100644
--- a/configdefinitions/src/vespa/stor-filestor.def
+++ b/configdefinitions/src/vespa/stor-filestor.def
@@ -24,7 +24,7 @@ disk_operation_timeout int default=0 restart
 ## PERFORMANCE PARAMETERS
 
 ## Number of threads to use for each mountpoint.
-num_threads int default=6 restart
+num_threads int default=8 restart
 
 ## When merging, if we find more than this number of documents that exist on all
 ## of the same copies, send a separate apply bucket diff with these entries
diff --git a/configserver/pom.xml b/configserver/pom.xml
index 1e242988327..8cd1b4b4254 100644
--- a/configserver/pom.xml
+++ b/configserver/pom.xml
@@ -215,11 +215,6 @@
       <artifactId>jersey-proxy-client</artifactId>
     </dependency>
     <dependency>
-      <groupId>net.jpountz.lz4</groupId>
-      <artifactId>lz4</artifactId>
-      <scope>compile</scope>
-    </dependency>
-    <dependency>
       <groupId>org.mockito</groupId>
       <artifactId>mockito-core</artifactId>
       <scope>test</scope>
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
index 92e7bf0300b..95c8733b540 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
@@ -314,8 +314,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
         tenant.getLocalSessionRepo().addSession(newSession);
 
         return Optional.of(Deployment.unprepared(newSession, this, hostProvisioner, tenant, timeout, clock,
-                                                 false /* don't validate as this is already deployed */, newSession.getVespaVersion(),
-                                                 bootstrap));
+                                                 false /* don't validate as this is already deployed */, bootstrap));
     }
 
     @Override
@@ -893,11 +892,10 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
         public void close() {
             metric.set(name,
                        Duration.between(start, clock.instant()).toMillis(),
-                       metric.createContext(Map.of("tenant", id.tenant().value(),
-                                                   "application", id.application().value(),
-                                                   "instance", id.instance().value(),
-                                                   "environment", environment,
-                                                   "region", region)));
+                       metric.createContext(Map.of("applicationId", id.toFullString(),
+                                                   "tenantName", id.tenant().value(),
+                                                   "app", id.application().value() + "." + id.instance().value(),
+                                                   "zone", environment + "." + region)));
         }
 
     }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java
index b2a10f4bb21..2a426e4cccc 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java
@@ -79,7 +79,7 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable
 
     // For testing only
     ConfigServerBootstrap(ApplicationRepository applicationRepository, RpcServer server, VersionState versionState,
-                          StateMonitor stateMonitor, VipStatus vipStatus, Mode mode,  VipStatusMode vipStatusMode) {
+                          StateMonitor stateMonitor, VipStatus vipStatus, Mode mode, VipStatusMode vipStatusMode) {
         this(applicationRepository, server, versionState, stateMonitor, vipStatus, mode, CONTINUE, vipStatusMode);
     }
 
@@ -231,20 +231,17 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable
                                                                 new DaemonThreadFactory("redeploy apps"));
         // Keep track of deployment per application
         Map<ApplicationId, Future<?>> futures = new HashMap<>();
-        Set<ApplicationId> failedDeployments = new HashSet<>();
-
-        for (ApplicationId appId : applicationIds) {
-            Optional<Deployment> deploymentOptional = applicationRepository.deployFromLocalActive(appId, true /* bootstrap */);
-            if (deploymentOptional.isEmpty()) continue;
-
-            futures.put(appId, executor.submit(deploymentOptional.get()::activate));
-        }
+        applicationIds.forEach(applicationId -> futures.put(applicationId, executor.submit(() -> {
+            applicationRepository.deployFromLocalActive(applicationId, true /* bootstrap */)
+                    .ifPresent(Deployment::activate);
+        })));
 
+        Set<ApplicationId> failedDeployments = new HashSet<>();
         for (Map.Entry<ApplicationId, Future<?>> f : futures.entrySet()) {
-            ApplicationId app = f.getKey();
             try {
                 f.getValue().get();
             } catch (ExecutionException e) {
+                ApplicationId app = f.getKey();
                 if (e.getCause() instanceof TransientException) {
                     log.log(LogLevel.INFO, "Redeploying " + app +
                             " failed with transient error, will retry after bootstrap: " + Exceptions.toMessageString(e));
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelManager.java b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelManager.java
index ea835206b7c..d46c551fa7e 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelManager.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelManager.java
@@ -11,6 +11,7 @@ import com.yahoo.config.model.api.SuperModelProvider;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.NodeFlavors;
 import com.yahoo.config.provision.Zone;
+import com.yahoo.log.LogLevel;
 import com.yahoo.vespa.config.GenerationCounter;
 import com.yahoo.vespa.config.server.application.ApplicationSet;
 import com.yahoo.vespa.config.server.model.SuperModelConfigProvider;
@@ -18,14 +19,19 @@ import com.yahoo.vespa.flags.FlagSource;
 
 import java.time.Instant;
 import java.util.ArrayList;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Optional;
+import java.util.Set;
+import java.util.logging.Logger;
 
 /**
  * Provides a SuperModel - a model of all application instances,  and makes it stays
  * up to date as applications are added, redeployed, and removed.
  */
 public class SuperModelManager implements SuperModelProvider {
+    private static final Logger logger = Logger.getLogger(SuperModelManager.class.getName());
+
     private final Zone zone;
 
     private final Object monitor = new Object();
@@ -38,6 +44,9 @@ public class SuperModelManager implements SuperModelProvider {
     private final long masterGeneration; // ConfigserverConfig's generation
     private final GenerationCounter generationCounter;
 
+    // The initial set of applications to be deployed on bootstrap.
+    private Optional<Set<ApplicationId>> bootstrapApplicationSet = Optional.empty();
+
     @Inject
     public SuperModelManager(ConfigserverConfig configserverConfig,
                              NodeFlavors nodeFlavors,
@@ -75,6 +84,10 @@ public class SuperModelManager implements SuperModelProvider {
             listeners.add(listener);
             SuperModel superModel = superModelConfigProvider.getSuperModel();
             superModel.getAllApplicationInfos().forEach(application -> listener.applicationActivated(superModel, application));
+
+            if (superModel.isComplete()) {
+                listener.notifyOfCompleteness(superModel);
+            }
         }
     }
 
@@ -89,28 +102,37 @@ public class SuperModelManager implements SuperModelProvider {
                     .getForVersionOrLatest(Optional.empty(), Instant.now())
                     .toApplicationInfo();
 
-            SuperModel newSuperModel = this.superModelConfigProvider
-                    .getSuperModel()
+            SuperModel newSuperModel = superModelConfigProvider.getSuperModel()
                     .cloneAndSetApplication(applicationInfo);
+
             generationCounter.increment();
             makeNewSuperModelConfigProvider(newSuperModel);
-            listeners.stream().forEach(listener ->
-                    listener.applicationActivated(newSuperModel, applicationInfo));
+            listeners.forEach(listener -> listener.applicationActivated(newSuperModel, applicationInfo));
         }
     }
 
     public void applicationRemoved(ApplicationId applicationId) {
         synchronized (monitor) {
+            bootstrapApplicationSet.ifPresent(set -> set.remove(applicationId));
+
             SuperModel newSuperModel = this.superModelConfigProvider
                     .getSuperModel()
                     .cloneAndRemoveApplication(applicationId);
             generationCounter.increment();
             makeNewSuperModelConfigProvider(newSuperModel);
-            listeners.stream().forEach(listener ->
-                    listener.applicationRemoved(newSuperModel, applicationId));
+            listeners.forEach(listener -> listener.applicationRemoved(newSuperModel, applicationId));
         }
     }
 
+    public void markAsComplete() {
+        // Invoked on component graph bootstrap (even before ConfigServerBootstrap),
+        // there is no need to bump generation counter.
+        logger.log(LogLevel.INFO, "Super model is complete");
+        SuperModel newSuperModel = getSuperModel().cloneAsComplete();
+        superModelConfigProvider = new SuperModelConfigProvider(newSuperModel, zone, flagSource);
+        listeners.forEach(listener -> listener.notifyOfCompleteness(newSuperModel));
+    }
+
     private void makeNewSuperModelConfigProvider(SuperModel newSuperModel) {
         generation = masterGeneration + generationCounter.get();
         superModelConfigProvider = new SuperModelConfigProvider(newSuperModel, zone, flagSource);
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelRequestHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelRequestHandler.java
index 520972f2fcf..51c5be1b1f0 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelRequestHandler.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelRequestHandler.java
@@ -125,5 +125,7 @@ public class SuperModelRequestHandler implements RequestHandler {
 
     public void enable() {
         enabled = true;
+        superModelManager.markAsComplete();
+        updateHandler();
     }
 }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java
index 7ec0f49ed60..7ff5d41485d 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java
@@ -21,10 +21,8 @@ import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent;
 
 import java.time.Duration;
 import java.util.List;
-import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.Executor;
 import java.util.concurrent.ExecutorService;
 import java.util.logging.Logger;
@@ -49,7 +47,6 @@ public class TenantApplications {
     private final Path locksPath;
     private final Curator.DirectoryCache directoryCache;
     private final ReloadHandler reloadHandler;
-    private final Map<ApplicationId, Lock> locks;
     private final Executor zkWatcherExecutor;
 
     private TenantApplications(Curator curator, ReloadHandler reloadHandler, TenantName tenant,
@@ -57,7 +54,6 @@ public class TenantApplications {
         this.curator = curator;
         this.applicationsPath = TenantRepository.getApplicationsPath(tenant);
         this.locksPath = TenantRepository.getLocksPath(tenant);
-        this.locks = new ConcurrentHashMap<>(2);
         this.reloadHandler = reloadHandler;
         this.zkWatcherExecutor = command -> zkWatcherExecutor.execute(tenant, command);
         this.directoryCache = curator.createDirectoryCache(applicationsPath.getAbsolute(), false, false, zkCacheExecutor);
@@ -148,10 +144,7 @@ public class TenantApplications {
 
     /** Returns the lock for changing the session status of the given application. */
     public Lock lock(ApplicationId id) {
-        curator.create(lockPath(id));
-        Lock lock = locks.computeIfAbsent(id, __ -> new Lock(lockPath(id).getAbsolute(), curator));
-        lock.acquire(Duration.ofMinutes(1)); // These locks shouldn't be held for very long.
-        return lock;
+        return curator.lock(lockPath(id), Duration.ofMinutes(1)); // These locks shouldn't be held for very long.
     }
 
     private void childEvent(CuratorFramework client, PathChildrenCacheEvent event) {
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java
index 89d7c349d6b..4bd5cf30cc6 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java
@@ -4,10 +4,9 @@ package com.yahoo.vespa.config.server.deploy;
 import com.yahoo.component.Version;
 import com.yahoo.config.application.api.DeployLogger;
 import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.AthenzDomain;
 import com.yahoo.config.provision.HostFilter;
 import com.yahoo.config.provision.Provisioner;
-import com.yahoo.config.provision.zone.ZoneId;
-import com.yahoo.jdisc.Metric;
 import com.yahoo.log.LogLevel;
 import com.yahoo.transaction.NestedTransaction;
 import com.yahoo.transaction.Transaction;
@@ -25,7 +24,6 @@ import com.yahoo.vespa.curator.Lock;
 
 import java.time.Clock;
 import java.time.Duration;
-import java.time.Instant;
 import java.util.Optional;
 import java.util.logging.Logger;
 
@@ -49,13 +47,19 @@ public class Deployment implements com.yahoo.config.provision.Deployment {
     private final Duration timeout;
     private final Clock clock;
     private final DeployLogger logger = new SilentDeployLogger();
-    
+
+    /** The repository part of docker image this application should run on. Version is separate from image repo */
+    Optional<String> dockerImageRepository;
+
     /** The Vespa version this application should run on */
     private final Version version;
 
     /** True if this deployment is done to bootstrap the config server */
     private final boolean isBootstrap;
 
+    /** The (optional) Athenz domain this application should use */
+    private final Optional<AthenzDomain> athenzDomain;
+
     private boolean prepared = false;
     
     /** Whether this model should be validated (only takes effect if prepared=false) */
@@ -64,9 +68,8 @@ public class Deployment implements com.yahoo.config.provision.Deployment {
     private boolean ignoreSessionStaleFailure = false;
 
     private Deployment(LocalSession session, ApplicationRepository applicationRepository,
-                       Optional<Provisioner> hostProvisioner, Tenant tenant,
-                       Duration timeout, Clock clock, boolean prepared, boolean validate, Version version,
-                       boolean isBootstrap) {
+                       Optional<Provisioner> hostProvisioner, Tenant tenant, Duration timeout,
+                       Clock clock, boolean prepared, boolean validate, boolean isBootstrap) {
         this.session = session;
         this.applicationRepository = applicationRepository;
         this.hostProvisioner = hostProvisioner;
@@ -75,23 +78,24 @@ public class Deployment implements com.yahoo.config.provision.Deployment {
         this.clock = clock;
         this.prepared = prepared;
         this.validate = validate;
-        this.version = version;
+        this.dockerImageRepository = session.getDockerImageRepository();
+        this.version = session.getVespaVersion();
         this.isBootstrap = isBootstrap;
+        this.athenzDomain = session.getAthenzDomain();
     }
 
     public static Deployment unprepared(LocalSession session, ApplicationRepository applicationRepository,
                                         Optional<Provisioner> hostProvisioner, Tenant tenant,
-                                        Duration timeout, Clock clock, boolean validate, Version version,
-                                        boolean isBootstrap) {
-        return new Deployment(session, applicationRepository, hostProvisioner, tenant,
-                              timeout, clock, false, validate, version, isBootstrap);
+                                        Duration timeout, Clock clock, boolean validate, boolean isBootstrap) {
+        return new Deployment(session, applicationRepository, hostProvisioner, tenant, timeout, clock, false,
+                              validate, isBootstrap);
     }
 
     public static Deployment prepared(LocalSession session, ApplicationRepository applicationRepository,
                                       Optional<Provisioner> hostProvisioner, Tenant tenant,
                                       Duration timeout, Clock clock, boolean isBootstrap) {
         return new Deployment(session, applicationRepository, hostProvisioner, tenant,
-                              timeout, clock, true, true, session.getVespaVersion(), isBootstrap);
+                              timeout, clock, true, true, isBootstrap);
     }
 
     public void setIgnoreSessionStaleFailure(boolean ignoreSessionStaleFailure) {
@@ -105,16 +109,14 @@ public class Deployment implements com.yahoo.config.provision.Deployment {
         try (ActionTimer timer = applicationRepository.timerFor(session.getApplicationId(), "deployment.prepareMillis")) {
             TimeoutBudget timeoutBudget = new TimeoutBudget(clock, timeout);
 
-            session.prepare(logger,
-                            new PrepareParams.Builder().applicationId(session.getApplicationId())
-                                                       .timeoutBudget(timeoutBudget)
-                                                       .ignoreValidationErrors(!validate)
-                                                       .vespaVersion(version.toString())
-                                                       .isBootstrap(isBootstrap)
-                                                       .build(),
-                            Optional.empty(),
-                            tenant.getPath(),
-                            clock.instant());
+            PrepareParams.Builder params = new PrepareParams.Builder().applicationId(session.getApplicationId())
+                    .timeoutBudget(timeoutBudget)
+                    .ignoreValidationErrors(!validate)
+                    .vespaVersion(version.toString())
+                    .isBootstrap(isBootstrap);
+            dockerImageRepository.ifPresent(params::dockerImageRepository);
+            athenzDomain.ifPresent(params::athenzDomain);
+            session.prepare(logger, params.build(), Optional.empty(), tenant.getPath(), clock.instant());
             this.prepared = true;
         }
     }
@@ -129,10 +131,12 @@ public class Deployment implements com.yahoo.config.provision.Deployment {
             TimeoutBudget timeoutBudget = new TimeoutBudget(clock, timeout);
 
             ApplicationId applicationId = session.getApplicationId();
+            LocalSession previousActiveSession;
             try (Lock lock = tenant.getApplicationRepo().lock(applicationId)) {
                 validateSessionStatus(session);
                 NestedTransaction transaction = new NestedTransaction();
-                transaction.add(deactivateCurrentActivateNew(applicationRepository.getActiveSession(applicationId), session, ignoreSessionStaleFailure));
+                previousActiveSession = applicationRepository.getActiveSession(applicationId);
+                transaction.add(deactivateCurrentActivateNew(previousActiveSession, session, ignoreSessionStaleFailure));
                 hostProvisioner.ifPresent(provisioner -> provisioner.activate(transaction, applicationId, session.getAllocatedHosts().getHosts()));
                 transaction.commit();
             }
@@ -147,8 +151,9 @@ public class Deployment implements com.yahoo.config.provision.Deployment {
 
             log.log(LogLevel.INFO, session.logPre() + "Session " + session.getSessionId() +
                                    " activated successfully using " +
-                                   (hostProvisioner.isPresent() ? hostProvisioner.get() : "no host provisioner") +
+                                   (hostProvisioner.isPresent() ? hostProvisioner.get().getClass().getSimpleName() : "no host provisioner") +
                                    ". Config generation " + session.getMetaData().getGeneration() +
+                                   (previousActiveSession != null ? ". Activated session based on previous active session " + previousActiveSession.getSessionId() : "") +
                                    ". File references used: " + applicationRepository.getFileReferences(applicationId));
         }
     }
@@ -166,14 +171,13 @@ public class Deployment implements com.yahoo.config.provision.Deployment {
     /** Exposes the session of this for testing only */
     public LocalSession session() { return session; }
 
-    private long validateSessionStatus(LocalSession localSession) {
+    private void validateSessionStatus(LocalSession localSession) {
         long sessionId = localSession.getSessionId();
         if (Session.Status.NEW.equals(localSession.getStatus())) {
             throw new IllegalStateException(localSession.logPre() + "Session " + sessionId + " is not prepared");
         } else if (Session.Status.ACTIVATE.equals(localSession.getStatus())) {
             throw new IllegalStateException(localSession.logPre() + "Session " + sessionId + " is already active");
         }
-        return sessionId;
     }
 
     private Transaction deactivateCurrentActivateNew(LocalSession active, LocalSession prepared, boolean ignoreStaleSessionFailure) {
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java
index 23bdf972a06..400460d2ce6 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java
@@ -12,8 +12,10 @@ import com.yahoo.config.model.api.HostProvisioner;
 import com.yahoo.config.model.api.Model;
 import com.yahoo.config.model.api.ModelContext;
 import com.yahoo.config.model.api.EndpointCertificateSecrets;
+import com.yahoo.config.model.api.Provisioned;
 import com.yahoo.config.model.api.TlsSecrets;
 import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.AthenzDomain;
 import com.yahoo.config.provision.HostName;
 import com.yahoo.config.provision.Zone;
 import com.yahoo.vespa.flags.FetchVector;
@@ -40,9 +42,12 @@ public class ModelContextImpl implements ModelContext {
     private final ConfigDefinitionRepo configDefinitionRepo;
     private final FileRegistry fileRegistry;
     private final Optional<HostProvisioner> hostProvisioner;
+    private final Provisioned provisioned;
     private final ModelContext.Properties properties;
     private final Optional<File> appDir;
 
+    private final Optional<String> wantedDockerImageRepository;
+
     /** The version of Vespa we are building a model for */
     private final Version modelVespaVersion;
 
@@ -62,8 +67,10 @@ public class ModelContextImpl implements ModelContext {
                             ConfigDefinitionRepo configDefinitionRepo,
                             FileRegistry fileRegistry,
                             Optional<HostProvisioner> hostProvisioner,
+                            Provisioned provisioned,
                             ModelContext.Properties properties,
                             Optional<File> appDir,
+                            Optional<String> wantedDockerImageRepository,
                             Version modelVespaVersion,
                             Version wantedNodeVespaVersion) {
         this.applicationPackage = applicationPackage;
@@ -73,8 +80,10 @@ public class ModelContextImpl implements ModelContext {
         this.configDefinitionRepo = configDefinitionRepo;
         this.fileRegistry = fileRegistry;
         this.hostProvisioner = hostProvisioner;
+        this.provisioned = provisioned;
         this.properties = properties;
         this.appDir = appDir;
+        this.wantedDockerImageRepository = wantedDockerImageRepository;
         this.modelVespaVersion = modelVespaVersion;
         this.wantedNodeVespaVersion = wantedNodeVespaVersion;
     }
@@ -97,6 +106,9 @@ public class ModelContextImpl implements ModelContext {
     public Optional<HostProvisioner> hostProvisioner() { return hostProvisioner; }
 
     @Override
+    public Provisioned provisioned() { return provisioned; }
+
+    @Override
     public DeployLogger deployLogger() { return deployLogger; }
 
     @Override
@@ -112,6 +124,9 @@ public class ModelContextImpl implements ModelContext {
     public Optional<File> appDir() { return appDir; }
 
     @Override
+    public Optional<String> wantedDockerImageRepository() { return wantedDockerImageRepository; }
+
+    @Override
     public Version modelVespaVersion() { return modelVespaVersion; }
 
     @Override
@@ -131,8 +146,12 @@ public class ModelContextImpl implements ModelContext {
         private final boolean isBootstrap;
         private final boolean isFirstTimeDeployment;
         private final boolean useAdaptiveDispatch;
+        private final double defaultTopKprobability;
         private final Optional<EndpointCertificateSecrets> endpointCertificateSecrets;
         private final double defaultTermwiseLimit;
+        private final double defaultSoftStartSeconds;
+        private final String proxyProtocol;
+        private final Optional<AthenzDomain> athenzDomain;
 
         public Properties(ApplicationId applicationId,
                           boolean multitenantFromConfig,
@@ -146,7 +165,8 @@ public class ModelContextImpl implements ModelContext {
                           boolean isBootstrap,
                           boolean isFirstTimeDeployment,
                           FlagSource flagSource,
-                          Optional<EndpointCertificateSecrets> endpointCertificateSecrets) {
+                          Optional<EndpointCertificateSecrets> endpointCertificateSecrets,
+                          Optional<AthenzDomain> athenzDomain) {
             this.applicationId = applicationId;
             this.multitenant = multitenantFromConfig || hostedVespa || Boolean.getBoolean("multitenant");
             this.configServerSpecs = configServerSpecs;
@@ -163,6 +183,13 @@ public class ModelContextImpl implements ModelContext {
             this.endpointCertificateSecrets = endpointCertificateSecrets;
             defaultTermwiseLimit = Flags.DEFAULT_TERM_WISE_LIMIT.bindTo(flagSource)
                     .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value();
+            defaultSoftStartSeconds = Flags.DEFAULT_SOFT_START_SECONDS.bindTo(flagSource)
+                    .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value();
+            defaultTopKprobability = Flags.DEFAULT_TOP_K_PROBABILITY.bindTo(flagSource)
+                    .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value();
+            this.proxyProtocol = Flags.PROXY_PROTOCOL.bindTo(flagSource)
+                    .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value();
+            this.athenzDomain = athenzDomain;
         }
 
         @Override
@@ -214,9 +241,28 @@ public class ModelContextImpl implements ModelContext {
         @Override
         public double defaultTermwiseLimit() { return defaultTermwiseLimit; }
 
+        public double defaultSoftStartSeconds() {
+            return 0;
+        }
+
+        @Override
+        public double defaultTopKProbability() {
+            return defaultTopKprobability;
+        }
+
         // TODO: Remove
         @Override
         public boolean useBucketSpaceMetric() { return true; }
+
+        @Override
+        public boolean useNewAthenzFilter() { return true; }
+
+        @Override
+        public String proxyProtocol() { return proxyProtocol; }
+
+        @Override
+        public Optional<AthenzDomain> athenzDomain() { return athenzDomain; }
+
     }
 
 }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java
index 13ef19f5f5d..09f5178cf6b 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java
@@ -87,17 +87,16 @@ public class ZooKeeperClient {
 
     /** Sets the app id and attempts to set up zookeeper. The app id must be ordered for purge to work OK. */
     private void createZooKeeperNodes() {
-        if (!configCurator.exists(rootPath.getAbsolute())) {
+        if ( ! configCurator.exists(rootPath.getAbsolute()))
             configCurator.createNode(rootPath.getAbsolute());
-        }
 
-        for (String subPath : Arrays.asList(
-                ConfigCurator.DEFCONFIGS_ZK_SUBPATH,
-                ConfigCurator.USER_DEFCONFIGS_ZK_SUBPATH,
-                ConfigCurator.USERAPP_ZK_SUBPATH,
-                ZKApplicationPackage.fileRegistryNode)) {
-            // TODO The replaceFirst below is hackish.
-            configCurator.createNode(getZooKeeperAppPath(null).getAbsolute(), subPath.replaceFirst("/", ""));
+        for (String subPath : Arrays.asList(ConfigCurator.DEFCONFIGS_ZK_SUBPATH,
+                                            ConfigCurator.USER_DEFCONFIGS_ZK_SUBPATH,
+                                            ConfigCurator.USERAPP_ZK_SUBPATH,
+                                            ZKApplicationPackage.fileRegistryNode)) {
+            // TODO: The replaceFirst below is hackish.
+            configCurator.createNode(getZooKeeperAppPath(null).getAbsolute(),
+                                     subPath.replaceFirst("/", ""));
         }
     }
 
@@ -108,7 +107,6 @@ public class ZooKeeperClient {
      */
     void write(ApplicationPackage app) {
         logFine("Feeding application config into ZooKeeper");
-        // gives lots and lots of debug output: // BasicConfigurator.configure();
         try {
             logFine("Feeding user def files into ZooKeeper");
             writeUserDefs(app);
@@ -121,43 +119,34 @@ public class ZooKeeperClient {
             write(app.getMetaData());
         } catch (Exception e) {
             throw new IllegalStateException("Unable to write vespa model to config server(s) " + System.getProperty("configsources") + "\n" +
-                    "Please ensure that cloudconfig_server is started on the config server node(s), " +
-                    "and check the vespa log for configserver errors. ", e);
+                                            "Please ensure that cloudconfig_server is started on the config server node(s), " +
+                                            "and check the vespa log for configserver errors. ", e);
         }
     }
 
     private void writeSearchDefinitions(ApplicationPackage app) throws IOException {
         Collection<NamedReader> sds = app.getSearchDefinitions();
-        if (sds.isEmpty()) {
-            return;
-        }
+        if (sds.isEmpty()) return;
+
+        // TODO: Change to SCHEMAS_DIR after March 2020
         Path zkPath = getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH).append(ApplicationPackage.SEARCH_DEFINITIONS_DIR);
         configCurator.createNode(zkPath.getAbsolute());
-        // Ensures that ranking expressions and other files are also fed.
+        // Ensures that ranking expressions and other files are also written
         writeDir(app.getFile(ApplicationPackage.SEARCH_DEFINITIONS_DIR), zkPath, false);
+        writeDir(app.getFile(ApplicationPackage.SCHEMAS_DIR), zkPath, false);
         for (NamedReader sd : sds) {
-            String name = sd.getName();
-            Reader reader = sd.getReader();
-            String data = com.yahoo.io.IOUtils.readAll(reader);
-            reader.close();
-            configCurator.putData(zkPath.getAbsolute(), name, data);
+            configCurator.putData(zkPath.getAbsolute(), sd.getName(), com.yahoo.io.IOUtils.readAll(sd.getReader()));
+            sd.getReader().close();
         }
     }
 
     /**
      * Puts some of the application package files into ZK - see write(app).
      *
-     * @param app The application package to use as input.
-     * @throws java.io.IOException  if not able to write to Zookeeper
+     * @param app the application package to use as input.
+     * @throws java.io.IOException if not able to write to Zookeeper
      */
     private void writeSomeOf(ApplicationPackage app) throws IOException {
-        ApplicationFile.PathFilter srFilter = new ApplicationFile.PathFilter() {
-            @Override
-            public boolean accept(Path path) {
-                return path.getName().endsWith(ApplicationPackage.RULES_NAME_SUFFIX);
-            }
-        };
-        // Copy app package files and subdirs into zk
         // TODO: We should have a way of doing this which doesn't require repeating all the content
         writeFile(app.getFile(Path.fromString(ApplicationPackage.SERVICES)),
                   getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH));
@@ -169,7 +158,8 @@ public class ZooKeeperClient {
                   getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH));
         writeDir(app.getFile(ApplicationPackage.RULES_DIR),
                  getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH).append(ApplicationPackage.RULES_DIR),
-                 srFilter, true);
+                 (path) -> path.getName().endsWith(ApplicationPackage.RULES_NAME_SUFFIX),
+                 true);
         writeDir(app.getFile(ApplicationPackage.QUERY_PROFILES_DIR),
                  getZooKeeperAppPath(ConfigCurator.USERAPP_ZK_SUBPATH).append(ApplicationPackage.QUERY_PROFILES_DIR),
                  xmlFilter, true);
@@ -194,20 +184,12 @@ public class ZooKeeperClient {
     }
 
     private void writeDir(ApplicationFile file, Path zooKeeperAppPath, boolean recurse) throws IOException {
-        writeDir(file, zooKeeperAppPath, new ApplicationFile.PathFilter() {
-            @Override
-            public boolean accept(Path path) {
-                return true;
-            }
-        }, recurse);
+        writeDir(file, zooKeeperAppPath, (__) -> true, recurse);
     }
 
     private void writeDir(ApplicationFile dir, Path path, ApplicationFile.PathFilter filenameFilter, boolean recurse) throws IOException {
-        if (!dir.isDirectory()) {
-            logger.log(LogLevel.FINE, dir.getPath().getAbsolute()+" is not a directory. Not feeding the files into ZooKeeper.");
-            return;
-        }
-        for (ApplicationFile file: listFiles(dir, filenameFilter)) {
+        if ( ! dir.isDirectory()) return;
+        for (ApplicationFile file : listFiles(dir, filenameFilter)) {
             String name = file.getPath().getName();
             if (name.startsWith(".")) continue; //.svn , .git ...
             if ("CVS".equals(name)) continue;
@@ -223,7 +205,8 @@ public class ZooKeeperClient {
     }
 
     /**
-     * Like {@link ApplicationFile#listFiles(com.yahoo.config.application.api.ApplicationFile.PathFilter)} with a slightly different semantic. Never filter out directories.
+     * Like {@link ApplicationFile#listFiles(com.yahoo.config.application.api.ApplicationFile.PathFilter)}
+     * with slightly different semantics: Never filter out directories.
      */
     private List<ApplicationFile> listFiles(ApplicationFile dir, ApplicationFile.PathFilter filter) {
         List<ApplicationFile> rawList = dir.listFiles();
@@ -243,9 +226,8 @@ public class ZooKeeperClient {
     }
 
     private void writeFile(ApplicationFile file, Path zkPath) throws IOException {
-        if (!file.exists()) {
-            return;
-        }
+        if ( ! file.exists()) return;
+
         ByteArrayOutputStream baos = new ByteArrayOutputStream();
         try (InputStream inputStream = file.createInputStream()) {
             inputStream.transferTo(baos);
@@ -292,8 +274,8 @@ public class ZooKeeperClient {
         String exportedRegistry = PreGeneratedFileRegistry.exportRegistry(fileRegistry);
 
         configCurator.putData(getZooKeeperAppPath(null).append(ZKApplicationPackage.fileRegistryNode).getAbsolute(),
-                vespaVersion.toFullString(),
-                exportedRegistry);
+                              vespaVersion.toFullString(),
+                              exportedRegistry);
     }
 
     /**
@@ -343,9 +325,8 @@ public class ZooKeeperClient {
     }
 
     public void write(AllocatedHosts hosts) throws IOException {
-        configCurator.putData(
-                rootPath.append(ZKApplicationPackage.allocatedHostsNode).getAbsolute(),
-                AllocatedHostsSerializer.toJson(hosts));
+        configCurator.putData(rootPath.append(ZKApplicationPackage.allocatedHostsNode).getAbsolute(),
+                              AllocatedHostsSerializer.toJson(hosts));
     }
 
     public void write(Map<Version, FileRegistry> fileRegistryMap) {
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/TesterClient.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/TesterClient.java
index 1418b2a0bcf..2d5cce2d4f1 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/TesterClient.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/TesterClient.java
@@ -59,8 +59,7 @@ public class TesterClient {
     }
 
     private HttpResponse execute(HttpUriRequest request, String messageIfRequestFails) {
-        // TODO: Change log level to DEBUG
-        logger.log(LogLevel.INFO, "Sending request to tester container " + request.getURI().toString());
+        logger.log(LogLevel.DEBUG, "Sending request to tester container " + request.getURI().toString());
         try {
             return new ProxyResponse(httpClient.execute(request));
         } catch (IOException e) {
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/status/StatusHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/status/StatusHandler.java
index bb752f9b804..0014d66026b 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/status/StatusHandler.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/status/StatusHandler.java
@@ -8,7 +8,7 @@ import com.yahoo.container.jdisc.HttpRequest;
 import com.yahoo.container.jdisc.HttpResponse;
 import com.yahoo.slime.Cursor;
 import com.yahoo.vespa.config.ConfigPayload;
-import com.yahoo.vespa.config.SlimeUtils;
+import com.yahoo.slime.SlimeUtils;
 import com.yahoo.vespa.config.server.GlobalComponentRegistry;
 import com.yahoo.vespa.config.server.http.HttpHandler;
 import com.yahoo.vespa.config.server.http.JSONResponse;
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java
index 833c9ca4fba..3c909730902 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java
@@ -245,7 +245,6 @@ public class ApplicationHandler extends HttpHandler {
     }
 
     private static boolean isTesterStartTestsRequest(HttpRequest request) {
-        System.out.println(getBindingMatch(request).groupCount());
         return getBindingMatch(request).groupCount() == 9 &&
                request.getUri().getPath().contains("/tester/run/");
     }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/Maintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/Maintainer.java
index d2fd4efa7dc..e40d9153ad1 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/Maintainer.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/Maintainer.java
@@ -48,7 +48,7 @@ public abstract class Maintainer extends AbstractComponent implements Runnable {
     @Override
     @SuppressWarnings({"try", "unused"})
     public void run() {
-        try (Lock lock = lock(lockRoot.append(name()))) {
+        try (Lock lock = curator.lock(lockRoot.append(name()), Duration.ofSeconds(1))) {
             maintain();
         } catch (UncheckedTimeoutException e) {
             // another config server instance is running this job at the moment; ok
@@ -57,12 +57,6 @@ public abstract class Maintainer extends AbstractComponent implements Runnable {
         }
     }
 
-    private Lock lock(Path path) {
-        Lock lock = new Lock(path.getAbsolute(), curator);
-        lock.acquire(Duration.ofSeconds(1));
-        return lock;
-    }
-
     @Override
     public void deconstruct() {
         this.service.shutdown();
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterMetricsRetriever.java b/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterMetricsRetriever.java
index 34c7dced404..4f64b73b403 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterMetricsRetriever.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterMetricsRetriever.java
@@ -6,12 +6,13 @@ import com.yahoo.log.LogLevel;
 import com.yahoo.slime.ArrayTraverser;
 import com.yahoo.slime.Inspector;
 import com.yahoo.slime.Slime;
-import com.yahoo.vespa.config.SlimeUtils;
+import com.yahoo.slime.SlimeUtils;
 import com.yahoo.yolean.Exceptions;
 import org.apache.http.client.config.RequestConfig;
 import org.apache.http.client.methods.CloseableHttpResponse;
 import org.apache.http.client.methods.HttpGet;
 import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
 
 import java.io.IOException;
 import java.io.InputStream;
@@ -42,12 +43,15 @@ public class ClusterMetricsRetriever {
     private static final List<String> WANTED_METRIC_SERVICES = List.of(VESPA_CONTAINER, VESPA_QRSERVER, VESPA_DISTRIBUTOR);
 
 
-    private static final CloseableHttpClient httpClient = VespaHttpClientBuilder.create()
-            .setDefaultRequestConfig(RequestConfig.custom()
-                    .setConnectTimeout(10 * 1000)
-                    .setSocketTimeout(10 * 1000)
-                    .build())
-            .build();
+
+    private static final CloseableHttpClient httpClient = VespaHttpClientBuilder
+                                                            .create(PoolingHttpClientConnectionManager::new)
+                                                            .setDefaultRequestConfig(
+                                                                    RequestConfig.custom()
+                                                                            .setConnectTimeout(10 * 1000)
+                                                                            .setSocketTimeout(10 * 1000)
+                                                                            .build())
+                                                            .build();
 
     /**
      * Call the metrics API on each host and aggregate the metrics
@@ -62,7 +66,7 @@ public class ClusterMetricsRetriever {
                     getHostMetrics(host, clusterMetricsMap)
                 );
 
-        ForkJoinPool threadPool = new ForkJoinPool(5);
+        ForkJoinPool threadPool = new ForkJoinPool(10);
         threadPool.submit(retrieveMetricsJob);
         threadPool.shutdown();
 
@@ -102,7 +106,7 @@ public class ClusterMetricsRetriever {
             return slime;
         } catch (IOException e) {
             // Usually caused by applications being deleted during metric retrieval
-            log.warning("Was unable to fetch metrics from " + hostURI + " : " + Exceptions.toMessageString(e));
+            log.info("Was unable to fetch metrics from " + hostURI + " : " + Exceptions.toMessageString(e));
             return new Slime();
         }
     }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/model/LbServicesProducer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/model/LbServicesProducer.java
index d88fae0a8ef..4fe2ec129d3 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/model/LbServicesProducer.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/model/LbServicesProducer.java
@@ -10,7 +10,6 @@ import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.TenantName;
 import com.yahoo.config.provision.Zone;
 import com.yahoo.vespa.flags.BooleanFlag;
-import com.yahoo.vespa.flags.FetchVector;
 import com.yahoo.vespa.flags.FlagSource;
 import com.yahoo.vespa.flags.Flags;
 
@@ -35,16 +34,17 @@ public class LbServicesProducer implements LbServicesConfig.Producer {
 
     private final Map<TenantName, Set<ApplicationInfo>> models;
     private final Zone zone;
-    private final BooleanFlag use4443Upstream;
+    private final BooleanFlag nginxUpstreamProxyProtocol;
 
     public LbServicesProducer(Map<TenantName, Set<ApplicationInfo>> models, Zone zone, FlagSource flagSource) {
         this.models = models;
         this.zone = zone;
-        this.use4443Upstream = Flags.USE_4443_UPSTREAM.bindTo(flagSource);
+        this.nginxUpstreamProxyProtocol = Flags.NGINX_UPSTREAM_PROXY_PROTOCOL.bindTo(flagSource);
     }
 
     @Override
     public void getConfig(LbServicesConfig.Builder builder) {
+        builder.nginxUpstreamProxyProtocol(nginxUpstreamProxyProtocol.value());
         models.keySet().stream()
                 .sorted()
                 .forEach(tenant -> {
@@ -56,10 +56,15 @@ public class LbServicesProducer implements LbServicesConfig.Producer {
         LbServicesConfig.Tenants.Builder tb = new LbServicesConfig.Tenants.Builder();
         apps.stream()
                 .sorted(Comparator.comparing(ApplicationInfo::getApplicationId))
+                .filter(applicationInfo -> generateRoutingConfig(applicationInfo.getApplicationId()))
                 .forEach(applicationInfo -> tb.applications(createLbAppIdKey(applicationInfo.getApplicationId()), getAppConfig(applicationInfo)));
         return tb;
     }
 
+    private boolean generateRoutingConfig(ApplicationId applicationId) {
+        return ( ! applicationId.instance().isTester());
+    }
+
     private String createLbAppIdKey(ApplicationId applicationId) {
         return applicationId.application().value() + ":" + zone.environment().value() + ":" + zone.region().value() + ":" + applicationId.instance().value();
     }
@@ -67,8 +72,6 @@ public class LbServicesProducer implements LbServicesConfig.Producer {
     private LbServicesConfig.Tenants.Applications.Builder getAppConfig(ApplicationInfo app) {
         LbServicesConfig.Tenants.Applications.Builder ab = new LbServicesConfig.Tenants.Applications.Builder();
         ab.activeRotation(getActiveRotation(app));
-        ab.use4443Upstream(
-                use4443Upstream.with(FetchVector.Dimension.APPLICATION_ID, app.getApplicationId().serializedForm()).value());
         app.getModel().getHosts().stream()
                 .sorted((a, b) -> a.getHostname().compareTo(b.getHostname()))
                 .forEach(hostInfo -> ab.hosts(hostInfo.getHostname(), getHostsConfig(hostInfo)));
@@ -92,10 +95,7 @@ public class LbServicesProducer implements LbServicesConfig.Producer {
     private LbServicesConfig.Tenants.Applications.Hosts.Builder getHostsConfig(HostInfo hostInfo) {
         LbServicesConfig.Tenants.Applications.Hosts.Builder hb = new LbServicesConfig.Tenants.Applications.Hosts.Builder();
         hb.hostname(hostInfo.getHostname());
-        hostInfo.getServices().stream()
-                 .forEach(serviceInfo -> {
-                     hb.services(serviceInfo.getServiceName(), getServiceConfig(serviceInfo));
-                 });
+        hostInfo.getServices().forEach(serviceInfo -> hb.services(serviceInfo.getServiceName(), getServiceConfig(serviceInfo)));
         return hb;
     }
 
@@ -114,12 +114,11 @@ public class LbServicesProducer implements LbServicesConfig.Producer {
                                 filter(prop -> !"".equals(prop)).sorted((a, b) -> a.compareTo(b)).collect(Collectors.toList()))
                 .endpointaliases(endpointAliases)
                 .index(Integer.parseInt(serviceInfo.getProperty("index").orElse("999999")));
-        serviceInfo.getPorts().stream()
-                .forEach(portInfo -> {
-                    LbServicesConfig.Tenants.Applications.Hosts.Services.Ports.Builder pb = new LbServicesConfig.Tenants.Applications.Hosts.Services.Ports.Builder()
-                        .number(portInfo.getPort())
-                        .tags(Joiner.on(" ").join(portInfo.getTags()));
-                    sb.ports(pb);
+        serviceInfo.getPorts().forEach(portInfo -> {
+            LbServicesConfig.Tenants.Applications.Hosts.Services.Ports.Builder pb = new LbServicesConfig.Tenants.Applications.Hosts.Services.Ports.Builder()
+                    .number(portInfo.getPort())
+                    .tags(Joiner.on(" ").join(portInfo.getTags()));
+            sb.ports(pb);
                 });
         return sb;
     }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java
index a2fc2bfd6a0..99bec2c9db7 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java
@@ -8,6 +8,7 @@ import com.yahoo.config.application.api.DeployLogger;
 import com.yahoo.config.model.api.ConfigDefinitionRepo;
 import com.yahoo.config.model.api.ModelContext;
 import com.yahoo.config.model.api.ModelFactory;
+import com.yahoo.config.model.api.Provisioned;
 import com.yahoo.config.model.application.provider.MockFileRegistry;
 import com.yahoo.config.provision.AllocatedHosts;
 import com.yahoo.config.provision.ApplicationId;
@@ -83,12 +84,14 @@ public class ActivatedModelsBuilder extends ModelsBuilder<Application> {
     protected Application buildModelVersion(ModelFactory modelFactory,
                                             ApplicationPackage applicationPackage,
                                             ApplicationId applicationId,
+                                            Optional<String> wantedDockerImageRepository,
                                             Version wantedNodeVespaVersion,
                                             Optional<AllocatedHosts> ignored, // Ignored since we have this in the app package for activated models
                                             Instant now) {
         log.log(LogLevel.DEBUG, String.format("Loading model version %s for session %s application %s",
                                               modelFactory.version(), appGeneration, applicationId));
         ModelContext.Properties modelContextProperties = createModelContextProperties(applicationId);
+        Provisioned provisioned = new Provisioned();
         ModelContext modelContext = new ModelContextImpl(
                 applicationPackage,
                 Optional.empty(),
@@ -96,9 +99,13 @@ public class ActivatedModelsBuilder extends ModelsBuilder<Application> {
                 logger,
                 configDefinitionRepo,
                 getForVersionOrLatest(applicationPackage.getFileRegistries(), modelFactory.version()).orElse(new MockFileRegistry()),
-                createStaticProvisioner(applicationPackage.getAllocatedHosts(), modelContextProperties),
+                createStaticProvisioner(applicationPackage.getAllocatedHosts(),
+                                        modelContextProperties.applicationId(),
+                                        provisioned),
+                provisioned,
                 modelContextProperties,
                 Optional.empty(),
+                wantedDockerImageRepository,
                 modelFactory.version(),
                 wantedNodeVespaVersion);
         MetricUpdater applicationMetricUpdater = metrics.getOrCreateMetricUpdater(Metrics.createDimensions(applicationId));
@@ -138,7 +145,8 @@ public class ActivatedModelsBuilder extends ModelsBuilder<Application> {
                                                flagSource,
                                                new EndpointCertificateMetadataStore(curator, TenantRepository.getTenantPath(tenant))
                                                        .readEndpointCertificateMetadata(applicationId)
-                                                       .flatMap(new EndpointCertificateRetriever(secretStore)::readEndpointCertificateSecrets));
+                                                       .flatMap(new EndpointCertificateRetriever(secretStore)::readEndpointCertificateSecrets),
+                                               zkClient.readAthenzDomain());
 
     }
 
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java
index 2de64ed145c..cc4378ae05e 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java
@@ -7,6 +7,7 @@ import com.yahoo.config.application.api.ApplicationPackage;
 import com.yahoo.config.model.api.HostProvisioner;
 import com.yahoo.config.model.api.ModelContext;
 import com.yahoo.config.model.api.ModelFactory;
+import com.yahoo.config.model.api.Provisioned;
 import com.yahoo.config.provision.AllocatedHosts;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.ApplicationLockException;
@@ -72,6 +73,7 @@ public abstract class ModelsBuilder<MODELRESULT extends ModelResult> {
      *                       and assigns to this SettableOptional such that it can be used after this method returns
      */
     public List<MODELRESULT> buildModels(ApplicationId applicationId,
+                                         Optional<String> dockerImageRepository,
                                          Version wantedNodeVespaVersion,
                                          ApplicationPackage applicationPackage,
                                          SettableOptional<AllocatedHosts> allocatedHosts,
@@ -104,8 +106,9 @@ public abstract class ModelsBuilder<MODELRESULT extends ModelResult> {
             int majorVersion = majorVersions.get(i);
             try {
                 allApplicationModels.addAll(buildModelVersions(keepMajorVersion(majorVersion, versions),
-                                                               applicationId, wantedNodeVespaVersion, applicationPackage,
-                                                               allocatedHosts, now, buildLatestModelForThisMajor, majorVersion));
+                                                               applicationId, dockerImageRepository, wantedNodeVespaVersion,
+                                                               applicationPackage, allocatedHosts, now,
+                                                               buildLatestModelForThisMajor, majorVersion));
                 buildLatestModelForThisMajor = false; // We have successfully built latest model version, do it only for this major
             }
             catch (OutOfCapacityException | ApplicationLockException | TransientException e) {
@@ -146,6 +149,7 @@ public abstract class ModelsBuilder<MODELRESULT extends ModelResult> {
     // versions is the set of versions for one particular major version
     private List<MODELRESULT> buildModelVersions(Set<Version> versions,
                                                  ApplicationId applicationId,
+                                                 Optional<String> wantedDockerImageRepository,
                                                  Version wantedNodeVespaVersion,
                                                  ApplicationPackage applicationPackage,
                                                  SettableOptional<AllocatedHosts> allocatedHosts,
@@ -160,6 +164,7 @@ public abstract class ModelsBuilder<MODELRESULT extends ModelResult> {
             MODELRESULT latestModelVersion = buildModelVersion(modelFactoryRegistry.getFactory(latest.get()),
                                                                applicationPackage,
                                                                applicationId,
+                                                               wantedDockerImageRepository,
                                                                wantedNodeVespaVersion,
                                                                allocatedHosts.asOptional(),
                                                                now);
@@ -182,6 +187,7 @@ public abstract class ModelsBuilder<MODELRESULT extends ModelResult> {
                 modelVersion = buildModelVersion(modelFactoryRegistry.getFactory(version),
                                                  applicationPackage,
                                                  applicationId,
+                                                 wantedDockerImageRepository,
                                                  wantedNodeVespaVersion,
                                                  allocatedHosts.asOptional(),
                                                  now);
@@ -192,8 +198,10 @@ public abstract class ModelsBuilder<MODELRESULT extends ModelResult> {
                 // config models (which is always true for manually deployed zones)
                 if (allApplicationVersions.size() > 0 && allApplicationVersions.get(0).getModel().skipOldConfigModels(now))
                     log.log(LogLevel.INFO, applicationId + ": Skipping old version (due to validation override)");
-                else
+                else {
+                    log.log(LogLevel.ERROR, applicationId + ": Failed to build version " + version);
                     throw e;
+                }
             }
         }
         return allApplicationVersions;
@@ -236,9 +244,8 @@ public abstract class ModelsBuilder<MODELRESULT extends ModelResult> {
     }
 
     protected abstract MODELRESULT buildModelVersion(ModelFactory modelFactory, ApplicationPackage applicationPackage,
-                                                     ApplicationId applicationId, 
-                                                     Version wantedNodeVespaVersion,
-                                                     Optional<AllocatedHosts> allocatedHosts,
+                                                     ApplicationId applicationId, Optional<String> dockerImageRepository,
+                                                     Version wantedNodeVespaVersion, Optional<AllocatedHosts> allocatedHosts,
                                                      Instant now);
 
     /**
@@ -246,15 +253,17 @@ public abstract class ModelsBuilder<MODELRESULT extends ModelResult> {
      * returns empty otherwise, which may either mean that no hosts are allocated or that we are running
      * non-hosted and should default to use hosts defined in the application package, depending on context
      */
-    Optional<HostProvisioner> createStaticProvisioner(Optional<AllocatedHosts> allocatedHosts, ModelContext.Properties properties) {
+    Optional<HostProvisioner> createStaticProvisioner(Optional<AllocatedHosts> allocatedHosts,
+                                                      ApplicationId applicationId,
+                                                      Provisioned provisioned) {
         if (hosted && allocatedHosts.isPresent())
-            return Optional.of(new StaticProvisioner(allocatedHosts.get(), createNodeRepositoryProvisioner(properties).get()));
+            return Optional.of(new StaticProvisioner(allocatedHosts.get(), createNodeRepositoryProvisioner(applicationId, provisioned).get()));
         return Optional.empty();
     }
 
-    Optional<HostProvisioner> createNodeRepositoryProvisioner(ModelContext.Properties properties) {
+    Optional<HostProvisioner> createNodeRepositoryProvisioner(ApplicationId applicationId, Provisioned provisioned) {
         return hostProvisionerProvider.getHostProvisioner().map(
-                provisioner -> new ProvisionerAdapter(provisioner, properties.applicationId()));
+                provisioner -> new ProvisionerAdapter(provisioner, applicationId, provisioned));
     }
 
 }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java
index 9cacb78e53c..6d0403f21c8 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java
@@ -11,6 +11,7 @@ import com.yahoo.config.model.api.Model;
 import com.yahoo.config.model.api.ModelContext;
 import com.yahoo.config.model.api.ModelCreateResult;
 import com.yahoo.config.model.api.ModelFactory;
+import com.yahoo.config.model.api.Provisioned;
 import com.yahoo.config.model.api.ValidationParameters;
 import com.yahoo.config.model.api.ValidationParameters.IgnoreValidationErrors;
 import com.yahoo.config.model.application.provider.FilesApplicationPackage;
@@ -82,8 +83,9 @@ public class PreparedModelsBuilder extends ModelsBuilder<PreparedModelsBuilder.P
     @Override
     protected PreparedModelResult buildModelVersion(ModelFactory modelFactory, 
                                                     ApplicationPackage applicationPackage,
-                                                    ApplicationId applicationId, 
-                                                    com.yahoo.component.Version wantedNodeVespaVersion,
+                                                    ApplicationId applicationId,
+                                                    Optional<String> wantedDockerImageRepository,
+                                                    Version wantedNodeVespaVersion,
                                                     Optional<AllocatedHosts> allocatedHosts,
                                                     Instant now) {
         Version modelVersion = modelFactory.version();
@@ -91,6 +93,7 @@ public class PreparedModelsBuilder extends ModelsBuilder<PreparedModelsBuilder.P
         FileDistributionProvider fileDistributionProvider = fileDistributionFactory.createProvider(context.getServerDBSessionDir());
 
         // Use empty on non-hosted systems, use already allocated hosts if available, create connection to a host provisioner otherwise
+        Provisioned provisioned = new Provisioned();
         ModelContext modelContext = new ModelContextImpl(
                 applicationPackage,
                 modelOf(modelVersion),
@@ -98,9 +101,11 @@ public class PreparedModelsBuilder extends ModelsBuilder<PreparedModelsBuilder.P
                 logger,
                 configDefinitionRepo,
                 fileDistributionProvider.getFileRegistry(),
-                createHostProvisioner(allocatedHosts),
+                createHostProvisioner(allocatedHosts, provisioned),
+                provisioned,
                 properties,
                 getAppDir(applicationPackage),
+                wantedDockerImageRepository,
                 modelVersion,
                 wantedNodeVespaVersion);
 
@@ -120,11 +125,15 @@ public class PreparedModelsBuilder extends ModelsBuilder<PreparedModelsBuilder.P
 
     // This method is an excellent demonstration of what happens when one is too liberal with Optional   
     // -bratseth, who had to write the below  :-\
-    private Optional<HostProvisioner> createHostProvisioner(Optional<AllocatedHosts> allocatedHosts) {
-        Optional<HostProvisioner> nodeRepositoryProvisioner = createNodeRepositoryProvisioner(properties);
+    private Optional<HostProvisioner> createHostProvisioner(Optional<AllocatedHosts> allocatedHosts,
+                                                            Provisioned provisioned) {
+        Optional<HostProvisioner> nodeRepositoryProvisioner = createNodeRepositoryProvisioner(properties.applicationId(),
+                                                                                              provisioned);
         if ( ! allocatedHosts.isPresent()) return nodeRepositoryProvisioner;
         
-        Optional<HostProvisioner> staticProvisioner = createStaticProvisioner(allocatedHosts, properties);
+        Optional<HostProvisioner> staticProvisioner = createStaticProvisioner(allocatedHosts,
+                                                                              properties.applicationId(),
+                                                                              provisioned);
         if ( ! staticProvisioner.isPresent()) return Optional.empty(); // Since we have hosts allocated this means we are on non-hosted
             
         // Nodes are already allocated by a model and we should use them unless this model requests hosts from a
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/monitoring/Metrics.java b/configserver/src/main/java/com/yahoo/vespa/config/server/monitoring/Metrics.java
index 87e9fa287a4..249c8639ea9 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/monitoring/Metrics.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/monitoring/Metrics.java
@@ -3,6 +3,8 @@ package com.yahoo.vespa.config.server.monitoring;
 
 import com.google.inject.Inject;
 import com.yahoo.cloud.config.ZookeeperServerConfig;
+import com.yahoo.component.AbstractComponent;
+import com.yahoo.concurrent.DaemonThreadFactory;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.TenantName;
 import com.yahoo.container.jdisc.config.HealthMonitorConfig;
@@ -14,14 +16,17 @@ import com.yahoo.statistics.Counter;
 
 import java.util.*;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
 
 /**
- * Statistics for server. The statistics framework takes care of logging.
+ * Metrics for config server. The statistics framework takes care of logging.
  *
  * @author Harald Musum
- * @since 4.2
  */
-public class Metrics extends TimerTask implements MetricUpdaterFactory {
+public class Metrics extends AbstractComponent implements MetricUpdaterFactory, Runnable {
 
     private static final java.util.logging.Logger log = java.util.logging.Logger.getLogger(Metrics.class.getName());
     private static final String METRIC_REQUESTS = getMetricName("requests");
@@ -33,23 +38,34 @@ public class Metrics extends TimerTask implements MetricUpdaterFactory {
     private final Counter failedRequests;
     private final Counter procTimeCounter;
     private final Metric metric;
-    private final ZKMetricUpdater zkMetricUpdater;
+    private final Optional<ZKMetricUpdater> zkMetricUpdater;
 
     // TODO The map is the key for now
     private final Map<Map<String, String>, MetricUpdater> metricUpdaters = new ConcurrentHashMap<>();
-    private final Timer timer = new Timer();
+    private final Optional<ScheduledExecutorService> executorService;
 
     @Inject
     public Metrics(Metric metric, Statistics statistics, HealthMonitorConfig healthMonitorConfig, ZookeeperServerConfig zkServerConfig) {
+        this(metric, statistics, healthMonitorConfig, zkServerConfig, true);
+    }
+
+    private Metrics(Metric metric, Statistics statistics, HealthMonitorConfig healthMonitorConfig,
+                    ZookeeperServerConfig zkServerConfig, boolean createZkMetricUpdater) {
         this.metric = metric;
         requests = createCounter(METRIC_REQUESTS, statistics);
         failedRequests = createCounter(METRIC_FAILED_REQUESTS, statistics);
         procTimeCounter = createCounter("procTime", statistics);
 
-        log.log(LogLevel.DEBUG, "Metric update interval is " + healthMonitorConfig.snapshot_interval() + " seconds");
-        long intervalMs = (long) (healthMonitorConfig.snapshot_interval() * 1000);
-        timer.scheduleAtFixedRate(this, 20000, intervalMs);
-        zkMetricUpdater = new ZKMetricUpdater(zkServerConfig, 19500, intervalMs);
+        if (createZkMetricUpdater) {
+            log.log(LogLevel.DEBUG, "Metric update interval is " + healthMonitorConfig.snapshot_interval() + " seconds");
+            long intervalMs = (long) (healthMonitorConfig.snapshot_interval() * 1000);
+            executorService = Optional.of(new ScheduledThreadPoolExecutor(1, new DaemonThreadFactory("configserver-metrics")));
+            executorService.get().scheduleAtFixedRate(this, 20000, intervalMs, TimeUnit.MILLISECONDS);
+            zkMetricUpdater = Optional.of(new ZKMetricUpdater(zkServerConfig, 19500, intervalMs));
+        } else {
+            executorService = Optional.empty();
+            zkMetricUpdater = Optional.empty();
+        }
     }
 
     public static Metrics createTestMetrics() {
@@ -58,14 +74,13 @@ public class Metrics extends TimerTask implements MetricUpdaterFactory {
         HealthMonitorConfig.Builder builder = new HealthMonitorConfig.Builder();
         builder.snapshot_interval(60.0);
         ZookeeperServerConfig.Builder zkBuilder = new ZookeeperServerConfig.Builder().myid(1);
-        return new Metrics(metric, statistics, new HealthMonitorConfig(builder), new ZookeeperServerConfig(zkBuilder));
+        return new Metrics(metric, statistics, new HealthMonitorConfig(builder), new ZookeeperServerConfig(zkBuilder), false);
     }
 
     private Counter createCounter(String name, Statistics statistics) {
         return new Counter(name, statistics, false);
     }
 
-
     void incrementRequests(Metric.Context metricContext) {
         requests.increment(1);
         metric.add(METRIC_REQUESTS, 1, metricContext);
@@ -126,8 +141,12 @@ public class Metrics extends TimerTask implements MetricUpdaterFactory {
             }
         }
         setRegularMetrics();
-        zkMetricUpdater.getZKMetrics().forEach((attr, val) -> metric.set(attr, val, null));
-        timer.purge();
+        zkMetricUpdater.ifPresent(updater -> updater.getZKMetrics().forEach((attr, val) -> metric.set(attr, val, null)));
+    }
+
+    public void deconstruct() {
+        executorService.ifPresent(ExecutorService::shutdown);
+        zkMetricUpdater.ifPresent(ZKMetricUpdater::shutdown);
     }
 
     private void setRegularMetrics() {
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/monitoring/ZKMetricUpdater.java b/configserver/src/main/java/com/yahoo/vespa/config/server/monitoring/ZKMetricUpdater.java
index b2813be5456..62da6fcffbe 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/monitoring/ZKMetricUpdater.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/monitoring/ZKMetricUpdater.java
@@ -2,6 +2,7 @@
 package com.yahoo.vespa.config.server.monitoring;
 
 import com.yahoo.cloud.config.ZookeeperServerConfig;
+import com.yahoo.concurrent.DaemonThreadFactory;
 
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
@@ -17,6 +18,8 @@ import java.util.Timer;
 import java.util.TimerTask;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Future;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 import java.util.concurrent.atomic.AtomicReference;
@@ -26,7 +29,7 @@ import java.util.regex.Pattern;
 
 import static com.yahoo.vespa.config.server.monitoring.Metrics.getMetricName;
 
-public class ZKMetricUpdater extends TimerTask {
+public class ZKMetricUpdater implements Runnable {
     private static final Logger log = Logger.getLogger(ZKMetricUpdater.class.getName());
 
     public static final String METRIC_ZK_ZNODES = getMetricName("zkZNodes");
@@ -35,19 +38,19 @@ public class ZKMetricUpdater extends TimerTask {
     public static final String METRIC_ZK_CONNECTIONS = getMetricName("zkConnections");
     public static final String METRIC_ZK_OUTSTANDING_REQUESTS = getMetricName("zkOutstandingRequests");
 
-    private final int CONNECTION_TIMEOUT_MS = 500;
-    private final int WRITE_TIMEOUT_MS = 250;
-    private final int READ_TIMEOUT_MS = 500;
+    private static final int CONNECTION_TIMEOUT_MS = 500;
+    private static final int WRITE_TIMEOUT_MS = 250;
+    private static final int READ_TIMEOUT_MS = 500;
 
     private AtomicReference<Map<String, Long>> zkMetrics = new AtomicReference<>(new HashMap<>());
-    private final Timer timer = new Timer();
+    private final ScheduledExecutorService executorService;
     private final int zkPort;
 
     public ZKMetricUpdater(ZookeeperServerConfig zkServerConfig, long delayMS, long intervalMS) {
         this.zkPort = zkServerConfig.clientPort();
-        if (intervalMS > 0) {
-            timer.scheduleAtFixedRate(this, delayMS, intervalMS);
-        }
+        if (intervalMS <= 0 ) throw new IllegalArgumentException("interval must be positive, was " + intervalMS + " ms");
+        this.executorService = new ScheduledThreadPoolExecutor(1, new DaemonThreadFactory("zkmetricupdater"));
+        this.executorService.scheduleAtFixedRate(this, delayMS, intervalMS, TimeUnit.MILLISECONDS);
     }
 
     private void setMetricAttribute(String attribute, long value, Map<String, Long> data) {
@@ -74,7 +77,10 @@ public class ZKMetricUpdater extends TimerTask {
     public void run() {
         Optional<String> report = retrieveReport();
         report.ifPresent(this::parseReport);
-        timer.purge();
+    }
+
+    public void shutdown() {
+        executorService.shutdown();
     }
 
     private Optional<String> retrieveReport() {
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/provision/ProvisionerAdapter.java b/configserver/src/main/java/com/yahoo/vespa/config/server/provision/ProvisionerAdapter.java
index ee4cc4a3043..7ce255910e6 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/provision/ProvisionerAdapter.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/provision/ProvisionerAdapter.java
@@ -2,6 +2,7 @@
 package com.yahoo.vespa.config.server.provision;
 
 import com.yahoo.config.model.api.HostProvisioner;
+import com.yahoo.config.model.api.Provisioned;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.Capacity;
 import com.yahoo.config.provision.ClusterSpec;
@@ -23,10 +24,12 @@ public class ProvisionerAdapter implements HostProvisioner {
 
     private final Provisioner provisioner;
     private final ApplicationId applicationId;
+    private final Provisioned provisioned;
 
-    public ProvisionerAdapter(Provisioner provisioner, ApplicationId applicationId) {
+    public ProvisionerAdapter(Provisioner provisioner, ApplicationId applicationId, Provisioned provisioned) {
         this.provisioner = provisioner;
         this.applicationId = applicationId;
+        this.provisioned = provisioned;
     }
 
     @Override
@@ -36,8 +39,18 @@ public class ProvisionerAdapter implements HostProvisioner {
     }
 
     @Override
+    @Deprecated // TODO: Remove after April 2020
     public List<HostSpec> prepare(ClusterSpec cluster, Capacity capacity, int groups, ProvisionLogger logger) {
-        return provisioner.prepare(applicationId, cluster, capacity, groups, logger);
+        provisioned.add(cluster.id(), capacity);
+        return provisioner.prepare(applicationId, cluster, capacity.withGroups(groups), logger);
     }
 
+    @Override
+    public List<HostSpec> prepare(ClusterSpec cluster, Capacity capacity, ProvisionLogger logger) {
+        provisioned.add(cluster.id(), capacity);
+        return provisioner.prepare(applicationId, cluster, capacity, logger);
+    }
+
+
+
 }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/provision/StaticProvisioner.java b/configserver/src/main/java/com/yahoo/vespa/config/server/provision/StaticProvisioner.java
index 26d322665f0..8ddfb1e8b09 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/provision/StaticProvisioner.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/provision/StaticProvisioner.java
@@ -33,8 +33,15 @@ public class StaticProvisioner implements HostProvisioner {
         throw new UnsupportedOperationException("Allocating a single host from provisioning info is not supported");
     }
 
+
     @Override
+    @Deprecated // TODO: Remove after April 2020
     public List<HostSpec> prepare(ClusterSpec cluster, Capacity capacity, int groups, ProvisionLogger logger) {
+        return prepare(cluster, capacity.withGroups(groups), logger);
+    }
+
+    @Override
+    public List<HostSpec> prepare(ClusterSpec cluster, Capacity capacity, ProvisionLogger logger) {
         List<HostSpec> hostsAlreadyAllocatedToCluster = 
                 allocatedHosts.getHosts().stream()
                                          .filter(host -> host.membership().isPresent() && matches(host.membership().get().cluster(), cluster))
@@ -42,7 +49,7 @@ public class StaticProvisioner implements HostProvisioner {
         if ( ! hostsAlreadyAllocatedToCluster.isEmpty()) 
             return hostsAlreadyAllocatedToCluster;
         else
-            return fallback.prepare(cluster, capacity, groups, logger);
+            return fallback.prepare(cluster, capacity, logger);
     }
 
     private boolean matches(ClusterSpec nodeCluster, ClusterSpec requestedCluster) {
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java
index 67aa49ea0b9..48580dbc6f4 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java
@@ -167,7 +167,7 @@ class GetConfigProcessor implements Runnable {
         log.log(LogLevel.DEBUG, () -> "Returning empty sentinel config for request from " + request.getClientHostName());
         ConfigPayload emptyPayload = ConfigPayload.empty();
         String configMd5 = ConfigUtils.getMd5(emptyPayload);
-        ConfigResponse config = SlimeConfigResponse.fromConfigPayload(emptyPayload, null, 0, false, configMd5);
+        ConfigResponse config = SlimeConfigResponse.fromConfigPayload(emptyPayload, 0, false, configMd5);
         request.addOkResponse(request.payloadFromResponse(config), config.getGeneration(), false, config.getConfigMd5());
         respond(request);
     }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/LZ4ConfigResponseFactory.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/LZ4ConfigResponseFactory.java
index dcbc21e536c..5235a2bcadd 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/LZ4ConfigResponseFactory.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/LZ4ConfigResponseFactory.java
@@ -29,7 +29,7 @@ public class LZ4ConfigResponseFactory implements ConfigResponseFactory {
         String configMd5 = ConfigUtils.getMd5(rawPayload);
         CompressionInfo info = CompressionInfo.create(CompressionType.LZ4, rawPayload.getByteLength());
         Utf8Array compressed = new Utf8Array(compressor.compress(rawPayload.getBytes()));
-        return new SlimeConfigResponse(compressed, defFile, generation, internalRedeploy, configMd5, info);
+        return new SlimeConfigResponse(compressed, generation, internalRedeploy, configMd5, info);
     }
 
 }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/UncompressedConfigResponseFactory.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/UncompressedConfigResponseFactory.java
index 91809f90a70..bd0b117c3db 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/UncompressedConfigResponseFactory.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/UncompressedConfigResponseFactory.java
@@ -25,7 +25,7 @@ public class UncompressedConfigResponseFactory implements ConfigResponseFactory
         Utf8Array rawPayload = payload.toUtf8Array(true);
         String configMd5 = ConfigUtils.getMd5(rawPayload);
         CompressionInfo info = CompressionInfo.create(CompressionType.UNCOMPRESSED, rawPayload.getByteLength());
-        return new SlimeConfigResponse(rawPayload, defFile, generation, internalRedeploy, configMd5, info);
+        return new SlimeConfigResponse(rawPayload, generation, internalRedeploy, configMd5, info);
     }
 
 }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSession.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSession.java
index 2c6d7de8b0c..825ae0d8d92 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSession.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSession.java
@@ -7,6 +7,7 @@ import com.yahoo.config.application.api.ApplicationPackage;
 import com.yahoo.config.application.api.ApplicationMetaData;
 import com.yahoo.config.application.api.DeployLogger;
 import com.yahoo.config.provision.AllocatedHosts;
+import com.yahoo.config.provision.AthenzDomain;
 import com.yahoo.transaction.AbstractTransaction;
 import com.yahoo.transaction.NestedTransaction;
 import com.yahoo.transaction.Transaction;
@@ -138,6 +139,14 @@ public class LocalSession extends Session implements Comparable<LocalSession> {
         zooKeeperClient.writeVespaVersion(version);
     }
 
+    public void setDockerImageRepository(Optional<String> dockerImageRepository) {
+        zooKeeperClient.writeDockerImageRepository(dockerImageRepository);
+    }
+
+    public void setAthenzDomain(Optional<AthenzDomain> athenzDomain) {
+        zooKeeperClient.writeAthenzDomain(athenzDomain);
+    }
+
     public enum Mode {
         READ, WRITE
     }
@@ -148,8 +157,12 @@ public class LocalSession extends Session implements Comparable<LocalSession> {
 
     public ApplicationId getApplicationId() { return zooKeeperClient.readApplicationId(); }
 
+    public Optional<String> getDockerImageRepository() { return zooKeeperClient.readDockerImageRepository(); }
+
     public Version getVespaVersion() { return zooKeeperClient.readVespaVersion(); }
 
+    public Optional<AthenzDomain> getAthenzDomain() { return zooKeeperClient.readAthenzDomain(); }
+
     public AllocatedHosts getAllocatedHosts() {
         return zooKeeperClient.getAllocatedHosts();
     }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java
index 1a41c1efd7a..f7ed801ddbd 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java
@@ -5,10 +5,11 @@ import com.yahoo.component.Version;
 import com.yahoo.config.model.api.ContainerEndpoint;
 import com.yahoo.config.model.api.EndpointCertificateMetadata;
 import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.AthenzDomain;
 import com.yahoo.config.provision.TenantName;
 import com.yahoo.container.jdisc.HttpRequest;
 import com.yahoo.slime.Slime;
-import com.yahoo.vespa.config.SlimeUtils;
+import com.yahoo.slime.SlimeUtils;
 import com.yahoo.vespa.config.server.TimeoutBudget;
 import com.yahoo.vespa.config.server.http.SessionHandler;
 import com.yahoo.vespa.config.server.tenant.ContainerEndpointSerializer;
@@ -35,6 +36,8 @@ public final class PrepareParams {
     static final String CONTAINER_ENDPOINTS_PARAM_NAME = "containerEndpoints";
     static final String TLS_SECRETS_KEY_NAME_PARAM_NAME = "tlsSecretsKeyName";
     static final String ENDPOINT_CERTIFICATE_METADATA_PARAM_NAME = "endpointCertificateMetadata";
+    static final String DOCKER_IMAGE_REPOSITORY = "dockerImageRepository";
+    static final String ATHENZ_DOMAIN = "athenzDomain";
 
     private final ApplicationId applicationId;
     private final TimeoutBudget timeoutBudget;
@@ -46,11 +49,14 @@ public final class PrepareParams {
     private final List<ContainerEndpoint> containerEndpoints;
     private final Optional<String> tlsSecretsKeyName;
     private final Optional<EndpointCertificateMetadata> endpointCertificateMetadata;
+    private final Optional<String> dockerImageRepository;
+    private final Optional<AthenzDomain> athenzDomain;
 
     private PrepareParams(ApplicationId applicationId, TimeoutBudget timeoutBudget, boolean ignoreValidationErrors,
                           boolean dryRun, boolean verbose, boolean isBootstrap, Optional<Version> vespaVersion,
                           List<ContainerEndpoint> containerEndpoints, Optional<String> tlsSecretsKeyName,
-                          Optional<EndpointCertificateMetadata> endpointCertificateMetadata) {
+                          Optional<EndpointCertificateMetadata> endpointCertificateMetadata,
+                          Optional<String> dockerImageRepository, Optional<AthenzDomain> athenzDomain) {
         this.timeoutBudget = timeoutBudget;
         this.applicationId = applicationId;
         this.ignoreValidationErrors = ignoreValidationErrors;
@@ -61,6 +67,8 @@ public final class PrepareParams {
         this.containerEndpoints = containerEndpoints;
         this.tlsSecretsKeyName = tlsSecretsKeyName;
         this.endpointCertificateMetadata = endpointCertificateMetadata;
+        this.dockerImageRepository = dockerImageRepository;
+        this.athenzDomain = athenzDomain;
     }
 
     public static class Builder {
@@ -75,6 +83,8 @@ public final class PrepareParams {
         private List<ContainerEndpoint> containerEndpoints = List.of();
         private Optional<String> tlsSecretsKeyName = Optional.empty();
         private Optional<EndpointCertificateMetadata> endpointCertificateMetadata = Optional.empty();
+        private Optional<String> dockerImageRepository = Optional.empty();
+        private Optional<AthenzDomain> athenzDomain = Optional.empty();
 
         public Builder() { }
 
@@ -142,9 +152,26 @@ public final class PrepareParams {
             return this;
         }
 
+        public Builder dockerImageRepository(String dockerImageRepository) {
+            if (dockerImageRepository == null) return this;
+            this.dockerImageRepository = Optional.of(dockerImageRepository);
+            return this;
+        }
+
+        public Builder athenzDomain(String athenzDomain) {
+            this.athenzDomain = Optional.ofNullable(athenzDomain).map(AthenzDomain::from);
+            return this;
+        }
+
+        public Builder athenzDomain(AthenzDomain athenzDomain) {
+            this.athenzDomain = Optional.of(athenzDomain);
+            return this;
+        }
+
         public PrepareParams build() {
             return new PrepareParams(applicationId, timeoutBudget, ignoreValidationErrors, dryRun,
-                                     verbose, isBootstrap, vespaVersion, containerEndpoints, tlsSecretsKeyName, endpointCertificateMetadata);
+                                     verbose, isBootstrap, vespaVersion, containerEndpoints, tlsSecretsKeyName,
+                                     endpointCertificateMetadata, dockerImageRepository, athenzDomain);
         }
 
     }
@@ -159,6 +186,8 @@ public final class PrepareParams {
                             .containerEndpoints(request.getProperty(CONTAINER_ENDPOINTS_PARAM_NAME))
                             .tlsSecretsKeyName(request.getProperty(TLS_SECRETS_KEY_NAME_PARAM_NAME))
                             .endpointCertificateMetadata(request.getProperty(ENDPOINT_CERTIFICATE_METADATA_PARAM_NAME))
+                            .dockerImageRepository(request.getProperty(DOCKER_IMAGE_REPOSITORY))
+                            .athenzDomain(request.getProperty(ATHENZ_DOMAIN))
                             .build();
     }
 
@@ -219,4 +248,11 @@ public final class PrepareParams {
     public Optional<EndpointCertificateMetadata> endpointCertificateMetadata() {
         return endpointCertificateMetadata;
     }
+
+    public Optional<String> dockerImageRepository() {
+        return dockerImageRepository;
+    }
+
+    public Optional<AthenzDomain> athenzDomain() { return athenzDomain; }
+
 }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java
index 26f437920ad..39c15474f0e 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java
@@ -63,6 +63,7 @@ public class RemoteSession extends Session {
         Optional<AllocatedHosts> allocatedHosts = applicationPackage.getAllocatedHosts();
 
         return ApplicationSet.fromList(applicationLoader.buildModels(zooKeeperClient.readApplicationId(),
+                                                                     zooKeeperClient.readDockerImageRepository(),
                                                                      zooKeeperClient.readVespaVersion(),
                                                                      applicationPackage,
                                                                      new SettableOptional<>(allocatedHosts),
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactoryImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactoryImpl.java
index cb1f6519f57..5ae1289033d 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactoryImpl.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactoryImpl.java
@@ -127,10 +127,13 @@ public class SessionFactoryImpl implements SessionFactory, LocalSessionLoader {
         ApplicationId existingApplicationId = existingSession.getApplicationId();
 
         long activeSessionId = getActiveSessionId(existingApplicationId);
-        logger.log(LogLevel.DEBUG, "Create from existing application id " + existingApplicationId + ", active session id is " + activeSessionId);
+        logger.log(LogLevel.DEBUG, "Create new session for application id '" + existingApplicationId + "' from existing active session " + activeSessionId);
         LocalSession session = create(existingApp, existingApplicationId, activeSessionId, internalRedeploy, timeoutBudget);
+        // Note: Needs to be kept in sync with calls in SessionPreparer.writeStateToZooKeeper()
         session.setApplicationId(existingApplicationId);
         session.setVespaVersion(existingSession.getVespaVersion());
+        session.setDockerImageRepository(existingSession.getDockerImageRepository());
+        session.setAthenzDomain(existingSession.getAthenzDomain());
         return session;
     }
 
@@ -138,7 +141,6 @@ public class SessionFactoryImpl implements SessionFactory, LocalSessionLoader {
                                 boolean internalRedeploy, TimeoutBudget timeoutBudget) {
         long sessionId = sessionCounter.nextSessionId();
         Path sessionIdPath = sessionsPath.append(String.valueOf(sessionId));
-        log.log(LogLevel.DEBUG, TenantRepository.logPre(tenant) + "Next session id is " + sessionId + " , sessionIdPath=" + sessionIdPath.getAbsolute());
         try {
             ensureZKPathDoesNotExist(sessionIdPath);
             SessionZooKeeperClient sessionZooKeeperClient = new SessionZooKeeperClient(curator,
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java
index 0115876ded9..6c77964a58b 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java
@@ -17,6 +17,7 @@ import com.yahoo.config.model.api.ModelContext;
 import com.yahoo.config.model.api.EndpointCertificateSecrets;
 import com.yahoo.config.provision.AllocatedHosts;
 import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.AthenzDomain;
 import com.yahoo.config.provision.HostName;
 import com.yahoo.config.provision.Zone;
 import com.yahoo.container.jdisc.secretstore.SecretStore;
@@ -139,8 +140,11 @@ public class SessionPreparer {
         final Path tenantPath;
         final ApplicationId applicationId;
 
+        /** The repository part of docker image to be used for this deployment */
+        final Optional<String> dockerImageRepository;
+
         /** The version of Vespa the application to be prepared specifies for its nodes */
-        final com.yahoo.component.Version vespaVersion;
+        final Version vespaVersion;
 
         final ContainerEndpointsCache containerEndpoints;
         final Set<ContainerEndpoint> endpointsSet;
@@ -149,6 +153,7 @@ public class SessionPreparer {
         private final EndpointCertificateRetriever endpointCertificateRetriever;
         private final Optional<EndpointCertificateMetadata> endpointCertificateMetadata;
         private final Optional<EndpointCertificateSecrets> endpointCertificateSecrets;
+        private final Optional<AthenzDomain> athenzDomain;
 
         private ApplicationPackage applicationPackage;
         private List<PreparedModelsBuilder.PreparedModelResult> modelResultList;
@@ -165,6 +170,7 @@ public class SessionPreparer {
             this.tenantPath = tenantPath;
 
             this.applicationId = params.getApplicationId();
+            this.dockerImageRepository = params.dockerImageRepository();
             this.vespaVersion = params.vespaVersion().orElse(Vtag.currentVersion);
             this.containerEndpoints = new ContainerEndpointsCache(tenantPath, curator);
             this.endpointCertificateMetadataStore = new EndpointCertificateMetadataStore(curator, tenantPath);
@@ -178,6 +184,7 @@ public class SessionPreparer {
                     .flatMap(endpointCertificateRetriever::readEndpointCertificateSecrets);
 
             this.endpointsSet = getEndpoints(params.containerEndpoints());
+            this.athenzDomain = params.athenzDomain();
 
             this.properties = new ModelContextImpl.Properties(params.getApplicationId(),
                                                               configserverConfig.multitenant(),
@@ -191,7 +198,8 @@ public class SessionPreparer {
                                                               params.isBootstrap(),
                                                               ! currentActiveApplicationSet.isPresent(),
                                                               context.getFlagSource(),
-                                                              endpointCertificateSecrets);
+                                                              endpointCertificateSecrets,
+                                                              athenzDomain);
             this.preparedModelsBuilder = new PreparedModelsBuilder(modelFactoryRegistry,
                                                                    permanentApplicationPackage,
                                                                    configDefinitionRepo,
@@ -223,7 +231,7 @@ public class SessionPreparer {
 
         AllocatedHosts buildModels(Instant now) {
             SettableOptional<AllocatedHosts> allocatedHosts = new SettableOptional<>();
-            this.modelResultList = preparedModelsBuilder.buildModels(applicationId, vespaVersion, 
+            this.modelResultList = preparedModelsBuilder.buildModels(applicationId, dockerImageRepository, vespaVersion,
                                                                      applicationPackage, allocatedHosts, now);
             checkTimeout("build models");
             return allocatedHosts.get();
@@ -239,10 +247,12 @@ public class SessionPreparer {
             writeStateToZooKeeper(context.getSessionZooKeeperClient(), 
                                   applicationPackage,
                                   applicationId,
+                                  dockerImageRepository,
                                   vespaVersion,
                                   logger,
                                   prepareResult.getFileRegistries(), 
-                                  prepareResult.allocatedHosts());
+                                  prepareResult.allocatedHosts(),
+                                  athenzDomain);
             checkTimeout("write state to zookeeper");
         }
 
@@ -281,15 +291,20 @@ public class SessionPreparer {
     private void writeStateToZooKeeper(SessionZooKeeperClient zooKeeperClient,
                                        ApplicationPackage applicationPackage,
                                        ApplicationId applicationId,
-                                       com.yahoo.component.Version vespaVersion,
+                                       Optional<String> dockerImageRepository,
+                                       Version vespaVersion,
                                        DeployLogger deployLogger,
                                        Map<Version, FileRegistry> fileRegistryMap,
-                                       AllocatedHosts allocatedHosts) {
+                                       AllocatedHosts allocatedHosts,
+                                       Optional<AthenzDomain> athenzDomain) {
         ZooKeeperDeployer zkDeployer = zooKeeperClient.createDeployer(deployLogger);
         try {
             zkDeployer.deploy(applicationPackage, fileRegistryMap, allocatedHosts);
+            // Note: When changing the below you need to also change similar calls in SessionFactoryImpl.createSessionFromExisting()
             zooKeeperClient.writeApplicationId(applicationId);
             zooKeeperClient.writeVespaVersion(vespaVersion);
+            zooKeeperClient.writeDockerImageRepository(dockerImageRepository);
+            zooKeeperClient.writeAthenzDomain(athenzDomain);
         } catch (RuntimeException | IOException e) {
             zkDeployer.cleanup();
             throw new RuntimeException("Error preparing session", e);
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java
index 727d4e6d7bc..81777bf3642 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java
@@ -5,23 +5,24 @@ import com.yahoo.component.Version;
 import com.yahoo.component.Vtag;
 import com.yahoo.config.application.api.ApplicationPackage;
 import com.yahoo.config.application.api.DeployLogger;
-import com.yahoo.config.provision.NodeFlavors;
+import com.yahoo.config.model.api.ConfigDefinitionRepo;
 import com.yahoo.config.provision.AllocatedHosts;
-import com.yahoo.transaction.NestedTransaction;
-import com.yahoo.transaction.Transaction;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.AthenzDomain;
+import com.yahoo.config.provision.NodeFlavors;
 import com.yahoo.log.LogLevel;
 import com.yahoo.path.Path;
-import com.yahoo.config.model.api.ConfigDefinitionRepo;
 import com.yahoo.text.Utf8;
-import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.transaction.NestedTransaction;
+import com.yahoo.transaction.Transaction;
 import com.yahoo.vespa.config.server.UserConfigDefinitionRepo;
 import com.yahoo.vespa.config.server.deploy.ZooKeeperClient;
 import com.yahoo.vespa.config.server.deploy.ZooKeeperDeployer;
+import com.yahoo.vespa.config.server.zookeeper.ConfigCurator;
 import com.yahoo.vespa.config.server.zookeeper.ZKApplicationPackage;
 import com.yahoo.vespa.curator.Curator;
 import com.yahoo.vespa.curator.transaction.CuratorOperations;
 import com.yahoo.vespa.curator.transaction.CuratorTransaction;
-import com.yahoo.vespa.config.server.zookeeper.ConfigCurator;
 
 import java.util.Optional;
 import java.util.concurrent.TimeUnit;
@@ -41,6 +42,8 @@ public class SessionZooKeeperClient {
     static final String APPLICATION_ID_PATH = "applicationId";
     private static final String VERSION_PATH = "version";
     private static final String CREATE_TIME_PATH = "createTime";
+    private static final String DOCKER_IMAGE_REPOSITORY_PATH = "dockerImageRepository";
+    private static final String ATHENZ_DOMAIN = "athenzDomain";
     private final Curator curator;
     private final ConfigCurator configCurator;
     private final Path sessionPath;
@@ -165,6 +168,14 @@ public class SessionZooKeeperClient {
         return sessionPath.append(VERSION_PATH).getAbsolute();
     }
 
+    private String dockerImageRepositoryPath() {
+        return sessionPath.append(DOCKER_IMAGE_REPOSITORY_PATH).getAbsolute();
+    }
+
+    private String athenzDomainPath() {
+        return sessionPath.append(ATHENZ_DOMAIN).getAbsolute();
+    }
+
     public void writeVespaVersion(Version version) {
         configCurator.putData(versionPath(), version.toString());
     }
@@ -174,6 +185,16 @@ public class SessionZooKeeperClient {
         return new Version(configCurator.getData(versionPath()));
     }
 
+    public Optional<String> readDockerImageRepository() {
+        if ( ! configCurator.exists(dockerImageRepositoryPath())) return Optional.empty();
+        String dockerImageRepository = configCurator.getData(dockerImageRepositoryPath());
+        return dockerImageRepository.isEmpty() ? Optional.empty() : Optional.of(dockerImageRepository);
+    }
+
+    public void writeDockerImageRepository(Optional<String> dockerImageRepository) {
+        dockerImageRepository.ifPresent(repo -> configCurator.putData(dockerImageRepositoryPath(), repo));
+    }
+
     // in seconds
     public long readCreateTime() {
         String path = getCreateTimePath();
@@ -206,6 +227,17 @@ public class SessionZooKeeperClient {
         return transaction;
     }
 
+    public void writeAthenzDomain(Optional<AthenzDomain> athenzDomain) {
+        athenzDomain.ifPresent(domain -> configCurator.putData(athenzDomainPath(), domain.value()));
+    }
+
+    public Optional<AthenzDomain> readAthenzDomain() {
+        if ( ! configCurator.exists(athenzDomainPath())) return Optional.empty();
+        return Optional.ofNullable(configCurator.getData(athenzDomainPath()))
+                .filter(domain -> ! domain.isBlank())
+                .map(AthenzDomain::from);
+    }
+
     /**
      * Create necessary paths atomically for a new session.
      *
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointsCache.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointsCache.java
index 65ebb38c2d0..97179e5234b 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointsCache.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointsCache.java
@@ -4,7 +4,7 @@ package com.yahoo.vespa.config.server.tenant;
 import com.yahoo.config.model.api.ContainerEndpoint;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.path.Path;
-import com.yahoo.vespa.config.SlimeUtils;
+import com.yahoo.slime.SlimeUtils;
 import com.yahoo.vespa.curator.Curator;
 import com.yahoo.vespa.curator.transaction.CuratorOperations;
 import com.yahoo.vespa.curator.transaction.CuratorTransaction;
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataStore.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataStore.java
index 6500449e557..8e51ac424f9 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataStore.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataStore.java
@@ -5,7 +5,7 @@ import com.yahoo.config.model.api.EndpointCertificateMetadata;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.path.Path;
 import com.yahoo.slime.Slime;
-import com.yahoo.vespa.config.SlimeUtils;
+import com.yahoo.slime.SlimeUtils;
 import com.yahoo.vespa.curator.Curator;
 import com.yahoo.vespa.curator.transaction.CuratorOperations;
 import com.yahoo.vespa.curator.transaction.CuratorTransaction;
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplication.java b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplication.java
index 3ae678969eb..5f180bdaee1 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplication.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplication.java
@@ -9,14 +9,12 @@ import java.io.StringReader;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
+import java.util.Optional;
 
 /**
  * Responsible for providing data from an application subtree in zookeeper.
  * (i.e. /config/v2/tenants/x/session/&lt;session id for an application&gt;/).
  *
- * Takes care of
- *
- *
  * @author Tony Vaagenes
  */
 public class ZKApplication {
@@ -84,6 +82,10 @@ public class ZKApplication {
         return reader(data);
     }
 
+    Optional<Reader> getOptionalDataReader(String path, String node) {
+        return Optional.ofNullable(getData(path, node)).map(data -> reader(data));
+    }
+
     public String getData(String path, String node) {
         try {
             return zk.getData(getFullPath(path), node);
@@ -181,10 +183,9 @@ public class ZKApplication {
     }
 
     Reader getDataReader(String path) {
-        final String data = getData(path);
-        if (data == null) {
+        String data = getData(path);
+        if (data == null)
             throw new IllegalArgumentException("No node for " + getFullPath(path) + " exists");
-        }
         return reader(data);
     }
 
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java
index bcb958c4b58..c7ec2657996 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java
@@ -139,13 +139,16 @@ public class ZKApplicationPackage implements ApplicationPackage {
 
     @Override
     public List<NamedReader> searchDefinitionContents() {
-        List<NamedReader> ret = new ArrayList<>();
-        for (String sd : zkApplication.getChildren(ConfigCurator.USERAPP_ZK_SUBPATH+"/"+SEARCH_DEFINITIONS_DIR)) {
-            if (sd.endsWith(ApplicationPackage.SD_NAME_SUFFIX)) {
-                ret.add(new NamedReader(sd, new StringReader(zkApplication.getData(ConfigCurator.USERAPP_ZK_SUBPATH+"/"+SEARCH_DEFINITIONS_DIR, sd))));
-            }
+        List<NamedReader> schemas = new ArrayList<>();
+        for (String sd : zkApplication.getChildren(ConfigCurator.USERAPP_ZK_SUBPATH + "/" + SEARCH_DEFINITIONS_DIR)) {
+            if (sd.endsWith(ApplicationPackage.SD_NAME_SUFFIX))
+                schemas.add(new NamedReader(sd, new StringReader(zkApplication.getData(ConfigCurator.USERAPP_ZK_SUBPATH + "/" + SEARCH_DEFINITIONS_DIR, sd))));
         }
-        return ret;
+        for (String sd : zkApplication.getChildren(ConfigCurator.USERAPP_ZK_SUBPATH + "/" + SCHEMAS_DIR)) {
+            if (sd.endsWith(ApplicationPackage.SD_NAME_SUFFIX))
+                schemas.add(new NamedReader(sd, new StringReader(zkApplication.getData(ConfigCurator.USERAPP_ZK_SUBPATH + "/" + SCHEMAS_DIR, sd))));
+        }
+        return schemas;
     }
 
     @Override
@@ -176,7 +179,7 @@ public class ZKApplicationPackage implements ApplicationPackage {
         try {
             return zkApplication.getDataReader(ConfigCurator.DEFCONFIGS_ZK_SUBPATH, def);
         } catch (IllegalArgumentException e) {
-            throw new IllegalArgumentException("Could not retrieve config definition " + def + ".", e);
+            throw new IllegalArgumentException("Could not retrieve config definition " + def, e);
         }
     }
 
@@ -264,7 +267,10 @@ public class ZKApplicationPackage implements ApplicationPackage {
 
     @Override
     public Reader getRankingExpression(String name) {
-        return zkApplication.getDataReader(ConfigCurator.USERAPP_ZK_SUBPATH+"/"+SEARCH_DEFINITIONS_DIR, name);
+        Optional<Reader> reader = zkApplication.getOptionalDataReader(ConfigCurator.USERAPP_ZK_SUBPATH + "/" + SCHEMAS_DIR, name);
+        if (reader.isPresent())
+            return reader.get();
+        return zkApplication.getDataReader(ConfigCurator.USERAPP_ZK_SUBPATH + "/" + SEARCH_DEFINITIONS_DIR, name);
     }
 
     @Override
diff --git a/configserver/src/main/resources/configserver-app/services.xml b/configserver/src/main/resources/configserver-app/services.xml
index 970dd49e865..0cd283ca62b 100644
--- a/configserver/src/main/resources/configserver-app/services.xml
+++ b/configserver/src/main/resources/configserver-app/services.xml
@@ -63,8 +63,7 @@
     <component id="com.yahoo.vespa.service.manager.UnionMonitorManager" bundle="service-monitor" />
     <component id="com.yahoo.vespa.service.model.ServiceMonitorImpl" bundle="service-monitor" />
     <component id="com.yahoo.vespa.service.duper.DuperModelManager" bundle="service-monitor" />
-    <component id="com.yahoo.vespa.orchestrator.ServiceMonitorInstanceLookupService" bundle="orchestrator" />
-    <component id="com.yahoo.vespa.orchestrator.status.ZookeeperStatusService" bundle="orchestrator" />
+    <component id="com.yahoo.vespa.orchestrator.status.ZkStatusService" bundle="orchestrator" />
     <component id="com.yahoo.vespa.orchestrator.controller.RetryingClusterControllerClientFactory" bundle="orchestrator" />
     <component id="com.yahoo.vespa.orchestrator.OrchestratorImpl" bundle="orchestrator" />
     
diff --git a/configserver/src/main/sh/start-configserver b/configserver/src/main/sh/start-configserver
index bec206214f8..bded46dbebe 100755
--- a/configserver/src/main/sh/start-configserver
+++ b/configserver/src/main/sh/start-configserver
@@ -174,6 +174,7 @@ vespa-run-as-vespa-user vespa-runserver -s configserver -r 30 -p $pidfile -- \
     --add-opens=java.base/java.lang=ALL-UNNAMED \
     --add-opens=java.base/java.net=ALL-UNNAMED \
     --add-opens=java.base/jdk.internal.loader=ALL-UNNAMED \
+	-Djava.io.tmpdir=${VESPA_HOME}/tmp \
 	-Djava.library.path=${VESPA_HOME}/lib64 \
 	-Djava.awt.headless=true \
 	-Dsun.rmi.dgc.client.gcInterval=3600000 \
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java
index a963252d7ca..0e076d60d52 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java
@@ -342,11 +342,10 @@ public class ApplicationRepositoryTest {
                                                           new MockTesterClient(),
                                                           actual);
         deployApp(testAppLogServerWithContainer);
-        Map<String, ?> context = Map.of("tenant", "test1",
-                                        "application", "testapp",
-                                        "instance", "default",
-                                        "environment", "prod",
-                                        "region", "default");
+        Map<String, ?> context = Map.of("applicationId", "test1.testapp.default",
+                                        "tenantName", "test1",
+                                        "app", "testapp.default",
+                                        "zone", "prod.default");
         MockMetric expected = new MockMetric();
         expected.set("deployment.prepareMillis", 0L, expected.createContext(context));
         expected.set("deployment.activateMillis", 0L, expected.createContext(context));
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java
index 2a1254d0d8d..654d811a31f 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ConfigServerBootstrapTest.java
@@ -12,6 +12,7 @@ import com.yahoo.config.provision.RegionName;
 import com.yahoo.component.Version;
 import com.yahoo.config.provision.Zone;
 import com.yahoo.container.QrSearchersConfig;
+import com.yahoo.container.core.VipStatusConfig;
 import com.yahoo.container.handler.ClustersStatus;
 import com.yahoo.container.handler.VipStatus;
 import com.yahoo.container.jdisc.config.HealthMonitorConfig;
@@ -235,6 +236,7 @@ public class ConfigServerBootstrapTest {
 
     private VipStatus createVipStatus(StateMonitor stateMonitor) {
         return new VipStatus(new QrSearchersConfig.Builder().build(),
+                             new VipStatusConfig.Builder().build(),
                              new ClustersStatus(),
                              stateMonitor);
     }
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/MockTesterClient.java b/configserver/src/test/java/com/yahoo/vespa/config/server/MockTesterClient.java
index db687857dae..c835bb29ec0 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/MockTesterClient.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/MockTesterClient.java
@@ -15,70 +15,40 @@ public class MockTesterClient extends TesterClient {
 
     @Override
     public HttpResponse getStatus(String testerHostname, int port) {
-        return new MockStatusResponse();
+        return new HttpResponse(200) {
+            @Override
+            public void render(OutputStream outputStream) throws IOException {
+                outputStream.write("OK".getBytes(StandardCharsets.UTF_8));
+            }
+        };
     }
 
     @Override
     public HttpResponse getLog(String testerHostname, int port, Long after) {
-        return new MockLogResponse();
+        return new HttpResponse(200) {
+            @Override
+            public void render(OutputStream outputStream) throws IOException {
+                outputStream.write("log".getBytes(StandardCharsets.UTF_8));
+            }
+        };
     }
 
     @Override
     public HttpResponse startTests(String testerHostname, int port, String suite, byte[] config) {
-        return new MockStartTestsResponse();
+        return new HttpResponse(200) {
+            @Override
+            public void render(OutputStream outputStream) { }
+        };
     }
 
     @Override
     public HttpResponse isTesterReady(String testerHostname, int port) {
-        return new MockTesterReadyResponse();
-    }
-
-    private static class MockStatusResponse extends HttpResponse {
-
-        private MockStatusResponse() {
-            super(200);
-        }
-
-        @Override
-        public void render(OutputStream outputStream) throws IOException {
-            outputStream.write("OK".getBytes(StandardCharsets.UTF_8));
-        }
-
-    }
-
-    private static class MockLogResponse extends HttpResponse {
-
-        private MockLogResponse() {
-            super(200);
-        }
-
-        @Override
-        public void render(OutputStream outputStream) throws IOException {
-            outputStream.write("log".getBytes(StandardCharsets.UTF_8));
-        }
-
-    }
-
-    private static class MockStartTestsResponse extends HttpResponse {
-
-        private MockStartTestsResponse() {
-            super(200);
-        }
-
-        @Override
-        public void render(OutputStream outputStream) { }
-
-    }
-
-    private static class MockTesterReadyResponse extends HttpResponse {
-
-        private MockTesterReadyResponse() {
-            super(200);
-        }
-
-        @Override
-        public void render(OutputStream outputStream) { }
-
+        return new HttpResponse(200) {
+            @Override
+            public void render(OutputStream outputStream) throws IOException {
+                outputStream.write("{ \"message\": \"OK\" } ".getBytes(StandardCharsets.UTF_8));
+            }
+        };
     }
 
 }
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java
index 370e0bd3c0e..03c6bad79a8 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java
@@ -4,6 +4,7 @@ package com.yahoo.vespa.config.server;
 import com.yahoo.component.Version;
 import com.yahoo.config.model.api.ContainerEndpoint;
 import com.yahoo.config.model.api.ModelContext;
+import com.yahoo.config.model.api.Provisioned;
 import com.yahoo.config.model.application.provider.BaseDeployLogger;
 import com.yahoo.config.model.application.provider.MockFileRegistry;
 import com.yahoo.config.model.test.MockApplicationPackage;
@@ -20,6 +21,7 @@ import java.util.Set;
 
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
+import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertThat;
@@ -45,6 +47,7 @@ public class ModelContextImplTest {
                 new StaticConfigDefinitionRepo(),
                 new MockFileRegistry(),
                 Optional.empty(),
+                new Provisioned(),
                 new ModelContextImpl.Properties(
                         ApplicationId.defaultId(),
                         true,
@@ -58,10 +61,12 @@ public class ModelContextImplTest {
                         false,
                         false,
                         flagSource,
-                        null),
+                        null,
+                        Optional.empty()),
+                Optional.empty(),
                 Optional.empty(),
-                new Version(6), 
-                new Version(6));
+                new Version(7),
+                new Version(8));
         assertTrue(context.applicationPackage() instanceof MockApplicationPackage);
         assertFalse(context.hostProvisioner().isPresent());
         assertFalse(context.permanentApplicationPackage().isPresent());
@@ -75,6 +80,13 @@ public class ModelContextImplTest {
         assertFalse(context.properties().hostedVespa());
         assertThat(context.properties().endpoints(), equalTo(endpoints));
         assertThat(context.properties().isFirstTimeDeployment(), equalTo(false));
+
+        assertEquals(Optional.empty(), context.wantedDockerImageRepository());
+        assertEquals(new Version(7), context.modelVespaVersion());
+        assertEquals(new Version(8), context.wantedNodeVespaVersion());
+        assertEquals(1.0, context.properties().defaultTermwiseLimit(), 0.0);
+        assertEquals(1.0, context.properties().defaultTopKProbability(), 0.0);
+        assertFalse(context.properties().useAdaptiveDispatch());
     }
 
 }
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ServerCacheTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ServerCacheTest.java
index f2ee7815df3..9a18570db2d 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/ServerCacheTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ServerCacheTest.java
@@ -27,7 +27,6 @@ public class ServerCacheTest {
     private static String configMd5 = "mymd5";
     private static String configMd5_2 = "mymd5_2";
     private static ConfigDefinition def = new ConfigDefinition("mypayload", new String[0]);
-    private static ConfigDefinition def_2 = new ConfigDefinition("otherpayload", new String[0]);
 
     private static ConfigDefinitionKey fooBarDefKey = new ConfigDefinitionKey("foo", "bar");
     private static ConfigDefinitionKey fooBazDefKey = new ConfigDefinitionKey("foo", "baz");
@@ -49,9 +48,9 @@ public class ServerCacheTest {
 
         cache = new ServerCache(new TestConfigDefinitionRepo(), userConfigDefinitionRepo);
 
-        cache.put(fooBarCacheKey, SlimeConfigResponse.fromConfigPayload(ConfigPayload.empty(), def.getCNode(), 2, false, configMd5), configMd5);
-        cache.put(bazQuuxCacheKey, SlimeConfigResponse.fromConfigPayload(ConfigPayload.empty(), def.getCNode(), 2, false, configMd5), configMd5);
-        cache.put(fooBarCacheKeyDifferentMd5, SlimeConfigResponse.fromConfigPayload(ConfigPayload.empty(), def_2.getCNode(), 2, false, configMd5_2), configMd5_2);
+        cache.put(fooBarCacheKey, SlimeConfigResponse.fromConfigPayload(ConfigPayload.empty(), 2, false, configMd5), configMd5);
+        cache.put(bazQuuxCacheKey, SlimeConfigResponse.fromConfigPayload(ConfigPayload.empty(), 2, false, configMd5), configMd5);
+        cache.put(fooBarCacheKeyDifferentMd5, SlimeConfigResponse.fromConfigPayload(ConfigPayload.empty(), 2, false, configMd5_2), configMd5_2);
     }
 
     @Test
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelControllerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelControllerTest.java
index b7486dc7951..561422c1cf8 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelControllerTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/SuperModelControllerTest.java
@@ -55,7 +55,7 @@ public class SuperModelControllerTest {
         ApplicationId app = ApplicationId.from(TenantName.from("a"),
                                                ApplicationName.from("foo"), InstanceName.defaultName());
         models.put(app, new ApplicationInfo(app, 4L, new VespaModel(FilesApplicationPackage.fromFile(testApp))));
-        SuperModel superModel = new SuperModel(models);
+        SuperModel superModel = new SuperModel(models, true);
         handler = new SuperModelController(new SuperModelConfigProvider(superModel, Zone.defaultZone(), new InMemoryFlagSource()), new TestConfigDefinitionRepo(), 2, new UncompressedConfigResponseFactory());
     }
     
@@ -98,7 +98,7 @@ public class SuperModelControllerTest {
         models.put(advanced, createApplicationInfo(testApp2, advanced, 4L));
         models.put(tooAdvanced, createApplicationInfo(testApp3, tooAdvanced, 4L));
 
-        SuperModel superModel = new SuperModel(models);
+        SuperModel superModel = new SuperModel(models, true);
         SuperModelController han = new SuperModelController(new SuperModelConfigProvider(superModel, Zone.defaultZone(), new InMemoryFlagSource()), new TestConfigDefinitionRepo(), 2, new UncompressedConfigResponseFactory());
         LbServicesConfig.Builder lb = new LbServicesConfig.Builder();
         han.getSuperModel().getConfig(lb);
@@ -126,7 +126,7 @@ public class SuperModelControllerTest {
         models.put(advanced, createApplicationInfo(testApp2, advanced, 4L));
         models.put(tooAdvanced, createApplicationInfo(testApp3, tooAdvanced, 4L));
 
-        SuperModel superModel = new SuperModel(models);
+        SuperModel superModel = new SuperModel(models, true);
         SuperModelController han = new SuperModelController(new SuperModelConfigProvider(superModel, Zone.defaultZone(), new InMemoryFlagSource()), new TestConfigDefinitionRepo(), 2, new UncompressedConfigResponseFactory());
         LbServicesConfig.Builder lb = new LbServicesConfig.Builder();
         han.getSuperModel().getConfig(lb);
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationTest.java
index 405fff3e190..ad910c2afc2 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationTest.java
@@ -4,14 +4,15 @@ package com.yahoo.vespa.config.server.application;
 import com.yahoo.cloud.config.ModelConfig;
 import com.yahoo.cloud.config.SlobroksConfig;
 import com.yahoo.cloud.config.log.LogdConfig;
+import com.yahoo.component.Version;
 import com.yahoo.config.SimpletypesConfig;
 import com.yahoo.config.model.application.provider.FilesApplicationPackage;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.ApplicationName;
 import com.yahoo.config.provision.InstanceName;
 import com.yahoo.config.provision.TenantName;
-import com.yahoo.component.Version;
 import com.yahoo.jrt.Request;
+import com.yahoo.text.Utf8;
 import com.yahoo.vespa.config.ConfigDefinitionKey;
 import com.yahoo.vespa.config.ConfigKey;
 import com.yahoo.vespa.config.GetConfigRequest;
@@ -33,12 +34,14 @@ import org.junit.Before;
 import org.junit.Test;
 import org.xml.sax.SAXException;
 
+import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.IOException;
-import java.util.List;
+import java.nio.charset.StandardCharsets;
 import java.util.Optional;
 
 import static org.hamcrest.CoreMatchers.is;
+import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertThat;
 import static org.junit.Assert.assertTrue;
@@ -111,22 +114,27 @@ public class ApplicationTest {
     }
 
     @Test
-    public void require_that_build_config_can_be_resolved() {
-        List<String> payload = handler.resolveConfig(createRequest(ModelConfig.CONFIG_DEF_NAME, ModelConfig.CONFIG_DEF_NAMESPACE, ModelConfig.CONFIG_DEF_MD5, ModelConfig.CONFIG_DEF_SCHEMA)).getLegacyPayload();
-        assertTrue(payload.get(1).contains("host"));
+    public void require_that_build_config_can_be_resolved() throws IOException {
+        ByteArrayOutputStream baos = new ByteArrayOutputStream();
+        handler.resolveConfig(createRequest(ModelConfig.CONFIG_DEF_NAME, ModelConfig.CONFIG_DEF_NAMESPACE,
+                                                                   ModelConfig.CONFIG_DEF_MD5, ModelConfig.CONFIG_DEF_SCHEMA))
+                .serialize(baos, CompressionType.UNCOMPRESSED);
+        assertTrue(baos.toString().startsWith("{\"vespaVersion\":\"1.0.0\",\"hosts\":[{\"name\":\"mytesthost\""));
     }
 
     @Test
-    public void require_that_non_existent_fields_in_schema_is_skipped() {
+    public void require_that_non_existent_fields_in_schema_is_skipped() throws IOException {
         // Ask for config without schema and check that we get correct default value back
-        List<String> payload = handler.resolveConfig(createSimpleConfigRequest()).getLegacyPayload();
-        assertThat(payload.get(0), is("boolval false"));
+        ByteArrayOutputStream baos = new ByteArrayOutputStream();
+        handler.resolveConfig(createSimpleConfigRequest()).serialize(baos, CompressionType.UNCOMPRESSED);;
+        assertEquals("{\"boolval\":false,\"doubleval\":0.0,\"enumval\":\"VAL1\",\"intval\":0,\"longval\":0,\"stringval\":\"s\"}", baos.toString(StandardCharsets.UTF_8));
         // Ask for config with wrong schema
         String[] schema = new String[1];
         schema[0] = "boolval bool default=true"; // changed to be true, original is false
-        payload = handler.resolveConfig(createRequest(SimpletypesConfig.CONFIG_DEF_NAME, SimpletypesConfig.CONFIG_DEF_NAMESPACE, "", schema)).getLegacyPayload();
-        assertThat(payload.size(), is(1));
-        assertThat(payload.get(0), is("boolval true"));
+        baos = new ByteArrayOutputStream();
+        handler.resolveConfig(createRequest(SimpletypesConfig.CONFIG_DEF_NAME, SimpletypesConfig.CONFIG_DEF_NAMESPACE, "", schema))
+                                        .serialize(baos, CompressionType.UNCOMPRESSED);
+        assertEquals("{\"boolval\":true,\"doubleval\":0.0,\"enumval\":\"VAL1\",\"intval\":0,\"longval\":0,\"stringval\":\"s\"}", baos.toString(Utf8.getCharset()));
     }
 
     @Test
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/ConfigConvergenceCheckerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/ConfigConvergenceCheckerTest.java
index 71ae6955e56..1f034a92cbb 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/application/ConfigConvergenceCheckerTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/ConfigConvergenceCheckerTest.java
@@ -10,7 +10,7 @@ import com.yahoo.config.provision.TenantName;
 import com.yahoo.component.Version;
 import com.yahoo.container.jdisc.HttpResponse;
 import com.yahoo.slime.Slime;
-import com.yahoo.vespa.config.SlimeUtils;
+import com.yahoo.slime.SlimeUtils;
 import com.yahoo.vespa.config.server.ServerCache;
 import com.yahoo.vespa.config.server.monitoring.MetricUpdater;
 import org.junit.Before;
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/OrchestratorMock.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/OrchestratorMock.java
index beb8abb7be6..88d486cef87 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/application/OrchestratorMock.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/OrchestratorMock.java
@@ -2,15 +2,20 @@
 package com.yahoo.vespa.config.server.application;
 
 import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference;
 import com.yahoo.vespa.applicationmodel.HostName;
 import com.yahoo.vespa.orchestrator.Host;
 import com.yahoo.vespa.orchestrator.Orchestrator;
 import com.yahoo.vespa.orchestrator.status.ApplicationInstanceStatus;
+import com.yahoo.vespa.orchestrator.status.HostInfo;
 import com.yahoo.vespa.orchestrator.status.HostStatus;
 
+import java.time.Instant;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
 import java.util.function.Function;
@@ -22,7 +27,7 @@ import java.util.function.Function;
  */
 public class OrchestratorMock implements Orchestrator {
 
-    private final Set<HostName> suspendedHosts = new HashSet<>();
+    private final Map<HostName, HostInfo> hostInfos = new HashMap<>();
     private final Set<ApplicationId> suspendedApplications = new HashSet<>();
 
     @Override
@@ -32,12 +37,19 @@ public class OrchestratorMock implements Orchestrator {
 
     @Override
     public HostStatus getNodeStatus(HostName hostName) {
-        return suspendedHosts.contains(hostName) ? HostStatus.ALLOWED_TO_BE_DOWN : HostStatus.NO_REMARKS;
+        HostInfo hostInfo = hostInfos.get(hostName);
+        return hostInfo == null ? HostStatus.NO_REMARKS : hostInfo.status();
     }
 
     @Override
-    public Function<HostName, Optional<HostStatus>> getNodeStatuses() {
-        return hostName -> Optional.of(getNodeStatus(hostName));
+    public HostInfo getHostInfo(ApplicationInstanceReference reference, HostName hostname) {
+        HostInfo hostInfo = hostInfos.get(hostname);
+        return hostInfo == null ? HostInfo.createNoRemarks() : hostInfo;
+    }
+
+    @Override
+    public Function<HostName, Optional<HostInfo>> getHostResolver() {
+        return hostName -> Optional.of(hostInfos.getOrDefault(hostName, HostInfo.createNoRemarks()));
     }
 
     @Override
@@ -45,12 +57,12 @@ public class OrchestratorMock implements Orchestrator {
 
     @Override
     public void resume(HostName hostName) {
-        suspendedHosts.remove(hostName);
+        hostInfos.remove(hostName);
     }
 
     @Override
     public void suspend(HostName hostName) {
-        suspendedHosts.add(hostName);
+        hostInfos.put(hostName, HostInfo.createSuspended(HostStatus.ALLOWED_TO_BE_DOWN, Instant.EPOCH));
     }
 
     @Override
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java
index 32b704dd551..84987bce32e 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java
@@ -216,16 +216,32 @@ public class DeployTester {
         return deployApp(applicationPath, vespaVersion, Instant.now());
     }
 
+
     /**
      * Do the initial "deploy" with the existing API-less code as the deploy API doesn't support first deploys yet.
      */
-    public PrepareResult deployApp(String applicationPath, String vespaVersion, Instant now)  {
-        PrepareParams.Builder paramsBuilder = new PrepareParams.Builder()
-                .applicationId(applicationId)
-                .timeoutBudget(new TimeoutBudget(clock, Duration.ofSeconds(60)));
+    public PrepareResult deployApp(String applicationPath, String vespaVersion, String dockerImageRepository) {
+        PrepareParams.Builder paramsBuilder = new PrepareParams.Builder();
         if (vespaVersion != null)
             paramsBuilder.vespaVersion(vespaVersion);
 
+        return deployApp(applicationPath, Instant.now(), paramsBuilder.dockerImageRepository(dockerImageRepository));
+    }
+
+    /**
+     * Do the initial "deploy" with the existing API-less code as the deploy API doesn't support first deploys yet.
+     */
+    public PrepareResult deployApp(String applicationPath, String vespaVersion, Instant now)  {
+        return deployApp(applicationPath, now, new PrepareParams.Builder().vespaVersion(vespaVersion));
+    }
+
+    /**
+     * Do the initial "deploy" with the existing API-less code as the deploy API doesn't support first deploys yet.
+     */
+    public PrepareResult deployApp(String applicationPath, Instant now, PrepareParams.Builder paramsBuilder)  {
+         paramsBuilder.applicationId(applicationId)
+                .timeoutBudget(new TimeoutBudget(clock, Duration.ofSeconds(60)));
+
         return applicationRepository.deploy(new File(applicationPath), paramsBuilder.build(), false, now);
     }
 
@@ -283,8 +299,14 @@ public class DeployTester {
         }
 
         @Override
+        @Deprecated // TODO: Remove after April 2020
         public List<HostSpec> prepare(ApplicationId applicationId, ClusterSpec cluster, Capacity capacity, int groups, ProvisionLogger logger) {
-            return hostProvisioner.prepare(cluster, capacity, groups, logger);
+            return hostProvisioner.prepare(cluster, capacity.withGroups(groups), logger);
+        }
+
+        @Override
+        public List<HostSpec> prepare(ApplicationId applicationId, ClusterSpec cluster, Capacity capacity, ProvisionLogger logger) {
+            return hostProvisioner.prepare(cluster, capacity, logger);
         }
 
         @Override
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java
index dd0c4eaf342..7e700b78bf7 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java
@@ -24,6 +24,7 @@ import com.yahoo.vespa.config.server.configchange.RestartActions;
 import com.yahoo.vespa.config.server.http.InvalidApplicationException;
 import com.yahoo.vespa.config.server.http.v2.PrepareResult;
 import com.yahoo.vespa.config.server.model.TestModelFactory;
+import com.yahoo.vespa.config.server.session.PrepareParams;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.TemporaryFolder;
@@ -87,6 +88,24 @@ public class HostedDeployTest {
     }
 
     @Test
+    public void testReDeployWithWantedDockerImageRepositoryAndAthenzDomain() throws IOException {
+        CountingModelFactory modelFactory = createHostedModelFactory(Version.fromString("4.5.6"), Clock.systemUTC());
+        DeployTester tester = new DeployTester(List.of(modelFactory), createConfigserverConfig());
+        String dockerImageRepository = "docker.foo.com:4443/bar/baz";
+        tester.deployApp("src/test/apps/hosted/", Instant.now(), new PrepareParams.Builder()
+                .vespaVersion("4.5.6")
+                .dockerImageRepository(dockerImageRepository)
+                .athenzDomain("foo"));
+
+        Optional<com.yahoo.config.provision.Deployment> deployment = tester.redeployFromLocalActive(tester.applicationId());
+        assertTrue(deployment.isPresent());
+        deployment.get().activate();
+        assertEquals("4.5.6", ((Deployment) deployment.get()).session().getVespaVersion().toString());
+        assertEquals(dockerImageRepository, ((Deployment) deployment.get()).session().getDockerImageRepository().get());
+        assertEquals("foo", ((Deployment) deployment.get()).session().getAthenzDomain().get().value());
+    }
+
+    @Test
     public void testDeployMultipleVersions() throws IOException {
         List<ModelFactory> modelFactories = List.of(createHostedModelFactory(Version.fromString("6.1.0")),
                                                     createHostedModelFactory(Version.fromString("6.2.0")),
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpConfigResponseTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpConfigResponseTest.java
index 9a371639f52..ef0a5f6113d 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpConfigResponseTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpConfigResponseTest.java
@@ -2,34 +2,25 @@
 package com.yahoo.vespa.config.server.http;
 
 import com.yahoo.config.SimpletypesConfig;
-import com.yahoo.config.codegen.DefParser;
-import com.yahoo.config.codegen.InnerCNode;
-import com.yahoo.text.StringUtilities;
 import com.yahoo.vespa.config.ConfigPayload;
 import com.yahoo.vespa.config.protocol.ConfigResponse;
-
 import com.yahoo.vespa.config.protocol.SlimeConfigResponse;
 import org.junit.Test;
 
 import java.io.IOException;
-import java.io.StringReader;
 
 import static org.hamcrest.core.Is.is;
 import static org.junit.Assert.assertThat;
 
 /**
  * @author Ulf Lilleengen
- * @since 5.1
  */
 public class HttpConfigResponseTest {
     @Test
     public void require_that_response_is_created_from_config() throws IOException {
         final long generation = 1L;
         ConfigPayload payload = ConfigPayload.fromInstance(new SimpletypesConfig(new SimpletypesConfig.Builder()));
-        // TODO: Hope to be able to remove this mess soon.
-        DefParser dParser = new DefParser(SimpletypesConfig.getDefName(), new StringReader(StringUtilities.implode(SimpletypesConfig.CONFIG_DEF_SCHEMA, "\n")));
-        InnerCNode targetDef = dParser.getTree();
-        ConfigResponse configResponse = SlimeConfigResponse.fromConfigPayload(payload, targetDef, generation, false, "mymd5");
+        ConfigResponse configResponse = SlimeConfigResponse.fromConfigPayload(payload, generation, false, "mymd5");
         HttpConfigResponse response = HttpConfigResponse.createFromConfig(configResponse);
         assertThat(SessionHandlerTest.getRenderedString(response), is("{\"boolval\":false,\"doubleval\":0.0,\"enumval\":\"VAL1\",\"intval\":0,\"longval\":0,\"stringval\":\"s\"}"));
     }
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandlerTest.java
index 3ae98c1b8f2..089b662b797 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandlerTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandlerTest.java
@@ -2,28 +2,27 @@
 package com.yahoo.vespa.config.server.http;
 
 import com.yahoo.config.SimpletypesConfig;
-import com.yahoo.config.codegen.DefParser;
-import com.yahoo.config.codegen.InnerCNode;
+import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.container.jdisc.HttpRequest;
 import com.yahoo.container.jdisc.HttpResponse;
-import com.yahoo.text.StringUtilities;
 import com.yahoo.vespa.config.ConfigKey;
 import com.yahoo.vespa.config.ConfigPayload;
-import com.yahoo.vespa.config.server.rpc.MockRequestHandler;
 import com.yahoo.vespa.config.protocol.SlimeConfigResponse;
-import com.yahoo.config.provision.ApplicationId;
-
+import com.yahoo.vespa.config.server.rpc.MockRequestHandler;
 import org.junit.Before;
 import org.junit.Test;
+
 import java.io.IOException;
-import java.io.StringReader;
 import java.util.Collections;
 import java.util.HashSet;
-import static org.hamcrest.core.Is.is;
-import static org.junit.Assert.*;
 
 import static com.yahoo.jdisc.http.HttpRequest.Method.GET;
-import static com.yahoo.jdisc.http.HttpResponse.Status.*;
+import static com.yahoo.jdisc.http.HttpResponse.Status.BAD_REQUEST;
+import static com.yahoo.jdisc.http.HttpResponse.Status.NOT_FOUND;
+import static org.hamcrest.core.Is.is;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
 
 /**
  * @author Ulf Lilleengen
@@ -48,8 +47,7 @@ public class HttpGetConfigHandlerTest {
         // Define config response for mock handler
         final long generation = 1L;
         ConfigPayload payload = ConfigPayload.fromInstance(new SimpletypesConfig(new SimpletypesConfig.Builder()));
-        InnerCNode targetDef = getInnerCNode();
-        mockRequestHandler.responses.put(ApplicationId.defaultId(), SlimeConfigResponse.fromConfigPayload(payload, targetDef, generation, false, "mymd5"));
+        mockRequestHandler.responses.put(ApplicationId.defaultId(), SlimeConfigResponse.fromConfigPayload(payload, generation, false, "mymd5"));
         HttpResponse response = handler.handle(HttpRequest.createTestRequest(configUri, GET));
         assertThat(SessionHandlerTest.getRenderedString(response), is("{\"boolval\":false,\"doubleval\":0.0,\"enumval\":\"VAL1\",\"intval\":0,\"longval\":0,\"stringval\":\"s\"}"));
     }
@@ -75,16 +73,10 @@ public class HttpGetConfigHandlerTest {
     public void require_that_nocache_property_works() throws IOException {
         long generation = 1L;
         ConfigPayload payload = ConfigPayload.fromInstance(new SimpletypesConfig(new SimpletypesConfig.Builder()));
-        InnerCNode targetDef = getInnerCNode();
-        mockRequestHandler.responses.put(ApplicationId.defaultId(), SlimeConfigResponse.fromConfigPayload(payload, targetDef, generation, false, "mymd5"));
+        mockRequestHandler.responses.put(ApplicationId.defaultId(), SlimeConfigResponse.fromConfigPayload(payload, generation, false, "mymd5"));
         final HttpRequest request = HttpRequest.createTestRequest(configUri, GET, null, Collections.singletonMap("nocache", "true"));
         HttpResponse response = handler.handle(request);
         assertThat(SessionHandlerTest.getRenderedString(response), is("{\"boolval\":false,\"doubleval\":0.0,\"enumval\":\"VAL1\",\"intval\":0,\"longval\":0,\"stringval\":\"s\"}"));
     }
 
-    private InnerCNode getInnerCNode() {
-        // TODO: Hope to be able to remove this mess soon.
-        DefParser dParser = new DefParser(SimpletypesConfig.getDefName(), new StringReader(StringUtilities.implode(SimpletypesConfig.CONFIG_DEF_SCHEMA, "\n")));
-        return dParser.getTree();
-    }
 }
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/SessionHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/SessionHandlerTest.java
index 9a326a18dd5..5b0bb7885d8 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/SessionHandlerTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/SessionHandlerTest.java
@@ -42,6 +42,7 @@ import java.nio.charset.StandardCharsets;
 import java.time.Instant;
 import java.util.Collection;
 import java.util.List;
+import java.util.Map;
 import java.util.Optional;
 
 /**
@@ -55,15 +56,27 @@ public class SessionHandlerTest {
     public static final String hostname = "foo";
     public static final int port = 1337;
 
-    public static HttpRequest createTestRequest(String path, com.yahoo.jdisc.http.HttpRequest.Method method, Cmd cmd, Long id, String subPath, InputStream data) {
-        return HttpRequest.createTestRequest("http://" + hostname + ":" + port + path + "/" + id + "/" + cmd.toString() + subPath, method, data);
+
+    public static HttpRequest createTestRequest(String path, com.yahoo.jdisc.http.HttpRequest.Method method,
+                                                Cmd cmd, Long id, String subPath, InputStream data, Map<String, String> properties) {
+        return HttpRequest.createTestRequest("http://" + hostname + ":" + port + path + "/" + id + "/" +
+                                             cmd.toString() + subPath, method, data, properties);
+    }
+
+    public static HttpRequest createTestRequest(String path, com.yahoo.jdisc.http.HttpRequest.Method method,
+                                                Cmd cmd, Long id, String subPath, InputStream data) {
+        return HttpRequest.createTestRequest("http://" + hostname + ":" + port + path + "/" + id + "/" +
+                                             cmd.toString() + subPath, method, data);
     }
 
-    public static HttpRequest createTestRequest(String path, com.yahoo.jdisc.http.HttpRequest.Method method, Cmd cmd, Long id, String subPath) {
-        return HttpRequest.createTestRequest("http://" + hostname + ":" + port + path + "/" + id + "/" + cmd.toString() + subPath, method);
+    public static HttpRequest createTestRequest(String path, com.yahoo.jdisc.http.HttpRequest.Method method,
+                                                Cmd cmd, Long id, String subPath) {
+        return HttpRequest.createTestRequest("http://" + hostname + ":" + port + path + "/" + id + "/" +
+                                             cmd.toString() + subPath, method);
     }
 
-    public static HttpRequest createTestRequest(String path, com.yahoo.jdisc.http.HttpRequest.Method method, Cmd cmd, Long id) {
+    public static HttpRequest createTestRequest(String path, com.yahoo.jdisc.http.HttpRequest.Method method,
+                                                Cmd cmd, Long id) {
         return createTestRequest(path, method, cmd, id, "");
     }
 
@@ -88,6 +101,7 @@ public class SessionHandlerTest {
         private ConfigChangeActions actions = new ConfigChangeActions();
         private long createTime = System.currentTimeMillis() / 1000;
         private ApplicationId applicationId;
+        private Optional<String> dockerImageRepository;
 
         public MockSession(long id, ApplicationPackage app) {
             this(id, app, new InMemoryFlagSource());
@@ -115,6 +129,7 @@ public class SessionHandlerTest {
         @Override
         public ConfigChangeActions prepare(DeployLogger logger, PrepareParams params, Optional<ApplicationSet> application, Path tenantPath, Instant now) {
             status = Session.Status.PREPARE;
+            this.dockerImageRepository = params.dockerImageRepository();
             if (doVerboseLogging) {
                 logger.log(LogLevel.DEBUG, "debuglog");
             }
@@ -158,6 +173,10 @@ public class SessionHandlerTest {
         @Override
         public void delete(NestedTransaction transaction) {  }
 
+        @Override
+        public Optional<String> getDockerImageRepository() {
+            return dockerImageRepository;
+        }
     }
 
     public enum Cmd {
@@ -218,11 +237,17 @@ public class SessionHandlerTest {
         public Collection<HostSpec> lastHosts;
 
         @Override
+        @Deprecated // TODO: Remove after April 2020
         public List<HostSpec> prepare(ApplicationId applicationId, ClusterSpec cluster, Capacity capacity, int groups, ProvisionLogger logger) {
             throw new UnsupportedOperationException();
         }
 
         @Override
+        public List<HostSpec> prepare(ApplicationId applicationId, ClusterSpec cluster, Capacity capacity, ProvisionLogger logger) {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
         public void activate(NestedTransaction transaction, ApplicationId application, Collection<HostSpec> hosts) {
             activated = true;
             lastApplicationId = application;
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java
index 67677822317..70f66cf8fde 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java
@@ -28,12 +28,12 @@ import com.yahoo.vespa.config.server.session.PrepareParams;
 import com.yahoo.vespa.config.server.tenant.Tenant;
 import com.yahoo.vespa.config.server.tenant.TenantBuilder;
 import com.yahoo.vespa.config.server.tenant.TenantRepository;
+import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
 
 import javax.ws.rs.client.Client;
 import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
@@ -42,6 +42,7 @@ import java.nio.charset.StandardCharsets;
 import java.time.Clock;
 
 import static com.yahoo.config.model.api.container.ContainerServiceType.CLUSTERCONTROLLER_CONTAINER;
+import static com.yahoo.vespa.config.server.http.SessionHandlerTest.getRenderedString;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNull;
@@ -58,11 +59,16 @@ public class ApplicationHandlerTest {
 
     private static File testApp = new File("src/test/apps/app");
 
-    private ListApplicationsHandler listApplicationsHandler;
     private final static TenantName mytenantName = TenantName.from("mytenant");
     private final static TenantName foobar = TenantName.from("foobar");
-    private final static ApplicationId applicationId = new ApplicationId.Builder().applicationName(ApplicationName.defaultName()).tenant(mytenantName).build();
-
+    private final static ApplicationId myTenantApplicationId = new ApplicationId.Builder().applicationName(ApplicationName.defaultName()).tenant(mytenantName).build();
+    private final static ApplicationId applicationId = new ApplicationId.Builder().applicationName(ApplicationName.defaultName()).tenant(TenantName.defaultName()).build();
+    private final static MockTesterClient testerClient = new MockTesterClient();
+    private final static NullMetric metric = new NullMetric();
+    private final static ConfigserverConfig configserverConfig = new ConfigserverConfig(new ConfigserverConfig.Builder());
+    private static final MockLogRetriever logRetriever = new MockLogRetriever();
+
+    private TestComponentRegistry componentRegistry;
     private TenantRepository tenantRepository;
     private ApplicationRepository applicationRepository;
     private SessionHandlerTest.MockProvisioner provisioner;
@@ -71,27 +77,30 @@ public class ApplicationHandlerTest {
 
     @Before
     public void setup() {
-        TestComponentRegistry componentRegistry = new TestComponentRegistry.Builder().build();
+        componentRegistry = new TestComponentRegistry.Builder().build();
         tenantRepository = new TenantRepository(componentRegistry, false);
-        tenantRepository.addTenant(TenantBuilder.create(componentRegistry, mytenantName));
-        tenantRepository.addTenant(TenantBuilder.create(componentRegistry, foobar));
         provisioner = new SessionHandlerTest.MockProvisioner();
         orchestrator = new OrchestratorMock();
         applicationRepository = new ApplicationRepository(tenantRepository,
                                                           provisioner,
                                                           orchestrator,
-                                                          new ConfigserverConfig(new ConfigserverConfig.Builder()),
-                                                          new MockLogRetriever(),
+                                                          configserverConfig,
+                                                          logRetriever,
                                                           Clock.systemUTC(),
-                                                          new MockTesterClient(),
-                                                          new NullMetric());
-        listApplicationsHandler = new ListApplicationsHandler(ListApplicationsHandler.testOnlyContext(),
-                                                              tenantRepository,
-                                                              Zone.defaultZone());
+                                                          testerClient,
+                                                          metric);
+    }
+
+    @After
+    public void shutdown() {
+        tenantRepository.close();
     }
 
     @Test
     public void testDelete() throws Exception {
+        tenantRepository.addTenant(TenantBuilder.create(componentRegistry, foobar));
+        tenantRepository.addTenant(TenantBuilder.create(componentRegistry, mytenantName));
+
         {
             applicationRepository.deploy(testApp, prepareParams(applicationId));
             Tenant mytenant = tenantRepository.getTenant(applicationId.tenant());
@@ -132,7 +141,7 @@ public class ApplicationHandlerTest {
 
     @Test
     public void testDeleteNonExistent() throws Exception {
-        deleteAndAssertResponse(applicationId,
+        deleteAndAssertResponse(myTenantApplicationId,
                                 Zone.defaultZone(),
                                 Response.Status.NOT_FOUND,
                                 HttpErrorResponse.errorCodes.NOT_FOUND,
@@ -180,10 +189,10 @@ public class ApplicationHandlerTest {
                                                                                 InfraDeployerProvider.empty(),
                                                                                 new ConfigConvergenceChecker(stateApiFactory),
                                                                                 mockHttpProxy,
-                                                                                new ConfigserverConfig(new ConfigserverConfig.Builder()),
-                                                                                new OrchestratorMock(),
-                                                                                new MockTesterClient(),
-                                                                                new NullMetric());
+                                                                                configserverConfig,
+                                                                                orchestrator,
+                                                                                testerClient,
+                                                                                metric);
         ApplicationHandler mockHandler = createApplicationHandler(applicationRepository);
         when(mockHttpProxy.get(any(), eq(host), eq(CLUSTERCONTROLLER_CONTAINER.serviceName),eq("clustercontroller-status/v1/clusterName1")))
                 .thenReturn(new StaticResponse(200, "text/html", "<html>...</html>"));
@@ -204,16 +213,15 @@ public class ApplicationHandlerTest {
 
         HttpResponse response = fileDistributionStatus(applicationId, zone);
         assertEquals(200, response.getStatus());
-        SessionHandlerTest.getRenderedString(response);
         assertEquals("{\"hosts\":[{\"hostname\":\"mytesthost\",\"status\":\"UNKNOWN\",\"message\":\"error: Connection error(104)\",\"fileReferences\":[]}],\"status\":\"UNKNOWN\"}",
-                     SessionHandlerTest.getRenderedString(response));
+                     getRenderedString(response));
 
         // 404 for unknown application
-        ApplicationId unknown = new ApplicationId.Builder().applicationName("unknown").tenant(mytenantName).build();
+        ApplicationId unknown = new ApplicationId.Builder().applicationName("unknown").tenant("default").build();
         HttpResponse responseForUnknown = fileDistributionStatus(unknown, zone);
         assertEquals(404, responseForUnknown.getStatus());
-        assertEquals("{\"error-code\":\"NOT_FOUND\",\"message\":\"No such application id: 'mytenant.unknown'\"}",
-                     SessionHandlerTest.getRenderedString(responseForUnknown));
+        assertEquals("{\"error-code\":\"NOT_FOUND\",\"message\":\"No such application id: 'default.unknown'\"}",
+                     getRenderedString(responseForUnknown));
     }
 
     @Test
@@ -225,9 +233,7 @@ public class ApplicationHandlerTest {
         HttpResponse response = mockHandler.handle(HttpRequest.createTestRequest(url, com.yahoo.jdisc.http.HttpRequest.Method.GET));
         assertEquals(200, response.getStatus());
 
-        ByteArrayOutputStream baos = new ByteArrayOutputStream();
-        response.render(baos);
-        assertEquals("log line", baos.toString());
+        assertEquals("log line", getRenderedString(response));
     }
 
     @Test
@@ -235,13 +241,9 @@ public class ApplicationHandlerTest {
         applicationRepository.deploy(testApp, prepareParams(applicationId));
         String url = toUrlPath(applicationId, Zone.defaultZone(), true) + "/tester/status";
         ApplicationHandler mockHandler = createApplicationHandler();
-
         HttpResponse response = mockHandler.handle(HttpRequest.createTestRequest(url, com.yahoo.jdisc.http.HttpRequest.Method.GET));
         assertEquals(200, response.getStatus());
-
-        ByteArrayOutputStream baos = new ByteArrayOutputStream();
-        response.render(baos);
-        assertEquals("OK", baos.toString());
+        assertEquals("OK", getRenderedString(response));
     }
 
     @Test
@@ -252,10 +254,7 @@ public class ApplicationHandlerTest {
 
         HttpResponse response = mockHandler.handle(HttpRequest.createTestRequest(url, com.yahoo.jdisc.http.HttpRequest.Method.GET));
         assertEquals(200, response.getStatus());
-
-        ByteArrayOutputStream baos = new ByteArrayOutputStream();
-        response.render(baos);
-        assertEquals("log", baos.toString());
+        assertEquals("log", getRenderedString(response));
     }
 
     @Test
@@ -345,10 +344,13 @@ public class ApplicationHandlerTest {
                 "/environment/" + zone.environment().value() +
                 "/region/" + zone.region().value() +
                 "/instance/" + applicationId.instance().value() + "\"]";
+        ListApplicationsHandler listApplicationsHandler = new ListApplicationsHandler(ListApplicationsHandler.testOnlyContext(),
+                                                                                      tenantRepository,
+                                                                                      Zone.defaultZone());
         ListApplicationsHandlerTest.assertResponse(listApplicationsHandler, "http://myhost:14000/application/v2/tenant/" + tenantName + "/application/",
-                Response.Status.OK,
-                expected,
-                com.yahoo.jdisc.http.HttpRequest.Method.GET);
+                                                   Response.Status.OK,
+                                                   expected,
+                                                   com.yahoo.jdisc.http.HttpRequest.Method.GET);
     }
 
     private void restart(ApplicationId application, Zone zone) throws IOException {
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandlerTest.java
index fb09aa99039..b72785876bc 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandlerTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandlerTest.java
@@ -1,37 +1,36 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.config.server.http.v2;
 
-import static com.yahoo.jdisc.Response.Status.BAD_REQUEST;
-import static com.yahoo.jdisc.Response.Status.NOT_FOUND;
-import static com.yahoo.jdisc.http.HttpRequest.Method.GET;
-import static org.hamcrest.core.Is.is;
-import static org.junit.Assert.*;
-import java.io.IOException;
-import java.io.StringReader;
-import java.util.Collections;
-import java.util.HashSet;
-
-import com.yahoo.config.provision.TenantName;
-import com.yahoo.vespa.config.server.TestComponentRegistry;
-import com.yahoo.vespa.config.server.http.HttpErrorResponse;
-import com.yahoo.vespa.config.server.tenant.TenantBuilder;
-import com.yahoo.vespa.config.server.tenant.TenantRepository;
-import org.junit.Before;
-import org.junit.Test;
 import com.yahoo.config.SimpletypesConfig;
-import com.yahoo.config.codegen.DefParser;
-import com.yahoo.config.codegen.InnerCNode;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.TenantName;
 import com.yahoo.container.jdisc.HttpRequest;
 import com.yahoo.container.jdisc.HttpResponse;
-import com.yahoo.text.StringUtilities;
 import com.yahoo.vespa.config.ConfigKey;
 import com.yahoo.vespa.config.ConfigPayload;
 import com.yahoo.vespa.config.protocol.SlimeConfigResponse;
-import com.yahoo.vespa.config.server.rpc.MockRequestHandler;
-import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.vespa.config.server.TestComponentRegistry;
 import com.yahoo.vespa.config.server.http.HandlerTest;
 import com.yahoo.vespa.config.server.http.HttpConfigRequest;
+import com.yahoo.vespa.config.server.http.HttpErrorResponse;
 import com.yahoo.vespa.config.server.http.SessionHandlerTest;
+import com.yahoo.vespa.config.server.rpc.MockRequestHandler;
+import com.yahoo.vespa.config.server.tenant.TenantBuilder;
+import com.yahoo.vespa.config.server.tenant.TenantRepository;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashSet;
+
+import static com.yahoo.jdisc.Response.Status.BAD_REQUEST;
+import static com.yahoo.jdisc.Response.Status.NOT_FOUND;
+import static com.yahoo.jdisc.http.HttpRequest.Method.GET;
+import static org.hamcrest.core.Is.is;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
 
 public class HttpGetConfigHandlerTest {
 
@@ -58,9 +57,8 @@ public class HttpGetConfigHandlerTest {
         // Define config response for mock handler
         final long generation = 1L;
         ConfigPayload payload = ConfigPayload.fromInstance(new SimpletypesConfig(new SimpletypesConfig.Builder()));
-        InnerCNode targetDef = getInnerCNode();
         mockRequestHandler.responses.put(new ApplicationId.Builder().tenant(tenant).applicationName("myapplication").build(),
-                                         SlimeConfigResponse.fromConfigPayload(payload, targetDef, generation, false, "mymd5"));
+                                         SlimeConfigResponse.fromConfigPayload(payload, generation, false, "mymd5"));
         HttpResponse response = handler.handle(HttpRequest.createTestRequest(configUri, GET));
         assertThat(SessionHandlerTest.getRenderedString(response), is(EXPECTED_RENDERED_STRING));
     }
@@ -71,11 +69,10 @@ public class HttpGetConfigHandlerTest {
                               "/application/myapplication/environment/staging/region/myregion/instance/myinstance/foo.bar/myid";
         final long generation = 1L;
         ConfigPayload payload = ConfigPayload.fromInstance(new SimpletypesConfig(new SimpletypesConfig.Builder()));
-        InnerCNode targetDef = getInnerCNode();
         mockRequestHandler.responses.put(new ApplicationId.Builder()
                                          .tenant(tenant)
                                          .applicationName("myapplication").instanceName("myinstance").build(),
-                                         SlimeConfigResponse.fromConfigPayload(payload, targetDef, generation, false, "mymd5"));
+                                         SlimeConfigResponse.fromConfigPayload(payload, generation, false, "mymd5"));
         HttpResponse response = handler.handle(HttpRequest.createTestRequest(uriLongAppId, GET));
         assertThat(SessionHandlerTest.getRenderedString(response), is(EXPECTED_RENDERED_STRING));
     }
@@ -121,18 +118,11 @@ public class HttpGetConfigHandlerTest {
     public void require_that_nocache_property_works() throws IOException {
         long generation = 1L;
         ConfigPayload payload = ConfigPayload.fromInstance(new SimpletypesConfig(new SimpletypesConfig.Builder()));
-        InnerCNode targetDef = getInnerCNode();
         mockRequestHandler.responses.put(new ApplicationId.Builder().tenant(tenant).applicationName("myapplication").build(),
-                                         SlimeConfigResponse.fromConfigPayload(payload, targetDef, generation, false, "mymd5"));
+                                         SlimeConfigResponse.fromConfigPayload(payload, generation, false, "mymd5"));
         final HttpRequest request = HttpRequest.createTestRequest(configUri, GET, null, Collections.singletonMap("nocache", "true"));
         HttpResponse response = handler.handle(request);
         assertThat(SessionHandlerTest.getRenderedString(response), is(EXPECTED_RENDERED_STRING));
     }
 
-    private InnerCNode getInnerCNode() {
-        // TODO: Hope to be able to remove this mess soon.
-        DefParser dParser = new DefParser(SimpletypesConfig.getDefName(), new StringReader(StringUtilities.implode(SimpletypesConfig.CONFIG_DEF_SCHEMA, "\n")));
-        return dParser.getTree();
-    }
-
 }
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandlerTest.java
index 028f5f9eb8c..11cbdb03ccf 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandlerTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandlerTest.java
@@ -44,6 +44,7 @@ import java.time.Instant;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
+import java.util.Map;
 import java.util.Optional;
 
 import static com.yahoo.jdisc.Response.Status.BAD_REQUEST;
@@ -90,7 +91,7 @@ public class SessionPrepareHandlerTest extends SessionHandlerTest {
     @Test
     public void require_error_when_session_id_does_not_exist() throws Exception {
         // No session with this id exists
-        HttpResponse response = createHandler().handle(SessionHandlerTest.createTestRequest(pathPrefix, HttpRequest.Method.PUT, Cmd.PREPARED, 9999L));
+        HttpResponse response = request(HttpRequest.Method.PUT, 9999L);
         assertHttpStatusCodeErrorCodeAndMessage(response, NOT_FOUND, HttpErrorResponse.errorCodes.NOT_FOUND, "Session 9999 was not found");
     }
 
@@ -120,8 +121,7 @@ public class SessionPrepareHandlerTest extends SessionHandlerTest {
     public void require_that_activate_url_is_returned_on_success() throws Exception {
         MockSession session = new MockSession(1, null);
         localRepo.addSession(session);
-        HttpResponse response = createHandler().handle(
-                SessionHandlerTest.createTestRequest(pathPrefix, HttpRequest.Method.PUT, Cmd.PREPARED, 1L));
+        HttpResponse response = request(HttpRequest.Method.PUT, 1L);
         assertThat(session.getStatus(), is(Session.Status.PREPARE));
         assertNotNull(response);
         assertThat(response.getStatus(), is(OK));
@@ -155,13 +155,11 @@ public class SessionPrepareHandlerTest extends SessionHandlerTest {
     public void require_get_response_activate_url_on_ok() throws Exception {
         MockSession session = new MockSession(1, null);
         localRepo.addSession(session);
-        SessionHandler sessHandler = createHandler();
-        sessHandler.handle(SessionHandlerTest.createTestRequest(pathPrefix, HttpRequest.Method.PUT, Cmd.PREPARED, 1L));
+        request(HttpRequest.Method.PUT, 1L);
         session.setStatus(Session.Status.PREPARE);
         SessionZooKeeperClient zooKeeperClient = createSessionZooKeeperClient(session);
         zooKeeperClient.writeStatus(Session.Status.PREPARE);
-        HttpResponse getResponse = sessHandler.handle(
-                SessionHandlerTest.createTestRequest(pathPrefix, HttpRequest.Method.GET, Cmd.PREPARED, 1L));
+        HttpResponse getResponse = request(HttpRequest.Method.GET, 1L);
         assertResponseContains(getResponse, "\"activate\":\"http://foo:1337" + pathPrefix +
                 "1/active\",\"message\":\"Session 1" + preparedMessage);
     }
@@ -170,19 +168,16 @@ public class SessionPrepareHandlerTest extends SessionHandlerTest {
     public void require_get_response_error_on_not_prepared() throws Exception {
         MockSession session = new MockSession(1, null);
         localRepo.addSession(session);
-        SessionHandler sessHandler = createHandler();
         session.setStatus(Session.Status.NEW);
         SessionZooKeeperClient zooKeeperClient = createSessionZooKeeperClient(session);
         zooKeeperClient.writeStatus(Session.Status.NEW);
-        HttpResponse getResponse = sessHandler.handle(
-                SessionHandlerTest.createTestRequest(pathPrefix, HttpRequest.Method.GET, Cmd.PREPARED, 1L));
+        HttpResponse getResponse = request(HttpRequest.Method.GET, 1L);
         assertHttpStatusCodeErrorCodeAndMessage(getResponse, BAD_REQUEST,
                                                 HttpErrorResponse.errorCodes.BAD_REQUEST,
                                                 "Session not prepared: 1");
         session.setStatus(Session.Status.ACTIVATE);
         zooKeeperClient.writeStatus(Session.Status.ACTIVATE);
-        getResponse = sessHandler.handle(
-                SessionHandlerTest.createTestRequest(pathPrefix, HttpRequest.Method.GET, Cmd.PREPARED, 1L));
+        getResponse = request(HttpRequest.Method.GET, 1L);
         assertHttpStatusCodeErrorCodeAndMessage(getResponse, BAD_REQUEST,
                                                 HttpErrorResponse.errorCodes.BAD_REQUEST,
                                                 "Session is active: 1");
@@ -193,9 +188,7 @@ public class SessionPrepareHandlerTest extends SessionHandlerTest {
         MockSession session = new MockSession(1, null);
         localRepo.addSession(session);
         session.setStatus(Session.Status.ACTIVATE);
-        SessionHandler sessionHandler = createHandler();
-        HttpResponse putResponse = sessionHandler.handle(
-                SessionHandlerTest.createTestRequest(pathPrefix, HttpRequest.Method.PUT, Cmd.PREPARED, 1L));
+        HttpResponse putResponse = request(HttpRequest.Method.PUT, 1L);
         assertHttpStatusCodeErrorCodeAndMessage(putResponse, BAD_REQUEST,
                                                 HttpErrorResponse.errorCodes.BAD_REQUEST,
                                                 "Session is active: 1");
@@ -205,9 +198,7 @@ public class SessionPrepareHandlerTest extends SessionHandlerTest {
     public void require_get_response_error_when_session_id_does_not_exist() throws Exception {
         MockSession session = new MockSession(1, null);
         localRepo.addSession(session);
-        SessionHandler sessHandler = createHandler();
-        HttpResponse getResponse = sessHandler.handle(
-                SessionHandlerTest.createTestRequest(pathPrefix, HttpRequest.Method.GET, Cmd.PREPARED, 9999L));
+        HttpResponse getResponse = request(HttpRequest.Method.GET, 9999L);
         assertHttpStatusCodeErrorCodeAndMessage(getResponse, NOT_FOUND,
                                                 HttpErrorResponse.errorCodes.NOT_FOUND,
                                                 "Session 9999 was not found");
@@ -217,8 +208,7 @@ public class SessionPrepareHandlerTest extends SessionHandlerTest {
     public void require_that_tenant_is_in_response() throws Exception {
         MockSession session = new MockSession(1, null);
         localRepo.addSession(session);
-        HttpResponse response = createHandler().handle(
-                SessionHandlerTest.createTestRequest(pathPrefix, HttpRequest.Method.PUT, Cmd.PREPARED, 1L));
+        HttpResponse response = request(HttpRequest.Method.PUT, 1L);
         assertNotNull(response);
         assertThat(response.getStatus(), is(OK));
         assertThat(session.getStatus(), is(Session.Status.PREPARE));
@@ -242,8 +232,7 @@ public class SessionPrepareHandlerTest extends SessionHandlerTest {
         localRepoDefault.addSession(session);
         pathPrefix = "/application/v2/tenant/" + defaultTenant + "/session/";
 
-        HttpResponse response = handler.handle(
-                SessionHandlerTest.createTestRequest(pathPrefix, HttpRequest.Method.PUT, Cmd.PREPARED, sessionId));
+        HttpResponse response = request(HttpRequest.Method.PUT, sessionId);
         assertNotNull(response);
         assertThat(SessionHandlerTest.getRenderedString(response), response.getStatus(), is(OK));
         assertThat(session.getStatus(), is(Session.Status.PREPARE));
@@ -274,8 +263,7 @@ public class SessionPrepareHandlerTest extends SessionHandlerTest {
     public void require_that_config_change_actions_are_in_response() throws Exception {
         MockSession session = new MockSession(1, null);
         localRepo.addSession(session);
-        HttpResponse response = createHandler().handle(
-                SessionHandlerTest.createTestRequest(pathPrefix, HttpRequest.Method.PUT, Cmd.PREPARED, 1L));
+        HttpResponse response = request(HttpRequest.Method.PUT, 1L);
         assertResponseContains(response, "\"configChangeActions\":{\"restart\":[],\"refeed\":[]}");
     }
 
@@ -289,8 +277,7 @@ public class SessionPrepareHandlerTest extends SessionHandlerTest {
                 new MockRefeedAction("change-id", false, "other change", services, "test")));
         MockSession session = new MockSession(1, null, actions);
         localRepo.addSession(session);
-        HttpResponse response = createHandler().handle(
-                SessionHandlerTest.createTestRequest(pathPrefix, HttpRequest.Method.PUT, Cmd.PREPARED, 1L));
+        HttpResponse response = request(HttpRequest.Method.PUT, 1L);
         assertResponseContains(response,
                                "Change(s) between active and new application that require restart:\\nIn cluster 'foo' of type 'bar");
         assertResponseContains(response,
@@ -301,8 +288,7 @@ public class SessionPrepareHandlerTest extends SessionHandlerTest {
     public void require_that_config_change_actions_are_not_logged_if_not_existing() throws Exception {
         MockSession session = new MockSession(1, null);
         localRepo.addSession(session);
-        HttpResponse response = createHandler().handle(
-                SessionHandlerTest.createTestRequest(pathPrefix, HttpRequest.Method.PUT, Cmd.PREPARED, 1L));
+        HttpResponse response = request(HttpRequest.Method.PUT, 1L);
         assertResponseNotContains(response, "Change(s) between active and new application that require restart");
         assertResponseNotContains(response, "Change(s) between active and new application that require re-feed");
     }
@@ -312,8 +298,7 @@ public class SessionPrepareHandlerTest extends SessionHandlerTest {
         String message = "Internal error";
         SessionThrowingException session = new SessionThrowingException(new OutOfCapacityException(message));
         localRepo.addSession(session);
-        HttpResponse response = createHandler()
-                .handle(SessionHandlerTest.createTestRequest(pathPrefix, HttpRequest.Method.PUT, Cmd.PREPARED, 1L));
+        HttpResponse response = request(HttpRequest.Method.PUT, 1L);
         assertEquals(400, response.getStatus());
         Slime data = getData(response);
         assertThat(data.get().field("error-code").asString(), is(HttpErrorResponse.errorCodes.OUT_OF_CAPACITY.name()));
@@ -325,8 +310,7 @@ public class SessionPrepareHandlerTest extends SessionHandlerTest {
         String message = "No nodes available";
         SessionThrowingException session = new SessionThrowingException(new NullPointerException(message));
         localRepo.addSession(session);
-        HttpResponse response = createHandler()
-                .handle(SessionHandlerTest.createTestRequest(pathPrefix, HttpRequest.Method.PUT, Cmd.PREPARED, 1L));
+        HttpResponse response = request(HttpRequest.Method.PUT, 1L);
         assertEquals(500, response.getStatus());
         Slime data = getData(response);
         assertThat(data.get().field("error-code").asString(), is(HttpErrorResponse.errorCodes.INTERNAL_SERVER_ERROR.name()));
@@ -339,14 +323,22 @@ public class SessionPrepareHandlerTest extends SessionHandlerTest {
         SessionThrowingException session =
                 new SessionThrowingException(new ApplicationLockException(new UncheckedTimeoutException(message)));
         localRepo.addSession(session);
-        HttpResponse response = createHandler()
-                .handle(SessionHandlerTest.createTestRequest(pathPrefix, HttpRequest.Method.PUT, Cmd.PREPARED, 1L));
+        HttpResponse response = request(HttpRequest.Method.PUT, 1L);
         assertEquals(500, response.getStatus());
         Slime data = getData(response);
         assertThat(data.get().field("error-code").asString(), is(HttpErrorResponse.errorCodes.APPLICATION_LOCK_FAILURE.name()));
         assertThat(data.get().field("message").asString(), is(message));
     }
 
+    @Test
+    public void test_docker_image_repository() {
+        MockSession session = new MockSession(1, null);
+        localRepo.addSession(session);
+        String dockerImageRepository = "https://foo.bar.com:4443/baz";
+        request(HttpRequest.Method.PUT, 1L, Map.of("dockerImageRepository", dockerImageRepository));
+        assertEquals(dockerImageRepository, localRepo.getSession(1).getDockerImageRepository().get());
+    }
+
     private Slime getData(HttpResponse response) throws IOException {
         ByteArrayOutputStream baos = new ByteArrayOutputStream();
         response.render(baos);
@@ -374,6 +366,14 @@ public class SessionPrepareHandlerTest extends SessionHandlerTest {
 
     }
 
+    private HttpResponse request(HttpRequest.Method put, long l) {
+        return request(put, l, Map.of());
+    }
+
+    private HttpResponse request(HttpRequest.Method put, long l, Map<String, String> requestParameters) {
+        return createHandler().handle(SessionHandlerTest.createTestRequest(pathPrefix, put, Cmd.PREPARED, l, "", null, requestParameters));
+    }
+
     public static class SessionThrowingException extends LocalSession {
         private final RuntimeException exception;
 
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/metrics/ClusterMetricsRetrieverTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/metrics/ClusterMetricsRetrieverTest.java
index 0894e38ce09..3f67d8e2cac 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/metrics/ClusterMetricsRetrieverTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/metrics/ClusterMetricsRetrieverTest.java
@@ -14,13 +14,15 @@ import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.function.BiConsumer;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 import static com.github.tomakehurst.wiremock.client.WireMock.aResponse;
 import static com.github.tomakehurst.wiremock.client.WireMock.get;
 import static com.github.tomakehurst.wiremock.client.WireMock.stubFor;
 import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo;
 import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.options;
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
 
 
 /**
@@ -29,11 +31,13 @@ import static org.junit.Assert.*;
 public class ClusterMetricsRetrieverTest {
 
     @Rule
-    public final WireMockRule wireMock = new WireMockRule(options().port(8080), true);
+    public final WireMockRule wireMock = new WireMockRule(options().dynamicPort(), true);
 
     @Test
     public void testMetricAggregation() throws IOException {
-        List<URI> hosts = List.of(URI.create("http://localhost:8080/1"), URI.create("http://localhost:8080/2"), URI.create("http://localhost:8080/3"));
+        List<URI> hosts = Stream.of(1, 2, 3)
+                .map(item -> URI.create("http://localhost:" + wireMock.port() + "/" + item))
+                .collect(Collectors.toList());
 
         stubFor(get(urlEqualTo("/1"))
                 .willReturn(aResponse()
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/model/LbServicesProducerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/model/LbServicesProducerTest.java
index 9a7cb72804f..f75d11a145f 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/model/LbServicesProducerTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/model/LbServicesProducerTest.java
@@ -38,6 +38,7 @@ import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.is;
 import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertThat;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assume.assumeFalse;
@@ -53,7 +54,7 @@ public class LbServicesProducerTest {
     private static final Set<ContainerEndpoint> endpoints = Set.of(
             new ContainerEndpoint("mydisc", List.of("rotation-1", "rotation-2"))
     );
-    private final InMemoryFlagSource flagSource = new InMemoryFlagSource();
+    private InMemoryFlagSource flagSource = new InMemoryFlagSource();
     private final boolean useGlobalServiceId;
 
     @Parameterized.Parameters
@@ -141,6 +142,23 @@ public class LbServicesProducerTest {
         assertThat("Missing endpoints in list: " + services.endpointaliases(), services.endpointaliases(), containsInAnyOrder("foo1.bar1.com", "foo2.bar2.com", rotation1, rotation2));
     }
 
+
+    @Test
+    public void testRoutingConfigForTesterApplication() throws IOException, SAXException {
+        assumeFalse(useGlobalServiceId);
+
+        Map<TenantName, Set<ApplicationInfo>> testModel = createTestModel(new DeployState.Builder());
+        LbServicesConfig conf = getLbServicesConfig(Zone.defaultZone(), testModel);
+        LbServicesConfig.Tenants.Applications.Hosts.Services services = conf.tenants("foo").applications("foo:prod:default:default").hosts("foo.foo.yahoo.com").services(QRSERVER.serviceName);
+        assertThat(services.servicealiases().size(), is(1));
+        assertThat(services.endpointaliases().size(), is(2));
+
+        // No config for tester application
+        assertNull(getLbServicesConfig(Zone.defaultZone(), testModel)
+                           .tenants("foo")
+                           .applications("baz:prod:default:custom-t"));
+    }
+
     private Map<TenantName, Set<ApplicationInfo>> randomizeApplications(Map<TenantName, Set<ApplicationInfo>> testModel, int seed) {
         Map<TenantName, Set<ApplicationInfo>> randomizedApplications = new LinkedHashMap<>();
         List<TenantName> keys = new ArrayList<>(testModel.keySet());
@@ -166,7 +184,7 @@ public class LbServicesProducerTest {
         Set<ApplicationInfo> aMap = new LinkedHashSet<>();
         ApplicationId fooApp = new ApplicationId.Builder().tenant(tenant).applicationName("foo").build();
         ApplicationId barApp = new ApplicationId.Builder().tenant(tenant).applicationName("bar").build();
-        ApplicationId bazApp = new ApplicationId.Builder().tenant(tenant).applicationName("baz").build();
+        ApplicationId bazApp = new ApplicationId.Builder().tenant(tenant).applicationName("baz").instanceName("custom-t").build(); // tester app
         aMap.add(createApplication(fooApp, deploystateBuilder));
         aMap.add(createApplication(barApp, deploystateBuilder));
         aMap.add(createApplication(bazApp, deploystateBuilder));
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/monitoring/ZKMetricUpdaterTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/monitoring/ZKMetricUpdaterTest.java
index ed089109759..607a2dca6c6 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/monitoring/ZKMetricUpdaterTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/monitoring/ZKMetricUpdaterTest.java
@@ -57,12 +57,14 @@ public class ZKMetricUpdaterTest {
         assertThat(reportedMetrics.get(ZKMetricUpdater.METRIC_ZK_LATENCY_MAX), equalTo(1234L));
         assertThat(reportedMetrics.get(ZKMetricUpdater.METRIC_ZK_OUTSTANDING_REQUESTS), equalTo(12L));
         assertThat(reportedMetrics.get(ZKMetricUpdater.METRIC_ZK_ZNODES), equalTo(4L));
+
+        updater.shutdown();
     }
 
     private ZKMetricUpdater buildUpdater() {
         ZookeeperServerConfig zkServerConfig = new ZookeeperServerConfig(
                 new ZookeeperServerConfig.Builder().clientPort(serverPort).myid(12345));
-        return new ZKMetricUpdater(zkServerConfig, 0, -1);
+        return new ZKMetricUpdater(zkServerConfig, 0, 100000);
     }
 
     private void setupTcpServer(Supplier<String> reportProvider) throws IOException {
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcServerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcServerTest.java
index 086dfa5d0d3..0b33de2a42c 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcServerTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcServerTest.java
@@ -1,18 +1,15 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.config.server.rpc;
 
-import com.google.common.base.Joiner;
 import com.yahoo.cloud.config.ConfigserverConfig;
 import com.yahoo.cloud.config.LbServicesConfig;
 import com.yahoo.cloud.config.SentinelConfig;
+import com.yahoo.component.Version;
 import com.yahoo.config.SimpletypesConfig;
-import com.yahoo.config.codegen.DefParser;
-import com.yahoo.config.codegen.InnerCNode;
 import com.yahoo.config.model.test.MockApplicationPackage;
+import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.TenantName;
-import com.yahoo.component.Version;
 import com.yahoo.jrt.Request;
-import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.vespa.config.ConfigKey;
 import com.yahoo.vespa.config.ConfigPayload;
 import com.yahoo.vespa.config.ConfigPayloadApplier;
@@ -24,12 +21,11 @@ import com.yahoo.vespa.config.protocol.JRTClientConfigRequest;
 import com.yahoo.vespa.config.protocol.JRTClientConfigRequestV3;
 import com.yahoo.vespa.config.protocol.SlimeConfigResponse;
 import com.yahoo.vespa.config.protocol.Trace;
-import com.yahoo.vespa.config.server.application.ApplicationSet;
 import com.yahoo.vespa.config.server.ServerCache;
 import com.yahoo.vespa.config.server.application.Application;
+import com.yahoo.vespa.config.server.application.ApplicationSet;
 import com.yahoo.vespa.config.server.monitoring.MetricUpdater;
 import com.yahoo.vespa.config.util.ConfigUtils;
-
 import com.yahoo.vespa.model.VespaModel;
 import org.junit.Rule;
 import org.junit.Test;
@@ -37,11 +33,14 @@ import org.junit.rules.TemporaryFolder;
 import org.xml.sax.SAXException;
 
 import java.io.IOException;
-import java.io.StringReader;
 import java.util.Optional;
 
 import static org.hamcrest.core.Is.is;
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
 
 /**
  * @author Ulf Lilleengen
@@ -161,11 +160,7 @@ public class RpcServerTest {
         builder.intval(123);
         SimpletypesConfig responseConfig = new SimpletypesConfig(builder);
         ConfigPayload responsePayload = ConfigPayload.fromInstance(responseConfig);
-        InnerCNode targetDef = new DefParser(SimpletypesConfig.CONFIG_DEF_NAME,
-                                             new StringReader(Joiner.on("\n").join(SimpletypesConfig.CONFIG_DEF_SCHEMA)))
-                                       .getTree();
         return SlimeConfigResponse.fromConfigPayload(responsePayload,
-                                                     targetDef,
                                                      3L,
                                                      true, /* internalRedeploy */
                                                      ConfigUtils.getMd5(responsePayload));
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java
index 40115170b69..a3b0f3ec44a 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java
@@ -342,11 +342,17 @@ public class SessionPreparerTest {
     private static class FailWithTransientExceptionProvisioner implements Provisioner {
 
         @Override
+        @Deprecated // TODO: Remove after April 2020
         public List<HostSpec> prepare(ApplicationId applicationId, ClusterSpec cluster, Capacity capacity, int groups, ProvisionLogger logger) {
             throw new LoadBalancerServiceException("Unable to create load balancer", new Exception("some internal exception"));
         }
 
         @Override
+        public List<HostSpec> prepare(ApplicationId applicationId, ClusterSpec cluster, Capacity capacity, ProvisionLogger logger) {
+            throw new LoadBalancerServiceException("Unable to create load balancer", new Exception("some internal exception"));
+        }
+
+        @Override
         public void activate(NestedTransaction transaction, ApplicationId application, Collection<HostSpec> hosts) { }
 
         @Override
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java
index f2c6aac2bda..9c7da7134e6 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java
@@ -5,6 +5,7 @@ import com.yahoo.component.Version;
 import com.yahoo.config.application.api.DeploymentSpec;
 import com.yahoo.config.model.deploy.DeployState;
 import com.yahoo.config.provision.AllocatedHosts;
+import com.yahoo.config.provision.DockerImage;
 import com.yahoo.config.provision.Flavor;
 import com.yahoo.config.provision.HostSpec;
 import com.yahoo.config.provision.NodeFlavors;
@@ -39,7 +40,8 @@ public class ZKApplicationPackageTest {
     private static final Optional<Flavor> TEST_FLAVOR = new MockNodeFlavors().getFlavor(TEST_FLAVOR_NAME);
     private static final AllocatedHosts ALLOCATED_HOSTS = AllocatedHosts.withHosts(
             Collections.singleton(new HostSpec("foo.yahoo.com", Collections.emptyList(), TEST_FLAVOR, Optional.empty(),
-                                               Optional.of(com.yahoo.component.Version.fromString("6.0.1")))));
+                                               Optional.of(Version.fromString("6.0.1")), Optional.empty(),
+                                               Optional.empty(), Optional.of(DockerImage.fromString("docker repo")))));
 
     private ConfigCurator configCurator;
 
@@ -59,7 +61,7 @@ public class ZKApplicationPackageTest {
         assertTrue(Pattern.compile(".*<alias>.*",Pattern.MULTILINE+Pattern.DOTALL).matcher(IOUtils.readAll(zkApp.getHosts())).matches());
         assertTrue(Pattern.compile(".*<slobroks>.*",Pattern.MULTILINE+Pattern.DOTALL).matcher(IOUtils.readAll(zkApp.getFile(Path.fromString("services.xml")).createReader())).matches());
         DeployState deployState = new DeployState.Builder().applicationPackage(zkApp).build();
-        assertEquals(deployState.getSearchDefinitions().size(), 5);
+        assertEquals(deployState.getSchemas().size(), 5);
         assertEquals(zkApp.searchDefinitionContents().size(), 5);
         assertEquals(IOUtils.readAll(zkApp.getRankingExpression("foo.expression")), "foo()+1\n");
         assertEquals(zkApp.getFiles(Path.fromString(""), "xml").size(), 3);
@@ -80,6 +82,8 @@ public class ZKApplicationPackageTest {
         assertThat(Utf8.toString(toJson(readInfo)), is(Utf8.toString(toJson(ALLOCATED_HOSTS))));
         assertThat(readInfo.getHosts().iterator().next().flavor(), is(TEST_FLAVOR));
         assertEquals("6.0.1", readInfo.getHosts().iterator().next().version().get().toString());
+        // TODO: Enable when dockerImageRepo is written to zk
+        //assertEquals("docker repo", readInfo.getHosts().iterator().next().dockerImageRepo().get());
         assertTrue(zkApp.getDeployment().isPresent());
         assertEquals("mydisc", DeploymentSpec.fromXml(zkApp.getDeployment().get()).requireInstance("default").globalServiceId().get());
     }
@@ -87,7 +91,7 @@ public class ZKApplicationPackageTest {
     private void feed(ConfigCurator zk, File dirToFeed) throws IOException {
         assertTrue(dirToFeed.isDirectory());
         zk.feedZooKeeper(dirToFeed, "/0" + ConfigCurator.USERAPP_ZK_SUBPATH, null, true);
-        String metaData = "{\"deploy\":{\"user\":\"foo\",\"from\":\"bar\",\"timestamp\":1},\"application\":{\"name\":\"foo\",\"checksum\":\"abc\",\"generation\":4,\"previousActiveGeneration\":3}}";
+        String metaData = "{\"deploy\":{\"user\":\"foo\",\"from\":\"bar\",\"timestamp\":1},\"application\":{\"id\":\"foo:foo:default\",\"checksum\":\"abc\",\"generation\":4,\"previousActiveGeneration\":3}}";
         zk.putData("/0", ConfigCurator.META_ZK_PATH, metaData);
         zk.putData("/0/" + ZKApplicationPackage.fileRegistryNode + "/3.0.0", "dummyfiles");
         zk.putData("/0/" + ZKApplicationPackage.allocatedHostsNode, toJson(ALLOCATED_HOSTS));
diff --git a/container-accesslogging/pom.xml b/container-accesslogging/pom.xml
index 53c17257992..59dabba9efe 100644
--- a/container-accesslogging/pom.xml
+++ b/container-accesslogging/pom.xml
@@ -65,6 +65,11 @@
       <artifactId>jackson-databind</artifactId>
       <scope>provided</scope>
     </dependency>
+    <dependency>
+      <groupId>org.assertj</groupId>
+      <artifactId>assertj-core</artifactId>
+      <scope>test</scope>
+    </dependency>
   </dependencies>
   <build>
     <plugins>
@@ -73,6 +78,10 @@
         <artifactId>bundle-plugin</artifactId>
         <extensions>true</extensions>
       </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-compiler-plugin</artifactId>
+      </plugin>
     </plugins>
     <outputDirectory>${buildOutputDirectory}</outputDirectory>
   </build>
diff --git a/container-accesslogging/src/main/java/com/yahoo/container/logging/AccessLogEntry.java b/container-accesslogging/src/main/java/com/yahoo/container/logging/AccessLogEntry.java
index b943c03f48f..d8085cc808b 100644
--- a/container-accesslogging/src/main/java/com/yahoo/container/logging/AccessLogEntry.java
+++ b/container-accesslogging/src/main/java/com/yahoo/container/logging/AccessLogEntry.java
@@ -2,6 +2,7 @@
 package com.yahoo.container.logging;
 
 import com.yahoo.collections.ListMap;
+import com.yahoo.yolean.trace.TraceNode;
 
 import javax.security.auth.x500.X500Principal;
 import java.net.InetAddress;
@@ -69,6 +70,7 @@ public class AccessLogEntry {
     private X500Principal sslPrincipal;
     private String rawPath;
     private String rawQuery;
+    private TraceNode traceNode;
 
     private ListMap<String,String> keyValues=null;
 
@@ -452,6 +454,18 @@ public class AccessLogEntry {
         }
     }
 
+    public void setTrace(TraceNode traceNode) {
+        synchronized (monitor) {
+            requireNull(this.traceNode);
+            this.traceNode = traceNode;
+        }
+    }
+    public TraceNode getTrace() {
+        synchronized (monitor) {
+            return traceNode;
+        }
+    }
+
     @Override
     public String toString() {
         synchronized (monitor) {
@@ -481,6 +495,7 @@ public class AccessLogEntry {
                     ", sslPrincipal=" + sslPrincipal +
                     ", rawPath='" + rawPath + '\'' +
                     ", rawQuery='" + rawQuery + '\'' +
+                    ", trace='" + traceNode + '\'' +
                     ", keyValues=" + keyValues +
                     '}';
         }
diff --git a/container-accesslogging/src/main/java/com/yahoo/container/logging/JSONFormatter.java b/container-accesslogging/src/main/java/com/yahoo/container/logging/JSONFormatter.java
index 556b97ced62..ae794e5b60a 100644
--- a/container-accesslogging/src/main/java/com/yahoo/container/logging/JSONFormatter.java
+++ b/container-accesslogging/src/main/java/com/yahoo/container/logging/JSONFormatter.java
@@ -5,6 +5,7 @@ import com.fasterxml.jackson.core.JsonEncoding;
 import com.fasterxml.jackson.core.JsonFactory;
 import com.fasterxml.jackson.core.JsonGenerator;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import com.yahoo.yolean.trace.TraceNode;
 
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
@@ -55,8 +56,7 @@ public class JSONFormatter {
             generator.writeStartObject();
             generator.writeStringField("ip", accessLogEntry.getIpV4Address());
             generator.writeNumberField("time", toTimestampInSeconds(accessLogEntry.getTimeStampMillis()));
-            generator.writeNumberField("duration",
-                                       durationAsSeconds(accessLogEntry.getDurationBetweenRequestResponseMillis()));
+            generator.writeNumberField("duration", durationAsSeconds(accessLogEntry.getDurationBetweenRequestResponseMillis()));
             generator.writeNumberField("responsesize", accessLogEntry.getReturnedContentSize());
             generator.writeNumberField("code", accessLogEntry.getStatusCode());
             generator.writeStringField("method", accessLogEntry.getHttpMethod());
@@ -95,6 +95,15 @@ public class JSONFormatter {
                 }
             }
 
+            TraceNode trace = accessLogEntry.getTrace();
+            if (trace != null) {
+                long timestamp = trace.timestamp();
+                if (timestamp == 0L) {
+                    timestamp = accessLogEntry.getTimeStampMillis();
+                }
+                trace.accept(new TraceRenderer(generator, timestamp));
+            }
+
             // Only add search sub block of this is a search request
             if (isSearchRequest(accessLogEntry)) {
                 generator.writeObjectFieldStart("search");
diff --git a/container-accesslogging/src/main/java/com/yahoo/container/logging/LogFileHandler.java b/container-accesslogging/src/main/java/com/yahoo/container/logging/LogFileHandler.java
index 82c89276319..a3d34ae6a2c 100644
--- a/container-accesslogging/src/main/java/com/yahoo/container/logging/LogFileHandler.java
+++ b/container-accesslogging/src/main/java/com/yahoo/container/logging/LogFileHandler.java
@@ -11,7 +11,6 @@ import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileOutputStream;
 import java.io.IOException;
-import java.io.OutputStream;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.concurrent.ArrayBlockingQueue;
@@ -41,7 +40,7 @@ public class LogFileHandler extends StreamHandler {
     private String filePattern = "./log.%T";  // default to current directory, ms time stamp
     private long nextRotationTime = 0;
     private FileOutputStream currentOutputStream = null;
-    private String fileName;
+    private volatile String fileName;
     private String symlinkName = null;
     private ArrayBlockingQueue<LogRecord> logQueue = new ArrayBlockingQueue<>(100000);
     private LogRecord rotateCmd = new LogRecord(Level.SEVERE, "rotateNow");
diff --git a/container-accesslogging/src/main/java/com/yahoo/container/logging/TraceRenderer.java b/container-accesslogging/src/main/java/com/yahoo/container/logging/TraceRenderer.java
new file mode 100644
index 00000000000..295786aa15d
--- /dev/null
+++ b/container-accesslogging/src/main/java/com/yahoo/container/logging/TraceRenderer.java
@@ -0,0 +1,185 @@
+package com.yahoo.container.logging;
+
+import com.yahoo.data.access.Inspectable;
+import com.yahoo.data.access.Inspector;
+import com.yahoo.data.access.simple.JsonRender;
+import com.yahoo.yolean.trace.TraceNode;
+import com.yahoo.yolean.trace.TraceVisitor;
+import com.fasterxml.jackson.core.JsonGenerator;
+
+import java.io.IOException;
+
+public class TraceRenderer extends TraceVisitor {
+    private static final String TRACE_CHILDREN = "children";
+    private static final String TRACE_MESSAGE = "message";
+    private static final String TRACE_TIMESTAMP = "timestamp";
+    private static final String TRACE = "trace";
+
+    private final long basetime;
+    private final JsonGenerator generator;
+    private final FieldConsumer fieldConsumer;
+    private boolean hasFieldName = false;
+    int emittedChildNesting = 0;
+    int currentChildNesting = 0;
+    private boolean insideOpenObject = false;
+
+    public interface FieldConsumer {
+        void accept(Object object) throws IOException;
+    }
+
+    private static class Consumer implements FieldConsumer {
+        private final JsonGenerator generator;
+
+        Consumer(JsonGenerator generator) {
+            this.generator = generator;
+        }
+
+        @Override
+        public void accept(Object object) throws IOException {
+            if (object instanceof Inspectable) {
+                renderInspectorDirect(((Inspectable) object).inspect());
+            } else {
+                generator.writeObject(object);
+            }
+        }
+        private void renderInspectorDirect(Inspector data) throws IOException {
+            StringBuilder intermediate = new StringBuilder();
+            JsonRender.render(data, intermediate, true);
+            generator.writeRawValue(intermediate.toString());
+        }
+    }
+
+    TraceRenderer(JsonGenerator generator, long basetime) {
+        this(generator, new Consumer(generator), basetime);
+    }
+    public TraceRenderer(JsonGenerator generator, FieldConsumer consumer, long basetime) {
+        this.generator = generator;
+        this.fieldConsumer = consumer;
+        this.basetime = basetime;
+    }
+
+    @Override
+    public void entering(TraceNode node) {
+        ++currentChildNesting;
+    }
+
+    @Override
+    public void leaving(TraceNode node) {
+        conditionalEndObject();
+        if (currentChildNesting == emittedChildNesting) {
+            try {
+                generator.writeEndArray();
+                generator.writeEndObject();
+            } catch (IOException e) {
+                throw new TraceRenderWrapper(e);
+            }
+            --emittedChildNesting;
+        }
+        --currentChildNesting;
+    }
+
+    @Override
+    public void visit(TraceNode node) {
+        try {
+            doVisit(node.timestamp(), node.payload(), node.children().iterator().hasNext());
+        } catch (IOException e) {
+            throw new TraceRenderWrapper(e);
+        }
+    }
+
+    private void doVisit(long timestamp, Object payload, boolean hasChildren) throws IOException {
+        boolean dirty = false;
+        if (timestamp != 0L) {
+            header();
+            generator.writeStartObject();
+            generator.writeNumberField(TRACE_TIMESTAMP, timestamp - basetime);
+            dirty = true;
+        }
+        if (payload != null) {
+            if (!dirty) {
+                header();
+                generator.writeStartObject();
+            }
+            generator.writeFieldName(TRACE_MESSAGE);
+            fieldConsumer.accept(payload);
+            dirty = true;
+        }
+        if (dirty) {
+            if (!hasChildren) {
+                generator.writeEndObject();
+            } else {
+                setInsideOpenObject(true);
+            }
+        }
+    }
+    private void header() {
+        fieldName();
+        for (int i = 0; i < (currentChildNesting - emittedChildNesting); ++i) {
+            startChildArray();
+        }
+        emittedChildNesting = currentChildNesting;
+    }
+
+    private void startChildArray() {
+        try {
+            conditionalStartObject();
+            generator.writeArrayFieldStart(TRACE_CHILDREN);
+        } catch (IOException e) {
+            throw new TraceRenderWrapper(e);
+        }
+    }
+
+    private void conditionalStartObject() throws IOException {
+        if (!isInsideOpenObject()) {
+            generator.writeStartObject();
+        } else {
+            setInsideOpenObject(false);
+        }
+    }
+
+    private void conditionalEndObject() {
+        if (isInsideOpenObject()) {
+            // This triggers if we were inside a data node with payload and
+            // subnodes, but none of the subnodes contained data
+            try {
+                generator.writeEndObject();
+                setInsideOpenObject(false);
+            } catch (IOException e) {
+                throw new TraceRenderWrapper(e);
+            }
+        }
+    }
+
+    private void fieldName() {
+        if (hasFieldName) {
+            return;
+        }
+
+        try {
+            generator.writeFieldName(TRACE);
+        } catch (IOException e) {
+            throw new TraceRenderWrapper(e);
+        }
+        hasFieldName = true;
+    }
+
+    boolean isInsideOpenObject() {
+        return insideOpenObject;
+    }
+
+    void setInsideOpenObject(boolean insideOpenObject) {
+        this.insideOpenObject = insideOpenObject;
+    }
+    public static final class TraceRenderWrapper extends RuntimeException {
+
+        /**
+         * Should never be serialized, but this is still needed.
+         */
+        private static final long serialVersionUID = 2L;
+
+        TraceRenderWrapper(IOException wrapped) {
+            super(wrapped);
+        }
+
+    }
+}
diff --git a/container-accesslogging/src/test/java/com/yahoo/container/logging/JSONLogTestCase.java b/container-accesslogging/src/test/java/com/yahoo/container/logging/JSONLogTestCase.java
index 8bbb8500cfd..6c7878f99ed 100644
--- a/container-accesslogging/src/test/java/com/yahoo/container/logging/JSONLogTestCase.java
+++ b/container-accesslogging/src/test/java/com/yahoo/container/logging/JSONLogTestCase.java
@@ -1,8 +1,10 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.container.logging;
 
+import com.yahoo.yolean.trace.TraceNode;
 import org.junit.Test;
 
+import java.io.IOException;
 import java.net.URI;
 
 import static org.junit.Assert.assertEquals;
@@ -70,6 +72,36 @@ public class JSONLogTestCase {
     }
 
     @Test
+    public void test_json_of_trace() throws IOException {
+        TraceNode root = new TraceNode("root", 7);
+        AccessLogEntry entry = newAccessLogEntry("test");
+        entry.setTrace(root);
+
+        String expectedOutput =
+                "{\"ip\":\"152.200.54.243\"," +
+                "\"time\":920880005.023," +
+                "\"duration\":0.122," +
+                "\"responsesize\":9875," +
+                "\"code\":200," +
+                "\"method\":\"GET\"," +
+                "\"uri\":\"?query=test\"," +
+                "\"version\":\"HTTP/1.1\"," +
+                "\"agent\":\"Mozilla/4.05 [en] (Win95; I)\"," +
+                "\"host\":\"localhost\"," +
+                "\"scheme\":null," +
+                "\"localport\":0," +
+                "\"trace\":{\"timestamp\":0,\"message\":\"root\"}," +
+                "\"search\":{" +
+                "\"totalhits\":1234," +
+                "\"hits\":0," +
+                "\"coverage\":{\"coverage\":100,\"documents\":100}" +
+                "}" +
+                "}";
+
+        assertEquals(expectedOutput, new JSONFormatter(entry).format());
+    }
+
+    @Test
     public void test_with_keyvalues() {
         AccessLogEntry entry = newAccessLogEntry("test");
         entry.addKeyValue("singlevalue", "value1");
diff --git a/container-accesslogging/src/test/java/com/yahoo/container/logging/LogFileHandlerTestCase.java b/container-accesslogging/src/test/java/com/yahoo/container/logging/LogFileHandlerTestCase.java
index de4823e66c6..bc7257b1ca9 100644
--- a/container-accesslogging/src/test/java/com/yahoo/container/logging/LogFileHandlerTestCase.java
+++ b/container-accesslogging/src/test/java/com/yahoo/container/logging/LogFileHandlerTestCase.java
@@ -2,12 +2,16 @@
 package com.yahoo.container.logging;
 
 import com.yahoo.io.IOUtils;
+import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
 
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.InputStreamReader;
+import java.nio.file.Files;
+import java.nio.file.Paths;
 import java.text.DateFormat;
 import java.text.SimpleDateFormat;
 import java.util.Date;
@@ -17,24 +21,23 @@ import java.util.logging.LogRecord;
 import java.util.logging.SimpleFormatter;
 import java.util.zip.GZIPInputStream;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
+import static org.assertj.core.api.Assertions.assertThat;
 
 /**
- * @author <a href="mailto:travisb@yahoo-inc.com">Bob Travis</a>
+ * @author Bob Travis
+ * @author bjorncs
  */
-// TODO: Make these tests wait until the right things happen rather than waiting for a predetermined time
-// These tests take too long, and are not cleaning up properly. See how this should be done in YApacheLogTestCase
 public class LogFileHandlerTestCase {
 
-    /**
-     * The scenario
-     */
+    @Rule
+    public TemporaryFolder temporaryFolder = new TemporaryFolder();
+
     @Test
-    public void testIt() {
+    public void testIt() throws IOException {
+        File root = temporaryFolder.newFolder("logfilehandlertest");
+
         LogFileHandler h = new LogFileHandler();
-        h.setFilePattern("./logfilehandlertest.%Y%m%d%H%M%S");
+        h.setFilePattern(root.getAbsolutePath() + "/logfilehandlertest.%Y%m%d%H%M%S");
         h.setFormatter(new Formatter() {
                 public String format(LogRecord r) {
                     DateFormat df = new SimpleDateFormat("yyyy.MM.dd:HH:mm:ss.SSS");
@@ -46,42 +49,27 @@ public class LogFileHandlerTestCase {
         long millisPerDay = 60*60*24*1000;
         long tomorrowDays = (now / millisPerDay) +1;
         long tomorrowMillis = tomorrowDays * millisPerDay;
-        assertEquals (tomorrowMillis, h.getNextRotationTime(now));
+        assertThat(tomorrowMillis).isEqualTo(h.getNextRotationTime(now));
         long[] rTimes = {1000, 2000, 10000};
         h.setRotationTimes(rTimes);
-        assertEquals (tomorrowMillis+1000, h.getNextRotationTime(tomorrowMillis));
-        assertEquals (tomorrowMillis+10000, h.getNextRotationTime(tomorrowMillis+3000));
-        boolean okToWrite = false; // don't want regular unit tests to create tiles....
-        if (okToWrite) {
-            LogRecord lr = new LogRecord(Level.INFO, "test");
-            h.publish(lr);
-            h.publish(new LogRecord(Level.INFO, "another test"));
-            h.rotateNow();
-            h.publish(lr);
-            h.flush();
-        }
-    }
-
-    private boolean delete(String fileOrDir) {
-      File file = new File(fileOrDir);
-      return file.delete();
-    }
-
-    private void deleteOnExit(String fileOrDir) {
-        new File(fileOrDir).deleteOnExit();
-    }
-
-    private static void deleteRecursive(String directory) {
-       IOUtils.recursiveDeleteDir(new File(directory));
+        assertThat(tomorrowMillis+1000).isEqualTo(h.getNextRotationTime(tomorrowMillis));
+        assertThat(tomorrowMillis+10000).isEqualTo(h.getNextRotationTime(tomorrowMillis+3000));
+        LogRecord lr = new LogRecord(Level.INFO, "test");
+        h.publish(lr);
+        h.publish(new LogRecord(Level.INFO, "another test"));
+        h.rotateNow();
+        h.publish(lr);
+        h.flush();
+        h.shutdown();
     }
 
     @Test
-    public void testSimpleLogging() {
-      String logFilePattern = "./testLogFileG1.txt";
+    public void testSimpleLogging() throws IOException {
+        File logFile = temporaryFolder.newFile("testLogFileG1.txt");
 
       //create logfilehandler
       LogFileHandler h = new LogFileHandler();
-      h.setFilePattern(logFilePattern);
+      h.setFilePattern(logFile.getAbsolutePath());
       h.setFormatter(new SimpleFormatter());
       h.setRotationTimes("0 5 ...");
 
@@ -89,17 +77,16 @@ public class LogFileHandlerTestCase {
       LogRecord lr = new LogRecord(Level.INFO, "testDeleteFileFirst1");
       h.publish(lr);
       h.flush();
-
-      new File(logFilePattern).deleteOnExit();
+      h.shutdown();
     }
 
     @Test
-    public void testDeleteFileDuringLogging() {
-      String logFilePattern = "./testLogFileG2.txt";
+    public void testDeleteFileDuringLogging() throws IOException {
+      File logFile = temporaryFolder.newFile("testLogFileG2.txt");
 
       //create logfilehandler
       LogFileHandler h = new LogFileHandler();
-      h.setFilePattern(logFilePattern);
+      h.setFilePattern(logFile.getAbsolutePath());
       h.setFormatter(new SimpleFormatter());
       h.setRotationTimes("0 5 ...");
 
@@ -109,76 +96,63 @@ public class LogFileHandlerTestCase {
       h.flush();
 
       //delete log file
-      delete(logFilePattern);
+        logFile.delete();
 
       //write log again
       lr = new LogRecord(Level.INFO, "testDeleteFileDuringLogging2");
       h.publish(lr);
       h.flush();
-
-      new File(logFilePattern).deleteOnExit();
+      h.shutdown();
     }
 
-    @Test
-    public void testSymlink() {
-        LogFileHandler h = new LogFileHandler();
-        h.setFilePattern("./testlogforsymlinkchecking/logfilehandlertest.%Y%m%d%H%M%S%s");
-        h.setFormatter(new Formatter() {
+    @Test(timeout = /*5 minutes*/300_000)
+    public void testSymlink() throws IOException, InterruptedException {
+        File root = temporaryFolder.newFolder("testlogforsymlinkchecking");
+        LogFileHandler handler = new LogFileHandler();
+        handler.setFilePattern(root.getAbsolutePath() + "/logfilehandlertest.%Y%m%d%H%M%S%s");
+        handler.setFormatter(new Formatter() {
             public String format(LogRecord r) {
                 DateFormat df = new SimpleDateFormat("yyyy.MM.dd:HH:mm:ss.SSS");
                 String timeStamp = df.format(new Date(r.getMillis()));
                 return ("["+timeStamp+"]" + " " + formatMessage(r) + "\n");
             }
         } );
-        h.setSymlinkName("symlink");
-        LogRecord lr = new LogRecord(Level.INFO, "test");
-        h.publish(lr);
-        String f1 = h.getFileName();
-        String f2 = null;
-        try {
-            while (f1 == null) {
-                Thread.sleep(1);
-                f1 = h.getFileName();
-            }
-            h.rotateNow();
+        handler.setSymlinkName("symlink");
+
+        handler.publish(new LogRecord(Level.INFO, "test"));
+        String firstFile;
+        do {
+             Thread.sleep(1);
+             firstFile = handler.getFileName();
+        } while (firstFile == null);
+        handler.rotateNow();
+        String secondFileName;
+        do {
             Thread.sleep(1);
-            f2 = h.getFileName();
-            while (f1.equals(f2)) {
-                Thread.sleep(1);
-                f2 = h.getFileName();
-            }
-            lr = new LogRecord(Level.INFO, "string which is way longer than the word test");
-            h.publish(lr);
-            Thread.sleep(1000);
-            File f = new File(f1);
-            long first = f.length();
-            f = new File(f2);
-            long second = f.length();
-            final long secondLength = 72;
-            for (int n = 0; n < 20 && second != secondLength; ++n) {
-                Thread.sleep(1000);
-                second = f.length();
-            }
-            f = new File("./testlogforsymlinkchecking", "symlink");
-            long link = f.length();
-            assertEquals(secondLength, link);
-            assertEquals(31, first);
-            assertEquals(secondLength, second);
-        } catch (InterruptedException e) {
-            // just let the test pass
-        }
-        deleteOnExit("./testlogforsymlinkchecking");
-        deleteOnExit("./testlogforsymlinkchecking/symlink");
-        deleteOnExit(f1);
-        if (f2 != null)
-            deleteOnExit(f2);
+            secondFileName = handler.getFileName();
+        } while (firstFile.equals(secondFileName));
+
+        handler.publish(new LogRecord(Level.INFO, "string which is way longer than the word test"));
+        handler.waitDrained();
+        assertThat(Files.size(Paths.get(firstFile))).isEqualTo(31);
+        final long expectedSecondFileLength = 72;
+        long secondFileLength;
+        do {
+            Thread.sleep(1);
+            secondFileLength = Files.size(Paths.get(secondFileName));
+        } while (secondFileLength != expectedSecondFileLength);
+
+        long symlinkFileLength = Files.size(root.toPath().resolve("symlink"));
+        assertThat(symlinkFileLength).isEqualTo(expectedSecondFileLength);
+        handler.shutdown();
     }
 
     @Test
     public void testcompression() throws InterruptedException, IOException {
-        IOUtils.recursiveDeleteDir(new File("./testcompression"));
+        File root = temporaryFolder.newFolder("testcompression");
+
         LogFileHandler h = new LogFileHandler(true);
-        h.setFilePattern("./testcompression/logfilehandlertest.%Y%m%d%H%M%S%s");
+        h.setFilePattern(root.getAbsolutePath() + "/logfilehandlertest.%Y%m%d%H%M%S%s");
         h.setFormatter(new Formatter() {
             public String format(LogRecord r) {
                 DateFormat df = new SimpleDateFormat("yyyy.MM.dd:HH:mm:ss.SSS");
@@ -186,28 +160,28 @@ public class LogFileHandlerTestCase {
                 return ("["+timeStamp+"]" + " " + formatMessage(r) + "\n");
             }
         } );
-        for (int i=0; i < 10000; i++) {
+        int logEntries = 10000;
+        for (int i = 0; i < logEntries; i++) {
             LogRecord lr = new LogRecord(Level.INFO, "test");
             h.publish(lr);
         }
         h.waitDrained();
         String f1 = h.getFileName();
-        assertTrue(f1.startsWith("./testcompression/logfilehandlertest."));
+        assertThat(f1).startsWith(root.getAbsolutePath() + "/logfilehandlertest.");
         File uncompressed = new File(f1);
         File compressed = new File(f1 + ".gz");
-        assertTrue(uncompressed.exists());
-        assertFalse(compressed.exists());
+        assertThat(uncompressed).exists();
+        assertThat(compressed).doesNotExist();
         String content = IOUtils.readFile(uncompressed);
-        assertEquals(310000, content.length());
+        assertThat(content).hasLineCount(logEntries);
         h.rotateNow();
         while (uncompressed.exists()) {
-            Thread.sleep(10);
+            Thread.sleep(1);
         }
-        assertTrue(compressed.exists());
+        assertThat(compressed).exists();
         String unzipped = IOUtils.readAll(new InputStreamReader(new GZIPInputStream(new FileInputStream(compressed))));
-        assertEquals(content, unzipped);
-
-        IOUtils.recursiveDeleteDir(new File("./testcompression"));
+        assertThat(content).isEqualTo(unzipped);
+        h.shutdown();
     }
 
 }
diff --git a/container-core/CMakeLists.txt b/container-core/CMakeLists.txt
index 341155457a8..6132c253c13 100644
--- a/container-core/CMakeLists.txt
+++ b/container-core/CMakeLists.txt
@@ -8,6 +8,7 @@ install_config_definition(src/main/resources/configdefinitions/identity.def cont
 install_config_definition(src/main/resources/configdefinitions/log-handler.def container.core.log-handler.def)
 install_config_definition(src/main/resources/configdefinitions/metrics-packets-handler.def container.jdisc.state.metrics-packets-handler.def)
 install_config_definition(src/main/resources/configdefinitions/metrics-presentation.def metrics.metrics-presentation.def)
+install_config_definition(src/main/resources/configdefinitions/metrics-proxy-api.def container.handler.metrics.metrics-proxy-api.def)
 install_config_definition(src/main/resources/configdefinitions/mockservice.def container.handler.test.mockservice.def)
 install_config_definition(src/main/resources/configdefinitions/qr-searchers.def container.qr-searchers.def)
 install_config_definition(src/main/resources/configdefinitions/qr.def container.qr.def)
diff --git a/container-core/abi-spec.json b/container-core/abi-spec.json
index 91c0e4cc6bf..6d683c53984 100644
--- a/container-core/abi-spec.json
+++ b/container-core/abi-spec.json
@@ -11,6 +11,23 @@
     ],
     "fields": []
   },
+  "com.yahoo.container.handler.ClustersStatus$Require": {
+    "superClass": "java.lang.Enum",
+    "interfaces": [],
+    "attributes": [
+      "public",
+      "final",
+      "enum"
+    ],
+    "methods": [
+      "public static com.yahoo.container.handler.ClustersStatus$Require[] values()",
+      "public static com.yahoo.container.handler.ClustersStatus$Require valueOf(java.lang.String)"
+    ],
+    "fields": [
+      "public static final enum com.yahoo.container.handler.ClustersStatus$Require ONE",
+      "public static final enum com.yahoo.container.handler.ClustersStatus$Require ALL"
+    ]
+  },
   "com.yahoo.container.handler.ClustersStatus": {
     "superClass": "com.yahoo.component.AbstractComponent",
     "interfaces": [],
@@ -23,7 +40,8 @@
       "public void setReceiveTrafficByDefault(boolean)",
       "public void setUp(java.lang.Object)",
       "public void setDown(java.lang.Object)",
-      "public boolean containerShouldReceiveTraffic()"
+      "public boolean containerShouldReceiveTraffic()",
+      "public boolean containerShouldReceiveTraffic(com.yahoo.container.handler.ClustersStatus$Require)"
     ],
     "fields": []
   },
@@ -154,6 +172,7 @@
       "public void <init>(com.yahoo.container.handler.ThreadpoolConfig)",
       "public com.yahoo.container.handler.ThreadpoolConfig$Builder maxthreads(int)",
       "public com.yahoo.container.handler.ThreadpoolConfig$Builder maxThreadExecutionTimeSeconds(int)",
+      "public com.yahoo.container.handler.ThreadpoolConfig$Builder softStartSeconds(double)",
       "public final boolean dispatchGetConfig(com.yahoo.config.ConfigInstance$Producer)",
       "public final java.lang.String getDefMd5()",
       "public final java.lang.String getDefName()",
@@ -191,7 +210,8 @@
       "public static java.lang.String getDefVersion()",
       "public void <init>(com.yahoo.container.handler.ThreadpoolConfig$Builder)",
       "public int maxthreads()",
-      "public int maxThreadExecutionTimeSeconds()"
+      "public int maxThreadExecutionTimeSeconds()",
+      "public double softStartSeconds()"
     ],
     "fields": [
       "public static final java.lang.String CONFIG_DEF_MD5",
@@ -230,6 +250,7 @@
       "public void <init>(com.yahoo.container.QrSearchersConfig)",
       "public void <init>(com.yahoo.container.handler.ClustersStatus)",
       "public void <init>(com.yahoo.container.QrSearchersConfig, com.yahoo.container.handler.ClustersStatus)",
+      "public void <init>(com.yahoo.container.QrSearchersConfig, com.yahoo.container.core.VipStatusConfig, com.yahoo.container.handler.ClustersStatus, com.yahoo.container.jdisc.state.StateMonitor)",
       "public void <init>(com.yahoo.container.QrSearchersConfig, com.yahoo.container.handler.ClustersStatus, com.yahoo.container.jdisc.state.StateMonitor)",
       "public void <init>(com.yahoo.container.QrSearchersConfig, com.yahoo.container.core.VipStatusConfig, com.yahoo.container.handler.ClustersStatus)",
       "public void setInRotation(java.lang.Boolean)",
@@ -853,5 +874,55 @@
       "public bridge synthetic java.lang.Object clone()"
     ],
     "fields": []
+  },
+  "ai.vespa.cloud.Environment": {
+    "superClass": "java.lang.Enum",
+    "interfaces": [],
+    "attributes": [
+      "public",
+      "final",
+      "enum"
+    ],
+    "methods": [
+      "public static ai.vespa.cloud.Environment[] values()",
+      "public static ai.vespa.cloud.Environment valueOf(java.lang.String)"
+    ],
+    "fields": [
+      "public static final enum ai.vespa.cloud.Environment dev",
+      "public static final enum ai.vespa.cloud.Environment perf",
+      "public static final enum ai.vespa.cloud.Environment test",
+      "public static final enum ai.vespa.cloud.Environment staging",
+      "public static final enum ai.vespa.cloud.Environment prod"
+    ]
+  },
+  "ai.vespa.cloud.SystemInfo": {
+    "superClass": "java.lang.Object",
+    "interfaces": [],
+    "attributes": [
+      "public"
+    ],
+    "methods": [
+      "public void <init>(com.yahoo.cloud.config.ConfigserverConfig)",
+      "public void <init>(ai.vespa.cloud.Zone)",
+      "public ai.vespa.cloud.Zone zone()"
+    ],
+    "fields": []
+  },
+  "ai.vespa.cloud.Zone": {
+    "superClass": "java.lang.Object",
+    "interfaces": [],
+    "attributes": [
+      "public"
+    ],
+    "methods": [
+      "public void <init>(ai.vespa.cloud.Environment, java.lang.String)",
+      "public ai.vespa.cloud.Environment environment()",
+      "public java.lang.String region()",
+      "public java.lang.String toString()",
+      "public int hashCode()",
+      "public boolean equals(java.lang.Object)",
+      "public static ai.vespa.cloud.Zone from(java.lang.String)"
+    ],
+    "fields": []
   }
 }
\ No newline at end of file
diff --git a/container-core/pom.xml b/container-core/pom.xml
index f3861c92129..64e5ebb00d3 100644
--- a/container-core/pom.xml
+++ b/container-core/pom.xml
@@ -16,6 +16,16 @@
   <packaging>container-plugin</packaging>
   <dependencies>
     <dependency>
+      <groupId>com.yahoo.vespa</groupId>
+      <artifactId>http-utils</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.httpcomponents</groupId>
+      <artifactId>httpclient</artifactId>
+    </dependency>
+
+    <dependency>
       <groupId>com.google.guava</groupId>
       <artifactId>guava</artifactId>
       <scope>provided</scope>
@@ -227,6 +237,16 @@
       <version>${project.version}</version>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>com.github.tomakehurst</groupId>
+      <artifactId>wiremock-standalone</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.assertj</groupId>
+      <artifactId>assertj-core</artifactId>
+      <scope>test</scope>
+    </dependency>
   </dependencies>
   <build>
     <plugins>
diff --git a/container-core/src/main/java/ai/vespa/cloud/Environment.java b/container-core/src/main/java/ai/vespa/cloud/Environment.java
new file mode 100644
index 00000000000..8f1d9fc962a
--- /dev/null
+++ b/container-core/src/main/java/ai/vespa/cloud/Environment.java
@@ -0,0 +1,13 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package ai.vespa.cloud;
+
+/**
+ * The environments of a Vespa cloud instance
+ *
+ * @author bratseth
+ */
+public enum Environment {
+
+    dev, perf, test, staging, prod
+
+}
diff --git a/container-core/src/main/java/ai/vespa/cloud/SystemInfo.java b/container-core/src/main/java/ai/vespa/cloud/SystemInfo.java
new file mode 100644
index 00000000000..0524ae072cd
--- /dev/null
+++ b/container-core/src/main/java/ai/vespa/cloud/SystemInfo.java
@@ -0,0 +1,31 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package ai.vespa.cloud;
+
+import com.google.inject.Inject;
+import com.yahoo.cloud.config.ConfigserverConfig;
+
+/**
+ * Provides information about the system in which this container is running.
+ * This is available and can be injected when running in a cloud environment.
+ *
+ * @author bratseth
+ */
+public class SystemInfo {
+
+    private final Zone zone;
+
+    /** Do not use */
+    @Inject
+    public SystemInfo(ConfigserverConfig config) {
+        this.zone = new Zone(Environment.valueOf(config.environment()), config.region());
+    }
+
+    /** Create an instance for testing */
+    public SystemInfo(Zone zone) {
+        this.zone = zone;
+    }
+
+    /** Returns the zone this is running in */
+    public Zone zone() { return zone; }
+
+}
diff --git a/container-core/src/main/java/ai/vespa/cloud/Zone.java b/container-core/src/main/java/ai/vespa/cloud/Zone.java
new file mode 100644
index 00000000000..48293aa7908
--- /dev/null
+++ b/container-core/src/main/java/ai/vespa/cloud/Zone.java
@@ -0,0 +1,61 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package ai.vespa.cloud;
+
+import java.util.Objects;
+
+/**
+ * The zone in which a cloud deployment may be running.
+ * A zone is a combination of an environment and a region.
+ *
+ * @author bratseth
+ */
+public class Zone {
+
+    private final Environment environment;
+
+    private final String region;
+
+    public Zone(Environment environment, String region) {
+        this.environment = environment;
+        this.region = region;
+    }
+
+    public Environment environment() { return environment; }
+    public String region() { return region; }
+
+    /** Returns the string environment.region */
+    @Override
+    public String toString() { return environment + "." + region; }
+
+    @Override
+    public int hashCode() { return Objects.hash(environment, region); }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if ( ! (o instanceof Zone)) return false;
+        Zone other = (Zone)o;
+        return this.environment.equals(other.environment) && this.region.equals(other.region);
+    }
+
+    /**
+     * Creates a zone from a string on the form environment.region
+     *
+     * @throws IllegalArgumentException if the given string is not a valid zone
+     */
+    public static Zone from(String zoneString) {
+        String[] parts = zoneString.split("\\.");
+        if (parts.length != 2)
+            throw new IllegalArgumentException("A zone string must be on the form [environment].[region], but was '" + zoneString + "'");
+
+        Environment environment;
+        try {
+            environment = Environment.valueOf(parts[0]);
+        }
+        catch (IllegalArgumentException e) {
+            throw new IllegalArgumentException("Invalid zone '" + zoneString + "': No environment named '" + parts[0] + "'");
+        }
+        return new Zone(environment, parts[1]);
+    }
+
+}
diff --git a/container-core/src/main/java/ai/vespa/cloud/package-info.java b/container-core/src/main/java/ai/vespa/cloud/package-info.java
new file mode 100644
index 00000000000..259a2bda258
--- /dev/null
+++ b/container-core/src/main/java/ai/vespa/cloud/package-info.java
@@ -0,0 +1,10 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+/**
+ * Public API to the Vespa cloud, available when this container runs in a cloud.
+ */
+@ExportPackage
+@PublicApi
+package ai.vespa.cloud;
+
+import com.yahoo.api.annotations.PublicApi;
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/container-core/src/main/java/com/yahoo/container/Server.java b/container-core/src/main/java/com/yahoo/container/Server.java
deleted file mode 100644
index a4dec6de5a2..00000000000
--- a/container-core/src/main/java/com/yahoo/container/Server.java
+++ /dev/null
@@ -1,45 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.container;
-
-import com.yahoo.config.subscription.ConfigSubscriber;
-import com.yahoo.container.QrConfig.Rpc;
-
-/**
- * The http server singleton managing listeners for various ports,
- * and the threads used to respond to requests on the ports
- *
- * @author bratseth
- * @deprecated
- */
-@SuppressWarnings("deprecation")
-@Deprecated // TODO: Remove this when the last usage og getServerDiscriminator is removed
-public class Server {
-
-    //TODO: Make this final again.
-    private static final Server instance = new Server();
-
-    /** A short string which is different for all the qrserver instances on a given node. */
-    private String localServerDiscriminator = "qrserver.0";
-
-    private Server() { }
-
-    public static Server get() {
-        return instance;
-    }
-
-    public void initialize(QrConfig config) {
-        localServerDiscriminator = config.discriminator();
-    }
-
-    /**
-     * A string unique for this QRS on this server.
-     *
-     * @return a server specific string
-     * @deprecated do not use
-     */
-    @Deprecated
-    public String getServerDiscriminator() {
-        return localServerDiscriminator;
-    }
-
-}
diff --git a/container-core/src/main/java/com/yahoo/container/core/config/BundleInstaller.java b/container-core/src/main/java/com/yahoo/container/core/config/BundleInstaller.java
new file mode 100644
index 00000000000..fc919571b6c
--- /dev/null
+++ b/container-core/src/main/java/com/yahoo/container/core/config/BundleInstaller.java
@@ -0,0 +1,21 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.container.core.config;
+
+import com.yahoo.config.FileReference;
+import com.yahoo.osgi.Osgi;
+import org.osgi.framework.Bundle;
+
+import java.util.List;
+
+/**
+ * @author gjoranv
+ */
+public interface BundleInstaller {
+
+    /**
+     * Installs the bundle with the given file reference, plus all bundles in its X-JDisc-Preinstall-Bundle directive.
+     * Returns all bundles installed to the given OSGi framework as a result of this call.
+     */
+    List<Bundle> installBundles(FileReference reference, Osgi osgi) throws InterruptedException;
+
+}
diff --git a/container-core/src/main/java/com/yahoo/container/core/config/BundleLoader.java b/container-core/src/main/java/com/yahoo/container/core/config/BundleManager.java
index 4c3d76436dd..5e9b42a5fda 100644
--- a/container-core/src/main/java/com/yahoo/container/core/config/BundleLoader.java
+++ b/container-core/src/main/java/com/yahoo/container/core/config/BundleManager.java
@@ -1,4 +1,4 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.container.core.config;
 
 import com.yahoo.collections.PredicateSplit;
@@ -9,13 +9,12 @@ import com.yahoo.osgi.Osgi;
 import org.osgi.framework.Bundle;
 import org.osgi.framework.wiring.BundleRevision;
 
-import java.io.File;
+import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
-import java.util.concurrent.TimeUnit;
 import java.util.logging.Logger;
 import java.util.stream.Collectors;
 
@@ -23,12 +22,12 @@ import static com.yahoo.collections.PredicateSplit.partition;
 import static com.yahoo.container.core.BundleLoaderProperties.DISK_BUNDLE_PREFIX;
 
 /**
- * Manages the set of installed 3rd-party component bundles.
+ * Manages the set of installed and active/inactive bundles.
  *
- * @author Tony Vaagenes
  * @author gjoranv
+ * @author Tony Vaagenes
  */
-public class BundleLoader {
+public class BundleManager {
 
     /* Map of file refs of active bundles (not scheduled for uninstall) to a list of all bundles that were installed
      * (pre-install directive) by the bundle pointed to by the file ref (including itself).
@@ -40,20 +39,96 @@ public class BundleLoader {
      */
     private final Map<FileReference, List<Bundle>> reference2Bundles = new LinkedHashMap<>();
 
-    private final Logger log = Logger.getLogger(BundleLoader.class.getName());
+    private final Logger log = Logger.getLogger(BundleManager.class.getName());
     private final Osgi osgi;
 
-    public BundleLoader(Osgi osgi) {
+    // A custom bundle installer for non-disk bundles, to be used for testing
+    private BundleInstaller customBundleInstaller = null;
+
+    public BundleManager(Osgi osgi) {
         this.osgi = osgi;
     }
 
-    private void install(List<FileReference> references) {
+    /**
+     * Installs the given set of bundles and returns the set of bundles that is no longer used
+     * by the application, and should therefore be scheduled for uninstall.
+     */
+    public synchronized Set<Bundle> use(List<FileReference> newFileReferences) {
+        // Must be done before allowing duplicates because allowed duplicates affect osgi.getCurrentBundles
+        Set<Bundle> bundlesToUninstall = getObsoleteBundles(newFileReferences);
+
+        Set<FileReference> obsoleteReferences = getObsoleteFileReferences(newFileReferences);
+        allowDuplicateBundles(obsoleteReferences);
+        removeInactiveFileReferences(obsoleteReferences);
+
+        installBundles(newFileReferences);
+        startBundles();
+
+        bundlesToUninstall.removeAll(allActiveBundles());
+        log.info("Bundles to schedule for uninstall: " + bundlesToUninstall);
+
+        log.info(installedBundlesMessage());
+        return bundlesToUninstall;
+    }
+
+    /**
+     * Returns the bundles that are not assumed to be retained by the new application generation.
+     * Note that at this point we don't yet know the full set of new bundles, because of the potential
+     * pre-install directives in the new bundles. However, only "disk bundles" (file:) can be listed
+     * in the pre-install directive, so we know about all the obsolete application bundles.
+     */
+    private Set<Bundle> getObsoleteBundles(List<FileReference> newReferences) {
+        Set<Bundle> bundlesToRemove = new HashSet<>(osgi.getCurrentBundles());
+
+        for (FileReference fileReferenceToKeep : newReferences) {
+            if (reference2Bundles.containsKey(fileReferenceToKeep)) {
+                bundlesToRemove.removeAll(reference2Bundles.get(fileReferenceToKeep));
+            }
+        }
+        bundlesToRemove.removeAll(osgi.getInitialBundles());
+        return bundlesToRemove;
+    }
+
+
+    private Set<FileReference> getObsoleteFileReferences(List<FileReference> newReferences) {
+        Set<FileReference> obsoleteReferences = new HashSet<>(reference2Bundles.keySet());
+        obsoleteReferences.removeAll(newReferences);
+        return obsoleteReferences;
+    }
+
+    /**
+     * Allow duplicates (bsn+version) for each bundle that corresponds to obsolete file references,
+     * and avoid allowing duplicates for bundles that were installed via the
+     * X-JDisc-Preinstall-Bundle directive. These bundles are always "disk bundles" (library
+     * bundles installed on the node, and not transferred via file distribution).
+     * Such bundles will never have duplicates because they always have the same location id.
+     */
+    private void allowDuplicateBundles(Set<FileReference> obsoleteReferences) {
+        // The bundle at index 0 for each file reference always corresponds to the bundle at the file reference location
+        Set<Bundle> allowedDuplicates = obsoleteReferences.stream()
+                .filter(reference -> ! isDiskBundle(reference))
+                .map(reference -> reference2Bundles.get(reference).get(0))
+                .collect(Collectors.toSet());
+
+        log.info(() -> allowedDuplicates.isEmpty() ? "" : "Adding bundles to allowed duplicates: " + allowedDuplicates);
+        osgi.allowDuplicateBundles(allowedDuplicates);
+    }
+
+    /**
+     * Cleans up the map of active file references
+     */
+    private void removeInactiveFileReferences(Set<FileReference> fileReferencesToRemove) {
+        // Clean up the map of active bundles
+        fileReferencesToRemove.forEach(reference2Bundles::remove);
+    }
+
+    private void installBundles(List<FileReference> references) {
         Set<FileReference> bundlesToInstall = new HashSet<>(references);
 
         // This is just an optimization, as installing a bundle with the same location id returns the already installed bundle.
         bundlesToInstall.removeAll(reference2Bundles.keySet());
 
-        PredicateSplit<FileReference> bundlesToInstall_isDisk = partition(bundlesToInstall, BundleLoader::isDiskBundle);
+        PredicateSplit<FileReference> bundlesToInstall_isDisk = partition(bundlesToInstall, BundleManager::isDiskBundle);
         installBundlesFromDisk(bundlesToInstall_isDisk.trueValues);
         installBundlesFromFileDistribution(bundlesToInstall_isDisk.falseValues);
 
@@ -81,7 +156,9 @@ public class BundleLoader {
             FileAcquirer fileAcquirer = Container.get().getFileAcquirer();
             boolean hasFileDistribution = (fileAcquirer != null);
             if (hasFileDistribution) {
-                installWithFileDistribution(bundlesToInstall, fileAcquirer);
+                installWithFileDistribution(bundlesToInstall, new FileAcquirerBundleInstaller(fileAcquirer));
+            } else if (customBundleInstaller != null) {
+                installWithFileDistribution(bundlesToInstall, customBundleInstaller);
             } else {
                 log.warning("Can't retrieve bundles since file distribution is disabled.");
             }
@@ -89,25 +166,18 @@ public class BundleLoader {
     }
 
     private void installBundleFromDisk(FileReference reference) {
-        assert(reference.value().startsWith(DISK_BUNDLE_PREFIX));
-        String referenceFileName = reference.value().substring(DISK_BUNDLE_PREFIX.length());
         log.info("Installing bundle from disk with reference '" + reference.value() + "'");
 
-        File file = new File(referenceFileName);
-        if ( ! file.exists()) {
-            throw new IllegalArgumentException("Reference '" + reference.value() + "' not found on disk.");
-        }
-
-        List<Bundle> bundles = osgi.install(file.getAbsolutePath());
-
+        var bundleInstaller = new DiskBundleInstaller();
+        List<Bundle> bundles = bundleInstaller.installBundles(reference, osgi);
         reference2Bundles.put(reference, bundles);
     }
 
-    private void installWithFileDistribution(List<FileReference> bundlesToInstall, FileAcquirer fileAcquirer) {
+    private void installWithFileDistribution(List<FileReference> bundlesToInstall, BundleInstaller bundleInstaller) {
         for (FileReference reference : bundlesToInstall) {
             try {
                 log.info("Installing bundle with reference '" + reference.value() + "'");
-                List<Bundle> bundles = obtainBundles(reference, fileAcquirer);
+                List<Bundle> bundles = bundleInstaller.installBundles(reference, osgi);
                 reference2Bundles.put(reference, bundles);
             }
             catch(Exception e) {
@@ -116,11 +186,6 @@ public class BundleLoader {
         }
     }
 
-    private List<Bundle> obtainBundles(FileReference reference, FileAcquirer fileAcquirer) throws InterruptedException {
-        File file = fileAcquirer.waitFor(reference, 7, TimeUnit.DAYS);
-        return osgi.install(file.getAbsolutePath());
-    }
-
     /**
      * Resolves and starts (calls the Bundles BundleActivator) all bundles. Bundle resolution must take place
      * after all bundles are installed to ensure that the framework can resolve dependencies between bundles.
@@ -146,84 +211,12 @@ public class BundleLoader {
         return (bundleRevision.getTypes() & BundleRevision.TYPE_FRAGMENT) != 0;
     }
 
-    /**
-     * Returns the bundles that are not assumed to be retained by the new application generation.
-     * and cleans up the map of active file references. Note that at this point we don't yet know
-     * the full set of new bundles, because of the potential pre-install directives in the new bundles.
-     * However, only "disk bundles" (file:) can be listed in the pre-install directive, so we know
-     * about all the obsolete application bundles.
-     */
-    private Set<Bundle> getObsoleteBundles(List<FileReference> newReferences) {
-        Set<Bundle> bundlesToRemove = new HashSet<>(osgi.getCurrentBundles());
-
-        for (FileReference fileReferenceToKeep : newReferences) {
-            if (reference2Bundles.containsKey(fileReferenceToKeep)) {
-                bundlesToRemove.removeAll(reference2Bundles.get(fileReferenceToKeep));
-            }
-        }
-        bundlesToRemove.removeAll(osgi.getInitialBundles());
-        return bundlesToRemove;
-    }
-
-    private void removeInactiveFileReferences(List<FileReference> newReferences) {
-        // Clean up the map of active bundles
-        Set<FileReference> fileReferencesToRemove = getObsoleteFileReferences(newReferences);
-        fileReferencesToRemove.forEach(reference2Bundles::remove);
-    }
-
-
-    /**
-     * Allow duplicates (bsn+version) for each bundle that corresponds to obsolete file references,
-     * and avoid allowing duplicates for bundles that were installed via the
-     * X-JDisc-Preinstall-Bundle directive. These bundles are always "disk bundles" (library
-     * bundles installed on the node, and not transferred via file distribution).
-     * Such bundles will never have duplicates because they always have the same location id.
-     */
-    private void allowDuplicateBundles(List<FileReference> newReferences) {
-        Set<FileReference> obsoleteReferences = getObsoleteFileReferences(newReferences);
-
-        // The bundle at index 0 for each file reference always corresponds to the bundle at the file reference location
-        Set<Bundle> allowedDuplicates = obsoleteReferences.stream()
-                .map(reference -> reference2Bundles.get(reference).get(0))
-                .collect(Collectors.toSet());
-
-        log.info(() -> allowedDuplicates.isEmpty() ? "" : "Adding bundles to allowed duplicates: " + allowedDuplicates);
-        osgi.allowDuplicateBundles(allowedDuplicates);
-    }
-
-    private Set<FileReference> getObsoleteFileReferences(List<FileReference> newReferences) {
-        Set<FileReference> obsoleteReferences = new HashSet<>(reference2Bundles.keySet());
-        obsoleteReferences.removeAll(newReferences);
-        return obsoleteReferences;
-    }
-
     private Set<Bundle> allActiveBundles() {
         return reference2Bundles.keySet().stream()
                 .flatMap(reference -> reference2Bundles.get(reference).stream())
                 .collect(Collectors.toSet());
     }
 
-    /**
-     * Installs the given set of bundles and returns the set of bundles that is no longer used
-     * by the application, and should therefore be scheduled for uninstall.
-     */
-    public synchronized Set<Bundle> use(List<FileReference> newBundles) {
-        // Must be done before allowing duplicates because allowed duplicates affect osgi.getCurrentBundles
-        Set<Bundle> bundlesToUninstall = getObsoleteBundles(newBundles);
-
-        allowDuplicateBundles(newBundles);
-        removeInactiveFileReferences(newBundles);
-
-        install(newBundles);
-        startBundles();
-
-        bundlesToUninstall.removeAll(allActiveBundles());
-        log.info("Bundles to schedule for uninstall: " + bundlesToUninstall);
-
-        log.info(installedBundlesMessage());
-        return bundlesToUninstall;
-    }
-
     private String installedBundlesMessage() {
         StringBuilder sb = new StringBuilder("Installed bundles: {" );
         for (Bundle b : osgi.getBundles())
@@ -233,4 +226,14 @@ public class BundleLoader {
         return sb.toString();
     }
 
+    // Only for testing
+    void useCustomBundleInstaller(BundleInstaller bundleInstaller) {
+        customBundleInstaller = bundleInstaller;
+    }
+
+    // Only for testing
+    List<FileReference> getActiveFileReferences() {
+        return new ArrayList<>(reference2Bundles.keySet());
+    }
+
 }
diff --git a/container-core/src/main/java/com/yahoo/container/core/config/DiskBundleInstaller.java b/container-core/src/main/java/com/yahoo/container/core/config/DiskBundleInstaller.java
new file mode 100644
index 00000000000..3edabe9f861
--- /dev/null
+++ b/container-core/src/main/java/com/yahoo/container/core/config/DiskBundleInstaller.java
@@ -0,0 +1,31 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.container.core.config;
+
+import com.yahoo.config.FileReference;
+import com.yahoo.osgi.Osgi;
+import org.osgi.framework.Bundle;
+
+import java.io.File;
+import java.util.List;
+
+import static com.yahoo.container.core.BundleLoaderProperties.DISK_BUNDLE_PREFIX;
+
+/**
+ * @author gjoranv
+ */
+public class DiskBundleInstaller implements BundleInstaller {
+
+    @Override
+    public List<Bundle> installBundles(FileReference reference, Osgi osgi) {
+        assert(reference.value().startsWith(DISK_BUNDLE_PREFIX));
+        String referenceFileName = reference.value().substring(DISK_BUNDLE_PREFIX.length());
+
+        File file = new File(referenceFileName);
+        if ( ! file.exists()) {
+            throw new IllegalArgumentException("Reference '" + reference.value() + "' not found on disk.");
+        }
+
+        return osgi.install(file.getAbsolutePath());
+    }
+
+}
diff --git a/container-core/src/main/java/com/yahoo/container/core/config/FileAcquirerBundleInstaller.java b/container-core/src/main/java/com/yahoo/container/core/config/FileAcquirerBundleInstaller.java
new file mode 100644
index 00000000000..72951e67b4e
--- /dev/null
+++ b/container-core/src/main/java/com/yahoo/container/core/config/FileAcquirerBundleInstaller.java
@@ -0,0 +1,53 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.container.core.config;
+
+import com.yahoo.config.FileReference;
+import com.yahoo.filedistribution.fileacquirer.FileAcquirer;
+import com.yahoo.osgi.Osgi;
+import org.osgi.framework.Bundle;
+
+import java.io.File;
+import java.nio.file.Files;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import java.util.logging.Logger;
+
+/**
+ * @author gjoranv
+ */
+public class FileAcquirerBundleInstaller implements BundleInstaller {
+    private static Logger log = Logger.getLogger(FileAcquirerBundleInstaller.class.getName());
+
+    private final FileAcquirer fileAcquirer;
+
+    public FileAcquirerBundleInstaller(FileAcquirer fileAcquirer) {
+        this.fileAcquirer = fileAcquirer;
+    }
+
+    @Override
+    public List<Bundle> installBundles(FileReference reference, Osgi osgi) throws InterruptedException {
+        File file = fileAcquirer.waitFor(reference, 7, TimeUnit.DAYS);
+
+        if (notReadable(file)) {
+            // Wait a few sec in case FileAcquirer returns right before the file is actually ready.
+            // This happened on rare occasions due to a (fixed) bug in file distribution.
+            log.warning("Unable to open bundle file with reference '" + reference + "'. Waiting for up to 5 sec.");
+            int retries = 0;
+            while (notReadable(file) && retries < 10) {
+                Thread.sleep(500);
+                retries++;
+            }
+            if (notReadable(file)) {
+                com.yahoo.protect.Process.logAndDie("Shutting down - unable to read bundle file with reference '" + reference
+                                                            + "' and path " + file.getAbsolutePath());
+            }
+        }
+
+        return osgi.install(file.getAbsolutePath());
+    }
+
+    private static boolean notReadable(File file) {
+        return ! Files.isReadable(file.toPath());
+    }
+
+}
diff --git a/container-core/src/main/java/com/yahoo/container/core/config/HandlersConfigurerDi.java b/container-core/src/main/java/com/yahoo/container/core/config/HandlersConfigurerDi.java
index ef132694e10..d87b38e8b18 100644
--- a/container-core/src/main/java/com/yahoo/container/core/config/HandlersConfigurerDi.java
+++ b/container-core/src/main/java/com/yahoo/container/core/config/HandlersConfigurerDi.java
@@ -9,7 +9,6 @@ import com.yahoo.component.ComponentSpecification;
 import com.yahoo.component.provider.ComponentRegistry;
 import com.yahoo.concurrent.ThreadFactoryFactory;
 import com.yahoo.config.FileReference;
-import com.yahoo.container.core.config.testutil.MockOsgiWrapper;
 import com.yahoo.container.di.ComponentDeconstructor;
 import com.yahoo.container.di.Container;
 import com.yahoo.container.di.componentgraph.core.ComponentGraph;
@@ -82,7 +81,7 @@ public class HandlersConfigurerDi {
                                 OsgiFramework osgiFramework) {
 
         this(subscriberFactory, vespaContainer, configId, deconstructor, discInjector,
-             new ContainerAndDiOsgi(osgiFramework, new BundleLoader(new OsgiImpl(osgiFramework))));
+             new ContainerAndDiOsgi(osgiFramework, new BundleManager(new OsgiImpl(osgiFramework))));
     }
 
     // Only public for testing
@@ -102,9 +101,9 @@ public class HandlersConfigurerDi {
     private static class ContainerAndDiOsgi extends OsgiImpl implements OsgiWrapper {
 
         private final OsgiFramework osgiFramework;
-        private final BundleLoader bundleLoader;
+        private final BundleManager bundleLoader;
 
-        public ContainerAndDiOsgi(OsgiFramework osgiFramework, BundleLoader bundleLoader) {
+        public ContainerAndDiOsgi(OsgiFramework osgiFramework, BundleManager bundleLoader) {
             super(osgiFramework);
             this.osgiFramework = osgiFramework;
             this.bundleLoader = bundleLoader;
diff --git a/container-core/src/main/java/com/yahoo/container/handler/ClustersStatus.java b/container-core/src/main/java/com/yahoo/container/handler/ClustersStatus.java
index aab13a1cc7b..0ed0daa2141 100644
--- a/container-core/src/main/java/com/yahoo/container/handler/ClustersStatus.java
+++ b/container-core/src/main/java/com/yahoo/container/handler/ClustersStatus.java
@@ -27,6 +27,8 @@ public class ClustersStatus extends AbstractComponent {
     @Inject
     public ClustersStatus() { }
 
+    public enum Require {ONE, ALL}
+
     /** Are there any (in-service influencing) clusters in this container? */
     private boolean containerHasClusters;
 
@@ -72,12 +74,25 @@ public class ClustersStatus extends AbstractComponent {
         setDown((String) clusterIdentifier);
     }
 
-    /** Returns whether this container should receive traffic based on the state of this */
+    @Deprecated // TODO: Remove on Vespa 8
     public boolean containerShouldReceiveTraffic() {
+        return containerShouldReceiveTraffic(Require.ONE);
+    }
+    /**
+     *  Returns whether this container should receive traffic based on the state of this
+     *  @param require Requirement for being up, ALL or ONE.
+     */
+    public boolean containerShouldReceiveTraffic(Require require) {
         synchronized (mutex) {
             if (containerHasClusters) {
-                // Should receive traffic when at least one cluster is up
-                return clusterStatus.values().stream().anyMatch(status -> status==true);
+                switch (require) {
+                    case ONE:
+                        // Should receive traffic when at least one cluster is up
+                        return clusterStatus.values().stream().anyMatch(status -> status == true);
+                    case ALL:
+                    default:
+                        return !clusterStatus.isEmpty() && clusterStatus.values().stream().allMatch(status -> status == true);
+                }
             }
             else {
                 return true;
diff --git a/container-core/src/main/java/com/yahoo/container/handler/ThreadPoolProvider.java b/container-core/src/main/java/com/yahoo/container/handler/ThreadPoolProvider.java
index 4cc3b48fd1a..b427a58c9b7 100644
--- a/container-core/src/main/java/com/yahoo/container/handler/ThreadPoolProvider.java
+++ b/container-core/src/main/java/com/yahoo/container/handler/ThreadPoolProvider.java
@@ -9,6 +9,7 @@ import com.yahoo.container.di.componentgraph.Provider;
 import com.yahoo.container.protect.ProcessTerminator;
 import com.yahoo.jdisc.Metric;
 
+import java.util.Map;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.Executor;
 import java.util.concurrent.ExecutorService;
@@ -43,7 +44,8 @@ public class ThreadPoolProvider extends AbstractComponent implements Provider<Ex
                                                              threadpoolConfig.maxthreads(),
                                                              0L, TimeUnit.SECONDS,
                                                              new SynchronousQueue<>(false),
-                                                             ThreadFactoryFactory.getThreadFactory("threadpool"));
+                                                             ThreadFactoryFactory.getThreadFactory("threadpool"),
+                                                             metric);
         // Prestart needed, if not all threads will be created by the fist N tasks and hence they might also
         // get the dreaded thread locals initialized even if they will never run.
         // That counters what we we want to achieve with the Q that will prefer thread locality.
@@ -161,17 +163,22 @@ public class ThreadPoolProvider extends AbstractComponent implements Provider<Ex
     /** A thread pool executor which maintains the last time a worker completed */
     private final static class WorkerCompletionTimingThreadPoolExecutor extends ThreadPoolExecutor {
 
+        private static final String UNHANDLED_EXCEPTIONS_METRIC = "jdisc.thread_pool.unhandled_exceptions";
+
         volatile long lastThreadAssignmentTimeMillis = System.currentTimeMillis();
         private final AtomicLong startedCount = new AtomicLong(0);
         private final AtomicLong completedCount = new AtomicLong(0);
+        private final Metric metric;
 
         public WorkerCompletionTimingThreadPoolExecutor(int corePoolSize,
                                                         int maximumPoolSize,
                                                         long keepAliveTime,
                                                         TimeUnit unit,
                                                         BlockingQueue<Runnable> workQueue,
-                                                        ThreadFactory threadFactory) {
+                                                        ThreadFactory threadFactory,
+                                                        Metric metric) {
             super(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, threadFactory);
+            this.metric = metric;
         }
 
         @Override
@@ -185,6 +192,9 @@ public class ThreadPoolProvider extends AbstractComponent implements Provider<Ex
         protected void afterExecute(Runnable r, Throwable t) {
             super.afterExecute(r, t);
             completedCount.incrementAndGet();
+            if (t != null) {
+                metric.add(UNHANDLED_EXCEPTIONS_METRIC, 1L, metric.createContext(Map.of("exception", t.getClass().getSimpleName())));
+            }
         }
 
         @Override
diff --git a/container-core/src/main/java/com/yahoo/container/handler/VipStatus.java b/container-core/src/main/java/com/yahoo/container/handler/VipStatus.java
index b9ef1627ce7..0bf86e8f440 100644
--- a/container-core/src/main/java/com/yahoo/container/handler/VipStatus.java
+++ b/container-core/src/main/java/com/yahoo/container/handler/VipStatus.java
@@ -24,6 +24,8 @@ public class VipStatus {
     /** If this is non-null, its value decides whether this container is in rotation */
     private Boolean rotationOverride = null;
 
+    private final boolean initiallyInRotation;
+
     /** The current state of this */
     private boolean currentlyInRotation;
 
@@ -44,20 +46,29 @@ public class VipStatus {
         this(new QrSearchersConfig.Builder().build(), clustersStatus);
     }
 
+    /** For testing */
     public VipStatus(QrSearchersConfig dispatchers, ClustersStatus clustersStatus) {
-        this(dispatchers, clustersStatus, new StateMonitor());
+        this(dispatchers, new VipStatusConfig.Builder().build(), clustersStatus, new StateMonitor());
     }
 
     @Inject
-    public VipStatus(QrSearchersConfig dispatchers, ClustersStatus clustersStatus, StateMonitor healthState) {
+    public VipStatus(QrSearchersConfig dispatchers,
+                     VipStatusConfig vipStatusConfig,
+                     ClustersStatus clustersStatus,
+                     StateMonitor healthState) {
         this.clustersStatus = clustersStatus;
         this.healthState = healthState;
+        initiallyInRotation = vipStatusConfig.initiallyInRotation();
         healthState.status(StateMonitor.Status.initializing);
         clustersStatus.setContainerHasClusters(! dispatchers.searchcluster().isEmpty());
         updateCurrentlyInRotation();
     }
 
-    /** @deprecated don't pass VipStatusConfig */
+    @Deprecated // TODO: Remove on Vespa 8
+    public VipStatus(QrSearchersConfig dispatchers, ClustersStatus clustersStatus, StateMonitor healthState) {
+        this(dispatchers, new VipStatusConfig.Builder().build(), clustersStatus, healthState);
+    }
+
     @Deprecated // TODO: Remove on Vespa 8
     public VipStatus(QrSearchersConfig dispatchers, VipStatusConfig ignored, ClustersStatus clustersStatus) {
         this(dispatchers, clustersStatus);
@@ -102,10 +113,20 @@ public class VipStatus {
 
     private void updateCurrentlyInRotation() {
         synchronized (mutex) {
-            if (rotationOverride != null)
+            if (rotationOverride != null) {
                 currentlyInRotation = rotationOverride;
-            else
-                currentlyInRotation = clustersStatus.containerShouldReceiveTraffic();
+            } else {
+                if (healthState.status() == StateMonitor.Status.up) {
+                    currentlyInRotation = clustersStatus.containerShouldReceiveTraffic(ClustersStatus.Require.ONE);
+                }
+                else if (healthState.status() == StateMonitor.Status.initializing) {
+                    currentlyInRotation = clustersStatus.containerShouldReceiveTraffic(ClustersStatus.Require.ALL)
+                                          && initiallyInRotation;
+                }
+                else {
+                    currentlyInRotation = clustersStatus.containerShouldReceiveTraffic(ClustersStatus.Require.ALL);
+                }
+            }
 
             // Change to/from 'up' when appropriate but don't change 'initializing' to 'down'
             if (currentlyInRotation)
diff --git a/container-core/src/main/java/com/yahoo/container/handler/metrics/MetricsV2Handler.java b/container-core/src/main/java/com/yahoo/container/handler/metrics/MetricsV2Handler.java
new file mode 100644
index 00000000000..78ea62e1b3a
--- /dev/null
+++ b/container-core/src/main/java/com/yahoo/container/handler/metrics/MetricsV2Handler.java
@@ -0,0 +1,77 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.container.handler.metrics;
+
+import ai.vespa.util.http.VespaHttpClientBuilder;
+import com.google.inject.Inject;
+import com.yahoo.container.jdisc.HttpResponse;
+import com.yahoo.restapi.Path;
+import com.yahoo.yolean.Exceptions;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.impl.client.BasicResponseHandler;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
+
+import java.io.IOException;
+import java.net.URI;
+import java.util.List;
+import java.util.Optional;
+import java.util.concurrent.Executor;
+
+import static com.yahoo.jdisc.Response.Status.INTERNAL_SERVER_ERROR;
+import static com.yahoo.jdisc.Response.Status.OK;
+
+/**
+ * @author gjoranv
+ */
+public class MetricsV2Handler extends HttpHandlerBase {
+
+    public static final String V2_PATH = "/metrics/v2";
+    static final String VALUES_PATH = V2_PATH + "/values";
+
+    private static final int HTTP_CONNECT_TIMEOUT = 5000;
+    private static final int HTTP_SOCKET_TIMEOUT = 30000;
+
+    private final String metricsProxyUri;
+    private final HttpClient httpClient = createHttpClient();
+
+    @Inject
+    public MetricsV2Handler(Executor executor,
+                            MetricsProxyApiConfig config) {
+        super(executor);
+        metricsProxyUri = "http://localhost:" + config.metricsPort() + config.metricsApiPath();
+    }
+
+    @Override
+    protected Optional<HttpResponse> doHandle(URI requestUri, Path apiPath, String consumer) {
+        if (apiPath.matches(V2_PATH)) return Optional.of(resourceListResponse(requestUri, List.of(VALUES_PATH)));
+        if (apiPath.matches(VALUES_PATH)) return Optional.of(valuesResponse(consumer));
+        return Optional.empty();
+    }
+
+    private JsonResponse valuesResponse(String consumer) {
+        try {
+            String uri = metricsProxyUri + consumerQuery(consumer);
+            String metricsJson = httpClient.execute(new HttpGet(uri), new BasicResponseHandler());
+            return new JsonResponse(OK, metricsJson);
+        } catch (IOException e) {
+            log.warning("Unable to retrieve metrics from " + metricsProxyUri + ": " + Exceptions.toMessageString(e));
+            return new ErrorResponse(INTERNAL_SERVER_ERROR, e.getMessage());
+        }
+    }
+
+    private static CloseableHttpClient createHttpClient() {
+        return VespaHttpClientBuilder.create()
+                .setUserAgent("application-metrics-retriever")
+                .setDefaultRequestConfig(RequestConfig.custom()
+                                                 .setConnectTimeout(HTTP_CONNECT_TIMEOUT)
+                                                 .setSocketTimeout(HTTP_SOCKET_TIMEOUT)
+                                                 .build())
+                .build();
+    }
+
+    static String consumerQuery(String consumer) {
+        return (consumer == null || consumer.isEmpty()) ? "" : "?consumer=" + consumer;
+    }
+}
diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedHttpRequestHandler.java b/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedHttpRequestHandler.java
index dddde1205ca..c58d49bf8c8 100644
--- a/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedHttpRequestHandler.java
+++ b/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedHttpRequestHandler.java
@@ -13,6 +13,7 @@ import com.yahoo.log.LogLevel;
 
 import java.io.IOException;
 import java.nio.ByteBuffer;
+import java.util.Map;
 import java.util.concurrent.Executor;
 import java.util.concurrent.TimeUnit;
 import java.util.logging.Logger;
@@ -31,6 +32,7 @@ public abstract class ThreadedHttpRequestHandler extends ThreadedRequestHandler
 
     public static final String CONTENT_TYPE = "Content-Type";
     private static final String RENDERING_ERRORS = "rendering_errors";
+    private static final String UNHANDLED_EXCEPTIONS_METRIC = "jdisc.http.handler.unhandled_exceptions";
 
     /** Logger for subclasses */
     protected final Logger log;
@@ -79,6 +81,7 @@ public abstract class ThreadedHttpRequestHandler extends ThreadedRequestHandler
             channel.setHttpResponse(httpResponse); // may or may not have already been done
             render(httpRequest, httpResponse, channel, jdiscRequest.creationTime(TimeUnit.MILLISECONDS));
         } catch (Exception e) {
+            metric.add(UNHANDLED_EXCEPTIONS_METRIC, 1L, contextFor(request, Map.of("exception", e.getClass().getSimpleName())));
             metric.add(RENDERING_ERRORS, 1, null);
             log.log(LogLevel.ERROR, "Uncaught exception handling request", e);
             if (channel != null) {
@@ -93,7 +96,7 @@ public abstract class ThreadedHttpRequestHandler extends ThreadedRequestHandler
 
     /** Render and return whether the channel was closed */
     private void render(HttpRequest request, HttpResponse httpResponse,
-                        LazyContentChannel channel, long startTime) throws IOException {
+                        LazyContentChannel channel, long startTime) {
         LoggingCompletionHandler logOnCompletion = null;
         ContentChannelOutputStream output = null;
         try {
@@ -168,7 +171,7 @@ public abstract class ThreadedHttpRequestHandler extends ThreadedRequestHandler
         @Override
         public void close(CompletionHandler completionHandler) {
             if ( closed ) return;
-            try { httpRequest.getData().close(); } catch (IOException e) {};
+            try { httpRequest.getData().close(); } catch (IOException e) {}
             if (channel == null)
                 channel = handleResponse();
             try {
diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedRequestHandler.java b/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedRequestHandler.java
index bfcecd61fa4..99732af9d31 100644
--- a/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedRequestHandler.java
+++ b/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedRequestHandler.java
@@ -17,6 +17,7 @@ import com.yahoo.jdisc.handler.ResponseDispatch;
 import com.yahoo.jdisc.handler.ResponseHandler;
 import com.yahoo.log.LogLevel;
 
+import java.net.URI;
 import java.time.Duration;
 import java.util.HashMap;
 import java.util.Map;
@@ -78,7 +79,7 @@ public abstract class ThreadedRequestHandler extends AbstractRequestHandler {
         this.allowAsyncResponse = allowAsyncResponse;
     }
 
-    private Metric.Context contextFor(Request request) {
+    Metric.Context contextFor(Request request, Map<String, String> extraDimensions) {
         BindingMatch match = request.getBindingMatch();
         if (match == null) return null;
         UriPattern matched = match.matched();
@@ -91,9 +92,17 @@ public abstract class ThreadedRequestHandler extends AbstractRequestHandler {
         if (endpoint != null) {
             dimensions.put("endpoint", endpoint);
         }
+        URI uri = request.getUri();
+        dimensions.put("scheme", uri.getScheme());
+        dimensions.put("port", Integer.toString(uri.getPort()));
+        String handlerClassName = getClass().getName();
+        dimensions.put("handler-name", handlerClassName);
+        dimensions.putAll(extraDimensions);
         return this.metric.createContext(dimensions);
     }
 
+    private Metric.Context contextFor(Request request) { return contextFor(request, Map.of()); }
+
     /**
      * Handles a request by assigning a worker thread to it.
      *
diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/state/StateMonitor.java b/container-core/src/main/java/com/yahoo/container/jdisc/state/StateMonitor.java
index f690c240537..faa08402cdc 100644
--- a/container-core/src/main/java/com/yahoo/container/jdisc/state/StateMonitor.java
+++ b/container-core/src/main/java/com/yahoo/container/jdisc/state/StateMonitor.java
@@ -14,6 +14,7 @@ import java.util.TreeSet;
 import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.concurrent.ThreadFactory;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.logging.Logger;
 
 /**
@@ -27,7 +28,7 @@ public class StateMonitor extends AbstractComponent {
 
     private final static Logger log = Logger.getLogger(StateMonitor.class.getName());
 
-    public enum Status {up, down, initializing};
+    public enum Status {up, down, initializing}
 
     private final CopyOnWriteArrayList<StateMetricConsumer> consumers = new CopyOnWriteArrayList<>();
     private final Thread thread;
@@ -37,6 +38,7 @@ public class StateMonitor extends AbstractComponent {
     private volatile MetricSnapshot snapshot;
     private volatile Status status;
     private final TreeSet<String> valueNames = new TreeSet<>();
+    private final AtomicBoolean stopped = new AtomicBoolean(false);
 
     /** For testing */
     public StateMonitor() {
@@ -53,10 +55,16 @@ public class StateMonitor extends AbstractComponent {
     }
 
     StateMonitor(HealthMonitorConfig config, Timer timer, ThreadFactory threadFactory) {
+        this((long)(config.snapshot_interval() * TimeUnit.SECONDS.toMillis(1)),
+             Status.valueOf(config.initialStatus()),
+             timer, threadFactory);
+    }
+    /* For Testing */
+    public StateMonitor(long snapshotIntervalMS, Status status, Timer timer, ThreadFactory threadFactory) {
         this.timer = timer;
-        this.snapshotIntervalMs = (long)(config.snapshot_interval() * TimeUnit.SECONDS.toMillis(1));
+        this.snapshotIntervalMs = snapshotIntervalMS;
         this.lastSnapshotTimeMs = timer.currentTimeMillis();
-        this.status = Status.valueOf(config.initialStatus());
+        this.status = status;
         thread = threadFactory.newThread(this::run);
         thread.start();
     }
@@ -99,13 +107,13 @@ public class StateMonitor extends AbstractComponent {
     private void run() {
         log.finest("StateMonitor started.");
         try {
-            while (!Thread.interrupted()) {
-                checkTime();
-                Thread.sleep((lastSnapshotTimeMs + snapshotIntervalMs) - timer.currentTimeMillis());
+            synchronized (stopped) {
+                while (!stopped.get()) {
+                    checkTime();
+                    stopped.wait((lastSnapshotTimeMs + snapshotIntervalMs) - timer.currentTimeMillis());
+                }
             }
-        } catch (InterruptedException e) {
-
-        }
+        } catch (InterruptedException e) { }
         log.finest("StateMonitor stopped.");
     }
 
@@ -137,12 +145,13 @@ public class StateMonitor extends AbstractComponent {
 
     @Override
     public void deconstruct() {
-        thread.interrupt();
+        synchronized (stopped) {
+            stopped.set(true);
+            stopped.notifyAll();
+        }
         try {
             thread.join(5000);
-        } catch (InterruptedException e) {
-            Thread.currentThread().interrupt();
-        }
+        } catch (InterruptedException e) { }
         if (thread.isAlive()) {
             log.warning("StateMonitor failed to terminate within 5 seconds of interrupt signal. Ignoring.");
         }
diff --git a/container-core/src/main/java/com/yahoo/restapi/ResourceResponse.java b/container-core/src/main/java/com/yahoo/restapi/ResourceResponse.java
index ff301d44798..0188136addb 100644
--- a/container-core/src/main/java/com/yahoo/restapi/ResourceResponse.java
+++ b/container-core/src/main/java/com/yahoo/restapi/ResourceResponse.java
@@ -1,46 +1,41 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.restapi;
 
 import com.yahoo.container.jdisc.HttpRequest;
-import com.yahoo.container.jdisc.HttpResponse;
-import com.yahoo.slime.Cursor;
-import com.yahoo.slime.JsonFormat;
 import com.yahoo.slime.Slime;
 
-import java.io.IOException;
-import java.io.OutputStream;
 import java.net.URI;
+import java.util.List;
 
 /**
  * Returns a response containing an array of links to sub-resources
  * 
  * @author bratseth
  */
-public class ResourceResponse extends HttpResponse {
+public class ResourceResponse extends SlimeJsonResponse {
 
-    private final Slime slime = new Slime();
+    public ResourceResponse(URI parentUrl, List<String> subResources) {
+        super(200, toSlime(parentUrl, subResources));
+    }
 
     public ResourceResponse(URI parentUrl, String ... subResources) {
-        super(200);
-        Cursor resourceArray = slime.setObject().setArray("resources");
-        for (String subResource : subResources) {
-            Cursor resourceEntry = resourceArray.addObject();
-            resourceEntry.setString("url", new Uri(parentUrl).append(subResource)
-                                                             .withTrailingSlash()
-                                                             .toString());
-        }
+        this(parentUrl, List.of(subResources));
     }
 
     public ResourceResponse(HttpRequest request, String ... subResources) {
         this(request.getUri(), subResources);
     }
 
-    @Override
-    public void render(OutputStream stream) throws IOException {
-        new JsonFormat(true).encode(stream, slime);
+    private static Slime toSlime(URI parentUrl, List<String> subResources) {
+        var slime = new Slime();
+        var resourceArray = slime.setObject().setArray("resources");
+        for (var subResource : subResources) {
+            var resourceEntry = resourceArray.addObject();
+            resourceEntry.setString("url", new Uri(parentUrl).append(subResource)
+                                                             .withTrailingSlash()
+                                                             .toString());
+        }
+        return slime;
     }
 
-    @Override
-    public String getContentType() { return "application/json"; }
-
 }
diff --git a/container-core/src/main/resources/configdefinitions/container-http.def b/container-core/src/main/resources/configdefinitions/container-http.def
index ccf559b862a..23edd402893 100644
--- a/container-core/src/main/resources/configdefinitions/container-http.def
+++ b/container-core/src/main/resources/configdefinitions/container-http.def
@@ -3,3 +3,6 @@ namespace=container.core
 
 ## If non-empty, handlers should emit a header containing this string as key and the local host name as value
 hostResponseHeaderKey string default=""
+
+## For debugging, number of requests to add trace and timing information too if debugging is enabled.
+numQueriesToTraceOnDebugAfterConstruction int default=1000
diff --git a/container-core/src/main/resources/configdefinitions/health-monitor.def b/container-core/src/main/resources/configdefinitions/health-monitor.def
index 5e70c72ae3f..4e91d85b2b8 100644
--- a/container-core/src/main/resources/configdefinitions/health-monitor.def
+++ b/container-core/src/main/resources/configdefinitions/health-monitor.def
@@ -6,4 +6,4 @@ namespace=container.jdisc.config
 snapshot_interval double default=300
 
 # Initial status used in /state/v1/health API (value for 'code' in 'status'). See StateMonitor for valid values
-initialStatus string default="up"
+initialStatus string default="initializing"
diff --git a/zookeeper-server/zookeeper-server-3.4/CMakeLists.txt b/container-core/src/main/resources/configdefinitions/metrics-proxy-api.def
index d80b48c9093..3e5b973e3f3 100644
--- a/zookeeper-server/zookeeper-server-3.4/CMakeLists.txt
+++ b/container-core/src/main/resources/configdefinitions/metrics-proxy-api.def
@@ -1,2 +1,6 @@
 # Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-install_fat_java_artifact(zookeeper-server-3.4)
+
+namespace=container.handler.metrics
+
+metricsPort int
+metricsApiPath string
diff --git a/container-core/src/main/resources/configdefinitions/threadpool.def b/container-core/src/main/resources/configdefinitions/threadpool.def
index 5b5e7e2f4a2..9bb9badd9b5 100644
--- a/container-core/src/main/resources/configdefinitions/threadpool.def
+++ b/container-core/src/main/resources/configdefinitions/threadpool.def
@@ -8,3 +8,8 @@ maxthreads int default=500
 # get out of a bad state. This should be set a bit higher than the expected max execution
 # time of each request when in a state of overload, i.e about "worst case execution time*2"
 maxThreadExecutionTimeSeconds int default=190
+
+# Length of period for soft start
+# During this period number of availble threads will be gradually increased.
+# Currently used to avoid feeding overload in container during cold start.
+softStartSeconds double default=0
diff --git a/container-core/src/test/java/ai/vespa/cloud/SystemInfoTest.java b/container-core/src/test/java/ai/vespa/cloud/SystemInfoTest.java
new file mode 100644
index 00000000000..6bc8b395e00
--- /dev/null
+++ b/container-core/src/test/java/ai/vespa/cloud/SystemInfoTest.java
@@ -0,0 +1,49 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package ai.vespa.cloud;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+/**
+ * @author bratseth
+ */
+public class SystemInfoTest {
+
+    @Test
+    public void testSystemInfo() {
+        Zone zone = new Zone(Environment.dev, "us-west-1");
+        SystemInfo info = new SystemInfo(zone);
+        assertEquals(zone, info.zone());
+    }
+
+    @Test
+    public void testZone() {
+        Zone zone = Zone.from("dev.us-west-1");
+        zone = Zone.from(zone.toString());
+        assertEquals(Environment.dev, zone.environment());
+        assertEquals("us-west-1", zone.region());
+        Zone sameZone = Zone.from("dev.us-west-1");
+        assertEquals(sameZone.hashCode(), zone.hashCode());
+        assertEquals(sameZone, zone);
+
+        try {
+            Zone.from("invalid");
+            fail("Expected exception");
+        }
+        catch (IllegalArgumentException e) {
+            assertEquals("A zone string must be on the form [environment].[region], but was 'invalid'",
+                         e.getMessage());
+        }
+
+        try {
+            Zone.from("invalid.us-west-1");
+            fail("Expected exception");
+        }
+        catch (IllegalArgumentException e) {
+            assertEquals("Invalid zone 'invalid.us-west-1': No environment named 'invalid'", e.getMessage());
+        }
+    }
+
+}
diff --git a/container-core/src/test/java/com/yahoo/container/core/config/BundleManagerTest.java b/container-core/src/test/java/com/yahoo/container/core/config/BundleManagerTest.java
new file mode 100644
index 00000000000..414e6b05128
--- /dev/null
+++ b/container-core/src/test/java/com/yahoo/container/core/config/BundleManagerTest.java
@@ -0,0 +1,106 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.container.core.config;
+
+import com.yahoo.config.FileReference;
+import org.junit.Before;
+import org.junit.Test;
+import org.osgi.framework.Bundle;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * @author gjoranv
+ */
+public class BundleManagerTest {
+
+    private static final FileReference BUNDLE_1_REF = new FileReference("bundle-1");
+    private static final Bundle BUNDLE_1 = new TestBundle(BUNDLE_1_REF.value());
+    private static final FileReference BUNDLE_2_REF = new FileReference("bundle-2");
+    private static final Bundle BUNDLE_2 = new TestBundle(BUNDLE_2_REF.value());
+
+    private BundleManager bundleLoader;
+    private TestOsgi osgi;
+
+    @Before
+    public void setup() {
+        osgi = new TestOsgi(testBundles());
+        var bundleInstaller = new TestBundleInstaller();
+        bundleLoader = new BundleManager(osgi);
+        bundleLoader.useCustomBundleInstaller(bundleInstaller);
+    }
+
+    @Test
+    public void bundles_are_installed_and_started() {
+        bundleLoader.use(List.of(BUNDLE_1_REF));
+        assertEquals(1, osgi.getInstalledBundles().size());
+
+        // The bundle is installed and started
+        TestBundle installedBundle = (TestBundle)osgi.getInstalledBundles().get(0);
+        assertEquals(BUNDLE_1.getSymbolicName(), installedBundle.getSymbolicName());
+        assertTrue(installedBundle.started);
+
+        // The file reference is active
+        assertEquals(1, bundleLoader.getActiveFileReferences().size());
+        assertEquals(BUNDLE_1_REF, bundleLoader.getActiveFileReferences().get(0));
+    }
+
+    @Test
+    public void new_bundle_can_be_installed_in_reconfig() {
+        bundleLoader.use(List.of(BUNDLE_1_REF));
+        Set<Bundle> obsoleteBundles = bundleLoader.use(List.of(BUNDLE_1_REF, BUNDLE_2_REF));
+
+        // No bundles are obsolete
+        assertTrue(obsoleteBundles.isEmpty());
+
+        // Both bundles are installed
+        assertEquals(2, osgi.getInstalledBundles().size());
+        assertEquals(BUNDLE_1.getSymbolicName(), osgi.getInstalledBundles().get(0).getSymbolicName());
+        assertEquals(BUNDLE_2.getSymbolicName(), osgi.getInstalledBundles().get(1).getSymbolicName());
+
+        // Both bundles are current
+        assertEquals(2, osgi.getCurrentBundles().size());
+        assertEquals(BUNDLE_1.getSymbolicName(), osgi.getCurrentBundles().get(0).getSymbolicName());
+        assertEquals(BUNDLE_2.getSymbolicName(), osgi.getCurrentBundles().get(1).getSymbolicName());
+
+
+        // Both file references are active
+        assertEquals(2, bundleLoader.getActiveFileReferences().size());
+        assertEquals(BUNDLE_1_REF, bundleLoader.getActiveFileReferences().get(0));
+        assertEquals(BUNDLE_2_REF, bundleLoader.getActiveFileReferences().get(1));
+    }
+
+    @Test
+    public void unused_bundle_is_marked_obsolete_after_reconfig() {
+        bundleLoader.use(List.of(BUNDLE_1_REF));
+        Set<Bundle> obsoleteBundles = bundleLoader.use(List.of(BUNDLE_2_REF));
+
+        // The returned set of obsolete bundles contains bundle-1
+        assertEquals(1, obsoleteBundles.size());
+        assertEquals(BUNDLE_1.getSymbolicName(), obsoleteBundles.iterator().next().getSymbolicName());
+
+        // Both bundles are installed
+        assertEquals(2, osgi.getInstalledBundles().size());
+        assertEquals(BUNDLE_1.getSymbolicName(), osgi.getInstalledBundles().get(0).getSymbolicName());
+        assertEquals(BUNDLE_2.getSymbolicName(), osgi.getInstalledBundles().get(1).getSymbolicName());
+
+        // Only bundle-2 is current
+        assertEquals(1, osgi.getCurrentBundles().size());
+        assertEquals(BUNDLE_2.getSymbolicName(), osgi.getCurrentBundles().get(0).getSymbolicName());
+
+        // Only the bundle-2 file reference is active
+        assertEquals(1, bundleLoader.getActiveFileReferences().size());
+        assertEquals(BUNDLE_2_REF, bundleLoader.getActiveFileReferences().get(0));
+    }
+
+
+    private static Map<String, Bundle> testBundles() {
+        return Map.of(BUNDLE_1_REF.value(), BUNDLE_1,
+                      BUNDLE_2_REF.value(), BUNDLE_2);
+    }
+
+}
diff --git a/container-core/src/test/java/com/yahoo/container/core/config/TestBundle.java b/container-core/src/test/java/com/yahoo/container/core/config/TestBundle.java
new file mode 100644
index 00000000000..421f4302c27
--- /dev/null
+++ b/container-core/src/test/java/com/yahoo/container/core/config/TestBundle.java
@@ -0,0 +1,102 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.container.core.config;
+
+import com.yahoo.container.bundle.MockBundle;
+import org.osgi.framework.Bundle;
+import org.osgi.framework.Version;
+import org.osgi.framework.wiring.BundleCapability;
+import org.osgi.framework.wiring.BundleRequirement;
+import org.osgi.framework.wiring.BundleRevision;
+import org.osgi.framework.wiring.BundleWiring;
+import org.osgi.resource.Capability;
+import org.osgi.resource.Requirement;
+
+import java.util.List;
+
+/**
+ * @author gjoranv
+ */
+class TestBundle extends MockBundle {
+
+    private static final BundleRevision revision = new TestBundleRevision();
+
+    private final String symbolicName;
+
+    boolean started = false;
+
+    TestBundle(String symbolicName) {
+        this.symbolicName = symbolicName;
+    }
+
+    @Override
+    public void start() {
+        started = true;
+    }
+
+    @Override
+    public String getSymbolicName() {
+        return symbolicName;
+    }
+
+
+    @SuppressWarnings("unchecked")
+    @Override
+    public <T> T adapt(Class<T> type) {
+        if (type.equals(BundleRevision.class)) {
+            return (T) revision;
+        } else {
+            throw new UnsupportedOperationException();
+        }
+    }
+
+
+    static class TestBundleRevision implements BundleRevision {
+
+        // Ensure this is not seen as a fragment bundle.
+        @Override
+        public int getTypes() {
+            return 0;
+        }
+
+        @Override
+        public String getSymbolicName() {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public Version getVersion() {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public List<BundleCapability> getDeclaredCapabilities(String namespace) {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public List<BundleRequirement> getDeclaredRequirements(String namespace) {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public BundleWiring getWiring() {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public List<Capability> getCapabilities(String namespace) {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public List<Requirement> getRequirements(String namespace) {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public Bundle getBundle() {
+            throw new UnsupportedOperationException();
+        }
+    }
+
+}
diff --git a/container-core/src/test/java/com/yahoo/container/core/config/TestBundleInstaller.java b/container-core/src/test/java/com/yahoo/container/core/config/TestBundleInstaller.java
new file mode 100644
index 00000000000..43a5268eabf
--- /dev/null
+++ b/container-core/src/test/java/com/yahoo/container/core/config/TestBundleInstaller.java
@@ -0,0 +1,20 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.container.core.config;
+
+import com.yahoo.config.FileReference;
+import com.yahoo.osgi.Osgi;
+import org.osgi.framework.Bundle;
+
+import java.util.List;
+
+/**
+ * @author gjoranv
+ */
+class TestBundleInstaller implements BundleInstaller {
+
+    @Override
+    public List<Bundle> installBundles(FileReference reference, Osgi osgi) {
+        return osgi.install(reference.value());
+    }
+
+}
diff --git a/container-core/src/test/java/com/yahoo/container/core/config/TestOsgi.java b/container-core/src/test/java/com/yahoo/container/core/config/TestOsgi.java
new file mode 100644
index 00000000000..54a3159239c
--- /dev/null
+++ b/container-core/src/test/java/com/yahoo/container/core/config/TestOsgi.java
@@ -0,0 +1,57 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.container.core.config;
+
+import com.yahoo.osgi.MockOsgi;
+import org.osgi.framework.Bundle;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * @author gjoranv
+ */
+class TestOsgi extends MockOsgi {
+
+    private final Map<String, Bundle> availableBundles;
+
+    private final List<Bundle> installedBundles = new ArrayList<>();
+    private final List<Bundle> allowedDuplicates = new ArrayList<>();
+
+    TestOsgi(Map<String, Bundle> availableBundles) {
+        this.availableBundles = availableBundles;
+    }
+
+    @Override
+    public List<Bundle> install(String fileReferenceValue) {
+        if (! availableBundles.containsKey(fileReferenceValue))
+            throw new IllegalArgumentException("No such bundle: " + fileReferenceValue);
+
+        Bundle bundle = availableBundles.get(fileReferenceValue);
+        installedBundles.add(bundle);
+        return List.of(bundle);
+    }
+
+    @Override
+    public Bundle[] getBundles() {
+        return installedBundles.toArray(new Bundle[0]);
+    }
+
+    public List<Bundle> getInstalledBundles() {
+        return installedBundles;
+    }
+
+    @Override
+    public List<Bundle> getCurrentBundles() {
+        var currentBundles = new ArrayList<>(installedBundles);
+        currentBundles.removeAll(allowedDuplicates);
+        return currentBundles;
+    }
+
+    @Override
+    public void allowDuplicateBundles(Collection<Bundle> bundles) {
+        allowedDuplicates.addAll(bundles);
+    }
+
+}
diff --git a/container-core/src/test/java/com/yahoo/container/handler/VipStatusTestCase.java b/container-core/src/test/java/com/yahoo/container/handler/VipStatusTestCase.java
index 52679c15957..e13debcddda 100644
--- a/container-core/src/test/java/com/yahoo/container/handler/VipStatusTestCase.java
+++ b/container-core/src/test/java/com/yahoo/container/handler/VipStatusTestCase.java
@@ -4,6 +4,9 @@ package com.yahoo.container.handler;
 import static org.junit.Assert.*;
 
 import com.yahoo.container.QrSearchersConfig;
+import com.yahoo.container.core.VipStatusConfig;
+import com.yahoo.container.jdisc.state.StateMonitor;
+import com.yahoo.jdisc.core.SystemTimer;
 import org.junit.Test;
 
 /**
@@ -13,44 +16,101 @@ import org.junit.Test;
  */
 public class VipStatusTestCase {
 
-    @Test
-    public void testVipStatusWorksWithClusters() {
+    private static QrSearchersConfig getSearchersConfig(String[] clusters) {
         var b = new QrSearchersConfig.Builder();
-        var searchClusterB = new QrSearchersConfig.Searchcluster.Builder();
-        searchClusterB.name("cluster1");
-        searchClusterB.name("cluster2");
-        searchClusterB.name("cluster3");
-        b.searchcluster(searchClusterB);
-        VipStatus v = new VipStatus(b.build());
+        if (clusters.length > 0) {
+            var searchClusterB = new QrSearchersConfig.Searchcluster.Builder();
+            for (String cluster : clusters) {
+                searchClusterB.name(cluster);
+            }
+            b.searchcluster(searchClusterB);
+        }
+        return b.build();
+    }
+
+    private static VipStatus getVipStatus(String[] clusters, StateMonitor.Status startState, boolean initiallyInRotation) {
+        return new VipStatus(getSearchersConfig(clusters),
+                             new VipStatusConfig.Builder().initiallyInRotation(initiallyInRotation).build(),
+                             new ClustersStatus(),
+                             new StateMonitor(1000, startState, new SystemTimer(), runnable -> {
+                                 Thread thread = new Thread(runnable, "StateMonitor");
+                                 thread.setDaemon(true);
+                                 return thread;
+                             }));
+    }
 
-        String cluster1 = "cluster1";
-        String cluster2 = "cluster2";
-        String cluster3 = "cluster3";
+    private static void remove(String[] clusters, VipStatus v) {
+        for (String s : clusters) {
+            v.removeFromRotation(s);
+        }
+    }
+
+    private static void add(String[] clusters, VipStatus v) {
+        for (String s : clusters) {
+            v.addToRotation(s);
+        }
+    }
 
+    private static void verifyUpOrDown(String[] clusters, StateMonitor.Status status) {
+        VipStatus v = getVipStatus(clusters, status, true);
+        remove(clusters, v);
         // initial state
         assertFalse(v.isInRotation());
+        v.addToRotation(clusters[0]);
+        assertFalse(v.isInRotation());
+        v.addToRotation(clusters[1]);
+        assertFalse(v.isInRotation());
+        v.addToRotation(clusters[2]);
+        assertTrue(v.isInRotation());
+    }
+
+    @Test
+    public void testInitializingOrDownRequireAllUp() {
+        String[] clusters = {"cluster1", "cluster2", "cluster3"};
+        verifyUpOrDown(clusters, StateMonitor.Status.initializing);
+        verifyUpOrDown(clusters, StateMonitor.Status.down);
+    }
+
+    @Test
+    public void testUpRequireAllDown() {
+        String[] clusters = {"cluster1", "cluster2", "cluster3"};
 
-        // one cluster becomes up
-        v.addToRotation(cluster1);
+        VipStatus v = getVipStatus(clusters, StateMonitor.Status.initializing, true);
+        assertFalse(v.isInRotation());
+        add(clusters, v);
         assertTrue(v.isInRotation());
 
-        // all clusters down
-        v.removeFromRotation(cluster1);
-        v.removeFromRotation(cluster2);
-        v.removeFromRotation(cluster3);
+        v.removeFromRotation(clusters[0]);
+        assertTrue(v.isInRotation());
+        v.removeFromRotation(clusters[1]);
+        assertTrue(v.isInRotation());
+        v.removeFromRotation(clusters[2]);
+        assertFalse(v.isInRotation());  // All down
+        v.addToRotation(clusters[1]);
         assertFalse(v.isInRotation());
-        // some clusters down
-        v.addToRotation(cluster2);
+        v.addToRotation(clusters[0]);
+        v.addToRotation(clusters[2]);
+        assertTrue(v.isInRotation());   // All up
+        v.removeFromRotation(clusters[0]);
+        v.removeFromRotation(clusters[2]);
         assertTrue(v.isInRotation());
-        // all clusters up
-        v.addToRotation(cluster1);
-        v.addToRotation(cluster3);
+        v.addToRotation(clusters[0]);
+        v.addToRotation(clusters[2]);
         assertTrue(v.isInRotation());
-        // and down again
-        v.removeFromRotation(cluster1);
-        v.removeFromRotation(cluster2);
-        v.removeFromRotation(cluster3);
+    }
+
+    @Test
+    public void testNoClustersConfiguringInitiallyInRotationFalse() {
+        String[] clusters = {};
+        VipStatus v = getVipStatus(clusters, StateMonitor.Status.initializing, false);
         assertFalse(v.isInRotation());
     }
 
-}
+    @Test
+    public void testNoClustersConfiguringInitiallyInRotationTrue() {
+        String[] clusters = {};
+        VipStatus v = getVipStatus(clusters, StateMonitor.Status.initializing, true);
+        assertTrue(v.isInRotation());
+    }
+
+}
\ No newline at end of file
diff --git a/container-core/src/test/java/com/yahoo/container/handler/metrics/MetricsV2HandlerTest.java b/container-core/src/test/java/com/yahoo/container/handler/metrics/MetricsV2HandlerTest.java
new file mode 100644
index 00000000000..b57814e50aa
--- /dev/null
+++ b/container-core/src/test/java/com/yahoo/container/handler/metrics/MetricsV2HandlerTest.java
@@ -0,0 +1,143 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.container.handler.metrics;
+
+import com.github.tomakehurst.wiremock.junit.WireMockRule;
+import com.yahoo.container.jdisc.RequestHandlerTestDriver;
+import org.json.JSONArray;
+import org.json.JSONException;
+import org.json.JSONObject;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Rule;
+import org.junit.Test;
+
+import java.io.BufferedReader;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.concurrent.Executors;
+import java.util.stream.Collectors;
+
+import static com.github.tomakehurst.wiremock.client.WireMock.aResponse;
+import static com.github.tomakehurst.wiremock.client.WireMock.equalTo;
+import static com.github.tomakehurst.wiremock.client.WireMock.get;
+import static com.github.tomakehurst.wiremock.client.WireMock.urlPathEqualTo;
+import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.options;
+import static com.yahoo.container.handler.metrics.MetricsV2Handler.V2_PATH;
+import static com.yahoo.container.handler.metrics.MetricsV2Handler.VALUES_PATH;
+import static com.yahoo.container.handler.metrics.MetricsV2Handler.consumerQuery;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+/**
+ * @author gjoranv
+ */
+public class MetricsV2HandlerTest {
+
+    private static final String URI_BASE = "http://localhost";
+
+    private static final String V2_URI = URI_BASE + V2_PATH;
+    private static final String VALUES_URI = URI_BASE + VALUES_PATH;
+
+    // Mock applicationmetrics api
+    private static final String MOCK_METRICS_PATH = "/node0";
+
+    private static final String TEST_FILE = "application-metrics.json";
+    private static final String RESPONSE = getFileContents(TEST_FILE);
+    private static final String CPU_METRIC = "cpu.util";
+    private static final String REPLACED_CPU_METRIC = "replaced_cpu_util";
+    private static final String CUSTOM_CONSUMER = "custom-consumer";
+
+    private static RequestHandlerTestDriver testDriver;
+
+    @Rule
+    public WireMockRule wireMockRule = new WireMockRule(options().dynamicPort());
+
+    @Before
+    public void setup() {
+        setupWireMock();
+        var handler = new MetricsV2Handler(Executors.newSingleThreadExecutor(),
+                                           new MetricsProxyApiConfig.Builder()
+                                                   .metricsPort(wireMockRule.port())
+                                                   .metricsApiPath(MOCK_METRICS_PATH)
+                                                   .build());
+        testDriver = new RequestHandlerTestDriver(handler);
+    }
+
+    private void setupWireMock() {
+        wireMockRule.stubFor(get(urlPathEqualTo(MOCK_METRICS_PATH))
+                                     .willReturn(aResponse().withBody(RESPONSE)));
+
+        // Add a slightly different response for a custom consumer.
+        String myConsumerResponse = RESPONSE.replaceAll(CPU_METRIC, REPLACED_CPU_METRIC);
+        wireMockRule.stubFor(get(urlPathEqualTo(MOCK_METRICS_PATH))
+                                     .withQueryParam("consumer", equalTo(CUSTOM_CONSUMER))
+                                     .willReturn(aResponse().withBody(myConsumerResponse)));
+    }
+
+    @Test
+    public void v2_response_contains_values_uri() throws Exception {
+        String response = testDriver.sendRequest(V2_URI).readAll();
+        JSONObject root = new JSONObject(response);
+        assertTrue(root.has("resources"));
+
+        JSONArray resources = root.getJSONArray("resources");
+        assertEquals(1, resources.length());
+
+        JSONObject valuesUri = resources.getJSONObject(0);
+        assertEquals(VALUES_URI, valuesUri.getString("url"));
+    }
+
+    @Ignore
+    @Test
+    public void visually_inspect_values_response() throws Exception {
+        JSONObject responseJson = getResponseAsJson(null);
+        System.out.println(responseJson.toString(4));
+    }
+
+    @Test
+    public void invalid_path_yields_error_response() throws Exception {
+        String response = testDriver.sendRequest(V2_URI + "/invalid").readAll();
+        JSONObject root = new JSONObject(response);
+        assertTrue(root.has("error"));
+        assertTrue(root.getString("error" ).startsWith("No content"));
+    }
+
+    @Test
+    public void values_response_is_equal_to_test_file() {
+        String response = testDriver.sendRequest(VALUES_URI).readAll();
+        assertEquals(RESPONSE, response);
+    }
+
+    @Test
+    public void consumer_is_propagated_to_metrics_proxy_api() throws JSONException {
+        JSONObject responseJson = getResponseAsJson(CUSTOM_CONSUMER);
+
+        JSONObject firstNodeMetricsValues =
+                responseJson.getJSONArray("nodes").getJSONObject(0)
+                        .getJSONObject("node")
+                        .getJSONArray("metrics").getJSONObject(0)
+                        .getJSONObject("values");
+
+        assertTrue(firstNodeMetricsValues.has(REPLACED_CPU_METRIC));
+    }
+
+    private JSONObject getResponseAsJson(String consumer) {
+        String response = testDriver.sendRequest(VALUES_URI + consumerQuery(consumer)).readAll();
+        try {
+            return new JSONObject(response);
+        } catch (JSONException e) {
+            fail("Failed to create json object: " + e.getMessage());
+            throw new RuntimeException(e);
+        }
+    }
+
+    private static String getFileContents(String filename) {
+        InputStream in = MetricsV2HandlerTest.class.getClassLoader().getResourceAsStream(filename);
+        if (in == null) {
+            throw new RuntimeException("File not found: " + filename);
+        }
+        return new BufferedReader(new InputStreamReader(in)).lines().collect(Collectors.joining("\n"));
+    }
+
+}
diff --git a/container-core/src/test/java/com/yahoo/container/jdisc/ThreadedHttpRequestHandlerTest.java b/container-core/src/test/java/com/yahoo/container/jdisc/ThreadedHttpRequestHandlerTest.java
new file mode 100644
index 00000000000..07dba21e5b6
--- /dev/null
+++ b/container-core/src/test/java/com/yahoo/container/jdisc/ThreadedHttpRequestHandlerTest.java
@@ -0,0 +1,57 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.container.jdisc;
+
+import com.yahoo.jdisc.Metric;
+import org.junit.Test;
+
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.Executors;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/**
+ * @author bjorncs
+ */
+public class ThreadedHttpRequestHandlerTest {
+
+    @Test
+    public void unhandled_exceptions_metric_is_incremented_if_subclassed_handler_throws_exception() {
+        MetricMock metricMock = new MetricMock();
+        ThreadedHttpRequestHandlerThrowingException handler = new ThreadedHttpRequestHandlerThrowingException(metricMock);
+        RequestHandlerTestDriver driver = new RequestHandlerTestDriver(handler);
+
+        driver.sendRequest("http://localhost/myhandler");
+        String expectedMetricName = "jdisc.http.handler.unhandled_exceptions";
+        assertThat(metricMock.addInvocations)
+                .containsKey(expectedMetricName);
+        assertThat(metricMock.addInvocations.get(expectedMetricName).dimensions)
+                .containsEntry("exception", "DummyException");
+    }
+
+    private static class MetricMock implements Metric {
+        final ConcurrentHashMap<String, SimpleMetricContext> addInvocations = new ConcurrentHashMap<>();
+
+        @Override public void add(String key, Number val, Context ctx) {
+            addInvocations.put(key, (SimpleMetricContext)ctx);
+        }
+        @Override public void set(String key, Number val, Context ctx) {}
+        @Override public Context createContext(Map<String, ?> properties) { return new SimpleMetricContext(properties); }
+    }
+
+    private static class SimpleMetricContext implements Metric.Context {
+        final Map<String, String> dimensions;
+
+        @SuppressWarnings("unchecked")
+        SimpleMetricContext(Map<String, ?> dimensions) { this.dimensions = (Map<String, String>)dimensions; }
+    }
+
+    private static class ThreadedHttpRequestHandlerThrowingException extends ThreadedHttpRequestHandler {
+        ThreadedHttpRequestHandlerThrowingException(Metric metric) {
+            super(Executors.newSingleThreadExecutor(), metric);
+        }
+        @Override public HttpResponse handle(HttpRequest request) { throw new DummyException(); }
+    }
+
+    private static class DummyException extends RuntimeException {}
+}
\ No newline at end of file
diff --git a/container-core/src/test/java/com/yahoo/container/jdisc/state/StateHandlerTestBase.java b/container-core/src/test/java/com/yahoo/container/jdisc/state/StateHandlerTestBase.java
index 78541137db5..8a1640e2c0e 100644
--- a/container-core/src/test/java/com/yahoo/container/jdisc/state/StateHandlerTestBase.java
+++ b/container-core/src/test/java/com/yahoo/container/jdisc/state/StateHandlerTestBase.java
@@ -57,7 +57,8 @@ public class StateHandlerTestBase {
         HealthMonitorConfig healthMonitorConfig =
                 new HealthMonitorConfig(
                         new HealthMonitorConfig.Builder()
-                                .snapshot_interval(TimeUnit.MILLISECONDS.toSeconds(SNAPSHOT_INTERVAL)));
+                                .snapshot_interval(TimeUnit.MILLISECONDS.toSeconds(SNAPSHOT_INTERVAL))
+                                .initialStatus("up"));
         ThreadFactory threadFactory = ignored -> mock(Thread.class);
         this.monitor = new StateMonitor(healthMonitorConfig, timer, threadFactory);
         builder.guiceModules().install(new AbstractModule() {
diff --git a/container-core/src/test/resources/application-metrics.json b/container-core/src/test/resources/application-metrics.json
new file mode 100644
index 00000000000..52cbb721bb1
--- /dev/null
+++ b/container-core/src/test/resources/application-metrics.json
@@ -0,0 +1,92 @@
+{
+  "nodes": [
+    {
+      "hostname": "node0",
+      "role": "role0",
+      "node": {
+        "timestamp": 1234,
+        "metrics": [
+          {
+            "values": {
+              "cpu.util": 16.222
+            },
+            "dimensions": {
+              "state": "active"
+            }
+          }
+        ]
+      },
+      "services": [
+        {
+          "name": "searchnode",
+          "timestamp": 1234,
+          "status": {
+            "code": "up"
+          },
+          "metrics": [
+            {
+              "values": {
+                "queries.count": 4
+              },
+              "dimensions": {
+                "documentType": "music"
+              }
+            }
+          ]
+        }
+      ]
+    },
+    {
+      "hostname": "node1",
+      "role": "role1",
+      "node": {
+        "timestamp": 1234,
+        "metrics": [
+          {
+            "values": {
+              "cpu.util": 32.444
+            },
+            "dimensions": {
+              "state": "active"
+            }
+          }
+        ]
+      },
+      "services": [
+        {
+          "name": "searchnode",
+          "timestamp": 1234,
+          "status": {
+            "code": "up"
+          },
+          "metrics": [
+            {
+              "values": {
+                "queries.count": 8
+              },
+              "dimensions": {
+                "documentType": "music"
+              }
+            }
+          ]
+        },
+        {
+          "name": "slobrok",
+          "timestamp": 1234,
+          "status": {
+            "code": "unknown",
+            "description": "Unable to fetch metrics from service 'slobrok'"
+          },
+          "metrics": [
+            {
+              "values": {},
+              "dimensions": {
+                "instance": "slobrok0"
+              }
+            }
+          ]
+        }
+      ]
+    }
+  ]
+}
diff --git a/container-dependencies-enforcer/pom.xml b/container-dependencies-enforcer/pom.xml
index 34ad2883625..224fd47b256 100644
--- a/container-dependencies-enforcer/pom.xml
+++ b/container-dependencies-enforcer/pom.xml
@@ -97,7 +97,7 @@
                                                 <include>javax.ws.rs:javax.ws.rs-api:[${javax.ws.rs-api.version}]:jar:provided</include>
                                                 <include>javax.xml.bind:jaxb-api:[${jaxb.version}]:jar:provided</include>
                                                 <include>net.jcip:jcip-annotations:[1.0]:jar:provided</include>
-                                                <include>net.jpountz.lz4:lz4:[${lz4.version}]:jar:provided</include>
+                                                <include>org.lz4:lz4-java:[${org.lz4.version}]:jar:provided</include>
                                                 <include>org.apache.felix:org.apache.felix.framework:[${felix.version}]:jar:provided</include>
                                                 <include>org.apache.felix:org.apache.felix.log:[${felix.log.version}]:jar:provided</include>
                                                 <include>org.apache.felix:org.apache.felix.main:[${felix.version}]:jar:provided</include>
diff --git a/container-dependency-versions/pom.xml b/container-dependency-versions/pom.xml
index a621545446f..b06734b3f87 100644
--- a/container-dependency-versions/pom.xml
+++ b/container-dependency-versions/pom.xml
@@ -179,9 +179,9 @@
                 <version>1.0</version>
             </dependency>
             <dependency>
-                <groupId>net.jpountz.lz4</groupId>
-                <artifactId>lz4</artifactId>
-                <version>${lz4.version}</version>
+                <groupId>org.lz4</groupId>
+                <artifactId>lz4-java</artifactId>
+                <version>${org.lz4.version}</version>
             </dependency>
             <dependency>
                 <groupId>org.apache.felix</groupId>
@@ -437,7 +437,7 @@
 
     <properties>
         <aopalliance.version>1.0</aopalliance.version>
-        <bouncycastle.version>1.63</bouncycastle.version>
+        <bouncycastle.version>1.65</bouncycastle.version>
         <felix.version>6.0.3</felix.version>
         <felix.log.version>1.0.1</felix.log.version>
         <findbugs.version>1.3.9</findbugs.version>
@@ -446,8 +446,8 @@
         <javax.inject.version>1</javax.inject.version>
         <javax.servlet-api.version>3.1.0</javax.servlet-api.version>
         <jaxb.version>2.3.0</jaxb.version>
-        <jetty.version>9.4.26.v20200117</jetty.version>
-        <lz4.version>1.3.0</lz4.version>
+        <jetty.version>9.4.27.v20200227</jetty.version>
+        <org.lz4.version>1.7.1</org.lz4.version>
         <org.json.version>20090211</org.json.version>
         <slf4j.version>1.7.5</slf4j.version>
         <xml-apis.version>1.4.01</xml-apis.version>
@@ -459,7 +459,7 @@
         <hk2.version>2.5.0-b32</hk2.version>
         <hk2.osgi-resource-locator.version>1.0.1</hk2.osgi-resource-locator.version>
         <jackson2.version>2.8.11</jackson2.version>
-        <jackson-databind.version>${jackson2.version}.2</jackson-databind.version>
+        <jackson-databind.version>${jackson2.version}.6</jackson-databind.version>
         <javassist.version>3.20.0-GA</javassist.version>
         <javax.annotation-api.version>1.2</javax.annotation-api.version>
         <javax.validation-api.version>1.1.0.Final</javax.validation-api.version>
diff --git a/container-dev/pom.xml b/container-dev/pom.xml
index a7217d05315..1bb06ab9694 100644
--- a/container-dev/pom.xml
+++ b/container-dev/pom.xml
@@ -106,6 +106,10 @@
           <groupId>commons-lang</groupId>
           <artifactId>commons-lang</artifactId>
         </exclusion>
+        <exclusion>
+          <groupId>org.apache.httpcomponents</groupId>
+          <artifactId>httpclient</artifactId>
+        </exclusion>
       </exclusions>
     </dependency>
     <dependency>
@@ -172,6 +176,10 @@
           <groupId>com.google.protobuf</groupId>
           <artifactId>protobuf-java</artifactId>
         </exclusion>
+        <exclusion>
+          <groupId>org.apache.commons</groupId>
+          <artifactId>commons-math3</artifactId>
+        </exclusion>
       </exclusions>
     </dependency>
     <dependency>
diff --git a/container-disc/abi-spec.json b/container-disc/abi-spec.json
index 35280e12146..d3ad495ff71 100644
--- a/container-disc/abi-spec.json
+++ b/container-disc/abi-spec.json
@@ -14,8 +14,11 @@
       "public abstract javax.net.ssl.SSLContext getRoleSslContext(java.lang.String, java.lang.String)",
       "public abstract java.lang.String getRoleToken(java.lang.String)",
       "public abstract java.lang.String getRoleToken(java.lang.String, java.lang.String)",
+      "public abstract java.lang.String getAccessToken(java.lang.String)",
+      "public abstract java.lang.String getAccessToken(java.lang.String, java.util.List)",
       "public abstract java.util.List getIdentityCertificate()",
-      "public abstract java.security.PrivateKey getPrivateKey()"
+      "public abstract java.security.PrivateKey getPrivateKey()",
+      "public abstract java.nio.file.Path trustStorePath()"
     ],
     "fields": []
   },
@@ -31,6 +34,17 @@
     ],
     "fields": []
   },
+  "com.yahoo.container.jdisc.secretstore.SecretNotFoundException": {
+    "superClass": "java.lang.RuntimeException",
+    "interfaces": [],
+    "attributes": [
+      "public"
+    ],
+    "methods": [
+      "public void <init>(java.lang.String)"
+    ],
+    "fields": []
+  },
   "com.yahoo.container.jdisc.secretstore.SecretStore": {
     "superClass": "java.lang.Object",
     "interfaces": [],
diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/AthenzIdentityProviderProvider.java b/container-disc/src/main/java/com/yahoo/container/jdisc/AthenzIdentityProviderProvider.java
index 9045db3eda5..0e3110e26a8 100644
--- a/container-disc/src/main/java/com/yahoo/container/jdisc/AthenzIdentityProviderProvider.java
+++ b/container-disc/src/main/java/com/yahoo/container/jdisc/AthenzIdentityProviderProvider.java
@@ -5,6 +5,7 @@ import com.yahoo.container.di.componentgraph.Provider;
 import com.yahoo.container.jdisc.athenz.AthenzIdentityProvider;
 
 import javax.net.ssl.SSLContext;
+import java.nio.file.Path;
 import java.security.PrivateKey;
 import java.security.cert.X509Certificate;
 import java.util.List;
@@ -59,6 +60,16 @@ public class AthenzIdentityProviderProvider implements Provider<AthenzIdentityPr
         }
 
         @Override
+        public String getAccessToken(String domain) {
+            throw new UnsupportedOperationException(message);
+        }
+
+        @Override
+        public String getAccessToken(String domain, List<String> roles) {
+            throw new UnsupportedOperationException(message);
+        }
+
+        @Override
         public List<X509Certificate> getIdentityCertificate() {
             throw new UnsupportedOperationException(message);
         }
@@ -67,5 +78,10 @@ public class AthenzIdentityProviderProvider implements Provider<AthenzIdentityPr
         public PrivateKey getPrivateKey() {
             throw new UnsupportedOperationException(message);
         }
+
+        @Override
+        public Path trustStorePath() {
+            throw new UnsupportedOperationException(message);
+        }
     }
 }
diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/AthenzIdentityProvider.java b/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/AthenzIdentityProvider.java
index 1db3f4a3b42..10bf96749e8 100644
--- a/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/AthenzIdentityProvider.java
+++ b/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/AthenzIdentityProvider.java
@@ -2,6 +2,7 @@
 package com.yahoo.container.jdisc.athenz;
 
 import javax.net.ssl.SSLContext;
+import java.nio.file.Path;
 import java.security.PrivateKey;
 import java.security.cert.X509Certificate;
 import java.util.List;
@@ -16,6 +17,9 @@ public interface AthenzIdentityProvider {
     SSLContext getRoleSslContext(String domain, String role);
     String getRoleToken(String domain);
     String getRoleToken(String domain, String role);
+    String getAccessToken(String domain);
+    String getAccessToken(String domain, List<String> roles);
     List<X509Certificate> getIdentityCertificate();
     PrivateKey getPrivateKey();
+    Path trustStorePath();
 }
diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/component/Deconstructor.java b/container-disc/src/main/java/com/yahoo/container/jdisc/component/Deconstructor.java
index 371f29e86a3..6a46e331762 100644
--- a/container-disc/src/main/java/com/yahoo/container/jdisc/component/Deconstructor.java
+++ b/container-disc/src/main/java/com/yahoo/container/jdisc/component/Deconstructor.java
@@ -19,6 +19,8 @@ import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
 import java.util.logging.Logger;
 
+import static com.yahoo.log.LogLevel.DEBUG;
+import static java.util.logging.Level.INFO;
 import static java.util.logging.Level.SEVERE;
 import static java.util.logging.Level.WARNING;
 
@@ -50,16 +52,16 @@ public class Deconstructor implements ComponentDeconstructor {
                 }
             } else if (component instanceof Provider) {
                 // TODO Providers should most likely be deconstructed similarly to AbstractComponent
-                log.info("Starting deconstruction of provider " + component);
+                log.log(DEBUG, () -> "Starting deconstruction of provider " + component);
                 ((Provider<?>) component).deconstruct();
-                log.info("Finished deconstruction of provider " + component);
+                log.log(DEBUG, () -> "Finished deconstruction of provider " + component);
             } else if (component instanceof SharedResource) {
-                log.info("Releasing container reference to resource " + component);
+                log.log(DEBUG, () -> "Releasing container reference to resource " + component);
                 // No need to delay release, as jdisc does ref-counting
                 ((SharedResource) component).release();
             }
         }
-        if (! destructibleComponents.isEmpty())
+        if (! destructibleComponents.isEmpty() || ! bundles.isEmpty())
             executor.schedule(new DestructComponentTask(destructibleComponents, bundles),
                               delay.getSeconds(), TimeUnit.SECONDS);
     }
@@ -70,8 +72,7 @@ public class Deconstructor implements ComponentDeconstructor {
         private final Collection<AbstractComponent> components;
         private final Collection<Bundle> bundles;
 
-        DestructComponentTask(Collection<AbstractComponent> components,
-                              Collection<Bundle> bundles) {
+        DestructComponentTask(Collection<AbstractComponent> components, Collection<Bundle> bundles) {
             this.components = components;
             this.bundles = bundles;
         }
@@ -88,10 +89,10 @@ public class Deconstructor implements ComponentDeconstructor {
         @Override
         public void run() {
             for (var component : components) {
-                log.info("Starting deconstruction of component " + component);
+                log.log(DEBUG, () -> "Starting deconstruction of component " + component);
                 try {
                     component.deconstruct();
-                    log.info("Finished deconstructing of component " + component);
+                    log.log(DEBUG, () -> "Finished deconstructing of component " + component);
                 } catch (Exception | NoClassDefFoundError e) { // May get class not found due to it being already unloaded
                     log.log(WARNING, "Exception thrown when deconstructing component " + component, e);
                 } catch (Error e) {
@@ -111,7 +112,7 @@ public class Deconstructor implements ComponentDeconstructor {
             // It should now be safe to uninstall the old bundles.
             for (var bundle : bundles) {
                 try {
-                    log.info("Uninstalling bundle " + bundle);
+                    log.log(INFO, "Uninstalling bundle " + bundle);
                     bundle.uninstall();
                 } catch (BundleException e) {
                     log.log(SEVERE, "Could not uninstall bundle " + bundle);
diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/secretstore/SecretNotFoundException.java b/container-disc/src/main/java/com/yahoo/container/jdisc/secretstore/SecretNotFoundException.java
new file mode 100644
index 00000000000..b9439432c06
--- /dev/null
+++ b/container-disc/src/main/java/com/yahoo/container/jdisc/secretstore/SecretNotFoundException.java
@@ -0,0 +1,12 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.container.jdisc.secretstore;
+
+/**
+ * @author mortent
+ */
+public class SecretNotFoundException extends RuntimeException {
+
+    public SecretNotFoundException(String message) {
+        super(message);
+    }
+}
diff --git a/container-disc/src/main/sh/vespa-start-container-daemon.sh b/container-disc/src/main/sh/vespa-start-container-daemon.sh
index 382843b5688..1ed2d5ccf29 100755
--- a/container-disc/src/main/sh/vespa-start-container-daemon.sh
+++ b/container-disc/src/main/sh/vespa-start-container-daemon.sh
@@ -64,6 +64,7 @@ configure_memory() {
     consider_fallback jvm_heapsize 1536
     consider_fallback jvm_stacksize 512
     consider_fallback jvm_baseMaxDirectMemorySize 75
+    consider_fallback jvm_compressedClassSpaceSize 32
     consider_fallback jvm_directMemorySizeCache 0
 
     # Update jvm_heapsize only if percentage is explicitly set (default is 0).
@@ -80,16 +81,20 @@ configure_memory() {
     fi
 
     # Safety measure against bad min vs max heapsize.
-   if ((jvm_minHeapsize > jvm_heapsize)); then
+    if ((jvm_minHeapsize > jvm_heapsize)); then
         jvm_minHeapsize=${jvm_heapsize}
         echo "Misconfigured heap size, jvm_minHeapsize(${jvm_minHeapsize} is larger than jvm_heapsize(${jvm_heapsize}). It has been capped."
-   fi
+    fi
 
     maxDirectMemorySize=$(( jvm_baseMaxDirectMemorySize + jvm_heapsize / 8 + jvm_directMemorySizeCache ))
 
     memory_options="-Xms${jvm_minHeapsize}m -Xmx${jvm_heapsize}m"
     memory_options="${memory_options} -XX:ThreadStackSize=${jvm_stacksize}"
-    memory_options="${memory_options} -XX:MaxDirectMemorySize=${maxDirectMemorySize}m"    
+    memory_options="${memory_options} -XX:MaxDirectMemorySize=${maxDirectMemorySize}m"
+
+    if ((jvm_compressedClassSpaceSize != 0)); then
+        memory_options="${memory_options} -XX:CompressedClassSpaceSize=${jvm_compressedClassSpaceSize}m"
+    fi
 
     if [ "${VESPA_USE_HUGEPAGES}" ]; then
         memory_options="${memory_options} -XX:+UseLargePages"
@@ -211,6 +216,7 @@ exec $numactlcmd $envcmd java \
         --add-opens=java.base/java.lang=ALL-UNNAMED \
         --add-opens=java.base/java.net=ALL-UNNAMED \
         --add-opens=java.base/jdk.internal.loader=ALL-UNNAMED \
+        -Djava.io.tmpdir="${VESPA_HOME}/tmp" \
         -Djava.library.path="${VESPA_HOME}/lib64" \
         -Djava.awt.headless=true \
         -Djavax.net.ssl.keyStoreType=JKS \
diff --git a/container-disc/src/main/ssl/jdisc_container.keydb b/container-disc/src/main/ssl/jdisc_container.keydb
deleted file mode 100644
index 23479a83442..00000000000
--- a/container-disc/src/main/ssl/jdisc_container.keydb
+++ /dev/null
@@ -1,11 +0,0 @@
-<keydb>
-  <keygroup name="jdisc_container" id="0">
-      <keyname name="jdisc_key" usage="all" type="transient">
-         <key version="0"
-               value = "2eMsYNKWBOWpsah8d57B65xvmFVZGiPAGv3pbI1mlZU-" current = "true"
-               timestamp = "20130320234320"
-               expiry = "20160319234320">
-        </key>
-      </keyname>
-  </keygroup>
-</keydb>
diff --git a/container-disc/src/test/java/com/yahoo/container/jdisc/component/DeconstructorTest.java b/container-disc/src/test/java/com/yahoo/container/jdisc/component/DeconstructorTest.java
index 345f75f7eb6..efdc8f44c17 100644
--- a/container-disc/src/test/java/com/yahoo/container/jdisc/component/DeconstructorTest.java
+++ b/container-disc/src/test/java/com/yahoo/container/jdisc/component/DeconstructorTest.java
@@ -2,14 +2,13 @@
 package com.yahoo.container.jdisc.component;
 
 import com.yahoo.component.AbstractComponent;
+import com.yahoo.container.bundle.MockBundle;
 import com.yahoo.container.di.componentgraph.Provider;
 import com.yahoo.jdisc.ResourceReference;
 import com.yahoo.jdisc.SharedResource;
 import org.junit.Before;
 import org.junit.Test;
 
-import java.util.Collections;
-
 import static java.util.Collections.emptyList;
 import static java.util.Collections.singleton;
 import static org.junit.Assert.assertTrue;
@@ -51,22 +50,41 @@ public class DeconstructorTest {
         assertTrue(sharedResource.released);
     }
 
+    @Test
+    public void bundles_are_uninstalled() throws InterruptedException {
+        var bundle = new UninstallableMockBundle();
+        // Done by executor, so it takes some time even with a 0 delay.
+        deconstructor.deconstruct(emptyList(), singleton(bundle));
+        int cnt = 0;
+        while (! bundle.uninstalled && (cnt++ < 12000)) {
+            Thread.sleep(10);
+        }
+        assertTrue(bundle.uninstalled);
+    }
+
     private static class TestAbstractComponent extends AbstractComponent {
         boolean destructed = false;
         @Override public void deconstruct() { destructed = true; }
     }
 
     private static class TestProvider implements Provider<Void> {
-        boolean destructed = false;
+        volatile boolean destructed = false;
 
         @Override public Void get() { return null; }
         @Override public void deconstruct() { destructed = true; }
     }
 
     private static class TestSharedResource implements SharedResource {
-        boolean released = false;
+        volatile boolean released = false;
 
         @Override public ResourceReference refer() { return null; }
         @Override public void release() { released = true; }
     }
+
+    private static class UninstallableMockBundle extends MockBundle {
+        boolean uninstalled = false;
+        @Override public void uninstall() {
+            uninstalled = true;
+        }
+    }
 }
diff --git a/container-integration-test/src/test/java/com/yahoo/search/query/gui/GUIHandlerTest.java b/container-integration-test/src/test/java/com/yahoo/search/query/gui/GUIHandlerTest.java
index 00272778a2b..7ca5d2a6b10 100644
--- a/container-integration-test/src/test/java/com/yahoo/search/query/gui/GUIHandlerTest.java
+++ b/container-integration-test/src/test/java/com/yahoo/search/query/gui/GUIHandlerTest.java
@@ -72,6 +72,7 @@ public class GUIHandlerTest {
 
     private String servicesXml() {
         return "<container version='1.0'>\n" +
+                "  <accesslog type='disabled'/>\n" +
                 "  <handler id='com.yahoo.search.query.gui.GUIHandler'>\n" +
                 "    <binding>http://*/querybuilder/*</binding>\n" +
                 "  </handler>\n" +
diff --git a/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/SessionCache.java b/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/SessionCache.java
index 113d99f77f9..7193433ccf7 100644
--- a/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/SessionCache.java
+++ b/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/SessionCache.java
@@ -120,7 +120,10 @@ public final class SessionCache extends AbstractComponent {
         RPCNetworkParams netParams = new RPCNetworkParams()
                 .setSlobrokConfigId(slobrokConfigId)
                 .setIdentity(new Identity(identity))
-                .setListenPort(mbusConfig.port());
+                .setListenPort(mbusConfig.port())
+                .setNumTargetsPerSpec(mbusConfig.numconnectionspertarget())
+                .setNumNetworkThreads(mbusConfig.numthreads())
+                .setOptimization(RPCNetworkParams.Optimization.valueOf(mbusConfig.optimize_for().name()));
         return SharedMessageBus.newInstance(mbusParams, netParams);
     }
 
diff --git a/container-messagebus/src/main/resources/configdefinitions/container-mbus.def b/container-messagebus/src/main/resources/configdefinitions/container-mbus.def
index b18bec66959..9aef2b32a66 100644
--- a/container-messagebus/src/main/resources/configdefinitions/container-mbus.def
+++ b/container-messagebus/src/main/resources/configdefinitions/container-mbus.def
@@ -2,9 +2,24 @@
 namespace=container.jdisc
 
 #settings for message bus in container
-enabled bool default=false
+
+# Which network port is used
 port int default=0
+
+# Number of connections per target
+numconnectionspertarget int default=1
+
+# Number network threads
+numthreads int default=2
+
+# Optimize for latency, or throughput.
+optimize_for enum {LATENCY, THROUGHPUT} default=LATENCY
+
+# Everying below is deprecated and will go away very soon.
+# Dynamic throttling is used, and works better than anything else.
 maxpendingcount int default=2048
+
+enabled bool default=false
 #maxpendingsize is set in megabytes!
 maxpendingsize int default=100
 
diff --git a/container-search/abi-spec.json b/container-search/abi-spec.json
index b5fbe235c43..51fee99a743 100644
--- a/container-search/abi-spec.json
+++ b/container-search/abi-spec.json
@@ -858,8 +858,12 @@
       "public void <init>(java.lang.String, java.lang.String)",
       "public int getTargetNumHits()",
       "public java.lang.String getIndexName()",
+      "public int getHnswExploreAdditionalHits()",
+      "public boolean getAllowApproximate()",
       "public java.lang.String getQueryTensorName()",
       "public void setTargetNumHits(int)",
+      "public void setHnswExploreAdditionalHits(int)",
+      "public void setAllowApproximate(boolean)",
       "public void setIndexName(java.lang.String)",
       "public com.yahoo.prelude.query.Item$ItemType getItemType()",
       "public java.lang.String getName()",
@@ -1954,6 +1958,7 @@
     "methods": [
       "public void <init>(com.yahoo.search.cluster.NodeManager)",
       "public void <init>(com.yahoo.search.cluster.NodeManager, boolean)",
+      "public void start()",
       "public com.yahoo.search.cluster.MonitorConfiguration getConfiguration()",
       "public void add(java.lang.Object, boolean)",
       "public com.yahoo.search.cluster.BaseNodeMonitor getNodeMonitor(java.lang.Object)",
@@ -1978,8 +1983,8 @@
     "methods": [
       "public void <init>(com.yahoo.component.ComponentId, java.util.List, boolean)",
       "public void <init>(com.yahoo.component.ComponentId, java.util.List, com.yahoo.search.cluster.Hasher, boolean)",
-      "public void <init>(com.yahoo.component.ComponentId, java.util.List, com.yahoo.search.cluster.Hasher, boolean, boolean)",
-      "public final void ping(java.lang.Object, java.util.concurrent.Executor)",
+      "protected void <init>(com.yahoo.component.ComponentId, java.util.List, com.yahoo.search.cluster.Hasher, boolean, boolean)",
+      "public final void ping(com.yahoo.search.cluster.ClusterMonitor, java.lang.Object, java.util.concurrent.Executor)",
       "protected abstract com.yahoo.prelude.Pong ping(com.yahoo.prelude.Ping, java.lang.Object)",
       "protected java.lang.Object getFirstConnection(com.yahoo.search.cluster.Hasher$NodeList, int, int, com.yahoo.search.Query)",
       "public final com.yahoo.search.Result search(com.yahoo.search.Query, com.yahoo.search.searchchain.Execution)",
@@ -2074,7 +2079,8 @@
     "methods": [
       "public abstract void working(java.lang.Object)",
       "public abstract void failed(java.lang.Object)",
-      "public abstract void ping(java.lang.Object, java.util.concurrent.Executor)",
+      "public void ping(java.lang.Object, java.util.concurrent.Executor)",
+      "public void ping(com.yahoo.search.cluster.ClusterMonitor, java.lang.Object, java.util.concurrent.Executor)",
       "public void pingIterationCompleted()"
     ],
     "fields": []
@@ -5788,6 +5794,7 @@
       "public com.yahoo.search.query.profile.compiled.CompiledQueryProfile getQueryProfile()",
       "public java.lang.Object get(com.yahoo.processing.request.CompoundName, java.util.Map, com.yahoo.processing.request.Properties)",
       "public void set(com.yahoo.processing.request.CompoundName, java.lang.Object, java.util.Map)",
+      "public void clearAll(com.yahoo.processing.request.CompoundName, java.util.Map)",
       "public java.util.Map listProperties(com.yahoo.processing.request.CompoundName, java.util.Map, com.yahoo.processing.request.Properties)",
       "public boolean isComplete(java.lang.StringBuilder, java.util.Map)",
       "public com.yahoo.search.query.profile.QueryProfileProperties clone()",
@@ -6283,6 +6290,7 @@
       "public boolean isOverridable(java.lang.String)",
       "public java.lang.Class getValueClass(java.lang.String)",
       "public com.yahoo.search.query.profile.types.QueryProfileType getType(java.lang.String)",
+      "public com.yahoo.search.query.profile.types.FieldType getFieldType(com.yahoo.processing.request.CompoundName)",
       "public com.yahoo.search.query.profile.types.FieldDescription getField(java.lang.String)",
       "public com.yahoo.search.query.profile.types.FieldDescription removeField(java.lang.String)",
       "public void addField(com.yahoo.search.query.profile.types.FieldDescription)",
@@ -6935,7 +6943,8 @@
   "com.yahoo.search.rendering.JsonRenderer$FieldConsumer": {
     "superClass": "java.lang.Object",
     "interfaces": [
-      "com.yahoo.search.result.Hit$RawUtf8Consumer"
+      "com.yahoo.search.result.Hit$RawUtf8Consumer",
+      "com.yahoo.container.logging.TraceRenderer$FieldConsumer"
     ],
     "attributes": [
       "public"
@@ -6947,6 +6956,7 @@
       "protected boolean shouldRender(java.lang.String, java.lang.Object)",
       "protected boolean shouldRenderUtf8Value(java.lang.String, int)",
       "protected void renderFieldContents(java.lang.Object)",
+      "public void accept(java.lang.Object)",
       "public bridge synthetic void accept(java.lang.Object, java.lang.Object)"
     ],
     "fields": []
diff --git a/container-search/pom.xml b/container-search/pom.xml
index 84ee5b2bc65..6fa32947869 100644
--- a/container-search/pom.xml
+++ b/container-search/pom.xml
@@ -132,6 +132,11 @@
       <scope>compile</scope>
     </dependency>
     <dependency>
+      <groupId>org.apache.commons</groupId>
+      <artifactId>commons-math3</artifactId>
+      <scope>compile</scope>
+    </dependency>
+    <dependency>
       <groupId>javax.xml.bind</groupId>
       <artifactId>jaxb-api</artifactId>
       <scope>test</scope>
diff --git a/container-search/src/main/java/com/yahoo/prelude/Index.java b/container-search/src/main/java/com/yahoo/prelude/Index.java
index 65d5879b004..0dfbf6470ad 100644
--- a/container-search/src/main/java/com/yahoo/prelude/Index.java
+++ b/container-search/src/main/java/com/yahoo/prelude/Index.java
@@ -1,7 +1,6 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.prelude;
 
-
 import com.yahoo.language.process.StemMode;
 
 import java.util.ArrayList;
@@ -10,7 +9,6 @@ import java.util.Iterator;
 import java.util.List;
 import java.util.Set;
 
-
 /**
  * Information about configured settings of a field or field collection (an actual index or not) in a search definition.
  * There are two types of settings:
@@ -26,6 +24,7 @@ import java.util.Set;
 public class Index {
 
     public static class Attribute {
+
         private boolean tokenizedContent = false;
         public final String name;
 
@@ -64,6 +63,7 @@ public class Index {
     private boolean normalize = false;
     private boolean literalBoost = false;
     private boolean numerical = false;
+    private boolean predicate = false;
     private long predicateUpperBound = Long.MAX_VALUE;
     private long predicateLowerBound = Long.MIN_VALUE;
 
@@ -73,8 +73,8 @@ public class Index {
     private boolean isNGram = false;
     private int gramSize = 2;
 
-    /** Whether implicit phrases should lead to a phrase item or an and item */
-    private boolean phraseSegmenting = true;
+    /** Whether implicit phrases should lead to a phrase item or an and item. */
+    private Boolean phraseSegmenting = false;
 
     /** The string terminating an exact token in this index, or null to use the default (space) */
     private String exactTerminator = null;
@@ -182,6 +182,8 @@ public class Index {
             setLiteralBoost(true);
         } else if (commandString.equals("numerical")) {
             setNumerical(true);
+        } else if (commandString.equals("predicate")) {
+            setPredicate(true);
         } else if (commandString.startsWith("predicate-bounds ")) {
             setPredicateBounds(commandString.substring(17));
         } else if (commandString.equals("phrase-segmenting")) {
@@ -207,20 +209,12 @@ public class Index {
         }
     }
 
-    /**
-     * Whether terms in this field are lower cased when indexing.
-     *
-     * @param lowercase true if terms are lowercased
-     */
+    /** Sets whether terms in this field are lowercased when indexing. */
     public void setLowercase(boolean lowercase) {
         this.lowercase = lowercase;
     }
 
-    /**
-     * Whether terms in this field are lower cased when indexing.
-     *
-     * @return true if terms are lowercased
-     */
+    /** Returns whether terms in this field are lowercased when indexing. */
     public boolean isLowercase() {
         return lowercase;
     }
@@ -313,6 +307,10 @@ public class Index {
 
     public boolean isNumerical() { return numerical; }
 
+    public void setPredicate(boolean isPredicate) { this.predicate = isPredicate; }
+
+    public boolean isPredicate() { return predicate; }
+
     public long getPredicateUpperBound() { return predicateUpperBound; }
 
     public long getPredicateLowerBound() { return predicateLowerBound; }
diff --git a/container-search/src/main/java/com/yahoo/prelude/IndexFacts.java b/container-search/src/main/java/com/yahoo/prelude/IndexFacts.java
index 76eef33d6c0..aa3d6a2c0f8 100644
--- a/container-search/src/main/java/com/yahoo/prelude/IndexFacts.java
+++ b/container-search/src/main/java/com/yahoo/prelude/IndexFacts.java
@@ -48,7 +48,6 @@ public class IndexFacts {
     static final String unionName = "unionOfAllKnown";
 
     /** A search definition which contains the union of all settings. */
-    @SuppressWarnings("deprecation")
     private SearchDefinition unionSearchDefinition = new SearchDefinition(unionName);
 
     private boolean frozen;
diff --git a/container-search/src/main/java/com/yahoo/prelude/IndexModel.java b/container-search/src/main/java/com/yahoo/prelude/IndexModel.java
index 062a514056b..00935392683 100644
--- a/container-search/src/main/java/com/yahoo/prelude/IndexModel.java
+++ b/container-search/src/main/java/com/yahoo/prelude/IndexModel.java
@@ -109,7 +109,6 @@ public final class IndexModel {
         return searchDefinitions;
     }
 
-    @SuppressWarnings("deprecation")
     private SearchDefinition unionOf(Collection<SearchDefinition> searchDefinitions) {
         SearchDefinition union = new SearchDefinition(IndexFacts.unionName);
 
diff --git a/container-search/src/main/java/com/yahoo/prelude/Location.java b/container-search/src/main/java/com/yahoo/prelude/Location.java
index 37284bd6bcc..908bf835e3c 100644
--- a/container-search/src/main/java/com/yahoo/prelude/Location.java
+++ b/container-search/src/main/java/com/yahoo/prelude/Location.java
@@ -126,8 +126,8 @@ public class Location {
         if (ns < -90.1 || ns > +90.1) {
             throw new IllegalArgumentException("n/s location must be in range [-90,+90]");
         }
-        if (radius_in_degrees < 0 || radius_in_degrees > 180.0) {
-            throw new IllegalArgumentException("radius must be in range [0,180] degrees, approximately upto 20000km");
+        if (radius_in_degrees < 0) {
+            pr = 512 * 1024 * 1024;
         }
         x = px;
         y = py;
diff --git a/container-search/src/main/java/com/yahoo/prelude/query/AndSegmentItem.java b/container-search/src/main/java/com/yahoo/prelude/query/AndSegmentItem.java
index bac227ac3e3..55e16804602 100644
--- a/container-search/src/main/java/com/yahoo/prelude/query/AndSegmentItem.java
+++ b/container-search/src/main/java/com/yahoo/prelude/query/AndSegmentItem.java
@@ -31,14 +31,17 @@ public class AndSegmentItem extends SegmentItem implements BlockItem {
         }
     }
 
+    @Override
     public ItemType getItemType() {
         return ItemType.AND;
     }
 
+    @Override
     public String getName() {
         return "SAND";
     }
 
+    @Override
     public String getIndexName() {
         if (getItemCount() == 0) {
             return "";
@@ -54,4 +57,5 @@ public class AndSegmentItem extends SegmentItem implements BlockItem {
             i.next().setWeight(w);
         }
     }
+
 }
diff --git a/container-search/src/main/java/com/yahoo/prelude/query/BlockItem.java b/container-search/src/main/java/com/yahoo/prelude/query/BlockItem.java
index 13673144a0a..d0ffcd2d0e0 100644
--- a/container-search/src/main/java/com/yahoo/prelude/query/BlockItem.java
+++ b/container-search/src/main/java/com/yahoo/prelude/query/BlockItem.java
@@ -3,10 +3,9 @@ package com.yahoo.prelude.query;
 
 
 /**
- * An interface used for anything which represents a single block
- * of query input.
+ * An interface used for anything which represents a single block of query input.
  *
- * @author  <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ * @author Steinar Knutsen
  */
 public interface BlockItem extends HasIndexItem {
 
@@ -39,4 +38,5 @@ public interface BlockItem extends HasIndexItem {
      * is necessary to change operator?
      */
     SegmentingRule getSegmentingRule();
+
 }
diff --git a/container-search/src/main/java/com/yahoo/prelude/query/IndexedSegmentItem.java b/container-search/src/main/java/com/yahoo/prelude/query/IndexedSegmentItem.java
index a06009e642a..300d40d4366 100644
--- a/container-search/src/main/java/com/yahoo/prelude/query/IndexedSegmentItem.java
+++ b/container-search/src/main/java/com/yahoo/prelude/query/IndexedSegmentItem.java
@@ -79,4 +79,5 @@ public abstract class IndexedSegmentItem extends TaggableSegmentItem implements
         super.disclose(discloser);
         discloser.addProperty("index", index);
     }
+
 }
diff --git a/container-search/src/main/java/com/yahoo/prelude/query/NearestNeighborItem.java b/container-search/src/main/java/com/yahoo/prelude/query/NearestNeighborItem.java
index 35b87ec0190..e8fa70afd1b 100644
--- a/container-search/src/main/java/com/yahoo/prelude/query/NearestNeighborItem.java
+++ b/container-search/src/main/java/com/yahoo/prelude/query/NearestNeighborItem.java
@@ -20,6 +20,8 @@ import java.nio.ByteBuffer;
 public class NearestNeighborItem extends SimpleTaggableItem {
 
     private int targetNumHits = 0;
+    private int hnswExploreAdditionalHits = 0;
+    private boolean approximate = true;
     private String field;
     private String queryTensorName;
 
@@ -34,12 +36,24 @@ public class NearestNeighborItem extends SimpleTaggableItem {
     /** Returns the field name */
     public String getIndexName() { return field; }
 
+    /** Returns the number of extra hits to explore in HNSW algorithm */
+    public int getHnswExploreAdditionalHits() { return hnswExploreAdditionalHits; }
+
+    /** Returns whether approximation is allowed */
+    public boolean getAllowApproximate() { return approximate; }
+
     /** Returns the name of the query tensor */
     public String getQueryTensorName() { return queryTensorName; }
 
     /** Set the K number of hits to produce */
     public void setTargetNumHits(int target) { this.targetNumHits = target; }
 
+    /** Set the number of extra hits to explore in HNSW algorithm */
+    public void setHnswExploreAdditionalHits(int num) { this.hnswExploreAdditionalHits = num; }
+
+    /** Set whether approximation is allowed */
+    public void setAllowApproximate(boolean value) { this.approximate = value; }
+
     @Override
     public void setIndexName(String index) { this.field = index; }
 
@@ -58,6 +72,8 @@ public class NearestNeighborItem extends SimpleTaggableItem {
         putString(field, buffer);
         putString(queryTensorName, buffer);
         IntegerCompressor.putCompressedPositiveNumber(targetNumHits, buffer);
+        IntegerCompressor.putCompressedPositiveNumber((approximate ? 1 : 0), buffer);
+        IntegerCompressor.putCompressedPositiveNumber(hnswExploreAdditionalHits, buffer);
         return 1;  // number of encoded stack dump items
     }
 
@@ -65,6 +81,9 @@ public class NearestNeighborItem extends SimpleTaggableItem {
     protected void appendBodyString(StringBuilder buffer) {
         buffer.append("{field=").append(field);
         buffer.append(",queryTensorName=").append(queryTensorName);
+        buffer.append(",hnsw.exploreAdditionalHits=").append(hnswExploreAdditionalHits);
+        buffer.append(",approximate=").append(String.valueOf(approximate));
         buffer.append(",targetNumHits=").append(targetNumHits).append("}");
     }
+
 }
diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/AllParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/AllParser.java
index d9b969757c2..49bdba2c90f 100644
--- a/container-search/src/main/java/com/yahoo/prelude/query/parser/AllParser.java
+++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/AllParser.java
@@ -30,6 +30,7 @@ public class AllParser extends SimpleParser {
         super(environment);
     }
 
+    @Override
     protected Item parseItems() {
         int position = tokens.getPosition();
         try {
diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/AnyParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/AnyParser.java
index dd836e9c8e1..b714a1d8b34 100644
--- a/container-search/src/main/java/com/yahoo/prelude/query/parser/AnyParser.java
+++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/AnyParser.java
@@ -35,21 +35,12 @@ public class AnyParser extends SimpleParser {
         return anyItems(true);
     }
 
-    Item parseFilter(String filter, Language queryLanguage, Set<String> searchDefinitions) {
-        return parseFilter(filter, queryLanguage, environment.getIndexFacts().newSession(searchDefinitions, Collections.emptySet()));
-    }
-
     Item parseFilter(String filter, Language queryLanguage, IndexFacts.Session indexFacts) {
-        Item filterRoot;
-
         setState(queryLanguage, indexFacts);
         tokenize(filter, null, indexFacts, queryLanguage);
 
-        filterRoot = anyItems(true);
-
-        if (filterRoot == null) {
-            return null;
-        }
+        Item filterRoot = anyItems(true);
+        if (filterRoot == null) return null;
 
         markAllTermsAsFilters(filterRoot);
         return filterRoot;
@@ -61,18 +52,10 @@ public class AnyParser extends SimpleParser {
 
         try {
             tokens.skipMultiple(PLUS);
+            if ( ! tokens.skipMultiple(MINUS)) return null;
+            if (tokens.currentIsNoIgnore(SPACE)) return null;
 
-            if (!tokens.skipMultiple(MINUS)) {
-                return null;
-            }
-
-            if (tokens.currentIsNoIgnore(SPACE)) {
-                return null;
-            }
-
-            if (item == null) {
-                item = indexableItem();
-            }
+            item = indexableItem();
 
             if (item == null) {
                 item = compositeItem();
@@ -88,13 +71,13 @@ public class AnyParser extends SimpleParser {
                     }
                 }
             }
-            if (item!=null)
+            if (item != null)
                 item.setProtected(true);
+
             return item;
         } finally {
-            if (item == null) {
+            if (item == null)
                 tokens.setPosition(position);
-            }
         }
     }
 
diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/SimpleParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/SimpleParser.java
index 9ddfea6dffb..0686a4bdb43 100644
--- a/container-search/src/main/java/com/yahoo/prelude/query/parser/SimpleParser.java
+++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/SimpleParser.java
@@ -50,32 +50,28 @@ abstract class SimpleParser extends StructuredParser {
     private Item anyItemsBody(boolean topLevel) {
         Item topLevelItem = null;
         NotItem not = null;
-        Item item;
+        Item item = null;
 
         do {
-            item = null;
-
-            if (item == null) {
-                item = positiveItem();
-                if (item != null) {
-                    if (not == null) {
-                        not = new NotItem();
-                        not.addPositiveItem(item);
-                        topLevelItem = combineItems(topLevelItem, not);
-                    } else {
-                        not.addPositiveItem(item);
-                    }
+            item = positiveItem();
+            if (item != null) {
+                if (not == null) {
+                    not = new NotItem();
+                    not.addPositiveItem(item);
+                    topLevelItem = combineItems(topLevelItem, not);
+                } else {
+                    not.addPositiveItem(item);
                 }
             }
 
             if (item == null) {
                 item = negativeItem();
                 if (item != null) {
-                    if (not == null && item != null) {
+                    if (not == null) {
                         not = new NotItem();
                         not.addNegativeItem(item);
                         topLevelItem = combineItems(topLevelItem, not);
-                    } else if (item != null) {
+                    } else {
                         not.addNegativeItem(item);
                     }
                 }
@@ -97,9 +93,8 @@ abstract class SimpleParser extends StructuredParser {
                 if (item != null) {
                     if (topLevelItem == null) {
                         topLevelItem = item;
-                    } else if (needNewTopLevel(topLevelItem, item)) {
+                    } else if (needNewORTopLevel(topLevelItem, item)) {
                         CompositeItem newTop = new OrItem();
-
                         newTop.addItem(topLevelItem);
                         newTop.addItem(item);
                         topLevelItem = newTop;
@@ -144,21 +139,13 @@ abstract class SimpleParser extends StructuredParser {
         }
     }
 
-
-    /** Says whether we need a new top level item given the new item */
-    private boolean needNewTopLevel(Item topLevelItem, Item item) {
-        if (item == null) {
-            return false;
-        }
-        if (topLevelItem instanceof TermItem) {
-            return true;
-        }
-        if (topLevelItem instanceof PhraseItem) {
-            return true;
-        }
-        if (topLevelItem instanceof BlockItem) {
-            return true;
-        }
+    /** Says whether we need a new top level OR item given the new item */
+    private boolean needNewORTopLevel(Item topLevelItem, Item item) {
+        if (item == null) return false;
+        if (topLevelItem instanceof TermItem) return true;
+        if (topLevelItem instanceof PhraseItem) return true;
+        if (topLevelItem instanceof BlockItem) return true;
+        if ( topLevelItem instanceof AndItem) return true;
         return false;
     }
 
diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/SpecialTokens.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/SpecialTokens.java
index c206ff7567e..8020088c2e3 100644
--- a/container-search/src/main/java/com/yahoo/prelude/query/parser/SpecialTokens.java
+++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/SpecialTokens.java
@@ -159,7 +159,7 @@ public class SpecialTokens {
         @Override
         public int hashCode() { return token.hashCode(); }
 
-        public Token toToken(int start,String rawSource) {
+        public Token toToken(int start, String rawSource) {
             return new Token(Token.Kind.WORD, replace(), true, new Substring(start, start + token.length(), rawSource)); // XXX: Unsafe?
         }
 
diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/StructuredParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/StructuredParser.java
index ee4c0d4d9f0..9ba6c1a8101 100644
--- a/container-search/src/main/java/com/yahoo/prelude/query/parser/StructuredParser.java
+++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/StructuredParser.java
@@ -442,9 +442,9 @@ abstract class StructuredParser extends AbstractParser {
         Item item = null;
 
         try {
-            if (!tokens.currentIs(WORD)
-                    && ((!tokens.currentIs(NUMBER) && !tokens.currentIs(MINUS)
-                    && !tokens.currentIs(UNDERSCORE)) || (!submodes.url && !submodes.site))) {
+            if ( ! tokens.currentIs(WORD)
+                && ((!tokens.currentIs(NUMBER) && !tokens.currentIs(MINUS)
+                && !tokens.currentIs(UNDERSCORE)) || (!submodes.url && !submodes.site))) {
                 return null;
             }
             Token word = tokens.next();
@@ -520,7 +520,7 @@ abstract class StructuredParser extends AbstractParser {
     /** Returns a word, a phrase, or another composite */
     private Item phraseBody(String indexName) {
         boolean quoted = false;
-        PhraseItem phrase = null;
+        CompositeItem composite = null;
         Item firstWord = null;
         boolean starAfterFirst = false;
         boolean starBeforeFirst;
@@ -539,7 +539,7 @@ abstract class StructuredParser extends AbstractParser {
                 quoted = !quoted;
             }
 
-            Item word = phraseWord(indexName, (firstWord != null) || (phrase != null));
+            Item word = phraseWord(indexName, (firstWord != null) || (composite != null));
 
             if (word == null) {
                 if (tokens.skipMultiple(QUOTE)) {
@@ -555,34 +555,39 @@ abstract class StructuredParser extends AbstractParser {
                 ((PhraseSegmentItem) word).setExplicit(true);
             }
 
-            if (phrase != null) {
-                phrase.addItem(word);
+            if (composite != null) {
+                composite.addItem(word);
+                connectLastTermsIn(composite);
             } else if (firstWord != null) {
                 if (submodes.site || submodes.url) {
                     UriItem uriItem = new UriItem();
                     if (submodes.site)
                         uriItem.setEndAnchorDefault(true);
-                    phrase = uriItem;
+                    composite = uriItem;
                 }
                 else {
-                    phrase = new PhraseItem();
+                    if (quoted || indexFacts.getIndex(indexName).getPhraseSegmenting())
+                        composite = new PhraseItem();
+                    else
+                        composite = new AndItem();
                 }
 
-                if (quoted || submodes.site || submodes.url) {
-                    phrase.setExplicit(true);
+                if ( (quoted || submodes.site || submodes.url) && composite instanceof PhraseItem) {
+                    ((PhraseItem)composite).setExplicit(true);
                 }
                 if (addStartOfHostMarker) {
-                    phrase.addItem(MarkerWordItem.createStartOfHost());
+                    composite.addItem(MarkerWordItem.createStartOfHost());
                 }
                 if (firstWord instanceof IntItem) {
                     IntItem asInt = (IntItem) firstWord;
                     firstWord = new WordItem(asInt.stringValue(), asInt.getIndexName(),
                             true, asInt.getOrigin());
                 }
-                phrase.addItem(firstWord);
-                phrase.addItem(word);
+                composite.addItem(firstWord);
+                composite.addItem(word);
+                connectLastTermsIn(composite);
             } else if (word instanceof PhraseItem) {
-                phrase = (PhraseItem) word;
+                composite = (PhraseItem)word;
             } else {
                 firstWord = word;
                 starAfterFirst = tokens.skipNoIgnore(STAR);
@@ -609,29 +614,29 @@ abstract class StructuredParser extends AbstractParser {
 
         braceLevelURL = 0;
 
-        if (phrase != null) {
+        if (composite != null) {
             if (addEndMarking()) {
-                phrase.addItem(MarkerWordItem.createEndOfHost());
+                composite.addItem(MarkerWordItem.createEndOfHost());
             }
-            return phrase;
+            return composite;
         } else if (firstWord != null && submodes.site) {
             if (starAfterFirst && !addStartOfHostMarker) {
                 return firstWord;
             } else {
-                phrase = new PhraseItem();
+                composite = new PhraseItem();
+                ((PhraseItem)composite).setExplicit(true);
                 if (addStartOfHostMarker) {
-                    phrase.addItem(MarkerWordItem.createStartOfHost());
+                    composite.addItem(MarkerWordItem.createStartOfHost());
                 }
                 if (firstWord instanceof IntItem) {
                     IntItem asInt = (IntItem) firstWord;
                     firstWord = new WordItem(asInt.stringValue(), asInt.getIndexName(), true, asInt.getOrigin());
                 }
-                phrase.addItem(firstWord);
+                composite.addItem(firstWord);
                 if (!starAfterFirst) {
-                    phrase.addItem(MarkerWordItem.createEndOfHost());
+                    composite.addItem(MarkerWordItem.createEndOfHost());
                 }
-                phrase.setExplicit(true);
-                return phrase;
+                return composite;
             }
         } else {
             if (firstWord != null && firstWord instanceof TermItem && (starAfterFirst || starBeforeFirst)) {
@@ -651,6 +656,15 @@ abstract class StructuredParser extends AbstractParser {
         }
     }
 
+    private void connectLastTermsIn(CompositeItem composite) {
+        int items = composite.items().size();
+        if (items < 2) return;
+        Item nextToLast = composite.items().get(items - 2);
+        Item last = composite.items().get(items - 1);
+        if ( ! (nextToLast instanceof TermItem)) return;
+        ((TermItem)nextToLast).setConnectivity(last, 1);
+    }
+
     private boolean addStartMarking() {
         if (submodes.explicitAnchoring() && tokens.currentIs(HAT)) {
             tokens.skip();
diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/Tokenizer.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/Tokenizer.java
index 61f09e2f7b7..5e243e52057 100644
--- a/container-search/src/main/java/com/yahoo/prelude/query/parser/Tokenizer.java
+++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/Tokenizer.java
@@ -108,8 +108,7 @@ public final class Tokenizer {
             if (i >= source.length()) break;
 
             int c = source.codePointAt(i);
-            if (characterClasses.isLetterOrDigit(c)
-                    || (c == '\'' && acceptApostropheAsWordCharacter(currentIndex))) {
+            if (characterClasses.isLetterOrDigit(c) || (c == '\'' && acceptApostropheAsWordCharacter(currentIndex))) {
                 i = consumeWordOrNumber(i, currentIndex);
             } else if (Character.isWhitespace(c)) {
                 addToken(SPACE, " ", i, i + 1);
@@ -187,7 +186,6 @@ public final class Tokenizer {
         return true;
     }
 
-    @SuppressWarnings({"deprecation"})
     private Index determineCurrentIndex(Index defaultIndex, IndexFacts.Session indexFacts) {
         int backtrack = tokens.size();
         int tokencnt = 0;
@@ -328,7 +326,6 @@ public final class Tokenizer {
                     wantEndQuote = true;
                     actualStart = curPos+1;
                 } else if (wantEndQuote && looksLikeExactEnd(curPos+1)) {
-                    // System.err.println("seen quoted token from "+actualStart+" to "+curPos);
                     seenSome = true;
                     wantEndQuote = false;
                     isQuoted = true;
@@ -435,7 +432,7 @@ public final class Tokenizer {
         if (suffStar) {
             addToken(STAR, "*", starPos, starPos + 1);
         }
-        tokens.add(new Token(WORD, source.substring(actualStart, end), true, new Substring(actualStart, end, source))); // XXX: Unsafe?
+        tokens.add(new Token(WORD, source.substring(actualStart, end), true, new Substring(actualStart, end, source)));
 
         // skip terminating quote
         if (isQuoted) {
@@ -451,17 +448,17 @@ public final class Tokenizer {
                 break;
             end++;
         }
-        tokens.add(new Token(WORD, source.substring(start, end), true, new Substring(start, end, source))); // XXX: Unsafe start?
-        if (end>=source.length())
+        tokens.add(new Token(WORD, source.substring(start, end), true, new Substring(start, end, source)));
+        if (end >= source.length())
             return end;
         else
-            return end+terminator.length(); // Don't create a token for the terminator
+            return end + terminator.length(); // Don't create a token for the terminator
     }
 
     private boolean terminatorStartsAt(int start,String terminator) {
-        int terminatorPosition=0;
-        while ((terminatorPosition+start)<source.length()) {
-            if (source.charAt(start+terminatorPosition)!=terminator.charAt(terminatorPosition))
+        int terminatorPosition = 0;
+        while ((terminatorPosition + start) < source.length()) {
+            if (source.charAt(start+terminatorPosition) != terminator.charAt(terminatorPosition))
                 return false;
             terminatorPosition++;
             if (terminatorPosition >= terminator.length())
@@ -481,8 +478,8 @@ public final class Tokenizer {
 
         while (tokenEnd < source.length()) {
             if (substringSpecialTokens) {
-                substringSpecialToken=getSpecialToken(tokenEnd);
-                if (substringSpecialToken!=null) break;
+                substringSpecialToken = getSpecialToken(tokenEnd);
+                if (substringSpecialToken != null) break;
             }
 
             int c = source.codePointAt(tokenEnd);
@@ -506,7 +503,7 @@ public final class Tokenizer {
                 // underscoresOnly = false;
                 quotesOnly = false;
             } else if (c == '\'') {
-                if (!acceptApostropheAsWordCharacter(currentIndex)) {
+                if ( ! acceptApostropheAsWordCharacter(currentIndex)) {
                     break;
                 }
                 // Otherwise consume apostrophes...
@@ -530,15 +527,15 @@ public final class Tokenizer {
             }
         }
 
-        if (substringSpecialToken==null)
+        if (substringSpecialToken == null)
             return --tokenEnd;
         // TODO: test the logic around tokenEnd with friends
-        addToken(substringSpecialToken.toToken(tokenEnd,source));
-        return --tokenEnd+substringSpecialToken.token().length();
+        addToken(substringSpecialToken.toToken(tokenEnd, source));
+        return --tokenEnd + substringSpecialToken.token().length();
     }
 
     private void addToken(Token.Kind kind, String word, int start, int end) {
-        addToken(new Token(kind, word, false, new Substring(start, end, source))); // XXX: Unsafe?
+        addToken(new Token(kind, word, false, new Substring(start, end, source)));
     }
 
     private void addToken(Token token) {
diff --git a/container-search/src/main/java/com/yahoo/prelude/querytransform/NormalizingSearcher.java b/container-search/src/main/java/com/yahoo/prelude/querytransform/NormalizingSearcher.java
index fdd6ad47a98..ce13045b518 100644
--- a/container-search/src/main/java/com/yahoo/prelude/querytransform/NormalizingSearcher.java
+++ b/container-search/src/main/java/com/yahoo/prelude/querytransform/NormalizingSearcher.java
@@ -111,25 +111,17 @@ public class NormalizingSearcher extends Searcher {
     }
 
     private void normalizeAlternatives(Language language, Session indexFacts, WordAlternativesItem block) {
-        if (!block.isNormalizable()) {
-            return;
-        }
-        {
-            Index index = indexFacts.getIndex(block.getIndexName());
-            if (index.isAttribute()) {
-                return;
-            }
-            if (!index.getNormalize()) {
-                return;
-            }
-        }
+        if ( ! block.isNormalizable()) return;
+
+        Index index = indexFacts.getIndex(block.getIndexName());
+        if (index.isAttribute()) return;
+        if ( ! index.getNormalize()) return;
 
         List<Alternative> terms = block.getAlternatives();
         for (Alternative term : terms) {
             String accentDropped = linguistics.getTransformer().accentDrop(term.word, language);
-            if (!term.word.equals(accentDropped) && accentDropped.length() > 0) {
+            if ( ! term.word.equals(accentDropped) && accentDropped.length() > 0)
                 block.addTerm(accentDropped, term.exactness * .7d);
-            }
         }
     }
 
diff --git a/container-search/src/main/java/com/yahoo/prelude/querytransform/QueryRewrite.java b/container-search/src/main/java/com/yahoo/prelude/querytransform/QueryRewrite.java
index 84c793a6df1..5a936d42ccc 100644
--- a/container-search/src/main/java/com/yahoo/prelude/querytransform/QueryRewrite.java
+++ b/container-search/src/main/java/com/yahoo/prelude/querytransform/QueryRewrite.java
@@ -4,6 +4,8 @@ package com.yahoo.prelude.querytransform;
 import com.yahoo.prelude.query.AndItem;
 import com.yahoo.prelude.query.CompositeItem;
 import com.yahoo.prelude.query.EquivItem;
+import com.yahoo.prelude.query.HasIndexItem;
+import com.yahoo.prelude.query.IndexedItem;
 import com.yahoo.prelude.query.Item;
 import com.yahoo.prelude.query.NearItem;
 import com.yahoo.prelude.query.NotItem;
@@ -169,7 +171,9 @@ public class QueryRewrite {
                         removeOtherNonrankedChildren(item, i);
                         recall = Recall.RECALLS_EVERYTHING;
                     } else if ((item instanceof AndItem) || (item instanceof NearItem)) {
-                        item.removeItem(i);
+                        if ( ! isRanked(item.getItem(i))) {
+                            item.removeItem(i);
+                        }
                     } else if (item instanceof RankItem) {
                         // empty
                     } else {
@@ -200,6 +204,20 @@ public class QueryRewrite {
                 parent.removeItem(i);
         }
     }
+
+    private static boolean isRanked(Item item) {
+        if (item instanceof CompositeItem) {
+            for (Item child : ((CompositeItem)item).items())
+                if (isRanked(child)) return true;
+            return false;
+        }
+        else if (item instanceof HasIndexItem && Hit.SDDOCNAME_FIELD.equals(((HasIndexItem)item).getIndexName())) {
+            return false; // No point in ranking by sddocname
+        }
+        else {
+            return item.isRanked();
+        }
+    }
     
     private static Item collapseSingleComposites(Item item) {
         if (!(item instanceof CompositeItem)) {
diff --git a/container-search/src/main/java/com/yahoo/prelude/querytransform/StemmingSearcher.java b/container-search/src/main/java/com/yahoo/prelude/querytransform/StemmingSearcher.java
index 655fbf6acc3..9a9044def2d 100644
--- a/container-search/src/main/java/com/yahoo/prelude/querytransform/StemmingSearcher.java
+++ b/container-search/src/main/java/com/yahoo/prelude/querytransform/StemmingSearcher.java
@@ -188,13 +188,10 @@ public class StemmingSearcher extends Searcher {
             return (Item) w;
         }
 
-        if (context.isCJK) {
-            composite = chooseCompositeForCJK(current,
-                                              ((Item) current).getParent(),
-                                              indexName);
-        } else {
-            composite = phraseSegment(current, indexName);
-        }
+        if (context.isCJK)
+            composite = chooseCompositeForCJK(current, ((Item) current).getParent(), indexName);
+        else
+            composite = chooseComposite(current, ((Item) current).getParent(), indexName);
 
         for (StemList segment : segments) {
             TaggableItem w = singleWordSegment(current, segment, index, substring, context.insidePhrase);
@@ -331,39 +328,34 @@ public class StemmingSearcher extends Searcher {
         }
     }
 
+    private CompositeItem chooseComposite(BlockItem current, CompositeItem parent, String indexName) {
+        if (parent instanceof PhraseItem || current instanceof PhraseSegmentItem)
+            return createPhraseSegment(current, indexName);
+        else
+            return createAndSegment(current);
+
+    }
+
     private CompositeItem chooseCompositeForCJK(BlockItem current, CompositeItem parent, String indexName) {
-        CompositeItem composite;
-        if (current.getSegmentingRule() == SegmentingRule.LANGUAGE_DEFAULT) {
-            if (parent instanceof PhraseItem || current instanceof PhraseSegmentItem) {
-                composite = phraseSegment(current, indexName);
-            } else
-                composite = createAndSegment(current);
-        } else {
-            switch (current.getSegmentingRule()) {
-            case PHRASE:
-                composite = phraseSegment(current, indexName);
-                break;
-            case BOOLEAN_AND:
-                composite = createAndSegment(current);
-                break;
+        if (current.getSegmentingRule() == SegmentingRule.LANGUAGE_DEFAULT)
+            return chooseComposite(current, parent, indexName);
+
+        switch (current.getSegmentingRule()) { // TODO: Why for CJK only? The segmentingRule says nothing about being for CJK only
+            case PHRASE: return createPhraseSegment(current, indexName);
+            case BOOLEAN_AND: return createAndSegment(current);
             default:
-                throw new IllegalArgumentException(
-                        "Unknown segmenting rule: "
-                                + current.getSegmentingRule()
-                                + ". This is a bug in Vespa, as the implementation has gotten out of sync."
-                                + " Please create a ticket as soon as possible.");
-            }
+                throw new IllegalArgumentException("Unknown segmenting rule: " + current.getSegmentingRule() +
+                                                   ". This is a bug in Vespa, as the implementation has gotten out of sync." +
+                                                   " Please create a ticket as soon as possible.");
         }
-        return composite;
     }
 
     private AndSegmentItem createAndSegment(BlockItem current) {
         return new AndSegmentItem(current.stringValue(), true, true);
     }
 
-    private CompositeItem phraseSegment(BlockItem current, String indexName) {
-        CompositeItem composite;
-        composite = new PhraseSegmentItem(current.getRawWord(), current.stringValue(), true, true);
+    private CompositeItem createPhraseSegment(BlockItem current, String indexName) {
+        CompositeItem composite = new PhraseSegmentItem(current.getRawWord(), current.stringValue(), true, true);
         composite.setIndexName(indexName);
         return composite;
     }
diff --git a/container-search/src/main/java/com/yahoo/prelude/searcher/PosSearcher.java b/container-search/src/main/java/com/yahoo/prelude/searcher/PosSearcher.java
index 43717ecf6cd..37561d3a0f5 100644
--- a/container-search/src/main/java/com/yahoo/prelude/searcher/PosSearcher.java
+++ b/container-search/src/main/java/com/yahoo/prelude/searcher/PosSearcher.java
@@ -147,7 +147,9 @@ public class PosSearcher extends Searcher {
         String radius = query.properties().getString(posRadius);
         int radiusUnits;
         if (radius == null) {
-            radiusUnits = 5000;
+            double radiuskm = 50.0;
+            double radiusdegrees = radiuskm * km2deg;
+            radiusUnits = (int)(radiusdegrees * 1000000);
         } else if (radius.endsWith("km")) {
             double radiuskm = Double.valueOf(radius.substring(0, radius.length()-2));
             double radiusdegrees = radiuskm * km2deg;
diff --git a/container-search/src/main/java/com/yahoo/prelude/searcher/ValidatePredicateSearcher.java b/container-search/src/main/java/com/yahoo/prelude/searcher/ValidatePredicateSearcher.java
index 9b6f5926b61..a8b3c76fe00 100644
--- a/container-search/src/main/java/com/yahoo/prelude/searcher/ValidatePredicateSearcher.java
+++ b/container-search/src/main/java/com/yahoo/prelude/searcher/ValidatePredicateSearcher.java
@@ -1,12 +1,12 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.prelude.searcher;
 
-import java.util.Optional;
 import com.yahoo.component.chain.dependencies.After;
 import com.yahoo.prelude.Index;
 import com.yahoo.prelude.IndexFacts;
 import com.yahoo.prelude.query.Item;
 import com.yahoo.prelude.query.PredicateQueryItem;
+import com.yahoo.prelude.query.SimpleIndexedItem;
 import com.yahoo.prelude.query.ToolBox;
 import com.yahoo.search.Query;
 import com.yahoo.search.Result;
@@ -15,7 +15,8 @@ import com.yahoo.search.querytransform.BooleanSearcher;
 import com.yahoo.search.result.ErrorMessage;
 import com.yahoo.search.searchchain.Execution;
 
-import java.util.Collection;
+import java.util.ArrayList;
+import java.util.List;
 
 /**
  * Checks that predicate queries don't use values outside the defined upper/lower bounds.
@@ -27,26 +28,26 @@ public class ValidatePredicateSearcher extends Searcher {
 
     @Override
     public Result search(Query query, Execution execution) {
-        Optional<ErrorMessage> e = validate(query, execution.context().getIndexFacts().newSession(query));
-        if (e.isPresent()) {
+        List<ErrorMessage> errorMessages = validate(query, execution.context().getIndexFacts().newSession(query));
+        if (!errorMessages.isEmpty()) {
             Result r = new Result(query);
-            r.hits().addError(e.get());
+            errorMessages.forEach(msg -> r.hits().addError(msg));
             return r;
         }
         return execution.search(query);
     }
 
-    private Optional<ErrorMessage> validate(Query query, IndexFacts.Session indexFacts) {
+    private List<ErrorMessage> validate(Query query, IndexFacts.Session indexFacts) {
         ValidatePredicateVisitor visitor = new ValidatePredicateVisitor(indexFacts);
         ToolBox.visit(visitor, query.getModel().getQueryTree().getRoot());
-        return visitor.errorMessage;
+        return visitor.errorMessages;
     }
 
     private static class ValidatePredicateVisitor extends ToolBox.QueryVisitor {
 
         private final IndexFacts.Session indexFacts;
 
-        public Optional<ErrorMessage> errorMessage = Optional.empty();
+        final List<ErrorMessage> errorMessages = new ArrayList<>();
 
         public ValidatePredicateVisitor(IndexFacts.Session indexFacts) {
             this.indexFacts = indexFacts;
@@ -57,22 +58,37 @@ public class ValidatePredicateSearcher extends Searcher {
             if (item instanceof PredicateQueryItem) {
                 visit((PredicateQueryItem) item);
             }
+            if (item instanceof SimpleIndexedItem) {
+                visit((SimpleIndexedItem) item);
+            }
             return true;
         }
 
         private void visit(PredicateQueryItem item) {
-            Index index = getIndexFromUnionOfDocumentTypes(item);
+            Index index = getIndexFromUnionOfDocumentTypes(item.getIndexName());
+            if (!index.isPredicate()) {
+                errorMessages.add(ErrorMessage.createIllegalQuery(String.format("Index '%s' is not a predicate attribute.", index.getName())));
+            }
             for (PredicateQueryItem.RangeEntry entry : item.getRangeFeatures()) {
                 long value = entry.getValue();
                 if (value < index.getPredicateLowerBound() || value > index.getPredicateUpperBound()) {
-                    errorMessage = Optional.of(ErrorMessage.createIllegalQuery(
-                            String.format("%s=%d outside configured predicate bounds.", entry.getKey(), value)));
+                    errorMessages.add(
+                            ErrorMessage.createIllegalQuery(String.format("%s=%d outside configured predicate bounds.", entry.getKey(), value)));
                 }
             }
         }
 
-        private Index getIndexFromUnionOfDocumentTypes(PredicateQueryItem item) {
-            return indexFacts.getIndex(item.getIndexName());
+        private void visit(SimpleIndexedItem item) {
+            String indexName = item.getIndexName();
+            Index index = getIndexFromUnionOfDocumentTypes(indexName);
+            if (index.isPredicate()) {
+                errorMessages.add(
+                        ErrorMessage.createIllegalQuery(String.format("Index '%s' is predicate attribute and can only be used in conjunction with a predicate query operator.", indexName)));
+            }
+        }
+
+        private Index getIndexFromUnionOfDocumentTypes(String indexName) {
+            return indexFacts.getIndex(indexName);
         }
 
         @Override
diff --git a/container-search/src/main/java/com/yahoo/prelude/searcher/ValidateSortingSearcher.java b/container-search/src/main/java/com/yahoo/prelude/searcher/ValidateSortingSearcher.java
index bbdb3b796a2..82148cf54e6 100644
--- a/container-search/src/main/java/com/yahoo/prelude/searcher/ValidateSortingSearcher.java
+++ b/container-search/src/main/java/com/yahoo/prelude/searcher/ValidateSortingSearcher.java
@@ -119,6 +119,9 @@ public class ValidateSortingSearcher extends Searcher {
             String name = f.getFieldName();
             if ("[rank]".equals(name) || "[docid]".equals(name)) {
                 // built-in constants - ok
+            } else if ("[relevance]".equals(name)) {
+                // built-in constant '[relevance]' must map to '[rank]'
+                f.getSorter().setName("[rank]");
             } else if ("[relevancy]".equals(name)) {
                 // built-in constant '[relevancy]' must map to '[rank]'
                 f.getSorter().setName("[rank]");
diff --git a/container-search/src/main/java/com/yahoo/search/Query.java b/container-search/src/main/java/com/yahoo/search/Query.java
index 395d8853603..dc8e2b70740 100644
--- a/container-search/src/main/java/com/yahoo/search/Query.java
+++ b/container-search/src/main/java/com/yahoo/search/Query.java
@@ -288,7 +288,6 @@ public class Query extends com.yahoo.processing.Request implements Cloneable {
         this("");
     }
 
-
     /**
      * Construct a query from a string formatted in the http style, e.g <code>?query=test&amp;offset=10&amp;hits=13</code>
      * The query must be uri encoded.
@@ -297,7 +296,6 @@ public class Query extends com.yahoo.processing.Request implements Cloneable {
         this(query, null);
     }
 
-
     /**
      * Creates a query from a request
      *
@@ -665,7 +663,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable {
         // getQueryTree isn't exception safe
         try {
             queryTree = model.getQueryTree().toString();
-        } catch (Exception e) {
+        } catch (Exception | StackOverflowError e) {
             queryTree = "[Could not parse user input: " + model.getQueryString() + "]";
         }
         return "query '" + queryTree + "'";
@@ -677,7 +675,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable {
         // getQueryTree isn't exception safe
         try {
             queryTree = model.getQueryTree().toString();
-        } catch (Exception e) {
+        } catch (Exception | StackOverflowError e) {
             queryTree = "Could not parse user input: " + model.getQueryString();
         }
         return "query=[" + queryTree + "]" + " offset=" + getOffset() + " hits=" + getHits() + "]";
diff --git a/container-search/src/main/java/com/yahoo/search/Result.java b/container-search/src/main/java/com/yahoo/search/Result.java
index 4080b09f40b..ab48d5797b2 100644
--- a/container-search/src/main/java/com/yahoo/search/Result.java
+++ b/container-search/src/main/java/com/yahoo/search/Result.java
@@ -89,7 +89,6 @@ public final class Result extends com.yahoo.processing.Response implements Clone
      * with a result. It should <b>always</b> be called when adding
      * hits from a result, but there is no constraints on the order of the calls.
      */
-    @SuppressWarnings("deprecation")
     public void mergeWith(Result result) {
         totalHitCount += result.getTotalHitCount();
         deepHitCount += result.getDeepHitCount();
diff --git a/container-search/src/main/java/com/yahoo/search/cluster/BaseNodeMonitor.java b/container-search/src/main/java/com/yahoo/search/cluster/BaseNodeMonitor.java
index dd01d895963..0d491d2f0c1 100644
--- a/container-search/src/main/java/com/yahoo/search/cluster/BaseNodeMonitor.java
+++ b/container-search/src/main/java/com/yahoo/search/cluster/BaseNodeMonitor.java
@@ -79,6 +79,8 @@ public abstract class BaseNodeMonitor<T> {
      */
     public abstract void responded();
 
+    /** @deprecated Not used */
+    @Deprecated // TODO: Remove on Vespa 8
     public boolean isIdle() {
         return (now()-respondedAt) >= configuration.getIdleLimit();
     }
diff --git a/container-search/src/main/java/com/yahoo/search/cluster/ClusterMonitor.java b/container-search/src/main/java/com/yahoo/search/cluster/ClusterMonitor.java
index d4b6279be89..15cf4995b77 100644
--- a/container-search/src/main/java/com/yahoo/search/cluster/ClusterMonitor.java
+++ b/container-search/src/main/java/com/yahoo/search/cluster/ClusterMonitor.java
@@ -38,6 +38,9 @@ public class ClusterMonitor<T> {
     /** A map from Node to corresponding MonitoredNode */
     private final Map<T, TrafficNodeMonitor<T>> nodeMonitors = Collections.synchronizedMap(new java.util.LinkedHashMap<>());
 
+    /** @deprecated It is not advised to start the monitoring thread in the constructor.
+     * Use ClusterMonitor(NodeManager manager, false) and explicit start(). */
+    @Deprecated
     public ClusterMonitor(NodeManager<T> manager) {
         this(manager, true);
     }
@@ -50,6 +53,12 @@ public class ClusterMonitor<T> {
         }
     }
 
+    public void start() {
+        if ( ! monitorThread.isAlive()) {
+            monitorThread.start();
+        }
+    }
+
     /** Returns the configuration of this cluster monitor */
     public MonitorConfiguration getConfiguration() { return configuration; }
 
@@ -92,7 +101,7 @@ public class ClusterMonitor<T> {
         Boolean wasWorking = monitor.isKnownWorking();
         monitor.responded();
         if (wasWorking != monitor.isKnownWorking())
-            nodeManager.working(monitor.getNode());
+            nodeManager.working(node);
     }
 
     /**
@@ -101,7 +110,7 @@ public class ClusterMonitor<T> {
     public void ping(Executor executor) {
         for (Iterator<BaseNodeMonitor<T>> i = nodeMonitorIterator(); i.hasNext() && !closed.get(); ) {
             BaseNodeMonitor<T> monitor= i.next();
-            nodeManager.ping(monitor.getNode(), executor); // Cause call to failed or responded
+            nodeManager.ping(this, monitor.getNode(), executor); // Cause call to failed or responded
         }
         if (closed.get()) return; // Do nothing to change state if close has started.
         nodeManager.pingIterationCompleted();
diff --git a/container-search/src/main/java/com/yahoo/search/cluster/ClusterSearcher.java b/container-search/src/main/java/com/yahoo/search/cluster/ClusterSearcher.java
index 20f56c86f7b..645c6446ef1 100644
--- a/container-search/src/main/java/com/yahoo/search/cluster/ClusterSearcher.java
+++ b/container-search/src/main/java/com/yahoo/search/cluster/ClusterSearcher.java
@@ -58,7 +58,7 @@ public abstract class ClusterSearcher<T> extends PingableSearcher implements Nod
         this(id, connections, hasher, internal, true);
     }
 
-    public ClusterSearcher(ComponentId id, List<T> connections, Hasher<T> hasher, boolean internal, boolean startPingThread) {
+    protected ClusterSearcher(ComponentId id, List<T> connections, Hasher<T> hasher, boolean internal, boolean startPingThread) {
         super(id);
         this.hasher = hasher;
         this.monitor = new ClusterMonitor<>(this, startPingThread);
@@ -70,7 +70,7 @@ public abstract class ClusterSearcher<T> extends PingableSearcher implements Nod
 
     /** Pinging a node, called from ClusterMonitor */
     @Override
-    public final void ping(T p, Executor executor) {
+    public final void ping(ClusterMonitor<T> clusterMonitor, T p, Executor executor) {
         log(LogLevel.FINE, "Sending ping to: ", p);
         Pinger pinger = new Pinger(p);
         FutureTask<Pong> future = new FutureTask<>(pinger);
@@ -80,7 +80,7 @@ public abstract class ClusterSearcher<T> extends PingableSearcher implements Nod
         Throwable logThrowable = null;
 
         try {
-            pong = future.get(monitor.getConfiguration().getFailLimit(), TimeUnit.MILLISECONDS);
+            pong = future.get(clusterMonitor.getConfiguration().getFailLimit(), TimeUnit.MILLISECONDS);
         } catch (InterruptedException e) {
             pong = new Pong(ErrorMessage.createUnspecifiedError("Ping was interrupted: " + p));
             logThrowable = e;
@@ -96,10 +96,10 @@ public abstract class ClusterSearcher<T> extends PingableSearcher implements Nod
         future.cancel(true);
 
         if (pong.badResponse()) {
-            monitor.failed(p, pong.error().get());
+            clusterMonitor.failed(p, pong.error().get());
             log(LogLevel.FINE, "Failed ping - ", pong);
         } else {
-            monitor.responded(p);
+            clusterMonitor.responded(p);
             log(LogLevel.FINE, "Answered ping - ", p);
         }
 
@@ -220,14 +220,6 @@ public abstract class ClusterSearcher<T> extends PingableSearcher implements Nod
         if (result == null)
             result = new Result(query, ErrorMessage.createBackendCommunicationError("No result returned in " + this +
                                                                                     " from " + connection + " for " + query));
-
-        if (result.hits().getError() != null) {
-            log(LogLevel.FINE, "FAILED: ", query);
-        } else if ( ! result.isCached()) {
-            log(LogLevel.FINE, "WORKING: ", query);
-        } else {
-            log(LogLevel.FINE, "CACHE HIT: ", query);
-        }
         return result;
     }
 
@@ -263,13 +255,6 @@ public abstract class ClusterSearcher<T> extends PingableSearcher implements Nod
             result.hits().addError(ErrorMessage.createBackendCommunicationError("Error filling " + result + " from " + connection + ": " +
                                                                                 Exceptions.toMessageString(e)));
         }
-        if (result.hits().getError() != null) {
-            log(LogLevel.FINE, "FAILED: ", result.getQuery());
-        } else if ( ! result.isCached()) {
-            log(LogLevel.FINE, "WORKING: ", result.getQuery());
-        } else {
-            log(LogLevel.FINE, "CACHE HIT: " + result.getQuery());
-        }
     }
 
     /**
diff --git a/container-search/src/main/java/com/yahoo/search/cluster/Hasher.java b/container-search/src/main/java/com/yahoo/search/cluster/Hasher.java
index 46752b0bedb..6c83e1c64e3 100644
--- a/container-search/src/main/java/com/yahoo/search/cluster/Hasher.java
+++ b/container-search/src/main/java/com/yahoo/search/cluster/Hasher.java
@@ -11,6 +11,7 @@ package com.yahoo.search.cluster;
 public class Hasher<T> {
 
     public static class NodeFactor<T> {
+
         private final T node;
 
         /**
diff --git a/container-search/src/main/java/com/yahoo/search/cluster/MonitorConfiguration.java b/container-search/src/main/java/com/yahoo/search/cluster/MonitorConfiguration.java
index 226e0180d2e..a2fb982e3c5 100644
--- a/container-search/src/main/java/com/yahoo/search/cluster/MonitorConfiguration.java
+++ b/container-search/src/main/java/com/yahoo/search/cluster/MonitorConfiguration.java
@@ -8,85 +8,45 @@ package com.yahoo.search.cluster;
  */
 public class MonitorConfiguration  {
 
-    /**
-     * The interval in ms between consecutive checks of the monitored
-     * nodes
-     */
+    /** The interval in ms between consecutive checks of the monitored nodes */
     private long checkInterval=1000;
 
-    /**
-     * The number of times a failed node must respond before getting
-     * traffic again
-     */
-    private int responseAfterFailLimit=3;
+    /** The number of milliseconds to attempt to complete a request before giving up */
+    private final long requestTimeout = 980;
 
-    /**
-     * The number of ms a node is allowed to stay idle before it is
-     * pinged
-     */
-    private long idleLimit=3000;
+    /** The number of milliseconds a node is allowed to fail before we mark it as not working */
+    private long failLimit = 5000;
 
-    /**
-     * The number of milliseconds to attempt to complete a request
-     * before giving up
-     */
-    private long requestTimeout = 5000;
+    /** Sets the interval between each ping of idle or failing nodes. Default is 1000 ms. */
+    public void setCheckInterval(long intervalMs) { this.checkInterval = intervalMs; }
 
-    /**
-     * The number of milliseconds a node is allowed to fail before we
-     * mark it as not working
-     */
-    private long failLimit=5000;
+    /** Returns the interval between each ping of idle or failing nodes. Default is 1000 ms. */
+    public long getCheckInterval() { return checkInterval; }
 
     /**
-     * The number of times a node is allowed to fail in one hour
-     * before it is quarantined for an hour
+     * Sets the number of times a failed node must respond before it is put  back in service. Default is 3.
+     *
+     * @deprecated will go away in Vespa 8
      */
-    private int failQuarantineLimit=3;
+    @Deprecated // TODO: Remove on Vespa 8
+    public void setResponseAfterFailLimit(int responseAfterFailLimit) { }
 
     /**
-     * The number of ms to quarantine an unstable node
+     * Sets the number of ms a node (failing or working) is allowed to stay idle before it is pinged. Default is 3000.
+     *
+     * @deprecated Will go away in Vespa 8
      */
-    private long quarantineTime=1000*60*60;
-
-    /**
-     * Sets the interval between each ping of idle or failing nodes
-     * Default is 1000ms
-     */
-    public void setCheckInterval(long intervalMs) {
-        this.checkInterval=intervalMs;
-    }
+    @Deprecated // TODO: Remove on Vespa 8
+    public void setIdleLimit(int idleLimit) { }
 
     /**
-     * Returns the interval between each ping of idle or failing nodes
-     * Default is 1000ms
-     */
-    public long getCheckInterval() {
-        return checkInterval;
-    }
-
-    /**
-     * Sets the number of times a failed node must respond before it is put
-     * back in service. Default is 3.
-     */
-    public void setResponseAfterFailLimit(int responseAfterFailLimit) {
-        this.responseAfterFailLimit=responseAfterFailLimit;
-    }
-
-    /**
-     * Sets the number of ms a node (failing or working) is allowed to
-     * stay idle before it is pinged. Default is 3000
-     */
-    public void setIdleLimit(int idleLimit) {
-        this.idleLimit=idleLimit;
-    }
-
-    /**
-     * Gets the number of ms a node (failing or working)
-     * is allowed to stay idle before it is pinged. Default is 3000
+     * Gets the number of ms a node (failing or working) is allowed to stay idle before it is pinged. Default is 3000.
+     *
+     * @deprecated Will go away in Vespa 8
      */
+    @Deprecated // TODO: Remove on Vespa 8
     public long getIdleLimit() {
-        return idleLimit;
+        return 3000;
     }
 
     /**
@@ -112,29 +72,26 @@ public class MonitorConfiguration  {
      * in quarantine.  Once in quarantine it won't be put back in
      * productuion before quarantineTime has expired even if it is
      * working. Default is 3
+     *
+     * @deprecated Will go away in Vespa 8
      */
-    public void setFailQuarantineLimit(int failQuarantineLimit) {
-        this.failQuarantineLimit=failQuarantineLimit;
-    }
+    @Deprecated // TODO: Remove on Vespa 8
+    public void setFailQuarantineLimit(int failQuarantineLimit) { }
 
     /**
-     * The number of ms an unstable node is quarantined. Default is
-     * 100*60*60
+     * The number of ms an unstable node is quarantined. Default is 100*60*60
+     *
+     * @deprecated Will go away in Vespa 8
      */
-    public void setQuarantineTime(long quarantineTime) {
-        this.quarantineTime=quarantineTime;
-    }
+    @Deprecated // TODO: Remove on Vespa 8
+    public void setQuarantineTime(long quarantineTime) { }
 
     public String toString() {
         return "monitor configuration [" +
-            "checkInterval: " + checkInterval +
-            " responseAfterFailLimit: " + responseAfterFailLimit +
-            " idleLimit: " + idleLimit +
-            " requestTimeout " + requestTimeout +
-            " feilLimit " + failLimit +
-            " failQuerantineLimit " + failQuarantineLimit +
-            " quarantineTime " + quarantineTime +
-            "]";
+               "checkInterval: " + checkInterval +
+               " requestTimeout " + requestTimeout +
+               " failLimit " + failLimit +
+               "]";
     }
 
 }
diff --git a/container-search/src/main/java/com/yahoo/search/cluster/NodeManager.java b/container-search/src/main/java/com/yahoo/search/cluster/NodeManager.java
index 9b20139e3c5..836c71089c1 100644
--- a/container-search/src/main/java/com/yahoo/search/cluster/NodeManager.java
+++ b/container-search/src/main/java/com/yahoo/search/cluster/NodeManager.java
@@ -7,7 +7,7 @@ import java.util.concurrent.Executor;
  * Must be implemented by a node collection which wants
  * it's node state monitored by a ClusterMonitor
  *
- * @author  bratseth
+ * @author bratseth
  */
 public interface NodeManager<T> {
 
@@ -19,9 +19,22 @@ public interface NodeManager<T> {
 
     /** 
      * Called when a node should be pinged. 
-     * This *must* lead to either a call to NodeMonitor.failed or NodeMonitor.responded 
+     * This *must* lead to either a call to NodeMonitor.failed or NodeMonitor.responded
+     *
+     * @deprecated Use ping(ClusterMonitor clusterMonitor, T node, Executor executor) instead.
      */
-    void ping(T node, Executor executor);
+    @Deprecated // TODO: Remove on Vespa 8
+    default void ping(T node, Executor executor) {
+        throw new IllegalStateException("If you have not overrriden ping(ClusterMonitor<T> clusterMonitor, T node, Executor executor), you should at least have overriden this method.");
+    }
+
+    /**
+     * Called when a node should be pinged.
+     * This *must* lead to either a call to ClusterMonitor.failed or ClusterMonitor.responded
+     */
+    default void ping(ClusterMonitor<T> clusterMonitor, T node, Executor executor) {
+        ping(node, executor);
+    }
     
     /** Called right after a ping has been issued to each node. This default implementation does nothing. */
     default void pingIterationCompleted() {}
diff --git a/container-search/src/main/java/com/yahoo/search/cluster/TrafficNodeMonitor.java b/container-search/src/main/java/com/yahoo/search/cluster/TrafficNodeMonitor.java
index ccf3e863ff3..d17f6bfbaa8 100644
--- a/container-search/src/main/java/com/yahoo/search/cluster/TrafficNodeMonitor.java
+++ b/container-search/src/main/java/com/yahoo/search/cluster/TrafficNodeMonitor.java
@@ -52,8 +52,8 @@ public class TrafficNodeMonitor<T> extends BaseNodeMonitor<T> {
      * Called when a response is received from this node.
      */
     public void responded() {
-        respondedAt=now();
-        succeededAt=respondedAt;
+        respondedAt = now();
+        succeededAt = respondedAt;
 
         setWorking(true,"Responds correctly");
     }
@@ -69,20 +69,20 @@ public class TrafficNodeMonitor<T> extends BaseNodeMonitor<T> {
         atStartUp = false;
         if (this.isWorking == working) return; // Old news
 
-        if (explanation==null) {
-            explanation="";
+        if (explanation == null) {
+            explanation = "";
         } else {
-            explanation=": " + explanation;
+            explanation = ": " + explanation;
         }
 
         if (working) {
             log.info("Putting " + node + " in service" + explanation);
         } else {
             log.warning("Taking " + node + " out of service" + explanation);
-            failedAt=now();
+            failedAt = now();
         }
 
-        this.isWorking=working;
+        this.isWorking = working;
     }
 
 }
diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java
index 03b51fbaf70..626cf087aca 100644
--- a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java
+++ b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java
@@ -5,18 +5,20 @@ import com.google.inject.Inject;
 import com.yahoo.cloud.config.ClusterInfoConfig;
 import com.yahoo.component.AbstractComponent;
 import com.yahoo.component.ComponentId;
+import com.yahoo.compress.Compressor;
 import com.yahoo.container.handler.VipStatus;
 import com.yahoo.jdisc.Metric;
 import com.yahoo.prelude.fastsearch.VespaBackEndSearcher;
 import com.yahoo.processing.request.CompoundName;
 import com.yahoo.search.Query;
 import com.yahoo.search.Result;
+import com.yahoo.search.cluster.ClusterMonitor;
 import com.yahoo.search.dispatch.SearchPath.InvalidSearchPathException;
 import com.yahoo.search.dispatch.rpc.RpcInvokerFactory;
+import com.yahoo.search.dispatch.rpc.RpcPingFactory;
 import com.yahoo.search.dispatch.rpc.RpcResourcePool;
 import com.yahoo.search.dispatch.searchcluster.Group;
 import com.yahoo.search.dispatch.searchcluster.Node;
-import com.yahoo.search.dispatch.searchcluster.PingFactory;
 import com.yahoo.search.dispatch.searchcluster.SearchCluster;
 import com.yahoo.search.query.profile.types.FieldDescription;
 import com.yahoo.search.query.profile.types.FieldType;
@@ -30,6 +32,7 @@ import java.util.List;
 import java.util.Optional;
 import java.util.OptionalInt;
 import java.util.Set;
+import java.util.stream.Collectors;
 
 /**
  * A dispatcher communicates with search nodes to perform queries and fill hits.
@@ -48,6 +51,7 @@ public class Dispatcher extends AbstractComponent {
     public static final String DISPATCH = "dispatch";
     private static final String INTERNAL = "internal";
     private static final String PROTOBUF = "protobuf";
+    private static final String TOP_K_PROBABILITY = "topKProbability";
 
     private static final String INTERNAL_METRIC = "dispatch_internal";
 
@@ -56,8 +60,12 @@ public class Dispatcher extends AbstractComponent {
     /** If enabled, search queries will use protobuf rpc */
     public static final CompoundName dispatchProtobuf = CompoundName.fromComponents(DISPATCH, PROTOBUF);
 
+    /** If set will control computation of how many hits will be fetched from each partition.*/
+    public static final CompoundName topKProbability = CompoundName.fromComponents(DISPATCH, TOP_K_PROBABILITY);
+
     /** A model of the search cluster this dispatches to */
     private final SearchCluster searchCluster;
+    private final ClusterMonitor clusterMonitor;
 
     private final LoadBalancer loadBalancer;
 
@@ -76,55 +84,77 @@ public class Dispatcher extends AbstractComponent {
         argumentType.setBuiltin(true);
         argumentType.addField(new FieldDescription(INTERNAL, FieldType.booleanType));
         argumentType.addField(new FieldDescription(PROTOBUF, FieldType.booleanType));
+        argumentType.addField(new FieldDescription(TOP_K_PROBABILITY, FieldType.doubleType));
         argumentType.freeze();
     }
 
     public static QueryProfileType getArgumentType() { return argumentType; }
 
     @Inject
-    public Dispatcher(ComponentId clusterId,
+    public Dispatcher(RpcResourcePool resourcePool,
+                      ComponentId clusterId,
                       DispatchConfig dispatchConfig,
                       ClusterInfoConfig clusterInfoConfig,
                       VipStatus vipStatus,
                       Metric metric) {
-        this(new SearchCluster(clusterId.stringValue(), dispatchConfig, clusterInfoConfig.nodeCount(), vipStatus),
-             dispatchConfig,
-             metric);
-    }
+        this(resourcePool, new SearchCluster(clusterId.stringValue(), dispatchConfig,clusterInfoConfig.nodeCount(),
+                                             vipStatus, new RpcPingFactory(resourcePool)),
+             dispatchConfig, metric);
 
-    private Dispatcher(SearchCluster searchCluster, DispatchConfig dispatchConfig, Metric metric) {
-        this(searchCluster,
-             dispatchConfig,
-             new RpcInvokerFactory(new RpcResourcePool(dispatchConfig), searchCluster),
-             metric);
     }
 
-    /* Protected for simple mocking in tests. Beware that searchCluster is shutdown on in deconstruct() */
-    protected Dispatcher(SearchCluster searchCluster,
-                         DispatchConfig dispatchConfig,
-                         RpcInvokerFactory rcpInvokerFactory,
-                         Metric metric) {
-        this(searchCluster, dispatchConfig, rcpInvokerFactory, rcpInvokerFactory, metric);
+    private Dispatcher(RpcResourcePool resourcePool, SearchCluster searchCluster, DispatchConfig dispatchConfig, Metric metric) {
+        this(new ClusterMonitor<>(searchCluster, true), searchCluster, dispatchConfig, new RpcInvokerFactory(resourcePool, searchCluster), metric);
     }
 
     /* Protected for simple mocking in tests. Beware that searchCluster is shutdown on in deconstruct() */
-    protected Dispatcher(SearchCluster searchCluster,
+    protected Dispatcher(ClusterMonitor clusterMonitor,
+                         SearchCluster searchCluster,
                          DispatchConfig dispatchConfig,
                          InvokerFactory invokerFactory,
-                         PingFactory pingFactory,
                          Metric metric) {
         if (dispatchConfig.useMultilevelDispatch())
             throw new IllegalArgumentException(searchCluster + " is configured with multilevel dispatch, but this is not supported");
 
         this.searchCluster = searchCluster;
+        this.clusterMonitor = clusterMonitor;
         this.loadBalancer = new LoadBalancer(searchCluster,
                                   dispatchConfig.distributionPolicy() == DispatchConfig.DistributionPolicy.ROUNDROBIN);
         this.invokerFactory = invokerFactory;
         this.metric = metric;
         this.metricContext = metric.createContext(null);
         this.maxHitsPerNode = dispatchConfig.maxHitsPerNode();
+        searchCluster.addMonitoring(clusterMonitor);
+        Thread warmup = new Thread(new Runnable() {
+            @Override
+            public void run() {
+                warmup(dispatchConfig.warmuptime());
+            }
+        });
+        warmup.start();
+        try {
+            while ( ! searchCluster.hasInformationAboutAllNodes()) {
+                Thread.sleep(1);
+            }
+            warmup.join();
+        } catch (InterruptedException e) {}
+
+        /*
+         * No we have information from all nodes and a ping iteration has completed.
+         * Instead of waiting until next ping interval to update coverage and group state,
+         * we should compute the state ourselves, so that when the dispatcher is ready the state
+         * of its groups are also known.
+         */
+        searchCluster.pingIterationCompleted();
+    }
 
-        searchCluster.startClusterMonitoring(pingFactory);
+    /*
+     Will run important code in order to trigger JIT compilation and avoid cold start issues.
+     Currently warms up lz4 compression code.
+     */
+
+    private static long warmup(double seconds) {
+        return new Compressor().warmup(seconds);
     }
 
     /** Returns the search cluster this dispatches to */
@@ -134,8 +164,8 @@ public class Dispatcher extends AbstractComponent {
 
     @Override
     public void deconstruct() {
-        /* The seach cluster must be shutdown first as it uses the invokerfactory. */
-        searchCluster.shutDown();
+        /* The clustermonitor must be shutdown first as it uses the invokerfactory through the searchCluster. */
+        clusterMonitor.shutdown();
         invokerFactory.release();
     }
 
@@ -191,7 +221,7 @@ public class Dispatcher extends AbstractComponent {
         int covered = searchCluster.groupsWithSufficientCoverage();
         int groups = searchCluster.orderedGroups().size();
         int max = Integer.min(Integer.min(covered + 1, groups), MAX_GROUP_SELECTION_ATTEMPTS);
-        Set<Integer> rejected = null;
+        Set<Integer> rejected = rejectGroupBlockingFeed(searchCluster.orderedGroups());
         for (int i = 0; i < max; i++) {
             Optional<Group> groupInCluster = loadBalancer.takeGroup(rejected);
             if (groupInCluster.isEmpty()) break; // No groups available
@@ -220,4 +250,21 @@ public class Dispatcher extends AbstractComponent {
         throw new IllegalStateException("No suitable groups to dispatch query. Rejected: " + rejected);
     }
 
+    /**
+     * We want to avoid groups blocking feed because their data may be out of date.
+     * If there is a single group blocking feed, we want to reject it.
+     * If multiple groups are blocking feed we should use them anyway as we may not have remaining
+     * capacity otherwise. Same if there are no other groups.
+     *
+     * @return a modifiable set containing the single group to reject, or null otherwise
+     */
+    private Set<Integer> rejectGroupBlockingFeed(List<Group> groups) {
+        if (groups.size() == 1) return null;
+        List<Group> groupsRejectingFeed = groups.stream().filter(Group::isBlockingWrites).collect(Collectors.toList());
+        if (groupsRejectingFeed.size() != 1) return null;
+        Set<Integer> rejected = new HashSet<>();
+        rejected.add(groupsRejectingFeed.get(0).id());
+        return rejected;
+    }
+
 }
diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/InterleavedSearchInvoker.java b/container-search/src/main/java/com/yahoo/search/dispatch/InterleavedSearchInvoker.java
index cec3e94d551..e62848a7f9e 100644
--- a/container-search/src/main/java/com/yahoo/search/dispatch/InterleavedSearchInvoker.java
+++ b/container-search/src/main/java/com/yahoo/search/dispatch/InterleavedSearchInvoker.java
@@ -81,7 +81,12 @@ public class InterleavedSearchInvoker extends SearchInvoker implements ResponseM
 
         int originalHits = query.getHits();
         int originalOffset = query.getOffset();
-        query.setHits(query.getHits() + query.getOffset());
+        int neededHits = originalHits + originalOffset;
+        Double topkProbabilityOverrride = query.properties().getDouble(Dispatcher.topKProbability);
+        int q = (topkProbabilityOverrride != null)
+                ? searchCluster.estimateHitsToFetch(neededHits, invokers.size(), topkProbabilityOverrride)
+                : searchCluster.estimateHitsToFetch(neededHits, invokers.size());
+        query.setHits(q);
         query.setOffset(0);
 
         for (SearchInvoker invoker : invokers) {
@@ -321,4 +326,7 @@ public class InterleavedSearchInvoker extends SearchInvoker implements ResponseM
     protected LinkedBlockingQueue<SearchInvoker> newQueue() {
         return new LinkedBlockingQueue<>();
     }
+
+    // For testing
+    Collection<SearchInvoker> invokers() { return invokers; }
 }
diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/InvokerFactory.java b/container-search/src/main/java/com/yahoo/search/dispatch/InvokerFactory.java
index 1c3a90ac6ab..03160e6c9c7 100644
--- a/container-search/src/main/java/com/yahoo/search/dispatch/InvokerFactory.java
+++ b/container-search/src/main/java/com/yahoo/search/dispatch/InvokerFactory.java
@@ -43,7 +43,7 @@ public abstract class InvokerFactory {
      * @param nodes pre-selected list of content nodes
      * @param acceptIncompleteCoverage if some of the nodes are unavailable and this parameter is
      *                                 false, verify that the remaining set of nodes has sufficient coverage
-     * @return Optional containing the SearchInvoker or empty if some node in the
+     * @return the invoker or empty if some node in the
      *         list is invalid and the remaining coverage is not sufficient
      */
     public Optional<SearchInvoker> createSearchInvoker(VespaBackEndSearcher searcher,
@@ -82,7 +82,7 @@ public abstract class InvokerFactory {
             if ( ! searchCluster.isPartialGroupCoverageSufficient(groupId, success) && !acceptIncompleteCoverage) {
                 return Optional.empty();
             }
-            if(invokers.size() == 0) {
+            if (invokers.size() == 0) {
                 return Optional.of(createCoverageErrorInvoker(nodes, failed));
             }
         }
diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/LeanHit.java b/container-search/src/main/java/com/yahoo/search/dispatch/LeanHit.java
index 86f1999d8b4..8a90557fa3b 100644
--- a/container-search/src/main/java/com/yahoo/search/dispatch/LeanHit.java
+++ b/container-search/src/main/java/com/yahoo/search/dispatch/LeanHit.java
@@ -12,15 +12,11 @@ public class LeanHit implements Comparable<LeanHit> {
     private final int distributionKey;
 
     public LeanHit(byte [] gid, int partId, int distributionKey, double relevance) {
-        this.gid = gid;
-        this.relevance = Double.isNaN(relevance) ? Double.NEGATIVE_INFINITY : relevance;
-        this.sortData = null;
-        this.partId = partId;
-        this.distributionKey = distributionKey;
+        this(gid, partId, distributionKey, relevance, null);
     }
-    public LeanHit(byte [] gid, int partId, int distributionKey, byte [] sortData) {
+    public LeanHit(byte [] gid, int partId, int distributionKey, double relevance, byte [] sortData) {
         this.gid = gid;
-        this.relevance = 0.0;
+        this.relevance = Double.isNaN(relevance) ? Double.NEGATIVE_INFINITY : relevance;
         this.sortData = sortData;
         this.partId = partId;
         this.distributionKey = distributionKey;
diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/LoadBalancer.java b/container-search/src/main/java/com/yahoo/search/dispatch/LoadBalancer.java
index 210ab5777d2..05e1ea6e2f9 100644
--- a/container-search/src/main/java/com/yahoo/search/dispatch/LoadBalancer.java
+++ b/container-search/src/main/java/com/yahoo/search/dispatch/LoadBalancer.java
@@ -134,6 +134,7 @@ public class LoadBalancer {
     }
 
     private static class RoundRobinScheduler implements GroupScheduler {
+
         private int needle = 0;
         private final List<GroupStatus> scoreboard;
 
@@ -204,6 +205,7 @@ public class LoadBalancer {
     }
 
     static class AdaptiveScheduler implements GroupScheduler {
+
         private final Random random;
         private final List<GroupStatus> scoreboard;
 
@@ -251,4 +253,5 @@ public class LoadBalancer {
             return selectGroup(needle, false, rejectedGroups);
         }
     }
+
 }
diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/TopKEstimator.java b/container-search/src/main/java/com/yahoo/search/dispatch/TopKEstimator.java
new file mode 100644
index 00000000000..8003d9c6744
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/dispatch/TopKEstimator.java
@@ -0,0 +1,42 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.dispatch;
+
+import org.apache.commons.math3.distribution.TDistribution;
+
+/**
+ * Use StudentT distribution and estimate how many hits you need from each partition
+ * to to get the globally top-k documents with the desired probability
+ * @author baldersheim
+ */
+public class TopKEstimator {
+    private final TDistribution studentT;
+    private final double defaultP;
+    private final boolean estimate;
+
+    private static boolean needEstimate(double p) {
+        return (0.0 < p) && (p < 1.0);
+    }
+    public TopKEstimator(double freedom, double defaultProbability) {
+        this.studentT = new TDistribution(null, freedom);
+        defaultP = defaultProbability;
+        estimate = needEstimate(defaultP);
+    }
+    double estimateExactK(double k, double n, double p) {
+        double variance = k * 1/n * (1 - 1/n);
+        double p_inverse = 1 - (1 - p)/n;
+        return k/n + studentT.inverseCumulativeProbability(p_inverse) * Math.sqrt(variance);
+    }
+    double estimateExactK(double k, double n) {
+        return estimateExactK(k, n, defaultP);
+    }
+    public int estimateK(int k, int n) {
+        return (estimate && n > 1)
+                ? (int)Math.ceil(estimateExactK(k, n, defaultP))
+                : k;
+    }
+    public int estimateK(int k, int n, double p) {
+        return (needEstimate(p) && (n > 1))
+                ? (int)Math.ceil(estimateExactK(k, n, p))
+                : k;
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/ProtobufSerialization.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/ProtobufSerialization.java
index ae2258c4546..51290c245ac 100644
--- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/ProtobufSerialization.java
+++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/ProtobufSerialization.java
@@ -214,7 +214,7 @@ public class ProtobufSerialization {
         for (var replyHit : protobuf.getHitsList()) {
             LeanHit hit = (replyHit.getSortData().isEmpty())
                     ? new LeanHit(replyHit.getGlobalId().toByteArray(), partId, distKey, replyHit.getRelevance())
-                    : new LeanHit(replyHit.getGlobalId().toByteArray(), partId, distKey, replyHit.getSortData().toByteArray());
+                    : new LeanHit(replyHit.getGlobalId().toByteArray(), partId, distKey, replyHit.getRelevance(), replyHit.getSortData().toByteArray());
             result.getLeanHits().add(hit);
         }
 
diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcClient.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcClient.java
index a2821892358..52cb2b4c061 100644
--- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcClient.java
+++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcClient.java
@@ -44,13 +44,14 @@ class RpcClient implements Client {
 
         // The current shared connection. This will be recycled when it becomes invalid.
         // All access to this must be synchronized
-        private Target target = null;
+        private Target target;
 
         public RpcNodeConnection(String hostname, int port, Supervisor supervisor) {
             this.supervisor = supervisor;
             this.hostname = hostname;
             this.port = port;
             description = "rpc node connection to " + hostname + ":" + port;
+            target = supervisor.connect(new Spec(hostname, port));
         }
 
         @Override
@@ -79,17 +80,16 @@ class RpcClient implements Client {
         private void invokeAsync(Request req, double timeout, RequestWaiter waiter) {
             // TODO: Consider replacing this by a watcher on the target
             synchronized(this) { // ensure we have exactly 1 valid connection across threads
-                if (target == null || ! target.isValid())
+                if (! target.isValid()) {
                     target = supervisor.connect(new Spec(hostname, port));
+                }
             }
             target.invokeAsync(req, timeout, waiter);
         }
 
         @Override
         public void close() {
-            if (target != null) {
-                target.close();
-            }
+            target.close();
         }
 
         @Override
diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcInvokerFactory.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcInvokerFactory.java
index 5c9928de924..74bc9e8bfbb 100644
--- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcInvokerFactory.java
+++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcInvokerFactory.java
@@ -1,28 +1,24 @@
 // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.search.dispatch.rpc;
 
-import com.yahoo.prelude.Pong;
 import com.yahoo.prelude.fastsearch.DocumentDatabase;
 import com.yahoo.prelude.fastsearch.VespaBackEndSearcher;
 import com.yahoo.processing.request.CompoundName;
 import com.yahoo.search.Query;
 import com.yahoo.search.Result;
-import com.yahoo.search.cluster.ClusterMonitor;
 import com.yahoo.search.dispatch.Dispatcher;
 import com.yahoo.search.dispatch.FillInvoker;
 import com.yahoo.search.dispatch.InvokerFactory;
 import com.yahoo.search.dispatch.SearchInvoker;
 import com.yahoo.search.dispatch.searchcluster.Node;
-import com.yahoo.search.dispatch.searchcluster.PingFactory;
 import com.yahoo.search.dispatch.searchcluster.SearchCluster;
 
 import java.util.Optional;
-import java.util.concurrent.Callable;
 
 /**
  * @author ollivir
  */
-public class RpcInvokerFactory extends InvokerFactory implements PingFactory {
+public class RpcInvokerFactory extends InvokerFactory {
 
     /** Unless turned off this will fill summaries by dispatching directly to search nodes over RPC when possible */
     private final static CompoundName dispatchSummaries = new CompoundName("dispatch.summaries");
@@ -60,12 +56,4 @@ public class RpcInvokerFactory extends InvokerFactory implements PingFactory {
         return new RpcFillInvoker(rpcResourcePool, documentDb);
     }
 
-    public void release() {
-        rpcResourcePool.release();
-    }
-
-    @Override
-    public Callable<Pong> createPinger(Node node, ClusterMonitor<Node> monitor) {
-        return new RpcPing(node, monitor, rpcResourcePool);
-    }
 }
diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcPing.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcPing.java
index e0f1dc5e675..5e04f1d7a3e 100644
--- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcPing.java
+++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcPing.java
@@ -10,55 +10,63 @@ import com.yahoo.search.cluster.ClusterMonitor;
 import com.yahoo.search.dispatch.rpc.Client.ProtobufResponse;
 import com.yahoo.search.dispatch.rpc.Client.ResponseOrError;
 import com.yahoo.search.dispatch.searchcluster.Node;
+import com.yahoo.search.dispatch.searchcluster.Pinger;
+import com.yahoo.search.dispatch.searchcluster.PongHandler;
 import com.yahoo.search.result.ErrorMessage;
 import com.yahoo.yolean.Exceptions;
 
-import java.util.concurrent.Callable;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.TimeUnit;
+import java.util.logging.Logger;
 
-public class RpcPing implements Callable<Pong> {
+public class RpcPing implements Pinger, Client.ResponseReceiver {
 
+    private static final Logger log = Logger.getLogger(RpcPing.class.getName());
     private static final String RPC_METHOD = "vespa.searchprotocol.ping";
     private static final CompressionType PING_COMPRESSION = CompressionType.NONE;
 
     private final Node node;
     private final RpcResourcePool resourcePool;
     private final ClusterMonitor<Node> clusterMonitor;
+    private final long pingSequenceId;
+    private final PongHandler pongHandler;
 
-    public RpcPing(Node node, ClusterMonitor<Node> clusterMonitor, RpcResourcePool rpcResourcePool) {
+    public RpcPing(Node node, ClusterMonitor<Node> clusterMonitor, RpcResourcePool rpcResourcePool, PongHandler pongHandler) {
         this.node = node;
         this.resourcePool = rpcResourcePool;
         this.clusterMonitor = clusterMonitor;
+        pingSequenceId = node.createPingSequenceId();
+        this.pongHandler = pongHandler;
     }
 
     @Override
-    public Pong call() throws Exception {
+    public void ping() {
         try {
-            var queue = new LinkedBlockingQueue<ResponseOrError<ProtobufResponse>>(1);
-
-            sendPing(queue);
+            sendPing();
+        } catch (RuntimeException e) {
+            pongHandler.handle(new Pong(ErrorMessage.createBackendCommunicationError("Exception when pinging " + node +
+                                                                                 ": " + Exceptions.toMessageString(e))));
+        }
+    }
 
-            var responseOrError = queue.poll(clusterMonitor.getConfiguration().getRequestTimeout(), TimeUnit.MILLISECONDS);
-            if (responseOrError == null) {
-                return new Pong(ErrorMessage.createNoAnswerWhenPingingNode("Timed out waiting for pong from " + node));
-            } else if (responseOrError.error().isPresent()) {
-                return new Pong(ErrorMessage.createBackendCommunicationError(responseOrError.error().get()));
-            }
+    private Pong toPong(ResponseOrError<ProtobufResponse> responseOrError) {
+        if (responseOrError == null) {
+            return new Pong(ErrorMessage.createNoAnswerWhenPingingNode("Timed out waiting for pong from " + node));
+        } else if (responseOrError.error().isPresent()) {
+            return new Pong(ErrorMessage.createBackendCommunicationError(responseOrError.error().get()));
+        }
 
+        try {
             return decodeReply(responseOrError.response().get());
-        } catch (RuntimeException e) {
-            return new Pong(
-                    ErrorMessage.createBackendCommunicationError("Exception when pinging " + node + ": " + Exceptions.toMessageString(e)));
+        } catch (InvalidProtocolBufferException e) {
+            return new Pong(ErrorMessage.createBackendCommunicationError(e.getMessage()));
         }
     }
 
-    private void sendPing(LinkedBlockingQueue<ResponseOrError<ProtobufResponse>> queue) {
+    private void sendPing() {
         var connection = resourcePool.getConnection(node.key());
         var ping = SearchProtocol.MonitorRequest.newBuilder().build().toByteArray();
         double timeoutSeconds = ((double) clusterMonitor.getConfiguration().getRequestTimeout()) / 1000.0;
         Compressor.Compression compressionResult = resourcePool.compressor().compress(PING_COMPRESSION, ping);
-        connection.request(RPC_METHOD, compressionResult.type(), ping.length, compressionResult.data(), rsp -> queue.add(rsp), timeoutSeconds);
+        connection.request(RPC_METHOD, compressionResult.type(), ping.length, compressionResult.data(),this, timeoutSeconds);
     }
 
     private Pong decodeReply(ProtobufResponse response) throws InvalidProtocolBufferException {
@@ -76,4 +84,14 @@ public class RpcPing implements Callable<Pong> {
         }
     }
 
+    @Override
+    public void receive(ResponseOrError<ProtobufResponse> response) {
+        if (node.isLastReceivedPong(pingSequenceId)) {
+            pongHandler.handle(toPong(response));
+        } else {
+            //TODO Reduce to debug or remove once we have enumerated what happens here.
+            log.info("Pong " + pingSequenceId + " from node " + node.key() + " in group " + node.group() +
+                     " with hostname " + node.hostname() + " received too late, latest is " + node.getLastReceivedPongId());
+        }
+    }
 }
diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcPingFactory.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcPingFactory.java
new file mode 100644
index 00000000000..ac8f0a59c20
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcPingFactory.java
@@ -0,0 +1,18 @@
+package com.yahoo.search.dispatch.rpc;
+
+import com.yahoo.search.cluster.ClusterMonitor;
+import com.yahoo.search.dispatch.searchcluster.Node;
+import com.yahoo.search.dispatch.searchcluster.PingFactory;
+import com.yahoo.search.dispatch.searchcluster.Pinger;
+import com.yahoo.search.dispatch.searchcluster.PongHandler;
+
+public class RpcPingFactory implements PingFactory {
+    private final RpcResourcePool rpcResourcePool;
+    public RpcPingFactory(RpcResourcePool rpcResourcePool) {
+        this.rpcResourcePool = rpcResourcePool;
+    }
+    @Override
+    public Pinger createPinger(Node node, ClusterMonitor<Node> monitor, PongHandler pongHandler) {
+        return new RpcPing(node, monitor, rpcResourcePool, pongHandler);
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcResourcePool.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcResourcePool.java
index ca2a0c9bfb0..065489ef9a0 100644
--- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcResourcePool.java
+++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcResourcePool.java
@@ -2,6 +2,9 @@
 package com.yahoo.search.dispatch.rpc;
 
 import com.google.common.collect.ImmutableMap;
+import com.google.inject.Inject;
+import com.yahoo.component.AbstractComponent;
+import com.yahoo.component.ComponentId;
 import com.yahoo.compress.CompressionType;
 import com.yahoo.compress.Compressor;
 import com.yahoo.compress.Compressor.Compression;
@@ -23,7 +26,7 @@ import java.util.Random;
  *
  * @author ollivir
  */
-public class RpcResourcePool {
+public class RpcResourcePool extends AbstractComponent {
     /** The compression method which will be used with rpc dispatch. "lz4" (default) and "none" is supported. */
     public final static CompoundName dispatchCompression = new CompoundName("dispatch.compression");
 
@@ -33,13 +36,15 @@ public class RpcResourcePool {
     /** Connections to the search nodes this talks to, indexed by node id ("partid") */
     private final ImmutableMap<Integer, NodeConnectionPool> nodeConnectionPools;
 
-    public RpcResourcePool(Map<Integer, NodeConnection> nodeConnections) {
+    RpcResourcePool(Map<Integer, NodeConnection> nodeConnections) {
         var builder = new ImmutableMap.Builder<Integer, NodeConnectionPool>();
         nodeConnections.forEach((key, connection) -> builder.put(key, new NodeConnectionPool(Collections.singletonList(connection))));
         this.nodeConnectionPools = builder.build();
     }
 
+    @Inject
     public RpcResourcePool(DispatchConfig dispatchConfig) {
+        super();
         var client = new RpcClient(dispatchConfig.numJrtTransportThreads());
 
         // Create rpc node connection pools indexed by the node distribution key
@@ -73,7 +78,9 @@ public class RpcResourcePool {
         }
     }
 
-    public void release() {
+    @Override
+    public void deconstruct() {
+        super.deconstruct();
         nodeConnectionPools.values().forEach(NodeConnectionPool::release);
     }
 
diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcSearchInvoker.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcSearchInvoker.java
index 07d8439ff46..76240e55c98 100644
--- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcSearchInvoker.java
+++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcSearchInvoker.java
@@ -5,7 +5,6 @@ import com.yahoo.compress.CompressionType;
 import com.yahoo.compress.Compressor;
 import com.yahoo.prelude.fastsearch.VespaBackEndSearcher;
 import com.yahoo.search.Query;
-import com.yahoo.search.Result;
 import com.yahoo.search.dispatch.InvokerResult;
 import com.yahoo.search.dispatch.SearchInvoker;
 import com.yahoo.search.dispatch.rpc.Client.ProtobufResponse;
diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Group.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Group.java
index 0e4e87b9a6a..ec616a18e09 100644
--- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Group.java
+++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Group.java
@@ -21,6 +21,7 @@ public class Group {
     private final AtomicBoolean hasSufficientCoverage = new AtomicBoolean(true);
     private final AtomicBoolean hasFullCoverage = new AtomicBoolean(true);
     private final AtomicLong activeDocuments = new AtomicLong(0);
+    private final AtomicBoolean isBlockingWrites = new AtomicBoolean(false);
 
     public Group(int id, List<Node> nodes) {
         this.id = id;
@@ -61,21 +62,16 @@ public class Group {
         return nodesUp;
     }
 
-    void aggregateActiveDocuments() {
-        long activeDocumentsInGroup = 0;
-        for (Node node : nodes) {
-            if (node.isWorking() == Boolean.TRUE) {
-                activeDocumentsInGroup += node.getActiveDocuments();
-            }
-        }
-        activeDocuments.set(activeDocumentsInGroup);
-
+    void aggregateNodeValues() {
+        activeDocuments.set(nodes.stream().filter(node -> node.isWorking() == Boolean.TRUE).mapToLong(Node::getActiveDocuments).sum());
+        isBlockingWrites.set(nodes.stream().anyMatch(node -> node.isBlockingWrites()));
     }
 
     /** Returns the active documents on this node. If unknown, 0 is returned. */
-    long getActiveDocuments() {
-        return this.activeDocuments.get();
-    }
+    long getActiveDocuments() { return activeDocuments.get(); }
+
+    /** Returns whether any node in this group is currently blocking write operations */
+    public boolean isBlockingWrites() { return isBlockingWrites.get(); }
 
     public boolean isFullCoverageStatusChanged(boolean hasFullCoverageNow) {
         boolean previousState = hasFullCoverage.getAndSet(hasFullCoverageNow);
diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Node.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Node.java
index 2f70c37cd48..8f465070de4 100644
--- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Node.java
+++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Node.java
@@ -21,6 +21,9 @@ public class Node {
     private final AtomicBoolean statusIsKnown = new AtomicBoolean(false);
     private final AtomicBoolean working = new AtomicBoolean(true);
     private final AtomicLong activeDocuments = new AtomicLong(0);
+    private final AtomicLong pingSequence = new AtomicLong(0);
+    private final AtomicLong lastPong = new AtomicLong(0);
+    private final AtomicBoolean isBlockingWrites = new AtomicBoolean(false);
 
     public Node(int key, String hostname, int group) {
         this.key = key;
@@ -28,6 +31,18 @@ public class Node {
         this.group = group;
     }
 
+    /** Give a monotonically increasing sequence number.*/
+    public long createPingSequenceId() { return pingSequence.incrementAndGet(); }
+    /** Checks if this pong is received in line and accepted, or out of band and should be ignored..*/
+    public boolean isLastReceivedPong(long pingId ) {
+        long last = lastPong.get();
+        while ((pingId > last) && ! lastPong.compareAndSet(last, pingId)) {
+            last = lastPong.get();
+        }
+        return last < pingId;
+    }
+    public long getLastReceivedPongId() { return lastPong.get(); }
+
     /** Returns the unique and stable distribution key of this node */
     public int key() { return key; }
 
@@ -56,14 +71,14 @@ public class Node {
     }
 
     /** Updates the active documents on this node */
-    void setActiveDocuments(long activeDocuments) {
-        this.activeDocuments.set(activeDocuments);
-    }
+    void setActiveDocuments(long activeDocuments) { this.activeDocuments.set(activeDocuments); }
 
     /** Returns the active documents on this node. If unknown, 0 is returned. */
-    long getActiveDocuments() {
-        return activeDocuments.get();
-    }
+    long getActiveDocuments() { return activeDocuments.get(); }
+
+    public void setBlockingWrites(boolean isBlockingWrites) { this.isBlockingWrites.set(isBlockingWrites); }
+
+    boolean isBlockingWrites() { return isBlockingWrites.get(); }
 
     @Override
     public int hashCode() { return Objects.hash(hostname, key, pathIndex, group); }
diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/PingFactory.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/PingFactory.java
index b16fa941f68..2e07d8d61e6 100644
--- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/PingFactory.java
+++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/PingFactory.java
@@ -1,13 +1,11 @@
 // Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.search.dispatch.searchcluster;
 
-import com.yahoo.prelude.Pong;
 import com.yahoo.search.cluster.ClusterMonitor;
 
-import java.util.concurrent.Callable;
 
 public interface PingFactory {
 
-    Callable<Pong> createPinger(Node node, ClusterMonitor<Node> monitor);
+    Pinger createPinger(Node node, ClusterMonitor<Node> monitor, PongHandler pongHandler);
 
 }
diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Pinger.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Pinger.java
new file mode 100644
index 00000000000..b4a7ccbf98c
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Pinger.java
@@ -0,0 +1,12 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.dispatch.searchcluster;
+
+/**
+ * Send a ping and ensure that the pong is propagated to the ponghandler.
+ * Should not wait as this should be done in parallel on all nodes.
+ *
+ * @author baldersheim
+ */
+public interface Pinger {
+    void ping();
+}
diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/PongHandler.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/PongHandler.java
new file mode 100644
index 00000000000..c0579b5d36e
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/PongHandler.java
@@ -0,0 +1,12 @@
+package com.yahoo.search.dispatch.searchcluster;
+
+import com.yahoo.prelude.Pong;
+
+/**
+ * Handle the Pong result of a Ping.
+ *
+ * @author baldersheim
+ */
+public interface PongHandler {
+    void handle(Pong pong);
+}
diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java
index 5f211c37917..7dfc03fd2d7 100644
--- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java
+++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java
@@ -10,7 +10,7 @@ import com.yahoo.net.HostName;
 import com.yahoo.prelude.Pong;
 import com.yahoo.search.cluster.ClusterMonitor;
 import com.yahoo.search.cluster.NodeManager;
-import com.yahoo.search.result.ErrorMessage;
+import com.yahoo.search.dispatch.TopKEstimator;
 import com.yahoo.vespa.config.search.DispatchConfig;
 
 import java.util.LinkedHashMap;
@@ -18,13 +18,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.OptionalInt;
-import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Executor;
-import java.util.concurrent.FutureTask;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-import java.util.function.Predicate;
-import java.util.logging.Level;
 import java.util.logging.Logger;
 import java.util.stream.Collectors;
 
@@ -43,9 +37,9 @@ public class SearchCluster implements NodeManager<Node> {
     private final ImmutableMap<Integer, Group> groups;
     private final ImmutableMultimap<String, Node> nodesByHost;
     private final ImmutableList<Group> orderedGroups;
-    private final ClusterMonitor<Node> clusterMonitor;
     private final VipStatus vipStatus;
-    private PingFactory pingFactory;
+    private final PingFactory pingFactory;
+    private final TopKEstimator hitEstimator;
     private long nextLogTime = 0;
 
     /**
@@ -58,10 +52,12 @@ public class SearchCluster implements NodeManager<Node> {
      */
     private final Optional<Node> localCorpusDispatchTarget;
 
-    public SearchCluster(String clusterId, DispatchConfig dispatchConfig, int containerClusterSize, VipStatus vipStatus) {
+    public SearchCluster(String clusterId, DispatchConfig dispatchConfig, int containerClusterSize,
+                         VipStatus vipStatus, PingFactory pingFactory) {
         this.clusterId = clusterId;
         this.dispatchConfig = dispatchConfig;
         this.vipStatus = vipStatus;
+        this.pingFactory = pingFactory;
 
         List<Node> nodes = toNodes(dispatchConfig);
         this.size = nodes.size();
@@ -82,31 +78,28 @@ public class SearchCluster implements NodeManager<Node> {
         for (Node node : nodes)
             nodesByHostBuilder.put(node.hostname(), node);
         this.nodesByHost = nodesByHostBuilder.build();
+        hitEstimator = new TopKEstimator(30.0, dispatchConfig.topKProbability());
 
         this.localCorpusDispatchTarget = findLocalCorpusDispatchTarget(HostName.getLocalhost(),
                                                                        size,
                                                                        containerClusterSize,
                                                                        nodesByHost,
                                                                        groups);
-
-        this.clusterMonitor = new ClusterMonitor<>(this);
     }
 
-    public void shutDown() {
-        clusterMonitor.shutdown();
+    /* Testing only */
+    public SearchCluster(String clusterId, DispatchConfig dispatchConfig,
+                         VipStatus vipStatus, PingFactory pingFactory) {
+        this(clusterId, dispatchConfig, 1, vipStatus, pingFactory);
     }
 
-    public void startClusterMonitoring(PingFactory pingFactory) {
-        this.pingFactory = pingFactory;
-
-        for (var group : orderedGroups) {
+    public void addMonitoring(ClusterMonitor clusterMonitor) {
+        for (var group : orderedGroups()) {
             for (var node : group.nodes())
                 clusterMonitor.add(node, true);
         }
     }
 
-    ClusterMonitor<Node> clusterMonitor() { return clusterMonitor; }
-
     private static Optional<Node> findLocalCorpusDispatchTarget(String selfHostname,
                                                                 int searchClusterSize,
                                                                 int containerClusterSize,
@@ -157,8 +150,8 @@ public class SearchCluster implements NodeManager<Node> {
 
     /** Returns the n'th (zero-indexed) group in the cluster if possible */
     public Optional<Group> group(int n) {
-        if (orderedGroups.size() > n) {
-            return Optional.of(orderedGroups.get(n));
+        if (orderedGroups().size() > n) {
+            return Optional.of(orderedGroups().get(n));
         } else {
             return Optional.empty();
         }
@@ -166,13 +159,13 @@ public class SearchCluster implements NodeManager<Node> {
 
     /** Returns the number of nodes per group - size()/groups.size() */
     public int groupSize() {
-        if (groups.size() == 0) return size();
-        return size() / groups.size();
+        if (groups().size() == 0) return size();
+        return size() / groups().size();
     }
 
     public int groupsWithSufficientCoverage() {
         int covered = 0;
-        for (Group g : orderedGroups) {
+        for (Group g : orderedGroups()) {
             if (g.hasSufficientCoverage()) {
                 covered++;
             }
@@ -188,7 +181,7 @@ public class SearchCluster implements NodeManager<Node> {
         if ( localCorpusDispatchTarget.isEmpty()) return Optional.empty();
 
         // Only use direct dispatch if the local group has sufficient coverage
-        Group localSearchGroup = groups.get(localCorpusDispatchTarget.get().group());
+        Group localSearchGroup = groups().get(localCorpusDispatchTarget.get().group());
         if ( ! localSearchGroup.hasSufficientCoverage()) return Optional.empty();
 
         // Only use direct dispatch if the local search node is not down
@@ -227,7 +220,10 @@ public class SearchCluster implements NodeManager<Node> {
                 setInRotationOnlyIf(hasWorkingNodes());
         }
         else if (usesLocalCorpusIn(node)) { // follow the status of this node
-            setInRotationOnlyIf(nodeIsWorking);
+            // Do not take this out of rotation if we're a combined cluster of size 1,
+            // as that can't be helpful, and leads to a deadlock where this node is never taken back in servic e
+            if (nodeIsWorking || size() > 1)
+                setInRotationOnlyIf(nodeIsWorking);
         }
     }
 
@@ -247,7 +243,14 @@ public class SearchCluster implements NodeManager<Node> {
             vipStatus.removeFromRotation(clusterId);
     }
 
-    private boolean hasInformationAboutAllNodes() {
+    public int estimateHitsToFetch(int wantedHits, int numPartitions) {
+        return hitEstimator.estimateK(wantedHits, numPartitions);
+    }
+    public int estimateHitsToFetch(int wantedHits, int numPartitions, double topKProbability) {
+        return hitEstimator.estimateK(wantedHits, numPartitions, topKProbability);
+    }
+
+    public boolean hasInformationAboutAllNodes() {
         return nodesByHost.values().stream().allMatch(node -> node.isWorking() != null);
     }
 
@@ -263,29 +266,41 @@ public class SearchCluster implements NodeManager<Node> {
         return localCorpusDispatchTarget.isPresent() && localCorpusDispatchTarget.get().group() == group.id();
     }
 
-    /** Used by the cluster monitor to manage node status */
-    @Override
-    public void ping(Node node, Executor executor) {
-        if (pingFactory == null) return; // not initialized yet
+    private static class PongCallback implements PongHandler {
 
-        FutureTask<Pong> futurePong = new FutureTask<>(pingFactory.createPinger(node, clusterMonitor));
-        executor.execute(futurePong);
-        Pong pong = getPong(futurePong, node);
-        futurePong.cancel(true);
+        private final ClusterMonitor<Node> clusterMonitor;
+        private final Node node;
 
-        if (pong.badResponse()) {
-            clusterMonitor.failed(node, pong.error().get());
-        } else {
-            if (pong.activeDocuments().isPresent()) {
-                node.setActiveDocuments(pong.activeDocuments().get());
+        PongCallback(Node node, ClusterMonitor<Node> clusterMonitor) {
+            this.node = node;
+            this.clusterMonitor = clusterMonitor;
+        }
+
+        @Override
+        public void handle(Pong pong) {
+            if (pong.badResponse()) {
+                clusterMonitor.failed(node, pong.error().get());
+            } else {
+                if (pong.activeDocuments().isPresent()) {
+                    node.setActiveDocuments(pong.activeDocuments().get());
+                    node.setBlockingWrites(pong.isBlockingWrites());
+                }
+                clusterMonitor.responded(node);
             }
-            clusterMonitor.responded(node);
         }
+
+    }
+
+    /** Used by the cluster monitor to manage node status */
+    @Override
+    public void ping(ClusterMonitor clusterMonitor, Node node, Executor executor) {
+        Pinger pinger = pingFactory.createPinger(node, clusterMonitor, new PongCallback(node, clusterMonitor));
+        pinger.ping();
     }
 
     private void pingIterationCompletedSingleGroup() {
-        Group group = groups.values().iterator().next();
-        group.aggregateActiveDocuments();
+        Group group = groups().values().iterator().next();
+        group.aggregateNodeValues();
         // With just one group sufficient coverage may not be the same as full coverage, as the
         // group will always be marked sufficient for use.
         updateSufficientCoverage(group, true);
@@ -295,21 +310,20 @@ public class SearchCluster implements NodeManager<Node> {
     }
 
     private void pingIterationCompletedMultipleGroups() {
-        int numGroups = orderedGroups.size();
+        int numGroups = orderedGroups().size();
         // Update active documents per group and use it to decide if the group should be active
-
         long[] activeDocumentsInGroup = new long[numGroups];
         long sumOfActiveDocuments = 0;
         for(int i = 0; i < numGroups; i++) {
-            Group group = orderedGroups.get(i);
-            group.aggregateActiveDocuments();
+            Group group = orderedGroups().get(i);
+            group.aggregateNodeValues();
             activeDocumentsInGroup[i] = group.getActiveDocuments();
             sumOfActiveDocuments += activeDocumentsInGroup[i];
         }
 
         boolean anyGroupsSufficientCoverage = false;
         for (int i = 0; i < numGroups; i++) {
-            Group group = orderedGroups.get(i);
+            Group group = orderedGroups().get(i);
             long activeDocuments = activeDocumentsInGroup[i];
             long averageDocumentsInOtherGroups = (sumOfActiveDocuments - activeDocuments) / (numGroups - 1);
             boolean sufficientCoverage = isGroupCoverageSufficient(group.workingNodes(), group.nodes().size(), activeDocuments, averageDocumentsInOtherGroups);
@@ -326,7 +340,7 @@ public class SearchCluster implements NodeManager<Node> {
      */
     @Override
     public void pingIterationCompleted() {
-        int numGroups = orderedGroups.size();
+        int numGroups = orderedGroups().size();
         if (numGroups == 1) {
             pingIterationCompletedSingleGroup();
         } else {
@@ -353,25 +367,11 @@ public class SearchCluster implements NodeManager<Node> {
         return workingNodes + nodesAllowedDown >= nodesInGroup;
     }
 
-    private Pong getPong(FutureTask<Pong> futurePong, Node node) {
-        try {
-            return futurePong.get(clusterMonitor.getConfiguration().getFailLimit(), TimeUnit.MILLISECONDS);
-        } catch (InterruptedException e) {
-            log.log(Level.WARNING, "Exception pinging " + node, e);
-            return new Pong(ErrorMessage.createUnspecifiedError("Ping was interrupted: " + node));
-        } catch (ExecutionException e) {
-            log.log(Level.WARNING, "Exception pinging " + node, e);
-            return new Pong(ErrorMessage.createUnspecifiedError("Execution was interrupted: " + node));
-        } catch (TimeoutException e) {
-            return new Pong(ErrorMessage.createNoAnswerWhenPingingNode("Ping thread timed out"));
-        }
-    }
-
     /**
      * Calculate whether a subset of nodes in a group has enough coverage
      */
     public boolean isPartialGroupCoverageSufficient(OptionalInt knownGroupId, List<Node> nodes) {
-        if (orderedGroups.size() == 1) {
+        if (orderedGroups().size() == 1) {
             boolean sufficient = nodes.size() >= groupSize() - dispatchConfig.maxNodesDownPerGroup();
             return sufficient;
         }
@@ -380,14 +380,14 @@ public class SearchCluster implements NodeManager<Node> {
             return false;
         }
         int groupId = knownGroupId.getAsInt();
-        Group group = groups.get(groupId);
+        Group group = groups().get(groupId);
         if (group == null) {
             return false;
         }
         int nodesInGroup = group.nodes().size();
         long sumOfActiveDocuments = 0;
         int otherGroups = 0;
-        for (Group g : orderedGroups) {
+        for (Group g : orderedGroups()) {
             if (g.id() != groupId) {
                 sumOfActiveDocuments += g.getActiveDocuments();
                 otherGroups++;
@@ -402,6 +402,7 @@ public class SearchCluster implements NodeManager<Node> {
     }
 
     private void trackGroupCoverageChanges(int index, Group group, boolean fullCoverage, long averageDocuments) {
+        if ( ! hasInformationAboutAllNodes()) return; // Be silent until we know what we are talking about.
         boolean changed = group.isFullCoverageStatusChanged(fullCoverage);
         if (changed || (!fullCoverage && System.currentTimeMillis() > nextLogTime)) {
             nextLogTime = System.currentTimeMillis() + 30 * 1000;
diff --git a/container-search/src/main/java/com/yahoo/search/federation/FederationResult.java b/container-search/src/main/java/com/yahoo/search/federation/FederationResult.java
index 5f1cfccf549..6243dc694c2 100644
--- a/container-search/src/main/java/com/yahoo/search/federation/FederationResult.java
+++ b/container-search/src/main/java/com/yahoo/search/federation/FederationResult.java
@@ -39,8 +39,8 @@ class FederationResult {
     }
 
     /**
-     * Wait on each target for that targets timeout
-     * On the worst case this is the same as waiting for the max target timeout,
+     * Wait on each target for that targets timeout.
+     * In the worst case this is the same as waiting for the max target timeout,
      * in the average case it may be much better because lower timeout sources do not get to
      * drive the timeout above their own timeout value.
      * When this completes, results can be accessed from the TargetResults with no blocking
diff --git a/container-search/src/main/java/com/yahoo/search/federation/FederationSearcher.java b/container-search/src/main/java/com/yahoo/search/federation/FederationSearcher.java
index c3ede4fe20a..60c5d42c531 100644
--- a/container-search/src/main/java/com/yahoo/search/federation/FederationSearcher.java
+++ b/container-search/src/main/java/com/yahoo/search/federation/FederationSearcher.java
@@ -78,7 +78,6 @@ public class FederationSearcher extends ForkingSearcher {
     /** The name of the query property containing the source name added to the query to each source by this */
     public final static CompoundName SOURCENAME = new CompoundName("sourceName");
     public final static CompoundName PROVIDERNAME = new CompoundName("providerName");
-
     /** Logging field name constants */
     public static final String LOG_COUNT_PREFIX = "count_";
 
@@ -110,7 +109,7 @@ public class FederationSearcher extends ForkingSearcher {
 
     // for testing
     public FederationSearcher(ComponentId id, SearchChainResolver searchChainResolver) {
-        this(searchChainResolver, false, PropagateSourceProperties.ALL, null);
+        this(searchChainResolver, false, PropagateSourceProperties.EVERY, null);
     }
 
     private FederationSearcher(SearchChainResolver searchChainResolver,
@@ -271,7 +270,10 @@ public class FederationSearcher extends ForkingSearcher {
         outgoing.setTimeout(timeout);
 
         switch (propagateSourceProperties) {
-            case ALL:
+            case EVERY:
+                propagatePerSourceQueryProperties(query, outgoing, window, sourceName, providerName, null);
+                break;
+            case NATIVE: case ALL:
                 propagatePerSourceQueryProperties(query, outgoing, window, sourceName, providerName, Query.nativeProperties);
                 break;
             case OFFSET_HITS:
@@ -288,10 +290,21 @@ public class FederationSearcher extends ForkingSearcher {
     private void propagatePerSourceQueryProperties(Query original, Query outgoing, Window window,
                                                    String sourceName, String providerName,
                                                    List<CompoundName> queryProperties) {
-        for (CompoundName key : queryProperties) {
-            Object value = getSourceOrProviderProperty(original, key, sourceName, providerName, window.get(key));
-            if (value != null)
-                outgoing.properties().set(key, value);
+        if (queryProperties == null) {
+            outgoing.setHits(window.hits);
+            outgoing.setOffset(window.offset);
+            original.properties().listProperties(CompoundName.fromComponents("provider", providerName)).forEach((k, v) ->
+                outgoing.properties().set(k, v));
+            original.properties().listProperties(CompoundName.fromComponents("source", sourceName)).forEach((k, v) ->
+                outgoing.properties().set(k, v));
+        }
+        else {
+            for (CompoundName key : queryProperties) {
+                Object value = getSourceOrProviderProperty(original, key, sourceName, providerName, window.get(key));
+                if (value != null)
+                    outgoing.properties().set(key, value);
+                if (value != null) System.out.println("Setting " + key + " = " + value);
+            }
         }
     }
 
@@ -319,7 +332,7 @@ public class FederationSearcher extends ForkingSearcher {
 
     private ErrorMessage missingSearchChainsErrorMessage(List<UnresolvedSearchChainException> unresolvedSearchChainExceptions) {
         String message = String.join(" ", getMessagesSet(unresolvedSearchChainExceptions)) +
-                      " Valid source refs are " + String.join(", ", allSourceRefDescriptions()) +'.';
+                                     " Valid source refs are " + String.join(", ", allSourceRefDescriptions()) +'.';
         return ErrorMessage.createInvalidQueryParameter(message);
     }
 
@@ -341,7 +354,7 @@ public class FederationSearcher extends ForkingSearcher {
     }
 
     private void warnIfUnresolvedSearchChains(List<UnresolvedSearchChainException> missingTargets,
-                                      HitGroup errorHitGroup) {
+                                              HitGroup errorHitGroup) {
         if (!missingTargets.isEmpty()) {
             errorHitGroup.addError(missingSearchChainsErrorMessage(missingTargets));
         }
@@ -479,9 +492,9 @@ public class FederationSearcher extends ForkingSearcher {
      * TODO This is probably a dirty hack for bug 4711376. There are probably better ways.
      * But I will leave that to trd-processing@
      *
-     * @param group  The merging hitgroup to be updated if necessary
-     * @param orderer The per provider hit orderer.
-     * @return The hitorderer chosen
+     * @param group the merging hitgroup to be updated if necessary
+     * @param orderer the per provider hit orderer
+     * @return he hitorderer chosen
      */
     private HitOrderer dirtyCopyIfModifiedOrderer(HitGroup group, HitOrderer orderer) {
         if (orderer != null) {
diff --git a/container-search/src/main/java/com/yahoo/search/handler/HttpSearchResponse.java b/container-search/src/main/java/com/yahoo/search/handler/HttpSearchResponse.java
index 3602d21f7d8..d636d3bc925 100644
--- a/container-search/src/main/java/com/yahoo/search/handler/HttpSearchResponse.java
+++ b/container-search/src/main/java/com/yahoo/search/handler/HttpSearchResponse.java
@@ -23,6 +23,7 @@ import com.yahoo.processing.rendering.Renderer;
 import com.yahoo.search.Query;
 import com.yahoo.search.Result;
 import com.yahoo.search.query.context.QueryContext;
+import com.yahoo.yolean.trace.TraceNode;
 
 /**
  * Wrap the result of a query as an HTTP response.
@@ -36,8 +37,13 @@ public class HttpSearchResponse extends ExtendedResponse {
     private final Renderer<Result> rendererCopy;
     private final Timing timing;
     private final HitCounts hitCounts;
+    private final TraceNode trace;
 
     public HttpSearchResponse(int status, Result result, Query query, Renderer renderer) {
+        this(status, result, query, renderer, null);
+    }
+
+    HttpSearchResponse(int status, Result result, Query query, Renderer renderer, TraceNode trace) {
         super(status);
         this.query = query;
         this.result = result;
@@ -45,6 +51,7 @@ public class HttpSearchResponse extends ExtendedResponse {
 
         this.timing = SearchResponse.createTiming(query, result);
         this.hitCounts = SearchResponse.createHitCounts(query, result);
+        this.trace = trace;
         populateHeaders(headers(), result.getHeaders(false));
     }
 
@@ -107,6 +114,9 @@ public class HttpSearchResponse extends ExtendedResponse {
     @Override
     public void populateAccessLogEntry(final AccessLogEntry accessLogEntry) {
         super.populateAccessLogEntry(accessLogEntry);
+        if (trace != null) {
+            accessLogEntry.setTrace(trace);
+        }
         populateAccessLogEntry(accessLogEntry, getHitCounts());
     }
 
diff --git a/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java b/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java
index 0981c6e8dad..5e3b79c1545 100644
--- a/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java
+++ b/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java
@@ -25,11 +25,12 @@ import com.yahoo.prelude.query.QueryException;
 import com.yahoo.prelude.query.parser.ParseException;
 import com.yahoo.processing.rendering.Renderer;
 import com.yahoo.processing.request.CompoundName;
+import com.yahoo.search.query.context.QueryContext;
 import com.yahoo.search.query.ranking.SoftTimeout;
 import com.yahoo.search.searchchain.ExecutionFactory;
 import com.yahoo.slime.Inspector;
 import com.yahoo.slime.ObjectTraverser;
-import com.yahoo.vespa.config.SlimeUtils;
+import com.yahoo.slime.SlimeUtils;
 import com.yahoo.yolean.Exceptions;
 import com.yahoo.search.Query;
 import com.yahoo.search.Result;
@@ -49,6 +50,7 @@ import com.yahoo.statistics.Handle;
 import com.yahoo.statistics.Statistics;
 import com.yahoo.statistics.Value;
 import com.yahoo.vespa.configdefinition.SpecialtokensConfig;
+import com.yahoo.yolean.trace.TraceNode;
 
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
@@ -58,6 +60,7 @@ import java.util.Optional;
 import java.util.concurrent.Executor;
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
@@ -98,6 +101,8 @@ public class SearchHandler extends LoggingRequestHandler {
 
     private final ExecutionFactory executionFactory;
 
+    private final AtomicLong numRequestsLeftToTrace;
+
     private final class MeanConnections implements Callback {
 
         @Override
@@ -125,6 +130,7 @@ public class SearchHandler extends LoggingRequestHandler {
              accessLog,
              QueryProfileConfigurer.createFromConfig(queryProfileConfig).compile(),
              executionFactory,
+             containerHttpConfig.numQueriesToTraceOnDebugAfterConstruction(),
              containerHttpConfig.hostResponseHeaderKey().equals("") ?
              Optional.empty() : Optional.of( containerHttpConfig.hostResponseHeaderKey()));
     }
@@ -136,6 +142,17 @@ public class SearchHandler extends LoggingRequestHandler {
                          CompiledQueryProfileRegistry queryProfileRegistry,
                          ExecutionFactory executionFactory,
                          Optional<String> hostResponseHeaderKey) {
+        this(statistics, metric, executor, accessLog, queryProfileRegistry, executionFactory, 0, hostResponseHeaderKey);
+    }
+
+    private SearchHandler(Statistics statistics,
+                         Metric metric,
+                         Executor executor,
+                         AccessLog accessLog,
+                         CompiledQueryProfileRegistry queryProfileRegistry,
+                         ExecutionFactory executionFactory,
+                         long numQueriesToTraceOnDebugAfterStartup,
+                         Optional<String> hostResponseHeaderKey) {
         super(executor, accessLog, metric, true);
         log.log(LogLevel.DEBUG, "SearchHandler.init " + System.identityHashCode(this));
         this.queryProfileRegistry = queryProfileRegistry;
@@ -150,6 +167,7 @@ public class SearchHandler extends LoggingRequestHandler {
                                                             .setCallback(new MeanConnections()));
 
         this.hostResponseHeaderKey = hostResponseHeaderKey;
+        this.numRequestsLeftToTrace = new AtomicLong(numQueriesToTraceOnDebugAfterStartup);
     }
 
     /** @deprecated use the other constructor */
@@ -215,7 +233,6 @@ public class SearchHandler extends LoggingRequestHandler {
 
     }
 
-    @SuppressWarnings("unchecked")
     private HttpResponse errorResponse(HttpRequest request, ErrorMessage errorMessage) {
         Query query = new Query();
         Result result = new Result(query, errorMessage);
@@ -281,7 +298,8 @@ public class SearchHandler extends LoggingRequestHandler {
         // Transform result to response
         Renderer renderer = toRendererCopy(query.getPresentation().getRenderer());
         HttpSearchResponse response = new HttpSearchResponse(getHttpResponseStatus(request, result),
-                                                             result, query, renderer);
+                                                             result, query, renderer,
+                                                             extractTraceNode(query));
         if (hostResponseHeaderKey.isPresent())
             response.headers().add(hostResponseHeaderKey.get(), selfHostname);
 
@@ -292,6 +310,19 @@ public class SearchHandler extends LoggingRequestHandler {
         return response;
     }
 
+    private static TraceNode extractTraceNode(Query query) {
+        if (log.isLoggable(Level.FINE)) {
+            QueryContext queryContext = query.getContext(false);
+            if (queryContext != null) {
+                Execution.Trace trace = queryContext.getTrace();
+                if (trace != null) {
+                    return trace.traceNode();
+                }
+            }
+        }
+        return null;
+    }
+
     private static int getErrors(Result result) {
         return result.hits().getErrorHit() == null ? 0 : 1;
     }
@@ -330,7 +361,13 @@ public class SearchHandler extends LoggingRequestHandler {
 
         Execution execution = executionFactory.newExecution(searchChain);
         query.getModel().setExecution(execution);
-        execution.trace().setForceTimestamps(query.properties().getBoolean(FORCE_TIMESTAMPS, false));
+        if (log.isLoggable(Level.FINE) && (numRequestsLeftToTrace.getAndDecrement() > 0)) {
+            query.setTraceLevel(Math.max(1, query.getTraceLevel()));
+            execution.trace().setForceTimestamps(true);
+
+        } else {
+            execution.trace().setForceTimestamps(query.properties().getBoolean(FORCE_TIMESTAMPS, false));
+        }
         if (query.properties().getBoolean(DETAILED_TIMING_LOGGING, false)) {
             // check and set (instead of set directly) to avoid overwriting stuff from prepareForBreakdownAnalysis()
             execution.context().setDetailedDiagnostics(true);
@@ -389,7 +426,7 @@ public class SearchHandler extends LoggingRequestHandler {
         } catch (ParseException e) {
             ErrorMessage error = ErrorMessage.createIllegalQuery("Could not parse query [" + request + "]: "
                                                                  + Exceptions.toMessageString(e));
-            log.log(LogLevel.DEBUG, () -> error.getDetailedMessage());
+            log.log(LogLevel.DEBUG, error::getDetailedMessage);
             return new Result(query, error);
         } catch (IllegalArgumentException e) {
             if ("Comparison method violates its general contract!".equals(e.getMessage())) {
@@ -401,7 +438,7 @@ public class SearchHandler extends LoggingRequestHandler {
             else {
                 ErrorMessage error = ErrorMessage.createBadRequest("Invalid search request [" + request + "]: "
                                                                    + Exceptions.toMessageString(e));
-                log.log(LogLevel.DEBUG, () -> error.getDetailedMessage());
+                log.log(LogLevel.DEBUG, error::getDetailedMessage);
                 return new Result(query, error);
             }
         } catch (LinkageError | StackOverflowError e) {
diff --git a/container-search/src/main/java/com/yahoo/search/query/Properties.java b/container-search/src/main/java/com/yahoo/search/query/Properties.java
index a1a70b4c3ba..a0cd4137e9f 100644
--- a/container-search/src/main/java/com/yahoo/search/query/Properties.java
+++ b/container-search/src/main/java/com/yahoo/search/query/Properties.java
@@ -30,8 +30,9 @@ public abstract class Properties extends com.yahoo.processing.request.Properties
         return (Properties)super.clone();
     }
 
-    /** The query owning this property object.
-     *  Only guaranteed to work if this instance is accessible as query.properties()
+    /**
+     * Returns the query owning this property object.
+     * Only guaranteed to work if this instance is accessible as query.properties()
      */
     public Query getParentQuery() {
         if (chained() == null) {
@@ -48,4 +49,5 @@ public abstract class Properties extends com.yahoo.processing.request.Properties
         if (chained() != null)
             chained().setParentQuery(query);
     }
+
 }
diff --git a/container-search/src/main/java/com/yahoo/search/query/Ranking.java b/container-search/src/main/java/com/yahoo/search/query/Ranking.java
index 7444c94f491..830a3f4ef81 100644
--- a/container-search/src/main/java/com/yahoo/search/query/Ranking.java
+++ b/container-search/src/main/java/com/yahoo/search/query/Ranking.java
@@ -47,7 +47,7 @@ public class Ranking implements Cloneable {
     public static final String PROPERTIES = "properties";
 
     static {
-        argumentType =new QueryProfileType(RANKING);
+        argumentType = new QueryProfileType(RANKING);
         argumentType.setStrict(true);
         argumentType.setBuiltin(true);
         argumentType.addField(new FieldDescription(LOCATION, "string", "location"));
@@ -63,7 +63,7 @@ public class Ranking implements Cloneable {
         argumentType.addField(new FieldDescription(FEATURES, "query-profile", "rankfeature"));
         argumentType.addField(new FieldDescription(PROPERTIES, "query-profile", "rankproperty"));
         argumentType.freeze();
-        argumentTypeName=new CompoundName(argumentType.getId().getName());
+        argumentTypeName = new CompoundName(argumentType.getId().getName());
     }
     public static QueryProfileType getArgumentType() { return argumentType; }
 
diff --git a/container-search/src/main/java/com/yahoo/search/query/Select.java b/container-search/src/main/java/com/yahoo/search/query/Select.java
index cb662dcd671..65ffd29efe0 100644
--- a/container-search/src/main/java/com/yahoo/search/query/Select.java
+++ b/container-search/src/main/java/com/yahoo/search/query/Select.java
@@ -57,12 +57,13 @@ public class Select implements Cloneable {
     }
 
     public Select(String where, String grouping, Query query) {
-        this(where, grouping, query, Collections.emptyList());
+        this(where, grouping, null, query, Collections.emptyList());
     }
 
-    private Select(String where, String grouping, Query query, List<GroupingRequest> groupingRequests) {
+    private Select(String where, String grouping, String groupingExpressionString, Query query, List<GroupingRequest> groupingRequests) {
         this.where = Objects.requireNonNull(where, "A Select must have a where string (possibly the empty string)");
         this.grouping = Objects.requireNonNull(grouping, "A Select must have a select string (possibly the empty string)");
+        this.groupingExpressionString = groupingExpressionString;
         this.parent = Objects.requireNonNull(query, "A Select must have a parent query");
         this.groupingRequests = deepCopy(groupingRequests, this);
     }
@@ -136,11 +137,11 @@ public class Select implements Cloneable {
 
     @Override
     public Object clone() {
-        return new Select(where, grouping, parent, groupingRequests);
+        return new Select(where, grouping, groupingExpressionString, parent, groupingRequests);
     }
 
     public Select cloneFor(Query parent)  {
-        return new Select(where, grouping, parent, groupingRequests);
+        return new Select(where, grouping, groupingExpressionString, parent, groupingRequests);
     }
 
 }
diff --git a/container-search/src/main/java/com/yahoo/search/query/SelectParser.java b/container-search/src/main/java/com/yahoo/search/query/SelectParser.java
index ae50756331c..775dca7c444 100644
--- a/container-search/src/main/java/com/yahoo/search/query/SelectParser.java
+++ b/container-search/src/main/java/com/yahoo/search/query/SelectParser.java
@@ -45,8 +45,7 @@ import com.yahoo.search.yql.VespaGroupingStep;
 import com.yahoo.slime.ArrayTraverser;
 import com.yahoo.slime.Inspector;
 import com.yahoo.slime.ObjectTraverser;
-import com.yahoo.slime.Type;
-import com.yahoo.vespa.config.SlimeUtils;
+import com.yahoo.slime.SlimeUtils;
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.Arrays;
diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/BackedOverridableQueryProfile.java b/container-search/src/main/java/com/yahoo/search/query/profile/BackedOverridableQueryProfile.java
index 99f1e26b221..11864e60cec 100644
--- a/container-search/src/main/java/com/yahoo/search/query/profile/BackedOverridableQueryProfile.java
+++ b/container-search/src/main/java/com/yahoo/search/query/profile/BackedOverridableQueryProfile.java
@@ -135,8 +135,8 @@ public class BackedOverridableQueryProfile extends OverridableQueryProfile imple
 
     @Override
     public List<String> getDimensions() {
-        List<String> dimensions=super.getDimensions();
-        if (dimensions!=null) return dimensions;
+        List<String> dimensions = super.getDimensions();
+        if (dimensions != null) return dimensions;
         return backingProfile.getDimensions();
     }
 
diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/DimensionBinding.java b/container-search/src/main/java/com/yahoo/search/query/profile/DimensionBinding.java
index 50bd2c58da8..0cbfdc5dca0 100644
--- a/container-search/src/main/java/com/yahoo/search/query/profile/DimensionBinding.java
+++ b/container-search/src/main/java/com/yahoo/search/query/profile/DimensionBinding.java
@@ -3,7 +3,6 @@ package com.yahoo.search.query.profile;
 
 import java.util.ArrayList;
 import java.util.Collections;
-import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
@@ -22,7 +21,7 @@ public class DimensionBinding {
     private DimensionValues values;
 
     /** The binding from those dimensions to values, and possibly other values */
-    private Map<String, String> context; // TODO: This is not needed any more
+    private Map<String, String> context;
 
     public static final DimensionBinding nullBinding =
         new DimensionBinding(Collections.unmodifiableList(Collections.emptyList()), DimensionValues.empty, null);
@@ -34,13 +33,13 @@ public class DimensionBinding {
     private boolean containsAllNulls;
 
     // NOTE: Map must be ordered
-    public static DimensionBinding createFrom(Map<String,String> values) {
+    public static DimensionBinding createFrom(Map<String, String> values) {
         return createFrom(new ArrayList<>(values.keySet()), values);
     }
 
     /** Creates a binding from a variant and a context. Any of the arguments may be null. */
     // NOTE: Map must be ordered
-    public static DimensionBinding createFrom(List<String> dimensions, Map<String,String> context) {
+    public static DimensionBinding createFrom(List<String> dimensions, Map<String, String> context) {
         if (dimensions == null || dimensions.size() == 0) {
             if (context == null) return nullBinding;
             if (dimensions == null) return new DimensionBinding(null, DimensionValues.empty, context); // Null, but must preserve context
@@ -51,7 +50,7 @@ public class DimensionBinding {
 
     /** Creates a binding from a variant and a context. Any of the arguments may be null. */
     public static DimensionBinding createFrom(List<String> dimensions, DimensionValues dimensionValues) {
-        if (dimensionValues==null || dimensionValues == DimensionValues.empty) return nullBinding;
+        if (dimensionValues == null || dimensionValues == DimensionValues.empty) return nullBinding;
 
         // If null, preserve raw material for creating a context later (in createFor)
         if (dimensions == null) return new DimensionBinding(null, dimensionValues, null);
@@ -61,14 +60,13 @@ public class DimensionBinding {
 
     /** Returns a binding for a (possibly) new set of variants. Variants may be null, but not bindings */
     public DimensionBinding createFor(List<String> newDimensions) {
-        if (newDimensions==null) return this; // Note: Not necessarily null - if no new variants then keep the existing binding
-        // if (this.context==null && values.length==0) return nullBinding; // No data from which to create a non-null binding
-        if (this.dimensions==newDimensions) return this; // Avoid creating a new object if the dimensions are the same
-
-        Map<String,String> context=this.context;
-        if (context==null)
-            context=this.values.asContext(this.dimensions !=null ? this.dimensions : newDimensions);
-        return new DimensionBinding(newDimensions,extractDimensionValues(newDimensions,context),context);
+        if (newDimensions == null) return this; // Note: Not necessarily null - if no new variants then keep the existing binding
+        if (this.dimensions == newDimensions) return this; // Avoid creating a new object if the dimensions are the same
+
+        Map<String,String> context = this.context;
+        if (context == null)
+            context = this.values.asContext(this.dimensions != null ? this.dimensions : newDimensions);
+        return new DimensionBinding(newDimensions, extractDimensionValues(newDimensions, context), context);
     }
 
     /**
@@ -76,20 +74,20 @@ public class DimensionBinding {
      * The array will not be modified. The context is needed in order to convert this binding to another
      * given another set of variant dimensions.
      */
-    private DimensionBinding(List<String> dimensions, DimensionValues values, Map<String,String> context) {
-        this.dimensions=dimensions;
-        this.values=values;
+    private DimensionBinding(List<String> dimensions, DimensionValues values, Map<String, String> context) {
+        this.dimensions = dimensions;
+        this.values = values;
         this.context = context;
-        containsAllNulls=values.isEmpty();
+        containsAllNulls = values.isEmpty();
     }
 
     /** Returns a read-only list of the dimensions of this. This value is undefined if this isNull() */
     public List<String> getDimensions() { return dimensions; }
 
     /** Returns a context created from the dimensions and values of this */
-    public Map<String,String> getContext() {
-        if (context !=null) return context;
-        context =values.asContext(dimensions);
+    public Map<String, String> getContext() {
+        if (context != null) return context;
+        context = values.asContext(dimensions);
         return context;
     }
 
@@ -102,7 +100,7 @@ public class DimensionBinding {
     public DimensionValues getValues() { return values; }
 
     /** Returns true only if this binding is null (contains no values for its dimensions (if any) */
-    public boolean isNull() { return dimensions==null || containsAllNulls; }
+    public boolean isNull() { return dimensions == null || containsAllNulls; }
 
     /**
      * Returns an array of the dimension values corresponding to the dimensions of this from the given context,
@@ -110,10 +108,10 @@ public class DimensionBinding {
      * Dimensions which are not set in this context get a null value.
      */
     private static DimensionValues extractDimensionValues(List<String> dimensions, Map<String,String> context) {
-        String[] dimensionValues=new String[dimensions.size()];
-        if (context==null || context.size()==0) return DimensionValues.createFrom(dimensionValues);
-        for (int i=0; i<dimensions.size(); i++)
-            dimensionValues[i]=context.get(dimensions.get(i));
+        String[] dimensionValues = new String[dimensions.size()];
+        if (context == null || context.size() == 0) return DimensionValues.createFrom(dimensionValues);
+        for (int i = 0; i < dimensions.size(); i++)
+            dimensionValues[i] = context.get(dimensions.get(i));
         return DimensionValues.createFrom(dimensionValues);
     }
 
@@ -138,16 +136,6 @@ public class DimensionBinding {
         return DimensionBinding.createFrom(combinedDimensions, combinedValues);
     }
 
-    /** Returns the binding of this (dimension->value) as a map */
-    private Map<String, String> asMap() {
-        Map<String, String> map = new LinkedHashMap<>();
-        for (int i = 0; i < Math.min(dimensions.size(), values.size()); i++) {
-            if (values.getValues()[i] != null)
-                map.put(dimensions.get(i), values.getValues()[i]);
-        }
-        return map;
-    }
-
     /**
      * Returns a combined list of dimensions from two separate lists,
      * or null if they are incompatible.
@@ -155,8 +143,12 @@ public class DimensionBinding {
      * (or return null if impossible).
      */
     private List<String> combineDimensions(List<String> d1, List<String> d2) {
+        if (d1.equals(d2)) return d1;
+        if (d1.isEmpty()) return d2;
+        if (d2.isEmpty()) return d1;
+
         List<String> combined = new ArrayList<>();
-        int d1Index = 0, d2Index=0;
+        int d1Index = 0, d2Index = 0;
         while (d1Index < d1.size() && d2Index < d2.size()) {
             if (d1.get(d1Index).equals(d2.get(d2Index))) { // agreement on next element
                 combined.add(d1.get(d1Index));
@@ -186,27 +178,22 @@ public class DimensionBinding {
      * or null if they are incompatible.
      */
     private Map<String, String> combineValues(Map<String, String> m1, Map<String, String> m2) {
-        Map<String, String> combinedValues = new LinkedHashMap<>(m1);
+        if (m1.isEmpty()) return m2;
+        if (m2.isEmpty()) return m1;
+        Map<String, String> combinedValues = null;
         for (Map.Entry<String, String> m2Entry : m2.entrySet()) {
             if (m2Entry.getValue() == null) continue;
             String m1Value = m1.get(m2Entry.getKey());
             if (m1Value != null && ! m1Value.equals(m2Entry.getValue()))
                 return null; // conflicting values of a key
+            if (combinedValues == null)
+                combinedValues = new LinkedHashMap<>(m1);
             combinedValues.put(m2Entry.getKey(), m2Entry.getValue());
         }
-        return combinedValues;
+        return combinedValues == null ? m1 : combinedValues;
     }
 
-    private boolean intersects(List<String> l1, List<String> l2) {
-        for (String l1Item : l1)
-            if (l2.contains(l1Item))
-                return true;
-        return false;
-    }
-
-    /**
-     * Returns true if <code>this == invalidBinding</code>
-     */
+    /** Returns true if this == invalidBinding */
     public boolean isInvalid() { return this == invalidBinding; }
 
     @Override
@@ -226,7 +213,7 @@ public class DimensionBinding {
     /** Two bindings are equal if they contain the same dimensions and the same non-null values */
     @Override
     public boolean equals(Object o) {
-        if (o==this) return true;
+        if (o == this) return true;
         if (! (o instanceof DimensionBinding)) return false;
         DimensionBinding other = (DimensionBinding)o;
         if ( ! this.dimensions.equals(other.dimensions)) return false;
diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/ModelObjectMap.java b/container-search/src/main/java/com/yahoo/search/query/profile/ModelObjectMap.java
index 4dc9ade62e5..e32d2dc226d 100644
--- a/container-search/src/main/java/com/yahoo/search/query/profile/ModelObjectMap.java
+++ b/container-search/src/main/java/com/yahoo/search/query/profile/ModelObjectMap.java
@@ -22,8 +22,7 @@ public class ModelObjectMap extends PropertyMap {
      */
     @Override
     protected boolean shouldSet(CompoundName name, Object value) {
-        if (value == null) return true;
-        return ! FieldType.isLegalFieldValue(value);
+        return value != null && ! FieldType.isLegalFieldValue(value);
     }
 
 }
diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfile.java b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfile.java
index e9ccdd22f98..ab0f129f1e9 100644
--- a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfile.java
+++ b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfile.java
@@ -100,7 +100,7 @@ public class QueryProfile extends FreezableSimpleComponent implements Cloneable
     public QueryProfileType getType() { return type; }
 
     /** Sets the type of this, or set to null to not use any type checking in this profile */
-    public void setType(QueryProfileType type) { this.type=type; }
+    public void setType(QueryProfileType type) { this.type = type; }
 
     /** Returns the virtual variants of this, or null if none */
     public QueryProfileVariants getVariants() { return variants; }
@@ -564,7 +564,7 @@ public class QueryProfile extends FreezableSimpleComponent implements Cloneable
         if (visitor.isDone()) return;
 
         if (allowContent) {
-            visitContent(visitor,dimensionBinding);
+            visitContent(visitor, dimensionBinding);
             if (visitor.isDone()) return;
         }
 
@@ -601,7 +601,7 @@ public class QueryProfile extends FreezableSimpleComponent implements Cloneable
             visitor.acceptValue(contentKey, getContent(contentKey), dimensionBinding, this, null);
         }
         else { // get all content in this
-            for (Map.Entry<String,Object> entry : getContent().entrySet()) {
+            for (Map.Entry<String, Object> entry : getContent().entrySet()) {
                 visitor.acceptValue(entry.getKey(), entry.getValue(), dimensionBinding, this, null);
                 if (visitor.isDone()) return;
             }
@@ -614,7 +614,7 @@ public class QueryProfile extends FreezableSimpleComponent implements Cloneable
     }
 
     /** Returns all the content from this as an unmodifiable map */
-    protected Map<String,Object> getContent() {
+    protected Map<String, Object> getContent() {
         return content.unmodifiableMap();
     }
 
diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileCompiler.java b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileCompiler.java
index cffe941b912..f1fc90dee09 100644
--- a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileCompiler.java
+++ b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileCompiler.java
@@ -71,6 +71,7 @@ public class QueryProfileCompiler {
      */
     private static Set<DimensionBindingForPath> collectVariants(CompoundName path, QueryProfile profile, DimensionBinding currentVariant) {
         Set<DimensionBindingForPath> variants = new HashSet<>();
+
         variants.addAll(collectVariantsFromValues(path, profile.getContent(), currentVariant));
         variants.addAll(collectVariantsInThis(path, profile, currentVariant));
         if (profile instanceof BackedOverridableQueryProfile)
@@ -157,6 +158,7 @@ public class QueryProfileCompiler {
 
                 if (combinedVariant.isInvalid()) continue; // values at this point in the graph are unreachable
 
+                variants.add(new DimensionBindingForPath(combinedVariant, path));
                 variants.addAll(collectVariantsFromValues(path, variant.values(), combinedVariant));
                 for (QueryProfile variantInheritedProfile : variant.inherited())
                     variants.addAll(collectVariants(path, variantInheritedProfile, combinedVariant));
@@ -169,9 +171,6 @@ public class QueryProfileCompiler {
                                                                            Map<String, Object> values,
                                                                            DimensionBinding currentVariant) {
         Set<DimensionBindingForPath> variants = new HashSet<>();
-        if ( ! values.isEmpty())
-            variants.add(new DimensionBindingForPath(currentVariant, path)); // there are actual values for this variant
-
         for (Map.Entry<String, Object> entry : values.entrySet()) {
             if (entry.getValue() instanceof QueryProfile)
                 variants.addAll(collectVariants(path.append(entry.getKey()), (QueryProfile)entry.getValue(), currentVariant));
@@ -202,7 +201,7 @@ public class QueryProfileCompiler {
 
         @Override
         public int hashCode() {
-           return binding.hashCode() + 17*path.hashCode();
+           return binding.hashCode() + 17 * path.hashCode();
         }
 
         @Override
diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java
index b250560e2f3..701ea7690f4 100644
--- a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java
+++ b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java
@@ -9,6 +9,7 @@ import com.yahoo.search.query.Properties;
 import com.yahoo.search.query.profile.compiled.CompiledQueryProfile;
 import com.yahoo.search.query.profile.compiled.DimensionalValue;
 import com.yahoo.search.query.profile.types.FieldDescription;
+import com.yahoo.search.query.profile.types.QueryProfileFieldType;
 import com.yahoo.search.query.profile.types.QueryProfileType;
 
 import java.util.ArrayList;
@@ -31,7 +32,12 @@ public class QueryProfileProperties extends Properties {
 
     /** Values which has been overridden at runtime, or null if none */
     private Map<CompoundName, Object> values = null;
-    /** Query profile references which has been overridden at runtime, or null if none. Earlier values has precedence */
+
+    /**
+     * Query profile references which has been overridden at runtime, possibly to the null value to clear values,
+     * or null if none (i.e this is lazy).
+     * Earlier values has precedence
+     */
     private List<Pair<CompoundName, CompiledQueryProfile>> references = null;
 
     /** Creates an instance from a profile, throws an exception if the given profile is null */
@@ -48,20 +54,21 @@ public class QueryProfileProperties extends Properties {
     public Object get(CompoundName name, Map<String, String> context,
                       com.yahoo.processing.request.Properties substitution) {
         name = unalias(name, context);
-        Object value = null;
-        if (values != null)
-            value = values.get(name);
-        if (value == null) {
-            Pair<CompoundName, CompiledQueryProfile> reference = findReference(name);
-            if (reference != null)
-                return reference.getSecond().get(name.rest(reference.getFirst().size()), context, substitution); // yes; even if null
+        if (values != null && values.containsKey(name))
+            return values.get(name); // Returns this value, even if null
+
+        Pair<CompoundName, CompiledQueryProfile> reference = findReference(name);
+        if (reference != null) {
+            if (reference.getSecond() == null)
+                return null; // cleared
+            else
+                return reference.getSecond().get(name.rest(reference.getFirst().size()), context, substitution); // even if null
         }
 
-        if (value == null)
-            value = profile.get(name, context, substitution);
-        if (value == null)
-            value = super.get(name, context, substitution);
-        return value;
+        Object value = profile.get(name, context, substitution);
+        if (value != null)
+            return value;
+        return super.get(name, context, substitution);
     }
 
     /**
@@ -70,7 +77,7 @@ public class QueryProfileProperties extends Properties {
      * @throws IllegalArgumentException if this property cannot be set in the wrapped query profile
      */
     @Override
-    public void set(CompoundName name, Object value, Map<String,String> context) {
+    public void set(CompoundName name, Object value, Map<String, String> context) {
         // TODO: Refactor
         try {
             name = unalias(name, context);
@@ -87,8 +94,10 @@ public class QueryProfileProperties extends Properties {
 
             // Check types
             if ( ! profile.getTypes().isEmpty()) {
-                for (int i = 0; i<name.size(); i++) {
-                    QueryProfileType type = profile.getType(name.first(i), context);
+                QueryProfileType type = null;
+                for (int i = 0; i < name.size(); i++) {
+                    if (type == null) // We're on the first iteration, or no type is explicitly specified
+                        type = profile.getType(name.first(i), context);
                     if (type == null) continue;
                     String localName = name.get(i);
                     FieldDescription fieldDescription = type.getField(localName);
@@ -97,12 +106,19 @@ public class QueryProfileProperties extends Properties {
 
                     // TODO: In addition to strictness, check legality along the way
 
-                    if (i == name.size()-1 && fieldDescription != null) { // at the end of the path, check the assignment type
-                        value = fieldDescription.getType().convertFrom(value, profile.getRegistry());
-                        if (value == null)
-                            throw new IllegalArgumentException("'" + value + "' is not a " +
-                                                               fieldDescription.getType().toInstanceDescription());
+                    if (fieldDescription != null) {
+                        if (i == name.size() - 1) { // at the end of the path, check the assignment type
+                            value = fieldDescription.getType().convertFrom(value, profile.getRegistry());
+                            if (value == null)
+                                throw new IllegalArgumentException("'" + value + "' is not a " +
+                                                                   fieldDescription.getType().toInstanceDescription());
+                        }
+                        else if (fieldDescription.getType() instanceof QueryProfileFieldType) {
+                            // If a type is specified, use that instead of the type implied by the name
+                            type = ((QueryProfileFieldType) fieldDescription.getType()).getQueryProfileType();
+                        }
                     }
+
                 }
             }
 
@@ -133,12 +149,26 @@ public class QueryProfileProperties extends Properties {
     }
 
     @Override
-    public Map<String, Object> listProperties(CompoundName path, Map<String,String> context,
+    public void clearAll(CompoundName name, Map<String, String> context) {
+        if (references == null)
+            references = new ArrayList<>();
+        references.add(new Pair<>(name, null));
+
+        if (values != null)
+            values.keySet().removeIf(key -> key.hasPrefix(name));
+    }
+
+    @Override
+    public Map<String, Object> listProperties(CompoundName path, Map<String, String> context,
                                               com.yahoo.processing.request.Properties substitution) {
         path = unalias(path, context);
         if (context == null) context = Collections.emptyMap();
 
-        Map<String, Object> properties = profile.listValues(path, context, substitution);
+        Map<String, Object> properties = new HashMap<>();
+        for (var entry : profile.listValues(path, context, substitution).entrySet()) {
+            if (references != null && containsNullParentOf(path, references)) continue;
+            properties.put(entry.getKey(), entry.getValue());
+        }
         properties.putAll(super.listProperties(path, context, substitution));
 
         if (references != null) {
@@ -155,8 +185,14 @@ public class QueryProfileProperties extends Properties {
                     pathInReference = path.rest(refEntry.getFirst().size());
                     prefixToReferenceKeys = CompoundName.empty;
                 }
-                for (Map.Entry<String, Object> valueEntry : refEntry.getSecond().listValues(pathInReference, context, substitution).entrySet()) {
-                    properties.put(prefixToReferenceKeys.append(new CompoundName(valueEntry.getKey())).toString(), valueEntry.getValue());
+                if (refEntry.getSecond() == null) {
+                    if (refEntry.getFirst().hasPrefix(path))
+                        properties.put(prefixToReferenceKeys.toString(), null);
+                }
+                else {
+                    for (Map.Entry<String, Object> valueEntry : refEntry.getSecond().listValues(pathInReference, context, substitution).entrySet()) {
+                        properties.put(prefixToReferenceKeys.append(new CompoundName(valueEntry.getKey())).toString(), valueEntry.getValue());
+                    }
                 }
             }
 
@@ -231,6 +267,12 @@ public class QueryProfileProperties extends Properties {
         return null;
     }
 
+    private boolean containsNullParentOf(CompoundName path, List<Pair<CompoundName, CompiledQueryProfile>> properties) {
+        if (properties.contains(new Pair<>(path, (CompiledQueryProfile)null))) return true;
+        if (path.size() > 0 && containsNullParentOf(path.first(path.size() - 1), properties)) return true;
+        return false;
+    }
+
     CompoundName unalias(CompoundName name, Map<String,String> context) {
         if (profile.getTypes().isEmpty()) return name;
 
diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/compiled/Binding.java b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/Binding.java
index 2774bd4ebf2..88014eef46d 100644
--- a/container-search/src/main/java/com/yahoo/search/query/profile/compiled/Binding.java
+++ b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/Binding.java
@@ -106,7 +106,7 @@ public class Binding implements Comparable<Binding> {
      * Returns true if all the dimension values in this have the same values
      * in the given context.
      */
-    public boolean matches(Map<String,String> context) {
+    public boolean matches(Map<String, String> context) {
         for (int i = 0; i < dimensions.length; i++) {
             if ( ! dimensionValues[i].equals(context.get(dimensions[i]))) return false;
         }
diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfile.java b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfile.java
index 644d366e7d0..1c7a7cf3e97 100644
--- a/container-search/src/main/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfile.java
+++ b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfile.java
@@ -102,7 +102,7 @@ public class CompiledQueryProfile extends AbstractComponent implements Cloneable
      * For example, if {a.d =&gt; "a.d-value" ,a.e =&gt; "a.e-value", b.d =&gt; "b.d-value", then calling listValues("a")
      * will return {"d" =&gt; "a.d-value","e" =&gt; "a.e-value"}
      */
-    public final Map<String, Object> listValues(CompoundName prefix) {  return listValues(prefix, Collections.<String,String>emptyMap()); }
+    public final Map<String, Object> listValues(CompoundName prefix) {  return listValues(prefix, Collections.emptyMap()); }
     public final Map<String, Object> listValues(String prefix) { return listValues(new CompoundName(prefix)); }
 
     /**
@@ -134,7 +134,6 @@ public class CompiledQueryProfile extends AbstractComponent implements Cloneable
     public Map<String, Object> listValues(CompoundName prefix, Map<String, String> context, Properties substitution) {
         Map<String, Object> values = new HashMap<>();
         for (Map.Entry<CompoundName, DimensionalValue<ValueWithSource>> entry : entries.entrySet()) {
-            if ( entry.getKey().size() <= prefix.size()) continue;
             if ( ! entry.getKey().hasPrefix(prefix)) continue;
 
             ValueWithSource valueWithSource = entry.getValue().get(context);
diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/compiled/DimensionalValue.java b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/DimensionalValue.java
index b5481059ac0..0137b848dac 100644
--- a/container-search/src/main/java/com/yahoo/search/query/profile/compiled/DimensionalValue.java
+++ b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/DimensionalValue.java
@@ -96,7 +96,7 @@ public class DimensionalValue<VALUE> {
         private VALUE value = null;
 
         /** The minimal binding this holds for */
-        private Binding binding = null;
+        private Binding binding;
 
         public Value(VALUE value, Binding binding) {
             this.value = value;
diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/config/QueryProfileXMLReader.java b/container-search/src/main/java/com/yahoo/search/query/profile/config/QueryProfileXMLReader.java
index 33f07a58195..1b1cdce5890 100644
--- a/container-search/src/main/java/com/yahoo/search/query/profile/config/QueryProfileXMLReader.java
+++ b/container-search/src/main/java/com/yahoo/search/query/profile/config/QueryProfileXMLReader.java
@@ -33,7 +33,7 @@ public class QueryProfileXMLReader {
      * Reads all query profile xml files in a given directory,
      * and all type xml files from the immediate subdirectory "types/" (if any)
      *
-     * @throws RuntimeException if <code>directory</code> is not a readable directory, or if there is some error in the XML
+     * @throws IllegalArgumentException if the directory is not readable, or if there is some error in the XML
      */
     public QueryProfileRegistry read(String directory) {
         List<NamedReader> queryProfileReaders = new ArrayList<>();
@@ -58,7 +58,7 @@ public class QueryProfileXMLReader {
             return read(queryProfileTypeReaders,queryProfileReaders);
         }
         catch (IOException e) {
-            throw new IllegalArgumentException("Could not read query profiles from '" + directory + "'",e);
+            throw new IllegalArgumentException("Could not read query profiles from '" + directory + "'", e);
         }
         finally {
             closeAll(queryProfileReaders);
@@ -105,14 +105,14 @@ public class QueryProfileXMLReader {
                                                    "' must be 'query-profile-type', not '" + root.getNodeName() + "'");
             }
 
-            String idString=root.getAttribute("id");
+            String idString = root.getAttribute("id");
             if (idString == null || idString.equals(""))
                 throw new IllegalArgumentException("'" + reader.getName() + "' has no 'id' attribute in the root element");
             ComponentId id = new ComponentId(idString);
-            validateFileNameToId(reader.getName(),id,"query profile type");
+            validateFileNameToId(reader.getName(), id,"query profile type");
             QueryProfileType type = new QueryProfileType(id);
-            type.setMatchAsPath(XML.getChild(root,"match") != null);
-            type.setStrict(XML.getChild(root,"strict") != null);
+            type.setMatchAsPath(XML.getChild(root, "match") != null);
+            type.setStrict(XML.getChild(root, "strict") != null);
             registry.register(type);
             queryProfileTypeElements.add(root);
         }
@@ -145,7 +145,7 @@ public class QueryProfileXMLReader {
                 queryProfile.setType(type);
             }
 
-            Element dimensions = XML.getChild(root,"dimensions");
+            Element dimensions = XML.getChild(root, "dimensions");
             if (dimensions != null)
                 queryProfile.setDimensions(toArray(XML.getValue(dimensions)));
 
@@ -215,7 +215,7 @@ public class QueryProfileXMLReader {
             try {
                 String fieldTypeName = field.getAttribute("type");
                 if (fieldTypeName == null) throw new IllegalArgumentException("Field '" + field + "' has no 'type' attribute");
-                FieldType fieldType=FieldType.fromString(fieldTypeName,registry);
+                FieldType fieldType = FieldType.fromString(fieldTypeName, registry);
                 type.addField(new FieldDescription(name,
                                                    fieldType,
                                                    field.getAttribute("alias"),
diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/types/FieldDescription.java b/container-search/src/main/java/com/yahoo/search/query/profile/types/FieldDescription.java
index b8290fa092b..6c30f1a8b05 100644
--- a/container-search/src/main/java/com/yahoo/search/query/profile/types/FieldDescription.java
+++ b/container-search/src/main/java/com/yahoo/search/query/profile/types/FieldDescription.java
@@ -97,7 +97,7 @@ public class FieldDescription implements Comparable<FieldDescription> {
         this.type = type;
 
         // Forbidden until we can figure out the right semantics
-        if (name.isCompound() && ! aliases.isEmpty()) throw new IllegalArgumentException("Aliases is not allowed with compound names");
+        if (name.isCompound() && ! aliases.isEmpty()) throw new IllegalArgumentException("Aliases are not allowed with compound names");
 
         this.aliases = ImmutableList.copyOf(aliases);
         this.mandatory = mandatory;
diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryProfileType.java b/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryProfileType.java
index 07c9e4475ec..c02aada2062 100644
--- a/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryProfileType.java
+++ b/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryProfileType.java
@@ -6,6 +6,7 @@ import com.google.common.collect.ImmutableMap;
 import com.yahoo.component.ComponentId;
 import com.yahoo.component.provider.FreezableSimpleComponent;
 import com.yahoo.processing.request.CompoundName;
+import com.yahoo.search.query.profile.OverridableQueryProfile;
 import com.yahoo.search.query.profile.QueryProfile;
 
 import java.util.ArrayList;
@@ -24,6 +25,7 @@ import static com.yahoo.text.Lowercase.toLowerCase;
 public class QueryProfileType extends FreezableSimpleComponent {
 
     private final CompoundName componentIdAsCompoundName;
+
     /** The fields of this query profile type */
     private Map<String, FieldDescription> fields;
 
@@ -217,25 +219,38 @@ public class QueryProfileType extends FreezableSimpleComponent {
 
     /** Returns the type of the given query profile type declared as a field in this */
     public QueryProfileType getType(String localName) {
-        FieldDescription fieldDescription=getField(localName);
-        if (fieldDescription ==null) return null;
+        FieldDescription fieldDescription = getField(localName);
+        if (fieldDescription == null) return null;
         if ( ! (fieldDescription.getType() instanceof QueryProfileFieldType)) return null;
         return ((QueryProfileFieldType) fieldDescription.getType()).getQueryProfileType();
     }
 
+    /** Returns the field type of the given name under this, of null if none */
+    public FieldType getFieldType(CompoundName name) {
+        FieldDescription field = getField(name.first());
+        if (field == null) return null;
+
+        FieldType fieldType = field.getType();
+        if (name.size() == 1) return fieldType;
+
+        if ( ! (fieldType instanceof QueryProfileFieldType)) return null;
+
+        return ((QueryProfileFieldType)fieldType).getQueryProfileType().getFieldType(name.rest());
+    }
+
     /**
      * Returns the description of the field with the given name in this type or an inherited type
      * (depth first left to right search). Returns null if the field is not defined in this or an inherited profile.
      */
     public FieldDescription getField(String name) {
-        FieldDescription field=fields.get(name);
-        if ( field!=null ) return field;
+        FieldDescription field = fields.get(name);
+        if ( field != null ) return field;
 
         if ( isFrozen() ) return null; // Inherited are collapsed into this
 
         for (QueryProfileType inheritedType : this.inherited() ) {
-            field=inheritedType.getField(name);
-            if (field!=null) return field;
+            field = inheritedType.getField(name);
+            if (field != null) return field;
         }
 
         return null;
@@ -276,7 +291,7 @@ public class QueryProfileType extends FreezableSimpleComponent {
             // Add (/to) a query profile type containing the rest of the name.
             // (we do not need the field description settings for intermediate query profile types
             // as the leaf entry will enforce them)
-            QueryProfileType type = getOrCreateQueryProfileType(name.first(), registry);
+            QueryProfileType type = extendOrCreateQueryProfileType(name.first(), registry);
             type.addField(fieldDescription.withName(name.rest()), registry);
         }
         else {
@@ -288,27 +303,42 @@ public class QueryProfileType extends FreezableSimpleComponent {
             addAlias(alias, fieldDescription.getName());
     }
 
-    private QueryProfileType getOrCreateQueryProfileType(String name, QueryProfileTypeRegistry registry) {
+    private QueryProfileType extendOrCreateQueryProfileType(String name, QueryProfileTypeRegistry registry) {
+        QueryProfileType type = null;
         FieldDescription fieldDescription = getField(name);
         if (fieldDescription != null) {
-            if ( ! ( fieldDescription.getType() instanceof QueryProfileFieldType))
+            if ( ! (fieldDescription.getType() instanceof QueryProfileFieldType))
                 throw new IllegalArgumentException("Cannot use name '" + name + "' as a prefix because it is " +
                                                    "already a " + fieldDescription.getType());
             QueryProfileFieldType fieldType = (QueryProfileFieldType) fieldDescription.getType();
-            QueryProfileType type = fieldType.getQueryProfileType();
-            if (type == null) { // an as-yet untyped reference; add type
-                type = new QueryProfileType(name);
-                registry.register(type.getId(), type);
-                fields.put(name, fieldDescription.withType(new QueryProfileFieldType(type)));
-            }
-            return type;
+            type = fieldType.getQueryProfileType();
+        }
+
+        if (type == null) {
+            type = registry.getComponent(name);
+        }
+
+        // found in registry but not already added in *this* type (getField also checks parents): extend it
+        if (type != null && ! fields.containsKey(name)) {
+            type = new QueryProfileType(ComponentId.createAnonymousComponentId(type.getIdString()),
+                                        new HashMap<>(),
+                                        List.of(type));
+        }
+
+        if (type == null) { // create it
+            type = new QueryProfileType(ComponentId.createAnonymousComponentId(name));
+        }
+
+        if (fieldDescription == null) {
+            fieldDescription = new FieldDescription(name, new QueryProfileFieldType(type));
         }
         else {
-            QueryProfileType type = new QueryProfileType(name);
-            registry.register(type.getId(), type);
-            fields.put(name, new FieldDescription(name, new QueryProfileFieldType(type)));
-            return type;
+            fieldDescription = fieldDescription.withType(new QueryProfileFieldType(type));
         }
+
+        registry.register(type);
+        fields.put(name, fieldDescription);
+        return type;
     }
 
     private void addAlias(String alias, String field) {
@@ -362,6 +392,7 @@ public class QueryProfileType extends FreezableSimpleComponent {
         return other.getId().equals(this.getId());
     }
 
+    @Override
     public String toString() {
         return "query profile type '" + getId() + "'";
     }
diff --git a/container-search/src/main/java/com/yahoo/search/query/properties/PropertyAliases.java b/container-search/src/main/java/com/yahoo/search/query/properties/PropertyAliases.java
index a4a82d27f8e..83e8dd530ad 100644
--- a/container-search/src/main/java/com/yahoo/search/query/properties/PropertyAliases.java
+++ b/container-search/src/main/java/com/yahoo/search/query/properties/PropertyAliases.java
@@ -37,6 +37,8 @@ public class PropertyAliases extends Properties {
      * @return the real name if an alias or the input name itself
      */
     protected CompoundName unalias(CompoundName nameOrAlias) {
+        if (aliases.isEmpty()) return nameOrAlias;
+        if (nameOrAlias.size() > 1) return nameOrAlias; // aliases are simple names
         CompoundName properName = aliases.get(nameOrAlias.getLowerCasedName());
         return (properName != null) ? properName : nameOrAlias;
     }
diff --git a/container-search/src/main/java/com/yahoo/search/query/properties/PropertyMap.java b/container-search/src/main/java/com/yahoo/search/query/properties/PropertyMap.java
index 30fc98ac6b1..643e215daef 100644
--- a/container-search/src/main/java/com/yahoo/search/query/properties/PropertyMap.java
+++ b/container-search/src/main/java/com/yahoo/search/query/properties/PropertyMap.java
@@ -26,7 +26,10 @@ public class PropertyMap extends Properties {
     /** The properties of this */
     private Map<CompoundName, Object> properties = new LinkedHashMap<>();
 
-    public void set(CompoundName name, Object value, Map<String,String> context) {
+    public void set(CompoundName name, Object value, Map<String, String> context) {
+        if (value == null) // Both clear and forward
+            properties.remove(name);
+
         if (shouldSet(name, value))
             properties.put(name, value);
         else
@@ -37,7 +40,7 @@ public class PropertyMap extends Properties {
      * Return true if this value should be set in this map, false if the set should be propagated instead
      * This default implementation always returns true.
      */
-    protected boolean shouldSet(CompoundName name,Object value) { return true; }
+    protected boolean shouldSet(CompoundName name, Object value) { return true; }
 
     @Override
     public Object get(CompoundName name, Map<String,String> context,
diff --git a/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java b/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java
index a4c150b606e..96f73e925af 100644
--- a/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java
+++ b/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java
@@ -44,7 +44,7 @@ public class QueryProperties extends Properties {
 
     @Override
     public Object get(CompoundName key,
-                      Map<String,String> context,
+                      Map<String, String> context,
                       com.yahoo.processing.request.Properties substitution) {
         if (key.size() == 2 && key.first().equals(Model.MODEL)) {
             Model model = query.getModel();
@@ -294,7 +294,7 @@ public class QueryProperties extends Properties {
                 super.set(key,value,context);
         }
         catch (Exception e) { // Make sure error messages are informative. This should be moved out of this properties implementation
-            if (e.getMessage().startsWith("Could not set"))
+            if (e.getMessage() != null && e.getMessage().startsWith("Could not set"))
                 throw e;
             else
                 throw new IllegalArgumentException("Could not set '" + key + "' to '" + value + "'", e);
diff --git a/container-search/src/main/java/com/yahoo/search/query/properties/RequestContextProperties.java b/container-search/src/main/java/com/yahoo/search/query/properties/RequestContextProperties.java
index ee09521fa74..6cf27fc9a3e 100644
--- a/container-search/src/main/java/com/yahoo/search/query/properties/RequestContextProperties.java
+++ b/container-search/src/main/java/com/yahoo/search/query/properties/RequestContextProperties.java
@@ -7,7 +7,7 @@ import com.yahoo.search.query.Properties;
 import java.util.Map;
 
 /**
- * Turns get(name) into get(name,request) using the request given at construction time.
+ * Turns get(name) into get(name, request) using the request given at construction time.
  * This is used to allow the query's request to be supplied to all property requests
  * without forcing users of the query.properties() to supply this explicitly.
  *
@@ -22,18 +22,18 @@ public class RequestContextProperties extends Properties {
     }
 
     @Override
-    public Object get(CompoundName name,Map<String,String> context,
+    public Object get(CompoundName name, Map<String,String> context,
                       com.yahoo.processing.request.Properties substitution) {
         return super.get(name, context == null ? requestMap : context, substitution);
     }
 
     @Override
-    public void set(CompoundName name,Object value,Map<String,String> context) {
+    public void set(CompoundName name, Object value, Map<String,String> context) {
         super.set(name, value, context == null ? requestMap : context);
     }
 
     @Override
-    public Map<String, Object> listProperties(CompoundName path,Map<String,String> context,
+    public Map<String, Object> listProperties(CompoundName path, Map<String,String> context,
                                               com.yahoo.processing.request.Properties substitution) {
         return super.listProperties(path, context == null ? requestMap : context, substitution);
     }
diff --git a/container-search/src/main/java/com/yahoo/search/querytransform/VespaLowercasingSearcher.java b/container-search/src/main/java/com/yahoo/search/querytransform/VespaLowercasingSearcher.java
index 1e8f436a05a..25488aa7bbc 100644
--- a/container-search/src/main/java/com/yahoo/search/querytransform/VespaLowercasingSearcher.java
+++ b/container-search/src/main/java/com/yahoo/search/querytransform/VespaLowercasingSearcher.java
@@ -44,4 +44,5 @@ public class VespaLowercasingSearcher extends LowercasingSearcher {
         Index index = indexFacts.getIndex(sb.toString());
         return index.isLowercase() || index.isAttribute();
     }
+
 }
diff --git a/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java b/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java
index af453983f89..31f8194b3b7 100644
--- a/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java
+++ b/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java
@@ -7,6 +7,7 @@ import com.fasterxml.jackson.core.JsonGenerator;
 import com.fasterxml.jackson.core.TreeNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.base.Preconditions;
+import com.yahoo.container.logging.TraceRenderer;
 import com.yahoo.data.JsonProducer;
 import com.yahoo.data.access.Inspectable;
 import com.yahoo.data.access.Inspector;
@@ -44,8 +45,6 @@ import com.yahoo.search.result.Hit;
 import com.yahoo.search.result.HitGroup;
 import com.yahoo.search.result.NanNumber;
 import com.yahoo.tensor.Tensor;
-import com.yahoo.yolean.trace.TraceNode;
-import com.yahoo.yolean.trace.TraceVisitor;
 import org.json.JSONArray;
 import org.json.JSONObject;
 
@@ -111,10 +110,6 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> {
     private static final String ROOT = "root";
     private static final String SOURCE = "source";
     private static final String TOTAL_COUNT = "totalCount";
-    private static final String TRACE = "trace";
-    private static final String TRACE_CHILDREN = "children";
-    private static final String TRACE_MESSAGE = "message";
-    private static final String TRACE_TIMESTAMP = "timestamp";
     private static final String TIMING = "timing";
     private static final String QUERY_TIME = "querytime";
     private static final String SUMMARY_FETCH_TIME = "summaryfetchtime";
@@ -132,145 +127,6 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> {
     private LongSupplier timeSource;
     private OutputStream stream;
 
-    private class TraceRenderer extends TraceVisitor {
-        private final long basetime;
-        private boolean hasFieldName = false;
-        int emittedChildNesting = 0;
-        int currentChildNesting = 0;
-        private boolean insideOpenObject = false;
-
-        TraceRenderer(long basetime) {
-            this.basetime = basetime;
-        }
-
-        @Override
-        public void entering(TraceNode node) {
-            ++currentChildNesting;
-        }
-
-        @Override
-        public void leaving(TraceNode node) {
-            conditionalEndObject();
-            if (currentChildNesting == emittedChildNesting) {
-                try {
-                    generator.writeEndArray();
-                    generator.writeEndObject();
-                } catch (IOException e) {
-                    throw new TraceRenderWrapper(e);
-                }
-                --emittedChildNesting;
-            }
-            --currentChildNesting;
-        }
-
-        @Override
-        public void visit(TraceNode node) {
-            try {
-                doVisit(node.timestamp(), node.payload(), node.children().iterator().hasNext());
-            } catch (IOException e) {
-                throw new TraceRenderWrapper(e);
-            }
-        }
-
-        private void doVisit(long timestamp, Object payload, boolean hasChildren) throws IOException {
-            boolean dirty = false;
-            if (timestamp != 0L) {
-                header();
-                generator.writeStartObject();
-                generator.writeNumberField(TRACE_TIMESTAMP, timestamp - basetime);
-                dirty = true;
-            }
-            if (payload != null) {
-                if (!dirty) {
-                    header();
-                    generator.writeStartObject();
-                }
-                generator.writeFieldName(TRACE_MESSAGE);
-                fieldConsumer.renderFieldContentsDirect(payload);
-                dirty = true;
-            }
-            if (dirty) {
-                if (!hasChildren) {
-                    generator.writeEndObject();
-                } else {
-                    setInsideOpenObject(true);
-                }
-            }
-        }
-
-        private void header() {
-            fieldName();
-            for (int i = 0; i < (currentChildNesting - emittedChildNesting); ++i) {
-                startChildArray();
-            }
-            emittedChildNesting = currentChildNesting;
-        }
-
-        private void startChildArray() {
-            try {
-                conditionalStartObject();
-                generator.writeArrayFieldStart(TRACE_CHILDREN);
-            } catch (IOException e) {
-                throw new TraceRenderWrapper(e);
-            }
-        }
-
-        private void conditionalStartObject() throws IOException {
-            if (!isInsideOpenObject()) {
-                generator.writeStartObject();
-            } else {
-                setInsideOpenObject(false);
-            }
-        }
-
-        private void conditionalEndObject() {
-            if (isInsideOpenObject()) {
-                // This triggers if we were inside a data node with payload and
-                // subnodes, but none of the subnodes contained data
-                try {
-                    generator.writeEndObject();
-                    setInsideOpenObject(false);
-                } catch (IOException e) {
-                    throw new TraceRenderWrapper(e);
-                }
-            }
-        }
-
-        private void fieldName() {
-            if (hasFieldName) {
-                return;
-            }
-
-            try {
-                generator.writeFieldName(TRACE);
-            } catch (IOException e) {
-                throw new TraceRenderWrapper(e);
-            }
-            hasFieldName = true;
-        }
-
-        boolean isInsideOpenObject() {
-            return insideOpenObject;
-        }
-
-        void setInsideOpenObject(boolean insideOpenObject) {
-            this.insideOpenObject = insideOpenObject;
-        }
-    }
-
-    private static final class TraceRenderWrapper extends RuntimeException {
-
-        /**
-         * Should never be serialized, but this is still needed.
-         */
-        private static final long serialVersionUID = 2L;
-
-        TraceRenderWrapper(IOException wrapped) {
-            super(wrapped);
-        }
-
-    }
-
     public JsonRenderer() {
         this(null);
     }
@@ -352,8 +208,8 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> {
             long basetime = trace.traceNode().timestamp();
             if (basetime == 0L)
                 basetime = getResult().getElapsedTime().first();
-            trace.accept(new TraceRenderer(basetime));
-        } catch (TraceRenderWrapper e) {
+            trace.accept(new TraceRenderer(generator, fieldConsumer, basetime));
+        } catch (TraceRenderer.TraceRenderWrapper e) {
             throw new IOException(e);
         }
     }
@@ -641,11 +497,9 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> {
 
     private String getJsonCallback() {
         Result result = getResult();
-        if (result != null) {
-            Query query = result.getQuery();
-            if (query != null) {
-                return query.properties().getString(JSON_CALLBACK, null);
-            }
+        Query query = result.getQuery();
+        if (query != null) {
+            return query.properties().getString(JSON_CALLBACK, null);
         }
         return null;
     }
@@ -671,7 +525,7 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> {
      * This instance is reused for all hits of a Result since we are in a single-threaded context
      * and want to limit object creation.
      */
-    public static class FieldConsumer implements Hit.RawUtf8Consumer {
+    public static class FieldConsumer implements Hit.RawUtf8Consumer, TraceRenderer.FieldConsumer {
 
         private final JsonGenerator generator;
         private final boolean debugRendering;
@@ -788,11 +642,12 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> {
             if (field instanceof Inspectable && ! (field instanceof FeatureData)) {
                 renderInspector(((Inspectable)field).inspect());
             } else {
-                renderFieldContentsDirect(field);
+                accept(field);
             }
         }
 
-        private void renderFieldContentsDirect(Object field) throws IOException {
+        @Override
+        public void accept(Object field) throws IOException {
             if (field == null) {
                 generator.writeNull();
             } else if (field instanceof Boolean) {
diff --git a/container-search/src/main/java/com/yahoo/search/result/Hit.java b/container-search/src/main/java/com/yahoo/search/result/Hit.java
index fc416c0d930..c14b3f39bc1 100644
--- a/container-search/src/main/java/com/yahoo/search/result/Hit.java
+++ b/container-search/src/main/java/com/yahoo/search/result/Hit.java
@@ -473,7 +473,7 @@ public class Hit extends ListenableFreezableClass implements Data, Comparable<Hi
     private Map<String, Object> getFieldMap(int minSize) {
         if (fields == null) {
             // Compensate for loadfactor and then some, rounded up....
-            fields = new LinkedHashMap<>(2*minSize);
+            fields = new LinkedHashMap<>(2 * minSize);
         }
         return fields;
     }
@@ -505,7 +505,7 @@ public class Hit extends ListenableFreezableClass implements Data, Comparable<Hi
     }
 
     /** Returns the types of this as a modifiable set. Modifications to this set are directly reflected in this hit */
-    //TODO This shoudld not be exposed as a modifiable set
+    // TODO: This should not be exposed as a modifiable set
     public Set<String> types() {
         if (types == null)
             types = new ArraySet<>(1);
diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java b/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java
index 5cc34ff5b28..84fe88d0292 100644
--- a/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java
+++ b/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java
@@ -3,7 +3,6 @@ package com.yahoo.search.searchchain;
 
 import com.yahoo.component.chain.Chain;
 import com.yahoo.language.Linguistics;
-import com.yahoo.log.LogLevel;
 import com.yahoo.prelude.IndexFacts;
 import com.yahoo.prelude.Ping;
 import com.yahoo.prelude.Pong;
@@ -11,7 +10,6 @@ import com.yahoo.prelude.query.parser.SpecialTokenRegistry;
 import com.yahoo.processing.Processor;
 import com.yahoo.processing.Request;
 import com.yahoo.processing.Response;
-import com.yahoo.protect.Validator;
 import com.yahoo.search.Query;
 import com.yahoo.search.Result;
 import com.yahoo.search.Searcher;
diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/testutil/DocumentSourceSearcher.java b/container-search/src/main/java/com/yahoo/search/searchchain/testutil/DocumentSourceSearcher.java
index d39a488626b..e346a766738 100644
--- a/container-search/src/main/java/com/yahoo/search/searchchain/testutil/DocumentSourceSearcher.java
+++ b/container-search/src/main/java/com/yahoo/search/searchchain/testutil/DocumentSourceSearcher.java
@@ -97,11 +97,10 @@ public class DocumentSourceSearcher extends Searcher {
     public Result search(Query query, Execution execution)  {
         queryCount++;
         Result r = unFilledResults.get(getQueryKeyClone(query));
-        if (r == null) {
+        if (r == null)
             r = defaultFilledResult.clone();
-        } else {
+        else
             r = r.clone();
-        }
 
         r.setQuery(query);
         r.hits().trim(query.getOffset(), query.getHits());
@@ -182,11 +181,8 @@ public class DocumentSourceSearcher extends Searcher {
      * reset. For testing - not reliable if multiple threads makes
      * queries simultaneously
      */
-    public int getQueryCount() {
-        return queryCount;
-    }
+    public int getQueryCount() { return queryCount; }
+
+    public void resetQueryCount() { queryCount = 0; }
 
-    public void resetQueryCount() {
-        queryCount=0;
-    }
 }
diff --git a/container-search/src/main/java/com/yahoo/search/searchers/ValidateNearestNeighborSearcher.java b/container-search/src/main/java/com/yahoo/search/searchers/ValidateNearestNeighborSearcher.java
index 8cae081cada..76b8c1ef8a2 100644
--- a/container-search/src/main/java/com/yahoo/search/searchers/ValidateNearestNeighborSearcher.java
+++ b/container-search/src/main/java/com/yahoo/search/searchers/ValidateNearestNeighborSearcher.java
@@ -4,37 +4,33 @@ package com.yahoo.search.searchers;
 
 import com.google.common.annotations.Beta;
 
-import com.yahoo.container.QrSearchersConfig;
 import com.yahoo.prelude.query.Item;
 import com.yahoo.prelude.query.NearestNeighborItem;
-import com.yahoo.prelude.query.QueryCanonicalizer;
 import com.yahoo.prelude.query.ToolBox;
 import com.yahoo.search.Query;
 import com.yahoo.search.Result;
 import com.yahoo.search.Searcher;
+import com.yahoo.search.grouping.vespa.GroupingExecutor;
 import com.yahoo.search.query.ranking.RankProperties;
 import com.yahoo.search.result.ErrorMessage;
 import com.yahoo.search.searchchain.Execution;
 import com.yahoo.tensor.Tensor;
 import com.yahoo.tensor.TensorType;
 import com.yahoo.vespa.config.search.AttributesConfig;
-import com.yahoo.yolean.chain.After;
+import com.yahoo.yolean.chain.Before;
 
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 
-// This depends on tensors in query.getRanking which are moved to rank.properties during query.prepare()
-// Query.prepare is done at the same time as canonicalization (by GroupingExecutor), so use that constraint.
-@After(QueryCanonicalizer.queryCanonicalization)
-
 /**
  * Validates any NearestNeighborItem query items.
  *
  * @author arnej
  */
 @Beta
+@Before(GroupingExecutor.COMPONENT_NAME) // Must happen before query.prepare()
 public class ValidateNearestNeighborSearcher extends Searcher {
 
     private Map<String, TensorType> validAttributes = new HashMap<>();
@@ -56,7 +52,7 @@ public class ValidateNearestNeighborSearcher extends Searcher {
     }
 
     private Optional<ErrorMessage> validate(Query query) {
-        NNVisitor visitor = new NNVisitor(query.getRanking().getProperties(), validAttributes);
+        NNVisitor visitor = new NNVisitor(query.getRanking().getProperties(), validAttributes, query);
         ToolBox.visit(visitor, query.getModel().getQueryTree().getRoot());
         return visitor.errorMessage;
     }
@@ -65,26 +61,26 @@ public class ValidateNearestNeighborSearcher extends Searcher {
 
         public Optional<ErrorMessage> errorMessage = Optional.empty();
 
-        private RankProperties rankProperties;
-        private Map<String, TensorType> validAttributes;
+        private final RankProperties rankProperties;
+        private final Map<String, TensorType> validAttributes;
+        private final Query query;
 
-        public NNVisitor(RankProperties rankProperties, Map<String, TensorType> validAttributes) {
+        public NNVisitor(RankProperties rankProperties, Map<String, TensorType> validAttributes, Query query) {
             this.rankProperties = rankProperties;
             this.validAttributes = validAttributes;
+            this.query = query;
         }
 
         @Override
         public boolean visit(Item item) {
             if (item instanceof NearestNeighborItem) {
-                validate((NearestNeighborItem) item);
+                String error = validate((NearestNeighborItem)item);
+                if (error != null)
+                    errorMessage = Optional.of(ErrorMessage.createIllegalQuery(error));
             }
             return true;
         }
 
-        private void setError(String description) {
-            errorMessage = Optional.of(ErrorMessage.createIllegalQuery(description));
-        }
-
         private static boolean isCompatible(TensorType lhs, TensorType rhs) {
             return lhs.dimensions().equals(rhs.dimensions());
         }
@@ -98,50 +94,27 @@ public class ValidateNearestNeighborSearcher extends Searcher {
             return true;
         }
 
-        private void validate(NearestNeighborItem item) {
-            int target = item.getTargetNumHits();
-            if (target < 1) {
-                setError(item.toString() + " has invalid targetNumHits");
-                return;
-            }
-            String qprop = item.getQueryTensorName();
-            List<Object> rankPropValList = rankProperties.asMap().get(qprop);
-            if (rankPropValList == null) {
-                setError(item.toString() + " query tensor not found");
-                return;
-            }
-            if (rankPropValList.size() != 1) {
-                setError(item.toString() + " query tensor does not have a single value");
-                return;
-            }
-            Object rankPropValue = rankPropValList.get(0);
-            if (! (rankPropValue instanceof Tensor)) {
-                setError(item.toString() + " query tensor should be a tensor, was: "+
-                    (rankPropValue == null ? "null" : rankPropValue.getClass().toString()));
-                return;
-            }
-            Tensor qTensor = (Tensor)rankPropValue;
-            TensorType qTensorType = qTensor.type();
-
-            String field = item.getIndexName();
-            if (validAttributes.containsKey(field)) {
-                TensorType fTensorType = validAttributes.get(field);
-                if (fTensorType == null) {
-                    setError(item.toString() + " field is not a tensor");
-                    return;
-                }
-                if (! isCompatible(fTensorType, qTensorType)) {
-                    setError(item.toString() + " field type "+fTensorType+" does not match query tensor type "+qTensorType);
-                    return;
-                }
-                if (! isDenseVector(fTensorType)) {
-                    setError(item.toString() + " tensor type "+fTensorType+" is not a dense vector");
-                    return;
-                }
-            } else {
-                setError(item.toString() + " field is not an attribute");
-                return;
-            }
+        /** Returns an error message if this is invalid, or null if it is valid */
+        private String validate(NearestNeighborItem item) {
+            if (item.getTargetNumHits() < 1)
+                return item + " has invalid targetNumHits " + item.getTargetNumHits() + ": Must be >= 1";
+
+            String queryFeatureName = "query(" + item.getQueryTensorName() + ")";
+            Optional<Tensor> queryTensor = query.getRanking().getFeatures().getTensor(queryFeatureName);
+            if (queryTensor.isEmpty())
+                return item + " requires a tensor rank feature " + queryFeatureName + " but this is not present";
+
+            if ( ! validAttributes.containsKey(item.getIndexName()))
+                return item + " field is not an attribute";
+            TensorType fieldType = validAttributes.get(item.getIndexName());
+            if (fieldType == null) return item + " field is not a tensor";
+            if ( ! isDenseVector(fieldType))
+                return item + " tensor type " + fieldType + " is not a dense vector";
+
+            if ( ! isCompatible(fieldType, queryTensor.get().type()))
+                return item + " field type " + fieldType + " does not match query type " + queryTensor.get().type();
+
+            return null;
         }
 
         @Override
diff --git a/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java b/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java
index 6eef1252998..dd52b9e19b8 100644
--- a/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java
+++ b/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java
@@ -701,7 +701,18 @@ public class VespaSerializer {
             destination.append(leafAnnotations(item));
             comma(destination, initLen);
             int targetNumHits = item.getTargetNumHits();
-            destination.append("\"targetNumHits\": ").append(targetNumHits);
+            annotationKey(destination, "targetNumHits").append(targetNumHits);
+            int explore = item.getHnswExploreAdditionalHits();
+            if (explore != 0) {
+                comma(destination, initLen);
+                String key = YqlParser.HNSW_EXPLORE_ADDITIONAL_HITS;
+                annotationKey(destination, key).append(explore);
+            }
+            boolean allow_approx = item.getAllowApproximate();
+            if (! allow_approx) {
+                comma(destination, initLen);
+                annotationKey(destination, "approximate").append(allow_approx);
+            }
             destination.append("}]");
             destination.append(NEAREST_NEIGHBOR).append('(');
             destination.append(item.getIndexName()).append(", ");
@@ -1347,6 +1358,11 @@ public class VespaSerializer {
         }
     }
 
+    private static StringBuilder annotationKey(StringBuilder annotation, String val) {
+        annotation.append("\"").append(val).append("\": ");
+        return annotation;
+    }
+
     private static void comma(StringBuilder annotation, int initLen) {
         if (annotation.length() > initLen) {
             annotation.append(", ");
diff --git a/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java
index 8d013e501e8..f4560806dd2 100644
--- a/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java
+++ b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java
@@ -137,6 +137,7 @@ public class YqlParser implements Parser {
     static final String ACCENT_DROP = "accentDrop";
     static final String ALTERNATIVES = "alternatives";
     static final String AND_SEGMENTING = "andSegmenting";
+    static final String APPROXIMATE = "approximate";
     static final String BOUNDS = "bounds";
     static final String BOUNDS_LEFT_OPEN = "leftOpen";
     static final String BOUNDS_OPEN = "open";
@@ -149,6 +150,7 @@ public class YqlParser implements Parser {
     static final String EQUIV = "equiv";
     static final String FILTER = "filter";
     static final String HIT_LIMIT = "hitLimit";
+    static final String HNSW_EXPLORE_ADDITIONAL_HITS = "hnsw.exploreAdditionalHits";
     static final String IMPLICIT_TRANSFORMS = "implicitTransforms";
     static final String LABEL = "label";
     static final String NEAR = "near";
@@ -421,6 +423,14 @@ public class YqlParser implements Parser {
         if (targetNumHits != null) {
             item.setTargetNumHits(targetNumHits);
         }
+        Integer hnswExploreAdditionalHits = getAnnotation(ast, HNSW_EXPLORE_ADDITIONAL_HITS,
+                Integer.class, null, "number of extra hits to explore for HNSW algorithm");
+        if (hnswExploreAdditionalHits != null) {
+            item.setHnswExploreAdditionalHits(hnswExploreAdditionalHits);
+        }
+        Boolean allowApproximate = getAnnotation(ast, APPROXIMATE,
+                Boolean.class, Boolean.TRUE, "allow approximate nearest neighbor search");
+        item.setAllowApproximate(allowApproximate);
         String label = getAnnotation(ast, LABEL, String.class, null, "item label");
         if (label != null) {
                 item.setLabel(label);
diff --git a/container-search/src/main/resources/configdefinitions/qr-start.def b/container-search/src/main/resources/configdefinitions/qr-start.def
index 031877ada81..95e9d4575dd 100644
--- a/container-search/src/main/resources/configdefinitions/qr-start.def
+++ b/container-search/src/main/resources/configdefinitions/qr-start.def
@@ -20,6 +20,9 @@ jvm.minHeapsize int default=1536 restart
 ## Stack size (in kilobytes)
 jvm.stacksize int default=512 restart
 
+## CompressedOOps size in megabytes
+jvm.compressedClassSpaceSize int default=32 restart
+
 ## Base value of maximum direct memory size (in megabytes)
 jvm.baseMaxDirectMemorySize int default=75 restart
 
diff --git a/container-search/src/main/resources/configdefinitions/strict-contracts.def b/container-search/src/main/resources/configdefinitions/strict-contracts.def
index f9dd788c054..5ceb37db8d1 100644
--- a/container-search/src/main/resources/configdefinitions/strict-contracts.def
+++ b/container-search/src/main/resources/configdefinitions/strict-contracts.def
@@ -1,6 +1,7 @@
 # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 namespace=search.federation
 
+## DEPRECATED: This config will be removed on Vespa 8
 ## A config to control whether to activate strict adherence to public contracts
 ## in the container. Usually, the container tries to do a best effort of hiding
 ## some undesirable effects of the the public contracts. Modifying this config
@@ -11,11 +12,9 @@ namespace=search.federation
 ## can be construed to be unnecessary.
 searchchains bool default=false
 
-# WARNING: Beta feature, might be removed soon.
-# Propagate source.(sourceName).{QueryProperties.PER_SOURCE_QUERY_PROPERTIES} and
-#           provider.(providerName).{QueryProperties.PER_SOURCE_QUERY_PROPERTIES}
-# to the outgoing query.
-# All means all in QueryProperties.PER_SOURCE_QUERY_PROPERTIES
-# OFFSET_HITS means  {Query.HITS, Query.OFFSET}
-# NONE means {}
-propagateSourceProperties enum {ALL, OFFSET_HITS, NONE} default=ALL
+# EVERY, // Propagate any property starting by source.[sourceName] and provider.[providerName]
+# NATIVE, // Propagate native properties only
+# ALL, // Deprecated synonym of NATIVE
+# OFFSET_HITS, // Propagate offset ands hits only
+# NONE // propagate no properties
+propagateSourceProperties enum {EVERY, NATIVE, ALL, OFFSET_HITS, NONE} default=EVERY
diff --git a/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java
index 5b6a4b68930..8b7a57c38e7 100644
--- a/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java
+++ b/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java
@@ -22,6 +22,7 @@ import com.yahoo.search.Query;
 import com.yahoo.search.Result;
 import com.yahoo.search.config.ClusterConfig;
 import com.yahoo.search.dispatch.Dispatcher;
+import com.yahoo.search.dispatch.rpc.RpcResourcePool;
 import com.yahoo.search.result.Hit;
 import com.yahoo.search.searchchain.Execution;
 import com.yahoo.vespa.config.search.DispatchConfig;
@@ -514,8 +515,10 @@ public class ClusterSearcherTestCase {
         DocumentdbInfoConfig.Builder documentDbConfig = new DocumentdbInfoConfig.Builder();
         documentDbConfig.documentdb(new DocumentdbInfoConfig.Documentdb.Builder().name("type1"));
 
-        Dispatcher dispatcher = new Dispatcher(new ComponentId("test-id"),
-                                               new DispatchConfig.Builder().build(),
+        DispatchConfig dispatchConfig = new DispatchConfig.Builder().build();
+        Dispatcher dispatcher = new Dispatcher(new RpcResourcePool(dispatchConfig),
+                                               ComponentId.createAnonymousComponentId("test-id"),
+                                               dispatchConfig,
                                                createClusterInfoConfig(),
                                                vipStatus,
                                                new MockMetric());
diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java
index b5e54b46e5a..63475c9c189 100644
--- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java
+++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java
@@ -14,7 +14,6 @@ import com.yahoo.prelude.fastsearch.SummaryParameters;
 import com.yahoo.search.Query;
 import com.yahoo.search.Result;
 import com.yahoo.search.Searcher;
-import com.yahoo.search.dispatch.Dispatcher;
 import com.yahoo.search.dispatch.rpc.RpcResourcePool;
 import com.yahoo.search.dispatch.searchcluster.Node;
 import com.yahoo.search.grouping.GroupingRequest;
@@ -151,7 +150,8 @@ public class FastSearcherTestCase {
         VipStatus vipStatus = new VipStatus(b.build());
         List<Node> nodes_1 = ImmutableList.of(new Node(0, "host0", 0));
         RpcResourcePool rpcPool_1 = new RpcResourcePool(MockDispatcher.toDispatchConfig(nodes_1));
-        Dispatcher dispatch_1 = MockDispatcher.create(nodes_1, rpcPool_1, 1, vipStatus);
+        MockDispatcher dispatch_1 = MockDispatcher.create(nodes_1, rpcPool_1, vipStatus);
+        dispatch_1.clusterMonitor.shutdown();
         vipStatus.addToRotation(clusterName);
         assertTrue(vipStatus.isInRotation());
         dispatch_1.deconstruct();
diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/MockDispatcher.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/MockDispatcher.java
index 440e3b8d78f..2a2c8410b2c 100644
--- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/MockDispatcher.java
+++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/MockDispatcher.java
@@ -2,8 +2,10 @@
 package com.yahoo.prelude.fastsearch.test;
 
 import com.yahoo.container.handler.VipStatus;
+import com.yahoo.search.cluster.ClusterMonitor;
 import com.yahoo.search.dispatch.Dispatcher;
 import com.yahoo.search.dispatch.rpc.RpcInvokerFactory;
+import com.yahoo.search.dispatch.rpc.RpcPingFactory;
 import com.yahoo.search.dispatch.rpc.RpcResourcePool;
 import com.yahoo.search.dispatch.searchcluster.Node;
 import com.yahoo.search.dispatch.searchcluster.SearchCluster;
@@ -13,25 +15,27 @@ import java.util.List;
 
 class MockDispatcher extends Dispatcher {
 
+    public final ClusterMonitor clusterMonitor;
+
     public static MockDispatcher create(List<Node> nodes) {
         var rpcResourcePool = new RpcResourcePool(toDispatchConfig(nodes));
 
-        return create(nodes, rpcResourcePool, 1, new VipStatus());
+        return create(nodes, rpcResourcePool, new VipStatus());
     }
 
-    public static MockDispatcher create(List<Node> nodes, RpcResourcePool rpcResourcePool,
-            int containerClusterSize, VipStatus vipStatus) {
+    public static MockDispatcher create(List<Node> nodes, RpcResourcePool rpcResourcePool, VipStatus vipStatus) {
         var dispatchConfig = toDispatchConfig(nodes);
-        var searchCluster = new SearchCluster("a", dispatchConfig, containerClusterSize, vipStatus);
-        return new MockDispatcher(searchCluster, dispatchConfig, rpcResourcePool);
+        var searchCluster = new SearchCluster("a", dispatchConfig, vipStatus, new RpcPingFactory(rpcResourcePool));
+        return new MockDispatcher(new ClusterMonitor<>(searchCluster, true), searchCluster, dispatchConfig, rpcResourcePool);
     }
 
-    private MockDispatcher(SearchCluster searchCluster, DispatchConfig dispatchConfig, RpcResourcePool rpcResourcePool) {
-        this(searchCluster, dispatchConfig, new RpcInvokerFactory(rpcResourcePool, searchCluster));
+    private MockDispatcher(ClusterMonitor clusterMonitor, SearchCluster searchCluster, DispatchConfig dispatchConfig, RpcResourcePool rpcResourcePool) {
+        this(clusterMonitor, searchCluster, dispatchConfig, new RpcInvokerFactory(rpcResourcePool, searchCluster));
     }
 
-    private MockDispatcher(SearchCluster searchCluster, DispatchConfig dispatchConfig, RpcInvokerFactory invokerFactory) {
-        super(searchCluster, dispatchConfig, invokerFactory, new MockMetric());
+    private MockDispatcher(ClusterMonitor clusterMonitor, SearchCluster searchCluster, DispatchConfig dispatchConfig, RpcInvokerFactory invokerFactory) {
+        super(clusterMonitor, searchCluster, dispatchConfig, invokerFactory, new MockMetric());
+        this.clusterMonitor = clusterMonitor;
     }
 
     static DispatchConfig toDispatchConfig(List<Node> nodes) {
diff --git a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ExactMatchAndDefaultIndexTestCase.java b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ExactMatchAndDefaultIndexTestCase.java
index 5cae40bd10d..df35d8dbdea 100644
--- a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ExactMatchAndDefaultIndexTestCase.java
+++ b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ExactMatchAndDefaultIndexTestCase.java
@@ -11,6 +11,7 @@ import org.junit.Test;
 
 import java.io.UnsupportedEncodingException;
 import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
 import java.util.Collections;
 
 import static org.junit.Assert.assertEquals;
@@ -34,7 +35,7 @@ public class ExactMatchAndDefaultIndexTestCase {
         q.getModel().setExecution(new Execution(new Execution.Context(null, facts, null, null, null)));
         assertEquals("AND testexact:a/b testexact:foo.com", q.getModel().getQueryTree().getRoot().toString());
         q = new Query("?query=" + enc("a/b foo.com"));
-        assertEquals("AND \"a b\" \"foo com\"", q.getModel().getQueryTree().getRoot().toString());
+        assertEquals("AND a b foo com", q.getModel().getQueryTree().getRoot().toString());
     }
 
     @Test
@@ -44,11 +45,7 @@ public class ExactMatchAndDefaultIndexTestCase {
     }
 
     private String enc(String s) {
-        try {
-            return URLEncoder.encode(s, "utf-8");
-        } catch (UnsupportedEncodingException e) {
-            throw new RuntimeException(e);
-        }
+        return URLEncoder.encode(s, StandardCharsets.UTF_8);
     }
 
 }
diff --git a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParseTestCase.java b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParseTestCase.java
index 0fdad1a1f9c..c1db7d73561 100644
--- a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParseTestCase.java
+++ b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParseTestCase.java
@@ -7,6 +7,7 @@ import com.yahoo.prelude.IndexFacts;
 import com.yahoo.prelude.IndexModel;
 import com.yahoo.prelude.SearchDefinition;
 import com.yahoo.prelude.query.AndItem;
+import com.yahoo.prelude.query.AndSegmentItem;
 import com.yahoo.prelude.query.CompositeItem;
 import com.yahoo.prelude.query.IntItem;
 import com.yahoo.prelude.query.Item;
@@ -48,7 +49,9 @@ public class ParseTestCase {
 
     @Test
     public void testTermWithIndexPrefix() {
-        tester.assertParsed("url:foobar", "url:foobar", Query.Type.ANY);
+        tester.assertParsed("url:foobar",
+                            "url:foobar",
+                            Query.Type.ANY);
     }
 
     @Test
@@ -59,104 +62,98 @@ public class ParseTestCase {
     @Test
     public void testMultipleTermsWithUTF8EncodingOred() {
         tester.assertParsed("OR l\u00e5gen delta M\u00dcNICH M\u00fcnchen",
-                "l\u00e5gen delta M\u00dcNICH M\u00fcnchen", Query.Type.ANY);
+                            "l\u00e5gen delta M\u00dcNICH M\u00fcnchen",
+                            Query.Type.ANY);
     }
 
     @Test
     public void testMultipleTermsWithMultiplePrefixes() {
         tester.assertParsed("RANK (+bar -normal.title:foo -baz) url:foobar",
-                "url:foobar +bar -normal.title:foo -baz", Query.Type.ANY);
+                            "url:foobar +bar -normal.title:foo -baz", Query.Type.ANY);
     }
 
     @Test
     public void testSimpleQueryDefaultOr() {
-        tester.assertParsed("OR foobar foo bar baz", "foobar foo bar baz",
-                Query.Type.ANY);
+        tester.assertParsed("OR foobar foo bar baz", "foobar foo bar baz", Query.Type.ANY);
     }
 
     @Test
     public void testOrAndNot() {
         tester.assertParsed("RANK (+(AND baz bar) -xyzzy -foobaz) foobar foo",
-                "foobar +baz foo -xyzzy -foobaz +bar", Query.Type.ANY);
+                            "foobar +baz foo -xyzzy -foobaz +bar", Query.Type.ANY);
     }
 
     @Test
     public void testSimpleOrNestedAnd() {
         tester.assertParsed("RANK (OR foo bar baz) foobar xyzzy",
-                "foobar +(foo bar baz) xyzzy", Query.Type.ANY);
+                            "foobar +(foo bar baz) xyzzy", Query.Type.ANY);
     }
 
     @Test
     public void testSimpleOrNestedNot() {
         tester.assertParsed("+(OR foobar xyzzy) -(AND foo bar baz)",
-                "foobar -(foo bar baz) xyzzy", Query.Type.ANY);
+                            "foobar -(foo bar baz) xyzzy", Query.Type.ANY);
     }
 
     @Test
     public void testOrNotNestedAnd() {
-        tester.assertParsed(
-                "RANK (+(AND baz (OR foo bar baz) bar) -xyzzy -foobaz) foobar foo",
-                "foobar +baz foo -xyzzy +(foo bar baz) -foobaz +bar",
-                Query.Type.ANY);
+        tester.assertParsed("RANK (+(AND baz (OR foo bar baz) bar) -xyzzy -foobaz) foobar foo",
+                            "foobar +baz foo -xyzzy +(foo bar baz) -foobaz +bar",
+                            Query.Type.ANY);
     }
 
     @Test
     public void testOrAndNotNestedNot() {
-        tester.assertParsed(
-                "RANK (+(AND baz bar) -xyzzy -(AND foo bar baz) -foobaz) foobar foo",
-                "foobar +baz foo -xyzzy -(foo bar baz) -foobaz +bar",
-                Query.Type.ANY);
+        tester.assertParsed("RANK (+(AND baz bar) -xyzzy -(AND foo bar baz) -foobaz) foobar foo",
+                            "foobar +baz foo -xyzzy -(foo bar baz) -foobaz +bar",
+                            Query.Type.ANY);
     }
 
     @Test
     public void testOrMultipleNestedAnd() {
-        tester.assertParsed(
-                "RANK (AND (OR fo ba foba) (OR foz baraz)) foobar foo bar baz",
-                "foobar +(fo ba foba) foo bar +(foz baraz) baz", Query.Type.ANY);
+        tester.assertParsed("RANK (AND (OR fo ba foba) (OR foz baraz)) foobar foo bar baz",
+                            "foobar +(fo ba foba) foo bar +(foz baraz) baz",
+                            Query.Type.ANY);
     }
 
     @Test
     public void testOrMultipleNestedNot() {
-        tester.assertParsed(
-                "+(OR foobar foo bar baz) -(AND fo ba foba) -(AND foz baraz)",
-                "foobar -(fo ba foba) foo bar -(foz baraz) baz", Query.Type.ANY);
+        tester.assertParsed("+(OR foobar foo bar baz) -(AND fo ba foba) -(AND foz baraz)",
+                            "foobar -(fo ba foba) foo bar -(foz baraz) baz",
+                            Query.Type.ANY);
     }
 
     @Test
     public void testOrAndNotMultipleNestedAnd() {
-        tester.assertParsed(
-                "RANK (+(AND baz (OR foo bar baz) (OR foz bazaz) bar) -xyzzy -foobaz) foobar foo",
-                "foobar +baz foo -xyzzy +(foo bar baz) -foobaz +(foz bazaz) +bar",
-                Query.Type.ANY);
+        tester.assertParsed("RANK (+(AND baz (OR foo bar baz) (OR foz bazaz) bar) -xyzzy -foobaz) foobar foo",
+                            "foobar +baz foo -xyzzy +(foo bar baz) -foobaz +(foz bazaz) +bar",
+                            Query.Type.ANY);
     }
 
     @Test
     public void testOrAndNotMultipleNestedNot() {
-        tester.assertParsed(
-                "RANK (+(AND baz bar) -xyzzy -(AND foo bar baz) -foobaz -(AND foz bazaz)) foobar foo",
-                "foobar +baz foo -xyzzy -(foo bar baz) -foobaz -(foz bazaz) +bar",
-                Query.Type.ANY);
+        tester.assertParsed("RANK (+(AND baz bar) -xyzzy -(AND foo bar baz) -foobaz -(AND foz bazaz)) foobar foo",
+                            "foobar +baz foo -xyzzy -(foo bar baz) -foobaz -(foz bazaz) +bar",
+                            Query.Type.ANY);
     }
 
     @Test
     public void testOrMultipleNestedAndNot() {
-        tester.assertParsed(
-                "RANK (+(AND (OR ffoooo bbaarr) (OR oof rab raboof)) -(AND fo ba foba) -(AND foz baraz)) foobar foo bar baz",
-                "foobar -(fo ba foba) foo +(ffoooo bbaarr) bar +(oof rab raboof) -(foz baraz) baz",
-                Query.Type.ANY);
+        tester.assertParsed("RANK (+(AND (OR ffoooo bbaarr) (OR oof rab raboof)) -(AND fo ba foba) -(AND foz baraz)) foobar foo bar baz",
+                            "foobar -(fo ba foba) foo +(ffoooo bbaarr) bar +(oof rab raboof) -(foz baraz) baz",
+                            Query.Type.ANY);
     }
 
     @Test
     public void testOrAndNotMultipleNestedAndNot() {
-        tester.assertParsed(
-                "RANK (+(AND (OR ffoooo bbaarr) (OR oof rab raboof) baz xyxyzzy) -(AND fo ba foba) -foo -bar -(AND foz baraz)) foobar",
-                "foobar -(fo ba foba) -foo +(ffoooo bbaarr) -bar +(oof rab raboof) -(foz baraz) +baz +xyxyzzy",
-                Query.Type.ANY);
+        tester.assertParsed("RANK (+(AND (OR ffoooo bbaarr) (OR oof rab raboof) baz xyxyzzy) -(AND fo ba foba) -foo -bar -(AND foz baraz)) foobar",
+                            "foobar -(fo ba foba) -foo +(ffoooo bbaarr) -bar +(oof rab raboof) -(foz baraz) +baz +xyxyzzy",
+                            Query.Type.ANY);
     }
 
     @Test
     public void testExplicitPhrase() {
-        Item root=tester.assertParsed("\"foo bar foobar\"", "\"foo bar foobar\"", Query.Type.ANY);
+        Item root = tester.assertParsed("\"foo bar foobar\"", "\"foo bar foobar\"", Query.Type.ANY);
         assertTrue(root instanceof PhraseItem);
         assertTrue(((PhraseItem)root).isExplicit());
     }
@@ -164,21 +161,20 @@ public class ParseTestCase {
     @Test
     public void testPhraseWithIndex() {
         tester.assertParsed("normal.title:\"foo bar foobar\"",
-                "normal.title:\"foo bar foobar\"", Query.Type.ANY);
+                            "normal.title:\"foo bar foobar\"", Query.Type.ANY);
     }
 
     @Test
     public void testPhrasesAndTerms() {
         tester.assertParsed("OR \"foo bar foobar\" xyzzy \"baz gaz faz\"",
-                "\"foo bar foobar\" xyzzy \"baz gaz faz\"", Query.Type.ANY);
+                            "\"foo bar foobar\" xyzzy \"baz gaz faz\"", Query.Type.ANY);
     }
 
     @Test
     public void testPhrasesAndTermsWithOperators() {
-        tester.assertParsed(
-                "RANK (+(AND \"baz gaz faz\" bazar) -\"foo bar foobar\") foofoo xyzzy",
-                "foofoo -\"foo bar foobar\" xyzzy +\"baz gaz faz\" +bazar",
-                Query.Type.ANY);
+        tester.assertParsed("RANK (+(AND \"baz gaz faz\" bazar) -\"foo bar foobar\") foofoo xyzzy",
+                            "foofoo -\"foo bar foobar\" xyzzy +\"baz gaz faz\" +bazar",
+                            Query.Type.ANY);
     }
 
     @Test
@@ -188,38 +184,40 @@ public class ParseTestCase {
 
     @Test
     public void testTermWithCatalogAndIndexPrefixDefaultAnd() {
-        tester.assertParsed("normal.title:foobar", "normal.title:foobar",
-                Query.Type.ALL);
+        tester.assertParsed("normal.title:foobar", "normal.title:foobar", Query.Type.ALL);
     }
 
     @Test
     public void testMultipleTermsWithMultiplePrefixesDefaultAnd() {
         tester.assertParsed("+(AND url:foobar bar) -normal.title:foo -baz",
-                "url:foobar +bar -normal.title:foo -baz", Query.Type.ALL);
+                            "url:foobar +bar -normal.title:foo -baz",
+                            Query.Type.ALL);
     }
 
     @Test
     public void testSimpleQueryDefaultAnd() {
-        tester.assertParsed("AND foobar foo bar baz", "foobar foo bar baz",
-                Query.Type.ALL);
+        tester.assertParsed("AND foobar foo bar baz", "foobar foo bar baz", Query.Type.ALL);
     }
 
     @Test
     public void testNotDefaultAnd() {
-        tester.assertParsed(
-                "+(AND foobar (OR foo bar baz) xyzzy) -(AND foz baraz bazar)",
-                "foobar +(foo bar baz) xyzzy -(foz baraz bazar)", Query.Type.ALL);
+        tester.assertParsed("+(AND foobar (OR foo bar baz) xyzzy) -(AND foz baraz bazar)",
+                            "foobar +(foo bar baz) xyzzy -(foz baraz bazar)",
+                            Query.Type.ALL);
     }
 
     @Test
     public void testSimpleTermQueryDefaultPhrase() {
-        tester.assertParsed("foobar", "foobar", Query.Type.PHRASE);
+        tester.assertParsed("foobar",
+                            "foobar",
+                            Query.Type.PHRASE);
     }
 
     @Test
     public void testSimpleQueryDefaultPhrase() {
-        Item root=tester.assertParsed("\"foobar foo bar baz\"", "foobar foo bar baz",
-                Query.Type.PHRASE);
+        Item root = tester.assertParsed("\"foobar foo bar baz\"",
+                                        "foobar foo bar baz",
+                                        Query.Type.PHRASE);
         assertTrue(root instanceof PhraseItem);
         assertFalse(((PhraseItem)root).isExplicit());
     }
@@ -227,23 +225,25 @@ public class ParseTestCase {
     @Test
     public void testMultipleTermsWithMultiplePrefixesDefaultPhrase() {
         tester.assertParsed("\"url foobar bar normal title foo baz\"",
-                "url:foobar +bar -normal.title:foo -baz", Query.Type.PHRASE);
+                            "url:foobar +bar -normal.title:foo -baz",
+                            Query.Type.PHRASE);
     }
 
     @Test
     public void testOdd1() {
-        tester.assertParsed("AND \"window print\" error", "+window.print() +error",Query.Type.ALL);
+        tester.assertParsed("AND window print error", "+window.print() +error",
+                            Query.Type.ALL);
     }
 
     @Test
     public void testOdd2() {
-        tester.assertParsed("normal.title:kaboom", "normal.title:\"kaboom\"",Query.Type.ALL);
+        tester.assertParsed("normal.title:kaboom", "normal.title:\"kaboom\"",
+                            Query.Type.ALL);
     }
 
     @Test
     public void testOdd2Uppercase() {
-        tester.assertParsed("normal.title:KABOOM", "NORMAL.TITLE:\"KABOOM\"",
-                Query.Type.ALL);
+        tester.assertParsed("normal.title:KABOOM", "NORMAL.TITLE:\"KABOOM\"", Query.Type.ALL);
     }
 
     @Test
@@ -280,19 +280,19 @@ public class ParseTestCase {
     @Test
     public void testNestedCompositesDefaultOr() {
         tester.assertParsed("RANK (OR foobar bar baz) foo xyzzy",
-                "foo +(foobar +(bar baz)) xyzzy", Query.Type.ANY);
+                            "foo +(foobar +(bar baz)) xyzzy", Query.Type.ANY);
     }
 
     @Test
     public void testNestedCompositesDefaultAnd() {
         tester.assertParsed("AND foo (OR foobar bar baz) xyzzy",
-                "foo +(foobar +(bar baz)) xyzzy", Query.Type.ALL);
+                            "foo +(foobar +(bar baz)) xyzzy", Query.Type.ALL);
     }
 
     @Test
     public void testNestedCompositesPhraseDefault() {
         tester.assertParsed("\"foo foobar bar baz xyzzy\"",
-                "foo +(foobar +(bar baz)) xyzzy", Query.Type.PHRASE);
+                            "foo +(foobar +(bar baz)) xyzzy", Query.Type.PHRASE);
     }
 
     @Test
@@ -349,8 +349,7 @@ public class ParseTestCase {
 
     @Test
     public void testNumericWithIndex() {
-        tester.assertParsed("document.size:[34;454]", "document.size:[34;454]",
-                Query.Type.ANY);
+        tester.assertParsed("document.size:[34;454]", "document.size:[34;454]", Query.Type.ANY);
     }
 
     @Test
@@ -361,14 +360,14 @@ public class ParseTestCase {
     @Test
     public void testMultipleIntegerWithIndex() {
         tester.assertParsed("OR document.size:[34;454] date:>1234567890",
-                "document.size:[34;454] date:>1234567890", Query.Type.ANY);
+                            "document.size:[34;454] date:>1234567890", Query.Type.ANY);
     }
 
     @Test
     public void testMixedNumericAndOtherTerms() {
         tester.assertParsed("RANK (AND document.size:<1024 xyzzy) foo date:>123456890",
-                "foo +document.size:<1024 +xyzzy date:>123456890",
-                Query.Type.ANY);
+                            "foo +document.size:<1024 +xyzzy date:>123456890",
+                            Query.Type.ANY);
     }
 
     @Test
@@ -378,20 +377,18 @@ public class ParseTestCase {
 
     @Test
     public void testItemPhraseEmptyPhrase() {
-        tester.assertParsed("RANK to \"or not to be\"", "+to\"or not to be\"\"\"",
-                Query.Type.ANY);
+        tester.assertParsed("RANK to \"or not to be\"", "+to\"or not to be\"\"\"", Query.Type.ANY);
     }
 
     @Test
     public void testSimpleQuery() {
-        tester.assertParsed("OR if am \"f g 4 2\" maybe", "if am \"  f g 4 2\"\" maybe",
-                Query.Type.ANY);
+        tester.assertParsed("OR if am \"f g 4 2\" maybe", "if am \"  f g 4 2\"\" maybe", Query.Type.ANY);
     }
 
     @Test
     public void testExcessivePluses() {
         tester.assertParsed("+(AND other is nothing) -test",
-                "++other +++++is ++++++nothing -test", Query.Type.ANY);
+                            "++other +++++is ++++++nothing -test", Query.Type.ANY);
     }
 
     @Test
@@ -401,39 +398,38 @@ public class ParseTestCase {
 
     @Test
     public void testPlusesAndMinuses() {
-        Item root=tester.assertParsed("\"a b c d d\"", "a+b+c+d--d", Query.Type.ANY);
-        assertTrue(root instanceof PhraseItem);
-        assertFalse(((PhraseItem)root).isExplicit());
+        tester.assertParsed("AND a b c d d", "a+b+c+d--d", Query.Type.ANY);
     }
 
     @Test
     public void testNumbers() {
-        tester.assertParsed("\"123 2132odfd 934032 32423\"",
-                "123+2132odfd.934032,,32423", Query.Type.ANY);
+        tester.assertParsed("AND 123 2132odfd 934032 32423", "123+2132odfd.934032,,32423", Query.Type.ANY);
     }
 
     @Test
     public void testOtherSignsInQuote() {
-        tester.assertParsed("\"0032 4 320 24329043\"", "0032+4\\320.24329043",
-                Query.Type.ANY);
+        tester.assertParsed("AND 0032 4 320 24329043", "0032+4\\320.24329043", Query.Type.ANY);
     }
 
     @Test
     public void testGribberish() {
         tester.assertParsed("1349832840234l3040roer\u00e6lf12",
-                ",1349832840234l3040roer\u00e6lf12", Query.Type.ANY);
+                            ",1349832840234l3040roer\u00e6lf12",
+                            Query.Type.ANY);
     }
 
     @Test
     public void testUrl() {
-        tester.assertParsed("www:\"www hotelaiguablava com\"",
-                "+www:www.hotelaiguablava:com", Query.Type.ANY);
+        tester.assertParsed("AND www:www www:hotelaiguablava www:com",
+                            "+www:www.hotelaiguablava:com",
+                            Query.Type.ANY);
     }
 
     @Test
     public void testUrlGribberish() {
-        tester.assertParsed("OR \"3 16\" fast.type:lycosoffensive",
-                "[ 3:16 fast.type:lycosoffensive", Query.Type.ANY);
+        tester.assertParsed("OR (AND 3 16) fast.type:lycosoffensive",
+                            "[ 3:16 fast.type:lycosoffensive",
+                            Query.Type.ANY);
     }
 
     @Test
@@ -475,8 +471,7 @@ public class ParseTestCase {
 
     @Test
     public void testPrefixWithDotAdvanced() {
-        tester.assertParsed("normal.title:foobar", "normal.title:foobar",
-                Query.Type.ADVANCED);
+        tester.assertParsed("normal.title:foobar", "normal.title:foobar", Query.Type.ADVANCED);
     }
 
     @Test
@@ -486,20 +481,21 @@ public class ParseTestCase {
 
     @Test
     public void testSimplePhraseAdvanced() {
-        tester.assertParsed("\"foo bar foobar\"", "\"foo bar foobar\"",
-                Query.Type.ADVANCED);
+        tester.assertParsed("\"foo bar foobar\"", "\"foo bar foobar\"", Query.Type.ADVANCED);
     }
 
     @Test
     public void testSimplePhraseWithIndexAdvanced() {
         tester.assertParsed("normal.title:\"foo bar foobar\"",
-                "normal.title:\"foo bar foobar\"", Query.Type.ADVANCED);
+                            "normal.title:\"foo bar foobar\"",
+                            Query.Type.ADVANCED);
     }
 
     @Test
     public void testMultiplePhrasesAdvanced() {
         tester.assertParsed("AND \"foo bar foobar\" \"baz gaz faz\"",
-                "\"foo bar foobar\" and \"baz gaz faz\"", Query.Type.ADVANCED);
+                            "\"foo bar foobar\" and \"baz gaz faz\"",
+                            Query.Type.ADVANCED);
     }
 
     @Test
@@ -661,23 +657,23 @@ public class ParseTestCase {
 
     @Test
     public void testImplicitPhrase1Advanced() {
-        tester.assertParsed("\"test if\"", "--test+-if", Query.Type.ADVANCED);
+        tester.assertParsed("AND test if", "--test+-if", Query.Type.ADVANCED);
     }
 
     @Test
     public void testImplicitPhrase2Advanced() {
-        tester.assertParsed("\"a b c d d\"", "a+b+c+d--d", Query.Type.ADVANCED);
+        tester.assertParsed("AND a b c d d", "a+b+c+d--d", Query.Type.ADVANCED);
     }
 
     @Test
     public void testImplicitPhrase3Advanced() {
-        tester.assertParsed("\"123 2132odfd 934032 32423\"",
+        tester.assertParsed("AND 123 2132odfd 934032 32423",
                 "123+2132odfd.934032,,32423", Query.Type.ADVANCED);
     }
 
     @Test
     public void testImplicitPhrase4Advanced() {
-        tester.assertParsed("\"0032 4 320 24329043\"", "0032+4\\320.24329043", Query.Type.ADVANCED);
+        tester.assertParsed("AND 0032 4 320 24329043", "0032+4\\320.24329043", Query.Type.ADVANCED);
     }
 
     @Test
@@ -730,7 +726,7 @@ public class ParseTestCase {
 
     @Test
     public void testSingleHyphen() {
-        tester.assertParsed("\"a b\"", "a-b", Query.Type.ALL);
+        tester.assertParsed("AND a b", "a-b", Query.Type.ALL);
     }
 
     @Test
@@ -883,27 +879,27 @@ public class ParseTestCase {
 
     @Test
     public void testSimpleDotPhraseAny() {
-        tester.assertParsed("OR a \"b c\" d", "a b.c d", Query.Type.ANY);
+        tester.assertParsed("OR a (AND b c) d", "a b.c d", Query.Type.ANY);
     }
 
     @Test
     public void testSimpleHyphenPhraseAny() {
-        tester.assertParsed("OR a \"b c\" d", "a b-c d", Query.Type.ANY);
+        tester.assertParsed("OR a (AND b c) d", "a b-c d", Query.Type.ANY);
     }
 
     @Test
     public void testAnotherSimpleDotPhraseAny() {
-        tester.assertParsed("OR \"a b\" c d", "a.b c d", Query.Type.ANY);
+        tester.assertParsed("OR (AND a b) c d", "a.b c d", Query.Type.ANY);
     }
 
     @Test
     public void testYetAnotherSimpleDotPhraseAny() {
-        tester.assertParsed("OR a b \"c d\"", "a b c.d", Query.Type.ANY);
+        tester.assertParsed("OR a b (AND c d)", "a b c.d", Query.Type.ANY);
     }
 
     @Test
     public void testVariousSeparatorsPhraseAny() {
-        tester.assertParsed("\"a b c d\"", "a-b.c%d", Query.Type.ANY);
+        tester.assertParsed("AND a b c d", "a-b.c%d", Query.Type.ANY);
     }
 
     @Test
@@ -918,45 +914,44 @@ public class ParseTestCase {
 
     @Test
     public void testIndexedDottedPhraseAny() {
-        tester.assertParsed("OR a url:\"b c\" d", "a url:b.c d", Query.Type.ANY);
+        tester.assertParsed("OR a (AND url:b url:c) d", "a url:b.c d", Query.Type.ANY);
     }
 
     @Test
     public void testIndexedPlusedPhraseAny() {
-        tester.assertParsed("OR a normal.title:\"b c\" d", "a normal.title:b+c d",
-                Query.Type.ANY);
+        tester.assertParsed("OR a (AND normal.title:b normal.title:c) d", "a normal.title:b+c d", Query.Type.ANY);
     }
 
     @Test
     public void testNestedNotAny() {
         tester.assertParsed(
-                "RANK (+(OR normal.title:foobar url:\"www pvv org\") -foo) a",
+                "RANK (+(OR normal.title:foobar (AND url:www url:pvv url:org)) -foo) a",
                 "a +(normal.title:foobar url:www.pvv.org) -foo", Query.Type.ANY);
     }
 
     @Test
     public void testDottedPhraseAdvanced() {
-        tester.assertParsed("OR a \"b c\"", "a or b.c", Query.Type.ADVANCED);
+        tester.assertParsed("OR a (AND b c)", "a or b.c", Query.Type.ADVANCED);
     }
 
     @Test
     public void testHyphenPhraseAdvanced() {
-        tester.assertParsed("OR (AND a \"b c\") d", "a and b-c or d", Query.Type.ADVANCED);
+        tester.assertParsed("OR (AND a (AND b c)) d", "a and b-c or d", Query.Type.ADVANCED);
     }
 
     @Test
     public void testAnotherDottedPhraseAdvanced() {
-        tester.assertParsed("OR \"a b\" c", "a.b or c", Query.Type.ADVANCED);
+        tester.assertParsed("OR (AND a b) c", "a.b or c", Query.Type.ADVANCED);
     }
 
     @Test
     public void testNottedDottedPhraseAdvanced() {
-        tester.assertParsed("+a -\"c d\"", "a andnot c.d", Query.Type.ADVANCED);
+        tester.assertParsed("+a -(AND c d)", "a andnot c.d", Query.Type.ADVANCED);
     }
 
     @Test
     public void testVariousSeparatorsPhraseAdvanced() {
-        tester.assertParsed("\"a b c d\"", "a-b.c%d", Query.Type.ADVANCED);
+        tester.assertParsed("AND a b c d", "a-b.c%d", Query.Type.ADVANCED);
     }
 
     @Test
@@ -976,14 +971,14 @@ public class ParseTestCase {
 
     @Test
     public void testNestedPlussedPhraseAdvanced() {
-        tester.assertParsed("AND (OR a normal.title:\"b c\") d",
+        tester.assertParsed("AND (OR a (AND normal.title:b normal.title:c)) d",
                 "a or normal.title:b+c and d", Query.Type.ADVANCED);
     }
 
     @Test
     public void testNottedNestedDottedPhraseAdvanced() {
         tester.assertParsed(
-                "+(AND a (OR normal.title:foobar url:\"www pvv org\")) -foo",
+                "+(AND a (OR normal.title:foobar (AND url:www url:pvv url:org))) -foo",
                 "a and (normal.title:foobar or url:www.pvv.org) andnot foo",
                 Query.Type.ADVANCED);
     }
@@ -995,7 +990,7 @@ public class ParseTestCase {
 
     @Test
     public void testPlusedTwiceThenQuotedPhraseAny() {
-        tester.assertParsed("\"a b c d\"", "a+b+\"c d\"", Query.Type.ANY);
+        tester.assertParsed("AND a b c d", "a+b+\"c d\"", Query.Type.ANY);
     }
 
     @Test
@@ -1005,7 +1000,7 @@ public class ParseTestCase {
 
     @Test
     public void testPhrasesInBraces() {
-        tester.assertParsed("url.domain:\"microsoft com\"",
+        tester.assertParsed("AND url.domain:microsoft url.domain:com",
                 "+(url.domain:microsoft.com)", Query.Type.ALL);
     }
 
@@ -1053,17 +1048,17 @@ public class ParseTestCase {
 
     @Test
     public void testPhraseNotPrefix() {
-        tester.assertParsed("OR foo \"prefix bar\"", "foo prefix*bar", Query.Type.ANY);
+        tester.assertParsed("OR foo (AND prefix bar)", "foo prefix*bar", Query.Type.ANY);
     }
 
     @Test
     public void testPhraseNotSubstring() {
-        tester.assertParsed("OR foo \"substring bar\"", "foo *substring*bar", Query.Type.ANY);
+        tester.assertParsed("OR foo (AND substring bar)", "foo *substring*bar", Query.Type.ANY);
     }
 
     @Test
     public void testPhraseNotSuffix() {
-        tester.assertParsed("OR \"foo suffix\" bar", "foo*suffix bar", Query.Type.ANY);
+        tester.assertParsed("OR (AND foo suffix) bar", "foo*suffix bar", Query.Type.ANY);
     }
 
     @Test
@@ -1086,20 +1081,17 @@ public class ParseTestCase {
 
     @Test
     public void testIndexedPhraseNotPrefix() {
-        tester.assertParsed("foo.bar:\"prefix xyzzy\"", "foo.bar:prefix*xyzzy",
-                     Query.Type.ANY);
+        tester.assertParsed("AND foo.bar:prefix foo.bar:xyzzy", "foo.bar:prefix*xyzzy", Query.Type.ANY);
     }
 
     @Test
     public void testIndexedPhraseNotSubstring() {
-        tester.assertParsed("foo.bar:\"substring xyzzy\"", "foo.bar:*substring*xyzzy",
-                     Query.Type.ANY);
+        tester.assertParsed("AND foo.bar:substring foo.bar:xyzzy", "foo.bar:*substring*xyzzy", Query.Type.ANY);
     }
 
     @Test
     public void testIndexedPhraseNotSuffix() {
-        tester.assertParsed("foo.bar:\"xyzzy suffix\"", "foo.bar:xyzzy*suffix",
-                     Query.Type.ANY);
+        tester.assertParsed("AND foo.bar:xyzzy foo.bar:suffix", "foo.bar:xyzzy*suffix", Query.Type.ANY);
     }
 
     @Test
@@ -1120,20 +1112,20 @@ public class ParseTestCase {
         assertTrue(root instanceof SuffixItem);
     }
 
-    /** Non existing index → phrase **/
+    /** Non existing index → and **/
     @Test
     public void testNonIndexPhraseNotPrefix() {
-        tester.assertParsed("\"void prefix\"", "void:prefix*", Query.Type.ANY);
+        tester.assertParsed("AND void prefix", "void:prefix*", Query.Type.ANY);
     }
 
     @Test
     public void testNonIndexPhraseNotSubstring() {
-        tester.assertParsed("\"void substring\"", "void:*substring*", Query.Type.ANY);
+        tester.assertParsed("AND void substring", "void:*substring*", Query.Type.ANY);
     }
 
     @Test
     public void testNonIndexPhraseNotSuffix() {
-        tester.assertParsed("\"void suffix\"", "void:*suffix", Query.Type.ANY);
+        tester.assertParsed("AND void suffix", "void:*suffix", Query.Type.ANY);
     }
 
     /** Explicit phrase → remove '*' **/
@@ -1198,7 +1190,7 @@ public class ParseTestCase {
     /** Extra spaces with index **/
     @Test
     public void testIndexPrefixExtraSpace() {
-        tester.assertParsed("\"foo prefix\"", "foo:prefix *", Query.Type.ANY);
+        tester.assertParsed("AND foo prefix", "foo:prefix *", Query.Type.ANY);
     }
 
     @Test
@@ -1419,7 +1411,7 @@ public class ParseTestCase {
 
     @Test
     public void testMultipleDifferentPhraseSeparators() {
-        tester.assertParsed("\"foo bar\"", "foo.-.bar", Query.Type.ANY);
+        tester.assertParsed("AND foo bar", "foo.-.bar", Query.Type.ANY);
     }
 
     @Test
@@ -1430,19 +1422,17 @@ public class ParseTestCase {
 
     @Test
     public void testReallyNoisyQuery1() {
-        tester.assertParsed("AND word another", "&word\"()/&#)(/&another!\"",
-                Query.Type.ALL);
+        tester.assertParsed("AND word another", "&word\"()/&#)(/&another!\"", Query.Type.ALL);
     }
 
     @Test
     public void testReallyNoisyQuery2() {
-        tester.assertParsed("\"\u03bc\u03bc hei\"", "&&&`\u00b5\u00b5=@hei", Query.Type.ALL);
+        tester.assertParsed("AND \u03bc\u03bc hei", "&&&`\u00b5\u00b5=@hei", Query.Type.ALL);
     }
 
     @Test
     public void testReallyNoisyQuery3() {
-        tester.assertParsed("AND \"hei hallo\" du der", "hei-hallo;du;der",
-                     Query.Type.ALL);
+        tester.assertParsed("AND hei hallo du der", "hei-hallo;du;der", Query.Type.ALL);
     }
 
     @Test
@@ -1478,7 +1468,7 @@ public class ParseTestCase {
 
     @Test
     public void testTheStupidSymbolsWhichAreNowWordCharactersInUnicode() {
-        tester.assertParsed("\"yz a\"", "yz\u00A8\u00AA\u00AF", Query.Type.ANY);
+        tester.assertParsed("AND yz a", "yz\u00A8\u00AA\u00AF", Query.Type.ANY);
     }
 
     @Test
@@ -1498,7 +1488,7 @@ public class ParseTestCase {
 
     @Test
     public void testImplicitPhrasingWithIndex() {
-        tester.assertParsed("a:\"b c\"", "a:/b/c", Query.Type.ANY);
+        tester.assertParsed("AND a:b a:c", "a:/b/c", Query.Type.ANY);
     }
 
     @Test
@@ -1508,7 +1498,7 @@ public class ParseTestCase {
 
     @Test
     public void testSingleNoisyPhraseWithIndex() {
-        tester.assertParsed("mail:\"yahoo com\"", "mail:@yahoo.com", Query.Type.ANY);
+        tester.assertParsed("AND mail:yahoo mail:com", "mail:@yahoo.com", Query.Type.ANY);
     }
 
     @Test
@@ -1599,7 +1589,7 @@ public class ParseTestCase {
                 "url.all:http://www.newsadvance.com/servlet/Satellite?pagename=LNA/MGArticle/IMD_BasicArticle&c=MGArticle&cid=1031782787014&path=!mgnetwork!diversions",
                 Query.Type.ALL);
         tester.assertParsed(
-                "AND ull:\"http www neue oz de information pub Boulevard index html file a 3 s 4 file\" s:\"37 iptc bdt 20050607 294 dpa 9001170 txt\" s:\"3 dir\" s:\"26 opt DPA parsed boulevard\" s:\"7 bereich\" s:\"9 Boulevard\"",
+                "AND ull:http ull:www ull:neue ull:oz ull:de ull:information ull:pub ull:Boulevard ull:index ull:html ull:file ull:a ull:3 ull:s ull:4 ull:file s:\"37 iptc bdt 20050607 294 dpa 9001170 txt\" s:\"3 dir\" s:\"26 opt DPA parsed boulevard\" s:\"7 bereich\" s:\"9 Boulevard\"",
                 "ull:http://www.neue-oz.de/information/pub_Boulevard/index.html?file=a:3:{s:4:\"file\";s:37:\"iptc-bdt-20050607-294-dpa_9001170.txt\";s:3:\"dir\";s:26:\"/opt/DPA/parsed/boulevard/\";s:7:\"bereich\";s:9:\"Boulevard\";}",
                 Query.Type.ALL);
     }
@@ -1640,7 +1630,7 @@ public class ParseTestCase {
 
     @Test
     public void testTooLongQueryTerms() {
-        tester.assertParsed("AND \"545558598787gggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggcfffffffffffffffffffffffffffffffffffffffffffccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccclllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyytttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrreeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeewwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqlkjhcxxdfffxdzzaqwwsxedcrfvtgbyhnujmikkiloolpppof filter ew 545558598787gggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggcfffffffffffffffffffffffffffffffffffffffffffccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccclllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyytttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrreeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeewwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqlkjhcxxdfffxdzzaqwwsxedcrfvtgbyhnujmikkiloolpppof\"!1000 \"2b 2f 545558598787gggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggcfffffffffffffffffffffffffffffffffffffffffffccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccclllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyytttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrreeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeewwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqlkjhcxxdfffxdzzaqwwsxedcrfvtgbyhnujmikkiloolpppof\"",
+        tester.assertParsed("AND 545558598787gggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggcfffffffffffffffffffffffffffffffffffffffffffccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccclllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyytttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrreeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeewwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqlkjhcxxdfffxdzzaqwwsxedcrfvtgbyhnujmikkiloolpppof filter ew 545558598787gggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggcfffffffffffffffffffffffffffffffffffffffffffccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccclllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyytttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrreeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeewwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqlkjhcxxdfffxdzzaqwwsxedcrfvtgbyhnujmikkiloolpppof 2b 2f 545558598787gggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggcfffffffffffffffffffffffffffffffffffffffffffccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccclllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyytttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrreeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeewwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqlkjhcxxdfffxdzzaqwwsxedcrfvtgbyhnujmikkiloolpppof",
                 "+/545558598787gggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggcfffffffffffffffffffffffffffffffffffffffffffccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccclllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyytttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrreeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeewwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqlkjhcxxdfffxdzzaqwwsxedcrfvtgbyhnujmikkiloolpppof&filter=ew:545558598787gggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggcfffffffffffffffffffffffffffffffffffffffffffccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccclllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyytttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrreeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeewwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqlkjhcxxdfffxdzzaqwwsxedcrfvtgbyhnujmikkiloolpppof!1000 =.2b..2f.545558598787gggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggcfffffffffffffffffffffffffffffffffffffffffffccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccclllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyytttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrreeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeewwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqlkjhcxxdfffxdzzaqwwsxedcrfvtgbyhnujmikkiloolpppof",
                 Query.Type.ALL);
     }
@@ -1648,22 +1638,22 @@ public class ParseTestCase {
     @Test
     public void testNonSpecialTokenParsing() {
         ParsingTester customTester = new ParsingTester(new SpecialTokens("default"));
-        customTester.assertParsed("OR c or c with \"tcp ip\"", "c# or c++ with tcp/ip", Query.Type.ANY);
+        customTester.assertParsed("OR c or c with (AND tcp ip)", "c# or c++ with tcp/ip", Query.Type.ANY);
     }
 
     @Test
     public void testNonIndexWithColons1() {
-        tester.assertParsed("OR this is \"notan iindex\"", "this is notan:iindex", Query.Type.ANY);
+        tester.assertParsed("OR this is (AND notan iindex)", "this is notan:iindex", Query.Type.ANY);
     }
 
     @Test
     public void testNonIndexWithColons2() {
-        tester.assertParsed("OR this is \"notan iindex either\"", "this is notan:iindex:either", Query.Type.ANY);
+        tester.assertParsed("OR this is (AND notan iindex either)", "this is notan:iindex:either", Query.Type.ANY);
     }
 
     @Test
     public void testIndexThenUnderscoreTermBecomesIndex() {
-        tester.assertParsed("name:\"batch article\"", "name:batch_article", Query.Type.ANY);
+        tester.assertParsed("AND name:batch name:article", "name:batch_article", Query.Type.ANY);
     }
 
     @Test
@@ -1671,17 +1661,17 @@ public class ParseTestCase {
         // "first" "second" and "third" are segments in the test language
         Item item = tester.parseQuery("name:firstsecondthird", null, Language.CHINESE_SIMPLIFIED, Query.Type.ANY, TestLinguistics.INSTANCE);
 
-        assertTrue(item instanceof PhraseSegmentItem);
-        PhraseSegmentItem phrase = (PhraseSegmentItem) item;
+        assertTrue(item instanceof AndSegmentItem);
+        AndSegmentItem segment = (AndSegmentItem) item;
 
-        assertEquals(3, phrase.getItemCount());
-        assertEquals("name:first", phrase.getItem(0).toString());
-        assertEquals("name:second", phrase.getItem(1).toString());
-        assertEquals("name:third", phrase.getItem(2).toString());
+        assertEquals(3, segment.getItemCount());
+        assertEquals("name:first", segment.getItem(0).toString());
+        assertEquals("name:second", segment.getItem(1).toString());
+        assertEquals("name:third", segment.getItem(2).toString());
 
-        assertEquals("name", ((WordItem) phrase.getItem(0)).getIndexName());
-        assertEquals("name", ((WordItem) phrase.getItem(1)).getIndexName());
-        assertEquals("name", ((WordItem) phrase.getItem(2)).getIndexName());
+        assertEquals("name", ((WordItem) segment.getItem(0)).getIndexName());
+        assertEquals("name", ((WordItem) segment.getItem(1)).getIndexName());
+        assertEquals("name", ((WordItem) segment.getItem(2)).getIndexName());
     }
 
     @Test
@@ -1689,22 +1679,22 @@ public class ParseTestCase {
         // "first" "second" and "third" are segments in the test language
         Item item = tester.parseQuery("name:\"firstsecondthird\"", null, Language.CHINESE_SIMPLIFIED, Query.Type.ANY, TestLinguistics.INSTANCE);
 
-        assertTrue(item instanceof PhraseSegmentItem);
-        PhraseSegmentItem phrase = (PhraseSegmentItem) item;
+        assertTrue(item instanceof AndSegmentItem);
+        AndSegmentItem segment = (AndSegmentItem) item;
 
-        assertEquals(3, phrase.getItemCount());
-        assertEquals("name:first", phrase.getItem(0).toString());
-        assertEquals("name:second", phrase.getItem(1).toString());
-        assertEquals("name:third", phrase.getItem(2).toString());
+        assertEquals(3, segment.getItemCount());
+        assertEquals("name:first", segment.getItem(0).toString());
+        assertEquals("name:second", segment.getItem(1).toString());
+        assertEquals("name:third", segment.getItem(2).toString());
 
-        assertEquals("name", ((WordItem) phrase.getItem(0)).getIndexName());
-        assertEquals("name", ((WordItem) phrase.getItem(1)).getIndexName());
-        assertEquals("name", ((WordItem)phrase.getItem(2)).getIndexName());
+        assertEquals("name", ((WordItem) segment.getItem(0)).getIndexName());
+        assertEquals("name", ((WordItem) segment.getItem(1)).getIndexName());
+        assertEquals("name", ((WordItem)segment.getItem(2)).getIndexName());
     }
 
     @Test
     public void testAndItemAndImplicitPhrase() {
-        tester.assertParsed("\"\u00d8 \u00d8 \u00d8 \u00d9\"",
+        tester.assertParsed("AND \u00d8 \u00d8 \u00d8 \u00d9",
                             "\u00d8\u00b9\u00d8\u00b1\u00d8\u00a8\u00d9", "",
                             Query.Type.ALL, Language.CHINESE_SIMPLIFIED);
     }
@@ -1736,7 +1726,7 @@ public class ParseTestCase {
     @Test
     public void testFakeCJKSegmentingOfMultiplePhrases() {
         Item item = tester.parseQuery("name:firstsecond.s", null, Language.CHINESE_SIMPLIFIED, Query.Type.ANY, TestLinguistics.INSTANCE);
-        assertEquals("name:\"'first second' s\"", item.toString());
+        assertEquals("AND (SAND name:first name:second) name:s", item.toString());
     }
 
     @Test
@@ -1801,7 +1791,7 @@ public class ParseTestCase {
 
     @Test
     public void testCommaOnlyLeadsToImplicitPhrasing() {
-        tester.assertParsed("\"A B C\"", "A,B,C", Query.Type.ALL);
+        tester.assertParsed("AND A B C", "A,B,C", Query.Type.ALL);
     }
 
     @Test
@@ -1873,8 +1863,8 @@ public class ParseTestCase {
 
     @Test
     public void testJPMobileExceptionQuery() {
-        tester.assertParsed("OR concat and \"make string\" 1 47 or",
-                     "(concat \"and\" (make-string 1 47) \"or\")", Query.Type.ALL);
+        tester.assertParsed("OR concat and (AND make string) 1 47 or",
+                            "(concat \"and\" (make-string 1 47) \"or\")", Query.Type.ALL);
     }
 
     @Test
@@ -1882,7 +1872,7 @@ public class ParseTestCase {
         tester.assertParsed("b", "a: b", Query.Type.ALL);
         tester.assertParsed("AND a b", "a : b", Query.Type.ALL);
         tester.assertParsed("AND a b", "a :b", Query.Type.ALL);
-        tester.assertParsed("\"a b\"", "a.:b", Query.Type.ALL);
+        tester.assertParsed("AND a b", "a.:b", Query.Type.ALL);
         tester.assertParsed("a:b", "a:b", Query.Type.ALL);
     }
 
@@ -1917,8 +1907,7 @@ public class ParseTestCase {
         tester.assertParsed("AND ringtone AND (OR a:\"Delivery SMAF large max 150kB 063\" OR a:\"RealMusic Delivery\")",
                 "ringtone AND (a:\"Delivery SMAF large max.150kB (063)\" OR a:\"RealMusic Delivery\" )",
                 Query.Type.ALL);
-        // The last one here is a little weird, but it's not a problem,
-        // so I let it pass for now...
+        // The last one here is a little weird, but it's not a problem, so let it pass for now...
         tester.assertParsed("OR (OR ringtone AND) (OR a:\"Delivery SMAF large max 150kB 063\" OR a:\"RealMusic Delivery\")",
                 "ringtone AND (a:\"Delivery SMAF large max.150kB (063)\" OR a:\"RealMusic Delivery\" )",
                 Query.Type.ANY);
@@ -1926,7 +1915,7 @@ public class ParseTestCase {
 
     @Test
     public void testMixedCaseIndexNames() {
-        tester.assertParsed("AND mixedCase:a mixedCase:b \"notAnIndex c\" mixedCase:d",
+        tester.assertParsed("AND mixedCase:a mixedCase:b notAnIndex c mixedCase:d",
                 "mixedcase:a MIXEDCASE:b notAnIndex:c mixedCase:d",
                 Query.Type.ALL);
     }
@@ -1934,7 +1923,7 @@ public class ParseTestCase {
     /** CJK special tokens should be recognized also on non-boundaries */
     @Test
     public void testChineseSpecialTokens() {
-        tester.assertParsed("AND \"cat tcp/ip zu\" \"foo dotnet bar dotnet dotnet c# c++ bar dotnet dotnet wiz\"",
+        tester.assertParsed("AND cat tcp/ip zu foo dotnet bar dotnet dotnet c# c++ bar dotnet dotnet wiz",
                      "cattcp/ipzu foo.netbar.net.netC#c++bar.net.netwiz","", Query.Type.ALL, Language.CHINESE_SIMPLIFIED);
     }
 
@@ -1945,7 +1934,7 @@ public class ParseTestCase {
     @Test
     public void testChineseSpecialTokensWithMultiSegmentReplace() {
         // special-token-fs is a special token, to be replaced by firstsecond, first and second are segments in test
-        tester.assertParsed("AND \"tcp/ip firstsecond dotnet\" firstsecond 'first second'","tcp/ipspecial-token-fs.net special-token-fs firstsecond",
+        tester.assertParsed("AND tcp/ip firstsecond dotnet firstsecond (SAND first second)","tcp/ipspecial-token-fs.net special-token-fs firstsecond",
                             "", Query.Type.ALL, Language.CHINESE_SIMPLIFIED, TestLinguistics.INSTANCE);
     }
 
@@ -2014,7 +2003,7 @@ public class ParseTestCase {
 
     @Test
     public void testVersionNumbers() {
-        tester.assertParsed("\"1 0 9\"", "1.0.9", Query.Type.ALL);
+        tester.assertParsed("AND 1 0 9", "1.0.9", Query.Type.ALL);
     }
 
     @Test
@@ -2321,7 +2310,7 @@ public class ParseTestCase {
 
     @Test
     public void testOdd1Web() {
-        tester.assertParsed("AND \"window print\" error", "+window.print() +error",Query.Type.WEB);
+        tester.assertParsed("AND window print error", "+window.print() +error",Query.Type.WEB);
     }
 
     @Test
@@ -2351,13 +2340,13 @@ public class ParseTestCase {
 
     @Test
     public void testDefaultWebIndices() {
-        tester.assertParsed("\"notanindex b\"","notanindex:b",Query.Type.WEB);
-        tester.assertParsed("site:\"b $\"","site:b",Query.Type.WEB);
-        tester.assertParsed("hostname:b","hostname:b",Query.Type.WEB);
-        tester.assertParsed("link:b","link:b",Query.Type.WEB);
-        tester.assertParsed("url:b","url:b",Query.Type.WEB);
-        tester.assertParsed("inurl:b","inurl:b",Query.Type.WEB);
-        tester.assertParsed("intitle:b","intitle:b",Query.Type.WEB);
+        tester.assertParsed("AND notanindex b","notanindex:b", Query.Type.WEB);
+        tester.assertParsed("site:\"b $\"","site:b", Query.Type.WEB);
+        tester.assertParsed("hostname:b","hostname:b", Query.Type.WEB);
+        tester.assertParsed("link:b","link:b", Query.Type.WEB);
+        tester.assertParsed("url:b","url:b", Query.Type.WEB);
+        tester.assertParsed("inurl:b","inurl:b", Query.Type.WEB);
+        tester.assertParsed("intitle:b","intitle:b", Query.Type.WEB);
     }
 
     @Test
@@ -2527,7 +2516,7 @@ public class ParseTestCase {
 
     @Test
     public void testSiteAndSegmentPhrasesFollowedByText() {
-        tester.assertParsed("AND host.all:\"www abc com x y-z $\" 'a b'",
+        tester.assertParsed("AND host.all:\"www abc com x y-z $\" (SAND a b)",
                            "host.all:www.abc.com/x'y-z a'b", "",
                            Query.Type.ALL, Language.ENGLISH);
     }
@@ -2544,7 +2533,7 @@ public class ParseTestCase {
 
     @Test
     public void testNonAsciiNumber() {
-        tester.assertParsed("title:\"199 119 201 149\"", "title:１９９．１１９．２０１．１４９", Query.Type.ALL);
+        tester.assertParsed("AND title:199 title:119 title:201 title:149", "title:１９９．１１９．２０１．１４９", Query.Type.ALL);
     }
 
 }
diff --git a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/TokenizerTestCase.java b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/TokenizerTestCase.java
index 12d993e8d41..aa2e9dbcf75 100644
--- a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/TokenizerTestCase.java
+++ b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/TokenizerTestCase.java
@@ -46,7 +46,7 @@ public class TokenizerTestCase {
         Tokenizer tokenizer = new Tokenizer(new SimpleLinguistics());
 
         tokenizer.setSpecialTokens(createSpecialTokens());
-        List<?> tokens = tokenizer.tokenize("drive (to hwy88, 88) +or language:en ugcapi_1");
+        List<?> tokens = tokenizer.tokenize("drive (to hwy88, 88) +or language:en ugcapi_1 & &a");
 
         assertEquals(new Token(WORD, "drive"), tokens.get(0));
         assertEquals(new Token(SPACE, " "), tokens.get(1));
@@ -69,6 +69,11 @@ public class TokenizerTestCase {
         assertEquals(new Token(WORD, "ugcapi"), tokens.get(18));
         assertEquals(new Token(UNDERSCORE, "_"), tokens.get(19));
         assertEquals(new Token(NUMBER, "1"), tokens.get(20));
+        assertEquals(new Token(SPACE, " "), tokens.get(21));
+        assertEquals(new Token(NOISE, "<NOISE>"), tokens.get(22));
+        assertEquals(new Token(SPACE, " "), tokens.get(23));
+        assertEquals(new Token(NOISE, "<NOISE>"), tokens.get(24));
+        assertEquals(new Token(WORD, "a"), tokens.get(25));
     }
 
     @Test
diff --git a/container-search/src/test/java/com/yahoo/prelude/querytransform/test/CJKSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/querytransform/test/CJKSearcherTestCase.java
index 91cf5015cba..0ca4b8aa615 100644
--- a/container-search/src/test/java/com/yahoo/prelude/querytransform/test/CJKSearcherTestCase.java
+++ b/container-search/src/test/java/com/yahoo/prelude/querytransform/test/CJKSearcherTestCase.java
@@ -45,7 +45,7 @@ public class CJKSearcherTestCase {
     @Test
     public void testCjkQueryWithOverlappingTokens() {
         // The test language segmenter will segment "bcd" into the overlapping tokens "bc" "cd"
-        assertTransformed("bcd", "'bc cd'", Query.Type.ALL, Language.CHINESE_SIMPLIFIED, Language.CHINESE_TRADITIONAL,
+        assertTransformed("bcd", "SAND bc cd", Query.Type.ALL, Language.CHINESE_SIMPLIFIED, Language.CHINESE_TRADITIONAL,
                           TestLinguistics.INSTANCE);
 
         // While "efg" will be segmented into one of the standard options, "e" "fg"
diff --git a/container-search/src/test/java/com/yahoo/prelude/querytransform/test/LiteralBoostSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/querytransform/test/LiteralBoostSearcherTestCase.java
index 12e756a07ee..023cd3c2849 100644
--- a/container-search/src/test/java/com/yahoo/prelude/querytransform/test/LiteralBoostSearcherTestCase.java
+++ b/container-search/src/test/java/com/yahoo/prelude/querytransform/test/LiteralBoostSearcherTestCase.java
@@ -71,7 +71,7 @@ public class LiteralBoostSearcherTestCase {
 
     @Test
     public void testQueryWithoutBoost() {
-        assertEquals("RANK (AND \"nonexistant a\" \"nonexistant b\") default_literal:nonexistant default_literal:a default_literal:nonexistant default_literal:b",
+        assertEquals("RANK (AND nonexistant a nonexistant b) default_literal:nonexistant default_literal:a default_literal:nonexistant default_literal:b",
                      transformQuery("?query=nonexistant:a nonexistant:b&source=cluster1&restrict=type1"));
     }
 
diff --git a/container-search/src/test/java/com/yahoo/prelude/querytransform/test/QueryRewriteTestCase.java b/container-search/src/test/java/com/yahoo/prelude/querytransform/test/QueryRewriteTestCase.java
index 11922cf640a..36137abd9b8 100644
--- a/container-search/src/test/java/com/yahoo/prelude/querytransform/test/QueryRewriteTestCase.java
+++ b/container-search/src/test/java/com/yahoo/prelude/querytransform/test/QueryRewriteTestCase.java
@@ -1,15 +1,22 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.prelude.querytransform.test;
 
+import com.yahoo.prelude.IndexFacts;
 import com.yahoo.prelude.query.AndItem;
 import com.yahoo.prelude.query.NotItem;
 import com.yahoo.prelude.query.OrItem;
+import com.yahoo.prelude.query.QueryCanonicalizer;
 import com.yahoo.prelude.query.WordItem;
 import com.yahoo.prelude.querytransform.QueryRewrite;
+import com.yahoo.prelude.querytransform.RecallSearcher;
 import com.yahoo.search.Query;
+import com.yahoo.search.Result;
+import com.yahoo.search.searchchain.Execution;
+import com.yahoo.search.test.QueryTestCase;
 import org.junit.Test;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
 
 /**
@@ -38,9 +45,17 @@ public class QueryRewriteTestCase {
         assertRewritten(query, "OR sddocname:per foo bar");
         ((OrItem)query.getModel().getQueryTree().getRoot()).getItem(2).setRanked(false); // set 'bar' unranked
         assertRewritten(query, "OR sddocname:per foo");
-
         assertRewritten("sddocname:per OR foo OR (bar AND fuz)", "per", "OR sddocname:per foo (AND bar fuz)");
+    }
 
+    @Test
+    public void testRankContributingTermsAreNotRemovedOnFullRecall() {
+        Query query = new Query(QueryTestCase.httpEncode("?query=default:term1 OR default:term2 OR default:term3 OR sddocname:per&type=adv&recall=+id:1&restrict=per"));
+        RecallSearcher searcher = new RecallSearcher();
+        Result result = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())).search(query);
+        assertNull(result.hits().getError());
+        assertNull(QueryCanonicalizer.canonicalize(query));
+        assertRewritten(query, "AND (OR default:term1 default:term2 default:term3 sddocname:per) |id:1");
     }
 
     @Test
@@ -88,6 +103,7 @@ public class QueryRewriteTestCase {
 
     private static void assertRewritten(Query query, String expectedOptimizedQuery) {
         QueryRewrite.optimizeByRestrict(query);
+        QueryRewrite.optimizeAndNot(query);
         QueryRewrite.collapseSingleComposites(query);
         assertEquals(expectedOptimizedQuery, query.getModel().getQueryTree().toString());
     }
diff --git a/container-search/src/test/java/com/yahoo/prelude/searcher/test/BlendingSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/searcher/test/BlendingSearcherTestCase.java
index 9da1c184505..0086f1b3571 100644
--- a/container-search/src/test/java/com/yahoo/prelude/searcher/test/BlendingSearcherTestCase.java
+++ b/container-search/src/test/java/com/yahoo/prelude/searcher/test/BlendingSearcherTestCase.java
@@ -108,7 +108,7 @@ public class BlendingSearcherTestCase {
                                 entry.getValue()));
             }
 
-            StrictContractsConfig contracts = new StrictContractsConfig(new StrictContractsConfig.Builder());
+            StrictContractsConfig contracts = new StrictContractsConfig.Builder().build();
 
             FederationSearcher fedSearcher =
                     new FederationSearcher(new FederationConfig(builder), contracts, new ComponentRegistry<>());
@@ -124,7 +124,6 @@ public class BlendingSearcherTestCase {
 
     @Test
     public void testitTwoPhase() {
-
         DocumentSourceSearcher chain1 = new DocumentSourceSearcher();
         DocumentSourceSearcher chain2 = new DocumentSourceSearcher();
         DocumentSourceSearcher chain3 = new DocumentSourceSearcher();
diff --git a/container-search/src/test/java/com/yahoo/prelude/searcher/test/FieldCollapsingSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/searcher/test/FieldCollapsingSearcherTestCase.java
index 4875121a501..12619bf0a5e 100644
--- a/container-search/src/test/java/com/yahoo/prelude/searcher/test/FieldCollapsingSearcherTestCase.java
+++ b/container-search/src/test/java/com/yahoo/prelude/searcher/test/FieldCollapsingSearcherTestCase.java
@@ -40,34 +40,8 @@ import static org.junit.Assert.assertTrue;
  *
  * @author Steinar Knutsen
  */
-@SuppressWarnings("deprecation")
 public class FieldCollapsingSearcherTestCase {
 
-    private FastHit createHit(String uri,int relevancy,int mid) {
-        FastHit hit = new FastHit(uri,relevancy);
-        hit.setField("amid", String.valueOf(mid));
-        return hit;
-    }
-
-    private void assertHit(String uri,int relevancy,int mid,Hit hit) {
-        assertEquals(uri,hit.getId().toString());
-        assertEquals(relevancy, ((int) hit.getRelevance().getScore()));
-        assertEquals(mid,Integer.parseInt((String) hit.getField("amid")));
-    }
-
-    private static class ZeroHitsControl extends com.yahoo.search.Searcher {
-        public int queryCount = 0;
-        public com.yahoo.search.Result search(com.yahoo.search.Query query,
-                com.yahoo.search.searchchain.Execution execution) {
-            ++queryCount;
-            if (query.getHits() == 0) {
-                return new Result(query);
-            } else {
-                return new Result(query, ErrorMessage.createIllegalQuery("Did not request zero hits."));
-            }
-        }
-    }
-
     @Test
     public void testFieldCollapsingWithoutHits() {
         // Set up
@@ -116,14 +90,14 @@ public class FieldCollapsingSearcherTestCase {
         // The searcher turns off collapsing further on in the chain
         q.properties().set("collapse", "0");
         Result r = new Result(q);
-        r.hits().add(createHit("http://acme.org/a.html",10,0));
-        r.hits().add(createHit("http://acme.org/b.html", 9,0));
-        r.hits().add(createHit("http://acme.org/c.html", 9,1));
-        r.hits().add(createHit("http://acme.org/d.html", 8,1));
-        r.hits().add(createHit("http://acme.org/e.html", 8,2));
-        r.hits().add(createHit("http://acme.org/f.html", 7,2));
-        r.hits().add(createHit("http://acme.org/g.html", 7,3));
-        r.hits().add(createHit("http://acme.org/h.html", 6,3));
+        r.hits().add(createHit("http://acme.org/a.html",10, 0));
+        r.hits().add(createHit("http://acme.org/b.html", 9, 0));
+        r.hits().add(createHit("http://acme.org/c.html", 9, 1));
+        r.hits().add(createHit("http://acme.org/d.html", 8, 1));
+        r.hits().add(createHit("http://acme.org/e.html", 8, 2));
+        r.hits().add(createHit("http://acme.org/f.html", 7, 2));
+        r.hits().add(createHit("http://acme.org/g.html", 7, 3));
+        r.hits().add(createHit("http://acme.org/h.html", 6, 3));
         r.setTotalHitCount(8);
         docsource.addResult(q, r);
 
@@ -133,10 +107,10 @@ public class FieldCollapsingSearcherTestCase {
 
         assertEquals(4, r.getHitCount());
         assertEquals(1, docsource.getQueryCount());
-        assertHit("http://acme.org/a.html",10,0,r.hits().get(0));
-        assertHit("http://acme.org/c.html", 9,1,r.hits().get(1));
-        assertHit("http://acme.org/e.html", 8,2,r.hits().get(2));
-        assertHit("http://acme.org/g.html", 7,3,r.hits().get(3));
+        assertHit("http://acme.org/a.html",10, 0, r.hits().get(0));
+        assertHit("http://acme.org/c.html", 9, 1, r.hits().get(1));
+        assertHit("http://acme.org/e.html", 8, 2, r.hits().get(2));
+        assertHit("http://acme.org/g.html", 7, 3, r.hits().get(3));
     }
 
     @Test
@@ -152,14 +126,14 @@ public class FieldCollapsingSearcherTestCase {
         // The searcher turns off collapsing further on in the chain
         q.properties().set("collapse", "0");
         Result r = new Result(q);
-        r.hits().add(createHit("http://acme.org/a.html",10,0));
-        r.hits().add(createHit("http://acme.org/b.html", 9,0));
-        r.hits().add(createHit("http://acme.org/c.html", 9,1));
-        r.hits().add(createHit("http://acme.org/d.html", 8,1));
-        r.hits().add(createHit("http://acme.org/e.html", 8,2));
-        r.hits().add(createHit("http://acme.org/f.html", 7,2));
-        r.hits().add(createHit("http://acme.org/g.html", 7,3));
-        r.hits().add(createHit("http://acme.org/h.html", 6,3));
+        r.hits().add(createHit("http://acme.org/a.html",10, 0));
+        r.hits().add(createHit("http://acme.org/b.html", 9, 0));
+        r.hits().add(createHit("http://acme.org/c.html", 9, 1));
+        r.hits().add(createHit("http://acme.org/d.html", 8, 1));
+        r.hits().add(createHit("http://acme.org/e.html", 8, 2));
+        r.hits().add(createHit("http://acme.org/f.html", 7, 2));
+        r.hits().add(createHit("http://acme.org/g.html", 7, 3));
+        r.hits().add(createHit("http://acme.org/h.html", 6, 3));
         r.setTotalHitCount(8);
         docsource.addResult(q, r);
 
@@ -169,10 +143,10 @@ public class FieldCollapsingSearcherTestCase {
 
         assertEquals(4, r.getHitCount());
         assertEquals(1, docsource.getQueryCount());
-        assertHit("http://acme.org/a.html",10,0,r.hits().get(0));
-        assertHit("http://acme.org/c.html", 9,1,r.hits().get(1));
-        assertHit("http://acme.org/e.html", 8,2,r.hits().get(2));
-        assertHit("http://acme.org/g.html", 7,3,r.hits().get(3));
+        assertHit("http://acme.org/a.html",10,0, r.hits().get(0));
+        assertHit("http://acme.org/c.html", 9,1, r.hits().get(1));
+        assertHit("http://acme.org/e.html", 8,2, r.hits().get(2));
+        assertHit("http://acme.org/g.html", 7,3, r.hits().get(3));
     }
 
     @Test
@@ -185,14 +159,14 @@ public class FieldCollapsingSearcherTestCase {
 
         Query q = new Query("?query=test_collapse");
         Result r = new Result(q);
-        r.hits().add(createHit("http://acme.org/a.html",10,0));
-        r.hits().add(createHit("http://acme.org/b.html", 9,0));
-        r.hits().add(createHit("http://acme.org/c.html", 9,1));
-        r.hits().add(createHit("http://acme.org/d.html", 8,1));
-        r.hits().add(createHit("http://acme.org/e.html", 8,2));
-        r.hits().add(createHit("http://acme.org/f.html", 7,2));
-        r.hits().add(createHit("http://acme.org/g.html", 7,3));
-        r.hits().add(createHit("http://acme.org/h.html", 6,3));
+        r.hits().add(createHit("http://acme.org/a.html",10, 0));
+        r.hits().add(createHit("http://acme.org/b.html", 9, 0));
+        r.hits().add(createHit("http://acme.org/c.html", 9, 1));
+        r.hits().add(createHit("http://acme.org/d.html", 8, 1));
+        r.hits().add(createHit("http://acme.org/e.html", 8, 2));
+        r.hits().add(createHit("http://acme.org/f.html", 7, 2));
+        r.hits().add(createHit("http://acme.org/g.html", 7, 3));
+        r.hits().add(createHit("http://acme.org/h.html", 6, 3));
         r.setTotalHitCount(8);
         docsource.addResult(q, r);
 
@@ -220,16 +194,16 @@ public class FieldCollapsingSearcherTestCase {
         // The searcher turns off collapsing further on in the chain
         q.properties().set("collapse", "0");
         Result r = new Result(q);
-        r.hits().add(createHit("http://acme.org/a.html",10,0));
-        r.hits().add(createHit("http://acme.org/b.html", 9,0));
-        r.hits().add(createHit("http://acme.org/c.html", 9,0));
-        r.hits().add(createHit("http://acme.org/d.html", 8,0));
-        r.hits().add(createHit("http://acme.org/e.html", 8,0));
-        r.hits().add(createHit("http://acme.org/f.html", 7,0));
-        r.hits().add(createHit("http://acme.org/g.html", 7,0));
-        r.hits().add(createHit("http://acme.org/h.html", 6,0));
-        r.hits().add(createHit("http://acme.org/i.html", 5,1));
-        r.hits().add(createHit("http://acme.org/j.html", 4,2));
+        r.hits().add(createHit("http://acme.org/a.html",10, 0));
+        r.hits().add(createHit("http://acme.org/b.html", 9, 0));
+        r.hits().add(createHit("http://acme.org/c.html", 9, 0));
+        r.hits().add(createHit("http://acme.org/d.html", 8, 0));
+        r.hits().add(createHit("http://acme.org/e.html", 8, 0));
+        r.hits().add(createHit("http://acme.org/f.html", 7, 0));
+        r.hits().add(createHit("http://acme.org/g.html", 7, 0));
+        r.hits().add(createHit("http://acme.org/h.html", 6, 0));
+        r.hits().add(createHit("http://acme.org/i.html", 5, 1));
+        r.hits().add(createHit("http://acme.org/j.html", 4, 2));
         r.setTotalHitCount(10);
         docsource.addResult(q, r);
 
@@ -239,15 +213,15 @@ public class FieldCollapsingSearcherTestCase {
 
         assertEquals(2, r.getHitCount());
         assertEquals(2, docsource.getQueryCount());
-        assertHit("http://acme.org/a.html",10,0,r.hits().get(0));
-        assertHit("http://acme.org/i.html", 5,1,r.hits().get(1));
+        assertHit("http://acme.org/a.html",10, 0, r.hits().get(0));
+        assertHit("http://acme.org/i.html", 5, 1, r.hits().get(1));
 
         // Next results
         docsource.resetQueryCount();
         r = doSearch(collapse, q, 2, 2, chained);
         assertEquals(1, r.getHitCount());
         assertEquals(2, docsource.getQueryCount());
-        assertHit("http://acme.org/j.html",4,2,r.hits().get(0));
+        assertHit("http://acme.org/j.html",4, 2, r.hits().get(0));
     }
 
     /**
@@ -265,16 +239,16 @@ public class FieldCollapsingSearcherTestCase {
         // The searcher turns off collapsing further on in the chain
         q.properties().set("collapse", "0");
         Result r = new Result(q);
-        r.hits().add(createHit("http://acme.org/a.html",10,1));
-        r.hits().add(createHit("http://acme.org/b.html",10,1));
-        r.hits().add(createHit("http://acme.org/c.html",10,0));
-        r.hits().add(createHit("http://acme.org/d.html",10,0));
-        r.hits().add(createHit("http://acme.org/e.html",10,0));
-        r.hits().add(createHit("http://acme.org/f.html",10,0));
-        r.hits().add(createHit("http://acme.org/g.html",10,0));
-        r.hits().add(createHit("http://acme.org/h.html",10,0));
-        r.hits().add(createHit("http://acme.org/i.html",10,0));
-        r.hits().add(createHit("http://acme.org/j.html",10,1));
+        r.hits().add(createHit("http://acme.org/a.html", 10, 1));
+        r.hits().add(createHit("http://acme.org/b.html", 10, 1));
+        r.hits().add(createHit("http://acme.org/c.html", 10, 0));
+        r.hits().add(createHit("http://acme.org/d.html", 10, 0));
+        r.hits().add(createHit("http://acme.org/e.html", 10, 0));
+        r.hits().add(createHit("http://acme.org/f.html", 10, 0));
+        r.hits().add(createHit("http://acme.org/g.html", 10, 0));
+        r.hits().add(createHit("http://acme.org/h.html", 10, 0));
+        r.hits().add(createHit("http://acme.org/i.html", 10, 0));
+        r.hits().add(createHit("http://acme.org/j.html", 10, 1));
         r.setTotalHitCount(10);
         docsource.addResult(q, r);
 
@@ -287,17 +261,6 @@ public class FieldCollapsingSearcherTestCase {
         assertHit("http://acme.org/c.html",10,0,r.hits().get(1));
     }
 
-    public static class QueryMessupSearcher extends Searcher {
-        public Result search(com.yahoo.search.Query query, Execution execution) {
-            AndItem a = new AndItem();
-            a.addItem(query.getModel().getQueryTree().getRoot());
-            a.addItem(new WordItem("b"));
-            query.getModel().getQueryTree().setRoot(a);
-
-            return execution.search(query);
-        }
-    }
-
     @Test
     public void testQueryTransformAndCollapsing() {
         // Set up
@@ -309,9 +272,9 @@ public class FieldCollapsingSearcherTestCase {
         chained.put(collapse, messUp);
         chained.put(messUp, docsource);
 
-        // Caveat: Collapse is set to false, because that's what the
-        // collapser asks for
-        Query q = new Query("?query=test_collapse+b&collapsefield=amid");
+        // Caveat: Collapse is set to false, because that's what the collapser asks for
+        Query q = new Query("?query=%22test%20collapse%22+b&collapsefield=amid");
+        System.out.println(q);
         // The searcher turns off collapsing further on in the chain
         q.properties().set("collapse", "0");
         Result r = new Result(q);
@@ -327,13 +290,13 @@ public class FieldCollapsingSearcherTestCase {
         docsource.addResult(q, r);
 
         // Test basic collapsing on mid
-        q = new Query("?query=test_collapse&collapsefield=amid");
+        q = new Query("?query=%22test%20collapse%22&collapsefield=amid");
         r = doSearch(collapse, q, 0, 2, chained);
 
         assertEquals(2, docsource.getQueryCount());
         assertEquals(2, r.getHitCount());
-        assertHit("http://acme.org/a.html",10,0,r.hits().get(0));
-        assertHit("http://acme.org/h.html", 6,1,r.hits().get(1));
+        assertHit("http://acme.org/a.html",10, 0, r.hits().get(0));
+        assertHit("http://acme.org/h.html", 6, 1, r.hits().get(1));
     }
 
     @Test
@@ -367,10 +330,10 @@ public class FieldCollapsingSearcherTestCase {
         assertEquals(4, r.getHitCount());
         assertEquals(1, docsource.getQueryCount());
         assertTrue(r.isFilled("placeholder"));
-        assertHit("http://acme.org/a.html",10,0,r.hits().get(0));
-        assertHit("http://acme.org/c.html", 9,1,r.hits().get(1));
-        assertHit("http://acme.org/e.html", 8,2,r.hits().get(2));
-        assertHit("http://acme.org/g.html", 7,3,r.hits().get(3));
+        assertHit("http://acme.org/a.html",10, 0, r.hits().get(0));
+        assertHit("http://acme.org/c.html", 9, 1, r.hits().get(1));
+        assertHit("http://acme.org/e.html", 8, 2, r.hits().get(2));
+        assertHit("http://acme.org/g.html", 7, 3, r.hits().get(3));
 
         docsource.resetQueryCount();
         // Test basic collapsing on mid
@@ -381,10 +344,10 @@ public class FieldCollapsingSearcherTestCase {
         assertEquals(1, docsource.getQueryCount());
         assertFalse(r.isFilled("placeholder"));
         assertTrue(r.isFilled("short"));
-        assertHit("http://acme.org/a.html",10,0,r.hits().get(0));
-        assertHit("http://acme.org/c.html", 9,1,r.hits().get(1));
-        assertHit("http://acme.org/e.html", 8,2,r.hits().get(2));
-        assertHit("http://acme.org/g.html", 7,3,r.hits().get(3));
+        assertHit("http://acme.org/a.html",10, 0, r.hits().get(0));
+        assertHit("http://acme.org/c.html", 9, 1, r.hits().get(1));
+        assertHit("http://acme.org/e.html", 8, 2, r.hits().get(2));
+        assertHit("http://acme.org/g.html", 7, 3, r.hits().get(3));
     }
 
     @Test
@@ -400,14 +363,14 @@ public class FieldCollapsingSearcherTestCase {
         // The searcher turns off collapsing further on in the chain
         q.properties().set("collapse", "0");
         Result r = new Result(q);
-        r.hits().add(createHit("http://acme.org/a.html",10,0));
-        r.hits().add(createHit("http://acme.org/b.html", 9,0));
-        r.hits().add(createHit("http://acme.org/c.html", 9,1));
-        r.hits().add(createHit("http://acme.org/d.html", 8,1));
-        r.hits().add(createHit("http://acme.org/e.html", 8,2));
-        r.hits().add(createHit("http://acme.org/f.html", 7,2));
-        r.hits().add(createHit("http://acme.org/g.html", 7,3));
-        r.hits().add(createHit("http://acme.org/h.html", 6,3));
+        r.hits().add(createHit("http://acme.org/a.html",10, 0));
+        r.hits().add(createHit("http://acme.org/b.html", 9, 0));
+        r.hits().add(createHit("http://acme.org/c.html", 9, 1));
+        r.hits().add(createHit("http://acme.org/d.html", 8, 1));
+        r.hits().add(createHit("http://acme.org/e.html", 8, 2));
+        r.hits().add(createHit("http://acme.org/f.html", 7, 2));
+        r.hits().add(createHit("http://acme.org/g.html", 7, 3));
+        r.hits().add(createHit("http://acme.org/h.html", 6, 3));
         r.setTotalHitCount(8);
         docsource.addResult(q, r);
 
@@ -416,29 +379,28 @@ public class FieldCollapsingSearcherTestCase {
         Result result = new Execution(chain, Execution.Context.createContextStub()).search(query);
 
         // Assert that the regular hits are collapsed
-        assertEquals(4+1, result.getHitCount());
+        assertEquals(4 + 1, result.getHitCount());
         assertEquals(1, docsource.getQueryCount());
-        assertHit("http://acme.org/a.html",10,0,result.hits().get(0));
-        assertHit("http://acme.org/c.html", 9,1,result.hits().get(1));
-        assertHit("http://acme.org/e.html", 8,2,result.hits().get(2));
-        assertHit("http://acme.org/g.html", 7,3,result.hits().get(3));
+        assertHit("http://acme.org/a.html",10, 0, result.hits().get(0));
+        assertHit("http://acme.org/c.html", 9, 1, result.hits().get(1));
+        assertHit("http://acme.org/e.html", 8, 2, result.hits().get(2));
+        assertHit("http://acme.org/g.html", 7, 3, result.hits().get(3));
 
         // Assert that the aggregation group hierarchy is left intact
-        HitGroup root= getFirstGroupIn(result.hits());
+        HitGroup root = getFirstGroupIn(result.hits());
         assertNotNull(root);
-        assertEquals("group:root:",root.getId().stringValue().substring(0,11)); // The id ends by a global counter currently
-        assertEquals(1,root.size());
-        HitGroup groupList= (GroupList)root.get("grouplist:g1");
+        assertEquals("group:root:",root.getId().stringValue().substring(0, 11)); // The id ends by a global counter currently
+        assertEquals(1, root.size());
+        HitGroup groupList = (GroupList)root.get("grouplist:g1");
         assertNotNull(groupList);
-        assertEquals(1,groupList.size());
-        HitGroup group= (HitGroup)groupList.get("group:long:37");
+        assertEquals(1, groupList.size());
+        HitGroup group = (HitGroup)groupList.get("group:long:37");
         assertNotNull(group);
     }
 
     private Group getFirstGroupIn(HitGroup hits) {
-        for (Hit h : hits) {
+        for (Hit h : hits)
             if (h instanceof Group) return (Group)h;
-        }
         return null;
     }
 
@@ -450,9 +412,8 @@ public class FieldCollapsingSearcherTestCase {
 
     private Chain<Searcher> chainedAsSearchChain(Searcher topOfChain, Map<Searcher, Searcher> chained) {
         List<Searcher> searchers = new ArrayList<>();
-        for (Searcher current = topOfChain; current != null; current = chained.get(current)) {
+        for (Searcher current = topOfChain; current != null; current = chained.get(current))
             searchers.add(current);
-        }
         return new Chain<>(searchers);
     }
 
@@ -470,7 +431,7 @@ public class FieldCollapsingSearcherTestCase {
 
         @Override
         public Result search(Query query, Execution execution) {
-            Result r=execution.search(query);
+            Result r = execution.search(query);
             r.hits().add(createAggregationGroup("g1"));
             return r;
         }
@@ -479,10 +440,51 @@ public class FieldCollapsingSearcherTestCase {
             Group root = new Group(new RootId(0), new Relevance(1));
             GroupList groupList = new GroupList(label);
             root.add(groupList);
-            Group value=new Group(new LongId(37l),new Relevance(2.11));
+            Group value = new Group(new LongId(37l), new Relevance(2.11));
             groupList.add(value);
             return root;
         }
     }
 
+    private FastHit createHit(String uri,int relevancy,int mid) {
+        FastHit hit = new FastHit(uri,relevancy);
+        hit.setField("amid", String.valueOf(mid));
+        return hit;
+    }
+
+    private void assertHit(String uri,int relevancy,int mid,Hit hit) {
+        assertEquals(uri,hit.getId().toString());
+        assertEquals(relevancy, ((int) hit.getRelevance().getScore()));
+        assertEquals(mid,Integer.parseInt((String) hit.getField("amid")));
+    }
+
+    private static class ZeroHitsControl extends com.yahoo.search.Searcher {
+
+        public int queryCount = 0;
+
+        @Override
+        public Result search(Query query, Execution execution) {
+            ++queryCount;
+            if (query.getHits() == 0) {
+                return new Result(query);
+            } else {
+                return new Result(query, ErrorMessage.createIllegalQuery("Did not request zero hits."));
+            }
+        }
+    }
+
+    public static class QueryMessupSearcher extends Searcher {
+
+        @Override
+        public Result search(com.yahoo.search.Query query, Execution execution) {
+            AndItem a = new AndItem();
+            a.addItem(query.getModel().getQueryTree().getRoot());
+            a.addItem(new WordItem("b"));
+            query.getModel().getQueryTree().setRoot(a);
+
+            return execution.search(query);
+        }
+
+    }
+
 }
diff --git a/container-search/src/test/java/com/yahoo/prelude/searcher/test/PosSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/searcher/test/PosSearcherTestCase.java
index e2973f8ed65..aa3fa53119e 100644
--- a/container-search/src/test/java/com/yahoo/prelude/searcher/test/PosSearcherTestCase.java
+++ b/container-search/src/test/java/com/yahoo/prelude/searcher/test/PosSearcherTestCase.java
@@ -118,6 +118,12 @@ public class PosSearcherTestCase {
         q.properties().set("pos.units", "udeg");
         doSearch(searcher, q, 0, 10);
         assertEquals("(2,0,0,18026,0,1,0,4294967295)", q.getRanking().getLocation().toString());
+
+        q = new Query();
+        q.properties().set("pos.ll", "N0;E0");
+        q.properties().set("pos.radius", "-1");
+        doSearch(searcher, q, 0, 10);
+        assertEquals("(2,0,0,536870912,0,1,0,4294967295)", q.getRanking().getLocation().toString());
     }
 
     /**
@@ -128,13 +134,13 @@ public class PosSearcherTestCase {
         Query q = new Query();
         q.properties().set("pos.xy", "22500;22500");
         doSearch(searcher, q, 0, 10);
-        assertEquals("(2,22500,22500,5000,0,1,0)", q.getRanking().getLocation().toString());
+        assertEquals("(2,22500,22500,450668,0,1,0)", q.getRanking().getLocation().toString());
 
         q = new Query();
         q.properties().set("pos.xy", "22500;22500");
         q.properties().set("pos.units", "unknown");
         doSearch(searcher, q, 0, 10);
-        assertEquals("(2,22500,22500,5000,0,1,0)", q.getRanking().getLocation().toString());
+        assertEquals("(2,22500,22500,450668,0,1,0)", q.getRanking().getLocation().toString());
     }
 
     @Test
diff --git a/container-search/src/test/java/com/yahoo/prelude/searcher/test/ValidatePredicateSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/searcher/test/ValidatePredicateSearcherTestCase.java
index 1b9ca1cd29b..2187cb89ae2 100644
--- a/container-search/src/test/java/com/yahoo/prelude/searcher/test/ValidatePredicateSearcherTestCase.java
+++ b/container-search/src/test/java/com/yahoo/prelude/searcher/test/ValidatePredicateSearcherTestCase.java
@@ -2,7 +2,6 @@
 package com.yahoo.prelude.searcher.test;
 
 import com.google.common.util.concurrent.MoreExecutors;
-import com.yahoo.language.Linguistics;
 import com.yahoo.language.simple.SimpleLinguistics;
 import com.yahoo.prelude.Index;
 import com.yahoo.prelude.IndexFacts;
@@ -20,8 +19,6 @@ import com.yahoo.search.searchchain.Execution;
 import com.yahoo.search.yql.YqlParser;
 import org.junit.Test;
 
-import java.util.*;
-
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNull;
 
@@ -46,6 +43,14 @@ public class ValidatePredicateSearcherTestCase {
         assertEquals(ErrorMessage.createIllegalQuery("age=200 outside configured predicate bounds."), r.hits().getError());
     }
 
+    @Test
+    public void queryFailsWhenPredicateFieldIsUsedInTermSearch() {
+        ValidatePredicateSearcher searcher = new ValidatePredicateSearcher();
+        String q = "select * from sources * where predicate_field CONTAINS \"true\";";
+        Result r = doSearch(searcher, q, "predicate-bounds [0..99]");
+        assertEquals(ErrorMessage.createIllegalQuery("Index 'predicate_field' is predicate attribute and can only be used in conjunction with a predicate query operator."), r.hits().getError());
+    }
+
     private static Result doSearch(ValidatePredicateSearcher searcher, String yqlQuery, String command) {
         QueryTree queryTree = new YqlParser(new ParserEnvironment()).parse(new Parsable().setQuery(yqlQuery));
         Query query = new Query();
@@ -53,6 +58,7 @@ public class ValidatePredicateSearcherTestCase {
 
         SearchDefinition searchDefinition = new SearchDefinition("document");
         Index index = new Index("predicate_field");
+        index.setPredicate(true);
         index.addCommand(command);
         searchDefinition.addIndex(index);
         IndexFacts indexFacts = new IndexFacts(new IndexModel(searchDefinition));
diff --git a/container-search/src/test/java/com/yahoo/prelude/searcher/test/ValidateSortingSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/searcher/test/ValidateSortingSearcherTestCase.java
index 65011ffb562..f4bf957e29a 100644
--- a/container-search/src/test/java/com/yahoo/prelude/searcher/test/ValidateSortingSearcherTestCase.java
+++ b/container-search/src/test/java/com/yahoo/prelude/searcher/test/ValidateSortingSearcherTestCase.java
@@ -52,6 +52,7 @@ public class ValidateSortingSearcherTestCase {
         assertEquals("[ASCENDING:[rank]]", quoteAndTransform("+[rank]"));
         assertEquals("[ASCENDING:[docid]]", quoteAndTransform("+[docid]"));
         assertEquals("[ASCENDING:[rank]]", quoteAndTransform("+[relevancy]"));
+        assertEquals("[ASCENDING:[rank]]", quoteAndTransform("+[relevance]"));
     }
 
     @Test
diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/SegmentSubstitutionTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/SegmentSubstitutionTestCase.java
index b8db5e4d90f..a4cf7d8c380 100644
--- a/container-search/src/test/java/com/yahoo/prelude/semantics/test/SegmentSubstitutionTestCase.java
+++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/SegmentSubstitutionTestCase.java
@@ -23,7 +23,7 @@ public class SegmentSubstitutionTestCase extends RuleBaseAbstractTestCase {
         Query q = new Query("?query=ignored&tracelevel=0&tracelevel.rules=0");
         q.getModel().getQueryTree().setRoot(a);
 
-        assertSemantics("\"first third\"", q);
+        assertSemantics("AND first third", q);
     }
 
     @Test
@@ -32,7 +32,7 @@ public class SegmentSubstitutionTestCase extends RuleBaseAbstractTestCase {
         Query q = new Query("?query=ignored&tracelevel=0&tracelevel.rules=0");
         q.getModel().getQueryTree().setRoot(a);
 
-        assertSemantics("\"bc first third fg\"", q);
+        assertSemantics("AND bc first third fg", q);
     }
 
     @Test
@@ -41,7 +41,7 @@ public class SegmentSubstitutionTestCase extends RuleBaseAbstractTestCase {
         Query q = new Query("?query=ignored&tracelevel=0&tracelevel.rules=0");
         q.getModel().getQueryTree().setRoot(a);
 
-        assertSemantics("+bc -\"first third\"", q);
+        assertSemantics("+bc -(AND first third)", q);
     }
 
     @Test
@@ -50,7 +50,7 @@ public class SegmentSubstitutionTestCase extends RuleBaseAbstractTestCase {
         Query q = new Query("?query=ignored&tracelevel=0&tracelevel.rules=0");
         q.getModel().getQueryTree().setRoot(a);
 
-        assertSemantics("\"9 2 7 0 bc third 2 3 8 9\"", q);
+        assertSemantics("AND 9 2 7 0 bc third 2 3 8 9", q);
     }
 
     private static Item parseQuery(String query) {
diff --git a/container-search/src/test/java/com/yahoo/prelude/test/IndexFactsTestCase.java b/container-search/src/test/java/com/yahoo/prelude/test/IndexFactsTestCase.java
index 82a5a0c7a24..e2ac44316e7 100644
--- a/container-search/src/test/java/com/yahoo/prelude/test/IndexFactsTestCase.java
+++ b/container-search/src/test/java/com/yahoo/prelude/test/IndexFactsTestCase.java
@@ -73,7 +73,7 @@ public class IndexFactsTestCase {
         Query q = newQuery("?query=a:b", indexFacts);
         assertEquals("a:b",  q.getModel().getQueryTree().getRoot().toString());
         q = newQuery("?query=notarealindex:b", indexFacts);
-        assertEquals("\"notarealindex b\"",  q.getModel().getQueryTree().getRoot().toString());
+        assertEquals("AND notarealindex b",  q.getModel().getQueryTree().getRoot().toString());
     }
 
     @Test
@@ -302,8 +302,8 @@ public class IndexFactsTestCase {
         IndexFacts.Session session2 = indexFacts.newSession(query2.getModel().getSources(), query2.getModel().getRestrict());
         assertTrue(session1.getIndex("url").isUriIndex());
         assertTrue(session2.getIndex("url").isUriIndex());
-        assertEquals("url:\"https foo bar\"", query1.getModel().getQueryTree().toString());
-        assertEquals("url:\"https foo bar\"", query2.getModel().getQueryTree().toString());
+        assertEquals("AND url:https url:foo url:bar", query1.getModel().getQueryTree().toString());
+        assertEquals("AND url:https url:foo url:bar", query2.getModel().getQueryTree().toString());
     }
 
     @Test
diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java
index de6bafa267a..eaafb2d8b8a 100644
--- a/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java
+++ b/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java
@@ -1,22 +1,21 @@
 // Copyright 2019 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.search.dispatch;
 
-import com.yahoo.prelude.Pong;
 import com.yahoo.prelude.fastsearch.VespaBackEndSearcher;
 import com.yahoo.prelude.fastsearch.test.MockMetric;
-import com.yahoo.processing.request.CompoundName;
 import com.yahoo.search.Query;
 import com.yahoo.search.Result;
 import com.yahoo.search.cluster.ClusterMonitor;
 import com.yahoo.search.dispatch.searchcluster.Node;
 import com.yahoo.search.dispatch.searchcluster.PingFactory;
+import com.yahoo.search.dispatch.searchcluster.Pinger;
+import com.yahoo.search.dispatch.searchcluster.PongHandler;
 import com.yahoo.search.dispatch.searchcluster.SearchCluster;
 import org.junit.Test;
 
 import java.util.List;
 import java.util.Optional;
 import java.util.OptionalInt;
-import java.util.concurrent.Callable;
 
 import static com.yahoo.search.dispatch.MockSearchCluster.createDispatchConfig;
 import static org.junit.Assert.assertEquals;
@@ -36,12 +35,13 @@ public class DispatcherTest {
         q.getModel().setSearchPath("1/0"); // second node in first group
         MockInvokerFactory invokerFactory = new MockInvokerFactory(cl, (nodes, a) -> {
             assertEquals(1, nodes.size());
-            assertEquals(2, nodes.get(0).key());
+            assertEquals(1, nodes.get(0).key());
             return true;
         });
-        Dispatcher disp = new Dispatcher(cl, createDispatchConfig(), invokerFactory, invokerFactory, new MockMetric());
+        Dispatcher disp = new Dispatcher(new ClusterMonitor(cl, false), cl, createDispatchConfig(), invokerFactory, new MockMetric());
         SearchInvoker invoker = disp.getSearchInvoker(q, null);
         invokerFactory.verifyAllEventsProcessed();
+        disp.deconstruct();
     }
 
     @Test
@@ -53,9 +53,10 @@ public class DispatcherTest {
             }
         };
         MockInvokerFactory invokerFactory = new MockInvokerFactory(cl, (n, a) -> true);
-        Dispatcher disp = new Dispatcher(cl, createDispatchConfig(), invokerFactory, invokerFactory, new MockMetric());
+        Dispatcher disp = new Dispatcher(new ClusterMonitor(cl, false), cl, createDispatchConfig(), invokerFactory, new MockMetric());
         SearchInvoker invoker = disp.getSearchInvoker(new Query(), null);
         invokerFactory.verifyAllEventsProcessed();
+        disp.deconstruct();
     }
 
     @Test
@@ -69,9 +70,10 @@ public class DispatcherTest {
             assertTrue(acceptIncompleteCoverage);
             return true;
         });
-        Dispatcher disp = new Dispatcher(cl, createDispatchConfig(), invokerFactory, invokerFactory, new MockMetric());
+        Dispatcher disp = new Dispatcher(new ClusterMonitor(cl, false), cl, createDispatchConfig(), invokerFactory, new MockMetric());
         SearchInvoker invoker = disp.getSearchInvoker(new Query(), null);
         invokerFactory.verifyAllEventsProcessed();
+        disp.deconstruct();
     }
 
     @Test
@@ -80,8 +82,9 @@ public class DispatcherTest {
             SearchCluster cl = new MockSearchCluster("1", 2, 1);
 
             MockInvokerFactory invokerFactory = new MockInvokerFactory(cl, (n, a) -> false, (n, a) -> false);
-            Dispatcher disp = new Dispatcher(cl, createDispatchConfig(), invokerFactory, invokerFactory, new MockMetric());
+            Dispatcher disp = new Dispatcher(new ClusterMonitor(cl, false), cl, createDispatchConfig(), invokerFactory, new MockMetric());
             disp.getSearchInvoker(new Query(), null);
+            disp.deconstruct();
             fail("Expected exception");
         }
         catch (IllegalStateException e) {
@@ -89,6 +92,53 @@ public class DispatcherTest {
         }
     }
 
+    @Test
+    public void testGroup0IsSelected() {
+        SearchCluster cluster = new MockSearchCluster("1", 3, 1);
+        Dispatcher dispatcher = new Dispatcher(new ClusterMonitor(cluster, false), cluster, createDispatchConfig(), new MockInvokerFactory(cluster, (n, a) -> true), new MockMetric());
+        cluster.pingIterationCompleted();
+        assertEquals(0,
+                     dispatcher.getSearchInvoker(new Query(), null).distributionKey().get().longValue());
+        dispatcher.deconstruct();
+    }
+
+    @Test
+    public void testGroup0IsSkippedWhenItIsBlockingFeed() {
+        SearchCluster cluster = new MockSearchCluster("1", 3, 1);
+        Dispatcher dispatcher = new Dispatcher(new ClusterMonitor(cluster, false), cluster, createDispatchConfig(), new MockInvokerFactory(cluster, (n, a) -> true), new MockMetric());
+        cluster.group(0).get().nodes().get(0).setBlockingWrites(true);
+        cluster.pingIterationCompleted();
+        assertEquals("Blocking group is avoided",
+                     1,
+                     (dispatcher.getSearchInvoker(new Query(), null).distributionKey().get()).longValue());
+        dispatcher.deconstruct();
+    }
+
+    @Test
+    public void testGroup0IsSelectedWhenMoreAreBlockingFeed() {
+        SearchCluster cluster = new MockSearchCluster("1", 3, 1);
+        Dispatcher dispatcher = new Dispatcher(new ClusterMonitor(cluster, false), cluster, createDispatchConfig(), new MockInvokerFactory(cluster, (n, a) -> true), new MockMetric());
+        cluster.group(0).get().nodes().get(0).setBlockingWrites(true);
+        cluster.group(1).get().nodes().get(0).setBlockingWrites(true);
+        cluster.pingIterationCompleted();
+        assertEquals("Blocking group is used when multiple groups are blocking",
+                     0,
+                     dispatcher.getSearchInvoker(new Query(), null).distributionKey().get().longValue());
+        dispatcher.deconstruct();
+    }
+
+    @Test
+    public void testGroup0IsSelectedWhenItIsBlockingFeedWhenNoOthers() {
+        SearchCluster cluster = new MockSearchCluster("1", 1, 1);
+        Dispatcher dispatcher = new Dispatcher(new ClusterMonitor(cluster, false), cluster, createDispatchConfig(), new MockInvokerFactory(cluster, (n, a) -> true), new MockMetric());
+        cluster.group(0).get().nodes().get(0).setBlockingWrites(true);
+        cluster.pingIterationCompleted();
+        assertEquals("Blocking group is used when there is no alternative",
+                     0,
+                     (dispatcher.getSearchInvoker(new Query(), null).distributionKey().get()).longValue());
+        dispatcher.deconstruct();
+    }
+
     interface FactoryStep {
         boolean returnInvoker(List<Node> nodes, boolean acceptIncompleteCoverage);
     }
@@ -116,7 +166,7 @@ public class DispatcherTest {
             boolean nonEmpty = events[step].returnInvoker(nodes, acceptIncompleteCoverage);
             step++;
             if (nonEmpty) {
-                return Optional.of(new MockInvoker(1));
+                return Optional.of(new MockInvoker(nodes.get(0).key()));
             } else {
                 return Optional.empty();
             }
@@ -142,9 +192,10 @@ public class DispatcherTest {
         }
 
         @Override
-        public Callable<Pong> createPinger(Node node, ClusterMonitor<Node> monitor) {
+        public Pinger createPinger(Node node, ClusterMonitor<Node> monitor, PongHandler pongHandler) {
             fail("Unexpected call to createPinger");
             return null;
         }
     }
+
 }
diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/InterleavedSearchInvokerTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/InterleavedSearchInvokerTest.java
index 27685426cf8..e16f09a58ab 100644
--- a/container-search/src/test/java/com/yahoo/search/dispatch/InterleavedSearchInvokerTest.java
+++ b/container-search/src/test/java/com/yahoo/search/dispatch/InterleavedSearchInvokerTest.java
@@ -204,6 +204,33 @@ public class InterleavedSearchInvokerTest {
     private static final List<Double> A5Aux = Arrays.asList(-1.0,11.0,8.5,7.5,-7.0,3.0,2.0);
     private static final List<Double> B5Aux = Arrays.asList(9.0,8.0,-3.0,7.0,6.0,1.0, -1.0);
 
+    private void validateThatTopKProbabilityOverrideTakesEffect(Double topKProbability, int expectedK) throws IOException {
+        InterleavedSearchInvoker invoker = createInterLeavedTestInvoker(A5, B5);
+        query.setHits(8);
+        query.properties().set(Dispatcher.topKProbability, topKProbability);
+        SearchInvoker [] invokers = invoker.invokers().toArray(new SearchInvoker[0]);
+        Result result = invoker.search(query, null);
+        assertEquals(2, invokers.length);
+        assertEquals(expectedK, ((MockInvoker)invokers[0]).hitsRequested);
+        assertEquals(8, result.hits().size());
+        assertEquals(11.0, result.hits().get(0).getRelevance().getScore(), DELTA);
+        assertEquals(9.0, result.hits().get(1).getRelevance().getScore(), DELTA);
+        assertEquals(8.5, result.hits().get(2).getRelevance().getScore(), DELTA);
+        assertEquals(8.0, result.hits().get(3).getRelevance().getScore(), DELTA);
+        assertEquals(7.5, result.hits().get(4).getRelevance().getScore(), DELTA);
+        assertEquals(7.0, result.hits().get(5).getRelevance().getScore(), DELTA);
+        assertEquals(6.0, result.hits().get(6).getRelevance().getScore(), DELTA);
+        assertEquals(3.0, result.hits().get(7).getRelevance().getScore(), DELTA);
+        assertEquals(0, result.getQuery().getOffset());
+        assertEquals(8, result.getQuery().getHits());
+    }
+
+    @Test
+    public void requireThatTopKProbabilityOverrideTakesEffect() throws IOException {
+        validateThatTopKProbabilityOverrideTakesEffect(null, 8);
+        validateThatTopKProbabilityOverrideTakesEffect(0.8, 6);
+    }
+
     @Test
     public void requireThatMergeOfConcreteHitsObeySorting() throws IOException {
         InterleavedSearchInvoker invoker = createInterLeavedTestInvoker(A5, B5);
diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/LeanHitTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/LeanHitTest.java
index 085a9b24993..8d81c5d8521 100644
--- a/container-search/src/test/java/com/yahoo/search/dispatch/LeanHitTest.java
+++ b/container-search/src/test/java/com/yahoo/search/dispatch/LeanHitTest.java
@@ -37,10 +37,14 @@ public class LeanHitTest {
     }
     @Test
     public void testOrderingBySortData() {
-        assertEquals(0, new LeanHit(gidA, 0, 0, gidA).compareTo(new LeanHit(gidA, 0, 0, gidA)));
-        verifyTransitiveOrdering(new LeanHit(gidA, 0, 0, gidA),
-                new LeanHit(gidA, 0, 0, gidB),
-                new LeanHit(gidA, 0, 0, gidC));
+        assertEquals(0, new LeanHit(gidA, 0, 0, 0.0, gidA).compareTo(new LeanHit(gidA, 0, 0, 0.0, gidA)));
+        verifyTransitiveOrdering(new LeanHit(gidA, 0, 0, 0.0, gidA),
+                new LeanHit(gidA, 0, 0, 0.0, gidB),
+                new LeanHit(gidA, 0, 0, 0.0, gidC));
+    }
+    @Test
+    public void testRelevanceIsKeptEvenWithBySortData() {
+        assertEquals(1.3, new LeanHit(gidA, 0, 0, 1.3, gidA).getRelevance(), 0.0);
     }
     @Test
     public void testNaN2negativeInfinity() {
diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java
index 0496194f8ed..36b476e2936 100644
--- a/container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java
+++ b/container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java
@@ -29,7 +29,7 @@ public class LoadBalancerTest {
     @Test
     public void requireThatLoadBalancerServesSingleNodeSetups() {
         Node n1 = new Node(0, "test-node1", 0);
-        SearchCluster cluster = new SearchCluster("a", createDispatchConfig(n1), 1, null);
+        SearchCluster cluster = new SearchCluster("a", createDispatchConfig(n1), null, null);
         LoadBalancer lb = new LoadBalancer(cluster, true);
 
         Optional<Group> grp = lb.takeGroup(null);
@@ -43,7 +43,7 @@ public class LoadBalancerTest {
     public void requireThatLoadBalancerServesMultiGroupSetups() {
         Node n1 = new Node(0, "test-node1", 0);
         Node n2 = new Node(1, "test-node2", 1);
-        SearchCluster cluster = new SearchCluster("a", createDispatchConfig(n1, n2), 1, null);
+        SearchCluster cluster = new SearchCluster("a", createDispatchConfig(n1, n2), null, null);
         LoadBalancer lb = new LoadBalancer(cluster, true);
 
         Optional<Group> grp = lb.takeGroup(null);
@@ -59,7 +59,7 @@ public class LoadBalancerTest {
         Node n2 = new Node(1, "test-node2", 0);
         Node n3 = new Node(0, "test-node3", 1);
         Node n4 = new Node(1, "test-node4", 1);
-        SearchCluster cluster = new SearchCluster("a", createDispatchConfig(n1, n2, n3, n4), 2, null);
+        SearchCluster cluster = new SearchCluster("a", createDispatchConfig(n1, n2, n3, n4), null, null);
         LoadBalancer lb = new LoadBalancer(cluster, true);
 
         Optional<Group> grp = lb.takeGroup(null);
@@ -70,7 +70,7 @@ public class LoadBalancerTest {
     public void requireThatLoadBalancerReturnsDifferentGroups() {
         Node n1 = new Node(0, "test-node1", 0);
         Node n2 = new Node(1, "test-node2", 1);
-        SearchCluster cluster = new SearchCluster("a", createDispatchConfig(n1, n2), 1, null);
+        SearchCluster cluster = new SearchCluster("a", createDispatchConfig(n1, n2), null,null);
         LoadBalancer lb = new LoadBalancer(cluster, true);
 
         // get first group
diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/MockInvoker.java b/container-search/src/test/java/com/yahoo/search/dispatch/MockInvoker.java
index c5fbda7c2f5..c159293d7d9 100644
--- a/container-search/src/test/java/com/yahoo/search/dispatch/MockInvoker.java
+++ b/container-search/src/test/java/com/yahoo/search/dispatch/MockInvoker.java
@@ -17,6 +17,7 @@ class MockInvoker extends SearchInvoker {
     private final Coverage coverage;
     private Query query;
     private List<Hit> hits;
+    int hitsRequested;
 
     protected MockInvoker(int key, Coverage coverage) {
         super(Optional.of(new Node(key, "?", 0)));
@@ -35,6 +36,7 @@ class MockInvoker extends SearchInvoker {
     @Override
     protected void sendSearchRequest(Query query) throws IOException {
         this.query = query;
+        hitsRequested = query.getHits();
     }
 
     @Override
diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java b/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java
index a976b287f63..cafba58662e 100644
--- a/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java
+++ b/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java
@@ -30,20 +30,19 @@ public class MockSearchCluster extends SearchCluster {
     }
 
     public MockSearchCluster(String clusterId, DispatchConfig dispatchConfig, int groups, int nodesPerGroup) {
-        super(clusterId, dispatchConfig, 1, null);
+        super(clusterId, dispatchConfig, null, null);
 
         ImmutableList.Builder<Group> orderedGroupBuilder = ImmutableList.builder();
         ImmutableMap.Builder<Integer, Group> groupBuilder = ImmutableMap.builder();
         ImmutableMultimap.Builder<String, Node> hostBuilder = ImmutableMultimap.builder();
-        int dk = 1;
+        int distributionKey = 0;
         for (int group = 0; group < groups; group++) {
             List<Node> nodes = new ArrayList<>();
             for (int node = 0; node < nodesPerGroup; node++) {
-                Node n = new Node(dk, "host" + dk, group);
-                n.setWorking(true);
+                Node n = new Node(distributionKey, "host" + distributionKey, group);
                 nodes.add(n);
                 hostBuilder.put(n.hostname(), n);
-                dk++;
+                distributionKey++;
             }
             Group g = new Group(group, nodes);
             groupBuilder.put(group, g);
diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/SearchPathTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/SearchPathTest.java
index 5a4457780e2..58042dcf228 100644
--- a/container-search/src/test/java/com/yahoo/search/dispatch/SearchPathTest.java
+++ b/container-search/src/test/java/com/yahoo/search/dispatch/SearchPathTest.java
@@ -18,6 +18,7 @@ import static org.junit.Assert.assertThat;
  * @author ollivir
  */
 public class SearchPathTest {
+
     @Test
     public void requreThatSearchPathsAreParsedCorrectly() {
         assertThat(SearchPath.fromString("0/0").get().toString(), equalTo("0/0"));
@@ -71,11 +72,11 @@ public class SearchPathTest {
     public void searchPathMustFilterNodesBasedOnDefinition() {
         MockSearchCluster cluster = new MockSearchCluster("a",3, 3);
 
-        assertThat(distKeysAsString(SearchPath.selectNodes("1/1", cluster)), equalTo("5"));
-        assertThat(distKeysAsString(SearchPath.selectNodes("/1", cluster)), equalTo("4,5,6"));
-        assertThat(distKeysAsString(SearchPath.selectNodes("0,1/2", cluster)), equalTo("7,8"));
-        assertThat(distKeysAsString(SearchPath.selectNodes("[1,3>/1", cluster)), equalTo("5,6"));
-        assertThat(distKeysAsString(SearchPath.selectNodes("[1,88>/1", cluster)), equalTo("5,6"));
+        assertThat(distKeysAsString(SearchPath.selectNodes("1/1", cluster)), equalTo("4"));
+        assertThat(distKeysAsString(SearchPath.selectNodes("/1", cluster)), equalTo("3,4,5"));
+        assertThat(distKeysAsString(SearchPath.selectNodes("0,1/2", cluster)), equalTo("6,7"));
+        assertThat(distKeysAsString(SearchPath.selectNodes("[1,3>/1", cluster)), equalTo("4,5"));
+        assertThat(distKeysAsString(SearchPath.selectNodes("[1,88>/1", cluster)), equalTo("4,5"));
     }
 
     private static String distKeysAsString(Collection<Node> nodes) {
diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/TopKEstimatorTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/TopKEstimatorTest.java
new file mode 100644
index 00000000000..c14e4f984f1
--- /dev/null
+++ b/container-search/src/test/java/com/yahoo/search/dispatch/TopKEstimatorTest.java
@@ -0,0 +1,28 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.dispatch;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+public class TopKEstimatorTest {
+    @Test
+    public void requireHitsAreEstimatedAccordingToPartitionsAndProbability() {
+        TopKEstimator estimator = new TopKEstimator(30, 0.999);
+        assertEquals(91.97368471911312, estimator.estimateExactK(200, 3), 0.0);
+        assertEquals(92, estimator.estimateK(200, 3));
+        assertEquals(37.96328109101396, estimator.estimateExactK(200, 10), 0.0);
+        assertEquals(38, estimator.estimateK(200, 10));
+        assertEquals(23.815737601023095, estimator.estimateExactK(200, 20), 0.0);
+        assertEquals(24, estimator.estimateK(200, 20));
+
+        assertEquals(37.96328109101396, estimator.estimateExactK(200, 10, 0.999), 0.0);
+        assertEquals(38, estimator.estimateK(200, 10, 0.999));
+        assertEquals(34.36212304875885, estimator.estimateExactK(200, 10, 0.99), 0.0);
+        assertEquals(35, estimator.estimateK(200, 10, 0.99));
+        assertEquals(41.44244358524574, estimator.estimateExactK(200, 10, 0.9999), 0.0);
+        assertEquals(42, estimator.estimateK(200, 10, 0.9999));
+        assertEquals(44.909040374464155, estimator.estimateExactK(200, 10, 0.99999), 0.0);
+        assertEquals(45, estimator.estimateK(200, 10, 0.99999));
+    }
+}
diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTest.java
index 10a579b0e4f..09024150a9a 100644
--- a/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTest.java
+++ b/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTest.java
@@ -8,18 +8,19 @@ import com.yahoo.net.HostName;
 import com.yahoo.prelude.Pong;
 import com.yahoo.search.cluster.ClusterMonitor;
 import com.yahoo.search.dispatch.MockSearchCluster;
+import com.yahoo.search.dispatch.TopKEstimator;
 import com.yahoo.search.result.ErrorMessage;
 import org.junit.Test;
 
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
-import java.util.concurrent.Callable;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 
+import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 
@@ -34,6 +35,7 @@ public class SearchClusterTest {
         final int nodesPerGroup;
         final VipStatus vipStatus;
         final SearchCluster searchCluster;
+        final ClusterMonitor clusterMonitor;
         final List<AtomicInteger> numDocsPerNode;
         List<AtomicInteger> pingCounts;
 
@@ -57,74 +59,76 @@ public class SearchClusterTest {
                 numDocsPerNode.add(new AtomicInteger(1));
                 pingCounts.add(new AtomicInteger(0));
             }
-            searchCluster = new SearchCluster(clusterId, MockSearchCluster.createDispatchConfig(nodes), nodes.size() / nodesPerGroup, vipStatus);
+            searchCluster = new SearchCluster(clusterId, MockSearchCluster.createDispatchConfig(nodes), nodes.size() / nodesPerGroup,
+                                              vipStatus, new Factory(nodesPerGroup, numDocsPerNode, pingCounts));
+            clusterMonitor = new ClusterMonitor(searchCluster, false);
+            searchCluster.addMonitoring(clusterMonitor);
         }
 
-        void startMonitoring() {
-            searchCluster.startClusterMonitoring(new Factory(nodesPerGroup, numDocsPerNode, pingCounts));
-        }
-
-        static private int maxFrom(List<AtomicInteger> list) {
-            int max = list.get(0).get();
-            for (AtomicInteger v : list) {
-                if (v.get() > max) {
-                    max = v.get();
+        private int maxPingCount() {
+            int max = pingCounts.get(0).get();
+            for (AtomicInteger count : pingCounts) {
+                if (count.get() > max) {
+                    max = count.get();
                 }
             }
             return max;
         }
 
-        private static int minFrom(List<AtomicInteger> list) {
-            int min = list.get(0).get();
-            for (AtomicInteger v : list) {
-                if (v.get() < min) {
-                    min = v.get();
+        private int minPingCount() {
+            int min = pingCounts.get(0).get();
+            for (AtomicInteger count : pingCounts) {
+                if (count.get() < min) {
+                    min = count.get();
                 }
             }
             return min;
         }
 
-        private void waitAtLeast(int atLeast, List<AtomicInteger> list) {
-            while (minFrom(list) < atLeast) {
+        void waitOneFullPingRound() {
+            int minPingCount = minPingCount();
+            int atLeast = maxPingCount() + 1;
+            while (minPingCount < atLeast) {
                 ExecutorService executor = Executors.newCachedThreadPool();
-                searchCluster.clusterMonitor().ping(executor);
+                clusterMonitor.ping(executor);
                 executor.shutdown();
                 try {
                     boolean completed = executor.awaitTermination(120, TimeUnit.SECONDS);
                     if ( ! completed )
                         throw new IllegalStateException("Ping thread timed out");
+                    // Since a separate thread will be modifying values in pingCounts, we need to wait for the thread to
+                    // finish before re-reading the minimum value
+                    minPingCount = minPingCount();
                 } catch (InterruptedException e) {
-                    System.out.println("Ping thread interrupted");
+                    throw new RuntimeException(e);
                 }
             }
         }
 
-        void waitOneFullPingRound() {
-            waitAtLeast(maxFrom(pingCounts) + 1, pingCounts);
-        }
-
         @Override
         public void close() {
-            searchCluster.shutDown();
+            clusterMonitor.shutdown();
         }
 
         static class Factory implements PingFactory {
 
-            static class Pinger implements Callable<Pong> {
+            static class PingJob implements Pinger {
 
                 private final AtomicInteger numDocs;
                 private final AtomicInteger pingCount;
-                Pinger(AtomicInteger numDocs, AtomicInteger pingCount) {
+                private final PongHandler pongHandler;
+                PingJob(AtomicInteger numDocs, AtomicInteger pingCount, PongHandler pongHandler) {
                     this.numDocs = numDocs;
                     this.pingCount = pingCount;
+                    this.pongHandler = pongHandler;
                 }
                 @Override
-                public Pong call() {
+                public void ping() {
                     int docs = numDocs.get();
-                    pingCount.incrementAndGet();
-                    return (docs < 0)
+                    pongHandler.handle ((docs < 0)
                             ? new Pong(ErrorMessage.createBackendCommunicationError("Negative numDocs = " + docs))
-                            : new Pong(docs);
+                            : new Pong(docs));
+                    pingCount.incrementAndGet();
                 }
             }
 
@@ -139,9 +143,9 @@ public class SearchClusterTest {
             }
 
             @Override
-            public Callable<Pong> createPinger(Node node, ClusterMonitor<Node> monitor) {
+            public Pinger createPinger(Node node, ClusterMonitor<Node> monitor, PongHandler pongHandler) {
                 int index = node.group() * numPerGroup + node.key();
-                return new Pinger(activeDocs.get(index), pingCounts.get(index));
+                return new PingJob(activeDocs.get(index), pingCounts.get(index), pongHandler);
             }
         }
 
@@ -153,7 +157,6 @@ public class SearchClusterTest {
             assertTrue(test.searchCluster.localCorpusDispatchTarget().isEmpty());
 
             assertFalse(test.vipStatus.isInRotation());
-            test.startMonitoring();
             test.waitOneFullPingRound();
             assertTrue(test.vipStatus.isInRotation());
         }
@@ -162,7 +165,6 @@ public class SearchClusterTest {
     @Test
     public void requireThatZeroDocsAreFine() {
         try (State test = new State("cluster.1", 2, "a", "b")) {
-            test.startMonitoring();
             test.waitOneFullPingRound();
 
             assertTrue(test.vipStatus.isInRotation());
@@ -184,19 +186,31 @@ public class SearchClusterTest {
             assertTrue(test.searchCluster.localCorpusDispatchTarget().isPresent());
 
             assertFalse(test.vipStatus.isInRotation());
-            test.startMonitoring();
             test.waitOneFullPingRound();
             assertTrue(test.vipStatus.isInRotation());
         }
     }
 
     @Test
-    public void requireThatVipStatusIsDefaultDownWithOnlySingleLocalDispatch() {
+    public void requireThatVipStatusStaysUpWithLocalDispatchAndClusterSize1() {
         try (State test = new State("cluster.1", 1, HostName.getLocalhost())) {
             assertTrue(test.searchCluster.localCorpusDispatchTarget().isPresent());
 
             assertFalse(test.vipStatus.isInRotation());
-            test.startMonitoring();
+            test.waitOneFullPingRound();
+            assertTrue(test.vipStatus.isInRotation());
+            test.numDocsPerNode.get(0).set(-1);
+            test.waitOneFullPingRound();
+            assertTrue(test.vipStatus.isInRotation());
+        }
+    }
+
+    @Test
+    public void requireThatVipStatusIsDefaultDownWithLocalDispatchAndClusterSize2() {
+        try (State test = new State("cluster.1", 1, HostName.getLocalhost(), "otherhost")) {
+            assertTrue(test.searchCluster.localCorpusDispatchTarget().isPresent());
+
+            assertFalse(test.vipStatus.isInRotation());
             test.waitOneFullPingRound();
             assertTrue(test.vipStatus.isInRotation());
             test.numDocsPerNode.get(0).set(-1);
@@ -209,7 +223,6 @@ public class SearchClusterTest {
     public void requireThatVipStatusDownWhenLocalIsDown() {
         try (State test = new State("cluster.1",1,HostName.getLocalhost(), "b")) {
 
-            test.startMonitoring();
             test.waitOneFullPingRound();
             assertTrue(test.vipStatus.isInRotation());
             assertTrue(test.searchCluster.localCorpusDispatchTarget().isPresent());
@@ -245,7 +258,6 @@ public class SearchClusterTest {
         List<String> nodeNames = generateNodeNames(numGroups, nodesPerGroup);
 
         try (State test = new State("cluster.1", nodesPerGroup, nodeNames)) {
-            test.startMonitoring();
             test.waitOneFullPingRound();
             assertTrue(test.vipStatus.isInRotation());
             assertTrue(test.searchCluster.localCorpusDispatchTarget().isEmpty());
@@ -273,8 +285,8 @@ public class SearchClusterTest {
     static private List<String> generateNodeNames(int numGroups, int nodesPerGroup) {
         List<String> nodeNames = new ArrayList<>(numGroups*nodesPerGroup);
         for (int g = 0; g < numGroups; g++) {
-            for (int n=0; n < nodesPerGroup; n++) {
-                nodeNames.add(new StringBuilder("node.").append(g).append('.').append(n).toString());
+            for (int n = 0; n < nodesPerGroup; n++) {
+                nodeNames.add("node." + g + '.' + n);
             }
         }
         return nodeNames;
@@ -284,7 +296,6 @@ public class SearchClusterTest {
         List<String> nodeNames = generateNodeNames(numGroups, nodesPerGroup);
 
         try (State test = new State("cluster.1", nodesPerGroup, nodeNames)) {
-            test.startMonitoring();
             test.waitOneFullPingRound();
             assertTrue(test.vipStatus.isInRotation());
             assertTrue(test.searchCluster.localCorpusDispatchTarget().isEmpty());
@@ -310,4 +321,18 @@ public class SearchClusterTest {
         verifyThatVipStatusUpRequireOnlyOneOnlineNode(3, 3);
     }
 
+    @Test
+    public void requireThatPingSequenceIsUpHeld() {
+        Node node = new Node(1, "n", 1);
+        assertEquals(1, node.createPingSequenceId());
+        assertEquals(2, node.createPingSequenceId());
+        assertEquals(0, node.getLastReceivedPongId());
+        assertTrue(node.isLastReceivedPong(2));
+        assertEquals(2, node.getLastReceivedPongId());
+        assertFalse(node.isLastReceivedPong(1));
+        assertFalse(node.isLastReceivedPong(2));
+        assertTrue(node.isLastReceivedPong(3));
+        assertEquals(3, node.getLastReceivedPongId());
+    }
+
 }
diff --git a/container-search/src/test/java/com/yahoo/search/federation/test/FederationSearcherTestCase.java b/container-search/src/test/java/com/yahoo/search/federation/test/FederationSearcherTestCase.java
index 111b7a8eb69..65cb4dff1f8 100644
--- a/container-search/src/test/java/com/yahoo/search/federation/test/FederationSearcherTestCase.java
+++ b/container-search/src/test/java/com/yahoo/search/federation/test/FederationSearcherTestCase.java
@@ -201,13 +201,33 @@ public class FederationSearcherTestCase {
     }
 
     @Test
-    public void testPropertyPropagation() {
-        Result result = searchWithPropertyPropagation(PropagateSourceProperties.ALL);
+    public void testPropertyPropagation_native() {
+        Result result = searchWithPropertyPropagation(PropagateSourceProperties.NATIVE);
 
         assertEquals("source:mySource1", result.hits().get(0).getId().stringValue());
         assertEquals("source:mySource2", result.hits().get(1).getId().stringValue());
         assertEquals("nalle", result.hits().get(0).getQuery().getPresentation().getSummary());
         assertNull(result.hits().get(1).getQuery().getPresentation().getSummary());
+        assertEquals(null, result.hits().get(0).getQuery().properties().get("custom"));
+    }
+
+    @Test
+    public void testPropertyPropagation_every() {
+        Result result = searchWithPropertyPropagation(PropagateSourceProperties.EVERY);
+
+        assertEquals("source:mySource1", result.hits().get(0).getId().stringValue());
+        assertEquals("source:mySource2", result.hits().get(1).getId().stringValue());
+        assertEquals("nalle", result.hits().get(0).getQuery().getPresentation().getSummary());
+        assertEquals("foo", result.hits().get(0).getQuery().properties().get("customSourceProperty"));
+        assertEquals(null,  result.hits().get(1).getQuery().properties().get("customSourceProperty"));
+        assertEquals(null,  result.hits().get(0).getQuery().properties().get("custom.source.property"));
+        assertEquals("bar", result.hits().get(1).getQuery().properties().get("custom.source.property"));
+        assertEquals(13, result.hits().get(0).getQuery().properties().get("hits"));
+        assertEquals(1, result.hits().get(0).getQuery().properties().get("offset"));
+        assertEquals(10, result.hits().get(1).getQuery().properties().get("hits"));
+        assertEquals(0, result.hits().get(1).getQuery().properties().get("offset"));
+
+        assertNull(result.hits().get(1).getQuery().getPresentation().getSummary());
     }
 
     private Result searchWithPropertyPropagation(PropagateSourceProperties.Enum propagateSourceProperties) {
@@ -215,7 +235,7 @@ public class FederationSearcherTestCase {
         addChained(new MockSearcher(), "mySource2");
         Chain<Searcher> mainChain = new Chain<>("default", createFederationSearcher(propagateSourceProperties));
 
-        Query q = new Query(QueryTestCase.httpEncode("?query=test&source.mySource1.presentation.summary=nalle"));
+        Query q = new Query(QueryTestCase.httpEncode("?query=test&source.mySource1.presentation.summary=nalle&source.mySource1.customSourceProperty=foo&source.mySource2.custom.source.property=bar&source.mySource1.hits=13&source.mySource1.offset=1"));
 
         Result result = new Execution(mainChain, Execution.Context.createContextStub(chainRegistry, null)).search(q);
         assertNull(result.hits().getError());
diff --git a/container-search/src/test/java/com/yahoo/search/handler/test/JSONSearchHandlerTestCase.java b/container-search/src/test/java/com/yahoo/search/handler/test/JSONSearchHandlerTestCase.java
index 02e2152d7c9..272092b6fc0 100644
--- a/container-search/src/test/java/com/yahoo/search/handler/test/JSONSearchHandlerTestCase.java
+++ b/container-search/src/test/java/com/yahoo/search/handler/test/JSONSearchHandlerTestCase.java
@@ -12,7 +12,7 @@ import com.yahoo.net.HostName;
 import com.yahoo.search.handler.SearchHandler;
 import com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase;
 import com.yahoo.slime.Inspector;
-import com.yahoo.vespa.config.SlimeUtils;
+import com.yahoo.slime.SlimeUtils;
 import org.json.JSONArray;
 import org.json.JSONObject;
 import org.junit.After;
diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/XmlReadingTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/XmlReadingTestCase.java
index 0f4a22ef368..6d7d6f38e99 100644
--- a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/XmlReadingTestCase.java
+++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/XmlReadingTestCase.java
@@ -299,8 +299,6 @@ public class XmlReadingTestCase {
         String queryString="tiled?query=india&queryProfile=myprofile&source.common.intl=tw&source.common.mode=adv";
 
         Query query=new Query(HttpRequest.createTestRequest(queryString, Method.GET), registry.getComponent("myprofile"));
-        for (Map.Entry e : query.properties().listProperties().entrySet())
-            System.out.println(e);
         assertEquals("news",query.properties().listProperties().get("source.common.provider"));
         assertEquals("news",query.properties().get("source.common.provider"));
     }
@@ -384,43 +382,64 @@ public class XmlReadingTestCase {
             assertNull(query.properties().get("profileRef.myProfile1Only"));
 
             // later assignment
-            query.properties().set("profileRef.name","newName");
-            assertEquals("newName",query.properties().get("profileRef.name"));
+            query.properties().set("profileRef.name", "newName");
+            assertEquals("newName", query.properties().get("profileRef.name"));
             // ...will not impact others
-            query=new Query(HttpRequest.createTestRequest("?query=test&profileRef=ref:MyProfile2", Method.GET),registry.getComponent("default"));
-            assertEquals("MyProfile2",query.properties().get("profileRef.name"));
+            query=new Query(HttpRequest.createTestRequest("?query=test&profileRef=ref:MyProfile2", Method.GET), registry.getComponent("default"));
+            assertEquals("MyProfile2", query.properties().get("profileRef.name"));
         }
 
     }
 
     @Test
     public void testRefOverrideTyped() {
-        CompiledQueryProfileRegistry registry=new QueryProfileXMLReader().read("src/test/java/com/yahoo/search/query/profile/config/test/refoverridetyped").compile();
+        CompiledQueryProfileRegistry registry = new QueryProfileXMLReader().read("src/test/java/com/yahoo/search/query/profile/config/test/refoverridetyped").compile();
 
         {
             // Original reference
-            Query query=new Query(HttpRequest.createTestRequest("?query=test", Method.GET),registry.getComponent("default"));
-            assertEquals(null,query.properties().get("profileRef"));
-            assertEquals("MyProfile1",query.properties().get("profileRef.name"));
-            assertEquals("myProfile1Only",query.properties().get("profileRef.myProfile1Only"));
+            Query query = new Query(HttpRequest.createTestRequest("?query=test", Method.GET), registry.getComponent("default"));
+            assertEquals(null, query.properties().get("profileRef"));
+            assertEquals("MyProfile1", query.properties().get("profileRef.name"));
+            assertEquals("myProfile1Only", query.properties().get("profileRef.myProfile1Only"));
             assertNull(query.properties().get("profileRef.myProfile2Only"));
         }
 
         {
             // Overridden reference
-            Query query=new Query(HttpRequest.createTestRequest("?query=test&profileRef=MyProfile2", Method.GET),registry.getComponent("default"));
-            assertEquals(null,query.properties().get("profileRef"));
-            assertEquals("MyProfile2",query.properties().get("profileRef.name"));
-            assertEquals("myProfile2Only",query.properties().get("profileRef.myProfile2Only"));
+            Query query = new Query(HttpRequest.createTestRequest("?query=test&profileRef=MyProfile2", Method.GET), registry.getComponent("default"));
+            assertEquals(null, query.properties().get("profileRef"));
+            assertEquals("MyProfile2", query.properties().get("profileRef.name"));
+            assertEquals("myProfile2Only", query.properties().get("profileRef.myProfile2Only"));
             assertNull(query.properties().get("profileRef.myProfile1Only"));
 
             // later assignment
-            query.properties().set("profileRef.name","newName");
-            assertEquals("newName",query.properties().get("profileRef.name"));
+            query.properties().set("profileRef.name", "newName");
+            assertEquals("newName", query.properties().get("profileRef.name"));
             // ...will not impact others
-            query=new Query(HttpRequest.createTestRequest("?query=test&profileRef=ref:MyProfile2", Method.GET),registry.getComponent("default"));
-            assertEquals("MyProfile2",query.properties().get("profileRef.name"));
+            query = new Query(HttpRequest.createTestRequest("?query=test&profileRef=ref:MyProfile2", Method.GET), registry.getComponent("default"));
+            assertEquals("MyProfile2", query.properties().get("profileRef.name"));
         }
     }
 
+    @Test
+    public void testTensorTypes() {
+        CompiledQueryProfileRegistry registry = new QueryProfileXMLReader().read("src/test/java/com/yahoo/search/query/profile/config/test/tensortypes").compile();
+
+        QueryProfileType type1 = registry.getTypeRegistry().getComponent("type1");
+        assertEquals("tensor<float>(x[1])", type1.getFieldType(new CompoundName("ranking.features.query(tensor_1)")).stringValue());
+        assertNull(type1.getFieldType(new CompoundName("ranking.features.query(tensor_2)")));
+        assertNull(type1.getFieldType(new CompoundName("ranking.features.query(tensor_3)")));
+
+        QueryProfileType type2 = registry.getTypeRegistry().getComponent("type2");
+        assertNull(type2.getFieldType(new CompoundName("ranking.features.query(tensor_1)")));
+        assertEquals("tensor<float>(x[2])", type2.getFieldType(new CompoundName("ranking.features.query(tensor_2)")).stringValue());
+        assertEquals("tensor<float>(x[3])", type2.getFieldType(new CompoundName("ranking.features.query(tensor_3)")).stringValue());
+
+        Query queryProfile1 = new Query("?query=test&ranking.features.query(tensor_1)=[1.200]", registry.getComponent("profile1"));
+        assertEquals("Is received as a tensor tensor", "tensor<float>(x[1]):[1.2]", queryProfile1.properties().get("ranking.features.query(tensor_1)").toString());
+
+        Query queryProfile2 = new Query("?query=test&ranking.features.query(tensor_1)=[1.200]", registry.getComponent("profile2"));
+        assertEquals("Is received as a string", "[1.200]", queryProfile2.properties().get("ranking.features.query(tensor_1)").toString());
+    }
+
 }
diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/tensortypes/profile1.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/tensortypes/profile1.xml
new file mode 100644
index 00000000000..000fd3e1c5b
--- /dev/null
+++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/tensortypes/profile1.xml
@@ -0,0 +1,2 @@
+<query-profile id="profile1" type="type1">
+</query-profile>
diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/tensortypes/profile2.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/tensortypes/profile2.xml
new file mode 100644
index 00000000000..f6539da23e8
--- /dev/null
+++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/tensortypes/profile2.xml
@@ -0,0 +1,2 @@
+<query-profile id="profile2" type="type2">
+</query-profile>
diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/tensortypes/types/type1.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/tensortypes/types/type1.xml
new file mode 100644
index 00000000000..3dfaab9c5f2
--- /dev/null
+++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/tensortypes/types/type1.xml
@@ -0,0 +1,3 @@
+<query-profile-type id="type1">
+    <field name="ranking.features.query(tensor_1)" type="tensor&lt;float&gt;(x[1])" />
+</query-profile-type>
diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/tensortypes/types/type2.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/tensortypes/types/type2.xml
new file mode 100644
index 00000000000..ed7cf23e464
--- /dev/null
+++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/tensortypes/types/type2.xml
@@ -0,0 +1,4 @@
+<query-profile-type id="type2">
+    <field name="ranking.features.query(tensor_2)" type="tensor&lt;float&gt;(x[2])" />
+    <field name="ranking.features.query(tensor_3)" type="tensor&lt;float&gt;(x[3])" />
+</query-profile-type>
diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileTestCase.java
index 46efb736918..eb1584efe84 100644
--- a/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileTestCase.java
+++ b/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileTestCase.java
@@ -326,14 +326,14 @@ public class QueryProfileTestCase {
         assertEquals("mormor-model.b", annetBarnMap.get("venn.model.b"));
     }
 
-    /** Tests that dots are followed when setting overridability */
+    /** Dots are followed when setting overridability */
     @Test
     public void testInstanceOverridable() {
         QueryProfile profile = new QueryProfile("root/unoverridableIndex");
         profile.set("model.defaultIndex","default", null);
         profile.setOverridable("model.defaultIndex", false,null);
 
-        assertFalse(profile.isDeclaredOverridable("model.defaultIndex",null).booleanValue());
+        assertFalse(profile.isDeclaredOverridable("model.defaultIndex",null));
 
         // Parameters should be ignored
         Query query = new Query(HttpRequest.createTestRequest("?model.defaultIndex=title", Method.GET), profile.compile(null));
@@ -345,7 +345,7 @@ public class QueryProfileTestCase {
         assertEquals("de", query.getModel().getLanguage().languageCode());
     }
 
-    /** Tests that dots are followed when setting overridability...also with variants */
+    /** Dots are followed when setting overridability, also with variants */
     @Test
     public void testInstanceOverridableWithVariants() {
         QueryProfile profile = new QueryProfile("root/unoverridableIndex");
@@ -504,7 +504,8 @@ public class QueryProfileTestCase {
         p.set("a","a-value", null);
         p.set("a.b","a.b-value", null);
         Map<String, Object> values = p.compile(null).listValues("a");
-        assertEquals(1, values.size());
+        assertEquals(2, values.size());
+        p.set("a","a-value", null);
         assertEquals("a.b-value", values.get("b"));
     }
 
diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/types/test/QueryProfileTypeTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/types/test/QueryProfileTypeTestCase.java
index c05c3589a30..3c200debcaf 100644
--- a/container-search/src/test/java/com/yahoo/search/query/profile/types/test/QueryProfileTypeTestCase.java
+++ b/container-search/src/test/java/com/yahoo/search/query/profile/types/test/QueryProfileTypeTestCase.java
@@ -397,7 +397,7 @@ public class QueryProfileTypeTestCase {
 
     @Test
     public void testTensorRankFeatureInRequest() throws UnsupportedEncodingException {
-        QueryProfile profile=new QueryProfile("test");
+        QueryProfile profile = new QueryProfile("test");
         profile.setType(type);
         registry.register(profile);
 
@@ -447,25 +447,25 @@ public class QueryProfileTypeTestCase {
      */
     @Test
     public void testTypedOverridingOfQueryProfileReferencesNonStrictThroughQueryNestedInAnUntypedProfile() {
-        QueryProfile topMap=new QueryProfile("topMap");
+        QueryProfile topMap = new QueryProfile("topMap");
 
-        QueryProfile subMap=new QueryProfile("topSubMap");
-        topMap.set("subMap",subMap, registry);
+        QueryProfile subMap = new QueryProfile("topSubMap");
+        topMap.set("subMap", subMap, registry);
 
-        QueryProfile test=new QueryProfile("test");
+        QueryProfile test = new QueryProfile("test");
         test.setType(type);
-        subMap.set("typeProfile",test, registry);
+        subMap.set("typeProfile", test, registry);
 
-        QueryProfile myUser=new QueryProfile("myUser");
+        QueryProfile myUser = new QueryProfile("myUser");
         myUser.setType(user);
-        myUser.set("myUserString","userValue1", registry);
-        myUser.set("myUserInteger",442, registry);
-        test.set("myUserQueryProfile",myUser, registry);
+        myUser.set("myUserString", "userValue1", registry);
+        myUser.set("myUserInteger", 442, registry);
+        test.set("myUserQueryProfile", myUser, registry);
 
-        QueryProfile newUser=new QueryProfile("newUser");
+        QueryProfile newUser = new QueryProfile("newUser");
         newUser.setType(user);
-        newUser.set("myUserString","newUserValue1", registry);
-        newUser.set("myUserInteger",845, registry);
+        newUser.set("myUserString", "newUserValue1", registry);
+        newUser.set("myUserInteger", 845, registry);
 
         registry.register(topMap);
         registry.register(subMap);
diff --git a/container-search/src/test/java/com/yahoo/search/searchers/ValidateNearestNeighborTestCase.java b/container-search/src/test/java/com/yahoo/search/searchers/ValidateNearestNeighborTestCase.java
index 0cbf3a6f92c..2c849a9b52c 100644
--- a/container-search/src/test/java/com/yahoo/search/searchers/ValidateNearestNeighborTestCase.java
+++ b/container-search/src/test/java/com/yahoo/search/searchers/ValidateNearestNeighborTestCase.java
@@ -1,14 +1,11 @@
 // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-package com.yahoo.prelude.searcher;
+package com.yahoo.search.searchers;
 
 import com.google.common.util.concurrent.MoreExecutors;
 
 import com.yahoo.config.subscription.ConfigGetter;
 import com.yahoo.config.subscription.RawSource;
-import com.yahoo.language.Linguistics;
 import com.yahoo.language.simple.SimpleLinguistics;
-import com.yahoo.prelude.Index;
 import com.yahoo.prelude.IndexFacts;
 import com.yahoo.prelude.IndexModel;
 import com.yahoo.prelude.SearchDefinition;
@@ -20,15 +17,11 @@ import com.yahoo.search.rendering.RendererRegistry;
 import com.yahoo.search.Result;
 import com.yahoo.search.result.ErrorMessage;
 import com.yahoo.search.searchchain.Execution;
-import com.yahoo.search.Searcher;
-import com.yahoo.search.searchers.ValidateNearestNeighborSearcher;
 import com.yahoo.search.yql.YqlParser;
 import com.yahoo.tensor.Tensor;
 import com.yahoo.tensor.TensorType;
 import com.yahoo.vespa.config.search.AttributesConfig;
 
-import java.util.*;
-
 import org.junit.Test;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNull;
@@ -139,12 +132,24 @@ public class ValidateNearestNeighborTestCase {
         assertEquals(ErrorMessage.createIllegalQuery(message), r.hits().getError());
     }
 
+    static String desc(String field, String qt, int th, String errmsg) {
+        StringBuilder r = new StringBuilder();
+        r.append("NEAREST_NEIGHBOR {");
+        r.append("field=").append(field);
+        r.append(",queryTensorName=").append(qt);
+        r.append(",hnsw.exploreAdditionalHits=0");
+        r.append(",approximate=true");
+        r.append(",targetNumHits=").append(th);
+        r.append("} ").append(errmsg);
+        return r.toString();
+    }
+
     @Test
     public void testMissingTargetNumHits() {
         String q = "select * from sources * where nearestNeighbor(dvector,qvector);";
         Tensor t = makeTensor(tt_dense_dvector_3);
         Result r = doSearch(searcher, q, t);
-        assertErrMsg("NEAREST_NEIGHBOR {field=dvector,queryTensorName=qvector,targetNumHits=0} has invalid targetNumHits", r);
+        assertErrMsg(desc("dvector", "qvector", 0, "has invalid targetNumHits 0: Must be >= 1"), r);
     }
 
     @Test
@@ -152,16 +157,7 @@ public class ValidateNearestNeighborTestCase {
         String q = makeQuery("dvector", "foo");
         Tensor t = makeTensor(tt_dense_dvector_3);
         Result r = doSearch(searcher, q, t);
-        assertErrMsg("NEAREST_NEIGHBOR {field=dvector,queryTensorName=foo,targetNumHits=1} query tensor not found", r);
-    }
-
-    @Test
-    public void testQueryTensorWrongType() {
-        String q = makeQuery("dvector", "qvector");
-        Result r = doSearch(searcher, q, "tensor string");
-        assertErrMsg("NEAREST_NEIGHBOR {field=dvector,queryTensorName=qvector,targetNumHits=1} query tensor should be a tensor, was: class java.lang.String", r);
-        r = doSearch(searcher, q, null);
-        assertErrMsg("NEAREST_NEIGHBOR {field=dvector,queryTensorName=qvector,targetNumHits=1} query tensor should be a tensor, was: null", r);
+        assertErrMsg(desc("dvector", "foo", 1, "requires a tensor rank feature query(foo) but this is not present"), r);
     }
 
     @Test
@@ -169,7 +165,7 @@ public class ValidateNearestNeighborTestCase {
         String q = makeQuery("dvector", "qvector");
         Tensor t = makeTensor(tt_dense_dvector_2, 2);
         Result r = doSearch(searcher, q, t);
-        assertErrMsg("NEAREST_NEIGHBOR {field=dvector,queryTensorName=qvector,targetNumHits=1} field type tensor(x[3]) does not match query tensor type tensor(x[2])", r);
+        assertErrMsg(desc("dvector", "qvector", 1, "field type tensor(x[3]) does not match query type tensor(x[2])"), r);
     }
 
     @Test
@@ -177,7 +173,7 @@ public class ValidateNearestNeighborTestCase {
         String q = makeQuery("foo", "qvector");
         Tensor t = makeTensor(tt_dense_dvector_3);
         Result r = doSearch(searcher, q, t);
-        assertErrMsg("NEAREST_NEIGHBOR {field=foo,queryTensorName=qvector,targetNumHits=1} field is not an attribute", r);
+        assertErrMsg(desc("foo", "qvector", 1, "field is not an attribute"), r);
     }
 
     @Test
@@ -185,7 +181,7 @@ public class ValidateNearestNeighborTestCase {
         String q = makeQuery("simple", "qvector");
         Tensor t = makeTensor(tt_dense_dvector_3);
         Result r = doSearch(searcher, q, t);
-        assertErrMsg("NEAREST_NEIGHBOR {field=simple,queryTensorName=qvector,targetNumHits=1} field is not a tensor", r);
+        assertErrMsg(desc("simple", "qvector", 1, "field is not a tensor"), r);
     }
 
     @Test
@@ -193,7 +189,7 @@ public class ValidateNearestNeighborTestCase {
         String q = makeQuery("sparse", "qvector");
         Tensor t = makeTensor(tt_sparse_vector_x);
         Result r = doSearch(searcher, q, t);
-        assertErrMsg("NEAREST_NEIGHBOR {field=sparse,queryTensorName=qvector,targetNumHits=1} tensor type tensor(x{}) is not a dense vector", r);
+        assertErrMsg(desc("sparse", "qvector", 1, "tensor type tensor(x{}) is not a dense vector"), r);
     }
 
     @Test
@@ -201,14 +197,14 @@ public class ValidateNearestNeighborTestCase {
         String q = makeQuery("matrix", "qvector");
         Tensor t = makeMatrix(tt_dense_matrix_xy);
         Result r = doSearch(searcher, q, t);
-        assertErrMsg("NEAREST_NEIGHBOR {field=matrix,queryTensorName=qvector,targetNumHits=1} tensor type tensor(x[3],y[1]) is not a dense vector", r);
+        assertErrMsg(desc("matrix", "qvector", 1, "tensor type tensor(x[3],y[1]) is not a dense vector"), r);
     }
 
-    private static Result doSearch(ValidateNearestNeighborSearcher searcher, String yqlQuery, Object qTensor) {
+    private static Result doSearch(ValidateNearestNeighborSearcher searcher, String yqlQuery, Tensor qTensor) {
         QueryTree queryTree = new YqlParser(new ParserEnvironment()).parse(new Parsable().setQuery(yqlQuery));
         Query query = new Query();
         query.getModel().getQueryTree().setRoot(queryTree.getRoot());
-        query.getRanking().getProperties().put("qvector", qTensor);
+        query.getRanking().getFeatures().put("query(qvector)", qTensor);
         SearchDefinition searchDefinition = new SearchDefinition("document");
         IndexFacts indexFacts = new IndexFacts(new IndexModel(searchDefinition));
         Execution.Context context = new Execution.Context(null, indexFacts, null, new RendererRegistry(MoreExecutors.directExecutor()), new SimpleLinguistics());
diff --git a/container-search/src/test/java/com/yahoo/search/searchers/test/InputCheckingSearcherTestCase.java b/container-search/src/test/java/com/yahoo/search/searchers/test/InputCheckingSearcherTestCase.java
index dbeced57c52..aa507d38be5 100644
--- a/container-search/src/test/java/com/yahoo/search/searchers/test/InputCheckingSearcherTestCase.java
+++ b/container-search/src/test/java/com/yahoo/search/searchers/test/InputCheckingSearcherTestCase.java
@@ -5,6 +5,7 @@ import static org.junit.Assert.*;
 
 import java.io.UnsupportedEncodingException;
 import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
 
 import org.junit.After;
 import org.junit.Before;
@@ -23,50 +24,50 @@ import com.yahoo.text.Utf8;
 /**
  * Functional test for InputCheckingSearcher.
  *
- * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ * @author Steinar Knutsen
  */
 public class InputCheckingSearcherTestCase {
 
     Execution execution;
 
     @Before
-    public void setUp() throws Exception {
+    public void setUp() {
         execution = new Execution(new Chain<Searcher>(new InputCheckingSearcher(MetricReceiver.nullImplementation)),
-                Execution.Context.createContextStub(new IndexFacts()));
+                                  Execution.Context.createContextStub(new IndexFacts()));
     }
 
     @After
-    public void tearDown() throws Exception {
+    public void tearDown() {
         execution = null;
     }
 
     @Test
-    public final void testCommonCase() {
+    public void testCommonCase() {
         Result r = execution.search(new Query("/search/?query=three+blind+mice"));
         assertNull(r.hits().getErrorHit());
     }
 
     @Test
-    public final void candidateButAsciiOnly() {
+    public void candidateButAsciiOnly() {
         Result r = execution.search(new Query("/search/?query=a+a+a+a+a+a"));
         assertNull(r.hits().getErrorHit());
     }
 
     @Test
-    public final void candidateButValid() throws UnsupportedEncodingException {
+    public void candidateButValid() throws UnsupportedEncodingException {
         Result r = execution.search(new Query("/search/?query=" + URLEncoder.encode("å å å å å å", "UTF-8")));
         assertNull(r.hits().getErrorHit());
     }
 
     @Test
-    public final void candidateButValidAndOutsideFirst256() throws UnsupportedEncodingException {
+    public void candidateButValidAndOutsideFirst256() throws UnsupportedEncodingException {
         Result r = execution.search(new Query("/search/?query=" + URLEncoder.encode("œ œ œ œ œ œ", "UTF-8")));
         assertNull(r.hits().getErrorHit());
     }
 
 
     @Test
-    public final void testDoubleEncoded() throws UnsupportedEncodingException {
+    public void testDoubleEncoded() throws UnsupportedEncodingException {
         String rawQuery = "å å å å å å";
         byte[] encodedOnce = Utf8.toBytes(rawQuery);
         char[] secondEncodingBuffer = new char[encodedOnce.length];
@@ -74,33 +75,42 @@ public class InputCheckingSearcherTestCase {
             secondEncodingBuffer[i] = (char) (encodedOnce[i] & 0xFF);
         }
         String query = new String(secondEncodingBuffer);
-        Result r = execution.search(new Query("/search/?query=" + URLEncoder.encode(query, "UTF-8")));
+        Result r = execution.search(new Query("/search/?query=" + URLEncoder.encode(query, StandardCharsets.UTF_8)));
         assertEquals(1, r.hits().getErrorHit().errors().size());
     }
 
     @Test
-    public final void testRepeatedConsecutiveTermsInPhrase() {
-        Result r = execution.search(new Query("/search/?query=a.b.0.0.0.0.0.c"));
+    public void testRepeatedConsecutiveTermsInPhrase() {
+        Result r = execution.search(new Query("/search/?query=%22a.b.0.0.0.0.0.c%22"));
         assertNull(r.hits().getErrorHit());
-        r = execution.search(new Query("/search/?query=a.b.0.0.0.0.0.0.c"));
+        r = execution.search(new Query("/search/?query=%22a.b.0.0.0.0.0.0.c%22"));
         assertNotNull(r.hits().getErrorHit());
+        assertEquals("More than 5 ocurrences of term '0' in a row detected in phrase : \"a b 0 0 0 0 0 0 c\"",
+                     r.hits().getErrorHit().errorIterator().next().getDetailedMessage());
         r = execution.search(new Query("/search/?query=a.b.0.0.0.1.0.0.0.c"));
         assertNull(r.hits().getErrorHit());
     }
+
     @Test
-    public final void testThatMaxRepeatedConsecutiveTermsInPhraseIs5() {
-        Result r = execution.search(new Query("/search/?query=a.b.0.0.0.0.0.c"));
+    public void testThatMaxRepeatedConsecutiveTermsInPhraseIs5() {
+        Result r = execution.search(new Query("/search/?query=%22a.b.0.0.0.0.0.c%22"));
         assertNull(r.hits().getErrorHit());
-        r = execution.search(new Query("/search/?query=a.b.0.0.0.0.0.0.c"));
+        r = execution.search(new Query("/search/?query=%22a.b.0.0.0.0.0.0.c%22"));
         assertNotNull(r.hits().getErrorHit());
-        r = execution.search(new Query("/search/?query=a.b.0.0.0.1.0.0.0.c"));
+        assertEquals("More than 5 ocurrences of term '0' in a row detected in phrase : \"a b 0 0 0 0 0 0 c\"",
+                     r.hits().getErrorHit().errorIterator().next().getDetailedMessage());
+        r = execution.search(new Query("/search/?query=%22a.b.0.0.0.1.0.0.0.c%22"));
         assertNull(r.hits().getErrorHit());
     }
+
     @Test
-    public final void testThatMaxRepeatedTermsInPhraseIs10() {
-        Result r = execution.search(new Query("/search/?query=0.a.1.a.2.a.3.a.4.a.5.a.6.a.7.a.9.a"));
+    public void testThatMaxRepeatedTermsInPhraseIs10() {
+        Result r = execution.search(new Query("/search/?query=%220.a.1.a.2.a.3.a.4.a.5.a.6.a.7.a.9.a%22"));
         assertNull(r.hits().getErrorHit());
-        r = execution.search(new Query("/search/?query=0.a.1.a.2.a.3.a.4.a.5.a.6.a.7.a.8.a.9.a.10.a"));
+        r = execution.search(new Query("/search/?query=%220.a.1.a.2.a.3.a.4.a.5.a.6.a.7.a.8.a.9.a.10.a%22"));
         assertNotNull(r.hits().getErrorHit());
+        assertEquals("Phrase contains more than 10 occurrences of term 'a' in phrase : \"0 a 1 a 2 a 3 a 4 a 5 a 6 a 7 a 8 a 9 a 10 a\"",
+                     r.hits().getErrorHit().errorIterator().next().getDetailedMessage());
     }
+
 }
diff --git a/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java b/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java
index ea34be5cf37..c831ee29631 100644
--- a/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java
+++ b/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java
@@ -16,6 +16,7 @@ import com.yahoo.prelude.IndexFacts;
 import com.yahoo.prelude.IndexModel;
 import com.yahoo.prelude.SearchDefinition;
 import com.yahoo.prelude.query.AndItem;
+import com.yahoo.prelude.query.AndSegmentItem;
 import com.yahoo.prelude.query.CompositeItem;
 import com.yahoo.prelude.query.Highlight;
 import com.yahoo.prelude.query.IndexedItem;
@@ -45,6 +46,8 @@ import org.junit.Test;
 import java.io.UnsupportedEncodingException;
 import java.net.URLEncoder;
 import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
@@ -345,6 +348,61 @@ public class QueryTestCase {
     }
 
     @Test
+    public void testQueryProfileClearAndSet() {
+        QueryProfile profile = new QueryProfile("myProfile");
+        profile.set("b", "b-value", null);
+        Query q = new Query(QueryTestCase.httpEncode("/search?queryProfile=myProfile"), profile.compile(null));
+        assertEquals("b-value", q.properties().get("b"));
+        assertContains(q.properties().listProperties("b"), "b-value");
+
+        q.properties().set("b", null, null);
+        assertContains(q.properties().listProperties("b"), (Object)null);
+
+        q.properties().set("b", "b-value", null);
+        assertEquals("b-value", q.properties().get("b"));
+        assertContains(q.properties().listProperties("b"), "b-value");
+    }
+
+    @Test
+    public void testQueryProfileClearValue() {
+        QueryProfile profile = new QueryProfile("myProfile");
+        profile.set("a", "a-value", null);
+        profile.set("b", "b-value", null);
+        profile.set("b.c", "b.c-value", null);
+        profile.set("b.d", "b.d-value", null);
+        Query q = new Query(QueryTestCase.httpEncode("/search?queryProfile=myProfile"), profile.compile(null));
+        assertEquals("a-value", q.properties().get("a"));
+        assertEquals("b-value", q.properties().get("b"));
+        assertEquals("b.c-value", q.properties().get("b.c"));
+        assertEquals("b.d-value", q.properties().get("b.d"));
+        assertContains(q.properties().listProperties("b"), "b-value", "b.c-value", "b.d-value");
+
+        q.properties().set("a", null, null);
+        assertEquals(null, q.properties().get("a"));
+
+        q.properties().set("b", null, null);
+        assertEquals(null, q.properties().get("b"));
+        assertEquals("b.c-value", q.properties().get("b.c"));
+        assertEquals("b.d-value", q.properties().get("b.d"));
+        assertContains(q.properties().listProperties("b"), null, "b.c-value", "b.d-value");
+
+        q.properties().set("b", "b-value", null);
+        q.properties().set("b.e", "b.e-value", null);
+        q.properties().set("b.f", "b.f-value", null);
+        assertEquals("b-value", q.properties().get("b"));
+        assertEquals("b.e-value", q.properties().get("b.e"));
+        assertContains(q.properties().listProperties("b"), "b-value", "b.c-value", "b.d-value", "b.e-value", "b.f-value");
+
+        q.properties().clearAll("b");
+        assertEquals(null, q.properties().get("b"));
+        assertEquals(null, q.properties().get("b.c"));
+        assertEquals(null, q.properties().get("b.d"));
+        assertEquals(null, q.properties().get("b.e"));
+        assertEquals(null, q.properties().get("b.f"));
+        assertContains(q.properties().listProperties("b"), (Object)null);
+    }
+
+    @Test
     public void testNotEqual() {
         Query q = new Query("/?query=something+test&nocache");
         Query p = new Query("/?query=something+test");
@@ -884,12 +942,12 @@ public class QueryTestCase {
     @Test
     public void testImplicitPhraseIsDefault() {
         Query query = new Query(httpEncode("?query=it's fine"));
-        assertEquals("AND 'it s' fine", query.getModel().getQueryTree().toString());
+        assertEquals("AND (SAND it s) fine", query.getModel().getQueryTree().toString());
     }
 
     @Test
     public void testImplicitPhrase() {
-        Query query = new Query(httpEncode("?query=myfield:it's myfield:fine"));
+        Query query = new Query(httpEncode("?query=myfield:it's myfield:a.b myfield:c"));
 
         SearchDefinition test = new SearchDefinition("test");
         Index myField = new Index("myfield");
@@ -899,12 +957,12 @@ public class QueryTestCase {
         IndexModel indexModel = new IndexModel(test);
         query.getModel().setExecution(new Execution(Execution.Context.createContextStub(new IndexFacts(indexModel))));
 
-        assertEquals("AND myfield:'it s' myfield:fine", query.getModel().getQueryTree().toString());
+        assertEquals("AND myfield:'it s' myfield:\"a b\" myfield:c", query.getModel().getQueryTree().toString());
     }
 
     @Test
     public void testImplicitAnd() {
-        Query query = new Query(httpEncode("?query=myfield:it's myfield:fine"));
+        Query query = new Query(httpEncode("?query=myfield:it's myfield:a.b myfield:c"));
 
         SearchDefinition test = new SearchDefinition("test");
         Index myField = new Index("myfield");
@@ -914,7 +972,57 @@ public class QueryTestCase {
         IndexModel indexModel = new IndexModel(test);
         query.getModel().setExecution(new Execution(Execution.Context.createContextStub(new IndexFacts(indexModel))));
 
-        assertEquals("AND (SAND myfield:it myfield:s) myfield:fine", query.getModel().getQueryTree().toString());
+        assertEquals("AND (SAND myfield:it myfield:s) myfield:a myfield:b myfield:c", query.getModel().getQueryTree().toString());
+        // 'it' and 's' should have connectivity 1
+        AndItem root = (AndItem)query.getModel().getQueryTree().getRoot();
+        AndSegmentItem sand = (AndSegmentItem)root.getItem(0);
+        WordItem it = (WordItem)sand.getItem(0);
+        assertEquals("it", it.getWord());
+        WordItem s = (WordItem)sand.getItem(1);
+        assertEquals("s", s.getWord());
+        assertEquals(s, it.getConnectedItem());
+        assertEquals(1.0, it.getConnectivity(), 0.00000001);
+    }
+
+    @Test
+    public void testImplicitAndConnectivity() {
+        SearchDefinition test = new SearchDefinition("test");
+        Index myField = new Index("myfield");
+        myField.addCommand("phrase-segmenting false");
+        test.addIndex(myField);
+        IndexModel indexModel = new IndexModel(test);
+
+        {
+            Query query = new Query(httpEncode("?query=myfield:b.c.d"));
+            query.getModel().setExecution(new Execution(Execution.Context.createContextStub(new IndexFacts(indexModel))));
+            assertEquals("AND myfield:b myfield:c myfield:d", query.getModel().getQueryTree().toString());
+            AndItem root = (AndItem) query.getModel().getQueryTree().getRoot();
+            WordItem b = (WordItem) root.getItem(0);
+            WordItem c = (WordItem) root.getItem(1);
+            WordItem d = (WordItem) root.getItem(2);
+            assertEquals(c, b.getConnectedItem());
+            assertEquals(1.0, b.getConnectivity(), 0.00000001);
+            assertEquals(d, c.getConnectedItem());
+            assertEquals(1.0, c.getConnectivity(), 0.00000001);
+        }
+
+        {
+            Query query = new Query(httpEncode("?query=myfield:a myfield:b.c.d myfield:e"));
+            query.getModel().setExecution(new Execution(Execution.Context.createContextStub(new IndexFacts(indexModel))));
+            assertEquals("AND myfield:a myfield:b myfield:c myfield:d myfield:e", query.getModel().getQueryTree().toString());
+            AndItem root = (AndItem) query.getModel().getQueryTree().getRoot();
+            WordItem a = (WordItem) root.getItem(0);
+            WordItem b = (WordItem) root.getItem(1);
+            WordItem c = (WordItem) root.getItem(2);
+            WordItem d = (WordItem) root.getItem(3);
+            WordItem e = (WordItem) root.getItem(4);
+            assertNull(a.getConnectedItem());
+            assertEquals(c, b.getConnectedItem());
+            assertEquals(1.0, b.getConnectivity(), 0.00000001);
+            assertEquals(d, c.getConnectedItem());
+            assertEquals(1.0, c.getConnectivity(), 0.00000001);
+            assertNull(d.getConnectedItem());
+        }
     }
 
     @Test
@@ -985,6 +1093,19 @@ public class QueryTestCase {
         assertEquals(expectedDetectionText, mockLinguistics.detector.lastDetectionText);
     }
 
+    private void assertContains(Map<String, Object> properties, Object ... expectedValues) {
+        if (expectedValues == null) {
+            assertEquals(1, properties.size());
+            assertTrue("Contains value null", properties.containsValue(null));
+        }
+        else {
+            assertEquals(properties + " contains values " + Arrays.toString(expectedValues),
+                         expectedValues.length, properties.size());
+            for (Object expectedValue : expectedValues)
+                assertTrue("Contains value " + expectedValue, properties.containsValue(expectedValue));
+        }
+    }
+
     /** A linguistics instance which records the last language detection text passed to it */
     private static class MockLinguistics extends SimpleLinguistics {
 
diff --git a/container-search/src/test/java/com/yahoo/search/yql/VespaSerializerTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/VespaSerializerTestCase.java
index 1106d8c3999..d770b08d31a 100644
--- a/container-search/src/test/java/com/yahoo/search/yql/VespaSerializerTestCase.java
+++ b/container-search/src/test/java/com/yahoo/search/yql/VespaSerializerTestCase.java
@@ -128,6 +128,9 @@ public class VespaSerializerTestCase {
     public void testNearestNeighbor() {
         parseAndConfirm("[{\"label\": \"foo\", \"targetNumHits\": 1000}]nearestNeighbor(semantic_embedding, my_property)");
         parseAndConfirm("[{\"targetNumHits\": 42}]nearestNeighbor(semantic_embedding, my_property)");
+        parseAndConfirm("[{\"targetNumHits\": 1, \"hnsw.exploreAdditionalHits\": 76}]nearestNeighbor(semantic_embedding, my_property)");
+        parseAndConfirm("[{\"targetNumHits\": 2, \"approximate\": false}]nearestNeighbor(semantic_embedding, my_property)");
+        parseAndConfirm("[{\"targetNumHits\": 3, \"hnsw.exploreAdditionalHits\": 67, \"approximate\": false}]nearestNeighbor(semantic_embedding, my_property)");
     }
 
     @Test
diff --git a/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java
index 5eb1f3e3de1..e43dbd4e266 100644
--- a/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java
+++ b/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java
@@ -550,9 +550,11 @@ public class YqlParserTestCase {
     @Test
     public void testNearestNeighbor() {
         assertParse("select foo from bar where nearestNeighbor(semantic_embedding, my_vector);",
-                    "NEAREST_NEIGHBOR {field=semantic_embedding,queryTensorName=my_vector,targetNumHits=0}");
+                    "NEAREST_NEIGHBOR {field=semantic_embedding,queryTensorName=my_vector,hnsw.exploreAdditionalHits=0,approximate=true,targetNumHits=0}");
         assertParse("select foo from bar where [{\"targetNumHits\": 37}]nearestNeighbor(semantic_embedding, my_vector);",
-                    "NEAREST_NEIGHBOR {field=semantic_embedding,queryTensorName=my_vector,targetNumHits=37}");
+                    "NEAREST_NEIGHBOR {field=semantic_embedding,queryTensorName=my_vector,hnsw.exploreAdditionalHits=0,approximate=true,targetNumHits=37}");
+        assertParse("select foo from bar where [{\"approximate\": false, \"hnsw.exploreAdditionalHits\": 8, \"targetNumHits\": 3}]nearestNeighbor(semantic_embedding, my_vector);",
+                    "NEAREST_NEIGHBOR {field=semantic_embedding,queryTensorName=my_vector,hnsw.exploreAdditionalHits=8,approximate=false,targetNumHits=3}");
     }
 
     @Test
diff --git a/container-search/src/test/java/com/yahoo/select/SelectTestCase.java b/container-search/src/test/java/com/yahoo/select/SelectTestCase.java
index 96392a4f29b..1715ed38964 100644
--- a/container-search/src/test/java/com/yahoo/select/SelectTestCase.java
+++ b/container-search/src/test/java/com/yahoo/select/SelectTestCase.java
@@ -667,6 +667,13 @@ public class SelectTestCase {
         assertGrouping(expected, parseGrouping(grouping));
     }
 
+    @Test
+    public void testMultipleOutputs() {
+        String grouping = "[ { \"all\" : { \"group\" : \"b\", \"each\" : {\"output\": [ \"count()\", \"avg(foo)\" ] } } } ]";
+        String expected = "[[]all(group(b) each(output(count(), avg(foo))))]";
+        assertGrouping(expected, parseGrouping(grouping));
+    }
+
     //------------------------------------------------------------------- Other tests
 
     @Test
@@ -717,6 +724,7 @@ public class SelectTestCase {
         assertEquals("all(group(time.dayofmonth(a)) each(output(count())))", query.getSelect().getGrouping().get(0).toString());
 
         Query clone = query.clone();
+        assertEquals(clone.getSelect().getGroupingExpressionString(), query.getSelect().getGroupingExpressionString());
         assertNotSame(query.getSelect(), clone.getSelect());
         assertNotSame(query.getSelect().getGrouping(), clone.getSelect().getGrouping());
         assertNotSame(query.getSelect().getGrouping().get(0), clone.getSelect().getGrouping().get(0));
@@ -725,8 +733,15 @@ public class SelectTestCase {
         assertEquals(query.getSelect().getGroupingString(), clone.getSelect().getGroupingString());
         assertEquals(query.getSelect().getGrouping().get(0).toString(), clone.getSelect().getGrouping().get(0).toString());
         assertEquals(query.getSelect().getGrouping().get(1).toString(), clone.getSelect().getGrouping().get(1).toString());
+    }
 
+    @Test
+    public void testCloneWithGroupingExpressionString() {
+        Query query = new Query();
+        query.getSelect().setGroupingExpressionString("all(group(foo) each(output(count())))");
 
+        Query clone = query.clone();
+        assertEquals(clone.getSelect().getGroupingExpressionString(), query.getSelect().getGroupingExpressionString());
     }
 
     //------------------------------------------------------------------- Assert methods
diff --git a/controller-api/pom.xml b/controller-api/pom.xml
index 5c744db2d6d..5d0bbff1c62 100644
--- a/controller-api/pom.xml
+++ b/controller-api/pom.xml
@@ -53,6 +53,13 @@
             <version>${project.version}</version>
         </dependency>
 
+        <dependency>
+            <groupId>com.yahoo.vespa</groupId>
+            <artifactId>vespajlib</artifactId>
+            <scope>provided</scope>
+            <version>${project.version}</version>
+        </dependency>
+
         <!-- compile -->
 
         <dependency>
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/UserResource.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/UserResource.java
deleted file mode 100644
index a290323a245..00000000000
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/UserResource.java
+++ /dev/null
@@ -1,27 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.controller.api.application.v4;
-
-import com.fasterxml.jackson.annotation.JsonInclude;
-import com.yahoo.vespa.hosted.controller.api.application.v4.model.UserInfo;
-import com.yahoo.vespa.hosted.controller.api.identifiers.UserId;
-
-import javax.ws.rs.GET;
-import javax.ws.rs.PUT;
-import javax.ws.rs.Path;
-import javax.ws.rs.Produces;
-import javax.ws.rs.QueryParam;
-import javax.ws.rs.core.MediaType;
-
-/**
- * @author gv
- */
-@Path("/v4/user")
-@Produces(MediaType.APPLICATION_JSON)
-public interface UserResource {
-    @GET
-    @JsonInclude(value = JsonInclude.Include.NON_NULL)
-    UserInfo whoAmI(@QueryParam("userOverride") UserId userOverride);
-
-    @PUT
-    void createUserTenant();
-}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/DeploymentData.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/DeploymentData.java
new file mode 100644
index 00000000000..0afe9347341
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/DeploymentData.java
@@ -0,0 +1,78 @@
+package com.yahoo.vespa.hosted.controller.api.application.v4.model;
+
+import com.yahoo.component.Version;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.DockerImage;
+import com.yahoo.config.provision.zone.ZoneId;
+import com.yahoo.vespa.athenz.api.AthenzDomain;
+import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMetadata;
+import com.yahoo.vespa.hosted.controller.api.integration.configserver.ContainerEndpoint;
+
+import java.util.Optional;
+import java.util.Set;
+
+import static java.util.Objects.requireNonNull;
+
+/**
+ * Data pertaining to a deployment to be done on a config server.
+ *
+ * @author jonmv
+ */
+public class DeploymentData {
+
+    private final ApplicationId instance;
+    private final ZoneId zone;
+    private final byte[] applicationPackage;
+    private final Version platform;
+    private final Set<ContainerEndpoint> containerEndpoints;
+    private final Optional<EndpointCertificateMetadata> endpointCertificateMetadata;
+    private final Optional<DockerImage> dockerImageRepo;
+    private final Optional<AthenzDomain> athenzDomain;
+
+    public DeploymentData(ApplicationId instance, ZoneId zone, byte[] applicationPackage, Version platform,
+                          Set<ContainerEndpoint> containerEndpoints,
+                          Optional<EndpointCertificateMetadata> endpointCertificateMetadata,
+                          Optional<DockerImage> dockerImageRepo,
+                          Optional<AthenzDomain> athenzDomain) {
+        this.instance = requireNonNull(instance);
+        this.zone = requireNonNull(zone);
+        this.applicationPackage = requireNonNull(applicationPackage);
+        this.platform = requireNonNull(platform);
+        this.containerEndpoints = requireNonNull(containerEndpoints);
+        this.endpointCertificateMetadata = requireNonNull(endpointCertificateMetadata);
+        this.dockerImageRepo = requireNonNull(dockerImageRepo);
+        this.athenzDomain = athenzDomain;
+    }
+
+    public ApplicationId instance() {
+        return instance;
+    }
+
+    public ZoneId zone() {
+        return zone;
+    }
+
+    public byte[] applicationPackage() {
+        return applicationPackage;
+    }
+
+    public Version platform() {
+        return platform;
+    }
+
+    public Set<ContainerEndpoint> containerEndpoints() {
+        return containerEndpoints;
+    }
+
+    public Optional<EndpointCertificateMetadata> endpointCertificateMetadata() {
+        return endpointCertificateMetadata;
+    }
+
+    public Optional<DockerImage> dockerImageRepo() {
+        return dockerImageRepo;
+    }
+
+    public Optional<AthenzDomain> athenzDomain() {
+        return athenzDomain;
+    }
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/InstanceInformation.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/InstanceInformation.java
index 5c279547e17..d888c5446a6 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/InstanceInformation.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/InstanceInformation.java
@@ -4,11 +4,7 @@ package com.yahoo.vespa.hosted.controller.api.application.v4.model;
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
 import com.fasterxml.jackson.annotation.JsonProperty;
-import com.yahoo.vespa.hosted.controller.api.identifiers.GitBranch;
-import com.yahoo.vespa.hosted.controller.api.identifiers.GitCommit;
-import com.yahoo.vespa.hosted.controller.api.identifiers.GitRepository;
-import com.yahoo.vespa.hosted.controller.api.identifiers.RevisionId;
-import com.yahoo.vespa.hosted.controller.api.identifiers.ScrewdriverId;
+import com.yahoo.config.provision.zone.RoutingMethod;
 
 import java.net.URI;
 import java.util.List;
@@ -18,40 +14,42 @@ import java.util.List;
  */
 @JsonIgnoreProperties(ignoreUnknown = true)
 public class InstanceInformation {
-    public List<URI> serviceUrls;
+
     public List<Endpoint> endpoints;
-    public URI nodes;
     public URI yamasUrl;
-    public RevisionId revision;
     public Long deployTimeEpochMs;
     public Long expiryTimeEpochMs;
 
-    public ScrewdriverId screwdriverId;
-    public GitRepository gitRepository;
-    public GitBranch gitBranch;
-    public GitCommit gitCommit;
-
     public static class Endpoint {
         public String cluster;
         public boolean tls;
         public URI url;
+        public String scope;
+        public RoutingMethod routingMethod;
 
         @JsonCreator
         public Endpoint(@JsonProperty("cluster") String cluster ,
                         @JsonProperty("tls") boolean tls,
-                        @JsonProperty("url") URI url) {
+                        @JsonProperty("url") URI url,
+                        @JsonProperty("scope") String scope,
+                        @JsonProperty("routingMethod") RoutingMethod routingMethod) {
             this.cluster = cluster;
             this.tls = tls;
             this.url = url;
+            this.scope = scope;
+            this.routingMethod = routingMethod;
         }
 
         @Override
         public String toString() {
-            return "Endpoint {" +
-                    "cluster=" + cluster+
-                    ", tls='" + tls + '\'' +
-                    ", url='" + url+ '\'' +
-                    '}';
+            return "Endpoint{" +
+                   "cluster='" + cluster + '\'' +
+                   ", tls=" + tls +
+                   ", url=" + url +
+                   ", scope='" + scope + '\'' +
+                   ", routingMethod=" + routingMethod +
+                   '}';
         }
     }
+
 }
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/InstancesReply.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/InstancesReply.java
index c063c339406..09d54d10845 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/InstancesReply.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/InstancesReply.java
@@ -3,9 +3,7 @@ package com.yahoo.vespa.hosted.controller.api.application.v4.model;
 
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
 
-import java.net.URI;
 import java.util.List;
-import java.util.Set;
 
 /**
  * @author Tony Vaagenes
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/UserInfo.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/UserInfo.java
deleted file mode 100644
index 2b2a089c543..00000000000
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/UserInfo.java
+++ /dev/null
@@ -1,17 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.controller.api.application.v4.model;
-
-import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
-import com.yahoo.vespa.hosted.controller.api.identifiers.UserId;
-
-import java.util.List;
-
-/**
- * @author gv
- */
-@JsonIgnoreProperties(ignoreUnknown = true)
-public class UserInfo {
-    public UserId user;
-    public boolean tenantExists;
-    public List<TenantInfo> tenants;
-}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/bcp/BcpStatus.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/bcp/BcpStatus.java
deleted file mode 100644
index 679d5fc5727..00000000000
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/bcp/BcpStatus.java
+++ /dev/null
@@ -1,18 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.controller.api.bcp;
-
-import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
-
-@JsonIgnoreProperties(ignoreUnknown = true)
-public class BcpStatus {
-    public String rotationStatus;
-    public String reason;
-
-    // For jackson
-    public BcpStatus() {}
-
-    public BcpStatus(String rotationStatus, String reason) {
-        this.rotationStatus = rotationStatus;
-        this.reason = reason;
-    }
-}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/bcp/package-info.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/bcp/package-info.java
deleted file mode 100644
index 2bb442c3db8..00000000000
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/bcp/package-info.java
+++ /dev/null
@@ -1,5 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-@ExportPackage
-package com.yahoo.vespa.hosted.controller.api.bcp;
-
-import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/TenantId.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/TenantId.java
index 4974192e213..3ac24bac7ca 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/TenantId.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/TenantId.java
@@ -10,10 +10,6 @@ public class TenantId extends NonDefaultIdentifier {
         super(id);
     }
 
-    public boolean isUser() {
-        return id().startsWith("by-");
-    }
-
     @Override
     public void validate() {
         super.validate();
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/UserId.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/UserId.java
index d2effc76827..f1a8e57ab03 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/UserId.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/identifiers/UserId.java
@@ -10,8 +10,4 @@ public class UserId extends NonDefaultIdentifier {
         super(id);
     }
 
-    public TenantId toTenantId() {
-        return new TenantId("by-" + id().replace('_', '-'));
-    }
-
 }
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ApplicationIdSnapshot.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ApplicationIdSnapshot.java
deleted file mode 100644
index 3f5a56207c5..00000000000
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ApplicationIdSnapshot.java
+++ /dev/null
@@ -1,72 +0,0 @@
-// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.controller.api.integration;
-
-import com.yahoo.config.provision.ApplicationId;
-import com.yahoo.config.provision.ApplicationName;
-import com.yahoo.config.provision.InstanceName;
-import com.yahoo.config.provision.TenantName;
-
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Optional;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-/**
- * @author freva
- */
-public class ApplicationIdSnapshot {
-    private final Map<TenantName, Map<ApplicationName, Set<InstanceName>>> instanceByApplicationByTenantName;
-
-    public ApplicationIdSnapshot(Map<TenantName, Map<ApplicationName, Set<InstanceName>>> instanceByApplicationByTenantName) {
-        this.instanceByApplicationByTenantName = instanceByApplicationByTenantName.entrySet().stream()
-                .collect(Collectors.toUnmodifiableMap(Map.Entry::getKey, e -> e.getValue().entrySet().stream()
-                        .collect(Collectors.toUnmodifiableMap(Map.Entry::getKey, f -> Set.copyOf(f.getValue())))));
-    }
-
-    public Set<TenantName> tenants() {
-        return instanceByApplicationByTenantName.keySet();
-    }
-
-    public Set<ApplicationName> applications(TenantName tenantName) {
-        return Optional.ofNullable(instanceByApplicationByTenantName.get(tenantName))
-                .map(Map::keySet)
-                .orElseGet(Set::of);
-    }
-
-    public Set<InstanceName> instances(TenantName tenantName, ApplicationName applicationName) {
-        return instanceByApplicationByTenantName.getOrDefault(tenantName, Map.of())
-                .getOrDefault(applicationName, Set.of());
-    }
-
-
-    public static class Builder {
-        private final Map<TenantName, Map<ApplicationName, Set<InstanceName>>> instanceByApplicationByTenantName = new HashMap<>();
-
-        public Builder add(TenantName tenantName) {
-            instanceByApplicationByTenantName.computeIfAbsent(tenantName, t -> new HashMap<>());
-            return this;
-        }
-
-        public Builder add(TenantName tenantName, ApplicationName applicationName) {
-            instanceByApplicationByTenantName.computeIfAbsent(tenantName, t -> new HashMap<>())
-                    .computeIfAbsent(applicationName, a -> new HashSet<>());
-            return this;
-        }
-
-        public Builder add(TenantName tenantName, ApplicationName applicationName, InstanceName instanceName) {
-            add(tenantName, applicationName);
-            instanceByApplicationByTenantName.get(tenantName).get(applicationName).add(instanceName);
-            return this;
-        }
-
-        public Builder add(ApplicationId applicationId) {
-            return add(applicationId.tenant(), applicationId.application(), applicationId.instance());
-        }
-
-        public ApplicationIdSnapshot build() {
-            return new ApplicationIdSnapshot(instanceByApplicationByTenantName);
-        }
-    }
-}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ApplicationIdSource.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ApplicationIdSource.java
deleted file mode 100644
index 4ee3e849bc7..00000000000
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ApplicationIdSource.java
+++ /dev/null
@@ -1,8 +0,0 @@
-// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.controller.api.integration;
-
-public interface ApplicationIdSource {
-
-    /** Returns a snapshot of all known tenants, applications and instances */
-    ApplicationIdSnapshot applicationIdSnapshot();
-}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ServiceRegistry.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ServiceRegistry.java
index 6ca5cae0455..ca939023245 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ServiceRegistry.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ServiceRegistry.java
@@ -3,7 +3,7 @@ package com.yahoo.vespa.hosted.controller.api.integration;
 
 import com.yahoo.vespa.hosted.controller.api.integration.aws.AwsEventFetcher;
 import com.yahoo.vespa.hosted.controller.api.integration.aws.ResourceTagger;
-import com.yahoo.vespa.hosted.controller.api.integration.certificates.ApplicationCertificateProvider;
+import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateProvider;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServer;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationStore;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.ArtifactRepository;
@@ -20,7 +20,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.resource.CostReportCons
 import com.yahoo.vespa.hosted.controller.api.integration.resource.MeteringClient;
 import com.yahoo.vespa.hosted.controller.api.integration.resource.TenantCost;
 import com.yahoo.vespa.hosted.controller.api.integration.routing.GlobalRoutingService;
-import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingGenerator;
 import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry;
 
 import java.time.Clock;
@@ -31,7 +30,6 @@ import java.time.Clock;
  *
  * @author mpolden
  */
-// TODO(mpolden): Access all services through this
 public interface ServiceRegistry {
 
     ConfigServer configServer();
@@ -42,11 +40,9 @@ public interface ServiceRegistry {
 
     GlobalRoutingService globalRoutingService();
 
-    RoutingGenerator routingGenerator();
-
     Mailer mailer();
 
-    ApplicationCertificateProvider applicationCertificateProvider();
+    EndpointCertificateProvider endpointCertificateProvider();
 
     MeteringClient meteringService();
 
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AthenzClientFactory.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AthenzClientFactory.java
index fffa849f7d3..bd65443b265 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AthenzClientFactory.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AthenzClientFactory.java
@@ -2,7 +2,6 @@
 package com.yahoo.vespa.hosted.controller.api.integration.athenz;
 
 import com.yahoo.vespa.athenz.api.AthenzIdentity;
-import com.yahoo.vespa.athenz.api.AthenzService;
 import com.yahoo.vespa.athenz.client.zms.ZmsClient;
 import com.yahoo.vespa.athenz.client.zts.ZtsClient;
 
@@ -16,4 +15,7 @@ public interface AthenzClientFactory {
     ZmsClient createZmsClient();
 
     ZtsClient createZtsClient();
+
+    default boolean cacheLookups() { return false; }
+
 }
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AthenzClientFactoryMock.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AthenzClientFactoryMock.java
index c228a196228..2995e1a28e7 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AthenzClientFactoryMock.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AthenzClientFactoryMock.java
@@ -7,7 +7,6 @@ import com.yahoo.vespa.athenz.api.AthenzService;
 import com.yahoo.vespa.athenz.client.zms.ZmsClient;
 import com.yahoo.vespa.athenz.client.zts.ZtsClient;
 
-import java.util.logging.Level;
 import java.util.logging.Logger;
 
 /**
@@ -47,8 +46,4 @@ public class AthenzClientFactoryMock extends AbstractComponent implements Athenz
         return new ZtsClientMock(athenz);
     }
 
-    private static void log(String format, Object... args) {
-        log.log(Level.INFO, String.format(format, args));
-    }
-
 }
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZtsClientMock.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZtsClientMock.java
index ec60f11060d..53e80fcb2ed 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZtsClientMock.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZtsClientMock.java
@@ -2,6 +2,7 @@
 package com.yahoo.vespa.hosted.controller.api.integration.athenz;
 
 import com.yahoo.security.Pkcs10Csr;
+import com.yahoo.vespa.athenz.api.AthenzAccessToken;
 import com.yahoo.vespa.athenz.api.AthenzDomain;
 import com.yahoo.vespa.athenz.api.AthenzIdentity;
 import com.yahoo.vespa.athenz.api.AthenzRole;
@@ -74,6 +75,16 @@ public class ZtsClientMock implements ZtsClient {
     }
 
     @Override
+    public AthenzAccessToken getAccessToken(AthenzDomain domain) {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public AthenzAccessToken getAccessToken(List<AthenzRole> athenzRole) {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
     public X509Certificate getRoleCertificate(AthenzRole role, Pkcs10Csr csr, Duration expiry) {
         throw new UnsupportedOperationException();
     }
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/ResourceTagger.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/ResourceTagger.java
index 9d0e6481d38..5b3cd18403b 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/ResourceTagger.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/aws/ResourceTagger.java
@@ -18,11 +18,7 @@ public interface ResourceTagger {
     int tagResources(ZoneApi zone, Map<HostName, ApplicationId> tenantOfHosts);
 
     static ResourceTagger empty() {
-        return new ResourceTagger() {
-            @Override
-            public int tagResources(ZoneApi zone, Map<HostName, ApplicationId> tenantOfHosts) {
-                return 0;
-            }
-        };
+        return (zone, tenantOfHosts) -> 0;
     }
+
 }
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/ApplicationCertificate.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/ApplicationCertificate.java
deleted file mode 100644
index 41f5b65d263..00000000000
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/ApplicationCertificate.java
+++ /dev/null
@@ -1,43 +0,0 @@
-// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.controller.api.integration.certificates;
-
-import java.util.Objects;
-
-/**
- * Represents a reference to a certificate and private key.
- *
- * @author mortent
- * @author andreer
- */
-public class ApplicationCertificate {
-
-    private final String secretsKeyNamePrefix;
-
-    public ApplicationCertificate(String secretsKeyNamePrefix) {
-        this.secretsKeyNamePrefix = Objects.requireNonNull(secretsKeyNamePrefix, "secretsKeyNamePrefix must be non-null");
-    }
-
-    /** The prefix of keys identifying this certificate and its private key in a key store */
-    public String secretsKeyNamePrefix() {
-        return secretsKeyNamePrefix;
-    }
-
-    @Override
-    public boolean equals(Object o) {
-        if (this == o) return true;
-        if (o == null || getClass() != o.getClass()) return false;
-        ApplicationCertificate that = (ApplicationCertificate) o;
-        return Objects.equals(secretsKeyNamePrefix, that.secretsKeyNamePrefix);
-    }
-
-    @Override
-    public int hashCode() {
-        return Objects.hash(secretsKeyNamePrefix);
-    }
-
-    @Override
-    public String toString() {
-        return "application certificate '" + secretsKeyNamePrefix + "'";
-    }
-
-}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/ApplicationCertificateMock.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/ApplicationCertificateMock.java
deleted file mode 100644
index cc2d08c3fcd..00000000000
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/ApplicationCertificateMock.java
+++ /dev/null
@@ -1,31 +0,0 @@
-// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.controller.api.integration.certificates;
-
-import com.yahoo.config.provision.ApplicationId;
-
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.UUID;
-
-/**
- * @author tokle
- */
-public class ApplicationCertificateMock implements ApplicationCertificateProvider {
-
-    private final Map<ApplicationId, List<String>> dnsNames = new HashMap<>();
-
-    public List<String> dnsNamesOf(ApplicationId application) {
-        return Collections.unmodifiableList(dnsNames.getOrDefault(application, List.of()));
-    }
-
-    @Override
-    public ApplicationCertificate requestCaSignedCertificate(ApplicationId applicationId, List<String> dnsNames) {
-        this.dnsNames.put(applicationId, dnsNames);
-        return new ApplicationCertificate(String.format("vespa.tls.%s.%s@%s", applicationId.tenant(),
-                                                        applicationId.application(),
-                                                        UUID.randomUUID().toString()));
-    }
-
-}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/ApplicationCertificateProvider.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/ApplicationCertificateProvider.java
deleted file mode 100644
index b6ad1701449..00000000000
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/ApplicationCertificateProvider.java
+++ /dev/null
@@ -1,17 +0,0 @@
-// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.controller.api.integration.certificates;
-
-import com.yahoo.config.provision.ApplicationId;
-
-import java.util.List;
-
-/**
- * Generates a certificate.
- *
- * @author andreer
- */
-public interface ApplicationCertificateProvider {
-
-    ApplicationCertificate requestCaSignedCertificate(ApplicationId applicationId, List<String> dnsNames);
-
-}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateMetadata.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateMetadata.java
index 56a50faa4fa..53366c9b922 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateMetadata.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateMetadata.java
@@ -1,11 +1,13 @@
 package com.yahoo.vespa.hosted.controller.api.integration.certificates;
 
+import java.util.List;
 import java.util.Objects;
+import java.util.Optional;
 
 /**
  * This class is used for metadata about an application's endpoint certificate on the controller.
  * <p>
- * It is a copy of com.yahoo.config.model.api.EndpointCertificateMetadata, but will soon be extended.
+ * It has more properties than com.yahoo.config.model.api.EndpointCertificateMetadata.
  *
  * @author andreer
  */
@@ -14,11 +16,22 @@ public class EndpointCertificateMetadata {
     private final String keyName;
     private final String certName;
     private final int version;
+    // TODO: make these fields required once all certs have them stored
+    private final Optional<String> request_id;
+    private final Optional<List<String>> requestedDnsSans;
+    private final Optional<String> issuer;
 
     public EndpointCertificateMetadata(String keyName, String certName, int version) {
+        this(keyName, certName, version, Optional.empty(), Optional.empty(), Optional.empty());
+    }
+
+    public EndpointCertificateMetadata(String keyName, String certName, int version, Optional<String> request_id, Optional<List<String>> requestedDnsSans, Optional<String> issuer) {
         this.keyName = keyName;
         this.certName = certName;
         this.version = version;
+        this.request_id = request_id;
+        this.requestedDnsSans = requestedDnsSans;
+        this.issuer = issuer;
     }
 
     public String keyName() {
@@ -33,12 +46,38 @@ public class EndpointCertificateMetadata {
         return version;
     }
 
+    public Optional<String> request_id() {
+        return request_id;
+    }
+
+    public Optional<List<String>> requestedDnsSans() {
+        return requestedDnsSans;
+    }
+
+    public Optional<String> issuer() {
+        return issuer;
+    }
+
+    public EndpointCertificateMetadata withVersion(int version) {
+        return new EndpointCertificateMetadata(
+                this.keyName,
+                this.certName,
+                version,
+                this.request_id,
+                this.requestedDnsSans,
+                this.issuer
+        );
+    }
+
     @Override
     public String toString() {
         return "EndpointCertificateMetadata{" +
                 "keyName='" + keyName + '\'' +
                 ", certName='" + certName + '\'' +
                 ", version=" + version +
+                ", request_id=" + request_id +
+                ", requestedDnsSans=" + requestedDnsSans +
+                ", issuer=" + issuer +
                 '}';
     }
 
@@ -48,12 +87,15 @@ public class EndpointCertificateMetadata {
         if (o == null || getClass() != o.getClass()) return false;
         EndpointCertificateMetadata that = (EndpointCertificateMetadata) o;
         return version == that.version &&
-                Objects.equals(keyName, that.keyName) &&
-                Objects.equals(certName, that.certName);
+                keyName.equals(that.keyName) &&
+                certName.equals(that.certName) &&
+                request_id.equals(that.request_id) &&
+                requestedDnsSans.equals(that.requestedDnsSans) &&
+                issuer.equals(that.issuer);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(keyName, certName, version);
+        return Objects.hash(keyName, certName, version, request_id, requestedDnsSans, issuer);
     }
 }
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateMock.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateMock.java
new file mode 100644
index 00000000000..23799f48f91
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateMock.java
@@ -0,0 +1,36 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.integration.certificates;
+
+import com.yahoo.config.provision.ApplicationId;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+/**
+ * @author tokle
+ */
+public class EndpointCertificateMock implements EndpointCertificateProvider {
+
+    private final Map<ApplicationId, List<String>> dnsNames = new HashMap<>();
+
+    public List<String> dnsNamesOf(ApplicationId application) {
+        return Collections.unmodifiableList(dnsNames.getOrDefault(application, List.of()));
+    }
+
+    @Override
+    public EndpointCertificateMetadata requestCaSignedCertificate(ApplicationId applicationId, List<String> dnsNames, Optional<EndpointCertificateMetadata> currentMetadata) {
+        this.dnsNames.put(applicationId, dnsNames);
+        String endpointCertificatePrefix = String.format("vespa.tls.%s.%s.%s", applicationId.tenant(),
+                applicationId.application(), applicationId.instance());
+        return new EndpointCertificateMetadata(endpointCertificatePrefix + "-key", endpointCertificatePrefix + "-cert", 0);
+    }
+
+    @Override
+    public List<EndpointCertificateMetadata> listCertificates() {
+        return Collections.emptyList();
+    }
+
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateProvider.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateProvider.java
new file mode 100644
index 00000000000..9c5c25c1c71
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateProvider.java
@@ -0,0 +1,19 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.integration.certificates;
+
+import com.yahoo.config.provision.ApplicationId;
+
+import java.util.List;
+import java.util.Optional;
+
+/**
+ * Generates an endpoint certificate for an application instance.
+ *
+ * @author andreer
+ */
+public interface EndpointCertificateProvider  {
+
+    EndpointCertificateMetadata requestCaSignedCertificate(ApplicationId applicationId, List<String> dnsNames, Optional<EndpointCertificateMetadata> currentMetadata);
+
+    List<EndpointCertificateMetadata> listCertificates();
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java
index ed83c83a1a2..16464f7cea6 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java
@@ -6,12 +6,11 @@ import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.zone.ZoneId;
 import com.yahoo.vespa.flags.json.FlagData;
 import com.yahoo.vespa.hosted.controller.api.application.v4.model.ClusterMetrics;
-import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions;
+import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeploymentData;
 import com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus;
 import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
 import com.yahoo.vespa.hosted.controller.api.identifiers.Hostname;
 import com.yahoo.vespa.hosted.controller.api.integration.LogEntry;
-import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMetadata;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud;
 import com.yahoo.vespa.serviceview.bindings.ApplicationView;
 
@@ -19,7 +18,6 @@ import java.io.InputStream;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
-import java.util.Set;
 
 /**
  * The API controllers use when communicating with config servers.
@@ -32,9 +30,7 @@ public interface ConfigServer {
         PrepareResponse prepareResponse();
     }
 
-    PreparedApplication deploy(DeploymentId deployment, DeployOptions deployOptions,
-                               Set<ContainerEndpoint> containerEndpoints, Optional<EndpointCertificateMetadata> endpointCertificateMetadata,
-                               byte[] content);
+    PreparedApplication deploy(DeploymentData deployment);
 
     void restart(DeploymentId deployment, Optional<Hostname> hostname);
 
@@ -44,7 +40,9 @@ public interface ConfigServer {
 
     ApplicationView getApplicationView(String tenantName, String applicationName, String instanceName, String environment, String region);
 
-    Map<?,?> getServiceApiResponse(String tenantName, String applicationName, String instanceName, String environment, String region, String serviceName, String restPath);
+    Map<?,?> getServiceApiResponse(DeploymentId deployment, String serviceName, String restPath);
+
+    String getClusterControllerStatus(DeploymentId deployment, String restPath);
 
     /**
      * Gets the Vespa logs of the given deployment.
@@ -111,15 +109,12 @@ public interface ConfigServer {
     TesterCloud.Status getTesterStatus(DeploymentId deployment);
 
     /** Starts tests on tester node */
-    // TODO: Remove default implementation when implemented in internal repo
-    default void startTests(DeploymentId deployment, TesterCloud.Suite suite, byte[] config) { }
+    String startTests(DeploymentId deployment, TesterCloud.Suite suite, byte[] config);
 
     /** Gets log from tester node */
-    // TODO: Remove default implementation when implemented in internal repo
-    default List<LogEntry> getTesterLog(DeploymentId deployment, long after) { return List.of(); }
+    List<LogEntry> getTesterLog(DeploymentId deployment, long after);
 
     /** Is tester node ready */
-    // TODO: Remove default implementation when implemented in internal repo
-    default boolean isTesterReady(DeploymentId deployment) { return false; }
+    boolean isTesterReady(DeploymentId deployment);
 
 }
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Node.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Node.java
index d8103c864df..07e411cd5cd 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Node.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Node.java
@@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.controller.api.integration.configserver;
 
 import com.yahoo.component.Version;
 import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.DockerImage;
 import com.yahoo.config.provision.HostName;
 import com.yahoo.config.provision.NodeResources;
 import com.yahoo.config.provision.NodeType;
@@ -30,6 +31,8 @@ public class Node {
     private final Version wantedVersion;
     private final Version currentOsVersion;
     private final Version wantedOsVersion;
+    private final DockerImage currentDockerImage;
+    private final DockerImage wantedDockerImage;
     private final ServiceState serviceState;
     private final Optional<Instant> suspendedSince;
     private final Optional<Instant> currentFirmwareCheck;
@@ -51,7 +54,7 @@ public class Node {
                 Optional<Instant> currentFirmwareCheck, Optional<Instant> wantedFirmwareCheck, ServiceState serviceState,
                 Optional<Instant> suspendedSince, long restartGeneration, long wantedRestartGeneration, long rebootGeneration, long wantedRebootGeneration,
                 int cost, String flavor, String clusterId, ClusterType clusterType, boolean wantToRetire, boolean wantToDeprovision,
-                Optional<TenantName> reservedTo) {
+                Optional<TenantName> reservedTo, DockerImage wantedDockerImage, DockerImage currentDockerImage) {
         this.hostname = hostname;
         this.parentHostname = parentHostname;
         this.state = state;
@@ -77,6 +80,8 @@ public class Node {
         this.wantToRetire = wantToRetire;
         this.wantToDeprovision = wantToDeprovision;
         this.reservedTo = reservedTo;
+        this.wantedDockerImage = wantedDockerImage;
+        this.currentDockerImage = currentDockerImage;
     }
 
     public HostName hostname() {
@@ -117,6 +122,14 @@ public class Node {
         return wantedOsVersion;
     }
 
+    public DockerImage currentDockerImage() {
+        return currentDockerImage;
+    }
+
+    public DockerImage wantedDockerImage() {
+        return wantedDockerImage;
+    }
+
     public Optional<Instant> currentFirmwareCheck() {
         return currentFirmwareCheck;
     }
@@ -228,6 +241,8 @@ public class Node {
         private Version wantedVersion;
         private Version currentOsVersion;
         private Version wantedOsVersion;
+        private DockerImage currentDockerImage;
+        private DockerImage wantedDockerImage;
         private Optional<Instant> currentFirmwareCheck = Optional.empty();
         private Optional<Instant> wantedFirmwareCheck = Optional.empty();
         private ServiceState serviceState;
@@ -257,6 +272,8 @@ public class Node {
             this.wantedVersion = node.wantedVersion;
             this.currentOsVersion = node.currentOsVersion;
             this.wantedOsVersion = node.wantedOsVersion;
+            this.currentDockerImage = node.currentDockerImage;
+            this.wantedDockerImage = node.wantedDockerImage;
             this.currentFirmwareCheck = node.currentFirmwareCheck;
             this.wantedFirmwareCheck = node.wantedFirmwareCheck;
             this.serviceState = node.serviceState;
@@ -324,6 +341,16 @@ public class Node {
             return this;
         }
 
+        public Builder currentDockerImage(DockerImage currentDockerImage) {
+            this.currentDockerImage = currentDockerImage;
+            return this;
+        }
+
+        public Builder wantedDockerImage(DockerImage wantedDockerImage) {
+            this.wantedDockerImage = wantedDockerImage;
+            return this;
+        }
+
         public Builder currentFirmwareCheck(Instant currentFirmwareCheck) {
             this.currentFirmwareCheck = Optional.ofNullable(currentFirmwareCheck);
             return this;
@@ -403,7 +430,8 @@ public class Node {
             return new Node(hostname, parentHostname, state, type, resources, owner, currentVersion, wantedVersion,
                             currentOsVersion, wantedOsVersion, currentFirmwareCheck, wantedFirmwareCheck, serviceState,
                             suspendedSince, restartGeneration, wantedRestartGeneration, rebootGeneration, wantedRebootGeneration,
-                            cost, flavor, clusterId, clusterType, wantToRetire, wantToDeprovision, reservedTo);
+                            cost, flavor, clusterId, clusterType, wantToRetire, wantToDeprovision, reservedTo,
+                            wantedDockerImage, currentDockerImage);
         }
 
     }
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java
index 22c373e97ee..05e9707473c 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java
@@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.controller.api.integration.configserver;
 
 import com.yahoo.component.Version;
 import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.DockerImage;
 import com.yahoo.config.provision.HostName;
 import com.yahoo.config.provision.NodeResources;
 import com.yahoo.config.provision.NodeType;
@@ -111,7 +112,7 @@ public interface NodeRepository {
                         Optional.ofNullable(node.getCurrentFirmwareCheck()).map(Instant::ofEpochMilli),
                         Optional.ofNullable(node.getWantedFirmwareCheck()).map(Instant::ofEpochMilli),
                         fromBoolean(node.getAllowedToBeDown()),
-                        Optional.ofNullable(node.suspendedSince()).map(Instant::ofEpochMilli),
+                        Optional.ofNullable(node.suspendedSinceMillis()).map(Instant::ofEpochMilli),
                         toInt(node.getCurrentRestartGeneration()),
                         toInt(node.getRestartGeneration()),
                         toInt(node.getCurrentRebootGeneration()),
@@ -122,7 +123,9 @@ public interface NodeRepository {
                         clusterTypeOf(node.getMembership()),
                         node.getWantToRetire(),
                         node.getWantToDeprovision(),
-                        Optional.ofNullable(node.getReservedTo()).map(name -> TenantName.from(name)));
+                        Optional.ofNullable(node.getReservedTo()).map(TenantName::from),
+                        dockerImageFrom(node.getWantedDockerImage()),
+                        dockerImageFrom(node.getCurrentDockerImage()));
     }
 
     private static String clusterIdOf(NodeMembership nodeMembership) {
@@ -205,4 +208,8 @@ public interface NodeRepository {
         return s == null ? Version.emptyVersion : Version.fromString(s);
     }
 
+    private static DockerImage dockerImageFrom(String s) {
+        return s == null ? DockerImage.EMPTY : DockerImage.fromString(s);
+    }
+
 }
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/ApplicationVersion.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/ApplicationVersion.java
index bbe533db9b5..30fd8fad1bd 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/ApplicationVersion.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/ApplicationVersion.java
@@ -3,7 +3,6 @@ package com.yahoo.vespa.hosted.controller.api.integration.deployment;
 
 import com.yahoo.component.Version;
 
-import java.net.URI;
 import java.time.Instant;
 import java.util.Objects;
 import java.util.Optional;
@@ -100,7 +99,7 @@ public class ApplicationVersion implements Comparable<ApplicationVersion> {
         return String.format("%s.%d-%s",
                              majorVersion,
                              buildNumber.getAsLong(),
-                             source.map(SourceRevision::commit).map(commit -> abbreviateCommit(commit))
+                             source.map(SourceRevision::commit).map(ApplicationVersion::abbreviateCommit)
                                    .or(this::commit)
                                    .orElse("unknown"));
     }
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobType.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobType.java
index 7aac0826b2b..700be6d263a 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobType.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobType.java
@@ -88,6 +88,12 @@ public enum JobType {
                             Map.of(PublicCd, ZoneId.from("prod", "aws-us-east-1c"),
                                    Public,   ZoneId.from("prod", "aws-us-east-1c")), true),
 
+    productionAwsApNortheast1a  ("production-aws-ap-northeast-1a",
+                            Map.of(Public, ZoneId.from("prod", "aws-ap-northeast-1a"))),
+
+    testAwsApNortheast1a   ("test-aws-ap-northeast-1a",
+                            Map.of(Public, ZoneId.from("prod", "aws-ap-northeast-1a")), true),
+
     productionAwsUsWest2a  ("production-aws-us-west-2a",
                             Map.of(main, ZoneId.from("prod"   , "aws-us-west-2a"))),
 
@@ -143,15 +149,15 @@ public enum JobType {
 
     private final String jobName;
     private final Map<SystemName, ZoneId> zones;
-    private final boolean isTest;
+    private final boolean isProductionTest;
 
-    JobType(String jobName, Map<SystemName, ZoneId> zones, boolean isTest) {
+    JobType(String jobName, Map<SystemName, ZoneId> zones, boolean isProductionTest) {
         if (zones.values().stream().map(ZoneId::environment).distinct().count() > 1)
             throw new IllegalArgumentException("All zones of a job must be in the same environment");
 
         this.jobName = jobName;
         this.zones = zones;
-        this.isTest = isTest;
+        this.isProductionTest = isProductionTest;
     }
 
     JobType(String jobName, Map<SystemName, ZoneId> zones) {
@@ -176,10 +182,10 @@ public enum JobType {
     public boolean isProduction() { return environment() == Environment.prod; }
 
     /** Returns whether this job runs tests */
-    public boolean isTest() { return isTest; }
+    public boolean isTest() { return isProductionTest || environment().isTest(); }
 
     /** Returns whether this job deploys to a zone */
-    public boolean isDeployment() { return ! (isProduction() && isTest); }
+    public boolean isDeployment() { return ! (isProduction() && isProductionTest); }
 
     /** Returns the environment of this job type, or null if it does not have an environment */
     public Environment environment() {
@@ -206,7 +212,7 @@ public enum JobType {
 
     /** Returns the job type for the given zone */
     public static Optional<JobType> from(SystemName system, ZoneId zone) {
-        return from(system, zone, false);
+        return from(system, zone, zone.environment().isTest());
     }
 
     /** Returns the production test job type for the given environment and region or null if none */
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/TesterCloud.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/TesterCloud.java
index 36b13b3496a..b33917abcbe 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/TesterCloud.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/TesterCloud.java
@@ -1,11 +1,13 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.api.integration.deployment;
 
+import com.yahoo.config.provision.HostName;
 import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
 import com.yahoo.vespa.hosted.controller.api.integration.LogEntry;
 
 import java.net.URI;
 import java.util.List;
+import java.util.Optional;
 
 /**
  * Allows running some predefined tests -- typically remotely.
@@ -15,21 +17,10 @@ import java.util.List;
 public interface TesterCloud {
 
     /** Signals the tester to run its tests. */
-    void startTests(URI testerUrl, Suite suite, byte[] config);
-
-    /** Signals the tester to run its tests. */
-    // TODO: Remove default implementation when implementations have been updated
-    default void startTests(DeploymentId deploymentId, Suite suite, byte[] config) {}
-
-    /** Returns the log entries from the tester with ids after the given threshold. */
-    List<LogEntry> getLog(URI testerUrl, long after);
+    void startTests(DeploymentId deploymentId, Suite suite, byte[] config);
 
     /** Returns the log entries from the tester with ids after the given threshold. */
-    // TODO: Remove default implementation when implementations have been updated
-    default List<LogEntry> getLog(DeploymentId deploymentId, long after) { return List.of(); }
-
-    /** Returns the current status of the tester. */
-    Status getStatus(URI testerUrl);
+    List<LogEntry> getLog(DeploymentId deploymentId, long after);
 
     /** Returns the current status of the tester. */
     Status getStatus(DeploymentId deploymentId);
@@ -38,20 +29,13 @@ public interface TesterCloud {
     boolean ready(URI endpointUrl);
 
     /** Returns whether the test container is ready to serve */
-    boolean testerReady(URI endpointUrl);
-
-    /** Returns whether the test container is ready to serve */
-    // TODO: Remove default implementation when implementations have been updated
-    default boolean testerReady(DeploymentId deploymentId) { return false; }
+    boolean testerReady(DeploymentId deploymentId);
 
-    /** Returns whether the given URL is registered in DNS. */
-    boolean exists(URI endpointUrl);
+    /** Returns the IP address of the given host name, if any. */
+    Optional<String> resolveHostName(HostName hostname);
 
-    /**
-     * Returns whether the given URL is registered in DNS. Always returns true,
-     * as endpoints are not use in this case
-     */
-    default boolean exists(DeploymentId deploymentId) { return true; }
+    /** Returns the host name of the given CNAME, if any. */
+    Optional<HostName> resolveCname(HostName hostName);
 
     enum Status {
 
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/MemoryNameService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/MemoryNameService.java
index 75d49e542dc..b54446c071e 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/MemoryNameService.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/MemoryNameService.java
@@ -45,7 +45,7 @@ public class MemoryNameService implements NameService {
                              .map(target -> new Record(Record.Type.ALIAS, name, target.asData()))
                              .collect(Collectors.toList());
         // Satisfy idempotency contract of interface
-        removeRecords(findRecords(Record.Type.ALIAS, name));
+        removeRecords(records);
         records.forEach(this::add);
         return records;
     }
@@ -68,8 +68,23 @@ public class MemoryNameService implements NameService {
 
     @Override
     public List<Record> findRecords(Record.Type type, RecordData data) {
+        if (type == Record.Type.ALIAS && data.asString().contains("/")) {
+            // Validate the same expectation as of a real name service
+            throw new IllegalArgumentException("Finding " + Record.Type.ALIAS + " record by data should only include " +
+                                               "the FQDN name");
+        }
         return records.stream()
-                      .filter(record -> record.type() == type && record.data().equals(data))
+                      .filter(record -> {
+                          if (record.type() == type) {
+                              if (type == Record.Type.ALIAS) {
+                                  // Unpack ALIAS record and compare FQDN of data part
+                                  return RecordData.fqdn(AliasTarget.from(record.data()).name().value())
+                                                   .equals(data);
+                              }
+                              return record.data().equals(data);
+                          }
+                          return false;
+                      })
                       .collect(Collectors.toUnmodifiableList());
     }
 
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/maven/Metadata.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/maven/Metadata.java
index ef9c9bcd1f3..e08b0700653 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/maven/Metadata.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/maven/Metadata.java
@@ -3,13 +3,9 @@ package com.yahoo.vespa.hosted.controller.api.integration.maven;
 
 import com.yahoo.component.Version;
 import com.yahoo.text.XML;
-import org.w3c.dom.Document;
 import org.w3c.dom.Element;
 
-import java.io.InputStream;
-import java.io.InputStreamReader;
 import java.util.ArrayList;
-import java.util.Comparator;
 import java.util.List;
 import java.util.stream.Collectors;
 
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeHistory.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeHistory.java
index 30f5801dabb..a7b9cdc5e66 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeHistory.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeHistory.java
@@ -42,7 +42,8 @@ public class NodeHistory {
         FailedExpirer,
         InactiveExpirer,
         ProvisionedExpirer,
-        ReservationExpirer
+        ReservationExpirer,
+        DynamicProvisioningMaintainer
     }
 
 }
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeRepositoryNode.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeRepositoryNode.java
index 2abf40be527..d554bba6b9d 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeRepositoryNode.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeRepositoryNode.java
@@ -6,10 +6,8 @@ import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.databind.JsonNode;
 
-import java.time.Instant;
 import java.util.Arrays;
 import java.util.Map;
-import java.util.Optional;
 import java.util.Set;
 
 @JsonIgnoreProperties(ignoreUnknown = true)
@@ -82,8 +80,8 @@ public class NodeRepositoryNode {
     private NodeHistory[] history;
     @JsonProperty("allowedToBeDown")
     private Boolean allowedToBeDown;
-    @JsonProperty("suspendedSince")
-    private Long suspendedSince;
+    @JsonProperty("suspendedSinceMillis")
+    private Long suspendedSinceMillis;
     @JsonProperty("reports")
     private Map<String, JsonNode> reports;
     @JsonProperty("modelName")
@@ -317,12 +315,12 @@ public class NodeRepositoryNode {
         return allowedToBeDown;
     }
 
-    public Long suspendedSince() {
-        return suspendedSince;
+    public Long suspendedSinceMillis() {
+        return suspendedSinceMillis;
     }
 
-    public void setSuspendedSince(long suspendedSinceMillis) {
-        this.suspendedSince = suspendedSinceMillis;
+    public void setSuspendedSinceMillis(long suspendedSinceMillis) {
+        this.suspendedSinceMillis = suspendedSinceMillis;
     }
 
     public String getCurrentOsVersion() {
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeState.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeState.java
index 6271671ca2d..ab4ab07ce85 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeState.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeState.java
@@ -10,6 +10,7 @@ public enum NodeState {
     inactive,
     dirty,
     failed,
-    parked;
+    parked,
+    deprovisioned;
 
 }
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/Issue.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/Issue.java
index 08c913e9085..5127c67654d 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/Issue.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/Issue.java
@@ -2,12 +2,10 @@
 package com.yahoo.vespa.hosted.controller.api.integration.organization;
 
 import com.google.common.collect.ImmutableList;
-import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId;
 
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
-import java.util.Objects;
 import java.util.Optional;
 
 /**
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/IssueHandler.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/IssueHandler.java
index 80e23249daa..a49cee30015 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/IssueHandler.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/IssueHandler.java
@@ -76,6 +76,15 @@ public interface IssueHandler {
     boolean reassign(IssueId issueId, User assignee);
 
     /**
+     * Reassign the issue with the given ID to the given user, and returns the outcome of this.
+     *
+     * @param issueId ID of the issue to be watched.
+     * @param watcher watcher to add to the issue.
+     * @return Whether adding the watcher was successful.
+     */
+    boolean addWatcher(IssueId issueId, String watcher);
+
+    /**
      * Escalate an issue filed with the given property.
      *
      * @param issueId ID of the issue to escalate.
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/Mail.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/Mail.java
index 8d75b375044..8ee6ea03897 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/Mail.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/Mail.java
@@ -4,7 +4,6 @@ package com.yahoo.vespa.hosted.controller.api.integration.organization;
 import com.google.common.collect.ImmutableList;
 
 import java.util.Collection;
-import java.util.List;
 import java.util.Objects;
 
 /**
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/MockContactRetriever.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/MockContactRetriever.java
index dd0c143ae28..ab39d8e5660 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/MockContactRetriever.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/MockContactRetriever.java
@@ -8,23 +8,28 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Optional;
+import java.util.function.Supplier;
 
 /**
  * @author olaa
  */
 public class MockContactRetriever implements ContactRetriever{
 
-    private final Map<PropertyId, Contact> contacts = new HashMap<>();
+    private final Map<PropertyId, Supplier<Contact>> contacts = new HashMap<>();
 
     @Override
     public Contact getContact(Optional<PropertyId> propertyId) {
-        return contacts.getOrDefault(propertyId.get(), contact());
+        return contacts.getOrDefault(propertyId.get(), this::contact).get();
     }
 
-    public void addContact(PropertyId propertyId, Contact contact) {
+    public void addContact(PropertyId propertyId, Supplier<Contact> contact) {
         contacts.put(propertyId, contact);
     }
 
+    public void addContact(PropertyId propertyId, Contact contact) {
+        contacts.put(propertyId, () -> contact);
+    }
+
     public Contact contact() {
         return new Contact(URI.create("contacts.tld"), URI.create("properties.tld"), URI.create("issues.tld"), Collections.emptyList(), "queue", Optional.of("component"));
     }
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/MockIssueHandler.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/MockIssueHandler.java
index 2e92c993686..cf0b467c790 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/MockIssueHandler.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/MockIssueHandler.java
@@ -2,12 +2,12 @@
 package com.yahoo.vespa.hosted.controller.api.integration.organization;
 
 import com.google.inject.Inject;
-import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId;
 
 import java.net.URI;
 import java.time.Clock;
 import java.time.Duration;
 import java.time.Instant;
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
@@ -84,6 +84,12 @@ public class MockIssueHandler implements IssueHandler {
     }
 
     @Override
+    public boolean addWatcher(IssueId issueId, String watcher) {
+        issues.get(issueId).addWatcher(watcher);
+        return true;
+    }
+
+    @Override
     public Optional<User> escalate(IssueId issueId, Contact contact) {
         List<List<User>> contacts = getContactUsers(contact);
         Optional<User> assignee = assigneeOf(issueId);
@@ -131,7 +137,7 @@ public class MockIssueHandler implements IssueHandler {
         issues.get(issueId).updated = clock.instant();
     }
 
-    private class PropertyInfo {
+    private static class PropertyInfo {
 
         private List<List<User>> contacts = Collections.emptyList();
         private URI issueUrl = URI.create("issues.tld");
@@ -146,17 +152,21 @@ public class MockIssueHandler implements IssueHandler {
         private Instant updated;
         private boolean open;
         private User assignee;
+        private List<String> watchers;
 
         private MockIssue(Issue issue) {
             this.issue = issue;
             this.updated = clock.instant();
             this.open = true;
             this.assignee = issue.assignee().orElse(null);
+            this.watchers = new ArrayList<>();
         }
 
         public Issue issue() { return issue; }
         public User assignee() { return assignee; }
         public boolean isOpen() { return open; }
+        public List<String> watchers() { return List.copyOf(watchers); }
+        public void addWatcher(String watcher) { watchers.add(watcher); }
 
     }
 
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/MeteringClient.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/MeteringClient.java
index 3e06b24c6be..1047e1a02a4 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/MeteringClient.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/MeteringClient.java
@@ -4,7 +4,9 @@ package com.yahoo.vespa.hosted.controller.api.integration.resource;
 import com.yahoo.config.provision.ApplicationName;
 import com.yahoo.config.provision.TenantName;
 
+import java.time.YearMonth;
 import java.util.Collection;
+import java.util.List;
 
 /**
  * Consumes and retrieves snapshots of resources allocated per application.
@@ -15,6 +17,8 @@ public interface MeteringClient {
 
     void consume(Collection<ResourceSnapshot> resources);
 
-    MeteringInfo getResourceSnapshots(TenantName tenantName, ApplicationName applicationName);
+    MeteringData getMeteringData(TenantName tenantName, ApplicationName applicationName);
+
+    List<ResourceSnapshot> getSnapshotHistoryForTenant(TenantName tenantName, YearMonth yearMonth);
 
 }
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/MeteringInfo.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/MeteringData.java
index d6cb8f7fe76..e9a6c81e636 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/MeteringInfo.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/MeteringData.java
@@ -9,14 +9,14 @@ import java.util.Map;
 /**
  * @author olaa
  */
-public class MeteringInfo {
+public class MeteringData {
 
     private final ResourceAllocation thisMonth;
     private final ResourceAllocation lastMonth;
     private final ResourceAllocation currentSnapshot;
     Map<ApplicationId, List<ResourceSnapshot>> snapshotHistory;
 
-    public MeteringInfo(ResourceAllocation thisMonth, ResourceAllocation lastMonth, ResourceAllocation currentSnapshot, Map<ApplicationId, List<ResourceSnapshot>> snapshotHistory) {
+    public MeteringData(ResourceAllocation thisMonth, ResourceAllocation lastMonth, ResourceAllocation currentSnapshot, Map<ApplicationId, List<ResourceSnapshot>> snapshotHistory) {
         this.thisMonth = thisMonth;
         this.lastMonth = lastMonth;
         this.currentSnapshot = currentSnapshot;
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/MockTenantCost.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/MockTenantCost.java
index 03ba44e04c7..784f97d5296 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/MockTenantCost.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/MockTenantCost.java
@@ -3,7 +3,6 @@ package com.yahoo.vespa.hosted.controller.api.integration.resource;
 
 import com.yahoo.config.provision.TenantName;
 
-import java.time.LocalDate;
 import java.time.YearMonth;
 import java.util.Collections;
 import java.util.List;
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/TenantCost.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/TenantCost.java
index b4ca0cd7076..b0f8ca757b4 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/TenantCost.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/resource/TenantCost.java
@@ -3,7 +3,6 @@ package com.yahoo.vespa.hosted.controller.api.integration.resource;
 
 import com.yahoo.config.provision.TenantName;
 
-import java.time.LocalDate;
 import java.time.YearMonth;
 import java.util.Collections;
 import java.util.List;
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/RoutingEndpoint.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/RoutingEndpoint.java
deleted file mode 100644
index 59ad23aaa23..00000000000
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/RoutingEndpoint.java
+++ /dev/null
@@ -1,59 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.controller.api.integration.routing;
-
-import java.util.Objects;
-
-/**
- * @author smorgrav
- */
-public class RoutingEndpoint {
-
-    private final boolean isGlobal;
-    private final String endpoint;
-    private final String hostname;
-    private final String upstreamName;
-
-    public RoutingEndpoint(String endpoint, String hostname, boolean isGlobal, String upstreamName) {
-        this.endpoint = endpoint;
-        this.hostname = hostname;
-        this.isGlobal = isGlobal;
-        this.upstreamName = upstreamName;
-    }
-
-    /** Whether this is a global endpoint */
-    public boolean isGlobal() {
-        return isGlobal;
-    }
-
-    /** URL for this endpoint */
-    public String endpoint() {
-        return endpoint;
-    }
-
-    /** First hostname for an upstream behind this endpoint */
-    public String hostname() {
-        return hostname;
-    }
-
-    /** The upstream name of this endpoint */
-    public String upstreamName() {
-        return upstreamName;
-    }
-
-    @Override
-    public boolean equals(Object o) {
-        if (this == o) return true;
-        if (o == null || getClass() != o.getClass()) return false;
-        RoutingEndpoint that = (RoutingEndpoint) o;
-        return isGlobal == that.isGlobal &&
-               endpoint.equals(that.endpoint) &&
-               hostname.equals(that.hostname) &&
-               upstreamName.equals(that.upstreamName);
-    }
-
-    @Override
-    public int hashCode() {
-        return Objects.hash(isGlobal, endpoint, hostname, upstreamName);
-    }
-
-}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/RoutingGenerator.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/RoutingGenerator.java
deleted file mode 100644
index f5c82018ac6..00000000000
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/RoutingGenerator.java
+++ /dev/null
@@ -1,26 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.controller.api.integration.routing;
-
-import com.yahoo.config.provision.ClusterSpec;
-import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
-
-import java.net.URI;
-import java.util.List;
-import java.util.Map;
-
-/**
- * @author bratseth
- * @author smorgrav
- */
-public interface RoutingGenerator {
-
-    /**
-     * @param deploymentId Specifying an application in a zone
-     * @return List of endpoints for that deploymentId
-     */
-    List<RoutingEndpoint> endpoints(DeploymentId deploymentId);
-
-    /** Returns the endpoints of each cluster in the given deployment — nothing global. */
-    Map<ClusterSpec.Id, URI> clusterEndpoints(DeploymentId deploymentId);
-
-}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/RoutingGeneratorMock.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/RoutingGeneratorMock.java
deleted file mode 100644
index a7dc5f81bc0..00000000000
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/RoutingGeneratorMock.java
+++ /dev/null
@@ -1,61 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.controller.api.integration.routing;
-
-import com.yahoo.config.provision.ClusterSpec;
-import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
-
-import java.net.URI;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.stream.Collectors;
-
-/**
- * Returns a default set of endpoints on every query if it has no mappings, or those added by the user, otherwise.
- *
- * @author bratseth
- * @author jonmv
- */
-public class RoutingGeneratorMock implements RoutingGenerator {
-
-    public static final List<RoutingEndpoint> TEST_ENDPOINTS =
-            List.of(new RoutingEndpoint("http://old-endpoint.vespa.yahooapis.com:4080", "host1", false, "upstream3"),
-                    new RoutingEndpoint("http://qrs-endpoint.vespa.yahooapis.com:4080", "host1", false, "upstream1"),
-                    new RoutingEndpoint("http://feeding-endpoint.vespa.yahooapis.com:4080", "host2", false, "upstream2"),
-                    new RoutingEndpoint("http://global-endpoint.vespa.yahooapis.com:4080", "host1", true, "upstream1"),
-                    new RoutingEndpoint("http://alias-endpoint.vespa.yahooapis.com:4080", "host1", true, "upstream1"));
-
-    private final Map<DeploymentId, List<RoutingEndpoint>> routingTable = new ConcurrentHashMap<>();
-    private final List<RoutingEndpoint> defaultEndpoints;
-
-    public RoutingGeneratorMock() {
-        this(List.of());
-    }
-
-    public RoutingGeneratorMock(List<RoutingEndpoint> endpoints) {
-        this.defaultEndpoints = List.copyOf(endpoints);
-    }
-
-    @Override
-    public List<RoutingEndpoint> endpoints(DeploymentId deployment) {
-        if (routingTable.isEmpty()) return defaultEndpoints;
-        return routingTable.getOrDefault(deployment, List.of());
-    }
-
-    @Override
-    public Map<ClusterSpec.Id, URI> clusterEndpoints(DeploymentId deployment) {
-        return endpoints(deployment).stream()
-                                    .limit(1)
-                                    .collect(Collectors.toMap(__ -> ClusterSpec.Id.from("default"),
-                                                              endpoint -> URI.create(endpoint.endpoint())));
-    }
-
-    public void putEndpoints(DeploymentId deployment, List<RoutingEndpoint> endpoints) {
-        routingTable.put(deployment, endpoints);
-    }
-
-    public void removeEndpoints(DeploymentId deployment) {
-        routingTable.remove(deployment);
-    }
-
-}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/status/StatusReply.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/status/StatusReply.java
deleted file mode 100644
index 2bf2a706ee6..00000000000
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/status/StatusReply.java
+++ /dev/null
@@ -1,33 +0,0 @@
-// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.controller.api.integration.routing.status;
-
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
-import com.fasterxml.jackson.annotation.JsonInclude;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.yahoo.vespa.hosted.controller.api.integration.routing.RotationStatus;
-
-/**
- * @author bjorncs
- */
-@JsonIgnoreProperties(ignoreUnknown = true)
-@JsonInclude(value = JsonInclude.Include.NON_NULL)
-public class StatusReply {
-
-    @JsonProperty("status") public final RotationStatus status;
-    @JsonProperty("lastUpdate") public final long lastUpdate;
-    @JsonProperty("cause") public final String cause;
-    @JsonProperty("agent") public final String agent;
-
-    @JsonCreator
-    public StatusReply(@JsonProperty("status") RotationStatus status,
-                       @JsonProperty("lastUpdate") long lastUpdate,
-                       @JsonProperty("cause") String cause,
-                       @JsonProperty("agent") String agent) {
-        this.status = status;
-        this.lastUpdate = lastUpdate;
-        this.cause = cause;
-        this.agent = agent;
-    }
-    
-}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/status/ZoneStatus.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/status/ZoneStatus.java
deleted file mode 100644
index aea5f6d928d..00000000000
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/status/ZoneStatus.java
+++ /dev/null
@@ -1,9 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.controller.api.integration.routing.status;
-
-/**
- * @author bjorncs
- */
-public enum ZoneStatus {
-    OUT, IN
-}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/status/ZoneStatusReply.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/status/ZoneStatusReply.java
deleted file mode 100644
index d2e94e73b39..00000000000
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/status/ZoneStatusReply.java
+++ /dev/null
@@ -1,21 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.controller.api.integration.routing.status;
-
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-/**
- * @author bjorncs
- */
-@JsonIgnoreProperties(ignoreUnknown = true)
-public class ZoneStatusReply {
-
-    public final ZoneStatus status;
-
-    @JsonCreator
-    public ZoneStatusReply(@JsonProperty("status") ZoneStatus status) {
-        this.status = status;
-    }
-
-}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/status/package-info.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/status/package-info.java
deleted file mode 100644
index 0b6117d516f..00000000000
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/status/package-info.java
+++ /dev/null
@@ -1,8 +0,0 @@
-// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-/**
- * @author bjorncs
- */
-@ExportPackage
-package com.yahoo.vespa.hosted.controller.api.integration.routing.status;
-
-import com.yahoo.osgi.annotation.ExportPackage;
\ No newline at end of file
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/LoggingDeploymentIssues.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/LoggingDeploymentIssues.java
index f9ddb0027c0..3007bbf1697 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/LoggingDeploymentIssues.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/LoggingDeploymentIssues.java
@@ -5,7 +5,6 @@ package com.yahoo.vespa.hosted.controller.api.integration.stubs;
 import com.google.inject.Inject;
 import com.yahoo.component.Version;
 import com.yahoo.config.provision.ApplicationId;
-import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId;
 import com.yahoo.vespa.hosted.controller.api.integration.organization.Contact;
 import com.yahoo.vespa.hosted.controller.api.integration.organization.DeploymentIssues;
 import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId;
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockMeteringClient.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockMeteringClient.java
index 45ead36f622..80caef1709d 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockMeteringClient.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockMeteringClient.java
@@ -3,16 +3,19 @@ package com.yahoo.vespa.hosted.controller.api.integration.stubs;
 
 import com.yahoo.config.provision.ApplicationName;
 import com.yahoo.config.provision.TenantName;
-import com.yahoo.vespa.hosted.controller.api.integration.resource.MeteringInfo;
+import com.yahoo.vespa.hosted.controller.api.integration.resource.MeteringData;
 import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceAllocation;
 import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceSnapshot;
 import com.yahoo.vespa.hosted.controller.api.integration.resource.MeteringClient;
 
+import java.time.YearMonth;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
+import java.util.Map;
 import java.util.Optional;
+import java.util.stream.Collectors;
 
 /**
  * @author olaa
@@ -20,7 +23,7 @@ import java.util.Optional;
 public class MockMeteringClient implements MeteringClient {
 
     private Collection<ResourceSnapshot> resources = new ArrayList<>();
-    private Optional<MeteringInfo> meteringInfo;
+    private Optional<MeteringData> meteringData;
 
     @Override
     public void consume(Collection<ResourceSnapshot> resources){
@@ -28,18 +31,25 @@ public class MockMeteringClient implements MeteringClient {
     }
 
     @Override
-    public MeteringInfo getResourceSnapshots(TenantName tenantName, ApplicationName applicationName) {
-        return meteringInfo.orElseGet(() -> {
+    public MeteringData getMeteringData(TenantName tenantName, ApplicationName applicationName) {
+        return meteringData.orElseGet(() -> {
             ResourceAllocation emptyAllocation = new ResourceAllocation(0, 0, 0);
-            return new MeteringInfo(emptyAllocation, emptyAllocation, emptyAllocation, Collections.emptyMap());
+            return new MeteringData(emptyAllocation, emptyAllocation, emptyAllocation, Collections.emptyMap());
         });
     }
 
+    @Override
+    public List<ResourceSnapshot> getSnapshotHistoryForTenant(TenantName tenantName, YearMonth yearMonth) {
+        return new ArrayList<>(resources);
+    }
+
     public Collection<ResourceSnapshot> consumedResources() {
         return this.resources;
     }
 
-    public void setMeteringInfo(MeteringInfo meteringInfo) {
-        this.meteringInfo = Optional.of(meteringInfo);
+    public void setMeteringData(MeteringData meteringData) {
+        this.meteringData = Optional.of(meteringData);
+        this.resources = meteringData.getSnapshotHistory().entrySet().stream().map(Map.Entry::getValue).flatMap(List::stream).collect(Collectors.toList());
+        boolean a = false;
     }
 }
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockTesterCloud.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockTesterCloud.java
index d2914f95360..eb5360bc4bc 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockTesterCloud.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockTesterCloud.java
@@ -1,13 +1,18 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.api.integration.stubs;
 
+import com.yahoo.config.provision.HostName;
 import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
 import com.yahoo.vespa.hosted.controller.api.integration.LogEntry;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud;
+import com.yahoo.vespa.hosted.controller.api.integration.dns.NameService;
+import com.yahoo.vespa.hosted.controller.api.integration.dns.Record;
+import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName;
 
 import java.net.URI;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Optional;
 import java.util.stream.Collectors;
 
 import static com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud.Status.NOT_STARTED;
@@ -15,28 +20,20 @@ import static com.yahoo.vespa.hosted.controller.api.integration.deployment.Teste
 
 public class MockTesterCloud implements TesterCloud {
 
+    private final NameService nameService;
+
     private List<LogEntry> log = new ArrayList<>();
     private Status status = NOT_STARTED;
     private byte[] config;
-    private URI testerUrl;
 
-    @Override
-    public void startTests(URI testerUrl, Suite suite, byte[] config) {
-        this.status = RUNNING;
-        this.config = config;
-        this.testerUrl = testerUrl;
+    public MockTesterCloud(NameService nameService) {
+        this.nameService = nameService;
     }
 
     @Override
     public void startTests(DeploymentId deploymentId, Suite suite, byte[] config) {
         this.status = RUNNING;
         this.config = config;
-        this.testerUrl = null;
-    }
-
-    @Override
-    public List<LogEntry> getLog(URI testerUrl, long after) {
-        return log.stream().filter(entry -> entry.id() > after).collect(Collectors.toList());
     }
 
     @Override
@@ -45,9 +42,6 @@ public class MockTesterCloud implements TesterCloud {
     }
 
     @Override
-    public Status getStatus(URI testerUrl) { return status; }
-
-    @Override
     public Status getStatus(DeploymentId deploymentId) { return status; }
 
     @Override
@@ -56,23 +50,20 @@ public class MockTesterCloud implements TesterCloud {
     }
 
     @Override
-    public boolean testerReady(URI testerUrl) {
-        return true;
-    }
-
-    @Override
     public boolean testerReady(DeploymentId deploymentId) {
         return true;
     }
 
     @Override
-    public boolean exists(URI endpointUrl) {
-        return true;
+    public Optional<String> resolveHostName(HostName hostname) {
+        return Optional.of("1.2.3.4");
     }
 
     @Override
-    public boolean exists(DeploymentId deploymentId) {
-        return true;
+    public Optional<HostName> resolveCname(HostName hostName) {
+        return nameService.findRecords(Record.Type.CNAME, RecordName.from(hostName.value())).stream()
+                          .findFirst()
+                          .map(record -> HostName.from(record.data().asString().substring(0, record.data().asString().length() - 1)));
     }
 
     public void add(LogEntry entry) {
@@ -87,8 +78,4 @@ public class MockTesterCloud implements TesterCloud {
         return config;
     }
 
-    public URI testerUrl() {
-        return testerUrl;
-    }
-
 }
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockUserManagement.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockUserManagement.java
index 6df44ceaf9d..295f8e8fd98 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockUserManagement.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockUserManagement.java
@@ -22,6 +22,14 @@ public class MockUserManagement implements UserManagement {
 
     private final Map<Role, Set<User>> memberships = new HashMap<>();
 
+    private Set<User> get(Role role) {
+        var membership = memberships.get(role);
+        if (membership == null) {
+            throw new IllegalArgumentException(role + " not found");
+        }
+        return membership;
+    }
+
     @Override
     public void createRole(Role role) {
         if (memberships.containsKey(role))
@@ -40,7 +48,7 @@ public class MockUserManagement implements UserManagement {
         List<User> userObjs = users.stream()
                                    .map(id -> new User(id.value(), id.value(), null, null))
                                    .collect(Collectors.toList());
-        memberships.get(role).addAll(userObjs);
+        get(role).addAll(userObjs);
     }
 
     @Override
@@ -52,7 +60,7 @@ public class MockUserManagement implements UserManagement {
 
     @Override
     public void removeUsers(Role role, Collection<UserId> users) {
-        memberships.get(role).removeIf(user -> users.contains(new UserId(user.email())));
+        get(role).removeIf(user -> users.contains(new UserId(user.email())));
     }
 
     @Override
@@ -64,7 +72,7 @@ public class MockUserManagement implements UserManagement {
 
     @Override
     public List<User> listUsers(Role role) {
-        return List.copyOf(memberships.get(role));
+        return List.copyOf(get(role));
     }
 
 }
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/user/Roles.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/user/Roles.java
index 77bd589f23b..578f516f01e 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/user/Roles.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/user/Roles.java
@@ -35,6 +35,7 @@ public class Roles {
     public static Role toRole(String value) {
         String[] parts = value.split("\\.");
         if (parts.length == 1 && parts[0].equals("hostedOperator")) return Role.hostedOperator();
+        if (parts.length == 1 && parts[0].equals("hostedSupporter")) return Role.hostedSupporter();
         if (parts.length == 2) return toRole(TenantName.from(parts[0]), parts[1]);
         if (parts.length == 3) return toRole(TenantName.from(parts[0]), ApplicationName.from(parts[1]), parts[2]);
         throw new IllegalArgumentException("Malformed or illegal role value '" + value + "'.");
@@ -43,9 +44,6 @@ public class Roles {
     /** Returns the {@link Role} the given tenant, application and role names correspond to. */
     public static Role toRole(TenantName tenant, String roleName) {
         switch (roleName) {
-            case "tenantOwner": return Role.tenantOwner(tenant);
-            case "tenantAdmin": return Role.tenantAdmin(tenant);
-            case "tenantOperator": return Role.tenantOperator(tenant);
             case "administrator": return Role.administrator(tenant);
             case "developer": return Role.developer(tenant);
             case "reader": return Role.reader(tenant);
@@ -56,10 +54,6 @@ public class Roles {
     /** Returns the {@link Role} the given tenant and role names correspond to. */
     public static Role toRole(TenantName tenant, ApplicationName application, String roleName) {
         switch (roleName) {
-            case "applicationAdmin": return Role.applicationAdmin(tenant, application);
-            case "applicationOperator": return Role.applicationOperator(tenant, application);
-            case "applicationDeveloper": return Role.applicationDeveloper(tenant, application);
-            case "applicationReader": return Role.applicationReader(tenant, application);
             case "headless": return Role.headless(tenant, application);
             default: throw new IllegalArgumentException("Malformed or illegal role name '" + roleName + "'.");
         }
@@ -96,13 +90,6 @@ public class Roles {
 
     private static String valueOf(RoleDefinition role) {
         switch (role) {
-            case tenantOwner:          return "tenantOwner";
-            case tenantAdmin:          return "tenantAdmin";
-            case tenantOperator:       return "tenantOperator";
-            case applicationAdmin:     return "applicationAdmin";
-            case applicationOperator:  return "applicationOperator";
-            case applicationDeveloper: return "applicationDeveloper";
-            case applicationReader:    return "applicationReader";
             case administrator:        return "administrator";
             case developer:            return "developer";
             case reader:               return "reader";
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneRegistry.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneRegistry.java
index d808b4b7adb..b9ee696431b 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneRegistry.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneRegistry.java
@@ -8,6 +8,7 @@ import com.yahoo.config.provision.Environment;
 import com.yahoo.config.provision.NodeType;
 import com.yahoo.config.provision.RegionName;
 import com.yahoo.config.provision.SystemName;
+import com.yahoo.config.provision.zone.RoutingMethod;
 import com.yahoo.config.provision.zone.UpgradePolicy;
 import com.yahoo.config.provision.zone.ZoneFilter;
 import com.yahoo.config.provision.zone.ZoneId;
@@ -72,6 +73,9 @@ public interface ZoneRegistry {
     /** Returns all OS upgrade policies */
     List<UpgradePolicy> osUpgradePolicies();
 
+    /** Returns the routing methods supported by given zone, with the most preferred method appearing first */
+    List<RoutingMethod> routingMethods(ZoneId zone);
+
     /** Returns a URL where an informative dashboard can be found. */
     URI dashboardUrl();
 
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/ApplicationRole.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/ApplicationRole.java
index 293ee208e13..d0de75abd21 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/ApplicationRole.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/ApplicationRole.java
@@ -2,7 +2,6 @@
 package com.yahoo.vespa.hosted.controller.api.role;
 
 import com.yahoo.config.provision.ApplicationName;
-import com.yahoo.config.provision.SystemName;
 import com.yahoo.config.provision.TenantName;
 
 /**
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Context.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Context.java
index c6e361426ba..2b10045dc19 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Context.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Context.java
@@ -3,7 +3,6 @@ package com.yahoo.vespa.hosted.controller.api.role;
 
 import com.yahoo.config.provision.ApplicationName;
 import com.yahoo.config.provision.InstanceName;
-import com.yahoo.config.provision.SystemName;
 import com.yahoo.config.provision.TenantName;
 
 import java.util.Objects;
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/InstanceRole.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/InstanceRole.java
deleted file mode 100644
index 6cc726f2ac3..00000000000
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/InstanceRole.java
+++ /dev/null
@@ -1,33 +0,0 @@
-// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.controller.api.role;
-
-import com.yahoo.config.provision.ApplicationName;
-import com.yahoo.config.provision.InstanceName;
-import com.yahoo.config.provision.TenantName;
-
-/**
- * A {@link Role} with a {@link Context} of a {@link TenantName}, an {@link ApplicationName}, and an {@link InstanceName}.
- *
- * @author jonmv
- */
-public class InstanceRole extends Role {
-
-    InstanceRole(RoleDefinition roleDefinition, TenantName tenant, ApplicationName application, InstanceName instance) {
-        super(roleDefinition, Context.limitedTo(tenant, application, instance));
-    }
-
-    /** Returns the {@link TenantName} this is bound to. */
-    public TenantName tenant() { return context.tenant().get(); }
-
-    /** Returns the {@link ApplicationName} this is bound to. */
-    public ApplicationName application() { return context.application().get(); }
-
-    /** Returns the {@link InstanceName} this is bound to. */
-    public InstanceName instance() { return context.instance().get(); }
-
-    @Override
-    public String toString() {
-        return "role '" + definition() + "' of instance '" + instance() + "' of '" + application() + "' owned by '" + tenant() + "'";
-    }
-
-}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java
index 998af030b6b..5c11dfc2a55 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java
@@ -6,6 +6,7 @@ import com.yahoo.restapi.Path;
 import java.net.URI;
 import java.util.EnumSet;
 import java.util.List;
+import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
 
@@ -20,54 +21,59 @@ import java.util.Set;
 enum PathGroup {
 
     /** Paths exclusive to operators (including read), used for system management. */
-    classifiedOperator(Optional.of("/api"),
+    classifiedOperator(PathPrefix.api,
                        "/configserver/v1/{*}"),
 
     /** Paths used for system management by operators. */
-    operator("/controller/v1/{*}",
+    operator(PathPrefix.none,
+             "/controller/v1/{*}",
              "/flags/v1/{*}",
              "/nodes/v2/{*}",
              "/orchestrator/v1/{*}",
              "/os/v1/{*}",
              "/provision/v2/{*}",
              "/zone/v2/{*}",
-             "/routing/v1/{*}"),
+             "/routing/v1/",
+             "/routing/v1/status/environment/{*}",
+             "/routing/v1/inactive/environment/{*}"),
 
     /** Paths used for creating and reading user resources. */
-    user(Optional.of("/api"),
+    user(PathPrefix.api,
          "/application/v4/user",
          "/athenz/v1/{*}"),
 
     /** Paths used for creating tenants with proper access control. */
     tenant(Matcher.tenant,
-           Optional.of("/api"),
+           PathPrefix.api,
            "/application/v4/tenant/{tenant}"),
 
     /** Paths used for user management on the tenant level. */
     tenantUsers(Matcher.tenant,
-                Optional.of("/api"),
+                PathPrefix.api,
                 "/user/v1/tenant/{tenant}"),
 
     /** Paths used by tenant administrators. */
     tenantInfo(Matcher.tenant,
-               Optional.of("/api"),
+               PathPrefix.api,
                "/application/v4/tenant/{tenant}/application/",
                "/application/v4/tenant/{tenant}/cost",
-               "/application/v4/tenant/{tenant}/cost/{date}"),
+               "/application/v4/tenant/{tenant}/cost/{date}",
+               "/routing/v1/status/tenant/{tenant}/{*}",
+               "/billing/v1/tenant/{tenant}/{*}"),
 
     tenantKeys(Matcher.tenant,
-               Optional.of("/api"),
+               PathPrefix.api,
                "/application/v4/tenant/{tenant}/key/"),
 
     applicationKeys(Matcher.tenant,
                     Matcher.application,
-                    Optional.of("/api"),
+                    PathPrefix.api,
                     "/application/v4/tenant/{tenant}/application/{application}/key/"),
 
     /** Path for the base application resource. */
     application(Matcher.tenant,
                 Matcher.application,
-                Optional.of("/api"),
+                PathPrefix.api,
                 "/application/v4/tenant/{tenant}/application/{application}",
                 "/application/v4/tenant/{tenant}/application/{application}/instance/",
                 "/application/v4/tenant/{tenant}/application/{application}/instance/{ignored}"),
@@ -75,14 +81,15 @@ enum PathGroup {
     /** Paths used for user management on the application level. */
     applicationUsers(Matcher.tenant,
                      Matcher.application,
-                     Optional.of("/api"),
+                     PathPrefix.api,
                      "/user/v1/tenant/{tenant}/application/{application}"),
 
     /** Paths used by application administrators. */
     applicationInfo(Matcher.tenant,
                     Matcher.application,
-                    Optional.of("/api"),
+                    PathPrefix.api,
                     "/application/v4/tenant/{tenant}/application/{application}/package",
+                    "/application/v4/tenant/{tenant}/application/{application}/compile-version",
                     "/application/v4/tenant/{tenant}/application/{application}/deployment",
                     "/application/v4/tenant/{tenant}/application/{application}/deploying/{*}",
                     "/application/v4/tenant/{tenant}/application/{application}/instance/{ignored}/deploying/{*}",
@@ -97,14 +104,15 @@ enum PathGroup {
                     "/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{ignored}/suspended",
                     "/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{ignored}/service/{*}",
                     "/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{ignored}/global-rotation/{*}",
-                    "/application/v4/tenant/{tenant}/application/{application}/metering"),
+                    "/application/v4/tenant/{tenant}/application/{application}/metering",
+                    "/routing/v1/inactive/tenant/{tenant}/application/{application}/instance/{ignored}/environment/prod/region/{region}"),
 
     // TODO jonmv: remove
     /** Path used to restart development nodes. */
     developmentRestart(Matcher.tenant,
                        Matcher.application,
                        Matcher.instance,
-                       Optional.of("/api"),
+                       PathPrefix.api,
                        "/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/dev/region/{region}/restart",
                        "/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/perf/region/{region}/restart",
                        "/application/v4/tenant/{tenant}/application/{application}/environment/dev/region/{region}/instance/{instance}/restart",
@@ -114,7 +122,7 @@ enum PathGroup {
     /** Path used to restart production nodes. */
     productionRestart(Matcher.tenant,
                       Matcher.application,
-                      Optional.of("/api"),
+                      PathPrefix.api,
                       "/application/v4/tenant/{tenant}/application/{application}/instance/{ignored}/environment/prod/region/{region}/restart",
                       "/application/v4/tenant/{tenant}/application/{application}/instance/{ignored}/environment/test/region/{region}/restart",
                       "/application/v4/tenant/{tenant}/application/{application}/instance/{ignored}/environment/staging/region/{region}/restart",
@@ -126,7 +134,7 @@ enum PathGroup {
     developmentDeployment(Matcher.tenant,
                           Matcher.application,
                           Matcher.instance,
-                          Optional.of("/api"),
+                          PathPrefix.api,
                           "/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploy/{job}",
                           "/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/dev/region/{region}",
                           "/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/dev/region/{region}/deploy",
@@ -141,7 +149,7 @@ enum PathGroup {
     /** Paths used for production deployments. */
     productionDeployment(Matcher.tenant,
                          Matcher.application,
-                         Optional.of("/api"),
+                         PathPrefix.api,
                          "/application/v4/tenant/{tenant}/application/{application}/instance/{ignored}/environment/prod/region/{region}",
                          "/application/v4/tenant/{tenant}/application/{application}/instance/{ignored}/environment/prod/region/{region}/deploy",
                          "/application/v4/tenant/{tenant}/application/{application}/instance/{ignored}/environment/test/region/{region}",
@@ -158,86 +166,78 @@ enum PathGroup {
     /** Paths used for continuous deployment to production. */
     submission(Matcher.tenant,
                Matcher.application,
-               Optional.of("/api"),
+               PathPrefix.api,
                "/application/v4/tenant/{tenant}/application/{application}/submit",
                "/application/v4/tenant/{tenant}/application/{application}/instance/{ignored}/submit"),
 
     /** Paths used for other tasks by build services. */ // TODO: This will vanish.
     buildService(Matcher.tenant,
                  Matcher.application,
-                 Optional.of("/api"),
+                 PathPrefix.api,
                  "/application/v4/tenant/{tenant}/application/{application}/jobreport",
                  "/application/v4/tenant/{tenant}/application/{application}/instance/{ignored}/jobreport"),
 
     /** Paths which contain (not very strictly) classified information about customers. */
-    classifiedTenantInfo(Optional.of("/api"),
+    classifiedTenantInfo(PathPrefix.api,
                          "/application/v4/",
                          "/application/v4/tenant/"),
 
     /** Paths which contain (not very strictly) classified information about, e.g., customers. */
-    classifiedInfo("/cost/v1/{*}",
+    classifiedInfo(PathPrefix.none,
                    "/",
                    "/d/{*}"),
 
     /** Same as classifiedInfo, but with optional /api prefix */
-    classifiedApiInfo(Optional.of("/api"),
+    classifiedApiInfo(PathPrefix.api,
                       "/deployment/v1/{*}",
                       "/user/v1/user"),
 
     /** Paths providing public information. */
-    publicInfo(Optional.of("/api"),
+    publicInfo(PathPrefix.api,
                "/badge/v1/{*}",
                "/zone/v1/{*}"),
 
     /** Paths used for deploying system-wide feature flags. */
-    systemFlagsDeploy("/system-flags/v1/deploy"),
+    systemFlagsDeploy(PathPrefix.none, "/system-flags/v1/deploy"),
 
 
     /** Paths used for "dry-running" system-wide feature flags. */
-    systemFlagsDryrun("/system-flags/v1/dryrun");
+    systemFlagsDryrun(PathPrefix.none, "/system-flags/v1/dryrun"),
+
+    /** Paths used for receiving payment callbacks */
+    paymentProcessor(PathPrefix.none, "/payment/notification");
+
 
     final List<String> pathSpecs;
-    final String prefix;
+    final PathPrefix prefix;
     final List<Matcher> matchers;
 
-    PathGroup(String... pathSpecs) {
-        this(List.of(), Optional.empty(), List.of(pathSpecs));
-    }
-
-    PathGroup(Optional<String> prefix, String... pathSpecs) {
+    PathGroup(PathPrefix prefix, String... pathSpecs) {
         this(List.of(), prefix, List.of(pathSpecs));
     }
 
-    PathGroup(Matcher first, String... pathSpecs) {
-        this(List.of(first), Optional.empty(), List.of(pathSpecs));
-    }
-
-    PathGroup(Matcher first, Optional<String> prefix, String... pathSpecs) {
+    PathGroup(Matcher first, PathPrefix prefix, String... pathSpecs) {
         this(List.of(first), prefix, List.of(pathSpecs));
     }
 
-    PathGroup(Matcher first, Matcher second, String... pathSpecs) {
-        this(List.of(first, second), Optional.empty(), List.of(pathSpecs));
-    }
-
-    PathGroup(Matcher first, Matcher second, Optional<String> prefix, String... pathSpecs) {
+    PathGroup(Matcher first, Matcher second, PathPrefix prefix, String... pathSpecs) {
         this(List.of(first, second), prefix, List.of(pathSpecs));
     }
 
-    PathGroup(Matcher first, Matcher second, Matcher third, Optional<String> prefix, String... pathSpecs) {
+    PathGroup(Matcher first, Matcher second, Matcher third, PathPrefix prefix, String... pathSpecs) {
         this(List.of(first, second, third), prefix, List.of(pathSpecs));
     }
 
     /** Creates a new path group, if the given context matchers are each present exactly once in each of the given specs. */
-    PathGroup(List<Matcher> matchers, Optional<String> prefix, List<String> pathSpecs) {
+    PathGroup(List<Matcher> matchers, PathPrefix prefix, List<String> pathSpecs) {
         this.matchers = matchers;
-        this.prefix = prefix.orElse("");
+        this.prefix = prefix;
         this.pathSpecs = pathSpecs;
     }
 
     /** Returns path if it matches any spec in this group, with match groups set by the match. */
     private Optional<Path> get(URI uri) {
-        Path matcher = new Path(uri, prefix);
+        Path matcher = new Path(uri, prefix.prefix);
         for (String spec : pathSpecs) // Iterate to be sure the Path's state is that of the match.
             if (matcher.matches(spec)) return Optional.of(matcher);
         return Optional.empty();
@@ -290,4 +290,21 @@ enum PathGroup {
 
     }
 
+    /**
+     * The valid prefixes of paths in a {@link PathGroup}. Provides flexibility in cases where paths are made available
+     * under a non-root path.
+     */
+    enum PathPrefix {
+
+        none(""),
+        api("/api");
+
+        private final String prefix;
+
+        PathPrefix(String prefix) {
+            this.prefix = Objects.requireNonNull(prefix);
+        }
+
+    }
+
 }
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Policy.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Policy.java
index e27fb0fbf27..cfe8d247e54 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Policy.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Policy.java
@@ -10,9 +10,11 @@ import java.util.Set;
 
 /**
  * Policies for REST APIs in the controller. A policy is only considered when defined in a {@link Role}.
- * A policy describes a set of {@link Privilege}s, which are valid for a set of {@link SystemName}s.
- * A policy is evaluated by an {@link Enforcer}, which holds the {@link SystemName} the evaluation is done in.
- * A policy is evaluated with a {@link Context}, which may limit it to a specific {@link TenantName} or {@link ApplicationName}.
+ *
+ * - A policy describes a set of {@link Privilege}s, which are valid for a set of {@link SystemName}s.
+ * - A policy is evaluated by an {@link Enforcer}, which holds the {@link SystemName} the evaluation is done in.
+ * - A policy is evaluated in a {@link Context}, which may limit it to a specific {@link TenantName} or
+ *   {@link ApplicationName}.
  *
  * @author mpolden
  */
@@ -23,6 +25,11 @@ enum Policy {
                       .on(PathGroup.all())
                       .in(SystemName.all())),
 
+    /** Full access to everything. */
+    supporter(Privilege.grant(Action.read)
+                       .on(PathGroup.all())
+                       .in(SystemName.all())),
+
     /** Full access to user management for a tenant in select systems. */
     tenantManager(Privilege.grant(Action.all())
                            .on(PathGroup.tenantUsers)
@@ -43,6 +50,11 @@ enum Policy {
                           .on(PathGroup.tenant)
                           .in(SystemName.main, SystemName.cd, SystemName.dev)), // TODO SystemName.all()
 
+    /** Access to create a tenant in public */
+    tenantCreatePublic(Privilege.grant(Action.create)
+                                .on(PathGroup.tenant)
+                                .in(SystemName.PublicCd, SystemName.Public)),
+
     /** Full access to tenant information and settings. */
     tenantDelete(Privilege.grant(Action.delete)
                           .on(PathGroup.tenant)
@@ -93,11 +105,6 @@ enum Policy {
                                    .on(PathGroup.developmentDeployment, PathGroup.developmentRestart)
                                    .in(SystemName.all())),
 
-    /** Full access to application production deployments. */
-    productionDeployment(Privilege.grant(Action.all())
-                                  .on(PathGroup.productionDeployment)
-                                  .in(SystemName.all())),
-
     /** Read access to all application deployments. */
     deploymentRead(Privilege.grant(Action.read)
                             .on(PathGroup.developmentDeployment, PathGroup.productionDeployment)
@@ -108,11 +115,6 @@ enum Policy {
                         .on(PathGroup.submission)
                         .in(SystemName.all())),
 
-    /** Full access to the additional tasks needed for continuous deployment. */
-    deploymentPipeline(Privilege.grant(Action.all()) // TODO remove when everyone is on new pipeline.
-                                .on(PathGroup.buildService, PathGroup.productionRestart)
-                                .in(SystemName.all())),
-
     /** Read access to all information in select systems. */
     classifiedRead(Privilege.grant(Action.read)
                             .on(PathGroup.allExcept(PathGroup.classifiedOperator))
@@ -135,7 +137,12 @@ enum Policy {
     /** Access to /system-flags/v1/dryrun. */
     systemFlagsDryrun(Privilege.grant(Action.update)
                                   .on(PathGroup.systemFlagsDryrun)
-                                  .in(SystemName.all()));
+                                  .in(SystemName.all())),
+
+    /** Access to /payment/notification */
+    paymentProcessor(Privilege.grant(Action.create)
+                                .on(PathGroup.paymentProcessor)
+                                .in(SystemName.PublicCd));
 
     private final Set<Privilege> privileges;
 
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Role.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Role.java
index b53cf9162e7..d3c5e412215 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Role.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Role.java
@@ -2,7 +2,6 @@
 package com.yahoo.vespa.hosted.controller.api.role;
 
 import com.yahoo.config.provision.ApplicationName;
-import com.yahoo.config.provision.InstanceName;
 import com.yahoo.config.provision.TenantName;
 
 import java.util.Objects;
@@ -28,6 +27,11 @@ public abstract class Role {
         return new UnboundRole(RoleDefinition.hostedOperator);
     }
 
+    /** Returns a {@link RoleDefinition#hostedSupporter} for the current system. */
+    public static UnboundRole hostedSupporter() {
+        return new UnboundRole(RoleDefinition.hostedSupporter);
+    }
+
     /** Returns a {@link RoleDefinition#everyone} for the current system. */
     public static UnboundRole everyone() {
         return new UnboundRole(RoleDefinition.everyone);
@@ -38,31 +42,6 @@ public abstract class Role {
         return new TenantRole(RoleDefinition.athenzTenantAdmin, tenant);
     }
 
-    /** Returns a {@link RoleDefinition#tenantPipeline} for the current system and given tenant and application. */
-    public static ApplicationRole tenantPipeline(TenantName tenant, ApplicationName application) {
-        return new ApplicationRole(RoleDefinition.tenantPipeline, tenant, application);
-    }
-
-    /** Returns a {@link RoleDefinition#athenzUser} for the current system and given tenant and application. */
-    public static InstanceRole athenzUser(TenantName tenant, ApplicationName application, InstanceName instance) {
-        return new InstanceRole(RoleDefinition.athenzUser, tenant, application, instance);
-    }
-
-    /** Returns a {@link RoleDefinition#tenantOwner} for the current system and given tenant. */
-    public static TenantRole tenantOwner(TenantName tenant) {
-        return new TenantRole(RoleDefinition.tenantOwner, tenant);
-    }
-
-    /** Returns a {@link RoleDefinition#tenantAdmin} for the current system and given tenant. */
-    public static TenantRole tenantAdmin(TenantName tenant) {
-        return new TenantRole(RoleDefinition.tenantAdmin, tenant);
-    }
-
-    /** Returns a {@link RoleDefinition#tenantOperator} for the current system and given tenant. */
-    public static TenantRole tenantOperator(TenantName tenant) {
-        return new TenantRole(RoleDefinition.tenantOperator, tenant);
-    }
-
     /** Returns a {@link RoleDefinition#reader} for the current system and given tenant. */
     public static TenantRole reader(TenantName tenant) {
         return new TenantRole(RoleDefinition.reader, tenant);
@@ -83,26 +62,6 @@ public abstract class Role {
         return new ApplicationRole(RoleDefinition.headless, tenant, application);
     }
 
-    /** Returns a {@link RoleDefinition#applicationAdmin} for the current system and given tenant and application. */
-    public static ApplicationRole applicationAdmin(TenantName tenant, ApplicationName application) {
-        return new ApplicationRole(RoleDefinition.applicationAdmin, tenant, application);
-    }
-
-    /** Returns a {@link RoleDefinition#applicationOperator} for the current system and given tenant and application. */
-    public static ApplicationRole applicationOperator(TenantName tenant, ApplicationName application) {
-        return new ApplicationRole(RoleDefinition.applicationOperator, tenant, application);
-    }
-
-    /** Returns a {@link RoleDefinition#applicationDeveloper} for the current system and given tenant and application. */
-    public static ApplicationRole applicationDeveloper(TenantName tenant, ApplicationName application) {
-        return new ApplicationRole(RoleDefinition.applicationDeveloper, tenant, application);
-    }
-
-    /** Returns a {@link RoleDefinition#applicationReader} for the current system and given tenant and application. */
-    public static ApplicationRole applicationReader(TenantName tenant, ApplicationName application) {
-        return new ApplicationRole(RoleDefinition.applicationReader, tenant, application);
-    }
-
     /** Returns a {@link RoleDefinition#buildService} for the current system and given tenant and application. */
     public static ApplicationRole buildService(TenantName tenant, ApplicationName application) {
         return new ApplicationRole(RoleDefinition.buildService, tenant, application);
@@ -114,6 +73,9 @@ public abstract class Role {
     /** Returns the role for system flag dryrun */
     public static UnboundRole systemFlagsDryrunner() { return new UnboundRole(RoleDefinition.systemFlagsDryrunner); }
 
+    /** Returns the role of the payment processor */
+    public static UnboundRole paymentProcessor() { return new UnboundRole(RoleDefinition.paymentProcessor); }
+
     /** Returns the role definition of this bound role. */
     public RoleDefinition definition() { return roleDefinition; }
 
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/RoleDefinition.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/RoleDefinition.java
index 58d69512feb..c05936ee593 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/RoleDefinition.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/RoleDefinition.java
@@ -21,6 +21,10 @@ public enum RoleDefinition {
     /** Deus ex machina. */
     hostedOperator(Policy.operator),
 
+    /** Machina autem exspiravit. */
+    hostedSupporter(Policy.supporter,
+                    Policy.tenantCreatePublic),
+
     /** Base role which every user is part of. */
     everyone(Policy.classifiedRead,
              Policy.classifiedApiRead,
@@ -28,50 +32,13 @@ public enum RoleDefinition {
              Policy.user,
              Policy.tenantCreate),
 
-    /** Application reader which can see all information about an application, its tenant and deployments. */
-    applicationReader(everyone,
-                      Policy.tenantRead,
-                      Policy.applicationRead,
-                      Policy.deploymentRead),
-
     /** Build service which may submit new applications for continuous deployment. */
-    buildService(applicationReader,
+    buildService(everyone,
+                 Policy.tenantRead,
+                 Policy.applicationRead,
+                 Policy.deploymentRead,
                  Policy.submission),
 
-    /** Application developer with access to deploy to development zones. */
-    applicationDeveloper(applicationReader,
-                         Policy.developmentDeployment),
-
-    /** Application operator with access to normal, operational tasks of an application. */
-    applicationOperator(applicationReader,
-                        Policy.applicationOperations),
-
-    /** Application administrator with full access to an already existing application, including emergency operations. */
-    applicationAdmin(applicationDeveloper,
-                     applicationOperator,
-                     Policy.applicationUpdate,
-                     Policy.applicationDelete,
-                     Policy.applicationManager,
-                     Policy.productionDeployment,
-                     Policy.submission),
-
-    /** Tenant operator with access to create application under a tenant, and to read the tenant's and public data. */
-    tenantOperator(everyone,
-                   Policy.tenantRead,
-                   Policy.applicationCreate,
-                   Policy.keyManagement),
-
-    /** Tenant admin with full access to all tenant resources, except deleting the tenant. */
-    tenantAdmin(tenantOperator,
-                applicationAdmin,
-                Policy.applicationDelete,
-                Policy.tenantManager,
-                Policy.tenantUpdate),
-
-    /** Tenant admin with full access to all tenant resources. */
-    tenantOwner(tenantAdmin,
-                Policy.tenantDelete),
-
     /** Reader — the base role for all tenant users */
     reader(Policy.tenantRead,
            Policy.applicationRead,
@@ -95,16 +62,6 @@ public enum RoleDefinition {
     /** Headless — the application specific role identified by deployment keys for production */
     headless(Policy.submission),
 
-    /** Build and continuous delivery service. */ // TODO replace with buildService, when everyone is on new pipeline.
-    tenantPipeline(everyone,
-                   Policy.submission,
-                   Policy.deploymentPipeline,
-                   Policy.productionDeployment),
-
-    /** Athenz user with access to development resources under its instances. */
-    athenzUser(everyone,
-               Policy.developmentDeployment),
-
     /** Tenant administrator with full access to all child resources. */
     athenzTenantAdmin(everyone,
                       Policy.tenantRead,
@@ -119,7 +76,9 @@ public enum RoleDefinition {
 
     systemFlagsDeployer(Policy.systemFlagsDeploy, Policy.systemFlagsDryrun),
 
-    systemFlagsDryrunner(Policy.systemFlagsDryrun);
+    systemFlagsDryrunner(Policy.systemFlagsDryrun),
+
+    paymentProcessor(Policy.paymentProcessor);
 
     private final Set<RoleDefinition> parents;
     private final Set<Policy> policies;
@@ -128,12 +87,8 @@ public enum RoleDefinition {
         this(Set.of(), policies);
     }
 
-    RoleDefinition(RoleDefinition first, Policy... policies) {
-        this(Set.of(first), policies);
-    }
-
-    RoleDefinition(RoleDefinition first, RoleDefinition second, Policy... policies) {
-        this(Set.of(first, second), policies);
+    RoleDefinition(RoleDefinition parent, Policy... policies) {
+        this(Set.of(parent), policies);
     }
 
     RoleDefinition(Set<RoleDefinition> parents, Policy... policies) {
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/TenantRole.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/TenantRole.java
index b53b0d1cc2d..ee7dd9d743e 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/TenantRole.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/TenantRole.java
@@ -1,7 +1,6 @@
 // Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.api.role;
 
-import com.yahoo.config.provision.SystemName;
 import com.yahoo.config.provision.TenantName;
 
 /**
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/UnboundRole.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/UnboundRole.java
index 64dc501e666..cc1f0b0b634 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/UnboundRole.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/UnboundRole.java
@@ -1,8 +1,6 @@
 // Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.api.role;
 
-import com.yahoo.config.provision.SystemName;
-
 /**
  * A {@link Role} with an unlimited {@link Context}.
  *
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/ControllerFlagsTarget.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/ControllerFlagsTarget.java
index a22a9cc63de..695a3c28761 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/ControllerFlagsTarget.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/ControllerFlagsTarget.java
@@ -1,10 +1,6 @@
 // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.api.systemflags.v1;
 
-/**
- * @author bjorncs
- */
-
 import com.yahoo.config.provision.SystemName;
 import com.yahoo.vespa.athenz.api.AthenzIdentity;
 
@@ -17,6 +13,9 @@ import static com.yahoo.vespa.hosted.controller.api.systemflags.v1.FlagsTarget.c
 import static com.yahoo.vespa.hosted.controller.api.systemflags.v1.FlagsTarget.defaultFile;
 import static com.yahoo.vespa.hosted.controller.api.systemflags.v1.FlagsTarget.systemFile;
 
+/**
+ * @author bjorncs
+ */
 class ControllerFlagsTarget implements FlagsTarget {
     private final SystemName system;
 
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchive.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchive.java
index 7e9da53b1c9..1e42efdd256 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchive.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchive.java
@@ -1,7 +1,16 @@
 // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.api.systemflags.v1;
 
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.NodeType;
+import com.yahoo.config.provision.SystemName;
+import com.yahoo.text.JSON;
+import com.yahoo.vespa.flags.FetchVector;
 import com.yahoo.vespa.flags.FlagId;
+import com.yahoo.vespa.flags.json.DimensionHelper;
 import com.yahoo.vespa.flags.json.FlagData;
 
 import java.io.BufferedInputStream;
@@ -13,11 +22,16 @@ import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.util.Arrays;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
 import java.util.Set;
 import java.util.TreeMap;
+import java.util.function.Consumer;
+import java.util.stream.Collectors;
 import java.util.stream.Stream;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipInputStream;
@@ -35,6 +49,8 @@ import static com.yahoo.yolean.Exceptions.uncheck;
  */
 public class SystemFlagsDataArchive {
 
+    private static final ObjectMapper mapper = new ObjectMapper();
+
     private final Map<FlagId, Map<String, FlagData>> files;
 
     private SystemFlagsDataArchive(Map<FlagId, Map<String, FlagData>> files) {
@@ -47,7 +63,7 @@ public class SystemFlagsDataArchive {
             ZipEntry entry;
             while ((entry = zipIn.getNextEntry()) != null) {
                 String name = entry.getName();
-                if (!entry.isDirectory() && name.startsWith("flags/") && name.endsWith(".json")) {
+                if (!entry.isDirectory() && name.startsWith("flags/")) {
                     Path filePath = Paths.get(name);
                     String rawData = new String(zipIn.readAllBytes(), StandardCharsets.UTF_8);
                     addFile(builder, rawData, filePath);
@@ -70,7 +86,7 @@ public class SystemFlagsDataArchive {
             directoryStream.forEach(absolutePath -> {
                 Path relativePath = root.relativize(absolutePath);
                 if (!Files.isDirectory(absolutePath) &&
-                        relativePath.startsWith("flags") && relativePath.toString().endsWith(".json")) {
+                        relativePath.startsWith("flags")) {
                     String rawData = uncheck(() -> Files.readString(absolutePath, StandardCharsets.UTF_8));
                     addFile(builder, rawData, relativePath);
                 }
@@ -86,7 +102,7 @@ public class SystemFlagsDataArchive {
         files.forEach((flagId, fileMap) -> {
             fileMap.forEach((filename, flagData) -> {
                 uncheck(() -> {
-                    zipOut.putNextEntry(new ZipEntry("flags/" + flagId.toString() + "/" + filename));
+                    zipOut.putNextEntry(new ZipEntry(toFilePath(flagId, filename)));
                     zipOut.write(flagData.serializeToUtf8Json());
                     zipOut.closeEntry();
                 });
@@ -112,23 +128,101 @@ public class SystemFlagsDataArchive {
         return targetData;
     }
 
+    public void validateAllFilesAreForTargets(SystemName currentSystem, Set<FlagsTarget> targets) throws IllegalArgumentException {
+        Set<String> validFiles = targets.stream()
+                .flatMap(target -> target.flagDataFilesPrioritized().stream())
+                .collect(Collectors.toSet());
+        Set<SystemName> otherSystems = Arrays.stream(SystemName.values())
+                .filter(systemName -> systemName != currentSystem)
+                .collect(Collectors.toSet());
+        files.forEach((flagId, fileMap) -> {
+            for (String filename : fileMap.keySet()) {
+                boolean isFileForOtherSystem = otherSystems.stream()
+                        .anyMatch(system -> filename.startsWith(system.value() + "."));
+                boolean isFileForCurrentSystem = validFiles.contains(filename);
+                if (!isFileForOtherSystem && !isFileForCurrentSystem) {
+                    throw new IllegalArgumentException("Unknown flag file: " + toFilePath(flagId, filename));
+                }
+            }
+        });
+    }
+
     private static void addFile(Builder builder, String rawData, Path filePath) {
         String filename = filePath.getFileName().toString();
+        if (filename.startsWith(".")) {
+            return; // Ignore files starting with '.'
+        }
+        if (!filename.endsWith(".json")) {
+            throw new IllegalArgumentException(String.format("Only JSON files are allowed in 'flags/' directory (found '%s')", filePath.toString()));
+        }
         FlagId directoryDeducedFlagId = new FlagId(filePath.getName(1).toString());
         FlagData flagData;
         if (rawData.isBlank()) {
             flagData = new FlagData(directoryDeducedFlagId);
         } else {
-            flagData = FlagData.deserialize(rawData);
+            String normalizedRawData = normalizeJson(rawData);
+            flagData = FlagData.deserialize(normalizedRawData);
             if (!directoryDeducedFlagId.equals(flagData.id())) {
                 throw new IllegalArgumentException(
                         String.format("Flag data file with flag id '%s' in directory for '%s'",
-                                      flagData.id(), directoryDeducedFlagId.toString()));
+                                flagData.id(), directoryDeducedFlagId.toString()));
+            }
+
+            String serializedData = flagData.serializeToJson();
+            if (!JSON.equals(serializedData, normalizedRawData)) {
+                throw new IllegalArgumentException(filePath + " contains unknown non-comment fields: " +
+                        "after removing any comment fields the JSON is:\n  " +
+                        normalizedRawData +
+                        "\nbut deserializing this ended up with a JSON that are missing some of the fields:\n  " +
+                        serializedData +
+                        "\nSee https://git.ouroath.com/vespa/hosted-feature-flags for more info on the JSON syntax");
             }
         }
         builder.addFile(filename, flagData);
     }
 
+    static String normalizeJson(String json) {
+        JsonNode root = uncheck(() -> mapper.readTree(json));
+        removeCommentsRecursively(root);
+        verifyValues(root);
+        return root.toString();
+    }
+
+    private static void verifyValues(JsonNode root) {
+        var cursor = new JsonAccessor(root);
+        cursor.get("rules").forEachArrayElement(rule -> rule.get("conditions").forEachArrayElement(condition -> {
+            var dimension = condition.get("dimension");
+            if (dimension.isEqualTo(DimensionHelper.toWire(FetchVector.Dimension.APPLICATION_ID))) {
+                condition.get("values").forEachArrayElement(conditionValue -> {
+                    String applicationIdString = conditionValue.asString()
+                            .orElseThrow(() -> new IllegalArgumentException("Non-string application ID: " + conditionValue));
+                    // Throws exception if not recognized
+                    ApplicationId.fromSerializedForm(applicationIdString);
+                });
+            } else if (dimension.isEqualTo(DimensionHelper.toWire(FetchVector.Dimension.NODE_TYPE))) {
+                condition.get("values").forEachArrayElement(conditionValue -> {
+                    String nodeTypeString = conditionValue.asString()
+                            .orElseThrow(() -> new IllegalArgumentException("Non-string node type: " + conditionValue));
+                    // Throws exception if not recognized
+                    NodeType.valueOf(nodeTypeString);
+                });
+            }
+        }));
+    }
+
+    private static void removeCommentsRecursively(JsonNode node) {
+        if (node instanceof ObjectNode) {
+            ObjectNode objectNode = (ObjectNode) node;
+            objectNode.remove("comment");
+        }
+
+        node.forEach(SystemFlagsDataArchive::removeCommentsRecursively);
+    }
+
+    private static String toFilePath(FlagId flagId, String filename) {
+        return "flags/" + flagId.toString() + "/" + filename;
+    }
+
     public static class Builder {
         private final Map<FlagId, Map<String, FlagData>> files = new TreeMap<>();
 
@@ -146,4 +240,40 @@ public class SystemFlagsDataArchive {
         }
 
     }
+
+    private static class JsonAccessor {
+        private final JsonNode jsonNode;
+
+        public JsonAccessor(JsonNode jsonNode) {
+            this.jsonNode = jsonNode;
+        }
+
+        public JsonAccessor get(String fieldName) {
+            if (jsonNode == null) {
+                return this;
+            } else {
+                return new JsonAccessor(jsonNode.get(fieldName));
+            }
+        }
+
+        public Optional<String> asString() {
+            return jsonNode != null && jsonNode.isTextual() ? Optional.of(jsonNode.textValue()) : Optional.empty();
+        }
+
+        public void forEachArrayElement(Consumer<JsonAccessor> consumer) {
+            if (jsonNode != null && jsonNode.isArray()) {
+                jsonNode.forEach(jsonNodeElement -> consumer.accept(new JsonAccessor(jsonNodeElement)));
+            }
+        }
+
+        /** Returns true if this (JsonNode) is a string and equal to value. */
+        public boolean isEqualTo(String value) {
+            return jsonNode != null && jsonNode.isTextual() && Objects.equals(jsonNode.textValue(), value);
+        }
+
+        @Override
+        public String toString() {
+            return jsonNode == null ? "undefined" : jsonNode.toString();
+        }
+    }
 }
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/wire/WireSystemFlagsDeployResult.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/wire/WireSystemFlagsDeployResult.java
index 69070d86ef7..17af0027893 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/wire/WireSystemFlagsDeployResult.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/wire/WireSystemFlagsDeployResult.java
@@ -18,6 +18,7 @@ import java.util.List;
 public class WireSystemFlagsDeployResult {
     @JsonProperty("changes") public List<WireFlagDataChange> changes;
     @JsonProperty("errors") public List<WireOperationFailure> errors;
+    @JsonProperty("warnings") public List<WireWarning> warnings;
 
     @JsonIgnoreProperties(ignoreUnknown = true)
     @JsonInclude(JsonInclude.Include.NON_NULL)
@@ -39,6 +40,14 @@ public class WireSystemFlagsDeployResult {
         @JsonProperty("data") public WireFlagData data;
     }
 
+    @JsonIgnoreProperties(ignoreUnknown = true)
+    @JsonInclude(JsonInclude.Include.NON_NULL)
+    public static class WireWarning {
+        @JsonProperty("flag-id") public String flagId;
+        @JsonProperty("message") public String message;
+        @JsonProperty("targets") public List<String> targets;
+    }
+
     public boolean hasErrors() { return errors != null && !errors.isEmpty(); }
 }
 
diff --git a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/identifiers/IdentifierTest.java b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/identifiers/IdentifierTest.java
index 8e278240a02..fdba1ab2680 100644
--- a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/identifiers/IdentifierTest.java
+++ b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/identifiers/IdentifierTest.java
@@ -107,11 +107,6 @@ public class IdentifierTest {
     }
 
     @Test
-    public void user_tenant_id_does_not_contain_underscore() {
-        assertEquals("by-under-score-user", new UserId("under_score_user").toTenantId().id());
-    }
-
-    @Test
     public void dns_names_has_no_underscore() {
         assertEquals("a-b-c", new ApplicationId("a_b_c").toDns());
     }
diff --git a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/integration/ApplicationIdSnapshotTest.java b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/integration/ApplicationIdSnapshotTest.java
deleted file mode 100644
index 53d9cdf8f55..00000000000
--- a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/integration/ApplicationIdSnapshotTest.java
+++ /dev/null
@@ -1,59 +0,0 @@
-// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.controller.api.integration;
-
-import com.yahoo.config.provision.ApplicationName;
-import com.yahoo.config.provision.InstanceName;
-import com.yahoo.config.provision.TenantName;
-import org.junit.Test;
-
-import java.util.Set;
-
-import static org.junit.Assert.assertEquals;
-
-/**
- * @author freva
- */
-public class ApplicationIdSnapshotTest {
-    private static final TenantName tenant1 = TenantName.from("tenant1");
-    private static final TenantName tenant2 = TenantName.from("tenant2");
-    private static final TenantName tenant3 = TenantName.from("tenant3");
-    private static final TenantName tenant4 = TenantName.from("tenant4");
-    private static final ApplicationName app1 = ApplicationName.from("app1");
-    private static final ApplicationName app2 = ApplicationName.from("app2");
-    private static final ApplicationName app3 = ApplicationName.from("app3");
-    private static final InstanceName instance1 = InstanceName.defaultName();
-    private static final InstanceName instance2 = InstanceName.from("instance2");
-    private static final InstanceName instance3 = InstanceName.from("instance3");
-
-    @Test
-    public void basic() {
-        ApplicationIdSnapshot snapshot = new ApplicationIdSnapshot.Builder()
-                .add(tenant1, app1, instance1)
-                .add(tenant1, app2, instance1)
-                .add(tenant1, app3)
-                .add(tenant1, app2, instance2)
-                .add(tenant2, app2, instance3)
-                .add(tenant3, app1)
-                .add(tenant4)
-                .build();
-
-        assertEquals(Set.of(tenant1, tenant2, tenant3, tenant4), snapshot.tenants());
-
-        assertEquals(Set.of(app1, app2, app3), snapshot.applications(tenant1));
-        assertEquals(Set.of(app2), snapshot.applications(tenant2));
-        assertEquals(Set.of(), snapshot.applications(tenant4));
-
-        assertEquals(Set.of(instance1), snapshot.instances(tenant1, app1));
-        assertEquals(Set.of(instance1, instance2), snapshot.instances(tenant1, app2));
-        assertEquals(Set.of(), snapshot.instances(tenant3, app1));
-    }
-
-    @Test
-    public void test_missing() {
-        ApplicationIdSnapshot snapshot = new ApplicationIdSnapshot.Builder().build();
-
-        assertEquals(Set.of(), snapshot.tenants());
-        assertEquals(Set.of(), snapshot.applications(tenant1));
-        assertEquals(Set.of(), snapshot.instances(tenant1, app1));
-    }
-}
\ No newline at end of file
diff --git a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/integration/user/RolesTest.java b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/integration/user/RolesTest.java
index cfb5462e50a..636825573f1 100644
--- a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/integration/user/RolesTest.java
+++ b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/integration/user/RolesTest.java
@@ -27,11 +27,8 @@ public class RolesTest {
 
         assertEquals(Role.hostedOperator(),
                      Roles.toRole("hostedOperator"));
-        assertEquals(Role.tenantOperator(tenant),
-                     Roles.toRole("my-tenant.tenantOperator"));
-        assertEquals(Role.applicationReader(tenant, application),
-                     Roles.toRole("my-tenant.my-application.applicationReader"));
-
+        assertEquals(Role.hostedSupporter(),
+                     Roles.toRole("hostedSupporter"));
         assertEquals(Role.administrator(tenant), Roles.toRole("my-tenant.administrator"));
         assertEquals(Role.developer(tenant), Roles.toRole("my-tenant.developer"));
         assertEquals(Role.reader(tenant), Roles.toRole("my-tenant.reader"));
@@ -40,17 +37,12 @@ public class RolesTest {
 
     @Test(expected = IllegalArgumentException.class)
     public void illegalTenantName() {
-        Roles.valueOf(Role.tenantAdmin(TenantName.from("my.tenant")));
+        Roles.valueOf(Role.developer(TenantName.from("my.tenant")));
     }
 
     @Test(expected = IllegalArgumentException.class)
     public void illegalApplicationName() {
-        Roles.valueOf(Role.applicationOperator(TenantName.from("my-tenant"), ApplicationName.from("my.app")));
-    }
-
-    @Test(expected = IllegalArgumentException.class)
-    public void illegalRole() {
-        Roles.valueOf(Role.tenantPipeline(TenantName.from("my-tenant"), ApplicationName.from("my-app")));
+        Roles.valueOf(Role.headless(TenantName.from("my-tenant"), ApplicationName.from("my.app")));
     }
 
     @Test(expected = IllegalArgumentException.class)
diff --git a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/role/RoleTest.java b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/role/RoleTest.java
index d153e218640..57b4af9d16c 100644
--- a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/role/RoleTest.java
+++ b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/role/RoleTest.java
@@ -2,12 +2,12 @@
 package com.yahoo.vespa.hosted.controller.api.role;
 
 import com.yahoo.config.provision.ApplicationName;
-import com.yahoo.config.provision.InstanceName;
 import com.yahoo.config.provision.SystemName;
 import com.yahoo.config.provision.TenantName;
 import org.junit.Test;
 
 import java.net.URI;
+import java.util.List;
 
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
@@ -30,6 +30,34 @@ public class RoleTest {
         assertTrue(mainEnforcer.allows(role, Action.update, URI.create("/os/v1/bar")));
         assertTrue(mainEnforcer.allows(role, Action.update, URI.create("/application/v4/tenant/t1/application/a1")));
         assertTrue(mainEnforcer.allows(role, Action.update, URI.create("/application/v4/tenant/t2/application/a2")));
+        assertTrue(mainEnforcer.allows(role, Action.read, URI.create("/routing/v1/")));
+        assertTrue(mainEnforcer.allows(role, Action.read, URI.create("/routing/v1/status/environment/")));
+        assertTrue(mainEnforcer.allows(role, Action.read, URI.create("/routing/v1/status/environment/prod")));
+        assertTrue(mainEnforcer.allows(role, Action.create, URI.create("/routing/v1/inactive/environment/prod/region/us-north-1")));
+    }
+
+    @Test
+    public void supporter_membership() {
+        Role role = Role.hostedSupporter();
+
+        // No create update or delete
+        assertFalse(mainEnforcer.allows(role, Action.create, URI.create("/not/explicitly/defined")));
+        assertFalse(mainEnforcer.allows(role, Action.create, URI.create("/controller/v1/foo")));
+        assertFalse(mainEnforcer.allows(role, Action.update, URI.create("/os/v1/bar")));
+        assertFalse(mainEnforcer.allows(role, Action.update, URI.create("/application/v4/tenant/t1/application/a1")));
+        assertFalse(mainEnforcer.allows(role, Action.update, URI.create("/application/v4/tenant/t2/application/a2")));
+        assertFalse(mainEnforcer.allows(role, Action.delete, URI.create("/application/v4/tenant/t8/application/a6/instance/i1/environment/dev/region/r1")));
+
+        // But reads is ok (but still only for valid paths)
+        assertFalse(mainEnforcer.allows(role, Action.read, URI.create("/not/explicitly/defined")));
+        assertTrue(mainEnforcer.allows(role, Action.read, URI.create("/controller/v1/foo")));
+        assertTrue(mainEnforcer.allows(role, Action.read, URI.create("/os/v1/bar")));
+        assertTrue(mainEnforcer.allows(role, Action.read, URI.create("/application/v4/tenant/t1/application/a1")));
+        assertTrue(mainEnforcer.allows(role, Action.read, URI.create("/application/v4/tenant/t2/application/a2")));
+        assertFalse(mainEnforcer.allows(role, Action.delete, URI.create("/application/v4/tenant/t8/application/a6/instance/i1/environment/dev/region/r1")));
+
+        // Check that we are allowed to create tenants in public
+        assertTrue(publicEnforcer.allows(role, Action.create, URI.create("/application/v4/tenant/t1")));
     }
 
     @Test
@@ -48,70 +76,14 @@ public class RoleTest {
 
     @Test
     public void build_service_membership() {
-        Role role = Role.tenantPipeline(TenantName.from("t1"), ApplicationName.from("a1"));
+        Role role = Role.buildService(TenantName.from("t1"), ApplicationName.from("a1"));
         assertFalse(publicEnforcer.allows(role, Action.create, URI.create("/not/explicitly/defined")));
         assertFalse(publicEnforcer.allows(role, Action.update, URI.create("/application/v4/tenant/t1/application/a1")));
-        assertTrue(publicEnforcer.allows(role, Action.create, URI.create("/application/v4/tenant/t1/application/a1/jobreport")));
+        assertTrue(publicEnforcer.allows(role, Action.create, URI.create("/application/v4/tenant/t1/application/a1/submit")));
         assertFalse("No global read access", publicEnforcer.allows(role, Action.read, URI.create("/controller/v1/foo")));
     }
 
     @Test
-    public void athenz_user_membership() {
-        Role role = Role.athenzUser(TenantName.from("t8"), ApplicationName.from("a6"), InstanceName.from("i1"));
-        assertTrue(mainEnforcer.allows(role, Action.create, URI.create("/application/v4/tenant/t8/application/a6/instance/i1/deploy/some-job")));
-        assertTrue(mainEnforcer.allows(role, Action.delete, URI.create("/application/v4/tenant/t8/application/a6/instance/i1/environment/dev/region/r1")));
-        assertFalse(mainEnforcer.allows(role, Action.delete, URI.create("/application/v4/tenant/t8/application/a6/instance/i1/environment/prod/region/r1")));
-    }
-
-    @Test
-    public void implications() {
-        TenantName tenant1 = TenantName.from("t1");
-        ApplicationName application1 = ApplicationName.from("a1");
-        TenantName tenant2 = TenantName.from("t2");
-        ApplicationName application2 = ApplicationName.from("a2");
-
-        Role tenantOwner1 = Role.tenantOwner(tenant1);
-        Role tenantAdmin1 = Role.tenantAdmin(tenant1);
-        Role tenantAdmin2 = Role.tenantAdmin(tenant2);
-        Role tenantOperator1 = Role.tenantOperator(tenant1);
-        Role applicationAdmin11 = Role.applicationAdmin(tenant1, application1);
-        Role applicationOperator11 = Role.applicationOperator(tenant1, application1);
-        Role applicationDeveloper11 = Role.applicationDeveloper(tenant1, application1);
-        Role applicationReader11 = Role.applicationReader(tenant1, application1);
-        Role applicationReader12 = Role.applicationReader(tenant1, application2);
-        Role applicationReader22 = Role.applicationReader(tenant2, application2);
-
-        assertFalse(tenantOwner1.implies(tenantOwner1));
-        assertTrue(tenantOwner1.implies(tenantAdmin1));
-        assertFalse(tenantOwner1.implies(tenantAdmin2));
-        assertTrue(tenantOwner1.implies(tenantOperator1));
-        assertTrue(tenantOwner1.implies(applicationAdmin11));
-        assertTrue(tenantOwner1.implies(applicationReader11));
-        assertTrue(tenantOwner1.implies(applicationReader12));
-        assertFalse(tenantOwner1.implies(applicationReader22));
-
-        assertFalse(tenantAdmin1.implies(tenantOwner1));
-        assertFalse(tenantAdmin1.implies(tenantAdmin2));
-        assertTrue(tenantAdmin1.implies(applicationDeveloper11));
-
-        assertFalse(tenantOperator1.implies(applicationReader11));
-
-        assertFalse(applicationAdmin11.implies(tenantAdmin1));
-        assertFalse(applicationAdmin11.implies(tenantOperator1));
-        assertTrue(applicationAdmin11.implies(applicationOperator11));
-        assertTrue(applicationAdmin11.implies(applicationDeveloper11));
-        assertTrue(applicationAdmin11.implies(applicationReader11));
-        assertFalse(applicationAdmin11.implies(applicationReader12));
-        assertFalse(applicationAdmin11.implies(applicationReader22));
-
-        assertFalse(applicationOperator11.implies(applicationDeveloper11));
-        assertTrue(applicationOperator11.implies(applicationReader11));
-
-        assertFalse(applicationDeveloper11.implies(applicationOperator11));
-        assertTrue(applicationDeveloper11.implies(applicationReader11));
-    }
-
-    @Test
     public void new_implications() {
         TenantName tenant1 = TenantName.from("t1");
         ApplicationName application1 = ApplicationName.from("a1");
@@ -133,13 +105,42 @@ public class RoleTest {
         Action action = Action.update;
         assertTrue(mainEnforcer.allows(Role.systemFlagsDeployer(), action, deployUri));
         assertTrue(mainEnforcer.allows(Role.hostedOperator(), action, deployUri));
+        assertFalse(mainEnforcer.allows(Role.hostedSupporter(), action, deployUri));
         assertFalse(mainEnforcer.allows(Role.systemFlagsDryrunner(), action, deployUri));
         assertFalse(mainEnforcer.allows(Role.everyone(), action, deployUri));
 
         URI dryrunUri = URI.create("/system-flags/v1/dryrun");
         assertTrue(mainEnforcer.allows(Role.systemFlagsDeployer(), action, dryrunUri));
         assertTrue(mainEnforcer.allows(Role.hostedOperator(), action, dryrunUri));
+        assertFalse(mainEnforcer.allows(Role.hostedSupporter(), action, dryrunUri));
         assertTrue(mainEnforcer.allows(Role.systemFlagsDryrunner(), action, dryrunUri));
         assertFalse(mainEnforcer.allows(Role.everyone(), action, dryrunUri));
     }
+
+    @Test
+    public void routing() {
+        var tenantUrl = URI.create("/routing/v1/status/tenant/t1");
+        var applicationUrl = URI.create("/routing/v1/status/tenant/t1/application/a1");
+        var instanceUrl = URI.create("/routing/v1/status/tenant/t1/application/a1/instance/i1");
+        var deploymentUrl = URI.create("/routing/v1/status/tenant/t1/application/a1/instance/i1/environment/prod/region/us-north-1");
+        // Read
+        for (var url : List.of(tenantUrl, applicationUrl, instanceUrl, deploymentUrl)) {
+            var allowedRole = Role.reader(TenantName.from("t1"));
+            var disallowedRole = Role.reader(TenantName.from("t2"));
+            assertTrue(allowedRole + " can read " + url, mainEnforcer.allows(allowedRole, Action.read, url));
+            assertFalse(disallowedRole + " cannot read " + url, mainEnforcer.allows(disallowedRole, Action.read, url));
+        }
+
+        // Write
+        {
+            var url = URI.create("/routing/v1/inactive/tenant/t1/application/a1/instance/i1/environment/prod/region/us-north-1");
+            var allowedRole = Role.developer(TenantName.from("t1"));
+            var disallowedRole = Role.developer(TenantName.from("t2"));
+            assertTrue(allowedRole + " can override status at " + url, mainEnforcer.allows(allowedRole, Action.create, url));
+            assertTrue(allowedRole + " can clear status at " + url, mainEnforcer.allows(allowedRole, Action.delete, url));
+            assertFalse(disallowedRole + " cannot override status at " + url, mainEnforcer.allows(disallowedRole, Action.create, url));
+            assertFalse(disallowedRole + " cannot clear status at " + url, mainEnforcer.allows(disallowedRole, Action.delete, url));
+        }
+    }
+
 }
diff --git a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchiveTest.java b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchiveTest.java
index 14584650d3b..35f6794e28d 100644
--- a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchiveTest.java
+++ b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/systemflags/v1/SystemFlagsDataArchiveTest.java
@@ -6,6 +6,7 @@ import com.yahoo.config.provision.Environment;
 import com.yahoo.config.provision.RegionName;
 import com.yahoo.config.provision.SystemName;
 import com.yahoo.config.provision.zone.ZoneId;
+import com.yahoo.text.JSON;
 import com.yahoo.vespa.athenz.api.AthenzService;
 import com.yahoo.vespa.flags.FetchVector;
 import com.yahoo.vespa.flags.FlagId;
@@ -13,6 +14,7 @@ import com.yahoo.vespa.flags.RawFlag;
 import com.yahoo.vespa.flags.json.FlagData;
 import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.ExpectedException;
 import org.junit.rules.TemporaryFolder;
 
 import java.io.BufferedInputStream;
@@ -27,9 +29,13 @@ import java.net.URI;
 import java.nio.file.Paths;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
 import static java.util.stream.Collectors.toList;
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
 
 /**
  * @author bjorncs
@@ -43,6 +49,9 @@ public class SystemFlagsDataArchiveTest {
     @Rule
     public final TemporaryFolder temporaryFolder = new TemporaryFolder();
 
+    @Rule
+    public final ExpectedException expectedException = ExpectedException.none();
+
     private static FlagsTarget mainControllerTarget = FlagsTarget.forController(SYSTEM);
     private static FlagsTarget prodUsWestCfgTarget = createConfigserverTarget(Environment.prod, "us-west-1");
     private static FlagsTarget prodUsEast3CfgTarget = createConfigserverTarget(Environment.prod, "us-east-3");
@@ -84,6 +93,113 @@ public class SystemFlagsDataArchiveTest {
         assertFlagDataHasValue(archive, FLAG_WITH_EMPTY_DATA, devUsEast1CfgTarget, "main");
     }
 
+    @Test
+    public void throws_exception_on_non_json_file() {
+        expectedException.expect(IllegalArgumentException.class);
+        expectedException.expectMessage("Only JSON files are allowed in 'flags/' directory (found 'flags/my-test-flag/file-name-without-dot-json')");
+        SystemFlagsDataArchive.fromDirectory(Paths.get("src/test/resources/system-flags-with-invalid-file-name/"));
+    }
+
+    @Test
+    public void throws_exception_on_unknown_file() {
+        SystemFlagsDataArchive archive = SystemFlagsDataArchive.fromDirectory(Paths.get("src/test/resources/system-flags-with-unknown-file-name/"));
+        expectedException.expect(IllegalArgumentException.class);
+        expectedException.expectMessage("Unknown flag file: flags/my-test-flag/main.prod.unknown-region.json");
+        archive.validateAllFilesAreForTargets(SystemName.main, Set.of(mainControllerTarget, prodUsWestCfgTarget));
+    }
+
+    @Test
+    public void throws_on_unknown_field() {
+        expectedException.expect(IllegalArgumentException.class);
+        expectedException.expectMessage(
+                "flags/my-test-flag/main.prod.us-west-1.json contains unknown non-comment fields: after removing any comment fields the JSON is:\n" +
+                "  {\"id\":\"my-test-flag\",\"rules\":[{\"condition\":[{\"type\":\"whitelist\",\"dimension\":\"hostname\",\"values\":[\"foo.com\"]}],\"value\":\"default\"}]}\n" +
+                "but deserializing this ended up with a JSON that are missing some of the fields:\n" +
+                "  {\"id\":\"my-test-flag\",\"rules\":[{\"value\":\"default\"}]}\n" +
+                "See https://git.ouroath.com/vespa/hosted-feature-flags for more info on the JSON syntax");
+        SystemFlagsDataArchive.fromDirectory(Paths.get("src/test/resources/system-flags-with-unknown-field-name/"));
+    }
+
+    @Test
+    public void remove_comments() {
+        assertTrue(JSON.equals("{\n" +
+                "    \"a\": {\n" +
+                "        \"b\": 1\n" +
+                "    },\n" +
+                "    \"list\": [\n" +
+                "        {\n" +
+                "            \"c\": 2\n" +
+                "        },\n" +
+                "        {\n" +
+                "        }\n" +
+                "    ]\n" +
+                "}",
+                SystemFlagsDataArchive.normalizeJson("{\n" +
+                "    \"comment\": \"comment a\",\n" +
+                "    \"a\": {\n" +
+                "        \"comment\": \"comment b\",\n" +
+                "        \"b\": 1\n" +
+                "    },\n" +
+                "    \"list\": [\n" +
+                "        {\n" +
+                "            \"comment\": \"comment c\",\n" +
+                "            \"c\": 2\n" +
+                "        },\n" +
+                "        {\n" +
+                "            \"comment\": \"comment d\"\n" +
+                "        }\n" +
+                "    ]\n" +
+                "}")));
+    }
+
+    @Test
+    public void normalize_json_fail_on_invalid_application() {
+        try {
+            SystemFlagsDataArchive.normalizeJson("{\n" +
+                    "    \"id\": \"foo\",\n" +
+                    "    \"rules\": [\n" +
+                    "        {\n" +
+                    "            \"conditions\": [\n" +
+                    "                {\n" +
+                    "                    \"type\": \"whitelist\",\n" +
+                    "                    \"dimension\": \"application\",\n" +
+                    "                    \"values\": [ \"a.b.c\" ]\n" +
+                    "                }\n" +
+                    "            ],\n" +
+                    "            \"value\": true\n" +
+                    "        }\n" +
+                    "    ]\n" +
+                    "}\n");
+            fail();
+        } catch (IllegalArgumentException e) {
+            assertEquals("Application ids must be on the form tenant:application:instance, but was a.b.c", e.getMessage());
+        }
+    }
+
+    @Test
+    public void normalize_json_fail_on_invalid_node_type() {
+        try {
+            SystemFlagsDataArchive.normalizeJson("{\n" +
+                    "    \"id\": \"foo\",\n" +
+                    "    \"rules\": [\n" +
+                    "        {\n" +
+                    "            \"conditions\": [\n" +
+                    "                {\n" +
+                    "                    \"type\": \"whitelist\",\n" +
+                    "                    \"dimension\": \"node-type\",\n" +
+                    "                    \"values\": [ \"footype\" ]\n" +
+                    "                }\n" +
+                    "            ],\n" +
+                    "            \"value\": true\n" +
+                    "        }\n" +
+                    "    ]\n" +
+                    "}\n");
+            fail();
+        } catch (IllegalArgumentException e) {
+            assertEquals("No enum constant com.yahoo.config.provision.NodeType.footype", e.getMessage());
+        }
+    }
+
     private static void assertArchiveReturnsCorrectTestFlagDataForTarget(SystemFlagsDataArchive archive) {
         assertFlagDataHasValue(archive, MY_TEST_FLAG, mainControllerTarget, "main.controller");
         assertFlagDataHasValue(archive, MY_TEST_FLAG, prodUsWestCfgTarget, "main.prod.us-west-1");
diff --git a/controller-api/src/test/resources/system-flags-with-invalid-file-name/flags/my-test-flag/file-name-without-dot-json b/controller-api/src/test/resources/system-flags-with-invalid-file-name/flags/my-test-flag/file-name-without-dot-json
new file mode 100644
index 00000000000..5924eb860c0
--- /dev/null
+++ b/controller-api/src/test/resources/system-flags-with-invalid-file-name/flags/my-test-flag/file-name-without-dot-json
@@ -0,0 +1,8 @@
+{
+  "id" : "my-test-flag",
+  "rules" : [
+    {
+      "value" : "default"
+    }
+  ]
+}
\ No newline at end of file
diff --git a/controller-api/src/test/resources/system-flags-with-unknown-field-name/flags/my-test-flag/main.prod.us-west-1.json b/controller-api/src/test/resources/system-flags-with-unknown-field-name/flags/my-test-flag/main.prod.us-west-1.json
new file mode 100644
index 00000000000..c41083fc7ab
--- /dev/null
+++ b/controller-api/src/test/resources/system-flags-with-unknown-field-name/flags/my-test-flag/main.prod.us-west-1.json
@@ -0,0 +1,15 @@
+{
+    "id" : "my-test-flag",
+    "rules" : [
+        {
+            "condition": [
+                {
+                    "type": "whitelist",
+                    "dimension": "hostname",
+                    "values": ["foo.com"]
+                }
+            ],
+            "value" : "default"
+        }
+    ]
+}
diff --git a/controller-api/src/test/resources/system-flags-with-unknown-file-name/flags/my-test-flag/main.prod.unknown-region.json b/controller-api/src/test/resources/system-flags-with-unknown-file-name/flags/my-test-flag/main.prod.unknown-region.json
new file mode 100644
index 00000000000..5924eb860c0
--- /dev/null
+++ b/controller-api/src/test/resources/system-flags-with-unknown-file-name/flags/my-test-flag/main.prod.unknown-region.json
@@ -0,0 +1,8 @@
+{
+  "id" : "my-test-flag",
+  "rules" : [
+    {
+      "value" : "default"
+    }
+  ]
+}
\ No newline at end of file
diff --git a/controller-api/src/test/resources/system-flags/flags/flag-with-empty-data/main.json b/controller-api/src/test/resources/system-flags/flags/flag-with-empty-data/main.json
index cef75be02b7..29160dc0081 100644
--- a/controller-api/src/test/resources/system-flags/flags/flag-with-empty-data/main.json
+++ b/controller-api/src/test/resources/system-flags/flags/flag-with-empty-data/main.json
@@ -1,7 +1,9 @@
 {
+  "comment": "comment a",
   "id" : "flag-with-empty-data",
   "rules" : [
     {
+      "comment": "comment b",
       "value" : "main"
     }
   ]
diff --git a/controller-server/pom.xml b/controller-server/pom.xml
index 0c545020449..158c1b5cd39 100644
--- a/controller-server/pom.xml
+++ b/controller-server/pom.xml
@@ -144,6 +144,13 @@
             <version>${project.version}</version>
         </dependency>
 
+        <dependency>
+            <groupId>com.yahoo.vespa</groupId>
+            <artifactId>http-utils</artifactId>
+            <version>${project.version}</version>
+            <scope>compile</scope>
+        </dependency>
+
         <!-- test -->
 
         <dependency>
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java
index e37d6accd89..0955a5388ce 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java
@@ -1,7 +1,6 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller;
 
-import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSortedMap;
 import com.yahoo.component.Version;
 import com.yahoo.config.application.api.DeploymentSpec;
@@ -11,7 +10,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationV
 import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId;
 import com.yahoo.vespa.hosted.controller.api.integration.organization.User;
 import com.yahoo.vespa.hosted.controller.application.ApplicationActivity;
-import com.yahoo.vespa.hosted.controller.application.Change;
 import com.yahoo.vespa.hosted.controller.application.Deployment;
 import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId;
 import com.yahoo.vespa.hosted.controller.metric.ApplicationMetrics;
@@ -32,7 +30,7 @@ import java.util.function.Function;
 import java.util.stream.Collectors;
 
 /**
- * An application. Belongs to a {@link Tenant}, and may have many {@link Instance}s.
+ * An application. Belongs to a {@link Tenant}, and may have multiple {@link Instance}s.
  *
  * This is immutable.
  *
@@ -107,6 +105,12 @@ public class Application {
     /** Returns the instances of this application */
     public Map<InstanceName, Instance> instances() { return instances; }
 
+    /** Returns the instances of this application which are defined in its deployment spec. */
+    public Map<InstanceName, Instance> productionInstances() {
+        return deploymentSpec.instanceNames().stream()
+                .collect(Collectors.toUnmodifiableMap(Function.identity(), instances::get));
+    }
+
     /** Returns the instance with the given name, if it exists. */
     public Optional<Instance> get(InstanceName instance) { return Optional.ofNullable(instances.get(instance)); }
 
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
index f7ae98befe4..e02c1b9f5dd 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
@@ -1,33 +1,35 @@
 // Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller;
 
-import com.google.common.collect.ImmutableList;
 import com.yahoo.component.Version;
-import com.yahoo.config.application.api.DeploymentInstanceSpec;
 import com.yahoo.config.application.api.DeploymentSpec;
 import com.yahoo.config.application.api.ValidationId;
 import com.yahoo.config.application.api.ValidationOverrides;
 import com.yahoo.config.provision.ApplicationId;
-import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.config.provision.DockerImage;
 import com.yahoo.config.provision.Environment;
 import com.yahoo.config.provision.InstanceName;
 import com.yahoo.config.provision.TenantName;
 import com.yahoo.config.provision.zone.ZoneId;
+import com.yahoo.container.jdisc.secretstore.SecretStore;
 import com.yahoo.vespa.athenz.api.AthenzDomain;
 import com.yahoo.vespa.athenz.api.AthenzIdentity;
 import com.yahoo.vespa.athenz.api.AthenzPrincipal;
 import com.yahoo.vespa.athenz.api.AthenzService;
 import com.yahoo.vespa.athenz.api.AthenzUser;
 import com.yahoo.vespa.curator.Lock;
+import com.yahoo.vespa.flags.FetchVector;
+import com.yahoo.vespa.flags.FlagSource;
+import com.yahoo.vespa.flags.Flags;
+import com.yahoo.vespa.flags.StringFlag;
 import com.yahoo.vespa.hosted.controller.api.ActivateResult;
 import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions;
-import com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus;
+import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeploymentData;
 import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings.ConfigChangeActions;
 import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
 import com.yahoo.vespa.hosted.controller.api.identifiers.Hostname;
 import com.yahoo.vespa.hosted.controller.api.identifiers.InstanceId;
 import com.yahoo.vespa.hosted.controller.api.identifiers.RevisionId;
-import com.yahoo.vespa.hosted.controller.api.integration.certificates.ApplicationCertificate;
 import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMetadata;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServer;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException;
@@ -39,19 +41,13 @@ import com.yahoo.vespa.hosted.controller.api.integration.configserver.PrepareRes
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationStore;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.ArtifactRepository;
+import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterId;
-import com.yahoo.vespa.hosted.controller.api.integration.dns.Record;
-import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData;
-import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName;
-import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingEndpoint;
-import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingGenerator;
 import com.yahoo.vespa.hosted.controller.application.ApplicationPackage;
 import com.yahoo.vespa.hosted.controller.application.ApplicationPackageValidator;
 import com.yahoo.vespa.hosted.controller.application.Deployment;
 import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics;
-import com.yahoo.vespa.hosted.controller.application.Endpoint;
-import com.yahoo.vespa.hosted.controller.application.EndpointId;
 import com.yahoo.vespa.hosted.controller.application.SystemApplication;
 import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId;
 import com.yahoo.vespa.hosted.controller.athenz.impl.AthenzFacade;
@@ -59,32 +55,23 @@ import com.yahoo.vespa.hosted.controller.concurrent.Once;
 import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger;
 import com.yahoo.vespa.hosted.controller.deployment.Run;
 import com.yahoo.vespa.hosted.controller.deployment.Versions;
-import com.yahoo.vespa.hosted.controller.dns.NameServiceQueue.Priority;
-import com.yahoo.vespa.hosted.controller.persistence.EndpointCertificateMetadataSerializer;
-import com.yahoo.vespa.hosted.controller.routing.RoutingPolicies;
+import com.yahoo.vespa.hosted.controller.certificate.EndpointCertificateManager;
 import com.yahoo.vespa.hosted.controller.persistence.CuratorDb;
-import com.yahoo.vespa.hosted.controller.rotation.RotationLock;
-import com.yahoo.vespa.hosted.controller.rotation.RotationRepository;
 import com.yahoo.vespa.hosted.controller.security.AccessControl;
 import com.yahoo.vespa.hosted.controller.security.Credentials;
 import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant;
 import com.yahoo.vespa.hosted.controller.tenant.Tenant;
 import com.yahoo.vespa.hosted.controller.versions.VespaVersion;
-import com.yahoo.vespa.hosted.rotation.config.RotationsConfig;
 
-import java.net.URI;
 import java.security.Principal;
 import java.time.Clock;
 import java.time.Duration;
 import java.time.Instant;
-import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.EnumSet;
 import java.util.HashMap;
-import java.util.HashSet;
-import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -94,8 +81,6 @@ import java.util.TreeMap;
 import java.util.function.Consumer;
 import java.util.logging.Level;
 import java.util.logging.Logger;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
 
 import static com.yahoo.vespa.hosted.controller.api.integration.configserver.Node.State.active;
 import static com.yahoo.vespa.hosted.controller.api.integration.configserver.Node.State.reserved;
@@ -121,44 +106,40 @@ public class ApplicationController {
 
     private final ArtifactRepository artifactRepository;
     private final ApplicationStore applicationStore;
-    private final RotationRepository rotationRepository;
     private final AccessControl accessControl;
     private final ConfigServer configServer;
-    private final RoutingGenerator routingGenerator;
-    private final RoutingPolicies routingPolicies;
     private final Clock clock;
     private final DeploymentTrigger deploymentTrigger;
     private final ApplicationPackageValidator applicationPackageValidator;
+    private final EndpointCertificateManager endpointCertificateManager;
+    private final StringFlag dockerImageRepoFlag;
+
+    ApplicationController(Controller controller, CuratorDb curator, AccessControl accessControl, Clock clock,
+                          SecretStore secretStore, FlagSource flagSource) {
 
-    ApplicationController(Controller controller, CuratorDb curator,
-                          AccessControl accessControl, RotationsConfig rotationsConfig,
-                          Clock clock) {
         this.controller = controller;
         this.curator = curator;
         this.accessControl = accessControl;
         this.configServer = controller.serviceRegistry().configServer();
-        this.routingGenerator = controller.serviceRegistry().routingGenerator();
         this.clock = clock;
         this.artifactRepository = controller.serviceRegistry().artifactRepository();
         this.applicationStore = controller.serviceRegistry().applicationStore();
+        this.dockerImageRepoFlag = Flags.DOCKER_IMAGE_REPO.bindTo(flagSource);
 
-        routingPolicies = new RoutingPolicies(controller);
-        rotationRepository = new RotationRepository(rotationsConfig, this, curator);
         deploymentTrigger = new DeploymentTrigger(controller, clock);
         applicationPackageValidator = new ApplicationPackageValidator(controller);
+        endpointCertificateManager = new EndpointCertificateManager(controller.zoneRegistry(), curator, secretStore,
+                controller.serviceRegistry().endpointCertificateProvider(), clock, flagSource);
 
         // Update serialization format of all applications
         Once.after(Duration.ofMinutes(1), () -> {
             Instant start = clock.instant();
             int count = 0;
-            for (TenantAndApplicationId id: curator.readApplicationIds()) {
+            for (TenantAndApplicationId id : curator.readApplicationIds()) {
                 lockApplicationIfPresent(id, application -> {
-                    if (id.tenant().value().startsWith("by-"))
-                        application = application.with(DeploymentSpec.empty);
-                    else
-                        for (InstanceName instance : application.get().deploymentSpec().instanceNames())
-                            if ( ! application.get().instances().containsKey(instance))
-                                application = withNewInstance(application, id.instance(instance));
+                    for (InstanceName instance : application.get().deploymentSpec().instanceNames())
+                        if (!application.get().instances().containsKey(instance))
+                            application = withNewInstance(application, id.instance(instance));
                     store(application);
                 });
                 count++;
@@ -166,8 +147,6 @@ public class ApplicationController {
             log.log(Level.INFO, String.format("Wrote %d applications in %s", count,
                                               Duration.between(start, clock.instant())));
         });
-
-        // TODO jonmv: Do the above for applications as well when they split writes.
     }
 
     /** Returns the application with the given id, or null if it is not present */
@@ -175,16 +154,10 @@ public class ApplicationController {
         return curator.readApplication(id);
     }
 
-    /** Returns the application with the given id, or null if it is not present */
-    // TODO jonmv: remove
-    public Optional<Application> getApplication(ApplicationId id) {
-        return getApplication(TenantAndApplicationId.from(id));
-    }
-
     /** Returns the instance with the given id, or null if it is not present */
     // TODO jonmv: remove or inline
     public Optional<Instance> getInstance(ApplicationId id) {
-        return getApplication(id).flatMap(application -> application.get(id.instance()));
+        return getApplication(TenantAndApplicationId.from(id)).flatMap(application -> application.get(id.instance()));
     }
 
     /**
@@ -208,7 +181,18 @@ public class ApplicationController {
 
     /** Returns a snapshot of all applications */
     public List<Application> asList() {
-        return curator.readApplications();
+        return curator.readApplications(false);
+    }
+
+    /**
+     * Returns a snapshot of all readable applications. Unlike {@link ApplicationController#asList()} this ignores
+     * applications that cannot currently be read (e.g. due to serialization issues) and may return an incomplete
+     * snapshot.
+     *
+     * This should only be used in cases where acting on a subset of applications is better than none.
+     */
+    public List<Application> readable() {
+        return curator.readApplications(true);
     }
 
     /** Returns the ID of all known applications. */
@@ -229,7 +213,7 @@ public class ApplicationController {
     public Map<ZoneId, List<String>> contentClustersByZone(Collection<DeploymentId> ids) {
         Map<ZoneId, List<String>> clusters = new TreeMap<>(Comparator.comparing(ZoneId::value));
         for (DeploymentId id : ids)
-            clusters.put(id.zoneId(), ImmutableList.copyOf(configServer.getContentClusters(id)));
+            clusters.put(id.zoneId(), List.copyOf(configServer.getContentClusters(id)));
         return Collections.unmodifiableMap(clusters);
     }
 
@@ -247,42 +231,12 @@ public class ApplicationController {
                                      .orElse(controller.systemVersion());
     }
 
-    /** Change status of all global endpoints for given deployment */
-    public void setGlobalRotationStatus(DeploymentId deployment, EndpointStatus status) {
-        var globalEndpoints = findGlobalEndpoints(deployment);
-        if (globalEndpoints.isEmpty()) throw new IllegalArgumentException(deployment + " has no global endpoints");
-        globalEndpoints.forEach(endpoint -> {
-            try {
-                configServer.setGlobalRotationStatus(deployment, endpoint.upstreamName(), status);
-            } catch (Exception e) {
-                throw new RuntimeException("Failed to set rotation status of " + endpoint + " in " + deployment, e);
-            }
-        });
-    }
-
-    /** Get global endpoint status for given deployment */
-    public Map<RoutingEndpoint, EndpointStatus> globalRotationStatus(DeploymentId deployment) {
-        var routingEndpoints = new LinkedHashMap<RoutingEndpoint, EndpointStatus>();
-        findGlobalEndpoints(deployment).forEach(endpoint -> {
-            var status = configServer.getGlobalRotationStatus(deployment, endpoint.upstreamName());
-            routingEndpoints.put(endpoint, status);
-        });
-        return Collections.unmodifiableMap(routingEndpoints);
-    }
-
-    /** Find the global endpoints of given deployment */
-    private List<RoutingEndpoint> findGlobalEndpoints(DeploymentId deployment) {
-        return routingGenerator.endpoints(deployment).stream()
-                               .filter(RoutingEndpoint::isGlobal)
-                               .collect(Collectors.toUnmodifiableList());
-    }
-
     /**
      * Creates a new application for an existing tenant.
      *
      * @throws IllegalArgumentException if the application already exists
      */
-    public Application createApplication(TenantAndApplicationId id, Optional<Credentials> credentials) {
+    public Application createApplication(TenantAndApplicationId id, Credentials credentials) {
         try (Lock lock = lock(id)) {
             if (getApplication(id).isPresent())
                 throw new IllegalArgumentException("Could not create '" + id + "': Application already exists");
@@ -291,14 +245,9 @@ public class ApplicationController {
 
             com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId.validate(id.application().value());
 
-            Optional<Tenant> tenant = controller.tenants().get(id.tenant());
-            if (tenant.isEmpty())
+            if (controller.tenants().get(id.tenant()).isEmpty())
                 throw new IllegalArgumentException("Could not create '" + id + "': This tenant does not exist");
-            if (tenant.get().type() != Tenant.Type.user) {
-                if (credentials.isEmpty())
-                    throw new IllegalArgumentException("Could not create '" + id + "': No credentials provided");
-                accessControl.createApplication(id, credentials.get());
-            }
+            accessControl.createApplication(id, credentials);
 
             LockedApplication locked = new LockedApplication(new Application(id, clock.instant()), lock);
             store(locked);
@@ -338,9 +287,65 @@ public class ApplicationController {
         return deploy(applicationId, zone, applicationPackageFromDeployer, Optional.empty(), options);
     }
 
-    /** Deploys an application. If the application does not exist it is created. */
-    // TODO: Get rid of the options arg
-    // TODO jonmv: Split this, and choose between deployDirectly and deploy in handler, excluding internally built from the latter.
+    /** Deploys an application package for an existing application instance. */
+    public ActivateResult deploy2(JobId job, boolean deploySourceVersions) { // TODO jonmv: make it number one!
+        if (job.application().instance().isTester())
+            throw new IllegalArgumentException("'" + job.application() + "' is a tester application!");
+
+        TenantAndApplicationId applicationId = TenantAndApplicationId.from(job.application());
+        ZoneId zone = job.type().zone(controller.system());
+
+        try (Lock deploymentLock = lockForDeployment(job.application(), zone)) {
+            Set<ContainerEndpoint> endpoints;
+            Optional<EndpointCertificateMetadata> endpointCertificateMetadata;
+
+            Run run = controller.jobController().last(job)
+                                .orElseThrow(() -> new IllegalStateException("No known run of '" + job + "'"));
+
+            if (run.hasEnded())
+                throw new IllegalStateException("No deployment expected for " + job + " now, as no job is running");
+
+            Version platform = run.versions().sourcePlatform().filter(__ -> deploySourceVersions).orElse(run.versions().targetPlatform());
+            ApplicationVersion revision = run.versions().sourceApplication().filter(__ -> deploySourceVersions).orElse(run.versions().targetApplication());
+            ApplicationPackage applicationPackage = getApplicationPackage(job.application(), zone, revision);
+
+            try (Lock lock = lock(applicationId)) {
+                LockedApplication application = new LockedApplication(requireApplication(applicationId), lock);
+                Instance instance = application.get().require(job.application().instance());
+
+                Deployment deployment = instance.deployments().get(zone);
+                if (   zone.environment().isProduction() && deployment != null
+                    && (   platform.compareTo(deployment.version()) < 0 && ! instance.change().isPinned()
+                        || revision.compareTo(deployment.applicationVersion()) < 0 && ! (revision.isUnknown() && controller.system().isCd())))
+                    throw new IllegalArgumentException(String.format("Rejecting deployment of application %s to %s, as the requested versions (platform: %s, application: %s)" +
+                                                                     " are older than the currently deployed (platform: %s, application: %s).",
+                                                                     job.application(), zone, platform, revision, deployment.version(), deployment.applicationVersion()));
+
+                if (   ! applicationPackage.trustedCertificates().isEmpty()
+                    &&   run.testerCertificate().isPresent())
+                    applicationPackage = applicationPackage.withTrustedCertificate(run.testerCertificate().get());
+
+                endpointCertificateMetadata = endpointCertificateManager.getEndpointCertificateMetadata(instance, zone);
+
+                endpoints = controller.routing().registerEndpointsInDns(application.get(), job.application().instance(), zone);
+            } // Release application lock while doing the deployment, which is a lengthy task.
+
+            // Carry out deployment without holding the application lock.
+            ActivateResult result = deploy(job.application(), applicationPackage, zone, platform, endpoints, endpointCertificateMetadata);
+
+            lockApplicationOrThrow(applicationId, application ->
+                    store(application.with(job.application().instance(),
+                                           instance -> instance.withNewDeployment(zone, revision, platform,
+                                                                                  clock.instant(), warningsFrom(result)))));
+            return result;
+        }
+    }
+
+    private ApplicationPackage getApplicationPackage(ApplicationId application, ZoneId zone, ApplicationVersion revision) {
+        return new ApplicationPackage(revision.isUnknown() ? applicationStore.getDev(application, zone)
+                                                           : applicationStore.get(application.tenant(), application.application(), revision));
+    }
+
     public ActivateResult deploy(ApplicationId instanceId, ZoneId zone,
                                  Optional<ApplicationPackage> applicationPackageFromDeployer,
                                  Optional<ApplicationVersion> applicationVersionFromDeployer,
@@ -349,10 +354,6 @@ public class ApplicationController {
             throw new IllegalArgumentException("'" + instanceId + "' is a tester application!");
 
         TenantAndApplicationId applicationId = TenantAndApplicationId.from(instanceId);
-        if (   getApplication(applicationId).isEmpty()
-            && controller.tenants().require(instanceId.tenant()).type() == Tenant.Type.user)
-            createApplication(applicationId, Optional.empty());
-
         if (getInstance(instanceId).isEmpty())
             createInstance(instanceId);
 
@@ -397,20 +398,14 @@ public class ApplicationController {
                     validateRun(application.get().require(instance), zone, platformVersion, applicationVersion);
                 }
 
-                if (controller.zoneRegistry().zones().directlyRouted().ids().contains(zone)) {
-                    // Provisions a new certificate if missing
-                    endpointCertificateMetadata = getEndpointCertificate(application.get().require(instance));
-                } else {
-                    endpointCertificateMetadata = Optional.empty();
-                }
+                endpointCertificateMetadata = endpointCertificateManager.getEndpointCertificateMetadata(application.get().require(instance), zone);
 
-                endpoints = registerEndpointsInDns(applicationPackage.deploymentSpec(), application.get().require(instanceId.instance()), zone);
+                endpoints = controller.routing().registerEndpointsInDns(application.get(), instance, zone);
             } // Release application lock while doing the deployment, which is a lengthy task.
 
             // Carry out deployment without holding the application lock.
-            options = withVersion(platformVersion, options);
-            ActivateResult result = deploy(instanceId, applicationPackage, zone, options, endpoints,
-                                           endpointCertificateMetadata);
+            ActivateResult result = deploy(instanceId, applicationPackage, zone, platformVersion,
+                                           endpoints, endpointCertificateMetadata);
 
             lockApplicationOrThrow(applicationId, application ->
                     store(application.with(instanceId.instance(),
@@ -460,7 +455,7 @@ public class ApplicationController {
 
         for (InstanceName instance : declaredInstances)
             if (applicationPackage.deploymentSpec().requireInstance(instance).concerns(Environment.prod))
-                application = withRotation(applicationPackage.deploymentSpec(), application, instance);
+                application = controller.routing().assignRotations(application, instance);
 
         store(application);
         return application;
@@ -482,126 +477,45 @@ public class ApplicationController {
             ApplicationPackage applicationPackage = new ApplicationPackage(
                     artifactRepository.getSystemApplicationPackage(application.id(), zone, version)
             );
-            DeployOptions options = withVersion(version, DeployOptions.none());
-            return deploy(application.id(), applicationPackage, zone, options, Set.of(), /* No application cert */ Optional.empty());
+            return deploy(application.id(), applicationPackage, zone, version, Set.of(), /* No application cert */ Optional.empty());
         } else {
            throw new RuntimeException("This system application does not have an application package: " + application.id().toShortString());
         }
     }
 
     /** Deploys the given tester application to the given zone. */
-    public ActivateResult deployTester(TesterId tester, ApplicationPackage applicationPackage, ZoneId zone, DeployOptions options) {
-        return deploy(tester.id(), applicationPackage, zone, options, Set.of(), /* No application cert for tester*/ Optional.empty());
+    public ActivateResult deployTester(TesterId tester, ApplicationPackage applicationPackage, ZoneId zone, Version platform) {
+        return deploy(tester.id(), applicationPackage, zone, platform, Set.of(), /* No application cert for tester*/ Optional.empty());
     }
 
     private ActivateResult deploy(ApplicationId application, ApplicationPackage applicationPackage,
-                                  ZoneId zone, DeployOptions deployOptions, Set<ContainerEndpoint> endpoints,
+                                  ZoneId zone, Version platform, Set<ContainerEndpoint> endpoints,
                                   Optional<EndpointCertificateMetadata> endpointCertificateMetadata) {
-        DeploymentId deploymentId = new DeploymentId(application, zone);
         try {
+            Optional<DockerImage> dockerImageRepo = Optional.ofNullable(
+                    dockerImageRepoFlag
+                            .with(FetchVector.Dimension.ZONE_ID, zone.value())
+                            .with(FetchVector.Dimension.APPLICATION_ID, application.serializedForm())
+                            .value())
+                    .filter(s -> !s.isBlank())
+                    .map(DockerImage::fromString);
+
+            Optional<AthenzDomain> domain = controller.tenants().get(application.tenant())
+                    .filter(tenant-> tenant instanceof AthenzTenant)
+                    .map(tenant -> ((AthenzTenant)tenant).domain());
+
             ConfigServer.PreparedApplication preparedApplication =
-                    configServer.deploy(deploymentId, deployOptions, endpoints, endpointCertificateMetadata, applicationPackage.zippedContent());
+                    configServer.deploy(new DeploymentData(application, zone, applicationPackage.zippedContent(), platform,
+                                                           endpoints, endpointCertificateMetadata, dockerImageRepo, domain));
             return new ActivateResult(new RevisionId(applicationPackage.hash()), preparedApplication.prepareResponse(),
                                       applicationPackage.zippedContent().length);
         } finally {
             // Even if prepare fails, a load balancer may have been provisioned. Always refresh routing policies so that
             // any DNS updates can be propagated as early as possible.
-            routingPolicies.refresh(application, applicationPackage.deploymentSpec(), zone);
+            controller.routing().policies().refresh(application, applicationPackage.deploymentSpec(), zone);
         }
     }
 
-    /** Makes sure the application has a global rotation, if eligible. */
-    private LockedApplication withRotation(DeploymentSpec deploymentSpec, LockedApplication application, InstanceName instanceName) {
-        try (RotationLock rotationLock = rotationRepository.lock()) {
-            var rotations = rotationRepository.getOrAssignRotations(deploymentSpec,
-                                                                    application.get().require(instanceName),
-                                                                    rotationLock);
-            application = application.with(instanceName, instance -> instance.with(rotations));
-            store(application); // store assigned rotation even if deployment fails
-        }
-        return application;
-    }
-
-    /**
-     * Register endpoints for rotations assigned to given application and zone in DNS.
-     *
-     * @return the registered endpoints
-     */
-    private Set<ContainerEndpoint> registerEndpointsInDns(DeploymentSpec deploymentSpec, Instance instance, ZoneId zone) {
-        var containerEndpoints = new HashSet<ContainerEndpoint>();
-        boolean registerLegacyNames = deploymentSpec.instance(instance.name())
-                                                    .flatMap(DeploymentInstanceSpec::globalServiceId)
-                                                    .isPresent();
-        for (var assignedRotation : instance.rotations()) {
-            var names = new ArrayList<String>();
-            var endpoints = instance.endpointsIn(controller.system(), assignedRotation.endpointId())
-                                    .scope(Endpoint.Scope.global);
-
-            // Skip rotations which do not apply to this zone. Legacy names always point to all zones
-            if (!registerLegacyNames && !assignedRotation.regions().contains(zone.region())) {
-                continue;
-            }
-
-            // Omit legacy DNS names when assigning rotations using <endpoints/> syntax
-            if (!registerLegacyNames) {
-                endpoints = endpoints.legacy(false);
-            }
-
-            // Register names in DNS
-            var rotation = rotationRepository.getRotation(assignedRotation.rotationId());
-            if (rotation.isPresent()) {
-                endpoints.asList().forEach(endpoint -> {
-                    controller.nameServiceForwarder().createCname(RecordName.from(endpoint.dnsName()),
-                                                                  RecordData.fqdn(rotation.get().name()),
-                                                                  Priority.normal);
-                    names.add(endpoint.dnsName());
-                });
-            }
-
-            // Include rotation ID as a valid name of this container endpoint (required by global routing health checks)
-            names.add(assignedRotation.rotationId().asString());
-            containerEndpoints.add(new ContainerEndpoint(assignedRotation.clusterId().value(), names));
-        }
-        return Collections.unmodifiableSet(containerEndpoints);
-    }
-
-    private Optional<EndpointCertificateMetadata> getEndpointCertificate(Instance instance) {
-        // Re-use certificate if already provisioned
-        Optional<EndpointCertificateMetadata> endpointCertificateMetadata = curator.readEndpointCertificateMetadata(instance.id());
-        if(endpointCertificateMetadata.isPresent())
-            return endpointCertificateMetadata;
-
-        ApplicationCertificate newCertificate = controller.serviceRegistry().applicationCertificateProvider().requestCaSignedCertificate(instance.id(), dnsNamesOf(instance.id()));
-        curator.writeApplicationCertificate(instance.id(), newCertificate);
-
-        return Optional.of(EndpointCertificateMetadataSerializer.fromTlsSecretsKeysString(newCertificate.secretsKeyNamePrefix()));
-    }
-
-    /** Returns all valid DNS names of given application */
-    private List<String> dnsNamesOf(ApplicationId applicationId) {
-        List<String> endpointDnsNames = new ArrayList<>();
-
-        // We add first an endpoint name based on a hash of the applicationId,
-        // as the certificate provider requires the first CN to be < 64 characters long.
-        endpointDnsNames.add(Endpoint.createHashedCn(applicationId, controller.system()));
-
-        var globalDefaultEndpoint = Endpoint.of(applicationId).named(EndpointId.defaultId());
-        var rotationEndpoints = Endpoint.of(applicationId).wildcard();
-
-        var zoneLocalEndpoints = controller.zoneRegistry().zones().directlyRouted().zones().stream().flatMap(zone -> Stream.of(
-                Endpoint.of(applicationId).target(ClusterSpec.Id.from("default"), zone.getId()),
-                Endpoint.of(applicationId).wildcard(zone.getId())
-        ));
-
-        Stream.concat(Stream.of(globalDefaultEndpoint, rotationEndpoints), zoneLocalEndpoints)
-              .map(Endpoint.EndpointBuilder::directRouting)
-              .map(endpoint -> endpoint.on(Endpoint.Port.tls()))
-              .map(endpointBuilder -> endpointBuilder.in(controller.system()))
-              .map(Endpoint::dnsName).forEach(endpointDnsNames::add);
-
-        return Collections.unmodifiableList(endpointDnsNames);
-    }
-
     private ActivateResult unexpectedDeployment(ApplicationId application, ZoneId zone) {
         Log logEntry = new Log();
         logEntry.level = "WARNING";
@@ -653,69 +567,12 @@ public class ApplicationController {
                                  options.deployCurrentVersion);
     }
 
-    /** Returns the endpoints of the deployment, or empty if the request fails */
-    public List<URI> getDeploymentEndpoints(DeploymentId deploymentId) {
-        if ( ! getInstance(deploymentId.applicationId())
-                .map(application -> application.deployments().containsKey(deploymentId.zoneId()))
-                .orElse(deploymentId.applicationId().instance().isTester()))
-            throw new NotExistsException("Deployment", deploymentId.toString());
-
-        try {
-            return ImmutableList.copyOf(routingGenerator.endpoints(deploymentId).stream()
-                                                        .map(RoutingEndpoint::endpoint)
-                                                        .map(URI::create)
-                                                        .iterator());
-        }
-        catch (RuntimeException e) {
-            log.log(Level.WARNING, "Failed to get endpoint information for " + deploymentId, e);
-            return Collections.emptyList();
-        }
-    }
-
-    /** Returns the non-empty endpoints per cluster in the given deployment, or empty if endpoints can't be found. */
-    public Map<ClusterSpec.Id, URI> clusterEndpoints(DeploymentId id) {
-        if ( ! getInstance(id.applicationId())
-                .map(application -> application.deployments().containsKey(id.zoneId()))
-                .orElse(id.applicationId().instance().isTester()))
-            throw new NotExistsException("Deployment", id.toString());
-
-        // TODO(jvenstad): Swap to use routingPolicies first, when this is ready.
-        try {
-            var endpoints = routingGenerator.clusterEndpoints(id);
-            if ( ! endpoints.isEmpty())
-                return endpoints;
-        }
-        catch (RuntimeException e) {
-            log.log(Level.WARNING, "Failed to get endpoint information for " + id, e);
-        }
-        return routingPolicies.get(id).values().stream()
-                              .filter(policy -> policy.endpointIn(controller.system()).scope() == Endpoint.Scope.zone)
-                              .collect(Collectors.toUnmodifiableMap(policy -> policy.id().cluster(),
-                                                                    policy -> policy.endpointIn(controller.system()).url()));
-    }
-
-    /** Returns all zone-specific cluster endpoints for the given application, in the given zones. */
-    public Map<ZoneId, Map<ClusterSpec.Id, URI>> clusterEndpoints(Collection<DeploymentId> ids) {
-        Map<ZoneId, Map<ClusterSpec.Id, URI>> deployments = new TreeMap<>(Comparator.comparing(ZoneId::value));
-        for (DeploymentId id : ids) {
-            var endpoints = clusterEndpoints(id);
-            if ( ! endpoints.isEmpty()) {
-                deployments.put(id.zoneId(), endpoints);
-            }
-        }
-        return Collections.unmodifiableMap(deployments);
-    }
-
     /**
      * Deletes the the given application. All known instances of the applications will be deleted.
      *
      * @throws IllegalArgumentException if the application has deployments or the caller is not authorized
      */
-    public void deleteApplication(TenantAndApplicationId id, Optional<Credentials> credentials) {
-        Tenant tenant = controller.tenants().require(id.tenant());
-        if (tenant.type() != Tenant.Type.user && credentials.isEmpty())
-            throw new IllegalArgumentException("Could not delete application '" + id + "': No credentials provided");
-
+    public void deleteApplication(TenantAndApplicationId id, Credentials credentials) {
         lockApplicationOrThrow(id, application -> {
             var deployments = application.get().instances().values().stream()
                                          .filter(instance -> ! instance.deployments().isEmpty())
@@ -727,15 +584,14 @@ public class ApplicationController {
                 throw new IllegalArgumentException("Could not delete '" + application + "': It has active deployments: " + deployments);
 
             for (Instance instance : application.get().instances().values()) {
-                removeEndpoints(instance);
+                controller.routing().removeEndpointsInDns(application.get(), instance.name());
                 application = application.without(instance.name());
             }
 
             applicationStore.removeAll(id.tenant(), id.application());
             applicationStore.removeAllTesters(id.tenant(), id.application());
 
-            if (tenant.type() != Tenant.Type.user)
-                accessControl.deleteApplication(id, credentials.get());
+            accessControl.deleteApplication(id, credentials);
             curator.removeApplication(id);
 
             controller.jobController().collectGarbage();
@@ -763,24 +619,13 @@ public class ApplicationController {
                 &&   application.get().deploymentSpec().instanceNames().contains(instanceId.instance()))
                 throw new IllegalArgumentException("Can not delete '" + instanceId + "', which is specified in 'deployment.xml'; remove it there first");
 
-            removeEndpoints(application.get().require(instanceId.instance()));
+            controller.routing().removeEndpointsInDns(application.get(), instanceId.instance());
             curator.writeApplication(application.without(instanceId.instance()).get());
             controller.jobController().collectGarbage();
             log.info("Deleted " + instanceId);
         });
     }
 
-    private void removeEndpoints(Instance instance) {
-        instance.rotations().forEach(assignedRotation -> {
-            var endpoints = instance.endpointsIn(controller.system(), assignedRotation.endpointId());
-            endpoints.asList().stream()
-                     .map(Endpoint::dnsName)
-                     .forEach(name -> {
-                         controller.nameServiceForwarder().removeRecords(Record.Type.CNAME, RecordName.from(name), Priority.normal);
-                     });
-        });
-    }
-
     /**
      * Replace any previous version of this application by this instance
      *
@@ -856,7 +701,7 @@ public class ApplicationController {
         } catch (NotFoundException ignored) {
             // ok; already gone
         } finally {
-            routingPolicies.refresh(application.get().id().instance(instanceName), application.get().deploymentSpec(), zone);
+            controller.routing().policies().refresh(application.get().id().instance(instanceName), application.get().deploymentSpec(), zone);
         }
         return application.with(instanceName, instance -> instance.withoutDeploymentIn(zone));
     }
@@ -898,15 +743,6 @@ public class ApplicationController {
                                                              instance.id(), zone, platformVersion, applicationVersion, deployment.version(), deployment.applicationVersion()));
     }
 
-    /** Returns the rotation repository, used for managing global rotation assignments */
-    public RotationRepository rotationRepository() {
-        return rotationRepository;
-    }
-
-    public RoutingPolicies routingPolicies() {
-        return routingPolicies;
-    }
-
     /**
      * Verifies that the application can be deployed to the tenant, following these rules:
      *
@@ -1008,7 +844,7 @@ public class ApplicationController {
     }
 
     /** Returns the latest known version within the given major. */
-    private Optional<Version> lastCompatibleVersion(int targetMajorVersion) {
+    public Optional<Version> lastCompatibleVersion(int targetMajorVersion) {
         return controller.versionStatus().versions().stream()
                          .map(VespaVersion::versionNumber)
                          .filter(version -> version.getMajor() == targetMajorVersion)
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java
index d3e21f0d399..017b8cfaf45 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java
@@ -9,11 +9,10 @@ import com.yahoo.config.provision.CloudName;
 import com.yahoo.config.provision.HostName;
 import com.yahoo.config.provision.SystemName;
 import com.yahoo.config.provision.zone.ZoneApi;
+import com.yahoo.container.jdisc.secretstore.SecretStore;
 import com.yahoo.jdisc.Metric;
 import com.yahoo.vespa.curator.Lock;
 import com.yahoo.vespa.flags.FlagSource;
-import com.yahoo.vespa.hosted.controller.api.integration.ApplicationIdSnapshot;
-import com.yahoo.vespa.hosted.controller.api.integration.ApplicationIdSource;
 import com.yahoo.vespa.hosted.controller.api.integration.ServiceRegistry;
 import com.yahoo.vespa.hosted.controller.api.integration.maven.MavenRepository;
 import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry;
@@ -54,7 +53,7 @@ import java.util.stream.Collectors;
  * 
  * @author bratseth
  */
-public class Controller extends AbstractComponent implements ApplicationIdSource {
+public class Controller extends AbstractComponent {
 
     private static final Logger log = Logger.getLogger(Controller.class.getName());
 
@@ -72,6 +71,7 @@ public class Controller extends AbstractComponent implements ApplicationIdSource
     private final NameServiceForwarder nameServiceForwarder;
     private final MavenRepository mavenRepository;
     private final Metric metric;
+    private final RoutingController routingController;
 
     /**
      * Creates a controller 
@@ -80,14 +80,14 @@ public class Controller extends AbstractComponent implements ApplicationIdSource
      */
     @Inject
     public Controller(CuratorDb curator, RotationsConfig rotationsConfig, AccessControl accessControl, FlagSource flagSource,
-                      MavenRepository mavenRepository, ServiceRegistry serviceRegistry, Metric metric) {
+                      MavenRepository mavenRepository, ServiceRegistry serviceRegistry, Metric metric, SecretStore secretStore) {
         this(curator, rotationsConfig, accessControl, com.yahoo.net.HostName::getLocalhost, flagSource,
-             mavenRepository, serviceRegistry, metric);
+             mavenRepository, serviceRegistry, metric, secretStore);
     }
 
     public Controller(CuratorDb curator, RotationsConfig rotationsConfig, AccessControl accessControl,
                       Supplier<String> hostnameSupplier, FlagSource flagSource, MavenRepository mavenRepository,
-                      ServiceRegistry serviceRegistry, Metric metric) {
+                      ServiceRegistry serviceRegistry, Metric metric, SecretStore secretStore) {
 
         this.hostnameSupplier = Objects.requireNonNull(hostnameSupplier, "HostnameSupplier cannot be null");
         this.curator = Objects.requireNonNull(curator, "Curator cannot be null");
@@ -101,11 +101,9 @@ public class Controller extends AbstractComponent implements ApplicationIdSource
         metrics = new ConfigServerMetrics(serviceRegistry.configServer());
         nameServiceForwarder = new NameServiceForwarder(curator);
         jobController = new JobController(this);
-        applicationController = new ApplicationController(this, curator, accessControl,
-                                                          Objects.requireNonNull(rotationsConfig, "RotationsConfig cannot be null"),
-                                                          clock
-        );
+        applicationController = new ApplicationController(this, curator, accessControl, clock, secretStore, flagSource);
         tenantController = new TenantController(this, curator, accessControl);
+        routingController = new RoutingController(this, Objects.requireNonNull(rotationsConfig, "RotationsConfig cannot be null"));
         auditLogger = new AuditLogger(curator, clock);
 
         // Record the version of this controller
@@ -123,6 +121,11 @@ public class Controller extends AbstractComponent implements ApplicationIdSource
     /** Returns the instance controlling deployment jobs. */
     public JobController jobController() { return jobController; }
 
+    /** Returns the instance controlling routing */
+    public RoutingController routing() {
+        return routingController;
+    }
+
     /** Returns the service registry of this */
     public ServiceRegistry serviceRegistry() {
         return serviceRegistry;
@@ -146,13 +149,6 @@ public class Controller extends AbstractComponent implements ApplicationIdSource
         return serviceRegistry.configServer().getApplicationView(tenantName, applicationName, instanceName, environment, region);
     }
 
-    // TODO: Model the response properly
-    public Map<?,?> getServiceApiResponse(String tenantName, String applicationName, String instanceName,
-                                          String environment, String region, String serviceName, String restPath) {
-        return serviceRegistry.configServer().getServiceApiResponse(tenantName, applicationName, instanceName, environment, region,
-                                                                    serviceName, restPath);
-    }
-
     /** Replace the current version status by a new one */
     public void updateVersionStatus(VersionStatus newStatus) {
         VersionStatus currentStatus = versionStatus();
@@ -274,15 +270,4 @@ public class Controller extends AbstractComponent implements ApplicationIdSource
         return vespaVersion.map(v -> v.versionNumber().toFullString()).orElse("unknown");
     }
 
-    @Override
-    public ApplicationIdSnapshot applicationIdSnapshot() {
-        ApplicationIdSnapshot.Builder snapshotBuilder = new ApplicationIdSnapshot.Builder();
-        tenants().asList().forEach(tenant -> snapshotBuilder.add(tenant.name()));
-        applications().asList().forEach(application -> {
-            snapshotBuilder.add(application.id().tenant(), application.id().application());
-            application.instances().values().stream().map(Instance::id).forEach(snapshotBuilder::add);
-        });
-
-        return snapshotBuilder.build();
-    }
 }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Instance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Instance.java
index a510275b98e..d12e7dbc0ca 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Instance.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Instance.java
@@ -4,20 +4,15 @@ package com.yahoo.vespa.hosted.controller;
 import com.google.common.collect.ImmutableMap;
 import com.yahoo.component.Version;
 import com.yahoo.config.provision.ApplicationId;
-import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.config.provision.Environment;
 import com.yahoo.config.provision.InstanceName;
-import com.yahoo.config.provision.SystemName;
 import com.yahoo.config.provision.zone.ZoneId;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
 import com.yahoo.vespa.hosted.controller.application.AssignedRotation;
 import com.yahoo.vespa.hosted.controller.application.Change;
-import com.yahoo.vespa.hosted.controller.application.ClusterInfo;
 import com.yahoo.vespa.hosted.controller.application.Deployment;
 import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics;
-import com.yahoo.vespa.hosted.controller.application.EndpointId;
-import com.yahoo.vespa.hosted.controller.application.EndpointList;
 import com.yahoo.vespa.hosted.controller.rotation.RotationStatus;
 
 import java.time.Instant;
@@ -72,7 +67,6 @@ public class Instance {
         Deployment previousDeployment = deployments.getOrDefault(zone, new Deployment(zone, applicationVersion,
                                                                                       version, instant));
         Deployment newDeployment = new Deployment(zone, applicationVersion, version, instant,
-                                                  previousDeployment.clusterInfo(),
                                                   previousDeployment.metrics().with(warnings),
                                                   previousDeployment.activity());
         return with(newDeployment);
@@ -88,12 +82,6 @@ public class Instance {
         return new Instance(id, deployments.values(), jobPauses, rotations, rotationStatus, change);
     }
 
-    public Instance withClusterInfo(ZoneId zone, Map<ClusterSpec.Id, ClusterInfo> clusterInfo) {
-        Deployment deployment = deployments.get(zone);
-        if (deployment == null) return this;    // No longer deployed in this zone.
-        return with(deployment.withClusterInfo(clusterInfo));
-    }
-
     public Instance recordActivityAt(Instant instant, ZoneId zone) {
         Deployment deployment = deployments.get(zone);
         if (deployment == null) return this;
@@ -166,20 +154,6 @@ public class Instance {
         return rotations;
     }
 
-    /** Returns the default global endpoints for this in given system - for a given endpoint ID */
-    public EndpointList endpointsIn(SystemName system, EndpointId endpointId) {
-        if (rotations.isEmpty()) return EndpointList.EMPTY;
-        return EndpointList.create(id, endpointId, system);
-    }
-
-    /** Returns the default global endpoints for this in given system */
-    public EndpointList endpointsIn(SystemName system) {
-        if (rotations.isEmpty()) return EndpointList.EMPTY;
-        final var endpointStream = rotations.stream()
-                .flatMap(rotation -> EndpointList.create(id, rotation.endpointId(), system).asList().stream());
-        return EndpointList.of(endpointStream);
-    }
-
     /** Returns the status of the global rotation(s) assigned to this */
     public RotationStatus rotationStatus() {
         return rotationStatus;
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java
index 35eb18ae5d8..92633fa969c 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java
@@ -8,7 +8,6 @@ import com.yahoo.vespa.curator.Lock;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion;
 import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId;
 import com.yahoo.vespa.hosted.controller.api.integration.organization.User;
-import com.yahoo.vespa.hosted.controller.application.Change;
 import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId;
 import com.yahoo.vespa.hosted.controller.metric.ApplicationMetrics;
 
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedTenant.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedTenant.java
index 6caf716aed4..12b985d1812 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedTenant.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedTenant.java
@@ -15,7 +15,6 @@ import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant;
 import com.yahoo.vespa.hosted.controller.api.integration.organization.BillingInfo;
 import com.yahoo.vespa.hosted.controller.tenant.CloudTenant;
 import com.yahoo.vespa.hosted.controller.tenant.Tenant;
-import com.yahoo.vespa.hosted.controller.tenant.UserTenant;
 
 import java.security.Principal;
 import java.security.PublicKey;
@@ -40,7 +39,6 @@ public abstract class LockedTenant {
     static LockedTenant of(Tenant tenant, Lock lock) {
         switch (tenant.type()) {
             case athenz: return new Athenz((AthenzTenant) tenant);
-            case user:   return new User((UserTenant) tenant);
             case cloud:  return new Cloud((CloudTenant) tenant);
             default:     throw new IllegalArgumentException("Unexpected tenant type '" + tenant.getClass().getName() + "'.");
         }
@@ -99,32 +97,6 @@ public abstract class LockedTenant {
     }
 
 
-    /** A locked UserTenant. */
-    public static class User extends LockedTenant {
-
-        private final Optional<Contact> contact;
-
-        private User(TenantName name, Optional<Contact> contact) {
-            super(name);
-            this.contact = contact;
-        }
-
-        private User(UserTenant tenant) {
-            this(tenant.name(), tenant.contact());
-        }
-
-        @Override
-        public UserTenant get() {
-            return new UserTenant(name, contact);
-        }
-
-        public User with(Contact contact) {
-            return new User(name, Optional.of(contact));
-        }
-
-    }
-
-
     /** A locked CloudTenant. */
     public static class Cloud extends LockedTenant {
 
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java
new file mode 100644
index 00000000000..463ffb58460
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java
@@ -0,0 +1,325 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller;
+
+import com.google.common.base.Suppliers;
+import com.yahoo.component.Version;
+import com.yahoo.config.application.api.DeploymentInstanceSpec;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.Environment;
+import com.yahoo.config.provision.InstanceName;
+import com.yahoo.config.provision.zone.RoutingMethod;
+import com.yahoo.config.provision.zone.ZoneId;
+import com.yahoo.vespa.flags.BooleanFlag;
+import com.yahoo.vespa.flags.FetchVector;
+import com.yahoo.vespa.flags.Flags;
+import com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus;
+import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
+import com.yahoo.vespa.hosted.controller.api.integration.configserver.ContainerEndpoint;
+import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion;
+import com.yahoo.vespa.hosted.controller.api.integration.dns.Record;
+import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData;
+import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName;
+import com.yahoo.vespa.hosted.controller.application.Deployment;
+import com.yahoo.vespa.hosted.controller.application.Endpoint;
+import com.yahoo.vespa.hosted.controller.application.Endpoint.Port;
+import com.yahoo.vespa.hosted.controller.application.EndpointList;
+import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId;
+import com.yahoo.vespa.hosted.controller.dns.NameServiceQueue.Priority;
+import com.yahoo.vespa.hosted.controller.rotation.RotationLock;
+import com.yahoo.vespa.hosted.controller.rotation.RotationRepository;
+import com.yahoo.vespa.hosted.controller.routing.RoutingId;
+import com.yahoo.vespa.hosted.controller.routing.RoutingPolicies;
+import com.yahoo.vespa.hosted.rotation.config.RotationsConfig;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.stream.Collectors;
+
+/**
+ * The routing controller encapsulates state and methods for inspecting and manipulating deployment endpoints in a
+ * hosted Vespa system.
+ *
+ * The one stop shop for all your routing needs!
+ *
+ * @author mpolden
+ */
+public class RoutingController {
+
+    /** The minimum Vespa version that supports directly routed endpoints */
+    public static final Version DIRECT_ROUTING_MIN_VERSION = new Version(7, 196, 4);
+
+    private final Controller controller;
+    private final RoutingPolicies routingPolicies;
+    private final RotationRepository rotationRepository;
+    private final BooleanFlag allowDirectRouting;
+
+    public RoutingController(Controller controller, RotationsConfig rotationsConfig) {
+        this.controller = Objects.requireNonNull(controller, "controller must be non-null");
+        this.routingPolicies = new RoutingPolicies(controller);
+        this.rotationRepository = new RotationRepository(rotationsConfig, controller.applications(),
+                                                         controller.curator());
+        this.allowDirectRouting = Flags.ALLOW_DIRECT_ROUTING.bindTo(controller.flagSource());
+    }
+
+    public RoutingPolicies policies() {
+        return routingPolicies;
+    }
+
+    public RotationRepository rotations() {
+        return rotationRepository;
+    }
+
+    /** Returns zone-scoped endpoints for given deployment */
+    public EndpointList endpointsOf(DeploymentId deployment) {
+        var endpoints = new LinkedHashSet<Endpoint>();
+        // Avoid reading application more than once per call to this
+        var application = Suppliers.memoize(() -> controller.applications().requireApplication(TenantAndApplicationId.from(deployment.applicationId())));
+        for (var policy : routingPolicies.get(deployment).values()) {
+            if (!policy.status().isActive()) continue;
+            for (var routingMethod :  controller.zoneRegistry().routingMethods(policy.id().zone())) {
+                if (routingMethod.isDirect() && !canRouteDirectlyTo(deployment, application.get())) continue;
+                endpoints.add(policy.endpointIn(controller.system(), routingMethod));
+            }
+        }
+        return EndpointList.copyOf(endpoints);
+    }
+
+    /** Returns global-scoped endpoints for given instance */
+    public EndpointList endpointsOf(ApplicationId instance) {
+        return endpointsOf(controller.applications().requireApplication(TenantAndApplicationId.from(instance)),
+                                                                        instance.instance());
+    }
+
+    /** Returns global-scoped endpoints for given instance */
+    public EndpointList endpointsOf(Application application, InstanceName instanceName) {
+        var endpoints = new LinkedHashSet<Endpoint>();
+        // Add global endpoints provided by rotations
+        var instance = application.require(instanceName);
+        for (var rotation : instance.rotations()) {
+            var deployments = rotation.regions().stream()
+                                      .map(region -> new DeploymentId(instance.id(), ZoneId.from(Environment.prod, region)))
+                                      .collect(Collectors.toList());
+            computeGlobalEndpoints(RoutingId.of(instance.id(), rotation.endpointId()),
+                                   application, deployments).requiresRotation()
+                                                            .forEach(endpoints::add);
+        }
+        // Add global endpoints provided by routing policies
+        var deploymentsByRoutingId = new LinkedHashMap<RoutingId, List<DeploymentId>>();
+        for (var policy : routingPolicies.get(instance.id()).values()) {
+            if (!policy.status().isActive()) continue;
+            for (var endpointId : policy.endpoints()) {
+                var routingId = RoutingId.of(instance.id(), endpointId);
+                deploymentsByRoutingId.putIfAbsent(routingId, new ArrayList<>());
+                deploymentsByRoutingId.get(routingId).add(new DeploymentId(instance.id(), policy.id().zone()));
+            }
+        }
+        deploymentsByRoutingId.forEach((routingId, deployments) -> {
+            computeGlobalEndpoints(routingId, application, deployments).not().requiresRotation().forEach(endpoints::add);
+        });
+        return EndpointList.copyOf(endpoints);
+    }
+
+    /** Returns all non-global endpoints and corresponding cluster IDs for given deployments, grouped by their zone */
+    public Map<ZoneId, List<Endpoint>> zoneEndpointsOf(Collection<DeploymentId> deployments) {
+        var endpoints = new TreeMap<ZoneId, List<Endpoint>>(Comparator.comparing(ZoneId::value));
+        for (var deployment : deployments) {
+            var zoneEndpoints = endpointsOf(deployment).scope(Endpoint.Scope.zone).asList();
+            if  ( ! zoneEndpoints.isEmpty()) {
+                endpoints.put(deployment.zoneId(), zoneEndpoints);
+            }
+        }
+        return Collections.unmodifiableMap(endpoints);
+    }
+
+    /** Change status of all global endpoints for given deployment */
+    public void setGlobalRotationStatus(DeploymentId deployment, EndpointStatus status) {
+        endpointsOf(deployment.applicationId()).requiresRotation().primary().ifPresent(endpoint -> {
+            try {
+                controller.serviceRegistry().configServer().setGlobalRotationStatus(deployment, endpoint.upstreamIdOf(deployment), status);
+            } catch (Exception e) {
+                throw new RuntimeException("Failed to set rotation status of " + endpoint + " in " + deployment, e);
+            }
+        });
+    }
+
+    /** Get global endpoint status for given deployment */
+    public Map<Endpoint, EndpointStatus> globalRotationStatus(DeploymentId deployment) {
+        var routingEndpoints = new LinkedHashMap<Endpoint, EndpointStatus>();
+        endpointsOf(deployment.applicationId()).requiresRotation().primary().ifPresent(endpoint -> {
+            var upstreamName = endpoint.upstreamIdOf(deployment);
+            var status = controller.serviceRegistry().configServer().getGlobalRotationStatus(deployment, upstreamName);
+            routingEndpoints.put(endpoint, status);
+        });
+        return Collections.unmodifiableMap(routingEndpoints);
+    }
+
+    /**
+     * Assigns one or more global rotations to given application, if eligible. The given application is implicitly
+     * stored, ensuring that the assigned rotation(s) are persisted when this returns.
+     */
+    public LockedApplication assignRotations(LockedApplication application, InstanceName instanceName) {
+        try (RotationLock rotationLock = rotationRepository.lock()) {
+            var rotations = rotationRepository.getOrAssignRotations(application.get().deploymentSpec(),
+                                                                    application.get().require(instanceName),
+                                                                    rotationLock);
+            application = application.with(instanceName, instance -> instance.with(rotations));
+            controller.applications().store(application); // store assigned rotation even if deployment fails
+        }
+        return application;
+    }
+
+    /**
+     * Register endpoints for rotations assigned to given application and zone in DNS.
+     *
+     * @return the registered endpoints
+     */
+    public Set<ContainerEndpoint> registerEndpointsInDns(Application application, InstanceName instanceName, ZoneId zone) {
+        var instance = application.require(instanceName);
+        var containerEndpoints = new HashSet<ContainerEndpoint>();
+        boolean registerLegacyNames = application.deploymentSpec().instance(instanceName)
+                                                 .flatMap(DeploymentInstanceSpec::globalServiceId)
+                                                 .isPresent();
+        for (var assignedRotation : instance.rotations()) {
+            var names = new ArrayList<String>();
+            var endpoints = endpointsOf(application, instanceName).named(assignedRotation.endpointId())
+                                                                  .requiresRotation();
+
+            // Skip rotations which do not apply to this zone. Legacy names always point to all zones
+            if (!registerLegacyNames && !assignedRotation.regions().contains(zone.region())) {
+                continue;
+            }
+
+            // Omit legacy DNS names when assigning rotations using <endpoints/> syntax
+            if (!registerLegacyNames) {
+                endpoints = endpoints.not().legacy();
+            }
+
+            // Register names in DNS
+            var rotation = rotationRepository.getRotation(assignedRotation.rotationId());
+            if (rotation.isPresent()) {
+                endpoints.forEach(endpoint -> {
+                    controller.nameServiceForwarder().createCname(RecordName.from(endpoint.dnsName()),
+                                                                  RecordData.fqdn(rotation.get().name()),
+                                                                  Priority.normal);
+                    names.add(endpoint.dnsName());
+                });
+            }
+
+            // Include rotation ID as a valid name of this container endpoint (required by global routing health checks)
+            names.add(assignedRotation.rotationId().asString());
+            containerEndpoints.add(new ContainerEndpoint(assignedRotation.clusterId().value(), names));
+        }
+        return Collections.unmodifiableSet(containerEndpoints);
+    }
+
+    /** Remove endpoints in DNS for all rotations assigned to given instance */
+    public void removeEndpointsInDns(Application application, InstanceName instanceName) {
+        endpointsOf(application, instanceName).requiresRotation()
+                                              .forEach(endpoint -> controller.nameServiceForwarder()
+                                                                             .removeRecords(Record.Type.CNAME,
+                                                                                            RecordName.from(endpoint.dnsName()),
+                                                                                            Priority.normal));
+    }
+
+    /** Returns the routing methods that are available across all given deployments */
+    private List<RoutingMethod> routingMethodsOfAll(List<DeploymentId> deployments, Application application) {
+        var deploymentsByMethod = new HashMap<RoutingMethod, Set<DeploymentId>>();
+        for (var deployment : deployments) {
+            for (var method : controller.zoneRegistry().routingMethods(deployment.zoneId())) {
+                deploymentsByMethod.putIfAbsent(method, new LinkedHashSet<>());
+                deploymentsByMethod.get(method).add(deployment);
+            }
+        }
+        var routingMethods = new ArrayList<RoutingMethod>();
+        deploymentsByMethod.forEach((method, supportedDeployments) -> {
+            if (supportedDeployments.containsAll(deployments)) {
+                if (method.isDirect() && !canRouteDirectlyTo(deployments, application)) return;
+                routingMethods.add(method);
+            }
+        });
+        return Collections.unmodifiableList(routingMethods);
+    }
+
+    /** Returns whether traffic can be directly routed to all given deployments */
+    private boolean canRouteDirectlyTo(List<DeploymentId> deployments, Application application) {
+        return deployments.stream().allMatch(deployment -> canRouteDirectlyTo(deployment, application));
+    }
+
+    /** Returns whether traffic can be directly routed to given deployment */
+    private boolean canRouteDirectlyTo(DeploymentId deploymentId, Application application) {
+        if (controller.system().isPublic()) return true; // Public always supports direct routing
+        if (controller.system().isCd()) return true; // CD deploys directly so we cannot enforce all requirements below
+
+        // Check Athenz service presence. The test framework uses this identity when sending requests to the
+        // deployment's container(s).
+        var athenzService = application.deploymentSpec().instance(deploymentId.applicationId().instance())
+                                       .flatMap(instance -> instance.athenzService(deploymentId.zoneId().environment(),
+                                                                                   deploymentId.zoneId().region()));
+        if (athenzService.isEmpty()) return false;
+
+        // Check minimum required compile-version
+        var instance = application.require(deploymentId.applicationId().instance());
+        var compileVersion = Optional.ofNullable(instance.deployments().get(deploymentId.zoneId()))
+                                     .map(Deployment::applicationVersion)
+                                     // Use compile version of the deployed version
+                                     .flatMap(ApplicationVersion::compileVersion)
+                                     // ... or compile version of the last submitted application package. This is the
+                                     //     case for initial deployments.
+                                     .or(() -> application.latestVersion().flatMap(ApplicationVersion::compileVersion));
+        if (compileVersion.isEmpty()) return false;
+        if (compileVersion.get().isBefore(DIRECT_ROUTING_MIN_VERSION)) return false;
+
+        // Check feature flag
+        // TODO(mpolden): Remove once we make this default
+        return this.allowDirectRouting.with(FetchVector.Dimension.APPLICATION_ID,
+                                            deploymentId.applicationId().serializedForm())
+                                      .value();
+    }
+
+    /** Compute global endpoints for given routing ID, application and deployments */
+    private EndpointList computeGlobalEndpoints(RoutingId routingId, Application application, List<DeploymentId> deployments) {
+        var endpoints = new ArrayList<Endpoint>();
+        var directMethods = 0;
+        var targets = deployments.stream().map(DeploymentId::zoneId).collect(Collectors.toList());
+        for (var method : routingMethodsOfAll(deployments, application)) {
+            if (method.isDirect() && ++directMethods > 1) {
+                throw new IllegalArgumentException("Invalid routing methods for " + routingId + ": Exceeded maximum " +
+                                                   "direct methods");
+            }
+            endpoints.add(Endpoint.of(routingId.application())
+                                  .named(routingId.endpointId(), targets)
+                                  .on(Port.fromRoutingMethod(method))
+                                  .routingMethod(method)
+                                  .in(controller.system()));
+            // TODO(mpolden): Remove this once all applications have migrated away from legacy endpoints
+            if (method == RoutingMethod.shared) {
+                endpoints.add(Endpoint.of(routingId.application())
+                                      .named(routingId.endpointId(), targets)
+                                      .on(Port.plain(4080))
+                                      .legacy()
+                                      .routingMethod(method)
+                                      .in(controller.system()));
+                endpoints.add(Endpoint.of(routingId.application())
+                                      .named(routingId.endpointId(), targets)
+                                      .on(Port.tls(4443))
+                                      .legacy()
+                                      .routingMethod(method)
+                                      .in(controller.system()));
+            }
+        }
+        return EndpointList.copyOf(endpoints);
+    }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java
index e794334232f..f64d79a2b80 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java
@@ -10,7 +10,6 @@ import com.yahoo.vespa.hosted.controller.security.AccessControl;
 import com.yahoo.vespa.hosted.controller.security.Credentials;
 import com.yahoo.vespa.hosted.controller.security.TenantSpec;
 import com.yahoo.vespa.hosted.controller.tenant.Tenant;
-import com.yahoo.vespa.hosted.controller.tenant.UserTenant;
 
 import java.time.Duration;
 import java.time.Instant;
@@ -94,14 +93,6 @@ public class TenantController {
         curator.writeTenant(tenant.get());
     }
 
-    /** Create an user tenant with given username */
-    public void createUser(UserTenant tenant) {
-        try (Lock lock = lock(tenant.name())) {
-            requireNonExistent(tenant.name());
-            curator.writeTenant(tenant);
-        }
-    }
-
     /** Create a tenant, provided the given credentials are valid. */
     public void create(TenantSpec tenantSpec, Credentials credentials) {
         try (Lock lock = lock(tenantSpec.tenant())) {
@@ -141,13 +132,6 @@ public class TenantController {
         }
     }
 
-    /** Deletes the given user tenant. */
-    public void deleteUser(UserTenant tenant) {
-        try (Lock lock = lock(tenant.name())) {
-            curator.removeTenant(tenant.name());
-        }
-    }
-
     private void requireNonExistent(TenantName name) {
         if (   "hosted-vespa".equals(name.value())
             || get(name).isPresent()
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationActivity.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationActivity.java
index 1be1e313945..8917c489c65 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationActivity.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationActivity.java
@@ -56,7 +56,7 @@ public class ApplicationActivity {
     public static ApplicationActivity from(Collection<Deployment> deployments) {
         Optional<DeploymentActivity> lastActivityByQuery = lastActivityBy(DeploymentActivity::lastQueried, deployments);
         Optional<DeploymentActivity> lastActivityByWrite = lastActivityBy(DeploymentActivity::lastWritten, deployments);
-        if (!lastActivityByQuery.isPresent() && !lastActivityByWrite.isPresent()) {
+        if (lastActivityByQuery.isEmpty() && lastActivityByWrite.isEmpty()) {
             return none;
         }
         return new ApplicationActivity(lastActivityByQuery.flatMap(DeploymentActivity::lastQueried),
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java
index 10f99395170..7ca6cb1e20b 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java
@@ -4,7 +4,6 @@ package com.yahoo.vespa.hosted.controller.application;
 import com.yahoo.collections.AbstractFilteringList;
 import com.yahoo.component.Version;
 import com.yahoo.config.application.api.DeploymentSpec;
-import com.yahoo.config.application.api.DeploymentSpec.UpgradePolicy;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.vespa.hosted.controller.Application;
 import com.yahoo.vespa.hosted.controller.ApplicationController;
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationPackage.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationPackage.java
index a8ab847cda6..083984bd13c 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationPackage.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationPackage.java
@@ -9,7 +9,7 @@ import com.yahoo.config.application.api.ValidationOverrides;
 import com.yahoo.security.X509CertificateUtils;
 import com.yahoo.slime.Inspector;
 import com.yahoo.slime.Slime;
-import com.yahoo.vespa.config.SlimeUtils;
+import com.yahoo.slime.SlimeUtils;
 import com.yahoo.yolean.Exceptions;
 
 import java.io.ByteArrayInputStream;
@@ -79,7 +79,7 @@ public class ApplicationPackage {
 
         Optional<Inspector> buildMetaObject = files.get("build-meta.json").map(SlimeUtils::jsonToSlime).map(Slime::get);
         if (requireFiles && buildMetaObject.isEmpty())
-            throw new IllegalArgumentException("Missing required file 'deployment.xml'");
+            throw new IllegalArgumentException("Missing required file 'build-meta.json'");
         this.compileVersion = buildMetaObject.flatMap(object -> parse(object, "compileVersion", field -> Version.fromString(field.asString())));
         this.buildTime = buildMetaObject.flatMap(object -> parse(object, "buildTime", field -> Instant.ofEpochMilli(field.asLong())));
 
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationPackageValidator.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationPackageValidator.java
index d7347b46f52..b245718171f 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationPackageValidator.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationPackageValidator.java
@@ -46,6 +46,13 @@ public class ApplicationPackageValidator {
         validateSteps(applicationPackage.deploymentSpec());
         validateEndpointRegions(applicationPackage.deploymentSpec());
         validateEndpointChange(application, applicationPackage, instant);
+        validateSecurityClientsPem(applicationPackage);
+    }
+
+    /** Verify that we have the security/clients.pem file for public systems */
+    private void validateSecurityClientsPem(ApplicationPackage applicationPackage) {
+        if (controller.system().isPublic() && applicationPackage.trustedCertificates().isEmpty())
+            throw new IllegalArgumentException("Missing required file 'security/clients.pem'");
     }
 
     /** Verify that each of the production zones listed in the deployment spec exist in this system */
@@ -101,7 +108,7 @@ public class ApplicationPackageValidator {
         var newEndpoints = allEndpointsOf(applicationPackage.deploymentSpec().requireInstance(instanceName));
 
         if (newEndpoints.containsAll(endpoints)) return; // Adding new endpoints is fine
-        if (containsAllRegions(newEndpoints, endpoints)) return; // Adding regions to endpoints is fine
+        if (containsAllDestinationsOf(endpoints, newEndpoints)) return; // Adding destinations is fine
 
         var removedEndpoints = new ArrayList<>(endpoints);
         removedEndpoints.removeAll(newEndpoints);
@@ -115,19 +122,23 @@ public class ApplicationPackageValidator {
                                            ". " + ValidationOverrides.toAllowMessage(validationId));
     }
 
-    /** Returns whether endpoint regions in newEndpoints contains all regions of corresponding endpoint in endpoints */
-    private static boolean containsAllRegions(List<Endpoint> newEndpoints, List<Endpoint> endpoints) {
+    /** Returns whether newEndpoints contains all destinations in endpoints */
+    private static boolean containsAllDestinationsOf(List<Endpoint> endpoints, List<Endpoint> newEndpoints) {
         var containsAllRegions = true;
+        var hasSameCluster = true;
         for (var endpoint : endpoints) {
             var endpointContainsAllRegions = false;
+            var endpointHasSameCluster = false;
             for (var newEndpoint : newEndpoints) {
                 if (endpoint.endpointId().equals(newEndpoint.endpointId())) {
                     endpointContainsAllRegions = newEndpoint.regions().containsAll(endpoint.regions());
+                    endpointHasSameCluster = newEndpoint.containerId().equals(endpoint.containerId());
                 }
             }
             containsAllRegions &= endpointContainsAllRegions;
+            hasSameCluster &= endpointHasSameCluster;
         }
-        return containsAllRegions;
+        return containsAllRegions && hasSameCluster;
     }
 
     /** Returns all configued endpoints of given deployment instance spec */
@@ -142,10 +153,10 @@ public class ApplicationPackageValidator {
         return instance.globalServiceId().map(globalServiceId -> {
             var regions = instance.zones().stream()
                                   .filter(zone -> zone.environment().isProduction())
-                                  .map(zone -> zone.region().get())
+                                  .flatMap(zone -> zone.region().stream())
                                   .map(RegionName::value)
                                   .collect(Collectors.toSet());
-            return new Endpoint(Optional.of(EndpointId.defaultId().id()), instance.globalServiceId().get(), regions);
+            return new Endpoint(Optional.of(EndpointId.defaultId().id()), globalServiceId, regions);
         });
     }
 
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ClusterCost.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ClusterCost.java
deleted file mode 100644
index fb675862320..00000000000
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ClusterCost.java
+++ /dev/null
@@ -1,95 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.controller.application;
-
-import java.util.Objects;
-
-/**
- * Calculate utilization relative to the target utilization,
- * tco and waste for one cluster of one deployment.
- *
- * The target utilization is defined the following assumptions:
- * 1. CPU contention starts to cause problems at 0.8
- * 2. Memory management starts to cause problems at 0.7
- * 3. Load is evenly divided between two deployments - each deployments can handle the other.
- * 4. Memory and disk are agnostic to query load.
- * 5. Peak utilization (daily variations) are twice the size of the average.
- *
- * With this in mind we get:
- *
- * CPU: 0.8/2/2 = 0.2
- * Mem: 0.7
- * Disk: 0.7
- * Disk busy: 0.3
- *
- * @author smorgrav
- */
-public class ClusterCost {
-
-    private final double tco;
-    private final double waste;
-    private final ClusterInfo clusterInfo;
-    private final ClusterUtilization systemUtilization;
-    private final ClusterUtilization targetUtilization;
-    private final ClusterUtilization resultUtilization;
-
-    /**
-     * @param clusterInfo       value object with cluster info e.g. the TCO for the hardware used
-     * @param systemUtilization utilization of system resources (as ratios)
-     */
-    public ClusterCost(ClusterInfo clusterInfo,
-                       ClusterUtilization systemUtilization) {
-
-        Objects.requireNonNull(clusterInfo, "Cluster info cannot be null");
-        Objects.requireNonNull(systemUtilization, "Cluster utilization cannot be null");
-
-        this.clusterInfo = clusterInfo;
-        this.systemUtilization = systemUtilization;
-        this.targetUtilization = new ClusterUtilization(0.7,0.2, 0.7, 0.3);
-        this.resultUtilization = calculateResultUtilization(systemUtilization, targetUtilization);
-
-        this.tco = clusterInfo.getHostnames().size() * clusterInfo.getFlavorCost();
-
-        double unusedUtilization = 1 - Math.min(1, resultUtilization.getMaxUtilization());
-        this.waste  = tco * unusedUtilization;
-    }
-
-    /** @return The TCO in dollars for this cluster (node tco * nodes) */
-    public double getTco() {
-        return tco;
-    }
-
-    /** @return The amount of dollars spent for unused resources in this cluster */
-    public double getWaste() {
-        return waste;
-    }
-
-    public ClusterInfo getClusterInfo() {
-        return clusterInfo;
-    }
-
-    public ClusterUtilization getSystemUtilization() {
-        return systemUtilization;
-    }
-
-    public ClusterUtilization getTargetUtilization() {
-        return targetUtilization;
-    }
-
-    public ClusterUtilization getResultUtilization() {
-        return resultUtilization;
-    }
-
-    static ClusterUtilization calculateResultUtilization(ClusterUtilization system, ClusterUtilization target) {
-        double cpu = ratio(system.getCpu(), target.getCpu());
-        double mem = ratio(system.getMemory(), target.getMemory());
-        double disk = ratio(system.getDisk(), target.getDisk());
-        double diskbusy = ratio(system.getDiskBusy(), target.getDiskBusy());
-
-        return new ClusterUtilization(mem, cpu, disk, diskbusy);
-    }
-
-    private static double ratio(double a, double b) {
-        if (b == 0) return 1;
-        return a/b;
-    }
-}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ClusterInfo.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ClusterInfo.java
deleted file mode 100644
index 40fc57acdc8..00000000000
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ClusterInfo.java
+++ /dev/null
@@ -1,71 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.controller.application;
-
-import com.yahoo.config.provision.ClusterSpec;
-
-import java.util.List;
-
-/**
- * Value object of static cluster information, in particular the TCO
- * of the hardware used for this cluster.
- *
- * Some duplication/flattening of flavor info is done to simplify client usage.
- *
- * @author smorgrav
- */
-public class ClusterInfo {
-    private final String flavor;
-    private final double flavorCPU;
-    private final double flavorMem;
-    private final double flavorDisk;
-    private final int flavorCost;
-    private final ClusterSpec.Type clusterType;
-    private final List<String> hostnames;
-
-    /**
-     * @param flavor The name of the flavor eg. 'C-2B/24/500'
-     * @param flavorCost The cost of one node in dollars
-     * @param flavorCPU The number of cpu cores granted
-     * @param flavorMem The memory granted in Gb
-     * @param flavorDisk The disk size granted in Gb
-     * @param clusterType The vespa cluster type e.g 'container' or 'content'
-     * @param hostnames All hostnames in this cluster
-     */
-    public ClusterInfo(String flavor, int flavorCost, double flavorCPU, double flavorMem,
-                       double flavorDisk, ClusterSpec.Type clusterType, List<String> hostnames) {
-        this.flavor = flavor;
-        this.flavorCost = flavorCost;
-        this.flavorCPU = flavorCPU;
-        this.flavorMem = flavorMem;
-        this.flavorDisk = flavorDisk;
-        this.clusterType = clusterType;
-        this.hostnames = hostnames;
-    }
-
-    /** @return The name of the flavor eg. 'C-2B/24/500' */
-    public String getFlavor() {
-        return flavor;
-    }
-
-    /** @return The cost of one node in dollars */
-    public int getFlavorCost() { return flavorCost; }
-
-    /** @return The disk size granted in Gb */
-    public double getFlavorDisk() { return flavorDisk; }
-
-    /** @return The number of cpu cores granted */
-    public double getFlavorCPU() { return flavorCPU; }
-
-    /** @return The memory granted in Gb */
-    public double getFlavorMem() { return flavorMem; }
-
-    /** @return The vespa cluster type e.g 'container' or 'content' */
-    public ClusterSpec.Type getClusterType() {
-        return clusterType;
-    }
-
-    /** @return All hostnames in this cluster */
-    public List<String> getHostnames() {
-        return hostnames;
-    }
-}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ClusterUtilization.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ClusterUtilization.java
deleted file mode 100644
index ff92ce36d1b..00000000000
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ClusterUtilization.java
+++ /dev/null
@@ -1,63 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.controller.application;
-
-/**
- * System resources as _ratios_ of available resources.
- *
- * Can be for actual readings or target numbers.
- *
- * @author smorgrav
- */
-public class ClusterUtilization {
-
-    private final double memory;
-    private final double cpu;
-    private final double disk;
-    private final double diskBusy;
-    private final double maxUtilization;
-
-    /**
-     * Resource utilization as ratios. The ratio is normally between 0 and 1 where
-     * one is fully utilized - but can be higher as it consumes more than it are guaranteed.
-     *
-     * @param memory Memory utilization
-     * @param cpu  CPU utilization
-     * @param disk  Disk utilization
-     * @param diskBusy Disk busy
-     */
-    public ClusterUtilization(double memory, double cpu, double disk, double diskBusy) {
-        this.memory = memory;
-        this.cpu = cpu;
-        this.disk = disk;
-        this.diskBusy = diskBusy;
-
-        double maxUtil = Math.max(cpu, disk);
-        maxUtil = Math.max(maxUtil, memory);
-        this.maxUtilization = Math.max(maxUtil, diskBusy);
-    }
-
-    /** @return The utilization ratio of the resource that is utilized the most. */
-    public double getMaxUtilization() {
-        return maxUtilization;
-    }
-
-    /** @return The utilization ratio for memory */
-    public double getMemory() {
-        return memory;
-    }
-
-    /** @return The utilization ratio for cpu */
-    public double getCpu() {
-        return cpu;
-    }
-
-    /** @return The utilization ratio for disk */
-    public double getDisk() {
-        return disk;
-    }
-
-    /** @return The disk busy ratio */
-    public double getDiskBusy() {
-        return diskBusy;
-    }
-}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java
index 361dcf9dbf9..200f41c8bcf 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java
@@ -2,13 +2,10 @@
 package com.yahoo.vespa.hosted.controller.application;
 
 import com.yahoo.component.Version;
-import com.yahoo.config.provision.ClusterSpec.Id;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion;
 import com.yahoo.config.provision.zone.ZoneId;
 
 import java.time.Instant;
-import java.util.Collections;
-import java.util.Map;
 import java.util.Objects;
 
 /**
@@ -23,24 +20,19 @@ public class Deployment {
     private final ApplicationVersion applicationVersion;
     private final Version version;
     private final Instant deployTime;
-    private final Map<Id, ClusterInfo> clusterInfo;
     private final DeploymentMetrics metrics;
     private final DeploymentActivity activity;
 
     public Deployment(ZoneId zone, ApplicationVersion applicationVersion, Version version, Instant deployTime) {
-        this(zone, applicationVersion, version, deployTime, Collections.emptyMap(),
-             DeploymentMetrics.none, DeploymentActivity.none);
+        this(zone, applicationVersion, version, deployTime, DeploymentMetrics.none, DeploymentActivity.none);
     }
 
     public Deployment(ZoneId zone, ApplicationVersion applicationVersion, Version version, Instant deployTime,
-                      Map<Id, ClusterInfo> clusterInfo,
-                      DeploymentMetrics metrics,
-                      DeploymentActivity activity) {
+                      DeploymentMetrics metrics,  DeploymentActivity activity) {
         this.zone = Objects.requireNonNull(zone, "zone cannot be null");
         this.applicationVersion = Objects.requireNonNull(applicationVersion, "applicationVersion cannot be null");
         this.version = Objects.requireNonNull(version, "version cannot be null");
         this.deployTime = Objects.requireNonNull(deployTime, "deployTime cannot be null");
-        this.clusterInfo = Map.copyOf(Objects.requireNonNull(clusterInfo, "clusterInfo cannot be null"));
         this.metrics = Objects.requireNonNull(metrics, "deploymentMetrics cannot be null");
         this.activity = Objects.requireNonNull(activity, "activity cannot be null");
     }
@@ -65,29 +57,13 @@ public class Deployment {
     /** Returns activity for this */
     public DeploymentActivity activity() { return activity; }
 
-    /** Returns information about the clusters allocated to this */
-    public Map<Id, ClusterInfo> clusterInfo() {
-        return clusterInfo;
-    }
-
     public Deployment recordActivityAt(Instant instant) {
-        return new Deployment(zone, applicationVersion, version, deployTime, clusterInfo, metrics,
+        return new Deployment(zone, applicationVersion, version, deployTime, metrics,
                               activity.recordAt(instant, metrics));
     }
 
-    public Deployment withClusterUtils() {
-        return new Deployment(zone, applicationVersion, version, deployTime, clusterInfo, metrics,
-                              activity);
-    }
-
-    public Deployment withClusterInfo(Map<Id, ClusterInfo> newClusterInfo) {
-        return new Deployment(zone, applicationVersion, version, deployTime, newClusterInfo, metrics,
-                              activity);
-    }
-
     public Deployment withMetrics(DeploymentMetrics metrics) {
-        return new Deployment(zone, applicationVersion, version, deployTime, clusterInfo, metrics,
-                              activity);
+        return new Deployment(zone, applicationVersion, version, deployTime, metrics, activity);
     }
 
     @Override
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentCost.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentCost.java
deleted file mode 100644
index 393c14b35d3..00000000000
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentCost.java
+++ /dev/null
@@ -1,61 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.controller.application;
-
-import java.util.HashMap;
-import java.util.Map;
-
-/**
- * Calculates cost for for an application deployment.
- *
- * @author smorgrav
- */
-public class DeploymentCost {
-
-    private final double utilization;
-    private final double waste;
-    private final double tco;
-
-    private final Map<String, ClusterCost> clusters;
-
-    public DeploymentCost(Map<String, ClusterCost> clusterCosts) {
-        clusters = new HashMap<>(clusterCosts);
-
-        double tco = 0;
-        double util = 0;
-        double waste = 0;
-        double maxWaste = -1;
-
-        for (ClusterCost costCluster : clusterCosts.values()) {
-            tco += costCluster.getTco();
-            waste += costCluster.getWaste();
-
-            if (costCluster.getWaste() > maxWaste) {
-                util = costCluster.getResultUtilization().getMaxUtilization();
-                maxWaste = costCluster.getWaste();
-            }
-        }
-
-        this.utilization = util;
-        this.waste = waste;
-        this.tco = tco;
-    }
-
-    public Map<String, ClusterCost> getCluster() {
-        return clusters;
-    }
-
-    /** Returns the total monthly cost of ownership for the deployment (sum of all clusters) */
-    public double getTco() {
-        return tco;
-    }
-
-    /** Returns the utilization of clusters that wastes most money in this deployment */
-    public double getUtilization() {
-        return utilization;
-    }
-
-    /** Returns the amount of dollars spent and not utilized */
-    public double getWaste() {
-        return waste;
-    }
-}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java
index 9a6daf026c3..ed5add8b98a 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java
@@ -1,16 +1,19 @@
 // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.application;
 
-import com.google.common.hash.Hashing;
-import com.google.common.io.BaseEncoding;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.config.provision.SystemName;
+import com.yahoo.config.provision.zone.RoutingMethod;
 import com.yahoo.config.provision.zone.ZoneId;
+import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
 
 import java.net.URI;
-import java.nio.charset.Charset;
+import java.util.List;
 import java.util.Objects;
+import java.util.function.Predicate;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 /**
  * Represents an application's endpoint. The endpoint scope can either be global or a specific zone. This is visible to
@@ -20,30 +23,49 @@ import java.util.Objects;
  */
 public class Endpoint {
 
-    public static final String YAHOO_DNS_SUFFIX = ".vespa.yahooapis.com";
-    public static final String OATH_DNS_SUFFIX = ".vespa.oath.cloud";
-    public static final String PUBLIC_DNS_SUFFIX = ".public.vespa.oath.cloud";
-    public static final String PUBLIC_CD_DNS_SUFFIX = ".public-cd.vespa.oath.cloud";
+    private static final String YAHOO_DNS_SUFFIX = ".vespa.yahooapis.com";
+    private static final String OATH_DNS_SUFFIX = ".vespa.oath.cloud";
+    private static final String PUBLIC_DNS_SUFFIX = ".public.vespa.oath.cloud";
+    private static final String PUBLIC_CD_DNS_SUFFIX = ".public-cd.vespa.oath.cloud";
 
+    private final String name;
     private final URI url;
+    private final List<ZoneId> zones;
     private final Scope scope;
     private final boolean legacy;
-    private final boolean directRouting;
+    private final RoutingMethod routingMethod;
     private final boolean tls;
-    private final boolean wildcard;
 
-    private Endpoint(String name, ApplicationId application, ZoneId zone, SystemName system, Port port, boolean legacy,
-                     boolean directRouting, boolean wildcard) {
+    private Endpoint(String name, ApplicationId application, List<ZoneId> zones, Scope scope, SystemName system,
+                     Port port, boolean legacy, RoutingMethod routingMethod) {
         Objects.requireNonNull(name, "name must be non-null");
         Objects.requireNonNull(application, "application must be non-null");
+        Objects.requireNonNull(zones, "zones must be non-null");
+        Objects.requireNonNull(scope, "scope must be non-null");
         Objects.requireNonNull(system, "system must be non-null");
         Objects.requireNonNull(port, "port must be non-null");
-        this.url = createUrl(name, application, zone, system, port, legacy, directRouting);
-        this.scope = zone == null ? Scope.global : Scope.zone;
+        Objects.requireNonNull(routingMethod, "routingMethod must be non-null");
+        if (scope == Scope.zone && zones.size() != 1) {
+            throw new IllegalArgumentException("A single zone must be given for zone-scoped endpoints");
+        }
+        this.name = name;
+        this.url = createUrl(name, application, zones, scope, system, port, legacy, routingMethod);
+        this.zones = List.copyOf(zones);
+        this.scope = scope;
         this.legacy = legacy;
-        this.directRouting = directRouting;
+        this.routingMethod = routingMethod;
         this.tls = port.tls;
-        this.wildcard = wildcard;
+    }
+
+    /**
+     * Returns the name of this endpoint (the first component of the DNS name). Depending on the endpoint type, this
+     * can be one of the following:
+     * - A wildcard (any scope)
+     * - A cluster name (only zone scope)
+     * - An endpoint ID (only global scope)
+     */
+    public String name() {
+        return name;
     }
 
     /** Returns the URL used to access this */
@@ -57,6 +79,11 @@ public class Endpoint {
         return url.getAuthority().replaceAll(":.*", "");
     }
 
+    /** Returns the zone(s) to which this routes traffic */
+    public List<ZoneId> zones() {
+        return zones;
+    }
+
     /** Returns the scope of this */
     public Scope scope() {
         return scope;
@@ -67,12 +94,9 @@ public class Endpoint {
         return legacy;
     }
 
-    /**
-     * Returns whether this endpoint supports direct routing. Direct routing means that this endpoint is served by an
-     * exclusive load balancer instead of a shared routing layer.
-     */
-    public boolean directRouting() {
-        return directRouting;
+    /** Returns the routing used for this */
+    public RoutingMethod routingMethod() {
+        return routingMethod;
     }
 
     /** Returns whether this endpoint supports TLS connections */
@@ -80,9 +104,16 @@ public class Endpoint {
         return tls;
     }
 
-    /** Returns whether this is a wildcard endpoint (used only in certificates) */
-    public boolean wildcard() {
-        return wildcard;
+    /** Returns whether this requires a rotation to be reachable */
+    public boolean requiresRotation() {
+        return routingMethod.isShared() && scope == Scope.global;
+    }
+
+    /** Returns the upstream ID of given deployment. This *must* match what the routing layer generates */
+    public String upstreamIdOf(DeploymentId deployment) {
+        if (scope != Scope.global) throw new IllegalArgumentException("Scope " + scope + " does not have upstream name");
+        if (!routingMethod.isShared()) throw new IllegalArgumentException("Routing method " + routingMethod + " does not have upstream name");
+        return upstreamIdOf(name, deployment.applicationId(), deployment.zoneId());
     }
 
     @Override
@@ -100,23 +131,28 @@ public class Endpoint {
 
     @Override
     public String toString() {
-        return String.format("endpoint %s [scope=%s, legacy=%s, directRouting=%s]", url, scope, legacy, directRouting);
+        return String.format("endpoint %s [scope=%s, legacy=%s, routingMethod=%s]", url, scope, legacy, routingMethod);
+    }
+
+    /** Returns the DNS suffix used for endpoints in given system */
+    public static String dnsSuffix(SystemName system) {
+        return dnsSuffix(system, false);
     }
 
-    private static URI createUrl(String name, ApplicationId application, ZoneId zone, SystemName system,
-                                 Port port, boolean legacy, boolean directRouting) {
+    private static URI createUrl(String name, ApplicationId application, List<ZoneId> zones, Scope scope,
+                                 SystemName system, Port port, boolean legacy, RoutingMethod routingMethod) {
         String scheme = port.tls ? "https" : "http";
-        String separator = separator(system, directRouting, port.tls);
+        String separator = separator(system, routingMethod, port.tls);
         String portPart = port.isDefault() ? "" : ":" + port.port;
         return URI.create(scheme + "://" +
                           sanitize(namePart(name, separator)) +
                           systemPart(system, separator) +
-                          sanitize(instancePart(application, zone, separator)) +
+                          sanitize(instancePart(application, separator)) +
                           sanitize(application.application().value()) +
                           separator +
                           sanitize(application.tenant().value()) +
                           "." +
-                          scopePart(zone, legacy) +
+                          scopePart(scope, zones, legacy) +
                           dnsSuffix(system, legacy) +
                           portPart +
                           "/");
@@ -126,9 +162,9 @@ public class Endpoint {
         return part.replace('_', '-');
     }
 
-    private static String separator(SystemName system, boolean directRouting, boolean tls) {
+    private static String separator(SystemName system, RoutingMethod routingMethod, boolean tls) {
         if (!tls) return ".";
-        if (directRouting) return ".";
+        if (routingMethod.isDirect()) return ".";
         if (system.isPublic()) return ".";
         return "--";
     }
@@ -138,13 +174,14 @@ public class Endpoint {
         return name + separator;
     }
 
-    private static String scopePart(ZoneId zone, boolean legacy) {
-        if (zone == null) return "global";
+    private static String scopePart(Scope scope, List<ZoneId> zones, boolean legacy) {
+        if (scope == Scope.global) return "global";
+        var zone = zones.get(0);
         if (!legacy && zone.environment().isProduction()) return zone.region().value(); // Skip prod environment for non-legacy endpoints
         return zone.region().value() + "." + zone.environment().value();
     }
 
-    private static String instancePart(ApplicationId application, ZoneId zone, String separator) {
+    private static String instancePart(ApplicationId application, String separator) {
         if (application.instance().isDefault()) return ""; // Skip "default"
         return application.instance().value() + separator;
     }
@@ -168,6 +205,30 @@ public class Endpoint {
         }
     }
 
+    private static String upstreamIdOf(String name, ApplicationId application, ZoneId zone) {
+        return Stream.of(namePart(name, ""),
+                         instancePart(application, ""),
+                         application.application().value(),
+                         application.tenant().value(),
+                         zone.region().value(),
+                         zone.environment().value())
+                     .filter(Predicate.not(String::isEmpty))
+                     .map(Endpoint::sanitizeUpstream)
+                     .collect(Collectors.joining("."));
+    }
+
+    /** Remove any invalid characters from a upstream part */
+    private static String sanitizeUpstream(String part) {
+        return truncate(part.toLowerCase()
+                            .replace('_', '-')
+                            .replaceAll("[^a-z0-9-]*", ""));
+    }
+
+    /** Truncate the given part at the front so its length does not exceed 63 characters */
+    private static String truncate(String part) {
+        return part.substring(Math.max(0, part.length() - 63));
+    }
+
     /** An endpoint's scope */
     public enum Scope {
 
@@ -202,6 +263,12 @@ public class Endpoint {
             return new Port(443, true);
         }
 
+        /** Returns default port for the given routing method */
+        public static Port fromRoutingMethod(RoutingMethod method) {
+            if (method.isDirect()) return Port.tls();
+            return Port.tls(4443);
+        }
+
         /** Create a HTTPS port */
         public static Port tls(int port) {
             return new Port(port, true);
@@ -214,13 +281,6 @@ public class Endpoint {
 
     }
 
-    /** Create a DNS name based on a hash of the ApplicationId. This should always be less than 64 characters long. */
-    public static String createHashedCn(ApplicationId application, SystemName system) {
-        var hashCode = Hashing.sha1().hashString(application.serializedForm(), Charset.defaultCharset());
-        var base32encoded = BaseEncoding.base32().omitPadding().lowerCase().encode(hashCode.asBytes());
-        return 'v' + base32encoded + dnsSuffix(system, false);
-    }
-
     /** Build an endpoint for given application */
     public static EndpointBuilder of(ApplicationId application) {
         return new EndpointBuilder(application);
@@ -230,12 +290,13 @@ public class Endpoint {
 
         private final ApplicationId application;
 
-        private ZoneId zone;
+        private Scope scope;
+        private List<ZoneId> zones;
         private ClusterSpec.Id cluster;
         private EndpointId endpointId;
         private Port port;
+        private RoutingMethod routingMethod = RoutingMethod.shared;
         private boolean legacy = false;
-        private boolean directRouting = false;
         private boolean wildcard = false;
 
         private EndpointBuilder(ApplicationId application) {
@@ -248,35 +309,44 @@ public class Endpoint {
                 throw new IllegalArgumentException("Cannot set multiple target types");
             }
             this.cluster = cluster;
-            this.zone = zone;
+            this.scope = Scope.zone;
+            this.zones = List.of(zone);
             return this;
         }
 
         /** Sets the endpoint target ID for this (as defined in deployments.xml) */
         public EndpointBuilder named(EndpointId endpointId) {
+           return named(endpointId, List.of());
+        }
+
+        /** Sets the endpoint ID for this (as defined in deployments.xml) */
+        public EndpointBuilder named(EndpointId endpointId, List<ZoneId> targets) {
             if (cluster != null || wildcard) {
                 throw new IllegalArgumentException("Cannot set multiple target types");
             }
             this.endpointId = endpointId;
+            this.zones = targets;
+            this.scope = Scope.global;
             return this;
         }
 
         /** Sets the global wildcard target for this */
         public EndpointBuilder wildcard() {
-            if (endpointId != null || cluster != null) {
-                throw new IllegalArgumentException("Cannot set multiple target types");
-            }
-            this.wildcard = true;
-            return this;
+            return wildcard(Scope.global, List.of());
         }
 
         /** Sets the zone wildcard target for this */
         public EndpointBuilder wildcard(ZoneId zone) {
-            if(endpointId != null || cluster != null) {
+            return wildcard(Scope.zone, List.of(zone));
+        }
+
+        private EndpointBuilder wildcard(Scope scope, List<ZoneId> zones) {
+            if (endpointId != null || cluster != null) {
                 throw new IllegalArgumentException("Cannot set multiple target types");
             }
-            this.zone = zone;
             this.wildcard = true;
+            this.scope = scope;
+            this.zones = zones;
             return this;
         }
 
@@ -292,9 +362,9 @@ public class Endpoint {
             return this;
         }
 
-        /** Enables direct routing support for this */
-        public EndpointBuilder directRouting() {
-            this.directRouting = true;
+        /** Sets the routing method for this */
+        public EndpointBuilder routingMethod(RoutingMethod method) {
+            this.routingMethod = method;
             return this;
         }
 
@@ -310,13 +380,13 @@ public class Endpoint {
             } else {
                 throw new IllegalArgumentException("Must set either cluster, rotation or wildcard target");
             }
-            if (system.isPublic() && !directRouting) {
-                throw new IllegalArgumentException("Public system only supports direct routing endpoints");
+            if (system.isPublic() && routingMethod != RoutingMethod.exclusive) {
+                throw new IllegalArgumentException("Public system only supports routing method " + RoutingMethod.exclusive);
             }
-            if (directRouting && !port.isDefault()) {
-                throw new IllegalArgumentException("Direct routing endpoints only support default port");
+            if (routingMethod.isDirect() && !port.isDefault()) {
+                throw new IllegalArgumentException("Routing method " + routingMethod + " can only use default port");
             }
-            return new Endpoint(name, application, zone, system, port, legacy, directRouting, wildcard);
+            return new Endpoint(name, application, zones, scope, system, port, legacy, routingMethod);
         }
 
     }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointList.java
index e12bb5cda7f..4da34dad737 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointList.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointList.java
@@ -2,16 +2,11 @@
 package com.yahoo.vespa.hosted.controller.application;
 
 import com.yahoo.collections.AbstractFilteringList;
-import com.yahoo.config.provision.ApplicationId;
-import com.yahoo.config.provision.SystemName;
-import com.yahoo.vespa.hosted.controller.application.Endpoint.Port;
+import com.yahoo.config.provision.zone.ZoneId;
 
 import java.util.Collection;
 import java.util.List;
 import java.util.Optional;
-import java.util.function.Predicate;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
 
 
 /**
@@ -21,8 +16,6 @@ import java.util.stream.Stream;
  */
 public class EndpointList extends AbstractFilteringList<Endpoint, EndpointList> {
 
-    public static final EndpointList EMPTY = new EndpointList(List.of());
-
     private EndpointList(Collection<? extends Endpoint> endpoints, boolean negate) {
         super(endpoints, negate, EndpointList::new);
         if (endpoints.stream().distinct().count() != endpoints.size()) {
@@ -30,18 +23,34 @@ public class EndpointList extends AbstractFilteringList<Endpoint, EndpointList>
         }
     }
 
-    private EndpointList(Collection<? extends Endpoint> endpoints) {
-        this(endpoints, false);
+    /** Returns the primary (non-legacy) endpoint, if any */
+    public Optional<Endpoint> primary() {
+        return not().legacy().asList().stream().findFirst();
+    }
+
+    /** Returns the subset of endpoints named according to given ID */
+    public EndpointList named(EndpointId id) {
+        return matching(endpoint -> endpoint.name().equals(id.id()));
+    }
+
+    /** Returns the subset of endpoints which target all of the given zones */
+    public EndpointList targets(List<ZoneId> zones) {
+        return matching(endpoint -> endpoint.zones().containsAll(zones));
     }
 
-    /** Returns the main endpoint, if any */
-    public Optional<Endpoint> main() {
-        return asList().stream().filter(Predicate.not(Endpoint::legacy)).findFirst();
+    /** Returns the subset of endpoints which target the given zones */
+    public EndpointList targets(ZoneId zone) {
+        return targets(List.of(zone));
     }
 
-    /** Returns the subset of endpoints are either legacy or not */
-    public EndpointList legacy(boolean legacy) {
-        return matching(endpoint -> endpoint.legacy() == legacy);
+    /** Returns the subset of endpoints that are considered legacy */
+    public EndpointList legacy() {
+        return matching(Endpoint::legacy);
+    }
+
+    /** Returns the subset of endpoints that require a rotation */
+    public EndpointList requiresRotation() {
+        return matching(Endpoint::requiresRotation);
     }
 
     /** Returns the subset of endpoints with given scope */
@@ -49,22 +58,8 @@ public class EndpointList extends AbstractFilteringList<Endpoint, EndpointList>
         return matching(endpoint -> endpoint.scope() == scope);
     }
 
-    public static EndpointList of(Stream<Endpoint> endpoints) {
-        return new EndpointList(endpoints.collect(Collectors.toUnmodifiableList()));
-    }
-
-    /** Returns the default global endpoints in given system. Default endpoints are served by a pre-provisioned routing layer */
-    public static EndpointList create(ApplicationId application, EndpointId endpointId, SystemName system) {
-        switch (system) {
-            case cd:
-            case main:
-                return new EndpointList(List.of(
-                        Endpoint.of(application).named(endpointId).on(Port.plain(4080)).legacy().in(system),
-                        Endpoint.of(application).named(endpointId).on(Port.tls(4443)).legacy().in(system),
-                        Endpoint.of(application).named(endpointId).on(Port.tls(4443)).in(system)
-                ));
-        }
-        return EMPTY;
+    public static EndpointList copyOf(Collection<Endpoint> endpoints) {
+        return new EndpointList(endpoints, false);
     }
 
 }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/InstanceList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/InstanceList.java
index 5e83d0a71a0..cc72074295f 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/InstanceList.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/InstanceList.java
@@ -8,10 +8,8 @@ import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.InstanceName;
 import com.yahoo.vespa.hosted.controller.Application;
 import com.yahoo.vespa.hosted.controller.Instance;
-import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion;
 import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatus;
 import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatusList;
-import com.yahoo.vespa.hosted.controller.deployment.RunStatus;
 
 import java.time.Instant;
 import java.util.Collection;
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzClientFactoryImpl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzClientFactoryImpl.java
index 447f9a462b1..173729c7472 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzClientFactoryImpl.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzClientFactoryImpl.java
@@ -3,7 +3,6 @@ package com.yahoo.vespa.hosted.controller.athenz.impl;
 
 import com.google.inject.Inject;
 import com.yahoo.vespa.athenz.api.AthenzIdentity;
-import com.yahoo.vespa.athenz.api.AthenzService;
 import com.yahoo.vespa.athenz.client.zms.DefaultZmsClient;
 import com.yahoo.vespa.athenz.client.zms.ZmsClient;
 import com.yahoo.vespa.athenz.client.zts.DefaultZtsClient;
@@ -49,4 +48,9 @@ public class AthenzClientFactoryImpl implements AthenzClientFactory {
         return new DefaultZtsClient(URI.create(config.ztsUrl()), identityProvider);
     }
 
+    @Override
+    public boolean cacheLookups() {
+        return true;
+    }
+
 }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java
index 7ace62ab44d..c3a9a8484c9 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java
@@ -1,6 +1,8 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.athenz.impl;
 
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
 import com.google.inject.Inject;
 import com.yahoo.config.provision.ApplicationName;
 import com.yahoo.config.provision.TenantName;
@@ -29,13 +31,16 @@ import com.yahoo.vespa.hosted.controller.security.Credentials;
 import com.yahoo.vespa.hosted.controller.security.TenantSpec;
 import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant;
 import com.yahoo.vespa.hosted.controller.tenant.Tenant;
-import com.yahoo.vespa.hosted.controller.tenant.UserTenant;
 
 import javax.ws.rs.ForbiddenException;
 import java.util.Arrays;
 import java.util.List;
+import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Function;
+import java.util.function.Predicate;
 import java.util.logging.Logger;
 import java.util.stream.Collectors;
 
@@ -49,16 +54,28 @@ public class AthenzFacade implements AccessControl {
     private final ZmsClient zmsClient;
     private final ZtsClient ztsClient;
     private final AthenzIdentity service;
+    private final Function<AthenzIdentity, List<AthenzDomain>> userDomains;
+    private final Predicate<AccessTuple> accessRights;
 
     @Inject
     public AthenzFacade(AthenzClientFactory factory) {
-        this(factory.createZmsClient(), factory.createZtsClient(), factory.getControllerIdentity());
+        this.zmsClient = factory.createZmsClient();
+        this.ztsClient = factory.createZtsClient();
+        this.service = factory.getControllerIdentity();
+        this.userDomains = factory.cacheLookups()
+                           ? CacheBuilder.newBuilder()
+                                         .expireAfterWrite(10, TimeUnit.MINUTES)
+                                         .build(CacheLoader.from(this::getUserDomains))::getUnchecked
+                           : this::getUserDomains;
+        this.accessRights = factory.cacheLookups()
+                            ? CacheBuilder.newBuilder()
+                                          .expireAfterWrite(10, TimeUnit.MINUTES)
+                                          .build(CacheLoader.from(this::lookupAccess))::getUnchecked
+                            : this::lookupAccess;
     }
 
-    public AthenzFacade(ZmsClient zmsClient, ZtsClient ztsClient, AthenzIdentity identity) {
-        this.zmsClient = zmsClient;
-        this.ztsClient = ztsClient;
-        this.service = identity;
+    private List<AthenzDomain> getUserDomains(AthenzIdentity userIdentity) {
+        return ztsClient.getTenantDomains(service, userIdentity, "admin");
     }
 
     @Override
@@ -184,10 +201,9 @@ public class AthenzFacade implements AccessControl {
     // TODO jonmv: Remove
     public List<Tenant> accessibleTenants(List<Tenant> tenants, Credentials credentials) {
         AthenzIdentity identity =  ((AthenzPrincipal) credentials.user()).getIdentity();
-        List<AthenzDomain> userDomains = ztsClient.getTenantDomains(service, identity, "admin");
         return tenants.stream()
-                      .filter(tenant ->    tenant.type() == Tenant.Type.user && ((UserTenant) tenant).is(identity.getName())
-                                        || tenant.type() == Tenant.Type.athenz && userDomains.contains(((AthenzTenant) tenant).domain()))
+                      .filter(tenant ->    tenant.type() == Tenant.Type.athenz
+                                        && userDomains.apply(identity).contains(((AthenzTenant) tenant).domain()))
                       .collect(Collectors.toUnmodifiableList());
     }
 
@@ -215,6 +231,10 @@ public class AthenzFacade implements AccessControl {
         return hasAccess("modify", service.getDomain().getName() + ":hosted-vespa", identity);
     }
 
+    public boolean hasHostedSupporterAccess(AthenzIdentity identity) {
+        return hasAccess("read", service.getDomain().getName() + ":hosted-vespa", identity);
+    }
+
     public boolean canLaunch(AthenzIdentity principal, AthenzService service) {
         return hasAccess("launch", service.getDomain().getName() + ":service."+service.getName(), principal);
     }
@@ -223,6 +243,10 @@ public class AthenzFacade implements AccessControl {
         return hasAccess(dryRun ? "dryrun" : "deploy", new AthenzResourceName(service.getDomain(), "system-flags").toResourceNameString(), identity);
     }
 
+    public boolean hasPaymentCallbackAccess(AthenzIdentity identity) {
+        return hasAccess("callback", new AthenzResourceName(service.getDomain().getName(), "payment-notification-resource").toResourceNameString(), identity);
+    }
+
     /**
      * Used when creating tenancies. As there are no tenancy policies at this point,
      * we cannot use {@link #hasTenantAdminAccess(AthenzIdentity, AthenzDomain)}
@@ -246,8 +270,12 @@ public class AthenzFacade implements AccessControl {
     }
 
     private boolean hasAccess(String action, String resource, AthenzIdentity identity) {
-        log("getAccess(action=%s, resource=%s, principal=%s)", action, resource, identity);
-        return zmsClient.hasAccess(AthenzResourceName.fromString(resource), action, identity);
+        return accessRights.test(new AccessTuple(resource, action, identity));
+    }
+
+    private boolean lookupAccess(AccessTuple tuple) {
+        log("getAccess(action=%s, resource=%s, principal=%s)", tuple.action, tuple.resource, tuple.identity);
+        return zmsClient.hasAccess(AthenzResourceName.fromString(tuple.resource), tuple.action, tuple.identity);
     }
 
     private static void log(String format, Object... args) {
@@ -273,4 +301,34 @@ public class AthenzFacade implements AccessControl {
         _modify_
     }
 
+
+    private static class AccessTuple {
+
+        private final String resource;
+        private final String action;
+        private final AthenzIdentity identity;
+
+        private AccessTuple(String resource, String action, AthenzIdentity identity) {
+            this.resource = resource;
+            this.action = action;
+            this.identity = identity;
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            AccessTuple that = (AccessTuple) o;
+            return resource.equals(that.resource) &&
+                   action.equals(that.action) &&
+                   identity.equals(that.identity);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(resource, action, identity);
+        }
+
+    }
+
 }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateException.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateException.java
new file mode 100644
index 00000000000..321eb783dab
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateException.java
@@ -0,0 +1,26 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.certificate;
+
+public class EndpointCertificateException extends RuntimeException {
+
+    private final Type type;
+
+    public EndpointCertificateException(Type type, String message) {
+        super(message);
+        this.type = type;
+    }
+
+    public EndpointCertificateException(Type type, String message, Throwable cause) {
+        super(message, cause);
+        this.type = type;
+    }
+
+    public Type type() {
+        return type;
+    }
+
+    public enum Type {
+        CERT_NOT_AVAILABLE,
+        VERIFICATION_FAILURE
+    }
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManager.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManager.java
new file mode 100644
index 00000000000..4cb62892555
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManager.java
@@ -0,0 +1,293 @@
+package com.yahoo.vespa.hosted.controller.certificate;
+
+import com.google.common.collect.Sets;
+import com.google.common.hash.Hashing;
+import com.google.common.io.BaseEncoding;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.config.provision.SystemName;
+import com.yahoo.config.provision.zone.RoutingMethod;
+import com.yahoo.config.provision.zone.ZoneApi;
+import com.yahoo.config.provision.zone.ZoneId;
+import com.yahoo.container.jdisc.secretstore.SecretNotFoundException;
+import com.yahoo.container.jdisc.secretstore.SecretStore;
+import com.yahoo.log.LogLevel;
+import com.yahoo.security.SubjectAlternativeName;
+import com.yahoo.security.X509CertificateUtils;
+import com.yahoo.vespa.flags.BooleanFlag;
+import com.yahoo.vespa.flags.FetchVector;
+import com.yahoo.vespa.flags.FlagSource;
+import com.yahoo.vespa.flags.Flags;
+import com.yahoo.vespa.flags.StringFlag;
+import com.yahoo.vespa.hosted.controller.Instance;
+import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateProvider;
+import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMetadata;
+import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry;
+import com.yahoo.vespa.hosted.controller.application.Endpoint;
+import com.yahoo.vespa.hosted.controller.application.EndpointId;
+import com.yahoo.vespa.hosted.controller.persistence.CuratorDb;
+import org.jetbrains.annotations.NotNull;
+
+import java.nio.charset.Charset;
+import java.security.cert.X509Certificate;
+import java.time.Clock;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.OptionalInt;
+import java.util.Set;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.logging.Logger;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+/**
+ * Looks up stored endpoint certificate metadata, provisions new certificates if none is found,
+ * re-provisions if zone is not covered, and uses refreshed certificates if a newer version is available.
+ *
+ * @author andreer
+ */
+public class EndpointCertificateManager {
+
+    private static final Logger log = Logger.getLogger(EndpointCertificateManager.class.getName());
+
+    private final ZoneRegistry zoneRegistry;
+    private final CuratorDb curator;
+    private final SecretStore secretStore;
+    private final EndpointCertificateProvider endpointCertificateProvider;
+    private final Clock clock;
+    private final BooleanFlag useRefreshedEndpointCertificate;
+    private final BooleanFlag validateEndpointCertificates;
+    private final StringFlag endpointCertificateBackfill;
+    private final BooleanFlag endpointCertInSharedRouting;
+
+    public EndpointCertificateManager(ZoneRegistry zoneRegistry,
+                                      CuratorDb curator,
+                                      SecretStore secretStore,
+                                      EndpointCertificateProvider endpointCertificateProvider,
+                                      Clock clock, FlagSource flagSource) {
+        this.zoneRegistry = zoneRegistry;
+        this.curator = curator;
+        this.secretStore = secretStore;
+        this.endpointCertificateProvider = endpointCertificateProvider;
+        this.clock = clock;
+        this.useRefreshedEndpointCertificate = Flags.USE_REFRESHED_ENDPOINT_CERTIFICATE.bindTo(flagSource);
+        this.validateEndpointCertificates = Flags.VALIDATE_ENDPOINT_CERTIFICATES.bindTo(flagSource);
+        this.endpointCertificateBackfill = Flags.ENDPOINT_CERTIFICATE_BACKFILL.bindTo(flagSource);
+        this.endpointCertInSharedRouting = Flags.ENDPOINT_CERT_IN_SHARED_ROUTING.bindTo(flagSource);
+        Executors.newSingleThreadScheduledExecutor().scheduleAtFixedRate(() -> {
+            try {
+                this.backfillCertificateMetadata();
+            } catch (Throwable t) {
+                log.log(LogLevel.INFO, "Unexpected Throwable caught while backfilling certificate metadata", t);
+            }
+        }, 1, 10, TimeUnit.MINUTES);
+    }
+
+    public Optional<EndpointCertificateMetadata> getEndpointCertificateMetadata(Instance instance, ZoneId zone) {
+        var t0 = Instant.now();
+        Optional<EndpointCertificateMetadata> metadata = getOrProvision(instance, zone);
+        Duration duration = Duration.between(t0, Instant.now());
+        if (duration.toSeconds() > 30) log.log(LogLevel.INFO, String.format("Getting endpoint certificate metadata for %s took %d seconds!", instance.id().serializedForm(), duration.toSeconds()));
+        return metadata;
+    }
+
+    @NotNull
+    private Optional<EndpointCertificateMetadata> getOrProvision(Instance instance, ZoneId zone) {
+        boolean endpointCertInSharedRouting = this.endpointCertInSharedRouting.with(FetchVector.Dimension.APPLICATION_ID, instance.id().serializedForm()).value();
+        if (!zoneRegistry.zones().directlyRouted().ids().contains(zone) && !endpointCertInSharedRouting)
+            return Optional.empty();
+
+        final var currentCertificateMetadata = curator.readEndpointCertificateMetadata(instance.id());
+
+        if (currentCertificateMetadata.isEmpty()) {
+            var provisionedCertificateMetadata = provisionEndpointCertificate(instance, Optional.empty());
+            // We do not verify the certificate if one has never existed before - because we do not want to
+            // wait for it to be available before we deploy. This allows the config server to start
+            // provisioning nodes ASAP, and the risk is small for a new deployment.
+            curator.writeEndpointCertificateMetadata(instance.id(), provisionedCertificateMetadata);
+            return Optional.of(provisionedCertificateMetadata);
+        }
+
+        // Reprovision certificate if it is missing SANs for the zone we are deploying to
+        var sansInCertificate = currentCertificateMetadata.get().requestedDnsSans();
+        var requiredSansForZone = dnsNamesOf(instance.id(), List.of(zone));
+        if (sansInCertificate.isPresent() && !sansInCertificate.get().containsAll(requiredSansForZone)) {
+            var reprovisionedCertificateMetadata = provisionEndpointCertificate(instance, currentCertificateMetadata);
+            curator.writeEndpointCertificateMetadata(instance.id(), reprovisionedCertificateMetadata);
+            // Verification is unlikely to succeed in this case, as certificate must be available first - controller will retry
+            validateEndpointCertificate(reprovisionedCertificateMetadata, instance, zone);
+            return Optional.of(reprovisionedCertificateMetadata);
+        }
+
+        // If feature flag set for application, look for and use refreshed certificate
+        if (useRefreshedEndpointCertificate.with(FetchVector.Dimension.APPLICATION_ID, instance.id().serializedForm()).value()) {
+            var latestAvailableVersion = latestVersionInSecretStore(currentCertificateMetadata.get());
+
+            if (latestAvailableVersion.isPresent() && latestAvailableVersion.getAsInt() > currentCertificateMetadata.get().version()) {
+                var refreshedCertificateMetadata = currentCertificateMetadata.get().withVersion(latestAvailableVersion.getAsInt());
+                validateEndpointCertificate(refreshedCertificateMetadata, instance, zone);
+                curator.writeEndpointCertificateMetadata(instance.id(), refreshedCertificateMetadata);
+                return Optional.of(refreshedCertificateMetadata);
+            }
+        }
+
+        validateEndpointCertificate(currentCertificateMetadata.get(), instance, zone);
+        return currentCertificateMetadata;
+    }
+
+    enum BackfillMode {
+        DISABLE,
+        DRYRUN,
+        ENABLE
+    }
+
+    private void backfillCertificateMetadata() {
+        BackfillMode mode = BackfillMode.valueOf(endpointCertificateBackfill.value());
+        if (mode == BackfillMode.DISABLE) return;
+
+        List<EndpointCertificateMetadata> allProviderCertificateMetadata = endpointCertificateProvider.listCertificates();
+        Map<String, EndpointCertificateMetadata> sanToEndpointCertificate = new HashMap<>();
+
+        allProviderCertificateMetadata.forEach((providerMetadata -> {
+            if (providerMetadata.request_id().isEmpty())
+                throw new RuntimeException("Backfill failed - provider metadata missing request_id");
+            if (providerMetadata.requestedDnsSans().isEmpty())
+                throw new RuntimeException("Backfill failed - provider metadata missing DNS SANs for " + providerMetadata.request_id().get());
+            providerMetadata.requestedDnsSans().get().forEach(san -> sanToEndpointCertificate.put(san, providerMetadata)
+            );
+        }));
+
+        Map<ApplicationId, EndpointCertificateMetadata> allEndpointCertificateMetadata = curator.readAllEndpointCertificateMetadata();
+
+        allEndpointCertificateMetadata.forEach((applicationId, storedMetaData) -> {
+            if (storedMetaData.requestedDnsSans().isPresent() && storedMetaData.request_id().isPresent() && storedMetaData.issuer().isPresent())
+                return;
+
+            var hashedCn = commonNameHashOf(applicationId, zoneRegistry.system()); // use as join key
+            EndpointCertificateMetadata providerMetadata = sanToEndpointCertificate.get(hashedCn);
+
+            if (providerMetadata == null) {
+                log.log(LogLevel.INFO, "No matching certificate provider metadata found for application " + applicationId.serializedForm());
+                return;
+            }
+
+            EndpointCertificateMetadata backfilledMetadata =
+                    new EndpointCertificateMetadata(
+                            storedMetaData.keyName(),
+                            storedMetaData.certName(),
+                            storedMetaData.version(),
+                            providerMetadata.request_id(),
+                            providerMetadata.requestedDnsSans(),
+                            providerMetadata.issuer());
+
+            if (mode == BackfillMode.DRYRUN) {
+                log.log(LogLevel.INFO, "Would update stored metadata " + storedMetaData + " with data from provider: " + backfilledMetadata);
+            } else if (mode == BackfillMode.ENABLE) {
+                curator.writeEndpointCertificateMetadata(applicationId, backfilledMetadata);
+            }
+        });
+    }
+
+    private OptionalInt latestVersionInSecretStore(EndpointCertificateMetadata originalCertificateMetadata) {
+        try {
+            var certVersions = new HashSet<>(secretStore.listSecretVersions(originalCertificateMetadata.certName()));
+            var keyVersions = new HashSet<>(secretStore.listSecretVersions(originalCertificateMetadata.keyName()));
+            return Sets.intersection(certVersions, keyVersions).stream().mapToInt(Integer::intValue).max();
+        } catch (SecretNotFoundException s) {
+            return OptionalInt.empty(); // Likely because the certificate is very recently provisioned - keep current version
+        }
+    }
+
+    private EndpointCertificateMetadata provisionEndpointCertificate(Instance instance, Optional<EndpointCertificateMetadata> currentMetadata) {
+        List<ZoneId> zones = zoneRegistry.zones().controllerUpgraded().zones().stream().map(ZoneApi::getId).collect(Collectors.toUnmodifiableList());
+        return endpointCertificateProvider.requestCaSignedCertificate(instance.id(), dnsNamesOf(instance.id(), zones), currentMetadata);
+    }
+
+    private void validateEndpointCertificate(EndpointCertificateMetadata endpointCertificateMetadata, Instance instance, ZoneId zone) {
+        if (validateEndpointCertificates.value())
+            try {
+                var pemEncodedEndpointCertificate = secretStore.getSecret(endpointCertificateMetadata.certName(), endpointCertificateMetadata.version());
+
+                if (pemEncodedEndpointCertificate == null)
+                    throw new EndpointCertificateException(EndpointCertificateException.Type.VERIFICATION_FAILURE, "Secret store returned null for certificate");
+
+                List<X509Certificate> x509CertificateList = X509CertificateUtils.certificateListFromPem(pemEncodedEndpointCertificate);
+
+                if (x509CertificateList.isEmpty())
+                    throw new EndpointCertificateException(EndpointCertificateException.Type.VERIFICATION_FAILURE, "Empty certificate list");
+                if (x509CertificateList.size() < 2)
+                    throw new EndpointCertificateException(EndpointCertificateException.Type.VERIFICATION_FAILURE, "Only a single certificate found in chain - intermediate certificates likely missing");
+
+                Instant now = clock.instant();
+                Instant firstExpiry = Instant.MAX;
+                for (X509Certificate x509Certificate : x509CertificateList) {
+                    Instant notBefore = x509Certificate.getNotBefore().toInstant();
+                    Instant notAfter = x509Certificate.getNotAfter().toInstant();
+                    if (now.isBefore(notBefore))
+                        throw new EndpointCertificateException(EndpointCertificateException.Type.VERIFICATION_FAILURE, "Certificate is not yet valid");
+                    if (now.isAfter(notAfter))
+                        throw new EndpointCertificateException(EndpointCertificateException.Type.VERIFICATION_FAILURE, "Certificate has expired");
+                    if (notAfter.isBefore(firstExpiry)) firstExpiry = notAfter;
+                }
+
+                X509Certificate endEntityCertificate = x509CertificateList.get(0);
+                Set<String> subjectAlternativeNames = X509CertificateUtils.getSubjectAlternativeNames(endEntityCertificate).stream()
+                        .filter(san -> san.getType().equals(SubjectAlternativeName.Type.DNS_NAME))
+                        .map(SubjectAlternativeName::getValue).collect(Collectors.toSet());
+
+                var dnsNamesOfZone = dnsNamesOf(instance.id(), List.of(zone));
+                if (!subjectAlternativeNames.containsAll(dnsNamesOfZone))
+                    throw new EndpointCertificateException(EndpointCertificateException.Type.VERIFICATION_FAILURE, "Certificate is missing required SANs for zone " + zone.value());
+
+            } catch (SecretNotFoundException s) {
+                // Normally because the cert is in the process of being provisioned - this will cause a retry in InternalStepRunner
+                throw new EndpointCertificateException(EndpointCertificateException.Type.CERT_NOT_AVAILABLE, "Certificate not found in secret store");
+            } catch (EndpointCertificateException e) {
+                log.log(LogLevel.WARNING, "Certificate validation failure for " + instance.id().serializedForm(), e);
+                throw e;
+            } catch (Exception e) {
+                log.log(LogLevel.WARNING, "Certificate validation failure for " + instance.id().serializedForm(), e);
+                throw new EndpointCertificateException(EndpointCertificateException.Type.VERIFICATION_FAILURE, "Certificate validation failure for app " + instance.id().serializedForm(), e);
+            }
+    }
+
+    private List<String> dnsNamesOf(ApplicationId applicationId, List<ZoneId> zones) {
+        List<String> endpointDnsNames = new ArrayList<>();
+
+        // We add first an endpoint name based on a hash of the applicationId,
+        // as the certificate provider requires the first CN to be < 64 characters long.
+        endpointDnsNames.add(commonNameHashOf(applicationId, zoneRegistry.system()));
+
+        var globalDefaultEndpoint = Endpoint.of(applicationId).named(EndpointId.defaultId());
+        var rotationEndpoints = Endpoint.of(applicationId).wildcard();
+
+        var zoneLocalEndpoints = zones.stream().flatMap(zone -> Stream.of(
+                Endpoint.of(applicationId).target(ClusterSpec.Id.from("default"), zone),
+                Endpoint.of(applicationId).wildcard(zone)
+        ));
+
+        Stream.concat(Stream.of(globalDefaultEndpoint, rotationEndpoints), zoneLocalEndpoints)
+                .map(endpoint -> endpoint.routingMethod(RoutingMethod.exclusive))
+                .map(endpoint -> endpoint.on(Endpoint.Port.tls()))
+                .map(endpointBuilder -> endpointBuilder.in(zoneRegistry.system()))
+                .map(Endpoint::dnsName).forEach(endpointDnsNames::add);
+
+        return Collections.unmodifiableList(endpointDnsNames);
+    }
+
+    /** Create a common name based on a hash of the ApplicationId. This should always be less than 64 characters long. */
+    private static String commonNameHashOf(ApplicationId application, SystemName system) {
+        var hashCode = Hashing.sha1().hashString(application.serializedForm(), Charset.defaultCharset());
+        var base32encoded = BaseEncoding.base32().omitPadding().lowerCase().encode(hashCode.asBytes());
+        return 'v' + base32encoded + Endpoint.dnsSuffix(system);
+    }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java
index 3a60c480100..0419b4038bc 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java
@@ -17,7 +17,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
 import com.yahoo.vespa.hosted.controller.application.Change;
 import com.yahoo.vespa.hosted.controller.application.Deployment;
-import com.yahoo.vespa.hosted.controller.versions.VersionStatus;
 
 import java.time.Duration;
 import java.time.Instant;
@@ -42,7 +41,6 @@ import static java.util.stream.Collectors.collectingAndThen;
 import static java.util.stream.Collectors.groupingBy;
 import static java.util.stream.Collectors.toMap;
 import static java.util.stream.Collectors.toUnmodifiableList;
-import static java.util.stream.Collectors.toUnmodifiableMap;
 
 /**
  * Status of the deployment jobs of an {@link Application}.
@@ -236,12 +234,19 @@ public class DeploymentStatus {
                     && testJobs.keySet().stream()
                                .noneMatch(test ->    test.type() == testType
                                                   && testJobs.get(test).contains(versions)))
-                    testJobs.merge(new JobId(job.application(), testType), List.of(versions), DeploymentStatus::union);
+                    testJobs.merge(anyDeclaredTest(testType).orElse(new JobId(job.application(), testType)), List.of(versions), DeploymentStatus::union);
             });
         }
         return ImmutableMap.copyOf(testJobs);
     }
 
+    private Optional<JobId> anyDeclaredTest(JobType testJob) {
+        return application.deploymentSpec().instanceNames().stream()
+                .map(application.id()::instance)
+                .flatMap(id -> declaredTest(id, testJob).stream())
+                .findFirst();
+    }
+
     /** JobId of any declared test of the given type, for the given instance. */
     private Optional<JobId> declaredTest(ApplicationId instanceId, JobType testJob) {
         JobId jobId = new JobId(instanceId, testJob);
@@ -579,8 +584,11 @@ public class DeploymentStatus {
                     Versions versions = Versions.from(change, status.application, status.deploymentFor(job.id()), status.systemVersion);
                     return job.lastSuccess()
                               .filter(run -> versions.targetsMatch(run.versions()))
-                              .filter(run -> status.instanceJobs(instance).get(prodType).lastCompleted()
-                                                   .map(last -> ! last.end().get().isAfter(run.start())).orElse(false))
+                              .filter(run -> ! status.jobs()
+                                                     .instance(instance)
+                                                     .type(prodType)
+                                                     .lastCompleted().endedNoLaterThan(run.start())
+                                                     .isEmpty())
                               .map(run -> run.end().get());
                 }
             };
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatusList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatusList.java
index c14493a0b72..efa21b71936 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatusList.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatusList.java
@@ -3,15 +3,9 @@ package com.yahoo.vespa.hosted.controller.deployment;
 
 import com.yahoo.collections.AbstractFilteringList;
 import com.yahoo.component.Version;
-import com.yahoo.config.application.api.DeploymentSpec;
-import com.yahoo.vespa.hosted.controller.Instance;
-import com.yahoo.vespa.hosted.controller.application.ApplicationList;
-import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId;
 
 import java.time.Instant;
 import java.util.Collection;
-import java.util.List;
-import java.util.Optional;
 
 /**
  * List for filtering deployment status of applications, for inspection and decision making.
@@ -48,14 +42,14 @@ public class DeploymentStatusList extends AbstractFilteringList<DeploymentStatus
 
     private static boolean failingUpgradeToVersionSince(JobList jobs, Version version, Instant threshold) {
         return ! jobs.not().failingApplicationChange()
-                     .firstFailing().endedBefore(threshold)
+                     .firstFailing().endedNoLaterThan(threshold)
                      .lastCompleted().on(version)
                      .isEmpty();
     }
 
     private static boolean failingApplicationChangeSince(JobList jobs, Instant threshold) {
         return ! jobs.failingApplicationChange()
-                     .firstFailing().endedBefore(threshold)
+                     .firstFailing().endedNoLaterThan(threshold)
                      .isEmpty();
     }
 
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java
index 35d478fb332..893956e5767 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java
@@ -17,7 +17,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
 import com.yahoo.vespa.hosted.controller.application.ApplicationList;
 import com.yahoo.vespa.hosted.controller.application.Change;
 import com.yahoo.vespa.hosted.controller.application.Deployment;
-import com.yahoo.vespa.hosted.controller.application.InstanceList;
 import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId;
 
 import java.time.Clock;
@@ -154,12 +153,7 @@ public class DeploymentTrigger {
                                  .parallel().map(Supplier::get).reduce(0L, Long::sum);
     }
 
-    /**
-     * Attempts to trigger the given job for the given application and returns the outcome.
-     *
-     * If the build service can not find the given job, or claims it is illegal to trigger it,
-     * the project id is removed from the application owning the job, to prevent further trigger attempts.
-     */
+    /** Attempts to trigger the given job. */
     public void trigger(Job job) {
         log.log(LogLevel.DEBUG, "Triggering " + job);
         applications().lockApplicationOrThrow(TenantAndApplicationId.from(job.applicationId()), application -> {
@@ -169,6 +163,18 @@ public class DeploymentTrigger {
         });
     }
 
+    /** Force triggering of a job for given instance, with same versions as last run. */
+    public JobId reTrigger(ApplicationId applicationId, JobType jobType) {
+        Application application = applications().requireApplication(TenantAndApplicationId.from(applicationId));
+        Instance instance = application.require(applicationId.instance());
+        DeploymentStatus status = jobs.deploymentStatus(application);
+        JobId job = new JobId(instance.id(), jobType);
+        JobStatus jobStatus = status.jobs().get(new JobId(applicationId, jobType)).get();
+        Versions versions = jobStatus.lastTriggered().get().versions();
+        trigger(deploymentJob(instance, versions, jobType, jobStatus, clock.instant()));
+        return job;
+    }
+
     /** Force triggering of a job for given instance. */
     public List<JobId> forceTrigger(ApplicationId applicationId, JobType jobType, String user, boolean requireTests) {
         Application application = applications().requireApplication(TenantAndApplicationId.from(applicationId));
@@ -181,7 +187,7 @@ public class DeploymentTrigger {
         if (jobs.isEmpty() || ! requireTests)
             jobs = Map.of(job, List.of(versions));
         jobs.forEach((jobId, versionsList) -> {
-            trigger(deploymentJob(instance, versionsList.get(0), jobId.type(), status.jobs().get(jobId).get(), reason, clock.instant()));
+            trigger(deploymentJob(instance, versionsList.get(0), jobId.type(), status.jobs().get(jobId).get(), clock.instant()));
         });
         return List.copyOf(jobs.keySet());
     }
@@ -252,7 +258,7 @@ public class DeploymentTrigger {
 
     /** Returns the set of all jobs which have changes to propagate from the upstream steps. */
     private List<Job> computeReadyJobs() {
-        return jobs.deploymentStatuses(ApplicationList.from(applications().asList())
+        return jobs.deploymentStatuses(ApplicationList.from(applications().readable())
                                                       .withProjectId() // Need to keep this, as we have applications with deployment spec that shouldn't be orchestrated.
                                                       .withDeploymentSpec())
                    .withChanges()
@@ -278,7 +284,6 @@ public class DeploymentTrigger {
                                                      versions,
                                                      job.type(),
                                                      status.instanceJobs(job.application().instance()).get(job.type()),
-                                                     "unknown reason",
                                                      readyAt));
                           });
         });
@@ -381,7 +386,7 @@ public class DeploymentTrigger {
 
     // ---------- Version and job helpers ----------
 
-    private Job deploymentJob(Instance instance, Versions versions, JobType jobType, JobStatus jobStatus, String reason, Instant availableSince) {
+    private Job deploymentJob(Instance instance, Versions versions, JobType jobType, JobStatus jobStatus, Instant availableSince) {
         return new Job(instance, versions, jobType, availableSince, jobStatus.isOutOfCapacity(), instance.change().application().isPresent());
     }
 
@@ -422,4 +427,3 @@ public class DeploymentTrigger {
     }
 
 }
-
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
index 100b66289bb..1f2d503326a 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
@@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.controller.deployment;
 
 import com.google.common.collect.ImmutableSet;
 import com.yahoo.component.Version;
+import com.yahoo.config.application.api.DeploymentInstanceSpec;
 import com.yahoo.config.application.api.DeploymentSpec;
 import com.yahoo.config.application.api.Notifications;
 import com.yahoo.config.application.api.Notifications.When;
@@ -10,7 +11,10 @@ import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.AthenzDomain;
 import com.yahoo.config.provision.AthenzService;
 import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.config.provision.HostName;
 import com.yahoo.config.provision.NodeResources;
+import com.yahoo.config.provision.SystemName;
+import com.yahoo.config.provision.zone.RoutingMethod;
 import com.yahoo.config.provision.zone.ZoneId;
 import com.yahoo.log.LogLevel;
 import com.yahoo.security.KeyAlgorithm;
@@ -18,16 +22,13 @@ import com.yahoo.security.KeyUtils;
 import com.yahoo.security.SignatureAlgorithm;
 import com.yahoo.security.X509CertificateBuilder;
 import com.yahoo.security.X509CertificateUtils;
-import com.yahoo.vespa.flags.BooleanFlag;
-import com.yahoo.vespa.flags.FetchVector;
-import com.yahoo.vespa.flags.Flags;
 import com.yahoo.vespa.hosted.controller.Application;
 import com.yahoo.vespa.hosted.controller.Controller;
 import com.yahoo.vespa.hosted.controller.Instance;
 import com.yahoo.vespa.hosted.controller.api.ActivateResult;
-import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions;
 import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
 import com.yahoo.vespa.hosted.controller.api.identifiers.Hostname;
+import com.yahoo.vespa.hosted.controller.api.integration.LogEntry;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.PrepareResponse;
@@ -38,10 +39,14 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterId;
 import com.yahoo.vespa.hosted.controller.api.integration.organization.DeploymentFailureMails;
+import com.yahoo.vespa.hosted.controller.api.integration.organization.Mail;
 import com.yahoo.vespa.hosted.controller.application.ApplicationPackage;
 import com.yahoo.vespa.hosted.controller.application.Deployment;
+import com.yahoo.vespa.hosted.controller.application.Endpoint;
 import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId;
+import com.yahoo.vespa.hosted.controller.certificate.EndpointCertificateException;
 import com.yahoo.vespa.hosted.controller.maintenance.JobRunner;
+import com.yahoo.vespa.hosted.controller.routing.RoutingPolicyId;
 import com.yahoo.yolean.Exceptions;
 
 import javax.security.auth.x500.X500Principal;
@@ -49,7 +54,6 @@ import java.io.ByteArrayOutputStream;
 import java.io.PrintStream;
 import java.io.UncheckedIOException;
 import java.math.BigInteger;
-import java.net.URI;
 import java.security.KeyPair;
 import java.security.cert.CertificateExpiredException;
 import java.security.cert.CertificateNotYetValidException;
@@ -86,12 +90,12 @@ import static com.yahoo.vespa.hosted.controller.deployment.Step.deactivateTester
 import static com.yahoo.vespa.hosted.controller.deployment.Step.deployInitialReal;
 import static com.yahoo.vespa.hosted.controller.deployment.Step.deployReal;
 import static com.yahoo.vespa.hosted.controller.deployment.Step.deployTester;
-import static com.yahoo.vespa.hosted.controller.deployment.Step.installInitialReal;
-import static com.yahoo.vespa.hosted.controller.deployment.Step.installReal;
 import static com.yahoo.vespa.hosted.controller.deployment.Step.installTester;
 import static java.nio.charset.StandardCharsets.UTF_8;
+import static java.util.Objects.requireNonNull;
 import static java.util.logging.Level.INFO;
 import static java.util.logging.Level.WARNING;
+import static java.util.stream.Collectors.joining;
 import static java.util.stream.Collectors.toList;
 
 /**
@@ -107,26 +111,24 @@ import static java.util.stream.Collectors.toList;
 public class InternalStepRunner implements StepRunner {
 
     private static final Logger logger = Logger.getLogger(InternalStepRunner.class.getName());
-    private static final NodeResources DEFAULT_TESTER_RESOURCES =
+
+    static final NodeResources DEFAULT_TESTER_RESOURCES =
             new NodeResources(1, 4, 50, 0.3, NodeResources.DiskSpeed.any);
     // Must match exactly the advertised resources of an AWS instance type. Also consider that the container
     // will have ~1.8 GB less memory than equivalent resources in AWS (VESPA-16259).
-    private static final NodeResources DEFAULT_TESTER_RESOURCES_AWS =
+    static final NodeResources DEFAULT_TESTER_RESOURCES_AWS =
             new NodeResources(2, 8, 50, 0.3, NodeResources.DiskSpeed.any);
 
-    static final Duration endpointTimeout = Duration.ofMinutes(15);
-    static final Duration testerTimeout = Duration.ofMinutes(30);
-    static final Duration installationTimeout = Duration.ofMinutes(60);
-    static final Duration certificateTimeout = Duration.ofMinutes(300);
-
     private final Controller controller;
     private final TestConfigSerializer testConfigSerializer;
     private final DeploymentFailureMails mails;
+    private final Timeouts timeouts;
 
     public InternalStepRunner(Controller controller) {
         this.controller = controller;
         this.testConfigSerializer = new TestConfigSerializer(controller.system());
         this.mails = new DeploymentFailureMails(controller.zoneRegistry());
+        this.timeouts = Timeouts.of(controller.system());
     }
 
     @Override
@@ -141,9 +143,9 @@ public class InternalStepRunner implements StepRunner {
                 case installTester: return installTester(id, logger);
                 case installReal: return installReal(id, logger);
                 case startStagingSetup: return startTests(id, true, logger);
-                case endStagingSetup: return endTests(id, logger);
+                case endStagingSetup:
+                case endTests:  return endTests(id, logger);
                 case startTests: return startTests(id, false, logger);
-                case endTests: return endTests(id, logger);
                 case copyVespaLogs: return copyVespaLogs(id, logger);
                 case deactivateReal: return deactivateReal(id, logger);
                 case deactivateTester: return deactivateTester(id, logger);
@@ -157,7 +159,7 @@ public class InternalStepRunner implements StepRunner {
         }
         catch (RuntimeException e) {
             logger.log(WARNING, "Unexpected exception running " + id, e);
-            if (JobProfile.of(id.type()).alwaysRun().contains(step.get())) {
+            if (step.get().alwaysRun()) {
                 logger.log("Will keep trying, as this is a cleanup step.");
                 return Optional.empty();
             }
@@ -171,34 +173,20 @@ public class InternalStepRunner implements StepRunner {
                    versions.sourcePlatform().orElse(versions.targetPlatform()) +
                    " and application version " +
                    versions.sourceApplication().orElse(versions.targetApplication()).id() + " ...");
-        return deployReal(id, true, versions, logger);
+        return deployReal(id, true, logger);
     }
 
     private Optional<RunStatus> deployReal(RunId id, DualLogger logger) {
         Versions versions = controller.jobController().run(id).get().versions();
         logger.log("Deploying platform version " + versions.targetPlatform() +
                          " and application version " + versions.targetApplication().id() + " ...");
-        return deployReal(id, false, versions, logger);
+        return deployReal(id, false, logger);
     }
 
-    private Optional<RunStatus> deployReal(RunId id, boolean setTheStage, Versions versions, DualLogger logger) {
-        Optional<ApplicationPackage> applicationPackage = id.type().environment().isManuallyDeployed()
-                ? Optional.of(new ApplicationPackage(controller.applications().applicationStore()
-                                                               .getDev(id.application(), id.type().zone(controller.system()))))
-                : Optional.empty();
-
-        Optional<Version> vespaVersion = id.type().environment().isManuallyDeployed()
-                ? Optional.of(versions.targetPlatform())
-                : Optional.empty();
+    private Optional<RunStatus> deployReal(RunId id, boolean setTheStage, DualLogger logger) {
         return deploy(id.application(),
                       id.type(),
-                      () -> controller.applications().deploy(id.application(),
-                                                             id.type().zone(controller.system()),
-                                                             applicationPackage,
-                                                             new DeployOptions(false,
-                                                                               vespaVersion,
-                                                                               false,
-                                                                               setTheStage)),
+                      () -> controller.applications().deploy2(id.job(), setTheStage),
                       controller.jobController().run(id).get()
                                 .stepInfo(setTheStage ? deployInitialReal : deployReal).get()
                                 .startTime().get(),
@@ -213,10 +201,7 @@ public class InternalStepRunner implements StepRunner {
                       () -> controller.applications().deployTester(id.tester(),
                                                                    testerPackage(id),
                                                                    id.type().zone(controller.system()),
-                                                                   new DeployOptions(true,
-                                                                                     Optional.of(platform),
-                                                                                     false,
-                                                                                     false)),
+                                                                   platform),
                       controller.jobController().run(id).get()
                                 .stepInfo(deployTester).get()
                                 .startTime().get(),
@@ -227,6 +212,13 @@ public class InternalStepRunner implements StepRunner {
                                        Instant startTime, DualLogger logger) {
         try {
             PrepareResponse prepareResponse = deployment.get().prepareResponse();
+            if (prepareResponse.log != null)
+                logger.logAll(prepareResponse.log.stream()
+                                                 .map(entry -> new LogEntry(0, // Sequenced by BufferedLogStore.
+                                                                            Instant.ofEpochMilli(entry.time),
+                                                                            LogEntry.typeOf(LogLevel.parse(entry.level)),
+                                                                            entry.message))
+                                                 .collect(toList()));
             if ( ! prepareResponse.configChangeActions.refeedActions.stream().allMatch(action -> action.allowed)) {
                 List<String> messages = new ArrayList<>();
                 messages.add("Deploy failed due to non-compatible changes that require re-feed.");
@@ -240,10 +232,6 @@ public class InternalStepRunner implements StepRunner {
                                                                  .filter(action -> ! action.allowed)
                                                                  .flatMap(action -> action.messages.stream())
                                                                  .forEach(messages::add);
-                messages.add("Details:");
-                prepareResponse.log.stream()
-                                   .map(entry -> entry.message)
-                                   .forEach(messages::add);
                 logger.log(messages);
                 return Optional.of(deploymentFailed);
             }
@@ -258,7 +246,7 @@ public class InternalStepRunner implements StepRunner {
                                                                   .map(Hostname::new)
                                                                   .forEach(hostname -> {
                                                                       controller.applications().restart(new DeploymentId(id, type.zone(controller.system())), Optional.of(hostname));
-                                                                      logger.log("Restarting services on host " + hostname.id() + ".");
+                                                                      logger.log("Schedule service restart on host " + hostname.id() + ".");
                                                                   });
             logger.log("Deployment successful.");
             if (prepareResponse.message != null)
@@ -270,9 +258,14 @@ public class InternalStepRunner implements StepRunner {
             Optional<RunStatus> result = startTime.isBefore(controller.clock().instant().minus(Duration.ofHours(1)))
                                          ? Optional.of(deploymentFailed) : Optional.empty();
             switch (e.getErrorCode()) {
+                case CERTIFICATE_NOT_READY:
+                    if (startTime.plus(timeouts.endpointCertificate()).isBefore(controller.clock().instant())) {
+                        logger.log("Deployment failed to find provisioned endpoint certificate after " + timeouts.endpointCertificate());
+                        return Optional.of(RunStatus.endpointCertificateTimeout);
+                    }
+                    return result;
                 case ACTIVATION_CONFLICT:
                 case APPLICATION_LOCK_FAILURE:
-                case CERTIFICATE_NOT_READY:
                     logger.log("Deployment failed with possibly transient error " + e.getErrorCode() +
                             ", will retry: " + e.getMessage());
                     return result;
@@ -282,7 +275,9 @@ public class InternalStepRunner implements StepRunner {
                     return result;
                 case OUT_OF_CAPACITY:
                     logger.log(e.getServerMessage());
-                    return Optional.of(outOfCapacity);
+                    return controller.system().isCd() && startTime.plus(timeouts.capacity()).isAfter(controller.clock().instant())
+                           ? Optional.empty()
+                           : Optional.of(outOfCapacity);
                 case INVALID_APPLICATION_PACKAGE:
                 case BAD_REQUEST:
                     logger.log(e.getMessage());
@@ -291,6 +286,19 @@ public class InternalStepRunner implements StepRunner {
 
             throw e;
         }
+        catch (EndpointCertificateException e) {
+            switch (e.type()) {
+                case CERT_NOT_AVAILABLE:
+                    // Same as CERTIFICATE_NOT_READY above, only from the controller
+                    if (startTime.plus(timeouts.endpointCertificate()).isBefore(controller.clock().instant())) {
+                        logger.log("Deployment failed to find provisioned endpoint certificate after " + timeouts.endpointCertificate());
+                        return Optional.of(RunStatus.endpointCertificateTimeout);
+                    }
+                    return Optional.empty();
+                default:
+                    throw e; // Should be surfaced / fail deployment
+            }
+        }
     }
 
     private Optional<RunStatus> installInitialReal(RunId id, DualLogger logger) {
@@ -316,10 +324,7 @@ public class InternalStepRunner implements StepRunner {
                                                                                                                Optional.of(platform));
         if (services.isEmpty()) {
             logger.log("Config status not currently available -- will retry.");
-            Step step = setTheStage ? installInitialReal : installReal;
-            return run.stepInfo(step).get().startTime().get().isBefore(controller.clock().instant().minus(Duration.ofMinutes(5)))
-                   ? Optional.of(error)
-                   : Optional.empty();
+            return Optional.empty();
         }
         List<Node> nodes = controller.serviceRegistry().configServer().nodeRepository().list(id.type().zone(controller.system()),
                                                                                              id.application(),
@@ -329,6 +334,7 @@ public class InternalStepRunner implements StepRunner {
         NodeList nodeList = NodeList.of(nodes, parents, services.get());
         boolean firstTick = run.convergenceSummary().isEmpty();
         if (firstTick) { // Run the first time (for each convergence step).
+            logger.log("######## Details for all nodes ########");
             logger.log(nodeList.asList().stream()
                                .flatMap(node -> nodeDetails(node, true))
                                .collect(toList()));
@@ -342,50 +348,61 @@ public class InternalStepRunner implements StepRunner {
                     return Optional.of(running);
                 }
             }
-            else if (timedOut(id, deployment.get(), endpointTimeout)) {
-                logger.log(WARNING, "Endpoints failed to show up within " + endpointTimeout.toMinutes() + " minutes!");
+            else if (timedOut(id, deployment.get(), timeouts.endpoint())) {
+                logger.log(WARNING, "Endpoints failed to show up within " + timeouts.endpoint().toMinutes() + " minutes!");
                 return Optional.of(error);
             }
         }
 
-        boolean failed = false;
+        String failureReason = null;
 
-        NodeList suspendedTooLong = nodeList.suspendedSince(controller.clock().instant().minus(installationTimeout));
+        NodeList suspendedTooLong = nodeList.suspendedSince(controller.clock().instant().minus(timeouts.nodesDown()));
         if ( ! suspendedTooLong.isEmpty()) {
-            logger.log(INFO, "Some nodes have been suspended for more than " + installationTimeout.toMinutes() + " minutes.");
-            failed = true;
+            failureReason = "Some nodes have been suspended for more than " + timeouts.nodesDown().toMinutes() + " minutes:\n" +
+                            suspendedTooLong.asList().stream().map(node -> node.node().hostname().value()).collect(joining("\n"));
         }
 
         if (run.noNodesDownSince()
-               .map(since -> since.isBefore(controller.clock().instant().minus(installationTimeout)))
+               .map(since -> since.isBefore(controller.clock().instant().minus(timeouts.noNodesDown())))
                .orElse(false)) {
             if (summary.needPlatformUpgrade() > 0 || summary.needReboot() > 0 || summary.needRestart() > 0)
-                logger.log(INFO, "No nodes allowed to suspend to progress installation for " + installationTimeout.toMinutes() + " minutes.");
+                failureReason = "No nodes allowed to suspend to progress installation for " + timeouts.noNodesDown().toMinutes() + " minutes.";
             else
-                logger.log(INFO, "Nodes not able to start with new application package.");
-            failed = true;
+                failureReason = "Nodes not able to start with new application package.";
         }
 
         Duration timeout = JobRunner.jobTimeout.minusHours(1); // Time out before job dies.
         if (timedOut(id, deployment.get(), timeout)) {
-            logger.log(INFO, "Installation failed to complete within " + timeout.toHours() + "hours!");
-            failed = true;
+            failureReason = "Installation failed to complete within " + timeout.toHours() + "hours!";
         }
 
-        if (failed) {
+        if (failureReason != null) {
+            logger.log("######## Details for all nodes ########");
             logger.log(nodeList.asList().stream()
                                .flatMap(node -> nodeDetails(node, true))
                                .collect(toList()));
+            logger.log("######## Details for nodes with pending changes ########");
+            logger.log(nodeList.not().in(nodeList.not().needsNewConfig()
+                                                 .not().needsPlatformUpgrade()
+                                                 .not().needsReboot()
+                                                 .not().needsRestart()
+                                                 .not().needsFirmwareUpgrade()
+                                                 .not().needsOsUpgrade())
+                               .asList().stream()
+                               .flatMap(node -> nodeDetails(node, true))
+                               .collect(toList()));
+            logger.log(INFO, failureReason);
             return Optional.of(installationFailed);
         }
 
         if ( ! firstTick)
-            logger.log(nodeList.allowedDown().asList().stream()
+            logger.log(nodeList.expectedDown().concat(nodeList.needsNewConfig()).asList().stream()
+                               .distinct()
                                .flatMap(node -> nodeDetails(node, false))
                                .collect(toList()));
 
         controller.jobController().locked(id, lockedRun -> {
-            Instant noNodesDownSince = summary.down() == 0 ? lockedRun.noNodesDownSince().orElse(controller.clock().instant()) : null;
+            Instant noNodesDownSince = nodeList.allowedDown().size() == 0 ? lockedRun.noNodesDownSince().orElse(controller.clock().instant()) : null;
             return lockedRun.noNodesDownSince(noNodesDownSince).withSummary(summary);
         });
 
@@ -415,21 +432,14 @@ public class InternalStepRunner implements StepRunner {
         logger.log(nodeList.asList().stream()
                            .flatMap(node -> nodeDetails(node, false))
                            .collect(toList()));
-        if (nodeList.summary().converged()) {
-            if (endpointsAvailable(testerId, zone, logger)) {
-                if (containersAreUp(testerId, zone, logger)) {
-                    logger.log("Tester container successfully installed!");
-                    return Optional.of(running);
-                }
-            }
-            else if (run.stepInfo(installTester).get().startTime().get().plus(endpointTimeout).isBefore(controller.clock().instant())) {
-                logger.log(WARNING, "Tester failed to show up within " + endpointTimeout.toMinutes() + " minutes!");
-                return Optional.of(error);
-            }
+
+        if (nodeList.summary().converged() && testerContainersAreUp(testerId, zone, logger)) {
+            logger.log("Tester container successfully installed!");
+            return Optional.of(running);
         }
 
-        if (run.stepInfo(installTester).get().startTime().get().plus(testerTimeout).isBefore(controller.clock().instant())) {
-            logger.log(WARNING, "Installation of tester failed to complete within " + testerTimeout.toMinutes() + " minutes!");
+        if (run.stepInfo(installTester).get().startTime().get().plus(timeouts.tester()).isBefore(controller.clock().instant())) {
+            logger.log(WARNING, "Installation of tester failed to complete within " + timeouts.tester().toMinutes() + " minutes!");
             return Optional.of(error);
         }
 
@@ -438,15 +448,13 @@ public class InternalStepRunner implements StepRunner {
 
     /** Returns true iff all containers in the deployment give 100 consecutive 200 OK responses on /status.html. */
     private boolean containersAreUp(ApplicationId id, ZoneId zoneId, DualLogger logger) {
-        var endpoints = controller.applications().clusterEndpoints(Set.of(new DeploymentId(id, zoneId)));
+        var endpoints = controller.routing().zoneEndpointsOf(Set.of(new DeploymentId(id, zoneId)));
         if ( ! endpoints.containsKey(zoneId))
             return false;
 
-        for (URI endpoint : endpoints.get(zoneId).values()) {
-            boolean ready = id.instance().isTester() ? controller.jobController().cloud().testerReady(endpoint)
-                                                     : controller.jobController().cloud().ready(endpoint);
-
-            if (!ready) {
+        for (var endpoint : endpoints.get(zoneId)) {
+            boolean ready = controller.jobController().cloud().ready(endpoint.url());
+            if ( ! ready) {
                 logger.log("Failed to get 100 consecutive OKs from " + endpoint);
                 return false;
             }
@@ -455,34 +463,69 @@ public class InternalStepRunner implements StepRunner {
         return true;
     }
 
+    /** Returns true iff all containers in the tester deployment give 100 consecutive 200 OK responses on /status.html. */
+    private boolean testerContainersAreUp(ApplicationId id, ZoneId zoneId, DualLogger logger) {
+        DeploymentId deploymentId = new DeploymentId(id, zoneId);
+        if (controller.jobController().cloud().testerReady(deploymentId)) {
+            return true;
+        } else {
+            logger.log("Failed to get 100 consecutive OKs from tester container for " + deploymentId);
+            return false;
+        }
+    }
+
     private boolean endpointsAvailable(ApplicationId id, ZoneId zone, DualLogger logger) {
-        var endpoints = controller.applications().clusterEndpoints(Set.of(new DeploymentId(id, zone)));
+        var endpoints = controller.routing().zoneEndpointsOf(Set.of(new DeploymentId(id, zone)));
         if ( ! endpoints.containsKey(zone)) {
             logger.log("Endpoints not yet ready.");
             return false;
         }
-        for (var endpoint : endpoints.get(zone).values())
-            if ( ! controller.jobController().cloud().exists(endpoint)) {
-                logger.log(INFO, "DNS lookup yielded no IP address for '" + endpoint + "'.");
+        var policies = controller.routing().policies().get(new DeploymentId(id, zone));
+        for (var endpoint : endpoints.get(zone)) {
+            HostName endpointName = HostName.from(endpoint.dnsName());
+            var ipAddress = controller.jobController().cloud().resolveHostName(endpointName);
+            if (ipAddress.isEmpty()) {
+                logger.log(INFO, "DNS lookup yielded no IP address for '" + endpointName + "'.");
                 return false;
             }
+            if (endpoint.routingMethod() == RoutingMethod.exclusive)  {
+                var policy = policies.get(new RoutingPolicyId(id, ClusterSpec.Id.from(endpoint.name()), zone));
+                if (policy == null)
+                    throw new IllegalStateException(endpoint + " has no matching policy in " + policies);
+
+                var cNameValue = controller.jobController().cloud().resolveCname(endpointName);
+                if ( ! cNameValue.map(policy.canonicalName()::equals).orElse(false)) {
+                    logger.log(INFO, "CNAME '" + endpointName + "' points at " +
+                                     cNameValue.map(name -> "'" + name + "'").orElse("nothing") +
+                                     " but should point at load balancer '" + policy.canonicalName() + "'");
+                    return false;
+                }
+                var loadBalancerAddress = controller.jobController().cloud().resolveHostName(policy.canonicalName());
+                if ( ! loadBalancerAddress.equals(ipAddress)) {
+                    logger.log(INFO, "IP address of CNAME '" + endpointName + "' (" + ipAddress.get() + ") and load balancer '" +
+                                     policy.canonicalName() + "' (" + loadBalancerAddress.orElse("empty") + ") are not equal");
+                    return false;
+                }
+            }
+        }
 
         logEndpoints(endpoints, logger);
         return true;
     }
 
-    private void logEndpoints(Map<ZoneId, Map<ClusterSpec.Id, URI>> endpoints, DualLogger logger) {
+    private void logEndpoints(Map<ZoneId, List<Endpoint>> zoneEndpoints, DualLogger logger) {
         List<String> messages = new ArrayList<>();
         messages.add("Found endpoints:");
-        endpoints.forEach((zone, uris) -> {
+        zoneEndpoints.forEach((zone, endpoints) -> {
             messages.add("- " + zone);
-            uris.forEach((cluster, uri) -> messages.add(" |-- " + uri + " (" + cluster + ")"));
+            for (Endpoint endpoint : endpoints)
+                messages.add(" |-- " + endpoint.url() + " (cluster '" + endpoint.name() + "')");
         });
         logger.log(messages);
     }
 
     private Stream<String> nodeDetails(NodeWithServices node, boolean printAllServices) {
-        return Stream.concat(Stream.of(node.node().hostname() + ": " + humanize(node.node().serviceState()),
+        return Stream.concat(Stream.of(node.node().hostname() + ": " + humanize(node.node().serviceState()) + (node.node().suspendedSince().map(since -> " since " + since).orElse("")),
                                        "--- platform " + node.node().wantedVersion() + (node.needsPlatformUpgrade()
                                                                                         ? " <-- " + (node.node().currentVersion().isEmpty() ? "not booted" : node.node().currentVersion())
                                                                                         : "") +
@@ -525,35 +568,30 @@ public class InternalStepRunner implements StepRunner {
                                     .productionDeployments().keySet().stream()
                                     .map(zone -> new DeploymentId(id.application(), zone))
                                     .collect(Collectors.toSet());
-        deployments.add(new DeploymentId(id.application(), id.type().zone(controller.system())));
+        ZoneId zoneId = id.type().zone(controller.system());
+        deployments.add(new DeploymentId(id.application(), zoneId));
 
         logger.log("Attempting to find endpoints ...");
-        var endpoints = controller.applications().clusterEndpoints(deployments);
-        if ( ! endpoints.containsKey(id.type().zone(controller.system()))) {
+        var endpoints = controller.routing().zoneEndpointsOf(deployments);
+        if ( ! endpoints.containsKey(zoneId)) {
             logger.log(WARNING, "Endpoints for the deployment to test vanished again, while it was still active!");
             return Optional.of(error);
         }
         logEndpoints(endpoints, logger);
 
-        Optional<URI> testerEndpoint = controller.jobController().testerEndpoint(id);
-        if (testerEndpoint.isEmpty()) {
-            logger.log(WARNING, "Endpoints for the tester container vanished again, while it was still active!");
-            return Optional.of(error);
-        }
-
-        if ( ! controller.jobController().cloud().testerReady(testerEndpoint.get())) {
+        if (!controller.jobController().cloud().testerReady(getTesterDeploymentId(id))) {
             logger.log(WARNING, "Tester container went bad!");
             return Optional.of(error);
         }
 
         logger.log("Starting tests ...");
-        controller.jobController().cloud().startTests(testerEndpoint.get(),
-                                                      TesterCloud.Suite.of(id.type(), isSetup),
-                                                      testConfigSerializer.configJson(id.application(),
-                                                                                      id.type(),
-                                                                                      true,
-                                                                                      endpoints,
-                                                                                      controller.applications().contentClustersByZone(deployments)));
+        TesterCloud.Suite suite = TesterCloud.Suite.of(id.type(), isSetup);
+        byte[] config = testConfigSerializer.configJson(id.application(),
+                                                        id.type(),
+                                                        true,
+                                                        endpoints,
+                                                        controller.applications().contentClustersByZone(deployments));
+        controller.jobController().cloud().startTests(getTesterDeploymentId(id), suite, config);
         return Optional.of(running);
     }
 
@@ -576,23 +614,7 @@ public class InternalStepRunner implements StepRunner {
 
         controller.jobController().updateTestLog(id);
 
-        BooleanFlag useConfigServerForTesterAPI = Flags.USE_CONFIG_SERVER_FOR_TESTER_API_CALLS.bindTo(controller.flagSource());
-        ZoneId zoneId = id.type().zone(controller.system());
-        TesterCloud.Status testStatus;
-        boolean useConfigServer = useConfigServerForTesterAPI.with(FetchVector.Dimension.ZONE_ID, zoneId.value()).value();
-        InternalStepRunner.logger.log(LogLevel.INFO, Flags.USE_CONFIG_SERVER_FOR_TESTER_API_CALLS.id().toString() +
-                                                     " has value " + useConfigServer + " in zone " + zoneId.value());
-        if (useConfigServer) {
-            testStatus = controller.serviceRegistry().configServer().getTesterStatus(getTesterDeploymentId(id, zoneId));
-        } else {
-            Optional<URI> testerEndpoint = controller.jobController().testerEndpoint(id);
-            if (testerEndpoint.isEmpty()) {
-                logger.log("Endpoints for tester not found -- trying again later.");
-                return Optional.empty();
-            }
-            testStatus = controller.jobController().cloud().getStatus(testerEndpoint.get());
-        }
-
+        TesterCloud.Status testStatus = controller.jobController().cloud().getStatus(getTesterDeploymentId(id));
         switch (testStatus) {
             case NOT_STARTED:
                 throw new IllegalStateException("Tester reports tests not started, even though they should have!");
@@ -685,22 +707,37 @@ public class InternalStepRunner implements StepRunner {
             return;
 
         try {
-            if (run.status() == outOfCapacity && run.id().type().isProduction())
-                controller.serviceRegistry().mailer().send(mails.outOfCapacity(run.id(), recipients));
-            if (run.status() == deploymentFailed)
-                controller.serviceRegistry().mailer().send(mails.deploymentFailure(run.id(), recipients));
-            if (run.status() == installationFailed)
-                controller.serviceRegistry().mailer().send(mails.installationFailure(run.id(), recipients));
-            if (run.status() == testFailure)
-                controller.serviceRegistry().mailer().send(mails.testFailure(run.id(), recipients));
-            if (run.status() == error)
-                controller.serviceRegistry().mailer().send(mails.systemError(run.id(), recipients));
+            logger.log(INFO, "Sending failure notification to " + String.join(", ", recipients));
+            mailOf(run, recipients).ifPresent(controller.serviceRegistry().mailer()::send);
         }
         catch (RuntimeException e) {
             logger.log(INFO, "Exception trying to send mail for " + run.id(), e);
         }
     }
 
+    private Optional<Mail> mailOf(Run run, List<String> recipients) {
+        switch (run.status()) {
+            case running:
+            case aborted:
+            case success:
+                return Optional.empty();
+            case outOfCapacity:
+                return run.id().type().isProduction() ? Optional.of(mails.outOfCapacity(run.id(), recipients)) : Optional.empty();
+            case deploymentFailed:
+                return Optional.of(mails.deploymentFailure(run.id(), recipients));
+            case installationFailed:
+                return Optional.of(mails.installationFailure(run.id(), recipients));
+            case testFailure:
+                return Optional.of(mails.testFailure(run.id(), recipients));
+            case error:
+            case endpointCertificateTimeout:
+                return Optional.of(mails.systemError(run.id(), recipients));
+            default:
+                logger.log(WARNING, "Don't know what mail to send for run status '" + run.status() + "'");
+                return Optional.of(mails.systemError(run.id(), recipients));
+        }
+    }
+
     /** Returns the deployment of the real application in the zone of the given job, if it exists. */
     private Optional<Deployment> deployment(ApplicationId id, JobType type) {
         return Optional.ofNullable(application(id).deployments().get(type.zone(controller.system())));
@@ -740,13 +777,9 @@ public class InternalStepRunner implements StepRunner {
         ZoneId zone = id.type().zone(controller.system());
         boolean useTesterCertificate = controller.system().isPublic() && id.type().environment().isTest();
 
-        byte[] servicesXml = servicesXml(controller.zoneRegistry().accessControlDomain(),
-                                         ! controller.system().isPublic(),
+        byte[] servicesXml = servicesXml(! controller.system().isPublic(),
                                          useTesterCertificate,
-                                         testerFlavorFor(id, spec)
-                                                 .map(NodeResources::fromLegacyName)
-                                                 .orElse(zone.region().value().contains("aws-") ?
-                                                         DEFAULT_TESTER_RESOURCES_AWS : DEFAULT_TESTER_RESOURCES));
+                                         testerResourcesFor(zone, spec.requireInstance(id.application().instance())));
         byte[] testPackage = controller.applications().applicationStore().getTester(id.application().tenant(), id.application().application(), version);
         byte[] deploymentXml = deploymentXml(id.tester(),
                                              spec.athenzDomain(),
@@ -770,7 +803,7 @@ public class InternalStepRunner implements StepRunner {
         X509Certificate certificate = X509CertificateBuilder.fromKeypair(keyPair,
                                                                          subject,
                                                                          controller.clock().instant(),
-                                                                         controller.clock().instant().plus(certificateTimeout),
+                                                                         controller.clock().instant().plus(timeouts.testerCertificate()),
                                                                          SignatureAlgorithm.SHA512_WITH_RSA,
                                                                          BigInteger.valueOf(1))
                                                             .build();
@@ -779,21 +812,23 @@ public class InternalStepRunner implements StepRunner {
         zipBuilder.add("artifacts/cert", X509CertificateUtils.toPem(certificate).getBytes(UTF_8));
     }
 
-    private DeploymentId getTesterDeploymentId(RunId runId, ZoneId zoneId) {
+    private DeploymentId getTesterDeploymentId(RunId runId) {
+        ZoneId zoneId = runId.type().zone(controller.system());
         return new DeploymentId(runId.tester().id(), zoneId);
     }
 
-    private static Optional<String> testerFlavorFor(RunId id, DeploymentSpec spec) {
-        for (DeploymentSpec.Step step : spec.steps())
-            if (step.concerns(id.type().environment()))
-                return step.zones().get(0).testerFlavor();
-
-        return Optional.empty();
+    static NodeResources testerResourcesFor(ZoneId zone, DeploymentInstanceSpec spec) {
+        return spec.steps().stream()
+                   .filter(step -> step.concerns(zone.environment()))
+                   .findFirst()
+                   .flatMap(step -> step.zones().get(0).testerFlavor())
+                   .map(NodeResources::fromLegacyName)
+                   .orElse(zone.region().value().contains("aws-") ?
+                           DEFAULT_TESTER_RESOURCES_AWS : DEFAULT_TESTER_RESOURCES);
     }
 
     /** Returns the generated services.xml content for the tester application. */
-    static byte[] servicesXml(AthenzDomain domain, boolean systemUsesAthenz, boolean useTesterCertificate,
-                              NodeResources resources) {
+    static byte[] servicesXml(boolean systemUsesAthenz, boolean useTesterCertificate, NodeResources resources) {
         int jdiscMemoryGb = 2; // 2Gb memory for tester application (excessive?).
         int jdiscMemoryPct = (int) Math.ceil(100 * jdiscMemoryGb / resources.memoryGb());
 
@@ -804,7 +839,6 @@ public class InternalStepRunner implements StepRunner {
                                               "<resources vcpu=\"%.2f\" memory=\"%.2fGb\" disk=\"%.2fGb\" disk-speed=\"%s\" storage-type=\"%s\"/>",
                                               resources.vcpu(), resources.memoryGb(), resources.diskGb(), resources.diskSpeed().name(), resources.storageType().name());
 
-        AthenzDomain idDomain = ("vespa.vespa.cd".equals(domain.value()) ? AthenzDomain.from("vespa.vespa") : domain);
         String servicesXml =
                 "<?xml version='1.0' encoding='UTF-8'?>\n" +
                 "<services xmlns:deploy='vespa' version='1.0'>\n" +
@@ -823,51 +857,6 @@ public class InternalStepRunner implements StepRunner {
                 "            <binding>http://*/tester/v1/*</binding>\n" +
                 "        </handler>\n" +
                 "\n" +
-                "        <http>\n" +
-                "            <!-- Make sure 4080 is the first port. This will be used by the config server. -->\n" +
-                "            <server id='default' port='4080'/>\n" +
-                "            <server id='testertls4443' port='4443'>\n" +
-                "                <config name=\"jdisc.http.connector\">\n" +
-                "                    <tlsClientAuthEnforcer>\n" +
-                "                        <enable>true</enable>\n" +
-                "                        <pathWhitelist>\n" +
-                "                            <item>/status.html</item>\n" +
-                "                            <item>/state/v1/config</item>\n" +
-                "                        </pathWhitelist>\n" +
-                "                    </tlsClientAuthEnforcer>\n" +
-                "                </config>\n" +
-                "                <ssl>\n" +
-                "                    <private-key-file>/var/lib/sia/keys/" + idDomain.value() + ".tenant.key.pem</private-key-file>\n" +
-                "                    <certificate-file>/var/lib/sia/certs/" + idDomain.value() + ".tenant.cert.pem</certificate-file>\n" +
-                "                    <ca-certificates-file>/opt/yahoo/share/ssl/certs/athenz_certificate_bundle.pem</ca-certificates-file>\n" +
-                "                    <client-authentication>want</client-authentication>\n" +
-                "                </ssl>\n" +
-                "            </server>\n" +
-                "            <filtering>\n" +
-                (systemUsesAthenz ?
-                "                <access-control domain='" + domain.value() + "'>\n" + // Set up dummy access control to pass validation :/
-                "                    <exclude>\n" +
-                "                        <binding>http://*/tester/v1/*</binding>\n" +
-                "                    </exclude>\n" +
-                "                </access-control>\n"
-                : "") +
-                "                <request-chain id=\"testrunner-api\">\n" +
-                "                    <filter id='authz-filter' class='com.yahoo.jdisc.http.filter.security.athenz.AthenzAuthorizationFilter' bundle=\"jdisc-security-filters\">\n" +
-                "                        <config name=\"jdisc.http.filter.security.athenz.athenz-authorization-filter\">\n" +
-                "                            <credentialsToVerify>TOKEN_ONLY</credentialsToVerify>\n" +
-                "                            <roleTokenHeaderName>Yahoo-Role-Auth</roleTokenHeaderName>\n" +
-                "                        </config>\n" +
-                "                        <component id=\"com.yahoo.jdisc.http.filter.security.athenz.StaticRequestResourceMapper\" bundle=\"jdisc-security-filters\">\n" +
-                "                            <config name=\"jdisc.http.filter.security.athenz.static-request-resource-mapper\">\n" +
-                "                                <resourceName>" + domain.value() + ":tester-application</resourceName>\n" +
-                "                                <action>deploy</action>\n" +
-                "                            </config>\n" +
-                "                        </component>\n" +
-                "                    </filter>\n" +
-                "                </request-chain>\n" +
-                "            </filtering>\n" +
-                "        </http>\n" +
-                "\n" +
                 "        <nodes count=\"1\" allocated-memory=\"" + jdiscMemoryPct + "%\">\n" +
                 "            " + resourceString + "\n" +
                 "        </nodes>\n" +
@@ -904,6 +893,10 @@ public class InternalStepRunner implements StepRunner {
             log(List.of(messages));
         }
 
+        private void logAll(List<LogEntry> messages) {
+            controller.jobController().log(id, step, messages);
+        }
+
         private void log(List<String> messages) {
             controller.jobController().log(id, step, INFO, messages);
         }
@@ -931,4 +924,27 @@ public class InternalStepRunner implements StepRunner {
 
     }
 
+
+    static class Timeouts {
+
+        private final SystemName system;
+
+        private Timeouts(SystemName system) {
+            this.system = requireNonNull(system);
+        }
+
+        public static Timeouts of(SystemName system) {
+            return new Timeouts(system);
+        }
+
+        Duration capacity() { return Duration.ofMinutes(system.isCd() ? 5 : 0); }
+        Duration endpoint() { return Duration.ofMinutes(15); }
+        Duration endpointCertificate() { return Duration.ofMinutes(20); }
+        Duration tester() { return Duration.ofMinutes(30); }
+        Duration nodesDown() { return Duration.ofMinutes(system.isCd() ? 20 : 60); }
+        Duration noNodesDown() { return Duration.ofMinutes(system.isCd() ? 20 : 120); }
+        Duration testerCertificate() { return Duration.ofMinutes(300); }
+
+    }
+
 }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
index c8cfc8ac286..9a5f9b1074a 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
@@ -1,10 +1,11 @@
 // Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.deployment;
 
+import com.google.common.collect.ImmutableSortedMap;
 import com.yahoo.component.Version;
+import com.yahoo.config.application.api.DeploymentSpec;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.zone.ZoneId;
-import com.yahoo.jdisc.Metric;
 import com.yahoo.vespa.curator.Lock;
 import com.yahoo.vespa.hosted.controller.Application;
 import com.yahoo.vespa.hosted.controller.Controller;
@@ -25,7 +26,6 @@ import com.yahoo.vespa.hosted.controller.application.Deployment;
 import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId;
 import com.yahoo.vespa.hosted.controller.persistence.BufferedLogStore;
 import com.yahoo.vespa.hosted.controller.persistence.CuratorDb;
-import com.yahoo.vespa.hosted.controller.tenant.Tenant;
 
 import java.net.URI;
 import java.security.cert.X509Certificate;
@@ -35,6 +35,7 @@ import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashSet;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.NavigableMap;
@@ -46,6 +47,7 @@ import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Consumer;
 import java.util.function.UnaryOperator;
 import java.util.logging.Level;
+import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
 import static com.google.common.collect.ImmutableList.copyOf;
@@ -54,6 +56,7 @@ import static com.yahoo.vespa.hosted.controller.deployment.Step.deactivateTester
 import static com.yahoo.vespa.hosted.controller.deployment.Step.endStagingSetup;
 import static com.yahoo.vespa.hosted.controller.deployment.Step.endTests;
 import static java.util.stream.Collectors.toList;
+import static java.util.stream.Collectors.toMap;
 import static java.util.stream.Collectors.toUnmodifiableList;
 import static java.util.stream.Collectors.toUnmodifiableMap;
 
@@ -72,8 +75,8 @@ import static java.util.stream.Collectors.toUnmodifiableMap;
  */
 public class JobController {
 
-    private static final int historyLength = 64;
-    private static final Duration maxHistoryAge = Duration.ofDays(60);
+    public static final int historyLength = 64;
+    public static final Duration maxHistoryAge = Duration.ofDays(60);
 
     private final Controller controller;
     private final CuratorDb curator;
@@ -82,7 +85,7 @@ public class JobController {
     private final Badges badges;
     private final JobMetrics metric;
 
-    private AtomicReference<Consumer<Run>> runner = new AtomicReference<>(__ -> { });
+    private final AtomicReference<Consumer<Run>> runner = new AtomicReference<>(__ -> { });
 
     public JobController(Controller controller) {
         this.controller = controller;
@@ -179,11 +182,8 @@ public class JobController {
             if (step.isEmpty())
                 return run;
 
-            Optional<URI> testerEndpoint = testerEndpoint(id);
-            if ( ! testerEndpoint.isPresent())
-                return run;
-
-            List<LogEntry> entries = cloud.getLog(testerEndpoint.get(), run.lastTestLogEntry());
+            List<LogEntry> entries = cloud.getLog(new DeploymentId(id.tester().id(), id.type().zone(controller.system())),
+                                                  run.lastTestLogEntry());
             if (entries.isEmpty())
                 return run;
 
@@ -197,16 +197,9 @@ public class JobController {
         locked(id, run -> run.with(testerCertificate));
     }
 
-    /** Returns a list of all applications which have registered. */
-    public List<TenantAndApplicationId> applications() {
-        return copyOf(controller.applications().asList().stream()
-                                .map(Application::id)
-                                .iterator());
-    }
-
     /** Returns a list of all instances of applications which have registered. */
     public List<ApplicationId> instances() {
-        return copyOf(controller.applications().asList().stream()
+        return copyOf(controller.applications().readable().stream()
                                 .flatMap(application -> application.instances().values().stream())
                                 .map(Instance::id)
                                 .iterator());
@@ -226,9 +219,14 @@ public class JobController {
 
     /** Returns an immutable map of all known runs for the given application and job type. */
     public NavigableMap<RunId, Run> runs(ApplicationId id, JobType type) {
-        NavigableMap<RunId, Run> runs = curator.readHistoricRuns(id, type);
-        last(id, type).ifPresent(run -> runs.put(run.id(), run));
-        return Collections.unmodifiableNavigableMap(runs);
+        ImmutableSortedMap.Builder<RunId, Run> runs = ImmutableSortedMap.orderedBy(Comparator.comparing(RunId::number));
+        Optional<Run> last = last(id, type);
+        curator.readHistoricRuns(id, type).forEach((runId, run) -> {
+            if (last.isEmpty() || ! runId.equals(last.get().id()))
+                runs.put(runId, run);
+        });
+        last.ifPresent(run -> runs.put(run.id(), run));
+        return runs.build();
     }
 
     /** Returns the run with the given id, if it exists. */
@@ -309,8 +307,10 @@ public class JobController {
     private DeploymentStatus deploymentStatus(Application application, Version systemVersion) {
         return new DeploymentStatus(application,
                                     DeploymentStatus.jobsFor(application, controller.system()).stream()
-                                                    .collect(toUnmodifiableMap(job -> job,
-                                                                               job -> jobStatus(job))),
+                                                    .collect(toMap(job -> job,
+                                                                   job -> jobStatus(job),
+                                                                   (j1, j2) -> { throw new IllegalArgumentException("Duplicate key " + j1.id()); },
+                                                                   LinkedHashMap::new)),
                                     controller.system(),
                                     systemVersion,
                                     controller.clock().instant());
@@ -353,7 +353,9 @@ public class JobController {
                      old = oldEntries.next()) {
 
                     // Make sure we keep the last success and the first failing
-                    if (successes == 1 && old.getValue().status() == RunStatus.success) {
+                    if (     successes == 1
+                        &&   old.getValue().status() == RunStatus.success
+                        && ! old.getValue().start().isBefore(controller.clock().instant().minus(maxHistoryAge))) {
                         oldEntries.next();
                         continue;
                     }
@@ -409,31 +411,24 @@ public class JobController {
 
     /** Orders a run of the given type, or throws an IllegalStateException if that job type is already running. */
     public void start(ApplicationId id, JobType type, Versions versions) {
-        if ( ! type.environment().isManuallyDeployed() && versions.targetApplication().isUnknown())
-            throw new IllegalArgumentException("Target application must be a valid reference.");
-
-        controller.applications().lockApplicationIfPresent(TenantAndApplicationId.from(id), application -> {
-            locked(id, type, __ -> {
-                Optional<Run> last = last(id, type);
-                if (last.flatMap(run -> active(run.id())).isPresent())
-                    throw new IllegalStateException("Can not start " + type + " for " + id + "; it is already running!");
-
-                RunId newId = new RunId(id, type, last.map(run -> run.id().number()).orElse(0L) + 1);
-                curator.writeLastRun(Run.initial(newId, versions, controller.clock().instant()));
-                metric.jobStarted(newId.job());
-            });
+        start(id, type, versions, JobProfile.of(type));
+    }
+
+    /** Orders a run of the given type, or throws an IllegalStateException if that job type is already running. */
+    public void start(ApplicationId id, JobType type, Versions versions, JobProfile profile) {
+        locked(id, type, __ -> {
+            Optional<Run> last = last(id, type);
+            if (last.flatMap(run -> active(run.id())).isPresent())
+                throw new IllegalStateException("Can not start " + type + " for " + id + "; it is already running!");
+
+            RunId newId = new RunId(id, type, last.map(run -> run.id().number()).orElse(0L) + 1);
+            curator.writeLastRun(Run.initial(newId, versions, controller.clock().instant(), profile));
+            metric.jobStarted(newId.job());
         });
     }
 
     /** Stores the given package and starts a deployment of it, after aborting any such ongoing deployment. */
     public void deploy(ApplicationId id, JobType type, Optional<Version> platform, ApplicationPackage applicationPackage) {
-        if ( ! type.environment().isManuallyDeployed())
-            throw new IllegalArgumentException("Direct deployments are only allowed to manually deployed environments.");
-
-        if (   controller.tenants().require(id.tenant()).type() == Tenant.Type.user
-            && controller.applications().getApplication(TenantAndApplicationId.from(id)).isEmpty())
-            controller.applications().createApplication(TenantAndApplicationId.from(id), Optional.empty());
-
         controller.applications().lockApplicationOrThrow(TenantAndApplicationId.from(id), application -> {
             if ( ! application.get().instances().containsKey(id.instance()))
                 application = controller.applications().withNewInstance(application, id);
@@ -442,13 +437,21 @@ public class JobController {
         });
 
         last(id, type).filter(run -> ! run.hasEnded()).ifPresent(run -> abortAndWait(run.id()));
-        locked(id, type, __ -> {
+
+        controller.applications().lockApplicationOrThrow(TenantAndApplicationId.from(id), application -> {
             controller.applications().applicationStore().putDev(id, type.zone(controller.system()), applicationPackage.zippedContent());
-            start(id, type, new Versions(platform.orElse(controller.systemVersion()),
-                                         ApplicationVersion.unknown,
-                                         Optional.empty(),
-                                         Optional.empty()));
+            start(id,
+                  type,
+                  new Versions(platform.orElse(applicationPackage.deploymentSpec().majorVersion()
+                                                                 .flatMap(controller.applications()::lastCompatibleVersion)
+                                                                 .orElseGet(controller::systemVersion)),
+                               ApplicationVersion.unknown,
+                               Optional.empty(),
+                               Optional.empty()),
+                  JobProfile.development);
+        });
 
+        locked(id, type, __ -> {
             runner.get().accept(last(id, type).get());
         });
     }
@@ -493,12 +496,17 @@ public class JobController {
                });
     }
 
+    // TODO(mpolden): Eliminate duplication in this and ApplicationController#deactivate
     public void deactivateTester(TesterId id, JobType type) {
+        var zone = type.zone(controller.system());
         try {
-            controller.serviceRegistry().configServer().deactivate(new DeploymentId(id.id(), type.zone(controller.system())));
-        }
-        catch (NotFoundException ignored) {
+            controller.serviceRegistry().configServer().deactivate(new DeploymentId(id.id(), zone));
+        } catch (NotFoundException ignored) {
             // Already gone -- great!
+        } finally {
+            // Passing an empty DeploymentSpec here is fine as it's used for registering global endpoint names, and
+            // tester instances have none.
+            controller.routing().policies().refresh(id.id(), DeploymentSpec.empty, zone);
         }
     }
 
@@ -526,21 +534,12 @@ public class JobController {
                                     .collect(toList()));
     }
 
-    /** Returns a URI of the tester endpoint retrieved from the routing generator, provided it matches an expected form. */
-    Optional<URI> testerEndpoint(RunId id) {
-        DeploymentId testerId = new DeploymentId(id.tester().id(), id.type().zone(controller.system()));
-        return controller.applications().getDeploymentEndpoints(testerId)
-                         .stream().findAny()
-                         .or(() -> controller.applications().routingPolicies().get(testerId).values().stream()
-                                             .findAny()
-                                             .map(policy -> policy.endpointIn(controller.system()).url()));
-    }
-
     private void prunePackages(TenantAndApplicationId id) {
         controller.applications().lockApplicationIfPresent(id, application -> {
             application.get().productionDeployments().values().stream()
                        .flatMap(List::stream)
                        .map(Deployment::applicationVersion)
+                       .filter(version -> ! version.isUnknown())
                        .min(Comparator.comparingLong(applicationVersion -> applicationVersion.buildNumber().getAsLong()))
                        .ifPresent(oldestDeployed -> {
                            controller.applications().applicationStore().prune(id.tenant(), id.application(), oldestDeployed);
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobList.java
index f353910163f..525eadb6eaf 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobList.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobList.java
@@ -117,7 +117,7 @@ public class JobList extends AbstractFilteringList<JobStatus, JobList> {
         return new RunFilter(JobStatus::firstFailing);
     }
 
-    /** Allows sub-filters for runs of the given kind */
+    /** Allows sub-filters for runs of the indicated kind */
     public class RunFilter {
 
         private final Function<JobStatus, Optional<Run>> which;
@@ -126,47 +126,32 @@ public class JobList extends AbstractFilteringList<JobStatus, JobList> {
             this.which = which;
         }
 
-        /** Returns the subset of jobs where the run of the given type exists */
+        /** Returns the subset of jobs where the run of the indicated type exists */
         public JobList present() {
             return matching(run -> true);
         }
 
-        /** Returns the runs of the given kind, mapped by the given function, as a list. */
+        /** Returns the runs of the indicated kind, mapped by the given function, as a list. */
         public <OtherType> List<OtherType> mapToList(Function<? super Run, OtherType> mapper) {
             return present().mapToList(which.andThen(Optional::get).andThen(mapper));
         }
 
-        /** Returns the runs of the given kind. */
+        /** Returns the runs of the indicated kind. */
         public List<Run> asList() {
             return mapToList(Function.identity());
         }
 
-        /** Returns the subset of jobs where the run of the given type occurred before the given instant */
-        public JobList endedBefore(Instant threshold) {
-            return matching(run -> run.end().orElse(Instant.MAX).isBefore(threshold));
+        /** Returns the subset of jobs where the run of the indicated type ended no later than the given instant */
+        public JobList endedNoLaterThan(Instant threshold) {
+            return matching(run -> ! run.end().orElse(Instant.MAX).isAfter(threshold));
         }
 
-        /** Returns the subset of jobs where the run of the given type occurred after the given instant */
-        public JobList endedAfter(Instant threshold) {
-            return matching(run -> run.end().orElse(Instant.MIN).isAfter(threshold));
-        }
-
-        /** Returns the subset of jobs where the run of the given type occurred before the given instant */
-        public JobList startedBefore(Instant threshold) {
-            return matching(run -> run.start().isBefore(threshold));
-        }
-
-        /** Returns the subset of jobs where the run of the given type occurred after the given instant */
-        public JobList startedAfter(Instant threshold) {
-            return matching(run -> run.start().isAfter(threshold));
-        }
-
-        /** Returns the subset of jobs where the run of the given type was on the given version */
+        /** Returns the subset of jobs where the run of the indicated type was on the given version */
         public JobList on(ApplicationVersion version) {
             return matching(run -> run.versions().targetApplication().equals(version));
         }
 
-        /** Returns the subset of jobs where the run of the given type was on the given version */
+        /** Returns the subset of jobs where the run of the indicated type was on the given version */
         public JobList on(Version version) {
             return matching(run -> run.versions().targetPlatform().equals(version));
         }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobMetrics.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobMetrics.java
index a6ffb56492f..80924c3c0aa 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobMetrics.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobMetrics.java
@@ -16,6 +16,7 @@ public class JobMetrics {
 
     public static final String start = "deployment.start";
     public static final String outOfCapacity = "deployment.outOfCapacity";
+    public static final String endpointCertificateTimeout = "deployment.endpointCertificateTimeout";
     public static final String deploymentFailure = "deployment.deploymentFailure";
     public static final String convergenceFailure = "deployment.convergenceFailure";
     public static final String testFailure = "deployment.testFailure";
@@ -40,17 +41,17 @@ public class JobMetrics {
     }
 
     Map<String, String> contextOf(JobId id) {
-        return Map.of("tenant", id.application().tenant().value(),
-                      "application", id.application().application().value(),
-                      "instance", id.application().instance().value(),
-                      "job", id.type().jobName(),
-                      "environment", id.type().environment().value(),
-                      "region", id.type().zone(system).region().value());
+        return Map.of("applicationId", id.application().toFullString(),
+                      "tenantName", id.application().tenant().value(),
+                      "app", id.application().application().value() + "." + id.application().instance().value(),
+                      "test", Boolean.toString(id.type().isTest()),
+                      "zone", id.type().zone(system).value());
     }
 
     static String valueOf(RunStatus status) {
         switch (status) {
             case outOfCapacity: return outOfCapacity;
+            case endpointCertificateTimeout: return endpointCertificateTimeout;
             case deploymentFailed: return deploymentFailure;
             case installationFailed: return convergenceFailure;
             case testFailure: return testFailure;
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobProfile.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobProfile.java
index 4c2b53d6ba2..1c1d60a2cf0 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobProfile.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobProfile.java
@@ -34,8 +34,8 @@ public enum JobProfile {
                           deployTester,
                           installTester,
                           startTests,
-                          endTests),
-               EnumSet.of(copyVespaLogs,
+                          endTests,
+                          copyVespaLogs,
                           deactivateTester,
                           deactivateReal,
                           report)),
@@ -49,42 +49,35 @@ public enum JobProfile {
                            deployReal,
                            installReal,
                            startTests,
-                           endTests),
-                EnumSet.of(copyVespaLogs,
+                           endTests,
+                           copyVespaLogs,
                            deactivateTester,
                            deactivateReal,
                            report)),
 
     production(EnumSet.of(deployReal,
                           installReal,
-                          deployTester,
-                          installTester,
-                          startTests,
-                          endTests),
-               EnumSet.of(deactivateTester,
                           report)),
 
     productionTest(EnumSet.of(deployTester,
                               installTester,
                               startTests,
-                              endTests),
-                   EnumSet.of(deactivateTester,
+                              endTests,
+                              deactivateTester,
                               report)),
 
     development(EnumSet.of(deployReal,
-                           installReal),
-                EnumSet.of(copyVespaLogs));
+                           installReal,
+                           copyVespaLogs));
 
 
     private final Set<Step> steps;
-    private final Set<Step> alwaysRun;
 
-    JobProfile(Set<Step> runWhileSuccess, Set<Step> alwaysRun) {
-        runWhileSuccess.addAll(alwaysRun);
-        this.steps = Collections.unmodifiableSet(runWhileSuccess);
-        this.alwaysRun = Collections.unmodifiableSet(alwaysRun);
+    JobProfile(Set<Step> steps) {
+        this.steps = Collections.unmodifiableSet(steps);
     }
 
+    // TODO jonmv: Let caller decide profile, and store with run?
     public static JobProfile of(JobType type) {
         switch (type.environment()) {
             case test: return systemTest;
@@ -99,7 +92,4 @@ public enum JobProfile {
     /** Returns all steps in this profile, the default for which is to run only when all prerequisites are successes. */
     public Set<Step> steps() { return steps; }
 
-    /** Returns the set of steps that should always be run, regardless of outcome. */
-    public Set<Step> alwaysRun() { return alwaysRun; }
-
 }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/NodeList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/NodeList.java
index d3533fc5200..fbfdac427e4 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/NodeList.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/NodeList.java
@@ -30,7 +30,7 @@ public class NodeList extends AbstractFilteringList<NodeWithServices, NodeList>
                                  .map(node -> new NodeWithServices(node,
                                                                    parentsByHostName.get(node.parentHostname().get()),
                                                                    services.wantedGeneration(),
-                                                                   servicesByHostName.get(node.hostname())))
+                                                                   servicesByHostName.getOrDefault(node.hostname(), List.of())))
                                  .collect(Collectors.toList()),
                             false,
                             services.wantedGeneration());
@@ -68,6 +68,11 @@ public class NodeList extends AbstractFilteringList<NodeWithServices, NodeList>
 
     /** The nodes currently allowed to be down. */
     public NodeList allowedDown() {
+        return matching(node -> node.isAllowedDown());
+    }
+
+    /** The nodes currently expected to be down. */
+    public NodeList expectedDown() {
         return matching(node -> node.isAllowedDown() || node.isNewlyProvisioned());
     }
 
@@ -83,7 +88,7 @@ public class NodeList extends AbstractFilteringList<NodeWithServices, NodeList>
 
     /** Returns a summary of the convergence status of the nodes in this list. */
     public ConvergenceSummary summary() {
-        NodeList allowedDown = allowedDown();
+        NodeList allowedDown = expectedDown();
         return new ConvergenceSummary(size(),
                                       allowedDown.size(),
                                       withParentDown().needsOsUpgrade().size(),
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/NodeWithServices.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/NodeWithServices.java
index 80c1fe0f40b..921bf045873 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/NodeWithServices.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/NodeWithServices.java
@@ -6,6 +6,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.configserver.ServiceCon
 
 import java.time.Instant;
 import java.util.List;
+import java.util.Objects;
 
 import static java.util.Objects.requireNonNull;
 
@@ -52,7 +53,8 @@ public class NodeWithServices {
     }
 
     public boolean needsPlatformUpgrade() {
-        return node.wantedVersion().isAfter(node.currentVersion());
+        return node.wantedVersion().isAfter(node.currentVersion())
+                || ! node.wantedDockerImage().equals(node.currentDockerImage());
     }
 
     public boolean needsReboot() {
@@ -79,4 +81,17 @@ public class NodeWithServices {
         return services.stream().anyMatch(service -> wantedConfigGeneration > service.currentGeneration());
     }
 
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        NodeWithServices that = (NodeWithServices) o;
+        return node.equals(that.node);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(node);
+    }
+
 }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java
index 8cd57fa7d3a..d2481cd97ad 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java
@@ -56,9 +56,9 @@ public class Run {
         this.testerCertificate = testerCertificate;
     }
 
-    public static Run initial(RunId id, Versions versions, Instant now) {
+    public static Run initial(RunId id, Versions versions, Instant now, JobProfile profile) {
         EnumMap<Step, StepInfo> steps = new EnumMap<>(Step.class);
-        JobProfile.of(id.type()).steps().forEach(step -> steps.put(step, StepInfo.initial(step)));
+        profile.steps().forEach(step -> steps.put(step, StepInfo.initial(step)));
         return new Run(id, steps, requireNonNull(versions), requireNonNull(now), Optional.empty(), running,
                        -1, Instant.EPOCH, Optional.empty(), Optional.empty(), Optional.empty());
     }
@@ -268,9 +268,9 @@ public class Run {
     private List<Step> forcedSteps() {
         return ImmutableList.copyOf(steps.entrySet().stream()
                                          .filter(entry ->    entry.getValue().status() == unfinished
-                                                          && JobProfile.of(id.type()).alwaysRun().contains(entry.getKey())
+                                                          && entry.getKey().alwaysRun()
                                                           && entry.getKey().prerequisites().stream()
-                                                                  .filter(JobProfile.of(id.type()).alwaysRun()::contains)
+                                                                  .filter(Step::alwaysRun)
                                                                   .allMatch(step ->    steps.get(step) == null
                                                                                     || steps.get(step).status() != unfinished))
                                          .map(Map.Entry::getKey)
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunLog.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunLog.java
index b5ee571a387..44299997b4f 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunLog.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunLog.java
@@ -8,7 +8,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.LogEntry;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
-import java.util.Optional;
 import java.util.OptionalLong;
 
 /**
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java
index 4d0b7ef3b90..fba3f7ae6e9 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java
@@ -17,6 +17,9 @@ public enum RunStatus {
     /** Deployment of the real application was rejected. */
     deploymentFailed,
 
+    /** Deployment timed out waiting for endpoint certificate */
+    endpointCertificateTimeout,
+
     /** Installation of the real application timed out. */
     installationFailed,
 
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java
index 44a93a655a8..17cfd1bdf1d 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java
@@ -26,54 +26,59 @@ import java.util.List;
 public enum Step {
 
     /** Download test-jar and assemble and deploy tester application. */
-    deployTester,
+    deployTester(false),
 
     /** See that tester is done deploying, and is ready to serve. */
-    installTester(deployTester),
+    installTester(false, deployTester),
 
     /** Download and deploy the initial real application, for staging tests. */
-    deployInitialReal(deployTester),
+    deployInitialReal(false, deployTester),
 
     /** See that the real application has had its nodes converge to the initial state. */
-    installInitialReal(deployInitialReal),
+    installInitialReal(false, deployInitialReal),
 
     /** Ask the tester to run its staging setup. */
-    startStagingSetup(installInitialReal, installTester),
+    startStagingSetup(false, installInitialReal, installTester),
 
     /** See that the staging setup is done. */
-    endStagingSetup(startStagingSetup),
+    endStagingSetup(false, startStagingSetup),
 
     /** Download and deploy real application, restarting services if required. */
-    deployReal(endStagingSetup, deployTester),
+    deployReal(false, endStagingSetup, deployTester),
 
     /** See that real application has had its nodes converge to the wanted version and generation. */
-    installReal(deployReal),
+    installReal(false, deployReal),
 
     /** Ask the tester to run its tests. */
-    startTests(installReal, installTester),
+    startTests(false, installReal, installTester),
 
     /** See that the tests are done running. */
-    endTests(startTests),
+    endTests(false, startTests),
 
     /** Fetch and store Vespa logs from the log server cluster of the deployment -- used for test and dev deployments. */
-    copyVespaLogs(installReal, endTests),
+    copyVespaLogs(true, installReal, endTests),
 
     /** Delete the real application -- used for test deployments. */
-    deactivateReal(deployInitialReal, deployReal, endTests, copyVespaLogs),
+    deactivateReal(true, deployInitialReal, deployReal, endTests, copyVespaLogs),
 
     /** Deactivate the tester. */
-    deactivateTester(deployTester, endTests),
+    deactivateTester(true, deployTester, endTests),
 
     /** Report completion to the deployment orchestration machinery. */
-    report(deactivateReal, deactivateTester);
+    report(true, installReal, deactivateReal, deactivateTester);
 
 
+    private final boolean alwaysRun;
     private final List<Step> prerequisites;
 
-    Step(Step... prerequisites) {
+    Step(boolean alwaysRun, Step... prerequisites) {
+        this.alwaysRun = alwaysRun;
         this.prerequisites = ImmutableList.copyOf(prerequisites);
     }
 
+    /** Returns whether this is a cleanup-step, and should always run, regardless of job outcome, when specified in a job. */
+    public boolean alwaysRun() { return alwaysRun; }
+
     /** Returns the prerequisite steps that must be successfully completed before this, assuming the job contains these steps. */
     public List<Step> prerequisites() { return prerequisites; }
 
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializer.java
index a083e2d0210..14e18b2c58a 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializer.java
@@ -7,8 +7,9 @@ import com.yahoo.config.provision.SystemName;
 import com.yahoo.config.provision.zone.ZoneId;
 import com.yahoo.slime.Cursor;
 import com.yahoo.slime.Slime;
-import com.yahoo.vespa.config.SlimeUtils;
+import com.yahoo.slime.SlimeUtils;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
+import com.yahoo.vespa.hosted.controller.application.Endpoint;
 
 import java.io.IOException;
 import java.io.UncheckedIOException;
@@ -32,7 +33,7 @@ public class TestConfigSerializer {
     public Slime configSlime(ApplicationId id,
                              JobType type,
                              boolean isCI,
-                             Map<ZoneId, Map<ClusterSpec.Id, URI>> deployments,
+                             Map<ZoneId, List<Endpoint>> deployments,
                              Map<ZoneId, List<String>> clusters) {
         Slime slime = new Slime();
         Cursor root = slime.setObject();
@@ -42,19 +43,19 @@ public class TestConfigSerializer {
         root.setString("system", system.value());
         root.setBool("isCI", isCI);
 
-        Cursor endpointsObject = root.setObject("endpoints"); // TODO jvenstad: remove.
+        // TODO jvenstad: remove when clients can be updated
+        Cursor endpointsObject = root.setObject("endpoints");
         deployments.forEach((zone, endpoints) -> {
             Cursor endpointArray = endpointsObject.setArray(zone.value());
-            for (URI endpoint : endpoints.values())
-                endpointArray.addString(endpoint.toString());
+            for (Endpoint endpoint : endpoints)
+                endpointArray.addString(endpoint.url().toString());
         });
 
         Cursor zoneEndpointsObject = root.setObject("zoneEndpoints");
         deployments.forEach((zone, endpoints) -> {
             Cursor clusterEndpointsObject = zoneEndpointsObject.setObject(zone.value());
-            endpoints.forEach((cluster, endpoint) -> {
-                clusterEndpointsObject.setString(cluster.value(), endpoint.toString());
-            });
+            for (Endpoint endpoint : endpoints)
+                clusterEndpointsObject.setString(endpoint.name(), endpoint.url().toString());
         });
 
         if ( ! clusters.isEmpty()) {
@@ -73,7 +74,7 @@ public class TestConfigSerializer {
     public byte[] configJson(ApplicationId id,
                              JobType type,
                              boolean isCI,
-                             Map<ZoneId, Map<ClusterSpec.Id, URI>> deployments,
+                             Map<ZoneId, List<Endpoint>> deployments,
                              Map<ZoneId, List<String>> clusters) {
         try {
             return SlimeUtils.toJsonBytes(configSlime(id, type, isCI, deployments, clusters));
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Versions.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Versions.java
index c4135120e3d..7a0349d7737 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Versions.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Versions.java
@@ -58,9 +58,9 @@ public class Versions {
 
     /** Returns whether source versions are present and match those of the given job other versions. */
     public boolean sourcesMatchIfPresent(Versions versions) {
-        return ( ! sourcePlatform.filter(version -> ! version.equals(targetPlatform)).isPresent() ||
+        return (sourcePlatform.filter(version -> ! version.equals(targetPlatform)).isEmpty() ||
                 sourcePlatform.equals(versions.sourcePlatform())) &&
-               ( ! sourceApplication.filter(version -> ! version.equals(targetApplication)).isPresent() ||
+               (sourceApplication.filter(version -> ! version.equals(targetApplication)).isEmpty() ||
                 sourceApplication.equals(versions.sourceApplication()));
     }
 
@@ -136,7 +136,7 @@ public class Versions {
     }
 
     private static <T extends Comparable<T>> Optional<T> max(Optional<T> o1, Optional<T> o2) {
-        return ! o1.isPresent() ? o2 : ! o2.isPresent() ? o1 : o1.get().compareTo(o2.get()) >= 0 ? o1 : o2;
+        return o1.isEmpty() ? o2 : o2.isEmpty() ? o1 : o1.get().compareTo(o2.get()) >= 0 ? o1 : o2;
     }
 
 }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java
index c854c5b45bf..9c426d23d82 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java
@@ -1,12 +1,11 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.maintenance;
 
-import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.zone.ZoneId;
 import com.yahoo.vespa.hosted.controller.Application;
-import com.yahoo.vespa.hosted.controller.Instance;
 import com.yahoo.vespa.hosted.controller.ApplicationController;
 import com.yahoo.vespa.hosted.controller.Controller;
+import com.yahoo.vespa.hosted.controller.Instance;
 import com.yahoo.vespa.hosted.controller.api.integration.organization.ApplicationSummary;
 import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId;
 import com.yahoo.vespa.hosted.controller.api.integration.organization.OwnershipIssues;
@@ -14,12 +13,10 @@ import com.yahoo.vespa.hosted.controller.api.integration.organization.User;
 import com.yahoo.vespa.hosted.controller.application.ApplicationList;
 import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId;
 import com.yahoo.vespa.hosted.controller.tenant.Tenant;
-import com.yahoo.vespa.hosted.controller.tenant.UserTenant;
 import com.yahoo.yolean.Exceptions;
 
 import java.time.Duration;
 import java.util.HashMap;
-import java.util.Optional;
 import java.util.logging.Level;
 
 /**
@@ -49,7 +46,7 @@ public class ApplicationOwnershipConfirmer extends Maintainer {
 
     /** File an ownership issue with the owners of all applications we know about. */
     private void confirmApplicationOwnerships() {
-        ApplicationList.from(controller().applications().asList())
+        applications()
                        .withProjectId()
                        .withProductionDeployment()
                        .asList()
@@ -57,14 +54,13 @@ public class ApplicationOwnershipConfirmer extends Maintainer {
                        .filter(application -> application.createdAt().isBefore(controller().clock().instant().minus(Duration.ofDays(90))))
                        .forEach(application -> {
                            try {
-                               Tenant tenant = tenantOf(application.id());
-                               tenant.contact().ifPresent(contact -> { // TODO jvenstad: Makes sense to require, and run this only in main?
-                                   ownershipIssues.confirmOwnership(application.ownershipIssueId(),
-                                                                    summaryOf(application.id()),
-                                                                    determineAssignee(tenant, application),
-                                                                    contact)
-                                                  .ifPresent(newIssueId -> store(newIssueId, application.id()));
-                               });
+                               // TODO jvenstad: Makes sense to require, and run this only in main?
+                               tenantOf(application.id()).contact().flatMap(contact -> {
+                                   return ownershipIssues.confirmOwnership(application.ownershipIssueId(),
+                                                                           summaryOf(application.id()),
+                                                                           determineAssignee(application),
+                                                                           contact);
+                               }).ifPresent(newIssueId -> store(newIssueId, application.id()));
                            }
                            catch (RuntimeException e) { // Catch errors due to wrong data in the controller, or issues client timeout.
                                log.log(Level.INFO, "Exception caught when attempting to file an issue for '" + application.id() + "': " + Exceptions.toMessageString(e));
@@ -90,11 +86,11 @@ public class ApplicationOwnershipConfirmer extends Maintainer {
 
     /** Escalate ownership issues which have not been closed before a defined amount of time has passed. */
     private void ensureConfirmationResponses() {
-        for (Application application : controller().applications().asList())
+        for (Application application : applications())
             application.ownershipIssueId().ifPresent(issueId -> {
                 try {
                     Tenant tenant = tenantOf(application.id());
-                    ownershipIssues.ensureResponse(issueId, tenant.type() == Tenant.Type.athenz ? tenant.contact() : Optional.empty());
+                    ownershipIssues.ensureResponse(issueId, tenant.contact());
                 }
                 catch (RuntimeException e) {
                     log.log(Level.INFO, "Exception caught when attempting to escalate issue with id '" + issueId + "': " + Exceptions.toMessageString(e));
@@ -103,7 +99,7 @@ public class ApplicationOwnershipConfirmer extends Maintainer {
     }
 
     private void updateConfirmedApplicationOwners() {
-        ApplicationList.from(controller().applications().asList())
+        applications()
                        .withProjectId()
                        .withProductionDeployment()
                        .asList()
@@ -118,8 +114,12 @@ public class ApplicationOwnershipConfirmer extends Maintainer {
                        });
     }
 
-    private User determineAssignee(Tenant tenant, Application application) {
-        return application.owner().orElse(tenant instanceof UserTenant ? userFor(tenant) : null);
+    private ApplicationList applications() {
+        return ApplicationList.from(controller().applications().readable());
+    }
+
+    private User determineAssignee(Application application) {
+        return application.owner().orElse(null);
     }
 
     private Tenant tenantOf(TenantAndApplicationId applicationId) {
@@ -127,10 +127,6 @@ public class ApplicationOwnershipConfirmer extends Maintainer {
                 .orElseThrow(() -> new IllegalStateException("No tenant found for application " + applicationId));
     }
 
-    protected User userFor(Tenant tenant) {
-        return User.from(tenant.name().value().replaceFirst(Tenant.userPrefix, ""));
-    }
-
     protected void store(IssueId issueId, TenantAndApplicationId applicationId) {
         controller().applications().lockApplicationIfPresent(applicationId, application ->
                 controller().applications().store(application.withOwnershipIssueId(issueId)));
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudEventReporter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudEventReporter.java
index bd8faaed2e2..199e9e4e7ae 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudEventReporter.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudEventReporter.java
@@ -49,6 +49,9 @@ public class CloudEventReporter extends Maintainer {
         for (var awsRegion : zonesByCloudNativeRegion.keySet()) {
             List<CloudEvent> events = eventFetcher.getEvents(awsRegion);
             for (var event : events) {
+                log.info(String.format("Retrieved event %s, affecting the following instances: %s",
+                        event.instanceEventId,
+                        event.affectedInstances));
                 List<String> deprovisionedHosts = deprovisionHosts(awsRegion, event);
                 submitIssue(event, deprovisionedHosts);
             }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterInfoMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterInfoMaintainer.java
deleted file mode 100644
index 88c1a48653a..00000000000
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterInfoMaintainer.java
+++ /dev/null
@@ -1,91 +0,0 @@
-// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.controller.maintenance;
-
-import com.yahoo.config.provision.ClusterSpec;
-import com.yahoo.config.provision.HostName;
-import com.yahoo.vespa.hosted.controller.Application;
-import com.yahoo.vespa.hosted.controller.Controller;
-import com.yahoo.vespa.hosted.controller.Instance;
-import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
-import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node;
-import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepository;
-import com.yahoo.vespa.hosted.controller.application.ClusterInfo;
-import com.yahoo.vespa.hosted.controller.application.Deployment;
-
-import java.time.Duration;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-import java.util.stream.Collectors;
-
-/**
- * Maintains information about hardware, hostnames and cluster specifications.
- *
- * This is used to calculate cost metrics for the application api.
- *
- * @author smorgrav
- */
-public class ClusterInfoMaintainer extends Maintainer {
-
-    private static final Logger log = Logger.getLogger(ClusterInfoMaintainer.class.getName());
-    
-    private final Controller controller;
-    private final NodeRepository nodeRepository;
-
-    ClusterInfoMaintainer(Controller controller, Duration duration, JobControl jobControl) {
-        super(controller, duration, jobControl);
-        this.controller = controller;
-        this.nodeRepository = controller.serviceRegistry().configServer().nodeRepository();
-    }
-
-    private Map<ClusterSpec.Id, ClusterInfo> getClusterInfo(List<Node> nodes) {
-        Map<ClusterSpec.Id, ClusterInfo> infoMap = new HashMap<>();
-
-        // Group nodes by clusterid
-        Map<String, List<Node>> clusters = nodes.stream().collect(Collectors.groupingBy(Node::clusterId));
-
-        // For each cluster - get info
-        for (String id : clusters.keySet()) {
-            List<Node> clusterNodes = clusters.get(id);
-
-            // Assume they are all equal and use first node as a representative for the cluster
-            Node node = clusterNodes.get(0);
-
-            // Add to map
-            List<String> hostnames = clusterNodes.stream()
-                                                 .map(Node::hostname)
-                                                 .map(HostName::value)
-                                                 .collect(Collectors.toList());
-            ClusterInfo info = new ClusterInfo(node.flavor(), node.cost(),
-                                               node.resources().vcpu(), node.resources().memoryGb(), node.resources().diskGb(),
-                                               ClusterSpec.Type.from(node.clusterType().name()), hostnames);
-            infoMap.put(new ClusterSpec.Id(id), info);
-        }
-
-        return infoMap;
-    }
-
-    @Override
-    protected void maintain() {
-        for (Application application : controller().applications().asList()) {
-            for (Instance instance : application.instances().values()) {
-                for (Deployment deployment : instance.deployments().values()) {
-                    DeploymentId deploymentId = new DeploymentId(instance.id(), deployment.zone());
-                    try {
-                        var nodes = nodeRepository.list(deploymentId.zoneId(), deploymentId.applicationId());
-                        Map<ClusterSpec.Id, ClusterInfo> clusterInfo = getClusterInfo(nodes);
-                        controller().applications().lockApplicationIfPresent(application.id(), lockedApplication ->
-                                controller.applications().store(lockedApplication.with(instance.name(),
-                                                                                       locked -> locked.withClusterInfo(deployment.zone(), clusterInfo))));
-                    }
-                    catch (Exception e) {
-                        log.log(Level.WARNING, "Failing getting cluster information for " + deploymentId, e);
-                    }
-                }
-            }
-        }
-    }
-
-}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainer.java
index cbe9d8c70c1..d1f2087ce12 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainer.java
@@ -6,6 +6,7 @@ import com.yahoo.log.LogLevel;
 import com.yahoo.vespa.hosted.controller.Controller;
 import com.yahoo.vespa.hosted.controller.LockedTenant;
 import com.yahoo.vespa.hosted.controller.TenantController;
+import com.yahoo.vespa.hosted.controller.api.integration.organization.Contact;
 import com.yahoo.vespa.hosted.controller.api.integration.organization.ContactRetriever;
 import com.yahoo.vespa.hosted.controller.tenant.Tenant;
 import com.yahoo.yolean.Exceptions;
@@ -15,6 +16,8 @@ import java.util.Optional;
 import java.util.function.Predicate;
 import java.util.logging.Logger;
 
+import static java.util.logging.Level.INFO;
+
 /**
  * Periodically fetch and store contact information for tenants.
  *
@@ -35,16 +38,21 @@ public class ContactInformationMaintainer extends Maintainer {
     protected void maintain() {
         TenantController tenants = controller().tenants();
         for (Tenant tenant : tenants.asList()) {
+            log.log(INFO, "Updating contact information for " + tenant);
             try {
                 switch (tenant.type()) {
-                    case athenz: tenants.lockIfPresent(tenant.name(), LockedTenant.Athenz.class, lockedTenant ->
-                            tenants.store(lockedTenant.with(contactRetriever.getContact(lockedTenant.get().propertyId()))));
-                        return;
-                    case user: tenants.lockIfPresent(tenant.name(), LockedTenant.User.class, lockedTenant ->
-                            tenants.store(lockedTenant.with(contactRetriever.getContact(Optional.empty()))));
-                        return;
-                    case cloud: return;
-                    default: throw new IllegalArgumentException("Unexpected tenant type '" + tenant.type() + "'.");
+                    case athenz:
+                        tenants.lockIfPresent(tenant.name(), LockedTenant.Athenz.class, lockedTenant -> {
+                            Contact contact = contactRetriever.getContact(lockedTenant.get().propertyId());
+                            log.log(INFO, "Contact found for " + tenant + " was " +
+                                          (Optional.of(contact).equals(tenant.contact()) ? "un" : "") + "changed");
+                            tenants.store(lockedTenant.with(contact));
+                        });
+                        break;
+                    case cloud:
+                        break;
+                    default:
+                        throw new IllegalArgumentException("Unexpected tenant type '" + tenant.type() + "'.");
                 }
             } catch (Exception e) {
                 log.log(LogLevel.WARNING, "Failed to update contact information for " + tenant + ": " +
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
index ec209c5ed98..18fe96fc9b2 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
@@ -32,7 +32,6 @@ public class ControllerMaintenance extends AbstractComponent {
     private final VersionStatusUpdater versionStatusUpdater;
     private final Upgrader upgrader;
     private final ReadyJobsTrigger readyJobsTrigger;
-    private final ClusterInfoMaintainer clusterInfoMaintainer;
     private final DeploymentMetricsMaintainer deploymentMetricsMaintainer;
     private final ApplicationOwnershipConfirmer applicationOwnershipConfirmer;
     private final SystemUpgrader systemUpgrader;
@@ -64,7 +63,6 @@ public class ControllerMaintenance extends AbstractComponent {
         versionStatusUpdater = new VersionStatusUpdater(controller, Duration.ofMinutes(3), jobControl);
         upgrader = new Upgrader(controller, maintenanceInterval, jobControl, curator);
         readyJobsTrigger = new ReadyJobsTrigger(controller, Duration.ofMinutes(1), jobControl);
-        clusterInfoMaintainer = new ClusterInfoMaintainer(controller, Duration.ofHours(2), jobControl);
         deploymentMetricsMaintainer = new DeploymentMetricsMaintainer(controller, Duration.ofMinutes(5), jobControl);
         applicationOwnershipConfirmer = new ApplicationOwnershipConfirmer(controller, Duration.ofHours(12), jobControl, controller.serviceRegistry().ownershipIssues());
         systemUpgrader = new SystemUpgrader(controller, Duration.ofMinutes(1), jobControl);
@@ -95,7 +93,6 @@ public class ControllerMaintenance extends AbstractComponent {
         versionStatusUpdater.deconstruct();
         upgrader.deconstruct();
         readyJobsTrigger.deconstruct();
-        clusterInfoMaintainer.deconstruct();
         deploymentMetricsMaintainer.deconstruct();
         applicationOwnershipConfirmer.deconstruct();
         systemUpgrader.deconstruct();
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainer.java
index ff88805f957..5b792f384e5 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainer.java
@@ -1,12 +1,11 @@
 // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.maintenance;
 
-import com.yahoo.config.provision.CloudName;
 import com.yahoo.config.provision.SystemName;
 import com.yahoo.vespa.hosted.controller.Controller;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepository;
 import com.yahoo.vespa.hosted.controller.api.integration.resource.CostReportConsumer;
-import com.yahoo.vespa.hosted.controller.restapi.cost.CostCalculator;
+import com.yahoo.vespa.hosted.controller.metric.CostCalculator;
 
 import java.time.Clock;
 import java.time.Duration;
@@ -34,7 +33,8 @@ public class CostReportMaintainer extends Maintainer {
 
     @Override
     protected void maintain() {
-        consumer.consume(CostCalculator.resourceShareByPropertyToCsv(nodeRepository, controller(), clock, consumer.fixedAllocations(), CloudName.from("yahoo")));
+        var csv = CostCalculator.resourceShareByPropertyToCsv(nodeRepository, controller(), clock, consumer.fixedAllocations());
+        consumer.consume(csv);
     }
 
 }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirer.java
index 7756e6b23a7..3160e7aef1d 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirer.java
@@ -24,7 +24,7 @@ public class DeploymentExpirer extends Maintainer {
 
     @Override
     protected void maintain() {
-        for (Application application : controller().applications().asList())
+        for (Application application : controller().applications().readable())
             for (Instance instance : application.instances().values())
                 for (Deployment deployment : instance.deployments().values()) {
                     if ( ! isExpired(deployment)) continue;
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java
index 3be15b67252..848bd451398 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java
@@ -19,9 +19,7 @@ import java.time.Duration;
 import java.util.Collection;
 import java.util.List;
 import java.util.Optional;
-import java.util.Set;
 import java.util.logging.Level;
-import java.util.stream.Collectors;
 
 import static com.yahoo.vespa.hosted.controller.versions.VespaVersion.Confidence.broken;
 
@@ -35,7 +33,7 @@ public class DeploymentIssueReporter extends Maintainer {
 
     static final Duration maxFailureAge = Duration.ofDays(2);
     static final Duration maxInactivity = Duration.ofDays(4);
-    static final Duration upgradeGracePeriod = Duration.ofHours(4);
+    static final Duration upgradeGracePeriod = Duration.ofHours(2);
 
     private final DeploymentIssues deploymentIssues;
 
@@ -53,7 +51,7 @@ public class DeploymentIssueReporter extends Maintainer {
 
     /** Returns the applications to maintain issue status for. */
     private List<Application> applications() {
-        return ApplicationList.from(controller().applications().asList())
+        return ApplicationList.from(controller().applications().readable())
                               .withProjectId()
                               .asList();
     }
@@ -105,16 +103,12 @@ public class DeploymentIssueReporter extends Maintainer {
                            .orElseThrow(() -> new IllegalStateException("No tenant found for application " + applicationId));
     }
 
-    private User userFor(Tenant tenant) {
-        return User.from(tenant.name().value().replaceFirst(Tenant.userPrefix, ""));
-    }
-
     /** File an issue for applicationId, if it doesn't already have an open issue associated with it. */
     private void fileDeploymentIssueFor(Application application) {
         try {
             Tenant tenant = ownerOf(application.id());
             tenant.contact().ifPresent(contact -> {
-                User assignee = tenant.type() == Tenant.Type.user ? userFor(tenant) : application.owner().orElse(null);
+                User assignee = application.owner().orElse(null);
                 Optional<IssueId> ourIssueId = application.deploymentIssueId();
                 IssueId issueId = deploymentIssues.fileUnlessOpen(ourIssueId, application.id().defaultInstance(), assignee, contact);
                 store(application.id(), issueId);
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java
index 162e46e19d2..1ce1e7f4c0c 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java
@@ -3,7 +3,6 @@ package com.yahoo.vespa.hosted.controller.maintenance;
 
 import com.yahoo.config.provision.SystemName;
 import com.yahoo.log.LogLevel;
-import com.yahoo.vespa.hosted.controller.Application;
 import com.yahoo.vespa.hosted.controller.Instance;
 import com.yahoo.vespa.hosted.controller.ApplicationController;
 import com.yahoo.vespa.hosted.controller.Controller;
@@ -12,7 +11,6 @@ import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics;
 import com.yahoo.yolean.Exceptions;
 
 import java.time.Duration;
-import java.util.List;
 import java.util.concurrent.ForkJoinPool;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -48,7 +46,7 @@ public class DeploymentMetricsMaintainer extends Maintainer {
         // Run parallel stream inside a custom ForkJoinPool so that we can control the number of threads used
         ForkJoinPool pool = new ForkJoinPool(applicationsToUpdateInParallel);
         pool.submit(() ->
-            applications.asList().parallelStream().forEach(application -> {
+            applications.readable().parallelStream().forEach(application -> {
                 for (Instance instance : application.instances().values())
                     for (Deployment deployment : instance.deployments().values()) {
                         attempts.incrementAndGet();
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java
index 0a890834c6e..056acd174a9 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java
@@ -112,7 +112,7 @@ public abstract class InfrastructureUpgrader extends Maintainer {
         }
     }
 
-    private class UnreachableNodeRepositoryException extends RuntimeException {
+    private static class UnreachableNodeRepositoryException extends RuntimeException {
         private UnreachableNodeRepositoryException(String reason) {
             super(reason);
         }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobControl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobControl.java
index 6aa1b89c605..898af474a8c 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobControl.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobControl.java
@@ -4,11 +4,9 @@ package com.yahoo.vespa.hosted.controller.maintenance;
 import com.yahoo.vespa.curator.Lock;
 import com.yahoo.vespa.hosted.controller.persistence.CuratorDb;
 
-import java.util.HashSet;
 import java.util.LinkedHashSet;
 import java.util.Set;
 import java.util.concurrent.ConcurrentSkipListSet;
-import java.util.logging.Logger;
 
 /**
  * Provides status and control over running maintenance jobs.
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Maintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Maintainer.java
index 9c3c1dc1f5e..3e283d6c54a 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Maintainer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Maintainer.java
@@ -6,6 +6,7 @@ import com.yahoo.config.provision.HostName;
 import com.yahoo.config.provision.SystemName;
 import com.yahoo.vespa.curator.Lock;
 import com.yahoo.vespa.hosted.controller.Controller;
+import com.yahoo.yolean.Exceptions;
 
 import java.time.Duration;
 import java.time.Instant;
@@ -51,7 +52,7 @@ public abstract class Maintainer extends AbstractComponent implements Runnable {
         this.name = name;
         this.activeSystems = Set.copyOf(activeSystems);
 
-        service = new ScheduledThreadPoolExecutor(1);
+        service = new ScheduledThreadPoolExecutor(1, r -> new Thread(r, name() + "-worker"));
         long delay = staggeredDelay(controller.curator().cluster(), controller.hostname(), controller.clock().instant(), interval);
         service.scheduleAtFixedRate(this, delay, interval.toMillis(), TimeUnit.MILLISECONDS);
         jobControl.started(name());
@@ -75,13 +76,23 @@ public abstract class Maintainer extends AbstractComponent implements Runnable {
             // another controller instance is running this job at the moment; ok
         }
         catch (Throwable t) {
-            log.log(Level.WARNING, this + " failed. Will retry in " + maintenanceInterval.toMinutes() + " minutes", t);
+            log.log(Level.WARNING, "Maintainer " + name() + " failed. Will retry in " +
+                                   maintenanceInterval + ": " + Exceptions.toMessageString(t));
         }
     }
 
     @Override
     public void deconstruct() {
-        this.service.shutdown();
+        var timeout = Duration.ofSeconds(30);
+        service.shutdown();
+        try {
+            if (!service.awaitTermination(timeout.toMillis(), TimeUnit.MILLISECONDS)) {
+                log.log(Level.WARNING, "Maintainer " + name() + " failed to shutdown " +
+                                       "within " + timeout);
+            }
+        } catch (InterruptedException e) {
+            throw new RuntimeException(e);
+        }
     }
 
     /** Called once each time this maintenance job should run */
@@ -104,8 +115,7 @@ public abstract class Maintainer extends AbstractComponent implements Runnable {
             return interval.toMillis();
 
         long offset = cluster.indexOf(host) * interval.toMillis() / cluster.size();
-        long timeUntilNextRun = Math.floorMod(offset - now.toEpochMilli(), interval.toMillis());
-        return timeUntilNextRun + interval.toMillis() / cluster.size();
+        return Math.floorMod(offset - now.toEpochMilli(), interval.toMillis());
     }
 
 }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java
index ccb802d314a..9c414ce8348 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java
@@ -43,7 +43,12 @@ public class MetricsReporter extends Maintainer {
     public static final String REMAINING_ROTATIONS = "remaining_rotations";
     public static final String NAME_SERVICE_REQUESTS_QUEUED = "dns.queuedRequests";
 
-    private static final Duration NODE_UPGRADE_TIMEOUT = Duration.ofHours(1);
+    // The time a node belonging to a system application can spend from being told to upgrade until the upgrade is
+    // completed. Nodes exceeding this time are counted as failures.
+    private static final Duration NODE_UPGRADE_TIMEOUT = Duration.ofMinutes(90);
+
+    // The time a single node can spend performing an OS upgrade after being told to upgrade. Nodes exceeding this time
+    // multiplied by the number of nodes upgrading are counted as failures.
     private static final Duration OS_UPGRADE_TIME_ALLOWANCE_PER_NODE = Duration.ofMinutes(30);
 
     private final Metric metric;
@@ -64,15 +69,15 @@ public class MetricsReporter extends Maintainer {
     }
 
     private void reportRemainingRotations() {
-        try (RotationLock lock = controller().applications().rotationRepository().lock()) {
-            int availableRotations = controller().applications().rotationRepository().availableRotations(lock).size();
+        try (RotationLock lock = controller().routing().rotations().lock()) {
+            int availableRotations = controller().routing().rotations().availableRotations(lock).size();
             metric.set(REMAINING_ROTATIONS, availableRotations, metric.createContext(Map.of()));
         }
     }
 
     private void reportDeploymentMetrics() {
-        ApplicationList applications = ApplicationList.from(controller().applications().asList())
-                                                  .withProductionDeployment();
+        ApplicationList applications = ApplicationList.from(controller().applications().readable())
+                                                      .withProductionDeployment();
         DeploymentStatusList deployments = controller().jobController().deploymentStatuses(applications);
 
         metric.set(DEPLOYMENT_FAIL_METRIC, deploymentFailRatio(deployments) * 100, metric.createContext(Map.of()));
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployer.java
index 37ecef8b41e..83deb71e663 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployer.java
@@ -1,18 +1,9 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.maintenance;
 
-import com.yahoo.config.application.api.DeploymentInstanceSpec;
-import com.yahoo.config.application.api.DeploymentSpec;
-import com.yahoo.config.provision.ApplicationId;
-import com.yahoo.config.provision.InstanceName;
 import com.yahoo.vespa.hosted.controller.Application;
-import com.yahoo.vespa.hosted.controller.Instance;
 import com.yahoo.vespa.hosted.controller.Controller;
 import com.yahoo.vespa.hosted.controller.application.ApplicationList;
-import com.yahoo.vespa.hosted.controller.application.Change;
-import com.yahoo.vespa.hosted.controller.application.InstanceList;
-import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatus;
-import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatusList;
 
 import java.time.Duration;
 
@@ -29,7 +20,7 @@ public class OutstandingChangeDeployer extends Maintainer {
 
     @Override
     protected void maintain() {
-        for (Application application : ApplicationList.from(controller().applications().asList())
+        for (Application application : ApplicationList.from(controller().applications().readable())
                                                       .withProductionDeployment()
                                                       .withDeploymentSpec()
                                                       .asList())
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java
index 866ea183991..34096ba6a10 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java
@@ -18,7 +18,6 @@ import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
 import java.util.Set;
-import java.util.function.Predicate;
 import java.util.stream.Collectors;
 
 /**
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RotationStatusUpdater.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RotationStatusUpdater.java
index ac5612b4e3f..35131d3db80 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RotationStatusUpdater.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RotationStatusUpdater.java
@@ -44,9 +44,9 @@ public class RotationStatusUpdater extends Maintainer {
         var failures = new AtomicInteger(0);
         var attempts = new AtomicInteger(0);
         var lastException = new AtomicReference<Exception>(null);
-        var instancesWithRotations = ApplicationList.from(applications.asList()).hasRotation().asList().stream()
-                .flatMap(application -> application.instances().values().stream())
-                .filter(instance -> ! instance.rotations().isEmpty());
+        var instancesWithRotations = ApplicationList.from(applications.readable()).hasRotation().asList().stream()
+                                                    .flatMap(application -> application.instances().values().stream())
+                                                    .filter(instance -> ! instance.rotations().isEmpty());
 
         // Run parallel stream inside a custom ForkJoinPool so that we can control the number of threads used
         var pool = new ForkJoinPool(applicationsToUpdateInParallel);
@@ -82,7 +82,7 @@ public class RotationStatusUpdater extends Maintainer {
     private RotationStatus getStatus(Instance instance) {
         var statusMap = new LinkedHashMap<RotationId, RotationStatus.Targets>();
         for (var assignedRotation : instance.rotations()) {
-            var rotation = applications.rotationRepository().getRotation(assignedRotation.rotationId());
+            var rotation = controller().routing().rotations().getRotation(assignedRotation.rotationId());
             if (rotation.isEmpty()) continue;
             var targets = service.getHealthStatus(rotation.get().name()).entrySet().stream()
                                  .collect(Collectors.toMap(Map.Entry::getKey, (kv) -> from(kv.getValue())));
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java
index da24d758320..1e22c59636a 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java
@@ -25,7 +25,6 @@ import java.util.logging.Logger;
 import java.util.stream.Collectors;
 
 import static com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger.ChangesToCancel.PLATFORM;
-import static java.util.Comparator.comparing;
 import static java.util.Comparator.naturalOrder;
 
 /**
@@ -104,7 +103,7 @@ public class Upgrader extends Maintainer {
 
     /** Returns a list of all production application instances, except those which are pinned, which we should not manipulate here. */
     private InstanceList instances() {
-        return InstanceList.from(controller().jobController().deploymentStatuses(ApplicationList.from(controller().applications().asList())))
+        return InstanceList.from(controller().jobController().deploymentStatuses(ApplicationList.from(controller().applications().readable())))
                            .withProductionDeployment()
                            .unpinned();
     }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostCalculator.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/metric/CostCalculator.java
index ec01b3817a7..4cc4ee0386c 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostCalculator.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/metric/CostCalculator.java
@@ -1,5 +1,5 @@
 // Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.controller.restapi.cost;
+package com.yahoo.vespa.hosted.controller.metric;
 
 import com.yahoo.config.provision.CloudName;
 import com.yahoo.config.provision.Environment;
@@ -28,12 +28,12 @@ import static com.yahoo.yolean.Exceptions.uncheck;
 public class CostCalculator {
 
     private static final double SELF_HOSTED_DISCOUNT = .5;
+    private static final CloudName cloudName = CloudName.from("yahoo");
 
     public static String resourceShareByPropertyToCsv(NodeRepository nodeRepository,
                                                       Controller controller,
                                                       Clock clock,
-                                                      Map<Property, ResourceAllocation> fixedAllocations,
-                                                      CloudName cloudName) {
+                                                      Map<Property, ResourceAllocation> fixedAllocations) {
 
         var date = DateTimeFormatter.ofPattern("yyyy-MM-dd").withZone(ZoneId.of("UTC")).format(clock.instant());
 
@@ -61,14 +61,12 @@ public class CostCalculator {
         }
 
         // Add fixed allocations from config
-        if (cloudName.equals(CloudName.from("yahoo"))) {
-            for (var kv : fixedAllocations.entrySet()) {
-                var property = kv.getKey();
-                var allocation = allocationByProperty.getOrDefault(property, ResourceAllocation.ZERO);
-                var discountedFixedAllocation = kv.getValue().multiply(SELF_HOSTED_DISCOUNT);
-                allocationByProperty.put(property, allocation.plus(discountedFixedAllocation));
-                totalAllocation = totalAllocation.plus(discountedFixedAllocation);
-            }
+        for (var kv : fixedAllocations.entrySet()) {
+            var property = kv.getKey();
+            var allocation = allocationByProperty.getOrDefault(property, ResourceAllocation.ZERO);
+            var discountedFixedAllocation = kv.getValue().multiply(SELF_HOSTED_DISCOUNT);
+            allocationByProperty.put(property, allocation.plus(discountedFixedAllocation));
+            totalAllocation = totalAllocation.plus(discountedFixedAllocation);
         }
 
         return toCsv(allocationByProperty, date, totalAllocation);
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java
index b730b63d426..121347c4b61 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java
@@ -6,7 +6,6 @@ import com.google.common.cache.CacheBuilder;
 import com.google.common.hash.Hashing;
 import com.google.common.util.concurrent.UncheckedExecutionException;
 import com.yahoo.component.Version;
-import com.yahoo.config.application.api.DeploymentInstanceSpec;
 import com.yahoo.config.application.api.DeploymentSpec;
 import com.yahoo.config.application.api.ValidationOverrides;
 import com.yahoo.config.provision.ClusterSpec;
@@ -19,7 +18,7 @@ import com.yahoo.slime.Cursor;
 import com.yahoo.slime.Inspector;
 import com.yahoo.slime.ObjectTraverser;
 import com.yahoo.slime.Slime;
-import com.yahoo.vespa.config.SlimeUtils;
+import com.yahoo.slime.SlimeUtils;
 import com.yahoo.vespa.hosted.controller.Application;
 import com.yahoo.vespa.hosted.controller.Instance;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion;
@@ -29,7 +28,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId;
 import com.yahoo.vespa.hosted.controller.api.integration.organization.User;
 import com.yahoo.vespa.hosted.controller.application.AssignedRotation;
 import com.yahoo.vespa.hosted.controller.application.Change;
-import com.yahoo.vespa.hosted.controller.application.ClusterInfo;
 import com.yahoo.vespa.hosted.controller.application.Deployment;
 import com.yahoo.vespa.hosted.controller.application.DeploymentActivity;
 import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics;
@@ -55,7 +53,6 @@ import java.util.OptionalInt;
 import java.util.OptionalLong;
 import java.util.Set;
 import java.util.concurrent.ExecutionException;
-import java.util.stream.Collectors;
 
 /**
  * Serializes {@link Application}s to/from slime.
@@ -92,6 +89,7 @@ public class ApplicationSerializer {
     private static final String pemDeployKeysField = "pemDeployKeys";
     private static final String assignedRotationClusterField = "clusterId";
     private static final String assignedRotationRotationField = "rotationId";
+    private static final String assignedRotationRegionsField = "regions";
     private static final String versionField = "version";
 
     // Instance fields
@@ -128,16 +126,6 @@ public class ApplicationSerializer {
     private static final String jobTypeField = "jobType";
     private static final String pausedUntilField = "pausedUntil";
 
-    // ClusterInfo fields
-    private static final String clusterInfoField = "clusterInfo";
-    private static final String clusterInfoFlavorField = "flavor";
-    private static final String clusterInfoCostField = "cost";
-    private static final String clusterInfoCpuField = "flavorCpu";
-    private static final String clusterInfoMemField = "flavorMem";
-    private static final String clusterInfoDiskField = "flavorDisk";
-    private static final String clusterInfoTypeField = "clusterType";
-    private static final String clusterInfoHostnamesField = "hostnames";
-
     // Deployment metrics fields
     private static final String deploymentMetricsField = "metrics";
     private static final String deploymentMetricsQPSField = "queriesPerSecond";
@@ -190,7 +178,7 @@ public class ApplicationSerializer {
             instanceObject.setString(instanceNameField, instance.name().value());
             deploymentsToSlime(instance.deployments().values(), instanceObject.setArray(deploymentsField));
             toSlime(instance.jobPauses(), instanceObject.setObject(deploymentJobsField));
-            assignedRotationsToSlime(instance.rotations(), instanceObject, assignedRotationsField);
+            assignedRotationsToSlime(instance.rotations(), instanceObject);
             toSlime(instance.rotationStatus(), instanceObject.setArray(rotationStatusField));
             toSlime(instance.change(), instanceObject, deployingField);
         }
@@ -210,7 +198,6 @@ public class ApplicationSerializer {
         object.setString(versionField, deployment.version().toString());
         object.setLong(deployTimeField, deployment.at().toEpochMilli());
         toSlime(deployment.applicationVersion(), object.setObject(applicationPackageRevisionField));
-        clusterInfoToSlime(deployment.clusterInfo(), object);
         deploymentMetricsToSlime(deployment.metrics(), object);
         deployment.activity().lastQueried().ifPresent(instant -> object.setLong(lastQueriedField, instant.toEpochMilli()));
         deployment.activity().lastWritten().ifPresent(instant -> object.setLong(lastWrittenField, instant.toEpochMilli()));
@@ -232,41 +219,19 @@ public class ApplicationSerializer {
         }
     }
 
-    private void clusterInfoToSlime(Map<ClusterSpec.Id, ClusterInfo> clusters, Cursor object) {
-        Cursor root = object.setObject(clusterInfoField);
-        for (Map.Entry<ClusterSpec.Id, ClusterInfo> entry : clusters.entrySet()) {
-            toSlime(entry.getValue(), root.setObject(entry.getKey().value()));
-        }
-    }
-
-    private void toSlime(ClusterInfo info, Cursor object) {
-        object.setString(clusterInfoFlavorField, info.getFlavor());
-        object.setLong(clusterInfoCostField, info.getFlavorCost());
-        object.setDouble(clusterInfoCpuField, info.getFlavorCPU());
-        object.setDouble(clusterInfoMemField, info.getFlavorMem());
-        object.setDouble(clusterInfoDiskField, info.getFlavorDisk());
-        object.setString(clusterInfoTypeField, info.getClusterType().name());
-        Cursor array = object.setArray(clusterInfoHostnamesField);
-        for (String host : info.getHostnames()) {
-            array.addString(host);
-        }
-    }
-
     private void zoneIdToSlime(ZoneId zone, Cursor object) {
         object.setString(environmentField, zone.environment().value());
         object.setString(regionField, zone.region().value());
     }
 
     private void toSlime(ApplicationVersion applicationVersion, Cursor object) {
-        if (applicationVersion.buildNumber().isPresent() && applicationVersion.source().isPresent()) {
-            object.setLong(applicationBuildNumberField, applicationVersion.buildNumber().getAsLong());
-            toSlime(applicationVersion.source().get(), object.setObject(sourceRevisionField));
-            applicationVersion.authorEmail().ifPresent(email -> object.setString(authorEmailField, email));
-            applicationVersion.compileVersion().ifPresent(version -> object.setString(compileVersionField, version.toString()));
-            applicationVersion.buildTime().ifPresent(time -> object.setLong(buildTimeField, time.toEpochMilli()));
-            applicationVersion.sourceUrl().ifPresent(url -> object.setString(sourceUrlField, url));
-            applicationVersion.commit().ifPresent(commit -> object.setString(commitField, commit));
-        }
+        applicationVersion.buildNumber().ifPresent(number -> object.setLong(applicationBuildNumberField, number));
+        applicationVersion.source().ifPresent(source -> toSlime(source, object.setObject(sourceRevisionField)));
+        applicationVersion.authorEmail().ifPresent(email -> object.setString(authorEmailField, email));
+        applicationVersion.compileVersion().ifPresent(version -> object.setString(compileVersionField, version.toString()));
+        applicationVersion.buildTime().ifPresent(time -> object.setLong(buildTimeField, time.toEpochMilli()));
+        applicationVersion.sourceUrl().ifPresent(url -> object.setString(sourceUrlField, url));
+        applicationVersion.commit().ifPresent(commit -> object.setString(commitField, commit));
     }
 
     private void toSlime(SourceRevision sourceRevision, Cursor object) {
@@ -310,13 +275,17 @@ public class ApplicationSerializer {
         });
     }
 
-    private void assignedRotationsToSlime(List<AssignedRotation> rotations, Cursor parent, String fieldName) {
-        var rotationsArray = parent.setArray(fieldName);
+    private void assignedRotationsToSlime(List<AssignedRotation> rotations, Cursor parent) {
+        var rotationsArray = parent.setArray(assignedRotationsField);
         for (var rotation : rotations) {
             var object = rotationsArray.addObject();
             object.setString(assignedRotationEndpointField, rotation.endpointId().id());
             object.setString(assignedRotationRotationField, rotation.rotationId().asString());
             object.setString(assignedRotationClusterField, rotation.clusterId().value());
+            var regionsArray = object.setArray(assignedRotationRegionsField);
+            for (var region : rotation.regions()) {
+                regionsArray.addString(region.value());
+            }
         }
     }
 
@@ -345,7 +314,7 @@ public class ApplicationSerializer {
         ApplicationMetrics metrics = new ApplicationMetrics(root.field(queryQualityField).asDouble(),
                                                             root.field(writeQualityField).asDouble());
         Set<PublicKey> deployKeys = deployKeysFromSlime(root.field(pemDeployKeysField));
-        List<Instance> instances = instancesFromSlime(id, deploymentSpec, root.field(instancesField));
+        List<Instance> instances = instancesFromSlime(id, root.field(instancesField));
         OptionalLong projectId = Serializers.optionalLong(root.field(projectIdField));
         Optional<ApplicationVersion> latestVersion = latestVersionFromSlime(root.field(latestVersionField));
 
@@ -355,19 +324,17 @@ public class ApplicationSerializer {
     }
 
     private Optional<ApplicationVersion> latestVersionFromSlime(Inspector latestVersionObject) {
-        if (latestVersionObject.valid())
-            return Optional.of(applicationVersionFromSlime(latestVersionObject));
-
-        return Optional.empty();
+        return Optional.of(applicationVersionFromSlime(latestVersionObject))
+                       .filter(version -> ! version.isUnknown());
     }
 
-    private List<Instance> instancesFromSlime(TenantAndApplicationId id, DeploymentSpec deploymentSpec, Inspector field) {
+    private List<Instance> instancesFromSlime(TenantAndApplicationId id, Inspector field) {
         List<Instance> instances = new ArrayList<>();
         field.traverse((ArrayTraverser) (name, object) -> {
             InstanceName instanceName = InstanceName.from(object.field(instanceNameField).asString());
             List<Deployment> deployments = deploymentsFromSlime(object.field(deploymentsField));
             Map<JobType, Instant> jobPauses = jobPausesFromSlime(object.field(deploymentJobsField));
-            List<AssignedRotation> assignedRotations = assignedRotationsFromSlime(deploymentSpec, instanceName, object);
+            List<AssignedRotation> assignedRotations = assignedRotationsFromSlime(object);
             RotationStatus rotationStatus = rotationStatusFromSlime(object);
             Change change = changeFromSlime(object.field(deployingField));
             instances.add(new Instance(id.instance(instanceName),
@@ -397,7 +364,6 @@ public class ApplicationSerializer {
                               applicationVersionFromSlime(deploymentObject.field(applicationPackageRevisionField)),
                               Version.fromString(deploymentObject.field(versionField).asString()),
                               Instant.ofEpochMilli(deploymentObject.field(deployTimeField).asLong()),
-                              clusterInfoMapFromSlime(deploymentObject.field(clusterInfoField)),
                               deploymentMetricsFromSlime(deploymentObject.field(deploymentMetricsField)),
                               DeploymentActivity.create(Serializers.optionalInstant(deploymentObject.field(lastQueriedField)),
                                                         Serializers.optionalInstant(deploymentObject.field(lastWrittenField)),
@@ -448,25 +414,6 @@ public class ApplicationSerializer {
         return Collections.unmodifiableMap(rotationStatus);
     }
 
-    private Map<ClusterSpec.Id, ClusterInfo> clusterInfoMapFromSlime   (Inspector object) {
-        Map<ClusterSpec.Id, ClusterInfo> map = new HashMap<>();
-        object.traverse((String name, Inspector value) -> map.put(new ClusterSpec.Id(name), clusterInfoFromSlime(value)));
-        return map;
-    }
-
-    private ClusterInfo clusterInfoFromSlime(Inspector inspector) {
-        String flavor = inspector.field(clusterInfoFlavorField).asString();
-        int cost = (int)inspector.field(clusterInfoCostField).asLong();
-        String type = inspector.field(clusterInfoTypeField).asString();
-        double flavorCpu = inspector.field(clusterInfoCpuField).asDouble();
-        double flavorMem = inspector.field(clusterInfoMemField).asDouble();
-        double flavorDisk = inspector.field(clusterInfoDiskField).asDouble();
-
-        List<String> hostnames = new ArrayList<>();
-        inspector.field(clusterInfoHostnamesField).traverse((ArrayTraverser)(int index, Inspector value) -> hostnames.add(value.asString()));
-        return new ClusterInfo(flavor, cost, flavorCpu, flavorMem, flavorDisk, ClusterSpec.Type.from(type), hostnames);
-    }
-
     private ZoneId zoneIdFromSlime(Inspector object) {
         return ZoneId.from(object.field(environmentField).asString(), object.field(regionField).asString());
     }
@@ -474,22 +421,16 @@ public class ApplicationSerializer {
     private ApplicationVersion applicationVersionFromSlime(Inspector object) {
         if ( ! object.valid()) return ApplicationVersion.unknown;
         OptionalLong applicationBuildNumber = Serializers.optionalLong(object.field(applicationBuildNumberField));
-        Optional<SourceRevision> sourceRevision = sourceRevisionFromSlime(object.field(sourceRevisionField));
-        if (sourceRevision.isEmpty() || applicationBuildNumber.isEmpty()) {
+        if (applicationBuildNumber.isEmpty())
             return ApplicationVersion.unknown;
-        }
+
+        Optional<SourceRevision> sourceRevision = sourceRevisionFromSlime(object.field(sourceRevisionField));
         Optional<String> authorEmail = Serializers.optionalString(object.field(authorEmailField));
         Optional<Version> compileVersion = Serializers.optionalString(object.field(compileVersionField)).map(Version::fromString);
         Optional<Instant> buildTime = Serializers.optionalInstant(object.field(buildTimeField));
         Optional<String> sourceUrl = Serializers.optionalString(object.field(sourceUrlField));
         Optional<String> commit = Serializers.optionalString(object.field(commitField));
 
-        if (authorEmail.isEmpty())
-            return ApplicationVersion.from(sourceRevision.get(), applicationBuildNumber.getAsLong());
-
-        if (compileVersion.isEmpty() || buildTime.isEmpty())
-            return ApplicationVersion.from(sourceRevision.get(), applicationBuildNumber.getAsLong(), authorEmail.get());
-
         return new ApplicationVersion(sourceRevision, applicationBuildNumber, authorEmail, compileVersion, buildTime, sourceUrl, commit);
     }
 
@@ -522,32 +463,20 @@ public class ApplicationSerializer {
         return change;
     }
 
-    private List<AssignedRotation> assignedRotationsFromSlime(DeploymentSpec deploymentSpec, InstanceName instance, Inspector root) {
+    private List<AssignedRotation> assignedRotationsFromSlime(Inspector root) {
         var assignedRotations = new LinkedHashMap<EndpointId, AssignedRotation>();
-
-        root.field(assignedRotationsField).traverse((ArrayTraverser) (idx, inspector) -> {
+        root.field(assignedRotationsField).traverse((ArrayTraverser) (i, inspector) -> {
             var clusterId = new ClusterSpec.Id(inspector.field(assignedRotationClusterField).asString());
             var endpointId = EndpointId.of(inspector.field(assignedRotationEndpointField).asString());
             var rotationId = new RotationId(inspector.field(assignedRotationRotationField).asString());
-            var regions = deploymentSpec.instance(instance)
-                                        .map(spec -> globalEndpointRegions(spec, endpointId))
-                                        .orElse(Set.of());
+            var regions = new LinkedHashSet<RegionName>();
+            inspector.field(assignedRotationRegionsField).traverse((ArrayTraverser) (j, regionInspector) -> {
+                regions.add(RegionName.from(regionInspector.asString()));
+            });
             assignedRotations.putIfAbsent(endpointId, new AssignedRotation(clusterId, endpointId, rotationId, regions));
         });
 
         return List.copyOf(assignedRotations.values());
     }
 
-    private Set<RegionName> globalEndpointRegions(DeploymentInstanceSpec spec, EndpointId endpointId) {
-        if (spec.globalServiceId().isPresent())
-            return spec.zones().stream()
-                       .flatMap(zone -> zone.region().stream())
-                       .collect(Collectors.toSet());
-
-        return spec.endpoints().stream()
-                   .filter(endpoint -> endpoint.endpointId().equals(endpointId.id()))
-                   .flatMap(endpoint -> endpoint.regions().stream())
-                   .collect(Collectors.toSet());
-    }
-
 }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java
index 01ef6db1dc8..77cdd8d342d 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java
@@ -9,23 +9,23 @@ import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.HostName;
 import com.yahoo.config.provision.TenantName;
 import com.yahoo.config.provision.zone.ZoneId;
+import com.yahoo.log.LogLevel;
 import com.yahoo.path.Path;
 import com.yahoo.slime.Slime;
-import com.yahoo.vespa.config.SlimeUtils;
+import com.yahoo.slime.SlimeUtils;
 import com.yahoo.vespa.curator.Curator;
 import com.yahoo.vespa.curator.Lock;
 import com.yahoo.vespa.hosted.controller.Application;
-import com.yahoo.vespa.hosted.controller.api.integration.certificates.ApplicationCertificate;
 import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMetadata;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId;
-import com.yahoo.vespa.hosted.controller.routing.GlobalRouting;
-import com.yahoo.vespa.hosted.controller.routing.RoutingPolicy;
 import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId;
 import com.yahoo.vespa.hosted.controller.auditlog.AuditLog;
 import com.yahoo.vespa.hosted.controller.deployment.Run;
 import com.yahoo.vespa.hosted.controller.deployment.Step;
 import com.yahoo.vespa.hosted.controller.dns.NameServiceQueue;
+import com.yahoo.vespa.hosted.controller.routing.GlobalRouting;
+import com.yahoo.vespa.hosted.controller.routing.RoutingPolicy;
 import com.yahoo.vespa.hosted.controller.routing.RoutingPolicyId;
 import com.yahoo.vespa.hosted.controller.routing.ZoneRoutingPolicy;
 import com.yahoo.vespa.hosted.controller.tenant.Tenant;
@@ -39,8 +39,10 @@ import java.io.IOException;
 import java.io.UncheckedIOException;
 import java.nio.ByteBuffer;
 import java.time.Duration;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
@@ -84,7 +86,7 @@ public class CuratorDb {
     private static final Path controllerRoot = root.append("controllers");
     private static final Path routingPoliciesRoot = root.append("routingPolicies");
     private static final Path zoneRoutingPoliciesRoot = root.append("zoneRoutingPolicies");
-    private static final Path applicationCertificateRoot = root.append("applicationCertificates");
+    private static final Path endpointCertificateRoot = root.append("applicationCertificates");
 
     private final StringSetSerializer stringSetSerializer = new StringSetSerializer();
     private final NodeVersionSerializer nodeVersionSerializer = new NodeVersionSerializer();
@@ -107,12 +109,6 @@ public class CuratorDb {
     // For each job id (path), store the ZK node version and its deserialised data — update when version changes. 
     private final Map<Path, Pair<Integer, NavigableMap<RunId, Run>>> cachedHistoricRuns = new ConcurrentHashMap<>();
 
-    /**
-     * All keys, to allow reentrancy.
-     * This will grow forever, but this should be too slow to be a problem.
-     */
-    private final ConcurrentHashMap<Path, Lock> locks = new ConcurrentHashMap<>();
-
     @Inject
     public CuratorDb(Curator curator) {
         this(curator, defaultTryLockTimeout);
@@ -134,28 +130,20 @@ public class CuratorDb {
 
     // -------------- Locks ---------------------------------------------------
 
-    /** Creates a reentrant lock */
-    private Lock lock(Path path, Duration timeout) {
-        curator.create(path);
-        Lock lock = locks.computeIfAbsent(path, (pathArg) -> new Lock(pathArg.getAbsolute(), curator));
-        lock.acquire(timeout);
-        return lock;
-    }
-
     public Lock lock(TenantName name) {
-        return lock(lockPath(name), defaultLockTimeout.multipliedBy(2));
+        return curator.lock(lockPath(name), defaultLockTimeout.multipliedBy(2));
     }
 
     public Lock lock(TenantAndApplicationId id) {
-        return lock(lockPath(id), defaultLockTimeout.multipliedBy(2));
+        return curator.lock(lockPath(id), defaultLockTimeout.multipliedBy(2));
     }
 
     public Lock lockForDeployment(ApplicationId id, ZoneId zone) {
-        return lock(lockPath(id, zone), deployLockTimeout);
+        return curator.lock(lockPath(id, zone), deployLockTimeout);
     }
 
     public Lock lock(ApplicationId id, JobType type) {
-        return lock(lockPath(id, type), defaultLockTimeout);
+        return curator.lock(lockPath(id, type), defaultLockTimeout);
     }
 
     public Lock lock(ApplicationId id, JobType type, Step step) throws TimeoutException {
@@ -163,15 +151,15 @@ public class CuratorDb {
     }
 
     public Lock lockRotations() {
-        return lock(lockRoot.append("rotations"), defaultLockTimeout);
+        return curator.lock(lockRoot.append("rotations"), defaultLockTimeout);
     }
 
     public Lock lockConfidenceOverrides() {
-        return lock(lockRoot.append("confidenceOverrides"), defaultLockTimeout);
+        return curator.lock(lockRoot.append("confidenceOverrides"), defaultLockTimeout);
     }
 
     public Lock lockInactiveJobs() {
-        return lock(lockRoot.append("inactiveJobsLock"), defaultLockTimeout);
+        return curator.lock(lockRoot.append("inactiveJobsLock"), defaultLockTimeout);
     }
 
     public Lock lockMaintenanceJob(String jobName) throws TimeoutException {
@@ -180,27 +168,27 @@ public class CuratorDb {
 
     @SuppressWarnings("unused") // Called by internal code
     public Lock lockProvisionState(String provisionStateId) {
-        return lock(lockPath(provisionStateId), Duration.ofSeconds(1));
+        return curator.lock(lockPath(provisionStateId), Duration.ofSeconds(1));
     }
 
     public Lock lockOsVersions() {
-        return lock(lockRoot.append("osTargetVersion"), defaultLockTimeout);
+        return curator.lock(lockRoot.append("osTargetVersion"), defaultLockTimeout);
     }
 
     public Lock lockOsVersionStatus() {
-        return lock(lockRoot.append("osVersionStatus"), defaultLockTimeout);
+        return curator.lock(lockRoot.append("osVersionStatus"), defaultLockTimeout);
     }
 
     public Lock lockRoutingPolicies() {
-        return lock(lockRoot.append("routingPolicies"), defaultLockTimeout);
+        return curator.lock(lockRoot.append("routingPolicies"), defaultLockTimeout);
     }
 
     public Lock lockAuditLog() {
-        return lock(lockRoot.append("auditLog"), defaultLockTimeout);
+        return curator.lock(lockRoot.append("auditLog"), defaultLockTimeout);
     }
 
     public Lock lockNameServiceQueue() {
-        return lock(lockRoot.append("nameServiceQueue"), defaultLockTimeout);
+        return curator.lock(lockRoot.append("nameServiceQueue"), defaultLockTimeout);
     }
 
     // -------------- Helpers ------------------------------------------
@@ -211,7 +199,7 @@ public class CuratorDb {
      */
     private Lock tryLock(Path path) throws TimeoutException {
         try {
-            return lock(path, tryLockTimeout);
+            return curator.lock(path, tryLockTimeout);
         }
         catch (UncheckedTimeoutException e) {
             throw new TimeoutException(e.getMessage());
@@ -352,26 +340,37 @@ public class CuratorDb {
         return read(applicationPath(application), applicationSerializer::fromSlime);
     }
 
-    public List<Application> readApplications() {
-        return readApplications(ignored -> true);
+    public List<Application> readApplications(boolean canFail) {
+        return readApplications(ignored -> true, canFail);
     }
 
     public List<Application> readApplications(TenantName name) {
-        return readApplications(application -> application.tenant().equals(name));
-    }
-
-    private List<Application> readApplications(Predicate<TenantAndApplicationId> applicationFilter) {
-        return readApplicationIds().stream()
-                                   .filter(applicationFilter)
-                                   .sorted()
-                                   .map(this::readApplication)
-                                   .flatMap(Optional::stream)
-                                   .collect(Collectors.toUnmodifiableList());
+        return readApplications(application -> application.tenant().equals(name), false);
+    }
+
+    private List<Application> readApplications(Predicate<TenantAndApplicationId> applicationFilter, boolean canFail) {
+        var applicationIds = readApplicationIds();
+        var applications = new ArrayList<Application>(applicationIds.size());
+        for (var id : applicationIds) {
+            if (!applicationFilter.test(id)) continue;
+            try {
+                readApplication(id).ifPresent(applications::add);
+            } catch (Exception e) {
+                if (canFail) {
+                    log.log(LogLevel.ERROR, "Failed to read application '" + id + "', this must be fixed through " +
+                                            "manual intervention", e);
+                } else {
+                    throw e;
+                }
+            }
+        }
+        return Collections.unmodifiableList(applications);
     }
 
     public List<TenantAndApplicationId> readApplicationIds() {
         return curator.getChildren(applicationRoot).stream()
                       .map(TenantAndApplicationId::fromSerialized)
+                      .sorted()
                       .collect(toUnmodifiableList());
     }
 
@@ -387,7 +386,6 @@ public class CuratorDb {
 
     public void writeHistoricRuns(ApplicationId id, JobType type, Iterable<Run> runs) {
         Path path = runsPath(id, type);
-        cachedHistoricRuns.remove(path);
         curator.set(path, asJson(runSerializer.toSlime(runs)));
     }
 
@@ -514,15 +512,25 @@ public class CuratorDb {
                                                      .orElse(new ZoneRoutingPolicy(zone, GlobalRouting.DEFAULT_STATUS));
     }
 
-    // -------------- Application web certificates ----------------------------
+    // -------------- Application endpoint certificates ----------------------------
 
-    public void writeApplicationCertificate(ApplicationId applicationId, ApplicationCertificate applicationCertificate) {
-        curator.set(applicationCertificatePath(applicationId), applicationCertificate.secretsKeyNamePrefix().getBytes());
+    public void writeEndpointCertificateMetadata(ApplicationId applicationId, EndpointCertificateMetadata endpointCertificateMetadata) {
+        curator.set(endpointCertificatePath(applicationId), asJson(EndpointCertificateMetadataSerializer.toSlime(endpointCertificateMetadata)));
     }
 
     public Optional<EndpointCertificateMetadata> readEndpointCertificateMetadata(ApplicationId applicationId) {
-        Optional<String> zkData = curator.getData(applicationCertificatePath(applicationId)).map(String::new);
-        return zkData.map(EndpointCertificateMetadataSerializer::fromJsonOrTlsSecretsKeysString);
+        return curator.getData(endpointCertificatePath(applicationId)).map(String::new).map(EndpointCertificateMetadataSerializer::fromJsonString);
+    }
+
+    public Map<ApplicationId, EndpointCertificateMetadata> readAllEndpointCertificateMetadata() {
+        Map<ApplicationId, EndpointCertificateMetadata> allEndpointCertificateMetadata = new HashMap<>();
+
+        for (String appIdString : curator.getChildren(endpointCertificateRoot)) {
+            ApplicationId applicationId = ApplicationId.fromSerializedForm(appIdString);
+            Optional<EndpointCertificateMetadata> endpointCertificateMetadata = readEndpointCertificateMetadata(applicationId);
+            allEndpointCertificateMetadata.put(applicationId, endpointCertificateMetadata.orElseThrow());
+        }
+        return allEndpointCertificateMetadata;
     }
 
     // -------------- Paths ---------------------------------------------------
@@ -642,8 +650,8 @@ public class CuratorDb {
         return controllerRoot.append(hostname);
     }
 
-    private static Path applicationCertificatePath(ApplicationId id) {
-        return applicationCertificateRoot.append(id.serializedForm());
+    private static Path endpointCertificatePath(ApplicationId id) {
+        return endpointCertificateRoot.append(id.serializedForm());
     }
 
 }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateMetadataSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateMetadataSerializer.java
index c2e901bbaea..80d8270eaaa 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateMetadataSerializer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateMetadataSerializer.java
@@ -3,9 +3,15 @@ package com.yahoo.vespa.hosted.controller.persistence;
 import com.yahoo.slime.Cursor;
 import com.yahoo.slime.Inspector;
 import com.yahoo.slime.Slime;
-import com.yahoo.vespa.config.SlimeUtils;
+import com.yahoo.slime.SlimeUtils;
+import com.yahoo.slime.Type;
 import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMetadata;
 
+import java.util.List;
+import java.util.Optional;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
 /**
  * (de)serializes endpoint certificate metadata
  * <p>
@@ -26,42 +32,53 @@ public class EndpointCertificateMetadataSerializer {
     private final static String keyNameField = "keyName";
     private final static String certNameField = "certName";
     private final static String versionField = "version";
+    private final static String requestIdField = "requestId";
+    private final static String requestedDnsSansField = "requestedDnsSans";
+    private final static String issuerField = "issuer";
 
-    public static void toSlime(EndpointCertificateMetadata metadata, Cursor object) {
+    public static Slime toSlime(EndpointCertificateMetadata metadata) {
+        Slime slime = new Slime();
+        Cursor object = slime.setObject();
         object.setString(keyNameField, metadata.keyName());
         object.setString(certNameField, metadata.certName());
         object.setLong(versionField, metadata.version());
+
+        metadata.request_id().ifPresent(id -> object.setString(requestIdField, id));
+        metadata.requestedDnsSans().ifPresent(sans -> {
+            Cursor cursor = object.setArray(requestedDnsSansField);
+            sans.forEach(cursor::addString);
+        });
+        metadata.issuer().ifPresent(id -> object.setString(issuerField, id));
+
+        return slime;
     }
 
     public static EndpointCertificateMetadata fromSlime(Inspector inspector) {
-        switch (inspector.type()) {
-            case STRING: // TODO: Remove once all are transmitted and stored as JSON
-                return new EndpointCertificateMetadata(
-                        inspector.asString() + "-key",
-                        inspector.asString() + "-cert",
-                        0
-                );
-            case OBJECT:
-                return new EndpointCertificateMetadata(
-                        inspector.field(keyNameField).asString(),
-                        inspector.field(certNameField).asString(),
-                        Math.toIntExact(inspector.field(versionField).asLong())
-                );
+        if (inspector.type() != Type.OBJECT)
+            throw new IllegalArgumentException("Unknown format encountered for endpoint certificate metadata!");
+        Optional<String> request_id = inspector.field(requestIdField).valid() ?
+                Optional.of(inspector.field(requestIdField).asString()) :
+                Optional.empty();
 
-            default:
-                throw new IllegalArgumentException("Unknown format encountered for endpoint certificate metadata!");
-        }
-    }
+        Optional<List<String>> requestedDnsSans = inspector.field(requestedDnsSansField).valid() ?
+                Optional.of(IntStream.range(0, inspector.field(requestedDnsSansField).entries())
+                        .mapToObj(i -> inspector.field(requestedDnsSansField).entry(i).asString()).collect(Collectors.toList())) :
+                Optional.empty();
+
+        Optional<String> issuer = inspector.field(issuerField).valid() ?
+                Optional.of(inspector.field(issuerField).asString()) :
+                Optional.empty();
 
-    public static EndpointCertificateMetadata fromTlsSecretsKeysString(String tlsSecretsKeys) {
-        return fromSlime(new Slime().setString(tlsSecretsKeys));
+        return new EndpointCertificateMetadata(
+                inspector.field(keyNameField).asString(),
+                inspector.field(certNameField).asString(),
+                Math.toIntExact(inspector.field(versionField).asLong()),
+                request_id,
+                requestedDnsSans,
+                issuer);
     }
 
-    public static EndpointCertificateMetadata fromJsonOrTlsSecretsKeysString(String zkdata) {
-        if(zkdata.strip().startsWith("{")) {
-            return fromSlime(SlimeUtils.jsonToSlime(zkdata).get());
-        } else {
-            return fromTlsSecretsKeysString(zkdata);
-        }
+    public static EndpointCertificateMetadata fromJsonString(String zkData) {
+        return fromSlime(SlimeUtils.jsonToSlime(zkData).get());
     }
 }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/LogSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/LogSerializer.java
index e32e8ceacca..fffe781e6e1 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/LogSerializer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/LogSerializer.java
@@ -6,7 +6,7 @@ import com.yahoo.slime.Cursor;
 import com.yahoo.slime.Inspector;
 import com.yahoo.slime.ObjectTraverser;
 import com.yahoo.slime.Slime;
-import com.yahoo.vespa.config.SlimeUtils;
+import com.yahoo.slime.SlimeUtils;
 import com.yahoo.vespa.hosted.controller.api.integration.LogEntry;
 import com.yahoo.vespa.hosted.controller.api.integration.LogEntry.Type;
 import com.yahoo.vespa.hosted.controller.deployment.Step;
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/MockCuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/MockCuratorDb.java
index 7044f4912e2..ab88e8fe019 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/MockCuratorDb.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/MockCuratorDb.java
@@ -2,7 +2,6 @@
 package com.yahoo.vespa.hosted.controller.persistence;
 
 import com.google.inject.Inject;
-import com.yahoo.vespa.curator.Curator;
 import com.yahoo.vespa.curator.mock.MockCurator;
 
 import java.time.Duration;
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java
index a4b0df31883..1aa229984a8 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java
@@ -31,6 +31,7 @@ import java.util.TreeMap;
 
 import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.aborted;
 import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.deploymentFailed;
+import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.endpointCertificateTimeout;
 import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.error;
 import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.installationFailed;
 import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.outOfCapacity;
@@ -161,23 +162,25 @@ class RunSerializer {
         if ( ! versionObject.field(buildField).valid())
             return ApplicationVersion.unknown;
 
-        SourceRevision revision = new SourceRevision(versionObject.field(repositoryField).asString(),
-                                                     versionObject.field(branchField).asString(),
-                                                     versionObject.field(commitField).asString());
         long buildNumber = versionObject.field(buildField).asLong();
+        // TODO jonmv: Remove source revision
+        Optional<SourceRevision> source = Optional.of(new SourceRevision(versionObject.field(repositoryField).asString(),
+                                                                         versionObject.field(branchField).asString(),
+                                                                         versionObject.field(commitField).asString()))
+                                                  .filter(revision -> ! revision.commit().isBlank() && ! revision.repository().isBlank() && ! revision.branch().isBlank());
         Optional<String> authorEmail = Serializers.optionalString(versionObject.field(authorEmailField));
         Optional<Version> compileVersion = Serializers.optionalString(versionObject.field(compileVersionField)).map(Version::fromString);
         Optional<Instant> buildTime = Serializers.optionalInstant(versionObject.field(buildTimeField));
         Optional<String> sourceUrl = Serializers.optionalString(versionObject.field(sourceUrlField));
         Optional<String> commit = Serializers.optionalString(versionObject.field(commitField));
 
-        return new ApplicationVersion(Optional.of(revision), OptionalLong.of(buildNumber), authorEmail,
+        return new ApplicationVersion(source, OptionalLong.of(buildNumber), authorEmail,
                                       compileVersion, buildTime, sourceUrl, commit);
     }
 
     // Don't change this — introduce a separate array instead.
     private Optional<ConvergenceSummary> convergenceSummaryFrom(Inspector summaryArray) {
-        if ( ! summaryArray.valid())
+        if ( ! summaryArray.valid() || summaryArray.entries() == 11) // TODO jonmv: fix
             return Optional.empty();
 
         if (summaryArray.entries() != 12)
@@ -244,12 +247,11 @@ class RunSerializer {
 
     private void toSlime(Version platformVersion, ApplicationVersion applicationVersion, Cursor versionsObject) {
         versionsObject.setString(platformVersionField, platformVersion.toString());
-        if ( ! applicationVersion.isUnknown()) {
-            versionsObject.setString(repositoryField, applicationVersion.source().get().repository());
-            versionsObject.setString(branchField, applicationVersion.source().get().branch());
-            versionsObject.setString(commitField, applicationVersion.source().get().commit());
-            versionsObject.setLong(buildField, applicationVersion.buildNumber().getAsLong());
-        }
+        applicationVersion.buildNumber().ifPresent(number -> versionsObject.setLong(buildField, number));
+        // TODO jonmv: Remove source revision.
+        applicationVersion.source().map(SourceRevision::repository).ifPresent(repository -> versionsObject.setString(repositoryField, repository));
+        applicationVersion.source().map(SourceRevision::branch).ifPresent(branch -> versionsObject.setString(branchField, branch));
+        applicationVersion.source().map(SourceRevision::commit).ifPresent(commit -> versionsObject.setString(commitField, commit));
         applicationVersion.authorEmail().ifPresent(email -> versionsObject.setString(authorEmailField, email));
         applicationVersion.compileVersion().ifPresent(version -> versionsObject.setString(compileVersionField, version.toString()));
         applicationVersion.buildTime().ifPresent(time -> versionsObject.setLong(buildTimeField, time.toEpochMilli()));
@@ -345,14 +347,15 @@ class RunSerializer {
 
     static String valueOf(RunStatus status) {
         switch (status) {
-            case running            : return "running";
-            case outOfCapacity      : return "outOfCapacity";
-            case deploymentFailed   : return "deploymentFailed";
-            case installationFailed : return "installationFailed";
-            case testFailure        : return "testFailure";
-            case error              : return "error";
-            case success            : return "success";
-            case aborted            : return "aborted";
+            case running                    : return "running";
+            case outOfCapacity              : return "outOfCapacity";
+            case endpointCertificateTimeout : return "endpointCertificateTimeout";
+            case deploymentFailed           : return "deploymentFailed";
+            case installationFailed         : return "installationFailed";
+            case testFailure                : return "testFailure";
+            case error                      : return "error";
+            case success                    : return "success";
+            case aborted                    : return "aborted";
 
             default: throw new AssertionError("No value defined for '" + status + "'!");
         }
@@ -360,14 +363,15 @@ class RunSerializer {
 
     static RunStatus runStatusOf(String status) {
         switch (status) {
-            case "running"            : return running;
-            case "outOfCapacity"      : return outOfCapacity;
-            case "deploymentFailed"   : return deploymentFailed;
-            case "installationFailed" : return installationFailed;
-            case "testFailure"        : return testFailure;
-            case "error"              : return error;
-            case "success"            : return success;
-            case "aborted"            : return aborted;
+            case "running"                    : return running;
+            case "outOfCapacity"              : return outOfCapacity;
+            case "endpointCertificateTimeout" : return endpointCertificateTimeout;
+            case "deploymentFailed"           : return deploymentFailed;
+            case "installationFailed"         : return installationFailed;
+            case "testFailure"                : return testFailure;
+            case "error"                      : return error;
+            case "success"                    : return success;
+            case "aborted"                    : return aborted;
 
             default: throw new IllegalArgumentException("No run status defined by '" + status + "'!");
         }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/Serializers.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/Serializers.java
index 841cb387e54..e5adccc850c 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/Serializers.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/Serializers.java
@@ -2,7 +2,7 @@
 package com.yahoo.vespa.hosted.controller.persistence;
 
 import com.yahoo.slime.Inspector;
-import com.yahoo.vespa.config.SlimeUtils;
+import com.yahoo.slime.SlimeUtils;
 
 import java.time.Instant;
 import java.util.Optional;
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/StringSetSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/StringSetSerializer.java
index 789f6393683..cfd92e8a7f4 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/StringSetSerializer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/StringSetSerializer.java
@@ -5,7 +5,7 @@ import com.yahoo.slime.ArrayTraverser;
 import com.yahoo.slime.Cursor;
 import com.yahoo.slime.Inspector;
 import com.yahoo.slime.Slime;
-import com.yahoo.vespa.config.SlimeUtils;
+import com.yahoo.slime.SlimeUtils;
 
 import java.io.IOException;
 import java.util.HashSet;
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java
index 35128466e4d..d4d5f4deb7b 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java
@@ -10,7 +10,7 @@ import com.yahoo.slime.Cursor;
 import com.yahoo.slime.Inspector;
 import com.yahoo.slime.Slime;
 import com.yahoo.vespa.athenz.api.AthenzDomain;
-import com.yahoo.vespa.config.SlimeUtils;
+import com.yahoo.slime.SlimeUtils;
 import com.yahoo.vespa.hosted.controller.api.identifiers.Property;
 import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId;
 import com.yahoo.vespa.hosted.controller.api.role.SimplePrincipal;
@@ -19,7 +19,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.organization.Contact;
 import com.yahoo.vespa.hosted.controller.api.integration.organization.BillingInfo;
 import com.yahoo.vespa.hosted.controller.tenant.CloudTenant;
 import com.yahoo.vespa.hosted.controller.tenant.Tenant;
-import com.yahoo.vespa.hosted.controller.tenant.UserTenant;
 
 import java.net.URI;
 import java.security.Principal;
@@ -68,7 +67,6 @@ public class TenantSerializer {
 
         switch (tenant.type()) {
             case athenz: toSlime((AthenzTenant) tenant, tenantObject); break;
-            case user:   toSlime((UserTenant) tenant, tenantObject);   break;
             case cloud:  toSlime((CloudTenant) tenant, tenantObject);  break;
             default:     throw new IllegalArgumentException("Unexpected tenant type '" + tenant.type() + "'.");
         }
@@ -85,13 +83,6 @@ public class TenantSerializer {
         });
     }
 
-    private void toSlime(UserTenant tenant, Cursor tenantObject) {
-        tenant.contact().ifPresent(contact -> {
-            Cursor contactCursor = tenantObject.setObject(contactField);
-            writeContact(contact, contactCursor);
-        });
-    }
-
     private void toSlime(CloudTenant tenant, Cursor root) {
         developerKeysToSlime(tenant.developerKeys(), root.setArray(pemDeveloperKeysField));
         toSlime(tenant.billingInfo(), root.setObject(billingInfoField));
@@ -117,7 +108,6 @@ public class TenantSerializer {
 
         switch (type) {
             case athenz: return athenzTenantFrom(tenantObject);
-            case user:   return userTenantFrom(tenantObject);
             case cloud:  return cloudTenantFrom(tenantObject);
             default:     throw new IllegalArgumentException("Unexpected tenant type '" + type + "'.");
         }
@@ -132,12 +122,6 @@ public class TenantSerializer {
         return new AthenzTenant(name, domain, property, propertyId, contact);
     }
 
-    private UserTenant userTenantFrom(Inspector tenantObject) {
-        TenantName name = TenantName.from(tenantObject.field(nameField).asString());
-        Optional<Contact> contact = contactFrom(tenantObject.field(contactField));
-        return new UserTenant(name, contact);
-    }
-
     private CloudTenant cloudTenantFrom(Inspector tenantObject) {
         TenantName name = TenantName.from(tenantObject.field(nameField).asString());
         BillingInfo billingInfo = billingInfoFrom(tenantObject.field(billingInfoField));
@@ -205,7 +189,6 @@ public class TenantSerializer {
     private static Tenant.Type typeOf(String value) {
         switch (value) {
             case "athenz": return Tenant.Type.athenz;
-            case "user":   return Tenant.Type.user;
             case "cloud":  return Tenant.Type.cloud;
             default: throw new IllegalArgumentException("Unknown tenant type '" + value + "'.");
         }
@@ -214,7 +197,6 @@ public class TenantSerializer {
     private static String valueOf(Tenant.Type type) {
         switch (type) {
             case athenz: return "athenz";
-            case user:   return "user";
             case cloud:  return "cloud";
             default: throw new IllegalArgumentException("Unexpected tenant type '" + type + "'.");
         }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializer.java
index 53f2d467e2d..6eb5b8fadcd 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializer.java
@@ -7,6 +7,7 @@ import com.yahoo.slime.ArrayTraverser;
 import com.yahoo.slime.Cursor;
 import com.yahoo.slime.Inspector;
 import com.yahoo.slime.Slime;
+import com.yahoo.vespa.hosted.controller.deployment.Run;
 import com.yahoo.vespa.hosted.controller.versions.DeploymentStatistics;
 import com.yahoo.vespa.hosted.controller.versions.NodeVersions;
 import com.yahoo.vespa.hosted.controller.versions.VersionStatus;
@@ -18,6 +19,7 @@ import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
+import java.util.stream.Collectors;
 
 /**
  * Serializer for {@link VersionStatus}.
@@ -82,7 +84,7 @@ public class VersionStatusSerializer {
         object.setBool(isControllerVersionField, version.isControllerVersion());
         object.setBool(isSystemVersionField, version.isSystemVersion());
         object.setBool(isReleasedField, version.isReleased());
-        deploymentStatisticsToSlime(version.statistics(), object.setObject(deploymentStatisticsField));
+        deploymentStatisticsToSlime(version.versionNumber(), object.setObject(deploymentStatisticsField));
         object.setString(confidenceField, version.confidence().name());
         nodeVersionsToSlime(version.nodeVersions(), object.setArray(nodeVersionsField));
     }
@@ -91,15 +93,12 @@ public class VersionStatusSerializer {
         nodeVersionSerializer.nodeVersionsToSlime(nodeVersions, array);
     }
 
-    private void deploymentStatisticsToSlime(DeploymentStatistics statistics, Cursor object) {
-        object.setString(versionField, statistics.version().toString());
-        applicationsToSlime(statistics.failing(), object.setArray(failingField));
-        applicationsToSlime(statistics.production(), object.setArray(productionField));
-        applicationsToSlime(statistics.deploying(), object.setArray(deployingField));
-    }
-
-    private void applicationsToSlime(Collection<ApplicationId> applications, Cursor array) {
-        applications.forEach(application -> array.addString(application.serializedForm()));
+    private void deploymentStatisticsToSlime(Version version, Cursor object) {
+        object.setString(versionField, version.toString());
+        // TODO jonmv: Remove the below.
+        object.setArray(failingField);
+        object.setArray(productionField);
+        object.setArray(deployingField);
     }
 
     private List<VespaVersion> vespaVersionsFromSlime(Inspector array) {
@@ -109,25 +108,18 @@ public class VersionStatusSerializer {
     }
 
     private VespaVersion vespaVersionFromSlime(Inspector object) {
-        var deploymentStatistics = deploymentStatisticsFromSlime(object.field(deploymentStatisticsField));
-        return new VespaVersion(deploymentStatistics,
+        var version = Version.fromString(object.field(deploymentStatisticsField).field(versionField).asString());
+        return new VespaVersion(version,
                                 object.field(releaseCommitField).asString(),
                                 Instant.ofEpochMilli(object.field(committedAtField).asLong()),
                                 object.field(isControllerVersionField).asBool(),
                                 object.field(isSystemVersionField).asBool(),
                                 object.field(isReleasedField).asBool(),
-                                nodeVersionSerializer.nodeVersionsFromSlime(object.field(nodeVersionsField), deploymentStatistics.version()),
+                                nodeVersionSerializer.nodeVersionsFromSlime(object.field(nodeVersionsField), version),
                                 VespaVersion.Confidence.valueOf(object.field(confidenceField).asString())
         );
     }
 
-    private DeploymentStatistics deploymentStatisticsFromSlime(Inspector object) {
-        return new DeploymentStatistics(Version.fromString(object.field(versionField).asString()),
-                                        applicationsFromSlime(object.field(failingField)),
-                                        applicationsFromSlime(object.field(productionField)),
-                                        applicationsFromSlime(object.field(deployingField)));
-    }
-
     private List<ApplicationId> applicationsFromSlime(Inspector array) {
         List<ApplicationId> applications = new ArrayList<>();
         array.traverse((ArrayTraverser) (i, entry) -> applications.add(
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
index b7b67b9ecb4..455759a3b41 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
@@ -13,6 +13,7 @@ import com.yahoo.config.provision.Environment;
 import com.yahoo.config.provision.InstanceName;
 import com.yahoo.config.provision.NodeResources;
 import com.yahoo.config.provision.TenantName;
+import com.yahoo.config.provision.zone.RoutingMethod;
 import com.yahoo.config.provision.zone.ZoneId;
 import com.yahoo.container.jdisc.HttpRequest;
 import com.yahoo.container.jdisc.HttpResponse;
@@ -27,11 +28,7 @@ import com.yahoo.security.KeyUtils;
 import com.yahoo.slime.Cursor;
 import com.yahoo.slime.Inspector;
 import com.yahoo.slime.Slime;
-import com.yahoo.vespa.athenz.api.AthenzIdentity;
-import com.yahoo.vespa.athenz.api.AthenzPrincipal;
-import com.yahoo.vespa.athenz.api.AthenzUser;
-import com.yahoo.vespa.config.SlimeUtils;
-import com.yahoo.vespa.hosted.controller.AlreadyExistsException;
+import com.yahoo.slime.SlimeUtils;
 import com.yahoo.vespa.hosted.controller.Application;
 import com.yahoo.vespa.hosted.controller.Controller;
 import com.yahoo.vespa.hosted.controller.Instance;
@@ -56,7 +53,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision;
 import com.yahoo.vespa.hosted.controller.api.integration.resource.CostInfo;
-import com.yahoo.vespa.hosted.controller.api.integration.resource.MeteringInfo;
+import com.yahoo.vespa.hosted.controller.api.integration.resource.MeteringData;
 import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceAllocation;
 import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceSnapshot;
 import com.yahoo.vespa.hosted.controller.api.role.Role;
@@ -65,10 +62,7 @@ import com.yahoo.vespa.hosted.controller.api.role.SecurityContext;
 import com.yahoo.vespa.hosted.controller.application.ApplicationPackage;
 import com.yahoo.vespa.hosted.controller.application.AssignedRotation;
 import com.yahoo.vespa.hosted.controller.application.Change;
-import com.yahoo.vespa.hosted.controller.application.ClusterCost;
-import com.yahoo.vespa.hosted.controller.application.ClusterUtilization;
 import com.yahoo.vespa.hosted.controller.application.Deployment;
-import com.yahoo.vespa.hosted.controller.application.DeploymentCost;
 import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics;
 import com.yahoo.vespa.hosted.controller.application.Endpoint;
 import com.yahoo.vespa.hosted.controller.application.SystemApplication;
@@ -89,7 +83,7 @@ import com.yahoo.vespa.hosted.controller.security.Credentials;
 import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant;
 import com.yahoo.vespa.hosted.controller.tenant.CloudTenant;
 import com.yahoo.vespa.hosted.controller.tenant.Tenant;
-import com.yahoo.vespa.hosted.controller.tenant.UserTenant;
+import com.yahoo.vespa.hosted.controller.versions.VersionStatus;
 import com.yahoo.vespa.hosted.controller.versions.VespaVersion;
 import com.yahoo.vespa.serviceview.bindings.ApplicationView;
 import com.yahoo.yolean.Exceptions;
@@ -209,13 +203,13 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
 
     private HttpResponse handleGET(Path path, HttpRequest request) {
         if (path.matches("/application/v4/")) return root(request);
-        if (path.matches("/application/v4/user")) return authenticatedUser(request);
         if (path.matches("/application/v4/tenant")) return tenants(request);
         if (path.matches("/application/v4/tenant/{tenant}")) return tenant(path.get("tenant"), request);
         if (path.matches("/application/v4/tenant/{tenant}/cost")) return tenantCost(path.get("tenant"), request);
         if (path.matches("/application/v4/tenant/{tenant}/cost/{month}")) return tenantCost(path.get("tenant"), path.get("month"), request);
         if (path.matches("/application/v4/tenant/{tenant}/application")) return applications(path.get("tenant"), Optional.empty(), request);
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}")) return application(path.get("tenant"), path.get("application"), request);
+        if (path.matches("/application/v4/tenant/{tenant}/application/{application}/compile-version")) return compileVersion(path.get("tenant"), path.get("application"));
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deployment")) return JobControllerApiHandlerHelper.overviewResponse(controller, TenantAndApplicationId.from(path.get("tenant"), path.get("application")), request.getUri());
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/package")) return applicationPackage(path.get("tenant"), path.get("application"), request);
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying")) return deploying(path.get("tenant"), path.get("application"), "default", request);
@@ -227,6 +221,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploying/pin")) return deploying(path.get("tenant"), path.get("application"), path.get("instance"), request);
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job")) return JobControllerApiHandlerHelper.jobTypeResponse(controller, appIdFromPath(path), request.getUri());
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}")) return JobControllerApiHandlerHelper.runResponse(controller.jobController().runs(appIdFromPath(path), jobTypeFromPath(path)), request.getUri());
+        if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}/package")) return devApplicationPackage(appIdFromPath(path), jobTypeFromPath(path));
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}/test-config")) return testConfig(appIdFromPath(path), jobTypeFromPath(path));
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}/run/{number}")) return JobControllerApiHandlerHelper.runDetailsResponse(controller.jobController(), runIdFromPath(path), request.getProperty("after"));
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}")) return deployment(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request);
@@ -250,7 +245,6 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
     }
 
     private HttpResponse handlePUT(Path path, HttpRequest request) {
-        if (path.matches("/application/v4/user")) return createUser(request);
         if (path.matches("/application/v4/tenant/{tenant}")) return updateTenant(path.get("tenant"), request);
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/global-rotation/override")) return setGlobalRotationOverride(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), false, request);
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/global-rotation/override")) return setGlobalRotationOverride(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), false, request);
@@ -327,27 +321,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
     private HttpResponse root(HttpRequest request) {
         return recurseOverTenants(request)
                 ? recursiveRoot(request)
-                : new ResourceResponse(request, "user", "tenant");
-    }
-
-    // TODO jonmv: Move to Athenz API.
-    private HttpResponse authenticatedUser(HttpRequest request) {
-        Principal user = requireUserPrincipal(request);
-        if (user == null)
-            throw new NotAuthorizedException("You must be authenticated.");
-
-        String userName = user instanceof AthenzPrincipal ? ((AthenzPrincipal) user).getIdentity().getName() : user.getName();
-        TenantName tenantName = TenantName.from(UserTenant.normalizeUser(userName));
-        List<Tenant> tenants = controller.tenants().asList(new Credentials(user));
-
-        Slime slime = new Slime();
-        Cursor response = slime.setObject();
-        response.setString("user", userName);
-        Cursor tenantsArray = response.setArray("tenants");
-        for (Tenant tenant : tenants)
-            tenantInTenantsListToSlime(tenant, request.getUri(), tenantsArray.addObject());
-        response.setBool("tenantExists", tenants.stream().anyMatch(tenant -> tenant.name().equals(tenantName)));
-        return new SlimeJsonResponse(slime);
+                : new ResourceResponse(request, "tenant");
     }
 
     private HttpResponse tenants(HttpRequest request) {
@@ -429,16 +403,45 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
 
     private HttpResponse applications(String tenantName, Optional<String> applicationName, HttpRequest request) {
         TenantName tenant = TenantName.from(tenantName);
+        if (controller.tenants().get(tenantName).isEmpty())
+            return ErrorResponse.notFoundError("Tenant '" + tenantName + "' does not exist");
+
         Slime slime = new Slime();
-        Cursor array = slime.setArray();
+        Cursor applicationArray = slime.setArray();
         for (Application application : controller.applications().asList(tenant)) {
-            if (applicationName.map(application.id().application().value()::equals).orElse(true))
-                for (InstanceName instance : application.instances().keySet())
-                    toSlime(application.id().instance(instance), array.addObject(), request);
+            if (applicationName.map(application.id().application().value()::equals).orElse(true)) {
+                Cursor applicationObject = applicationArray.addObject();
+                applicationObject.setString("tenant", application.id().tenant().value());
+                applicationObject.setString("application", application.id().application().value());
+                applicationObject.setString("url", withPath("/application/v4" +
+                                                            "/tenant/" + application.id().tenant().value() +
+                                                            "/application/" + application.id().application().value(),
+                                                            request.getUri()).toString());
+                Cursor instanceArray = applicationObject.setArray("instances");
+                for (InstanceName instance : showOnlyProductionInstances(request) ? application.productionInstances().keySet()
+                                                                                  : application.instances().keySet()) {
+                    Cursor instanceObject = instanceArray.addObject();
+                    instanceObject.setString("instance", instance.value());
+                    instanceObject.setString("url", withPath("/application/v4" +
+                                                             "/tenant/" + application.id().tenant().value() +
+                                                             "/application/" + application.id().application().value() +
+                                                             "/instance/" + instance.value(),
+                                                             request.getUri()).toString());
+                }
+            }
         }
         return new SlimeJsonResponse(slime);
     }
 
+    private HttpResponse devApplicationPackage(ApplicationId id, JobType type) {
+        if ( ! type.environment().isManuallyDeployed())
+            throw new IllegalArgumentException("Only manually deployed zones have dev packages");
+
+        ZoneId zone = type.zone(controller.system());
+        byte[] applicationPackage = controller.applications().applicationStore().getDev(id, zone);
+        return new ZipResponse(id.toFullString() + "." + zone.value() + ".zip", applicationPackage);
+    }
+
     private HttpResponse applicationPackage(String tenantName, String applicationName, HttpRequest request) {
         var tenantAndApplication = TenantAndApplicationId.from(tenantName, applicationName);
         var applicationId = ApplicationId.from(tenantName, applicationName, InstanceName.defaultName().value());
@@ -477,6 +480,13 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
         return new SlimeJsonResponse(slime);
     }
 
+    private HttpResponse compileVersion(String tenantName, String applicationName) {
+        Slime slime = new Slime();
+        slime.setObject().setString("compileVersion",
+                                    compileVersion(TenantAndApplicationId.from(tenantName, applicationName)).toFullString());
+        return new SlimeJsonResponse(slime);
+    }
+
     private HttpResponse instance(String tenantName, String applicationName, String instanceName, HttpRequest request) {
         Slime slime = new Slime();
         toSlime(slime.setObject(), getInstance(tenantName, applicationName, instanceName),
@@ -645,6 +655,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
             case admin: return "admin";
             case content: return "content";
             case container: return "container";
+            case combined: return "combined";
             default: throw new IllegalArgumentException("Unexpected node cluster type '" + type + "'.");
         }
     }
@@ -683,9 +694,13 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
     private HttpResponse trigger(ApplicationId id, JobType type, HttpRequest request) {
         Inspector requestObject = toSlime(request.getData()).get();
         boolean requireTests = ! requestObject.field("skipTests").asBool();
-        String triggered = controller.applications().deploymentTrigger()
-                                     .forceTrigger(id, type, request.getJDiscRequest().getUserPrincipal().getName(), requireTests)
-                                     .stream().map(job -> job.type().jobName()).collect(joining(", "));
+        boolean reTrigger = requestObject.field("reTrigger").asBool();
+        String triggered = reTrigger
+                           ? controller.applications().deploymentTrigger()
+                                       .reTrigger(id, type).type().jobName()
+                           : controller.applications().deploymentTrigger()
+                                       .forceTrigger(id, type, request.getJDiscRequest().getUserPrincipal().getName(), requireTests)
+                                       .stream().map(job -> job.type().jobName()).collect(joining(", "));
         return new MessageResponse(triggered.isEmpty() ? "Job " + type.jobName() + " for " + id + " not triggered"
                                                        : "Triggered " + triggered + " for " + id);
     }
@@ -726,13 +741,11 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
                 toSlime(object.setObject("outstandingChange"), status.outstandingChange(instance.name()));
         });
 
-        // Compile version. The version that should be used when building an application
-        object.setString("compileVersion", compileVersion(application.id()).toFullString());
-
         application.majorVersion().ifPresent(majorVersion -> object.setLong("majorVersion", majorVersion));
 
         Cursor instancesArray = object.setArray("instances");
-        for (Instance instance : application.instances().values())
+        for (Instance instance : showOnlyProductionInstances(request) ? application.productionInstances().values()
+                                                                      : application.instances().values())
             toSlime(instancesArray.addObject(), status, instance, application.deploymentSpec(), request);
 
         application.deployKeys().stream().map(KeyUtils::toPem).forEach(object.setArray("pemDeployKeys")::addString);
@@ -754,6 +767,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
         application.deploymentIssueId().ifPresent(issueId -> object.setString("deploymentIssueId", issueId.value()));
     }
 
+    // TODO: Eliminate duplicated code in this and toSlime(Cursor, Instance, DeploymentStatus, HttpRequest)
     private void toSlime(Cursor object, DeploymentStatus status, Instance instance, DeploymentSpec deploymentSpec, HttpRequest request) {
         object.setString("instance", instance.name().value());
 
@@ -770,17 +784,6 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
             if ( ! status.outstandingChange(instance.name()).isEmpty())
                 toSlime(object.setObject("outstandingChange"), status.outstandingChange(instance.name()));
 
-            Cursor deploymentJobsArray = object.setArray("deploymentJobs");
-            for (JobStatus job : jobStatus) {
-                Cursor jobObject = deploymentJobsArray.addObject();
-                jobObject.setString("type", job.id().type().jobName());
-                jobObject.setBool("success", job.isSuccess());
-                job.lastTriggered().ifPresent(jobRun -> toSlime(jobRun, jobObject.setObject("lastTriggered")));
-                job.lastCompleted().ifPresent(jobRun -> toSlime(jobRun, jobObject.setObject("lastCompleted")));
-                job.firstFailing().ifPresent(jobRun -> toSlime(jobRun, jobObject.setObject("firstFailing")));
-                job.lastSuccess().ifPresent(jobRun -> toSlime(jobRun, jobObject.setObject("lastSuccess")));
-            }
-
             // Change blockers
             Cursor changeBlockers = object.setArray("changeBlockers");
             deploymentSpec.instance(instance.name()).ifPresent(spec -> spec.changeBlocker().forEach(changeBlocker -> {
@@ -796,35 +799,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
         }
 
         // Global endpoints
-        var globalEndpointUrls = new LinkedHashSet<String>();
-
-        // Add default global endpoints. These are backed by rotations.
-        instance.endpointsIn(controller.system())
-                .scope(Endpoint.Scope.global)
-                .legacy(false) // Hide legacy names
-                .asList().stream()
-                .map(Endpoint::url)
-                .map(URI::toString)
-                .forEach(globalEndpointUrls::add);
-
-        // Per-cluster endpoints. These are backed by load balancers.
-        var routingPolicies = controller.applications().routingPolicies().get(instance.id()).values();
-        for (var policy : routingPolicies) {
-            policy.globalEndpointsIn(controller.system()).asList().stream()
-                  .map(Endpoint::url)
-                  .map(URI::toString)
-                  .forEach(globalEndpointUrls::add);
-        }
-
-        var globalRotationsArray = object.setArray("globalRotations");
-        globalEndpointUrls.forEach(globalRotationsArray::addString);
-
-        // Legacy field. Identifies the first assigned rotation, if any.
-        instance.rotations().stream()
-                .map(AssignedRotation::rotationId)
-                .findFirst()
-                .ifPresent(rotation -> object.setString("rotationId", rotation.asString()));
-
+        globalEndpointsToSlime(object, instance);
 
         // Deployments sorted according to deployment spec
         List<Deployment> deployments = deploymentSpec.instance(instance.name())
@@ -854,6 +829,30 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
         }
     }
 
+    // TODO(mpolden): Remove once legacy dashboard and integration tests stop expecting these fields
+    private void globalEndpointsToSlime(Cursor object, Instance instance) {
+        var globalEndpointUrls = new LinkedHashSet<String>();
+
+        // Add global endpoints backed by rotations
+        controller.routing().endpointsOf(instance.id())
+                  .requiresRotation()
+                  .not().legacy() // Hide legacy names
+                  .asList().stream()
+                  .map(Endpoint::url)
+                  .map(URI::toString)
+                  .forEach(globalEndpointUrls::add);
+
+
+        var globalRotationsArray = object.setArray("globalRotations");
+        globalEndpointUrls.forEach(globalRotationsArray::addString);
+
+        // Legacy field. Identifies the first assigned rotation, if any.
+        instance.rotations().stream()
+                .map(AssignedRotation::rotationId)
+                .findFirst()
+                .ifPresent(rotation -> object.setString("rotationId", rotation.asString()));
+    }
+
     private void toSlime(Cursor object, Instance instance, DeploymentStatus status, HttpRequest request) {
         Application application = status.application();
         object.setString("tenant", instance.id().tenant().value());
@@ -886,18 +885,6 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
             if ( ! status.outstandingChange(instance.name()).isEmpty())
                 toSlime(object.setObject("outstandingChange"), status.outstandingChange(instance.name()));
 
-            Cursor deploymentsArray = object.setArray("deploymentJobs");
-            for (JobStatus job : jobStatus) {
-                Cursor jobObject = deploymentsArray.addObject();
-                jobObject.setString("type", job.id().type().jobName());
-                jobObject.setBool("success", job.isSuccess());
-
-                job.lastTriggered().ifPresent(jobRun -> toSlime(jobRun, jobObject.setObject("lastTriggered")));
-                job.lastCompleted().ifPresent(jobRun -> toSlime(jobRun, jobObject.setObject("lastCompleted")));
-                job.firstFailing().ifPresent(jobRun -> toSlime(jobRun, jobObject.setObject("firstFailing")));
-                job.lastSuccess().ifPresent(jobRun -> toSlime(jobRun, jobObject.setObject("lastSuccess")));
-            }
-
             // Change blockers
             Cursor changeBlockers = object.setArray("changeBlockers");
             application.deploymentSpec().instance(instance.name()).ifPresent(spec -> spec.changeBlocker().forEach(changeBlocker -> {
@@ -912,35 +899,10 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
             }));
         }
 
-        // Compile version. The version that should be used when building an application
-        object.setString("compileVersion", compileVersion(application.id()).toFullString());
-
         application.majorVersion().ifPresent(majorVersion -> object.setLong("majorVersion", majorVersion));
 
-        // Rotation
-        Cursor globalRotationsArray = object.setArray("globalRotations");
-        instance.endpointsIn(controller.system())
-                .scope(Endpoint.Scope.global)
-                .legacy(false) // Hide legacy names
-                .asList().stream()
-                .map(Endpoint::url)
-                .map(URI::toString)
-                .forEach(globalRotationsArray::addString);
-
-        instance.rotations().stream()
-                .map(AssignedRotation::rotationId)
-                .findFirst()
-                .ifPresent(rotation -> object.setString("rotationId", rotation.asString()));
-
-        // Per-cluster rotations
-        var routingPolicies = controller.applications().routingPolicies().get(instance.id()).values();
-        for (var policy : routingPolicies) {
-            if (!policy.status().isActive()) continue;
-            policy.globalEndpointsIn(controller.system()).asList().stream()
-                  .map(Endpoint::url)
-                  .map(URI::toString)
-                  .forEach(globalRotationsArray::addString);
-        }
+        // Global endpoint
+        globalEndpointsToSlime(object, instance);
 
         // Deployments sorted according to deployment spec
         List<Deployment> deployments =
@@ -1038,30 +1000,35 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
               .ifPresent(version -> toSlime(version, object.setObject("revision")));
     }
 
+    private void toSlime(Endpoint endpoint, String cluster, Cursor object) {
+        object.setString("cluster", cluster);
+        object.setBool("tls", endpoint.tls());
+        object.setString("url", endpoint.url().toString());
+        object.setString("scope", endpointScopeString(endpoint.scope()));
+        object.setString("routingMethod", routingMethodString(endpoint.routingMethod()));
+    }
+
     private void toSlime(Cursor response, DeploymentId deploymentId, Deployment deployment, HttpRequest request) {
         response.setString("tenant", deploymentId.applicationId().tenant().value());
         response.setString("application", deploymentId.applicationId().application().value());
         response.setString("instance", deploymentId.applicationId().instance().value()); // pointless
         response.setString("environment", deploymentId.zoneId().environment().value());
         response.setString("region", deploymentId.zoneId().region().value());
+        var application = controller.applications().requireApplication(TenantAndApplicationId.from(deploymentId.applicationId()));
 
-        // Add endpoint(s) defined by routing policies
+        // Add zone endpoints
         var endpointArray = response.setArray("endpoints");
-        for (var policy : controller.applications().routingPolicies().get(deploymentId).values()) {
-            if (!policy.status().isActive()) continue;
-            Cursor endpointObject = endpointArray.addObject();
-            Endpoint endpoint = policy.endpointIn(controller.system());
-            endpointObject.setString("cluster", policy.id().cluster().value());
-            endpointObject.setBool("tls", endpoint.tls());
-            endpointObject.setString("url", endpoint.url().toString());
+        for (var endpoint : controller.routing().endpointsOf(deploymentId)) {
+            toSlime(endpoint, endpoint.name(), endpointArray.addObject());
+        }
+        // Add global endpoints
+        var globalEndpoints = controller.routing().endpointsOf(application, deploymentId.applicationId().instance())
+                                        .not().legacy()
+                                        .targets(deploymentId.zoneId());
+        for (var endpoint : globalEndpoints) {
+            // TODO(mpolden): Pass cluster name. Cluster that a global endpoint points to is not available at this level.
+            toSlime(endpoint, "", endpointArray.addObject());
         }
-
-        // serviceUrls contains zone/cluster-specific endpoints for this deployment. The name of these endpoints may
-        // contain  the cluster name (if non-default) and since the controller has no knowledge of clusters, we have to
-        // ask the routing layer here
-        Cursor serviceUrlArray = response.setArray("serviceUrls");
-        controller.applications().getDeploymentEndpoints(deploymentId)
-                  .forEach(endpoint -> serviceUrlArray.addString(endpoint.toString()));
 
         response.setString("nodes", withPath("/zone/v2/" + deploymentId.zoneId().environment() + "/" + deploymentId.zoneId().region() + "/nodes/v2/node/?&recursive=true&application=" + deploymentId.applicationId().tenant() + "." + deploymentId.applicationId().application() + "." + deploymentId.applicationId().instance(), request.getUri()).toString());
         response.setString("yamasUrl", monitoringSystemUri(deploymentId).toString());
@@ -1071,12 +1038,11 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
         controller.zoneRegistry().getDeploymentTimeToLive(deploymentId.zoneId())
                 .ifPresent(deploymentTimeToLive -> response.setLong("expiryTimeEpochMs", deployment.at().plus(deploymentTimeToLive).toEpochMilli()));
 
-        Application application = controller.applications().requireApplication(TenantAndApplicationId.from(deploymentId.applicationId()));
         DeploymentStatus status = controller.jobController().deploymentStatus(application);
         application.projectId().ifPresent(i -> response.setString("screwdriverId", String.valueOf(i)));
         sourceRevisionToSlime(deployment.applicationVersion().source(), response);
 
-        Instance instance = application.instances().get(deploymentId.applicationId().instance());
+        var instance = application.instances().get(deploymentId.applicationId().instance());
         if (instance != null) {
             if (!instance.rotations().isEmpty() && deployment.zone().environment() == Environment.prod)
                 toSlime(instance.rotations(), instance.rotationStatus(), deployment, response);
@@ -1103,11 +1069,6 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
         deployment.activity().lastQueriesPerSecond().ifPresent(value -> activity.setDouble("lastQueriesPerSecond", value));
         deployment.activity().lastWritesPerSecond().ifPresent(value -> activity.setDouble("lastWritesPerSecond", value));
 
-        // Cost
-        DeploymentCost appCost = new DeploymentCost(Map.of());
-        Cursor costObject = response.setObject("cost");
-        toSlime(appCost, costObject);
-
         // Metrics
         DeploymentMetrics metrics = deployment.metrics();
         Cursor metricsObject = response.setObject("metrics");
@@ -1130,7 +1091,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
     }
 
     private void sourceRevisionToSlime(Optional<SourceRevision> revision, Cursor object) {
-        if ( ! revision.isPresent()) return;
+        if (revision.isEmpty()) return;
         object.setString("gitRepository", revision.get().repository());
         object.setString("gitBranch", revision.get().branch());
         object.setString("gitCommit", revision.get().commit());
@@ -1166,20 +1127,21 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
      */
     private Version compileVersion(TenantAndApplicationId id) {
         Version oldestPlatform = controller.applications().oldestInstalledPlatform(id);
-        return controller.versionStatus().versions().stream()
-                         .filter(version -> version.confidence().equalOrHigherThan(VespaVersion.Confidence.low))
-                         .filter(VespaVersion::isReleased)
-                         .map(VespaVersion::versionNumber)
-                         .filter(version -> ! version.isAfter(oldestPlatform))
-                         .max(Comparator.naturalOrder())
-                         .orElseGet(() -> controller.mavenRepository().metadata().versions().stream()
-                                                    .filter(version -> ! version.isAfter(oldestPlatform))
-                                                    .filter(version -> ! controller.versionStatus().versions().stream()
-                                                                                   .map(VespaVersion::versionNumber)
-                                                                                   .collect(Collectors.toSet()).contains(version))
-                                                    .max(Comparator.naturalOrder())
-                                                    .orElseThrow(() -> new IllegalStateException("No available releases of " +
-                                                                                                 controller.mavenRepository().artifactId())));
+        VersionStatus versionStatus = controller.versionStatus();
+        return versionStatus.versions().stream()
+                            .filter(version -> version.confidence().equalOrHigherThan(VespaVersion.Confidence.low))
+                            .filter(VespaVersion::isReleased)
+                            .map(VespaVersion::versionNumber)
+                            .filter(version -> ! version.isAfter(oldestPlatform))
+                            .max(Comparator.naturalOrder())
+                            .orElseGet(() -> controller.mavenRepository().metadata().versions().stream()
+                                                  .filter(version -> ! version.isAfter(oldestPlatform))
+                                                  .filter(version -> ! versionStatus.versions().stream()
+                                                                                    .map(VespaVersion::versionNumber)
+                                                                                    .collect(Collectors.toSet()).contains(version))
+                                                  .max(Comparator.naturalOrder())
+                                                  .orElseThrow(() -> new IllegalStateException("No available releases of " +
+                                                                                               controller.mavenRepository().artifactId())));
     }
 
     private HttpResponse setGlobalRotationOverride(String tenantName, String applicationName, String instanceName, String environment, String region, boolean inService, HttpRequest request) {
@@ -1204,7 +1166,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
     private void setGlobalEndpointStatus(DeploymentId deployment, boolean inService, HttpRequest request) {
         var agent = isOperator(request) ? GlobalRouting.Agent.operator : GlobalRouting.Agent.tenant;
         var status = inService ? GlobalRouting.Status.in : GlobalRouting.Status.out;
-        controller.applications().routingPolicies().setGlobalRoutingStatus(deployment, status, agent);
+        controller.routing().policies().setGlobalRoutingStatus(deployment, status, agent);
     }
 
     /** Set the global rotation status for given deployment. This only applies to global endpoints backed by a rotation */
@@ -1215,7 +1177,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
         long timestamp = controller.clock().instant().getEpochSecond();
         var status = inService ? EndpointStatus.Status.in : EndpointStatus.Status.out;
         var endpointStatus = new EndpointStatus(status, reason, agent.name(), timestamp);
-        controller.applications().setGlobalRotationStatus(deployment, endpointStatus);
+        controller.routing().setGlobalRotationStatus(deployment, endpointStatus);
     }
 
     private HttpResponse getGlobalRotationOverride(String tenantName, String applicationName, String instanceName, String environment, String region) {
@@ -1223,9 +1185,9 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
                                                      ZoneId.from(environment, region));
         Slime slime = new Slime();
         Cursor array = slime.setObject().setArray("globalrotationoverride");
-        controller.applications().globalRotationStatus(deploymentId)
+        controller.routing().globalRotationStatus(deploymentId)
                   .forEach((endpoint, status) -> {
-                      array.addString(endpoint.upstreamName());
+                      array.addString(endpoint.upstreamIdOf(deploymentId));
                       Cursor statusObject = array.addObject();
                       statusObject.setString("status", status.getStatus().name());
                       statusObject.setString("reason", status.getReason() == null ? "" : status.getReason());
@@ -1252,33 +1214,34 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
     }
 
     private HttpResponse metering(String tenant, String application, HttpRequest request) {
+
         Slime slime = new Slime();
         Cursor root = slime.setObject();
 
-        MeteringInfo meteringInfo = controller.serviceRegistry()
+        MeteringData meteringData = controller.serviceRegistry()
                 .meteringService()
-                .getResourceSnapshots(TenantName.from(tenant), ApplicationName.from(application));
+                .getMeteringData(TenantName.from(tenant), ApplicationName.from(application));
 
-        ResourceAllocation currentSnapshot = meteringInfo.getCurrentSnapshot();
+        ResourceAllocation currentSnapshot = meteringData.getCurrentSnapshot();
         Cursor currentRate = root.setObject("currentrate");
         currentRate.setDouble("cpu", currentSnapshot.getCpuCores());
         currentRate.setDouble("mem", currentSnapshot.getMemoryGb());
         currentRate.setDouble("disk", currentSnapshot.getDiskGb());
 
-        ResourceAllocation thisMonth = meteringInfo.getThisMonth();
+        ResourceAllocation thisMonth = meteringData.getThisMonth();
         Cursor thismonth = root.setObject("thismonth");
         thismonth.setDouble("cpu", thisMonth.getCpuCores());
         thismonth.setDouble("mem", thisMonth.getMemoryGb());
         thismonth.setDouble("disk", thisMonth.getDiskGb());
 
-        ResourceAllocation lastMonth = meteringInfo.getLastMonth();
+        ResourceAllocation lastMonth = meteringData.getLastMonth();
         Cursor lastmonth = root.setObject("lastmonth");
         lastmonth.setDouble("cpu", lastMonth.getCpuCores());
         lastmonth.setDouble("mem", lastMonth.getMemoryGb());
         lastmonth.setDouble("disk", lastMonth.getDiskGb());
 
 
-        Map<ApplicationId, List<ResourceSnapshot>> history = meteringInfo.getSnapshotHistory();
+        Map<ApplicationId, List<ResourceSnapshot>> history = meteringData.getSnapshotHistory();
         Cursor details = root.setObject("details");
 
         Cursor detailsCpu = details.setObject("cpu");
@@ -1349,35 +1312,22 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
     }
 
     private HttpResponse service(String tenantName, String applicationName, String instanceName, String environment, String region, String serviceName, String restPath, HttpRequest request) {
-        Map<?,?> result = controller.getServiceApiResponse(tenantName, applicationName, instanceName, environment, region, serviceName, restPath);
-        ServiceApiResponse response = new ServiceApiResponse(ZoneId.from(environment, region),
-                                                             new ApplicationId.Builder().tenant(tenantName).applicationName(applicationName).instanceName(instanceName).build(),
-                                                             controller.zoneRegistry().getConfigServerApiUris(ZoneId.from(environment, region)),
+        DeploymentId deploymentId = new DeploymentId(ApplicationId.from(tenantName, applicationName, instanceName), ZoneId.from(environment, region));
+
+        if ("container-clustercontroller".equals((serviceName)) && restPath.contains("/status/")) {
+            String result = controller.serviceRegistry().configServer().getClusterControllerStatus(deploymentId, restPath);
+            return new HtmlResponse(result);
+        }
+
+        Map<?,?> result = controller.serviceRegistry().configServer().getServiceApiResponse(deploymentId, serviceName, restPath);
+        ServiceApiResponse response = new ServiceApiResponse(deploymentId.zoneId(),
+                                                             deploymentId.applicationId(),
+                                                             controller.zoneRegistry().getConfigServerApiUris(deploymentId.zoneId()),
                                                              request.getUri());
         response.setResponse(result, serviceName, restPath);
         return response;
     }
 
-    private HttpResponse createUser(HttpRequest request) {
-        String user = Optional.of(requireUserPrincipal(request))
-                              .filter(AthenzPrincipal.class::isInstance)
-                              .map(AthenzPrincipal.class::cast)
-                              .map(AthenzPrincipal::getIdentity)
-                              .filter(AthenzUser.class::isInstance)
-                              .map(AthenzIdentity::getName)
-                              .map(UserTenant::normalizeUser)
-                              .orElseThrow(() -> new ForbiddenException("Not authenticated or not a user."));
-
-        UserTenant tenant = UserTenant.create(user);
-        try {
-            controller.tenants().createUser(tenant);
-            return new MessageResponse("Created user '" + user + "'");
-        } catch (AlreadyExistsException e) {
-            // Ok
-            return new MessageResponse("User '" + user + "' already exists");
-        }
-    }
-
     private HttpResponse updateTenant(String tenantName, HttpRequest request) {
         getTenantOrThrow(tenantName);
         TenantName tenant = TenantName.from(tenantName);
@@ -1398,11 +1348,8 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
     private HttpResponse createApplication(String tenantName, String applicationName, HttpRequest request) {
         Inspector requestObject = toSlime(request.getData()).get();
         TenantAndApplicationId id = TenantAndApplicationId.from(tenantName, applicationName);
-        Optional<Credentials> credentials = controller.tenants().require(id.tenant()).type() == Tenant.Type.user
-                                            ? Optional.empty()
-                                            : Optional.of(accessControlRequests.credentials(id.tenant(), requestObject, request.getJDiscRequest()));
+        Credentials credentials = accessControlRequests.credentials(id.tenant(), requestObject, request.getJDiscRequest());
         Application application = controller.applications().createApplication(id, credentials);
-
         Slime slime = new Slime();
         toSlime(id, slime.setObject(), request);
         return new SlimeJsonResponse(slime);
@@ -1444,7 +1391,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
                 change = change.withPin();
 
             controller.applications().deploymentTrigger().forceChange(id, change);
-            response.append("Triggered " + change + " for " + id);
+            response.append("Triggered ").append(change).append(" for ").append(id);
         });
         return new MessageResponse(response.toString());
     }
@@ -1457,7 +1404,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
         controller.applications().lockApplicationOrThrow(TenantAndApplicationId.from(id), application -> {
             Change change = Change.of(application.get().latestVersion().get());
             controller.applications().deploymentTrigger().forceChange(id, change);
-            response.append("Triggered " + change + " for " + id);
+            response.append("Triggered ").append(change).append(" for ").append(id);
         });
         return new MessageResponse(response.toString());
     }
@@ -1469,14 +1416,13 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
         controller.applications().lockApplicationOrThrow(TenantAndApplicationId.from(id), application -> {
             Change change = application.get().require(id.instance()).change();
             if (change.isEmpty()) {
-                response.append("No deployment in progress for " + id + " at this time");
+                response.append("No deployment in progress for ").append(id).append(" at this time");
                 return;
             }
 
             ChangesToCancel cancel = ChangesToCancel.valueOf(choice.toUpperCase());
             controller.applications().deploymentTrigger().cancelChange(id, cancel);
-            response.append("Changed deployment from '" + change + "' to '" +
-                            controller.applications().requireInstance(id).change() + "' for " + id);
+            response.append("Changed deployment from '").append(change).append("' to '").append(controller.applications().requireInstance(id).change()).append("' for ").append(id);
         });
 
         return new MessageResponse(response.toString());
@@ -1495,6 +1441,9 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
     }
 
     private HttpResponse jobDeploy(ApplicationId id, JobType type, HttpRequest request) {
+        if ( ! type.environment().isManuallyDeployed() && ! isOperator(request))
+            throw new IllegalArgumentException("Direct deployments are only allowed to manually deployed environments.");
+
         Map<String, byte[]> dataParts = parseDataParts(request);
         if ( ! dataParts.containsKey("applicationZip"))
             throw new IllegalArgumentException("Missing required form part 'applicationZip'");
@@ -1628,16 +1577,13 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
 
     private HttpResponse deleteTenant(String tenantName, HttpRequest request) {
         Optional<Tenant> tenant = controller.tenants().get(tenantName);
-        if ( ! tenant.isPresent())
+        if (tenant.isEmpty())
             return ErrorResponse.notFoundError("Could not delete tenant '" + tenantName + "': Tenant not found");
 
-        if (tenant.get().type() == Tenant.Type.user)
-            controller.tenants().deleteUser((UserTenant) tenant.get());
-        else
-            controller.tenants().delete(tenant.get().name(),
-                                        accessControlRequests.credentials(tenant.get().name(),
-                                                                          toSlime(request.getData()).get(),
-                                                                          request.getJDiscRequest()));
+        controller.tenants().delete(tenant.get().name(),
+                                    accessControlRequests.credentials(tenant.get().name(),
+                                                                      toSlime(request.getData()).get(),
+                                                                      request.getJDiscRequest()));
 
         // TODO: Change to a message response saying the tenant was deleted
         return tenant(tenant.get(), request);
@@ -1645,32 +1591,27 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
 
     private HttpResponse deleteApplication(String tenantName, String applicationName, HttpRequest request) {
         TenantAndApplicationId id = TenantAndApplicationId.from(tenantName, applicationName);
-        Optional<Credentials> credentials = controller.tenants().require(id.tenant()).type() == Tenant.Type.user
-                                            ? Optional.empty()
-                                            : Optional.of(accessControlRequests.credentials(id.tenant(), toSlime(request.getData()).get(), request.getJDiscRequest()));
+        Credentials credentials = accessControlRequests.credentials(id.tenant(), toSlime(request.getData()).get(), request.getJDiscRequest());
         controller.applications().deleteApplication(id, credentials);
         return new MessageResponse("Deleted application " + id);
     }
 
     private HttpResponse deleteInstance(String tenantName, String applicationName, String instanceName, HttpRequest request) {
         TenantAndApplicationId id = TenantAndApplicationId.from(tenantName, applicationName);
-        Optional<Credentials> credentials = controller.tenants().require(id.tenant()).type() == Tenant.Type.user
-                ? Optional.empty()
-                : Optional.of(accessControlRequests.credentials(id.tenant(), toSlime(request.getData()).get(), request.getJDiscRequest()));
         controller.applications().deleteInstance(id.instance(instanceName));
-        if (controller.applications().requireApplication(id).instances().isEmpty())
+        if (controller.applications().requireApplication(id).instances().isEmpty()) {
+            Credentials credentials = accessControlRequests.credentials(id.tenant(), toSlime(request.getData()).get(), request.getJDiscRequest());
             controller.applications().deleteApplication(id, credentials);
+        }
         return new MessageResponse("Deleted instance " + id.instance(instanceName).toFullString());
     }
 
     private HttpResponse deactivate(String tenantName, String applicationName, String instanceName, String environment, String region, HttpRequest request) {
-        Instance instance = controller.applications().requireInstance(ApplicationId.from(tenantName, applicationName, instanceName));
-
+        DeploymentId id = new DeploymentId(ApplicationId.from(tenantName, applicationName, instanceName),
+                                           ZoneId.from(environment, region));
         // Attempt to deactivate application even if the deployment is not known by the controller
-        DeploymentId deploymentId = new DeploymentId(instance.id(), ZoneId.from(environment, region));
-        controller.applications().deactivate(deploymentId.applicationId(), deploymentId.zoneId());
-
-        return new MessageResponse("Deactivated " + deploymentId);
+        controller.applications().deactivate(id.applicationId(), id.zoneId());
+        return new MessageResponse("Deactivated " + id);
     }
 
     /** Returns test config for indicated job, with production deployments of the default instance. */
@@ -1692,7 +1633,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
         return new SlimeJsonResponse(testConfigSerializer.configSlime(id,
                                                                       type,
                                                                       false,
-                                                                      controller.applications().clusterEndpoints(deployments),
+                                                                      controller.routing().zoneEndpointsOf(deployments),
                                                                       controller.applications().contentClustersByZone(deployments)));
     }
 
@@ -1733,7 +1674,6 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
                     });
                 });
                 break;
-            case user: break;
             case cloud: {
                 CloudTenant cloudTenant = (CloudTenant) tenant;
 
@@ -1748,10 +1688,12 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
             }
             default: throw new IllegalArgumentException("Unexpected tenant type '" + tenant.type() + "'.");
         }
+        // TODO jonmv: This should list applications, not instances.
         Cursor applicationArray = object.setArray("applications");
         for (Application application : applications) {
             DeploymentStatus status = controller.jobController().deploymentStatus(application);
-            for (Instance instance : application.instances().values())
+            for (Instance instance : showOnlyProductionInstances(request) ? application.productionInstances().values()
+                                                                          : application.instances().values())
                 if (recurseOverApplications(request))
                     toSlime(applicationArray.addObject(), instance, status, request);
                 else
@@ -1770,7 +1712,6 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
                 metaData.setString("athensDomain", athenzTenant.domain().getName());
                 metaData.setString("property", athenzTenant.property().id());
                 break;
-            case user: break;
             case cloud: break;
             default: throw new IllegalArgumentException("Unexpected tenant type '" + tenant.type() + "'.");
         }
@@ -1920,57 +1861,6 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
         return controller.versionStatus().versions().stream().anyMatch(v -> v.versionNumber().equals(version));
     }
 
-    public static void toSlime(DeploymentCost deploymentCost, Cursor object) {
-        object.setLong("tco", (long)deploymentCost.getTco());
-        object.setLong("waste", (long)deploymentCost.getWaste());
-        object.setDouble("utilization", deploymentCost.getUtilization());
-        Cursor clustersObject = object.setObject("cluster");
-        for (Map.Entry<String, ClusterCost> clusterEntry : deploymentCost.getCluster().entrySet())
-            toSlime(clusterEntry.getValue(), clustersObject.setObject(clusterEntry.getKey()));
-    }
-
-    private static void toSlime(ClusterCost clusterCost, Cursor object) {
-        object.setLong("count", clusterCost.getClusterInfo().getHostnames().size());
-        object.setString("resource", getResourceName(clusterCost.getResultUtilization()));
-        object.setDouble("utilization", clusterCost.getResultUtilization().getMaxUtilization());
-        object.setLong("tco", (int)clusterCost.getTco());
-        object.setLong("waste", (int)clusterCost.getWaste());
-        object.setString("flavor", clusterCost.getClusterInfo().getFlavor());
-        object.setDouble("flavorCost", clusterCost.getClusterInfo().getFlavorCost());
-        object.setDouble("flavorCpu", clusterCost.getClusterInfo().getFlavorCPU());
-        object.setDouble("flavorMem", clusterCost.getClusterInfo().getFlavorMem());
-        object.setDouble("flavorDisk", clusterCost.getClusterInfo().getFlavorDisk());
-        object.setString("type", clusterCost.getClusterInfo().getClusterType().name());
-        Cursor utilObject = object.setObject("util");
-        utilObject.setDouble("cpu", clusterCost.getResultUtilization().getCpu());
-        utilObject.setDouble("mem", clusterCost.getResultUtilization().getMemory());
-        utilObject.setDouble("disk", clusterCost.getResultUtilization().getDisk());
-        utilObject.setDouble("diskBusy", clusterCost.getResultUtilization().getDiskBusy());
-        Cursor usageObject = object.setObject("usage");
-        usageObject.setDouble("cpu", clusterCost.getSystemUtilization().getCpu());
-        usageObject.setDouble("mem", clusterCost.getSystemUtilization().getMemory());
-        usageObject.setDouble("disk", clusterCost.getSystemUtilization().getDisk());
-        usageObject.setDouble("diskBusy", clusterCost.getSystemUtilization().getDiskBusy());
-        Cursor hostnamesArray = object.setArray("hostnames");
-        for (String hostname : clusterCost.getClusterInfo().getHostnames())
-            hostnamesArray.addString(hostname);
-    }
-
-    private static String getResourceName(ClusterUtilization utilization) {
-        String name = "cpu";
-        double max = utilization.getMaxUtilization();
-
-        if (utilization.getMemory() == max) {
-            name = "mem";
-        } else if (utilization.getDisk() == max) {
-            name = "disk";
-        } else if (utilization.getDiskBusy() == max) {
-            name = "diskbusy";
-        }
-
-        return name;
-    }
-
     private static boolean recurseOverTenants(HttpRequest request) {
         return recurseOverApplications(request) || "tenant".equals(request.getProperty("recursive"));
     }
@@ -1983,9 +1873,12 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
         return ImmutableSet.of("all", "true", "deployment").contains(request.getProperty("recursive"));
     }
 
+    private static boolean showOnlyProductionInstances(HttpRequest request) {
+        return "true".equals(request.getProperty("production"));
+    }
+
     private static String tenantType(Tenant tenant) {
         switch (tenant.type()) {
-            case user: return "USER";
             case athenz: return "ATHENS";
             case cloud: return "CLOUD";
             default: throw new IllegalArgumentException("Unknown tenant type: " + tenant.getClass().getSimpleName());
@@ -2008,7 +1901,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
     private HttpResponse submit(String tenant, String application, HttpRequest request) {
         Map<String, byte[]> dataParts = parseDataParts(request);
         Inspector submitOptions = SlimeUtils.jsonToSlime(dataParts.get(EnvironmentResource.SUBMIT_OPTIONS)).get();
-        long projectId = Math.max(1, submitOptions.field("projectId").asLong());
+        long projectId = Math.max(1, submitOptions.field("projectId").asLong()); // Absence of this means it's not a prod app :/
         Optional<String> repository = optional("repository", submitOptions);
         Optional<String> branch = optional("branch", submitOptions);
         Optional<String> commit = optional("commit", submitOptions);
@@ -2081,6 +1974,23 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
         return "UNKNOWN";
     }
 
+    private static String endpointScopeString(Endpoint.Scope scope) {
+        switch (scope) {
+            case global: return "global";
+            case zone: return "zone";
+        }
+        throw new IllegalArgumentException("Unknown endpoint scope " + scope);
+    }
+
+    private static String routingMethodString(RoutingMethod method) {
+        switch (method) {
+            case exclusive: return "exclusive";
+            case shared: return "shared";
+            case sharedLayer4: return "sharedLayer4";
+        }
+        throw new IllegalArgumentException("Unknown routing method " + method);
+    }
+
     private static <T> T getAttribute(HttpRequest request, String attributeName, Class<T> cls) {
         return Optional.ofNullable(request.getJDiscRequest().context().get(attributeName))
                        .filter(cls::isInstance)
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/HtmlResponse.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/HtmlResponse.java
new file mode 100644
index 00000000000..99884875a64
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/HtmlResponse.java
@@ -0,0 +1,30 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.restapi.application;
+
+import com.yahoo.container.jdisc.HttpResponse;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.charset.StandardCharsets;
+
+/**
+ * @author freva
+ */
+public class HtmlResponse extends HttpResponse {
+
+    private final String content;
+
+    public HtmlResponse(String content) {
+        super(200);
+        this.content = content;
+    }
+
+    @Override
+    public void render(OutputStream stream) throws IOException {
+        stream.write(content.getBytes(StandardCharsets.UTF_8));
+    }
+
+    @Override
+    public String getContentType() { return "text/html"; }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java
index 91a0455db11..f72439b694a 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java
@@ -5,6 +5,7 @@ import com.google.common.base.Joiner;
 import com.yahoo.component.Version;
 import com.yahoo.config.application.api.DeploymentInstanceSpec;
 import com.yahoo.config.application.api.DeploymentSpec;
+import com.yahoo.config.application.api.DeploymentSpec.ChangeBlocker;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.SystemName;
 import com.yahoo.config.provision.zone.ZoneId;
@@ -41,15 +42,22 @@ import com.yahoo.vespa.hosted.controller.deployment.Versions;
 import com.yahoo.vespa.hosted.controller.versions.VespaVersion;
 
 import java.net.URI;
+import java.time.Instant;
+import java.time.format.TextStyle;
 import java.util.ArrayDeque;
+import java.util.Arrays;
+import java.util.Collection;
 import java.util.Comparator;
 import java.util.Deque;
 import java.util.LinkedHashMap;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.Optional;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
+import static com.yahoo.config.application.api.DeploymentSpec.UpgradePolicy.canary;
 import static com.yahoo.config.application.api.DeploymentSpec.UpgradePolicy.conservative;
 import static com.yahoo.config.application.api.DeploymentSpec.UpgradePolicy.defaultPolicy;
 import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.stagingTest;
@@ -175,6 +183,20 @@ class JobControllerApiHandlerHelper {
                               devJobObject.setString("url", baseUriForJobs.resolve(baseUriForJobs.getPath() + "/" + type.jobName()).normalize().toString());
                           });
 
+        Cursor jobsArray = responseObject.setArray("deployment");
+        Arrays.stream(JobType.values())
+              .filter(type -> type.environment().isManuallyDeployed())
+              .map(devType -> new JobId(instance.id(), devType))
+              .forEach(job -> {
+                  Collection<Run> runs = controller.jobController().runs(job).descendingMap().values();
+                  if (runs.isEmpty())
+                      return;
+
+                  Cursor jobObject = jobsArray.addObject();
+                  jobObject.setString("jobName", job.type().jobName());
+                  toSlime(jobObject.setArray("runs"), runs, baseUriForJobs);
+              });
+
         return new SlimeJsonResponse(slime);
     }
 
@@ -415,9 +437,11 @@ class JobControllerApiHandlerHelper {
 
         versionObject.setLong("build", version.buildNumber().getAsLong());
         Cursor sourceObject = versionObject.setObject("source");
-        sourceObject.setString("gitRepository", version.source().get().repository());
-        sourceObject.setString("gitBranch", version.source().get().branch());
-        sourceObject.setString("gitCommit", version.source().get().commit());
+        version.source().ifPresent(source -> {
+            sourceObject.setString("gitRepository", source.repository());
+            sourceObject.setString("gitBranch", source.branch());
+            sourceObject.setString("gitCommit", source.commit());
+        });
         version.sourceUrl().ifPresent(url -> versionObject.setString("sourceUrl", url));
         version.commit().ifPresent(commit -> versionObject.setString("commit", commit));
     }
@@ -433,9 +457,11 @@ class JobControllerApiHandlerHelper {
                                .orElseThrow(() -> new IllegalStateException("Unknown run '" + runId + "'"));
         detailsObject.setBool("active", ! run.hasEnded());
         detailsObject.setString("status", nameOf(run.status()));
-        jobController.updateTestLog(runId);
-        try { jobController.updateVespaLog(runId); }
-        catch (RuntimeException ignored) { } // May be perfectly fine, e.g., when logserver isn't up yet.
+        try {
+            jobController.updateTestLog(runId);
+            jobController.updateVespaLog(runId);
+        }
+        catch (RuntimeException ignored) { } // Return response when this fails, which it does when, e.g., logserver is booting.
 
         RunLog runLog = (after == null ? jobController.details(runId) : jobController.details(runId, Long.parseLong(after)))
                 .orElseThrow(() -> new NotExistsException(String.format(
@@ -529,15 +555,16 @@ class JobControllerApiHandlerHelper {
 
     private static String nameOf(RunStatus status) {
         switch (status) {
-            case running:               return "running";
-            case aborted:               return "aborted";
-            case error:                 return "error";
-            case testFailure:           return "testFailure";
-            case outOfCapacity:         return "outOfCapacity";
-            case installationFailed:    return "installationFailed";
-            case deploymentFailed:      return "deploymentFailed";
-            case success:               return "success";
-            default:                    throw new IllegalArgumentException("Unexpected status '" + status + "'");
+            case running:                    return "running";
+            case aborted:                    return "aborted";
+            case error:                      return "error";
+            case testFailure:                return "testFailure";
+            case endpointCertificateTimeout: return "endpointCertificateTimeout";
+            case outOfCapacity:              return "outOfCapacity";
+            case installationFailed:         return "installationFailed";
+            case deploymentFailed:           return "deploymentFailed";
+            case success:                    return "success";
+            default:                         throw new IllegalArgumentException("Unexpected status '" + status + "'");
         }
     }
 
@@ -552,6 +579,7 @@ class JobControllerApiHandlerHelper {
         Cursor responseObject = slime.setObject();
         responseObject.setString("tenant", id.tenant().value());
         responseObject.setString("application", id.application().value());
+        application.projectId().ifPresent(projectId -> responseObject.setLong("projectId", projectId));
 
         Map<JobId, List<Versions>> jobsToRun = status.jobsToRun();
         Cursor stepsArray = responseObject.setArray("steps");
@@ -565,12 +593,55 @@ class JobControllerApiHandlerHelper {
             stepObject.setBool("declared", stepStatus.isDeclared());
             stepObject.setString("instance", stepStatus.instance().value());
 
+            stepStatus.readyAt(change).ifPresent(ready -> stepObject.setLong("readyAt", ready.toEpochMilli()));
+            stepStatus.readyAt(change)
+                      .filter(controller.clock().instant()::isBefore)
+                      .ifPresent(until -> stepObject.setLong("delayedUntil", until.toEpochMilli()));
+            stepStatus.pausedUntil().ifPresent(until -> stepObject.setLong("pausedUntil", until.toEpochMilli()));
+            stepStatus.coolingDownUntil(change).ifPresent(until -> stepObject.setLong("coolingDownUntil", until.toEpochMilli()));
+            stepStatus.blockedUntil(Change.of(controller.systemVersion())) // Dummy version — just anything with a platform.
+                      .ifPresent(until -> stepObject.setLong("platformBlockedUntil", until.toEpochMilli()));
+            application.latestVersion().map(Change::of).flatMap(stepStatus::blockedUntil) // Dummy version — just anything with an application.
+                      .ifPresent(until -> stepObject.setLong("applicationBlockedUntil", until.toEpochMilli()));
+
+            if (stepStatus.type() == DeploymentStatus.StepType.delay)
+                stepStatus.completedAt(change).ifPresent(completed -> stepObject.setLong("completedAt", completed.toEpochMilli()));
+
+            if (stepStatus.type() == DeploymentStatus.StepType.instance) {
+                Cursor deployingObject = stepObject.setObject("deploying");
+                if ( ! change.isEmpty()) {
+                    change.platform().ifPresent(version -> deployingObject.setString("platform", version.toString()));
+                    change.application().ifPresent(version -> toSlime(deployingObject.setObject("application"), version));
+                }
+
+                Cursor latestVersionsObject = stepObject.setObject("latestVersions");
+                List<ChangeBlocker> blockers = application.deploymentSpec().requireInstance(stepStatus.instance()).changeBlocker();
+                latestVersionWithCompatibleConfidenceAndNotNewerThanSystem(controller.versionStatus().versions(),
+                                                                           application.deploymentSpec().requireInstance(stepStatus.instance()).upgradePolicy())
+                          .ifPresent(latestPlatform -> {
+                              Cursor latestPlatformObject = latestVersionsObject.setObject("platform");
+                              latestPlatformObject.setString("platform", latestPlatform.versionNumber().toFullString());
+                              latestPlatformObject.setLong("at", latestPlatform.committedAt().toEpochMilli());
+                              latestPlatformObject.setBool("upgrade", application.require(stepStatus.instance()).productionDeployments().values().stream()
+                                                                                 .anyMatch(deployment -> deployment.version().isBefore(latestPlatform.versionNumber())));
+                              toSlime(latestPlatformObject.setArray("blockers"), blockers.stream().filter(ChangeBlocker::blocksVersions));
+                          });
+                application.latestVersion().ifPresent(latestApplication -> {
+                    Cursor latestApplicationObject = latestVersionsObject.setObject("application");
+                    toSlime(latestApplicationObject.setObject("application"), latestApplication);
+                    latestApplicationObject.setLong("at", latestApplication.buildTime().orElse(Instant.EPOCH).toEpochMilli());
+                    latestApplicationObject.setBool("upgrade", application.require(stepStatus.instance()).productionDeployments().values().stream()
+                                                                          .anyMatch(deployment -> deployment.applicationVersion().compareTo(latestApplication) < 0));
+                    toSlime(latestApplicationObject.setArray("blockers"), blockers.stream().filter(ChangeBlocker::blocksRevisions));
+                });
+            }
+
             stepStatus.job().ifPresent(job -> {
                 stepObject.setString("jobName", job.type().jobName());
-                String baseUriForJob = baseUriForDeployments.resolve(baseUriForDeployments.getPath() +
+                URI baseUriForJob = baseUriForDeployments.resolve(baseUriForDeployments.getPath() +
                                                                      "/../instance/" + job.application().instance().value() +
-                                                                     "/job/" + job.type().jobName()).normalize().toString();
-                stepObject.setString("url", baseUriForJob);
+                                                                     "/job/" + job.type().jobName()).normalize();
+                stepObject.setString("url", baseUriForJob.toString());
                 stepObject.setString("environment", job.type().environment().value());
                 stepObject.setString("region", job.type().zone(controller.system()).value());
 
@@ -595,41 +666,16 @@ class JobControllerApiHandlerHelper {
                     Cursor runObject = toRunArray.addObject();
                     toSlime(runObject.setObject("versions"), versions);
                 }
-                stepStatus.readyAt(change).ifPresent(ready -> stepObject.setLong("readyAt", ready.toEpochMilli()));
-                stepStatus.readyAt(change)
-                          .filter(controller.clock().instant()::isBefore)
-                          .ifPresent(until -> stepObject.setLong("delayedUntil", until.toEpochMilli()));
-                stepStatus.pausedUntil().ifPresent(until -> stepObject.setLong("pausedUntil", until.toEpochMilli()));
-                stepStatus.coolingDownUntil(change).ifPresent(until -> stepObject.setLong("coolingDownUntil", until.toEpochMilli()));
-                stepStatus.blockedUntil(change).ifPresent(until -> stepObject.setLong("blockedUntil", until.toEpochMilli()));
-
-                Cursor runsArray = stepObject.setArray("runs");
-                jobStatus.runs().descendingMap().values().stream().limit(10).forEach(run -> {
-                    Cursor runObject = runsArray.addObject();
-                    runObject.setLong("id", run.id().number());
-                    runObject.setString("url", baseUriForJob + "/run/" + run.id());
-                    runObject.setLong("start", run.start().toEpochMilli());
-                    run.end().ifPresent(end -> runObject.setLong("end", end.toEpochMilli()));
-                    runObject.setString("status", run.status().name());
-                    toSlime(runObject.setObject("versions"), run.versions());
-                    Cursor runStepsArray = runObject.setArray("steps");
-                    run.steps().forEach((step, info) -> {
-                        Cursor runStepObject = runStepsArray.addObject();
-                        runStepObject.setString("name", step.name());
-                        runStepObject.setString("status", info.status().name());
-                    });
-                });
+
+                toSlime(stepObject.setArray("runs"), jobStatus.runs().descendingMap().values(), baseUriForJob);
             });
         }
 
-        // TODO jonmv: Add latest platform and application status.
-
         return new SlimeJsonResponse(slime);
     }
 
     private static void toSlime(Cursor versionObject, ApplicationVersion version) {
-        version.buildNumber().ifPresent(id -> versionObject.setLong("id", id));
-        version.source().ifPresent(source -> versionObject.setString("commit", source.commit()));
+        version.buildNumber().ifPresent(id -> versionObject.setLong("build", id));
         version.compileVersion().ifPresent(platform -> versionObject.setString("compileVersion", platform.toFullString()));
         version.sourceUrl().ifPresent(url -> versionObject.setString("sourceUrl", url));
         version.commit().ifPresent(commit -> versionObject.setString("commit", commit));
@@ -642,5 +688,52 @@ class JobControllerApiHandlerHelper {
         versions.sourceApplication().ifPresent(application -> toSlime(versionsObject.setObject("sourceApplication"), application));
     }
 
-}
+    private static void toSlime(Cursor blockersArray, Stream<ChangeBlocker> blockers) {
+        blockers.forEach(blocker -> {
+            Cursor blockerObject = blockersArray.addObject();
+            blocker.window().days().stream()
+                   .map(day -> day.getDisplayName(TextStyle.SHORT, Locale.ENGLISH))
+                   .forEach(blockerObject.setArray("days")::addString);
+            blocker.window().hours()
+                   .forEach(blockerObject.setArray("hours")::addLong);
+            blockerObject.setString("zone", blocker.window().zone().toString());
+        });
+    }
+
+    private static Optional<VespaVersion> latestVersionWithCompatibleConfidenceAndNotNewerThanSystem(List<VespaVersion> versions,
+                                                                                                     DeploymentSpec.UpgradePolicy policy) {
+        int i;
+        for (i = versions.size(); i-- > 0; )
+            if (versions.get(i).isSystemVersion())
+                break;
 
+        if (i < 0)
+            return Optional.empty();
+
+        VespaVersion.Confidence required = policy == canary ? broken : normal;
+        for (int j = i; j >= 0; j--)
+            if (versions.get(j).confidence().equalOrHigherThan(required))
+                return Optional.of(versions.get(j));
+
+        return Optional.of(versions.get(i));
+    }
+
+    private static void toSlime(Cursor runsArray, Collection<Run> runs, URI baseUriForJob) {
+        runs.stream().limit(10).forEach(run -> {
+            Cursor runObject = runsArray.addObject();
+            runObject.setLong("id", run.id().number());
+            runObject.setString("url", baseUriForJob.resolve(baseUriForJob.getPath() + "/run/" + run.id().number()).toString());
+            runObject.setLong("start", run.start().toEpochMilli());
+            run.end().ifPresent(end -> runObject.setLong("end", end.toEpochMilli()));
+            runObject.setString("status", run.status().name());
+            toSlime(runObject.setObject("versions"), run.versions());
+            Cursor runStepsArray = runObject.setArray("steps");
+            run.steps().forEach((step, info) -> {
+                Cursor runStepObject = runStepsArray.addObject();
+                runStepObject.setString("name", step.name());
+                runStepObject.setString("status", info.status().name());
+            });
+        });
+    }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java
index 2cadd864df1..ea3559a92ae 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java
@@ -8,7 +8,7 @@ import com.yahoo.container.jdisc.LoggingRequestHandler;
 import com.yahoo.io.IOUtils;
 import com.yahoo.restapi.Path;
 import com.yahoo.slime.Inspector;
-import com.yahoo.vespa.config.SlimeUtils;
+import com.yahoo.slime.SlimeUtils;
 import com.yahoo.vespa.hosted.controller.Controller;
 import com.yahoo.vespa.hosted.controller.auditlog.AuditLoggingRequestHandler;
 import com.yahoo.vespa.hosted.controller.maintenance.ControllerMaintenance;
@@ -70,6 +70,7 @@ public class ControllerApiHandler extends AuditLoggingRequestHandler {
         if (path.matches("/controller/v1/auditlog/")) return new AuditLogResponse(controller.auditLogger().readLog());
         if (path.matches("/controller/v1/maintenance/")) return new JobsResponse(maintenance.jobControl());
         if (path.matches("/controller/v1/jobs/upgrader")) return new UpgraderResponse(maintenance.upgrader());
+        if (path.matches("/controller/v1/metering/tenant/{tenant}/month/{month}")) return new MeteringResponse(controller.serviceRegistry().meteringService(), path.get("tenant"), path.get("month"));
         return notFound(path);
     }
 
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/MeteringResponse.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/MeteringResponse.java
new file mode 100644
index 00000000000..23e3195db01
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/MeteringResponse.java
@@ -0,0 +1,39 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.restapi.controller;
+
+import com.yahoo.config.provision.TenantName;
+import com.yahoo.restapi.SlimeJsonResponse;
+import com.yahoo.slime.Cursor;
+import com.yahoo.slime.Slime;
+import com.yahoo.vespa.hosted.controller.api.integration.resource.MeteringClient;
+import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceSnapshot;
+
+import java.time.YearMonth;
+import java.util.List;
+
+/**
+ * @author olaa
+ */
+public class MeteringResponse extends SlimeJsonResponse {
+
+    public MeteringResponse(MeteringClient meteringClient, String tenantName, String month) {
+        super(toSlime(meteringClient, tenantName, month));
+    }
+
+    private static Slime toSlime(MeteringClient meteringClient, String tenantName, String month) {
+        Slime slime = new Slime();
+        Cursor root = slime.setArray();
+        List<ResourceSnapshot> snapshots = meteringClient.getSnapshotHistoryForTenant(TenantName.from(tenantName), YearMonth.parse(month));
+        snapshots.forEach(snapshot -> {
+            Cursor object = root.addObject();
+            object.setString("applicationId", snapshot.getApplicationId().toShortString());
+            object.setLong("timestamp", snapshot.getTimestamp().toEpochMilli());
+            object.setString("zoneId", snapshot.getZoneId().value());
+            object.setDouble("cpu", snapshot.getCpuCores());
+            object.setDouble("memory", snapshot.getMemoryGb());
+            object.setDouble("disk", snapshot.getDiskGb());
+        });
+        return slime;
+    }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostApiHandler.java
deleted file mode 100644
index 8bf3f4a6fd0..00000000000
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostApiHandler.java
+++ /dev/null
@@ -1,52 +0,0 @@
-// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.controller.restapi.cost;
-
-import com.yahoo.config.provision.CloudName;
-import com.yahoo.container.jdisc.HttpRequest;
-import com.yahoo.container.jdisc.HttpResponse;
-import com.yahoo.container.jdisc.LoggingRequestHandler;
-import com.yahoo.restapi.Path;
-import com.yahoo.vespa.hosted.controller.Controller;
-import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepository;
-import com.yahoo.vespa.hosted.controller.api.integration.resource.CostReportConsumer;
-import com.yahoo.restapi.ErrorResponse;
-import com.yahoo.restapi.StringResponse;
-
-import java.time.Clock;
-import java.util.Optional;
-
-import static com.yahoo.jdisc.http.HttpRequest.Method.GET;
-
-/**
- * @author ldalves
- */
-public class CostApiHandler extends LoggingRequestHandler {
-
-    private final Controller controller;
-    private final NodeRepository nodeRepository;
-    private final CostReportConsumer costReportConsumer;
-
-    public CostApiHandler(Context ctx, Controller controller) {
-        super(ctx);
-        this.controller = controller;
-        this.nodeRepository = controller.serviceRegistry().configServer().nodeRepository();
-        this.costReportConsumer = controller.serviceRegistry().costReportConsumer();
-    }
-
-    @Override
-    public HttpResponse handle(HttpRequest request) {
-        if (request.getMethod() != GET) {
-            return ErrorResponse.methodNotAllowed("Method '" + request.getMethod() + "' is not supported");
-        }
-
-        Path path = new Path(request.getUri());
-
-        if (path.matches("/cost/v1/csv")) {
-            Optional<String> cloudProperty = Optional.ofNullable(request.getProperty("cloud"));
-            CloudName cloud = cloudProperty.map(CloudName::from).orElse(CloudName.defaultName());
-            return new StringResponse(CostCalculator.resourceShareByPropertyToCsv(nodeRepository, controller, Clock.systemUTC(), costReportConsumer.fixedAllocations(), cloud));
-        }
-
-        return ErrorResponse.notFoundError("Nothing at " + path);
-    }
-}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/package-info.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/package-info.java
deleted file mode 100644
index a96ae5488fa..00000000000
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/package-info.java
+++ /dev/null
@@ -1,5 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-@ExportPackage
-package com.yahoo.vespa.hosted.controller.restapi.cost;
-
-import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiHandler.java
index f3c45ca7221..befec42e84e 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiHandler.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiHandler.java
@@ -18,8 +18,6 @@ import java.net.URI;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
-import static java.util.stream.Collectors.joining;
-
 /**
  * This API serves redirects to a badge server.
  * 
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java
index d27bc581f75..2e2f318a626 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java
@@ -1,7 +1,6 @@
 // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.restapi.deployment;
 
-import com.yahoo.component.Version;
 import com.yahoo.component.Vtag;
 import com.yahoo.config.application.api.DeploymentInstanceSpec;
 import com.yahoo.config.application.api.DeploymentSpec;
@@ -17,21 +16,29 @@ import com.yahoo.restapi.Uri;
 import com.yahoo.slime.Cursor;
 import com.yahoo.slime.Slime;
 import com.yahoo.vespa.hosted.controller.Controller;
+import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
 import com.yahoo.vespa.hosted.controller.application.ApplicationList;
+import com.yahoo.vespa.hosted.controller.application.Change;
 import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId;
-import com.yahoo.vespa.hosted.controller.deployment.JobList;
+import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatus;
 import com.yahoo.vespa.hosted.controller.deployment.Run;
-import com.yahoo.vespa.hosted.controller.deployment.RunStatus;
 import com.yahoo.vespa.hosted.controller.restapi.application.EmptyResponse;
+import com.yahoo.vespa.hosted.controller.versions.DeploymentStatistics;
 import com.yahoo.vespa.hosted.controller.versions.VespaVersion;
 import com.yahoo.yolean.Exceptions;
 
-import java.time.Instant;
-import java.util.Comparator;
+import java.util.LinkedHashMap;
 import java.util.Map;
 import java.util.Optional;
 import java.util.logging.Level;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static java.util.function.Function.identity;
+import static java.util.stream.Collectors.groupingBy;
+import static java.util.stream.Collectors.toList;
+import static java.util.stream.Collectors.toMap;
+import static java.util.stream.Collectors.toUnmodifiableMap;
 
 /**
  * This implements the deployment/v1 API which provides information about the status of Vespa platform and
@@ -89,10 +96,10 @@ public class DeploymentApiHandler extends LoggingRequestHandler {
         Cursor platformArray = root.setArray("versions");
         var versionStatus = controller.versionStatus();
         var systemVersion = versionStatus.systemVersion().map(VespaVersion::versionNumber).orElse(Vtag.currentVersion);
-        Map<ApplicationId, JobList> jobs = controller.jobController().deploymentStatuses(ApplicationList.from(controller.applications().asList()), systemVersion)
-                                                     .asList().stream()
-                                                     .flatMap(status -> status.instanceJobs().entrySet().stream())
-                                                     .collect(Collectors.toUnmodifiableMap(Map.Entry::getKey, Map.Entry::getValue));
+        var deploymentStatuses = controller.jobController().deploymentStatuses(ApplicationList.from(controller.applications().asList()), systemVersion);
+        var deploymentStatistics = DeploymentStatistics.compute(versionStatus.versions().stream().map(VespaVersion::versionNumber).collect(toList()),
+                                                                deploymentStatuses)
+                                                       .stream().collect(toMap(DeploymentStatistics::version, identity()));
         for (VespaVersion version : versionStatus.versions()) {
             Cursor versionObject = platformArray.addObject();
             versionObject.setString("version", version.versionNumber().toString());
@@ -108,42 +115,106 @@ public class DeploymentApiHandler extends LoggingRequestHandler {
                 configServerObject.setString("hostname", hostname.value());
             }
 
+            DeploymentStatistics statistics = deploymentStatistics.get(version.versionNumber());
             Cursor failingArray = versionObject.setArray("failingApplications");
-            for (ApplicationId id : version.statistics().failing()) {
-                if (jobs.containsKey(id))
-                    firstFailingOn(version.versionNumber(), jobs.get(id)).ifPresent(firstFailing -> {
-                        Cursor applicationObject = failingArray.addObject();
-                        toSlime(applicationObject, id, request);
-                        applicationObject.setString("failing", firstFailing.id().type().jobName());
-                        applicationObject.setString("status", firstFailing.status().name());
-                    });
+            for (Run run : statistics.failingUpgrades()) {
+                Cursor applicationObject = failingArray.addObject();
+                toSlime(applicationObject, run.id().application(), request);
+                applicationObject.setString("failing", run.id().type().jobName());
+                applicationObject.setString("status", run.status().name());
             }
 
+            var statusByInstance = deploymentStatuses.asList().stream()
+                                                     .flatMap(status -> status.instanceJobs().keySet().stream()
+                                                                              .map(instance -> Map.entry(instance, status)))
+                                                     .collect(toUnmodifiableMap(entry -> entry.getKey(), entry -> entry.getValue()));
+            var jobsByInstance = statusByInstance.entrySet().stream()
+                                                 .collect(toUnmodifiableMap(entry -> entry.getKey(),
+                                                                            entry -> entry.getValue().instanceJobs().get(entry.getKey())));
             Cursor productionArray = versionObject.setArray("productionApplications");
-            for (ApplicationId id : version.statistics().production()) {
-                if (jobs.containsKey(id)) {
-                    int successes = productionSuccessesFor(version.versionNumber(), jobs.get(id));
-                    if (successes == 0) continue; // Just upgraded to a newer version.
-                    Cursor applicationObject = productionArray.addObject();
-                    toSlime(applicationObject, id, request);
-                    applicationObject.setLong("productionJobs", jobs.get(id).production().size());
-                    applicationObject.setLong("productionSuccesses", productionSuccessesFor(version.versionNumber(), jobs.get(id)));
-                }
-            }
+            statistics.productionSuccesses().stream()
+                      .collect(groupingBy(run -> run.id().application()))
+                      .forEach((id, runs) -> {
+                          Cursor applicationObject = productionArray.addObject();
+                          toSlime(applicationObject, id, request);
+                          applicationObject.setLong("productionJobs", jobsByInstance.get(id).production().size());
+                          applicationObject.setLong("productionSuccesses", runs.size());
+                      });
 
             Cursor runningArray = versionObject.setArray("deployingApplications");
-            for (ApplicationId id : version.statistics().deploying()) {
-                if (jobs.containsKey(id))
-                    lastDeployingTo(version.versionNumber(), jobs.get(id)).ifPresent(lastDeploying -> {
-                        Cursor applicationObject = runningArray.addObject();
-                        toSlime(applicationObject, id, request);
-                        applicationObject.setString("running", lastDeploying.id().type().jobName());
-                    });
+            for (Run run : statistics.runningUpgrade()) {
+                Cursor applicationObject = runningArray.addObject();
+                toSlime(applicationObject, run.id().application(), request);
+                applicationObject.setString("running", run.id().type().jobName());
+            }
+
+            class RunInfo { //  ヽ༼ຈل͜ຈ༽━☆ﾟ.*･｡ﾟ
+                final Run run;
+                final boolean upgrade;
+                RunInfo(Run run, boolean upgrade) { this.run = run; this.upgrade = upgrade; }
+                @Override public String toString() { return run.id().toString(); }
             }
+            Cursor instancesArray = versionObject.setArray("applications");
+            Stream.of(statistics.failingUpgrades().stream().map(run -> new RunInfo(run, true)),
+                      statistics.otherFailing().stream().map(run -> new RunInfo(run, false)),
+                      statistics.runningUpgrade().stream().map(run -> new RunInfo(run, true)),
+                      statistics.otherRunning().stream().map(run -> new RunInfo(run, false)),
+                      statistics.productionSuccesses().stream().map(run -> new RunInfo(run, true)))
+                  .flatMap(identity())
+                  .collect(Collectors.groupingBy(run -> run.run.id().application(),
+                                                 LinkedHashMap::new, // Put apps with failing and running jobs first.
+                                                 groupingBy(run -> run.run.id().type(),
+                                                            LinkedHashMap::new,
+                                                            toList())))
+                  .forEach((instance, runs) -> {
+                      var status = statusByInstance.get(instance);
+                      Cursor instanceObject = instancesArray.addObject();
+                      instanceObject.setString("tenant", instance.tenant().value());
+                      instanceObject.setString("application", instance.application().value());
+                      instanceObject.setString("instance", instance.instance().value());
+                      instanceObject.setBool("upgrading", status.application().require(instance.instance()).change().platform().equals(Optional.of(statistics.version())));
+                      status.instanceSteps().get(instance.instance()).blockedUntil(Change.of(statistics.version()))
+                            .ifPresent(until -> instanceObject.setLong("blockedUntil", until.toEpochMilli()));
+                      instanceObject.setString("upgradePolicy", toString(status.application().deploymentSpec().instance(instance.instance())
+                                                                               .map(DeploymentInstanceSpec::upgradePolicy)
+                                                                               .orElse(DeploymentSpec.UpgradePolicy.defaultPolicy)));
+                      Cursor jobsArray = instanceObject.setArray("jobs");
+                      status.jobSteps().forEach((job, jobStatus) -> {
+                          if ( ! job.application().equals(instance)) return;
+                          Cursor jobObject = jobsArray.addObject();
+                          jobObject.setString("name", job.type().jobName());
+                          jobStatus.pausedUntil().ifPresent(until -> jobObject.setLong("pausedUntil", until.toEpochMilli()));
+                          jobStatus.coolingDownUntil(status.application().require(instance.instance()).change())
+                                   .ifPresent(until -> jobObject.setLong("coolingDownUntil", until.toEpochMilli()));
+                      });
+                      Cursor allRunsObject = instanceObject.setObject("allRuns");
+                      Cursor upgradeRunsObject = instanceObject.setObject("upgradeRuns");
+                      runs.forEach((type, rs) -> {
+                          Cursor runObject = allRunsObject.setObject(type.jobName());
+                          Cursor upgradeObject = upgradeRunsObject.setObject(type.jobName());
+                          for (RunInfo run : rs) {
+                              toSlime(runObject, run.run);
+                              if (run.upgrade)
+                                  toSlime(upgradeObject, run.run);
+                          }
+                      });
+                  });
         }
+        JobType.allIn(controller.system()).stream()
+               .filter(job -> ! job.environment().isManuallyDeployed())
+               .map(JobType::jobName).forEach(root.setArray("jobs")::addString);
         return new SlimeJsonResponse(slime);
     }
 
+    private void toSlime(Cursor jobObject, Run run) {
+        String key = run.hasFailed() ? "failing" : run.hasEnded() ? "success" : "running";
+        Cursor runObject = jobObject.setObject(key);
+        runObject.setLong("number", run.id().number());
+        runObject.setLong("start", run.start().toEpochMilli());
+        run.end().ifPresent(end -> runObject.setLong("end", end.toEpochMilli()));
+        runObject.setString("status", run.status().name());
+    }
+
     private void toSlime(Cursor object, ApplicationId id, HttpRequest request) {
         object.setString("tenant", id.tenant().value());
         object.setString("application", id.application().value());
@@ -164,33 +235,4 @@ public class DeploymentApiHandler extends LoggingRequestHandler {
         return upgradePolicy.name();
     }
 
-    // ----------------------------- Utilities to pick out the relevant JobStatus -- filter chains should mirror the ones in VersionStatus
-
-    /** The first upgrade job to fail on this version, for this application */
-    private Optional<Run> firstFailingOn(Version version, JobList jobs) {
-        return jobs.failing()
-                   .not().failingApplicationChange()
-                   .not().withStatus(RunStatus.outOfCapacity)
-                   .lastCompleted().on(version)
-                   .lastCompleted().asList().stream()
-                   .min(Comparator.<Run, Instant>comparing(run -> run.start())
-                                .thenComparing(run -> run.id().type()));
-    }
-
-    /** The number of production jobs with last success on the given version, for this application */
-    private int productionSuccessesFor(Version version, JobList jobs) {
-        return jobs.production()
-                   .lastSuccess().on(version)
-                   .size();
-    }
-
-    /** The last triggered upgrade to this version, for this application */
-    private Optional<Run> lastDeployingTo(Version version, JobList jobs) {
-        return jobs.upgrading()
-                   .lastTriggered().on(version)
-                   .lastTriggered().asList().stream()
-                   .max(Comparator.<Run, Instant>comparing(run -> run.start())
-                                .thenComparing(run -> run.id().type()));
-    }
-
 }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilter.java
index 1aaecb58a8d..4d3cfacab14 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilter.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilter.java
@@ -3,7 +3,6 @@ package com.yahoo.vespa.hosted.controller.restapi.filter;
 
 import com.google.inject.Inject;
 import com.yahoo.config.provision.ApplicationName;
-import com.yahoo.config.provision.InstanceName;
 import com.yahoo.config.provision.TenantName;
 import com.yahoo.jdisc.http.filter.DiscFilterRequest;
 import com.yahoo.jdisc.http.filter.security.base.JsonSecurityRequestFilterBase;
@@ -12,7 +11,6 @@ import com.yahoo.restapi.Path;
 import com.yahoo.vespa.athenz.api.AthenzDomain;
 import com.yahoo.vespa.athenz.api.AthenzIdentity;
 import com.yahoo.vespa.athenz.api.AthenzPrincipal;
-import com.yahoo.vespa.athenz.api.AthenzUser;
 import com.yahoo.vespa.athenz.client.zms.ZmsClientException;
 import com.yahoo.vespa.hosted.controller.Controller;
 import com.yahoo.vespa.hosted.controller.TenantController;
@@ -28,9 +26,15 @@ import com.yahoo.yolean.Exceptions;
 
 import java.net.URI;
 import java.security.Principal;
-import java.util.HashSet;
+import java.util.ArrayList;
+import java.util.List;
 import java.util.Optional;
 import java.util.Set;
+import java.util.concurrent.CopyOnWriteArraySet;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
 import java.util.logging.Logger;
 
 import static com.yahoo.vespa.hosted.controller.athenz.HostedAthenzIdentities.SCREWDRIVER_DOMAIN;
@@ -46,11 +50,13 @@ public class AthenzRoleFilter extends JsonSecurityRequestFilterBase {
 
     private final AthenzFacade athenz;
     private final TenantController tenants;
+    private final ExecutorService executor;
 
     @Inject
     public AthenzRoleFilter(AthenzClientFactory athenzClientFactory, Controller controller) {
         this.athenz = new AthenzFacade(athenzClientFactory);
         this.tenants = controller.tenants();
+        this.executor = Executors.newCachedThreadPool();
     }
 
     @Override
@@ -69,7 +75,7 @@ public class AthenzRoleFilter extends JsonSecurityRequestFilterBase {
         return Optional.empty();
     }
 
-    Set<Role> roles(AthenzPrincipal principal, URI uri) {
+    Set<Role> roles(AthenzPrincipal principal, URI uri) throws Exception {
         Path path = new Path(uri);
 
         path.matches("/application/v4/tenant/{tenant}/{*}");
@@ -78,38 +84,71 @@ public class AthenzRoleFilter extends JsonSecurityRequestFilterBase {
         path.matches("/application/v4/tenant/{tenant}/application/{application}/{*}");
         Optional<ApplicationName> application = Optional.ofNullable(path.get("application")).map(ApplicationName::from);
 
-        path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/{*}");
-        Optional<InstanceName> instance = Optional.ofNullable(path.get("instance")).map(InstanceName::from);
-
         AthenzIdentity identity = principal.getIdentity();
 
-        Set<Role> roleMemberships = new HashSet<>();
-        if (athenz.hasHostedOperatorAccess(identity))
-            roleMemberships.add(Role.hostedOperator());
+        Set<Role> roleMemberships = new CopyOnWriteArraySet<>();
+        List<Future<?>> futures = new ArrayList<>();
 
-        // Add all tenants that are accessible for this request
-        athenz.accessibleTenants(tenants.asList(), new Credentials(principal))
-                .forEach(accessibleTenant -> roleMemberships.add(Role.athenzTenantAdmin(accessibleTenant.name())));
+        futures.add(executor.submit(() -> {
+            if (athenz.hasHostedOperatorAccess(identity))
+                roleMemberships.add(Role.hostedOperator());
+        }));
 
-        if (identity.getDomain().equals(SCREWDRIVER_DOMAIN) && application.isPresent() && tenant.isPresent())
-            // NOTE: Only fine-grained deploy authorization for Athenz tenants
-            if (   tenant.get().type() != Tenant.Type.athenz
-                || hasDeployerAccess(identity, ((AthenzTenant) tenant.get()).domain(), application.get()))
-                    roleMemberships.add(Role.tenantPipeline(tenant.get().name(), application.get()));
+        futures.add(executor.submit(() -> {
+            if (athenz.hasHostedSupporterAccess(identity))
+                roleMemberships.add(Role.hostedSupporter());
+        }));
 
-        if (athenz.hasSystemFlagsAccess(identity, /*dryrun*/false)) {
-            roleMemberships.add(Role.systemFlagsDeployer());
-        }
+        futures.add(executor.submit(() -> {
+            // Add all tenants that are accessible for this request
+            athenz.accessibleTenants(tenants.asList(), new Credentials(principal))
+                  .forEach(accessibleTenant -> roleMemberships.add(Role.athenzTenantAdmin(accessibleTenant.name())));
+        }));
 
-        if (athenz.hasSystemFlagsAccess(identity, /*dryrun*/true)) {
+        if (identity.getDomain().equals(SCREWDRIVER_DOMAIN) && application.isPresent() && tenant.isPresent())
+            futures.add(executor.submit(() -> {
+                if (   tenant.get().type() == Tenant.Type.athenz
+                    && hasDeployerAccess(identity, ((AthenzTenant) tenant.get()).domain(), application.get()))
+                    roleMemberships.add(Role.buildService(tenant.get().name(), application.get()));
+            }));
+
+        futures.add(executor.submit(() -> {
+            if (athenz.hasSystemFlagsAccess(identity, /*dryrun*/false))
+                roleMemberships.add(Role.systemFlagsDeployer());
+        }));
+
+        futures.add(executor.submit(() -> {
+            if (athenz.hasPaymentCallbackAccess(identity))
+                roleMemberships.add(Role.paymentProcessor());
+        }));
+
+        // Run last request in handler thread to avoid creating extra thread.
+        if (athenz.hasSystemFlagsAccess(identity, /*dryrun*/true))
             roleMemberships.add(Role.systemFlagsDryrunner());
-        }
+
+        for (Future<?> future : futures)
+            future.get(30, TimeUnit.SECONDS);
 
         return roleMemberships.isEmpty()
                 ? Set.of(Role.everyone())
                 : Set.copyOf(roleMemberships);
     }
 
+    @Override
+    public void deconstruct() {
+        try {
+            executor.shutdown();
+            if ( ! executor.awaitTermination(30, TimeUnit.SECONDS)) {
+                executor.shutdownNow();
+                if ( ! executor.awaitTermination(10, TimeUnit.SECONDS))
+                    throw new IllegalStateException("Failed to shut down executor 40 seconds");
+            }
+        }
+        catch (InterruptedException e) {
+            throw new IllegalStateException("Interrupted while shutting down executor", e);
+        }
+    }
+
     private boolean hasDeployerAccess(AthenzIdentity identity, AthenzDomain tenantDomain, ApplicationName application) {
         try {
             return athenz.hasApplicationAccess(identity,
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilter.java
index a41103453eb..4916b7b24a1 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilter.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilter.java
@@ -29,8 +29,9 @@ import java.util.logging.Logger;
 import static java.nio.charset.StandardCharsets.UTF_8;
 
 /**
- * Assigns the {@link Role#buildService(TenantName, ApplicationName)} role to requests with a
- * Authorization header signature matching the public key of the indicated application.
+ * Assigns the {@link Role#headless(TenantName, ApplicationName)} role or
+ * {@link Role#developer(TenantName)} to requests with a X-Authorization header signature
+ * matching the public key of the indicated application.
  * Requests which already have a set of roles assigned to them are not modified.
  *
  * @author jonmv
@@ -63,14 +64,6 @@ public class SignatureFilter extends JsonSecurityRequestFilterBase {
         return Optional.empty();
     }
 
-    // TODO jonmv: Remove after October 2019.
-    private boolean anyDeployKeyMatches(TenantAndApplicationId id, DiscFilterRequest request) {
-        return controller.applications().getApplication(id).stream()
-                         .map(Application::deployKeys)
-                         .flatMap(Set::stream)
-                         .anyMatch(key -> keyVerifies(key, request));
-    }
-
     private boolean keyVerifies(PublicKey key, DiscFilterRequest request) {
         return new RequestVerifier(key, controller.clock()).verify(Method.valueOf(request.getMethod()),
                                                                    request.getUri(),
@@ -80,30 +73,23 @@ public class SignatureFilter extends JsonSecurityRequestFilterBase {
     }
 
     private Optional<SecurityContext> getSecurityContext(DiscFilterRequest request) {
-        ApplicationId id = ApplicationId.fromSerializedForm(request.getHeader("X-Key-Id"));
-        if (request.getHeader("X-Key") != null) { // TODO jonmv: Remove check and else branch after Oct 2019.
-            PublicKey key = KeyUtils.fromPemEncodedPublicKey(new String(Base64.getDecoder().decode(request.getHeader("X-Key")), UTF_8));
-            if (keyVerifies(key, request)) {
-                Optional<CloudTenant> tenant = controller.tenants().get(id.tenant())
-                                                         .filter(CloudTenant.class::isInstance)
-                                                         .map(CloudTenant.class::cast);
-                if (tenant.isPresent() && tenant.get().developerKeys().containsKey(key))
-                    return Optional.of(new SecurityContext(tenant.get().developerKeys().get(key),
-                                                           Set.of(Role.reader(id.tenant()),
-                                                                  Role.developer(id.tenant()))));
+        PublicKey key = KeyUtils.fromPemEncodedPublicKey(new String(Base64.getDecoder().decode(request.getHeader("X-Key")), UTF_8));
+        if (keyVerifies(key, request)) {
+            ApplicationId id = ApplicationId.fromSerializedForm(request.getHeader("X-Key-Id"));
+            Optional<CloudTenant> tenant = controller.tenants().get(id.tenant())
+                                                     .filter(CloudTenant.class::isInstance)
+                                                     .map(CloudTenant.class::cast);
+            if (tenant.isPresent() && tenant.get().developerKeys().containsKey(key))
+                return Optional.of(new SecurityContext(tenant.get().developerKeys().get(key),
+                                                       Set.of(Role.reader(id.tenant()),
+                                                              Role.developer(id.tenant()))));
 
-                Optional <Application> application = controller.applications().getApplication(TenantAndApplicationId.from(id));
-                if (application.isPresent() && application.get().deployKeys().contains(key))
-                    return Optional.of(new SecurityContext(new SimplePrincipal("headless@" + id.tenant() + "." + id.application()),
-                                                           Set.of(Role.reader(id.tenant()),
-                                                                  Role.developer(id.tenant())))); // TODO jonmv: Change to headless after Oct 10 2019.
-            }
+            Optional <Application> application = controller.applications().getApplication(TenantAndApplicationId.from(id));
+            if (application.isPresent() && application.get().deployKeys().contains(key))
+                return Optional.of(new SecurityContext(new SimplePrincipal("headless@" + id.tenant() + "." + id.application()),
+                                                       Set.of(Role.reader(id.tenant()),
+                                                              Role.headless(id.tenant(), id.application()))));
         }
-        else if (anyDeployKeyMatches(TenantAndApplicationId.from(id), request))
-            return Optional.of(new SecurityContext(new SimplePrincipal("headless@" + id.tenant() + "." + id.application()),
-                                                   Set.of(Role.reader(id.tenant()),
-                                                          Role.developer(id.tenant()))));
-
         return Optional.empty();
     }
 
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiHandler.java
index c168a057bfb..7d8c5922c3f 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiHandler.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiHandler.java
@@ -13,7 +13,7 @@ import com.yahoo.restapi.Path;
 import com.yahoo.slime.Cursor;
 import com.yahoo.slime.Inspector;
 import com.yahoo.slime.Slime;
-import com.yahoo.vespa.config.SlimeUtils;
+import com.yahoo.slime.SlimeUtils;
 import com.yahoo.vespa.hosted.controller.Controller;
 import com.yahoo.config.provision.zone.ZoneId;
 import com.yahoo.config.provision.zone.ZoneList;
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java
index cad206b0a69..ba40f9c2085 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java
@@ -2,27 +2,37 @@
 package com.yahoo.vespa.hosted.controller.restapi.routing;
 
 import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.ApplicationName;
+import com.yahoo.config.provision.InstanceName;
+import com.yahoo.config.provision.TenantName;
+import com.yahoo.config.provision.zone.RoutingMethod;
 import com.yahoo.config.provision.zone.ZoneId;
 import com.yahoo.container.jdisc.HttpRequest;
 import com.yahoo.container.jdisc.HttpResponse;
 import com.yahoo.restapi.ErrorResponse;
 import com.yahoo.restapi.MessageResponse;
 import com.yahoo.restapi.Path;
+import com.yahoo.restapi.ResourceResponse;
 import com.yahoo.restapi.SlimeJsonResponse;
 import com.yahoo.slime.Cursor;
 import com.yahoo.slime.Slime;
+import com.yahoo.vespa.hosted.controller.Application;
 import com.yahoo.vespa.hosted.controller.Controller;
 import com.yahoo.vespa.hosted.controller.Instance;
 import com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus;
 import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
+import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId;
 import com.yahoo.vespa.hosted.controller.auditlog.AuditLoggingRequestHandler;
 import com.yahoo.vespa.hosted.controller.routing.GlobalRouting;
-import com.yahoo.vespa.hosted.controller.routing.RoutingPolicy;
 import com.yahoo.yolean.Exceptions;
 
+import java.net.URI;
 import java.time.Instant;
+import java.util.Comparator;
+import java.util.List;
 import java.util.Objects;
 import java.util.logging.Level;
+import java.util.stream.Collectors;
 
 /**
  * This implements the /routing/v1 API, which provides operator with global routing control at both zone- and
@@ -44,7 +54,7 @@ public class RoutingApiHandler extends AuditLoggingRequestHandler {
         try {
             var path = new Path(request.getUri());
             switch (request.getMethod()) {
-                case GET: return get(path);
+                case GET: return get(path, request);
                 case POST: return post(path);
                 case DELETE: return delete(path);
                 default: return ErrorResponse.methodNotAllowed("Method '" + request.getMethod() + "' is not supported");
@@ -69,39 +79,123 @@ public class RoutingApiHandler extends AuditLoggingRequestHandler {
         return ErrorResponse.notFoundError("Nothing at " + path);
     }
 
-    private HttpResponse get(Path path) {
-        if (path.matches("/routing/v1/status/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}")) return deploymentStatus(path);
-        if (path.matches("/routing/v1/status/environment/{environment}/region/{region}")) return zoneStatus(path);
+    private HttpResponse get(Path path, HttpRequest request) {
+        if (path.matches("/routing/v1/")) return status(request.getUri());
+        if (path.matches("/routing/v1/status/tenant/{tenant}")) return tenant(path, request);
+        if (path.matches("/routing/v1/status/tenant/{tenant}/application/{application}")) return application(path, request);
+        if (path.matches("/routing/v1/status/tenant/{tenant}/application/{application}/instance/{instance}")) return instance(path, request);
+        if (path.matches("/routing/v1/status/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}")) return deployment(path);
+        if (path.matches("/routing/v1/status/environment")) return environment(request);
+        if (path.matches("/routing/v1/status/environment/{environment}/region/{region}")) return zone(path);
         return ErrorResponse.notFoundError("Nothing at " + path);
     }
 
+    private HttpResponse environment(HttpRequest request) {
+        var zones = controller.zoneRegistry().zones().all().ids();
+        if (isRecursive(request)) {
+            var slime = new Slime();
+            var root = slime.setObject();
+            var zonesArray = root.setArray("zones");
+            for (var zone : zones) {
+                toSlime(zone, zonesArray.addObject());
+            }
+            return new SlimeJsonResponse(slime);
+        }
+        var resources = controller.zoneRegistry().zones().all().ids().stream()
+                                  .map(zone -> zone.environment().value() +
+                                               "/region/" + zone.region().value())
+                                  .sorted()
+                                  .collect(Collectors.toList());
+        return new ResourceResponse(request.getUri(), resources);
+    }
+
+    private HttpResponse status(URI requestUrl) {
+        return new ResourceResponse(requestUrl, "status/tenant", "status/environment");
+    }
+
+    private HttpResponse tenant(Path path, HttpRequest request) {
+        var tenantName = tenantFrom(path);
+        if (isRecursive(request)) {
+            var slime = new Slime();
+            var root = slime.setObject();
+            toSlime(controller.applications().asList(tenantName), null, null, root);
+            return new SlimeJsonResponse(slime);
+        }
+        var resources = controller.applications().asList(tenantName).stream()
+                                  .map(Application::id)
+                                  .map(TenantAndApplicationId::application)
+                                  .map(ApplicationName::value)
+                                  .map(application -> "application/" + application)
+                                  .sorted()
+                                  .collect(Collectors.toList());
+        return new ResourceResponse(request.getUri(), resources);
+    }
+
+    private HttpResponse application(Path path, HttpRequest request) {
+        var tenantAndApplicationId = tenantAndApplicationIdFrom(path);
+        if (isRecursive(request)) {
+            var slime = new Slime();
+            var root = slime.setObject();
+            toSlime(List.of(controller.applications().requireApplication(tenantAndApplicationId)), null,
+                    null, root);
+            return new SlimeJsonResponse(slime);
+        }
+        var resources = controller.applications().requireApplication(tenantAndApplicationId).instances().keySet().stream()
+                                  .map(InstanceName::value)
+                                  .map(instance -> "instance/" + instance)
+                                  .sorted()
+                                  .collect(Collectors.toList());
+        return new ResourceResponse(request.getUri(), resources);
+    }
+
+    private HttpResponse instance(Path path, HttpRequest request) {
+        var instanceId = instanceFrom(path);
+        if (isRecursive(request)) {
+            var slime = new Slime();
+            var root = slime.setObject();
+            toSlime(List.of(controller.applications().requireApplication(TenantAndApplicationId.from(instanceId))),
+                    instanceId, null, root);
+            return new SlimeJsonResponse(slime);
+        }
+        var resources = controller.applications().requireInstance(instanceId).deployments().keySet().stream()
+                                  .map(zone -> "environment/" + zone.environment().value() +
+                                               "/region/" + zone.region().value())
+                                  .sorted()
+                                  .collect(Collectors.toList());
+        return new ResourceResponse(request.getUri(), resources);
+    }
+
     private HttpResponse setZoneStatus(Path path, boolean in) {
         var zone = zoneFrom(path);
         if (controller.zoneRegistry().zones().directlyRouted().ids().contains(zone)) {
             var status = in ? GlobalRouting.Status.in : GlobalRouting.Status.out;
-            controller.applications().routingPolicies().setGlobalRoutingStatus(zone, status);
+            controller.routing().policies().setGlobalRoutingStatus(zone, status);
         } else {
             controller.serviceRegistry().configServer().setGlobalRotationStatus(zone, in);
         }
-        return new MessageResponse("Set global routing status for deployments in " + zone + " to '" +
-                                   (in ? "in" : "out") + "'");
+        return new MessageResponse("Set global routing status for deployments in " + zone + " to " +
+                                   (in ? "IN" : "OUT"));
     }
 
-    private HttpResponse zoneStatus(Path path) {
+    private HttpResponse zone(Path path) {
         var zone = zoneFrom(path);
         var slime = new Slime();
         var root = slime.setObject();
+        toSlime(zone, root);
+        return new SlimeJsonResponse(slime);
+    }
+
+    private void toSlime(ZoneId zone, Cursor zoneObject) {
         if (controller.zoneRegistry().zones().directlyRouted().ids().contains(zone)) {
-            var zonePolicy = controller.applications().routingPolicies().get(zone);
-            zoneStatusToSlime(root, zonePolicy.zone(), zonePolicy.globalRouting(), RoutingType.policy);
+            var zonePolicy = controller.routing().policies().get(zone);
+            zoneStatusToSlime(zoneObject, zonePolicy.zone(), zonePolicy.globalRouting(), RoutingMethod.exclusive);
         } else {
             // Rotation status per zone only exposes in/out status, no agent or time of change.
             var in = controller.serviceRegistry().configServer().getGlobalRotationStatus(zone);
             var globalRouting = new GlobalRouting(in ? GlobalRouting.Status.in : GlobalRouting.Status.out,
                                                   GlobalRouting.Agent.operator, Instant.EPOCH);
-            zoneStatusToSlime(root, zone, globalRouting, RoutingType.rotation);
+            zoneStatusToSlime(zoneObject, zone, globalRouting, RoutingMethod.shared);
         }
-        return new SlimeJsonResponse(slime);
     }
 
     private HttpResponse setDeploymentStatus(Path path, boolean in) {
@@ -115,58 +209,81 @@ public class RoutingApiHandler extends AuditLoggingRequestHandler {
             var endpointStatus = new EndpointStatus(in ? EndpointStatus.Status.in : EndpointStatus.Status.out, "",
                                                     agent.name(),
                                                     controller.clock().instant().getEpochSecond());
-            controller.applications().setGlobalRotationStatus(deployment, endpointStatus);
+            controller.routing().setGlobalRotationStatus(deployment, endpointStatus);
         }
 
         // Set policy status
-        controller.applications().routingPolicies().setGlobalRoutingStatus(deployment, status, agent);
-        return new MessageResponse("Set global routing status for " + deployment + " to '" + (in ? "in" : "out") + "'");
+        controller.routing().policies().setGlobalRoutingStatus(deployment, status, agent);
+        return new MessageResponse("Set global routing status for " + deployment + " to " + (in ? "IN" : "OUT"));
     }
 
-    private HttpResponse deploymentStatus(Path path) {
-        var deployment = deploymentFrom(path);
-        var instance = controller.applications().requireInstance(deployment.applicationId());
+    private HttpResponse deployment(Path path) {
         var slime = new Slime();
-        var deploymentsObject = slime.setObject().setArray("deployments");
+        var root = slime.setObject();
+        var deploymentId = deploymentFrom(path);
+        var application = controller.applications().requireApplication(TenantAndApplicationId.from(deploymentId.applicationId()));
+        toSlime(List.of(application), deploymentId.applicationId(), deploymentId.zoneId(), root);
+        return new SlimeJsonResponse(slime);
+    }
 
-        // Include status from rotation
-        if (rotationCanRouteTo(deployment.zoneId(), instance)) {
-            var rotationStatus = controller.applications().globalRotationStatus(deployment);
-            // Status is equal across all global endpoints, as the status is per deployment, not per endpoint.
-            var endpointStatus = rotationStatus.values().stream().findFirst();
-            if (endpointStatus.isPresent()) {
-                var changedAt = Instant.ofEpochSecond(endpointStatus.get().getEpoch());
-                GlobalRouting.Agent agent;
-                try {
-                    agent = GlobalRouting.Agent.valueOf(endpointStatus.get().getAgent());
-                } catch (IllegalArgumentException e) {
-                    agent = GlobalRouting.Agent.unknown;
+    private void toSlime(List<Application> applications, ApplicationId instanceId, ZoneId zoneId, Cursor root) {
+        var deploymentsArray = root.setArray("deployments");
+        for (var application : applications) {
+            var instances = instanceId == null
+                    ? application.instances().values()
+                    : List.of(application.instances().get(instanceId.instance()));
+            for (var instance : instances) {
+                var zones = zoneId == null
+                        ? instance.deployments().keySet().stream().sorted(Comparator.comparing(ZoneId::value))
+                                  .collect(Collectors.toList())
+                        : List.of(zoneId);
+                for (var zone : zones) {
+                    var deploymentId = new DeploymentId(instance.id(), zone);
+                    // Include status from rotation
+                    if (rotationCanRouteTo(zone, instance)) {
+                        var rotationStatus = controller.routing().globalRotationStatus(deploymentId);
+                        // Status is equal across all global endpoints, as the status is per deployment, not per endpoint.
+                        var endpointStatus = rotationStatus.values().stream().findFirst();
+                        if (endpointStatus.isPresent()) {
+                            var changedAt = Instant.ofEpochSecond(endpointStatus.get().getEpoch());
+                            GlobalRouting.Agent agent;
+                            try {
+                                agent = GlobalRouting.Agent.valueOf(endpointStatus.get().getAgent());
+                            } catch (IllegalArgumentException e) {
+                                agent = GlobalRouting.Agent.unknown;
+                            }
+                            var status = endpointStatus.get().getStatus() == EndpointStatus.Status.in
+                                    ? GlobalRouting.Status.in
+                                    : GlobalRouting.Status.out;
+                            deploymentStatusToSlime(deploymentsArray.addObject(), deploymentId,
+                                                    new GlobalRouting(status, agent, changedAt),
+                                                    RoutingMethod.shared);
+                        }
+                    }
+
+                    // Include status from routing policies
+                    var routingPolicies = controller.routing().policies().get(deploymentId);
+                    for (var policy : routingPolicies.values()) {
+                        if (!controller.zoneRegistry().routingMethods(policy.id().zone()).contains(RoutingMethod.exclusive)) continue;
+                        deploymentStatusToSlime(deploymentsArray.addObject(), new DeploymentId(policy.id().owner(),
+                                                                                               policy.id().zone()),
+                                                policy.status().globalRouting(), RoutingMethod.exclusive);
+                    }
                 }
-                var status = endpointStatus.get().getStatus() == EndpointStatus.Status.in
-                        ? GlobalRouting.Status.in
-                        : GlobalRouting.Status.out;
-                deploymentStatusToSlime(deploymentsObject.addObject(), deployment,
-                                        new GlobalRouting(status, agent, changedAt),
-                                        RoutingType.rotation);
             }
         }
 
-        // Include status from routing policies
-        var routingPolicies = controller.applications().routingPolicies().get(deployment);
-        for (var policy : routingPolicies.values()) {
-            deploymentStatusToSlime(deploymentsObject.addObject(), policy);
-        }
-
-        return new SlimeJsonResponse(slime);
     }
 
-    /** Returns whether instance has an assigned rotation and a deployment in given zone */
-    private static boolean rotationCanRouteTo(ZoneId zone, Instance instance) {
-        return !instance.rotations().isEmpty() && instance.deployments().containsKey(zone);
+    /** Returns whether instance has an assigned rotation that can route to given zone */
+    private boolean rotationCanRouteTo(ZoneId zone, Instance instance) {
+        return !instance.rotations().isEmpty() &&
+               instance.deployments().containsKey(zone) &&
+               controller.zoneRegistry().routingMethods(zone).get(0).isShared();
     }
 
-    private static void zoneStatusToSlime(Cursor object, ZoneId zone, GlobalRouting globalRouting, RoutingType routingType) {
-        object.setString("routingType", routingType.name());
+    private static void zoneStatusToSlime(Cursor object, ZoneId zone, GlobalRouting globalRouting, RoutingMethod method) {
+        object.setString("routingMethod", asString(method));
         object.setString("environment", zone.environment().value());
         object.setString("region", zone.region().value());
         object.setString("status", asString(globalRouting.status()));
@@ -174,8 +291,8 @@ public class RoutingApiHandler extends AuditLoggingRequestHandler {
         object.setLong("changedAt", globalRouting.changedAt().toEpochMilli());
     }
 
-    private static void deploymentStatusToSlime(Cursor object, DeploymentId deployment, GlobalRouting globalRouting, RoutingType routingType) {
-        object.setString("routingType", routingType.name());
+    private static void deploymentStatusToSlime(Cursor object, DeploymentId deployment, GlobalRouting globalRouting, RoutingMethod method) {
+        object.setString("routingMethod", asString(method));
         object.setString("instance", deployment.applicationId().serializedForm());
         object.setString("environment", deployment.zoneId().environment().value());
         object.setString("region", deployment.zoneId().region().value());
@@ -184,14 +301,24 @@ public class RoutingApiHandler extends AuditLoggingRequestHandler {
         object.setLong("changedAt", globalRouting.changedAt().toEpochMilli());
     }
 
-    private static void deploymentStatusToSlime(Cursor object, RoutingPolicy policy) {
-        deploymentStatusToSlime(object, new DeploymentId(policy.id().owner(), policy.id().zone()),
-                                policy.status().globalRouting(), RoutingType.policy);
+    private TenantName tenantFrom(Path path) {
+        return TenantName.from(path.get("tenant"));
+    }
+
+    private ApplicationName applicationFrom(Path path) {
+        return ApplicationName.from(path.get("application"));
+    }
+
+    private TenantAndApplicationId tenantAndApplicationIdFrom(Path path) {
+       return TenantAndApplicationId.from(tenantFrom(path), applicationFrom(path));
+    }
+
+    private ApplicationId instanceFrom(Path path) {
+        return ApplicationId.from(tenantFrom(path), applicationFrom(path), InstanceName.from(path.get("instance")));
     }
 
     private DeploymentId deploymentFrom(Path path) {
-        return new DeploymentId(ApplicationId.from(path.get("tenant"), path.get("application"), path.get("instance")),
-                                zoneFrom(path));
+        return new DeploymentId(instanceFrom(path), zoneFrom(path));
     }
 
     private ZoneId zoneFrom(Path path) {
@@ -202,6 +329,10 @@ public class RoutingApiHandler extends AuditLoggingRequestHandler {
         return zone;
     }
 
+    private static boolean isRecursive(HttpRequest request) {
+        return "true".equals(request.getProperty("recursive"));
+    }
+
     private static String asString(GlobalRouting.Status status) {
         switch (status) {
             case in: return "in";
@@ -219,12 +350,13 @@ public class RoutingApiHandler extends AuditLoggingRequestHandler {
         }
     }
 
-    private enum RoutingType {
-        /** Global routing is configured by use of an {@link com.yahoo.vespa.hosted.controller.application.AssignedRotation} */
-        rotation,
-
-        /** Global routing is configured by a {@link com.yahoo.vespa.hosted.controller.routing.RoutingPolicy} */
-        policy,
+    private static String asString(RoutingMethod method) {
+        switch (method) {
+            case shared: return "shared";
+            case exclusive: return "exclusive";
+            case sharedLayer4: return "sharedLayer4";
+            default: return "unknown";
+        }
     }
 
 }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/FlagsClient.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/FlagsClient.java
index 5cd6b32d572..161d3734aae 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/FlagsClient.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/FlagsClient.java
@@ -1,6 +1,8 @@
 // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.restapi.systemflags;
 
+import ai.vespa.util.http.retry.DelayedConnectionLevelRetryHandler;
+import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.yahoo.vespa.athenz.api.AthenzIdentity;
 import com.yahoo.vespa.athenz.identity.ServiceIdentityProvider;
@@ -23,7 +25,6 @@ import org.apache.http.client.utils.URIBuilder;
 import org.apache.http.entity.ContentType;
 import org.apache.http.entity.StringEntity;
 import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.impl.client.DefaultHttpRequestRetryHandler;
 import org.apache.http.impl.client.HttpClientBuilder;
 import org.apache.http.message.BasicNameValuePair;
 import org.apache.http.util.EntityUtils;
@@ -35,6 +36,7 @@ import java.io.UncheckedIOException;
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.time.Duration;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Set;
 
@@ -65,6 +67,17 @@ class FlagsClient {
         });
     }
 
+    List<FlagId> listDefinedFlags(FlagsTarget target) {
+        HttpGet request = new HttpGet(createUri(target, "/defined", List.of()));
+        return executeRequest(request, response -> {
+            verifySuccess(response, null);
+            JsonNode json = mapper.readTree(response.getEntity().getContent());
+            List<FlagId> flagIds = new ArrayList<>();
+            json.fieldNames().forEachRemaining(fieldName -> flagIds.add(new FlagId(fieldName)));
+            return flagIds;
+        });
+    }
+
     void putFlagData(FlagsTarget target, FlagData flagData) throws FlagsException, UncheckedIOException {
         HttpPut request = new HttpPut(createUri(target, "/data/" + flagData.id().toString(), List.of()));
         request.setEntity(jsonContent(flagData.serializeToJson()));
@@ -82,11 +95,12 @@ class FlagsClient {
         });
     }
 
-
     private static CloseableHttpClient createClient(ServiceIdentityProvider identityProvider, Set<FlagsTarget> targets) {
+        DelayedConnectionLevelRetryHandler retryHandler = DelayedConnectionLevelRetryHandler.Builder
+                .withExponentialBackoff(Duration.ofSeconds(1), Duration.ofSeconds(20), 5)
+                .build();
         return HttpClientBuilder.create()
                 .setUserAgent("controller-flags-v1-client")
-                .setRetryHandler(new DefaultHttpRequestRetryHandler(5, /*retry on non-idempotent requests*/true))
                 .setSSLContext(identityProvider.getIdentitySslContext())
                 .setSSLHostnameVerifier(new FlagTargetsHostnameVerifier(targets))
                 .setDefaultRequestConfig(RequestConfig.custom()
@@ -96,6 +110,7 @@ class FlagsClient {
                                                  .build())
                 .setMaxConnPerRoute(2)
                 .setMaxConnTotal(100)
+                .setRetryHandler(retryHandler)
                 .build();
     }
 
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployResult.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployResult.java
index bc00090dc88..57d47757c5e 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployResult.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployResult.java
@@ -8,6 +8,7 @@ import com.yahoo.vespa.hosted.controller.api.systemflags.v1.FlagsTarget;
 import com.yahoo.vespa.hosted.controller.api.systemflags.v1.wire.WireSystemFlagsDeployResult;
 import com.yahoo.vespa.hosted.controller.api.systemflags.v1.wire.WireSystemFlagsDeployResult.WireFlagDataChange;
 import com.yahoo.vespa.hosted.controller.api.systemflags.v1.wire.WireSystemFlagsDeployResult.WireOperationFailure;
+import com.yahoo.vespa.hosted.controller.api.systemflags.v1.wire.WireSystemFlagsDeployResult.WireWarning;
 
 import java.util.ArrayList;
 import java.util.HashMap;
@@ -17,6 +18,8 @@ import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
+import java.util.function.BiFunction;
+import java.util.function.Function;
 
 import static java.util.stream.Collectors.toList;
 
@@ -27,14 +30,16 @@ class SystemFlagsDeployResult {
 
     private final List<FlagDataChange> flagChanges;
     private final List<OperationError> errors;
+    private final List<Warning> warnings;
 
-    SystemFlagsDeployResult(List<FlagDataChange> flagChanges, List<OperationError> errors) {
+    SystemFlagsDeployResult(List<FlagDataChange> flagChanges, List<OperationError> errors, List<Warning> warnings) {
         this.flagChanges = flagChanges;
         this.errors = errors;
+        this.warnings = warnings;
     }
 
     SystemFlagsDeployResult(List<OperationError> errors) {
-        this(List.of(), errors);
+        this(List.of(), errors, List.of());
     }
 
     List<FlagDataChange> flagChanges() {
@@ -45,43 +50,50 @@ class SystemFlagsDeployResult {
         return errors;
     }
 
+    List<Warning> warnings() { return warnings; }
+
     static SystemFlagsDeployResult merge(List<SystemFlagsDeployResult> results) {
         List<FlagDataChange> mergedChanges = mergeChanges(results);
         List<OperationError> mergedErrors = mergeErrors(results);
-        return new SystemFlagsDeployResult(mergedChanges, mergedErrors);
+        List<Warning> mergedWarnings = mergeWarnings(results);
+        return new SystemFlagsDeployResult(mergedChanges, mergedErrors, mergedWarnings);
     }
 
     private static List<OperationError> mergeErrors(List<SystemFlagsDeployResult> results) {
-        Map<OperationErrorWithoutTarget, Set<FlagsTarget>> targetsForError = new HashMap<>();
-        for (SystemFlagsDeployResult result : results) {
-            for (OperationError error : result.errors()) {
-                var errorWithoutTarget = new OperationErrorWithoutTarget(error);
-                targetsForError.computeIfAbsent(errorWithoutTarget, k -> new HashSet<>())
-                        .addAll(error.targets());
-            }
-        }
-        List<OperationError> mergedErrors = new ArrayList<>();
-        targetsForError.forEach(
-                (error, targets) -> mergedErrors.add(error.toOperationError(targets)));
-        return mergedErrors;
+        return merge(results, SystemFlagsDeployResult::errors, OperationError::targets,
+                OperationErrorWithoutTarget::new, OperationErrorWithoutTarget::toOperationError);
     }
 
     private static List<FlagDataChange> mergeChanges(List<SystemFlagsDeployResult> results) {
-        Map<FlagDataChangeWithoutTarget, Set<FlagsTarget>> targetsForChange = new HashMap<>();
+        return merge(results, SystemFlagsDeployResult::flagChanges, FlagDataChange::targets,
+                FlagDataChangeWithoutTarget::new, FlagDataChangeWithoutTarget::toFlagDataChange);
+    }
+
+    private static List<Warning> mergeWarnings(List<SystemFlagsDeployResult> results) {
+        return merge(results, SystemFlagsDeployResult::warnings, Warning::targets,
+                WarningWithoutTarget::new, WarningWithoutTarget::toWarning);
+    }
+
+    private static <VALUE, VALUE_WITHOUT_TARGET> List<VALUE> merge(
+            List<SystemFlagsDeployResult> results,
+            Function<SystemFlagsDeployResult, List<VALUE>> valuesGetter,
+            Function<VALUE, Set<FlagsTarget>> targetsGetter,
+            Function<VALUE, VALUE_WITHOUT_TARGET> transformer,
+            BiFunction<VALUE_WITHOUT_TARGET, Set<FlagsTarget>, VALUE> reverseTransformer) {
+        Map<VALUE_WITHOUT_TARGET, Set<FlagsTarget>> targetsForValue = new HashMap<>();
         for (SystemFlagsDeployResult result : results) {
-            for (FlagDataChange change : result.flagChanges()) {
-                var changeWithoutTarget = new FlagDataChangeWithoutTarget(change);
-                targetsForChange.computeIfAbsent(changeWithoutTarget, k -> new HashSet<>())
-                        .addAll(change.targets());
+            for (VALUE value : valuesGetter.apply(result)) {
+                VALUE_WITHOUT_TARGET valueWithoutTarget = transformer.apply(value);
+                targetsForValue.computeIfAbsent(valueWithoutTarget, k -> new HashSet<>())
+                        .addAll(targetsGetter.apply(value));
             }
         }
-        List<FlagDataChange> mergedChanges = new ArrayList<>();
-        targetsForChange.forEach(
-                (change, targets) -> mergedChanges.add(change.toFlagDataChange(targets)));
-        return mergedChanges;
+        List<VALUE> mergedValues = new ArrayList<>();
+        targetsForValue.forEach(
+                (value, targets) -> mergedValues.add(reverseTransformer.apply(value, targets)));
+        return mergedValues;
     }
 
-
     WireSystemFlagsDeployResult toWire() {
         var wireResult = new WireSystemFlagsDeployResult();
         wireResult.changes = new ArrayList<>();
@@ -104,6 +116,14 @@ class SystemFlagsDeployResult {
             wireError.data = error.flagData().map(FlagData::toWire).orElse(null);
             wireResult.errors.add(wireError);
         }
+        wireResult.warnings = new ArrayList<>();
+        for (Warning warning : warnings) {
+            var wireWarning = new WireWarning();
+            wireWarning.message = warning.message();
+            wireWarning.flagId = warning.flagId().toString();
+            wireWarning.targets = warning.targets().stream().map(FlagsTarget::asString).collect(toList());
+            wireResult.warnings.add(wireWarning);
+        }
         return wireResult;
     }
 
@@ -218,6 +238,10 @@ class SystemFlagsDeployResult {
             return new OperationError(message, Set.of(target), OperationType.DELETE, id, null);
         }
 
+        static OperationError archiveValidationFailed(String message) {
+            return new OperationError(message, Set.of(), OperationType.VALIDATE_ARCHIVE, null, null);
+        }
+
         String message() { return message; }
         Set<FlagsTarget> targets() { return targets; }
         OperationType operation() { return operation; }
@@ -251,7 +275,7 @@ class SystemFlagsDeployResult {
     }
 
     enum OperationType {
-        CREATE("create"), DELETE("delete"), UPDATE("update"), LIST("list");
+        CREATE("create"), DELETE("delete"), UPDATE("update"), LIST("list"), VALIDATE_ARCHIVE("validate-archive");
 
         private final String stringValue;
 
@@ -260,6 +284,38 @@ class SystemFlagsDeployResult {
         String asString() { return stringValue; }
     }
 
+    static class Warning {
+        final String message;
+        final Set<FlagsTarget> targets;
+        final FlagId flagId;
+
+        private Warning(String message, Set<FlagsTarget> targets, FlagId flagId) {
+            this.message = message;
+            this.targets = targets;
+            this.flagId = flagId;
+        }
+
+        static Warning dataForUndefinedFlag(FlagsTarget target, FlagId flagId) {
+            return new Warning("Flag data present for undefined flag", Set.of(target), flagId);
+        }
+
+        String message() { return message; }
+        Set<FlagsTarget> targets() { return targets; }
+        FlagId flagId() { return flagId; }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            Warning warning = (Warning) o;
+            return Objects.equals(message, warning.message) &&
+                    Objects.equals(targets, warning.targets) &&
+                    Objects.equals(flagId, warning.flagId);
+        }
+
+        @Override public int hashCode() { return Objects.hash(message, targets, flagId); }
+    }
+
     private static class FlagDataChangeWithoutTarget {
         final FlagId flagId;
         final OperationType operationType;
@@ -329,4 +385,30 @@ class SystemFlagsDeployResult {
 
         @Override public int hashCode() { return Objects.hash(message, operation, flagId, flagData); }
     }
+
+    private static class WarningWithoutTarget {
+        final String message;
+        final FlagId flagId;
+
+        WarningWithoutTarget(Warning warning) {
+            this.message = warning.message();
+            this.flagId = warning.flagId();
+        }
+
+        Warning toWarning(Set<FlagsTarget> targets) { return new Warning(message, targets, flagId); }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            WarningWithoutTarget that = (WarningWithoutTarget) o;
+            return Objects.equals(message, that.message) &&
+                    Objects.equals(flagId, that.flagId);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(message, flagId);
+        }
+    }
 }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployer.java
index b89a9dd7f5a..820384ce810 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployer.java
@@ -2,6 +2,7 @@
 package com.yahoo.vespa.hosted.controller.restapi.systemflags;
 
 import com.yahoo.concurrent.DaemonThreadFactory;
+import com.yahoo.config.provision.SystemName;
 import com.yahoo.log.LogLevel;
 import com.yahoo.vespa.athenz.identity.ServiceIdentityProvider;
 import com.yahoo.vespa.flags.FlagId;
@@ -10,6 +11,7 @@ import com.yahoo.vespa.flags.json.FlagData;
 import com.yahoo.vespa.hosted.controller.api.systemflags.v1.FlagsTarget;
 import com.yahoo.vespa.hosted.controller.api.systemflags.v1.SystemFlagsDataArchive;
 import com.yahoo.vespa.hosted.controller.restapi.systemflags.SystemFlagsDeployResult.OperationError;
+import com.yahoo.vespa.hosted.controller.restapi.systemflags.SystemFlagsDeployResult.Warning;
 
 import java.util.ArrayList;
 import java.util.Collection;
@@ -38,16 +40,18 @@ class SystemFlagsDeployer  {
     private static final Logger log = Logger.getLogger(SystemFlagsDeployer.class.getName());
 
     private final FlagsClient client;
+    private final SystemName system;
     private final Set<FlagsTarget> targets;
     private final ExecutorService executor = Executors.newCachedThreadPool(new DaemonThreadFactory("system-flags-deployer-"));
 
 
-    SystemFlagsDeployer(ServiceIdentityProvider identityProvider, Set<FlagsTarget> targets) {
-        this(new FlagsClient(identityProvider, targets), targets);
+    SystemFlagsDeployer(ServiceIdentityProvider identityProvider, SystemName system, Set<FlagsTarget> targets) {
+        this(new FlagsClient(identityProvider, targets), system, targets);
     }
 
-    SystemFlagsDeployer(FlagsClient client, Set<FlagsTarget> targets) {
+    SystemFlagsDeployer(FlagsClient client, SystemName system, Set<FlagsTarget> targets) {
         this.client = client;
+        this.system = system;
         this.targets = targets;
     }
 
@@ -65,14 +69,21 @@ class SystemFlagsDeployer  {
                 throw new RuntimeException(e);
             }
         });
+        try {
+            archive.validateAllFilesAreForTargets(system, targets);
+        } catch (IllegalArgumentException e) {
+            results.add(new SystemFlagsDeployResult(List.of(OperationError.archiveValidationFailed(e.getMessage()))));
+        }
         return SystemFlagsDeployResult.merge(results);
     }
 
     private SystemFlagsDeployResult deployFlags(FlagsTarget target, Set<FlagData> flagData, boolean dryRun) {
         Map<FlagId, FlagData> wantedFlagData = lookupTable(flagData);
         Map<FlagId, FlagData> currentFlagData;
+        List<FlagId> definedFlags;
         try {
             currentFlagData = lookupTable(client.listFlagData(target));
+            definedFlags = client.listDefinedFlags(target);
         } catch (Exception e) {
             log.log(LogLevel.WARNING, String.format("Failed to list flag data for target '%s': %s", target, e.getMessage()), e);
             return new SystemFlagsDeployResult(List.of(OperationError.listFailed(e.getMessage(), target)));
@@ -80,12 +91,13 @@ class SystemFlagsDeployer  {
 
         List<OperationError> errors = new ArrayList<>();
         List<FlagDataChange> results = new ArrayList<>();
+        List<Warning> warnings = new ArrayList<>();
 
         createNewFlagData(target, dryRun, wantedFlagData, currentFlagData, results, errors);
         updateExistingFlagData(target, dryRun, wantedFlagData, currentFlagData, results, errors);
         removeOldFlagData(target, dryRun, wantedFlagData, currentFlagData, results, errors);
-
-        return new SystemFlagsDeployResult(results, errors);
+        warnOnFlagDataForUndefinedFlags(target, wantedFlagData, currentFlagData, definedFlags, warnings);
+        return new SystemFlagsDeployResult(results, errors, warnings);
     }
 
     private void createNewFlagData(FlagsTarget target,
@@ -163,6 +175,18 @@ class SystemFlagsDeployer  {
         });
     }
 
+    private static void warnOnFlagDataForUndefinedFlags(FlagsTarget target,
+                                                        Map<FlagId, FlagData> wantedFlagData,
+                                                        Map<FlagId, FlagData> currentFlagData,
+                                                        List<FlagId> definedFlags,
+                                                        List<Warning> warnings) {
+        for (FlagId flagId : currentFlagData.keySet()) {
+            if (wantedFlagData.containsKey(flagId) && !definedFlags.contains(flagId)) {
+                warnings.add(Warning.dataForUndefinedFlag(target, flagId));
+            }
+        }
+    }
+
     private static void dryRunFlagDataValidation(FlagData data) {
         Flags.getFlag(data.id())
                 .ifPresent(definition -> data.validate(definition.getUnboundFlag().serializer()));
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsHandler.java
index fd594cd57bc..2f7716aaff0 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsHandler.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsHandler.java
@@ -35,7 +35,7 @@ public class SystemFlagsHandler extends LoggingRequestHandler {
                               Executor executor,
                               AccessLog accessLog) {
         super(executor, accessLog);
-        this.deployer = new SystemFlagsDeployer(identityProvider, FlagsTarget.getAllTargetsInSystem(zoneRegistry));
+        this.deployer = new SystemFlagsDeployer(identityProvider, zoneRegistry.system(), FlagsTarget.getAllTargetsInSystem(zoneRegistry));
     }
 
     @Override
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java
index ca989d3323e..0e3295b1143 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java
@@ -17,10 +17,9 @@ import com.yahoo.slime.Cursor;
 import com.yahoo.slime.Inspector;
 import com.yahoo.slime.Slime;
 import com.yahoo.slime.SlimeStream;
-import com.yahoo.vespa.config.SlimeUtils;
+import com.yahoo.slime.SlimeUtils;
 import com.yahoo.vespa.hosted.controller.Controller;
 import com.yahoo.vespa.hosted.controller.LockedTenant;
-import com.yahoo.vespa.hosted.controller.api.integration.ApplicationIdSnapshot;
 import com.yahoo.vespa.hosted.controller.api.integration.user.Roles;
 import com.yahoo.vespa.hosted.controller.api.integration.user.User;
 import com.yahoo.vespa.hosted.controller.api.integration.user.UserId;
@@ -125,7 +124,6 @@ public class UserApiHandler extends LoggingRequestHandler {
         User user = getAttribute(request, User.ATTRIBUTE_NAME, User.class);
         Set<Role> roles = getAttribute(request, SecurityContext.ATTRIBUTE_NAME, SecurityContext.class).roles();
 
-        ApplicationIdSnapshot snapshot = controller.applicationIdSnapshot();
         Map<TenantName, List<TenantRole>> tenantRolesByTenantName = roles.stream()
                 .flatMap(role -> filterTenantRoles(role).stream())
                 .distinct()
@@ -134,10 +132,11 @@ public class UserApiHandler extends LoggingRequestHandler {
 
         // List of operator roles, currently only one available, but possible to extend
         List<Role> operatorRoles = roles.stream()
-                .filter(role -> role.definition().equals(RoleDefinition.hostedOperator))
+                .filter(role -> role.definition().equals(RoleDefinition.hostedOperator) ||
+                        role.definition().equals(RoleDefinition.hostedSupporter))
+                .sorted(Comparator.comparing(Role::definition))
                 .collect(Collectors.toList());
 
-
         Slime slime = new Slime();
         Cursor root = slime.setObject();
 
@@ -155,17 +154,6 @@ public class UserApiHandler extends LoggingRequestHandler {
                     Cursor tenantRolesObject = tenantObject.setArray("roles");
                     tenantRolesByTenantName.getOrDefault(tenant, List.of())
                             .forEach(role -> tenantRolesObject.addString(role.definition().name()));
-
-                    Cursor tenantApplicationsObject = tenantObject.setObject("applications");
-                    snapshot.applications(tenant).stream()
-                            .sorted()
-                            .forEach(application -> {
-                                Cursor applicationObject = tenantApplicationsObject.setObject(application.value());
-                                Cursor applicationInstancesObject = applicationObject.setArray("instances");
-                                snapshot.instances(tenant, application).stream()
-                                        .sorted()
-                                        .forEach(instance -> applicationInstancesObject.addString(instance.value()));
-                            });
                 });
 
         if (!operatorRoles.isEmpty()) {
@@ -347,13 +335,6 @@ public class UserApiHandler extends LoggingRequestHandler {
 
     private static String valueOf(Role role) {
         switch (role.definition()) {
-            case tenantOwner:           return "tenantOwner";
-            case tenantAdmin:           return "tenantAdmin";
-            case tenantOperator:        return "tenantOperator";
-            case applicationAdmin:      return "applicationAdmin";
-            case applicationOperator:   return "applicationOperator";
-            case applicationDeveloper:  return "applicationDeveloper";
-            case applicationReader:     return "applicationReader";
             case administrator:         return "administrator";
             case developer:             return "developer";
             case reader:                return "reader";
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepository.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepository.java
index 1a135b8d9a2..71b1fb9b2e4 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepository.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepository.java
@@ -1,13 +1,11 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.rotation;
 
+import com.yahoo.config.application.api.DeploymentInstanceSpec;
 import com.yahoo.config.application.api.DeploymentSpec;
 import com.yahoo.config.application.api.Endpoint;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.ClusterSpec;
-import com.yahoo.config.provision.Environment;
-import com.yahoo.config.provision.InstanceName;
-import com.yahoo.config.provision.RegionName;
 import com.yahoo.vespa.hosted.controller.ApplicationController;
 import com.yahoo.vespa.hosted.controller.Instance;
 import com.yahoo.vespa.hosted.controller.application.AssignedRotation;
@@ -16,15 +14,14 @@ import com.yahoo.vespa.hosted.controller.persistence.CuratorDb;
 import com.yahoo.vespa.hosted.rotation.config.RotationsConfig;
 
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
-import java.util.Set;
 import java.util.function.Function;
-import java.util.function.Predicate;
 import java.util.logging.Logger;
 import java.util.stream.Collectors;
 
@@ -69,27 +66,30 @@ public class RotationRepository {
      * If a rotation is already assigned to the application, that rotation will be returned.
      * If no rotation is assigned, return an available rotation. The caller is responsible for assigning the rotation.
      *
-     * @param deploymentSpec the deployment spec for the application
+     * @param instanceSpec the instance deployment spec
      * @param instance the instance requesting a rotation
      * @param lock lock which must be acquired by the caller
      */
-    private Rotation getOrAssignRotation(DeploymentSpec deploymentSpec, Instance instance, RotationLock lock) {
-        if ( ! instance.rotations().isEmpty()) {
-            return allRotations.get(instance.rotations().get(0).rotationId());
+    private AssignedRotation assignRotationTo(String globalServiceId, DeploymentInstanceSpec instanceSpec,
+                                              Instance instance, RotationLock lock) {
+        RotationId rotation;
+        if (instance.rotations().isEmpty()) {
+            rotation = findAvailableRotation(instance.id(), lock).id();
+        } else {
+            rotation = instance.rotations().get(0).rotationId();
         }
-
-        if (deploymentSpec.requireInstance(instance.name()).globalServiceId().isEmpty()) {
-            throw new IllegalArgumentException("global-service-id is not set in deployment spec for instance '" +
-                                               instance.name() + "'");
-        }
-        long productionZones = deploymentSpec.requireInstance(instance.name()).zones().stream()
-                                                     .filter(zone -> zone.concerns(Environment.prod))
-                                                     .count();
-        if (productionZones < 2) {
+        var productionRegions = instanceSpec.zones().stream()
+                                            .filter(zone -> zone.environment().isProduction())
+                                            .flatMap(zone -> zone.region().stream())
+                                            .collect(Collectors.toSet());
+        if (productionRegions.size() < 2) {
             throw new IllegalArgumentException("global-service-id is set but less than 2 prod zones are defined " +
                                                "in instance '" + instance.name() + "'");
         }
-        return findAvailableRotation(instance.id(), lock);
+        return new AssignedRotation(new ClusterSpec.Id(globalServiceId),
+                                    EndpointId.defaultId(),
+                                    rotation,
+                                    productionRegions);
     }
 
     /**
@@ -111,100 +111,39 @@ public class RotationRepository {
         }
 
         // Only allow one kind of configuration syntax
-        if (     deploymentSpec.requireInstance(instance.name()).globalServiceId().isPresent()
-            && ! deploymentSpec.requireInstance(instance.name()).endpoints().isEmpty()) {
+        var instanceSpec = deploymentSpec.requireInstance(instance.name());
+        if (     instanceSpec.globalServiceId().isPresent()
+            && ! instanceSpec.endpoints().isEmpty()) {
             throw new IllegalArgumentException("Cannot provision rotations with both global-service-id and 'endpoints'");
         }
 
-        // Support the older case of setting global-service-id
-        if (deploymentSpec.requireInstance(instance.name()).globalServiceId().isPresent()) {
-            var regions = deploymentSpec.requireInstance(instance.name()).zones().stream()
-                                                .filter(zone -> zone.environment().isProduction())
-                                                .flatMap(zone -> zone.region().stream())
-                                                .collect(Collectors.toSet());
-
-            var rotation = getOrAssignRotation(deploymentSpec, instance, lock);
-
-            return List.of(
-                    new AssignedRotation(
-                            new ClusterSpec.Id(deploymentSpec.requireInstance(instance.name()).globalServiceId().get()),
-                            EndpointId.defaultId(),
-                            rotation.id(),
-                            regions
-                    )
-            );
+        // Support legacy global-service-id
+        if (instanceSpec.globalServiceId().isPresent()) {
+            return List.of(assignRotationTo(instanceSpec.globalServiceId().get(), instanceSpec, instance, lock));
         }
 
-        Map<EndpointId, AssignedRotation> existingAssignments = existingEndpointAssignments(deploymentSpec, instance);
-        Map<EndpointId, AssignedRotation> updatedAssignments = assignRotationsToEndpoints(deploymentSpec, existingAssignments, instance.name(), lock);
-
-        existingAssignments.putAll(updatedAssignments);
-
-        return List.copyOf(existingAssignments.values());
+        return assignRotationsTo(instanceSpec.endpoints(), instance, lock);
     }
 
-    private Map<EndpointId, AssignedRotation> assignRotationsToEndpoints(DeploymentSpec deploymentSpec,
-                                                                         Map<EndpointId, AssignedRotation> existingAssignments,
-                                                                         InstanceName instance,
-                                                                         RotationLock lock) {
+    private List<AssignedRotation> assignRotationsTo(List<Endpoint> endpoints, Instance instance, RotationLock lock) {
+        if (endpoints.isEmpty()) return List.of(); // No endpoints declared, nothing to assign.
         var availableRotations = new ArrayList<>(availableRotations(lock).values());
-
-        var neededRotations = deploymentSpec.requireInstance(instance).endpoints().stream()
-                                            .filter(Predicate.not(endpoint -> existingAssignments.containsKey(EndpointId.of(endpoint.endpointId()))))
-                                            .collect(Collectors.toSet());
-
-        if (neededRotations.size() > availableRotations.size()) {
-            throw new IllegalStateException("Hosted Vespa ran out of rotations, unable to assign rotation: need " + neededRotations.size() + ", have " + availableRotations.size());
+        var assignedRotationsByEndpointId = instance.rotations().stream()
+                                                    .collect(Collectors.toMap(AssignedRotation::endpointId,
+                                                                              Function.identity()));
+        var assignments = new ArrayList<AssignedRotation>();
+        for (var endpoint : endpoints) {
+            var endpointId = EndpointId.of(endpoint.endpointId());
+            var assignedRotation = assignedRotationsByEndpointId.get(endpointId);
+            RotationId rotationId;
+            if (assignedRotation == null) { // No rotation is assigned to this endpoint
+                rotationId = requireNonEmpty(availableRotations).remove(0).id();
+            } else { // Rotation already assigned to this endpoint, reuse it
+                rotationId = assignedRotation.rotationId();
+            }
+            assignments.add(new AssignedRotation(ClusterSpec.Id.from(endpoint.containerId()), endpointId, rotationId, endpoint.regions()));
         }
-
-        return neededRotations.stream()
-                .map(endpoint -> {
-                        return new AssignedRotation(
-                                new ClusterSpec.Id(endpoint.containerId()),
-                                EndpointId.of(endpoint.endpointId()),
-                                availableRotations.remove(0).id(),
-                                endpoint.regions()
-                        );
-                })
-                .collect(
-                        Collectors.toMap(
-                                AssignedRotation::endpointId,
-                                Function.identity(),
-                                (a, b) -> { throw new IllegalStateException("Duplicate entries:" + a + ", " + b); },
-                                LinkedHashMap::new
-                        )
-                );
-    }
-
-    private Map<EndpointId, AssignedRotation> existingEndpointAssignments(DeploymentSpec deploymentSpec, Instance instance) {
-        // Get the regions that has been configured for an endpoint.  Empty set if the endpoint
-        // is no longer mentioned in the configuration file.
-        Function<EndpointId, Set<RegionName>> configuredRegionsForEndpoint = endpointId ->
-            deploymentSpec.requireInstance(instance.name()).endpoints().stream()
-                                 .filter(endpoint -> endpointId.id().equals(endpoint.endpointId()))
-                                 .map(Endpoint::regions)
-                                 .findFirst()
-                                 .orElse(Set.of());
-
-        // Build a new AssignedRotation instance where we update set of regions from the configuration instead
-        // of using the one already mentioned in the assignment.  This allows us to overwrite the set of regions.
-        Function<AssignedRotation, AssignedRotation> assignedRotationWithConfiguredRegions = assignedRotation ->
-            new AssignedRotation(
-                    assignedRotation.clusterId(),
-                    assignedRotation.endpointId(),
-                    assignedRotation.rotationId(),
-                    configuredRegionsForEndpoint.apply(assignedRotation.endpointId()));
-
-        return instance.rotations().stream()
-                       .collect(Collectors.toMap(
-                                          AssignedRotation::endpointId,
-                                          assignedRotationWithConfiguredRegions,
-                                          (a, b) -> {
-                                              throw new IllegalStateException("Duplicate entries: " + a + ", " + b);
-                                          },
-                                          LinkedHashMap::new
-                                  )
-                          );
+        return Collections.unmodifiableList(assignments);
     }
 
     /**
@@ -224,12 +163,8 @@ public class RotationRepository {
 
     private Rotation findAvailableRotation(ApplicationId id, RotationLock lock) {
         Map<RotationId, Rotation> availableRotations = availableRotations(lock);
-        if (availableRotations.isEmpty()) {
-            throw new IllegalStateException("Unable to assign global rotation to " + id
-                                            + " - no rotations available");
-        }
         // Return first available rotation
-        RotationId rotation = availableRotations.keySet().iterator().next();
+        RotationId rotation = requireNonEmpty(availableRotations.keySet()).iterator().next();
         log.info(String.format("Offering %s to application %s", rotation, id));
         return allRotations.get(rotation);
     }
@@ -246,4 +181,9 @@ public class RotationRepository {
                                                         Collections::unmodifiableMap));
     }
 
+    private static <T extends Collection<?>> T requireNonEmpty(T rotations) {
+        if (rotations.isEmpty()) throw new IllegalStateException("Hosted Vespa ran out of rotations, unable to assign rotation");
+        return rotations;
+    }
+
 }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/GlobalRouting.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/GlobalRouting.java
index 244686c85e4..7ea651c7bd5 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/GlobalRouting.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/GlobalRouting.java
@@ -27,7 +27,10 @@ public class GlobalRouting {
         this.changedAt = Objects.requireNonNull(changedAt, "changedAt must be non-null");
     }
 
-    /** The current status of this */
+    /**
+     * The wanted status of this. The system will try to set this status, but there are constraints that may lead to
+     * the effective status not matching this. See {@link RoutingPolicies}.
+     */
     public Status status() {
         return status;
     }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingId.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingId.java
index 5543d0ea0b7..8b012b15cd3 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingId.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingId.java
@@ -48,4 +48,8 @@ public class RoutingId {
         return "routing id for " + endpointId + " of " + application;
     }
 
+    public static RoutingId of(ApplicationId application, EndpointId endpoint) {
+        return new RoutingId(application, endpoint);
+    }
+
 }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java
index 88f746a9c51..033539f64c4 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java
@@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.controller.routing;
 
 import com.yahoo.config.application.api.DeploymentSpec;
 import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.zone.RoutingMethod;
 import com.yahoo.config.provision.zone.ZoneId;
 import com.yahoo.vespa.curator.Lock;
 import com.yahoo.vespa.hosted.controller.Controller;
@@ -13,11 +14,11 @@ import com.yahoo.vespa.hosted.controller.api.integration.dns.Record;
 import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData;
 import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName;
 import com.yahoo.vespa.hosted.controller.application.EndpointId;
+import com.yahoo.vespa.hosted.controller.dns.NameServiceForwarder;
 import com.yahoo.vespa.hosted.controller.dns.NameServiceQueue.Priority;
 import com.yahoo.vespa.hosted.controller.persistence.CuratorDb;
 
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashSet;
@@ -57,14 +58,9 @@ public class RoutingPolicies {
 
     /** Read all known routing policies for given deployment */
     public Map<RoutingPolicyId, RoutingPolicy> get(DeploymentId deployment) {
-        return get(deployment.applicationId(), deployment.zoneId());
-    }
-
-    /** Read all known routing policies for given deployment */
-    public Map<RoutingPolicyId, RoutingPolicy> get(ApplicationId application, ZoneId zone) {
-        return db.readRoutingPolicies(application).entrySet()
+        return db.readRoutingPolicies(deployment.applicationId()).entrySet()
                  .stream()
-                 .filter(kv -> kv.getKey().zone().equals(zone))
+                 .filter(kv -> kv.getKey().zone().equals(deployment.zoneId()))
                  .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
     }
 
@@ -78,7 +74,6 @@ public class RoutingPolicies {
      * load balancers for given application have changed.
      */
     public void refresh(ApplicationId application, DeploymentSpec deploymentSpec, ZoneId zone) {
-        if (!controller.zoneRegistry().zones().directlyRouted().ids().contains(zone)) return;
         var loadBalancers = new AllocatedLoadBalancers(application, zone, controller.serviceRegistry().configServer()
                                                                                     .getLoadBalancers(application, zone),
                                                        deploymentSpec);
@@ -87,7 +82,7 @@ public class RoutingPolicies {
             removeGlobalDnsUnreferencedBy(loadBalancers, lock);
             storePoliciesOf(loadBalancers, lock);
             removePoliciesUnreferencedBy(loadBalancers, lock);
-            updateGlobalDnsOf(get(loadBalancers.application).values(), inactiveZones, lock);
+            updateGlobalDnsOf(get(loadBalancers.deployment.applicationId()).values(), inactiveZones, lock);
         }
     }
 
@@ -128,33 +123,42 @@ public class RoutingPolicies {
             var staleTargets = new LinkedHashSet<AliasTarget>();
             for (var policy : routeEntry.getValue()) {
                 if (policy.dnsZone().isEmpty()) continue;
+                if (!controller.zoneRegistry().routingMethods(policy.id().zone()).contains(RoutingMethod.exclusive)) continue;
                 var target = new AliasTarget(policy.canonicalName(), policy.dnsZone().get(), policy.id().zone());
                 var zonePolicy = db.readZoneRoutingPolicy(policy.id().zone());
                 // Remove target zone if global routing status is set out at:
                 // - zone level (ZoneRoutingPolicy)
                 // - deployment level (RoutingPolicy)
                 // - application package level (deployment.xml)
-                if (anyOut(zonePolicy.globalRouting(), policy.status().globalRouting()) ||
-                    inactiveZones.contains(policy.id().zone())) {
+                if (isConfiguredOut(policy, zonePolicy, inactiveZones)) {
                     staleTargets.add(target);
                 } else {
                     targets.add(target);
                 }
             }
+            // If all targets are configured out, all targets are set in. We do this because otherwise removing 100% of
+            // the ALIAS records would cause the global endpoint to stop resolving entirely (NXDOMAIN).
+            if (targets.isEmpty() && !staleTargets.isEmpty()) {
+                targets.addAll(staleTargets);
+                staleTargets.clear();
+            }
             if (!targets.isEmpty()) {
-                var endpoint = RoutingPolicy.globalEndpointOf(routeEntry.getKey().application(),
-                                                              routeEntry.getKey().endpointId(), controller.system());
-                controller.nameServiceForwarder().createAlias(RecordName.from(endpoint.dnsName()), targets, Priority.normal);
+                var endpoints = controller.routing().endpointsOf(routeEntry.getKey().application())
+                                          .named(routeEntry.getKey().endpointId())
+                                          .not().requiresRotation();
+                endpoints.forEach(endpoint -> controller.nameServiceForwarder().createAlias(RecordName.from(endpoint.dnsName()), targets, Priority.normal));
             }
-            staleTargets.forEach(t -> controller.nameServiceForwarder().removeRecords(Record.Type.ALIAS, t.asData(), Priority.normal));
+            staleTargets.forEach(t -> controller.nameServiceForwarder().removeRecords(Record.Type.ALIAS,
+                                                                                      RecordData.fqdn(t.name().value()),
+                                                                                      Priority.normal));
         }
     }
 
     /** Store routing policies for given load balancers */
     private void storePoliciesOf(AllocatedLoadBalancers loadBalancers, @SuppressWarnings("unused") Lock lock) {
-        var policies = new LinkedHashMap<>(get(loadBalancers.application));
+        var policies = new LinkedHashMap<>(get(loadBalancers.deployment.applicationId()));
         for (LoadBalancer loadBalancer : loadBalancers.list) {
-            var policyId = new RoutingPolicyId(loadBalancer.application(), loadBalancer.cluster(), loadBalancers.zone);
+            var policyId = new RoutingPolicyId(loadBalancer.application(), loadBalancer.cluster(), loadBalancers.deployment.zoneId());
             var existingPolicy = policies.get(policyId);
             var newPolicy = new RoutingPolicy(policyId, loadBalancer.hostname(), loadBalancer.dnsZone(),
                                               loadBalancers.endpointIdsOf(loadBalancer),
@@ -166,42 +170,45 @@ public class RoutingPolicies {
             updateZoneDnsOf(newPolicy);
             policies.put(newPolicy.id(), newPolicy);
         }
-        db.writeRoutingPolicies(loadBalancers.application, policies);
+        db.writeRoutingPolicies(loadBalancers.deployment.applicationId(), policies);
     }
 
     /** Update zone DNS record for given policy */
     private void updateZoneDnsOf(RoutingPolicy policy) {
-        var name = RecordName.from(policy.endpointIn(controller.system()).dnsName());
+        var name = RecordName.from(policy.endpointIn(controller.system(), RoutingMethod.exclusive).dnsName());
         var data = RecordData.fqdn(policy.canonicalName().value());
-        controller.nameServiceForwarder().createCname(name, data, Priority.normal);
+        nameUpdaterIn(policy.id().zone()).createCname(name, data);
     }
 
     /** Remove policies and zone DNS records unreferenced by given load balancers */
     private void removePoliciesUnreferencedBy(AllocatedLoadBalancers loadBalancers, @SuppressWarnings("unused") Lock lock) {
-        var policies = get(loadBalancers.application);
+        var policies = get(loadBalancers.deployment.applicationId());
         var newPolicies = new LinkedHashMap<>(policies);
         var activeLoadBalancers = loadBalancers.list.stream().map(LoadBalancer::hostname).collect(Collectors.toSet());
         for (var policy : policies.values()) {
             // Leave active load balancers and irrelevant zones alone
             if (activeLoadBalancers.contains(policy.canonicalName()) ||
-                !policy.id().zone().equals(loadBalancers.zone)) continue;
+                !policy.id().zone().equals(loadBalancers.deployment.zoneId())) continue;
 
-            var dnsName = policy.endpointIn(controller.system()).dnsName();
-            controller.nameServiceForwarder().removeRecords(Record.Type.CNAME, RecordName.from(dnsName), Priority.normal);
+            var dnsName = policy.endpointIn(controller.system(), RoutingMethod.exclusive).dnsName();
+            nameUpdaterIn(loadBalancers.deployment.zoneId()).removeRecords(Record.Type.CNAME, RecordName.from(dnsName));
             newPolicies.remove(policy.id());
         }
-        db.writeRoutingPolicies(loadBalancers.application, newPolicies);
+        db.writeRoutingPolicies(loadBalancers.deployment.applicationId(), newPolicies);
     }
 
     /** Remove unreferenced global endpoints from DNS */
     private void removeGlobalDnsUnreferencedBy(AllocatedLoadBalancers loadBalancers, @SuppressWarnings("unused") Lock lock) {
-        var zonePolicies = get(loadBalancers.application, loadBalancers.zone).values();
+        var zonePolicies = get(loadBalancers.deployment).values();
         var removalCandidates = new HashSet<>(routingTableFrom(zonePolicies).keySet());
         var activeRoutingIds = routingIdsFrom(loadBalancers);
         removalCandidates.removeAll(activeRoutingIds);
         for (var id : removalCandidates) {
-            var endpoint = RoutingPolicy.globalEndpointOf(id.application(), id.endpointId(), controller.system());
-            controller.nameServiceForwarder().removeRecords(Record.Type.ALIAS, RecordName.from(endpoint.dnsName()), Priority.normal);
+            var endpoints = controller.routing().endpointsOf(id.application())
+                                      .not().requiresRotation()
+                                      .named(id.endpointId());
+            var nameUpdater = nameUpdaterIn(loadBalancers.deployment.zoneId());
+            endpoints.forEach(endpoint -> nameUpdater.removeRecords(Record.Type.ALIAS, RecordName.from(endpoint.dnsName())));
         }
     }
 
@@ -229,10 +236,15 @@ public class RoutingPolicies {
         return Collections.unmodifiableMap(routingTable);
     }
 
-    private static boolean anyOut(GlobalRouting... globalRouting) {
-        return Arrays.stream(globalRouting)
-                     .map(GlobalRouting::status)
-                     .anyMatch(status -> status == GlobalRouting.Status.out);
+    /** Returns whether the global routing status of given policy is configured to be {@link GlobalRouting.Status#out} */
+    private static boolean isConfiguredOut(RoutingPolicy policy, ZoneRoutingPolicy zonePolicy, Set<ZoneId> inactiveZones) {
+        // A deployment is can be configured out at any of the following levels:
+        // - zone level (ZoneRoutingPolicy)
+        // - deployment level (RoutingPolicy)
+        // - application package level (deployment.xml)
+        return zonePolicy.globalRouting().status() == GlobalRouting.Status.out ||
+               policy.status().globalRouting().status() == GlobalRouting.Status.out ||
+               inactiveZones.contains(policy.id().zone());
     }
 
     private static boolean isActive(LoadBalancer loadBalancer) {
@@ -247,22 +259,20 @@ public class RoutingPolicies {
     /** Load balancers allocated to a deployment */
     private static class AllocatedLoadBalancers {
 
-        private final ApplicationId application;
-        private final ZoneId zone;
+        private final DeploymentId deployment;
         private final List<LoadBalancer> list;
         private final DeploymentSpec deploymentSpec;
 
         private AllocatedLoadBalancers(ApplicationId application, ZoneId zone, List<LoadBalancer> loadBalancers,
                                        DeploymentSpec deploymentSpec) {
-            this.application = application;
-            this.zone = zone;
+            this.deployment = new DeploymentId(application, zone);
             this.list = List.copyOf(loadBalancers);
             this.deploymentSpec = deploymentSpec;
         }
 
         /** Compute all endpoint IDs for given load balancer */
         private Set<EndpointId> endpointIdsOf(LoadBalancer loadBalancer) {
-            if (zone.environment().isManuallyDeployed()) { // Manual deployments do not have any configurable endpoints
+            if (!deployment.zoneId().environment().isProduction()) { // Only production deployments have configurable endpoints
                 return Set.of();
             }
             var instanceSpec = deploymentSpec.instance(loadBalancer.application().instance());
@@ -271,7 +281,7 @@ public class RoutingPolicies {
             }
             return instanceSpec.get().endpoints().stream()
                                .filter(endpoint -> endpoint.containerId().equals(loadBalancer.cluster().value()))
-                               .filter(endpoint -> endpoint.regions().contains(zone.region()))
+                               .filter(endpoint -> endpoint.regions().contains(deployment.zoneId().region()))
                                .map(com.yahoo.config.application.api.Endpoint::endpointId)
                                .map(EndpointId::of)
                                .collect(Collectors.toSet());
@@ -290,4 +300,53 @@ public class RoutingPolicies {
                            .collect(Collectors.toUnmodifiableSet());
     }
 
+    /** Returns the name updater to use for given zone */
+    private NameUpdater nameUpdaterIn(ZoneId zone) {
+        if (controller.zoneRegistry().routingMethods(zone).contains(RoutingMethod.exclusive)) {
+            return new NameUpdater(controller.nameServiceForwarder());
+        }
+        return new DiscardingNameUpdater();
+    }
+
+    /** A name updater that passes name service operations to the next handler */
+    private static class NameUpdater {
+
+        private final NameServiceForwarder forwarder;
+
+        public NameUpdater(NameServiceForwarder forwarder) {
+            this.forwarder = forwarder;
+        }
+
+        public void removeRecords(Record.Type type, RecordName name) {
+            forwarder.removeRecords(type, name, Priority.normal);
+        }
+
+        public void createAlias(RecordName name, Set<AliasTarget> targets) {
+            forwarder.createAlias(name, targets, Priority.normal);
+        }
+
+        public void createCname(RecordName name, RecordData data) {
+            forwarder.createCname(name, data, Priority.normal);
+        }
+
+    }
+
+    /** A name updater that does nothing */
+    private static class DiscardingNameUpdater extends NameUpdater {
+
+        private DiscardingNameUpdater() {
+            super(null);
+        }
+
+        @Override
+        public void removeRecords(Record.Type type, RecordName name) {}
+
+        @Override
+        public void createAlias(RecordName name, Set<AliasTarget> target) {}
+
+        @Override
+        public void createCname(RecordName name, RecordData data) {}
+
+    }
+
 }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java
index 9ef2d519c05..37027e8a8c9 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java
@@ -2,13 +2,12 @@
 package com.yahoo.vespa.hosted.controller.routing;
 
 import com.google.common.collect.ImmutableSortedSet;
-import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.HostName;
 import com.yahoo.config.provision.SystemName;
+import com.yahoo.config.provision.zone.RoutingMethod;
 import com.yahoo.vespa.hosted.controller.application.Endpoint;
 import com.yahoo.vespa.hosted.controller.application.Endpoint.Port;
 import com.yahoo.vespa.hosted.controller.application.EndpointId;
-import com.yahoo.vespa.hosted.controller.application.EndpointList;
 
 import java.util.Objects;
 import java.util.Optional;
@@ -69,13 +68,12 @@ public class RoutingPolicy {
     }
 
     /** Returns the endpoint of this */
-    public Endpoint endpointIn(SystemName system) {
-        return Endpoint.of(id.owner()).target(id.cluster(), id.zone()).on(Port.tls()).directRouting().in(system);
-    }
-
-    /** Returns global endpoints which this is a member of */
-    public EndpointList globalEndpointsIn(SystemName system) {
-        return EndpointList.of(endpoints.stream().map(endpointId -> globalEndpointOf(id.owner(), endpointId, system)));
+    public Endpoint endpointIn(SystemName system, RoutingMethod routingMethod) {
+        return Endpoint.of(id.owner())
+                       .target(id.cluster(), id.zone())
+                       .on(Port.fromRoutingMethod(routingMethod))
+                       .routingMethod(routingMethod)
+                       .in(system);
     }
 
     @Override
@@ -98,9 +96,4 @@ public class RoutingPolicy {
                              id.zone().value());
     }
 
-    /** Creates a global endpoint for given application */
-    public static Endpoint globalEndpointOf(ApplicationId application, EndpointId endpointId, SystemName system) {
-        return Endpoint.of(application).named(endpointId).on(Port.tls()).directRouting().in(system);
-    }
-
 }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/Tenant.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/Tenant.java
index e0c750dec80..bac43517f1a 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/Tenant.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/Tenant.java
@@ -64,11 +64,8 @@ public abstract class Tenant {
         /** Tenant authenticated through Athenz. */
         athenz,
 
-        /** Tenant authenticated through Okta, as a user. */
-        user,
-
         /** Tenant authenticated through some cloud identity provider. */
-        cloud;
+        cloud
 
     }
 
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/UserTenant.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/UserTenant.java
deleted file mode 100644
index a46d847f6f3..00000000000
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/UserTenant.java
+++ /dev/null
@@ -1,73 +0,0 @@
-// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.controller.tenant;
-
-import com.yahoo.config.provision.TenantName;
-import com.yahoo.vespa.hosted.controller.api.integration.organization.Contact;
-
-import java.util.Optional;
-
-/**
- * Represents an user tenant in hosted Vespa.
- *
- * @author mpolden
- */
-public class UserTenant extends Tenant {
-
-    /**
-     * This should only be used by serialization.
-     * Use {@link #create(String)}.
-     * */
-    public UserTenant(TenantName name, Optional<Contact> contact) {
-        super(name, contact);
-    }
-
-    @Override
-    public Type type() {
-        return Type.user;
-    }
-
-    public UserTenant(TenantName name) {
-        super(name, Optional.empty());
-    }
-
-    /** Returns true if this is the tenant for the given user name */
-    public boolean is(String username) {
-        return name().value().equals(normalizeUser(username));
-    }
-
-    @Override
-    public String toString() {
-        return "user tenant '" + name() + "'";
-    }
-
-    /** Create a new user tenant */
-    public static UserTenant create(String username) {
-        TenantName name = TenantName.from(username);
-        return new UserTenant(requireName(requireUser(name)));
-    }
-
-    public static UserTenant create(String username, Optional<Contact> contact) {
-        TenantName name = TenantName.from(username);
-        return new UserTenant(requireName(requireUser(name)), contact);
-    }
-
-    /** Normalize given username. E.g. foo_bar becomes by-foo-bar */
-    public static String normalizeUser(String username) {
-        int offset = 0;
-        if (username.startsWith(Tenant.userPrefix)) {
-            offset = Tenant.userPrefix.length();
-        }
-        return Tenant.userPrefix + username.substring(offset).replace('_', '-');
-    }
-
-    private static TenantName requireUser(TenantName name) {
-        if (!name.value().startsWith(Tenant.userPrefix)) {
-            throw new IllegalArgumentException("User tenant must have prefix '" + Tenant.userPrefix + "'");
-        }
-        if (name.value().substring(Tenant.userPrefix.length()).contains("_")) {
-            throw new IllegalArgumentException("User tenant cannot contain '_'");
-        }
-        return name;
-    }
-
-}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/DeploymentStatistics.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/DeploymentStatistics.java
index ae7223489c2..b91ed3734f1 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/DeploymentStatistics.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/DeploymentStatistics.java
@@ -1,90 +1,154 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.versions;
 
-import com.google.common.collect.ImmutableSet;
 import com.yahoo.component.Version;
-import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.vespa.hosted.controller.Instance;
+import com.yahoo.vespa.hosted.controller.application.Deployment;
+import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatus;
+import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatusList;
+import com.yahoo.vespa.hosted.controller.deployment.JobList;
+import com.yahoo.vespa.hosted.controller.deployment.JobStatus;
+import com.yahoo.vespa.hosted.controller.deployment.Run;
+import com.yahoo.vespa.hosted.controller.deployment.RunStatus;
 
+import java.util.ArrayList;
 import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static java.util.Comparator.naturalOrder;
+import static java.util.function.Function.identity;
 
 /**
  * Statistics about deployments on a platform version. This is immutable.
  *
- * @author bratseth
+ * @author jonmv
  */
 public class DeploymentStatistics {
     
     private final Version version;
-    private final ImmutableSet<ApplicationId> failing;
-    private final ImmutableSet<ApplicationId> production;
-    private final ImmutableSet<ApplicationId> deploying;
-
-    /** DO NOT USE. Public for serialization purposes */
-    public DeploymentStatistics(Version version, Collection<ApplicationId> failingApplications,
-                                Collection<ApplicationId> production, Collection<ApplicationId> deploying) {
-        this.version = version;
-        this.failing = ImmutableSet.copyOf(failingApplications);
-        this.production = ImmutableSet.copyOf(production);
-        this.deploying = ImmutableSet.copyOf(deploying);
-    }
+    private final List<Run> failingUpgrades;
+    private final List<Run> otherFailing;
+    private final List<Run> productionSuccesses;
+    private final List<Run> runningUpgrade;
+    private final List<Run> otherRunning;
 
-    /** Returns a statistics instance with the values as 0 */
-    public static DeploymentStatistics empty(Version version) { 
-        return new DeploymentStatistics(version, ImmutableSet.of(), ImmutableSet.of(), ImmutableSet.of());
+    public DeploymentStatistics(Version version, List<Run> failingUpgrades, List<Run> otherFailing,
+                                List<Run> productionSuccesses, List<Run> runningUpgrade, List<Run> otherRunning) {
+        this.version = Objects.requireNonNull(version);
+        this.failingUpgrades = List.copyOf(failingUpgrades);
+        this.otherFailing = List.copyOf(otherFailing);
+        this.productionSuccesses = List.copyOf(productionSuccesses);
+        this.runningUpgrade = List.copyOf(runningUpgrade);
+        this.otherRunning = List.copyOf(otherRunning);
     }
 
-    /** Returns the version these statistics are for */
+    /** Returns the version these statistics are for. */
     public Version version() { return version; }
-    
-    /** 
-     * Returns the applications which have at least one job (of any type) which fails on this version, 
-     * excluding errors known to not be caused by this version
-     */
-    public Set<ApplicationId> failing() { return failing; }
-    
-    /** Returns the applications which have this version in production in at least one zone */
-    public Set<ApplicationId> production() { return production; }
 
-    /** Returns the applications which are currently upgrading to this version */
-    public Set<ApplicationId> deploying() { return deploying; }
+    /** Returns the runs on the version of this, for currently failing instances, where the failure may be because of the upgrade. */
+    public List<Run> failingUpgrades() { return failingUpgrades; }
 
-    /** Returns a version of this with the given failing application added */
-    public DeploymentStatistics withFailing(ApplicationId application) { 
-        return new DeploymentStatistics(version, add(application, failing), production, deploying);
-    }
+    /** Returns all other failing runs on the version of this, for currently failing instances. */
+    public List<Run> otherFailing() { return otherFailing; }
 
-    /** Returns a version of this with the given production application added */
-    public DeploymentStatistics withProduction(ApplicationId application) {
-        return new DeploymentStatistics(version, failing, add(application, production), deploying);
-    }
+    /** Returns the production runs where the last success was on the version of this. */
+    public List<Run> productionSuccesses() { return productionSuccesses; }
 
-    /** Returns a version of this with the given deploying application added */
-    public DeploymentStatistics withDeploying(ApplicationId application) {
-        return new DeploymentStatistics(version, failing, production, add(application, deploying));
-    }
-    
-    private ImmutableSet<ApplicationId> add(ApplicationId application, ImmutableSet<ApplicationId> list) {
-        ImmutableSet.Builder<ApplicationId> b = new ImmutableSet.Builder<>();
-        b.addAll(list);
-        b.add(application);
-        return b.build();
-    }
+    /** Returns the currently running runs on the version of this, where an upgrade is attempted. */
+    public List<Run> runningUpgrade() { return runningUpgrade; }
 
-    @Override
-    public boolean equals(Object o) {
-        if (this == o) return true;
-        if (!(o instanceof DeploymentStatistics)) return false;
-        DeploymentStatistics that = (DeploymentStatistics) o;
-        return Objects.equals(version, that.version) &&
-               Objects.equals(failing, that.failing) &&
-               Objects.equals(production, that.production);
-    }
+    /** Returns all other currently running runs on the version on this. */
+    public List<Run> otherRunning() { return otherRunning; }
+
+    public static List<DeploymentStatistics> compute(Collection<Version> infrastructureVersions, DeploymentStatusList statuses) {
+
+        Set<Version> allVersions = new HashSet<>(infrastructureVersions);
+        Map<Version, List<Run>> failingUpgrade = new HashMap<>();
+        Map<Version, List<Run>> otherFailing = new HashMap<>();
+        Map<Version, List<Run>> productionSuccesses = new HashMap<>();
+        Map<Version, List<Run>> runningUpgrade = new HashMap<>();
+        Map<Version, List<Run>> otherRunning = new HashMap<>();
+
+        for (DeploymentStatus status : statuses.asList()) {
+            if (status.application().projectId().isEmpty())
+                continue;
+
+            for (Instance instance : status.application().instances().values())
+                for (Deployment deployment : instance.productionDeployments().values())
+                    allVersions.add(deployment.version());
+
+            JobList failing = status.jobs().failing();
+
+            // Add all unsuccessful runs for failing jobs as any run may have resulted in an incomplete deployment
+            // where a subset of nodes have upgraded.
+            // TODO jonmv: canary-pipeline.custom on 7.188.11, but not really, in staging ...
+            failing.not().failingApplicationChange()
+                   .not().withStatus(RunStatus.outOfCapacity)
+                   .mapToList(JobStatus::runs)
+                   .forEach(runs -> runs.descendingMap().values().stream()
+                                        .dropWhile(run -> ! run.hasEnded())
+                                        .takeWhile(run -> run.hasFailed())
+                                        .forEach(run -> {
+                                            failingUpgrade.putIfAbsent(run.versions().targetPlatform(), new ArrayList<>());
+                                            if (failingUpgrade.get(run.versions().targetPlatform()).stream().noneMatch(existing -> existing.id().job().equals(run.id().job())))
+                                                failingUpgrade.get(run.versions().targetPlatform()).add(run);
+                                        }));
+
+            failing.failingApplicationChange()
+                   .concat(failing.withStatus(RunStatus.outOfCapacity))
+                   .lastCompleted().asList()
+                   .forEach(run -> {
+                       otherFailing.putIfAbsent(run.versions().targetPlatform(), new ArrayList<>());
+                       otherFailing.get(run.versions().targetPlatform()).add(run);
+                   });
+
+            status.jobs().production()
+                  .lastSuccess().asList()
+                  .forEach(run -> {
+                      productionSuccesses.putIfAbsent(run.versions().targetPlatform(), new ArrayList<>());
+                      productionSuccesses.get(run.versions().targetPlatform()).add(run);
+                  });
+
+            JobList running = status.jobs().running();
+            running.upgrading()
+                   .lastTriggered().asList()
+                   .forEach(run -> {
+                       runningUpgrade.putIfAbsent(run.versions().targetPlatform(), new ArrayList<>());
+                       runningUpgrade.get(run.versions().targetPlatform()).add(run);
+                   });
+
+            running.not().upgrading()
+                   .lastTriggered().asList()
+                   .forEach(run -> {
+                       otherRunning.putIfAbsent(run.versions().targetPlatform(), new ArrayList<>());
+                       otherRunning.get(run.versions().targetPlatform()).add(run);
+                   });
+        }
+
+        return Stream.of(allVersions.stream(),
+                         failingUpgrade.keySet().stream(),
+                         otherFailing.keySet().stream(),
+                         productionSuccesses.keySet().stream(),
+                         runningUpgrade.keySet().stream(),
+                         otherRunning.keySet().stream())
+                     .flatMap(identity()) // Lol.
+                     .distinct()
+                     .sorted(naturalOrder())
+                     .map(version -> new DeploymentStatistics(version,
+                                                              failingUpgrade.getOrDefault(version, List.of()),
+                                                              otherFailing.getOrDefault(version, List.of()),
+                                                              productionSuccesses.getOrDefault(version, List.of()),
+                                                              runningUpgrade.getOrDefault(version, List.of()),
+                                                              otherRunning.getOrDefault(version, List.of())))
+                     .collect(Collectors.toUnmodifiableList());
 
-    @Override
-    public int hashCode() {
-        return Objects.hash(version, failing, production);
     }
 
 }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/NodeVersion.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/NodeVersion.java
index 8d0232afa58..d82cddb4779 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/NodeVersion.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/NodeVersion.java
@@ -95,8 +95,4 @@ public class NodeVersion {
         return Objects.hash(hostname, zone, currentVersion, wantedVersion, changedAt);
     }
 
-    public static NodeVersion empty(HostName hostname) {
-        return new NodeVersion(hostname, ZoneId.defaultId(), Version.emptyVersion, Version.emptyVersion, Instant.EPOCH);
-    }
-
 }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java
index 1b2928c57b1..0868d7ca695 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java
@@ -119,8 +119,9 @@ public class VersionStatus {
         }
 
 
-        var deploymentStatistics = computeDeploymentStatistics(infrastructureVersions.keySet(),
-                                                               controller.jobController().deploymentStatuses(ApplicationList.from(controller.applications().asList())));
+        var deploymentStatistics = DeploymentStatistics.compute(infrastructureVersions.keySet(),
+                                                                controller.jobController().deploymentStatuses(ApplicationList.from(controller.applications().asList())
+                                                                                                                            .withProjectId()));
         List<VespaVersion> versions = new ArrayList<>();
         List<Version> releasedVersions = controller.mavenRepository().metadata().versions();
 
@@ -187,50 +188,6 @@ public class VersionStatus {
         return versions;
     }
 
-    private static Collection<DeploymentStatistics> computeDeploymentStatistics(Set<Version> infrastructureVersions,
-                                                                                DeploymentStatusList statuses) {
-        Map<Version, DeploymentStatistics> versionMap = new HashMap<>();
-
-        for (Version infrastructureVersion : infrastructureVersions) {
-            versionMap.put(infrastructureVersion, DeploymentStatistics.empty(infrastructureVersion));
-        }
-
-        for (DeploymentStatus status : statuses.asList()) {
-            for (Instance instance : status.application().instances().values())
-                for (Deployment deployment : instance.productionDeployments().values())
-                    versionMap.computeIfAbsent(deployment.version(), DeploymentStatistics::empty);
-
-            status.instanceJobs().forEach((id, jobs) -> {
-                // Add all unsuccessful runs for failing jobs as any run may have resulted in an incomplete deployment
-                // where a subset of nodes have upgraded.
-                jobs.failing()
-                    .not().failingApplicationChange()
-                    .not().withStatus(RunStatus.outOfCapacity)
-                    .mapToList(JobStatus::runs)
-                    .forEach(runs -> runs.descendingMap().values().stream()
-                                         .dropWhile(run -> !run.hasEnded())
-                                         .takeWhile(run -> run.hasFailed())
-                                         .map(run -> run.versions().targetPlatform())
-                                         .forEach(version -> versionMap.put(version,
-                                                                            versionMap.getOrDefault(version, DeploymentStatistics.empty(version))
-                                                                                      .withFailing(id))));
-
-                jobs.production()
-                    .lastSuccess().mapToList(run -> run.versions().targetPlatform())
-                    .forEach(version -> versionMap.put(version,
-                                                       versionMap.getOrDefault(version, DeploymentStatistics.empty(version))
-                                                                 .withProduction(id)));
-
-                jobs.upgrading()
-                    .lastTriggered().mapToList(run -> run.versions().targetPlatform())
-                    .forEach(version -> versionMap.put(version,
-                                                       versionMap.getOrDefault(version, DeploymentStatistics.empty(version))
-                                                                 .withDeploying(id)));
-            });
-        }
-        return versionMap.values();
-    }
-
     private static VespaVersion createVersion(DeploymentStatistics statistics,
                                               Set<ControllerVersion> controllerVersions,
                                               Version systemVersion,
@@ -270,7 +227,7 @@ public class VersionStatus {
             }
         }
 
-        return new VespaVersion(statistics,
+        return new VespaVersion(statistics.version(),
                                 commitSha,
                                 commitDate,
                                 isControllerVersion,
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java
index 7c3c30738d6..7122b7dcc40 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java
@@ -5,6 +5,7 @@ import com.yahoo.component.Version;
 import com.yahoo.vespa.hosted.controller.Controller;
 import com.yahoo.vespa.hosted.controller.application.ApplicationList;
 import com.yahoo.vespa.hosted.controller.application.InstanceList;
+import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatus;
 
 import java.time.Instant;
 import java.time.ZoneOffset;
@@ -20,21 +21,21 @@ import static com.yahoo.config.application.api.DeploymentSpec.UpgradePolicy;
  * @author bratseth
  */
 public class VespaVersion implements Comparable<VespaVersion> {
-    
+
+    private final Version version;
     private final String releaseCommit;
     private final Instant committedAt;
     private final boolean isControllerVersion;
     private final boolean isSystemVersion;
     private final boolean isReleased;
-    private final DeploymentStatistics statistics;
     private final NodeVersions nodeVersions;
     private final Confidence confidence;
 
-    public VespaVersion(DeploymentStatistics statistics, String releaseCommit, Instant committedAt,
+    public VespaVersion(Version version, String releaseCommit, Instant committedAt,
                         boolean isControllerVersion, boolean isSystemVersion, boolean isReleased,
                         NodeVersions nodeVersions,
                         Confidence confidence) {
-        this.statistics = statistics;
+        this.version = version;
         this.releaseCommit = releaseCommit;
         this.committedAt = committedAt;
         this.isControllerVersion = isControllerVersion;
@@ -48,10 +49,10 @@ public class VespaVersion implements Comparable<VespaVersion> {
         InstanceList all = InstanceList.from(controller.jobController().deploymentStatuses(ApplicationList.from(controller.applications().asList())))
                                        .withProductionDeployment();
         // 'production on this': All deployment jobs upgrading to this version have completed without failure
-        InstanceList productionOnThis = all.matching(statistics.production()::contains)
-                                                   .not().failingUpgrade()
-                                                   .not().upgradingTo(statistics.version());
-        InstanceList failingOnThis = all.matching(statistics.failing()::contains);
+        InstanceList productionOnThis = all.matching(instance -> statistics.productionSuccesses().stream().anyMatch(run -> run.id().application().equals(instance)))
+                                           .not().failingUpgrade()
+                                           .not().upgradingTo(statistics.version());
+        InstanceList failingOnThis = all.matching(instance -> statistics.failingUpgrades().stream().anyMatch(run -> run.id().application().equals(instance)));
 
         // 'broken' if any Canary fails
         if  ( ! failingOnThis.with(UpgradePolicy.canary).isEmpty())
@@ -74,7 +75,7 @@ public class VespaVersion implements Comparable<VespaVersion> {
     }
 
     /** Returns the version number of this Vespa version */
-    public Version versionNumber() { return statistics.version(); }
+    public Version versionNumber() { return version; }
 
     /** Returns the sha of the release tag commit for this version in git */
     public String releaseCommit() { return releaseCommit; }
@@ -82,9 +83,6 @@ public class VespaVersion implements Comparable<VespaVersion> {
     /** Returns the time of the release commit */
     public Instant committedAt() { return committedAt; }
     
-    /** Statistics about deployment of this version */
-    public DeploymentStatistics statistics() { return statistics; }
-
     /** Returns whether this is the current version of controllers in this system (the lowest version across all
      * controllers) */
     public boolean isControllerVersion() {
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java
index 02c86f78ec0..0ff14e01874 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java
@@ -1,4 +1,4 @@
-// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller;
 
 import com.google.common.collect.Sets;
@@ -6,30 +6,37 @@ import com.yahoo.component.Version;
 import com.yahoo.config.application.api.DeploymentSpec;
 import com.yahoo.config.application.api.ValidationId;
 import com.yahoo.config.application.api.ValidationOverrides;
-import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.AthenzDomain;
+import com.yahoo.config.provision.AthenzService;
 import com.yahoo.config.provision.CloudName;
+import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.config.provision.Environment;
 import com.yahoo.config.provision.RegionName;
 import com.yahoo.config.provision.SystemName;
 import com.yahoo.config.provision.TenantName;
+import com.yahoo.config.provision.zone.RoutingMethod;
 import com.yahoo.config.provision.zone.ZoneId;
+import com.yahoo.path.Path;
+import com.yahoo.vespa.flags.Flags;
+import com.yahoo.vespa.flags.InMemoryFlagSource;
 import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions;
 import com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus;
 import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
 import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMetadata;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion;
-import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision;
 import com.yahoo.vespa.hosted.controller.api.integration.dns.Record;
-import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingEndpoint;
+import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData;
+import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName;
 import com.yahoo.vespa.hosted.controller.application.ApplicationPackage;
-import com.yahoo.vespa.hosted.controller.application.AssignedRotation;
 import com.yahoo.vespa.hosted.controller.application.Deployment;
 import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics;
+import com.yahoo.vespa.hosted.controller.application.Endpoint;
 import com.yahoo.vespa.hosted.controller.application.SystemApplication;
 import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder;
 import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext;
 import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester;
 import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock;
+import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb;
 import com.yahoo.vespa.hosted.controller.rotation.RotationId;
 import com.yahoo.vespa.hosted.controller.rotation.RotationLock;
 import org.junit.Test;
@@ -38,7 +45,6 @@ import java.time.Duration;
 import java.time.Instant;
 import java.util.Collection;
 import java.util.List;
-import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
 import java.util.function.Function;
@@ -176,51 +182,34 @@ public class ControllerTest {
 
     @Test
     public void testGlobalRotations() {
-        // Setup
-        ControllerTester tester = this.tester.controllerTester();
-        ZoneId zone = ZoneId.from(Environment.defaultEnvironment(), RegionName.defaultName());
-        ApplicationId app = ApplicationId.from("tenant", "app1", "default");
-        DeploymentId deployment = new DeploymentId(app, zone);
-        tester.serviceRegistry().routingGeneratorMock().putEndpoints(deployment, List.of(
-                new RoutingEndpoint("http://old-endpoint.vespa.yahooapis.com:4080", "host1", false, "upstream2"),
-                new RoutingEndpoint("http://qrs-endpoint.vespa.yahooapis.com:4080", "host1", false, "upstream1"),
-                new RoutingEndpoint("http://feeding-endpoint.vespa.yahooapis.com:4080", "host2", false, "upstream3"),
-                new RoutingEndpoint("http://global-endpoint-2.vespa.yahooapis.com:4080", "host2", true, "upstream4"),
-                new RoutingEndpoint("http://global-endpoint.vespa.yahooapis.com:4080", "host1", true, "upstream1"),
-                new RoutingEndpoint("http://alias-endpoint.vespa.yahooapis.com:4080", "host1", true, "upstream1")
-        ));
-
-        Supplier<Map<RoutingEndpoint, EndpointStatus>> globalRotationStatus = () -> tester.controller().applications().globalRotationStatus(deployment);
-        Supplier<List<EndpointStatus>> upstreamOneEndpoints = () -> {
-            return globalRotationStatus.get()
-                                       .entrySet().stream()
-                                       .filter(kv -> kv.getKey().upstreamName().equals("upstream1"))
-                                       .map(Map.Entry::getValue)
-                                       .collect(Collectors.toList());
-        };
+        var context = tester.newDeploymentContext();
+        var zone1 = ZoneId.from("prod", "us-west-1");
+        var zone2 = ZoneId.from("prod", "us-east-3");
+        var applicationPackage = new ApplicationPackageBuilder()
+                .region(zone1.region())
+                .region(zone2.region())
+                .endpoint("default", "default", zone1.region().value(), zone2.region().value())
+                .build();
+        context.submit(applicationPackage).deploy();
 
         // Check initial rotation status
-        assertEquals(3, globalRotationStatus.get().size());
-        assertEquals(2, upstreamOneEndpoints.get().size());
-        assertTrue("All upstreams are in", upstreamOneEndpoints.get().stream().allMatch(es -> es.getStatus() == EndpointStatus.Status.in));
-
-        // Set the global rotations out of service
-        EndpointStatus status = new EndpointStatus(EndpointStatus.Status.out, "unit-test", "Test", tester.clock().instant().getEpochSecond());
-        tester.controller().applications().setGlobalRotationStatus(deployment, status);
-        assertEquals(2, upstreamOneEndpoints.get().size());
-        assertTrue("All upstreams are out", upstreamOneEndpoints.get().stream().allMatch(es -> es.getStatus() == EndpointStatus.Status.out));
-        assertTrue("Reason is set", upstreamOneEndpoints.get().stream().allMatch(es -> es.getReason().equals("unit-test")));
-
-        // Deployment without a global endpoint
-        tester.serviceRegistry().routingGeneratorMock().putEndpoints(deployment, List.of(
-                new RoutingEndpoint("http://old-endpoint.vespa.yahooapis.com:4080", "host1", false, "upstream2"),
-                new RoutingEndpoint("http://qrs-endpoint.vespa.yahooapis.com:4080", "host1", false, "upstream1"),
-                new RoutingEndpoint("http://feeding-endpoint.vespa.yahooapis.com:4080", "host2", false, "upstream3")
-        ));
-        try {
-            tester.controller().applications().setGlobalRotationStatus(deployment, status);
-            fail("Expected exception");
-        } catch (IllegalArgumentException ignored) {}
+        var deployment1 = context.deploymentIdIn(zone1);
+        var status1 = tester.controller().routing().globalRotationStatus(deployment1);
+        assertEquals(1, status1.size());
+        assertTrue("All upstreams are in", status1.values().stream().allMatch(es -> es.getStatus() == EndpointStatus.Status.in));
+
+        // Set the deployment out of service in the global rotation
+        var newStatus = new EndpointStatus(EndpointStatus.Status.out, "unit-test", ControllerTest.class.getSimpleName(), tester.clock().instant().getEpochSecond());
+        tester.controller().routing().setGlobalRotationStatus(deployment1, newStatus);
+        status1 = tester.controller().routing().globalRotationStatus(deployment1);
+        assertEquals(1, status1.size());
+        assertTrue("All upstreams are out", status1.values().stream().allMatch(es -> es.getStatus() == EndpointStatus.Status.out));
+        assertTrue("Reason is set", status1.values().stream().allMatch(es -> es.getReason().equals("unit-test")));
+
+        // Other deployment remains in
+        var status2 = tester.controller().routing().globalRotationStatus(context.deploymentIdIn(zone2));
+        assertEquals(1, status2.size());
+        assertTrue("All upstreams are in", status2.values().stream().allMatch(es -> es.getStatus() == EndpointStatus.Status.in));
     }
 
     @Test
@@ -297,10 +286,10 @@ public class ControllerTest {
         var context = tester.newDeploymentContext("tenant1", "app1", "default");
         ApplicationPackage applicationPackage = new ApplicationPackageBuilder()
                 .environment(Environment.prod)
-                .endpoint("foobar", "qrs", "us-west-1", "us-central-1")
-                .endpoint("default", "qrs", "us-west-1", "us-central-1")
-                .endpoint("all", "qrs")
-                .endpoint("west", "qrs", "us-west-1")
+                .endpoint("foobar", "qrs", "us-west-1", "us-central-1")  // Rotation 01
+                .endpoint("default", "qrs", "us-west-1", "us-central-1") // Rotation 02
+                .endpoint("all", "qrs")                                  // Rotation 03
+                .endpoint("west", "qrs", "us-west-1")                    // Rotation 04
                 .region("us-west-1")
                 .region("us-central-1")
                 .build();
@@ -312,9 +301,9 @@ public class ControllerTest {
         var notWest = Set.of(
                 "rotation-id-01", "foobar--app1--tenant1.global.vespa.oath.cloud",
                 "rotation-id-02", "app1--tenant1.global.vespa.oath.cloud",
-                "rotation-id-04", "all--app1--tenant1.global.vespa.oath.cloud"
+                "rotation-id-03", "all--app1--tenant1.global.vespa.oath.cloud"
         );
-        var west = Sets.union(notWest, Set.of("rotation-id-03", "west--app1--tenant1.global.vespa.oath.cloud"));
+        var west = Sets.union(notWest, Set.of("rotation-id-04", "west--app1--tenant1.global.vespa.oath.cloud"));
 
         for (Deployment deployment : deployments) {
             assertEquals("Rotation names are passed to config server in " + deployment.zone(),
@@ -328,7 +317,7 @@ public class ControllerTest {
         var record1 = tester.controllerTester().findCname("app1--tenant1.global.vespa.oath.cloud");
         assertTrue(record1.isPresent());
         assertEquals("app1--tenant1.global.vespa.oath.cloud", record1.get().name().asString());
-        assertEquals("rotation-fqdn-04.", record1.get().data().asString());
+        assertEquals("rotation-fqdn-02.", record1.get().data().asString());
 
         var record2 = tester.controllerTester().findCname("foobar--app1--tenant1.global.vespa.oath.cloud");
         assertTrue(record2.isPresent());
@@ -338,12 +327,12 @@ public class ControllerTest {
         var record3 = tester.controllerTester().findCname("all--app1--tenant1.global.vespa.oath.cloud");
         assertTrue(record3.isPresent());
         assertEquals("all--app1--tenant1.global.vespa.oath.cloud", record3.get().name().asString());
-        assertEquals("rotation-fqdn-02.", record3.get().data().asString());
+        assertEquals("rotation-fqdn-03.", record3.get().data().asString());
 
         var record4 = tester.controllerTester().findCname("west--app1--tenant1.global.vespa.oath.cloud");
         assertTrue(record4.isPresent());
         assertEquals("west--app1--tenant1.global.vespa.oath.cloud", record4.get().name().asString());
-        assertEquals("rotation-fqdn-03.", record4.get().data().asString());
+        assertEquals("rotation-fqdn-04.", record4.get().data().asString());
     }
 
     @Test
@@ -475,23 +464,20 @@ public class ControllerTest {
                 .environment(Environment.prod)
                 .endpoint("default", "qrs", "us-west-1", "us-central-1")
                 .region("us-west-1")
-                .region("us-central-1") // Two deployments should result in each DNS alias being registered once
+                .region("us-central-1")
                 .build();
         context.submit(applicationPackage).deploy();
 
         ApplicationPackage applicationPackage2 = new ApplicationPackageBuilder()
                 .environment(Environment.prod)
                 .region("us-west-1")
-                .region("us-central-1") // Two deployments should result in each DNS alias being registered once
+                .region("us-central-1")
                 .allow(ValidationId.globalEndpointChange)
                 .build();
 
         context.submit(applicationPackage2).deploy();
 
-        assertEquals(
-                List.of(AssignedRotation.fromStrings("qrs", "default", "rotation-id-01", Set.of())),
-                context.instance().rotations()
-        );
+        assertEquals(List.of(), context.instance().rotations());
 
         assertEquals(
                 Set.of(),
@@ -528,9 +514,9 @@ public class ControllerTest {
             context.submit(applicationPackage);
             tester.applications().deleteApplication(context.application().id(),
                                                     tester.controllerTester().credentialsFor(context.application().id().tenant()));
-            try (RotationLock lock = tester.applications().rotationRepository().lock()) {
+            try (RotationLock lock = tester.controller().routing().rotations().lock()) {
                 assertTrue("Rotation is unassigned",
-                           tester.applications().rotationRepository().availableRotations(lock)
+                           tester.controller().routing().rotations().availableRotations(lock)
                                  .containsKey(new RotationId("rotation-id-01")));
             }
             context.flushDnsUpdates();
@@ -655,8 +641,6 @@ public class ControllerTest {
                                                         .region("us-west-1")
                                                         .region("us-east-3")
                                                         .build();
-        SourceRevision source = new SourceRevision("repo", "master", "commit1");
-
         context.submit(applicationPackage).deploy();
 
         DeploymentId deployment1 = context.deploymentIdIn(ZoneId.from(Environment.prod, RegionName.from("us-west-1")));
@@ -715,8 +699,12 @@ public class ControllerTest {
 
         // Create app1
         var context1 = tester.newDeploymentContext("tenant1", "app1", "default");
-        var applicationPackage = new ApplicationPackageBuilder().environment(Environment.prod)
-                                                                .region("us-west-1")
+        var prodZone = ZoneId.from("prod", "us-west-1");
+        tester.controllerTester().zoneRegistry().exclusiveRoutingIn(ZoneApiMock.from(prodZone));
+        var applicationPackage = new ApplicationPackageBuilder().athenzIdentity(AthenzDomain.from("domain"), AthenzService.from("service"))
+                                                                .compileVersion(RoutingController.DIRECT_ROUTING_MIN_VERSION)
+                                                                .environment(prodZone.environment())
+                                                                .region(prodZone.region())
                                                                 .build();
         // Deploy app1 in production
         context1.submit(applicationPackage).deploy();
@@ -725,13 +713,13 @@ public class ControllerTest {
         assertEquals(Stream.concat(Stream.of("vznqtz7a5ygwjkbhhj7ymxvlrekgt4l6g.vespa.oath.cloud",
                                              "app1.tenant1.global.vespa.oath.cloud",
                                              "*.app1.tenant1.global.vespa.oath.cloud"),
-                                   tester.controller().zoneRegistry().zones().all().ids().stream()
+                                   tester.controller().zoneRegistry().zones().controllerUpgraded().ids().stream()
                                          .flatMap(zone -> Stream.of("", "*.")
                                                                 .map(prefix -> prefix + "app1.tenant1." + zone.region().value() +
                                                                                (zone.environment() == Environment.prod ? "" :  "." + zone.environment().value()) +
                                                                                ".vespa.oath.cloud")))
                            .collect(Collectors.toUnmodifiableList()),
-                     tester.controllerTester().serviceRegistry().applicationCertificateMock().dnsNamesOf(context1.instanceId()));
+                     tester.controllerTester().serviceRegistry().endpointCertificateMock().dnsNamesOf(context1.instanceId()));
 
         // Next deployment reuses certificate
         context1.submit(applicationPackage).deploy();
@@ -739,13 +727,12 @@ public class ControllerTest {
 
         // Create app2
         var context2 = tester.newDeploymentContext("tenant1", "app2", "default");
-        ZoneId zone = ZoneId.from("dev", "us-east-1");
+        var devZone = ZoneId.from("dev", "us-east-1");
 
-        // Deploy app2, after "removing" direct routing everywhere
-        tester.controllerTester().zoneRegistry().setDirectlyRouted();
-        tester.controller().applications().deploy(context2.instanceId(), zone, Optional.of(applicationPackage), DeployOptions.none());
+        // Deploy app2 in a zone with shared routing
+        tester.controller().applications().deploy(context2.instanceId(), devZone, Optional.of(applicationPackage), DeployOptions.none());
         assertTrue("Application deployed and activated",
-                   tester.configServer().application(context2.instanceId(), zone).get().activated());
+                   tester.configServer().application(context2.instanceId(), devZone).get().activated());
         assertFalse("Does not provision certificate in zones with routing layer", certificate.apply(context2.instance()).isPresent());
     }
 
@@ -783,4 +770,195 @@ public class ControllerTest {
         }
     }
 
+    @Test
+    public void testDeployWithoutSourceRevision() {
+        var context = tester.newDeploymentContext();
+        var applicationPackage = new ApplicationPackageBuilder()
+                .upgradePolicy("default")
+                .environment(Environment.prod)
+                .region("us-west-1")
+                .build();
+
+        // Submit without source revision
+        context.submit(applicationPackage, Optional.empty())
+               .deploy();
+        assertEquals("Deployed application", 1, context.instance().deployments().size());
+    }
+
+    @Test
+    public void testDeployWithGlobalEndpointsAndMultipleRoutingMethods() {
+        var context = tester.newDeploymentContext();
+        var zone1 = ZoneId.from("prod", "us-west-1");
+        var zone2 = ZoneId.from("prod", "us-east-3");
+        var applicationPackage = new ApplicationPackageBuilder()
+                .athenzIdentity(AthenzDomain.from("domain"), AthenzService.from("service"))
+                .compileVersion(RoutingController.DIRECT_ROUTING_MIN_VERSION)
+                .endpoint("default", "default", zone1.region().value(), zone2.region().value())
+                .endpoint("east", "default", zone2.region().value())
+                .region(zone1.region())
+                .region(zone2.region())
+                .build();
+
+        // Zone 1 supports shared and sharedLayer4
+        tester.controllerTester().zoneRegistry().setRoutingMethod(ZoneApiMock.from(zone1), RoutingMethod.shared,
+                                                                  RoutingMethod.sharedLayer4);
+        // Zone 2 supports shared and exclusive
+        tester.controllerTester().zoneRegistry().setRoutingMethod(ZoneApiMock.from(zone2), RoutingMethod.shared,
+                                                                  RoutingMethod.exclusive);
+
+        context.submit(applicationPackage).deploy();
+        var expectedRecords = List.of(
+                // The 'east' global endpoint, pointing to zone 2 with exclusive routing
+                new Record(Record.Type.ALIAS,
+                           RecordName.from("east.application.tenant.global.vespa.oath.cloud"),
+                           RecordData.from("lb-0--tenant:application:default--prod.us-east-3/dns-zone-1/prod.us-east-3")),
+
+                // The 'default' global endpoint, pointing to both zones with shared routing, via rotation
+                new Record(Record.Type.CNAME,
+                           RecordName.from("application--tenant.global.vespa.oath.cloud"),
+                           RecordData.from("rotation-fqdn-01.")),
+
+                // The zone-scoped endpoint pointing to zone 2 with exclusive routing
+                new Record(Record.Type.CNAME,
+                           RecordName.from("application.tenant.us-east-3.vespa.oath.cloud"),
+                           RecordData.from("lb-0--tenant:application:default--prod.us-east-3.")),
+
+                // The 'east' global endpoint, pointing to zone 2 with shared routing, via rotation
+                new Record(Record.Type.CNAME,
+                           RecordName.from("east--application--tenant.global.vespa.oath.cloud"),
+                           RecordData.from("rotation-fqdn-02.")));
+        assertEquals(expectedRecords, List.copyOf(tester.controllerTester().nameService().records()));
+    }
+
+    @Test
+    public void testDirectRoutingSupport() {
+        var context = tester.newDeploymentContext();
+        var zone1 = ZoneId.from("prod", "us-west-1");
+        var zone2 = ZoneId.from("prod", "us-east-3");
+        var applicationPackageBuilder = new ApplicationPackageBuilder()
+                .region(zone1.region())
+                .region(zone2.region());
+        tester.controllerTester().zoneRegistry()
+              .setRoutingMethod(ZoneApiMock.from(zone1), RoutingMethod.shared, RoutingMethod.sharedLayer4)
+              .setRoutingMethod(ZoneApiMock.from(zone2), RoutingMethod.shared, RoutingMethod.sharedLayer4);
+        Supplier<Set<RoutingMethod>> routingMethods = () -> tester.controller().routing().endpointsOf(context.deploymentIdIn(zone1))
+                                                                  .asList()
+                                                                  .stream()
+                                                                  .map(Endpoint::routingMethod)
+                                                                  .collect(Collectors.toSet());
+        ((InMemoryFlagSource) tester.controller().flagSource()).withBooleanFlag(Flags.ALLOW_DIRECT_ROUTING.id(), false);
+
+        // Without everything
+        context.submit(applicationPackageBuilder.build()).deploy();
+        assertEquals(Set.of(RoutingMethod.shared), routingMethods.get());
+
+        // Without Athenz service
+        context.submit(applicationPackageBuilder.compileVersion(RoutingController.DIRECT_ROUTING_MIN_VERSION).build())
+               .deploy();
+        assertEquals(Set.of(RoutingMethod.shared), routingMethods.get());
+
+        // Without feature flag
+        applicationPackageBuilder = applicationPackageBuilder.compileVersion(RoutingController.DIRECT_ROUTING_MIN_VERSION)
+                                                             .athenzIdentity(AthenzDomain.from("domain"), AthenzService.from("service"));
+        context.submit(applicationPackageBuilder.build()).deploy();
+        assertEquals(Set.of(RoutingMethod.shared), routingMethods.get());
+
+        // With everything required
+        ((InMemoryFlagSource) tester.controller().flagSource()).withBooleanFlag(Flags.ALLOW_DIRECT_ROUTING.id(), true);
+        context.submit(applicationPackageBuilder.build()).deploy();
+        assertEquals(Set.of(RoutingMethod.shared, RoutingMethod.sharedLayer4), routingMethods.get());
+
+        // Global endpoint is configured and includes directly routed endpoint name
+        applicationPackageBuilder = applicationPackageBuilder.endpoint("default", "default");
+        context.submit(applicationPackageBuilder.build()).deploy();
+        for (var zone : List.of(zone1, zone2)) {
+            assertEquals(Set.of("rotation-id-01",
+                                "application.tenant.global.vespa.oath.cloud",
+                                "application--tenant.global.vespa.oath.cloud"),
+                         tester.configServer().rotationNames().get(context.deploymentIdIn(zone)));
+        }
+    }
+
+    @Test
+    public void testChangeEndpointCluster() {
+        var context = tester.newDeploymentContext();
+        var west = ZoneId.from("prod", "us-west-1");
+        var east = ZoneId.from("prod", "us-east-3");
+
+        // Deploy application
+        var applicationPackage = new ApplicationPackageBuilder()
+                .environment(Environment.prod)
+                .endpoint("default", "foo")
+                .region(west.region().value())
+                .region(east.region().value())
+                .build();
+        context.submit(applicationPackage).deploy();
+        assertEquals(ClusterSpec.Id.from("foo"), tester.applications().requireInstance(context.instanceId())
+                                                       .rotations().get(0).clusterId());
+
+        // Redeploy with endpoint cluster changed needs override
+        applicationPackage = new ApplicationPackageBuilder()
+                .environment(Environment.prod)
+                .endpoint("default", "bar")
+                .region(west.region().value())
+                .region(east.region().value())
+                .build();
+        try {
+            context.submit(applicationPackage).deploy();
+            fail("Expected exception");
+        } catch (IllegalArgumentException e) {
+            assertEquals("global-endpoint-change: application 'tenant.application' has endpoints [endpoint " +
+                         "'default' (cluster foo) -> us-east-3, us-west-1], but does not include all of these in " +
+                         "deployment.xml. Deploying given deployment.xml will remove " +
+                         "[endpoint 'default' (cluster foo) -> us-east-3, us-west-1] and add " +
+                         "[endpoint 'default' (cluster bar) -> us-east-3, us-west-1]. To allow this add " +
+                         "<allow until='yyyy-mm-dd'>global-endpoint-change</allow> to validation-overrides.xml, see " +
+                         "https://docs.vespa.ai/documentation/reference/validation-overrides.html", e.getMessage());
+        }
+
+        // Redeploy with override succeeds
+        applicationPackage = new ApplicationPackageBuilder()
+                .environment(Environment.prod)
+                .endpoint("default", "bar")
+                .region(west.region().value())
+                .region(east.region().value())
+                .allow(ValidationId.globalEndpointChange)
+                .build();
+        context.submit(applicationPackage).deploy();
+        assertEquals(ClusterSpec.Id.from("bar"), tester.applications().requireInstance(context.instanceId())
+                                                       .rotations().get(0).clusterId());
+    }
+
+    @Test
+    public void testReadableApplications() {
+        var db = new MockCuratorDb();
+        var tester = new DeploymentTester(new ControllerTester(db));
+
+        // Create and deploy two applications
+        var app1 = tester.newDeploymentContext("t1", "a1", "default")
+                         .submit()
+                         .deploy();
+        var app2 = tester.newDeploymentContext("t2", "a2", "default")
+                         .submit()
+                         .deploy();
+        assertEquals(2, tester.applications().readable().size());
+
+        // Write invalid data to one application
+        db.curator().set(Path.fromString("/controller/v1/applications/" + app2.application().id().serialized()),
+                         new byte[]{(byte) 0xDE, (byte) 0xAD});
+
+        // Can read the remaining readable
+        assertEquals(1, tester.applications().readable().size());
+
+        // Unconditionally reading all applications fails
+        try {
+            tester.applications().asList();
+            fail("Expected exception");
+        } catch (Exception ignored) {
+        }
+
+        // Deployment for readable application still succeeds
+        app1.submit().deploy();
+    }
+
 }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java
index c83463bc1ea..775eb2a4d75 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java
@@ -33,6 +33,7 @@ import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId;
 import com.yahoo.vespa.hosted.controller.athenz.impl.AthenzFacade;
 import com.yahoo.vespa.hosted.controller.integration.ConfigServerMock;
 import com.yahoo.vespa.hosted.controller.integration.MetricsMock;
+import com.yahoo.vespa.hosted.controller.integration.SecretStoreMock;
 import com.yahoo.vespa.hosted.controller.integration.ServiceRegistryMock;
 import com.yahoo.vespa.hosted.controller.integration.ZoneRegistryMock;
 import com.yahoo.vespa.hosted.controller.persistence.CuratorDb;
@@ -59,7 +60,6 @@ import java.util.OptionalLong;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.function.Consumer;
-import java.util.function.Supplier;
 import java.util.logging.Handler;
 import java.util.logging.Logger;
 
@@ -96,6 +96,10 @@ public final class ControllerTester {
              new ServiceRegistryMock());
     }
 
+    public ControllerTester(ServiceRegistryMock serviceRegistryMock) {
+        this(new AthenzDbMock(), new MockCuratorDb(), defaultRotationsConfig(), serviceRegistryMock);
+    }
+
     public ControllerTester(RotationsConfig rotationsConfig) {
         this(rotationsConfig, new MockCuratorDb());
     }
@@ -160,7 +164,9 @@ public final class ControllerTester {
 
     public AthenzDbMock athenzDb() { return athenzDb; }
 
-    public MemoryNameService nameService() { return serviceRegistry.nameServiceMock(); }
+    public MemoryNameService nameService() {
+        return serviceRegistry.nameService();
+    }
 
     public ZoneRegistryMock zoneRegistry() { return serviceRegistry.zoneRegistry(); }
 
@@ -188,27 +194,6 @@ public final class ControllerTester {
         controller = createController(curator, rotationsConfig, athenzDb, serviceRegistry);
     }
 
-    /** Creates the given tenant and application and deploys it */
-    public void createAndDeploy(String tenantName, String domainName, String applicationName, Environment environment, long projectId, Long propertyId) {
-        createAndDeploy(tenantName, domainName, applicationName, toZone(environment), projectId, propertyId);
-    }
-
-    /** Creates the given tenant and application and deploys it */
-    public void createAndDeploy(String tenantName, String domainName, String applicationName,
-                                    String instanceName, ZoneId zone, long projectId, Long propertyId) {
-        throw new AssertionError("Not supposed to use this");
-    }
-
-    /** Creates the given tenant and application and deploys it */
-    public void createAndDeploy(String tenantName, String domainName, String applicationName, ZoneId zone, long projectId, Long propertyId) {
-        createAndDeploy(tenantName, domainName, applicationName, "default", zone, projectId, propertyId);
-    }
-
-    /** Creates the given tenant and application and deploys it */
-    public void createAndDeploy(String tenantName, String domainName, String applicationName, Environment environment, long projectId) {
-        createAndDeploy(tenantName, domainName, applicationName, environment, projectId, null);
-    }
-
     /** Upgrade controller to given version */
     public void upgradeController(Version version, String commitSha, Instant commitDate) {
         for (var hostname : controller().curator().cluster()) {
@@ -308,9 +293,8 @@ public final class ControllerTester {
         AthenzCredentials credentials = new AthenzCredentials(
                 new AthenzPrincipal(user), domain, new OktaIdentityToken("okta-identity-token"), new OktaAccessToken("okta-access-token"));
         controller().tenants().create(tenantSpec, credentials);
-        if (contact.isPresent())
-            controller().tenants().lockOrThrow(name, LockedTenant.Athenz.class, tenant ->
-                    controller().tenants().store(tenant.with(contact.get())));
+        contact.ifPresent(value -> controller().tenants().lockOrThrow(name, LockedTenant.Athenz.class, tenant ->
+                controller().tenants().store(tenant.with(value))));
         assertNotNull(controller().tenants().get(name));
         return name;
     }
@@ -322,20 +306,20 @@ public final class ControllerTester {
         return tenant;
     }
 
-    public Optional<Credentials> credentialsFor(TenantName tenantName) {
+    public Credentials credentialsFor(TenantName tenantName) {
         Tenant tenant = controller().tenants().require(tenantName);
 
         switch (tenant.type()) {
             case athenz:
-                return Optional.of(new AthenzCredentials(new AthenzPrincipal(new AthenzUser("user")),
+                return new AthenzCredentials(new AthenzPrincipal(new AthenzUser("user")),
                                                                              ((AthenzTenant) tenant).domain(),
                                                                              new OktaIdentityToken("okta-identity-token"),
-                                                                             new OktaAccessToken("okta-access-token")));
+                                                                             new OktaAccessToken("okta-access-token"));
             case cloud:
-                return Optional.of(new Credentials(new SimplePrincipal("dev")));
+                return new Credentials(new SimplePrincipal("dev"));
 
             default:
-                return Optional.empty();
+                throw new IllegalArgumentException("Unexpected tenant type '" + tenant.type() + "'");
         }
     }
 
@@ -356,14 +340,6 @@ public final class ControllerTester {
         return application;
     }
 
-    public void deploy(ApplicationId id, ZoneId zone) {
-        deploy(id, zone, new ApplicationPackage(new byte[0]));
-    }
-
-    public void deploy(ApplicationId id, ZoneId zone, ApplicationPackage applicationPackage) {
-        deploy(id, zone, applicationPackage, false);
-    }
-
     public void deploy(ApplicationId id, ZoneId zone, ApplicationPackage applicationPackage, boolean deployCurrentVersion) {
         deploy(id, zone, Optional.of(applicationPackage), deployCurrentVersion);
     }
@@ -379,10 +355,6 @@ public final class ControllerTester {
                                            new DeployOptions(false, version, false, deployCurrentVersion));
     }
 
-    public Supplier<Instance> application(ApplicationId application) {
-        return () -> controller().applications().requireInstance(application);
-    }
-
     private static Controller createController(CuratorDb curator, RotationsConfig rotationsConfig,
                                                AthenzDbMock athensDb,
                                                ServiceRegistryMock serviceRegistry) {
@@ -393,7 +365,7 @@ public final class ControllerTester {
                                                new InMemoryFlagSource(),
                                                new MockMavenRepository(),
                                                serviceRegistry,
-                                               new MetricsMock());
+                                               new MetricsMock(), new SecretStoreMock());
         // Calculate initial versions
         controller.updateVersionStatus(VersionStatus.compute(controller));
         return controller;
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/ClusterCostTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/ClusterCostTest.java
deleted file mode 100644
index 6df2d55d52b..00000000000
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/ClusterCostTest.java
+++ /dev/null
@@ -1,35 +0,0 @@
-// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.controller.application;
-
-import com.yahoo.config.provision.ClusterSpec;
-import org.junit.Assert;
-import org.junit.Test;
-
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * @author smorgrav
- */
-public class ClusterCostTest {
-
-    @Test
-    public void clusterCost() {
-        List<String> hostnames = new ArrayList<>();
-        hostnames.add("host1");
-        hostnames.add("host2");
-        ClusterInfo info = new ClusterInfo("test", 100, 10, 10, 10, ClusterSpec.Type.container, hostnames);
-        ClusterUtilization util = new ClusterUtilization(0.3, 0.2, 0.5, 0.1);
-        ClusterCost cost = new ClusterCost(info, util);
-
-        // CPU is fully utilized
-        Assert.assertEquals(200, cost.getTco(), Double.MIN_VALUE);
-        Assert.assertEquals(0, cost.getWaste(), Double.MIN_VALUE);
-
-        // Set Disk as the most utilized resource
-        util = new ClusterUtilization(0.3, 0.1, 0.5, 0.1);
-        cost = new ClusterCost(info, util);
-        Assert.assertEquals(200, cost.getTco(), Double.MIN_NORMAL); // TCO is independent of utilization
-        Assert.assertEquals(57.1428571429, cost.getWaste(), 0.001); // Waste is not independent
-    }
-}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/ClusterUtilizationTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/ClusterUtilizationTest.java
deleted file mode 100644
index c930978eb1e..00000000000
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/ClusterUtilizationTest.java
+++ /dev/null
@@ -1,29 +0,0 @@
-// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.controller.application;
-
-import org.junit.Assert;
-import org.junit.Test;
-
-/**
- * @author smorgrav
- */
-public class ClusterUtilizationTest {
-
-    private static final double delta = Double.MIN_NORMAL;
-
-    @Test
-    public void getMaxUtilization() {
-        ClusterUtilization resources = new ClusterUtilization(0.3, 0.1, 0.4, 0.5);
-        Assert.assertEquals(0.5, resources.getMaxUtilization(), delta);
-
-        resources = new ClusterUtilization(0.3, 0.1, 0.4, 0.0);
-        Assert.assertEquals(0.4, resources.getMaxUtilization(), delta);
-
-        resources = new ClusterUtilization(0.4, 0.3, 0.3, 0.0);
-        Assert.assertEquals(0.4, resources.getMaxUtilization(), delta);
-
-        resources = new ClusterUtilization(0.1, 0.3, 0.3, 0.0);
-        Assert.assertEquals(0.3, resources.getMaxUtilization(), delta);
-    }
-
-}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/DeploymentCostTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/DeploymentCostTest.java
deleted file mode 100644
index 2e58253d768..00000000000
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/DeploymentCostTest.java
+++ /dev/null
@@ -1,38 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.controller.application;
-
-import com.yahoo.config.provision.ClusterSpec;
-import org.junit.Assert;
-import org.junit.Test;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-/**
- * @author smorgrav
- */
-public class DeploymentCostTest {
-
-    @Test
-    public void deploymentCost() {
-        Map<String, ClusterCost> clusters = new HashMap<>();
-        clusters.put("cluster1", createClusterCost(100, 0.2));
-        clusters.put("cluster2", createClusterCost(50, 0.1));
-
-        DeploymentCost cost = new DeploymentCost(clusters);
-        Assert.assertEquals(300, cost.getTco(), Double.MIN_VALUE); // 2*100 + 2*50
-        Assert.assertEquals(28.5714285714, cost.getWaste(), 0.001); // from cluster2
-        Assert.assertEquals(0.7142857142857143, cost.getUtilization(), Double.MIN_VALUE); // from cluster2
-    }
-
-    private ClusterCost createClusterCost(int flavorCost, double cpuUtil) {
-        List<String> hostnames = new ArrayList<>();
-        hostnames.add("host1");
-        hostnames.add("host2");
-        ClusterInfo info = new ClusterInfo("test", flavorCost, 10, 10, 10, ClusterSpec.Type.container, hostnames);
-        ClusterUtilization util = new ClusterUtilization(0.3, cpuUtil, 0.5, 0.1);
-        return new ClusterCost(info, util);
-    }
-}
\ No newline at end of file
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java
index ea97e3e6c71..f968b8c76d7 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java
@@ -4,7 +4,9 @@ package com.yahoo.vespa.hosted.controller.application;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.config.provision.SystemName;
+import com.yahoo.config.provision.zone.RoutingMethod;
 import com.yahoo.config.provision.zone.ZoneId;
+import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
 import com.yahoo.vespa.hosted.controller.application.Endpoint.Port;
 import org.junit.Test;
 
@@ -21,7 +23,7 @@ public class EndpointTest {
     private static final ApplicationId app2 = ApplicationId.from("t2", "a2", "i2");
 
     @Test
-    public void test_global_endpoints() {
+    public void global_endpoints() {
         EndpointId endpointId = EndpointId.defaultId();
 
         Map<String, Endpoint> tests = Map.of(
@@ -47,29 +49,29 @@ public class EndpointTest {
 
                 // Main endpoint with direct routing and default TLS port
                 "https://a1.t1.global.vespa.oath.cloud/",
-                Endpoint.of(app1).named(endpointId).on(Port.tls()).directRouting().in(SystemName.main),
+                Endpoint.of(app1).named(endpointId).on(Port.tls()).routingMethod(RoutingMethod.exclusive).in(SystemName.main),
 
                 // Main endpoint with custom rotation name
                 "https://r1.a1.t1.global.vespa.oath.cloud/",
-                Endpoint.of(app1).named(EndpointId.of("r1")).on(Port.tls()).directRouting().in(SystemName.main),
+                Endpoint.of(app1).named(EndpointId.of("r1")).on(Port.tls()).routingMethod(RoutingMethod.exclusive).in(SystemName.main),
 
                 // Main endpoint for custom instance in default rotation
                 "https://i2.a2.t2.global.vespa.oath.cloud/",
-                Endpoint.of(app2).named(endpointId).on(Port.tls()).directRouting().in(SystemName.main),
+                Endpoint.of(app2).named(endpointId).on(Port.tls()).routingMethod(RoutingMethod.exclusive).in(SystemName.main),
 
                 // Main endpoint for custom instance with custom rotation name
                 "https://r2.i2.a2.t2.global.vespa.oath.cloud/",
-                Endpoint.of(app2).named(EndpointId.of("r2")).on(Port.tls()).directRouting().in(SystemName.main),
+                Endpoint.of(app2).named(EndpointId.of("r2")).on(Port.tls()).routingMethod(RoutingMethod.exclusive).in(SystemName.main),
 
                 // Main endpoint in public system
                 "https://a1.t1.global.public.vespa.oath.cloud/",
-                Endpoint.of(app1).named(endpointId).on(Port.tls()).directRouting().in(SystemName.Public)
+                Endpoint.of(app1).named(endpointId).on(Port.tls()).routingMethod(RoutingMethod.exclusive).in(SystemName.Public)
         );
         tests.forEach((expected, endpoint) -> assertEquals(expected, endpoint.url().toString()));
     }
 
     @Test
-    public void test_global_endpoints_with_endpoint_id() {
+    public void global_endpoints_with_endpoint_id() {
         var endpointId = EndpointId.defaultId();
 
         Map<String, Endpoint> tests = Map.of(
@@ -95,29 +97,29 @@ public class EndpointTest {
 
                 // Main endpoint with direct routing and default TLS port
                 "https://a1.t1.global.vespa.oath.cloud/",
-                Endpoint.of(app1).named(endpointId).on(Port.tls()).directRouting().in(SystemName.main),
+                Endpoint.of(app1).named(endpointId).on(Port.tls()).routingMethod(RoutingMethod.exclusive).in(SystemName.main),
 
                 // Main endpoint with custom rotation name
                 "https://r1.a1.t1.global.vespa.oath.cloud/",
-                Endpoint.of(app1).named(EndpointId.of("r1")).on(Port.tls()).directRouting().in(SystemName.main),
+                Endpoint.of(app1).named(EndpointId.of("r1")).on(Port.tls()).routingMethod(RoutingMethod.exclusive).in(SystemName.main),
 
                 // Main endpoint for custom instance in default rotation
                 "https://i2.a2.t2.global.vespa.oath.cloud/",
-                Endpoint.of(app2).named(endpointId).on(Port.tls()).directRouting().in(SystemName.main),
+                Endpoint.of(app2).named(endpointId).on(Port.tls()).routingMethod(RoutingMethod.exclusive).in(SystemName.main),
 
                 // Main endpoint for custom instance with custom rotation name
                 "https://r2.i2.a2.t2.global.vespa.oath.cloud/",
-                Endpoint.of(app2).named(EndpointId.of("r2")).on(Port.tls()).directRouting().in(SystemName.main),
+                Endpoint.of(app2).named(EndpointId.of("r2")).on(Port.tls()).routingMethod(RoutingMethod.exclusive).in(SystemName.main),
 
                 // Main endpoint in public system
                 "https://a1.t1.global.public.vespa.oath.cloud/",
-                Endpoint.of(app1).named(endpointId).on(Port.tls()).directRouting().in(SystemName.Public)
+                Endpoint.of(app1).named(endpointId).on(Port.tls()).routingMethod(RoutingMethod.exclusive).in(SystemName.Public)
         );
         tests.forEach((expected, endpoint) -> assertEquals(expected, endpoint.url().toString()));
     }
 
     @Test
-    public void test_zone_endpoints() {
+    public void zone_endpoints() {
         var cluster = ClusterSpec.Id.from("default"); // Always default for non-direct routing
         var prodZone = ZoneId.from("prod", "us-north-1");
         var testZone = ZoneId.from("test", "us-north-2");
@@ -153,17 +155,21 @@ public class EndpointTest {
 
                 // Non-default cluster in public
                 "https://c1.a1.t1.us-north-1.public.vespa.oath.cloud/",
-                Endpoint.of(app1).target(ClusterSpec.Id.from("c1"), prodZone).on(Port.tls()).directRouting().in(SystemName.Public),
+                Endpoint.of(app1).target(ClusterSpec.Id.from("c1"), prodZone).on(Port.tls()).routingMethod(RoutingMethod.exclusive).in(SystemName.Public),
 
                 // Non-default cluster and instance in public
                 "https://c2.i2.a2.t2.us-north-1.public.vespa.oath.cloud/",
-                Endpoint.of(app2).target(ClusterSpec.Id.from("c2"), prodZone).on(Port.tls()).directRouting().in(SystemName.Public)
+                Endpoint.of(app2).target(ClusterSpec.Id.from("c2"), prodZone).on(Port.tls()).routingMethod(RoutingMethod.exclusive).in(SystemName.Public),
+
+                // Endpoint in main using shared layer 4
+                "https://a1.t1.us-north-1.vespa.oath.cloud/",
+                Endpoint.of(app1).target(cluster, prodZone).on(Port.tls()).routingMethod(RoutingMethod.sharedLayer4).in(SystemName.main)
         );
         tests.forEach((expected, endpoint) -> assertEquals(expected, endpoint.url().toString()));
     }
 
     @Test
-    public void test_wildcard_endpoints() {
+    public void wildcard_endpoints() {
         var defaultCluster = ClusterSpec.Id.from("default");
         var prodZone = ZoneId.from("prod", "us-north-1");
         var testZone = ZoneId.from("test", "us-north-2");
@@ -173,7 +179,7 @@ public class EndpointTest {
                 "https://a1.t1.global.public.vespa.oath.cloud/",
                 Endpoint.of(app1)
                         .named(EndpointId.defaultId())
-                        .directRouting()
+                        .routingMethod(RoutingMethod.exclusive)
                         .on(Port.tls())
                         .in(SystemName.Public),
 
@@ -181,7 +187,7 @@ public class EndpointTest {
                 "https://*.a1.t1.global.public.vespa.oath.cloud/",
                 Endpoint.of(app1)
                         .wildcard()
-                        .directRouting()
+                        .routingMethod(RoutingMethod.exclusive)
                         .on(Port.tls())
                         .in(SystemName.Public),
 
@@ -189,7 +195,7 @@ public class EndpointTest {
                 "https://a1.t1.us-north-1.public.vespa.oath.cloud/",
                 Endpoint.of(app1)
                         .target(defaultCluster, prodZone)
-                        .directRouting()
+                        .routingMethod(RoutingMethod.exclusive)
                         .on(Port.tls())
                         .in(SystemName.Public),
 
@@ -197,7 +203,7 @@ public class EndpointTest {
                 "https://*.a1.t1.us-north-1.public.vespa.oath.cloud/",
                 Endpoint.of(app1)
                         .wildcard(prodZone)
-                        .directRouting()
+                        .routingMethod(RoutingMethod.exclusive)
                         .on(Port.tls())
                         .in(SystemName.Public),
 
@@ -205,7 +211,7 @@ public class EndpointTest {
                 "https://a1.t1.us-north-2.test.public.vespa.oath.cloud/",
                 Endpoint.of(app1)
                         .target(defaultCluster, testZone)
-                        .directRouting()
+                        .routingMethod(RoutingMethod.exclusive)
                         .on(Port.tls())
                         .in(SystemName.Public),
 
@@ -213,11 +219,37 @@ public class EndpointTest {
                 "https://*.a1.t1.us-north-2.test.public.vespa.oath.cloud/",
                 Endpoint.of(app1)
                         .wildcard(testZone)
-                        .directRouting()
+                        .routingMethod(RoutingMethod.exclusive)
                         .on(Port.tls())
                         .in(SystemName.Public)
         );
 
         tests.forEach((expected, endpoint) -> assertEquals(expected, endpoint.url().toString()));
     }
+
+    @Test
+    public void upstream_name() {
+        var zone = ZoneId.from("prod", "us-north-1");
+        var tests1 = Map.of(
+                // With default cluster
+                "a1.t1.us-north-1.prod",
+                Endpoint.of(app1).named(EndpointId.defaultId()).on(Port.tls(4443)).in(SystemName.main),
+
+                // With non-default cluster
+                "c1.a1.t1.us-north-1.prod",
+                Endpoint.of(app1).named(EndpointId.of("c1")).on(Port.tls(4443)).in(SystemName.main)
+        );
+        var tests2 = Map.of(
+                // With non-default instance
+                "i2.a2.t2.us-north-1.prod",
+                Endpoint.of(app2).named(EndpointId.defaultId()).on(Port.tls(4443)).in(SystemName.main),
+
+                // With non-default instance and cluster
+                "c2.i2.a2.t2.us-north-1.prod",
+                Endpoint.of(app2).named(EndpointId.of("c2")).on(Port.tls(4443)).in(SystemName.main)
+        );
+        tests1.forEach((expected, endpoint) -> assertEquals(expected, endpoint.upstreamIdOf(new DeploymentId(app1, zone))));
+        tests2.forEach((expected, endpoint) -> assertEquals(expected, endpoint.upstreamIdOf(new DeploymentId(app2, zone))));
+    }
+
 }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManagerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManagerTest.java
new file mode 100644
index 00000000000..d7bc73adf37
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManagerTest.java
@@ -0,0 +1,124 @@
+package com.yahoo.vespa.hosted.controller.certificate;
+
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.SystemName;
+import com.yahoo.config.provision.zone.ZoneId;
+import com.yahoo.security.KeyAlgorithm;
+import com.yahoo.security.KeyUtils;
+import com.yahoo.security.SignatureAlgorithm;
+import com.yahoo.security.X509CertificateBuilder;
+import com.yahoo.security.X509CertificateUtils;
+import com.yahoo.vespa.flags.Flags;
+import com.yahoo.vespa.flags.InMemoryFlagSource;
+import com.yahoo.vespa.hosted.controller.Instance;
+import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMock;
+import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMetadata;
+import com.yahoo.vespa.hosted.controller.integration.SecretStoreMock;
+import com.yahoo.vespa.hosted.controller.integration.ZoneRegistryMock;
+import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb;
+import org.junit.Before;
+import org.junit.Test;
+
+import javax.security.auth.x500.X500Principal;
+import java.security.KeyPair;
+import java.security.cert.X509Certificate;
+import java.time.Clock;
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
+import java.util.List;
+import java.util.Optional;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * @author andreer
+ */
+public class EndpointCertificateManagerTest {
+
+    private final SecretStoreMock secretStore = new SecretStoreMock();
+    private final ZoneRegistryMock zoneRegistryMock = new ZoneRegistryMock(SystemName.main);
+    private final MockCuratorDb mockCuratorDb = new MockCuratorDb();
+    private final EndpointCertificateMock endpointCertificateMock = new EndpointCertificateMock();
+    private final InMemoryFlagSource inMemoryFlagSource = new InMemoryFlagSource();
+    private final Clock clock = Clock.systemUTC();
+    private final EndpointCertificateManager endpointCertificateManager =
+            new EndpointCertificateManager(zoneRegistryMock, mockCuratorDb, secretStore, endpointCertificateMock, clock, inMemoryFlagSource);
+
+    private static final KeyPair testKeyPair = KeyUtils.generateKeypair(KeyAlgorithm.EC, 192);
+    private static final X509Certificate testCertificate = X509CertificateBuilder
+            .fromKeypair(
+                    testKeyPair,
+                    new X500Principal("CN=test"),
+                    Instant.now(), Instant.now().plus(5, ChronoUnit.MINUTES),
+                    SignatureAlgorithm.SHA256_WITH_ECDSA,
+                    X509CertificateBuilder.generateRandomSerialNumber())
+            .addSubjectAlternativeName("vt2ktgkqme5zlnp4tj4ttyor7fj3v7q5o.vespa.oath.cloud")
+            .addSubjectAlternativeName("default.default.global.vespa.oath.cloud")
+            .addSubjectAlternativeName("*.default.default.global.vespa.oath.cloud")
+            .addSubjectAlternativeName("default.default.us-east-1.test.vespa.oath.cloud")
+            .addSubjectAlternativeName("*.default.default.us-east-1.test.vespa.oath.cloud")
+            .build();
+
+    private final Instance testInstance = new Instance(ApplicationId.defaultId());
+    private final String testKeyName = "testKeyName";
+    private final String testCertName = "testCertName";
+    private ZoneId testZone;
+
+    @Before
+    public void setUp() {
+        zoneRegistryMock.exclusiveRoutingIn(zoneRegistryMock.zones().all().zones());
+        testZone = zoneRegistryMock.zones().directlyRouted().zones().stream().findFirst().orElseThrow().getId();
+        inMemoryFlagSource.withBooleanFlag(Flags.VALIDATE_ENDPOINT_CERTIFICATES.id(), true);
+    }
+
+    @Test
+    public void provisions_new_certificate() {
+        Optional<EndpointCertificateMetadata> endpointCertificateMetadata = endpointCertificateManager.getEndpointCertificateMetadata(testInstance, testZone);
+        assertTrue(endpointCertificateMetadata.isPresent());
+        assertTrue(endpointCertificateMetadata.get().keyName().matches("vespa.tls.default.default.*-key"));
+        assertTrue(endpointCertificateMetadata.get().certName().matches("vespa.tls.default.default.*-cert"));
+        assertEquals(0, endpointCertificateMetadata.get().version());
+    }
+
+    @Test
+    public void reuses_stored_certificate_metadata() {
+        mockCuratorDb.writeEndpointCertificateMetadata(testInstance.id(), new EndpointCertificateMetadata(testKeyName, testCertName, 7));
+        secretStore.setSecret(testKeyName, KeyUtils.toPem(testKeyPair.getPrivate()), 7);
+        secretStore.setSecret(testCertName, X509CertificateUtils.toPem(testCertificate)+X509CertificateUtils.toPem(testCertificate), 7);
+        Optional<EndpointCertificateMetadata> endpointCertificateMetadata = endpointCertificateManager.getEndpointCertificateMetadata(testInstance, testZone);
+        assertTrue(endpointCertificateMetadata.isPresent());
+        assertEquals(testKeyName, endpointCertificateMetadata.get().keyName());
+        assertEquals(testCertName, endpointCertificateMetadata.get().certName());
+        assertEquals(7, endpointCertificateMetadata.get().version());
+    }
+
+    @Test
+    public void uses_refreshed_certificate_when_available_and_valid() {
+        inMemoryFlagSource.withBooleanFlag(Flags.USE_REFRESHED_ENDPOINT_CERTIFICATE.id(), true);
+
+        secretStore.setSecret(testKeyName, "secret-key", 7);
+        secretStore.setSecret(testCertName, "cert", 7);
+        secretStore.setSecret(testKeyName, KeyUtils.toPem(testKeyPair.getPrivate()), 8);
+        secretStore.setSecret(testKeyName, KeyUtils.toPem(testKeyPair.getPrivate()), 9);
+        secretStore.setSecret(testCertName, X509CertificateUtils.toPem(testCertificate)+X509CertificateUtils.toPem(testCertificate), 8);
+        mockCuratorDb.writeEndpointCertificateMetadata(testInstance.id(), new EndpointCertificateMetadata(testKeyName, testCertName, 7));
+        Optional<EndpointCertificateMetadata> endpointCertificateMetadata = endpointCertificateManager.getEndpointCertificateMetadata(testInstance, testZone);
+        assertTrue(endpointCertificateMetadata.isPresent());
+        assertEquals(testKeyName, endpointCertificateMetadata.get().keyName());
+        assertEquals(testCertName, endpointCertificateMetadata.get().certName());
+        assertEquals(8, endpointCertificateMetadata.get().version());
+    }
+
+    @Test
+    public void reprovisions_certificate_when_necessary() {
+        mockCuratorDb.writeEndpointCertificateMetadata(testInstance.id(), new EndpointCertificateMetadata(testKeyName, testCertName, -1, Optional.of("uuid"), Optional.of(List.of()), Optional.empty()));
+        secretStore.setSecret("vespa.tls.default.default.default-key", KeyUtils.toPem(testKeyPair.getPrivate()), 0);
+        secretStore.setSecret("vespa.tls.default.default.default-cert", X509CertificateUtils.toPem(testCertificate)+X509CertificateUtils.toPem(testCertificate), 0);
+        Optional<EndpointCertificateMetadata> endpointCertificateMetadata = endpointCertificateManager.getEndpointCertificateMetadata(testInstance, testZone);
+        assertTrue(endpointCertificateMetadata.isPresent());
+        assertEquals(0, endpointCertificateMetadata.get().version());
+        assertEquals(endpointCertificateMetadata, mockCuratorDb.readEndpointCertificateMetadata(testInstance.id()));
+    }
+
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java
index 9b0706d184f..2f2b1fbb364 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java
@@ -1,17 +1,23 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.deployment;
 
+import com.yahoo.component.Version;
 import com.yahoo.config.application.api.ValidationId;
 import com.yahoo.config.provision.AthenzDomain;
 import com.yahoo.config.provision.AthenzService;
 import com.yahoo.config.provision.Environment;
 import com.yahoo.config.provision.RegionName;
+import com.yahoo.security.SignatureAlgorithm;
+import com.yahoo.security.X509CertificateBuilder;
 import com.yahoo.security.X509CertificateUtils;
 import com.yahoo.vespa.hosted.controller.application.ApplicationPackage;
 
+import javax.security.auth.x500.X500Principal;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.UncheckedIOException;
+import java.security.KeyPairGenerator;
+import java.security.NoSuchAlgorithmException;
 import java.security.cert.X509Certificate;
 import java.text.SimpleDateFormat;
 import java.time.Duration;
@@ -52,6 +58,7 @@ public class ApplicationPackageBuilder {
     private String searchDefinition = "search test { }";
     private boolean explicitSystemTest = false;
     private boolean explicitStagingTest = false;
+    private Version compileVersion = Version.fromString("6.1");
 
     public ApplicationPackageBuilder majorVersion(int majorVersion) {
         this.majorVersion = OptionalInt.of(majorVersion);
@@ -159,6 +166,11 @@ public class ApplicationPackageBuilder {
         return this;
     }
 
+    public ApplicationPackageBuilder compileVersion(Version version) {
+        compileVersion = version;
+        return this;
+    }
+
     public ApplicationPackageBuilder athenzIdentity(AthenzDomain domain, AthenzService service) {
         this.athenzIdentityAttributes = String.format("athenz-domain='%s' athenz-service='%s'", domain.value(),
                                                       service.value());
@@ -186,6 +198,24 @@ public class ApplicationPackageBuilder {
         return this;
     }
 
+    public ApplicationPackageBuilder trustDefaultCertificate() {
+        try {
+            var generator = KeyPairGenerator.getInstance("RSA");
+            var builder = X509CertificateBuilder.fromKeypair(
+                    generator.generateKeyPair(),
+                    new X500Principal("CN=name"),
+                    Instant.now(),
+                    Instant.now().plusMillis(300_000),
+                    SignatureAlgorithm.SHA256_WITH_RSA,
+                    X509CertificateBuilder.generateRandomSerialNumber()
+            );
+            this.trustedCertificates.add(builder.build());
+        } catch (NoSuchAlgorithmException e) {
+            throw new RuntimeException(e);
+        }
+        return this;
+    }
+
     private byte[] deploymentSpec() {
         StringBuilder xml = new StringBuilder();
         xml.append("<deployment version='1.0' ");
@@ -201,11 +231,11 @@ public class ApplicationPackageBuilder {
             xml.append("'/>\n");
         }
         xml.append(notifications);
-        xml.append(blockChange);
         if (explicitSystemTest)
             xml.append("    <test />\n");
         if (explicitStagingTest)
             xml.append("    <staging />\n");
+        xml.append(blockChange);
         xml.append("    <");
         xml.append(environment.value());
         if (globalServiceId != null) {
@@ -237,8 +267,8 @@ public class ApplicationPackageBuilder {
         return searchDefinition.getBytes(UTF_8);
     }
 
-    private static byte[] buildMeta() {
-        return "{\"compileVersion\":\"6.1\",\"buildTime\":1000}".getBytes(UTF_8);
+    private static byte[] buildMeta(Version compileVersion) {
+        return ("{\"compileVersion\":\"" + compileVersion.toFullString() + "\",\"buildTime\":1000}").getBytes(UTF_8);
     }
 
     public ApplicationPackage build() {
@@ -262,7 +292,7 @@ public class ApplicationPackageBuilder {
             out.write(searchDefinition());
             out.closeEntry();
             out.putNextEntry(new ZipEntry(dir + "build-meta.json"));
-            out.write(buildMeta());
+            out.write(buildMeta(compileVersion));
             out.closeEntry();
             out.putNextEntry(new ZipEntry(dir + "security/clients.pem"));
             out.write(X509CertificateUtils.toPem(trustedCertificates).getBytes(UTF_8));
@@ -284,7 +314,7 @@ public class ApplicationPackageBuilder {
             out.write(deploymentXml.getBytes(UTF_8));
             out.closeEntry();
             out.putNextEntry(new ZipEntry("build-meta.json"));
-            out.write(buildMeta());
+            out.write(buildMeta(Version.fromString("6.1")));
             out.closeEntry();
         } catch (IOException e) {
             throw new UncheckedIOException(e);
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java
index 2792a59b523..a04dc6fb579 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java
@@ -6,12 +6,15 @@ import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.AthenzDomain;
 import com.yahoo.config.provision.AthenzService;
 import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.config.provision.Environment;
 import com.yahoo.config.provision.HostName;
 import com.yahoo.config.provision.zone.ZoneId;
 import com.yahoo.security.KeyAlgorithm;
 import com.yahoo.security.KeyUtils;
 import com.yahoo.security.SignatureAlgorithm;
 import com.yahoo.security.X509CertificateBuilder;
+import com.yahoo.vespa.flags.Flags;
+import com.yahoo.vespa.flags.InMemoryFlagSource;
 import com.yahoo.vespa.hosted.controller.Application;
 import com.yahoo.vespa.hosted.controller.ControllerTester;
 import com.yahoo.vespa.hosted.controller.Instance;
@@ -24,14 +27,14 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterId;
-import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingEndpoint;
-import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingGeneratorMock;
 import com.yahoo.vespa.hosted.controller.application.ApplicationPackage;
 import com.yahoo.vespa.hosted.controller.application.Deployment;
 import com.yahoo.vespa.hosted.controller.application.EndpointId;
 import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId;
 import com.yahoo.vespa.hosted.controller.integration.ConfigServerMock;
+import com.yahoo.vespa.hosted.controller.maintenance.JobControl;
 import com.yahoo.vespa.hosted.controller.maintenance.JobRunner;
+import com.yahoo.vespa.hosted.controller.maintenance.NameServiceDispatcher;
 import com.yahoo.vespa.hosted.controller.routing.GlobalRouting;
 import com.yahoo.vespa.hosted.controller.routing.RoutingPolicy;
 import com.yahoo.vespa.hosted.controller.routing.RoutingPolicyId;
@@ -42,8 +45,8 @@ import java.math.BigInteger;
 import java.net.URI;
 import java.security.KeyPair;
 import java.security.cert.X509Certificate;
+import java.time.Duration;
 import java.time.Instant;
-import java.util.Collections;
 import java.util.HashSet;
 import java.util.LinkedHashMap;
 import java.util.List;
@@ -90,13 +93,13 @@ public class DeploymentContext {
             .emailAddress("b@a")
             .trust(generateCertificate())
             .build();
+
     public static final SourceRevision defaultSourceRevision = new SourceRevision("repository1", "master", "commit1");
 
     private final TenantAndApplicationId applicationId;
     private final ApplicationId instanceId;
     private final TesterId testerId;
     private final JobController jobs;
-    private final RoutingGeneratorMock routing;
     private final JobRunner runner;
     private final DeploymentTester tester;
 
@@ -104,15 +107,14 @@ public class DeploymentContext {
     private boolean deferDnsUpdates = false;
 
     public DeploymentContext(ApplicationId instanceId, DeploymentTester tester) {
-
         this.applicationId = TenantAndApplicationId.from(instanceId);
         this.instanceId = instanceId;
         this.testerId = TesterId.of(instanceId);
         this.jobs = tester.controller().jobController();
         this.runner = tester.runner();
         this.tester = tester;
-        this.routing = tester.controllerTester().serviceRegistry().routingGeneratorMock();
         createTenantAndApplication();
+        ((InMemoryFlagSource) tester.controller().flagSource()).withBooleanFlag(Flags.ALLOW_DIRECT_ROUTING.id(), true);
     }
 
     private void createTenantAndApplication() {
@@ -190,6 +192,15 @@ public class DeploymentContext {
         return this;
     }
 
+    /** Defer provisioning of load balancers in zones in given environment */
+    public DeploymentContext deferLoadBalancerProvisioningIn(Environment... environment) {
+        return deferLoadBalancerProvisioningIn(Set.of(environment));
+    }
+
+    public DeploymentContext deferLoadBalancerProvisioningIn(Set<Environment> environments) {
+        configServer().deferLoadBalancerProvisioningIn(environments);
+        return this;
+    }
 
     /** Defer DNS updates */
     public DeploymentContext deferDnsUpdates() {
@@ -199,46 +210,45 @@ public class DeploymentContext {
 
     /** Flush all pending DNS updates */
     public DeploymentContext flushDnsUpdates() {
-        tester.nameServiceDispatcher().run();
+        flushDnsUpdates(Integer.MAX_VALUE);
         assertTrue("All name service requests dispatched",
                    tester.controller().curator().readNameServiceQueue().requests().isEmpty());
         return this;
     }
 
-    /** Add a routing policy for this in given zone, with status set to active */
-    public DeploymentContext addRoutingPolicy(ZoneId zone, boolean active) {
-        return addRoutingPolicy(instanceId, zone, active);
-    }
-
-    /** Add a routing policy for tester instance of this in given zone, with status set to active */
-    public DeploymentContext addTesterRoutingPolicy(ZoneId zone, boolean active) {
-        return addRoutingPolicy(testerId.id(), zone, active);
+    /** Flush count pending DNS updates */
+    public DeploymentContext flushDnsUpdates(int count) {
+        var dispatcher = new NameServiceDispatcher(tester.controller(), Duration.ofDays(1),
+                                                   new JobControl(tester.controller().curator()), count);
+        dispatcher.run();
+        return this;
     }
 
-    private DeploymentContext addRoutingPolicy(ApplicationId instance, ZoneId zone, boolean active) {
-        var clusterId = "default" + (!active ? "-inactive" : "");
-        var id = new RoutingPolicyId(instance, ClusterSpec.Id.from(clusterId), zone);
-        var policies = new LinkedHashMap<>(tester.controller().curator().readRoutingPolicies(instance));
+    /** Add a routing policy for this in given zone, with status set to inactive */
+    public DeploymentContext addInactiveRoutingPolicy(ZoneId zone) {
+        var clusterId = "default-inactive";
+        var id = new RoutingPolicyId(instanceId, ClusterSpec.Id.from(clusterId), zone);
+        var policies = new LinkedHashMap<>(tester.controller().curator().readRoutingPolicies(instanceId));
         policies.put(id, new RoutingPolicy(id, HostName.from("lb-host"),
                                            Optional.empty(),
-                                           Set.of(EndpointId.of("c0")),
-                                           new Status(active, GlobalRouting.DEFAULT_STATUS)));
-        tester.controller().curator().writeRoutingPolicies(instance, policies);
+                                           Set.of(EndpointId.of("default")),
+                                           new Status(false, GlobalRouting.DEFAULT_STATUS)));
+        tester.controller().curator().writeRoutingPolicies(instanceId, policies);
         return this;
     }
 
     /** Submit given application package for deployment */
     public DeploymentContext submit(ApplicationPackage applicationPackage) {
-        return submit(applicationPackage, defaultSourceRevision);
+        return submit(applicationPackage, Optional.of(defaultSourceRevision));
     }
 
     /** Submit given application package for deployment */
-    public DeploymentContext submit(ApplicationPackage applicationPackage, SourceRevision sourceRevision) {
+    public DeploymentContext submit(ApplicationPackage applicationPackage, Optional<SourceRevision> sourceRevision) {
         var projectId = tester.controller().applications()
                               .requireApplication(applicationId)
                               .projectId()
                               .orElse(1000); // These are really set through submission, so just pick one if it hasn't been set.
-        lastSubmission = jobs.submit(applicationId, Optional.of(sourceRevision), Optional.of("a@b"), Optional.empty(),
+        lastSubmission = jobs.submit(applicationId, sourceRevision, Optional.of("a@b"), Optional.empty(),
                                      Optional.empty(), projectId, applicationPackage, new byte[0]);
         return this;
     }
@@ -279,7 +289,6 @@ public class DeploymentContext {
         runner.advance(currentRun(job));
         assertTrue(jobs.run(id).get().hasFailed());
         assertTrue(jobs.run(id).get().hasEnded());
-        doTeardown(job);
         return this;
     }
 
@@ -303,9 +312,6 @@ public class DeploymentContext {
                     throw new AssertionError("Job '" + run.id() + "' was run twice");
 
         assertFalse("Change should have no targets, but was " + instance().change(), instance().change().hasTargets());
-        if (!deferDnsUpdates) {
-            flushDnsUpdates();
-        }
         return this;
     }
 
@@ -331,8 +337,8 @@ public class DeploymentContext {
             if (job.type().environment().isManuallyDeployed())
                 return this;
         }
-        doTests(job);
-        doTeardown(job);
+        if (job.type().isTest())
+            doTests(job);
         return this;
     }
 
@@ -361,11 +367,10 @@ public class DeploymentContext {
         triggerJobs();
         RunId id = currentRun(job).id();
         doDeploy(job);
-        tester.clock().advance(InternalStepRunner.installationTimeout.plusSeconds(1));
+        tester.clock().advance(InternalStepRunner.Timeouts.of(tester.controller().system()).noNodesDown().plusSeconds(1));
         runner.advance(currentRun(job));
         assertTrue(jobs.run(id).get().hasFailed());
         assertTrue(jobs.run(id).get().hasEnded());
-        doTeardown(job);
         return this;
     }
 
@@ -376,19 +381,13 @@ public class DeploymentContext {
         RunId id = currentRun(job).id();
         doDeploy(job);
         doUpgrade(job);
-        tester.clock().advance(InternalStepRunner.installationTimeout.plusSeconds(1));
+        tester.clock().advance(InternalStepRunner.Timeouts.of(tester.controller().system()).noNodesDown().plusSeconds(1));
         runner.advance(currentRun(job));
         assertTrue(jobs.run(id).get().hasFailed());
         assertTrue(jobs.run(id).get().hasEnded());
-        doTeardown(job);
         return this;
     }
 
-    /** Sets a single endpoint in the routing layer for the instance in this */
-    public DeploymentContext setEndpoints(ZoneId zone) {
-        return setEndpoints(zone, false);
-    }
-
     /** Deploy default application package, start a run for that change and return its ID */
     public RunId newRun(JobType type) {
         submit();
@@ -409,12 +408,13 @@ public class DeploymentContext {
 
     /** Start tests in system test stage */
     public RunId startSystemTestTests() {
-        RunId id = newRun(JobType.systemTest);
+        var id = newRun(JobType.systemTest);
+        var testZone = JobType.systemTest.zone(tester.controller().system());
         runner.run();
-        configServer().convergeServices(instanceId, JobType.systemTest.zone(tester.controller().system()));
-        configServer().convergeServices(testerId.id(), JobType.systemTest.zone(tester.controller().system()));
-        setEndpoints(JobType.systemTest.zone(tester.controller().system()));
-        setTesterEndpoints(JobType.systemTest.zone(tester.controller().system()));
+        if ( ! deferDnsUpdates)
+            flushDnsUpdates();
+        configServer().convergeServices(instanceId, testZone);
+        configServer().convergeServices(testerId.id(), testZone);
         runner.run();
         assertEquals(unfinished, jobs.run(id).get().stepStatuses().get(Step.endTests));
         assertTrue(jobs.run(id).get().steps().get(Step.endTests).startTime().isPresent());
@@ -440,7 +440,10 @@ public class DeploymentContext {
         // First step is always a deployment.
         runner.advance(currentRun(job));
 
-        if ( ! job.type().environment().isManuallyDeployed())
+        if ( ! deferDnsUpdates)
+            flushDnsUpdates();
+
+        if (job.type().isTest())
             doInstallTester(job);
 
         if (job.type() == JobType.stagingTest) { // Do the initial deployment and installation of the real application.
@@ -448,13 +451,11 @@ public class DeploymentContext {
             Versions versions = currentRun(job).versions();
             tester.configServer().nodeRepository().doUpgrade(deployment, Optional.empty(), versions.sourcePlatform().orElse(versions.targetPlatform()));
             configServer().convergeServices(id.application(), zone);
-            setEndpoints(zone);
             runner.advance(currentRun(job));
             assertEquals(Step.Status.succeeded, jobs.run(id).get().stepStatuses().get(Step.installInitialReal));
 
             // All installation is complete and endpoints are ready, so setup may begin.
-            if (job.type().isDeployment())
-                assertEquals(Step.Status.succeeded, jobs.run(id).get().stepStatuses().get(Step.installInitialReal));
+            assertEquals(Step.Status.succeeded, jobs.run(id).get().stepStatuses().get(Step.installInitialReal));
             assertEquals(Step.Status.succeeded, jobs.run(id).get().stepStatuses().get(Step.installTester));
             assertEquals(Step.Status.succeeded, jobs.run(id).get().stepStatuses().get(Step.startStagingSetup));
 
@@ -486,34 +487,6 @@ public class DeploymentContext {
         return run;
     }
 
-    /** Sets a single endpoint in the routing layer for the tester instance in this */
-    private DeploymentContext setTesterEndpoints(ZoneId zone) {
-        return setEndpoints(zone, true);
-    }
-
-    /** Sets a single endpoint in the routing layer; this matches that required for the tester */
-    private DeploymentContext setEndpoints(ZoneId zone, boolean tester) {
-        var id = instanceId;
-        if (tester) {
-            id = testerId.id();
-        }
-        routing.putEndpoints(new DeploymentId(id, zone),
-                             Collections.singletonList(new RoutingEndpoint(String.format("https://%s--%s--%s.%s.%s.vespa:43",
-                                                                                         id.instance().value(),
-                                                                                         id.application().value(),
-                                                                                         id.tenant().value(),
-                                                                                         zone.region().value(),
-                                                                                         zone.environment().value()),
-                                                                           "host1",
-                                                                           true,
-                                                                           String.format("cluster1.%s.%s.%s.%s",
-                                                                                         id.application().value(),
-                                                                                         id.tenant().value(),
-                                                                                         zone.region().value(),
-                                                                                         zone.environment().value()))));
-        return this;
-    }
-
     /** Lets nodes converge on new application version. */
     private void doConverge(JobId job) {
         RunId id = currentRun(job).id();
@@ -521,14 +494,13 @@ public class DeploymentContext {
 
         assertEquals(unfinished, jobs.run(id).get().stepStatuses().get(Step.installReal));
         configServer().convergeServices(id.application(), zone);
-        setEndpoints(zone);
         runner.advance(currentRun(job));
         if (job.type().environment().isManuallyDeployed()) {
             assertEquals(Step.Status.succeeded, jobs.run(id).get().stepStatuses().get(Step.installReal));
             assertTrue(jobs.run(id).get().hasEnded());
             return;
         }
-        assertEquals(Step.Status.succeeded, jobs.run(id).get().stepStatuses().get(Step.installReal));
+        assertEquals("Status of " + id, Step.Status.succeeded, jobs.run(id).get().stepStatuses().get(Step.installReal));
     }
 
     /** Installs tester and starts tests. */
@@ -542,8 +514,7 @@ public class DeploymentContext {
         assertEquals(unfinished, jobs.run(id).get().stepStatuses().get(Step.installTester));
         configServer().convergeServices(TesterId.of(id.application()).id(), zone);
         runner.advance(currentRun(job));
-        assertEquals(unfinished, jobs.run(id).get().stepStatuses().get(Step.installTester));
-        setTesterEndpoints(zone);
+        assertEquals(succeeded, jobs.run(id).get().stepStatuses().get(Step.installTester));
         runner.advance(currentRun(job));
     }
 
@@ -567,16 +538,6 @@ public class DeploymentContext {
         assertTrue(configServer().nodeRepository().list(zone, TesterId.of(id.application()).id()).isEmpty());
     }
 
-    /** Removes endpoints from routing layer — always call this. */
-    private void doTeardown(JobId job) {
-        ZoneId zone = zone(job);
-        DeploymentId deployment = new DeploymentId(job.application(), zone);
-
-        if ( ! instance().deployments().containsKey(zone))
-            routing.removeEndpoints(deployment);
-        routing.removeEndpoints(new DeploymentId(TesterId.of(job.application()).id(), zone));
-    }
-
     private JobId jobId(JobType type) {
         return new JobId(instanceId, type);
     }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java
index 5b5c6b61357..589229b32d4 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java
@@ -2,7 +2,6 @@
 package com.yahoo.vespa.hosted.controller.deployment;
 
 import com.yahoo.config.provision.ApplicationId;
-import com.yahoo.config.provision.zone.ZoneId;
 import com.yahoo.log.LogLevel;
 import com.yahoo.test.ManualClock;
 import com.yahoo.vespa.hosted.controller.Application;
@@ -10,17 +9,14 @@ import com.yahoo.vespa.hosted.controller.ApplicationController;
 import com.yahoo.vespa.hosted.controller.Controller;
 import com.yahoo.vespa.hosted.controller.ControllerTester;
 import com.yahoo.vespa.hosted.controller.Instance;
-import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
 import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzDbMock;
-import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterId;
-import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingGeneratorMock;
 import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockTesterCloud;
+import com.yahoo.vespa.hosted.controller.application.ApplicationList;
 import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId;
 import com.yahoo.vespa.hosted.controller.integration.ConfigServerMock;
 import com.yahoo.vespa.hosted.controller.maintenance.JobControl;
 import com.yahoo.vespa.hosted.controller.maintenance.JobRunner;
 import com.yahoo.vespa.hosted.controller.maintenance.JobRunnerTest;
-import com.yahoo.vespa.hosted.controller.maintenance.NameServiceDispatcher;
 import com.yahoo.vespa.hosted.controller.maintenance.OutstandingChangeDeployer;
 import com.yahoo.vespa.hosted.controller.maintenance.ReadyJobsTrigger;
 import com.yahoo.vespa.hosted.controller.maintenance.Upgrader;
@@ -30,7 +26,6 @@ import java.time.Duration;
 import java.time.Instant;
 import java.time.ZoneOffset;
 import java.time.temporal.TemporalAdjusters;
-import java.util.Collections;
 import java.util.logging.Logger;
 
 import static org.junit.Assert.assertTrue;
@@ -48,20 +43,16 @@ public class DeploymentTester {
 
     public static final TenantAndApplicationId appId = TenantAndApplicationId.from("tenant", "application");
     public static final ApplicationId instanceId = appId.defaultInstance();
-    public static final TesterId testerId = TesterId.of(instanceId);
 
     private final ControllerTester tester;
     private final JobController jobs;
-    private final RoutingGeneratorMock routing;
     private final MockTesterCloud cloud;
     private final JobRunner runner;
     private final Upgrader upgrader;
     private final ReadyJobsTrigger readyJobsTrigger;
     private final OutstandingChangeDeployer outstandingChangeDeployer;
-    private final NameServiceDispatcher nameServiceDispatcher;
 
     public JobController jobs() { return jobs; }
-    public RoutingGeneratorMock routing() { return routing; }
     public MockTesterCloud cloud() { return cloud; }
     public JobRunner runner() { return runner; }
     public ConfigServerMock configServer() { return tester.configServer(); }
@@ -75,6 +66,7 @@ public class DeploymentTester {
     public Application application(TenantAndApplicationId id ) { return applications().requireApplication(id); }
     public Instance instance() { return instance(instanceId); }
     public Instance instance(ApplicationId id) { return applications().requireInstance(id); }
+    public DeploymentStatusList deploymentStatuses() { return jobs.deploymentStatuses(ApplicationList.from(applications().asList())); }
 
     public DeploymentTester() {
         this(new ControllerTester());
@@ -83,7 +75,6 @@ public class DeploymentTester {
     public DeploymentTester(ControllerTester controllerTester) {
         tester = controllerTester;
         jobs = tester.controller().jobController();
-        routing = tester.serviceRegistry().routingGeneratorMock();
         cloud = (MockTesterCloud) tester.controller().jobController().cloud();
         var jobControl = new JobControl(tester.controller().curator());
         runner = new JobRunner(tester.controller(), Duration.ofDays(1), jobControl,
@@ -92,9 +83,6 @@ public class DeploymentTester {
         upgrader.setUpgradesPerMinute(1); // Anything that makes it at least one for any maintenance period is fine.
         readyJobsTrigger = new ReadyJobsTrigger(tester.controller(), maintenanceInterval, jobControl);
         outstandingChangeDeployer = new OutstandingChangeDeployer(tester.controller(), maintenanceInterval, jobControl);
-        nameServiceDispatcher = new NameServiceDispatcher(tester.controller(), maintenanceInterval, jobControl,
-                                                          Integer.MAX_VALUE);
-        routing.putEndpoints(new DeploymentId(null, null), Collections.emptyList()); // Turn off default behaviour for the mock.
 
         // Get deployment job logs to stderr.
         Logger.getLogger("").setLevel(LogLevel.DEBUG);
@@ -112,10 +100,6 @@ public class DeploymentTester {
 
     public OutstandingChangeDeployer outstandingChangeDeployer() { return outstandingChangeDeployer; }
 
-    public NameServiceDispatcher nameServiceDispatcher() {
-        return nameServiceDispatcher;
-    }
-
     public DeploymentTester atMondayMorning() {
         return at(tester.clock().instant().atZone(ZoneOffset.UTC)
                         .with(TemporalAdjusters.previousOrSame(DayOfWeek.MONDAY))
@@ -148,13 +132,6 @@ public class DeploymentTester {
         return newDeploymentContext(tenantName, applicationName, instanceName).application();
     }
 
-    /**
-     * Sets a single endpoint in the routing mock; this matches that required for the tester.
-     */
-    public void setEndpoints(ApplicationId id, ZoneId zone) {
-        newDeploymentContext(id).setEndpoints(zone);
-    }
-
     /** Aborts and finishes all running jobs. */
     public void abortAll() {
         triggerJobs();
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java
index 656afda149d..2b5e8b3e91c 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java
@@ -3,13 +3,19 @@ package com.yahoo.vespa.hosted.controller.deployment;
 
 import com.yahoo.component.Version;
 import com.yahoo.config.provision.Environment;
+import com.yahoo.config.provision.SystemName;
+import com.yahoo.config.provision.zone.RoutingMethod;
 import com.yahoo.config.provision.zone.ZoneId;
+import com.yahoo.vespa.hosted.controller.ControllerTester;
 import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId;
 import com.yahoo.vespa.hosted.controller.application.ApplicationPackage;
 import com.yahoo.vespa.hosted.controller.application.Change;
+import com.yahoo.vespa.hosted.controller.application.SystemApplication;
+import com.yahoo.vespa.hosted.controller.integration.ServiceRegistryMock;
+import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock;
 import com.yahoo.vespa.hosted.controller.versions.VespaVersion;
 import org.junit.Test;
 
@@ -27,6 +33,8 @@ import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobTy
 import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionApNortheast2;
 import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionApSoutheast1;
 import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionAwsUsEast1a;
+import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionCdAwsUsEast1a;
+import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionCdUsCentral1;
 import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionEuWest1;
 import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionUsCentral1;
 import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionUsEast3;
@@ -57,7 +65,7 @@ import static org.junit.Assert.assertTrue;
  */
 public class DeploymentTriggerTest {
 
-    private final DeploymentTester tester = new DeploymentTester();
+    private DeploymentTester tester = new DeploymentTester();
 
     @Test
     public void testTriggerFailing() {
@@ -965,7 +973,7 @@ public class DeploymentTriggerTest {
         tester.triggerJobs();
         assertEquals(List.of(), tester.jobs().active());
 
-        tester.atMondayMorning().clock().advance(Duration.ofDays(5)); // Inside block window for second instance.
+        tester.atMondayMorning().clock().advance(Duration.ofDays(5)); // Inside revision block window for second, conservative instance.
         Version version = Version.fromString("8.1");
         tester.controllerTester().upgradeSystem(version);
         tester.upgrader().maintain();
@@ -973,16 +981,18 @@ public class DeploymentTriggerTest {
         assertEquals(Change.empty(), app2.instance().change());
         assertEquals(Change.empty(), app3.instance().change());
 
-        // Upgrade instance 1; a failure allows an application change to accompany the upgrade.
+        // Upgrade instance 1; a failure in any instance allows an application change to accompany the upgrade.
         // The new platform won't roll out to the conservative instance until the normal one is upgraded.
-        app1.failDeployment(systemTest);
+        app2.failDeployment(systemTest);
         app1.submit(applicationPackage);
         assertEquals(Change.of(version).with(app1.application().latestVersion().get()), app1.instance().change());
-        app1.runJob(systemTest)
-            .jobAborted(stagingTest)
+        app2.runJob(systemTest);
+        app1.jobAborted(stagingTest)
             .runJob(stagingTest)
             .runJob(productionUsWest1)
             .runJob(productionUsEast3);
+        app1.runJob(stagingTest);   // Tests with only the outstanding application change.
+        app2.runJob(systemTest);    // Tests with only the outstanding application change.
         tester.clock().advance(Duration.ofHours(2));
         app1.runJob(productionEuWest1);
         tester.clock().advance(Duration.ofHours(1));
@@ -997,9 +1007,7 @@ public class DeploymentTriggerTest {
         app1.runJob(testUsEast3);
         app1.runJob(productionApSoutheast1);
 
-        app2.runJob(systemTest)     // Testing outstanding revision.
-            .runJob(stagingTest);   // Testing outstanding revision.
-
+        // Confidence rises to high, for the new version, and instance 2 starts to upgrade.
         tester.controllerTester().computeVersionStatus();
         tester.upgrader().maintain();
         tester.outstandingChangeDeployer().run();
@@ -1009,14 +1017,13 @@ public class DeploymentTriggerTest {
         assertEquals(Change.of(version), app2.instance().change());
         assertEquals(Change.empty(), app3.instance().change());
 
-        app2.runJob(systemTest)     // Explicitly defined for this instance.
-            .runJob(stagingTest)    // Never completed successfully with just the upgrade.
+        app1.runJob(stagingTest);   // Never completed successfully with just the upgrade.
+        app2.runJob(systemTest)     // Never completed successfully with just the upgrade.
             .runJob(productionEuWest1)
-            .failDeployment(testEuWest1)
-            .runJob(systemTest);    // Testing outstanding revision with currently deployed (upgraded) platform.
+            .failDeployment(testEuWest1);
 
+        // Instance 2 failed the last job, and now exist block window, letting application change roll out with the upgrade.
         tester.clock().advance(Duration.ofDays(1)); // Leave block window for revisions.
-        app2.abortJob(testEuWest1);
         tester.upgrader().maintain();
         tester.outstandingChangeDeployer().run();
         assertEquals(0, tester.jobs().active().size());
@@ -1067,4 +1074,58 @@ public class DeploymentTriggerTest {
         assertEquals(Change.of(app.lastSubmission().get()), app.instance().change());
     }
 
+    @Test
+    public void mixedDirectAndPipelineJobsInProduction() {
+        ApplicationPackage cdPackage = new ApplicationPackageBuilder().region("cd-us-central-1")
+                                                                      .region("cd-aws-us-east-1a")
+                                                                      .build();
+        ServiceRegistryMock services = new ServiceRegistryMock();
+        var zones = List.of(ZoneApiMock.fromId("test.cd-us-central-1"),
+                            ZoneApiMock.fromId("staging.cd-us-central-1"),
+                            ZoneApiMock.fromId("prod.cd-us-central-1"),
+                            ZoneApiMock.fromId("prod.cd-aws-us-east-1a"));
+        services.zoneRegistry()
+                .setSystemName(SystemName.cd)
+                .setZones(zones)
+                .setRoutingMethod(zones, RoutingMethod.shared);
+        tester = new DeploymentTester(new ControllerTester(services));
+        tester.configServer().bootstrap(services.zoneRegistry().zones().all().ids(), SystemApplication.values());
+        tester.controllerTester().upgradeSystem(Version.fromString("6.1"));
+        tester.controllerTester().computeVersionStatus();
+        var app = tester.newDeploymentContext();
+
+        app.runJob(productionCdUsCentral1, cdPackage);
+        app.submit(cdPackage);
+        app.runJob(systemTest);
+        // Staging test requires unknown initial version, and is broken.
+        tester.controller().applications().deploymentTrigger().forceTrigger(app.instanceId(), productionCdUsCentral1, "user", false);
+        app.runJob(productionCdUsCentral1)
+           .abortJob(stagingTest) // Complete failing run.
+           .runJob(stagingTest)
+           .runJob(productionCdAwsUsEast1a);
+
+        app.runJob(productionCdUsCentral1, cdPackage);
+        var version = new Version("7.1");
+        tester.controllerTester().upgradeSystem(version);
+        tester.upgrader().maintain();
+        // System and staging tests both require unknown versions, and are broken.
+        tester.controller().applications().deploymentTrigger().forceTrigger(app.instanceId(), productionCdUsCentral1, "user", false);
+        app.runJob(productionCdUsCentral1)
+           .abortJob(systemTest)
+           .abortJob(stagingTest)
+           .runJob(systemTest)
+           .runJob(stagingTest)
+           .runJob(productionCdAwsUsEast1a);
+
+        app.runJob(productionCdUsCentral1, cdPackage);
+        app.submit(cdPackage);
+        app.runJob(systemTest);
+        // Staging test requires unknown initial version, and is broken.
+        tester.controller().applications().deploymentTrigger().forceTrigger(app.instanceId(), productionCdUsCentral1, "user", false);
+        app.runJob(productionCdUsCentral1)
+           .jobAborted(stagingTest)
+           .runJob(stagingTest)
+           .runJob(productionCdAwsUsEast1a);
+    }
+
 }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java
index ff66ab38d32..99325ff909d 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java
@@ -5,18 +5,19 @@ import com.google.common.collect.ImmutableList;
 import com.yahoo.component.Version;
 import com.yahoo.config.application.api.DeploymentSpec;
 import com.yahoo.config.provision.AthenzDomain;
+import com.yahoo.config.provision.AthenzService;
 import com.yahoo.config.provision.HostName;
 import com.yahoo.config.provision.NodeResources;
 import com.yahoo.config.provision.SystemName;
+import com.yahoo.config.provision.zone.RoutingMethod;
 import com.yahoo.config.provision.zone.ZoneId;
-import com.yahoo.slime.ArrayTraverser;
 import com.yahoo.slime.Inspector;
-import com.yahoo.vespa.config.SlimeUtils;
+import com.yahoo.slime.SlimeUtils;
+import com.yahoo.vespa.hosted.controller.RoutingController;
 import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings.ConfigChangeActions;
 import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings.RefeedAction;
 import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings.RestartAction;
 import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings.ServiceInfo;
-import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
 import com.yahoo.vespa.hosted.controller.api.integration.LogEntry;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node;
@@ -54,13 +55,13 @@ import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.app
 import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.publicCdApplicationPackage;
 import static com.yahoo.vespa.hosted.controller.deployment.DeploymentTester.instanceId;
 import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.deploymentFailed;
+import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.installationFailed;
 import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.failed;
 import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.succeeded;
 import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.unfinished;
 import static java.util.Collections.singletonList;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
 
 /**
  * @author jonmv
@@ -140,9 +141,6 @@ public class InternalStepRunnerTest {
         ZoneId zone = id.type().zone(system());
         HostName host = tester.configServer().hostFor(instanceId, zone);
 
-        tester.setEndpoints(app.testerId().id(), JobType.productionUsCentral1.zone(system()));
-        tester.runner().run();
-
         tester.configServer().setConfigChangeActions(new ConfigChangeActions(singletonList(new RestartAction("cluster",
                                                                                                              "container",
                                                                                                              "search",
@@ -163,18 +161,22 @@ public class InternalStepRunnerTest {
         tester.runner().run();
         assertEquals(unfinished, tester.jobs().run(id).get().stepStatuses().get(Step.installReal));
 
-        tester.clock().advance(InternalStepRunner.installationTimeout.plus(Duration.ofSeconds(1)));
+        tester.clock().advance(InternalStepRunner.Timeouts.of(system()).noNodesDown().plus(Duration.ofSeconds(1)));
         tester.runner().run();
-        assertEquals(RunStatus.error, tester.jobs().run(id).get().status());
+        assertEquals(installationFailed, tester.jobs().run(id).get().status());
     }
 
     @Test
     public void waitsForEndpointsAndTimesOut() {
         app.newRun(JobType.systemTest);
 
-        // Tester fails to show up for staging tests, and the real deployment for system tests.
-        tester.setEndpoints(app.testerId().id(), JobType.systemTest.zone(system()));
-        tester.setEndpoints(app.instanceId(), JobType.stagingTest.zone(system()));
+        // Tester endpoint fails to show up for staging tests, and the real deployment for system tests.
+        var testZone = JobType.systemTest.zone(system());
+        var stagingZone = JobType.stagingTest.zone(system());
+        tester.newDeploymentContext(app.testerId().id())
+              .deferLoadBalancerProvisioningIn(testZone.environment());
+        tester.newDeploymentContext(app.instanceId())
+              .deferLoadBalancerProvisioningIn(stagingZone.environment());
 
         tester.runner().run();
         tester.configServer().convergeServices(app.instanceId(), JobType.stagingTest.zone(system()));
@@ -185,10 +187,9 @@ public class InternalStepRunnerTest {
         tester.configServer().convergeServices(app.testerId().id(), JobType.stagingTest.zone(system()));
         tester.runner().run();
 
-        tester.clock().advance(InternalStepRunner.endpointTimeout.plus(Duration.ofSeconds(1)));
+        tester.clock().advance(InternalStepRunner.Timeouts.of(system()).endpoint().plus(Duration.ofSeconds(1)));
         tester.runner().run();
         assertEquals(failed, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installReal));
-        assertEquals(failed, tester.jobs().last(app.instanceId(), JobType.stagingTest).get().stepStatuses().get(Step.installTester));
     }
 
     @Test
@@ -200,21 +201,17 @@ public class InternalStepRunnerTest {
 
         // Node is down too long in system test, and no nodes go down in staging.
         tester.runner().run();
-        tester.setEndpoints(app.testerId().id(), JobType.systemTest.zone(system()));
         tester.configServer().setVersion(app.testerId().id(), JobType.systemTest.zone(system()), tester.controller().systemVersion());
         tester.configServer().convergeServices(app.testerId().id(), JobType.systemTest.zone(system()));
-        tester.setEndpoints(app.instanceId(), JobType.systemTest.zone(system()));
-        tester.setEndpoints(app.testerId().id(), JobType.stagingTest.zone(system()));
         tester.configServer().setVersion(app.testerId().id(), JobType.stagingTest.zone(system()), tester.controller().systemVersion());
         tester.configServer().convergeServices(app.testerId().id(), JobType.stagingTest.zone(system()));
-        tester.setEndpoints(app.instanceId(), JobType.stagingTest.zone(system()));
         tester.runner().run();
         assertEquals(succeeded, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installTester));
         assertEquals(succeeded, tester.jobs().last(app.instanceId(), JobType.stagingTest).get().stepStatuses().get(Step.installTester));
 
         Node systemTestNode = tester.configServer().nodeRepository().list(JobType.systemTest.zone(system()),
                                                                           app.instanceId()).iterator().next();
-        tester.clock().advance(InternalStepRunner.installationTimeout.minus(Duration.ofSeconds(1)));
+        tester.clock().advance(InternalStepRunner.Timeouts.of(system()).noNodesDown().minus(Duration.ofSeconds(1)));
         tester.configServer().nodeRepository().putByHostname(JobType.systemTest.zone(system()),
                                                              new Node.Builder(systemTestNode)
                                                                      .serviceState(Node.ServiceState.allowedDown)
@@ -229,7 +226,7 @@ public class InternalStepRunnerTest {
         assertEquals(unfinished, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installReal));
         assertEquals(failed, tester.jobs().last(app.instanceId(), JobType.stagingTest).get().stepStatuses().get(Step.installInitialReal));
 
-        tester.clock().advance(InternalStepRunner.installationTimeout.minus(Duration.ofSeconds(3)));
+        tester.clock().advance(InternalStepRunner.Timeouts.of(system()).nodesDown().minus(Duration.ofSeconds(3)));
         tester.runner().run();
         assertEquals(unfinished, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installReal));
 
@@ -243,13 +240,11 @@ public class InternalStepRunnerTest {
         app.newRun(JobType.systemTest);
         tester.runner().run();
         tester.configServer().convergeServices(app.instanceId(), JobType.systemTest.zone(system()));
-        tester.setEndpoints(app.instanceId(), JobType.systemTest.zone(system()));
         tester.runner().run();
         assertEquals(succeeded, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installReal));
         assertEquals(unfinished, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installTester));
 
         tester.applications().deactivate(app.instanceId(), JobType.systemTest.zone(system()));
-        tester.setEndpoints(app.testerId().id(), JobType.systemTest.zone(system()));
         tester.configServer().convergeServices(app.testerId().id(), JobType.systemTest.zone(system()));
         tester.runner().run();
         assertEquals(succeeded, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installTester));
@@ -260,15 +255,27 @@ public class InternalStepRunnerTest {
 
     @Test
     public void alternativeEndpointsAreDetected() {
-        app.newRun(JobType.systemTest);
-        tester.runner().run();;
-        tester.configServer().convergeServices(app.instanceId(), JobType.systemTest.zone(system()));
-        tester.configServer().convergeServices(app.testerId().id(), JobType.systemTest.zone(system()));
+        var systemTestZone =  JobType.systemTest.zone(system());
+        var stagingZone =  JobType.stagingTest.zone(system());
+        tester.controllerTester().zoneRegistry().exclusiveRoutingIn(ZoneApiMock.from(systemTestZone), ZoneApiMock.from(stagingZone));
+        var applicationPackage = new ApplicationPackageBuilder()
+                .athenzIdentity(AthenzDomain.from("domain"), AthenzService.from("service"))
+                .upgradePolicy("default")
+                .region("us-central-1")
+                .parallel("us-west-1", "us-east-3")
+                .compileVersion(RoutingController.DIRECT_ROUTING_MIN_VERSION)
+                .build();
+        app.submit(applicationPackage)
+           .triggerJobs();
+
+        tester.runner().run();
         assertEquals(unfinished, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installReal));
         assertEquals(unfinished, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installTester));
-        app.addRoutingPolicy(JobType.systemTest.zone(system()), true);
-        app.addTesterRoutingPolicy(JobType.systemTest.zone(system()), true);
-        tester.runner().run();;
+
+        app.flushDnsUpdates();
+        tester.configServer().convergeServices(app.instanceId(), JobType.systemTest.zone(system()));
+        tester.configServer().convergeServices(app.testerId().id(), JobType.systemTest.zone(system()));
+        tester.runner().run();
         assertEquals(succeeded, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installReal));
         assertEquals(succeeded, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installTester));
     }
@@ -314,15 +321,13 @@ public class InternalStepRunnerTest {
         RunId id = app.startSystemTestTests();
         tester.runner().run();
         assertEquals(unfinished, tester.jobs().run(id).get().stepStatuses().get(Step.endTests));
-        assertEquals(URI.create(tester.routing().endpoints(new DeploymentId(app.testerId().id(), JobType.systemTest.zone(system()))).get(0).endpoint()),
-                     tester.cloud().testerUrl());
+        var testZone = JobType.systemTest.zone(system());
         Inspector configObject = SlimeUtils.jsonToSlime(tester.cloud().config()).get();
         assertEquals(app.instanceId().serializedForm(), configObject.field("application").asString());
-        assertEquals(JobType.systemTest.zone(system()).value(), configObject.field("zone").asString());
+        assertEquals(testZone.value(), configObject.field("zone").asString());
         assertEquals(system().value(), configObject.field("system").asString());
-        assertEquals(1, configObject.field("endpoints").children());
-        assertEquals(1, configObject.field("endpoints").field(JobType.systemTest.zone(system()).value()).entries());
-        configObject.field("endpoints").field(JobType.systemTest.zone(system()).value()).traverse((ArrayTraverser) (__, endpoint) -> assertEquals(tester.routing().endpoints(new DeploymentId(instanceId, JobType.systemTest.zone(system()))).get(0).endpoint(), endpoint.asString()));
+        assertEquals(1, configObject.field("zoneEndpoints").children());
+        assertEquals(1, configObject.field("zoneEndpoints").field(testZone.value()).children());
 
         long lastId = tester.jobs().details(id).get().lastId().getAsLong();
         tester.cloud().add(new LogEntry(0, Instant.ofEpochMilli(123), info, "Ready!"));
@@ -368,7 +373,6 @@ public class InternalStepRunnerTest {
 
         tester.runner().run(); // Job run order determined by JobType enum order per application.
         tester.configServer().convergeServices(app.instanceId(), zone);
-        tester.setEndpoints(app.instanceId(), zone);
         assertEquals(unfinished, tester.jobs().run(id).get().stepStatuses().get(Step.installReal));
         assertEquals(applicationPackage.hash(), tester.configServer().application(app.instanceId(), zone).get().applicationPackage().hash());
         assertEquals(otherPackage.hash(), tester.configServer().application(app.instanceId(), JobType.perfUsEast3.zone(system())).get().applicationPackage().hash());
@@ -377,12 +381,6 @@ public class InternalStepRunnerTest {
         tester.runner().run();
         assertEquals(1, tester.jobs().active().size());
         assertEquals(version, tester.instance(app.instanceId()).deployments().get(zone).version());
-
-        try {
-            tester.jobs().deploy(app.instanceId(), JobType.productionApNortheast1, Optional.empty(), applicationPackage);
-            fail("Deployments outside dev should not be allowed.");
-        }
-        catch (IllegalArgumentException expected) { }
     }
 
     @Test
@@ -424,10 +422,13 @@ public class InternalStepRunnerTest {
 
     @Test
     public void certificateTimeoutAbortsJob() {
-        tester.controllerTester().zoneRegistry().setSystemName(SystemName.PublicCd);
-        tester.controllerTester().zoneRegistry().setZones(ZoneApiMock.fromId("test.aws-us-east-1c"),
-                                                          ZoneApiMock.fromId("staging.aws-us-east-1c"),
-                                                          ZoneApiMock.fromId("prod.aws-us-east-1c"));
+        tester.controllerTester().zoneRegistry().setSystemName(SystemName.Public);
+        var zones = List.of(ZoneApiMock.fromId("test.aws-us-east-1c"),
+                            ZoneApiMock.fromId("staging.aws-us-east-1c"),
+                            ZoneApiMock.fromId("prod.aws-us-east-1c"));
+        tester.controllerTester().zoneRegistry()
+              .setZones(zones)
+              .setRoutingMethod(zones, RoutingMethod.exclusive);
         tester.configServer().bootstrap(tester.controllerTester().zoneRegistry().zones().all().ids(), SystemApplication.values());
         RunId id = app.startSystemTestTests();
 
@@ -435,7 +436,7 @@ public class InternalStepRunnerTest {
         trusted.add(tester.jobs().run(id).get().testerCertificate().get());
         assertEquals(trusted, tester.configServer().application(app.instanceId(), id.type().zone(system())).get().applicationPackage().trustedCertificates());
 
-        tester.clock().advance(InternalStepRunner.certificateTimeout.plus(Duration.ofSeconds(1)));
+        tester.clock().advance(InternalStepRunner.Timeouts.of(system()).testerCertificate().plus(Duration.ofSeconds(1)));
         tester.runner().run();
         assertEquals(RunStatus.aborted, tester.jobs().run(id).get().status());
     }
@@ -451,11 +452,45 @@ public class InternalStepRunnerTest {
                                            "3554970337.947845\t17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\t5480\tcontainer\tstderr\twarning\tjava.lang.NullPointerException\\n\\tat org.apache.felix.framework.BundleRevisionImpl.calculateContentPath(BundleRevisionImpl.java:438)\\n\\tat org.apache.felix.framework.BundleRevisionImpl.initializeContentPath(BundleRevisionImpl.java:371)";
 
     @Test
+    public void generates_correct_tester_flavor() {
+        DeploymentSpec spec = DeploymentSpec.fromXml("<deployment version='1.0' athenz-domain='domain' athenz-service='service'>\n" +
+                                                     "    <instance id='first'>\n" +
+                                                     "        <test tester-flavor=\"d-6-16-100\" />\n" +
+                                                     "        <prod>\n" +
+                                                     "            <region active=\"true\">us-west-1</region>\n" +
+                                                     "            <test>us-west-1</test>\n" +
+                                                     "        </prod>\n" +
+                                                     "    </instance>\n" +
+                                                     "    <instance id='second'>\n" +
+                                                     "        <test />\n" +
+                                                     "        <staging />\n" +
+                                                     "        <prod tester-flavor=\"d-6-16-100\">\n" +
+                                                     "            <parallel>\n" +
+                                                     "                <region active=\"true\">us-east-3</region>\n" +
+                                                     "                <region active=\"true\">us-central-1</region>\n" +
+                                                     "            </parallel>\n" +
+                                                     "            <region active=\"true\">us-west-1</region>\n" +
+                                                     "            <test>us-west-1</test>\n" +
+                                                     "        </prod>\n" +
+                                                     "    </instance>\n" +
+                                                     "</deployment>\n");
+
+        NodeResources firstResources = InternalStepRunner.testerResourcesFor(ZoneId.from("prod", "us-west-1"), spec.requireInstance("first"));
+        assertEquals(InternalStepRunner.DEFAULT_TESTER_RESOURCES, firstResources);
+
+        NodeResources secondResources = InternalStepRunner.testerResourcesFor(ZoneId.from("prod", "us-west-1"), spec.requireInstance("second"));
+        assertEquals(6, secondResources.vcpu(), 1e-9);
+        assertEquals(16, secondResources.memoryGb(), 1e-9);
+        assertEquals(100, secondResources.diskGb(), 1e-9);
+    }
+
+    @Test
     public void generates_correct_services_xml_test() {
-        assertFile("test_runner_services.xml-cd", new String(InternalStepRunner.servicesXml(AthenzDomain.from("vespa.vespa.cd"),
-                                                                                            true,
-                                                                                            false,
-                                                                                            new NodeResources(2, 12, 75, 1, NodeResources.DiskSpeed.fast, NodeResources.StorageType.local))));
+        assertFile("test_runner_services.xml-cd",
+                   new String(InternalStepRunner.servicesXml(
+                           true,
+                           false,
+                           new NodeResources(2, 12, 75, 1, NodeResources.DiskSpeed.fast, NodeResources.StorageType.local))));
     }
 
     private void assertFile(String resourceName, String actualContent) {
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializerTest.java
index ce22e0772ff..6a12c4457db 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializerTest.java
@@ -1,11 +1,14 @@
 // Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.deployment;
 
+import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.config.provision.SystemName;
 import com.yahoo.config.provision.zone.ZoneId;
-import com.yahoo.vespa.config.SlimeUtils;
+import com.yahoo.slime.SlimeUtils;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
+import com.yahoo.vespa.hosted.controller.application.Endpoint;
+import com.yahoo.vespa.hosted.controller.application.EndpointId;
 import org.junit.Test;
 
 import java.io.IOException;
@@ -29,8 +32,10 @@ public class TestConfigSerializerTest {
         byte[] json = new TestConfigSerializer(SystemName.PublicCd).configJson(instanceId,
                                                                                JobType.systemTest,
                                                                                true,
-                                                                               Map.of(zone, Map.of(ClusterSpec.Id.from("ai"),
-                                                                                                   URI.create("https://server/"))),
+                                                                               Map.of(zone, List.of(Endpoint.of(ApplicationId.defaultId())
+                                                                                                            .named(EndpointId.of("ai"))
+                                                                                                            .on(Endpoint.Port.tls())
+                                                                                                            .in(SystemName.main))),
                                                                                Map.of(zone, List.of("facts")));
         byte[] expected = Files.readAllBytes(Paths.get("src/test/resources/testConfig.json"));
         assertEquals(new String(SlimeUtils.toJsonBytes(SlimeUtils.jsonToSlime(expected))),
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ArtifactRepositoryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ArtifactRepositoryMock.java
index a7b9cbd1e7e..23d21d2b51a 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ArtifactRepositoryMock.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ArtifactRepositoryMock.java
@@ -55,7 +55,7 @@ public class ArtifactRepositoryMock extends AbstractComponent implements Artifac
         return Objects.hash(applicationId, applicationVersion);
     }
 
-    private class Artifact {
+    private static class Artifact {
 
         private final byte[] data;
         private int hits = 0;
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java
index 66c7334dc2d..9e9d27fd744 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java
@@ -5,20 +5,23 @@ import com.google.inject.Inject;
 import com.yahoo.component.AbstractComponent;
 import com.yahoo.component.Version;
 import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.config.provision.DockerImage;
+import com.yahoo.config.provision.Environment;
 import com.yahoo.config.provision.HostName;
 import com.yahoo.config.provision.NodeResources;
 import com.yahoo.config.provision.NodeType;
 import com.yahoo.config.provision.zone.ZoneId;
 import com.yahoo.vespa.flags.json.FlagData;
 import com.yahoo.vespa.hosted.controller.api.application.v4.model.ClusterMetrics;
-import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions;
+import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeploymentData;
 import com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus;
 import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings.ConfigChangeActions;
 import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
 import com.yahoo.vespa.hosted.controller.api.identifiers.Hostname;
 import com.yahoo.vespa.hosted.controller.api.identifiers.Identifier;
 import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId;
-import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMetadata;
+import com.yahoo.vespa.hosted.controller.api.integration.LogEntry;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServer;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.ContainerEndpoint;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.LoadBalancer;
@@ -44,10 +47,12 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
+import java.util.UUID;
 import java.util.logging.Level;
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
@@ -68,8 +73,10 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer
     private final Map<DeploymentId, ServiceConvergence> serviceStatus = new HashMap<>();
     private final Set<ApplicationId> disallowConvergenceCheckApplications = new HashSet<>();
     private final Version initialVersion = new Version(6, 1, 0);
+    private final DockerImage initialDockerImage = DockerImage.fromString("dockerImage:6.1.0");
     private final Set<DeploymentId> suspendedApplications = new HashSet<>();
-    private final Map<ZoneId, List<LoadBalancer>> loadBalancers = new HashMap<>();
+    private final Map<ZoneId, Set<LoadBalancer>> loadBalancers = new HashMap<>();
+    private final Set<Environment> deferLoadBalancerProvisioning = new HashSet<>();
     private final Map<DeploymentId, List<Log>> warnings = new HashMap<>();
     private final Map<DeploymentId, Set<String>> rotationNames = new HashMap<>();
     private final Map<DeploymentId, List<ClusterMetrics>> clusterMetrics = new HashMap<>();
@@ -100,6 +107,8 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer
                                                                .parentHostname(parent.hostname())
                                                                .currentVersion(initialVersion)
                                                                .wantedVersion(initialVersion)
+                                                               .currentDockerImage(initialDockerImage)
+                                                               .wantedDockerImage(initialDockerImage)
                                                                .currentOsVersion(Version.emptyVersion)
                                                                .wantedOsVersion(Version.emptyVersion)
                                                                .resources(new NodeResources(2, 8, 50, 1, slow, remote))
@@ -243,6 +252,10 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer
         this.clusterMetrics.put(deployment, clusterMetrics);
     }
 
+    public void deferLoadBalancerProvisioningIn(Set<Environment> environments) {
+        deferLoadBalancerProvisioning.addAll(environments);
+    }
+
     @Override
     public NodeRepositoryMock nodeRepository() {
         return nodeRepository;
@@ -260,8 +273,8 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer
         disallowConvergenceCheckApplications.add(applicationId);
     }
 
-    private List<LoadBalancer> getLoadBalancers(ZoneId zone) {
-        return loadBalancers.getOrDefault(zone, Collections.emptyList());
+    private Set<LoadBalancer> getLoadBalancers(ZoneId zone) {
+        return loadBalancers.getOrDefault(zone, new LinkedHashSet<>());
     }
 
     @Override
@@ -281,8 +294,24 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer
         return TesterCloud.Status.SUCCESS;
     }
 
-    public void addLoadBalancers(ZoneId zone, List<LoadBalancer> loadBalancers) {
-        this.loadBalancers.putIfAbsent(zone, new ArrayList<>());
+    @Override
+    public String startTests(DeploymentId deployment, TesterCloud.Suite suite, byte[] config) {
+        return "Tests started";
+    }
+
+    @Override
+    public List<LogEntry> getTesterLog(DeploymentId deployment, long after) {
+        return List.of();
+    }
+
+    @Override
+    public boolean isTesterReady(DeploymentId deployment) {
+        return false;
+    }
+
+    /** Add any of given loadBalancers that do not already exist to the load balancers in zone */
+    public void putLoadBalancers(ZoneId zone, List<LoadBalancer> loadBalancers) {
+        this.loadBalancers.putIfAbsent(zone, new LinkedHashSet<>());
         this.loadBalancers.get(zone).addAll(loadBalancers);
     }
 
@@ -291,43 +320,51 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer
     }
 
     @Override
-    public PreparedApplication deploy(DeploymentId deployment, DeployOptions deployOptions,
-                                      Set<ContainerEndpoint> containerEndpoints,
-                                      Optional<EndpointCertificateMetadata> endpointCertificateMetadata, byte[] content) {
-        lastPrepareVersion = deployOptions.vespaVersion.map(Version::fromString).orElse(null);
+    public PreparedApplication deploy(DeploymentData deployment) {
+        lastPrepareVersion = deployment.platform();
         if (prepareException != null) {
             RuntimeException prepareException = this.prepareException;
             this.prepareException = null;
             throw prepareException;
         }
-        applications.put(deployment, new Application(deployment.applicationId(), lastPrepareVersion, new ApplicationPackage(content)));
+        DeploymentId id = new DeploymentId(deployment.instance(), deployment.zone());
+        applications.put(id, new Application(id.applicationId(), lastPrepareVersion, new ApplicationPackage(deployment.applicationPackage())));
 
-        if (nodeRepository().list(deployment.zoneId(), deployment.applicationId()).isEmpty())
-            provision(deployment.zoneId(), deployment.applicationId());
+        if (nodeRepository().list(id.zoneId(), id.applicationId()).isEmpty())
+            provision(id.zoneId(), id.applicationId());
 
         this.rotationNames.put(
-                deployment,
-                containerEndpoints.stream()
-                                  .map(ContainerEndpoint::names)
-                                  .flatMap(Collection::stream)
-                                  .collect(Collectors.toSet())
+                id,
+                deployment.containerEndpoints().stream()
+                          .map(ContainerEndpoint::names)
+                          .flatMap(Collection::stream)
+                          .collect(Collectors.toSet())
         );
 
+        if (!deferLoadBalancerProvisioning.contains(id.zoneId().environment())) {
+            putLoadBalancers(id.zoneId(), List.of(new LoadBalancer(UUID.randomUUID().toString(),
+                                                                   id.applicationId(),
+                                                                   ClusterSpec.Id.from("default"),
+                                                                   HostName.from("lb-0--" + id.applicationId().serializedForm() + "--" + id.zoneId().toString()),
+                                                                   LoadBalancer.State.active,
+                                                                   Optional.of("dns-zone-1"))));
+        }
+
         return () -> {
-            Application application = applications.get(deployment);
+            Application application = applications.get(id);
             application.activate();
-            List<Node> nodes = nodeRepository.list(deployment.zoneId(), deployment.applicationId());
+            List<Node> nodes = nodeRepository.list(id.zoneId(), id.applicationId());
             for (Node node : nodes) {
-                nodeRepository.putByHostname(deployment.zoneId(), new Node.Builder(node)
+                nodeRepository.putByHostname(id.zoneId(), new Node.Builder(node)
                         .state(Node.State.active)
                         .wantedVersion(application.version().get())
                         .build());
             }
-            serviceStatus.put(deployment, new ServiceConvergence(deployment.applicationId(),
-                                                                 deployment.zoneId(),
-                                                                 false,
-                                                                 2,
-                                                                 nodes.stream()
+            serviceStatus.put(id, new ServiceConvergence(id.applicationId(),
+                                                         id.zoneId(),
+                                                         false,
+                                                         2,
+                                                         nodes.stream()
                                                                       .map(node -> new ServiceConvergence.Status(node.hostname(),
                                                                                                                  43,
                                                                                                                  "container",
@@ -342,7 +379,7 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer
                                               Collections.emptyList());
             setConfigChangeActions(null);
             prepareResponse.tenant = new TenantId("tenant");
-            prepareResponse.log = warnings.getOrDefault(deployment, Collections.emptyList());
+            prepareResponse.log = warnings.getOrDefault(id, Collections.emptyList());
             return prepareResponse;
         };
     }
@@ -366,6 +403,7 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer
             throw new NotFoundException("No application with id " + applicationId + " exists, cannot deactivate");
         applications.remove(deployment);
         serviceStatus.remove(deployment);
+        removeLoadBalancers(deployment.applicationId(), deployment.zoneId());
     }
 
     // Returns a canned example response
@@ -401,8 +439,7 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer
 
     // Returns a canned example response
     @Override
-    public Map<?,?> getServiceApiResponse(String tenantName, String applicationName, String instanceName,
-                                          String environment, String region, String serviceName, String restPath) {
+    public Map<?,?> getServiceApiResponse(DeploymentId deployment, String serviceName, String restPath) {
         Map<String,List<?>> root = new HashMap<>();
         List<Map<?,?>> resources = new ArrayList<>();
         Map<String,String> resource = new HashMap<>();
@@ -413,6 +450,11 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer
     }
 
     @Override
+    public String getClusterControllerStatus(DeploymentId deployment, String restPath) {
+        return "<h1>OK</h1>";
+    }
+
+    @Override
     public void setGlobalRotationStatus(DeploymentId deployment, String upstreamName, EndpointStatus status) {
         endpoints.put(upstreamName, status);
     }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/MetricsMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/MetricsMock.java
index 3dd6689dfdf..e2f598340be 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/MetricsMock.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/MetricsMock.java
@@ -21,7 +21,8 @@ public class MetricsMock implements Metric {
 
     @Override
     public void set(String key, Number val, Context ctx) {
-        Map<String, Number> metricsMap = metrics.getOrDefault(ctx, new HashMap<>());
+        metrics.putIfAbsent(ctx, new HashMap<>());
+        Map<String, Number> metricsMap = metrics.get(ctx);
         metricsMap.put(key, val);
     }
 
@@ -43,10 +44,6 @@ public class MetricsMock implements Metric {
         return ctx;
     }
 
-    public Map<Context, Map<String, Number>> getMetrics() {
-        return metrics;
-    }
-    
     /** Returns a zero-context metric by name, or null if it is not present */
     public Number getMetric(String name) {
         Map<String, Number> valuesForEmptyContext = metrics.get(createContext(Collections.emptyMap()));
@@ -64,9 +61,8 @@ public class MetricsMock implements Metric {
 
     /** Returns metric filtered by dimension and name */
     public Optional<Number> getMetric(Predicate<Map<String, String>> dimensionMatcher, String name) {
-        Map<String, Number> metrics = getMetrics(dimensionMatcher).entrySet()
+        Map<String, Number> metrics = getMetrics(dimensionMatcher).values()
                                                                   .stream()
-                                                                  .map(Map.Entry::getValue)
                                                                   .findFirst()
                                                                   .orElseGet(Collections::emptyMap);
         return Optional.ofNullable(metrics.get(name));
@@ -81,13 +77,16 @@ public class MetricsMock implements Metric {
         }
 
         @Override
-        public boolean equals(Object obj) {
-            return Objects.deepEquals(obj, dimensions);
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            MapContext that = (MapContext) o;
+            return dimensions.equals(that.dimensions);
         }
 
         @Override
         public int hashCode() {
-            return Objects.toString(dimensions).hashCode();
+            return Objects.hash(dimensions);
         }
 
         public Map<String, String> getDimensions() {
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java
index 632b8499e11..4aab21a44fe 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java
@@ -208,7 +208,10 @@ public class NodeRepositoryMock implements NodeRepository {
     public void doUpgrade(DeploymentId deployment, Optional<HostName> hostName, Version version) {
         modifyNodes(deployment, hostName, node -> {
             assert node.wantedVersion().equals(version);
-            return new Node.Builder(node).currentVersion(version).build();
+            return new Node.Builder(node)
+                    .currentVersion(version)
+                    .currentDockerImage(node.wantedDockerImage())
+                    .build();
         });
     }
 
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/SecretStoreMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/SecretStoreMock.java
index a14b7b82d67..c088965c2ad 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/SecretStoreMock.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/SecretStoreMock.java
@@ -5,6 +5,7 @@ import com.yahoo.component.AbstractComponent;
 import com.yahoo.container.jdisc.secretstore.SecretStore;
 
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.TreeMap;
 
@@ -45,4 +46,8 @@ public class SecretStoreMock extends AbstractComponent implements SecretStore {
         return secrets.getOrDefault(key, new TreeMap<>()).get(version);
     }
 
+    @Override
+    public List<Integer> listSecretVersions(String key) {
+        return List.copyOf(secrets.getOrDefault(key, new TreeMap<>()).keySet());
+    }
 }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java
index bd82807342e..98178f2a19f 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java
@@ -1,4 +1,4 @@
-// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.integration;
 
 import com.google.inject.Inject;
@@ -10,7 +10,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.ServiceRegistry;
 import com.yahoo.vespa.hosted.controller.api.integration.aws.MockAwsEventFetcher;
 import com.yahoo.vespa.hosted.controller.api.integration.aws.MockResourceTagger;
 import com.yahoo.vespa.hosted.controller.api.integration.aws.ResourceTagger;
-import com.yahoo.vespa.hosted.controller.api.integration.certificates.ApplicationCertificateMock;
+import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMock;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServer;
 import com.yahoo.vespa.hosted.controller.api.integration.dns.MemoryNameService;
 import com.yahoo.vespa.hosted.controller.api.integration.entity.MemoryEntityService;
@@ -21,8 +21,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.resource.CostReportCons
 import com.yahoo.vespa.hosted.controller.api.integration.resource.MockTenantCost;
 import com.yahoo.vespa.hosted.controller.api.integration.routing.GlobalRoutingService;
 import com.yahoo.vespa.hosted.controller.api.integration.routing.MemoryGlobalRoutingService;
-import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingGenerator;
-import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingGeneratorMock;
 import com.yahoo.vespa.hosted.controller.api.integration.stubs.DummyOwnershipIssues;
 import com.yahoo.vespa.hosted.controller.api.integration.stubs.LoggingDeploymentIssues;
 import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMailer;
@@ -42,9 +40,8 @@ public class ServiceRegistryMock extends AbstractComponent implements ServiceReg
     private final ConfigServerMock configServerMock;
     private final MemoryNameService memoryNameService = new MemoryNameService();
     private final MemoryGlobalRoutingService memoryGlobalRoutingService = new MemoryGlobalRoutingService();
-    private final RoutingGeneratorMock routingGeneratorMock = new RoutingGeneratorMock(RoutingGeneratorMock.TEST_ENDPOINTS);
     private final MockMailer mockMailer = new MockMailer();
-    private final ApplicationCertificateMock applicationCertificateMock = new ApplicationCertificateMock();
+    private final EndpointCertificateMock endpointCertificateMock = new EndpointCertificateMock();
     private final MockMeteringClient mockMeteringClient = new MockMeteringClient();
     private final MockContactRetriever mockContactRetriever = new MockContactRetriever();
     private final MockIssueHandler mockIssueHandler = new MockIssueHandler();
@@ -55,7 +52,7 @@ public class ServiceRegistryMock extends AbstractComponent implements ServiceReg
     private final MockBilling mockBilling = new MockBilling();
     private final MockAwsEventFetcher mockAwsEventFetcher = new MockAwsEventFetcher();
     private final ArtifactRepositoryMock artifactRepositoryMock = new ArtifactRepositoryMock();
-    private final MockTesterCloud mockTesterCloud = new MockTesterCloud();
+    private final MockTesterCloud mockTesterCloud;
     private final ApplicationStoreMock applicationStoreMock = new ApplicationStoreMock();
     private final MockRunDataStore mockRunDataStore = new MockRunDataStore();
     private final MockTenantCost mockTenantCost = new MockTenantCost();
@@ -64,6 +61,7 @@ public class ServiceRegistryMock extends AbstractComponent implements ServiceReg
     public ServiceRegistryMock(SystemName system) {
         this.zoneRegistryMock = new ZoneRegistryMock(system);
         this.configServerMock = new ConfigServerMock(zoneRegistryMock);
+        this.mockTesterCloud = new MockTesterCloud(nameService());
     }
 
     @Inject
@@ -91,18 +89,13 @@ public class ServiceRegistryMock extends AbstractComponent implements ServiceReg
     }
 
     @Override
-    public RoutingGenerator routingGenerator() {
-        return routingGeneratorMock;
-    }
-
-    @Override
     public MockMailer mailer() {
         return mockMailer;
     }
 
     @Override
-    public ApplicationCertificateMock applicationCertificateProvider() {
-        return applicationCertificateMock;
+    public EndpointCertificateMock endpointCertificateProvider() {
+        return endpointCertificateMock;
     }
 
     @Override
@@ -192,28 +185,16 @@ public class ServiceRegistryMock extends AbstractComponent implements ServiceReg
         return configServerMock;
     }
 
-    public MemoryNameService nameServiceMock() {
-        return memoryNameService;
-    }
-
     public MemoryGlobalRoutingService globalRoutingServiceMock() {
         return memoryGlobalRoutingService;
     }
 
-    public RoutingGeneratorMock routingGeneratorMock() {
-        return routingGeneratorMock;
-    }
-
     public MockContactRetriever contactRetrieverMock() {
         return mockContactRetriever;
     }
 
-    public ArtifactRepositoryMock artifactRepositoryMock() {
-        return artifactRepositoryMock;
-    }
-
-    public ApplicationCertificateMock applicationCertificateMock() {
-        return applicationCertificateMock;
+    public EndpointCertificateMock endpointCertificateMock() {
+        return endpointCertificateMock;
     }
 
 }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneApiMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneApiMock.java
index 269bdcc5dca..af0e3d5807d 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneApiMock.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneApiMock.java
@@ -8,6 +8,8 @@ import com.yahoo.config.provision.SystemName;
 import com.yahoo.config.provision.zone.ZoneApi;
 import com.yahoo.config.provision.zone.ZoneId;
 
+import java.util.Objects;
+
 /**
  * @author hakonhall
  */
@@ -34,6 +36,10 @@ public class ZoneApiMock implements ZoneApi {
         return newBuilder().with(ZoneId.from(environment, region)).build();
     }
 
+    public static ZoneApiMock from(ZoneId zone) {
+        return newBuilder().with(zone).build();
+    }
+
     @Override
     public SystemName getSystemName() { return systemName; }
 
@@ -46,8 +52,21 @@ public class ZoneApiMock implements ZoneApi {
     @Override
     public String getCloudNativeRegionName() { return cloudNativeRegionName; }
 
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        ZoneApiMock that = (ZoneApiMock) o;
+        return id.equals(that.id);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(id);
+    }
+
     public static class Builder {
-        private SystemName systemName = SystemName.defaultSystem();
+        private final SystemName systemName = SystemName.defaultSystem();
         private ZoneId id = ZoneId.defaultId();
         private CloudName cloudName = CloudName.defaultName();
         private String cloudNativeRegionName = id.region().value();
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneFilterMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneFilterMock.java
index 4cd50625ca3..23f59683f4b 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneFilterMock.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneFilterMock.java
@@ -1,9 +1,10 @@
-// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.integration;
 
 import com.yahoo.config.provision.CloudName;
 import com.yahoo.config.provision.Environment;
 import com.yahoo.config.provision.RegionName;
+import com.yahoo.config.provision.zone.RoutingMethod;
 import com.yahoo.config.provision.zone.ZoneApi;
 import com.yahoo.config.provision.zone.ZoneFilter;
 import com.yahoo.config.provision.zone.ZoneId;
@@ -11,6 +12,7 @@ import com.yahoo.config.provision.zone.ZoneList;
 
 import java.util.Collection;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 import java.util.function.Predicate;
 import java.util.stream.Collectors;
@@ -23,22 +25,22 @@ import java.util.stream.Collectors;
 public class ZoneFilterMock implements ZoneList {
 
     private final List<ZoneApi> zones;
-    private final Set<ZoneApi> directlyRouted;
+    private final Map<ZoneApi, List<RoutingMethod>> zoneRoutingMethods;
     private final boolean negate;
 
-    private ZoneFilterMock(List<ZoneApi> zones, Set<ZoneApi> directlyRouted, boolean negate) {
+    private ZoneFilterMock(List<ZoneApi> zones, Map<ZoneApi, List<RoutingMethod>> zoneRoutingMethods, boolean negate) {
         this.zones = zones;
-        this.directlyRouted = directlyRouted;
+        this.zoneRoutingMethods = zoneRoutingMethods;
         this.negate = negate;
     }
 
-    public static ZoneFilter from(Collection<ZoneApi> zones, Set<ZoneApi> directlyRouted) {
-        return new ZoneFilterMock(List.copyOf(zones), Set.copyOf(directlyRouted), false);
+    public static ZoneFilter from(Collection<? extends ZoneApi> zones, Map<ZoneApi, List<RoutingMethod>> routingMethods) {
+        return new ZoneFilterMock(List.copyOf(zones), Map.copyOf(routingMethods), false);
     }
 
     @Override
     public ZoneList not() {
-        return new ZoneFilterMock(zones, directlyRouted, ! negate);
+        return new ZoneFilterMock(zones, zoneRoutingMethods, ! negate);
     }
 
     @Override
@@ -53,7 +55,12 @@ public class ZoneFilterMock implements ZoneList {
 
     @Override
     public ZoneList directlyRouted() {
-        return filter(directlyRouted::contains);
+        return routingMethod(RoutingMethod.exclusive);
+    }
+
+    @Override
+    public ZoneList routingMethod(RoutingMethod method) {
+        return filter(zone -> zoneRoutingMethods.getOrDefault(zone, List.of()).contains(method));
     }
 
     @Override
@@ -93,7 +100,7 @@ public class ZoneFilterMock implements ZoneList {
                                 condition.negate().test(zone) :
                                 condition.test(zone))
                         .collect(Collectors.toList()),
-                directlyRouted, false);
+                zoneRoutingMethods, false);
     }
 
 }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java
index 6f8ff39456f..efc875b06f5 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java
@@ -1,4 +1,4 @@
-// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.integration;
 
 import com.yahoo.component.AbstractComponent;
@@ -9,6 +9,7 @@ import com.yahoo.config.provision.Environment;
 import com.yahoo.config.provision.NodeType;
 import com.yahoo.config.provision.RegionName;
 import com.yahoo.config.provision.SystemName;
+import com.yahoo.config.provision.zone.RoutingMethod;
 import com.yahoo.config.provision.zone.UpgradePolicy;
 import com.yahoo.config.provision.zone.ZoneApi;
 import com.yahoo.config.provision.zone.ZoneFilter;
@@ -25,7 +26,6 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
-import java.util.Set;
 
 /**
  * @author mpolden
@@ -34,11 +34,12 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry
 
     private final Map<ZoneId, Duration> deploymentTimeToLive = new HashMap<>();
     private final Map<Environment, RegionName> defaultRegionForEnvironment = new HashMap<>();
-    private List<ZoneApi> zones = List.of();
+    private final Map<CloudName, UpgradePolicy> osUpgradePolicies = new HashMap<>();
+    private final Map<ZoneApi, List<RoutingMethod>> zoneRoutingMethods = new HashMap<>();
+
+    private List<? extends ZoneApi> zones;
     private SystemName system;
     private UpgradePolicy upgradePolicy = null;
-    private Map<CloudName, UpgradePolicy> osUpgradePolicies = new HashMap<>();
-    private Set<ZoneApi> directlyRouted = Set.of();
 
     /**
      * This sets the default list of zones contained in this. If your test need a particular set of zones, use
@@ -46,21 +47,21 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry
      */
     public ZoneRegistryMock(SystemName system) {
         this.system = system;
-        setZones(List.of(
-                ZoneApiMock.fromId("test.us-east-1"),
-                ZoneApiMock.fromId("staging.us-east-3"),
-                ZoneApiMock.fromId("dev.us-east-1"),
-                ZoneApiMock.fromId("dev.aws-us-east-2a"),
-                ZoneApiMock.fromId("perf.us-east-3"),
-                ZoneApiMock.fromId("prod.aws-us-east-1a"),
-                ZoneApiMock.fromId("prod.ap-northeast-1"),
-                ZoneApiMock.fromId("prod.ap-northeast-2"),
-                ZoneApiMock.fromId("prod.ap-southeast-1"),
-                ZoneApiMock.fromId("prod.us-east-3"),
-                ZoneApiMock.fromId("prod.us-west-1"),
-                ZoneApiMock.fromId("prod.us-central-1"),
-                ZoneApiMock.fromId("prod.eu-west-1")));
-        setDirectlyRouted(Set.copyOf(this.zones));
+        this.zones = List.of(ZoneApiMock.fromId("test.us-east-1"),
+                             ZoneApiMock.fromId("staging.us-east-3"),
+                             ZoneApiMock.fromId("dev.us-east-1"),
+                             ZoneApiMock.fromId("dev.aws-us-east-2a"),
+                             ZoneApiMock.fromId("perf.us-east-3"),
+                             ZoneApiMock.fromId("prod.aws-us-east-1a"),
+                             ZoneApiMock.fromId("prod.ap-northeast-1"),
+                             ZoneApiMock.fromId("prod.ap-northeast-2"),
+                             ZoneApiMock.fromId("prod.ap-southeast-1"),
+                             ZoneApiMock.fromId("prod.us-east-3"),
+                             ZoneApiMock.fromId("prod.us-west-1"),
+                             ZoneApiMock.fromId("prod.us-central-1"),
+                             ZoneApiMock.fromId("prod.eu-west-1"));
+        // All zones use a shared routing method by default
+        setRoutingMethod(this.zones, RoutingMethod.shared);
     }
 
     public ZoneRegistryMock setDeploymentTimeToLive(ZoneId zone, Duration duration) {
@@ -73,7 +74,7 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry
         return this;
     }
 
-    public ZoneRegistryMock setZones(List<ZoneApi> zones) {
+    public ZoneRegistryMock setZones(List<? extends ZoneApi> zones) {
         this.zones = zones;
         return this;
     }
@@ -97,12 +98,28 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry
         return this;
     }
 
-    public ZoneRegistryMock setDirectlyRouted(ZoneApi... zones) {
-        return setDirectlyRouted(Set.of(zones));
+    public ZoneRegistryMock exclusiveRoutingIn(ZoneApi... zones) {
+        return exclusiveRoutingIn(List.of(zones));
+    }
+
+    public ZoneRegistryMock exclusiveRoutingIn(List<? extends ZoneApi> zones) {
+        return setRoutingMethod(zones, RoutingMethod.exclusive);
     }
 
-    public ZoneRegistryMock setDirectlyRouted(Set<ZoneApi> zones) {
-        directlyRouted = zones;
+    public ZoneRegistryMock setRoutingMethod(ZoneApi zone, RoutingMethod... routingMethods) {
+        return setRoutingMethod(zone, List.of(routingMethods));
+    }
+
+    public ZoneRegistryMock setRoutingMethod(List<? extends ZoneApi> zones, RoutingMethod... routingMethods) {
+        zones.forEach(zone -> setRoutingMethod(zone, List.of(routingMethods)));
+        return this;
+    }
+
+    public ZoneRegistryMock setRoutingMethod(ZoneApi zone, List<RoutingMethod> routingMethods) {
+        if (routingMethods.stream().distinct().count() != routingMethods.size()) {
+            throw new IllegalArgumentException("Routing methods must be distinct");
+        }
+        this.zoneRoutingMethods.put(zone, List.copyOf(routingMethods));
         return this;
     }
 
@@ -113,7 +130,7 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry
 
     @Override
     public ZoneFilter zones() {
-        return ZoneFilterMock.from(zones, directlyRouted);
+        return ZoneFilterMock.from(zones, zoneRoutingMethods);
     }
 
     @Override
@@ -147,6 +164,11 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry
     }
 
     @Override
+    public List<RoutingMethod> routingMethods(ZoneId zone) {
+        return List.copyOf(zoneRoutingMethods.getOrDefault(ZoneApiMock.from(zone), List.of()));
+    }
+
+    @Override
     public URI dashboardUrl() {
         return URI.create("https://dashboard.tld");
     }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmerTest.java
index 8997f34fb98..299c81fdb3c 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmerTest.java
@@ -10,7 +10,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.organization.OwnershipI
 import com.yahoo.vespa.hosted.controller.api.integration.organization.User;
 import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester;
 import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb;
-import com.yahoo.vespa.hosted.controller.tenant.UserTenant;
 import org.junit.Before;
 import org.junit.Test;
 
@@ -42,70 +41,69 @@ public class ApplicationOwnershipConfirmerTest {
     @Test
     public void testConfirmation() {
         Optional<Contact> contact = Optional.of(tester.controllerTester().serviceRegistry().contactRetrieverMock().contact());
-        var propertyApp = tester.newDeploymentContext();
+        var app = tester.newDeploymentContext();
         tester.controller().tenants().lockOrThrow(appId.tenant(), LockedTenant.Athenz.class, tenant ->
                 tester.controller().tenants().store(tenant.with(contact.get())));
-        propertyApp.submit().deploy();
+        app.submit().deploy();
 
-        UserTenant user = UserTenant.create("by-user", contact);
-        tester.controller().tenants().createUser(user);
-        var userApp = tester.newDeploymentContext("by-user", "application", "default");
-        userApp.submit().deploy();
+        var appWithoutContact = tester.newDeploymentContext("other", "application", "default");
+        appWithoutContact.submit().deploy();
 
-        assertFalse("No issue is initially stored for a new application.", propertyApp.application().ownershipIssueId().isPresent());
-        assertFalse("No issue is initially stored for a new application.", userApp.application().ownershipIssueId().isPresent());
-        assertFalse("No escalation has been attempted for a new application", issues.escalatedToContact || issues.escalatedToTerminator);
+        assertFalse("No issue is initially stored for a new application.", app.application().ownershipIssueId().isPresent());
+        assertFalse("No issue is initially stored for a new application.", appWithoutContact.application().ownershipIssueId().isPresent());
+        assertFalse("No escalation has been attempted for a new application", issues.escalated);
 
         // Set response from the issue mock, which will be obtained by the maintainer on issue filing.
         Optional<IssueId> issueId = Optional.of(IssueId.from("1"));
         issues.response = issueId;
         confirmer.maintain();
 
-        assertFalse("No issue is stored for an application newer than 3 months.", propertyApp.application().ownershipIssueId().isPresent());
-        assertFalse("No issue is stored for an application newer than 3 months.", userApp.application().ownershipIssueId().isPresent());
+        assertFalse("No issue is stored for an application newer than 3 months.", app.application().ownershipIssueId().isPresent());
+        assertFalse("No issue is stored for an application newer than 3 months.", appWithoutContact.application().ownershipIssueId().isPresent());
 
         tester.clock().advance(Duration.ofDays(91));
         confirmer.maintain();
 
-        assertEquals("Confirmation issue has been filed for property owned application.", issueId, propertyApp.application().ownershipIssueId());
-        assertEquals("Confirmation issue has been filed for user owned application.", issueId, userApp.application().ownershipIssueId());
-        assertTrue(issues.escalatedToTerminator);
-        assertTrue("Both applications have had their responses ensured.", issues.escalatedToContact);
+        assertEquals("Confirmation issue has been filed for application with contact.", issueId, app.application().ownershipIssueId());
+        assertTrue("The confirmation issue response has been ensured.", issues.escalated);
+        assertEquals("No confirmation issue has been filed for application without contact.", Optional.empty(), appWithoutContact.application().ownershipIssueId());
 
         // No new issue is created, so return empty now.
         issues.response = Optional.empty();
         confirmer.maintain();
 
-        assertEquals("Confirmation issue reference is not updated when no issue id is returned.", issueId, propertyApp.application().ownershipIssueId());
-        assertEquals("Confirmation issue reference is not updated when no issue id is returned.", issueId, userApp.application().ownershipIssueId());
-
-        // The user deletes all production deployments — see that the issue is forgotten.
-        assertEquals("Confirmation issue for user is still open.", issueId, userApp.application().ownershipIssueId());
-        userApp.application().productionDeployments().values().stream().flatMap(List::stream)
-               .forEach(deployment -> tester.controller().applications().deactivate(userApp.instanceId(), deployment.zone()));
-        assertTrue("No production deployments are listed for user.", userApp.application().require(InstanceName.defaultName()).productionDeployments().isEmpty());
-        confirmer.maintain();
+        assertEquals("Confirmation issue reference is not updated when no issue id is returned.", issueId, app.application().ownershipIssueId());
 
         // Time has passed, and a new confirmation issue is in order for the property which is still in production.
         Optional<IssueId> issueId2 = Optional.of(IssueId.from("2"));
         issues.response = issueId2;
         confirmer.maintain();
 
-        assertEquals("A new confirmation issue id is stored when something is returned to the maintainer.", issueId2, propertyApp.application().ownershipIssueId());
-        assertEquals("Confirmation issue for application without production deployments has not been filed.", issueId, userApp.application().ownershipIssueId());
+        assertEquals("A new confirmation issue id is stored when something is returned to the maintainer.", issueId2, app.application().ownershipIssueId());
 
-        assertFalse("No owner is stored for application", propertyApp.application().owner().isPresent());
+        assertFalse("No owner is stored for application", app.application().owner().isPresent());
         issues.owner = Optional.of(User.from("username"));
         confirmer.maintain();
-        assertEquals("Owner has been added to application", propertyApp.application().owner().get().username(), "username");
+        assertEquals("Owner has been added to application", app.application().owner().get().username(), "username");
+
+        // The app deletes all production deployments — see that the issue is forgotten.
+        assertEquals("Confirmation issue for application is still open.", issueId2, app.application().ownershipIssueId());
+        app.application().productionDeployments().values().stream().flatMap(List::stream)
+           .forEach(deployment -> tester.controller().applications().deactivate(app.instanceId(), deployment.zone()));
+        assertTrue("No production deployments are listed for user.", app.application().require(InstanceName.defaultName()).productionDeployments().isEmpty());
+        confirmer.maintain();
+
+        // Time has passed, and a new confirmation issue is in order for the property which is still in production.
+        issues.response = Optional.of(IssueId.from("3"));
+        confirmer.maintain();
 
+        assertEquals("Confirmation issue for application without production deployments has not been filed.", issueId2, app.application().ownershipIssueId());
     }
 
-    private class MockOwnershipIssues implements OwnershipIssues {
+    private static class MockOwnershipIssues implements OwnershipIssues {
 
         private Optional<IssueId> response;
-        private boolean escalatedToContact = false;
-        private boolean escalatedToTerminator = false;
+        private boolean escalated = false;
         private Optional<User> owner = Optional.empty();
 
         @Override
@@ -115,8 +113,7 @@ public class ApplicationOwnershipConfirmerTest {
 
         @Override
         public void ensureResponse(IssueId issueId, Optional<Contact> contact) {
-            if (contact.isPresent()) escalatedToContact = true;
-            else escalatedToTerminator = true;
+            escalated = true;
         }
 
         @Override
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CloudEventReporterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CloudEventReporterTest.java
index cd2a4fd8453..1c66a01db8e 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CloudEventReporterTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CloudEventReporterTest.java
@@ -27,10 +27,10 @@ import static org.junit.Assert.*;
  */
 public class CloudEventReporterTest {
 
-    private ControllerTester tester = new ControllerTester();
-    private ZoneApiMock nonAwsZone = createZone("prod.zone3", "region-1", "other");
-    private ZoneApiMock awsZone1 = createZone("prod.zone1", "region-1", "aws");
-    private ZoneApiMock awsZone2 = createZone("prod.zone2", "region-2", "aws");
+    private final ControllerTester tester = new ControllerTester();
+    private final ZoneApiMock nonAwsZone = createZone("prod.zone3", "region-1", "other");
+    private final ZoneApiMock awsZone1 = createZone("prod.zone1", "region-1", "aws");
+    private final ZoneApiMock awsZone2 = createZone("prod.zone2", "region-2", "aws");
 
 
     /**
@@ -153,4 +153,4 @@ public class CloudEventReporterTest {
                 .build();
     }
 
-}
\ No newline at end of file
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterInfoMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterInfoMaintainerTest.java
deleted file mode 100644
index ff72a2f7231..00000000000
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterInfoMaintainerTest.java
+++ /dev/null
@@ -1,90 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.controller.maintenance;
-
-import com.yahoo.component.Version;
-import com.yahoo.config.provision.ApplicationId;
-import com.yahoo.config.provision.ClusterSpec;
-import com.yahoo.config.provision.HostName;
-import com.yahoo.config.provision.NodeResources;
-import com.yahoo.config.provision.NodeType;
-import com.yahoo.config.provision.zone.ZoneId;
-import com.yahoo.vespa.hosted.controller.ControllerTester;
-import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node;
-import com.yahoo.vespa.hosted.controller.application.Deployment;
-import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb;
-import org.junit.Test;
-
-import java.time.Duration;
-import java.util.List;
-
-import static org.junit.Assert.assertEquals;
-
-/**
- * @author smorgrav
- */
-public class ClusterInfoMaintainerTest {
-
-    private final ControllerTester tester = new ControllerTester();
-
-    @Test
-    public void maintain() {
-        tester.createTenant("tenant1", "domain123", 321L);
-        ApplicationId app = tester.createApplication("tenant1", "app1", "default").id().defaultInstance();
-        ZoneId zone = ZoneId.from("dev", "us-east-1");
-        tester.deploy(app, zone);
-
-        // Precondition: no cluster info attached to the deployments
-        Deployment deployment = tester.controller().applications().getInstance(app).get().deployments().values().stream()
-                                      .findFirst()
-                                      .get();
-        assertEquals(0, deployment.clusterInfo().size());
-
-        addNodes(zone);
-        ClusterInfoMaintainer maintainer = new ClusterInfoMaintainer(tester.controller(), Duration.ofHours(1),
-                                                                     new JobControl(new MockCuratorDb()));
-        maintainer.maintain();
-
-        deployment = tester.controller().applications().getInstance(app).get().deployments().values().stream()
-                           .findFirst()
-                           .get();
-        assertEquals(2, deployment.clusterInfo().size());
-        assertEquals(10, deployment.clusterInfo().get(ClusterSpec.Id.from("clusterA")).getFlavorCost());
-    }
-
-    private void addNodes(ZoneId zone) {
-        var nodeA = new Node.Builder()
-                .hostname(HostName.from("hostA"))
-                .parentHostname(HostName.from("parentHostA"))
-                .state(Node.State.active)
-                .type(NodeType.tenant)
-                .owner(ApplicationId.from("tenant1", "app1", "default"))
-                .currentVersion(Version.fromString("7.42"))
-                .wantedVersion(Version.fromString("7.42"))
-                .currentOsVersion(Version.fromString("7.6"))
-                .wantedOsVersion(Version.fromString("7.6"))
-                .serviceState(Node.ServiceState.expectedUp)
-                .resources(new NodeResources(1, 1, 1, 1))
-                .cost(10)
-                .clusterId("clusterA")
-                .clusterType(Node.ClusterType.container)
-                .build();
-        var nodeB = new Node.Builder()
-                .hostname(HostName.from("hostB"))
-                .parentHostname(HostName.from("parentHostB"))
-                .state(Node.State.active)
-                .type(NodeType.tenant)
-                .owner(ApplicationId.from("tenant1", "app1", "default"))
-                .currentVersion(Version.fromString("7.42"))
-                .wantedVersion(Version.fromString("7.42"))
-                .currentOsVersion(Version.fromString("7.6"))
-                .wantedOsVersion(Version.fromString("7.6"))
-                .serviceState(Node.ServiceState.expectedUp)
-                .resources(new NodeResources(1, 1, 1, 1))
-                .cost(20)
-                .clusterId("clusterB")
-                .clusterType(Node.ClusterType.container)
-                .build();
-        tester.configServer().nodeRepository().addNodes(zone, List.of(nodeA, nodeB));
-    }
-
-}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainerTest.java
index 0db718080c2..32d2e2f35f4 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainerTest.java
@@ -37,22 +37,22 @@ public class ContactInformationMaintainerTest {
 
     @Test
     public void updates_contact_information() {
-        long propertyId = 1;
-        TenantName name = tester.createTenant("tenant1", "domain1", propertyId);
-        Supplier<AthenzTenant> tenant = () -> (AthenzTenant) tester.controller().tenants().require(name);
-        assertFalse("No contact information initially", tenant.get().contact().isPresent());
+        PropertyId propertyId1 = new PropertyId("1");
+        PropertyId propertyId2 = new PropertyId("2");
+        TenantName name1 = tester.createTenant("tenant1", "domain1", 1L);
+        TenantName name2 = tester.createTenant("zenant1", "domain2", 2L);
+        Supplier<AthenzTenant> tenant1 = () -> (AthenzTenant) tester.controller().tenants().require(name1);
+        Supplier<AthenzTenant> tenant2 = () -> (AthenzTenant) tester.controller().tenants().require(name2);
+        assertFalse("No contact information initially", tenant1.get().contact().isPresent());
+        assertFalse("No contact information initially", tenant2.get().contact().isPresent());
 
         Contact contact = testContact();
-        registerContact(propertyId, contact);
-        maintainer.run();
+        tester.serviceRegistry().contactRetriever().addContact(propertyId1, () -> { throw new RuntimeException("ERROR"); });
+        tester.serviceRegistry().contactRetriever().addContact(propertyId2, () -> contact);
+        maintainer.maintain();
 
-        assertTrue("Contact information added", tenant.get().contact().isPresent());
-        assertEquals(contact, tenant.get().contact().get());
-    }
-
-    private void registerContact(long propertyId, Contact contact) {
-        PropertyId p = new PropertyId(String.valueOf(propertyId));
-        tester.serviceRegistry().contactRetrieverMock().addContact(p, contact);
+        assertEquals("No contact information added due to error", Optional.empty(), tenant1.get().contact());
+        assertEquals("Contact information added", Optional.of(contact), tenant2.get().contact());
     }
 
     private static Contact testContact() {
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirerTest.java
index 258dad3b6d6..bef04b338e6 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirerTest.java
@@ -4,7 +4,6 @@ package com.yahoo.vespa.hosted.controller.maintenance;
 import com.yahoo.config.provision.Environment;
 import com.yahoo.config.provision.RegionName;
 import com.yahoo.config.provision.zone.ZoneId;
-import com.yahoo.vespa.hosted.controller.Application;
 import com.yahoo.vespa.hosted.controller.Instance;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
 import com.yahoo.vespa.hosted.controller.application.ApplicationPackage;
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporterTest.java
index ea946e132a9..ee5639dbfec 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporterTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporterTest.java
@@ -4,7 +4,6 @@ package com.yahoo.vespa.hosted.controller.maintenance;
 import com.yahoo.component.Version;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.Environment;
-import com.yahoo.vespa.hosted.controller.Application;
 import com.yahoo.vespa.hosted.controller.LockedTenant;
 import com.yahoo.vespa.hosted.controller.api.integration.organization.Contact;
 import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId;
@@ -13,7 +12,6 @@ import com.yahoo.vespa.hosted.controller.application.ApplicationPackage;
 import com.yahoo.vespa.hosted.controller.application.Change;
 import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId;
 import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder;
-import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext;
 import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester;
 import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb;
 import com.yahoo.vespa.hosted.controller.versions.VespaVersion;
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java
index ed8918786c5..931f22dd7f9 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java
@@ -6,7 +6,6 @@ import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.zone.ZoneId;
 import com.yahoo.vespa.hosted.controller.Application;
 import com.yahoo.vespa.hosted.controller.Controller;
-import com.yahoo.vespa.hosted.controller.Instance;
 import com.yahoo.vespa.hosted.controller.api.application.v4.model.ClusterMetrics;
 import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
@@ -40,7 +39,6 @@ public class DeploymentMetricsMaintainerTest {
 
         DeploymentMetricsMaintainer maintainer = maintainer(tester.controller());
         Supplier<Application> app = application::application;
-        Supplier<Instance> instance = application::instance;
         Supplier<Deployment> deployment = () -> application.deployment(ZoneId.from("dev", "us-east-1"));
 
         // No metrics gathered yet
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java
index 9de0020ce4a..d2946651619 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java
@@ -62,6 +62,7 @@ import static com.yahoo.vespa.hosted.controller.deployment.Step.report;
 import static com.yahoo.vespa.hosted.controller.deployment.Step.startTests;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotSame;
 import static org.junit.Assert.assertSame;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
@@ -98,7 +99,7 @@ public class JobRunnerTest {
             jobs.start(id, systemTest, versions);
             fail("Job is already running, so this should not be allowed!");
         }
-        catch (IllegalStateException e) { }
+        catch (IllegalStateException ignored) { }
         jobs.start(id, stagingTest, versions);
 
         assertTrue(jobs.last(id, systemTest).get().stepStatuses().values().stream().allMatch(unfinished::equals));
@@ -184,7 +185,7 @@ public class JobRunnerTest {
         runner.maintain();
         assertTrue(run.get().hasFailed());
         assertTrue(run.get().hasEnded());
-        assertTrue(run.get().status() == aborted);
+        assertSame(aborted, run.get().status());
 
         // A new run is attempted.
         jobs.start(id, systemTest, versions);
@@ -195,7 +196,7 @@ public class JobRunnerTest {
         runner.maintain();
         assertTrue(run.get().hasEnded());
         assertTrue(run.get().hasFailed());
-        assertFalse(run.get().status() == aborted);
+        assertNotSame(aborted, run.get().status());
         assertEquals(failed, run.get().stepStatuses().get(deployTester));
         assertEquals(unfinished, run.get().stepStatuses().get(installTester));
         assertEquals(succeeded, run.get().stepStatuses().get(report));
@@ -241,7 +242,7 @@ public class JobRunnerTest {
             jobs.locked(id, systemTest, deactivateTester, step -> { });
             fail("deployTester step should still be locked!");
         }
-        catch (TimeoutException e) { }
+        catch (TimeoutException ignored) { }
 
         // Thread is still trying to deploy tester -- delete application, and see all data is garbage collected.
         assertEquals(Collections.singletonList(runId), jobs.active().stream().map(run -> run.id()).collect(Collectors.toList()));
@@ -338,6 +339,25 @@ public class JobRunnerTest {
     }
 
     @Test
+    public void onlySuccessfulRunExpiresThenAnotherFails() {
+        DeploymentTester tester = new DeploymentTester();
+        JobController jobs = tester.controller().jobController();
+        var app = tester.newDeploymentContext().submit();
+        JobId jobId = new JobId(app.instanceId(), systemTest);
+        assertFalse(jobs.lastSuccess(jobId).isPresent());
+
+        app.runJob(systemTest);
+        assertTrue(jobs.lastSuccess(jobId).isPresent());
+        assertEquals(1, jobs.runs(jobId).size());
+
+        tester.clock().advance(JobController.maxHistoryAge.plusSeconds(1));
+        app.submit();
+        app.failDeployment(systemTest);
+        assertFalse(jobs.lastSuccess(jobId).isPresent());
+        assertEquals(1, jobs.runs(jobId).size());
+    }
+
+    @Test
     public void timeout() {
         DeploymentTester tester = new DeploymentTester();
         JobController jobs = tester.controller().jobController();
@@ -375,12 +395,11 @@ public class JobRunnerTest {
             jobs.finish(jobs.last(id, systemTest).get().id());
         }
 
-        Map<String, String> context = Map.of("tenant", "tenant",
-                                             "application", "real",
-                                             "instance", "default",
-                                             "job", "system-test",
-                                             "environment", "test",
-                                             "region", "us-east-1");
+        Map<String, String> context = Map.of("applicationId", "tenant.real.default",
+                                             "tenantName", "tenant",
+                                             "app", "real.default",
+                                             "test", "true",
+                                             "zone", "test.us-east-1");
         MetricsMock metric = ((MetricsMock) tester.controller().metric());
         assertEquals(RunStatus.values().length - 1, metric.getMetric(context::equals, JobMetrics.start).get().intValue());
         assertEquals(1, metric.getMetric(context::equals, JobMetrics.abort).get().intValue());
@@ -389,12 +408,13 @@ public class JobRunnerTest {
         assertEquals(1, metric.getMetric(context::equals, JobMetrics.convergenceFailure).get().intValue());
         assertEquals(1, metric.getMetric(context::equals, JobMetrics.deploymentFailure).get().intValue());
         assertEquals(1, metric.getMetric(context::equals, JobMetrics.outOfCapacity).get().intValue());
+        assertEquals(1, metric.getMetric(context::equals, JobMetrics.endpointCertificateTimeout).get().intValue());
         assertEquals(1, metric.getMetric(context::equals, JobMetrics.testFailure).get().intValue());
     }
 
     public static ExecutorService inThreadExecutor() {
         return new AbstractExecutorService() {
-            AtomicBoolean shutDown = new AtomicBoolean(false);
+            final AtomicBoolean shutDown = new AtomicBoolean(false);
             @Override public void shutdown() { shutDown.set(true); }
             @Override public List<Runnable> shutdownNow() { shutDown.set(true); return Collections.emptyList(); }
             @Override public boolean isShutdown() { return shutDown.get(); }
@@ -406,7 +426,7 @@ public class JobRunnerTest {
 
     private static ExecutorService phasedExecutor(Phaser phaser) {
         return new AbstractExecutorService() {
-            ExecutorService delegate = Executors.newFixedThreadPool(32);
+            final ExecutorService delegate = Executors.newFixedThreadPool(32);
             @Override public void shutdown() { delegate.shutdown(); }
             @Override public List<Runnable> shutdownNow() { return delegate.shutdownNow(); }
             @Override public boolean isShutdown() { return delegate.isShutdown(); }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MaintainerTest.java
index 3aa1f2b5af2..efd5d61ce56 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MaintainerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MaintainerTest.java
@@ -38,17 +38,22 @@ public class MaintainerTest {
     @Test
     public void staggering() {
         List<HostName> cluster = List.of(HostName.from("cfg1"), HostName.from("cfg2"), HostName.from("cfg3"));
-        Instant now = Instant.ofEpochMilli(1001);
         Duration interval = Duration.ofMillis(300);
 
-        assertEquals(299, Maintainer.staggeredDelay(cluster, HostName.from("cfg1"), now, interval));
-        assertEquals(399, Maintainer.staggeredDelay(cluster, HostName.from("cfg2"), now, interval));
-        assertEquals(199, Maintainer.staggeredDelay(cluster, HostName.from("cfg3"), now, interval));
+        Instant now = Instant.ofEpochMilli(1000);
+        assertEquals(200, Maintainer.staggeredDelay(cluster, HostName.from("cfg1"), now, interval));
+        assertEquals(  0, Maintainer.staggeredDelay(cluster, HostName.from("cfg2"), now, interval));
+        assertEquals(100, Maintainer.staggeredDelay(cluster, HostName.from("cfg3"), now, interval));
 
-        now = Instant.ofEpochMilli(1101);
+        now = Instant.ofEpochMilli(1001);
         assertEquals(199, Maintainer.staggeredDelay(cluster, HostName.from("cfg1"), now, interval));
         assertEquals(299, Maintainer.staggeredDelay(cluster, HostName.from("cfg2"), now, interval));
-        assertEquals(399, Maintainer.staggeredDelay(cluster, HostName.from("cfg3"), now, interval));
+        assertEquals( 99, Maintainer.staggeredDelay(cluster, HostName.from("cfg3"), now, interval));
+
+        now = Instant.ofEpochMilli(1101);
+        assertEquals( 99, Maintainer.staggeredDelay(cluster, HostName.from("cfg1"), now, interval));
+        assertEquals(199, Maintainer.staggeredDelay(cluster, HostName.from("cfg2"), now, interval));
+        assertEquals(299, Maintainer.staggeredDelay(cluster, HostName.from("cfg3"), now, interval));
 
         assertEquals(300, Maintainer.staggeredDelay(cluster, HostName.from("cfg0"), now, interval));
     }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java
index e0fb2aa0ee1..c00705149e9 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java
@@ -256,7 +256,7 @@ public class MetricsReporterTest {
                          tester.configServer().nodeRepository().list(zone1.getId(), SystemApplication.configServer.id()).stream()
                                .map(Node::wantedVersion).min(Comparator.naturalOrder()).get());
             tester.configServer().setVersion(SystemApplication.configServer.id(), zone1.getId(), version, 1);
-            tester.clock().advance(Duration.ofMinutes(30).plus(Duration.ofSeconds(1)));
+            tester.clock().advance(Duration.ofMinutes(60).plus(Duration.ofSeconds(1)));
             tester.computeVersionStatus();
             reporter.maintain();
             assertEquals(2, getNodesFailingUpgrade());
@@ -278,7 +278,7 @@ public class MetricsReporterTest {
         var cloud = CloudName.defaultName();
         tester.zoneRegistry().setOsUpgradePolicy(cloud, UpgradePolicy.create().upgrade(zone));
         var osUpgrader = new OsUpgrader(tester.controller(), Duration.ofDays(1),
-                                        new JobControl(tester.curator()), CloudName.defaultName());;
+                                        new JobControl(tester.curator()), CloudName.defaultName());
         var statusUpdater = new OsVersionStatusUpdater(tester.controller(), Duration.ofDays(1),
                                                        new JobControl(tester.controller().curator()));
         tester.configServer().bootstrap(List.of(zone.getId()), SystemApplication.configServerHost, SystemApplication.tenantHost);
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployerTest.java
index 442a2fd1853..314901d5e4b 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployerTest.java
@@ -2,9 +2,7 @@
 package com.yahoo.vespa.hosted.controller.maintenance;
 
 import com.yahoo.component.Version;
-import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.Environment;
-import com.yahoo.vespa.hosted.controller.Application;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision;
 import com.yahoo.vespa.hosted.controller.application.ApplicationPackage;
@@ -17,6 +15,7 @@ import org.junit.Test;
 
 import java.time.Duration;
 import java.util.List;
+import java.util.Optional;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
@@ -47,7 +46,7 @@ public class OutstandingChangeDeployerTest {
         assertFalse(app1.deploymentStatus().outstandingChange(app1.instance().name()).hasTargets());
 
         assertEquals(1, app1.application().latestVersion().get().buildNumber().getAsLong());
-        app1.submit(applicationPackage, new SourceRevision("repository1", "master", "cafed00d"));
+        app1.submit(applicationPackage, Optional.of(new SourceRevision("repository1", "master", "cafed00d")));
 
         assertTrue(app1.deploymentStatus().outstandingChange(app1.instance().name()).hasTargets());
         assertEquals("1.0.2-cafed00d", app1.deploymentStatus().outstandingChange(app1.instance().name()).application().get().id());
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java
index 26012443e3e..e0a97d32597 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java
@@ -28,9 +28,8 @@ import static org.junit.Assert.assertEquals;
 public class ResourceMeterMaintainerTest {
 
     private final ControllerTester tester = new ControllerTester();
-    private final double DELTA = Double.MIN_VALUE;
-    private MockMeteringClient snapshotConsumer = new MockMeteringClient();
-    private MetricsMock metrics = new MetricsMock();
+    private final MockMeteringClient snapshotConsumer = new MockMeteringClient();
+    private final MetricsMock metrics = new MetricsMock();
 
     @Test
     public void testMaintainer() {
@@ -45,16 +44,16 @@ public class ResourceMeterMaintainerTest {
         ResourceSnapshot app1 = consumedResources.stream().filter(snapshot -> snapshot.getApplicationId().equals(ApplicationId.from("tenant1", "app1", "default"))).findFirst().orElseThrow();
         ResourceSnapshot app2 = consumedResources.stream().filter(snapshot -> snapshot.getApplicationId().equals(ApplicationId.from("tenant2", "app2", "default"))).findFirst().orElseThrow();
 
-        assertEquals(24, app1.getCpuCores(), DELTA);
-        assertEquals(24, app1.getMemoryGb(), DELTA);
-        assertEquals(500, app1.getDiskGb(), DELTA);
+        assertEquals(24, app1.getCpuCores(), Double.MIN_VALUE);
+        assertEquals(24, app1.getMemoryGb(), Double.MIN_VALUE);
+        assertEquals(500, app1.getDiskGb(), Double.MIN_VALUE);
 
-        assertEquals(40, app2.getCpuCores(), DELTA);
-        assertEquals(24, app2.getMemoryGb(), DELTA);
-        assertEquals(500, app2.getDiskGb(), DELTA);
+        assertEquals(40, app2.getCpuCores(), Double.MIN_VALUE);
+        assertEquals(24, app2.getMemoryGb(), Double.MIN_VALUE);
+        assertEquals(500, app2.getDiskGb(), Double.MIN_VALUE);
 
         assertEquals(tester.clock().millis()/1000, metrics.getMetric("metering_last_reported"));
-        assertEquals(2224.0d, (Double) metrics.getMetric("metering_total_reported"), DELTA);
+        assertEquals(2224.0d, (Double) metrics.getMetric("metering_total_reported"), Double.MIN_VALUE);
     }
 
     private void setUpZones() {
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainerTest.java
index 0a0aa7c9ded..653a4c5394b 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainerTest.java
@@ -19,7 +19,7 @@ import static org.junit.Assert.*;
  */
 public class ResourceTagMaintainerTest {
 
-    ControllerTester tester = new ControllerTester();
+    final ControllerTester tester = new ControllerTester();
 
     @Test
     public void maintain() {
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RotationStatusUpdaterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RotationStatusUpdaterTest.java
index d7a28708049..77567a58ed2 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RotationStatusUpdaterTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RotationStatusUpdaterTest.java
@@ -65,7 +65,7 @@ public class RotationStatusUpdaterTest {
                 .region(zone1.region().value())
                 .region(zone2.region().value())
                 .region(zone3.region().value())
-                .endpoint("default", "default", "us-east-3", "us-west-1")
+                .endpoint("default", "foo", "us-east-3", "us-west-1")
                 .endpoint("eu", "default", "eu-west-1")
                 .build();
         context.submit(applicationPackage)
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java
index 249c2644170..78f198bf05f 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java
@@ -136,7 +136,7 @@ public class UpgraderTest {
 
         canary1.deployPlatform(version3);
 
-        tester.controllerTester().computeVersionStatus();;
+        tester.controllerTester().computeVersionStatus();
         assertEquals(VespaVersion.Confidence.normal, tester.controller().versionStatus().systemVersion().get().confidence());
         tester.upgrader().maintain();
         tester.triggerJobs();
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/testdata/application-without-project-id.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/testdata/application-without-project-id.json
deleted file mode 100644
index 31949cce282..00000000000
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/testdata/application-without-project-id.json
+++ /dev/null
@@ -1,19 +0,0 @@
-{
-  "id": "tenant1:app1:default",
-  "deploymentSpecField": "<deployment version='1.0'>\n  <test />\n  <staging />\n  <prod>\n     <region active=\"true\">us-central-1</region>\n     <region active=\"true\">us-west-1</region>\n  </prod>\n</deployment>\n",
-  "validationOverrides": "<deployment version='1.0'/>",
-  "deployments": [],
-  "deploymentJobs": {
-    "jobStatus": [
-      {
-        "jobType": "system-test",
-        "lastTriggered": {
-          "version": "6.42.1",
-          "upgrade": false,
-          "at": 1506330088050
-        }
-      }
-    ]
-  },
-  "outstandingChangeField": false
-}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java
index c4cb01f8164..a73445dfa5a 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java
@@ -5,10 +5,9 @@ import com.yahoo.component.Version;
 import com.yahoo.config.application.api.DeploymentSpec;
 import com.yahoo.config.application.api.ValidationOverrides;
 import com.yahoo.config.provision.ApplicationId;
-import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.config.provision.zone.ZoneId;
 import com.yahoo.security.KeyUtils;
-import com.yahoo.vespa.config.SlimeUtils;
+import com.yahoo.slime.SlimeUtils;
 import com.yahoo.vespa.hosted.controller.Application;
 import com.yahoo.vespa.hosted.controller.Instance;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion;
@@ -18,7 +17,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId;
 import com.yahoo.vespa.hosted.controller.api.integration.organization.User;
 import com.yahoo.vespa.hosted.controller.application.AssignedRotation;
 import com.yahoo.vespa.hosted.controller.application.Change;
-import com.yahoo.vespa.hosted.controller.application.ClusterInfo;
 import com.yahoo.vespa.hosted.controller.application.Deployment;
 import com.yahoo.vespa.hosted.controller.application.DeploymentActivity;
 import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics;
@@ -36,7 +34,6 @@ import java.security.PublicKey;
 import java.time.Instant;
 import java.time.temporal.ChronoUnit;
 import java.util.ArrayList;
-import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
@@ -99,7 +96,6 @@ public class ApplicationSerializerTest {
         Instant activityAt = Instant.parse("2018-06-01T10:15:30.00Z");
         deployments.add(new Deployment(zone1, applicationVersion1, Version.fromString("1.2.3"), Instant.ofEpochMilli(3))); // One deployment without cluster info and utils
         deployments.add(new Deployment(zone2, applicationVersion2, Version.fromString("1.2.3"), Instant.ofEpochMilli(5),
-                                       createClusterInfo(3, 4),
                                        new DeploymentMetrics(2, 3, 4, 5, 6,
                                                              Optional.of(Instant.now().truncatedTo(ChronoUnit.MILLIS)),
                                                              Map.of(DeploymentMetrics.Warning.all, 3)),
@@ -187,16 +183,6 @@ public class ApplicationSerializerTest {
         assertEquals(original.require(id1.instance()).change(), serialized.require(id1.instance()).change());
         assertEquals(original.require(id3.instance()).change(), serialized.require(id3.instance()).change());
 
-        // Test cluster info
-        assertEquals(3, serialized.require(id1.instance()).deployments().get(zone2).clusterInfo().size());
-        assertEquals(10, serialized.require(id1.instance()).deployments().get(zone2).clusterInfo().get(ClusterSpec.Id.from("id2")).getFlavorCost());
-        assertEquals(ClusterSpec.Type.content, serialized.require(id1.instance()).deployments().get(zone2).clusterInfo().get(ClusterSpec.Id.from("id2")).getClusterType());
-        assertEquals("flavor2", serialized.require(id1.instance()).deployments().get(zone2).clusterInfo().get(ClusterSpec.Id.from("id2")).getFlavor());
-        assertEquals(4, serialized.require(id1.instance()).deployments().get(zone2).clusterInfo().get(ClusterSpec.Id.from("id2")).getHostnames().size());
-        assertEquals(2, serialized.require(id1.instance()).deployments().get(zone2).clusterInfo().get(ClusterSpec.Id.from("id2")).getFlavorCPU(), Double.MIN_VALUE);
-        assertEquals(4, serialized.require(id1.instance()).deployments().get(zone2).clusterInfo().get(ClusterSpec.Id.from("id2")).getFlavorMem(), Double.MIN_VALUE);
-        assertEquals(50, serialized.require(id1.instance()).deployments().get(zone2).clusterInfo().get(ClusterSpec.Id.from("id2")).getFlavorDisk(), Double.MIN_VALUE);
-
         // Test metrics
         assertEquals(original.metrics().queryServiceQuality(), serialized.metrics().queryServiceQuality(), Double.MIN_VALUE);
         assertEquals(original.metrics().writeServiceQuality(), serialized.metrics().writeServiceQuality(), Double.MIN_VALUE);
@@ -209,21 +195,6 @@ public class ApplicationSerializerTest {
         assertEquals(original.require(id1.instance()).deployments().get(zone2).metrics().warnings(), serialized.require(id1.instance()).deployments().get(zone2).metrics().warnings());
     }
 
-    private Map<ClusterSpec.Id, ClusterInfo> createClusterInfo(int clusters, int hosts) {
-        Map<ClusterSpec.Id, ClusterInfo> result = new HashMap<>();
-
-        for (int cluster = 0; cluster < clusters; cluster++) {
-            List<String> hostnames = new ArrayList<>();
-            for (int host = 0; host < hosts; host++) {
-                hostnames.add("hostname" + cluster*host + host);
-            }
-
-            result.put(ClusterSpec.Id.from("id" + cluster), new ClusterInfo("flavor" + cluster, 10,
-                2, 4, 50, ClusterSpec.Type.content, hostnames));
-        }
-        return result;
-    }
-
     @Test
     public void testCompleteApplicationDeserialization() throws Exception {
         byte[] applicationJson = Files.readAllBytes(testData.resolve("complete-application.json"));
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateMetadataSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateMetadataSerializerTest.java
new file mode 100644
index 00000000000..3c80245c025
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateMetadataSerializerTest.java
@@ -0,0 +1,47 @@
+package com.yahoo.vespa.hosted.controller.persistence;
+
+import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMetadata;
+import org.junit.Test;
+
+import java.util.List;
+import java.util.Optional;
+
+import static org.junit.Assert.*;
+
+public class EndpointCertificateMetadataSerializerTest {
+
+    private final EndpointCertificateMetadata sample =
+            new EndpointCertificateMetadata("keyName", "certName", 1);
+    private final EndpointCertificateMetadata sampleWithRequestMetadata =
+            new EndpointCertificateMetadata("keyName", "certName", 1, Optional.of("requestId"), Optional.of(List.of("SAN1", "SAN2")), Optional.of("issuer"));
+
+    @Test
+    public void serialize() {
+        assertEquals(
+                "{\"keyName\":\"keyName\",\"certName\":\"certName\",\"version\":1}",
+                EndpointCertificateMetadataSerializer.toSlime(sample).toString());
+    }
+
+    @Test
+    public void serializeWithRequestMetadata() {
+        assertEquals(
+                "{\"keyName\":\"keyName\",\"certName\":\"certName\",\"version\":1,\"requestId\":\"requestId\",\"requestedDnsSans\":[\"SAN1\",\"SAN2\"],\"issuer\":\"issuer\"}",
+                EndpointCertificateMetadataSerializer.toSlime(sampleWithRequestMetadata).toString());
+    }
+
+    @Test
+    public void deserializeFromJson() {
+        assertEquals(
+                sample,
+                EndpointCertificateMetadataSerializer.fromJsonString(
+                        "{\"keyName\":\"keyName\",\"certName\":\"certName\",\"version\":1}"));
+    }
+
+    @Test
+    public void deserializeFromJsonWithRequestMetadata() {
+        assertEquals(
+                sampleWithRequestMetadata,
+                EndpointCertificateMetadataSerializer.fromJsonString(
+                        "{\"keyName\":\"keyName\",\"certName\":\"certName\",\"version\":1,\"requestId\":\"requestId\",\"requestedDnsSans\":[\"SAN1\",\"SAN2\"],\"issuer\":\"issuer\"}"));
+    }
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializerTest.java
index c9ec5adc98c..7ca964e06dd 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializerTest.java
@@ -6,7 +6,7 @@ import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.config.provision.HostName;
 import com.yahoo.config.provision.zone.ZoneId;
-import com.yahoo.vespa.config.SlimeUtils;
+import com.yahoo.slime.SlimeUtils;
 import com.yahoo.vespa.hosted.controller.application.EndpointId;
 import com.yahoo.vespa.hosted.controller.routing.GlobalRouting;
 import com.yahoo.vespa.hosted.controller.routing.RoutingPolicy;
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java
index e5757604caf..b0a0cdf6293 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java
@@ -5,12 +5,13 @@ import com.google.common.collect.ImmutableMap;
 import com.yahoo.component.Version;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.security.X509CertificateUtils;
-import com.yahoo.vespa.config.SlimeUtils;
+import com.yahoo.slime.SlimeUtils;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision;
 import com.yahoo.vespa.hosted.controller.deployment.ConvergenceSummary;
+import com.yahoo.vespa.hosted.controller.deployment.JobProfile;
 import com.yahoo.vespa.hosted.controller.deployment.Run;
 import com.yahoo.vespa.hosted.controller.deployment.RunStatus;
 import com.yahoo.vespa.hosted.controller.deployment.Step;
@@ -54,8 +55,8 @@ public class RunSerializerTest {
     private static final RunSerializer serializer = new RunSerializer();
     private static final Path runFile = Paths.get("src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json");
     private static final RunId id = new RunId(ApplicationId.from("tenant", "application", "default"),
-                                               JobType.productionUsEast3,
-                                               (long) 112358);
+                                              JobType.productionUsEast3,
+                                              112358);
     private static final Instant start = Instant.parse("2007-12-03T10:15:30.00Z");
 
     @Test
@@ -148,7 +149,7 @@ public class RunSerializerTest {
         assertEquals(run.versions(), phoenix.versions());
         assertEquals(run.steps(), phoenix.steps());
 
-        Run initial = Run.initial(id, run.versions(), run.start());
+        Run initial = Run.initial(id, run.versions(), run.start(), JobProfile.production);
         assertEquals(initial, serializer.runFromSlime(serializer.toSlime(initial)));
     }
 
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializerTest.java
index ff1c952c2a5..9c085bb72f7 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializerTest.java
@@ -12,7 +12,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.organization.Contact;
 import com.yahoo.vespa.hosted.controller.api.role.SimplePrincipal;
 import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant;
 import com.yahoo.vespa.hosted.controller.tenant.CloudTenant;
-import com.yahoo.vespa.hosted.controller.tenant.UserTenant;
 import org.junit.Test;
 
 import java.net.URI;
@@ -77,14 +76,6 @@ public class TenantSerializerTest {
     }
 
     @Test
-    public void user_tenant() {
-        UserTenant tenant = UserTenant.create("by-foo", Optional.of(contact()));
-        UserTenant serialized = (UserTenant) serializer.tenantFrom(serializer.toSlime(tenant));
-        assertEquals(tenant.name(), serialized.name());
-        assertEquals(contact(), serialized.contact().get());
-    }
-
-    @Test
     public void cloud_tenant() {
         CloudTenant tenant = new CloudTenant(TenantName.from("elderly-lady"),
                                              new BillingInfo("old cat lady", "vespa"),
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializerTest.java
index c224e24618e..0c1f94d90cf 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializerTest.java
@@ -28,18 +28,11 @@ public class VersionStatusSerializerTest {
     @Test
     public void testSerialization() {
         List<VespaVersion> vespaVersions = new ArrayList<>();
-        DeploymentStatistics statistics = new DeploymentStatistics(
-                Version.fromString("5.0"),
-                Collections.singletonList(ApplicationId.from("tenant1", "failing1", "default")),
-                List.of(ApplicationId.from("tenant2", "success1", "default"),
-                        ApplicationId.from("tenant2", "success2", "default")),
-                List.of(ApplicationId.from("tenant1", "failing1", "default"),
-                        ApplicationId.from("tenant2", "success2", "default"))
-        );
-        vespaVersions.add(new VespaVersion(statistics, "dead", Instant.now(), false, false,
+        Version version = Version.fromString("5.0");
+        vespaVersions.add(new VespaVersion(version, "dead", Instant.now(), false, false,
                                            true, nodeVersions(Version.fromString("5.0"), Version.fromString("5.1"),
                                                               Instant.ofEpochMilli(123), "cfg1", "cfg2", "cfg3"), VespaVersion.Confidence.normal));
-        vespaVersions.add(new VespaVersion(statistics, "cafe", Instant.now(), true, true,
+        vespaVersions.add(new VespaVersion(version, "cafe", Instant.now(), true, true,
                                            false, nodeVersions(Version.fromString("5.0"), Version.fromString("5.1"),
                                                                Instant.ofEpochMilli(456), "cfg1", "cfg2", "cfg3"), VespaVersion.Confidence.normal));
         VersionStatus status = new VersionStatus(vespaVersions);
@@ -55,7 +48,7 @@ public class VersionStatusSerializerTest {
             assertEquals(a.isControllerVersion(), b.isControllerVersion());
             assertEquals(a.isSystemVersion(), b.isSystemVersion());
             assertEquals(a.isReleased(), b.isReleased());
-            assertEquals(a.statistics(), b.statistics());
+            assertEquals(a.versionNumber(), b.versionNumber());
             assertEquals(a.nodeVersions(), b.nodeVersions());
             assertEquals(a.confidence(), b.confidence());
         }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerTester.java
index df8787a2a4b..62b52d0d087 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerTester.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerTester.java
@@ -5,24 +5,16 @@ import com.yahoo.application.container.JDisc;
 import com.yahoo.application.container.handler.Request;
 import com.yahoo.application.container.handler.Response;
 import com.yahoo.component.ComponentSpecification;
-import com.yahoo.component.Version;
 import com.yahoo.config.provision.ApplicationName;
-import com.yahoo.config.provision.zone.ZoneApi;
 import com.yahoo.container.http.filter.FilterChainRepository;
 import com.yahoo.jdisc.http.filter.SecurityRequestFilter;
 import com.yahoo.jdisc.http.filter.SecurityRequestFilterChain;
 import com.yahoo.vespa.athenz.api.AthenzDomain;
 import com.yahoo.vespa.athenz.api.AthenzIdentity;
 import com.yahoo.vespa.hosted.controller.Controller;
-import com.yahoo.vespa.hosted.controller.api.identifiers.ScrewdriverId;
 import com.yahoo.vespa.hosted.controller.api.integration.athenz.ApplicationAction;
 import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzClientFactoryMock;
-import com.yahoo.vespa.hosted.controller.application.SystemApplication;
-import com.yahoo.vespa.hosted.controller.athenz.HostedAthenzIdentities;
-import com.yahoo.vespa.hosted.controller.integration.ConfigServerMock;
 import com.yahoo.vespa.hosted.controller.integration.ServiceRegistryMock;
-import com.yahoo.vespa.hosted.controller.versions.ControllerVersion;
-import com.yahoo.vespa.hosted.controller.versions.VersionStatus;
 import org.junit.ComparisonFailure;
 
 import java.io.File;
@@ -32,7 +24,6 @@ import java.nio.charset.CharacterCodingException;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Paths;
-import java.time.Instant;
 import java.util.Optional;
 import java.util.function.Consumer;
 import java.util.function.Supplier;
@@ -54,17 +45,11 @@ public class ContainerTester {
         this.container = container;
         this.responseFilePath = responseFilePath;
     }
-    
-    public JDisc container() { return container; }
 
     public Controller controller() {
         return (Controller) container.components().getComponent(Controller.class.getName());
     }
 
-    public ConfigServerMock configServer() {
-        return serviceRegistry().configServerMock();
-    }
-
     public AthenzClientFactoryMock athenzClientFactory() {
         return (AthenzClientFactoryMock) container.components().getComponent(AthenzClientFactoryMock.class.getName());
     }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerCloudTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerCloudTest.java
index 16447f47ee8..8dd1f9ad10a 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerCloudTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerCloudTest.java
@@ -1,12 +1,14 @@
 // Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.restapi;
 
+import ai.vespa.hosted.api.MultiPartStreamer;
 import com.yahoo.application.container.handler.Request;
 import com.yahoo.config.provision.SystemName;
 import com.yahoo.vespa.hosted.controller.api.integration.user.User;
 import com.yahoo.vespa.hosted.controller.api.role.Role;
 import com.yahoo.vespa.hosted.controller.api.role.SecurityContext;
 import com.yahoo.vespa.hosted.controller.api.role.SimplePrincipal;
+import com.yahoo.yolean.Exceptions;
 
 import java.nio.charset.StandardCharsets;
 import java.security.Principal;
@@ -61,19 +63,24 @@ public class ControllerContainerCloudTest extends ControllerContainerTest {
     protected RequestBuilder request(String path) { return new RequestBuilder(path, Request.Method.GET); }
     protected RequestBuilder request(String path, Request.Method method) { return new RequestBuilder(path, method); }
 
-    protected class RequestBuilder implements Supplier<Request> {
+    protected static class RequestBuilder implements Supplier<Request> {
         private final String path;
         private final Request.Method method;
         private byte[] data = new byte[0];
         private Principal principal = () -> "user@test";
         private User user;
         private Set<Role> roles = Set.of(Role.everyone());
+        private String contentType;
 
         private RequestBuilder(String path, Request.Method method) {
             this.path = path;
             this.method = method;
         }
 
+        public RequestBuilder contentType(String contentType) { this.contentType = contentType; return this; }
+        public RequestBuilder data(MultiPartStreamer streamer) {
+            return Exceptions.uncheck(() -> data(streamer.data().readAllBytes()).contentType(streamer.contentType()));
+        }
         public RequestBuilder data(byte[] data) { this.data = data; return this; }
         public RequestBuilder data(String data) { this.data = data.getBytes(StandardCharsets.UTF_8); return this; }
         public RequestBuilder principal(String principal) { this.principal = new SimplePrincipal(principal); return this; }
@@ -85,7 +92,7 @@ public class ControllerContainerCloudTest extends ControllerContainerTest {
             Request request = new Request("http://localhost:8080" + path, data, method, principal);
             request.getAttributes().put(SecurityContext.ATTRIBUTE_NAME, new SecurityContext(principal, roles));
             if (user != null) request.getAttributes().put(User.ATTRIBUTE_NAME, user);
-            request.getHeaders().put("Content-Type", "application/json");
+            request.getHeaders().put("Content-Type", contentType);
             return request;
         }
     }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java
index b7adc3064fa..ad8a3409eef 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java
@@ -4,7 +4,6 @@ package com.yahoo.vespa.hosted.controller.restapi;
 import com.yahoo.application.Networking;
 import com.yahoo.application.container.JDisc;
 import com.yahoo.application.container.handler.Request;
-import com.yahoo.application.container.handler.Response;
 import com.yahoo.config.provision.SystemName;
 import com.yahoo.vespa.athenz.api.AthenzIdentity;
 import com.yahoo.vespa.athenz.api.AthenzUser;
@@ -14,13 +13,9 @@ import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzClientFact
 import org.junit.After;
 import org.junit.Before;
 
-import java.io.UncheckedIOException;
-import java.nio.charset.CharacterCodingException;
-
 import static com.yahoo.vespa.hosted.controller.integration.AthenzFilterMock.IDENTITY_HEADER_NAME;
 import static com.yahoo.vespa.hosted.controller.integration.AthenzFilterMock.OKTA_ACCESS_TOKEN_HEADER_NAME;
 import static com.yahoo.vespa.hosted.controller.integration.AthenzFilterMock.OKTA_IDENTITY_TOKEN_HEADER_NAME;
-import static org.junit.Assert.assertEquals;
 
 /**
  * Superclass of REST API tests which needs to set up a functional container instance.
@@ -65,6 +60,8 @@ public class ControllerContainerTest {
                "      <item key=\"rotation-id-5\">rotation-fqdn-5</item>\n" +
                "    </rotations>\n" +
                "  </config>\n" +
+               "  " +
+               "<accesslog type='disabled'/>\n" +
                "  <component id='com.yahoo.vespa.flags.InMemoryFlagSource'/>\n" +
                "  <component id='com.yahoo.vespa.configserver.flags.db.FlagsDbImpl'/>\n" +
                "  <component id='com.yahoo.vespa.curator.mock.MockCurator'/>\n" +
@@ -90,9 +87,6 @@ public class ControllerContainerTest {
                "  <handler id='com.yahoo.vespa.hosted.controller.restapi.os.OsApiHandler'>\n" +
                "    <binding>http://*/os/v1/*</binding>\n" +
                "  </handler>\n" +
-               "  <handler id='com.yahoo.vespa.hosted.controller.restapi.cost.CostApiHandler'>\n" +
-               "    <binding>http://*/cost/v1/*</binding>\n" +
-               "  </handler>\n" +
                "  <handler id='com.yahoo.vespa.hosted.controller.restapi.zone.v2.ZoneApiHandler'>\n" +
                "    <binding>http://*/zone/v2</binding>\n" +
                "    <binding>http://*/zone/v2/*</binding>\n" +
@@ -151,17 +145,6 @@ public class ControllerContainerTest {
                "  </http>\n";
     }
 
-    protected void assertResponse(Request request, int responseStatus, String responseMessage) {
-        Response response = container.handleRequest(request);
-        // Compare both status and message at once for easier diagnosis
-        try {
-            assertEquals("status: " + responseStatus + "\nmessage: " + responseMessage,
-                         "status: " + response.getStatus() + "\nmessage: " + response.getBodyAsString());
-        } catch (CharacterCodingException e) {
-            throw new UncheckedIOException(e);
-        }
-    }
-
     protected static Request authenticatedRequest(String uri) {
         return addIdentityToRequest(new Request(uri), defaultUser);
     }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ResponseHandlerToApplicationResponseWrapper.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ResponseHandlerToApplicationResponseWrapper.java
index 55cfc075c7c..1af5de3e4ea 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ResponseHandlerToApplicationResponseWrapper.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ResponseHandlerToApplicationResponseWrapper.java
@@ -39,7 +39,7 @@ public class ResponseHandlerToApplicationResponseWrapper implements ResponseHand
                 });
     }
 
-    private class SimpleContentChannel implements ContentChannel {
+    private static class SimpleContentChannel implements ContentChannel {
 
         private final Queue<ByteBuffer> buffers = new ConcurrentLinkedQueue<>();
         private final AtomicBoolean closed = new AtomicBoolean(false);
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java
new file mode 100644
index 00000000000..5c5dc4b5fe6
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java
@@ -0,0 +1,78 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.restapi.application;
+
+import com.yahoo.config.provision.ApplicationName;
+import com.yahoo.config.provision.Environment;
+import com.yahoo.config.provision.TenantName;
+import com.yahoo.vespa.hosted.controller.api.integration.user.User;
+import com.yahoo.vespa.hosted.controller.ControllerTester;
+import com.yahoo.vespa.hosted.controller.api.role.Role;
+import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId;
+import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder;
+import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester;
+import com.yahoo.vespa.hosted.controller.restapi.ContainerTester;
+import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerCloudTest;
+import com.yahoo.vespa.hosted.controller.security.CloudTenantSpec;
+import com.yahoo.vespa.hosted.controller.security.Credentials;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.Set;
+
+import static com.yahoo.application.container.handler.Request.Method.POST;
+import static com.yahoo.vespa.hosted.controller.restapi.application.ApplicationApiTest.createApplicationSubmissionData;
+
+public class ApplicationApiCloudTest extends ControllerContainerCloudTest {
+
+    private static final String responseFiles = "src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/";
+
+    private ContainerTester tester;
+    private DeploymentTester deploymentTester;
+
+    private static final TenantName tenantName = TenantName.from("scoober");
+    private static final ApplicationName applicationName = ApplicationName.from("albums");
+    private static final User developerUser = new User("developer", "Joe Developer", "", "");
+
+    @Before
+    public void before() {
+        tester = new ContainerTester(container, responseFiles);
+        deploymentTester = new DeploymentTester(new ControllerTester(tester));
+        deploymentTester.controllerTester().computeVersionStatus();
+    }
+
+    @Test
+    public void test_missing_security_clients_pem() {
+        setupTenantAndApplication();
+
+        var application = prodBuilder().build();
+
+        var deployRequest = request("/application/v4/tenant/scoober/application/albums/submit", POST)
+                .data(createApplicationSubmissionData(application, 0))
+                .roles(Set.of(Role.developer(tenantName)));
+
+        tester.assertResponse(
+                deployRequest,
+                "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Missing required file 'security/clients.pem'\"}",
+                400);
+    }
+
+
+    private ApplicationPackageBuilder prodBuilder() {
+        return new ApplicationPackageBuilder()
+                .instances("default")
+                .environment(Environment.prod)
+                .region("aws-us-east-1a");
+    }
+
+    private void setupTenantAndApplication() {
+        var tenantSpec = new CloudTenantSpec(tenantName, "");
+        tester.controller().tenants().create(tenantSpec, credentials("developer@scoober"));
+
+        var appId = TenantAndApplicationId.from(tenantName, applicationName);
+        tester.controller().applications().createApplication(appId, credentials("developer@scoober"));
+    }
+
+    private static Credentials credentials(String name) {
+        return new Credentials(() -> name);
+    }
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
index e96d25c2cab..2752ba64b61 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
@@ -9,10 +9,10 @@ import com.yahoo.config.application.api.ValidationId;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.ApplicationName;
 import com.yahoo.config.provision.AthenzService;
-import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.config.provision.Environment;
 import com.yahoo.config.provision.RegionName;
 import com.yahoo.config.provision.TenantName;
+import com.yahoo.config.provision.zone.RoutingMethod;
 import com.yahoo.config.provision.zone.ZoneId;
 import com.yahoo.vespa.athenz.api.AthenzDomain;
 import com.yahoo.vespa.athenz.api.AthenzIdentity;
@@ -24,6 +24,7 @@ import com.yahoo.vespa.hosted.controller.Application;
 import com.yahoo.vespa.hosted.controller.ControllerTester;
 import com.yahoo.vespa.hosted.controller.Instance;
 import com.yahoo.vespa.hosted.controller.LockedTenant;
+import com.yahoo.vespa.hosted.controller.RoutingController;
 import com.yahoo.vespa.hosted.controller.api.application.v4.EnvironmentResource;
 import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions;
 import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
@@ -40,15 +41,13 @@ import com.yahoo.vespa.hosted.controller.api.integration.organization.Contact;
 import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId;
 import com.yahoo.vespa.hosted.controller.api.integration.organization.User;
 import com.yahoo.vespa.hosted.controller.api.integration.resource.CostInfo;
-import com.yahoo.vespa.hosted.controller.api.integration.resource.MeteringInfo;
+import com.yahoo.vespa.hosted.controller.api.integration.resource.MeteringData;
 import com.yahoo.vespa.hosted.controller.api.integration.resource.MockTenantCost;
 import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceAllocation;
 import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceSnapshot;
-import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingEndpoint;
 import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMeteringClient;
 import com.yahoo.vespa.hosted.controller.application.ApplicationPackage;
 import com.yahoo.vespa.hosted.controller.application.Change;
-import com.yahoo.vespa.hosted.controller.application.ClusterInfo;
 import com.yahoo.vespa.hosted.controller.application.Deployment;
 import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics;
 import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId;
@@ -58,6 +57,7 @@ import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext;
 import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester;
 import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger;
 import com.yahoo.vespa.hosted.controller.integration.ConfigServerMock;
+import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock;
 import com.yahoo.vespa.hosted.controller.maintenance.JobControl;
 import com.yahoo.vespa.hosted.controller.maintenance.RotationStatusUpdater;
 import com.yahoo.vespa.hosted.controller.metric.ApplicationMetrics;
@@ -75,7 +75,6 @@ import org.junit.Test;
 import java.io.File;
 import java.math.BigDecimal;
 import java.net.URI;
-import java.nio.charset.StandardCharsets;
 import java.time.Duration;
 import java.time.Instant;
 import java.time.YearMonth;
@@ -96,6 +95,9 @@ import static com.yahoo.application.container.handler.Request.Method.GET;
 import static com.yahoo.application.container.handler.Request.Method.PATCH;
 import static com.yahoo.application.container.handler.Request.Method.POST;
 import static com.yahoo.application.container.handler.Request.Method.PUT;
+import static java.net.URLEncoder.encode;
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static java.util.stream.Collectors.joining;
 import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
@@ -174,21 +176,6 @@ public class ApplicationApiTest extends ControllerContainerTest {
                                       .oktaAccessToken(OKTA_AT).oktaIdentityToken(OKTA_IT)
                                       .data("{\"athensDomain\":\"domain1\", \"property\":\"property1\"}"),
                               new File("tenant-without-applications.json"));
-        // GET the authenticated user (with associated tenants)
-        tester.assertResponse(request("/application/v4/user", GET).userIdentity(USER_ID),
-                              new File("user.json"));
-        // PUT a user tenant
-        tester.assertResponse(request("/application/v4/user", PUT).userIdentity(USER_ID),
-                              "{\"message\":\"Created user 'by-myuser'\"}");
-        // GET the authenticated user which now exists (with associated tenants)
-        tester.assertResponse(request("/application/v4/user", GET).userIdentity(USER_ID),
-                              new File("user-which-exists.json"));
-        // DELETE the user
-        tester.assertResponse(request("/application/v4/tenant/by-myuser", DELETE).userIdentity(USER_ID),
-                              "{\"tenant\":\"by-myuser\",\"type\":\"USER\",\"applications\":[]}");
-        // GET all tenants
-        tester.assertResponse(request("/application/v4/tenant/", GET).userIdentity(USER_ID),
-                              new File("tenant-list.json"));
 
         // GET list of months for a tenant
         tester.assertResponse(request("/application/v4/tenant/tenant1/cost", GET).userIdentity(USER_ID).oktaAccessToken(OKTA_AT).oktaIdentityToken(OKTA_IT),
@@ -231,24 +218,59 @@ public class ApplicationApiTest extends ControllerContainerTest {
 
         // GET tenant applications
         tester.assertResponse(request("/application/v4/tenant/tenant1/application/", GET).userIdentity(USER_ID),
-                              new File("instance-list.json"));
+                              new File("application-list.json"));
         // GET tenant applications (instances of "application1" only)
         tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/", GET).userIdentity(USER_ID),
-                              new File("instance-list.json"));
+                              new File("application-list.json"));
+        // GET at a tenant, with "&recursive=true&production=true", recurses over no instances yet, as they are not in deployment spec.
+        tester.assertResponse(request("/application/v4/tenant/tenant1/", GET)
+                                      .userIdentity(USER_ID)
+                                      .properties(Map.of("recursive", "true",
+                                                         "production", "true")),
+                              new File("tenant-without-applications.json"));
+        // GET at an application, with "&recursive=true&production=true", recurses over no instances yet, as they are not in deployment spec.
+        tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", GET)
+                                      .userIdentity(USER_ID)
+                                      .properties(Map.of("recursive", "true",
+                                                         "production", "true")),
+                              new File("application-without-instances.json"));
 
         addUserToHostedOperatorRole(HostedAthenzIdentities.from(HOSTED_VESPA_OPERATOR));
 
         ApplicationId id = ApplicationId.from("tenant1", "application1", "instance1");
         var app1 = deploymentTester.newDeploymentContext(id);
 
-        // POST (deploy) an application to start a manual deployment to dev
+        // POST (deploy) an application to start a manual deployment in prod is not allowed
         MultiPartStreamer entity = createApplicationDeployData(applicationPackageInstance1, true);
+        tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploy/production-us-east-3/", POST)
+                                      .data(entity)
+                                      .userIdentity(USER_ID),
+                              "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Direct deployments are only allowed to manually deployed environments.\"}", 400);
+
+        // POST (deploy) an application to start a manual deployment in prod is allowed for operators
+        tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploy/production-us-east-3/", POST)
+                                      .data(entity)
+                                      .userIdentity(HOSTED_VESPA_OPERATOR),
+                              "{\"message\":\"Deployment started in run 1 of production-us-east-3 for tenant1.application1.instance1. This may take about 15 minutes the first time.\",\"run\":1}");
+        app1.runJob(JobType.productionUsEast3);
+        tester.controller().applications().deactivate(app1.instanceId(), ZoneId.from("prod", "us-east-3"));
+
+        // POST (deploy) an application to start a manual deployment to dev
         tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploy/dev-us-east-1/", POST)
                                       .data(entity)
                                       .userIdentity(USER_ID),
                               "{\"message\":\"Deployment started in run 1 of dev-us-east-1 for tenant1.application1.instance1. This may take about 15 minutes the first time.\",\"run\":1}");
         app1.runJob(JobType.devUsEast1);
 
+        // GET dev application package
+        tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/job/dev-us-east-1/package", GET)
+                                      .userIdentity(USER_ID),
+                              (response) -> {
+                                  assertEquals("attachment; filename=\"tenant1.application1.instance1.dev.us-east-1.zip\"", response.getHeaders().getFirst("Content-Disposition"));
+                                  assertArrayEquals(applicationPackageInstance1.zippedContent(), response.getBody());
+                              },
+                              200);
+
         // POST an application package is not generally allowed under user instance
         tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/otheruser/deploy/dev-us-east-1", POST)
                                       .userIdentity(OTHER_USER_ID)
@@ -355,6 +377,18 @@ public class ApplicationApiTest extends ControllerContainerTest {
                                       .data("{\"skipTests\":true}")
                                       .userIdentity(USER_ID),
                               "{\"message\":\"Triggered production-us-west-1 for tenant2.application2.instance1\"}");
+        app2.runJob(JobType.productionUsWest1);
+
+        // POST a re-triggering to force a production job to start with previous parameters
+        tester.assertResponse(request("/application/v4/tenant/tenant2/application/application2/instance/instance1/job/production-us-west-1", POST)
+                                      .data("{\"reTrigger\":true}")
+                                      .userIdentity(USER_ID),
+                              "{\"message\":\"Triggered production-us-west-1 for tenant2.application2.instance1\"}");
+
+        // DELETE manually deployed prod deployment again
+        tester.assertResponse(request("/application/v4/tenant/tenant2/application/application2/instance/instance1/environment/prod/region/us-west-1", DELETE)
+                                      .userIdentity(HOSTED_VESPA_OPERATOR),
+                              "{\"message\":\"Deactivated tenant2.application2.instance1 in prod.us-west-1\"}");
 
         // GET application having both change and outstanding change
         tester.assertResponse(request("/application/v4/tenant/tenant2/application/application2", GET)
@@ -395,6 +429,11 @@ public class ApplicationApiTest extends ControllerContainerTest {
                                       .data("{\"majorVersion\":null}"),
                               "{\"message\":\"Set major version to empty\"}");
 
+        // GET compile version for an application
+        tester.assertResponse(request("/application/v4/tenant/tenant2/application/application2/compile-version", GET)
+                                      .userIdentity(USER_ID),
+                              "{\"compileVersion\":\"6.1.0\"}");
+
         // DELETE the pem deploy key
         tester.assertResponse(request("/application/v4/tenant/tenant2/application/application2/key", DELETE)
                                       .userIdentity(USER_ID)
@@ -482,7 +521,7 @@ public class ApplicationApiTest extends ControllerContainerTest {
                               "{\"message\":\"Triggered pin to 6.1 for tenant1.application1.instance1\"}");
         assertTrue("Action is logged to audit log",
                    tester.controller().auditLogger().readLog().entries().stream()
-                         .anyMatch(entry -> entry.resource().equals("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploying/pin")));
+                         .anyMatch(entry -> entry.resource().equals("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploying/pin?")));
         tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploying", GET)
                                       .userIdentity(USER_ID), "{\"platform\":\"6.1\",\"pinned\":true}");
         tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploying/pin", GET)
@@ -539,9 +578,11 @@ public class ApplicationApiTest extends ControllerContainerTest {
 
         // POST a 'restart application' command
         tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/instance1/restart", POST)
-                                      .screwdriverIdentity(SCREWDRIVER_ID),
+                                      .userIdentity(HOSTED_VESPA_OPERATOR),
                               "{\"message\":\"Requested restart of tenant1.application1.instance1 in prod.us-central-1\"}");
 
+        addUserToHostedOperatorRole(HostedAthenzIdentities.from(SCREWDRIVER_ID));
+
         // POST a 'restart application' in staging environment command
         tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/staging/region/us-central-1/instance/instance1/restart", POST)
                                       .screwdriverIdentity(SCREWDRIVER_ID),
@@ -558,7 +599,8 @@ public class ApplicationApiTest extends ControllerContainerTest {
                               "{\"message\":\"Requested restart of tenant1.application1.instance1 in dev.us-central-1\"}");
 
         // POST a 'restart application' command with a host filter (other filters not supported yet)
-        tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/instance1/restart?hostname=node-1-tenant-host-prod.us-central-1", POST)
+        tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/instance1/restart", POST)
+                                      .properties(Map.of("hostname", "node-1-tenant-host-prod.us-central-1"))
                                       .screwdriverIdentity(SCREWDRIVER_ID),
                               "{\"message\":\"Requested restart of tenant1.application1.instance1 in prod.us-central-1\"}", 200);
 
@@ -607,10 +649,7 @@ public class ApplicationApiTest extends ControllerContainerTest {
                                                   ZoneId.from("dev", "us-east-1"),
                                                   Optional.of(applicationPackageDefault),
                                                   new DeployOptions(false, Optional.empty(), false, false));
-        tester.serviceRegistry().routingGeneratorMock().putEndpoints(new DeploymentId(ApplicationId.from("tenant1", "application1", "default"), ZoneId.from("prod", "us-central-1")),
-                                                                     List.of(new RoutingEndpoint("https://us-central-1.prod.default", "host", false, "upstream")));
-        tester.serviceRegistry().routingGeneratorMock().putEndpoints(new DeploymentId(ApplicationId.from("tenant1", "application1", "my-user"), ZoneId.from("dev", "us-east-1")),
-                                                                     List.of(new RoutingEndpoint("https://us-east-1.dev.my-user", "host", false, "upstream")));
+
         // GET test-config for local tests against a dev deployment
         tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/my-user/job/dev-us-east-1/test-config", GET)
                                       .userIdentity(USER_ID),
@@ -662,7 +701,9 @@ public class ApplicationApiTest extends ControllerContainerTest {
                               200);
 
         // GET application package for previous build
-        tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/package?build=1", GET).userIdentity(HOSTED_VESPA_OPERATOR),
+        tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/package", GET)
+                                      .properties(Map.of("build", "1"))
+                                      .userIdentity(HOSTED_VESPA_OPERATOR),
                               (response) -> {
                                   assertEquals("attachment; filename=\"tenant1.application1-build1.zip\"", response.getHeaders().getFirst("Content-Disposition"));
                                   assertArrayEquals(applicationPackageInstance1.zippedContent(), response.getBody());
@@ -724,18 +765,6 @@ public class ApplicationApiTest extends ControllerContainerTest {
                                       .userIdentity(USER_ID),
                               "{\"message\":\"Aborting run 2 of staging-test for tenant1.application1.instance1\"}");
 
-        // PUT (create) the authenticated user
-        byte[] data = new byte[0];
-        tester.assertResponse(request("/application/v4/user?user=new_user&domain=by", PUT)
-                                      .data(data)
-                                      .userIdentity(new UserId("new_user")), // Normalized to by-new-user by API
-                              new File("create-user-response.json"));
-
-        // GET user lists only tenants for the authenticated user
-        tester.assertResponse(request("/application/v4/user", GET)
-                                      .userIdentity(new UserId("other_user")),
-                              "{\"user\":\"other_user\",\"tenants\":[],\"tenantExists\":false}");
-
         // OPTIONS return 200 OK
         tester.assertResponse(request("/application/v4/", Request.Method.OPTIONS)
                                       .userIdentity(USER_ID),
@@ -776,8 +805,6 @@ public class ApplicationApiTest extends ControllerContainerTest {
         // Create tenant and deploy
         var app = deploymentTester.newDeploymentContext(createTenantAndApplication());
         app.submit(applicationPackage).deploy();
-        app.addRoutingPolicy(westZone, true);
-        app.addRoutingPolicy(eastZone, true);
 
         // Invalid application fails
         tester.assertResponse(request("/application/v4/tenant/tenant2/application/application2/environment/prod/region/us-west-1/instance/default/global-rotation", GET)
@@ -862,19 +889,22 @@ public class ApplicationApiTest extends ControllerContainerTest {
                               400);
 
         // GET global rotation status for us-west-1 in default endpoint
-        tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-west-1/global-rotation?endpointId=default", GET)
+        tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-west-1/global-rotation", GET)
+                                      .properties(Map.of("endpointId", "default"))
                                       .userIdentity(USER_ID),
                               "{\"bcpStatus\":{\"rotationStatus\":\"IN\"}}",
                               200);
 
         // GET global rotation status for us-west-1 in eu endpoint
-        tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-west-1/global-rotation?endpointId=eu", GET)
+        tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-west-1/global-rotation", GET)
+                                      .properties(Map.of("endpointId", "eu"))
                                       .userIdentity(USER_ID),
                               "{\"bcpStatus\":{\"rotationStatus\":\"UNKNOWN\"}}",
                               200);
 
         // GET global rotation status for eu-west-1 in eu endpoint
-        tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/eu-west-1/global-rotation?endpointId=eu", GET)
+        tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/eu-west-1/global-rotation", GET)
+                                      .properties(Map.of("endpointId", "eu"))
                                       .userIdentity(USER_ID),
                               "{\"bcpStatus\":{\"rotationStatus\":\"IN\"}}",
                               200);
@@ -898,10 +928,8 @@ public class ApplicationApiTest extends ControllerContainerTest {
                                       .oktaAccessToken(OKTA_AT).oktaIdentityToken(OKTA_IT),
                               new File("instance-reference.json"));
 
-        // Grant deploy access
-        addScrewdriverUserToDeployRole(SCREWDRIVER_ID,
-                                       ATHENZ_TENANT_DOMAIN,
-                                       ApplicationName.from("application1"));
+        // Add build service to operator role
+        addUserToHostedOperatorRole(HostedAthenzIdentities.from(SCREWDRIVER_ID));
 
         // POST (deploy) an application to a prod zone - allowed when project ID is not specified
         MultiPartStreamer entity = createApplicationDeployData(applicationPackageInstance1, true);
@@ -944,7 +972,7 @@ public class ApplicationApiTest extends ControllerContainerTest {
                 new ResourceSnapshot(applicationId, 1, 2,3, Instant.ofEpochMilli(246), ZoneId.defaultId()),
                 new ResourceSnapshot(applicationId, 1, 2,3, Instant.ofEpochMilli(492), ZoneId.defaultId())));
 
-        mockMeteringClient.setMeteringInfo(new MeteringInfo(thisMonth, lastMonth, currentSnapshot, snapshotHistory));
+        mockMeteringClient.setMeteringData(new MeteringData(thisMonth, lastMonth, currentSnapshot, snapshotHistory));
 
         tester.assertResponse(request("/application/v4/tenant/doesnotexist/application/doesnotexist/metering", GET)
                                       .userIdentity(USER_ID)
@@ -1008,6 +1036,12 @@ public class ApplicationApiTest extends ControllerContainerTest {
                               "{\"error-code\":\"NOT_FOUND\",\"message\":\"Tenant 'tenant1' does not exist\"}",
                               404);
 
+        // GET non-existing tenant's applications
+        tester.assertResponse(request("/application/v4/tenant/tenant1/application", GET)
+                                      .userIdentity(USER_ID),
+                              "{\"error-code\":\"NOT_FOUND\",\"message\":\"Tenant 'tenant1' does not exist\"}",
+                              404);
+
         // GET non-existing application
         tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", GET)
                                       .userIdentity(USER_ID),
@@ -1051,14 +1085,6 @@ public class ApplicationApiTest extends ControllerContainerTest {
                               "{\"error-code\":\"BAD_REQUEST\",\"message\":\"New tenant or application names must start with a letter, may contain no more than 20 characters, and may only contain lowercase letters, digits or dashes, but no double-dashes.\"}",
                               400);
 
-        // POST (add) an Athenz tenant with by- prefix
-        tester.assertResponse(request("/application/v4/tenant/by-tenant2", POST)
-                                      .userIdentity(USER_ID)
-                                      .data("{\"athensDomain\":\"domain1\", \"property\":\"property1\"}")
-                                      .oktaAccessToken(OKTA_AT).oktaIdentityToken(OKTA_IT),
-                              "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Athenz tenant name cannot have prefix 'by-'\"}",
-                              400);
-
         // POST (add) an Athenz tenant with a reserved name
         tester.assertResponse(request("/application/v4/tenant/hosted-vespa", POST)
                                       .userIdentity(USER_ID)
@@ -1089,12 +1115,16 @@ public class ApplicationApiTest extends ControllerContainerTest {
                               404);
 
         // GET non-existent application package of specific build
-        tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/package?build=42", GET).userIdentity(HOSTED_VESPA_OPERATOR),
+        tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/package", GET)
+                                      .properties(Map.of("build", "42"))
+                                      .userIdentity(HOSTED_VESPA_OPERATOR),
                               "{\"error-code\":\"NOT_FOUND\",\"message\":\"No application package found for 'tenant1.application1' with build number 42\"}",
                               404);
 
         // GET non-existent application package of invalid build
-        tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/package?build=foobar", GET).userIdentity(HOSTED_VESPA_OPERATOR),
+        tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/package", GET)
+                                      .properties(Map.of("build", "foobar"))
+                                      .userIdentity(HOSTED_VESPA_OPERATOR),
                               "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Invalid build number: For input string: \\\"foobar\\\"\"}",
                               400);
         
@@ -1334,26 +1364,13 @@ public class ApplicationApiTest extends ControllerContainerTest {
         createAthenzDomainWithAdmin(ATHENZ_TENANT_DOMAIN, tenantAdmin);
         allowLaunchOfService(new com.yahoo.vespa.athenz.api.AthenzService(ATHENZ_TENANT_DOMAIN, "service"));
 
-        // Create tenant
-        // PUT (create) the authenticated user
-        tester.assertResponse(request("/application/v4/user?user=new_user&domain=by", PUT)
-                                      .userIdentity(userId), // Normalized to by-new-user by API
-                              new File("create-user-response.json"));
-
         ApplicationPackage applicationPackage = new ApplicationPackageBuilder()
                 .athenzIdentity(com.yahoo.config.provision.AthenzDomain.from("domain1"), com.yahoo.config.provision.AthenzService.from("service"))
                 .build();
 
-        // POST (deploy) an application to a dev zone
-        MultiPartStreamer entity = createApplicationDeployData(applicationPackage, true);
-        tester.assertResponse(request("/application/v4/tenant/by-new-user/application/application1/environment/dev/region/us-west-1/instance/default", POST)
-                                      .data(entity)
-                                      .userIdentity(userId),
-                              "{\"error-code\":\"BAD_REQUEST\",\"message\":\"User user.new-user is not allowed to launch service domain1.service. Please reach out to the domain admin.\"}",
-                              400);
-
         createTenantAndApplication();
-        // POST (deploy) an application to dev through a deployment job
+        MultiPartStreamer entity = createApplicationDeployData(applicationPackage, true);
+        // POST (deploy) an application to dev through a deployment job, with user instance and a proper tenant
         tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/new-user/deploy/dev-us-east-1", POST)
                                       .data(entity)
                                       .userIdentity(userId),
@@ -1365,12 +1382,6 @@ public class ApplicationApiTest extends ControllerContainerTest {
                 .domains.get(ATHENZ_TENANT_DOMAIN)
                         .admin(HostedAthenzIdentities.from(userId));
 
-        // POST (deploy) an application to a dev zone
-        tester.assertResponse(request("/application/v4/tenant/by-new-user/application/application1/environment/dev/region/us-east-1/instance/default", POST)
-                                      .data(entity)
-                                      .userIdentity(userId),
-                              new File("deploy-result.json"));
-
         // POST (deploy) an application to dev through a deployment job
         tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/new-user/deploy/dev-us-east-1", POST)
                                       .data(entity)
@@ -1398,7 +1409,7 @@ public class ApplicationApiTest extends ControllerContainerTest {
         AthenzCredentials credentials = new AthenzCredentials(
                 new AthenzPrincipal(new AthenzUser(developer.id())), sandboxDomain, OKTA_IT, OKTA_AT);
         tester.controller().tenants().create(tenantSpec, credentials);
-        tester.controller().applications().createApplication(TenantAndApplicationId.from("sandbox", "myapp"), Optional.of(credentials));
+        tester.controller().applications().createApplication(TenantAndApplicationId.from("sandbox", "myapp"), credentials);
 
         // Create an application package referencing the service from the other domain
         ApplicationPackage applicationPackage = new ApplicationPackageBuilder()
@@ -1437,19 +1448,21 @@ public class ApplicationApiTest extends ControllerContainerTest {
 
     }
 
-
-
     @Test
     public void applicationWithRoutingPolicy() {
         var app = deploymentTester.newDeploymentContext(createTenantAndApplication());
+        var zone = ZoneId.from(Environment.prod, RegionName.from("us-west-1"));
+        deploymentTester.controllerTester().zoneRegistry().setRoutingMethod(ZoneApiMock.from(zone),
+                                                                            List.of(RoutingMethod.exclusive, RoutingMethod.shared));
         ApplicationPackage applicationPackage = new ApplicationPackageBuilder()
+                .athenzIdentity(com.yahoo.config.provision.AthenzDomain.from("domain"), AthenzService.from("service"))
+                .compileVersion(RoutingController.DIRECT_ROUTING_MIN_VERSION)
                 .environment(Environment.prod)
                 .instances("instance1")
-                .region("us-west-1")
+                .region(zone.region().value())
                 .build();
         app.submit(applicationPackage).deploy();
-        app.addRoutingPolicy(ZoneId.from(Environment.prod, RegionName.from("us-west-1")), true);
-        app.addRoutingPolicy(ZoneId.from(Environment.prod, RegionName.from("us-west-1")), false);
+        app.addInactiveRoutingPolicy(zone);
 
         // GET application
         tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1", GET)
@@ -1478,7 +1491,7 @@ public class ApplicationApiTest extends ControllerContainerTest {
         return streamer;
     }
 
-    private MultiPartStreamer createApplicationSubmissionData(ApplicationPackage applicationPackage, long projectId) {
+    static MultiPartStreamer createApplicationSubmissionData(ApplicationPackage applicationPackage, long projectId) {
         return new MultiPartStreamer().addJson(EnvironmentResource.SUBMIT_OPTIONS, "{\"repository\":\"repository1\",\"branch\":\"master\",\"commit\":\"commit1\","
                                                                                    + "\"projectId\":" + projectId + ",\"authorEmail\":\"a@b\"}")
                                       .addBytes(EnvironmentResource.APPLICATION_ZIP, applicationPackage.zippedContent())
@@ -1562,19 +1575,10 @@ public class ApplicationApiTest extends ControllerContainerTest {
 
                 for (Instance instance : application.instances().values()) {
                     for (Deployment deployment : instance.deployments().values()) {
-                        Map<ClusterSpec.Id, ClusterInfo> clusterInfo = new HashMap<>();
-                        List<String> hostnames = new ArrayList<>();
-                        hostnames.add("host1");
-                        hostnames.add("host2");
-                        clusterInfo.put(ClusterSpec.Id.from("cluster1"),
-                                        new ClusterInfo("flavor1", 37, 2, 4, 50,
-                                                        ClusterSpec.Type.content, hostnames));
                         DeploymentMetrics metrics = new DeploymentMetrics(1, 2, 3, 4, 5,
                                                                           Optional.of(Instant.ofEpochMilli(123123)), Map.of());
-
                         lockedApplication = lockedApplication.with(instance.name(),
-                                                                   lockedInstance -> lockedInstance.withClusterInfo(deployment.zone(), clusterInfo)
-                                                                                                   .with(deployment.zone(), metrics)
+                                                                   lockedInstance -> lockedInstance.with(deployment.zone(), metrics)
                                                                                                    .recordActivityAt(Instant.parse("2018-06-01T10:15:30.00Z"), deployment.zone()));
                     }
                     deploymentTester.applications().store(lockedApplication);
@@ -1610,7 +1614,7 @@ public class ApplicationApiTest extends ControllerContainerTest {
 
     private void assertGlobalRouting(DeploymentId deployment, GlobalRouting.Status status, GlobalRouting.Agent agent) {
         var changedAt = tester.controller().clock().instant();
-        var westPolicies = tester.controller().applications().routingPolicies().get(deployment);
+        var westPolicies = tester.controller().routing().policies().get(deployment);
         assertEquals(1, westPolicies.size());
         var westPolicy = westPolicies.values().iterator().next();
         assertEquals(status, westPolicy.status().globalRouting().status());
@@ -1627,8 +1631,8 @@ public class ApplicationApiTest extends ControllerContainerTest {
         private OktaIdentityToken oktaIdentityToken;
         private OktaAccessToken oktaAccessToken;
         private String contentType = "application/json";
-        private Map<String, List<String>> headers = new HashMap<>();
-        private String recursive;
+        private final Map<String, List<String>> headers = new HashMap<>();
+        private final Map<String, String> properties = new HashMap<>();
 
         private RequestBuilder(String path, Request.Method method) {
             this.path = path;
@@ -1636,7 +1640,7 @@ public class ApplicationApiTest extends ControllerContainerTest {
         }
 
         private RequestBuilder data(byte[] data) { this.data = data; return this; }
-        private RequestBuilder data(String data) { return data(data.getBytes(StandardCharsets.UTF_8)); }
+        private RequestBuilder data(String data) { return data(data.getBytes(UTF_8)); }
         private RequestBuilder data(MultiPartStreamer streamer) {
             return Exceptions.uncheck(() -> data(streamer.data().readAllBytes()).contentType(streamer.contentType()));
         }
@@ -1646,7 +1650,8 @@ public class ApplicationApiTest extends ControllerContainerTest {
         private RequestBuilder oktaIdentityToken(OktaIdentityToken oktaIdentityToken) { this.oktaIdentityToken = oktaIdentityToken; return this; }
         private RequestBuilder oktaAccessToken(OktaAccessToken oktaAccessToken) { this.oktaAccessToken = oktaAccessToken; return this; }
         private RequestBuilder contentType(String contentType) { this.contentType = contentType; return this; }
-        private RequestBuilder recursive(String recursive) { this.recursive = recursive; return this; }
+        private RequestBuilder recursive(String recursive) {return properties(Map.of("recursive", recursive)); }
+        private RequestBuilder properties(Map<String, String> properties) { this.properties.putAll(properties); return this; }
         private RequestBuilder header(String name, String value) {
             this.headers.putIfAbsent(name, new ArrayList<>());
             this.headers.get(name).add(value);
@@ -1656,11 +1661,13 @@ public class ApplicationApiTest extends ControllerContainerTest {
         @Override
         public Request get() {
             Request request = new Request("http://localhost:8080" + path +
-                                          // user and domain parameters are translated to a Principal by MockAuthorizer as we do not run HTTP filters
-                                          (recursive == null ? "" : "?recursive=" + recursive),
+                                          properties.entrySet().stream()
+                                                    .map(entry -> encode(entry.getKey(), UTF_8) + "=" + encode(entry.getValue(), UTF_8))
+                                                    .collect(joining("&", "?", "")),
                                           data, method);
             request.getHeaders().addAll(headers);
             request.getHeaders().put("Content-Type", contentType);
+            // user and domain parameters are translated to a Principal by MockAuthorizer as we do not run HTTP filters
             if (identity != null) {
                 addIdentityToRequest(request, identity);
             }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java
index db82adb4940..17d234b0c0c 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java
@@ -34,12 +34,10 @@ import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobTy
 import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.stagingTest;
 import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.systemTest;
 import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.testUsCentral1;
-import static com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud.Status.FAILURE;
 import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.applicationPackage;
 import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.deploymentFailed;
 import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.installationFailed;
 import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.running;
-import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.testFailure;
 import static org.junit.Assert.assertEquals;
 
 /**
@@ -52,6 +50,9 @@ public class JobControllerApiHandlerHelperTest {
     public void testResponses() {
         ApplicationPackage applicationPackage = new ApplicationPackageBuilder()
                 .stagingTest()
+                .blockChange(true, true, "mon,tue", "7-13", "UTC")
+                .blockChange(false, true, "sun", "0-23", "CET")
+                .blockChange(true, false, "fri-sat", "8", "America/Los_Angeles")
                 .region("us-central-1")
                 .test("us-central-1")
                 .parallel("us-west-1", "us-east-3")
@@ -80,15 +81,10 @@ public class JobControllerApiHandlerHelperTest {
         tester.runner().run();
         assertEquals(deploymentFailed, tester.jobs().last(app.instanceId(), productionUsEast3).get().status());
 
-        ZoneId usWest1 = productionUsWest1.zone(tester.controller().system());
-        tester.configServer().convergeServices(app.instanceId(), usWest1);
-        tester.configServer().convergeServices(app.testerId().id(), usWest1);
-        tester.setEndpoints(app.instanceId(), usWest1);
-        tester.setEndpoints(app.testerId().id(), usWest1);
         tester.runner().run();
-        tester.cloud().set(FAILURE);
+        tester.clock().advance(Duration.ofHours(2).plusSeconds(1));
         tester.runner().run();
-        assertEquals(testFailure, tester.jobs().last(app.instanceId(), productionUsWest1).get().status());
+        assertEquals(installationFailed, tester.jobs().last(app.instanceId(), productionUsWest1).get().status());
         assertEquals(revision2, app.deployment(productionUsCentral1.zone(tester.controller().system())).applicationVersion());
         assertEquals(revision1, app.deployment(productionUsEast3.zone(tester.controller().system())).applicationVersion());
         assertEquals(revision2, app.deployment(productionUsWest1.zone(tester.controller().system())).applicationVersion());
@@ -140,7 +136,6 @@ public class JobControllerApiHandlerHelperTest {
         userApp.runJob(devAwsUsEast2a, applicationPackage);
         assertResponse(JobControllerApiHandlerHelper.runResponse(tester.jobs().runs(userApp.instanceId(), devAwsUsEast2a), URI.create("https://some.url:43/root")), "dev-aws-us-east-2a-runs.json");
         assertResponse(JobControllerApiHandlerHelper.jobTypeResponse(tester.controller(), userApp.instanceId(), URI.create("https://some.url:43/root/")), "overview-user-instance.json");
-
         assertResponse(JobControllerApiHandlerHelper.overviewResponse(tester.controller(), app.application().id(), URI.create("https://some.url:43/root/")), "deployment-overview-2.json");
     }
 
@@ -157,7 +152,6 @@ public class JobControllerApiHandlerHelperTest {
 
         tester.configServer().setLogStream("Nope, this won't be logged");
         tester.configServer().convergeServices(app.instanceId(), zone);
-        tester.setEndpoints(app.instanceId(), zone);
         tester.runner().run();
 
         assertResponse(JobControllerApiHandlerHelper.jobTypeResponse(tester.controller(), app.instanceId(), URI.create("https://some.url:43/root")), "dev-overview.json");
@@ -183,7 +177,6 @@ public class JobControllerApiHandlerHelperTest {
     private void compare(HttpResponse response, String expected) throws JSONException, IOException {
         ByteArrayOutputStream baos = new ByteArrayOutputStream();
         response.render(baos);
-        System.err.println(baos.toString());
         JSONObject actualJSON = new JSONObject(new String(baos.toByteArray()));
         JSONObject expectedJSON = new JSONObject(expected);
         assertEquals(expectedJSON.toString(), actualJSON.toString());
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ServiceApiResponseTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ServiceApiResponseTest.java
index 9ef94cc9afd..57774c3f412 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ServiceApiResponseTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ServiceApiResponseTest.java
@@ -7,7 +7,7 @@ import com.yahoo.config.provision.RegionName;
 import com.yahoo.config.provision.zone.ZoneId;
 import com.yahoo.io.IOUtils;
 import com.yahoo.slime.Slime;
-import com.yahoo.vespa.config.SlimeUtils;
+import com.yahoo.slime.SlimeUtils;
 import com.yahoo.vespa.serviceview.bindings.ApplicationView;
 import com.yahoo.vespa.serviceview.bindings.ClusterView;
 import com.yahoo.vespa.serviceview.bindings.ServiceView;
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-list.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-list.json
new file mode 100644
index 00000000000..2479f127f92
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-list.json
@@ -0,0 +1,13 @@
+[
+  {
+    "tenant": "tenant1",
+    "application":"application1",
+    "url":"http://localhost:8080/application/v4/tenant/tenant1/application/application1",
+    "instances": [
+      {
+        "instance":"instance1",
+        "url":"http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1"
+      }
+    ]
+  }
+]
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-without-instances.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-without-instances.json
new file mode 100644
index 00000000000..2ee72f150e5
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-without-instances.json
@@ -0,0 +1,12 @@
+{
+  "tenant": "tenant1",
+  "application": "application1",
+  "deployments": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/job/",
+  "instances": [],
+  "pemDeployKeys": [],
+  "metrics": {
+    "queryServiceQuality": 0.0,
+    "writeServiceQuality": 0.0
+  },
+  "activity": {}
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application2-with-patches.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application2-with-patches.json
index 7b84780d64e..5c073173544 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application2-with-patches.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application2-with-patches.json
@@ -14,7 +14,6 @@
     "commit": "commit1"
   },
   "projectId": 1000,
-  "compileVersion": "6.1.0",
   "majorVersion": 7,
   "instances": [
     {
@@ -40,75 +39,6 @@
           "commit": "commit1"
         }
       },
-      "deploymentJobs": [
-        {
-          "type": "system-test",
-          "success": false,
-          "lastTriggered": {
-            "id": 1,
-            "version": "7.0.0",
-            "revision": {
-              "buildNumber": 1,
-              "hash": "1.0.1-commit1",
-              "source": {
-                "gitRepository": "repository1",
-                "gitBranch": "master",
-                "gitCommit": "commit1"
-              },
-              "sourceUrl": "repository1/tree/commit1",
-              "commit": "commit1"
-            },
-            "reason": "unknown reason",
-            "at": "(ignore)"
-          }
-        },
-        {
-          "type": "staging-test",
-          "success": false,
-          "lastTriggered": {
-            "id": 1,
-            "version": "7.0.0",
-            "revision": {
-              "buildNumber": 1,
-              "hash": "1.0.1-commit1",
-              "source": {
-                "gitRepository": "repository1",
-                "gitBranch": "master",
-                "gitCommit": "commit1"
-              },
-              "sourceUrl": "repository1/tree/commit1",
-              "commit": "commit1"
-            },
-            "reason": "unknown reason",
-            "at": "(ignore)"
-          }
-        },
-        {
-          "type": "production-us-west-1",
-          "success": false,
-          "lastTriggered": {
-            "id": 1,
-            "version": "7.0.0",
-            "revision": {
-              "buildNumber": 1,
-              "hash": "1.0.1-commit1",
-              "source": {
-                "gitRepository": "repository1",
-                "gitBranch": "master",
-                "gitCommit": "commit1"
-              },
-              "sourceUrl": "repository1/tree/commit1",
-              "commit": "commit1"
-            },
-            "reason": "unknown reason",
-            "at": "(ignore)"
-          }
-        },
-        {
-          "type": "production-us-east-3",
-          "success": false
-        }
-      ],
       "changeBlockers": [],
       "globalRotations": [
         "https://instance1--application2--tenant2.global.vespa.oath.cloud:4443/"
@@ -126,4 +56,3 @@
   },
   "activity": {}
 }
-
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application2.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application2.json
index 3242fd343de..01fd88a599f 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application2.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application2.json
@@ -14,7 +14,6 @@
     "commit": "commit1"
   },
   "projectId": 1000,
-  "compileVersion": "6.1.0",
   "instances": [
     {
       "instance": "default",
@@ -39,75 +38,6 @@
           "commit": "commit1"
         }
       },
-      "deploymentJobs": [
-        {
-          "type": "system-test",
-          "success": false,
-          "lastTriggered": {
-            "id": 1,
-            "version": "7.0.0",
-            "revision": {
-              "buildNumber": 1,
-              "hash": "1.0.1-commit1",
-              "source": {
-                "gitRepository": "repository1",
-                "gitBranch": "master",
-                "gitCommit": "commit1"
-              },
-              "sourceUrl": "repository1/tree/commit1",
-              "commit": "commit1"
-            },
-            "reason": "unknown reason",
-            "at": "(ignore)"
-          }
-        },
-        {
-          "type": "staging-test",
-          "success": false,
-          "lastTriggered": {
-            "id": 1,
-            "version": "7.0.0",
-            "revision": {
-              "buildNumber": 1,
-              "hash": "1.0.1-commit1",
-              "source": {
-                "gitRepository": "repository1",
-                "gitBranch": "master",
-                "gitCommit": "commit1"
-              },
-              "sourceUrl": "repository1/tree/commit1",
-              "commit": "commit1"
-            },
-            "reason": "unknown reason",
-            "at": "(ignore)"
-          }
-        },
-        {
-          "type": "production-us-west-1",
-          "success": false,
-          "lastTriggered": {
-            "id": 1,
-            "version": "7.0.0",
-            "revision": {
-              "buildNumber": 1,
-              "hash": "1.0.1-commit1",
-              "source": {
-                "gitRepository": "repository1",
-                "gitBranch": "master",
-                "gitCommit": "commit1"
-              },
-              "sourceUrl": "repository1/tree/commit1",
-              "commit": "commit1"
-            },
-            "reason": "unknown reason",
-            "at": "(ignore)"
-          }
-        },
-        {
-          "type": "production-us-east-3",
-          "success": false
-        }
-      ],
       "changeBlockers": [],
       "globalRotations": [
         "https://instance1--application2--tenant2.global.vespa.oath.cloud:4443/"
@@ -123,4 +53,3 @@
   },
   "activity": {}
 }
-
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-job-accepted.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-job-accepted.json
deleted file mode 100644
index d37e9120837..00000000000
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-job-accepted.json
+++ /dev/null
@@ -1,4 +0,0 @@
-{
-  "message": "Deployment started in run 1 of dev-us-east-1 for tenant1.application1.instance1. This may take about 15 minutes the first time.",
-  "run": 1
-}
\ No newline at end of file
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview-2.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview-2.json
index fd1e8bc5f20..e45bd190d5f 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview-2.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview-2.json
@@ -1,45 +1,150 @@
 {
   "tenant": "tenant",
   "application": "application",
+  "projectId": 1001,
   "steps": [
     {
       "type": "instance",
       "dependencies": [],
       "declared": true,
-      "instance": "default"
+      "instance": "default",
+      "readyAt": 0,
+      "deploying": {
+        "application": {
+          "build": 3,
+          "compileVersion": "6.1.0",
+          "sourceUrl": "repository1/tree/commit1",
+          "commit": "commit1"
+        }
+      },
+      "latestVersions": {
+        "platform": {
+          "platform": "7.1.0",
+          "at": 0,
+          "upgrade": true,
+          "blockers": [
+            {
+              "days": [
+                "Mon",
+                "Tue"
+              ],
+              "hours": [
+                7,
+                8,
+                9,
+                10,
+                11,
+                12,
+                13
+              ],
+              "zone": "UTC"
+            },
+            {
+              "days": [
+                "Sun"
+              ],
+              "hours": [
+                0,
+                1,
+                2,
+                3,
+                4,
+                5,
+                6,
+                7,
+                8,
+                9,
+                10,
+                11,
+                12,
+                13,
+                14,
+                15,
+                16,
+                17,
+                18,
+                19,
+                20,
+                21,
+                22,
+                23
+              ],
+              "zone": "CET"
+            }
+          ]
+        },
+        "application": {
+          "application": {
+            "build": 3,
+            "compileVersion": "6.1.0",
+            "sourceUrl": "repository1/tree/commit1",
+            "commit": "commit1"
+          },
+          "at": 1000,
+          "upgrade": true,
+          "blockers": [
+            {
+              "days": [
+                "Mon",
+                "Tue"
+              ],
+              "hours": [
+                7,
+                8,
+                9,
+                10,
+                11,
+                12,
+                13
+              ],
+              "zone": "UTC"
+            },
+            {
+              "days": [
+                "Fri",
+                "Sat"
+              ],
+              "hours": [
+                8
+              ],
+              "zone": "America/Los_Angeles"
+            }
+          ]
+        }
+      }
     },
     {
       "type": "test",
       "dependencies": [],
       "declared": false,
       "instance": "default",
+      "readyAt": 0,
       "jobName": "system-test",
       "url": "https://some.url:43/instance/default/job/system-test",
       "environment": "test",
       "region": "test.us-east-1",
       "toRun": [],
-      "readyAt": 0,
       "runs": [
         {
           "id": 3,
-          "url": "https://some.url:43/instance/default/job/system-test/run/run 3 of system-test for tenant.application",
-          "start": 2000,
-          "end": 2000,
+          "url": "https://some.url:43/instance/default/job/system-test/run/3",
+          "start": 7203000,
+          "end": 7203000,
           "status": "success",
           "versions": {
             "targetPlatform": "6.1.0",
             "targetApplication": {
-              "id": 3,
-              "commit": "commit1",
+              "build": 3,
               "compileVersion": "6.1.0",
-              "sourceUrl": "repository1/tree/commit1"
+              "sourceUrl": "repository1/tree/commit1",
+              "commit": "commit1"
             },
             "sourcePlatform": "6.1.0",
             "sourceApplication": {
-              "id": 2,
-              "commit": "commit1",
+              "build": 2,
               "compileVersion": "6.1.0",
-              "sourceUrl": "repository1/tree/commit1"
+              "sourceUrl": "repository1/tree/commit1",
+              "commit": "commit1"
             }
           },
           "steps": [
@@ -87,24 +192,24 @@
         },
         {
           "id": 2,
-          "url": "https://some.url:43/instance/default/job/system-test/run/run 2 of system-test for tenant.application",
+          "url": "https://some.url:43/instance/default/job/system-test/run/2",
           "start": 1000,
           "end": 1000,
           "status": "success",
           "versions": {
             "targetPlatform": "6.1.0",
             "targetApplication": {
-              "id": 2,
-              "commit": "commit1",
+              "build": 2,
               "compileVersion": "6.1.0",
-              "sourceUrl": "repository1/tree/commit1"
+              "sourceUrl": "repository1/tree/commit1",
+              "commit": "commit1"
             },
             "sourcePlatform": "6.1.0",
             "sourceApplication": {
-              "id": 1,
-              "commit": "commit1",
+              "build": 1,
               "compileVersion": "6.1.0",
-              "sourceUrl": "repository1/tree/commit1"
+              "sourceUrl": "repository1/tree/commit1",
+              "commit": "commit1"
             }
           },
           "steps": [
@@ -152,17 +257,17 @@
         },
         {
           "id": 1,
-          "url": "https://some.url:43/instance/default/job/system-test/run/run 1 of system-test for tenant.application",
+          "url": "https://some.url:43/instance/default/job/system-test/run/1",
           "start": 0,
           "end": 0,
           "status": "success",
           "versions": {
             "targetPlatform": "6.1.0",
             "targetApplication": {
-              "id": 1,
-              "commit": "commit1",
+              "build": 1,
               "compileVersion": "6.1.0",
-              "sourceUrl": "repository1/tree/commit1"
+              "sourceUrl": "repository1/tree/commit1",
+              "commit": "commit1"
             }
           },
           "steps": [
@@ -215,6 +320,9 @@
       "dependencies": [],
       "declared": true,
       "instance": "default",
+      "readyAt": 7953000,
+      "delayedUntil": 7953000,
+      "coolingDownUntil": 7953000,
       "jobName": "staging-test",
       "url": "https://some.url:43/instance/default/job/staging-test",
       "environment": "staging",
@@ -224,45 +332,42 @@
           "versions": {
             "targetPlatform": "6.1.0",
             "targetApplication": {
-              "id": 3,
-              "commit": "commit1",
+              "build": 3,
               "compileVersion": "6.1.0",
-              "sourceUrl": "repository1/tree/commit1"
+              "sourceUrl": "repository1/tree/commit1",
+              "commit": "commit1"
             },
             "sourcePlatform": "6.1.0",
             "sourceApplication": {
-              "id": 1,
-              "commit": "commit1",
+              "build": 1,
               "compileVersion": "6.1.0",
-              "sourceUrl": "repository1/tree/commit1"
+              "sourceUrl": "repository1/tree/commit1",
+              "commit": "commit1"
             }
           }
         }
       ],
-      "readyAt": 752000,
-      "delayedUntil": 752000,
-      "coolingDownUntil": 752000,
       "runs": [
         {
           "id": 5,
-          "url": "https://some.url:43/instance/default/job/staging-test/run/run 5 of staging-test for tenant.application",
-          "start": 102000,
-          "end": 102000,
+          "url": "https://some.url:43/instance/default/job/staging-test/run/5",
+          "start": 7303000,
+          "end": 7303000,
           "status": "installationFailed",
           "versions": {
             "targetPlatform": "6.1.0",
             "targetApplication": {
-              "id": 3,
-              "commit": "commit1",
+              "build": 3,
               "compileVersion": "6.1.0",
-              "sourceUrl": "repository1/tree/commit1"
+              "sourceUrl": "repository1/tree/commit1",
+              "commit": "commit1"
             },
             "sourcePlatform": "6.1.0",
             "sourceApplication": {
-              "id": 1,
-              "commit": "commit1",
+              "build": 1,
               "compileVersion": "6.1.0",
-              "sourceUrl": "repository1/tree/commit1"
+              "sourceUrl": "repository1/tree/commit1",
+              "commit": "commit1"
             }
           },
           "steps": [
@@ -326,24 +431,24 @@
         },
         {
           "id": 4,
-          "url": "https://some.url:43/instance/default/job/staging-test/run/run 4 of staging-test for tenant.application",
-          "start": 2000,
-          "end": 2000,
+          "url": "https://some.url:43/instance/default/job/staging-test/run/4",
+          "start": 7203000,
+          "end": 7203000,
           "status": "installationFailed",
           "versions": {
             "targetPlatform": "6.1.0",
             "targetApplication": {
-              "id": 3,
-              "commit": "commit1",
+              "build": 3,
               "compileVersion": "6.1.0",
-              "sourceUrl": "repository1/tree/commit1"
+              "sourceUrl": "repository1/tree/commit1",
+              "commit": "commit1"
             },
             "sourcePlatform": "6.1.0",
             "sourceApplication": {
-              "id": 1,
-              "commit": "commit1",
+              "build": 1,
               "compileVersion": "6.1.0",
-              "sourceUrl": "repository1/tree/commit1"
+              "sourceUrl": "repository1/tree/commit1",
+              "commit": "commit1"
             }
           },
           "steps": [
@@ -407,24 +512,24 @@
         },
         {
           "id": 3,
-          "url": "https://some.url:43/instance/default/job/staging-test/run/run 3 of staging-test for tenant.application",
-          "start": 2000,
-          "end": 2000,
+          "url": "https://some.url:43/instance/default/job/staging-test/run/3",
+          "start": 7203000,
+          "end": 7203000,
           "status": "success",
           "versions": {
             "targetPlatform": "6.1.0",
             "targetApplication": {
-              "id": 3,
-              "commit": "commit1",
+              "build": 3,
               "compileVersion": "6.1.0",
-              "sourceUrl": "repository1/tree/commit1"
+              "sourceUrl": "repository1/tree/commit1",
+              "commit": "commit1"
             },
             "sourcePlatform": "6.1.0",
             "sourceApplication": {
-              "id": 2,
-              "commit": "commit1",
+              "build": 2,
               "compileVersion": "6.1.0",
-              "sourceUrl": "repository1/tree/commit1"
+              "sourceUrl": "repository1/tree/commit1",
+              "commit": "commit1"
             }
           },
           "steps": [
@@ -488,24 +593,24 @@
         },
         {
           "id": 2,
-          "url": "https://some.url:43/instance/default/job/staging-test/run/run 2 of staging-test for tenant.application",
+          "url": "https://some.url:43/instance/default/job/staging-test/run/2",
           "start": 1000,
           "end": 1000,
           "status": "success",
           "versions": {
             "targetPlatform": "6.1.0",
             "targetApplication": {
-              "id": 2,
-              "commit": "commit1",
+              "build": 2,
               "compileVersion": "6.1.0",
-              "sourceUrl": "repository1/tree/commit1"
+              "sourceUrl": "repository1/tree/commit1",
+              "commit": "commit1"
             },
             "sourcePlatform": "6.1.0",
             "sourceApplication": {
-              "id": 1,
-              "commit": "commit1",
+              "build": 1,
               "compileVersion": "6.1.0",
-              "sourceUrl": "repository1/tree/commit1"
+              "sourceUrl": "repository1/tree/commit1",
+              "commit": "commit1"
             }
           },
           "steps": [
@@ -569,17 +674,17 @@
         },
         {
           "id": 1,
-          "url": "https://some.url:43/instance/default/job/staging-test/run/run 1 of staging-test for tenant.application",
+          "url": "https://some.url:43/instance/default/job/staging-test/run/1",
           "start": 0,
           "end": 0,
           "status": "success",
           "versions": {
             "targetPlatform": "6.1.0",
             "targetApplication": {
-              "id": 1,
-              "commit": "commit1",
+              "build": 1,
               "compileVersion": "6.1.0",
-              "sourceUrl": "repository1/tree/commit1"
+              "sourceUrl": "repository1/tree/commit1",
+              "commit": "commit1"
             }
           },
           "steps": [
@@ -651,51 +756,43 @@
       ],
       "declared": true,
       "instance": "default",
+      "readyAt": 7203000,
       "jobName": "production-us-central-1",
       "url": "https://some.url:43/instance/default/job/production-us-central-1",
       "environment": "prod",
       "region": "prod.us-central-1",
       "currentPlatform": "6.1.0",
       "currentApplication": {
-        "id": 3,
-        "commit": "commit1",
+        "build": 3,
         "compileVersion": "6.1.0",
-        "sourceUrl": "repository1/tree/commit1"
+        "sourceUrl": "repository1/tree/commit1",
+        "commit": "commit1"
       },
       "toRun": [],
-      "readyAt": 2000,
       "runs": [
         {
           "id": 3,
-          "url": "https://some.url:43/instance/default/job/production-us-central-1/run/run 3 of production-us-central-1 for tenant.application",
-          "start": 2000,
+          "url": "https://some.url:43/instance/default/job/production-us-central-1/run/3",
+          "start": 7203000,
           "status": "running",
           "versions": {
             "targetPlatform": "6.1.0",
             "targetApplication": {
-              "id": 3,
-              "commit": "commit1",
+              "build": 3,
               "compileVersion": "6.1.0",
-              "sourceUrl": "repository1/tree/commit1"
+              "sourceUrl": "repository1/tree/commit1",
+              "commit": "commit1"
             },
             "sourcePlatform": "6.1.0",
             "sourceApplication": {
-              "id": 2,
-              "commit": "commit1",
+              "build": 2,
               "compileVersion": "6.1.0",
-              "sourceUrl": "repository1/tree/commit1"
+              "sourceUrl": "repository1/tree/commit1",
+              "commit": "commit1"
             }
           },
           "steps": [
             {
-              "name": "deployTester",
-              "status": "succeeded"
-            },
-            {
-              "name": "installTester",
-              "status": "unfinished"
-            },
-            {
               "name": "deployReal",
               "status": "succeeded"
             },
@@ -704,18 +801,6 @@
               "status": "unfinished"
             },
             {
-              "name": "startTests",
-              "status": "unfinished"
-            },
-            {
-              "name": "endTests",
-              "status": "unfinished"
-            },
-            {
-              "name": "deactivateTester",
-              "status": "unfinished"
-            },
-            {
               "name": "report",
               "status": "unfinished"
             }
@@ -723,36 +808,28 @@
         },
         {
           "id": 2,
-          "url": "https://some.url:43/instance/default/job/production-us-central-1/run/run 2 of production-us-central-1 for tenant.application",
+          "url": "https://some.url:43/instance/default/job/production-us-central-1/run/2",
           "start": 1000,
           "end": 1000,
           "status": "success",
           "versions": {
             "targetPlatform": "6.1.0",
             "targetApplication": {
-              "id": 2,
-              "commit": "commit1",
+              "build": 2,
               "compileVersion": "6.1.0",
-              "sourceUrl": "repository1/tree/commit1"
+              "sourceUrl": "repository1/tree/commit1",
+              "commit": "commit1"
             },
             "sourcePlatform": "6.1.0",
             "sourceApplication": {
-              "id": 1,
-              "commit": "commit1",
+              "build": 1,
               "compileVersion": "6.1.0",
-              "sourceUrl": "repository1/tree/commit1"
+              "sourceUrl": "repository1/tree/commit1",
+              "commit": "commit1"
             }
           },
           "steps": [
             {
-              "name": "deployTester",
-              "status": "succeeded"
-            },
-            {
-              "name": "installTester",
-              "status": "succeeded"
-            },
-            {
               "name": "deployReal",
               "status": "succeeded"
             },
@@ -761,18 +838,6 @@
               "status": "succeeded"
             },
             {
-              "name": "startTests",
-              "status": "succeeded"
-            },
-            {
-              "name": "endTests",
-              "status": "succeeded"
-            },
-            {
-              "name": "deactivateTester",
-              "status": "succeeded"
-            },
-            {
               "name": "report",
               "status": "succeeded"
             }
@@ -780,29 +845,21 @@
         },
         {
           "id": 1,
-          "url": "https://some.url:43/instance/default/job/production-us-central-1/run/run 1 of production-us-central-1 for tenant.application",
+          "url": "https://some.url:43/instance/default/job/production-us-central-1/run/1",
           "start": 0,
           "end": 0,
           "status": "success",
           "versions": {
             "targetPlatform": "6.1.0",
             "targetApplication": {
-              "id": 1,
-              "commit": "commit1",
+              "build": 1,
               "compileVersion": "6.1.0",
-              "sourceUrl": "repository1/tree/commit1"
+              "sourceUrl": "repository1/tree/commit1",
+              "commit": "commit1"
             }
           },
           "steps": [
             {
-              "name": "deployTester",
-              "status": "succeeded"
-            },
-            {
-              "name": "installTester",
-              "status": "succeeded"
-            },
-            {
               "name": "deployReal",
               "status": "succeeded"
             },
@@ -811,18 +868,6 @@
               "status": "succeeded"
             },
             {
-              "name": "startTests",
-              "status": "succeeded"
-            },
-            {
-              "name": "endTests",
-              "status": "succeeded"
-            },
-            {
-              "name": "deactivateTester",
-              "status": "succeeded"
-            },
-            {
               "name": "report",
               "status": "succeeded"
             }
@@ -846,17 +891,17 @@
           "versions": {
             "targetPlatform": "6.1.0",
             "targetApplication": {
-              "id": 3,
-              "commit": "commit1",
+              "build": 3,
               "compileVersion": "6.1.0",
-              "sourceUrl": "repository1/tree/commit1"
+              "sourceUrl": "repository1/tree/commit1",
+              "commit": "commit1"
             },
             "sourcePlatform": "6.1.0",
             "sourceApplication": {
-              "id": 3,
-              "commit": "commit1",
+              "build": 3,
               "compileVersion": "6.1.0",
-              "sourceUrl": "repository1/tree/commit1"
+              "sourceUrl": "repository1/tree/commit1",
+              "commit": "commit1"
             }
           }
         }
@@ -864,24 +909,24 @@
       "runs": [
         {
           "id": 2,
-          "url": "https://some.url:43/instance/default/job/test-us-central-1/run/run 2 of test-us-central-1 for tenant.application",
+          "url": "https://some.url:43/instance/default/job/test-us-central-1/run/2",
           "start": 1000,
           "end": 1000,
           "status": "success",
           "versions": {
             "targetPlatform": "6.1.0",
             "targetApplication": {
-              "id": 2,
-              "commit": "commit1",
+              "build": 2,
               "compileVersion": "6.1.0",
-              "sourceUrl": "repository1/tree/commit1"
+              "sourceUrl": "repository1/tree/commit1",
+              "commit": "commit1"
             },
             "sourcePlatform": "6.1.0",
             "sourceApplication": {
-              "id": 2,
-              "commit": "commit1",
+              "build": 2,
               "compileVersion": "6.1.0",
-              "sourceUrl": "repository1/tree/commit1"
+              "sourceUrl": "repository1/tree/commit1",
+              "commit": "commit1"
             }
           },
           "steps": [
@@ -913,24 +958,24 @@
         },
         {
           "id": 1,
-          "url": "https://some.url:43/instance/default/job/test-us-central-1/run/run 1 of test-us-central-1 for tenant.application",
+          "url": "https://some.url:43/instance/default/job/test-us-central-1/run/1",
           "start": 0,
           "end": 0,
           "status": "success",
           "versions": {
             "targetPlatform": "6.1.0",
             "targetApplication": {
-              "id": 1,
-              "commit": "commit1",
+              "build": 1,
               "compileVersion": "6.1.0",
-              "sourceUrl": "repository1/tree/commit1"
+              "sourceUrl": "repository1/tree/commit1",
+              "commit": "commit1"
             },
             "sourcePlatform": "6.1.0",
             "sourceApplication": {
-              "id": 1,
-              "commit": "commit1",
+              "build": 1,
               "compileVersion": "6.1.0",
-              "sourceUrl": "repository1/tree/commit1"
+              "sourceUrl": "repository1/tree/commit1",
+              "commit": "commit1"
             }
           },
           "steps": [
@@ -975,27 +1020,27 @@
       "region": "prod.us-west-1",
       "currentPlatform": "6.1.0",
       "currentApplication": {
-        "id": 2,
-        "commit": "commit1",
+        "build": 2,
         "compileVersion": "6.1.0",
-        "sourceUrl": "repository1/tree/commit1"
+        "sourceUrl": "repository1/tree/commit1",
+        "commit": "commit1"
       },
       "toRun": [
         {
           "versions": {
             "targetPlatform": "6.1.0",
             "targetApplication": {
-              "id": 3,
-              "commit": "commit1",
+              "build": 3,
               "compileVersion": "6.1.0",
-              "sourceUrl": "repository1/tree/commit1"
+              "sourceUrl": "repository1/tree/commit1",
+              "commit": "commit1"
             },
             "sourcePlatform": "6.1.0",
             "sourceApplication": {
-              "id": 2,
-              "commit": "commit1",
+              "build": 2,
               "compileVersion": "6.1.0",
-              "sourceUrl": "repository1/tree/commit1"
+              "sourceUrl": "repository1/tree/commit1",
+              "commit": "commit1"
             }
           }
         }
@@ -1003,56 +1048,36 @@
       "runs": [
         {
           "id": 2,
-          "url": "https://some.url:43/instance/default/job/production-us-west-1/run/run 2 of production-us-west-1 for tenant.application",
+          "url": "https://some.url:43/instance/default/job/production-us-west-1/run/2",
           "start": 1000,
-          "end": 1000,
-          "status": "testFailure",
+          "end": 7202000,
+          "status": "installationFailed",
           "versions": {
             "targetPlatform": "6.1.0",
             "targetApplication": {
-              "id": 2,
-              "commit": "commit1",
+              "build": 2,
               "compileVersion": "6.1.0",
-              "sourceUrl": "repository1/tree/commit1"
+              "sourceUrl": "repository1/tree/commit1",
+              "commit": "commit1"
             },
             "sourcePlatform": "6.1.0",
             "sourceApplication": {
-              "id": 1,
-              "commit": "commit1",
+              "build": 1,
               "compileVersion": "6.1.0",
-              "sourceUrl": "repository1/tree/commit1"
+              "sourceUrl": "repository1/tree/commit1",
+              "commit": "commit1"
             }
           },
           "steps": [
             {
-              "name": "deployTester",
-              "status": "succeeded"
-            },
-            {
-              "name": "installTester",
-              "status": "succeeded"
-            },
-            {
               "name": "deployReal",
               "status": "succeeded"
             },
             {
               "name": "installReal",
-              "status": "succeeded"
-            },
-            {
-              "name": "startTests",
-              "status": "succeeded"
-            },
-            {
-              "name": "endTests",
               "status": "failed"
             },
             {
-              "name": "deactivateTester",
-              "status": "succeeded"
-            },
-            {
               "name": "report",
               "status": "succeeded"
             }
@@ -1060,29 +1085,21 @@
         },
         {
           "id": 1,
-          "url": "https://some.url:43/instance/default/job/production-us-west-1/run/run 1 of production-us-west-1 for tenant.application",
+          "url": "https://some.url:43/instance/default/job/production-us-west-1/run/1",
           "start": 0,
           "end": 0,
           "status": "success",
           "versions": {
             "targetPlatform": "6.1.0",
             "targetApplication": {
-              "id": 1,
-              "commit": "commit1",
+              "build": 1,
               "compileVersion": "6.1.0",
-              "sourceUrl": "repository1/tree/commit1"
+              "sourceUrl": "repository1/tree/commit1",
+              "commit": "commit1"
             }
           },
           "steps": [
             {
-              "name": "deployTester",
-              "status": "succeeded"
-            },
-            {
-              "name": "installTester",
-              "status": "succeeded"
-            },
-            {
               "name": "deployReal",
               "status": "succeeded"
             },
@@ -1091,18 +1108,6 @@
               "status": "succeeded"
             },
             {
-              "name": "startTests",
-              "status": "succeeded"
-            },
-            {
-              "name": "endTests",
-              "status": "succeeded"
-            },
-            {
-              "name": "deactivateTester",
-              "status": "succeeded"
-            },
-            {
               "name": "report",
               "status": "succeeded"
             }
@@ -1123,27 +1128,27 @@
       "region": "prod.us-east-3",
       "currentPlatform": "6.1.0",
       "currentApplication": {
-        "id": 1,
-        "commit": "commit1",
+        "build": 1,
         "compileVersion": "6.1.0",
-        "sourceUrl": "repository1/tree/commit1"
+        "sourceUrl": "repository1/tree/commit1",
+        "commit": "commit1"
       },
       "toRun": [
         {
           "versions": {
             "targetPlatform": "6.1.0",
             "targetApplication": {
-              "id": 3,
-              "commit": "commit1",
+              "build": 3,
               "compileVersion": "6.1.0",
-              "sourceUrl": "repository1/tree/commit1"
+              "sourceUrl": "repository1/tree/commit1",
+              "commit": "commit1"
             },
             "sourcePlatform": "6.1.0",
             "sourceApplication": {
-              "id": 1,
-              "commit": "commit1",
+              "build": 1,
               "compileVersion": "6.1.0",
-              "sourceUrl": "repository1/tree/commit1"
+              "sourceUrl": "repository1/tree/commit1",
+              "commit": "commit1"
             }
           }
         }
@@ -1151,56 +1156,36 @@
       "runs": [
         {
           "id": 2,
-          "url": "https://some.url:43/instance/default/job/production-us-east-3/run/run 2 of production-us-east-3 for tenant.application",
+          "url": "https://some.url:43/instance/default/job/production-us-east-3/run/2",
           "start": 1000,
           "end": 1000,
           "status": "deploymentFailed",
           "versions": {
             "targetPlatform": "6.1.0",
             "targetApplication": {
-              "id": 2,
-              "commit": "commit1",
+              "build": 2,
               "compileVersion": "6.1.0",
-              "sourceUrl": "repository1/tree/commit1"
+              "sourceUrl": "repository1/tree/commit1",
+              "commit": "commit1"
             },
             "sourcePlatform": "6.1.0",
             "sourceApplication": {
-              "id": 1,
-              "commit": "commit1",
+              "build": 1,
               "compileVersion": "6.1.0",
-              "sourceUrl": "repository1/tree/commit1"
+              "sourceUrl": "repository1/tree/commit1",
+              "commit": "commit1"
             }
           },
           "steps": [
             {
-              "name": "deployTester",
-              "status": "failed"
-            },
-            {
-              "name": "installTester",
-              "status": "unfinished"
-            },
-            {
               "name": "deployReal",
-              "status": "unfinished"
+              "status": "failed"
             },
             {
               "name": "installReal",
               "status": "unfinished"
             },
             {
-              "name": "startTests",
-              "status": "unfinished"
-            },
-            {
-              "name": "endTests",
-              "status": "unfinished"
-            },
-            {
-              "name": "deactivateTester",
-              "status": "succeeded"
-            },
-            {
               "name": "report",
               "status": "succeeded"
             }
@@ -1208,29 +1193,21 @@
         },
         {
           "id": 1,
-          "url": "https://some.url:43/instance/default/job/production-us-east-3/run/run 1 of production-us-east-3 for tenant.application",
+          "url": "https://some.url:43/instance/default/job/production-us-east-3/run/1",
           "start": 0,
           "end": 0,
           "status": "success",
           "versions": {
             "targetPlatform": "6.1.0",
             "targetApplication": {
-              "id": 1,
-              "commit": "commit1",
+              "build": 1,
               "compileVersion": "6.1.0",
-              "sourceUrl": "repository1/tree/commit1"
+              "sourceUrl": "repository1/tree/commit1",
+              "commit": "commit1"
             }
           },
           "steps": [
             {
-              "name": "deployTester",
-              "status": "succeeded"
-            },
-            {
-              "name": "installTester",
-              "status": "succeeded"
-            },
-            {
               "name": "deployReal",
               "status": "succeeded"
             },
@@ -1239,18 +1216,6 @@
               "status": "succeeded"
             },
             {
-              "name": "startTests",
-              "status": "succeeded"
-            },
-            {
-              "name": "endTests",
-              "status": "succeeded"
-            },
-            {
-              "name": "deactivateTester",
-              "status": "succeeded"
-            },
-            {
               "name": "report",
               "status": "succeeded"
             }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json
index 33c83a30e38..36aa8a87689 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json
@@ -1,37 +1,66 @@
 {
   "tenant": "tenant1",
   "application": "application1",
+  "projectId": 123,
   "steps": [
     {
       "type": "instance",
       "dependencies": [],
       "declared": true,
-      "instance": "instance1"
+      "instance": "instance1",
+      "readyAt": 0,
+      "deploying": {
+        "application": {
+          "build": 4,
+          "compileVersion": "6.1.0",
+          "sourceUrl": "repository1/tree/commit1",
+          "commit": "commit1"
+        }
+      },
+      "latestVersions": {
+        "platform": {
+          "platform": "6.1.0",
+          "at": "(ignore)",
+          "upgrade": false,
+          "blockers": []
+        },
+        "application": {
+          "application": {
+            "build": 4,
+            "compileVersion": "6.1.0",
+            "sourceUrl": "repository1/tree/commit1",
+            "commit": "commit1"
+          },
+          "at": 1000,
+          "upgrade": false,
+          "blockers": []
+        }
+      }
     },
     {
       "type": "test",
       "dependencies": [],
       "declared": false,
       "instance": "instance1",
+      "readyAt": 0,
       "jobName": "system-test",
       "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/system-test",
       "environment": "test",
       "region": "test.us-east-1",
       "toRun": [],
-      "readyAt": 0,
       "runs": [
         {
           "id": 2,
-          "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/system-test/run/run 2 of system-test for tenant1.application1.instance1",
+          "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/system-test/run/2",
           "start": "(ignore)",
           "status": "running",
           "versions": {
             "targetPlatform": "6.1.0",
             "targetApplication": {
-              "id": 4,
-              "commit": "commit1",
+              "build": 4,
               "compileVersion": "6.1.0",
-              "sourceUrl": "repository1/tree/commit1"
+              "sourceUrl": "repository1/tree/commit1",
+              "commit": "commit1"
             }
           },
           "steps": [
@@ -79,17 +108,17 @@
         },
         {
           "id": 1,
-          "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/system-test/run/run 1 of system-test for tenant1.application1.instance1",
+          "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/system-test/run/1",
           "start": "(ignore)",
           "end": "(ignore)",
           "status": "success",
           "versions": {
             "targetPlatform": "6.1.0",
             "targetApplication": {
-              "id": 1,
-              "commit": "commit1",
+              "build": 1,
               "compileVersion": "6.1.0",
-              "sourceUrl": "repository1/tree/commit1"
+              "sourceUrl": "repository1/tree/commit1",
+              "commit": "commit1"
             }
           },
           "steps": [
@@ -142,25 +171,25 @@
       "dependencies": [],
       "declared": false,
       "instance": "instance1",
+      "readyAt": 0,
       "jobName": "staging-test",
       "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/staging-test",
       "environment": "staging",
       "region": "staging.us-east-3",
       "toRun": [],
-      "readyAt": 0,
       "runs": [
         {
           "id": 2,
-          "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/staging-test/run/run 2 of staging-test for tenant1.application1.instance1",
+          "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/staging-test/run/2",
           "start": "(ignore)",
           "status": "running",
           "versions": {
             "targetPlatform": "6.1.0",
             "targetApplication": {
-              "id": 4,
-              "commit": "commit1",
+              "build": 4,
               "compileVersion": "6.1.0",
-              "sourceUrl": "repository1/tree/commit1"
+              "sourceUrl": "repository1/tree/commit1",
+              "commit": "commit1"
             }
           },
           "steps": [
@@ -224,17 +253,17 @@
         },
         {
           "id": 1,
-          "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/staging-test/run/run 1 of staging-test for tenant1.application1.instance1",
+          "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/staging-test/run/1",
           "start": "(ignore)",
           "end": "(ignore)",
           "status": "success",
           "versions": {
             "targetPlatform": "6.1.0",
             "targetApplication": {
-              "id": 1,
-              "commit": "commit1",
+              "build": 1,
               "compileVersion": "6.1.0",
-              "sourceUrl": "repository1/tree/commit1"
+              "sourceUrl": "repository1/tree/commit1",
+              "commit": "commit1"
             }
           },
           "steps": [
@@ -314,10 +343,10 @@
           "versions": {
             "targetPlatform": "6.1.0",
             "targetApplication": {
-              "id": 4,
-              "commit": "commit1",
+              "build": 4,
               "compileVersion": "6.1.0",
-              "sourceUrl": "repository1/tree/commit1"
+              "sourceUrl": "repository1/tree/commit1",
+              "commit": "commit1"
             }
           }
         }
@@ -325,29 +354,21 @@
       "runs": [
         {
           "id": 1,
-          "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/production-us-central-1/run/run 1 of production-us-central-1 for tenant1.application1.instance1",
+          "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/production-us-central-1/run/1",
           "start": "(ignore)",
           "end": "(ignore)",
           "status": "success",
           "versions": {
             "targetPlatform": "6.1.0",
             "targetApplication": {
-              "id": 1,
-              "commit": "commit1",
+              "build": 1,
               "compileVersion": "6.1.0",
-              "sourceUrl": "repository1/tree/commit1"
+              "sourceUrl": "repository1/tree/commit1",
+              "commit": "commit1"
             }
           },
           "steps": [
             {
-              "name": "deployTester",
-              "status": "succeeded"
-            },
-            {
-              "name": "installTester",
-              "status": "succeeded"
-            },
-            {
               "name": "deployReal",
               "status": "succeeded"
             },
@@ -356,18 +377,6 @@
               "status": "succeeded"
             },
             {
-              "name": "startTests",
-              "status": "succeeded"
-            },
-            {
-              "name": "endTests",
-              "status": "succeeded"
-            },
-            {
-              "name": "deactivateTester",
-              "status": "succeeded"
-            },
-            {
               "name": "report",
               "status": "succeeded"
             }
@@ -391,10 +400,10 @@
           "versions": {
             "targetPlatform": "6.1.0",
             "targetApplication": {
-              "id": 4,
-              "commit": "commit1",
+              "build": 4,
               "compileVersion": "6.1.0",
-              "sourceUrl": "repository1/tree/commit1"
+              "sourceUrl": "repository1/tree/commit1",
+              "commit": "commit1"
             }
           }
         }
@@ -402,27 +411,19 @@
       "runs": [
         {
           "id": 1,
-          "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/production-us-west-1/run/run 1 of production-us-west-1 for tenant1.application1.instance1",
+          "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/production-us-west-1/run/1",
           "start": "(ignore)",
           "status": "aborted",
           "versions": {
             "targetPlatform": "6.1.0",
             "targetApplication": {
-              "id": 1,
-              "commit": "commit1",
-              "sourceUrl": "repository1/tree/commit1"
+              "build": 1,
+              "sourceUrl": "repository1/tree/commit1",
+              "commit": "commit1"
             }
           },
           "steps": [
             {
-              "name": "deployTester",
-              "status": "unfinished"
-            },
-            {
-              "name": "installTester",
-              "status": "unfinished"
-            },
-            {
               "name": "deployReal",
               "status": "unfinished"
             },
@@ -431,18 +432,6 @@
               "status": "unfinished"
             },
             {
-              "name": "startTests",
-              "status": "unfinished"
-            },
-            {
-              "name": "endTests",
-              "status": "unfinished"
-            },
-            {
-              "name": "deactivateTester",
-              "status": "unfinished"
-            },
-            {
               "name": "report",
               "status": "unfinished"
             }
@@ -466,39 +455,31 @@
           "versions": {
             "targetPlatform": "6.1.0",
             "targetApplication": {
-              "id": 4,
-              "commit": "commit1",
+              "build": 4,
               "compileVersion": "6.1.0",
-              "sourceUrl": "repository1/tree/commit1"
+              "sourceUrl": "repository1/tree/commit1",
+              "commit": "commit1"
             }
           }
         }
       ],
       "runs": [
         {
-          "id": 1,
-          "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/production-us-east-3/run/run 1 of production-us-east-3 for tenant1.application1.instance1",
+          "id": 2,
+          "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/production-us-east-3/run/2",
           "start": "(ignore)",
           "status": "aborted",
           "versions": {
             "targetPlatform": "6.1.0",
             "targetApplication": {
-              "id": 1,
-              "commit": "commit1",
+              "build": 1,
               "compileVersion": "6.1.0",
-              "sourceUrl": "repository1/tree/commit1"
+              "sourceUrl": "repository1/tree/commit1",
+              "commit": "commit1"
             }
           },
           "steps": [
             {
-              "name": "deployTester",
-              "status": "unfinished"
-            },
-            {
-              "name": "installTester",
-              "status": "unfinished"
-            },
-            {
               "name": "deployReal",
               "status": "unfinished"
             },
@@ -507,20 +488,33 @@
               "status": "unfinished"
             },
             {
-              "name": "startTests",
+              "name": "report",
               "status": "unfinished"
-            },
+            }
+          ]
+        },
+        {
+          "id": 1,
+          "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/production-us-east-3/run/1",
+          "start": "(ignore)",
+          "end": "(ignore)",
+          "status": "success",
+          "versions": {
+            "targetPlatform": "6.1.0",
+            "targetApplication": {}
+          },
+          "steps": [
             {
-              "name": "endTests",
-              "status": "unfinished"
+              "name": "deployReal",
+              "status": "succeeded"
             },
             {
-              "name": "deactivateTester",
-              "status": "unfinished"
+              "name": "installReal",
+              "status": "succeeded"
             },
             {
-              "name": "report",
-              "status": "unfinished"
+              "name": "copyVespaLogs",
+              "status": "succeeded"
             }
           ]
         }
@@ -533,19 +527,39 @@
         5
       ],
       "declared": true,
-      "instance": "instance2"
+      "instance": "instance2",
+      "deploying": {},
+      "latestVersions": {
+        "platform": {
+          "platform": "6.1.0",
+          "at": "(ignore)",
+          "upgrade": false,
+          "blockers": []
+        },
+        "application": {
+          "application": {
+            "build": 4,
+            "compileVersion": "6.1.0",
+            "sourceUrl": "repository1/tree/commit1",
+            "commit": "commit1"
+          },
+          "at": 1000,
+          "upgrade": false,
+          "blockers": []
+        }
+      }
     },
     {
       "type": "test",
       "dependencies": [],
       "declared": false,
       "instance": "instance2",
+      "readyAt": 0,
       "jobName": "system-test",
       "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance2/job/system-test",
       "environment": "test",
       "region": "test.us-east-1",
       "toRun": [],
-      "readyAt": 0,
       "runs": []
     },
     {
@@ -553,12 +567,12 @@
       "dependencies": [],
       "declared": false,
       "instance": "instance2",
+      "readyAt": 0,
       "jobName": "staging-test",
       "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance2/job/staging-test",
       "environment": "staging",
       "region": "staging.us-east-3",
       "toRun": [],
-      "readyAt": 0,
       "runs": []
     },
     {
@@ -577,10 +591,10 @@
           "versions": {
             "targetPlatform": "6.1.0",
             "targetApplication": {
-              "id": 4,
-              "commit": "commit1",
+              "build": 4,
               "compileVersion": "6.1.0",
-              "sourceUrl": "repository1/tree/commit1"
+              "sourceUrl": "repository1/tree/commit1",
+              "commit": "commit1"
             }
           }
         }
@@ -603,10 +617,10 @@
           "versions": {
             "targetPlatform": "6.1.0",
             "targetApplication": {
-              "id": 4,
-              "commit": "commit1",
+              "build": 4,
               "compileVersion": "6.1.0",
-              "sourceUrl": "repository1/tree/commit1"
+              "sourceUrl": "repository1/tree/commit1",
+              "commit": "commit1"
             }
           }
         }
@@ -629,10 +643,10 @@
           "versions": {
             "targetPlatform": "6.1.0",
             "targetApplication": {
-              "id": 4,
-              "commit": "commit1",
+              "build": 4,
               "compileVersion": "6.1.0",
-              "sourceUrl": "repository1/tree/commit1"
+              "sourceUrl": "repository1/tree/commit1",
+              "commit": "commit1"
             }
           }
         }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-with-routing-policy.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-with-routing-policy.json
index 22ba83a4730..63e6e4b3937 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-with-routing-policy.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-with-routing-policy.json
@@ -8,12 +8,18 @@
     {
       "cluster": "default",
       "tls": true,
-      "url": "https://instance1.application1.tenant1.us-west-1.vespa.oath.cloud/"
+      "url": "https://instance1.application1.tenant1.us-west-1.vespa.oath.cloud/",
+      "scope": "zone",
+      "routingMethod": "exclusive"
+    },
+    {
+      "cluster": "default",
+      "tls": true,
+      "url": "https://instance1--application1--tenant1.us-west-1.vespa.oath.cloud:4443/",
+      "scope": "zone",
+      "routingMethod": "shared"
     }
   ],
-  "serviceUrls": [
-    "https://instance1--application1--tenant1.us-west-1.prod.vespa:43"
-  ],
   "nodes": "http://localhost:8080/zone/v2/prod/us-west-1/nodes/v2/node/%3F&recursive=true&application=tenant1.application1.instance1",
   "yamasUrl": "http://monitoring-system.test/?environment=prod&region=us-west-1&application=tenant1.application1.instance1",
   "version": "(ignore)",
@@ -36,12 +42,6 @@
   },
   "status": "complete",
   "activity": {},
-  "cost": {
-    "tco": 0,
-    "waste": 0,
-    "utilization": 0.0,
-    "cluster": {}
-  },
   "metrics": {
     "queriesPerSecond": 0.0,
     "writesPerSecond": 0.0,
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json
index f9692a4afb7..928525a20d1 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json
@@ -4,9 +4,21 @@
   "instance": "instance1",
   "environment": "prod",
   "region": "us-central-1",
-  "endpoints": [],
-  "serviceUrls": [
-    "https://instance1--application1--tenant1.us-central-1.prod.vespa:43"
+  "endpoints": [
+    {
+      "cluster": "default",
+      "tls": true,
+      "url": "https://instance1--application1--tenant1.us-central-1.vespa.oath.cloud:4443/",
+      "scope": "zone",
+      "routingMethod": "shared"
+    },
+    {
+      "cluster": "",
+      "tls": true,
+      "url": "https://instance1--application1--tenant1.global.vespa.oath.cloud:4443/",
+      "scope": "global",
+      "routingMethod": "shared"
+    }
   ],
   "nodes": "http://localhost:8080/zone/v2/prod/us-central-1/nodes/v2/node/%3F&recursive=true&application=tenant1.application1.instance1",
   "yamasUrl": "http://monitoring-system.test/?environment=prod&region=us-central-1&application=tenant1.application1.instance1",
@@ -44,12 +56,6 @@
     "lastQueriesPerSecond": 1.0,
     "lastWritesPerSecond": 2.0
   },
-  "cost": {
-    "tco": 0,
-    "waste": 0,
-    "utilization": 0.0,
-    "cluster": {}
-  },
   "metrics": {
     "queriesPerSecond": 1.0,
     "writesPerSecond": 2.0,
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-aws-us-east-2a-runs.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-aws-us-east-2a-runs.json
index fe90ddd772e..2053b5a80b1 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-aws-us-east-2a-runs.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-aws-us-east-2a-runs.json
@@ -2,8 +2,8 @@
   "1": {
     "id": 1,
     "status": "success",
-    "start": 102000,
-    "end": 102000,
+    "start": 7303000,
+    "end": 7303000,
     "wantedPlatform": "7.1",
     "wantedApplication": {
       "hash": "unknown"
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-overview.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-overview.json
index bfaa62a602d..b37d0d41ae4 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-overview.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-overview.json
@@ -29,5 +29,37 @@
       ],
       "url": "https://some.url:43/root/dev-us-east-1"
     }
-  }
+  },
+  "deployment": [
+    {
+      "jobName": "dev-us-east-1",
+      "runs": [
+        {
+          "id": 1,
+          "url": "https://some.url:43/root/run/1",
+          "start": 0,
+          "end": 0,
+          "status": "success",
+          "versions": {
+            "targetPlatform": "6.1.0",
+            "targetApplication": {}
+          },
+          "steps": [
+            {
+              "name": "deployReal",
+              "status": "succeeded"
+            },
+            {
+              "name": "installReal",
+              "status": "succeeded"
+            },
+            {
+              "name": "copyVespaLogs",
+              "status": "succeeded"
+            }
+          ]
+        }
+      ]
+    }
+  ]
 }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1-log-first-part.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1-log-first-part.json
index e1c2310ce7e..3a54ac70b0d 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1-log-first-part.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1-log-first-part.json
@@ -28,6 +28,11 @@
       {
         "at": 0,
         "type": "info",
+        "message": "######## Details for all nodes ########"
+      },
+      {
+        "at": 0,
+        "type": "info",
         "message": "host-tenant:application:default-dev.us-east-1: unorchestrated"
       },
       {
@@ -49,7 +54,7 @@
       }
     ]
   },
-  "lastId": 7,
+  "lastId": 8,
   "steps": {
     "deployReal": {
       "status": "succeeded",
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1-log-second-part.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1-log-second-part.json
index b63031fab4f..91b02e0193e 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1-log-second-part.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1-log-second-part.json
@@ -6,7 +6,12 @@
       {
         "at": 0,
         "type": "info",
-        "message": " |-- https://default--application--tenant.us-east-1.dev.vespa:43 (cluster 'default')"
+        "message": "- dev.us-east-1"
+      },
+      {
+        "at": 0,
+        "type": "info",
+        "message": " |-- https://application--tenant.us-east-1.dev.vespa.oath.cloud:4443/ (cluster 'default')"
       },
       {
         "at": 0,
@@ -15,7 +20,7 @@
       }
     ]
   },
-  "lastId": 11,
+  "lastId": 12,
   "steps": {
     "deployReal": {
       "status": "succeeded",
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1.json
index 204927f6954..83fa1983957 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1.json
@@ -4,9 +4,14 @@
   "instance": "instance1",
   "environment": "dev",
   "region": "us-east-1",
-  "endpoints": [],
-  "serviceUrls": [
-    "https://instance1--application1--tenant1.us-east-1.dev.vespa:43"
+  "endpoints": [
+    {
+      "cluster": "default",
+      "tls": true,
+      "url": "https://instance1--application1--tenant1.us-east-1.dev.vespa.oath.cloud:4443/",
+      "scope": "zone",
+      "routingMethod": "shared"
+    }
   ],
   "nodes": "http://localhost:8080/zone/v2/dev/us-east-1/nodes/v2/node/%3F&recursive=true&application=tenant1.application1.instance1",
   "yamasUrl": "http://monitoring-system.test/?environment=dev&region=us-east-1&application=tenant1.application1.instance1",
@@ -20,12 +25,6 @@
     "lastQueriesPerSecond": 1.0,
     "lastWritesPerSecond": 2.0
   },
-  "cost": {
-    "tco": 0,
-    "waste": 0,
-    "utilization": 0.0,
-    "cluster": {}
-  },
   "metrics": {
     "queriesPerSecond": 1.0,
     "writesPerSecond": 2.0,
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/global-rotation-get.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/global-rotation-get.json
index a21b1558aee..f7c512842fd 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/global-rotation-get.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/global-rotation-get.json
@@ -1 +1,11 @@
-{"globalrotationoverride":["cluster1.application1.tenant1.us-west-1.prod",{"status":"in","reason":"","agent":"","timestamp":1497618757}]}
+{
+  "globalrotationoverride": [
+    "instance1.application1.tenant1.us-west-1.prod",
+    {
+      "status": "in",
+      "reason": "",
+      "agent": "",
+      "timestamp": 1497618757
+    }
+  ]
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance-list.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance-list.json
deleted file mode 100644
index 024ca11dbe3..00000000000
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance-list.json
+++ /dev/null
@@ -1,3 +0,0 @@
-[
-  @include(instance-reference.json)
-]
\ No newline at end of file
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance-with-routing-policy.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance-with-routing-policy.json
index 18f5127718f..33b1d95b5ca 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance-with-routing-policy.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance-with-routing-policy.json
@@ -11,178 +11,8 @@
   "sourceUrl": "repository1/tree/commit1",
   "commit": "commit1",
   "projectId": 1000,
-  "deploymentJobs": [
-    {
-      "type": "system-test",
-      "success": true,
-      "lastTriggered": {
-        "id": 1,
-        "version": "(ignore)",
-        "revision": {
-          "buildNumber": "(ignore)",
-          "hash": "1.0.1-commit1",
-          "source": {
-            "gitRepository": "repository1",
-            "gitBranch": "master",
-            "gitCommit": "commit1"
-          },
-          "sourceUrl": "repository1/tree/commit1",
-          "commit": "commit1"
-        },
-        "reason": "unknown reason",
-        "at": "(ignore)"
-      },
-      "lastCompleted": {
-        "id": 1,
-        "version": "(ignore)",
-        "revision": {
-          "buildNumber": "(ignore)",
-          "hash": "1.0.1-commit1",
-          "source": {
-            "gitRepository": "repository1",
-            "gitBranch": "master",
-            "gitCommit": "commit1"
-          },
-          "sourceUrl": "repository1/tree/commit1",
-          "commit": "commit1"
-        },
-        "reason": "unknown reason",
-        "at": "(ignore)"
-      },
-      "lastSuccess": {
-        "id": 1,
-        "version": "(ignore)",
-        "revision": {
-          "buildNumber": "(ignore)",
-          "hash": "1.0.1-commit1",
-          "source": {
-            "gitRepository": "repository1",
-            "gitBranch": "master",
-            "gitCommit": "commit1"
-          },
-          "sourceUrl": "repository1/tree/commit1",
-          "commit": "commit1"
-        },
-        "reason": "unknown reason",
-        "at": "(ignore)"
-      }
-    },
-    {
-      "type": "staging-test",
-      "success": true,
-      "lastTriggered": {
-        "id": 1,
-        "version": "(ignore)",
-        "revision": {
-          "buildNumber": "(ignore)",
-          "hash": "1.0.1-commit1",
-          "source": {
-            "gitRepository": "repository1",
-            "gitBranch": "master",
-            "gitCommit": "commit1"
-          },
-          "sourceUrl": "repository1/tree/commit1",
-          "commit": "commit1"
-        },
-        "reason": "unknown reason",
-        "at": "(ignore)"
-      },
-      "lastCompleted": {
-        "id": 1,
-        "version": "(ignore)",
-        "revision": {
-          "buildNumber": "(ignore)",
-          "hash": "1.0.1-commit1",
-          "source": {
-            "gitRepository": "repository1",
-            "gitBranch": "master",
-            "gitCommit": "commit1"
-          },
-          "sourceUrl": "repository1/tree/commit1",
-          "commit": "commit1"
-        },
-        "reason": "unknown reason",
-        "at": "(ignore)"
-      },
-      "lastSuccess": {
-        "id": 1,
-        "version": "(ignore)",
-        "revision": {
-          "buildNumber": "(ignore)",
-          "hash": "1.0.1-commit1",
-          "source": {
-            "gitRepository": "repository1",
-            "gitBranch": "master",
-            "gitCommit": "commit1"
-          },
-          "sourceUrl": "repository1/tree/commit1",
-          "commit": "commit1"
-        },
-        "reason": "unknown reason",
-        "at": "(ignore)"
-      }
-    },
-    {
-      "type": "production-us-west-1",
-      "success": true,
-      "lastTriggered": {
-        "id": 1,
-        "version": "(ignore)",
-        "revision": {
-          "buildNumber": "(ignore)",
-          "hash": "1.0.1-commit1",
-          "source": {
-            "gitRepository": "repository1",
-            "gitBranch": "master",
-            "gitCommit": "commit1"
-          },
-          "sourceUrl": "repository1/tree/commit1",
-          "commit": "commit1"
-        },
-        "reason": "unknown reason",
-        "at": "(ignore)"
-      },
-      "lastCompleted": {
-        "id": 1,
-        "version": "(ignore)",
-        "revision": {
-          "buildNumber": "(ignore)",
-          "hash": "1.0.1-commit1",
-          "source": {
-            "gitRepository": "repository1",
-            "gitBranch": "master",
-            "gitCommit": "commit1"
-          },
-          "sourceUrl": "repository1/tree/commit1",
-          "commit": "commit1"
-        },
-        "reason": "unknown reason",
-        "at": "(ignore)"
-      },
-      "lastSuccess": {
-        "id": 1,
-        "version": "(ignore)",
-        "revision": {
-          "buildNumber": "(ignore)",
-          "hash": "1.0.1-commit1",
-          "source": {
-            "gitRepository": "repository1",
-            "gitBranch": "master",
-            "gitCommit": "commit1"
-          },
-          "sourceUrl": "repository1/tree/commit1",
-          "commit": "commit1"
-        },
-        "reason": "unknown reason",
-        "at": "(ignore)"
-      }
-    }
-  ],
   "changeBlockers": [],
-  "compileVersion": "(ignore)",
-  "globalRotations": [
-    "https://c0.instance1.application1.tenant1.global.vespa.oath.cloud/"
-  ],
+  "globalRotations": [],
   "instances": [
     {
       "environment": "prod",
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance-without-change-multiple-deployments.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance-without-change-multiple-deployments.json
deleted file mode 100644
index 6338306897c..00000000000
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance-without-change-multiple-deployments.json
+++ /dev/null
@@ -1,315 +0,0 @@
-{
-  "tenant": "tenant1",
-  "application": "application1",
-  "instance": "instance1",
-  "deployments": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/",
-  "source": {
-    "gitRepository": "repository1",
-    "gitBranch": "master",
-    "gitCommit": "commit1"
-  },
-  "sourceUrl": "repository1/tree/commit1",
-  "commit": "commit1",
-  "projectId": 1000,
-  "deploymentJobs": [
-    {
-      "type": "system-test",
-      "success": true,
-      "lastTriggered": {
-        "id": 1,
-        "version": "(ignore)",
-        "revision": {
-          "buildNumber": "(ignore)",
-          "hash": "(ignore)",
-          "source": {
-            "gitRepository": "repository1",
-            "gitBranch": "master",
-            "gitCommit": "commit1"
-          },
-          "sourceUrl": "repository1/tree/commit1",
-          "commit": "commit1"
-        },
-        "reason": "unknown reason",
-        "at": "(ignore)"
-      },
-      "lastCompleted": {
-        "id": 2,
-        "version": "(ignore)",
-        "revision": {
-          "buildNumber": "(ignore)",
-          "hash": "(ignore)",
-          "source": {
-            "gitRepository": "repository1",
-            "gitBranch": "master",
-            "gitCommit": "commit1"
-          },
-          "sourceUrl": "repository1/tree/commit1",
-          "commit": "commit1"
-        },
-        "reason": "unknown reason",
-        "at": "(ignore)"
-      },
-      "lastSuccess": {
-        "id": 2,
-        "version": "(ignore)",
-        "revision": {
-          "buildNumber": "(ignore)",
-          "hash": "(ignore)",
-          "source": {
-            "gitRepository": "repository1",
-            "gitBranch": "master",
-            "gitCommit": "commit1"
-          },
-          "sourceUrl": "repository1/tree/commit1",
-          "commit": "commit1"
-        },
-        "reason": "unknown reason",
-        "at": "(ignore)"
-      }
-    },
-    {
-      "type": "staging-test",
-      "success": true,
-      "lastTriggered": {
-        "id": 1,
-        "version": "(ignore)",
-        "revision": {
-          "buildNumber": "(ignore)",
-          "hash": "(ignore)",
-          "source": {
-            "gitRepository": "repository1",
-            "gitBranch": "master",
-            "gitCommit": "commit1"
-          },
-          "sourceUrl": "repository1/tree/commit1",
-          "commit": "commit1"
-        },
-        "reason": "unknown reason",
-        "at": "(ignore)"
-      },
-      "lastCompleted": {
-        "id": 3,
-        "version": "(ignore)",
-        "revision": {
-          "buildNumber": "(ignore)",
-          "hash": "(ignore)",
-          "source": {
-            "gitRepository": "repository1",
-            "gitBranch": "master",
-            "gitCommit": "commit1"
-          },
-          "sourceUrl": "repository1/tree/commit1",
-          "commit": "commit1"
-        },
-        "reason": "unknown reason",
-        "at": "(ignore)"
-      },
-      "lastSuccess": {
-        "id": 3,
-        "version": "(ignore)",
-        "revision": {
-          "buildNumber": "(ignore)",
-          "hash": "(ignore)",
-          "source": {
-            "gitRepository": "repository1",
-            "gitBranch": "master",
-            "gitCommit": "commit1"
-          },
-          "sourceUrl": "repository1/tree/commit1",
-          "commit": "commit1"
-        },
-        "reason": "unknown reason",
-        "at": "(ignore)"
-      }
-    },
-    {
-      "type": "production-us-west-1",
-      "success": true,
-      "lastTriggered": {
-        "id": 1,
-        "version": "(ignore)",
-        "revision": {
-          "buildNumber": "(ignore)",
-          "hash": "(ignore)",
-          "source": {
-            "gitRepository": "repository1",
-            "gitBranch": "master",
-            "gitCommit": "commit1"
-          },
-          "sourceUrl": "repository1/tree/commit1",
-          "commit": "commit1"
-        },
-        "reason": "unknown reason",
-        "at": "(ignore)"
-      },
-      "lastCompleted": {
-        "id": 1,
-        "version": "(ignore)",
-        "revision": {
-          "buildNumber": "(ignore)",
-          "hash": "(ignore)",
-          "source": {
-            "gitRepository": "repository1",
-            "gitBranch": "master",
-            "gitCommit": "commit1"
-          },
-          "sourceUrl": "repository1/tree/commit1",
-          "commit": "commit1"
-        },
-        "reason": "unknown reason",
-        "at": "(ignore)"
-      },
-      "lastSuccess": {
-        "id": 1,
-        "version": "(ignore)",
-        "revision": {
-          "buildNumber": "(ignore)",
-          "hash": "(ignore)",
-          "source": {
-            "gitRepository": "repository1",
-            "gitBranch": "master",
-            "gitCommit": "commit1"
-          },
-          "sourceUrl": "repository1/tree/commit1",
-          "commit": "commit1"
-        },
-        "reason": "unknown reason",
-        "at": "(ignore)"
-      }
-    },
-    {
-      "type": "production-us-east-3",
-      "success": true,
-      "lastTriggered": {
-        "id": 1,
-        "version": "(ignore)",
-        "revision": {
-          "buildNumber": "(ignore)",
-          "hash": "(ignore)",
-          "source": {
-            "gitRepository": "repository1",
-            "gitBranch": "master",
-            "gitCommit": "commit1"
-          },
-          "sourceUrl": "repository1/tree/commit1",
-          "commit": "commit1"
-        },
-        "reason": "unknown reason",
-        "at": "(ignore)"
-      },
-      "lastCompleted": {
-        "id": 2,
-        "version": "(ignore)",
-        "revision": {
-          "buildNumber": "(ignore)",
-          "hash": "(ignore)",
-          "source": {
-            "gitRepository": "repository1",
-            "gitBranch": "master",
-            "gitCommit": "commit1"
-          },
-          "sourceUrl": "repository1/tree/commit1",
-          "commit": "commit1"
-        },
-        "reason": "unknown reason",
-        "at": "(ignore)"
-      },
-      "lastSuccess": {
-        "id": 2,
-        "version": "(ignore)",
-        "revision": {
-          "buildNumber": "(ignore)",
-          "hash": "(ignore)",
-          "source": {
-            "gitRepository": "repository1",
-            "gitBranch": "master",
-            "gitCommit": "commit1"
-          },
-          "sourceUrl": "repository1/tree/commit1",
-          "commit": "commit1"
-        },
-        "reason": "unknown reason",
-        "at": "(ignore)"
-      }
-    }
-  ],
-  "changeBlockers": [],
-  "compileVersion": "(ignore)",
-  "globalRotations": [
-    "https://instance1--application1--tenant1.global.vespa.oath.cloud:4443/"
-  ],
-  "rotationId": "rotation-id-1",
-  "instances": [
-    {
-      "bcpStatus": {
-        "rotationStatus": "IN"
-      },
-      "endpointStatus": [
-        {
-          "endpointId": "default",
-          "rotationId": "rotation-id-1",
-          "clusterId": "foo",
-          "status": "IN",
-          "lastUpdated": "(ignore)"
-        }
-      ],
-      "applicationVersion": {
-        "hash": "1.0.1-commit1",
-        "build": 1,
-        "source": {
-          "gitRepository": "repository1",
-          "gitBranch": "master",
-          "gitCommit": "commit1"
-        },
-        "sourceUrl": "repository1/tree/commit1",
-        "commit": "commit1"
-      },
-      "status": "complete",
-      "environment": "prod",
-      "region": "us-west-1",
-      "instance": "instance1",
-      "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-west-1"
-    },
-    {
-      "bcpStatus": {
-        "rotationStatus": "UNKNOWN"
-      },
-      "endpointStatus": [
-        {
-          "endpointId": "default",
-          "rotationId": "rotation-id-1",
-          "clusterId": "foo",
-          "status": "UNKNOWN",
-          "lastUpdated": "(ignore)"
-        }
-      ],
-      "applicationVersion": {
-        "hash": "1.0.1-commit1",
-        "build": 1,
-        "source": {
-          "gitRepository": "repository1",
-          "gitBranch": "master",
-          "gitCommit": "commit1"
-        },
-        "sourceUrl": "repository1/tree/commit1",
-        "commit": "commit1"
-      },
-      "status": "complete",
-      "environment": "prod",
-      "region": "us-east-3",
-      "instance": "instance1",
-      "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-east-3"
-    }
-  ],
-  "pemDeployKeys": [],
-  "metrics": {
-    "queryServiceQuality": 0.5,
-    "writeServiceQuality": 0.7
-  },
-  "activity": {
-    "lastQueried": 1527848130000,
-    "lastWritten": 1527848130000,
-    "lastQueriesPerSecond": 1.0,
-    "lastWritesPerSecond": 2.0
-  }
-}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance.json
index fd8bc256ac5..1b2c0b4e237 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance.json
@@ -13,8 +13,8 @@
   "projectId": 123,
   "deploying": {
     "revision": {
-      "buildNumber": "(ignore)",
-      "hash": "(ignore)",
+      "buildNumber": 1,
+      "hash": "1.0.1-commit1",
       "source": {
         "gitRepository": "repository1",
         "gitBranch": "master",
@@ -24,198 +24,6 @@
       "commit": "commit1"
     }
   },
-  "deploymentJobs": [
-    {
-      "type": "system-test",
-      "success": true,
-      "lastTriggered": {
-        "id": 1,
-        "version": "(ignore)",
-        "revision": {
-          "buildNumber": "(ignore)",
-          "hash": "(ignore)",
-          "source": {
-            "gitRepository": "repository1",
-            "gitBranch": "master",
-            "gitCommit": "commit1"
-          },
-          "sourceUrl": "repository1/tree/commit1",
-          "commit": "commit1"
-        },
-        "reason": "unknown reason",
-        "at": "(ignore)"
-      },
-      "lastCompleted": {
-        "id": 1,
-        "version": "(ignore)",
-        "revision": {
-          "buildNumber": "(ignore)",
-          "hash": "(ignore)",
-          "source": {
-            "gitRepository": "repository1",
-            "gitBranch": "master",
-            "gitCommit": "commit1"
-          },
-          "sourceUrl": "repository1/tree/commit1",
-          "commit": "commit1"
-        },
-        "reason": "unknown reason",
-        "at": "(ignore)"
-      },
-      "lastSuccess": {
-        "id": 1,
-        "version": "(ignore)",
-        "revision": {
-          "buildNumber": "(ignore)",
-          "hash": "(ignore)",
-          "source": {
-            "gitRepository": "repository1",
-            "gitBranch": "master",
-            "gitCommit": "commit1"
-          },
-          "sourceUrl": "repository1/tree/commit1",
-          "commit": "commit1"
-        },
-        "reason": "unknown reason",
-        "at": "(ignore)"
-      }
-    },
-    {
-      "type": "staging-test",
-      "success": true,
-      "lastTriggered": {
-        "id": 1,
-        "version": "(ignore)",
-        "revision": {
-          "buildNumber": "(ignore)",
-          "hash": "(ignore)",
-          "source": {
-            "gitRepository": "repository1",
-            "gitBranch": "master",
-            "gitCommit": "commit1"
-          },
-          "sourceUrl": "repository1/tree/commit1",
-          "commit": "commit1"
-        },
-        "reason": "unknown reason",
-        "at": "(ignore)"
-      },
-      "lastCompleted": {
-        "id": 1,
-        "version": "(ignore)",
-        "revision": {
-          "buildNumber": "(ignore)",
-          "hash": "(ignore)",
-          "source": {
-            "gitRepository": "repository1",
-            "gitBranch": "master",
-            "gitCommit": "commit1"
-          },
-          "sourceUrl": "repository1/tree/commit1",
-          "commit": "commit1"
-        },
-        "reason": "unknown reason",
-        "at": "(ignore)"
-      },
-      "lastSuccess": {
-        "id": 1,
-        "version": "(ignore)",
-        "revision": {
-          "buildNumber": "(ignore)",
-          "hash": "(ignore)",
-          "source": {
-            "gitRepository": "repository1",
-            "gitBranch": "master",
-            "gitCommit": "commit1"
-          },
-          "sourceUrl": "repository1/tree/commit1",
-          "commit": "commit1"
-        },
-        "reason": "unknown reason",
-        "at": "(ignore)"
-      }
-    },
-    {
-      "type": "production-us-central-1",
-      "success": true,
-      "lastTriggered": {
-        "id": 1,
-        "version": "(ignore)",
-        "revision": {
-          "buildNumber": "(ignore)",
-          "hash": "(ignore)",
-          "source": {
-            "gitRepository": "repository1",
-            "gitBranch": "master",
-            "gitCommit": "commit1"
-          },
-          "sourceUrl": "repository1/tree/commit1",
-          "commit": "commit1"
-        },
-        "reason": "unknown reason",
-        "at": "(ignore)"
-      },
-      "lastCompleted": {
-        "id": 1,
-        "version": "(ignore)",
-        "revision": {
-          "buildNumber": "(ignore)",
-          "hash": "(ignore)",
-          "source": {
-            "gitRepository": "repository1",
-            "gitBranch": "master",
-            "gitCommit": "commit1"
-          },
-          "sourceUrl": "repository1/tree/commit1",
-          "commit": "commit1"
-        },
-        "reason": "unknown reason",
-        "at": "(ignore)"
-      },
-      "lastSuccess": {
-        "id": 1,
-        "version": "(ignore)",
-        "revision": {
-          "buildNumber": "(ignore)",
-          "hash": "(ignore)",
-          "source": {
-            "gitRepository": "repository1",
-            "gitBranch": "master",
-            "gitCommit": "commit1"
-          },
-          "sourceUrl": "repository1/tree/commit1",
-          "commit": "commit1"
-        },
-        "reason": "unknown reason",
-        "at": "(ignore)"
-      }
-    },
-    {
-      "type": "production-us-east-3",
-      "success": false,
-      "lastTriggered": {
-        "id": 1,
-        "version": "(ignore)",
-        "revision": {
-          "buildNumber": "(ignore)",
-          "hash": "(ignore)",
-          "source": {
-            "gitRepository": "repository1",
-            "gitBranch": "master",
-            "gitCommit": "commit1"
-          },
-          "sourceUrl": "repository1/tree/commit1",
-          "commit": "commit1"
-        },
-        "reason": "unknown reason",
-        "at": "(ignore)"
-      }
-    },
-    {
-      "type": "production-us-west-1",
-      "success": false
-    }
-  ],
   "changeBlockers": [
     {
       "versions": true,
@@ -241,7 +49,6 @@
       ]
     }
   ],
-  "compileVersion": "6.0.0",
   "globalRotations": [
     "https://instance1--application1--tenant1.global.vespa.oath.cloud:4443/"
   ],
@@ -263,7 +70,7 @@
           "rotationId": "rotation-id-1",
           "clusterId": "foo",
           "status": "IN",
-          "lastUpdated":"(ignore)"
+          "lastUpdated": "(ignore)"
         }
       ],
       "environment": "prod",
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance1-recursive.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance1-recursive.json
index ee75d129241..19676decdb8 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance1-recursive.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/instance1-recursive.json
@@ -13,8 +13,8 @@
   "projectId": 123,
   "deploying": {
     "revision": {
-      "buildNumber": "(ignore)",
-      "hash": "(ignore)",
+      "buildNumber": 1,
+      "hash": "1.0.1-commit1",
       "source": {
         "gitRepository": "repository1",
         "gitBranch": "master",
@@ -24,198 +24,6 @@
       "commit": "commit1"
     }
   },
-  "deploymentJobs": [
-    {
-      "type": "system-test",
-      "success": true,
-      "lastTriggered": {
-        "id": 1,
-        "version": "(ignore)",
-        "revision": {
-          "buildNumber": "(ignore)",
-          "hash": "(ignore)",
-          "source": {
-            "gitRepository": "repository1",
-            "gitBranch": "master",
-            "gitCommit": "commit1"
-          },
-          "sourceUrl": "repository1/tree/commit1",
-          "commit": "commit1"
-        },
-        "reason": "unknown reason",
-        "at": "(ignore)"
-      },
-      "lastCompleted": {
-        "id": 1,
-        "version": "(ignore)",
-        "revision": {
-          "buildNumber": "(ignore)",
-          "hash": "(ignore)",
-          "source": {
-            "gitRepository": "repository1",
-            "gitBranch": "master",
-            "gitCommit": "commit1"
-          },
-          "sourceUrl": "repository1/tree/commit1",
-          "commit": "commit1"
-        },
-        "reason": "unknown reason",
-        "at": "(ignore)"
-      },
-      "lastSuccess": {
-        "id": 1,
-        "version": "(ignore)",
-        "revision": {
-          "buildNumber": "(ignore)",
-          "hash": "(ignore)",
-          "source": {
-            "gitRepository": "repository1",
-            "gitBranch": "master",
-            "gitCommit": "commit1"
-          },
-          "sourceUrl": "repository1/tree/commit1",
-          "commit": "commit1"
-        },
-        "reason": "unknown reason",
-        "at": "(ignore)"
-      }
-    },
-    {
-      "type": "staging-test",
-      "success": true,
-      "lastTriggered": {
-        "id": 1,
-        "version": "(ignore)",
-        "revision": {
-          "buildNumber": "(ignore)",
-          "hash": "(ignore)",
-          "source": {
-            "gitRepository": "repository1",
-            "gitBranch": "master",
-            "gitCommit": "commit1"
-          },
-          "sourceUrl": "repository1/tree/commit1",
-          "commit": "commit1"
-        },
-        "reason": "unknown reason",
-        "at": "(ignore)"
-      },
-      "lastCompleted": {
-        "id": 1,
-        "version": "(ignore)",
-        "revision": {
-          "buildNumber": "(ignore)",
-          "hash": "(ignore)",
-          "source": {
-            "gitRepository": "repository1",
-            "gitBranch": "master",
-            "gitCommit": "commit1"
-          },
-          "sourceUrl": "repository1/tree/commit1",
-          "commit": "commit1"
-        },
-        "reason": "unknown reason",
-        "at": "(ignore)"
-      },
-      "lastSuccess": {
-        "id": 1,
-        "version": "(ignore)",
-        "revision": {
-          "buildNumber": "(ignore)",
-          "hash": "(ignore)",
-          "source": {
-            "gitRepository": "repository1",
-            "gitBranch": "master",
-            "gitCommit": "commit1"
-          },
-          "sourceUrl": "repository1/tree/commit1",
-          "commit": "commit1"
-        },
-        "reason": "unknown reason",
-        "at": "(ignore)"
-      }
-    },
-    {
-      "type": "production-us-central-1",
-      "success": true,
-      "lastTriggered": {
-        "id": 1,
-        "version": "(ignore)",
-        "revision": {
-          "buildNumber": "(ignore)",
-          "hash": "(ignore)",
-          "source": {
-            "gitRepository": "repository1",
-            "gitBranch": "master",
-            "gitCommit": "commit1"
-          },
-          "sourceUrl": "repository1/tree/commit1",
-          "commit": "commit1"
-        },
-        "reason": "unknown reason",
-        "at": "(ignore)"
-      },
-      "lastCompleted": {
-        "id": 1,
-        "version": "(ignore)",
-        "revision": {
-          "buildNumber": "(ignore)",
-          "hash": "(ignore)",
-          "source": {
-            "gitRepository": "repository1",
-            "gitBranch": "master",
-            "gitCommit": "commit1"
-          },
-          "sourceUrl": "repository1/tree/commit1",
-          "commit": "commit1"
-        },
-        "reason": "unknown reason",
-        "at": "(ignore)"
-      },
-      "lastSuccess": {
-        "id": 1,
-        "version": "(ignore)",
-        "revision": {
-          "buildNumber": "(ignore)",
-          "hash": "(ignore)",
-          "source": {
-            "gitRepository": "repository1",
-            "gitBranch": "master",
-            "gitCommit": "commit1"
-          },
-          "sourceUrl": "repository1/tree/commit1",
-          "commit": "commit1"
-        },
-        "reason": "unknown reason",
-        "at": "(ignore)"
-      }
-    },
-    {
-      "type": "production-us-east-3",
-      "success": false,
-      "lastTriggered": {
-        "id": 1,
-        "version": "(ignore)",
-        "revision": {
-          "buildNumber": "(ignore)",
-          "hash": "(ignore)",
-          "source": {
-            "gitRepository": "repository1",
-            "gitBranch": "master",
-            "gitCommit": "commit1"
-          },
-          "sourceUrl": "repository1/tree/commit1",
-          "commit": "commit1"
-        },
-        "reason": "unknown reason",
-        "at": "(ignore)"
-      }
-    },
-    {
-      "type": "production-us-west-1",
-      "success": false
-    }
-  ],
   "changeBlockers": [
     {
       "versions": true,
@@ -241,7 +49,6 @@
       ]
     }
   ],
-  "compileVersion": "(ignore)",
   "globalRotations": [
     "https://instance1--application1--tenant1.global.vespa.oath.cloud:4443/"
   ],
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs-direct-deployment.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs-direct-deployment.json
index 85a3245c308..1e43c6e2953 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs-direct-deployment.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs-direct-deployment.json
@@ -1,7 +1,8 @@
 {
-  "devJobs": {},
-  "deployments": [],
   "lastVersions": {},
   "deploying": {},
-  "jobs": {}
+  "deployments": [],
+  "jobs": {},
+  "devJobs": {},
+  "deployment": []
 }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs.json
index dc37c3b4bb4..b16ca4cc67c 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs.json
@@ -230,19 +230,13 @@
             "commit": "commit1"
           },
           "steps": {
-            "deployTester": "succeeded",
-            "installTester": "succeeded",
             "deployReal": "succeeded",
             "installReal": "succeeded",
-            "startTests": "succeeded",
-            "endTests": "succeeded",
-            "deactivateTester": "succeeded",
             "report": "succeeded"
           },
           "tasks": {
             "deploy": "succeeded",
-            "install": "succeeded",
-            "test": "succeeded"
+            "install": "succeeded"
           },
           "log": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/production-us-central-1/run/1"
         }
@@ -268,13 +262,8 @@
             "commit": "commit1"
           },
           "steps": {
-            "deployTester": "unfinished",
-            "installTester": "unfinished",
             "deployReal": "unfinished",
             "installReal": "unfinished",
-            "startTests": "unfinished",
-            "endTests": "unfinished",
-            "deactivateTester": "unfinished",
             "report": "unfinished"
           },
           "tasks": {},
@@ -286,7 +275,7 @@
     "us-east-3": {
       "runs": [
         {
-          "id": 1,
+          "id": 2,
           "status": "aborted",
           "start": "(ignore)",
           "wantedPlatform": "6.1",
@@ -302,16 +291,31 @@
             "commit": "commit1"
           },
           "steps": {
-            "deployTester": "unfinished",
-            "installTester": "unfinished",
             "deployReal": "unfinished",
             "installReal": "unfinished",
-            "startTests": "unfinished",
-            "endTests": "unfinished",
-            "deactivateTester": "unfinished",
             "report": "unfinished"
           },
           "tasks": {},
+          "log": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/production-us-east-3/run/2"
+        },
+        {
+          "id": 1,
+          "status": "success",
+          "start": "(ignore)",
+          "end": "(ignore)",
+          "wantedPlatform": "6.1",
+          "wantedApplication": {
+            "hash": "unknown"
+          },
+          "steps": {
+            "deployReal": "succeeded",
+            "installReal": "succeeded",
+            "copyVespaLogs": "succeeded"
+          },
+          "tasks": {
+            "deploy": "succeeded",
+            "install": "succeeded"
+          },
           "log": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/production-us-east-3/run/1"
         }
       ],
@@ -344,5 +348,37 @@
       ],
       "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/dev-us-east-1"
     }
-  }
+  },
+  "deployment": [
+    {
+      "jobName": "dev-us-east-1",
+      "runs": [
+        {
+          "id": 1,
+          "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/run/1",
+          "start": "(ignore)",
+          "end": "(ignore)",
+          "status": "success",
+          "versions": {
+            "targetPlatform": "6.1.0",
+            "targetApplication": {}
+          },
+          "steps": [
+            {
+              "name": "deployReal",
+              "status": "succeeded"
+            },
+            {
+              "name": "installReal",
+              "status": "succeeded"
+            },
+            {
+              "name": "copyVespaLogs",
+              "status": "succeeded"
+            }
+          ]
+        }
+      ]
+    }
+  ]
 }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview-user-instance.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview-user-instance.json
index cf47e8671b0..d46b396b8cd 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview-user-instance.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview-user-instance.json
@@ -1,55 +1,86 @@
 {
+  "lastVersions": {
+    "platform": {
+      "platform": "7.1",
+      "at": 0,
+      "completed": "0 of 0 complete"
+    },
+    "application": {
+      "application": {
+        "hash": "1.0.3-commit1",
+        "build": 3,
+        "source": {
+          "gitRepository": "repository1",
+          "gitBranch": "master",
+          "gitCommit": "commit1"
+        },
+        "sourceUrl": "repository1/tree/commit1",
+        "commit": "commit1"
+      },
+      "at": 1000,
+      "completed": "0 of 0 complete"
+    }
+  },
+  "deploying": {},
+  "deployments": [],
+  "jobs": {},
   "devJobs": {
     "dev-aws-us-east-2a": {
       "runs": [
         {
+          "id": 1,
+          "status": "success",
+          "start": 7303000,
+          "end": 7303000,
           "wantedPlatform": "7.1",
-          "log": "https://some.url:43/root/dev-aws-us-east-2a/run/1",
           "wantedApplication": {
             "hash": "unknown"
           },
-          "start": 102000,
-          "end": 102000,
-          "id": 1,
           "steps": {
             "deployReal": "succeeded",
             "installReal": "succeeded",
             "copyVespaLogs": "succeeded"
           },
           "tasks": {
-            "install": "succeeded",
-            "deploy": "succeeded"
+            "deploy": "succeeded",
+            "install": "succeeded"
           },
-          "status": "success"
+          "log": "https://some.url:43/root/dev-aws-us-east-2a/run/1"
         }
       ],
       "url": "https://some.url:43/root/dev-aws-us-east-2a"
     }
   },
-  "deployments": [],
-  "lastVersions": {
-    "application": {
-      "at": 1000,
-      "application": {
-        "build": 3,
-        "source": {
-          "gitRepository": "repository1",
-          "gitCommit": "commit1",
-          "gitBranch": "master"
-        },
-        "sourceUrl": "repository1/tree/commit1",
-        "commit": "commit1",
-        "hash": "1.0.3-commit1"
-      },
-      "completed": "0 of 0 complete"
-    },
-    "platform": {
-      "at": 0,
-      "completed": "0 of 0 complete",
-      "platform": "7.1"
+  "deployment": [
+    {
+      "jobName": "dev-aws-us-east-2a",
+      "runs": [
+        {
+          "id": 1,
+          "url": "https://some.url:43/root//run/1",
+          "start": 7303000,
+          "end": 7303000,
+          "status": "success",
+          "versions": {
+            "targetPlatform": "7.1.0",
+            "targetApplication": {}
+          },
+          "steps": [
+            {
+              "name": "deployReal",
+              "status": "succeeded"
+            },
+            {
+              "name": "installReal",
+              "status": "succeeded"
+            },
+            {
+              "name": "copyVespaLogs",
+              "status": "succeeded"
+            }
+          ]
+        }
+      ]
     }
-  },
-  "deploying": {},
-  "jobs": {}
+  ]
 }
-
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview.json
index ad00476154c..2c43aa45d06 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview.json
@@ -37,7 +37,7 @@
   "deployments": [
     {
       "us-central-1": {
-        "at": 2000,
+        "at": 7203000,
         "platform": "6.1",
         "application": {
           "hash": "1.0.3-commit1",
@@ -97,8 +97,8 @@
         {
           "id": 3,
           "status": "success",
-          "start": 2000,
-          "end": 2000,
+          "start": 7203000,
+          "end": 7203000,
           "wantedPlatform": "6.1",
           "wantedApplication": {
             "hash": "1.0.3-commit1",
@@ -265,8 +265,8 @@
         {
           "id": 5,
           "status": "installationFailed",
-          "start": 102000,
-          "end": 102000,
+          "start": 7303000,
+          "end": 7303000,
           "wantedPlatform": "6.1",
           "wantedApplication": {
             "hash": "1.0.3-commit1",
@@ -313,8 +313,8 @@
         {
           "id": 4,
           "status": "installationFailed",
-          "start": 2000,
-          "end": 2000,
+          "start": 7203000,
+          "end": 7203000,
           "wantedPlatform": "6.1",
           "wantedApplication": {
             "hash": "1.0.3-commit1",
@@ -361,8 +361,8 @@
         {
           "id": 3,
           "status": "success",
-          "start": 2000,
-          "end": 2000,
+          "start": 7203000,
+          "end": 7203000,
           "wantedPlatform": "6.1",
           "wantedApplication": {
             "hash": "1.0.3-commit1",
@@ -510,7 +510,7 @@
         {
           "id": 3,
           "status": "running",
-          "start": 2000,
+          "start": 7203000,
           "wantedPlatform": "6.1",
           "wantedApplication": {
             "hash": "1.0.3-commit1",
@@ -536,13 +536,8 @@
             "commit": "commit1"
           },
           "steps": {
-            "deployTester": "succeeded",
-            "installTester": "unfinished",
             "deployReal": "succeeded",
             "installReal": "unfinished",
-            "startTests": "unfinished",
-            "endTests": "unfinished",
-            "deactivateTester": "unfinished",
             "report": "unfinished"
           },
           "tasks": {
@@ -581,19 +576,13 @@
             "commit": "commit1"
           },
           "steps": {
-            "deployTester": "succeeded",
-            "installTester": "succeeded",
             "deployReal": "succeeded",
             "installReal": "succeeded",
-            "startTests": "succeeded",
-            "endTests": "succeeded",
-            "deactivateTester": "succeeded",
             "report": "succeeded"
           },
           "tasks": {
             "deploy": "succeeded",
-            "install": "succeeded",
-            "test": "succeeded"
+            "install": "succeeded"
           },
           "log": "https://some.url:43/root/production-us-central-1/run/2"
         },
@@ -615,19 +604,13 @@
             "commit": "commit1"
           },
           "steps": {
-            "deployTester": "succeeded",
-            "installTester": "succeeded",
             "deployReal": "succeeded",
             "installReal": "succeeded",
-            "startTests": "succeeded",
-            "endTests": "succeeded",
-            "deactivateTester": "succeeded",
             "report": "succeeded"
           },
           "tasks": {
             "deploy": "succeeded",
-            "install": "succeeded",
-            "test": "succeeded"
+            "install": "succeeded"
           },
           "log": "https://some.url:43/root/production-us-central-1/run/1"
         }
@@ -789,9 +772,9 @@
         },
         {
           "id": 2,
-          "status": "testFailure",
+          "status": "installationFailed",
           "start": 1000,
-          "end": 1000,
+          "end": 7202000,
           "wantedPlatform": "6.1",
           "wantedApplication": {
             "hash": "1.0.2-commit1",
@@ -817,19 +800,13 @@
             "commit": "commit1"
           },
           "steps": {
-            "deployTester": "succeeded",
-            "installTester": "succeeded",
             "deployReal": "succeeded",
-            "installReal": "succeeded",
-            "startTests": "succeeded",
-            "endTests": "failed",
-            "deactivateTester": "succeeded",
+            "installReal": "failed",
             "report": "succeeded"
           },
           "tasks": {
             "deploy": "succeeded",
-            "install": "succeeded",
-            "test": "failed"
+            "install": "failed"
           },
           "log": "https://some.url:43/root/production-us-west-1/run/2"
         },
@@ -851,19 +828,13 @@
             "commit": "commit1"
           },
           "steps": {
-            "deployTester": "succeeded",
-            "installTester": "succeeded",
             "deployReal": "succeeded",
             "installReal": "succeeded",
-            "startTests": "succeeded",
-            "endTests": "succeeded",
-            "deactivateTester": "succeeded",
             "report": "succeeded"
           },
           "tasks": {
             "deploy": "succeeded",
-            "install": "succeeded",
-            "test": "succeeded"
+            "install": "succeeded"
           },
           "log": "https://some.url:43/root/production-us-west-1/run/1"
         }
@@ -933,16 +904,13 @@
             "commit": "commit1"
           },
           "steps": {
-            "deployTester": "failed",
-            "installTester": "unfinished",
-            "deployReal": "unfinished",
+            "deployReal": "failed",
             "installReal": "unfinished",
-            "startTests": "unfinished",
-            "endTests": "unfinished",
-            "deactivateTester": "succeeded",
             "report": "succeeded"
           },
-          "tasks": {},
+          "tasks": {
+            "deploy": "failed"
+          },
           "log": "https://some.url:43/root/production-us-east-3/run/2"
         },
         {
@@ -963,19 +931,13 @@
             "commit": "commit1"
           },
           "steps": {
-            "deployTester": "succeeded",
-            "installTester": "succeeded",
             "deployReal": "succeeded",
             "installReal": "succeeded",
-            "startTests": "succeeded",
-            "endTests": "succeeded",
-            "deactivateTester": "succeeded",
             "report": "succeeded"
           },
           "tasks": {
             "deploy": "succeeded",
-            "install": "succeeded",
-            "test": "succeeded"
+            "install": "succeeded"
           },
           "log": "https://some.url:43/root/production-us-east-3/run/1"
         }
@@ -983,5 +945,6 @@
       "url": "https://some.url:43/root/production-us-east-3"
     }
   },
-  "devJobs": {}
+  "devJobs": {},
+  "deployment": []
 }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-us-central-1.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-us-central-1.json
index 39824e22928..4ffe809297d 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-us-central-1.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-us-central-1.json
@@ -7,9 +7,21 @@
   "instance": "instance1",
   "environment": "prod",
   "region": "us-central-1",
-  "endpoints": [],
-  "serviceUrls": [
-    "https://instance1--application1--tenant1.us-central-1.prod.vespa:43"
+  "endpoints": [
+    {
+      "cluster": "default",
+      "tls": true,
+      "url": "https://instance1--application1--tenant1.us-central-1.vespa.oath.cloud:4443/",
+      "scope": "zone",
+      "routingMethod": "shared"
+    },
+    {
+      "cluster": "",
+      "tls": true,
+      "url": "https://instance1--application1--tenant1.global.vespa.oath.cloud:4443/",
+      "scope": "global",
+      "routingMethod": "shared"
+    }
   ],
   "nodes": "http://localhost:8080/zone/v2/prod/us-central-1/nodes/v2/node/%3F&recursive=true&application=tenant1.application1.instance1",
   "yamasUrl": "http://monitoring-system.test/?environment=prod&region=us-central-1&application=tenant1.application1.instance1",
@@ -47,12 +59,6 @@
     "lastQueriesPerSecond": 1.0,
     "lastWritesPerSecond": 2.0
   },
-  "cost": {
-    "tco": 0,
-    "waste": 0,
-    "utilization": 0.0,
-    "cluster": {}
-  },
   "metrics": {
     "queriesPerSecond": 1.0,
     "writesPerSecond": 2.0,
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/root.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/root.json
index 986245decca..d63a7ba7d56 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/root.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/root.json
@@ -1,9 +1,6 @@
 {
   "resources":[
     {
-      "url":"http://localhost:8080/application/v4/user/"
-    },
-    {
       "url":"http://localhost:8080/application/v4/tenant/"
     }
   ]
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-runs.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-runs.json
index 4238ce4b834..2ad35968732 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-runs.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-runs.json
@@ -94,8 +94,8 @@
   "3": {
     "id": 3,
     "status": "success",
-    "start": 2000,
-    "end": 2000,
+    "start": 7203000,
+    "end": 7203000,
     "wantedPlatform": "6.1",
     "wantedApplication": {
       "hash": "1.0.3-commit1",
@@ -146,8 +146,8 @@
   "4": {
     "id": 4,
     "status": "installationFailed",
-    "start": 2000,
-    "end": 2000,
+    "start": 7203000,
+    "end": 7203000,
     "wantedPlatform": "6.1",
     "wantedApplication": {
       "hash": "1.0.3-commit1",
@@ -194,8 +194,8 @@
   "5": {
     "id": 5,
     "status": "installationFailed",
-    "start": 102000,
-    "end": 102000,
+    "start": 7303000,
+    "end": 7303000,
     "wantedPlatform": "6.1",
     "wantedApplication": {
       "hash": "1.0.3-commit1",
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-test-log.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-test-log.json
index 273887c26c4..ba51471d467 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-test-log.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-test-log.json
@@ -4,144 +4,149 @@
   "log": {
     "deployTester": [
       {
-        "at": 102000,
+        "at": 7303000,
         "type": "info",
         "message": "No services requiring restart."
       },
       {
-        "at": 102000,
+        "at": 7303000,
         "type": "info",
         "message": "Deployment successful."
       },
       {
-        "at": 102000,
+        "at": 7303000,
         "type": "info",
         "message": "foo"
       }
     ],
     "installTester": [
       {
-        "at": 102000,
+        "at": 7303000,
         "type": "info",
         "message": "host-tenant:application:default-t-staging.us-east-3: unorchestrated"
       },
       {
-        "at": 102000,
+        "at": 7303000,
         "type": "info",
         "message": "--- platform 6.1"
       },
       {
-        "at": 102000,
+        "at": 7303000,
         "type": "info",
         "message": "--- container on port 43 has config generation 1, wanted is 2"
       },
       {
-        "at": 102000,
+        "at": 7303000,
         "type": "info",
         "message": "host-tenant:application:default-t-staging.us-east-3: unorchestrated"
       },
       {
-        "at": 102000,
+        "at": 7303000,
         "type": "info",
         "message": "--- platform 6.1"
       },
       {
-        "at": 102000,
+        "at": 7303000,
         "type": "info",
         "message": "--- container on port 43 has config generation 1, wanted is 2"
       },
       {
-        "at": 102000,
+        "at": 7303000,
         "type": "info",
         "message": "host-tenant:application:default-t-staging.us-east-3: unorchestrated"
       },
       {
-        "at": 102000,
+        "at": 7303000,
         "type": "info",
         "message": "--- platform 6.1"
       },
       {
-        "at": 102000,
+        "at": 7303000,
         "type": "info",
         "message": "--- container on port 43 has config generation 1, wanted is 2"
       }
     ],
     "deployInitialReal": [
       {
-        "at": 102000,
+        "at": 7303000,
         "type": "info",
         "message": "Deploying platform version 6.1 and application version 1.0.1-commit1 ..."
       },
       {
-        "at": 102000,
+        "at": 7303000,
         "type": "info",
         "message": "No services requiring restart."
       },
       {
-        "at": 102000,
+        "at": 7303000,
         "type": "info",
         "message": "Deployment successful."
       },
       {
-        "at": 102000,
+        "at": 7303000,
         "type": "info",
         "message": "foo"
       }
     ],
     "installInitialReal": [
       {
-        "at": 102000,
+        "at": 7303000,
+        "type": "info",
+        "message": "######## Details for all nodes ########"
+      },
+      {
+        "at": 7303000,
         "type": "info",
         "message": "host-tenant:application:default-staging.us-east-3: unorchestrated"
       },
       {
-        "at": 102000,
+        "at": 7303000,
         "type": "info",
         "message": "--- platform 6.1"
       },
       {
-        "at": 102000,
+        "at": 7303000,
         "type": "info",
         "message": "--- container on port 43 has config generation 1, wanted is 2"
       },
       {
-        "at": 102000,
+        "at": 7303000,
         "type": "info",
         "message": "Deployment expired before installation was successful."
       }
     ],
     "deactivateReal": [
       {
-        "at": 102000,
+        "at": 7303000,
         "type": "info",
         "message": "Deactivating deployment of tenant.application in staging.us-east-3 ..."
       }
     ],
     "deactivateTester": [
       {
-        "at": 102000,
+        "at": 7303000,
         "type": "info",
         "message": "Deactivating tester of tenant.application in staging.us-east-3 ..."
       }
     ]
   },
-  "lastId": 22,
+  "lastId": 23,
   "steps": {
     "deployTester": {
       "status": "succeeded",
-      "startMillis": 102000
+      "startMillis": 7303000
     },
     "installTester": {
       "status": "unfinished",
-      "startMillis": 102000
+      "startMillis": 7303000
     },
     "deployInitialReal": {
       "status": "succeeded",
-      "startMillis": 102000
+      "startMillis": 7303000
     },
     "installInitialReal": {
       "status": "failed",
-      "startMillis": 102000,
+      "startMillis": 7303000,
       "convergence": {
         "nodes": 1,
         "down": 0,
@@ -177,19 +182,19 @@
     },
     "copyVespaLogs": {
       "status": "succeeded",
-      "startMillis": 102000
+      "startMillis": 7303000
     },
     "deactivateReal": {
       "status": "succeeded",
-      "startMillis": 102000
+      "startMillis": 7303000
     },
     "deactivateTester": {
       "status": "succeeded",
-      "startMillis": 102000
+      "startMillis": 7303000
     },
     "report": {
       "status": "succeeded",
-      "startMillis": 102000
+      "startMillis": 7303000
     }
   }
 }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-details.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-details.json
index dd3d16fc721..489d6a11b6a 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-details.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-details.json
@@ -83,12 +83,41 @@
       {
         "at": "(ignore)",
         "type": "info",
-        "message": "Endpoints not yet ready."
+        "message": "Tester container successfully installed!"
+      }
+    ],
+    "deployReal": [
+      {
+        "at": "(ignore)",
+        "type": "info",
+        "message": "Deploying platform version 6.1 and application version 1.0.1-commit1 ..."
       },
       {
         "at": "(ignore)",
         "type": "info",
-        "message": "host-tenant1:application1:instance1-t-test.us-east-1: unorchestrated"
+        "message": "No services requiring restart."
+      },
+      {
+        "at": "(ignore)",
+        "type": "info",
+        "message": "Deployment successful."
+      },
+      {
+        "at": "(ignore)",
+        "type": "info",
+        "message": "foo"
+      }
+    ],
+    "installReal": [
+      {
+        "at": "(ignore)",
+        "type": "info",
+        "message": "######## Details for all nodes ########"
+      },
+      {
+        "at": "(ignore)",
+        "type": "info",
+        "message": "host-tenant1:application1:instance1-test.us-east-1: unorchestrated"
       },
       {
         "at": "(ignore)",
@@ -98,47 +127,68 @@
       {
         "at": "(ignore)",
         "type": "info",
-        "message": "Found endpoints:"
+        "message": "--- container on port 43 has config generation 1, wanted is 2"
       },
       {
         "at": "(ignore)",
         "type": "info",
-        "message": "- test.us-east-1"
+        "message": "host-tenant1:application1:instance1-test.us-east-1: unorchestrated"
       },
       {
         "at": "(ignore)",
         "type": "info",
-        "message": " |-- https://instance1-t--application1--tenant1.us-east-1.test.vespa:43 (cluster 'default')"
+        "message": "--- platform 6.1"
       },
       {
         "at": "(ignore)",
         "type": "info",
-        "message": "Tester container successfully installed!"
-      }
-    ],
-    "deployReal": [
+        "message": "--- container on port 43 has config generation 1, wanted is 2"
+      },
       {
         "at": "(ignore)",
         "type": "info",
-        "message": "Deploying platform version 6.1 and application version 1.0.1-commit1 ..."
+        "message": "host-tenant1:application1:instance1-test.us-east-1: unorchestrated"
       },
       {
         "at": "(ignore)",
         "type": "info",
-        "message": "No services requiring restart."
+        "message": "--- platform 6.1"
       },
       {
         "at": "(ignore)",
         "type": "info",
-        "message": "Deployment successful."
+        "message": "--- container on port 43 has config generation 1, wanted is 2"
       },
       {
         "at": "(ignore)",
         "type": "info",
-        "message": "foo"
-      }
-    ],
-    "installReal": [
+        "message": "host-tenant1:application1:instance1-test.us-east-1: unorchestrated"
+      },
+      {
+        "at": "(ignore)",
+        "type": "info",
+        "message": "--- platform 6.1"
+      },
+      {
+        "at": "(ignore)",
+        "type": "info",
+        "message": "--- container on port 43 has config generation 1, wanted is 2"
+      },
+      {
+        "at": "(ignore)",
+        "type": "info",
+        "message": "host-tenant1:application1:instance1-test.us-east-1: unorchestrated"
+      },
+      {
+        "at": "(ignore)",
+        "type": "info",
+        "message": "--- platform 6.1"
+      },
+      {
+        "at": "(ignore)",
+        "type": "info",
+        "message": "--- container on port 43 has config generation 1, wanted is 2"
+      },
       {
         "at": "(ignore)",
         "type": "info",
@@ -167,7 +217,7 @@
       {
         "at": "(ignore)",
         "type": "info",
-        "message": " |-- https://instance1--application1--tenant1.us-east-1.test.vespa:43 (cluster 'default')"
+        "message": " |-- https://instance1--application1--tenant1.us-east-1.test.vespa.oath.cloud:4443/ (cluster 'default')"
       },
       {
         "at": "(ignore)",
@@ -194,7 +244,7 @@
       {
         "at": "(ignore)",
         "type": "info",
-        "message": " |-- https://instance1--application1--tenant1.us-east-1.test.vespa:43 (cluster 'default')"
+        "message": " |-- https://instance1--application1--tenant1.us-east-1.test.vespa.oath.cloud:4443/ (cluster 'default')"
       },
       {
         "at": "(ignore)",
@@ -224,7 +274,7 @@
       }
     ]
   },
-  "lastId": 40,
+  "lastId": 50,
   "steps": {
     "deployTester": {
       "status": "succeeded",
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/test-config-dev.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/test-config-dev.json
index 818dc72a9d9..0632ab7a67b 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/test-config-dev.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/test-config-dev.json
@@ -5,18 +5,18 @@
   "isCI": false,
   "endpoints": {
     "dev.us-east-1": [
-      "https://us-east-1.dev.my-user"
+      "https://my-user--application1--tenant1.us-east-1.dev.vespa.oath.cloud:4443/"
     ],
     "prod.us-central-1": [
-      "https://us-central-1.prod.default"
+      "https://application1--tenant1.us-central-1.vespa.oath.cloud:4443/"
     ]
   },
   "zoneEndpoints": {
     "dev.us-east-1": {
-      "default": "https://us-east-1.dev.my-user"
+      "default": "https://my-user--application1--tenant1.us-east-1.dev.vespa.oath.cloud:4443/"
     },
     "prod.us-central-1": {
-      "default": "https://us-central-1.prod.default"
+      "default": "https://application1--tenant1.us-central-1.vespa.oath.cloud:4443/"
     }
   },
   "clusters": {
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/test-config.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/test-config.json
index c1ccbc7100f..c81ed767239 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/test-config.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/test-config.json
@@ -5,12 +5,12 @@
   "isCI": false,
   "endpoints": {
     "prod.us-central-1": [
-      "https://us-central-1.prod.default"
+      "https://application1--tenant1.us-central-1.vespa.oath.cloud:4443/"
     ]
   },
   "zoneEndpoints": {
     "prod.us-central-1": {
-      "default": "https://us-central-1.prod.default"
+      "default": "https://application1--tenant1.us-central-1.vespa.oath.cloud:4443/"
     }
   },
   "clusters": {
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/us-east-3-log-without-first.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/us-east-3-log-without-first.json
index 0fa4c541832..6c9315ca64b 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/us-east-3-log-without-first.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/us-east-3-log-without-first.json
@@ -1,50 +1,27 @@
 {
+  "active": false,
+  "status": "deploymentFailed",
   "log": {
-    "deployTester": [
+    "deployReal": [
       {
         "at": 1000,
         "type": "info",
         "message": "Failed to deploy application: ERROR!"
       }
-    ],
-    "deactivateTester": [
-      {
-        "at": 1000,
-        "type": "info",
-        "message": "Deactivating tester of tenant.application in prod.us-east-3 ..."
-      }
     ]
   },
-  "active": false,
-  "lastId": 2,
+  "lastId": 1,
   "steps": {
-    "startTests": {
-      "status": "unfinished"
-    },
-    "deployTester": {
-      "startMillis": 1000,
-      "status": "failed"
-    },
-    "report": {
-      "startMillis": 1000,
-      "status": "succeeded"
-    },
-    "installTester": {
-      "status": "unfinished"
-    },
     "deployReal": {
-      "status": "unfinished"
+      "status": "failed",
+      "startMillis": 1000
     },
     "installReal": {
       "status": "unfinished"
     },
-    "deactivateTester": {
-      "startMillis": 1000,
-      "status": "succeeded"
-    },
-    "endTests": {
-      "status": "unfinished"
+    "report": {
+      "status": "succeeded",
+      "startMillis": 1000
     }
-  },
-  "status": "deploymentFailed"
+  }
 }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/user-which-exists.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/user-which-exists.json
deleted file mode 100644
index f2703677738..00000000000
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/user-which-exists.json
+++ /dev/null
@@ -1,5 +0,0 @@
-{
-  "user": "myuser",
-  "tenants": @include(tenant-list-with-user.json),
-  "tenantExists": true
-}
\ No newline at end of file
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/user.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/user.json
deleted file mode 100644
index 79b9a785801..00000000000
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/user.json
+++ /dev/null
@@ -1,5 +0,0 @@
-{
-  "user": "myuser",
-  "tenants": @include(tenant-list.json),
-  "tenantExists": false
-}
\ No newline at end of file
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/configserver/ConfigServerApiHandlerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/configserver/ConfigServerApiHandlerTest.java
index 8aea26f21e3..c414a3680fc 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/configserver/ConfigServerApiHandlerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/configserver/ConfigServerApiHandlerTest.java
@@ -7,7 +7,6 @@ import com.yahoo.config.provision.RegionName;
 import com.yahoo.config.provision.zone.ZoneApi;
 import com.yahoo.vespa.hosted.controller.integration.ConfigServerProxyMock;
 import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock;
-import com.yahoo.vespa.hosted.controller.integration.ZoneRegistryMock;
 import com.yahoo.vespa.hosted.controller.proxy.ProxyRequest;
 import com.yahoo.vespa.hosted.controller.restapi.ContainerTester;
 import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerTest;
@@ -140,4 +139,4 @@ public class ConfigServerApiHandlerTest extends ControllerContainerTest {
         assertEquals(List.of(URI.create(target)), last.getTargets());
         assertEquals(com.yahoo.jdisc.http.HttpRequest.Method.valueOf(method), last.getMethod());
     }
-}
\ No newline at end of file
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java
index b21c588235e..8029d650b75 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java
@@ -2,8 +2,11 @@
 package com.yahoo.vespa.hosted.controller.restapi.controller;
 
 import com.yahoo.application.container.handler.Request;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.zone.ZoneId;
 import com.yahoo.container.jdisc.HttpRequest;
 import com.yahoo.test.ManualClock;
+import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceSnapshot;
 import com.yahoo.vespa.hosted.controller.auditlog.AuditLogger;
 import com.yahoo.vespa.hosted.controller.restapi.ContainerTester;
 import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerTest;
@@ -15,6 +18,7 @@ import java.io.File;
 import java.nio.charset.StandardCharsets;
 import java.time.Duration;
 import java.time.Instant;
+import java.util.List;
 import java.util.Set;
 
 import static org.junit.Assert.assertFalse;
@@ -156,4 +160,20 @@ public class ControllerApiTest extends ControllerContainerTest {
         tester.assertResponse(authenticatedRequest("http://localhost:8080/controller/v1/auditlog/"), new File("auditlog.json"));
     }
 
+    @Test
+    public void testMeteringApi() {
+        ApplicationId applicationId = ApplicationId.from("tenant", "app", "instance");
+        Instant timestamp = Instant.ofEpochMilli(123456789);
+        ZoneId zoneId = ZoneId.defaultId();
+        List<ResourceSnapshot> snapshots = List.of(
+                new ResourceSnapshot(applicationId, 12,48,1200, timestamp, zoneId),
+                new ResourceSnapshot(applicationId, 24, 96,2400, timestamp, zoneId)
+        );
+        tester.controller().serviceRegistry().meteringService().consume(snapshots);
+        tester.assertResponse(
+                operatorRequest("http://localhost:8080/controller/v1/metering/tenant/tenantName/month/2020-02", "", Request.Method.GET),
+                new File("metering.json")
+        );
+    }
+
 }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json
index 3371c5563c9..fbdf8caaed7 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json
@@ -10,9 +10,6 @@
       "name": "CloudEventReporter"
     },
     {
-      "name": "ClusterInfoMaintainer"
-    },
-    {
       "name": "ContactInformationMaintainer"
     },
     {
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/metering.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/metering.json
new file mode 100644
index 00000000000..b64e8f26a63
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/metering.json
@@ -0,0 +1,18 @@
+[
+  {
+    "applicationId": "tenant.app.instance",
+    "timestamp": 123456789,
+    "zoneId": "prod.default",
+    "cpu": 12.0,
+    "memory": 48.0,
+    "disk": 1200.0
+  },
+  {
+    "applicationId": "tenant.app.instance",
+    "timestamp": 123456789,
+    "zoneId": "prod.default",
+    "cpu": 24.0,
+    "memory": 96.0,
+    "disk": 2400.0
+  }
+]
\ No newline at end of file
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostApiTest.java
deleted file mode 100644
index f992a54a114..00000000000
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostApiTest.java
+++ /dev/null
@@ -1,49 +0,0 @@
-// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.controller.restapi.cost;
-
-import com.yahoo.application.container.handler.Request;
-import com.yahoo.config.provision.CloudName;
-import com.yahoo.config.provision.SystemName;
-import com.yahoo.config.provision.zone.ZoneApi;
-import com.yahoo.vespa.athenz.api.AthenzIdentity;
-import com.yahoo.vespa.athenz.api.AthenzUser;
-import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock;
-import com.yahoo.vespa.hosted.controller.restapi.ContainerTester;
-import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerTest;
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- * @author andreer
- */
-public class CostApiTest extends ControllerContainerTest {
-
-    private static final String responses = "src/test/java/com/yahoo/vespa/hosted/controller/restapi/cost/responses/";
-    private static final AthenzIdentity operator = AthenzUser.fromUserId("operatorUser");
-    private static final CloudName cloud1 = CloudName.from("yahoo");
-    private static final CloudName cloud2 = CloudName.from("cloud2");
-    private static final ZoneApi zone1 = ZoneApiMock.newBuilder().withId("prod.us-east-3").with(cloud1).build();
-    private static final ZoneApi zone2 = ZoneApiMock.newBuilder().withId("prod.us-west-1").with(cloud1).build();
-    private static final ZoneApi zone3 = ZoneApiMock.newBuilder().withId("prod.eu-west-1").with(cloud2).build();
-
-    private ContainerTester tester;
-
-    @Before
-    public void before() {
-        tester = new ContainerTester(container, responses);
-        tester.serviceRegistry().zoneRegistry().setSystemName(SystemName.cd)
-              .setZones(zone1, zone2, zone3);
-    }
-
-    @Test
-    public void test_api() {
-        assertResponse(new Request("http://localhost:8080/cost/v1/csv"),
-                "Date,Property,Reserved Cpu Cores,Reserved Memory GB,Reserved Disk Space GB,Usage Fraction\n", 200);
-    }
-
-    private void assertResponse(Request request, String body, int statusCode) {
-        addIdentityToRequest(request, operator);
-        tester.assertResponse(request, body, statusCode);
-    }
-
-}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java
index 6df2b00c9e5..c8036676fa9 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java
@@ -65,10 +65,13 @@ public class DeploymentApiTest extends ControllerContainerTest {
         deploymentTester.upgrader().maintain();
         deploymentTester.triggerJobs();
         productionApp.runJob(JobType.systemTest).runJob(JobType.stagingTest).runJob(JobType.productionUsWest1);
-        failingApp.runJob(JobType.systemTest).failDeployment(JobType.stagingTest);
+        failingApp.failDeployment(JobType.systemTest).failDeployment(JobType.stagingTest);
         deploymentTester.upgrader().maintain();
         deploymentTester.triggerJobs();
 
+        // Application fails application change
+        productionApp.submit(multiInstancePackage).failDeployment(JobType.systemTest);
+
         tester.controller().updateVersionStatus(censorConfigServers(VersionStatus.compute(tester.controller())));
         tester.assertResponse(authenticatedRequest("http://localhost:8080/deployment/v1/"), new File("root.json"));
         tester.assertResponse(authenticatedRequest("http://localhost:8080/api/deployment/v1/"), new File("root.json"));
@@ -78,7 +81,7 @@ public class DeploymentApiTest extends ControllerContainerTest {
         List<VespaVersion> censored = new ArrayList<>();
         for (VespaVersion version : versionStatus.versions()) {
             if (version.nodeVersions().size() > 0) {
-                version = new VespaVersion(version.statistics(),
+                version = new VespaVersion(version.versionNumber(),
                                            version.releaseCommit(),
                                            version.committedAt(),
                                            version.isControllerVersion(),
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json
index 2579eede1ae..dd953a3b6cf 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json
@@ -29,7 +29,87 @@
           "productionSuccesses": 1
         }
       ],
-      "deployingApplications": []
+      "deployingApplications": [],
+      "applications": [
+        {
+          "tenant": "tenant1",
+          "application": "application1",
+          "instance": "default",
+          "upgrading": false,
+          "upgradePolicy": "default",
+          "jobs": [
+            {
+              "name": "system-test",
+              "coolingDownUntil": "(ignore)"
+            },
+            {
+              "name": "staging-test",
+              "coolingDownUntil": "(ignore)"
+            },
+            {
+              "name": "production-us-west-1"
+            }
+          ],
+          "allRuns": {
+            "production-us-west-1": {
+              "success": {
+                "number": 1,
+                "start": "(ignore)",
+                "end": "(ignore)",
+                "status": "success"
+              }
+            }
+          },
+          "upgradeRuns": {
+            "production-us-west-1": {
+              "success": {
+                "number": 1,
+                "start": "(ignore)",
+                "end": "(ignore)",
+                "status": "success"
+              }
+            }
+          }
+        },
+        {
+          "tenant": "tenant2",
+          "application": "application2",
+          "instance": "i2",
+          "upgrading": false,
+          "upgradePolicy": "default",
+          "jobs": [
+            {
+              "name": "system-test"
+            },
+            {
+              "name": "staging-test"
+            },
+            {
+              "name": "production-us-west-1"
+            }
+          ],
+          "allRuns": {
+            "production-us-west-1": {
+              "success": {
+                "number": 1,
+                "start": "(ignore)",
+                "end": "(ignore)",
+                "status": "success"
+              }
+            }
+          },
+          "upgradeRuns": {
+            "production-us-west-1": {
+              "success": {
+                "number": 1,
+                "start": "(ignore)",
+                "end": "(ignore)",
+                "status": "success"
+              }
+            }
+          }
+        }
+      ]
     },
     {
       "version": "5.1",
@@ -53,6 +133,15 @@
           "instance": "default",
           "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1",
           "upgradePolicy": "default",
+          "failing": "system-test",
+          "status": "error"
+        },
+        {
+          "tenant": "tenant1",
+          "application": "application1",
+          "instance": "default",
+          "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1",
+          "upgradePolicy": "default",
           "failing": "staging-test",
           "status": "error"
         }
@@ -75,6 +164,14 @@
           "instance": "default",
           "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1",
           "upgradePolicy": "default",
+          "running": "system-test"
+        },
+        {
+          "tenant": "tenant1",
+          "application": "application1",
+          "instance": "default",
+          "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1",
+          "upgradePolicy": "default",
           "running": "staging-test"
         },
         {
@@ -85,8 +182,217 @@
           "upgradePolicy": "default",
           "running": "production-us-west-1"
         }
+      ],
+      "applications": [
+        {
+          "tenant": "tenant1",
+          "application": "application1",
+          "instance": "default",
+          "upgrading": true,
+          "upgradePolicy": "default",
+          "jobs": [
+            {
+              "name": "system-test",
+              "coolingDownUntil": "(ignore)"
+            },
+            {
+              "name": "staging-test",
+              "coolingDownUntil": "(ignore)"
+            },
+            {
+              "name": "production-us-west-1"
+            }
+          ],
+          "allRuns": {
+            "system-test": {
+              "failing": {
+                "number": 2,
+                "start": "(ignore)",
+                "end": "(ignore)",
+                "status": "error"
+              },
+              "running": {
+                "number": 3,
+                "start": "(ignore)",
+                "status": "running"
+              }
+            },
+            "staging-test": {
+              "failing": {
+                "number": 2,
+                "start": "(ignore)",
+                "end": "(ignore)",
+                "status": "error"
+              },
+              "running": {
+                "number": 3,
+                "start": "(ignore)",
+                "status": "running"
+              }
+            }
+          },
+          "upgradeRuns": {
+            "system-test": {
+              "failing": {
+                "number": 2,
+                "start": "(ignore)",
+                "end": "(ignore)",
+                "status": "error"
+              },
+              "running": {
+                "number": 3,
+                "start": "(ignore)",
+                "status": "running"
+              }
+            },
+            "staging-test": {
+              "failing": {
+                "number": 2,
+                "start": "(ignore)",
+                "end": "(ignore)",
+                "status": "error"
+              },
+              "running": {
+                "number": 3,
+                "start": "(ignore)",
+                "status": "running"
+              }
+            }
+          }
+        },
+        {
+          "tenant": "tenant2",
+          "application": "application2",
+          "instance": "i1",
+          "upgrading": false,
+          "upgradePolicy": "default",
+          "jobs": [
+            {
+              "name": "system-test",
+              "coolingDownUntil": "(ignore)"
+            },
+            {
+              "name": "staging-test"
+            },
+            {
+              "name": "production-us-west-1"
+            }
+          ],
+          "allRuns": {
+            "system-test": {
+              "failing": {
+                "number": 3,
+                "start": "(ignore)",
+                "end": "(ignore)",
+                "status": "error"
+              }
+            },
+            "staging-test": {
+              "running": {
+                "number": 3,
+                "start": "(ignore)",
+                "status": "running"
+              }
+            },
+            "production-us-west-1": {
+              "success": {
+                "number": 2,
+                "start": "(ignore)",
+                "end": "(ignore)",
+                "status": "success"
+              }
+            }
+          },
+          "upgradeRuns": {
+            "system-test": {},
+            "staging-test": {},
+            "production-us-west-1": {
+              "success": {
+                "number": 2,
+                "start": "(ignore)",
+                "end": "(ignore)",
+                "status": "success"
+              }
+            }
+          }
+        },
+        {
+          "tenant": "tenant2",
+          "application": "application2",
+          "instance": "i2",
+          "upgrading": true,
+          "upgradePolicy": "default",
+          "jobs": [
+            {
+              "name": "system-test"
+            },
+            {
+              "name": "staging-test"
+            },
+            {
+              "name": "production-us-west-1"
+            }
+          ],
+          "allRuns": {
+            "production-us-west-1": {
+              "running": {
+                "number": 2,
+                "start": "(ignore)",
+                "status": "running"
+              }
+            },
+            "system-test": {
+              "running": {
+                "number": 1,
+                "start": "(ignore)",
+                "status": "running"
+              }
+            },
+            "staging-test": {
+              "running": {
+                "number": 1,
+                "start": "(ignore)",
+                "status": "running"
+              }
+            }
+          },
+          "upgradeRuns": {
+            "production-us-west-1": {
+              "running": {
+                "number": 2,
+                "start": "(ignore)",
+                "status": "running"
+              }
+            },
+            "system-test": {},
+            "staging-test": {}
+          }
+        }
       ]
     }
+  ],
+  "jobs": [
+    "system-test",
+    "staging-test",
+    "production-us-east-3",
+    "test-us-east-3",
+    "production-us-west-1",
+    "test-us-west-1",
+    "production-us-central-1",
+    "test-us-central-1",
+    "production-ap-northeast-1",
+    "test-ap-northeast-1",
+    "production-ap-northeast-2",
+    "test-ap-northeast-2",
+    "production-ap-southeast-1",
+    "test-ap-southeast-1",
+    "production-eu-west-1",
+    "test-eu-west-1",
+    "production-aws-us-east-1a",
+    "test-aws-us-east-1a",
+    "production-aws-us-west-2a",
+    "test-aws-us-west-2a",
+    "production-aws-us-east-1b",
+    "test-aws-us-east-1b"
   ]
 }
-
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilterTest.java
index 82b97a5b144..5e50e80b7a7 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilterTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilterTest.java
@@ -2,7 +2,6 @@
 package com.yahoo.vespa.hosted.controller.restapi.filter;
 
 import com.yahoo.config.provision.ApplicationName;
-import com.yahoo.config.provision.InstanceName;
 import com.yahoo.config.provision.TenantName;
 import com.yahoo.vespa.athenz.api.AthenzDomain;
 import com.yahoo.vespa.athenz.api.AthenzPrincipal;
@@ -39,7 +38,6 @@ public class AthenzRoleFilterTest {
     private static final TenantName TENANT = TenantName.from("mytenant");
     private static final TenantName TENANT2 = TenantName.from("othertenant");
     private static final ApplicationName APPLICATION = ApplicationName.from("myapp");
-    private static final InstanceName INSTANCE = InstanceName.from("john");
     private static final URI NO_CONTEXT_PATH = URI.create("/application/v4/");
     private static final URI TENANT_CONTEXT_PATH = URI.create("/application/v4/tenant/mytenant/");
     private static final URI APPLICATION_CONTEXT_PATH = URI.create("/application/v4/tenant/mytenant/application/myapp/");
@@ -48,12 +46,11 @@ public class AthenzRoleFilterTest {
     private static final URI INSTANCE_CONTEXT_PATH = URI.create("/application/v4/tenant/mytenant/application/myapp/instance/john");
     private static final URI INSTANCE2_CONTEXT_PATH = URI.create("/application/v4/tenant/mytenant/application/myapp/instance/jane");
 
-    private ControllerTester tester;
     private AthenzRoleFilter filter;
 
     @Before
     public void setup() {
-        tester = new ControllerTester();
+        ControllerTester tester = new ControllerTester();
         filter = new AthenzRoleFilter(new AthenzClientFactoryMock(tester.athenzDb()),
                                       tester.controller());
 
@@ -70,16 +67,16 @@ public class AthenzRoleFilterTest {
     }
 
     @Test
-    public void testTranslations() {
+    public void testTranslations() throws Exception {
 
         // Hosted operators are always members of the hostedOperator role.
-        assertEquals(Set.of(Role.hostedOperator(), Role.systemFlagsDeployer(), Role.systemFlagsDryrunner()),
+        assertEquals(Set.of(Role.hostedOperator(), Role.systemFlagsDeployer(), Role.systemFlagsDryrunner(), Role.paymentProcessor(), Role.hostedSupporter()),
                      filter.roles(HOSTED_OPERATOR, NO_CONTEXT_PATH));
 
-        assertEquals(Set.of(Role.hostedOperator(), Role.systemFlagsDeployer(), Role.systemFlagsDryrunner()),
+        assertEquals(Set.of(Role.hostedOperator(), Role.systemFlagsDeployer(), Role.systemFlagsDryrunner(), Role.paymentProcessor(), Role.hostedSupporter()),
                      filter.roles(HOSTED_OPERATOR, TENANT_CONTEXT_PATH));
 
-        assertEquals(Set.of(Role.hostedOperator(), Role.systemFlagsDeployer(), Role.systemFlagsDryrunner()),
+        assertEquals(Set.of(Role.hostedOperator(), Role.systemFlagsDeployer(), Role.systemFlagsDryrunner(), Role.paymentProcessor(), Role.hostedSupporter()),
                      filter.roles(HOSTED_OPERATOR, APPLICATION_CONTEXT_PATH));
 
         // Tenant admins are members of the athenzTenantAdmin role within their tenant subtree.
@@ -98,14 +95,14 @@ public class AthenzRoleFilterTest {
         assertEquals(Set.of(Role.athenzTenantAdmin(TENANT)),
                      filter.roles(TENANT_ADMIN, APPLICATION2_CONTEXT_PATH));
 
-        // Build services are members of the tenantPipeline role within their application subtree.
+        // Build services are members of the buildService role within their application subtree.
         assertEquals(Set.of(Role.everyone()),
                      filter.roles(TENANT_PIPELINE, NO_CONTEXT_PATH));
 
         assertEquals(Set.of(Role.everyone()),
                      filter.roles(TENANT_PIPELINE, TENANT_CONTEXT_PATH));
 
-        assertEquals(Set.of(Role.tenantPipeline(TENANT, APPLICATION)),
+        assertEquals(Set.of(Role.buildService(TENANT, APPLICATION)),
                      filter.roles(TENANT_PIPELINE, APPLICATION_CONTEXT_PATH));
 
         assertEquals(Set.of(Role.everyone()),
@@ -115,7 +112,7 @@ public class AthenzRoleFilterTest {
         assertEquals(Set.of(Role.athenzTenantAdmin(TENANT)),
                      filter.roles(TENANT_ADMIN_AND_PIPELINE, TENANT_CONTEXT_PATH));
 
-        assertEquals(Set.of(Role.athenzTenantAdmin(TENANT), Role.tenantPipeline(TENANT, APPLICATION)),
+        assertEquals(Set.of(Role.athenzTenantAdmin(TENANT), Role.buildService(TENANT, APPLICATION)),
                      filter.roles(TENANT_ADMIN_AND_PIPELINE, APPLICATION_CONTEXT_PATH));
 
         // Users have nothing special under their instance
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilterTest.java
index a5520b42459..c95691fc120 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilterTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilterTest.java
@@ -43,6 +43,16 @@ public class ControllerAuthorizationFilterTest {
     }
 
     @Test
+    public void supporter() {
+        ControllerTester tester = new ControllerTester();
+        SecurityContext securityContext = new SecurityContext(() -> "operator", Set.of(Role.hostedSupporter()));
+        ControllerAuthorizationFilter filter = createFilter(tester);
+
+        assertIsForbidden(invokeFilter(filter, createRequest(Method.POST, "/zone/v2/path", securityContext)));
+        assertIsAllowed(invokeFilter(filter, createRequest(Method.GET, "/zone/v1/path", securityContext)));
+    }
+
+    @Test
     public void unprivileged() {
         ControllerTester tester = new ControllerTester();
         SecurityContext securityContext = new SecurityContext(() -> "user", Set.of(Role.everyone()));
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilterTest.java
index 0a1e996696b..6e1480d45e7 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilterTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilterTest.java
@@ -30,7 +30,6 @@ import java.security.PublicKey;
 import java.util.Set;
 
 import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
 
 public class SignatureFilterTest {
@@ -95,21 +94,14 @@ public class SignatureFilterTest {
         verifySecurityContext(requestOf(signer.signed(request.copy(), Method.GET, InputStream::nullInputStream), emptyBody),
                               new SecurityContext(new SimplePrincipal("headless@my-tenant.my-app"),
                                                   Set.of(Role.reader(id.tenant()),
-                                                         Role.developer(id.tenant())))); // TODO jonmv: Change to headless.
-
-        // TODO jonmv: remove after Oct 2019.
-        // Signed request gets a build service role when a matching key is stored for the application and no X-Key header is provided.
-        verifySecurityContext(requestOf(signer.legacySigned(request.copy(), Method.GET, InputStream::nullInputStream), emptyBody),
-                              new SecurityContext(new SimplePrincipal("headless@my-tenant.my-app"),
-                                                  Set.of(Role.reader(id.tenant()),
-                                                         Role.developer(id.tenant()))));
+                                                         Role.headless(id.tenant(), id.application()))));
 
         // Signed POST request with X-Key header gets a headless role.
         byte[] hiBytes = new byte[]{0x48, 0x69};
         verifySecurityContext(requestOf(signer.signed(request.copy(), Method.POST, () -> new ByteArrayInputStream(hiBytes)), hiBytes),
                               new SecurityContext(new SimplePrincipal("headless@my-tenant.my-app"),
                                                   Set.of(Role.reader(id.tenant()),
-                                                         Role.developer(id.tenant())))); // TODO jonmv: Change to headless.
+                                                         Role.headless(id.tenant(), id.application()))));
 
         // Signed request gets a developer role when a matching developer key is stored for the tenant.
         tester.curator().writeTenant(new CloudTenant(appId.tenant(),
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-initial.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-initial.json
deleted file mode 100644
index dbaa6623fae..00000000000
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-initial.json
+++ /dev/null
@@ -1,153 +0,0 @@
-{
-  "versions": [
-    {
-      "version": "0.0.0",
-      "targetVersion": false,
-      "cloud": "cloud1",
-      "nodes": [
-        {
-          "hostname": "node-2-configserver-host-prod.us-east-3",
-          "environment": "prod",
-          "region": "us-east-3"
-        },
-        {
-          "hostname": "node-1-configserver-host-prod.us-east-3",
-          "environment": "prod",
-          "region": "us-east-3"
-        },
-        {
-          "hostname": "node-3-configserver-host-prod.us-east-3",
-          "environment": "prod",
-          "region": "us-east-3"
-        },
-        {
-          "hostname": "node-1-configserver-host-prod.us-west-1",
-          "environment": "prod",
-          "region": "us-west-1"
-        },
-        {
-          "hostname": "node-2-configserver-host-prod.us-west-1",
-          "environment": "prod",
-          "region": "us-west-1"
-        },
-        {
-          "hostname": "node-3-configserver-host-prod.us-west-1",
-          "environment": "prod",
-          "region": "us-west-1"
-        },
-        {
-          "hostname": "node-2-proxy-host-prod.us-east-3",
-          "environment": "prod",
-          "region": "us-east-3"
-        },
-        {
-          "hostname": "node-3-proxy-host-prod.us-east-3",
-          "environment": "prod",
-          "region": "us-east-3"
-        },
-        {
-          "hostname": "node-1-proxy-host-prod.us-east-3",
-          "environment": "prod",
-          "region": "us-east-3"
-        },
-        {
-          "hostname": "node-2-proxy-host-prod.us-west-1",
-          "environment": "prod",
-          "region": "us-west-1"
-        },
-        {
-          "hostname": "node-1-proxy-host-prod.us-west-1",
-          "environment": "prod",
-          "region": "us-west-1"
-        },
-        {
-          "hostname": "node-3-proxy-host-prod.us-west-1",
-          "environment": "prod",
-          "region": "us-west-1"
-        },
-        {
-          "hostname": "node-3-tenant-host-prod.us-east-3",
-          "environment": "prod",
-          "region": "us-east-3"
-        },
-        {
-          "hostname": "node-2-tenant-host-prod.us-east-3",
-          "environment": "prod",
-          "region": "us-east-3"
-        },
-        {
-          "hostname": "node-1-tenant-host-prod.us-east-3",
-          "environment": "prod",
-          "region": "us-east-3"
-        },
-        {
-          "hostname": "node-3-tenant-host-prod.us-west-1",
-          "environment": "prod",
-          "region": "us-west-1"
-        },
-        {
-          "hostname": "node-2-tenant-host-prod.us-west-1",
-          "environment": "prod",
-          "region": "us-west-1"
-        },
-        {
-          "hostname": "node-1-tenant-host-prod.us-west-1",
-          "environment": "prod",
-          "region": "us-west-1"
-        }
-      ]
-    },
-    {
-      "version": "0.0.0",
-      "targetVersion": false,
-      "cloud": "cloud2",
-      "nodes": [
-        {
-          "hostname": "node-1-configserver-host-prod.eu-west-1",
-          "environment": "prod",
-          "region": "eu-west-1"
-        },
-        {
-          "hostname": "node-2-configserver-host-prod.eu-west-1",
-          "environment": "prod",
-          "region": "eu-west-1"
-        },
-        {
-          "hostname": "node-3-configserver-host-prod.eu-west-1",
-          "environment": "prod",
-          "region": "eu-west-1"
-        },
-        {
-          "hostname": "node-1-proxy-host-prod.eu-west-1",
-          "environment": "prod",
-          "region": "eu-west-1"
-        },
-        {
-          "hostname": "node-3-proxy-host-prod.eu-west-1",
-          "environment": "prod",
-          "region": "eu-west-1"
-        },
-        {
-          "hostname": "node-2-proxy-host-prod.eu-west-1",
-          "environment": "prod",
-          "region": "eu-west-1"
-        },
-        {
-          "hostname": "node-1-tenant-host-prod.eu-west-1",
-          "environment": "prod",
-          "region": "eu-west-1"
-        },
-        {
-          "hostname": "node-3-tenant-host-prod.eu-west-1",
-          "environment": "prod",
-          "region": "eu-west-1"
-        },
-        {
-          "hostname": "node-2-tenant-host-prod.eu-west-1",
-          "environment": "prod",
-          "region": "eu-west-1"
-        }
-      ]
-    }
-  ]
-}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiTest.java
index b7edcef97e4..fefd23eb67c 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiTest.java
@@ -2,10 +2,15 @@
 package com.yahoo.vespa.hosted.controller.restapi.routing;
 
 import com.yahoo.application.container.handler.Request;
+import com.yahoo.config.provision.AthenzDomain;
+import com.yahoo.config.provision.AthenzService;
+import com.yahoo.config.provision.zone.RoutingMethod;
 import com.yahoo.config.provision.zone.ZoneId;
 import com.yahoo.vespa.hosted.controller.ControllerTester;
+import com.yahoo.vespa.hosted.controller.RoutingController;
 import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder;
 import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester;
+import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock;
 import com.yahoo.vespa.hosted.controller.restapi.ContainerTester;
 import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerTest;
 import org.junit.Before;
@@ -13,7 +18,6 @@ import org.junit.Test;
 
 import java.io.File;
 import java.util.List;
-import java.util.Set;
 
 import static org.junit.Assert.assertNotEquals;
 
@@ -34,19 +38,104 @@ public class RoutingApiTest extends ControllerContainerTest {
     }
 
     @Test
-    public void policy_based_routing() {
-        var context = deploymentTester.newDeploymentContext();
+    public void discovery() {
+        // Deploy
+        var context = deploymentTester.newDeploymentContext("t1", "a1", "default");
+        var westZone = ZoneId.from("prod", "us-west-1");
+        var eastZone = ZoneId.from("prod", "us-east-3");
+        var applicationPackage = new ApplicationPackageBuilder()
+                .region(westZone.region())
+                .region(eastZone.region())
+                .endpoint("default", "default", eastZone.region().value(), westZone.region().value())
+                .build();
+        context.submit(applicationPackage).deploy();
 
-        // Deploy application
+        // GET root
+        tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/", "",
+                                              Request.Method.GET),
+                              new File("discovery/root.json"));
+
+        // GET tenant
+        tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/tenant/t1", "",
+                                              Request.Method.GET),
+                              new File("discovery/tenant.json"));
+
+        // GET application
+        tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/tenant/t1/application/a1/",
+                                              "",
+                                              Request.Method.GET),
+                              new File("discovery/application.json"));
+
+        // GET instance
+        tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/tenant/t1/application/a1/instance/default/",
+                                              "",
+                                              Request.Method.GET),
+                              new File("discovery/instance.json"));
+
+        // GET environment
+        tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/environment/", "",
+                                              Request.Method.GET),
+                              new File("discovery/environment.json"));
+    }
+
+    @Test
+    public void recursion() {
+        var context1 = deploymentTester.newDeploymentContext("t1", "a1", "default");
         var westZone = ZoneId.from("prod", "us-west-1");
         var eastZone = ZoneId.from("prod", "us-east-3");
+        var package1 = new ApplicationPackageBuilder()
+                .region(westZone.region())
+                .region(eastZone.region())
+                .endpoint("default", "default", eastZone.region().value(), westZone.region().value())
+                .build();
+        context1.submit(package1).deploy();
+
+        var context2 = deploymentTester.newDeploymentContext("t1", "a2", "default");
+        var package2 = new ApplicationPackageBuilder()
+                .region(westZone.region())
+                .region(eastZone.region())
+                .endpoint("default", "default", eastZone.region().value(), westZone.region().value())
+                .build();
+        context2.submit(package2).deploy();
+
+        // GET tenant recursively
+        tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/tenant/t1?recursive=true", "",
+                                              Request.Method.GET),
+                              new File("recursion/tenant.json"));
+
+        // GET application recursively
+        tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/tenant/t1/application/a1?recursive=true", "",
+                                              Request.Method.GET),
+                              new File("recursion/application.json"));
+
+        // GET instance recursively
+        tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/tenant/t1/application/a1/instance/default?recursive=true", "",
+                                              Request.Method.GET),
+                              new File("recursion/application.json"));
+
+        // GET environment recursively
+        tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/environment?recursive=true", "",
+                                              Request.Method.GET),
+                              new File("recursion/environment.json"));
+    }
+
+    @Test
+    public void exclusive_routing() {
+        var context = deploymentTester.newDeploymentContext();
+        // Zones support direct routing
+        var westZone = ZoneId.from("prod", "us-west-1");
+        var eastZone = ZoneId.from("prod", "us-east-3");
+        deploymentTester.controllerTester().zoneRegistry().exclusiveRoutingIn(ZoneApiMock.from(westZone),
+                                                                              ZoneApiMock.from(eastZone));
+        // Deploy application
         var applicationPackage = new ApplicationPackageBuilder()
+                .athenzIdentity(AthenzDomain.from("domain"), AthenzService.from("service"))
+                .compileVersion(RoutingController.DIRECT_ROUTING_MIN_VERSION)
                 .region(westZone.region())
                 .region(eastZone.region())
+                .endpoint("default", "default", eastZone.region().value(), westZone.region().value())
                 .build();
         context.submit(applicationPackage).deploy();
-        context.addRoutingPolicy(westZone, true);
-        context.addRoutingPolicy(eastZone, true);
 
         // GET initial deployment status
         tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/tenant/tenant/application/application/instance/default/environment/prod/region/us-west-1",
@@ -56,7 +145,7 @@ public class RoutingApiTest extends ControllerContainerTest {
         // POST sets deployment out
         tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/inactive/tenant/tenant/application/application/instance/default/environment/prod/region/us-west-1",
                                               "", Request.Method.POST),
-                              "{\"message\":\"Set global routing status for tenant.application in prod.us-west-1 to 'out'\"}");
+                              "{\"message\":\"Set global routing status for tenant.application in prod.us-west-1 to OUT\"}");
         tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/tenant/tenant/application/application/instance/default/environment/prod/region/us-west-1",
                                               "", Request.Method.GET),
                               new File("policy/deployment-status-out.json"));
@@ -64,7 +153,7 @@ public class RoutingApiTest extends ControllerContainerTest {
         // DELETE sets deployment in
         tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/inactive/tenant/tenant/application/application/instance/default/environment/prod/region/us-west-1",
                                               "", Request.Method.DELETE),
-                              "{\"message\":\"Set global routing status for tenant.application in prod.us-west-1 to 'in'\"}");
+                              "{\"message\":\"Set global routing status for tenant.application in prod.us-west-1 to IN\"}");
         tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/tenant/tenant/application/application/instance/default/environment/prod/region/us-west-1",
                                               "", Request.Method.GET),
                               new File("policy/deployment-status-in.json"));
@@ -77,7 +166,7 @@ public class RoutingApiTest extends ControllerContainerTest {
         // POST sets zone out
         tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/inactive/environment/prod/region/us-west-1",
                                               "", Request.Method.POST),
-                              "{\"message\":\"Set global routing status for deployments in prod.us-west-1 to 'out'\"}");
+                              "{\"message\":\"Set global routing status for deployments in prod.us-west-1 to OUT\"}");
         tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/environment/prod/region/us-west-1",
                                               "", Request.Method.GET),
                               new File("policy/zone-status-out.json"));
@@ -85,16 +174,14 @@ public class RoutingApiTest extends ControllerContainerTest {
         // DELETE sets zone in
         tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/inactive/environment/prod/region/us-west-1",
                                               "", Request.Method.DELETE),
-                              "{\"message\":\"Set global routing status for deployments in prod.us-west-1 to 'in'\"}");
+                              "{\"message\":\"Set global routing status for deployments in prod.us-west-1 to IN\"}");
         tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/environment/prod/region/us-west-1",
                                               "", Request.Method.GET),
                               new File("policy/zone-status-in.json"));
     }
 
     @Test
-    public void rotation_based_routing() {
-        // No zones support direct routing
-        deploymentTester.controllerTester().zoneRegistry().setDirectlyRouted(Set.of());
+    public void shared_routing() {
         // Deploy application
         var context = deploymentTester.newDeploymentContext();
         var westZone = ZoneId.from("prod", "us-west-1");
@@ -102,7 +189,7 @@ public class RoutingApiTest extends ControllerContainerTest {
         var applicationPackage = new ApplicationPackageBuilder()
                 .region(westZone.region())
                 .region(eastZone.region())
-                .endpoint("default", "qrs", eastZone.region().value(), westZone.region().value())
+                .endpoint("default", "default", eastZone.region().value(), westZone.region().value())
                 .build();
         context.submit(applicationPackage).deploy();
 
@@ -116,7 +203,7 @@ public class RoutingApiTest extends ControllerContainerTest {
         // POST sets deployment out
         tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/inactive/tenant/tenant/application/application/instance/default/environment/prod/region/us-west-1",
                                               "", Request.Method.POST),
-                              "{\"message\":\"Set global routing status for tenant.application in prod.us-west-1 to 'out'\"}");
+                              "{\"message\":\"Set global routing status for tenant.application in prod.us-west-1 to OUT\"}");
         tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/tenant/tenant/application/application/instance/default/environment/prod/region/us-west-1",
                                               "", Request.Method.GET),
                               new File("rotation/deployment-status-out.json"));
@@ -124,7 +211,7 @@ public class RoutingApiTest extends ControllerContainerTest {
         // DELETE sets deployment in
         tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/inactive/tenant/tenant/application/application/instance/default/environment/prod/region/us-west-1",
                                               "", Request.Method.DELETE),
-                              "{\"message\":\"Set global routing status for tenant.application in prod.us-west-1 to 'in'\"}");
+                              "{\"message\":\"Set global routing status for tenant.application in prod.us-west-1 to IN\"}");
         tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/tenant/tenant/application/application/instance/default/environment/prod/region/us-west-1",
                                               "", Request.Method.GET),
                               new File("rotation/deployment-status-in.json"));
@@ -137,7 +224,7 @@ public class RoutingApiTest extends ControllerContainerTest {
         // POST sets zone out
         tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/inactive/environment/prod/region/us-west-1",
                                               "", Request.Method.POST),
-                              "{\"message\":\"Set global routing status for deployments in prod.us-west-1 to 'out'\"}");
+                              "{\"message\":\"Set global routing status for deployments in prod.us-west-1 to OUT\"}");
         tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/environment/prod/region/us-west-1",
                                               "", Request.Method.GET),
                               new File("rotation/zone-status-out.json"));
@@ -145,15 +232,33 @@ public class RoutingApiTest extends ControllerContainerTest {
         // DELETE sets zone in
         tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/inactive/environment/prod/region/us-west-1",
                                               "", Request.Method.DELETE),
-                              "{\"message\":\"Set global routing status for deployments in prod.us-west-1 to 'in'\"}");
+                              "{\"message\":\"Set global routing status for deployments in prod.us-west-1 to IN\"}");
         tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/environment/prod/region/us-west-1",
                                               "", Request.Method.GET),
                               new File("rotation/zone-status-in.json"));
+    }
 
-        // TODO(mpolden): Remove the following once a zone supports either of routing policy and rotation
+    // TODO(mpolden): Remove this once a zone supports either of routing policy and rotation
+    @Test
+    public void mixed_routing() {
+        var westZone = ZoneId.from("prod", "us-west-1");
+        var eastZone = ZoneId.from("prod", "us-east-3");
+
+        // One zone supports multiple routing methods
+        deploymentTester.controllerTester().zoneRegistry().setRoutingMethod(ZoneApiMock.from(westZone),
+                                                                            RoutingMethod.shared,
+                                                                            RoutingMethod.exclusive);
+
+        // Deploy application
+        var context = deploymentTester.newDeploymentContext();
+        var applicationPackage = new ApplicationPackageBuilder()
+                .region(westZone.region())
+                .region(eastZone.region())
+                .endpoint("default", "default", eastZone.region().value(), westZone.region().value())
+                .build();
+        context.submit(applicationPackage).deploy();
 
         // GET status with both policy and rotation assigned
-        context.addRoutingPolicy(westZone, true);
         tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/tenant/tenant/application/application/instance/default/environment/prod/region/us-west-1",
                                               "", Request.Method.GET),
                               new File("multi-status-initial.json"));
@@ -161,7 +266,7 @@ public class RoutingApiTest extends ControllerContainerTest {
         // POST sets deployment out
         tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/inactive/tenant/tenant/application/application/instance/default/environment/prod/region/us-west-1",
                                               "", Request.Method.POST),
-                              "{\"message\":\"Set global routing status for tenant.application in prod.us-west-1 to 'out'\"}");
+                              "{\"message\":\"Set global routing status for tenant.application in prod.us-west-1 to OUT\"}");
         tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/tenant/tenant/application/application/instance/default/environment/prod/region/us-west-1",
                                               "", Request.Method.GET),
                               new File("multi-status-out.json"));
@@ -169,7 +274,7 @@ public class RoutingApiTest extends ControllerContainerTest {
         // DELETE sets deployment in
         tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/inactive/tenant/tenant/application/application/instance/default/environment/prod/region/us-west-1",
                                               "", Request.Method.DELETE),
-                              "{\"message\":\"Set global routing status for tenant.application in prod.us-west-1 to 'in'\"}");
+                              "{\"message\":\"Set global routing status for tenant.application in prod.us-west-1 to IN\"}");
         tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/tenant/tenant/application/application/instance/default/environment/prod/region/us-west-1",
                                               "", Request.Method.GET),
                               new File("multi-status-in.json"));
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/discovery/application.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/discovery/application.json
new file mode 100644
index 00000000000..deda734cbbf
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/discovery/application.json
@@ -0,0 +1,7 @@
+{
+  "resources": [
+    {
+      "url": "http://localhost:8080/routing/v1/status/tenant/t1/application/a1/instance/default/"
+    }
+  ]
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/discovery/environment.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/discovery/environment.json
new file mode 100644
index 00000000000..1e06b279873
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/discovery/environment.json
@@ -0,0 +1,43 @@
+{
+  "resources": [
+    {
+      "url": "http://localhost:8080/routing/v1/status/environment/dev/region/aws-us-east-2a/"
+    },
+    {
+      "url": "http://localhost:8080/routing/v1/status/environment/dev/region/us-east-1/"
+    },
+    {
+      "url": "http://localhost:8080/routing/v1/status/environment/perf/region/us-east-3/"
+    },
+    {
+      "url": "http://localhost:8080/routing/v1/status/environment/prod/region/ap-northeast-1/"
+    },
+    {
+      "url": "http://localhost:8080/routing/v1/status/environment/prod/region/ap-northeast-2/"
+    },
+    {
+      "url": "http://localhost:8080/routing/v1/status/environment/prod/region/ap-southeast-1/"
+    },
+    {
+      "url": "http://localhost:8080/routing/v1/status/environment/prod/region/aws-us-east-1a/"
+    },
+    {
+      "url": "http://localhost:8080/routing/v1/status/environment/prod/region/eu-west-1/"
+    },
+    {
+      "url": "http://localhost:8080/routing/v1/status/environment/prod/region/us-central-1/"
+    },
+    {
+      "url": "http://localhost:8080/routing/v1/status/environment/prod/region/us-east-3/"
+    },
+    {
+      "url": "http://localhost:8080/routing/v1/status/environment/prod/region/us-west-1/"
+    },
+    {
+      "url": "http://localhost:8080/routing/v1/status/environment/staging/region/us-east-3/"
+    },
+    {
+      "url": "http://localhost:8080/routing/v1/status/environment/test/region/us-east-1/"
+    }
+  ]
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/discovery/instance.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/discovery/instance.json
new file mode 100644
index 00000000000..1a3ad823e14
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/discovery/instance.json
@@ -0,0 +1,10 @@
+{
+  "resources": [
+    {
+      "url": "http://localhost:8080/routing/v1/status/tenant/t1/application/a1/instance/default/environment/prod/region/us-east-3/"
+    },
+    {
+      "url": "http://localhost:8080/routing/v1/status/tenant/t1/application/a1/instance/default/environment/prod/region/us-west-1/"
+    }
+  ]
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/discovery/root.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/discovery/root.json
new file mode 100644
index 00000000000..9b5630335aa
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/discovery/root.json
@@ -0,0 +1,10 @@
+{
+  "resources": [
+    {
+      "url": "http://localhost:8080/routing/v1/status/tenant/"
+    },
+    {
+      "url": "http://localhost:8080/routing/v1/status/environment/"
+    }
+  ]
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/discovery/tenant.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/discovery/tenant.json
new file mode 100644
index 00000000000..acd05d35c8d
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/discovery/tenant.json
@@ -0,0 +1,7 @@
+{
+  "resources": [
+    {
+      "url": "http://localhost:8080/routing/v1/status/tenant/t1/application/a1/"
+    }
+  ]
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/multi-status-in.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/multi-status-in.json
index b3f3e90a9cd..1c23c6bb569 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/multi-status-in.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/multi-status-in.json
@@ -1,7 +1,7 @@
 {
   "deployments": [
     {
-      "routingType": "rotation",
+      "routingMethod": "shared",
       "instance": "tenant:application:default",
       "environment": "prod",
       "region": "us-west-1",
@@ -10,7 +10,7 @@
       "changedAt": "(ignore)"
     },
     {
-      "routingType": "policy",
+      "routingMethod": "exclusive",
       "instance": "tenant:application:default",
       "environment": "prod",
       "region": "us-west-1",
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/multi-status-initial.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/multi-status-initial.json
index a15a0cc8a99..eea78c1b963 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/multi-status-initial.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/multi-status-initial.json
@@ -1,16 +1,16 @@
 {
   "deployments": [
     {
-      "routingType": "rotation",
+      "routingMethod": "shared",
       "instance": "tenant:application:default",
       "environment": "prod",
       "region": "us-west-1",
       "status": "in",
-      "agent": "operator",
+      "agent": "unknown",
       "changedAt": "(ignore)"
     },
     {
-      "routingType": "policy",
+      "routingMethod": "exclusive",
       "instance": "tenant:application:default",
       "environment": "prod",
       "region": "us-west-1",
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/multi-status-out.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/multi-status-out.json
index 373d7076ffc..6cb90bdb673 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/multi-status-out.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/multi-status-out.json
@@ -1,7 +1,7 @@
 {
   "deployments": [
     {
-      "routingType": "rotation",
+      "routingMethod": "shared",
       "instance": "tenant:application:default",
       "environment": "prod",
       "region": "us-west-1",
@@ -10,7 +10,7 @@
       "changedAt": "(ignore)"
     },
     {
-      "routingType": "policy",
+      "routingMethod": "exclusive",
       "instance": "tenant:application:default",
       "environment": "prod",
       "region": "us-west-1",
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/deployment-status-in.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/deployment-status-in.json
index 74c8173d132..59519c33d06 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/deployment-status-in.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/deployment-status-in.json
@@ -1,7 +1,7 @@
 {
   "deployments": [
     {
-      "routingType": "policy",
+      "routingMethod": "exclusive",
       "instance": "tenant:application:default",
       "environment": "prod",
       "region": "us-west-1",
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/deployment-status-initial.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/deployment-status-initial.json
index 444b6a825ea..e95d9bcdc42 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/deployment-status-initial.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/deployment-status-initial.json
@@ -1,7 +1,7 @@
 {
   "deployments": [
     {
-      "routingType": "policy",
+      "routingMethod": "exclusive",
       "instance": "tenant:application:default",
       "environment": "prod",
       "region": "us-west-1",
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/deployment-status-out.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/deployment-status-out.json
index 3f5353505df..49b85775e63 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/deployment-status-out.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/deployment-status-out.json
@@ -1,7 +1,7 @@
 {
   "deployments": [
     {
-      "routingType": "policy",
+      "routingMethod": "exclusive",
       "instance": "tenant:application:default",
       "environment": "prod",
       "region": "us-west-1",
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/zone-status-in.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/zone-status-in.json
index 376b6c9c902..abf0a46ae3e 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/zone-status-in.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/zone-status-in.json
@@ -1,5 +1,5 @@
 {
-  "routingType": "policy",
+  "routingMethod": "exclusive",
   "environment": "prod",
   "region": "us-west-1",
   "status": "in",
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/zone-status-initial.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/zone-status-initial.json
index 482fd920070..8328e1ffab1 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/zone-status-initial.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/zone-status-initial.json
@@ -1,5 +1,5 @@
 {
-  "routingType": "policy",
+  "routingMethod": "exclusive",
   "environment": "prod",
   "region": "us-west-1",
   "status": "in",
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/zone-status-out.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/zone-status-out.json
index 5ab261067bf..d86ca2d56e6 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/zone-status-out.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/policy/zone-status-out.json
@@ -1,5 +1,5 @@
 {
-  "routingType": "policy",
+  "routingMethod": "exclusive",
   "environment": "prod",
   "region": "us-west-1",
   "status": "out",
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/recursion/application.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/recursion/application.json
new file mode 100644
index 00000000000..e0b0e5e9b7a
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/recursion/application.json
@@ -0,0 +1,22 @@
+{
+  "deployments": [
+    {
+      "routingMethod": "shared",
+      "instance": "t1:a1:default",
+      "environment": "prod",
+      "region": "us-east-3",
+      "status": "in",
+      "agent": "unknown",
+      "changedAt": "(ignore)"
+    },
+    {
+      "routingMethod": "shared",
+      "instance": "t1:a1:default",
+      "environment": "prod",
+      "region": "us-west-1",
+      "status": "in",
+      "agent": "unknown",
+      "changedAt": "(ignore)"
+    }
+  ]
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/recursion/environment.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/recursion/environment.json
new file mode 100644
index 00000000000..f0dd0b7310d
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/recursion/environment.json
@@ -0,0 +1,108 @@
+{
+  "zones": [
+    {
+      "routingMethod": "shared",
+      "environment": "test",
+      "region": "us-east-1",
+      "status": "in",
+      "agent": "operator",
+      "changedAt": 0
+    },
+    {
+      "routingMethod": "shared",
+      "environment": "staging",
+      "region": "us-east-3",
+      "status": "in",
+      "agent": "operator",
+      "changedAt": 0
+    },
+    {
+      "routingMethod": "shared",
+      "environment": "dev",
+      "region": "us-east-1",
+      "status": "in",
+      "agent": "operator",
+      "changedAt": 0
+    },
+    {
+      "routingMethod": "shared",
+      "environment": "dev",
+      "region": "aws-us-east-2a",
+      "status": "in",
+      "agent": "operator",
+      "changedAt": 0
+    },
+    {
+      "routingMethod": "shared",
+      "environment": "perf",
+      "region": "us-east-3",
+      "status": "in",
+      "agent": "operator",
+      "changedAt": 0
+    },
+    {
+      "routingMethod": "shared",
+      "environment": "prod",
+      "region": "aws-us-east-1a",
+      "status": "in",
+      "agent": "operator",
+      "changedAt": 0
+    },
+    {
+      "routingMethod": "shared",
+      "environment": "prod",
+      "region": "ap-northeast-1",
+      "status": "in",
+      "agent": "operator",
+      "changedAt": 0
+    },
+    {
+      "routingMethod": "shared",
+      "environment": "prod",
+      "region": "ap-northeast-2",
+      "status": "in",
+      "agent": "operator",
+      "changedAt": 0
+    },
+    {
+      "routingMethod": "shared",
+      "environment": "prod",
+      "region": "ap-southeast-1",
+      "status": "in",
+      "agent": "operator",
+      "changedAt": 0
+    },
+    {
+      "routingMethod": "shared",
+      "environment": "prod",
+      "region": "us-east-3",
+      "status": "in",
+      "agent": "operator",
+      "changedAt": 0
+    },
+    {
+      "routingMethod": "shared",
+      "environment": "prod",
+      "region": "us-west-1",
+      "status": "in",
+      "agent": "operator",
+      "changedAt": 0
+    },
+    {
+      "routingMethod": "shared",
+      "environment": "prod",
+      "region": "us-central-1",
+      "status": "in",
+      "agent": "operator",
+      "changedAt": 0
+    },
+    {
+      "routingMethod": "shared",
+      "environment": "prod",
+      "region": "eu-west-1",
+      "status": "in",
+      "agent": "operator",
+      "changedAt": 0
+    }
+  ]
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/recursion/tenant.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/recursion/tenant.json
new file mode 100644
index 00000000000..1ee4e1b82ba
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/recursion/tenant.json
@@ -0,0 +1,40 @@
+{
+  "deployments": [
+    {
+      "routingMethod": "shared",
+      "instance": "t1:a1:default",
+      "environment": "prod",
+      "region": "us-east-3",
+      "status": "in",
+      "agent": "unknown",
+      "changedAt": "(ignore)"
+    },
+    {
+      "routingMethod": "shared",
+      "instance": "t1:a1:default",
+      "environment": "prod",
+      "region": "us-west-1",
+      "status": "in",
+      "agent": "unknown",
+      "changedAt": "(ignore)"
+    },
+    {
+      "routingMethod": "shared",
+      "instance": "t1:a2:default",
+      "environment": "prod",
+      "region": "us-east-3",
+      "status": "in",
+      "agent": "unknown",
+      "changedAt": "(ignore)"
+    },
+    {
+      "routingMethod": "shared",
+      "instance": "t1:a2:default",
+      "environment": "prod",
+      "region": "us-west-1",
+      "status": "in",
+      "agent": "unknown",
+      "changedAt": "(ignore)"
+    }
+  ]
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/deployment-status-in.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/deployment-status-in.json
index fd30a27fba5..5b15b72752c 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/deployment-status-in.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/deployment-status-in.json
@@ -1,7 +1,7 @@
 {
   "deployments": [
     {
-      "routingType": "rotation",
+      "routingMethod": "shared",
       "instance": "tenant:application:default",
       "environment": "prod",
       "region": "us-west-1",
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/deployment-status-initial.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/deployment-status-initial.json
index 59ea77cf7ae..90b2317c1b3 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/deployment-status-initial.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/deployment-status-initial.json
@@ -1,7 +1,7 @@
 {
   "deployments": [
     {
-      "routingType": "rotation",
+      "routingMethod": "shared",
       "instance": "tenant:application:default",
       "environment": "prod",
       "region": "us-west-1",
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/deployment-status-out.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/deployment-status-out.json
index 0240079f154..85e345c01d0 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/deployment-status-out.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/deployment-status-out.json
@@ -1,7 +1,7 @@
 {
   "deployments": [
     {
-      "routingType": "rotation",
+      "routingMethod": "shared",
       "instance": "tenant:application:default",
       "environment": "prod",
       "region": "us-west-1",
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/zone-status-in.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/zone-status-in.json
index a883fd3f342..eb06e9ee11d 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/zone-status-in.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/zone-status-in.json
@@ -1,5 +1,5 @@
 {
-  "routingType": "rotation",
+  "routingMethod": "shared",
   "environment": "prod",
   "region": "us-west-1",
   "status": "in",
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/zone-status-initial.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/zone-status-initial.json
index a883fd3f342..eb06e9ee11d 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/zone-status-initial.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/zone-status-initial.json
@@ -1,5 +1,5 @@
 {
-  "routingType": "rotation",
+  "routingMethod": "shared",
   "environment": "prod",
   "region": "us-west-1",
   "status": "in",
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/zone-status-out.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/zone-status-out.json
index 18803f56de3..440b80bc4d0 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/zone-status-out.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/rotation/zone-status-out.json
@@ -1,5 +1,5 @@
 {
-  "routingType": "rotation",
+  "routingMethod": "shared",
   "environment": "prod",
   "region": "us-west-1",
   "status": "out",
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployResultTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployResultTest.java
index 104bb91a8cb..b2a835b6b55 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployResultTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployResultTest.java
@@ -27,7 +27,8 @@ public class SystemFlagsDeployResultTest {
                 List.of(
                         FlagDataChange.deleted(flagOne, controllerTarget)),
                 List.of(
-                        OperationError.deleteFailed("delete failed", controllerTarget, flagTwo)));
+                        OperationError.deleteFailed("delete failed", controllerTarget, flagTwo)),
+                List.of());
         WireSystemFlagsDeployResult wire = result.toWire();
 
         assertThat(wire.changes).hasSize(1);
@@ -49,11 +50,13 @@ public class SystemFlagsDeployResultTest {
         SystemFlagsDeployResult resultController =
                 new SystemFlagsDeployResult(
                         List.of(FlagDataChange.deleted(flagOne, controllerTarget)),
-                        List.of(OperationError.deleteFailed("message", controllerTarget, flagTwo)));
+                        List.of(OperationError.deleteFailed("message", controllerTarget, flagTwo)),
+                        List.of());
         SystemFlagsDeployResult resultProdUsWest1 =
                 new SystemFlagsDeployResult(
                         List.of(FlagDataChange.deleted(flagOne, prodUsWest1Target)),
-                        List.of(OperationError.deleteFailed("message", prodUsWest1Target, flagTwo)));
+                        List.of(OperationError.deleteFailed("message", prodUsWest1Target, flagTwo)),
+                        List.of());
 
         var results = List.of(resultController, resultProdUsWest1);
         SystemFlagsDeployResult mergedResult = merge(results);
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployerTest.java
index 22ed079b501..475ac12f2fd 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployerTest.java
@@ -58,7 +58,7 @@ public class SystemFlagsDeployerTest {
                 .build();
 
         SystemFlagsDeployer deployer =
-                new SystemFlagsDeployer(flagsClient, Set.of(controllerTarget, prodUsWest1Target, prodUsEast3Target));
+                new SystemFlagsDeployer(flagsClient, SYSTEM, Set.of(controllerTarget, prodUsWest1Target, prodUsEast3Target));
 
 
         SystemFlagsDeployResult result = deployer.deployFlags(archive, false);
@@ -83,7 +83,7 @@ public class SystemFlagsDeployerTest {
                 .addFile("main.json", defaultData)
                 .build();
 
-        SystemFlagsDeployer deployer = new SystemFlagsDeployer(flagsClient, Set.of(controllerTarget));
+        SystemFlagsDeployer deployer = new SystemFlagsDeployer(flagsClient, SYSTEM, Set.of(controllerTarget));
         SystemFlagsDeployResult result = deployer.deployFlags(archive, true);
 
         verify(flagsClient, times(1)).listFlagData(controllerTarget);
@@ -107,20 +107,50 @@ public class SystemFlagsDeployerTest {
                 .addFile("main.json", defaultData)
                 .build();
 
-        SystemFlagsDeployer deployer = new SystemFlagsDeployer(flagsClient, Set.of(prodUsWest1Target, prodUsEast3Target));
+        SystemFlagsDeployer deployer = new SystemFlagsDeployer(flagsClient, SYSTEM, Set.of(prodUsWest1Target, prodUsEast3Target));
 
         SystemFlagsDeployResult result = deployer.deployFlags(archive, false);
 
-        System.out.println(result);
-
         assertThat(result.errors()).containsOnly(
                 OperationError.listFailed(exception.getMessage(), prodUsWest1Target));
         assertThat(result.flagChanges()).containsOnly(
                 FlagDataChange.created(FLAG_ID, prodUsEast3Target, defaultData));
     }
 
+    @Test
+    public void creates_error_entry_for_invalid_flag_archive() throws IOException {
+        FlagsClient flagsClient = mock(FlagsClient.class);
+        FlagData defaultData = flagData("flags/my-flag/main.json");
+        SystemFlagsDataArchive archive = new SystemFlagsDataArchive.Builder()
+                .addFile("main.prod.unknown-region.json", defaultData)
+                .build();
+        SystemFlagsDeployer deployer = new SystemFlagsDeployer(flagsClient, SYSTEM, Set.of(controllerTarget));
+        SystemFlagsDeployResult result = deployer.deployFlags(archive, false);
+        assertThat(result.flagChanges())
+                .isEmpty();
+        assertThat(result.errors())
+                .containsOnly(OperationError.archiveValidationFailed("Unknown flag file: flags/my-flag/main.prod.unknown-region.json"));
+    }
+
+    @Test
+    public void creates_warning_entry_for_existing_flag_data_for_undefined_flag() throws IOException {
+        FlagData prodUsEast3Data = flagData("flags/my-flag/main.prod.us-east-3.json");
+        FlagsClient flagsClient = mock(FlagsClient.class);
+        when(flagsClient.listFlagData(prodUsEast3Target))
+                .thenReturn(List.of(prodUsEast3Data));
+        when(flagsClient.listDefinedFlags(prodUsEast3Target))
+                .thenReturn(List.of());
+        SystemFlagsDataArchive archive = new SystemFlagsDataArchive.Builder()
+                .addFile("main.prod.us-east-3.json", prodUsEast3Data)
+                .build();
+        SystemFlagsDeployer deployer = new SystemFlagsDeployer(flagsClient, SYSTEM, Set.of(prodUsEast3Target));
+        SystemFlagsDeployResult result = deployer.deployFlags(archive, true);
+        assertThat(result.warnings())
+                .containsOnly(SystemFlagsDeployResult.Warning.dataForUndefinedFlag(prodUsEast3Target, new FlagId("my-flag")));
+    }
+
     private static FlagData flagData(String filename) throws IOException {
         return FlagData.deserializeUtf8Json(Files.readAllBytes(Paths.get("src/test/resources/system-flags/" + filename)));
     }
 
-}
\ No newline at end of file
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java
index d1dd50cfb4c..51466e5b1e2 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java
@@ -71,11 +71,6 @@ public class UserApiTest extends ControllerContainerCloudTest {
                                       .data("{\"token\":\"hello\"}"),
                               new File("tenant-without-applications.json"));
 
-        // PUT a tenant is not available to anyone.
-        tester.assertResponse(request("/application/v4/user/", PUT)
-                                      .roles(operator),
-                              "{\"error-code\":\"FORBIDDEN\",\"message\":\"Not authenticated or not a user.\"}", 403);
-
         // GET at user/v1 root fails as no access control is defined there.
         tester.assertResponse(request("/user/v1/"),
                               accessDenied, 403);
@@ -102,7 +97,7 @@ public class UserApiTest extends ControllerContainerCloudTest {
         tester.assertResponse(request("/user/v1/tenant/my-tenant/application/my-app", POST)
                                       .roles(Set.of(Role.administrator(TenantName.from("my-tenant"))))
                                       .data("{\"user\":\"headless@app\",\"roleName\":\"headless\"}"),
-                              "{\"error-code\":\"INTERNAL_SERVER_ERROR\",\"message\":\"NullPointerException\"}", 500);
+                              "{\"error-code\":\"BAD_REQUEST\",\"message\":\"role 'headless' of 'my-app' owned by 'my-tenant' not found\"}", 400);
 
         // POST an application is allowed for a tenant developer.
         tester.assertResponse(request("/application/v4/tenant/my-tenant/application/my-app", POST)
@@ -193,17 +188,20 @@ public class UserApiTest extends ControllerContainerCloudTest {
                              .data("{\"user\":\"administrator@tenant\",\"roleName\":\"administrator\"}"),
                               "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Can't remove the last administrator of a tenant.\"}", 400);
 
-        // DELETE the tenant is available to the tenant owner.
+        // DELETE the tenant is not allowed
         tester.assertResponse(request("/application/v4/tenant/my-tenant", DELETE)
-                                      .roles(Set.of(Role.tenantOwner(id.tenant()))),
-                              new File("tenant-without-applications.json"));
+                                      .roles(Set.of(Role.developer(id.tenant()))),
+                              "{\n" +
+                              "  \"code\" : 403,\n" +
+                              "  \"message\" : \"Access denied\"\n" +
+                              "}", 403);
     }
 
     @Test
     public void userMetadataTest() {
         ContainerTester tester = new ContainerTester(container, responseFiles);
         ControllerTester controller = new ControllerTester(tester);
-        Set<Role> operator = Set.of(Role.hostedOperator());
+        Set<Role> operator = Set.of(Role.hostedOperator(), Role.hostedSupporter());
         User user = new User("dev@domail", "Joe Developer", "dev", null);
 
         tester.assertResponse(request("/api/user/v1/user")
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-with-applications-athenz.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-with-applications-athenz.json
index 079e2c9c388..56108dce94f 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-with-applications-athenz.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-with-applications-athenz.json
@@ -11,57 +11,21 @@
         "administrator",
         "developer",
         "reader"
-      ],
-      "applications": {
-        "app1": {
-          "instances": [
-            "default"
-          ]
-        },
-        "app2": {
-          "instances": [
-            "default",
-            "dev"
-          ]
-        }
-      }
+      ]
     },
     "tenant1": {
       "roles": [
         "administrator",
         "developer",
         "reader"
-      ],
-      "applications": {
-        "app1": {
-          "instances": [
-            "default"
-          ]
-        },
-        "app2": {
-          "instances": [
-            "default",
-            "myinstance"
-          ]
-        },
-        "app3": {
-          "instances": []
-        }
-      }
+      ]
     },
     "tenant2": {
       "roles": [
         "administrator",
         "developer",
         "reader"
-      ],
-      "applications": {
-        "app2": {
-          "instances": [
-            "test"
-          ]
-        }
-      }
+      ]
     }
   }
 }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-with-applications-cloud.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-with-applications-cloud.json
index f0ea10ed888..ea76aa977ce 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-with-applications-cloud.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-with-applications-cloud.json
@@ -10,53 +10,17 @@
       "roles": [
         "developer",
         "reader"
-      ],
-      "applications": {
-        "app1": {
-          "instances": [
-            "default"
-          ]
-        },
-        "app2": {
-          "instances": [
-            "default",
-            "dev"
-          ]
-        }
-      }
+      ]
     },
     "tenant1": {
       "roles": [
         "administrator"
-      ],
-      "applications": {
-        "app1": {
-          "instances": [
-            "default"
-          ]
-        },
-        "app2": {
-          "instances": [
-            "default",
-            "myinstance"
-          ]
-        },
-        "app3": {
-          "instances": []
-        }
-      }
+      ]
     },
     "tenant2": {
       "roles": [
         "developer"
-      ],
-      "applications": {
-        "app2": {
-          "instances": [
-            "test"
-          ]
-        }
-      }
+      ]
     }
   }
 }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-without-applications.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-without-applications.json
index 17489bb15d8..400fe8d4d9b 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-without-applications.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-without-applications.json
@@ -7,6 +7,7 @@
   },
   "tenants": {},
   "operator": [
-    "hostedOperator"
+    "hostedOperator",
+    "hostedSupporter"
   ]
 }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiTest.java
index eaeba420bc9..d5031267b27 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiTest.java
@@ -6,7 +6,6 @@ import com.yahoo.config.provision.RegionName;
 import com.yahoo.config.provision.zone.ZoneApi;
 import com.yahoo.vespa.hosted.controller.api.role.Role;
 import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock;
-import com.yahoo.vespa.hosted.controller.integration.ZoneRegistryMock;
 import com.yahoo.vespa.hosted.controller.restapi.ContainerTester;
 import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerCloudTest;
 import org.junit.Before;
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiTest.java
index f00363989e6..a7adac7f89d 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiTest.java
@@ -8,7 +8,6 @@ import com.yahoo.config.provision.zone.ZoneApi;
 import com.yahoo.config.provision.zone.ZoneId;
 import com.yahoo.vespa.hosted.controller.integration.ConfigServerProxyMock;
 import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock;
-import com.yahoo.vespa.hosted.controller.integration.ZoneRegistryMock;
 import com.yahoo.vespa.hosted.controller.proxy.ProxyRequest;
 import com.yahoo.vespa.hosted.controller.restapi.ContainerTester;
 import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerTest;
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepositoryTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepositoryTest.java
index 674f084a8b7..136ed508a33 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepositoryTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepositoryTest.java
@@ -1,21 +1,24 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.rotation;
 
+import com.yahoo.config.provision.RegionName;
 import com.yahoo.config.provision.SystemName;
+import com.yahoo.config.provision.zone.RoutingMethod;
 import com.yahoo.vespa.hosted.controller.ControllerTester;
 import com.yahoo.vespa.hosted.controller.application.ApplicationPackage;
 import com.yahoo.vespa.hosted.controller.application.AssignedRotation;
 import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder;
 import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext;
 import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester;
+import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock;
 import com.yahoo.vespa.hosted.rotation.config.RotationsConfig;
-import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.ExpectedException;
 
 import java.net.URI;
 import java.util.List;
+import java.util.Set;
 import java.util.stream.Collectors;
 
 import static org.junit.Assert.assertEquals;
@@ -48,16 +51,9 @@ public class RotationRepositoryTest {
             .region("us-west-1")
             .build();
 
-    private DeploymentTester tester;
-    private RotationRepository repository;
-    private DeploymentContext application;
-
-    @Before
-    public void before() {
-        tester = new DeploymentTester(new ControllerTester(rotationsConfig));
-        repository = tester.applications().rotationRepository();
-        application = tester.newDeploymentContext("tenant1", "app1", "default");
-    }
+    private final DeploymentTester tester = new DeploymentTester(new ControllerTester(rotationsConfig));
+    private final RotationRepository repository = tester.controller().routing().rotations();
+    private final DeploymentContext application = tester.newDeploymentContext("tenant1", "app1", "default");
 
     @Test
     public void assigns_and_reuses_rotation() {
@@ -67,23 +63,37 @@ public class RotationRepositoryTest {
 
         assertEquals(List.of(expected.id()), rotationIds(application.instance().rotations()));
         assertEquals(URI.create("https://app1--tenant1.global.vespa.oath.cloud:4443/"),
-                     application.instance().endpointsIn(SystemName.main).main().get().url());
+                     tester.controller().routing().endpointsOf(application.instanceId()).primary().get().url());
         try (RotationLock lock = repository.lock()) {
             List<AssignedRotation> rotations = repository.getOrAssignRotations(application.application().deploymentSpec(),
                                                                                application.instance(),
                                                                               lock);
             assertSingleRotation(expected, rotations, repository);
+            assertEquals(Set.of(RegionName.from("us-west-1"), RegionName.from("us-east-3")),
+                         application.instance().rotations().get(0).regions());
         }
 
         // Submitting once more assigns same rotation
-        application.submit(applicationPackage);
+        application.submit(applicationPackage).deploy();
         assertEquals(List.of(expected.id()), rotationIds(application.instance().rotations()));
+
+        // Adding region updates rotation
+        var applicationPackage = new ApplicationPackageBuilder()
+                .globalServiceId("foo")
+                .region("us-east-3")
+                .region("us-west-1")
+                .region("us-central-1")
+                .build();
+        application.submit(applicationPackage).deploy();
+        assertEquals(Set.of(RegionName.from("us-west-1"), RegionName.from("us-east-3"),
+                     RegionName.from("us-central-1")),
+                     application.instance().rotations().get(0).regions());
     }
     
     @Test
     public void strips_whitespace_in_rotation_fqdn() {
-        tester = new DeploymentTester(new ControllerTester(rotationsConfigWhitespaces));
-        RotationRepository repository = tester.controller().applications().rotationRepository();
+        var tester = new DeploymentTester(new ControllerTester(rotationsConfigWhitespaces));
+        RotationRepository repository = tester.controller().routing().rotations();
         var application2 = tester.newDeploymentContext("tenant1", "app2", "default");
 
         application2.submit(applicationPackage);
@@ -106,7 +116,7 @@ public class RotationRepositoryTest {
 
         // We're now out of rotations
         thrown.expect(IllegalStateException.class);
-        thrown.expectMessage("no rotations available");
+        thrown.expectMessage("out of rotations");
         var application3 = tester.newDeploymentContext("tenant3", "app3", "default");
         application3.submit(applicationPackage);
     }
@@ -136,14 +146,19 @@ public class RotationRepositoryTest {
     public void prefixes_system_when_not_main() {
         ApplicationPackage applicationPackage = new ApplicationPackageBuilder()
                 .globalServiceId("foo")
-                .region("us-east-3")
-                .region("us-west-1")
+                .region("cd-us-central-1")
+                .region("cd-us-west-1")
                 .build();
+        var zones = List.of(ZoneApiMock.fromId("prod.cd-us-central-1"), ZoneApiMock.fromId("prod.cd-us-west-1"));
+        tester.controllerTester().zoneRegistry()
+              .setZones(zones)
+              .setRoutingMethod(zones, RoutingMethod.shared)
+              .setSystemName(SystemName.cd);
         var application2 = tester.newDeploymentContext("tenant2", "app2", "default");
         application2.submit(applicationPackage);
         assertEquals(List.of(new RotationId("foo-1")), rotationIds(application2.instance().rotations()));
         assertEquals("https://cd--app2--tenant2.global.vespa.oath.cloud:4443/",
-                     application2.instance().endpointsIn(SystemName.cd).main().get().url().toString());
+                     tester.controller().routing().endpointsOf(application2.instanceId()).primary().get().url().toString());
     }
 
     @Test
@@ -159,9 +174,9 @@ public class RotationRepositoryTest {
         assertEquals(List.of(new RotationId("foo-1")), rotationIds(instance1.instance().rotations()));
         assertEquals(List.of(new RotationId("foo-2")), rotationIds(instance2.instance().rotations()));
         assertEquals(URI.create("https://instance1--application1--tenant1.global.vespa.oath.cloud:4443/"),
-                     instance1.instance().endpointsIn(SystemName.main).main().get().url());
+                     tester.controller().routing().endpointsOf(instance1.instanceId()).primary().get().url());
         assertEquals(URI.create("https://instance2--application1--tenant1.global.vespa.oath.cloud:4443/"),
-                     instance2.instance().endpointsIn(SystemName.main).main().get().url());
+                     tester.controller().routing().endpointsOf(instance2.instanceId()).primary().get().url());
     }
 
     @Test
@@ -179,9 +194,9 @@ public class RotationRepositoryTest {
         assertEquals(List.of(new RotationId("foo-2")), rotationIds(instance2.instance().rotations()));
 
         assertEquals(URI.create("https://instance1--application1--tenant1.global.vespa.oath.cloud:4443/"),
-                     instance1.instance().endpointsIn(SystemName.main).main().get().url());
+                     tester.controller().routing().endpointsOf(instance1.instanceId()).primary().get().url());
         assertEquals(URI.create("https://instance2--application1--tenant1.global.vespa.oath.cloud:4443/"),
-                     instance2.instance().endpointsIn(SystemName.main).main().get().url());
+                     tester.controller().routing().endpointsOf(instance2.instanceId()).primary().get().url());
     }
 
     private void assertSingleRotation(Rotation expected, List<AssignedRotation> assignedRotations, RotationRepository repository) {
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java
index ec169633433..e44a1364185 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java
@@ -1,43 +1,53 @@
 // Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.routing;
 
+import com.google.common.collect.Sets;
 import com.yahoo.config.application.api.DeploymentSpec;
 import com.yahoo.config.application.api.ValidationId;
 import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.AthenzDomain;
+import com.yahoo.config.provision.AthenzService;
 import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.config.provision.Environment;
 import com.yahoo.config.provision.HostName;
+import com.yahoo.config.provision.RegionName;
+import com.yahoo.config.provision.SystemName;
+import com.yahoo.config.provision.zone.RoutingMethod;
 import com.yahoo.config.provision.zone.ZoneId;
 import com.yahoo.vespa.hosted.controller.ControllerTester;
 import com.yahoo.vespa.hosted.controller.Instance;
+import com.yahoo.vespa.hosted.controller.RoutingController;
 import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.LoadBalancer;
+import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
 import com.yahoo.vespa.hosted.controller.api.integration.dns.Record;
 import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData;
 import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName;
 import com.yahoo.vespa.hosted.controller.application.ApplicationPackage;
+import com.yahoo.vespa.hosted.controller.application.Endpoint;
 import com.yahoo.vespa.hosted.controller.application.EndpointId;
+import com.yahoo.vespa.hosted.controller.application.SystemApplication;
 import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder;
 import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext;
 import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester;
+import com.yahoo.vespa.hosted.controller.integration.ServiceRegistryMock;
 import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock;
 import com.yahoo.vespa.hosted.rotation.config.RotationsConfig;
 import org.junit.Test;
 
-import java.net.URI;
 import java.time.Duration;
 import java.time.Instant;
 import java.time.temporal.ChronoUnit;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
-import java.util.Collections;
 import java.util.List;
-import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertSame;
 import static org.junit.Assert.assertTrue;
 
 /**
@@ -50,10 +60,9 @@ public class RoutingPoliciesTest {
     private final ZoneId zone2 = ZoneId.from("prod", "us-central-1");
     private final ZoneId zone3 = ZoneId.from("prod", "us-east-3");
 
-    private final ApplicationPackage applicationPackage = new ApplicationPackageBuilder()
-            .region(zone1.region())
-            .region(zone2.region())
-            .build();
+    private final ApplicationPackage applicationPackage = applicationPackageBuilder().region(zone1.region())
+                                                                                     .region(zone2.region())
+                                                                                     .build();
 
     @Test
     public void global_routing_policies() {
@@ -62,7 +71,7 @@ public class RoutingPoliciesTest {
         var context2 = tester.newDeploymentContext("tenant1", "app2", "default");
         int clustersPerZone = 2;
         int numberOfDeployments = 2;
-        var applicationPackage = new ApplicationPackageBuilder()
+        var applicationPackage = applicationPackageBuilder()
                 .region(zone1.region())
                 .region(zone2.region())
                 .endpoint("r0", "c0")
@@ -72,7 +81,7 @@ public class RoutingPoliciesTest {
         tester.provisionLoadBalancers(clustersPerZone, context1.instanceId(), zone1, zone2);
 
         // Creates alias records
-        context1.submit(applicationPackage).deploy();
+        context1.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy();
         tester.assertTargets(context1.instanceId(), EndpointId.of("r0"), 0, zone1, zone2);
         tester.assertTargets(context1.instanceId(), EndpointId.of("r1"), 0, zone1);
         tester.assertTargets(context1.instanceId(), EndpointId.of("r2"), 1, zone1, zone2);
@@ -81,7 +90,7 @@ public class RoutingPoliciesTest {
                      tester.policiesOf(context1.instance().id()).size());
 
         // Applications gains a new deployment
-        ApplicationPackage applicationPackage2 = new ApplicationPackageBuilder()
+        ApplicationPackage applicationPackage2 = applicationPackageBuilder()
                 .region(zone1.region())
                 .region(zone2.region())
                 .region(zone3.region())
@@ -91,7 +100,7 @@ public class RoutingPoliciesTest {
                 .build();
         numberOfDeployments++;
         tester.provisionLoadBalancers(clustersPerZone, context1.instanceId(), zone3);
-        context1.submit(applicationPackage2).deploy();
+        context1.submit(applicationPackage2).deferLoadBalancerProvisioningIn(Environment.prod).deploy();
 
         // Endpoints are updated to contain cluster in new deployment
         tester.assertTargets(context1.instanceId(), EndpointId.of("r0"), 0, zone1, zone2, zone3);
@@ -101,22 +110,22 @@ public class RoutingPoliciesTest {
         // Another application is deployed with a single cluster and global endpoint
         var endpoint4 = "r0.app2.tenant1.global.vespa.oath.cloud";
         tester.provisionLoadBalancers(1, context2.instanceId(), zone1, zone2);
-        var applicationPackage3 = new ApplicationPackageBuilder()
+        var applicationPackage3 = applicationPackageBuilder()
                 .region(zone1.region())
                 .region(zone2.region())
                 .endpoint("r0", "c0")
                 .build();
-        context2.submit(applicationPackage3).deploy();
+        context2.submit(applicationPackage3).deferLoadBalancerProvisioningIn(Environment.prod).deploy();
         tester.assertTargets(context2.instanceId(), EndpointId.of("r0"), 0, zone1, zone2);
 
         // All endpoints for app1 are removed
-        ApplicationPackage applicationPackage4 = new ApplicationPackageBuilder()
+        ApplicationPackage applicationPackage4 = applicationPackageBuilder()
                 .region(zone1.region())
                 .region(zone2.region())
                 .region(zone3.region())
                 .allow(ValidationId.globalEndpointChange)
                 .build();
-        context1.submit(applicationPackage4).deploy();
+        context1.submit(applicationPackage4).deferLoadBalancerProvisioningIn(Environment.prod).deploy();
         tester.assertTargets(context1.instanceId(), EndpointId.of("r0"), 0);
         tester.assertTargets(context1.instanceId(), EndpointId.of("r1"), 0);
         tester.assertTargets(context1.instanceId(), EndpointId.of("r2"), 0);
@@ -136,7 +145,7 @@ public class RoutingPoliciesTest {
         // Deploy application
         int clustersPerZone = 2;
         tester.provisionLoadBalancers(clustersPerZone, context1.instanceId(), zone1, zone2);
-        context1.submit(applicationPackage).deploy();
+        context1.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy();
 
         // Deployment creates records and policies for all clusters in all zones
         Set<String> expectedRecords = Set.of(
@@ -149,13 +158,13 @@ public class RoutingPoliciesTest {
         assertEquals(4, tester.policiesOf(context1.instanceId()).size());
 
         // Next deploy does nothing
-        context1.submit(applicationPackage).deploy();
+        context1.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy();
         assertEquals(expectedRecords, tester.recordNames());
         assertEquals(4, tester.policiesOf(context1.instanceId()).size());
 
         // Add 1 cluster in each zone and deploy
         tester.provisionLoadBalancers(clustersPerZone + 1, context1.instanceId(), zone1, zone2);
-        context1.submit(applicationPackage).deploy();
+        context1.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy();
         expectedRecords = Set.of(
                 "c0.app1.tenant1.us-west-1.vespa.oath.cloud",
                 "c1.app1.tenant1.us-west-1.vespa.oath.cloud",
@@ -169,7 +178,7 @@ public class RoutingPoliciesTest {
 
         // Deploy another application
         tester.provisionLoadBalancers(clustersPerZone, context2.instanceId(), zone1, zone2);
-        context2.submit(applicationPackage).deploy();
+        context2.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy();
         expectedRecords = Set.of(
                 "c0.app1.tenant1.us-west-1.vespa.oath.cloud",
                 "c1.app1.tenant1.us-west-1.vespa.oath.cloud",
@@ -182,12 +191,12 @@ public class RoutingPoliciesTest {
                 "c0.app2.tenant1.us-west-1.vespa.oath.cloud",
                 "c1.app2.tenant1.us-west-1.vespa.oath.cloud"
         );
-        assertEquals(expectedRecords, tester.recordNames());
+        assertEquals(expectedRecords.stream().sorted().collect(Collectors.toList()), tester.recordNames().stream().sorted().collect(Collectors.toList()));
         assertEquals(4, tester.policiesOf(context2.instanceId()).size());
 
         // Deploy removes cluster from app1
         tester.provisionLoadBalancers(clustersPerZone, context1.instanceId(), zone1, zone2);
-        context1.submit(applicationPackage).deploy();
+        context1.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy();
         expectedRecords = Set.of(
                 "c0.app1.tenant1.us-west-1.vespa.oath.cloud",
                 "c1.app1.tenant1.us-west-1.vespa.oath.cloud",
@@ -224,11 +233,11 @@ public class RoutingPoliciesTest {
         var context = tester.newDeploymentContext("tenant1", "app1", "default");
         tester.provisionLoadBalancers(1, context.instanceId(), zone1, zone2);
 
-        var applicationPackage = new ApplicationPackageBuilder()
+        var applicationPackage = applicationPackageBuilder()
                 .region(zone1.region().value())
                 .endpoint("r0", "c0")
                 .build();
-        context.submit(applicationPackage).deploy();
+        context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy();
 
         var endpoint = "r0.app1.tenant1.global.vespa.oath.cloud";
         assertEquals(endpoint + " points to c0 in all regions",
@@ -240,22 +249,6 @@ public class RoutingPoliciesTest {
     }
 
     @Test
-    public void cluster_endpoints_resolve_from_policies() {
-        var tester = new RoutingPoliciesTester();
-        var context = tester.newDeploymentContext("tenant1", "app1", "default");
-        tester.provisionLoadBalancers(3, context.instanceId(), zone1);
-        context.submit(applicationPackage).deploy();
-        tester.controllerTester().serviceRegistry().routingGeneratorMock().putEndpoints(context.deploymentIdIn(zone1), Collections.emptyList());
-        assertEquals(Map.of(ClusterSpec.Id.from("c0"),
-                            URI.create("https://c0.app1.tenant1.us-west-1.vespa.oath.cloud/"),
-                            ClusterSpec.Id.from("c1"),
-                            URI.create("https://c1.app1.tenant1.us-west-1.vespa.oath.cloud/"),
-                            ClusterSpec.Id.from("c2"),
-                            URI.create("https://c2.app1.tenant1.us-west-1.vespa.oath.cloud/")),
-                     tester.controllerTester().controller().applications().clusterEndpoints(context.deploymentIdIn(zone1)));
-    }
-
-    @Test
     public void manual_deployment_creates_routing_policy() {
         // Empty application package is valid in manually deployed environments
         var tester = new RoutingPoliciesTester();
@@ -265,8 +258,7 @@ public class RoutingPoliciesTest {
         var zoneApi = ZoneApiMock.from(zone.environment(), zone.region());
         tester.controllerTester().serviceRegistry().zoneRegistry()
               .setZones(zoneApi)
-              .setDirectlyRouted(zoneApi);
-        tester.provisionLoadBalancers(1, context.instanceId(), zone);
+              .exclusiveRoutingIn(zoneApi);
 
         // Deploy to dev
         tester.controllerTester().controller().applications().deploy(context.instanceId(), zone, Optional.of(emptyApplicationPackage), DeployOptions.none());
@@ -275,7 +267,7 @@ public class RoutingPoliciesTest {
 
         // Routing policy is created and DNS is updated
         assertEquals(1, tester.policiesOf(context.instanceId()).size());
-        assertEquals(Set.of("c0.app1.tenant1.us-east-1.dev.vespa.oath.cloud"), tester.recordNames());
+        assertEquals(Set.of("app1.tenant1.us-east-1.dev.vespa.oath.cloud"), tester.recordNames());
     }
 
     @Test
@@ -288,19 +280,19 @@ public class RoutingPoliciesTest {
         var zoneApi = ZoneApiMock.from(zone.environment(), zone.region());
         tester.controllerTester().serviceRegistry().zoneRegistry()
               .setZones(zoneApi)
-              .setDirectlyRouted(zoneApi);
-
+              .exclusiveRoutingIn(zoneApi);
+        var prodRecords = Set.of("app1.tenant1.us-central-1.vespa.oath.cloud", "app1.tenant1.us-west-1.vespa.oath.cloud");
+        assertEquals(prodRecords, tester.recordNames());
 
         // Deploy to dev under different instance
         var devInstance = context.application().id().instance("user");
-        tester.provisionLoadBalancers(1, devInstance, zone);
         tester.controllerTester().controller().applications().deploy(devInstance, zone, Optional.of(applicationPackage), DeployOptions.none());
         assertEquals("DeploymentSpec is persisted", applicationPackage.deploymentSpec(), context.application().deploymentSpec());
         context.flushDnsUpdates();
 
         // Routing policy is created and DNS is updated
         assertEquals(1, tester.policiesOf(devInstance).size());
-        assertEquals(Set.of("c0.user.app1.tenant1.us-east-1.dev.vespa.oath.cloud"), tester.recordNames());
+        assertEquals(Sets.union(prodRecords, Set.of("user.app1.tenant1.us-east-1.dev.vespa.oath.cloud")), tester.recordNames());
     }
 
     @Test
@@ -310,12 +302,12 @@ public class RoutingPoliciesTest {
 
         // Initial load balancer is provisioned
         tester.provisionLoadBalancers(1, context.instanceId(), zone1);
-        var applicationPackage = new ApplicationPackageBuilder()
+        var applicationPackage = applicationPackageBuilder()
                 .region(zone1.region())
                 .build();
 
         // Application is deployed
-        context.submit(applicationPackage).deploy();
+        context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy();
         var expectedRecords = Set.of(
                 "c0.app1.tenant1.us-west-1.vespa.oath.cloud"
         );
@@ -334,10 +326,10 @@ public class RoutingPoliciesTest {
                                             newHostname,
                                             LoadBalancer.State.active,
                                             Optional.of("dns-zone-1"));
-        tester.controllerTester().configServer().addLoadBalancers(zone1, List.of(loadBalancer));
+        tester.controllerTester().configServer().putLoadBalancers(zone1, List.of(loadBalancer));
 
         // Application redeployment preserves DNS record
-        context.submit(applicationPackage).deploy();
+        context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy();
         assertEquals(expectedRecords, tester.recordNames());
         assertEquals(1, tester.policiesOf(context.instanceId()).size());
         assertEquals("CNAME points to current load blancer", newHostname.value() + ".",
@@ -351,13 +343,13 @@ public class RoutingPoliciesTest {
 
         // Provision load balancers and deploy application
         tester.provisionLoadBalancers(1, context.instanceId(), zone1, zone2);
-        var applicationPackage = new ApplicationPackageBuilder()
+        var applicationPackage = applicationPackageBuilder()
                 .region(zone1.region())
                 .region(zone2.region())
                 .endpoint("r0", "c0", zone1.region().value(), zone2.region().value())
                 .endpoint("r1", "c0", zone1.region().value(), zone2.region().value())
                 .build();
-        context.submit(applicationPackage).deploy();
+        context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy();
 
         // Global DNS record is created
         tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1, zone2);
@@ -386,7 +378,7 @@ public class RoutingPoliciesTest {
         assertEquals(Instant.EPOCH, policy2.status().globalRouting().changedAt());
 
         // Next deployment does not affect status
-        context.submit(applicationPackage).deploy();
+        context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy();
         context.flushDnsUpdates();
         tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone2);
         tester.assertTargets(context.instanceId(), EndpointId.of("r1"), 0, zone2);
@@ -405,24 +397,24 @@ public class RoutingPoliciesTest {
         assertEquals(changedAt.truncatedTo(ChronoUnit.MILLIS), policy1.status().globalRouting().changedAt());
 
         // Deployment is set out through a new deployment.xml
-        var applicationPackage2 = new ApplicationPackageBuilder()
+        var applicationPackage2 = applicationPackageBuilder()
                 .region(zone1.region())
                 .region(zone2.region(), false)
                 .endpoint("r0", "c0", zone1.region().value(), zone2.region().value())
                 .endpoint("r1", "c0", zone1.region().value(), zone2.region().value())
                 .build();
-        context.submit(applicationPackage2).deploy();
+        context.submit(applicationPackage2).deferLoadBalancerProvisioningIn(Environment.prod).deploy();
         tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1);
         tester.assertTargets(context.instanceId(), EndpointId.of("r1"), 0, zone1);
 
         // ... back in
-        var applicationPackage3 = new ApplicationPackageBuilder()
+        var applicationPackage3 = applicationPackageBuilder()
                 .region(zone1.region())
                 .region(zone2.region())
                 .endpoint("r0", "c0", zone1.region().value(), zone2.region().value())
                 .endpoint("r1", "c0", zone1.region().value(), zone2.region().value())
                 .build();
-        context.submit(applicationPackage3).deploy();
+        context.submit(applicationPackage3).deferLoadBalancerProvisioningIn(Environment.prod).deploy();
         tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1, zone2);
         tester.assertTargets(context.instanceId(), EndpointId.of("r1"), 0, zone1, zone2);
     }
@@ -435,14 +427,14 @@ public class RoutingPoliciesTest {
         var contexts = List.of(context1, context2);
 
         // Deploy applications
-        var applicationPackage = new ApplicationPackageBuilder()
+        var applicationPackage = applicationPackageBuilder()
                 .region(zone1.region())
                 .region(zone2.region())
                 .endpoint("default", "c0", zone1.region().value(), zone2.region().value())
                 .build();
         for (var context : contexts) {
             tester.provisionLoadBalancers(1, context.instanceId(), zone1, zone2);
-            context.submit(applicationPackage).deploy();
+            context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy();
             tester.assertTargets(context.instanceId(), EndpointId.defaultId(), 0, zone1, zone2);
         }
 
@@ -482,6 +474,112 @@ public class RoutingPoliciesTest {
         tester.assertTargets(context1.instanceId(), EndpointId.defaultId(), 0, zone1);
         tester.assertTargets(context2.instanceId(), EndpointId.defaultId(), 0, zone1, zone2);
     }
+
+    @Test
+    public void non_production_deployment_is_not_registered_in_global_endpoint() {
+        var tester = new RoutingPoliciesTester(SystemName.Public);
+
+        // Configure the system to use the same region for test, staging and prod
+        var sharedRegion = RegionName.from("aws-us-east-1c");
+        var prodZone = ZoneId.from(Environment.prod, sharedRegion);
+        var stagingZone = ZoneId.from(Environment.staging, sharedRegion);
+        var testZone = ZoneId.from(Environment.test, sharedRegion);
+        var zones = List.of(ZoneApiMock.from(prodZone),
+                            ZoneApiMock.from(stagingZone),
+                            ZoneApiMock.from(testZone));
+        tester.controllerTester().zoneRegistry()
+              .setZones(zones)
+              .setRoutingMethod(zones, RoutingMethod.exclusive);
+        tester.controllerTester().configServer().bootstrap(List.of(prodZone, stagingZone, testZone),
+                                                           SystemApplication.all());
+
+        var context = tester.tester.newDeploymentContext();
+        var endpointId = EndpointId.of("r0");
+        var applicationPackage = applicationPackageBuilder()
+                .trustDefaultCertificate()
+                .region(sharedRegion)
+                .endpoint(endpointId.id(), "default")
+                .build();
+
+        // Application starts deployment
+        context = context.submit(applicationPackage);
+        for (var testJob : List.of(JobType.systemTest, JobType.stagingTest)) {
+            context = context.runJob(testJob);
+            // Since runJob implicitly tears down the deployment and immediately deletes DNS records associated with the
+            // deployment, we consume only one DNS update at a time here
+            do {
+                context = context.flushDnsUpdates(1);
+                tester.assertTargets(context.instanceId(), endpointId, 0);
+            } while (!tester.recordNames().isEmpty());
+        }
+
+        // Deployment completes
+        context.completeRollout();
+        tester.assertTargets(context.instanceId(), endpointId, 0, prodZone);
+    }
+
+    @Test
+    public void changing_global_routing_status_never_removes_all_members() {
+        var tester = new RoutingPoliciesTester();
+        var context = tester.newDeploymentContext("tenant1", "app1", "default");
+
+        // Provision load balancers and deploy application
+        tester.provisionLoadBalancers(1, context.instanceId(), zone1, zone2);
+        var applicationPackage = applicationPackageBuilder()
+                .region(zone1.region())
+                .region(zone2.region())
+                .endpoint("r0", "c0", zone1.region().value(), zone2.region().value())
+                .build();
+        context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy();
+
+        // Global DNS record is created, pointing to all configured zones
+        tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1, zone2);
+
+        // Global routing status is overridden for one deployment
+        tester.routingPolicies().setGlobalRoutingStatus(context.deploymentIdIn(zone1), GlobalRouting.Status.out,
+                                                        GlobalRouting.Agent.tenant);
+        context.flushDnsUpdates();
+        tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone2);
+
+        // Setting other deployment out implicitly sets all deployments in
+        tester.routingPolicies().setGlobalRoutingStatus(context.deploymentIdIn(zone2), GlobalRouting.Status.out,
+                                                        GlobalRouting.Agent.tenant);
+        context.flushDnsUpdates();
+        tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1, zone2);
+
+        // One inactive deployment is put back in. Global DNS record now points to the only active deployment
+        tester.routingPolicies().setGlobalRoutingStatus(context.deploymentIdIn(zone1), GlobalRouting.Status.in,
+                                                        GlobalRouting.Agent.tenant);
+        context.flushDnsUpdates();
+        tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1);
+
+        // Setting zone (containing active deployment) out puts all deployments in
+        tester.routingPolicies().setGlobalRoutingStatus(zone1, GlobalRouting.Status.out);
+        context.flushDnsUpdates();
+        assertEquals(GlobalRouting.Status.out, tester.routingPolicies().get(zone1).globalRouting().status());
+        tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1, zone2);
+
+        // Setting zone back in removes the currently inactive deployment
+        tester.routingPolicies().setGlobalRoutingStatus(zone1, GlobalRouting.Status.in);
+        context.flushDnsUpdates();
+        tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1);
+
+        // Inactive deployment is set in
+        tester.routingPolicies().setGlobalRoutingStatus(context.deploymentIdIn(zone2), GlobalRouting.Status.in,
+                                                        GlobalRouting.Agent.tenant);
+        context.flushDnsUpdates();
+        for (var policy : tester.routingPolicies().get(context.instanceId()).values()) {
+            assertSame(GlobalRouting.Status.in, policy.status().globalRouting().status());
+        }
+        tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1, zone2);
+    }
+
+    /** Returns an application package builder that satisfies requirements for a directly routed endpoint */
+    private static ApplicationPackageBuilder applicationPackageBuilder() {
+        return new ApplicationPackageBuilder()
+                .athenzIdentity(AthenzDomain.from("domain"), AthenzService.from("service"))
+                .compileVersion(RoutingController.DIRECT_ROUTING_MIN_VERSION);
+    }
     
     private static List<LoadBalancer> createLoadBalancers(ZoneId zone, ApplicationId application, int count) {
         List<LoadBalancer> loadBalancers = new ArrayList<>();
@@ -503,11 +601,15 @@ public class RoutingPoliciesTest {
         private final DeploymentTester tester;
 
         public RoutingPoliciesTester() {
-            this(new DeploymentTester());
+            this(SystemName.main);
+        }
+
+        public RoutingPoliciesTester(SystemName system) {
+            this(new DeploymentTester(new ControllerTester(new ServiceRegistryMock(system))));
         }
 
         public RoutingPolicies routingPolicies() {
-            return tester.controllerTester().controller().applications().routingPolicies();
+            return tester.controllerTester().controller().routing().policies();
         }
 
         public DeploymentContext newDeploymentContext(String tenant, String application, String instance) {
@@ -520,12 +622,14 @@ public class RoutingPoliciesTest {
 
         public RoutingPoliciesTester(DeploymentTester tester) {
             this.tester = tester;
+            // Make all zones directly routed
+            tester.controllerTester().zoneRegistry().exclusiveRoutingIn(tester.controllerTester().zoneRegistry().zones().all().zones());
         }
 
         private void provisionLoadBalancers(int clustersPerZone, ApplicationId application, ZoneId... zones) {
             for (ZoneId zone : zones) {
                 tester.configServer().removeLoadBalancers(application, zone);
-                tester.configServer().addLoadBalancers(zone, createLoadBalancers(zone, application, clustersPerZone));
+                tester.configServer().putLoadBalancers(zone, createLoadBalancers(zone, application, clustersPerZone));
             }
         }
 
@@ -555,12 +659,12 @@ public class RoutingPoliciesTest {
         }
 
         private void assertTargets(ApplicationId application, EndpointId endpointId, int loadBalancerId, ZoneId ...zone) {
-            var prefix = "";
-            if (!endpointId.equals(EndpointId.defaultId())) {
-                prefix = endpointId.id() + ".";
-            }
-            var endpoint = prefix + application.application().value() + "." + application.tenant().value() +
-                           ".global.vespa.oath.cloud";
+            var endpoint = tester.controller().routing().endpointsOf(application)
+                                 .named(endpointId)
+                                 .targets(List.of(zone))
+                                 .primary()
+                                 .map(Endpoint::dnsName)
+                                 .orElse("<none>");
             var zoneTargets = Arrays.stream(zone)
                                     .map(z -> "lb-" + loadBalancerId + "--" + application.serializedForm() + "--" +
                                               z.value() + "/dns-zone-1/" + z.value())
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java
index c983d3610e8..f60d11693d8 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java
@@ -1,7 +1,6 @@
 // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.versions;
 
-import com.google.common.collect.ImmutableSet;
 import com.yahoo.component.Version;
 import com.yahoo.component.Vtag;
 import com.yahoo.config.provision.ApplicationId;
@@ -11,10 +10,13 @@ import com.yahoo.config.provision.zone.ZoneApi;
 import com.yahoo.vespa.hosted.controller.Controller;
 import com.yahoo.vespa.hosted.controller.ControllerTester;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node;
+import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId;
+import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
 import com.yahoo.vespa.hosted.controller.application.ApplicationPackage;
 import com.yahoo.vespa.hosted.controller.application.SystemApplication;
 import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder;
 import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester;
+import com.yahoo.vespa.hosted.controller.deployment.Run;
 import com.yahoo.vespa.hosted.controller.persistence.CuratorDb;
 import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb;
 import com.yahoo.vespa.hosted.controller.versions.VespaVersion.Confidence;
@@ -23,6 +25,7 @@ import org.junit.Test;
 import java.time.Duration;
 import java.time.Instant;
 import java.util.List;
+import java.util.Map;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
@@ -30,6 +33,7 @@ import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobTy
 import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionUsWest1;
 import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.stagingTest;
 import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.systemTest;
+import static java.util.stream.Collectors.toSet;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotNull;
@@ -146,36 +150,68 @@ public class VersionStatusTest {
         tester.triggerJobs();
 
         // - app1 is in production on version1, but then fails in system test on version2
-        context1.submit(applicationPackage)
-                .timeOutConvergence(systemTest);
+        context1.timeOutConvergence(systemTest);
         // - app2 is partially in production on version1 and partially on version2
-        context2.submit(applicationPackage)
-                .runJob(systemTest)
+        context2.runJob(systemTest)
                 .runJob(stagingTest)
                 .runJob(productionUsWest1)
                 .failDeployment(productionUsEast3);
         // - app3 is in production on version1, but then fails in staging test on version2
-        context3.submit(applicationPackage)
-                .timeOutUpgrade(stagingTest);
+        context3.timeOutUpgrade(stagingTest);
 
+        tester.triggerJobs();
         tester.controllerTester().computeVersionStatus();
         List<VespaVersion> versions = tester.controller().versionStatus().versions();
         assertEquals("The two versions above exist", 2, versions.size());
 
         VespaVersion v1 = versions.get(0);
         assertEquals(version1, v1.versionNumber());
-        assertEquals("No applications are failing on version1.", ImmutableSet.of(), v1.statistics().failing());
-        assertEquals("All applications have at least one active production deployment on version 1.", ImmutableSet.of(context1.instanceId(), context2.instanceId(), context3.instanceId()), v1.statistics().production());
-        assertEquals("No applications have active deployment jobs on version1.", ImmutableSet.of(), v1.statistics().deploying());
+        var statistics = DeploymentStatistics.compute(List.of(version1, version2), tester.deploymentStatuses());
+        var statistics1 = statistics.get(0);
+        assertJobsRun("No runs are failing on version1.",
+                      Map.of(context1.instanceId(), List.of(),
+                             context2.instanceId(), List.of(),
+                             context3.instanceId(), List.of()),
+                      statistics1.failingUpgrades());
+        assertJobsRun("All applications have at least one active production deployment on version 1.",
+                     Map.of(context1.instanceId(), List.of(productionUsWest1, productionUsEast3),
+                            context2.instanceId(), List.of(productionUsEast3),
+                            context3.instanceId(), List.of(productionUsWest1, productionUsEast3)),
+                     statistics1.productionSuccesses());
+        assertEquals("No applications have active deployment jobs on version1.",
+                     List.of(),
+                     statistics1.runningUpgrade());
 
         VespaVersion v2 = versions.get(1);
         assertEquals(version2, v2.versionNumber());
-        assertEquals("All applications have failed on version2 in at least one zone.", ImmutableSet.of(context1.instanceId(), context2.instanceId(), context3.instanceId()), v2.statistics().failing());
-        assertEquals("Only app2 has successfully deployed to production on version2.", ImmutableSet.of(context2.instanceId()), v2.statistics().production());
-        // Should test the below, but can't easily be done with current test framework. This test passes in DeploymentApiTest.
-        // assertEquals("All applications are being retried on version2.", ImmutableSet.of(app1.id(), app2.id(), app3.id()), v2.statistics().deploying());
+        var statistics2 = statistics.get(1);
+        assertJobsRun("All applications have failed on version2 in at least one zone.",
+                     Map.of(context1.instanceId(), List.of(systemTest),
+                            context2.instanceId(), List.of(productionUsEast3),
+                            context3.instanceId(), List.of(stagingTest)),
+                     statistics2.failingUpgrades());
+        assertJobsRun("Only app2 has successfully deployed to production on version2.",
+                      Map.of(context1.instanceId(), List.of(),
+                             context2.instanceId(), List.of(productionUsWest1),
+                             context3.instanceId(), List.of()),
+                     statistics2.productionSuccesses());
+        assertJobsRun("All applications are being retried on version2.",
+                      Map.of(context1.instanceId(), List.of(systemTest, stagingTest),
+                             context2.instanceId(), List.of(productionUsEast3),
+                             context3.instanceId(), List.of(systemTest, stagingTest)),
+                     statistics2.runningUpgrade());
     }
-    
+
+    private static void assertJobsRun(String assertion, Map<ApplicationId, List<JobType>> jobs, List<Run> runs) {
+        assertEquals(assertion,
+                     jobs.entrySet().stream()
+                         .flatMap(entry -> entry.getValue().stream().map(type -> new JobId(entry.getKey(), type)))
+                         .collect(toSet()),
+                     runs.stream()
+                         .map(run -> run.id().job())
+                         .collect(toSet()));
+    }
+
     @Test
     public void testVersionConfidence() {
         DeploymentTester tester = new DeploymentTester().atMondayMorning();
@@ -340,10 +376,7 @@ public class VersionStatusTest {
 
         // Test version order
         List<VespaVersion> versions = tester.controller().versionStatus().versions();
-        assertEquals(3, versions.size());
-        assertEquals("6.2", versions.get(0).versionNumber().toString());
-        assertEquals("6.4", versions.get(1).versionNumber().toString());
-        assertEquals("6.5", versions.get(2).versionNumber().toString());
+        assertEquals(List.of("6.2", "6.4", "6.5"), versions.stream().map(version -> version.versionNumber().toString()).collect(Collectors.toList()));
 
         // Check release status is correct (static data in MockMavenRepository).
         assertTrue(versions.get(0).isReleased());
@@ -379,7 +412,7 @@ public class VersionStatusTest {
 
         // Stale override was removed
         assertFalse("Stale override removed", tester.controller().curator().readConfidenceOverrides()
-                                                    .keySet().contains(version0));
+                                                    .containsKey(version0));
     }
 
     @Test
@@ -516,7 +549,7 @@ public class VersionStatusTest {
                .failDeployment(productionUsWest1);
         tester.controllerTester().computeVersionStatus();
         for (var version : List.of(version0, version1)) {
-            assertOnVersion(version, context.instanceId(), tester.controllerTester());
+            assertOnVersion(version, context.instanceId(), tester);
         }
 
         // System is upgraded and application starts upgrading to next version
@@ -531,14 +564,14 @@ public class VersionStatusTest {
                .failDeployment(productionUsWest1);
         tester.controllerTester().computeVersionStatus();
         for (var version : List.of(version0, version1, version2)) {
-            assertOnVersion(version, context.instanceId(), tester.controllerTester());
+            assertOnVersion(version, context.instanceId(), tester);
         }
 
         // Upgrade succeeds
         context.deployPlatform(version2);
         tester.controllerTester().computeVersionStatus();
         assertEquals(1, tester.controller().versionStatus().versions().size());
-        assertOnVersion(version2, context.instanceId(), tester.controllerTester());
+        assertOnVersion(version2, context.instanceId(), tester);
 
         // System is upgraded and application starts upgrading to next version
         var version3 = Version.fromString("7.4");
@@ -552,17 +585,17 @@ public class VersionStatusTest {
         tester.controllerTester().computeVersionStatus();
         assertEquals(2, tester.controller().versionStatus().versions().size());
         for (var version : List.of(version2, version3)) {
-            assertOnVersion(version, context.instanceId(), tester.controllerTester());
+            assertOnVersion(version, context.instanceId(), tester);
         }
     }
 
-    private void assertOnVersion(Version version, ApplicationId instance, ControllerTester tester) {
+    private void assertOnVersion(Version version, ApplicationId instance, DeploymentTester tester) {
         var vespaVersion = tester.controller().versionStatus().version(version);
         assertNotNull("Statistics for version " + version + " exist", vespaVersion);
-        var statistics = vespaVersion.statistics();
-        assertTrue("Application is on version " + version, statistics.production().contains(instance) ||
-                                                           statistics.failing().contains(instance) ||
-                                                           statistics.deploying().contains(instance));
+        var statistics = DeploymentStatistics.compute(List.of(version), tester.deploymentStatuses()).get(0);
+        assertTrue("Application is on version " + version,
+                   Stream.of(statistics.productionSuccesses(), statistics.failingUpgrades(), statistics.runningUpgrade())
+                         .anyMatch(runs -> runs.stream().anyMatch(run -> run.id().application().equals(instance))));
     }
 
     private static void writeControllerVersion(HostName hostname, Version version, CuratorDb db) {
@@ -571,7 +604,7 @@ public class VersionStatusTest {
 
     private Confidence confidence(Controller controller, Version version) {
         return controller.versionStatus().versions().stream()
-                .filter(v -> v.statistics().version().equals(version))
+                .filter(v -> v.versionNumber().equals(version))
                 .findFirst()
                 .map(VespaVersion::confidence)
                 .orElseThrow(() -> new IllegalArgumentException("Expected to find version: " + version));
diff --git a/controller-server/src/test/resources/testConfig.json b/controller-server/src/test/resources/testConfig.json
index 3b01f7f39ca..48bf4792176 100644
--- a/controller-server/src/test/resources/testConfig.json
+++ b/controller-server/src/test/resources/testConfig.json
@@ -5,12 +5,12 @@
   "isCI": true,
   "endpoints": {
     "test.aws-us-east-1c": [
-      "https://server/"
+      "https://ai--default--default.global.vespa.oath.cloud/"
     ]
   },
   "zoneEndpoints": {
     "test.aws-us-east-1c": {
-      "ai": "https://server/"
+      "ai": "https://ai--default--default.global.vespa.oath.cloud/"
     }
   },
   "clusters": {
diff --git a/controller-server/src/test/resources/test_runner_services.xml-cd b/controller-server/src/test/resources/test_runner_services.xml-cd
index 235ca7cb698..125c5004d25 100644
--- a/controller-server/src/test/resources/test_runner_services.xml-cd
+++ b/controller-server/src/test/resources/test_runner_services.xml-cd
@@ -15,49 +15,6 @@
             <binding>http://*/tester/v1/*</binding>
         </handler>
 
-        <http>
-            <!-- Make sure 4080 is the first port. This will be used by the config server. -->
-            <server id='default' port='4080'/>
-            <server id='testertls4443' port='4443'>
-                <config name="jdisc.http.connector">
-                    <tlsClientAuthEnforcer>
-                        <enable>true</enable>
-                        <pathWhitelist>
-                            <item>/status.html</item>
-                            <item>/state/v1/config</item>
-                        </pathWhitelist>
-                    </tlsClientAuthEnforcer>
-                </config>
-                <ssl>
-                    <private-key-file>/var/lib/sia/keys/vespa.vespa.tenant.key.pem</private-key-file>
-                    <certificate-file>/var/lib/sia/certs/vespa.vespa.tenant.cert.pem</certificate-file>
-                    <ca-certificates-file>/opt/yahoo/share/ssl/certs/athenz_certificate_bundle.pem</ca-certificates-file>
-                    <client-authentication>want</client-authentication>
-                </ssl>
-            </server>
-            <filtering>
-                <access-control domain='vespa.vespa.cd'>
-                    <exclude>
-                        <binding>http://*/tester/v1/*</binding>
-                    </exclude>
-                </access-control>
-                <request-chain id="testrunner-api">
-                    <filter id='authz-filter' class='com.yahoo.jdisc.http.filter.security.athenz.AthenzAuthorizationFilter' bundle="jdisc-security-filters">
-                        <config name="jdisc.http.filter.security.athenz.athenz-authorization-filter">
-                            <credentialsToVerify>TOKEN_ONLY</credentialsToVerify>
-                            <roleTokenHeaderName>Yahoo-Role-Auth</roleTokenHeaderName>
-                        </config>
-                        <component id="com.yahoo.jdisc.http.filter.security.athenz.StaticRequestResourceMapper" bundle="jdisc-security-filters">
-                            <config name="jdisc.http.filter.security.athenz.static-request-resource-mapper">
-                                <resourceName>vespa.vespa.cd:tester-application</resourceName>
-                                <action>deploy</action>
-                            </config>
-                        </component>
-                    </filter>
-                </request-chain>
-            </filtering>
-        </http>
-
         <nodes count="1" allocated-memory="17%">
             <resources vcpu="2.00" memory="12.00Gb" disk="75.00Gb" disk-speed="fast" storage-type="local"/>
         </nodes>
diff --git a/default_build_settings.cmake b/default_build_settings.cmake
index 21622ade9fc..6ed2e06e060 100644
--- a/default_build_settings.cmake
+++ b/default_build_settings.cmake
@@ -4,13 +4,14 @@ include(VespaExtendedDefaultBuildSettings OPTIONAL)
 
 function(setup_vespa_default_build_settings_rhel_6_10)
   message("-- Setting up default build settings for rhel 6.10")
+  set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" "/usr/include/openblas" PARENT_SCOPE)
 endfunction()
 
 function(setup_vespa_default_build_settings_rhel_7_7)
   message("-- Setting up default build settings for rhel 7.7")
-  set(DEFAULT_EXTRA_LINK_DIRECTORY "${VESPA_DEPS}/lib64" "/usr/lib64/llvm5.0/lib" PARENT_SCOPE)
-  set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" "/usr/include/llvm5.0" PARENT_SCOPE)
-  set(DEFAULT_VESPA_LLVM_VERSION "5.0" PARENT_SCOPE)
+  set(DEFAULT_EXTRA_LINK_DIRECTORY "${VESPA_DEPS}/lib64" "/usr/lib64/llvm7.0/lib" PARENT_SCOPE)
+  set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" "/usr/include/llvm7.0" "/usr/include/openblas" PARENT_SCOPE)
+  set(DEFAULT_VESPA_LLVM_VERSION "7" PARENT_SCOPE)
 endfunction()
 
 function(setup_vespa_default_build_settings_rhel_8_1)
@@ -20,14 +21,15 @@ endfunction()
 
 function(setup_vespa_default_build_settings_centos_7)
   message("-- Setting up default build settings for centos 7")
-  set(DEFAULT_EXTRA_LINK_DIRECTORY "${VESPA_DEPS}/lib64" "/usr/lib64/llvm5.0/lib" PARENT_SCOPE)
-  set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" "/usr/include/llvm5.0" PARENT_SCOPE)
-  set(DEFAULT_VESPA_LLVM_VERSION "5.0" PARENT_SCOPE)
+  set(DEFAULT_EXTRA_LINK_DIRECTORY "${VESPA_DEPS}/lib64" "/usr/lib64/llvm7.0/lib" PARENT_SCOPE)
+  set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" "/usr/include/llvm7.0" "/usr/include/openblas" PARENT_SCOPE)
+  set(DEFAULT_VESPA_LLVM_VERSION "7" PARENT_SCOPE)
 endfunction()
 
 function(setup_vespa_default_build_settings_centos_8)
   message("-- Setting up default build settings for centos 8")
-  set(DEFAULT_VESPA_LLVM_VERSION "7" PARENT_SCOPE)
+  set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" "/usr/include/openblas" PARENT_SCOPE)
+  set(DEFAULT_VESPA_LLVM_VERSION "8" PARENT_SCOPE)
 endfunction()
 
 function(setup_vespa_default_build_settings_darwin)
@@ -35,18 +37,18 @@ function(setup_vespa_default_build_settings_darwin)
   if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang" OR "${CMAKE_CXX_COMPILER_ID}" STREQUAL "AppleClang")
     set(DEFAULT_LLVM_INCLUDE_DIRECTORY "/usr/local/opt/llvm/include")
     set(DEFAULT_LLVM_LINK_DIRECTORY "/usr/local/opt/llvm/lib")
-    set(DEFAULT_VESPA_LLVM_VERSION "9" PARENT_SCOPE)
+    set(DEFAULT_VESPA_LLVM_VERSION "10" PARENT_SCOPE)
   else()
-    set(DEFAULT_VESPA_LLVM_VERSION "8" PARENT_SCOPE)
+    set(DEFAULT_VESPA_LLVM_VERSION "10" PARENT_SCOPE)
   endif()
-  set(DEFAULT_CMAKE_PREFIX_PATH "${VESPA_DEPS}" "/usr/local/opt/bison" "/usr/local/opt/flex" "/usr/local/opt/openssl@1.1" PARENT_SCOPE)
-  set(DEFAULT_EXTRA_LINK_DIRECTORY "${VESPA_DEPS}/lib" "/usr/local/opt/bison/lib" "/usr/local/opt/flex/lib" "/usr/local/opt/icu4c/lib" "/usr/local/opt/openssl@1.1/lib")
+  set(DEFAULT_CMAKE_PREFIX_PATH "${VESPA_DEPS}" "/usr/local/opt/bison" "/usr/local/opt/flex" "/usr/local/opt/openssl@1.1" "/usr/local/opt/openblas" PARENT_SCOPE)
+  set(DEFAULT_EXTRA_LINK_DIRECTORY "${VESPA_DEPS}/lib" "/usr/local/opt/bison/lib" "/usr/local/opt/flex/lib" "/usr/local/opt/icu4c/lib" "/usr/local/opt/openssl@1.1/lib" "/usr/local/opt/openblas/lib")
   if(DEFINED DEFAULT_LLVM_LINK_DIRECTORY)
     list(APPEND DEFAULT_EXTRA_LINK_DIRECTORY "${DEFAULT_LLVM_LINK_DIRECTORY}")
   endif()
   list(APPEND DEFAULT_EXTRA_LINK_DIRECTORY "/usr/local/lib")
   set(DEFAULT_EXTRA_LINK_DIRECTORY "${DEFAULT_EXTRA_LINK_DIRECTORY}" PARENT_SCOPE)
-  set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" "/usr/local/opt/flex/include" "/usr/local/opt/icu4c/include" "/usr/local/opt/openssl@1.1/include")
+  set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" "/usr/local/opt/flex/include" "/usr/local/opt/icu4c/include" "/usr/local/opt/openssl@1.1/include" "/usr/local/opt/openblas/include")
   if(DEFINED DEFAULT_LLVM_INCLUDE_DIRECTORY)
     list(APPEND DEFAULT_EXTRA_INCLUDE_DIRECTORY "${DEFAULT_LLVM_INCLUDE_DIRECTORY}")
   endif()
@@ -54,36 +56,76 @@ function(setup_vespa_default_build_settings_darwin)
   set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${DEFAULT_EXTRA_INCLUDE_DIRECTORY}" PARENT_SCOPE)
 endfunction()
 
-function(setup_vespa_default_build_settings_fedora_29)
-  message("-- Setting up default build settings for fedora 29")
-  set(DEFAULT_VESPA_LLVM_VERSION "7" PARENT_SCOPE)
-endfunction()
-
 function(setup_vespa_default_build_settings_fedora_30)
   message("-- Setting up default build settings for fedora 30")
+  set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" "/usr/include/openblas" PARENT_SCOPE)
   set(DEFAULT_VESPA_LLVM_VERSION "8" PARENT_SCOPE)
 endfunction()
 
 function(setup_vespa_default_build_settings_fedora_31)
   message("-- Setting up default build settings for fedora 31")
+  set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" "/usr/include/openblas" PARENT_SCOPE)
   set(DEFAULT_VESPA_LLVM_VERSION "9" PARENT_SCOPE)
 endfunction()
 
 function(setup_vespa_default_build_settings_fedora_32)
   message("-- Setting up default build settings for fedora 32")
+  set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" "/usr/include/openblas" PARENT_SCOPE)
+  set(DEFAULT_VESPA_LLVM_VERSION "10" PARENT_SCOPE)
+endfunction()
+
+function(setup_vespa_default_build_settings_fedora_33)
+  message("-- Setting up default build settings for fedora 33")
+  set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" "/usr/include/openblas" PARENT_SCOPE)
+  set(DEFAULT_VESPA_LLVM_VERSION "10" PARENT_SCOPE)
+endfunction()
+
+function(setup_vespa_default_build_settings_ubuntu_19_10)
+  message("-- Setting up default build settings for ubuntu 19.10")
+  set(DEFAULT_EXTRA_LINK_DIRECTORY "${VESPA_DEPS}/lib" "/usr/lib/llvm-9/lib" PARENT_SCOPE)
+  set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" "/usr/lib/llvm-9/include" PARENT_SCOPE)
   set(DEFAULT_VESPA_LLVM_VERSION "9" PARENT_SCOPE)
 endfunction()
 
-function(setup_vespa_default_build_settings_ubuntu_18_10)
-  message("-- Setting up default build settings for ubuntu 18.10")
-  set(DEFAULT_EXTRA_LINK_DIRECTORY "${VESPA_DEPS}/lib" "/usr/lib/llvm-6.0/lib" PARENT_SCOPE)
-  set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" "/usr/lib/llvm-6.0/include" PARENT_SCOPE)
-  set(DEFAULT_VESPA_LLVM_VERSION "6.0" PARENT_SCOPE)
+function(vespa_use_default_vespa_unprivileged)
+  if(NOT DEFINED VESPA_UNPRIVILEGED)
+    message("-- Setting VESPA_UNPRIVILEGED to yes")
+    set(VESPA_UNPRIVILEGED "yes" PARENT_SCOPE)
+  endif()
+endfunction()
+
+function(vespa_use_default_cmake_install_prefix)
+  if(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT)
+    if(VESPA_UNPRIVILEGED STREQUAL "no")
+      set(DEFAULT_CMAKE_INSTALL_PREFIX "/opt/vespa")
+      if(COMMAND vespa_use_specific_install_prefix)
+        vespa_use_specific_install_prefix()
+      endif()	 
+    else()
+      set(DEFAULT_CMAKE_INSTALL_PREFIX "$ENV{HOME}/vespa")
+    endif()
+    message("-- Setting CMAKE_INSTALL_PREFIX to ${DEFAULT_CMAKE_INSTALL_PREFIX}")
+    set(CMAKE_INSTALL_PREFIX "${DEFAULT_CMAKE_INSTALL_PREFIX}" CACHE PATH "Install prefix for vespa project" FORCE)
+  endif()
+endfunction()
+
+function(vespa_use_default_vespa_user)
+  if(NOT DEFINED VESPA_USER)
+    if(VESPA_UNPRIVILEGED STREQUAL "no")
+      set(DEFAULT_VESPA_USER "vespa")
+      if(COMMAND vespa_use_specific_vespa_user)
+        vespa_use_specific_vespa_user()
+      endif()
+    else()
+      set(DEFAULT_VESPA_USER "$ENV{USER}")
+    endif()
+    message("-- Setting VESPA_USER to ${DEFAULT_VESPA_USER}")
+    set(VESPA_USER "${DEFAULT_VESPA_USER}" PARENT_SCOPE)
+  endif()
 endfunction()
 
 function(vespa_use_default_build_settings)
-  if (DEFINED CMAKE_INSTALL_PREFIX AND DEFINED CMAKE_PREFIX_PATH AND
-      NOT CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT AND
+  if (DEFINED CMAKE_PREFIX_PATH AND
       DEFINED VESPA_LLVM_VERSION AND
       DEFINED EXTRA_INCLUDE_DIRECTORY AND DEFINED EXTRA_LINK_DIRECTORY AND
       DEFINED CMAKE_INSTALL_RPATH AND DEFINED CMAKE_BUILD_RPATH)
@@ -94,26 +136,7 @@ function(vespa_use_default_build_settings)
   unset(DEFAULT_CMAKE_PREFIX_PATH)
   unset(DEFAULT_EXTRA_LINK_DIRECTORY)
   unset(DEFAULT_EXTRA_INCLUDE_DIRECTORY)
-  unset(DEFAULT_VESPA_USER)
   unset(DEFAULT_VESPA_CPU_ARCH_FLAGS)
-  if(NOT DEFINED VESPA_UNPRIVILEGED)
-    message("-- Setting VESPA_UNPRIVILEGED to yes")
-    set(VESPA_UNPRIVILEGED "yes" PARENT_SCOPE)
-    set(VESPA_UNPRIVILEGED "yes")
-  endif()
-  if(VESPA_UNPRIVILEGED STREQUAL "no")
-    set(DEFAULT_CMAKE_INSTALL_PREFIX "/opt/vespa")
-    set(DEFAULT_VESPA_USER "vespa")
-    if(COMMAND vespa_use_specific_install_prefix)
-      vespa_use_specific_install_prefix()
-    endif()
-    if(COMMAND vespa_use_specific_vespa_user)
-      vespa_use_specific_vespa_user()
-    endif()
-  else()
-    set(DEFAULT_CMAKE_INSTALL_PREFIX "$ENV{HOME}/vespa")
-    set(DEFAULT_VESPA_USER "$ENV{USER}")
-  endif()
   if(APPLE)
     if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang")
       set(VESPA_DEPS "/opt/vespa-deps-clang")
@@ -142,16 +165,16 @@ function(vespa_use_default_build_settings)
     setup_vespa_default_build_settings_centos_8()
   elseif(VESPA_OS_DISTRO STREQUAL "darwin")
     setup_vespa_default_build_settings_darwin()
-  elseif(VESPA_OS_DISTRO_COMBINED STREQUAL "fedora 29")
-    setup_vespa_default_build_settings_fedora_29()
   elseif(VESPA_OS_DISTRO_COMBINED STREQUAL "fedora 30")
     setup_vespa_default_build_settings_fedora_30()
   elseif(VESPA_OS_DISTRO_COMBINED STREQUAL "fedora 31")
     setup_vespa_default_build_settings_fedora_31()
   elseif(VESPA_OS_DISTRO_COMBINED STREQUAL "fedora 32")
     setup_vespa_default_build_settings_fedora_32()
-  elseif(VESPA_OS_DISTRO_COMBINED STREQUAL "ubuntu 18.10")
-    setup_vespa_default_build_settings_ubuntu_18_10()
+  elseif(VESPA_OS_DISTRO_COMBINED STREQUAL "fedora 33")
+    setup_vespa_default_build_settings_fedora_33()
+  elseif(VESPA_OS_DISTRO_COMBINED STREQUAL "ubuntu 19.10")
+    setup_vespa_default_build_settings_ubuntu_19_10()
   else()
     message(FATAL_ERROR "-- Unkonwn vespa build platform ${VESPA_OS_DISTRO_COMBINED}")
   endif()
@@ -174,9 +197,6 @@ function(vespa_use_default_build_settings)
       set(DEFAULT_VESPA_CPU_ARCH_FLAGS "-mtune=intel")
     endif()
   endif()
-  if(DEFINED DEFAULT_CMAKE_INSTALL_PREFIX)
-    message("-- DEFAULT_CMAKE_INSTALL_PREFIX is ${DEFAULT_CMAKE_INSTALL_PREFIX}")
-  endif()
   if(DEFINED DEFAULT_CMAKE_PREFIX_PATH)
     message("-- DEFAULT_CMAKE_PREFIX_PATH is ${DEFAULT_CMAKE_PREFIX_PATH}")
   endif()
@@ -189,14 +209,7 @@ function(vespa_use_default_build_settings)
   if(DEFINED DEFAULT_VESPA_LLVM_VERSION)
     message("-- DEFAULT_VESPA_LLVM_VERSION is ${DEFAULT_VESPA_LLVM_VERSION}")
   endif()
-  if(DEFINED DEFAULT_VESPA_USER)
-    message("-- DEFAULT_VESPA_USER is ${DEFAULT_VESPA_USER}")
-  endif()
   message("-- DEFAULT_VESPA_CPU_ARCH_FLAGS is ${DEFAULT_VESPA_CPU_ARCH_FLAGS}")
-  if(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT AND DEFINED DEFAULT_CMAKE_INSTALL_PREFIX)
-    message("-- Setting CMAKE_INSTALL_PREFIX to ${DEFAULT_CMAKE_INSTALL_PREFIX}")
-    set(CMAKE_INSTALL_PREFIX "${DEFAULT_CMAKE_INSTALL_PREFIX}" CACHE PATH "Install prefix for vespa project" FORCE)
-  endif()
   if(NOT DEFINED CMAKE_PREFIX_PATH AND DEFINED DEFAULT_CMAKE_PREFIX_PATH)
     message("-- Setting CMAKE_PREFIX_PATH to ${DEFAULT_CMAKE_PREFIX_PATH}")
     set(CMAKE_PREFIX_PATH "${DEFAULT_CMAKE_PREFIX_PATH}" PARENT_SCOPE)
@@ -237,10 +250,6 @@ function(vespa_use_default_build_settings)
     message("-- Setting VESPA_LLVM_VERSION to ${DEFAULT_VESPA_LLVM_VERSION}")
     set(VESPA_LLVM_VERSION "${DEFAULT_VESPA_LLVM_VERSION}" PARENT_SCOPE)
   endif()
-  if(NOT DEFINED VESPA_USER AND DEFINED DEFAULT_VESPA_USER)
-    message("-- Setting VESPA_USER to ${DEFAULT_VESPA_USER}")
-    set(VESPA_USER "${DEFAULT_VESPA_USER}" PARENT_SCOPE)
-  endif()
   if(NOT DEFINED VESPA_CPU_ARCH_FLAGS AND DEFINED DEFAULT_VESPA_CPU_ARCH_FLAGS)
     message("-- Setting VESPA_CPU_ARCH_FLAGS to ${DEFAULT_VESPA_CPU_ARCH_FLAGS}")
     set(VESPA_CPU_ARCH_FLAGS "${DEFAULT_VESPA_CPU_ARCH_FLAGS}" PARENT_SCOPE)
diff --git a/defaults/src/main/java/com/yahoo/vespa/defaults/Defaults.java b/defaults/src/main/java/com/yahoo/vespa/defaults/Defaults.java
index 6fb6e4f0860..0565b1cff09 100644
--- a/defaults/src/main/java/com/yahoo/vespa/defaults/Defaults.java
+++ b/defaults/src/main/java/com/yahoo/vespa/defaults/Defaults.java
@@ -39,6 +39,7 @@ public class Defaults {
         vespaPortConfigServerHttp = vespaPortConfigServerRpc + 1;
         vespaPortConfigProxyRpc = findConfigProxyPort(vespaPortBase + 90);
     }
+
     static private String findVespaHome(String defHome) {
         Optional<String> vespaHomeEnv = Optional.ofNullable(System.getenv("VESPA_HOME"));
         if ( ! vespaHomeEnv.isPresent() || vespaHomeEnv.get().trim().isEmpty()) {
diff --git a/dist/vespa.spec b/dist/vespa.spec
index 36d7b8334f1..b2f3d1d5df4 100644
--- a/dist/vespa.spec
+++ b/dist/vespa.spec
@@ -6,6 +6,13 @@
 # Force special prefix for Vespa
 %define _prefix /opt/vespa
 %define _vespa_deps_prefix /opt/vespa-deps
+%define _vespa_user vespa
+%define _vespa_group vespa
+%undefine _vespa_user_uid
+%define _create_vespa_group 1
+%define _create_vespa_user 1
+%define _create_vespa_service 1
+%define _defattr_is_vespa_vespa 0
 
 Name:           vespa
 Version:        _VESPA_VERSION_
@@ -23,11 +30,11 @@ BuildRequires: centos-release-scl
 %endif
 %endif
 %if 0%{?el7}
-BuildRequires: devtoolset-8-gcc-c++
-BuildRequires: devtoolset-8-libatomic-devel
-BuildRequires: devtoolset-8-binutils
+BuildRequires: devtoolset-9-gcc-c++
+BuildRequires: devtoolset-9-libatomic-devel
+BuildRequires: devtoolset-9-binutils
 BuildRequires: rh-maven35
-%define _devtoolset_enable /opt/rh/devtoolset-8/enable
+%define _devtoolset_enable /opt/rh/devtoolset-9/enable
 %define _rhmaven35_enable /opt/rh/rh-maven35/enable
 %endif
 %if 0%{?el8}
@@ -42,15 +49,16 @@ BuildRequires: libatomic
 BuildRequires: Judy-devel
 %if 0%{?el7}
 BuildRequires: cmake3
-BuildRequires: llvm5.0-devel
+BuildRequires: llvm7.0-devel
 BuildRequires: vespa-boost-devel >= 1.59.0-6
 BuildRequires: vespa-gtest >= 1.8.1-1
 BuildRequires: vespa-protobuf-devel >= 3.7.0-4
 BuildRequires: vespa-openssl-devel >= 1.1.1c-1
+BuildRequires: vespa-icu-devel >= 65.1.0-1
 %endif
 %if 0%{?el8}
 BuildRequires: cmake >= 3.11.4-3
-BuildRequires: llvm-devel >= 7.0.1-3
+BuildRequires: llvm-devel >= 8.0.1
 BuildRequires: boost-devel >= 1.66
 BuildRequires: openssl-devel
 BuildRequires: vespa-gtest >= 1.8.1-1
@@ -59,38 +67,45 @@ BuildRequires: vespa-protobuf-devel >= 3.7.0-4
 %if 0%{?fedora}
 BuildRequires: cmake >= 3.9.1
 BuildRequires: maven
-BuildRequires: vespa-protobuf-devel >= 3.7.0-4
 BuildRequires: openssl-devel
-%if 0%{?fc29}
-BuildRequires: llvm-devel >= 7.0.0
-BuildRequires: boost-devel >= 1.66
-BuildRequires: gtest-devel
-BuildRequires: gmock-devel
-%endif
 %if 0%{?fc30}
+BuildRequires: vespa-protobuf-devel >= 3.7.0-4
 BuildRequires: llvm-devel >= 8.0.0
 BuildRequires: boost-devel >= 1.69
 BuildRequires: gtest-devel
 BuildRequires: gmock-devel
 %endif
 %if 0%{?fc31}
+BuildRequires: vespa-protobuf-devel >= 3.7.0-4
 BuildRequires: llvm-devel >= 9.0.0
 BuildRequires: boost-devel >= 1.69
 BuildRequires: gtest-devel
 BuildRequires: gmock-devel
 %endif
 %if 0%{?fc32}
-BuildRequires: llvm-devel >= 9.0.0
+BuildRequires: protobuf-devel
+BuildRequires: llvm-devel >= 10.0.0
 BuildRequires: boost-devel >= 1.69
 BuildRequires: gtest-devel
 BuildRequires: gmock-devel
 %endif
+%if 0%{?fc33}
+BuildRequires: protobuf-devel
+BuildRequires: llvm-devel >= 10.0.0
+BuildRequires: boost-devel >= 1.69
+BuildRequires: gtest-devel
+BuildRequires: gmock-devel
 %endif
-BuildRequires: xxhash-devel >= 0.6.5
+%endif
+BuildRequires: xxhash-devel >= 0.7.3
+BuildRequires: openblas-devel
 BuildRequires: lz4-devel
 BuildRequires: libzstd-devel
 BuildRequires: zlib-devel
+BuildRequires: re2-devel
+%if ! 0%{?el7}
 BuildRequires: libicu-devel
+%endif
 BuildRequires: java-11-openjdk-devel
 BuildRequires: rpm-build
 BuildRequires: make
@@ -122,66 +137,169 @@ Requires: perl-URI
 Requires: valgrind
 Requires: Judy
 Requires: xxhash
-Requires: xxhash-libs >= 0.6.5
+Requires: xxhash-libs >= 0.7.3
+%if 0%{?el8}
+Requires: openblas
+%else
+Requires: openblas-serial
+%endif
 Requires: lz4
 Requires: libzstd
 Requires: zlib
+Requires: re2
+%if ! 0%{?el7}
 Requires: libicu
+%endif
 Requires: perf
 Requires: gdb
 Requires: net-tools
 %if 0%{?el7}
-Requires: llvm5.0
+Requires: llvm7.0
 Requires: vespa-openssl >= 1.1.1c-1
+Requires: vespa-icu >= 65.1.0-1
 Requires: vespa-protobuf >= 3.7.0-4
-%define _vespa_llvm_version 5.0
-%define _extra_link_directory /usr/lib64/llvm5.0/lib;%{_vespa_deps_prefix}/lib64
-%define _extra_include_directory /usr/include/llvm5.0;%{_vespa_deps_prefix}/include
+Requires: vespa-telegraf >= 1.1.1-1
+%define _vespa_llvm_version 7
+%define _extra_link_directory /usr/lib64/llvm7.0/lib;%{_vespa_deps_prefix}/lib64
+%define _extra_include_directory /usr/include/llvm7.0;%{_vespa_deps_prefix}/include;/usr/include/openblas
 %endif
 %if 0%{?el8}
-Requires: llvm-libs >= 7.0.1-3
+Requires: llvm-libs >= 8.0.1
 Requires: vespa-protobuf >= 3.7.0-4
 Requires: openssl-libs
-%define _vespa_llvm_version 7.0
+%define _vespa_llvm_version 8
 %define _extra_link_directory %{_vespa_deps_prefix}/lib64
-%define _extra_include_directory %{_vespa_deps_prefix}/include
+%define _extra_include_directory %{_vespa_deps_prefix}/include;/usr/include/openblas
 %endif
 %if 0%{?fedora}
-Requires: vespa-protobuf >= 3.7.0-4
 Requires: openssl-libs
-%if 0%{?fc29}
-Requires: llvm-libs >= 7.0.0
-%define _vespa_llvm_version 7
-%endif
 %if 0%{?fc30}
+Requires: vespa-protobuf >= 3.7.0-4
 Requires: llvm-libs >= 8.0.0
 %define _vespa_llvm_version 8
 %endif
 %if 0%{?fc31}
+Requires: vespa-protobuf >= 3.7.0-4
 Requires: llvm-libs >= 9.0.0
 %define _vespa_llvm_version 9
 %endif
 %if 0%{?fc32}
-Requires: llvm-libs >= 9.0.0
-%define _vespa_llvm_version 9
+Requires: protobuf
+Requires: llvm-libs >= 10.0.0
+%define _vespa_llvm_version 10
+%endif
+%if 0%{?fc33}
+Requires: protobuf
+Requires: llvm-libs >= 10.0.0
+%define _vespa_llvm_version 10
 %endif
 %define _extra_link_directory %{_vespa_deps_prefix}/lib64
-%define _extra_include_directory %{_vespa_deps_prefix}/include
+%define _extra_include_directory %{_vespa_deps_prefix}/include;/usr/include/openblas
 %endif
-Requires: java-11-openjdk
-Requires(pre): shadow-utils
+Requires: %{name}-base = %{version}-%{release}
+Requires: %{name}-base-libs = %{version}-%{release}
+Requires: %{name}-clients = %{version}-%{release}
+Requires: %{name}-config-model-fat = %{version}-%{release}
+Requires: %{name}-jars = %{version}-%{release}
+Requires: %{name}-malloc = %{version}-%{release}
+Requires: %{name}-tools = %{version}-%{release}
 
 # Ugly workaround because vespamalloc/src/vespamalloc/malloc/mmap.cpp uses the private
 # _dl_sym function.
-Provides: libc.so.6(GLIBC_PRIVATE)(64bit)
+%global __requires_exclude ^libc\\.so\\.6\\(GLIBC_PRIVATE\\)\\(64bit\\)$
+
 
 %description
 
 Vespa - The open big data serving engine
 
+%package base
+
+Summary: Vespa - The open big data serving engine - base
+
+Requires: java-11-openjdk-devel
+Requires: perl
+Requires: perl-Getopt-Long
+Requires(pre): shadow-utils
+
+%description base
+
+Vespa - The open big data serving engine - base
+
+%package base-libs
+
+Summary: Vespa - The open big data serving engine - base C++ libs
+
+Requires: xxhash-libs >= 0.7.3
+Requires: lz4
+Requires: libzstd
+%if 0%{?el7}
+Requires: vespa-openssl >= 1.1.1c-1
+%else
+Requires: openssl-libs
+%endif
+
+%description base-libs
+
+Vespa - The open big data serving engine - base C++ libs
+
+%package clients
+
+Summary: Vespa - The open big data serving engine - clients
+
+%description clients
+
+Vespa - The open big data serving engine - clients
+
+%package config-model-fat
+
+Summary: Vespa - The open big data serving engine - config models
+
+%description config-model-fat
+
+Vespa - The open big data serving engine - config models
+
+%package node-admin
+
+Summary: Vespa - The open big data serving engine - node-admin
+
+Requires: %{name}-base = %{version}-%{release}
+Requires: %{name}-jars = %{version}-%{release}
+
+%description node-admin
+
+Vespa - The open big data serving engine - node-admin
+
+%package jars
+
+Summary: Vespa - The open big data serving engine - shared java jar files
+
+%description jars
+
+Vespa - The open big data serving engine - shared java jar files
+
+%package malloc
+
+Summary: Vespa - The open big data serving engine - malloc library
+
+%description malloc
+
+Vespa - The open big data serving engine - malloc library
+
+%package tools
+
+Summary: Vespa - The open big data serving engine - tools
+
+Requires: %{name}-base = %{version}-%{release}
+Requires: %{name}-base-libs = %{version}-%{release}
+
+%description tools
+
+Vespa - The open big data serving engine - tools
+
 %prep
 %if 0%{?installdir:1}
-%setup -D -T
+%setup -c -D -T
 %else
 %setup -q
 %endif
@@ -206,8 +324,9 @@ cmake3 -DCMAKE_INSTALL_PREFIX=%{_prefix} \
        -DCMAKE_PREFIX_PATH=%{_vespa_deps_prefix} \
        -DEXTRA_LINK_DIRECTORY="%{_extra_link_directory}" \
        -DEXTRA_INCLUDE_DIRECTORY="%{_extra_include_directory}" \
-       -DCMAKE_INSTALL_RPATH="%{_prefix}/lib64%{?_extra_link_directory:;%{_extra_link_directory}};/usr/lib/jvm/jre-11-openjdk/lib" \
+       -DCMAKE_INSTALL_RPATH="%{_prefix}/lib64%{?_extra_link_directory:;%{_extra_link_directory}}" \
        %{?_vespa_llvm_version:-DVESPA_LLVM_VERSION="%{_vespa_llvm_version}"} \
+       -DVESPA_USER=%{_vespa_user} \
        -DVESPA_UNPRIVILEGED=no \
        .
 
@@ -223,49 +342,313 @@ cp -r %{installdir} %{buildroot}
 make install DESTDIR=%{buildroot}
 %endif
 
+%if %{_create_vespa_service}
 mkdir -p %{buildroot}/usr/lib/systemd/system
 cp %{buildroot}/%{_prefix}/etc/systemd/system/vespa.service %{buildroot}/usr/lib/systemd/system
 cp %{buildroot}/%{_prefix}/etc/systemd/system/vespa-configserver.service %{buildroot}/usr/lib/systemd/system
+%endif
+
+ln -s /usr/lib/jvm/jre-11-openjdk %{buildroot}/%{_prefix}/jdk
 
 %clean
 rm -rf $RPM_BUILD_ROOT
 
-%pre
-getent group vespa >/dev/null || groupadd -r vespa
-getent passwd vespa >/dev/null || \
-    useradd -r -g vespa --home-dir %{_prefix} --create-home -s /sbin/nologin \
-    -c "Create owner of all Vespa data files" vespa
-# Home dir created with rwx on user only.
-chmod a+rx %{_prefix}
+%pre base
+%if %{_create_vespa_group}
+getent group %{_vespa_group} >/dev/null || groupadd -r %{_vespa_group}
+%endif
+%if %{_create_vespa_user}
+getent passwd %{_vespa_user} >/dev/null || \
+    useradd -r %{?_vespa_user_uid:-u %{_vespa_user_uid}} -g %{_vespa_group} --home-dir %{_prefix} -s /sbin/nologin \
+    -c "Create owner of all Vespa data files" %{_vespa_user}
+%endif
 echo "pathmunge %{_prefix}/bin" > /etc/profile.d/vespa.sh
 echo "export VESPA_HOME=%{_prefix}" >> /etc/profile.d/vespa.sh
 exit 0
 
+%if %{_create_vespa_service}
 %post
 %systemd_post vespa-configserver.service
 %systemd_post vespa.service
+%endif
 
+%if %{_create_vespa_service}
 %preun
 %systemd_preun vespa.service
 %systemd_preun vespa-configserver.service
+%endif
 
+%if %{_create_vespa_service}
 %postun
 %systemd_postun_with_restart vespa.service
 %systemd_postun_with_restart vespa-configserver.service
+%endif
+
+%postun base
 if [ $1 -eq 0 ]; then # this is an uninstallation
     rm -f /etc/profile.d/vespa.sh
-    ! getent passwd vespa >/dev/null || userdel vespa
-    ! getent group vespa >/dev/null || groupdel vespa
+%if %{_create_vespa_user}
+    ! getent passwd %{_vespa_user} >/dev/null || userdel %{_vespa_user}
+%endif
+%if %{_create_vespa_group}
+    ! getent group %{_vespa_group} >/dev/null || groupdel %{_vespa_group}
+%endif
+fi
+# Keep modifications to conf/vespa/default-env.txt across
+# package uninstall + install.
+if test -f %{_prefix}/conf/vespa/default-env.txt.rpmsave
+then
+    if test -f %{_prefix}/conf/vespa/default-env.txt
+    then
+	# Temporarily remove default-env.txt.rpmsave when
+	# default-env.txt exists
+	rm -f %{_prefix}/conf/vespa/default-env.txt.rpmsave
+    else
+	mv %{_prefix}/conf/vespa/default-env.txt.rpmsave %{_prefix}/conf/vespa/default-env.txt
+    fi
 fi
 
 %files
-%defattr(-,vespa,vespa,-)
+%if %{_defattr_is_vespa_vespa}
+%defattr(-,%{_vespa_user},%{_vespa_group},-)
+%endif
 %doc
-%{_prefix}/*
+%dir %{_prefix}
+%{_prefix}/bin
+%exclude %{_prefix}/bin/vespa-destination
+%exclude %{_prefix}/bin/vespa-document-statistics
+%exclude %{_prefix}/bin/vespa-fbench
+%exclude %{_prefix}/bin/vespa-feeder
+%exclude %{_prefix}/bin/vespa-get
+%exclude %{_prefix}/bin/vespa-logfmt
+%exclude %{_prefix}/bin/vespa-query-profile-dump-tool
+%exclude %{_prefix}/bin/vespa-stat
+%exclude %{_prefix}/bin/vespa-security-env
+%exclude %{_prefix}/bin/vespa-summary-benchmark
+%exclude %{_prefix}/bin/vespa-visit
+%exclude %{_prefix}/bin/vespa-visit-target
+%dir %{_prefix}/conf
+%{_prefix}/conf/configserver
+%{_prefix}/conf/configserver-app
+%exclude %{_prefix}/conf/configserver-app/components/config-model-fat.jar
+%exclude %{_prefix}/conf/configserver-app/config-models.xml
+%dir %{_prefix}/conf/logd
+%dir %{_prefix}/conf/vespa
+%dir %attr(-,%{_vespa_user},-) %{_prefix}/conf/zookeeper
+%dir %{_prefix}/etc
+%{_prefix}/etc/systemd
+%{_prefix}/etc/vespa
+%exclude %{_prefix}/etc/vespamalloc.conf
+%{_prefix}/include
+%dir %{_prefix}/lib
+%dir %{_prefix}/lib/jars
+%{_prefix}/lib/jars/application-model-jar-with-dependencies.jar
+%{_prefix}/lib/jars/application-preprocessor-jar-with-dependencies.jar
+%{_prefix}/lib/jars/athenz-identity-provider-service-jar-with-dependencies.jar
+%{_prefix}/lib/jars/clustercontroller-apps-jar-with-dependencies.jar
+%{_prefix}/lib/jars/clustercontroller-apputil-jar-with-dependencies.jar
+%{_prefix}/lib/jars/clustercontroller-core-jar-with-dependencies.jar
+%{_prefix}/lib/jars/clustercontroller-utils-jar-with-dependencies.jar
+%{_prefix}/lib/jars/config-models
+%{_prefix}/lib/jars/config-proxy-jar-with-dependencies.jar
+%{_prefix}/lib/jars/configserver-flags-jar-with-dependencies.jar
+%{_prefix}/lib/jars/configserver-jar-with-dependencies.jar
+%{_prefix}/lib/jars/document.jar
+%{_prefix}/lib/jars/filedistribution-jar-with-dependencies.jar
+%{_prefix}/lib/jars/jdisc_jetty.jar
+%{_prefix}/lib/jars/logserver-jar-with-dependencies.jar
+%{_prefix}/lib/jars/metrics-proxy-jar-with-dependencies.jar
+%{_prefix}/lib/jars/node-repository-jar-with-dependencies.jar
+%{_prefix}/lib/jars/orchestrator-jar-with-dependencies.jar
+%{_prefix}/lib/jars/predicate-search-jar-with-dependencies.jar
+%{_prefix}/lib/jars/searchlib.jar
+%{_prefix}/lib/jars/searchlib-jar-with-dependencies.jar
+%{_prefix}/lib/jars/service-monitor-jar-with-dependencies.jar
+%{_prefix}/lib/jars/vespa_feed_perf-jar-with-dependencies.jar
+%{_prefix}/lib/jars/vespa-testrunner-components.jar
+%{_prefix}/lib/jars/vespa-testrunner-components-jar-with-dependencies.jar
+%{_prefix}/lib/jars/zookeeper-command-line-client-jar-with-dependencies.jar
+%{_prefix}/lib/perl5
+%{_prefix}/lib64
+%exclude %{_prefix}/lib64/libfastos.so
+%exclude %{_prefix}/lib64/libfnet.so
+%exclude %{_prefix}/lib64/libstaging_vespalib.so
+%exclude %{_prefix}/lib64/libvespadefaults.so
+%exclude %{_prefix}/lib64/libvespalib.so
+%exclude %{_prefix}/lib64/libvespalog.so
+%exclude %{_prefix}/lib64/vespa
+%{_prefix}/libexec
+%exclude %{_prefix}/libexec/vespa/common-env.sh
+%exclude %{_prefix}/libexec/vespa/node-admin.sh
+%exclude %{_prefix}/libexec/vespa/standalone-container.sh
+%exclude %{_prefix}/libexec/vespa/vespa-curl-wrapper
+%dir %attr(1777,-,-) %{_prefix}/logs
+%dir %attr(1777,%{_vespa_user},-) %{_prefix}/logs/vespa
+%dir %attr(-,%{_vespa_user},-) %{_prefix}/logs/vespa/configserver
+%dir %attr(-,%{_vespa_user},-) %{_prefix}/logs/vespa/node-admin
+%dir %attr(-,%{_vespa_user},-) %{_prefix}/logs/vespa/search
+%{_prefix}/man
+%{_prefix}/sbin
+%{_prefix}/share
+%dir %attr(1777,-,-) %{_prefix}/tmp
+%dir %attr(1777,%{_vespa_user},-) %{_prefix}/tmp/vespa
+%dir %{_prefix}/var
+%dir %{_prefix}/var/db
+%dir %attr(-,%{_vespa_user},-) %{_prefix}/var/db/vespa
+%dir %attr(-,%{_vespa_user},-) %{_prefix}/var/db/vespa/logcontrol
+%dir %attr(-,%{_vespa_user},-) %{_prefix}/var/zookeeper
 %config(noreplace) %{_prefix}/conf/logd/logd.cfg
-%config(noreplace) %{_prefix}/conf/vespa/default-env.txt
-%config(noreplace) %{_prefix}/etc/vespamalloc.conf
+%if %{_create_vespa_service}
 %attr(644,root,root) /usr/lib/systemd/system/vespa.service
 %attr(644,root,root) /usr/lib/systemd/system/vespa-configserver.service
+%endif
+
+%files base
+%if %{_defattr_is_vespa_vespa}
+%defattr(-,%{_vespa_user},%{_vespa_group},-)
+%endif
+%dir %{_prefix}
+%dir %{_prefix}/bin
+%{_prefix}/bin/vespa-logfmt
+%{_prefix}/bin/vespa-security-env
+%dir %{_prefix}/conf
+%dir %{_prefix}/conf/vespa
+%config(noreplace) %{_prefix}/conf/vespa/default-env.txt
+%{_prefix}/jdk
+%dir %{_prefix}/lib
+%dir %{_prefix}/lib/jars
+%{_prefix}/lib/jars/security-tools-jar-with-dependencies.jar
+%dir %{_prefix}/libexec
+%dir %{_prefix}/libexec/vespa
+%{_prefix}/libexec/vespa/common-env.sh
+%{_prefix}/libexec/vespa/vespa-curl-wrapper
+
+%files base-libs
+%if %{_defattr_is_vespa_vespa}
+%defattr(-,%{_vespa_user},%{_vespa_group},-)
+%endif
+%dir %{_prefix}
+%dir %{_prefix}/lib64
+%{_prefix}/lib64/libfastos.so
+%{_prefix}/lib64/libfnet.so
+%{_prefix}/lib64/libstaging_vespalib.so
+%{_prefix}/lib64/libvespadefaults.so
+%{_prefix}/lib64/libvespalib.so
+%{_prefix}/lib64/libvespalog.so
+
+%files clients
+%if %{_defattr_is_vespa_vespa}
+%defattr(-,%{_vespa_user},%{_vespa_group},-)
+%endif
+%dir %{_prefix}
+%dir %{_prefix}/lib
+%dir %{_prefix}/lib/jars
+%{_prefix}/lib/jars/vespa-http-client-jar-with-dependencies.jar
+
+%files config-model-fat
+%if %{_defattr_is_vespa_vespa}
+%defattr(-,%{_vespa_user},%{_vespa_group},-)
+%endif
+%dir %{_prefix}
+%dir %{_prefix}/conf
+%dir %{_prefix}/conf/configserver-app
+%dir %{_prefix}/conf/configserver-app/components
+%{_prefix}/conf/configserver-app/components/config-model-fat.jar
+%{_prefix}/conf/configserver-app/config-models.xml
+%dir %{_prefix}/lib
+%dir %{_prefix}/lib/jars
+%{_prefix}/lib/jars/config-model-fat.jar
+
+%files node-admin
+%if %{_defattr_is_vespa_vespa}
+%defattr(-,%{_vespa_user},%{_vespa_group},-)
+%endif
+%dir %{_prefix}
+%dir %{_prefix}/conf
+%{_prefix}/conf/node-admin-app
+%dir %{_prefix}/libexec
+%dir %{_prefix}/libexec/vespa
+%{_prefix}/libexec/vespa/node-admin.sh
+
+%files jars
+%if %{_defattr_is_vespa_vespa}
+%defattr(-,%{_vespa_user},%{_vespa_group},-)
+%endif
+%dir %{_prefix}
+%dir %{_prefix}/lib
+%dir %{_prefix}/lib/jars
+%{_prefix}/lib/jars/aopalliance-repackaged-*.jar
+%{_prefix}/lib/jars/bcpkix-jdk15on-*.jar
+%{_prefix}/lib/jars/bcprov-jdk15on-*.jar
+%{_prefix}/lib/jars/component-jar-with-dependencies.jar
+%{_prefix}/lib/jars/config-bundle-jar-with-dependencies.jar
+%{_prefix}/lib/jars/configdefinitions-jar-with-dependencies.jar
+%{_prefix}/lib/jars/config-model-api-jar-with-dependencies.jar
+%{_prefix}/lib/jars/config-model-jar-with-dependencies.jar
+%{_prefix}/lib/jars/config-provisioning-jar-with-dependencies.jar
+%{_prefix}/lib/jars/container-disc-jar-with-dependencies.jar
+%{_prefix}/lib/jars/container-jersey2-jar-with-dependencies.jar
+%{_prefix}/lib/jars/container-search-and-docproc-jar-with-dependencies.jar
+%{_prefix}/lib/jars/container-search-gui-jar-with-dependencies.jar
+%{_prefix}/lib/jars/defaults-jar-with-dependencies.jar
+%{_prefix}/lib/jars/docprocs-jar-with-dependencies.jar
+%{_prefix}/lib/jars/flags-jar-with-dependencies.jar
+%{_prefix}/lib/jars/hk2-*.jar
+%{_prefix}/lib/jars/jackson-*.jar
+%{_prefix}/lib/jars/javassist-*.jar
+%{_prefix}/lib/jars/javax.*.jar
+%{_prefix}/lib/jars/jdisc_core-jar-with-dependencies.jar
+%{_prefix}/lib/jars/jdisc_http_service-jar-with-dependencies.jar
+%{_prefix}/lib/jars/jdisc-security-filters-jar-with-dependencies.jar
+%{_prefix}/lib/jars/jersey-*.jar
+%{_prefix}/lib/jars/jetty-*.jar
+%{_prefix}/lib/jars/mimepull-*.jar
+%{_prefix}/lib/jars/model-evaluation-jar-with-dependencies.jar
+%{_prefix}/lib/jars/model-integration-jar-with-dependencies.jar
+%{_prefix}/lib/jars/osgi-resource-locator-*.jar
+%{_prefix}/lib/jars/security-utils-jar-with-dependencies.jar
+%{_prefix}/lib/jars/simplemetrics-jar-with-dependencies.jar
+%{_prefix}/lib/jars/standalone-container-jar-with-dependencies.jar
+%{_prefix}/lib/jars/validation-api-*.jar
+%{_prefix}/lib/jars/vespa-athenz-jar-with-dependencies.jar
+%{_prefix}/lib/jars/vespaclient-container-plugin-jar-with-dependencies.jar
+%{_prefix}/lib/jars/vespajlib.jar
+%{_prefix}/lib/jars/zkfacade-jar-with-dependencies.jar
+%{_prefix}/lib/jars/zookeeper-server-*-jar-with-dependencies.jar
+%{_prefix}/lib/jars/zookeeper-server-common-jar-with-dependencies.jar
+%{_prefix}/lib/jars/zookeeper-server-jar-with-dependencies.jar
+%dir %{_prefix}/libexec
+%dir %{_prefix}/libexec/vespa
+%{_prefix}/libexec/vespa/standalone-container.sh
+
+%files malloc
+%if %{_defattr_is_vespa_vespa}
+%defattr(-,%{_vespa_user},%{_vespa_group},-)
+%endif
+%dir %{_prefix}
+%dir %{_prefix}/etc
+%config(noreplace) %{_prefix}/etc/vespamalloc.conf
+%dir %{_prefix}/lib64
+%{_prefix}/lib64/vespa
+
+%files tools
+%if %{_defattr_is_vespa_vespa}
+%defattr(-,%{_vespa_user},%{_vespa_group},-)
+%endif
+%dir %{_prefix}
+%dir %{_prefix}/bin
+%{_prefix}/bin/vespa-destination
+%{_prefix}/bin/vespa-document-statistics
+%{_prefix}/bin/vespa-fbench
+%{_prefix}/bin/vespa-feeder
+%{_prefix}/bin/vespa-get
+%{_prefix}/bin/vespa-query-profile-dump-tool
+%{_prefix}/bin/vespa-stat
+%{_prefix}/bin/vespa-summary-benchmark
+%{_prefix}/bin/vespa-visit
+%{_prefix}/bin/vespa-visit-target
+%dir %{_prefix}/lib
+%dir %{_prefix}/lib/jars
+%{_prefix}/lib/jars/vespaclient-java-jar-with-dependencies.jar
 
 %changelog
diff --git a/docker-api/CMakeLists.txt b/docker-api/CMakeLists.txt
index 1288c624890..edcdcfb5bfc 100644
--- a/docker-api/CMakeLists.txt
+++ b/docker-api/CMakeLists.txt
@@ -1,4 +1,2 @@
 # Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-install_fat_java_artifact(docker-api)
-install(DIRECTORY DESTINATION conf/node-admin-app/components)
-install_symlink(lib/jars/docker-api-jar-with-dependencies.jar conf/node-admin-app/components/docker-api.jar)
+install(FILES target/docker-api-jar-with-dependencies.jar DESTINATION conf/node-admin-app/components)
diff --git a/docker-api/pom.xml b/docker-api/pom.xml
index 2257437adfa..749eca97c53 100644
--- a/docker-api/pom.xml
+++ b/docker-api/pom.xml
@@ -94,11 +94,6 @@
             </exclusions>
         </dependency>
         <dependency>
-            <groupId>net.jpountz.lz4</groupId>
-            <artifactId>lz4</artifactId>
-            <scope>compile</scope>
-        </dependency>
-        <dependency>
             <groupId>org.apache.httpcomponents</groupId>
             <artifactId>httpcore</artifactId>
             <!-- We explicitly specify the version of httpcore to be used by
diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/ContainerResources.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/ContainerResources.java
index ffd5dffece9..c7b9d342043 100644
--- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/ContainerResources.java
+++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/ContainerResources.java
@@ -124,4 +124,8 @@ public class ContainerResources {
     public ContainerResources withMemoryBytes(long memoryBytes) {
         return new ContainerResources(cpus, cpuShares, memoryBytes);
     }
+
+    public ContainerResources withUnlimitedCpus() {
+        return new ContainerResources(0, 0, memoryBytes);
+    }
 }
diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/CreateContainerCommandImpl.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/CreateContainerCommandImpl.java
index 32302a98757..30d79c57296 100644
--- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/CreateContainerCommandImpl.java
+++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/CreateContainerCommandImpl.java
@@ -34,6 +34,7 @@ class CreateContainerCommandImpl implements Docker.CreateContainerCommand {
     private final Map<String, String> labels = new HashMap<>();
     private final List<String> environmentAssignments = new ArrayList<>();
     private final List<String> volumeBindSpecs = new ArrayList<>();
+    private final List<String> dnsOptions = new ArrayList<>();
     private final List<Ulimit> ulimits = new ArrayList<>();
     private final Set<Capability> addCapabilities = new HashSet<>();
     private final Set<Capability> dropCapabilities = new HashSet<>();
@@ -90,12 +91,18 @@ class CreateContainerCommandImpl implements Docker.CreateContainerCommand {
     }
 
     @Override
-    public Docker.CreateContainerCommand withSecurityOpts(String securityOpt) {
+    public Docker.CreateContainerCommand withSecurityOpt(String securityOpt) {
         securityOpts.add(securityOpt);
         return this;
     }
 
     @Override
+    public Docker.CreateContainerCommand withDnsOption(String dnsOption) {
+        dnsOptions.add(dnsOption);
+        return this;
+    }
+
+    @Override
     public Docker.CreateContainerCommand withPrivileged(boolean privileged) {
         this.privileged = privileged;
         return this;
@@ -171,6 +178,7 @@ class CreateContainerCommandImpl implements Docker.CreateContainerCommand {
                 .withPidsLimit(-1L)
                 .withCapAdd(addCapabilities.toArray(new Capability[0]))
                 .withCapDrop(dropCapabilities.toArray(new Capability[0]))
+                .withDnsOptions(dnsOptions)
                 .withPrivileged(privileged);
 
         containerResources.ifPresent(cr -> hostConfig
@@ -241,6 +249,7 @@ class CreateContainerCommandImpl implements Docker.CreateContainerCommand {
                 toRepeatedOption("--cap-add", addCapabilitiesList),
                 toRepeatedOption("--cap-drop", dropCapabilitiesList),
                 toRepeatedOption("--security-opt", securityOpts),
+                toRepeatedOption("--dns-option", dnsOptions),
                 toOptionalOption("--net", networkMode),
                 toOptionalOption("--ip", ipv4Address),
                 toOptionalOption("--ip6", ipv6Address),
diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/Docker.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/Docker.java
index 4e7ef5a1ff6..477d0edf8be 100644
--- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/Docker.java
+++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/Docker.java
@@ -52,7 +52,8 @@ public interface Docker {
         CreateContainerCommand withManagedBy(String manager);
         CreateContainerCommand withAddCapability(String capabilityName);
         CreateContainerCommand withDropCapability(String capabilityName);
-        CreateContainerCommand withSecurityOpts(String securityOpt);
+        CreateContainerCommand withSecurityOpt(String securityOpt);
+        CreateContainerCommand withDnsOption(String dnsOption);
         CreateContainerCommand withPrivileged(boolean privileged);
 
         void create();
diff --git a/docker/enter-build-container.sh b/docker/enter-build-container.sh
index 45188cb4df3..ff7a30ecc8f 100755
--- a/docker/enter-build-container.sh
+++ b/docker/enter-build-container.sh
@@ -10,7 +10,7 @@ fi
 DIR=$(cd $(dirname "${BASH_SOURCE[0]}") && pwd)
 cd $DIR
 
-DOCKER_IMAGE="vespaengine/vespa-dev:latest"
+DOCKER_IMAGE="vespaengine/vespa-build-centos7:latest"
 
 docker pull ${DOCKER_IMAGE}
 docker run -ti --rm -v $(pwd)/..:/vespa --entrypoint /vespa/docker/build/enter-build-container-internal.sh "$DOCKER_IMAGE"
diff --git a/docker/vespa-ci.sh b/docker/vespa-ci.sh
index f4c46f8919a..9ac12f155cc 100755
--- a/docker/vespa-ci.sh
+++ b/docker/vespa-ci.sh
@@ -12,7 +12,7 @@ DIR=$(cd "${RELATIVE_DIR}" && pwd)
 cd "${DIR}"
 
 GIT_COMMIT=$1
-DOCKER_IMAGE="vespaengine/vespa-dev:latest"
+DOCKER_IMAGE="vespaengine/vespa-build-centos7:latest"
 INTERNAL_DIR=/vespa
 LOG_PREFIX=vespa-ci-$(date +%Y-%m-%dT%H:%M:%S)
 
diff --git a/docproc/src/main/java/com/yahoo/docproc/jdisc/DocprocThreadManager.java b/docproc/src/main/java/com/yahoo/docproc/jdisc/DocprocThreadManager.java
deleted file mode 100644
index 6fd4beac056..00000000000
--- a/docproc/src/main/java/com/yahoo/docproc/jdisc/DocprocThreadManager.java
+++ /dev/null
@@ -1,50 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.docproc.jdisc;
-
-import com.yahoo.document.DocumentUtil;
-import com.yahoo.log.LogLevel;
-
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.logging.Logger;
-
-/**
- * @author Einar M R Rosenvinge
- */
-class DocprocThreadManager {
-
-    private static Logger log = Logger.getLogger(DocprocThreadManager.class.getName());
-
-    private final long maxConcurrentByteSize;
-    private final AtomicLong bytesStarted = new AtomicLong(0);
-    private final AtomicLong bytesFinished = new AtomicLong(0);
-
-    DocprocThreadManager(double maxConcurrentFactor, double documentExpansionFactor, int containerCoreMemoryMb) {
-        this((long) (((double) DocumentUtil.calculateMaxPendingSize(maxConcurrentFactor, documentExpansionFactor,
-                                                      containerCoreMemoryMb)) * maxConcurrentFactor));
-    }
-
-    DocprocThreadManager(long maxConcurrentByteSize) {
-        final int MINCONCURRENTBYTES=256*1024*1024;    //256M
-        if (maxConcurrentByteSize < MINCONCURRENTBYTES) {
-            maxConcurrentByteSize = MINCONCURRENTBYTES;
-        }
-
-        this.maxConcurrentByteSize = maxConcurrentByteSize;
-        log.log(LogLevel.CONFIG, "Docproc service allowed to concurrently process "
-                               + (((double) maxConcurrentByteSize) / 1024.0d / 1024.0d) + " megabytes of input data.");
-    }
-
-    boolean isAboveLimit() {
-        return (bytesFinished.get() - bytesStarted.get() > maxConcurrentByteSize);
-    }
-    void beforeExecute(DocumentProcessingTask task) {
-        bytesStarted.getAndAdd(task.getApproxSize());
-    }
-
-    void afterExecute(DocumentProcessingTask task) {
-        bytesFinished.getAndAdd(task.getApproxSize());
-    }
-    void shutdown() {
-    }
-
-}
diff --git a/docproc/src/main/java/com/yahoo/docproc/jdisc/DocprocThreadPoolExecutor.java b/docproc/src/main/java/com/yahoo/docproc/jdisc/DocprocThreadPoolExecutor.java
deleted file mode 100644
index e1a902c8d5c..00000000000
--- a/docproc/src/main/java/com/yahoo/docproc/jdisc/DocprocThreadPoolExecutor.java
+++ /dev/null
@@ -1,59 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.docproc.jdisc;
-
-import com.yahoo.concurrent.DaemonThreadFactory;
-import com.yahoo.log.LogLevel;
-
-import java.util.List;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-import java.util.logging.Logger;
-
-/**
- * @author Einar M R Rosenvinge
- */
-public class DocprocThreadPoolExecutor extends ThreadPoolExecutor {
-
-    private static Logger log = Logger.getLogger(DocprocThreadPoolExecutor.class.getName());
-    private DocprocThreadManager threadManager;
-
-    public DocprocThreadPoolExecutor(int maxNumThreads, BlockingQueue<Runnable> queue, DocprocThreadManager threadMgr) {
-        super((maxNumThreads > 0) ? maxNumThreads : Runtime.getRuntime().availableProcessors(),
-              (maxNumThreads > 0) ? maxNumThreads : 2048,
-              1, TimeUnit.SECONDS,
-              queue,
-              new DaemonThreadFactory("docproc-"));
-        this.threadManager = threadMgr;
-        allowCoreThreadTimeOut(false);
-        log.log(LogLevel.DEBUG, "Created docproc thread pool with " + super.getCorePoolSize() + " worker threads.");
-    }
-
-    @Override
-    protected void beforeExecute(Thread thread, Runnable runnable) {
-        threadManager.beforeExecute((DocumentProcessingTask) runnable);
-    }
-
-    @Override
-    protected void afterExecute(Runnable runnable, Throwable throwable) {
-        threadManager.afterExecute((DocumentProcessingTask) runnable);
-    }
-
-    @Override
-    public void shutdown() {
-        super.shutdown();
-        threadManager.shutdown();
-    }
-
-    @Override
-    public List<Runnable> shutdownNow() {
-        List<Runnable> list =  super.shutdownNow();
-        threadManager.shutdown();
-        return list;
-    }
-
-    boolean isAboveLimit() {
-        return threadManager.isAboveLimit();
-    }
-
-}
diff --git a/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingHandler.java b/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingHandler.java
index 5b7b9d85a91..d34c300b8d0 100644
--- a/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingHandler.java
+++ b/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingHandler.java
@@ -35,6 +35,7 @@ import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.RejectedExecutionException;
 import java.util.concurrent.ScheduledThreadPoolExecutor;
 import java.util.concurrent.SynchronousQueue;
+import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 import java.util.logging.Logger;
 
@@ -52,7 +53,7 @@ public class DocumentProcessingHandler extends AbstractRequestHandler {
     private final ComponentRegistry<DocprocService> docprocServiceRegistry;
     private final ComponentRegistry<AbstractConcreteDocumentFactory> docFactoryRegistry;
     private final ChainRegistry<DocumentProcessor> chainRegistry = new ChainRegistry<>();
-    private DocprocThreadPoolExecutor threadPool;
+    private ThreadPoolExecutor threadPool;
     private final ScheduledThreadPoolExecutor laterExecutor =
             new ScheduledThreadPoolExecutor(2, new DaemonThreadFactory("docproc-later-"));
     private ContainerDocumentConfig containerDocConfig;
@@ -61,7 +62,7 @@ public class DocumentProcessingHandler extends AbstractRequestHandler {
     public DocumentProcessingHandler(ComponentRegistry<DocprocService> docprocServiceRegistry,
                                      ComponentRegistry<DocumentProcessor> documentProcessorComponentRegistry,
                                      ComponentRegistry<AbstractConcreteDocumentFactory> docFactoryRegistry,
-                                     DocprocThreadPoolExecutor threadPool, DocumentTypeManager documentTypeManager,
+                                     ThreadPoolExecutor threadPool, DocumentTypeManager documentTypeManager,
                                      ChainsModel chainsModel, SchemaMap schemaMap, Statistics statistics,
                                      Metric metric,
                                      ContainerDocumentConfig containerDocConfig) {
@@ -88,28 +89,27 @@ public class DocumentProcessingHandler extends AbstractRequestHandler {
         }
     }
 
+    private static int computeNumThreads(int maxThreads) {
+        return (maxThreads > 0) ? maxThreads : Runtime.getRuntime().availableProcessors();
+    }
+
     public DocumentProcessingHandler(ComponentRegistry<DocprocService> docprocServiceRegistry,
                                      ComponentRegistry<DocumentProcessor> documentProcessorComponentRegistry,
                                      ComponentRegistry<AbstractConcreteDocumentFactory> docFactoryRegistry,
                                      DocumentProcessingHandlerParameters params) {
         this(docprocServiceRegistry, documentProcessorComponentRegistry, docFactoryRegistry,
-             new DocprocThreadPoolExecutor(params.getMaxNumThreads(),
-                                           chooseQueueType(params.getMaxNumThreads()),
-                                           new DocprocThreadManager(params.getMaxConcurrentFactor(),
-                                                                    params.getDocumentExpansionFactor(),
-                                                                    params.getContainerCoreMemoryMb())),
+             new ThreadPoolExecutor(computeNumThreads(params.getMaxNumThreads()),
+                                    computeNumThreads(params.getMaxNumThreads()),
+                                   0,TimeUnit.SECONDS,
+                                    new LinkedBlockingQueue<>(),
+                                    new DaemonThreadFactory("docproc-")
+                                           ),
              params.getDocumentTypeManager(), params.getChainsModel(), params.getSchemaMap(),
              params.getStatisticsManager(),
              params.getMetric(),
              params.getContainerDocConfig());
     }
 
-    private static BlockingQueue<Runnable> chooseQueueType(int maxNumThreads) {
-        return (maxNumThreads > 0)
-                ? new LinkedBlockingQueue<>()
-                : new SynchronousQueue<>();
-    }
-
     @Inject
     public DocumentProcessingHandler(ComponentRegistry<DocumentProcessor> documentProcessorComponentRegistry,
                                      ComponentRegistry<AbstractConcreteDocumentFactory> docFactoryRegistry,
@@ -124,9 +124,6 @@ public class DocumentProcessingHandler extends AbstractRequestHandler {
         this(new ComponentRegistry<>(),
              documentProcessorComponentRegistry, docFactoryRegistry, new DocumentProcessingHandlerParameters().setMaxNumThreads
                 (docprocConfig.numthreads())
-                     .setMaxConcurrentFactor(containerMbusConfig.maxConcurrentFactor())
-                     .setDocumentExpansionFactor(containerMbusConfig.documentExpansionFactor())
-                     .setContainerCoreMemoryMb(containerMbusConfig.containerCoreMemory())
                      .setDocumentTypeManager(new DocumentTypeManager(docManConfig))
                      .setChainsModel(buildFromConfig(chainsConfig)).setSchemaMap(configureMapping(mappingConfig))
                      .setStatisticsManager(manager)
@@ -198,14 +195,10 @@ public class DocumentProcessingHandler extends AbstractRequestHandler {
     }
 
     private void submit(DocumentProcessingTask task) {
-        if (threadPool.isAboveLimit()) {
+        try {
+            threadPool.execute(task);
+        } catch (RejectedExecutionException ree) {
             task.queueFull();
-        } else {
-            try {
-                threadPool.execute(task);
-            } catch (RejectedExecutionException ree) {
-                task.queueFull();
-            }
         }
     }
 
diff --git a/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingHandlerParameters.java b/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingHandlerParameters.java
index bf308e39218..b8a6aa9c105 100644
--- a/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingHandlerParameters.java
+++ b/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingHandlerParameters.java
@@ -18,9 +18,6 @@ import com.yahoo.statistics.Statistics;
 public class DocumentProcessingHandlerParameters {
 
     private int maxNumThreads = 0;
-    private double maxConcurrentFactor = 0.2;
-    private double documentExpansionFactor = 20.0;
-    private int containerCoreMemoryMb = 50;
     private DocumentTypeManager documentTypeManager = null;
     private ChainsModel chainsModel = null;
     private SchemaMap schemaMap = null;
@@ -28,22 +25,7 @@ public class DocumentProcessingHandlerParameters {
     private Metric metric = new NullMetric();
     private ContainerDocumentConfig containerDocConfig;
 
-    public DocumentProcessingHandlerParameters() {
-    }
-
-    /**
-     * Returns the number of megabytes of memory reserved for container core classes and data.
-     *
-     * @return the number of megabytes of memory reserved for container core classes and data.
-     */
-    public int getContainerCoreMemoryMb() {
-        return containerCoreMemoryMb;
-    }
 
-    public DocumentProcessingHandlerParameters setContainerCoreMemoryMb(int containerCoreMemoryMb) {
-        this.containerCoreMemoryMb = containerCoreMemoryMb;
-        return this;
-    }
 
     public Metric getMetric() {
         return metric;
@@ -55,36 +37,6 @@ public class DocumentProcessingHandlerParameters {
     }
 
     /**
-     * Returns the document expansion factor, i.e.&nbsp;by what factor a serialized and possibly compressed
-     * input document is expected to expand during deserialization, including any temporary memory needed
-     * when processing it.
-     *
-     * @return the document expansion factor.
-     */
-    public double getDocumentExpansionFactor() {
-        return documentExpansionFactor;
-    }
-
-    public DocumentProcessingHandlerParameters setDocumentExpansionFactor(double documentExpansionFactor) {
-        this.documentExpansionFactor = documentExpansionFactor;
-        return this;
-    }
-
-    /**
-     * Returns the max concurrent factor.
-     *
-     * @return the max concurrent factor.
-     */
-    public double getMaxConcurrentFactor() {
-        return maxConcurrentFactor;
-    }
-
-    public DocumentProcessingHandlerParameters setMaxConcurrentFactor(double maxConcurrentFactor) {
-        this.maxConcurrentFactor = maxConcurrentFactor;
-        return this;
-    }
-
-    /**
      * Returns the maximum number of thread that the thread pool will ever attempt to run simultaneously.
      *
      * @return the maximum number of thread that the thread pool will ever attempt to run simultaneously.
diff --git a/docproc/src/test/java/com/yahoo/docproc/ProcessingUpdateTestCase.java b/docproc/src/test/java/com/yahoo/docproc/ProcessingUpdateTestCase.java
index 6868bc8ecd4..4160f366fdb 100644
--- a/docproc/src/test/java/com/yahoo/docproc/ProcessingUpdateTestCase.java
+++ b/docproc/src/test/java/com/yahoo/docproc/ProcessingUpdateTestCase.java
@@ -37,8 +37,8 @@ public class ProcessingUpdateTestCase {
     @Test
     public void testProcessingUpdates() {
         DocumentType articleType = new DocumentType("article");
-        Field bodyField = new Field("body", DataType.STRING, true);
-        Field titleField = new Field("title", DataType.STRING, true);
+        Field bodyField = new Field("body", DataType.STRING);
+        Field titleField = new Field("title", DataType.STRING);
         articleType.addField(bodyField);
         articleType.addField(titleField);
         dtm = new DocumentTypeManager();
diff --git a/docproc/src/test/java/com/yahoo/docproc/jdisc/DocprocThreadPoolExecutorTestCase.java b/docproc/src/test/java/com/yahoo/docproc/jdisc/DocprocThreadPoolExecutorTestCase.java
deleted file mode 100644
index a74fb9a3edf..00000000000
--- a/docproc/src/test/java/com/yahoo/docproc/jdisc/DocprocThreadPoolExecutorTestCase.java
+++ /dev/null
@@ -1,83 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.docproc.jdisc;
-
-import org.junit.Test;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.TimeUnit;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-/**
- * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
- */
-public class DocprocThreadPoolExecutorTestCase {
-    private final Set<Long> threadIds = Collections.synchronizedSet(new HashSet<Long>());
-
-    @Test
-    public void threadPool() throws InterruptedException {
-        int numThreads = 8;
-        int numTasks = 200;
-
-        LinkedBlockingQueue<Runnable> q = new LinkedBlockingQueue<>();
-        DocprocThreadManager mgr = new DocprocThreadManager(1000l);
-        DocprocThreadPoolExecutor pool = new DocprocThreadPoolExecutor(numThreads, q, mgr);
-
-        List<MockedDocumentProcessingTask> tasks = new ArrayList<>(numTasks);
-        for (int i = 0; i < numTasks; i++) {
-            tasks.add(new MockedDocumentProcessingTask());
-        }
-
-        for (int i = 0; i < numTasks; i++) {
-            pool.execute(tasks.get(i));
-        }
-        pool.shutdown();
-        pool.awaitTermination(120L, TimeUnit.SECONDS);
-
-        for (int i = 0; i < numTasks; i++) {
-            assertTrue(tasks.get(i).hasBeenRun());
-        }
-
-        System.err.println(threadIds);
-        assertEquals(numThreads, threadIds.size());
-    }
-
-    private class MockedDocumentProcessingTask extends DocumentProcessingTask {
-        private boolean hasBeenRun = false;
-
-        public MockedDocumentProcessingTask() {
-            super(null, null, null);
-        }
-
-        @Override
-        public void run() {
-            threadIds.add(Thread.currentThread().getId());
-            System.err.println(System.currentTimeMillis() + "   MOCK Thread " + Thread.currentThread().getId() + " running task " + this);
-            for (int i = 0; i < 100000; i++) {
-                Math.sin((double) (System.currentTimeMillis() / 10000L));
-            }
-            System.err.println(System.currentTimeMillis() + "   MOCK Thread " + Thread.currentThread().getId() + " DONE task " + this);
-            hasBeenRun = true;
-        }
-
-        @Override
-        public int getApproxSize() {
-            return 333;
-        }
-
-        @Override
-        public String toString() {
-            return "seqNum " + getSeqNum();
-        }
-
-        public boolean hasBeenRun() {
-            return hasBeenRun;
-        }
-    }
-}
diff --git a/document/abi-spec.json b/document/abi-spec.json
index 6ce2543b4c2..3764015b917 100644
--- a/document/abi-spec.json
+++ b/document/abi-spec.json
@@ -427,6 +427,8 @@
     "methods": [
       "public void <init>(java.lang.String)",
       "public void <init>(java.lang.String, com.yahoo.document.StructDataType, com.yahoo.document.StructDataType)",
+      "public void <init>(java.lang.String, com.yahoo.document.StructDataType, com.yahoo.document.StructDataType, java.util.Set)",
+      "public void <init>(java.lang.String, java.util.Set)",
       "public com.yahoo.document.DocumentType clone()",
       "public com.yahoo.document.Document createFieldValue()",
       "public java.lang.Class getValueClass()",
@@ -448,6 +450,8 @@
       "public com.yahoo.document.Field getField(int)",
       "public boolean hasField(java.lang.String)",
       "public int getFieldCount()",
+      "public java.util.Set getImportedFieldNames()",
+      "public boolean hasImportedField(java.lang.String)",
       "public com.yahoo.document.Field removeField(java.lang.String)",
       "public java.util.Collection getFields()",
       "public java.util.Set fieldSet()",
@@ -712,6 +716,8 @@
       "public com.yahoo.document.DocumenttypesConfig$Documenttype$Builder fieldsets(java.util.Map)",
       "public com.yahoo.document.DocumenttypesConfig$Documenttype$Builder referencetype(com.yahoo.document.DocumenttypesConfig$Documenttype$Referencetype$Builder)",
       "public com.yahoo.document.DocumenttypesConfig$Documenttype$Builder referencetype(java.util.List)",
+      "public com.yahoo.document.DocumenttypesConfig$Documenttype$Builder importedfield(com.yahoo.document.DocumenttypesConfig$Documenttype$Importedfield$Builder)",
+      "public com.yahoo.document.DocumenttypesConfig$Documenttype$Builder importedfield(java.util.List)",
       "public com.yahoo.document.DocumenttypesConfig$Documenttype build()"
     ],
     "fields": [
@@ -719,7 +725,8 @@
       "public java.util.List datatype",
       "public java.util.List annotationtype",
       "public java.util.Map fieldsets",
-      "public java.util.List referencetype"
+      "public java.util.List referencetype",
+      "public java.util.List importedfield"
     ]
   },
   "com.yahoo.document.DocumenttypesConfig$Documenttype$Datatype$Annotationref$Annotation$Builder": {
@@ -1264,6 +1271,35 @@
     ],
     "fields": []
   },
+  "com.yahoo.document.DocumenttypesConfig$Documenttype$Importedfield$Builder": {
+    "superClass": "java.lang.Object",
+    "interfaces": [
+      "com.yahoo.config.ConfigBuilder"
+    ],
+    "attributes": [
+      "public"
+    ],
+    "methods": [
+      "public void <init>()",
+      "public void <init>(com.yahoo.document.DocumenttypesConfig$Documenttype$Importedfield)",
+      "public com.yahoo.document.DocumenttypesConfig$Documenttype$Importedfield$Builder name(java.lang.String)",
+      "public com.yahoo.document.DocumenttypesConfig$Documenttype$Importedfield build()"
+    ],
+    "fields": []
+  },
+  "com.yahoo.document.DocumenttypesConfig$Documenttype$Importedfield": {
+    "superClass": "com.yahoo.config.InnerNode",
+    "interfaces": [],
+    "attributes": [
+      "public",
+      "final"
+    ],
+    "methods": [
+      "public void <init>(com.yahoo.document.DocumenttypesConfig$Documenttype$Importedfield$Builder)",
+      "public java.lang.String name()"
+    ],
+    "fields": []
+  },
   "com.yahoo.document.DocumenttypesConfig$Documenttype$Inherits$Builder": {
     "superClass": "java.lang.Object",
     "interfaces": [
@@ -1347,7 +1383,9 @@
       "public java.util.Map fieldsets()",
       "public com.yahoo.document.DocumenttypesConfig$Documenttype$Fieldsets fieldsets(java.lang.String)",
       "public java.util.List referencetype()",
-      "public com.yahoo.document.DocumenttypesConfig$Documenttype$Referencetype referencetype(int)"
+      "public com.yahoo.document.DocumenttypesConfig$Documenttype$Referencetype referencetype(int)",
+      "public java.util.List importedfield()",
+      "public com.yahoo.document.DocumenttypesConfig$Documenttype$Importedfield importedfield(int)"
     ],
     "fields": []
   },
@@ -1457,8 +1495,10 @@
     ],
     "methods": [
       "public void <init>(java.lang.String, int, com.yahoo.document.DataType, boolean)",
+      "public void <init>(java.lang.String, int, com.yahoo.document.DataType)",
       "public void <init>(java.lang.String)",
       "public void <init>(java.lang.String, com.yahoo.document.DataType, boolean, com.yahoo.document.DocumentType)",
+      "public void <init>(java.lang.String, com.yahoo.document.DataType, com.yahoo.document.DocumentType)",
       "public void <init>(java.lang.String, com.yahoo.document.DataType, boolean)",
       "public void <init>(java.lang.String, com.yahoo.document.DataType)",
       "public void <init>(java.lang.String, com.yahoo.document.Field)",
@@ -1887,7 +1927,8 @@
       "public void <init>(java.lang.String)",
       "public java.lang.String getSelection()",
       "public boolean isPresent()",
-      "public static com.yahoo.document.TestAndSetCondition fromConditionString(java.util.Optional)"
+      "public static com.yahoo.document.TestAndSetCondition fromConditionString(java.util.Optional)",
+      "public java.lang.String toString()"
     ],
     "fields": [
       "public static final com.yahoo.document.TestAndSetCondition NOT_PRESENT_CONDITION"
@@ -3083,6 +3124,7 @@
       "public final com.yahoo.document.datatypes.FieldValue setFieldValue(java.lang.String, java.lang.Integer)",
       "public final com.yahoo.document.datatypes.FieldValue setFieldValue(java.lang.String, java.lang.Long)",
       "public final com.yahoo.document.datatypes.FieldValue setFieldValue(java.lang.String, java.lang.Byte)",
+      "public final com.yahoo.document.datatypes.FieldValue setFieldValue(java.lang.String, java.lang.Boolean)",
       "public abstract com.yahoo.document.datatypes.FieldValue removeFieldValue(com.yahoo.document.Field)",
       "public com.yahoo.document.datatypes.FieldValue removeFieldValue(java.lang.String)",
       "public abstract void clear()",
diff --git a/document/pom.xml b/document/pom.xml
index f5c5dec5b7c..acc24aca823 100644
--- a/document/pom.xml
+++ b/document/pom.xml
@@ -32,10 +32,6 @@
             <scope>test</scope>
         </dependency>
         <dependency>
-            <groupId>net.jpountz.lz4</groupId>
-            <artifactId>lz4</artifactId>
-        </dependency>
-        <dependency>
             <groupId>org.mockito</groupId>
             <artifactId>mockito-core</artifactId>
             <scope>test</scope>
diff --git a/document/src/main/java/com/yahoo/document/DocumentOperation.java b/document/src/main/java/com/yahoo/document/DocumentOperation.java
index 8209322c472..bb1e5d2b357 100644
--- a/document/src/main/java/com/yahoo/document/DocumentOperation.java
+++ b/document/src/main/java/com/yahoo/document/DocumentOperation.java
@@ -4,10 +4,7 @@ package com.yahoo.document;
 /**
  * Base class for "document operations".
  * These include "put" (DocumentPut), "update" (DocumentUpdate), "remove" (DocumentRemove)
- * and "get" (DocumentGet).  The latter only for internal use.
- * Historically, put operations were represented by the Document class alone,
- * but since it doesn't make much sense to put a *test and set* condition in Document,
- * a more uniform interface for document operations was needed.
+ * and "get" (DocumentGet).
  *
  * @author Vegard Sjonfjell
  */
diff --git a/document/src/main/java/com/yahoo/document/DocumentPut.java b/document/src/main/java/com/yahoo/document/DocumentPut.java
index 5906a9ca0ba..c5ce2e7e181 100644
--- a/document/src/main/java/com/yahoo/document/DocumentPut.java
+++ b/document/src/main/java/com/yahoo/document/DocumentPut.java
@@ -30,7 +30,8 @@ public class DocumentPut extends DocumentOperation {
 
     /**
      * Copy constructor
-     * @param other DocumentPut to copy
+     *
+     * @param other the DocumentPut to copy
      */
     public DocumentPut(DocumentPut other) {
         super(other);
diff --git a/document/src/main/java/com/yahoo/document/DocumentType.java b/document/src/main/java/com/yahoo/document/DocumentType.java
index f3ff6e4ed30..23559878fbb 100755
--- a/document/src/main/java/com/yahoo/document/DocumentType.java
+++ b/document/src/main/java/com/yahoo/document/DocumentType.java
@@ -41,6 +41,7 @@ public class DocumentType extends StructuredDataType {
     private StructDataType bodyType;
     private List<DocumentType> inherits = new ArrayList<>(1);
     private Map<String, Set<Field>> fieldSets = new HashMap<>();
+    private final Set<String> importedFieldNames;
 
     /**
      * Creates a new document type and registers it with the document type manager.
@@ -51,8 +52,7 @@ public class DocumentType extends StructuredDataType {
      * @param name The name of the new document type
      */
     public DocumentType(String name) {
-        this(name, new StructDataType(name + ".header"),
-                   new StructDataType(name + ".body"));
+        this(name, createHeaderStructType(name), createBodyStructType(name));
     }
 
     /**
@@ -65,9 +65,27 @@ public class DocumentType extends StructuredDataType {
      * @param bodyType   The type of the body struct
      */
     public DocumentType(String name, StructDataType headerType, StructDataType bodyType) {
+        this(name, headerType, bodyType, Collections.emptySet());
+    }
+
+    public DocumentType(String name, StructDataType headerType,
+                        StructDataType bodyType, Set<String> importedFieldNames) {
         super(name);
         this.headerType = headerType;
         this.bodyType = bodyType;
+        this.importedFieldNames = Collections.unmodifiableSet(importedFieldNames);
+    }
+
+    public DocumentType(String name, Set<String> importedFieldNames) {
+        this(name, createHeaderStructType(name), createBodyStructType(name), importedFieldNames);
+    }
+
+    private static StructDataType createHeaderStructType(String name) {
+        return new StructDataType(name + ".header");
+    }
+
+    private static StructDataType createBodyStructType(String name) {
+        return new StructDataType(name + ".body");
     }
 
     @Override
@@ -181,8 +199,7 @@ public class DocumentType extends StructuredDataType {
         if (isRegistered()) {
             throw new IllegalStateException("You cannot add fields to a document type that is already registered.");
         }
-        StructDataType struct = (field.isHeader() ? headerType : bodyType);
-        struct.addField(field);
+        headerType.addField(field);
     }
 
     // Do not use, public only for testing
@@ -221,8 +238,8 @@ public class DocumentType extends StructuredDataType {
         if (isRegistered()) {
             throw new IllegalStateException("You cannot add fields to a document type that is already registered.");
         }
-        Field field = new Field(name, type, false);
-        bodyType.addField(field);
+        Field field = new Field(name, type);
+        headerType.addField(field);
         return field;
     }
 
@@ -234,13 +251,9 @@ public class DocumentType extends StructuredDataType {
      * @return The field created
      *         TODO Fix searchdefinition so that exception can be thrown if filed is already registerd
      */
+    @Deprecated
     public Field addHeaderField(String name, DataType type) {
-        if (isRegistered()) {
-            throw new IllegalStateException("You cannot add fields to a document type that is already registered.");
-        }
-        Field field = new Field(name, type, true);
-        headerType.addField(field);
-        return field;
+        return addField(name, type);
     }
 
     /**
@@ -374,6 +387,14 @@ public class DocumentType extends StructuredDataType {
         return headerType.getFieldCount() + bodyType.getFieldCount();
     }
 
+    public Set<String> getImportedFieldNames() {
+        return importedFieldNames;
+    }
+
+    public boolean hasImportedField(String fieldName) {
+        return importedFieldNames.contains(fieldName);
+    }
+
     /**
      * Removes an field from the DocumentType.
      *
diff --git a/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java b/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java
index 154c25880a9..21a163aa2b9 100644
--- a/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java
+++ b/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java
@@ -10,8 +10,10 @@ import com.yahoo.log.LogLevel;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Map;
 import java.util.logging.Logger;
+import java.util.stream.Collectors;
 
 /**
  * Configures the Vespa document manager from a config id.
@@ -144,7 +146,10 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub
         for (Field field : body.getFields()) {
             field.setHeader(false);
         }
-        DocumentType type = new DocumentType(doc.name(), header, body);
+        var importedFields = doc.importedfield().stream()
+                .map(f -> f.name())
+                .collect(Collectors.toUnmodifiableSet());
+        DocumentType type = new DocumentType(doc.name(), header, body, importedFields);
         for (Object j : doc.inherits()) {
             DocumentmanagerConfig.Datatype.Documenttype.Inherits parent =
                     (DocumentmanagerConfig.Datatype.Documenttype.Inherits) j;
@@ -180,9 +185,9 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub
                                : manager.getDataType(field.datatype(), field.detailedtype());
 
             if (field.id().size() == 1) {
-                type.addField(new Field(field.name(), field.id().get(0).id(), fieldType, true));
+                type.addField(new Field(field.name(), field.id().get(0).id(), fieldType));
             } else {
-                type.addField(new Field(field.name(), fieldType, true));
+                type.addField(new Field(field.name(), fieldType));
             }
         }
         manager.register(type);
diff --git a/document/src/main/java/com/yahoo/document/Field.java b/document/src/main/java/com/yahoo/document/Field.java
index 671c8c7f763..9be4036174c 100644
--- a/document/src/main/java/com/yahoo/document/Field.java
+++ b/document/src/main/java/com/yahoo/document/Field.java
@@ -21,7 +21,6 @@ public class Field extends FieldBase implements FieldSet, Comparable, Serializab
 
     protected DataType dataType;
     protected int fieldId;
-    private boolean isHeader;
     private boolean forcedId;
 
     /**
@@ -32,11 +31,14 @@ public class Field extends FieldBase implements FieldSet, Comparable, Serializab
      * @param isHeader Whether this is a "header" field or a "content" field
      *                 (true = "header").
      */
+    @Deprecated
     public Field(String name, int id, DataType dataType, boolean isHeader) {
+        this(name, id, dataType);
+    }
+    public Field(String name, int id, DataType dataType) {
         super(name);
         this.fieldId = id;
         this.dataType = dataType;
-        this.isHeader = isHeader;
         this.forcedId = true;
         validateId(id, null);
     }
@@ -55,8 +57,13 @@ public class Field extends FieldBase implements FieldSet, Comparable, Serializab
      *                 (true = "header").
      * @param owner    the owning document (used to check for id collisions)
      */
+    @Deprecated
     public Field(String name, DataType dataType, boolean isHeader, DocumentType owner) {
-        this(name, 0, dataType, isHeader);
+        this(name, dataType, owner);
+    }
+
+    public Field(String name, DataType dataType, DocumentType owner) {
+        this(name, 0, dataType);
         this.fieldId = calculateIdV7(owner);
         this.forcedId = false;
     }
@@ -69,8 +76,9 @@ public class Field extends FieldBase implements FieldSet, Comparable, Serializab
      * @param isHeader Whether this is a "header" field or a "content" field
      *                 (true = "header").
      */
+    @Deprecated
     public Field(String name, DataType dataType, boolean isHeader) {
-        this(name, dataType, isHeader, null);
+        this(name, dataType);
     }
 
     /**
@@ -80,7 +88,7 @@ public class Field extends FieldBase implements FieldSet, Comparable, Serializab
      * @param dataType The datatype of the field
      */
     public Field(String name, DataType dataType) {
-        this(name, dataType, true);
+        this(name, dataType, null);
     }
 
     /**
@@ -89,7 +97,7 @@ public class Field extends FieldBase implements FieldSet, Comparable, Serializab
      */
     // TODO: Decide on one copy/clone idiom and do it for this and all it is calling
     public Field(String name, Field field) {
-        this(name, field.dataType, field.isHeader, null);
+        this(name, field.dataType, null);
     }
 
     public int compareTo(Object o) {
@@ -196,14 +204,12 @@ public class Field extends FieldBase implements FieldSet, Comparable, Serializab
     /** @deprecated this has no longer any semantic meaning as this is no longer an aspect with a field */
     @Deprecated // TODO: Remove on Vespa 8
     public boolean isHeader() {
-        return isHeader;
+        return true;
     }
 
     /** @deprecated this has no longer any semantic meaning as this is no longer an aspect with a field */
     @Deprecated // TODO: Remove on Vespa 8
-    public void setHeader(boolean header) {
-        this.isHeader = header;
-    }
+    public void setHeader(boolean header) { }
 
     /** Two fields are equal if they have the same name and the same data type */
     @Override
diff --git a/document/src/main/java/com/yahoo/document/FieldPath.java b/document/src/main/java/com/yahoo/document/FieldPath.java
index 295b1c9dd5c..134681a4028 100755
--- a/document/src/main/java/com/yahoo/document/FieldPath.java
+++ b/document/src/main/java/com/yahoo/document/FieldPath.java
@@ -10,7 +10,7 @@ import java.util.List;
  * This class represents a path into a document, that can be used to iterate through the document and extract the field
  * values you're interested in.
  *
- * @author <a href="mailto:thomasg@yahoo-inc.com">Thomas Gundersen</a>
+ * @author Thomas Gundersen
  */
 public class FieldPath implements Iterable<FieldPathEntry> {
 
diff --git a/document/src/main/java/com/yahoo/document/TestAndSetCondition.java b/document/src/main/java/com/yahoo/document/TestAndSetCondition.java
index 767fd6672d4..def1c05a011 100644
--- a/document/src/main/java/com/yahoo/document/TestAndSetCondition.java
+++ b/document/src/main/java/com/yahoo/document/TestAndSetCondition.java
@@ -45,4 +45,13 @@ public class TestAndSetCondition {
                 .orElse(TestAndSetCondition.NOT_PRESENT_CONDITION);
     }
 
+    @Override
+    public String toString() {
+        StringBuilder string = new StringBuilder();
+        string.append("condition '");
+        string.append(conditionStr);
+        string.append("'");
+
+        return string.toString();
+    }
 }
diff --git a/document/src/main/java/com/yahoo/document/datatypes/StructuredFieldValue.java b/document/src/main/java/com/yahoo/document/datatypes/StructuredFieldValue.java
index b3ea93d8467..43016187954 100644
--- a/document/src/main/java/com/yahoo/document/datatypes/StructuredFieldValue.java
+++ b/document/src/main/java/com/yahoo/document/datatypes/StructuredFieldValue.java
@@ -139,6 +139,10 @@ public abstract class StructuredFieldValue extends CompositeFieldValue {
     public final FieldValue setFieldValue(String field, Byte value) {
         return setFieldValue(field, new ByteFieldValue(value));
     }
+
+    public final FieldValue setFieldValue(String field, Boolean value) {
+        return setFieldValue(field, new BoolFieldValue(value));
+    }
     /**
      * Removes and returns a field value.
      *
diff --git a/document/src/main/java/com/yahoo/document/idstring/IdIdString.java b/document/src/main/java/com/yahoo/document/idstring/IdIdString.java
index 4c7f71dd712..9c75cf6828b 100644
--- a/document/src/main/java/com/yahoo/document/idstring/IdIdString.java
+++ b/document/src/main/java/com/yahoo/document/idstring/IdIdString.java
@@ -11,11 +11,13 @@ import com.yahoo.text.Utf8;
  * Time: 11:02 AM
  */
 public class IdIdString extends IdString {
-    private String type;
+    private final String type;
     private String group;
     private long location;
     private boolean hasGroup;
     private boolean hasNumber;
+    private static final int SIZE_OF_ID_AND_3_COLONS = 2 + 3; // "id:::"
+    private static final int MAX_LENGTH = IdString.MAX_LENGTH_EXCEPT_NAMESPACE_SPECIFIC - SIZE_OF_ID_AND_3_COLONS;
 
     public static String replaceType(String id, String typeName) {
         int typeStartPos = id.indexOf(":", 3) + 1;
@@ -47,6 +49,10 @@ public class IdIdString extends IdString {
         super(Scheme.id, namespace, localId);
         this.type = type;
         boolean hasSetLocation = false;
+        if (namespace.length() + type.length() + keyValues.length() >= MAX_LENGTH) {
+            throw new IllegalArgumentException("Length of namespace(" + namespace.length() +  ") + doctype(" + type.length() +
+                    ") + key/values(" + keyValues.length() +"), is longer than " + MAX_LENGTH);
+        }
         for(String pair : keyValues.split(",")) {
             int pos = pair.indexOf('=');
             if (pos == -1) {
diff --git a/document/src/main/java/com/yahoo/document/idstring/IdString.java b/document/src/main/java/com/yahoo/document/idstring/IdString.java
index 0fe382be914..2114a480ec3 100644
--- a/document/src/main/java/com/yahoo/document/idstring/IdString.java
+++ b/document/src/main/java/com/yahoo/document/idstring/IdString.java
@@ -42,6 +42,8 @@ public abstract class IdString {
     private final String namespace;
     private final String namespaceSpecific;
     private Utf8String cache;
+    // This max unsigned 16 bit integer - 1 as the offset will be length + 1
+    static final int MAX_LENGTH_EXCEPT_NAMESPACE_SPECIFIC = 0xfffe;
 
     /**
      * Creates a IdString based on the given document id string.
@@ -115,6 +117,8 @@ public abstract class IdString {
             colonPos = id.indexOf(":", currPos);
             if (colonPos < 0) {
                 throw new IllegalArgumentException("Unparseable id '" + id + "': Key/value section missing");
+            } else if (colonPos >= MAX_LENGTH_EXCEPT_NAMESPACE_SPECIFIC) {
+                throw new IllegalArgumentException("Document id prior to the namespace specific part, " + colonPos + ", is longer than " + MAX_LENGTH_EXCEPT_NAMESPACE_SPECIFIC + " id: " + id);
             }
             String keyValues = id.substring(currPos, colonPos);
 
diff --git a/document/src/main/java/com/yahoo/document/select/rule/AttributeNode.java b/document/src/main/java/com/yahoo/document/select/rule/AttributeNode.java
index 0cedda7c4f0..9e2759e1590 100644
--- a/document/src/main/java/com/yahoo/document/select/rule/AttributeNode.java
+++ b/document/src/main/java/com/yahoo/document/select/rule/AttributeNode.java
@@ -7,6 +7,7 @@ import com.yahoo.document.Document;
 import com.yahoo.document.DocumentGet;
 import com.yahoo.document.DocumentPut;
 import com.yahoo.document.DocumentRemove;
+import com.yahoo.document.DocumentType;
 import com.yahoo.document.DocumentUpdate;
 import com.yahoo.document.FieldPath;
 import com.yahoo.document.datatypes.FieldPathIteratorHandler;
@@ -131,10 +132,37 @@ public class AttributeNode implements ExpressionNode {
         throw new IllegalStateException("Function '" + function + "' is not supported.");
     }
 
-    private static Object evaluateFieldPath(String fieldPth, Object value) {
+    private static boolean looksLikeComplexFieldPath(String path) {
+        for (int i = 0; i < path.length(); ++i) {
+            switch (path.charAt(i)) {
+                case '.':
+                case '{':
+                case '[':
+                    return true;
+            }
+        }
+        return false;
+    }
+
+    private static boolean isSimpleImportedField(String path, DocumentType documentType) {
+        if (looksLikeComplexFieldPath(path)) {
+            return false;
+        }
+        return documentType.hasImportedField(path);
+    }
+
+    private static Object evaluateFieldPath(String fieldPathStr, Object value) {
         if (value instanceof DocumentPut) {
             final Document doc = ((DocumentPut) value).getDocument();
-            FieldPath fieldPath = doc.getDataType().buildFieldPath(fieldPth);
+            if (isSimpleImportedField(fieldPathStr, doc.getDataType())) {
+                // Imported fields can only be meaningfully evaluated in the backend, so we
+                // explicitly treat them as if they are valid fields with missing values. This
+                // will be treated the same as if it's a normal field by the selection operators.
+                // This avoids any awkward interaction with Invalid values or having to
+                // augment the FieldPath code with knowledge of imported fields.
+                return null;
+            }
+            FieldPath fieldPath = doc.getDataType().buildFieldPath(fieldPathStr);
             IteratorHandler handler = new IteratorHandler();
             doc.iterateNested(fieldPath, 0, handler);
             if (handler.values.isEmpty()) {
diff --git a/document/src/main/java/com/yahoo/document/select/rule/ComparisonNode.java b/document/src/main/java/com/yahoo/document/select/rule/ComparisonNode.java
index 47599e53ece..8f52c29e84d 100644
--- a/document/src/main/java/com/yahoo/document/select/rule/ComparisonNode.java
+++ b/document/src/main/java/com/yahoo/document/select/rule/ComparisonNode.java
@@ -144,9 +144,9 @@ public class ComparisonNode implements ExpressionNode {
                 return new ResultList(Result.INVALID);
             }
         } else if (oLeft instanceof AttributeNode.VariableValueList) {
-            return evaluateListAndSingle((AttributeNode.VariableValueList)oLeft, oRight);
+            return evaluateLhsListAndRhsSingle((AttributeNode.VariableValueList)oLeft, oRight);
         } else if (oRight instanceof AttributeNode.VariableValueList) {
-            return evaluateListAndSingle((AttributeNode.VariableValueList)oRight, oLeft);
+            return evaluateLhsSingleAndRhsList(oLeft, (AttributeNode.VariableValueList)oRight);
         }
         return new ResultList(evaluateBool(oLeft, oRight));
     }
@@ -197,7 +197,7 @@ public class ComparisonNode implements ExpressionNode {
         }
     }
 
-    private ResultList evaluateListAndSingle(AttributeNode.VariableValueList lhs, Object rhs) {
+    private ResultList evaluateLhsListAndRhsSingle(AttributeNode.VariableValueList lhs, Object rhs) {
         if (rhs == null && lhs == null) {
             return new ResultList(Result.TRUE);
         }
@@ -215,6 +215,24 @@ public class ComparisonNode implements ExpressionNode {
         return retVal;
     }
 
+    private ResultList evaluateLhsSingleAndRhsList(Object lhs, AttributeNode.VariableValueList rhs) {
+        if (rhs == null && lhs == null) {
+            return new ResultList(Result.TRUE);
+        }
+
+        if (rhs == null || lhs == null) {
+            return new ResultList(Result.FALSE);
+        }
+
+        ResultList retVal = new ResultList();
+        for (ResultList.VariableValue value : rhs) {
+            Result result = evaluateBool(lhs, value.getValue());
+            retVal.add((FieldPathIteratorHandler.VariableMap)value.getVariables().clone(), result);
+        }
+
+        return retVal;
+    }
+
     /**
      * Evaluate this expression on two operands, given that they are not invalid.
      *
diff --git a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializer6.java b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializer6.java
index c2111edfd10..630f204c44d 100644
--- a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializer6.java
+++ b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializer6.java
@@ -82,7 +82,7 @@ public class VespaDocumentSerializer6 extends BufferSerializer implements Docume
     }
 
     public void write(Document doc) {
-        write(new Field(doc.getDataType().getName(), 0, doc.getDataType(), true), doc);
+        write(new Field(doc.getDataType().getName(), 0, doc.getDataType()), doc);
     }
 
     @SuppressWarnings("deprecation")
diff --git a/document/src/main/java/com/yahoo/document/serialization/XmlDocumentWriter.java b/document/src/main/java/com/yahoo/document/serialization/XmlDocumentWriter.java
index 0d6b0cae926..5db98f26141 100644
--- a/document/src/main/java/com/yahoo/document/serialization/XmlDocumentWriter.java
+++ b/document/src/main/java/com/yahoo/document/serialization/XmlDocumentWriter.java
@@ -320,7 +320,7 @@ public final class XmlDocumentWriter implements DocumentWriter {
         buffer = new XmlStream();
         buffer.setIndent(indent);
         optionalWrapperMarker.clear();
-        write(new Field(document.getDataType().getName(), 0, document.getDataType(), true), document);
+        write(new Field(document.getDataType().getName(), 0, document.getDataType()), document);
     }
 
     @Override
diff --git a/document/src/test/document/documentmanager.importedfields.cfg b/document/src/test/document/documentmanager.importedfields.cfg
new file mode 100644
index 00000000000..765e290ae82
--- /dev/null
+++ b/document/src/test/document/documentmanager.importedfields.cfg
@@ -0,0 +1,65 @@
+datatype[7]
+datatype[0].id -1365874599
+datatype[0].arraytype[0]
+datatype[0].weightedsettype[0]
+datatype[0].structtype[1]
+datatype[0].structtype[0].name referenced_type.header
+datatype[0].structtype[0].version 9
+datatype[0].structtype[0].field[0]
+datatype[0].documenttype[0]
+datatype[1].id 278604398
+datatype[1].arraytype[0]
+datatype[1].weightedsettype[0]
+datatype[1].structtype[1]
+datatype[1].structtype[0].name referenced_type.body
+datatype[1].structtype[0].version 9
+datatype[1].structtype[0].field[0]
+datatype[1].documenttype[0]
+datatype[2].id 124647170
+datatype[2].arraytype[0]
+datatype[2].weightedsettype[0]
+datatype[2].structtype[0]
+datatype[2].documenttype[1]
+datatype[2].documenttype[0].name referenced_type
+datatype[2].documenttype[0].version 9
+datatype[2].documenttype[0].inherits[0]
+datatype[2].documenttype[0].headerstruct -1365874599
+datatype[2].documenttype[0].bodystruct 278604398
+datatype[3].id 12345678
+datatype[3].arraytype[0]
+datatype[3].weightedsettype[0]
+datatype[3].structtype[0]
+datatype[3].documenttype[0]
+datatype[3].referencetype[1]
+datatype[3].referencetype[0].target_type_id 124647170
+datatype[4].id 673066331
+datatype[4].arraytype[0]
+datatype[4].weightedsettype[0]
+datatype[4].structtype[1]
+datatype[4].structtype[0].name type_with_ref.header
+datatype[4].structtype[0].version 234
+datatype[4].structtype[0].field[1]
+datatype[4].structtype[0].field[0].name my_ref_field
+datatype[4].structtype[0].field[0].id[0]
+datatype[4].structtype[0].field[0].datatype 12345678
+datatype[4].documenttype[0]
+datatype[5].id -176986064
+datatype[5].arraytype[0]
+datatype[5].weightedsettype[0]
+datatype[5].structtype[1]
+datatype[5].structtype[0].name type_with_ref.body
+datatype[5].structtype[0].version 234
+datatype[5].structtype[0].field[0]
+datatype[5].documenttype[0]
+datatype[6].id -1293964543
+datatype[6].arraytype[0]
+datatype[6].weightedsettype[0]
+datatype[6].structtype[0]
+datatype[6].documenttype[1]
+datatype[6].documenttype[0].name type_with_ref
+datatype[6].documenttype[0].version 234
+datatype[6].documenttype[0].inherits[0]
+datatype[6].documenttype[0].headerstruct 673066331
+datatype[6].documenttype[0].bodystruct -176986064
+datatype[6].documenttype[0].importedfield[0].name "my_cool_imported_field"
+datatype[6].documenttype[0].importedfield[1].name "my_awesome_imported_field"
\ No newline at end of file
diff --git a/document/src/test/java/com/yahoo/document/DocumentCalculatorTestCase.java b/document/src/test/java/com/yahoo/document/DocumentCalculatorTestCase.java
index fb2d478d38b..36cc18ebd6b 100755
--- a/document/src/test/java/com/yahoo/document/DocumentCalculatorTestCase.java
+++ b/document/src/test/java/com/yahoo/document/DocumentCalculatorTestCase.java
@@ -27,11 +27,11 @@ public class DocumentCalculatorTestCase {
         docMan = new DocumentTypeManager();
         testDocType = new DocumentType("testdoc");
 
-        testDocType.addHeaderField("byteattr", DataType.BYTE);
-        testDocType.addHeaderField("intattr", DataType.INT);
-        testDocType.addHeaderField("longattr", DataType.LONG);
-        testDocType.addHeaderField("doubleattr", DataType.DOUBLE);
-        testDocType.addHeaderField("missingattr", DataType.INT);
+        testDocType.addField("byteattr", DataType.BYTE);
+        testDocType.addField("intattr", DataType.INT);
+        testDocType.addField("longattr", DataType.LONG);
+        testDocType.addField("doubleattr", DataType.DOUBLE);
+        testDocType.addField("missingattr", DataType.INT);
 
         docMan.registerDocumentType(testDocType);
         doc = new Document(testDocType, new DocumentId("id:ns:testdoc::testdoc:http://www.ntnu.no/"));
diff --git a/document/src/test/java/com/yahoo/document/DocumentIdTestCase.java b/document/src/test/java/com/yahoo/document/DocumentIdTestCase.java
index 08abd6e6a2d..fea3b265b6d 100644
--- a/document/src/test/java/com/yahoo/document/DocumentIdTestCase.java
+++ b/document/src/test/java/com/yahoo/document/DocumentIdTestCase.java
@@ -34,11 +34,11 @@ public class DocumentIdTestCase {
     public void setUp() {
         DocumentType testDocType = new DocumentType("testdoc");
 
-        testDocType.addHeaderField("intattr", DataType.INT);
+        testDocType.addField("intattr", DataType.INT);
         testDocType.addField("rawattr", DataType.RAW);
         testDocType.addField("floatattr", DataType.FLOAT);
-        testDocType.addHeaderField("stringattr", DataType.STRING);
-        testDocType.addHeaderField("Minattr", DataType.INT);
+        testDocType.addField("stringattr", DataType.STRING);
+        testDocType.addField("Minattr", DataType.INT);
 
         manager.registerDocumentType(testDocType);
     }
diff --git a/document/src/test/java/com/yahoo/document/DocumentSerializationTestCase.java b/document/src/test/java/com/yahoo/document/DocumentSerializationTestCase.java
index bc1224ca8ea..fa47c80c6fb 100644
--- a/document/src/test/java/com/yahoo/document/DocumentSerializationTestCase.java
+++ b/document/src/test/java/com/yahoo/document/DocumentSerializationTestCase.java
@@ -62,25 +62,25 @@ public class DocumentSerializationTestCase extends AbstractTypesTest {
     public void testSerializationAllVersions() throws IOException {
 
         DocumentType docInDocType = new DocumentType("docindoc");
-        docInDocType.addField(new Field("stringindocfield", DataType.STRING, false));
+        docInDocType.addField(new Field("stringindocfield", DataType.STRING));
 
         DocumentType docType = new DocumentType("serializetest");
-        docType.addField(new Field("floatfield", DataType.FLOAT, true));
-        docType.addField(new Field("stringfield", DataType.STRING, true));
-        docType.addField(new Field("longfield", DataType.LONG, true));
-        docType.addField(new Field("urifield", DataType.URI, true));
-        docType.addField(new Field("intfield", DataType.INT, false));
-        docType.addField(new Field("rawfield", DataType.RAW, false));
-        docType.addField(new Field("doublefield", DataType.DOUBLE, false));
-        docType.addField(new Field("bytefield", DataType.BYTE, false));
-        docType.addField(new Field("boolfield", DataType.BOOL, false));
+        docType.addField(new Field("floatfield", DataType.FLOAT));
+        docType.addField(new Field("stringfield", DataType.STRING));
+        docType.addField(new Field("longfield", DataType.LONG));
+        docType.addField(new Field("urifield", DataType.URI));
+        docType.addField(new Field("intfield", DataType.INT));
+        docType.addField(new Field("rawfield", DataType.RAW));
+        docType.addField(new Field("doublefield", DataType.DOUBLE));
+        docType.addField(new Field("bytefield", DataType.BYTE));
+        docType.addField(new Field("boolfield", DataType.BOOL));
         DataType arrayOfFloatDataType = new ArrayDataType(DataType.FLOAT);
-        docType.addField(new Field("arrayoffloatfield", arrayOfFloatDataType, false));
+        docType.addField(new Field("arrayoffloatfield", arrayOfFloatDataType));
         DataType arrayOfArrayOfFloatDataType = new ArrayDataType(arrayOfFloatDataType);
-        docType.addField(new Field("arrayofarrayoffloatfield", arrayOfArrayOfFloatDataType, false));
-        docType.addField(new Field("docfield", DataType.DOCUMENT, false));
+        docType.addField(new Field("arrayofarrayoffloatfield", arrayOfArrayOfFloatDataType));
+        docType.addField(new Field("docfield", DataType.DOCUMENT));
         DataType weightedSetDataType = DataType.getWeightedSet(DataType.STRING, false, false);
-        docType.addField(new Field("wsfield", weightedSetDataType, false));
+        docType.addField(new Field("wsfield", weightedSetDataType));
 
         DocumentTypeManager docMan = new DocumentTypeManager();
         docMan.register(docInDocType);
diff --git a/document/src/test/java/com/yahoo/document/DocumentTestCase.java b/document/src/test/java/com/yahoo/document/DocumentTestCase.java
index 141a74a24fe..dcd4622b3f4 100644
--- a/document/src/test/java/com/yahoo/document/DocumentTestCase.java
+++ b/document/src/test/java/com/yahoo/document/DocumentTestCase.java
@@ -107,26 +107,26 @@ public class DocumentTestCase extends DocumentTestCaseBase {
         docMan = new DocumentTypeManager();
 
         DocumentType docInDocType = new DocumentType("docindoc");
-        docInDocType.addField(new Field("tull", 2, docMan.getDataType(2), true));
+        docInDocType.addField(new Field("tull", 2, docMan.getDataType(2)));
 
         docMan.registerDocumentType(docInDocType);
 
         DocumentType sertestDocType = new DocumentType("sertest");
-        sertestDocType.addField(new Field("mailid", 2, docMan.getDataType(2), true));
-        sertestDocType.addField(new Field("date", 3, docMan.getDataType(0), true));
-        sertestDocType.addField(new Field("from", 4, docMan.getDataType(2), true));
-        sertestDocType.addField(new Field("to", 6, docMan.getDataType(2), true));
-        sertestDocType.addField(new Field("subject", 9, docMan.getDataType(2), true));
-        sertestDocType.addField(new Field("body", 10, docMan.getDataType(2), false));
-        sertestDocType.addField(new Field("attachmentcount", 11, docMan.getDataType(0), false));
-        sertestDocType.addField(new Field("attachments", 1081629685, DataType.getArray(docMan.getDataType(2)), false));
-        sertestDocType.addField(new Field("rawfield", 879, DataType.RAW, false));
-        sertestDocType.addField(new Field("weightedfield", 880, DataType.getWeightedSet(DataType.STRING), false));
-        sertestDocType.addField(new Field("weightedfieldCreate", 881, DataType.getWeightedSet(DataType.STRING, true, false), false));
-        sertestDocType.addField(new Field("docindoc", 882, docInDocType, false));
-        sertestDocType.addField(new Field("mapfield", 883, new MapDataType(DataType.STRING, DataType.STRING), false));
-        sertestDocType.addField(new Field("myposfield", 884, PositionDataType.INSTANCE, false));
-        sertestDocType.addField(new Field("myboolfield", 885, DataType.BOOL, false));
+        sertestDocType.addField(new Field("mailid", 2, docMan.getDataType(2)));
+        sertestDocType.addField(new Field("date", 3, docMan.getDataType(0)));
+        sertestDocType.addField(new Field("from", 4, docMan.getDataType(2)));
+        sertestDocType.addField(new Field("to", 6, docMan.getDataType(2)));
+        sertestDocType.addField(new Field("subject", 9, docMan.getDataType(2)));
+        sertestDocType.addField(new Field("body", 10, docMan.getDataType(2)));
+        sertestDocType.addField(new Field("attachmentcount", 11, docMan.getDataType(0)));
+        sertestDocType.addField(new Field("attachments", 1081629685, DataType.getArray(docMan.getDataType(2))));
+        sertestDocType.addField(new Field("rawfield", 879, DataType.RAW));
+        sertestDocType.addField(new Field("weightedfield", 880, DataType.getWeightedSet(DataType.STRING)));
+        sertestDocType.addField(new Field("weightedfieldCreate", 881, DataType.getWeightedSet(DataType.STRING, true, false)));
+        sertestDocType.addField(new Field("docindoc", 882, docInDocType));
+        sertestDocType.addField(new Field("mapfield", 883, new MapDataType(DataType.STRING, DataType.STRING)));
+        sertestDocType.addField(new Field("myposfield", 884, PositionDataType.INSTANCE));
+        sertestDocType.addField(new Field("myboolfield", 885, DataType.BOOL));
 
         docMan.registerDocumentType(sertestDocType);
     }
@@ -880,13 +880,13 @@ public class DocumentTestCase extends DocumentTestCaseBase {
     public void testInheritance() {
         // Create types that inherit each other.. And test that it works..
         DocumentType parentType = new DocumentType("parent");
-        parentType.addField(new Field("parentbodyint", DataType.INT, false));
-        parentType.addField(new Field("parentheaderint", DataType.INT, true));
-        parentType.addField(new Field("overwritten", DataType.INT, true));
+        parentType.addField(new Field("parentbodyint", DataType.INT));
+        parentType.addField(new Field("parentheaderint", DataType.INT));
+        parentType.addField(new Field("overwritten", DataType.INT));
         DocumentType childType = new DocumentType("child");
-        childType.addField(new Field("childbodyint", DataType.INT, false));
-        childType.addField(new Field("childheaderint", DataType.INT, true));
-        childType.addField(new Field("overwritten", DataType.INT, true));
+        childType.addField(new Field("childbodyint", DataType.INT));
+        childType.addField(new Field("childheaderint", DataType.INT));
+        childType.addField(new Field("overwritten", DataType.INT));
         childType.inherit(parentType);
 
         DocumentTypeManager manager = new DocumentTypeManager();
@@ -914,13 +914,13 @@ public class DocumentTestCase extends DocumentTestCaseBase {
     @Test
     public void testInheritanceTypeMismatch() {
         DocumentType parentType = new DocumentType("parent");
-        parentType.addField(new Field("parentbodyint", DataType.INT, false));
-        parentType.addField(new Field("parentheaderint", DataType.INT, true));
-        parentType.addField(new Field("overwritten", DataType.STRING, true));
+        parentType.addField(new Field("parentbodyint", DataType.INT));
+        parentType.addField(new Field("parentheaderint", DataType.INT));
+        parentType.addField(new Field("overwritten", DataType.STRING));
         DocumentType childType = new DocumentType("child");
-        childType.addField(new Field("childbodyint", DataType.INT, false));
-        childType.addField(new Field("childheaderint", DataType.INT, true));
-        childType.addField(new Field("overwritten", DataType.INT, true));
+        childType.addField(new Field("childbodyint", DataType.INT));
+        childType.addField(new Field("childheaderint", DataType.INT));
+        childType.addField(new Field("overwritten", DataType.INT));
         try {
             childType.inherit(parentType);
             fail("Inheritance with conflicting types worked.");
@@ -934,7 +934,7 @@ public class DocumentTestCase extends DocumentTestCaseBase {
     public void testFieldValueImplementations() {
         docMan = new DocumentTypeManager();
         DocumentType docType = new DocumentType("impl");
-        docType.addField(new Field("something", DataType.getArray(DataType.STRING), false));
+        docType.addField(new Field("something", DataType.getArray(DataType.STRING)));
         docMan.register(docType);
 
         //just checks that isAssignableFrom() in Document.setFieldValue() goes the right way
@@ -1276,9 +1276,9 @@ public class DocumentTestCase extends DocumentTestCaseBase {
     public void testDocumentComparisonDoesNotCorruptStateBug6394548() {
         DocumentTypeManager docMan = new DocumentTypeManager();
         DocumentType docType = new DocumentType("bug2354045");
-        docType.addField(new Field("string", 2, DataType.STRING, true));
-        docType.addField(new Field("int", 1, DataType.INT, true));
-        docType.addField(new Field("float", 0, DataType.FLOAT, true));
+        docType.addField(new Field("string", 2, DataType.STRING));
+        docType.addField(new Field("int", 1, DataType.INT));
+        docType.addField(new Field("float", 0, DataType.FLOAT));
         docMan.register(docType);
 
         Document doc1 = new Document(docType, new DocumentId("id:ns:bug2354045::bug6394548"));
diff --git a/document/src/test/java/com/yahoo/document/DocumentTestCaseBase.java b/document/src/test/java/com/yahoo/document/DocumentTestCaseBase.java
index 68fe1c8cc57..6f95f77f08c 100644
--- a/document/src/test/java/com/yahoo/document/DocumentTestCaseBase.java
+++ b/document/src/test/java/com/yahoo/document/DocumentTestCaseBase.java
@@ -23,14 +23,14 @@ public class DocumentTestCaseBase {
         docMan = new DocumentTypeManager();
         testDocType = new DocumentType("testdoc");
 
-        testDocType.addHeaderField("byteattr", DataType.BYTE);
-        testDocType.addHeaderField("intattr", DataType.INT);
+        testDocType.addField("byteattr", DataType.BYTE);
+        testDocType.addField("intattr", DataType.INT);
         testDocType.addField("rawattr", DataType.RAW);
         testDocType.addField("floatattr", DataType.FLOAT);
-        testDocType.addHeaderField("stringattr", DataType.STRING);
-        testDocType.addHeaderField("Minattr", DataType.INT);
-        testDocType.addHeaderField("Minattr2", DataType.INT);
-        testDocType.addHeaderField("primitive1", DataType.INT);
+        testDocType.addField("stringattr", DataType.STRING);
+        testDocType.addField("Minattr", DataType.INT);
+        testDocType.addField("Minattr2", DataType.INT);
+        testDocType.addField("primitive1", DataType.INT);
 
         StructDataType sdt = new StructDataType("struct1");
         sdt.addField(new Field("primitive1", DataType.INT));
diff --git a/document/src/test/java/com/yahoo/document/DocumentTypeManagerTestCase.java b/document/src/test/java/com/yahoo/document/DocumentTypeManagerTestCase.java
index 65c217e09e1..57b36d9758f 100644
--- a/document/src/test/java/com/yahoo/document/DocumentTypeManagerTestCase.java
+++ b/document/src/test/java/com/yahoo/document/DocumentTypeManagerTestCase.java
@@ -8,6 +8,7 @@ import com.yahoo.document.datatypes.StringFieldValue;
 import com.yahoo.document.datatypes.StructuredFieldValue;
 import org.junit.Test;
 
+import java.util.HashSet;
 import java.util.Iterator;
 
 import static org.junit.Assert.assertEquals;
@@ -30,7 +31,7 @@ public class DocumentTypeManagerTestCase {
 
         DocumentType newDocType = new DocumentType("testdoc");
         newDocType.addField("Fjomp", DataType.INT);
-        newDocType.addHeaderField("Fjols", DataType.STRING);
+        newDocType.addField("Fjols", DataType.STRING);
 
         manager.registerDocumentType(newDocType);
 
@@ -40,7 +41,6 @@ public class DocumentTypeManagerTestCase {
 
         assertEquals("Fjomp", fetched4.getName());
         assertEquals(fetched4.getDataType(), DataType.INT);
-        assertEquals(fetched4.isHeader(), false);
     }
 
     @Test
@@ -64,8 +64,8 @@ public class DocumentTypeManagerTestCase {
         StructDataType struct = new StructDataType("mystruct");
         DataType wset1 = DataType.getWeightedSet(DataType.getArray(DataType.INT));
         DataType wset2 = DataType.getWeightedSet(DataType.getArray(DataType.TAG));
-        struct.addField(new Field("foo", wset1, true));
-        struct.addField(new Field("bar", wset2, false));
+        struct.addField(new Field("foo", wset1));
+        struct.addField(new Field("bar", wset2));
         DataType array = DataType.getArray(struct);
         DocumentType docType = new DocumentType("mydoc");
         docType.addField("hmm", array);
@@ -148,7 +148,6 @@ public class DocumentTypeManagerTestCase {
         assertTrue(type.hasField("foobarfield1"));
 
         Field foobarfield0 = type.getField("foobarfield0");
-        assertTrue(!foobarfield0.isHeader());
         assertTrue(foobarfield0.getDataType().getCode() == 2);
 
         Field foobarfield1 = type.getField("foobarfield1");
@@ -190,7 +189,6 @@ public class DocumentTypeManagerTestCase {
         assertTrue(type.hasField("arrayarrayfloat"));
 
         Field arrayfloat = type.getField("arrayfloat");
-        assertTrue(!arrayfloat.isHeader());
         ArrayDataType dataType = (ArrayDataType) arrayfloat.getDataType();
         assertTrue(dataType.getCode() == 99);
         assertTrue(dataType.getValueClass().equals(Array.class));
@@ -200,7 +198,6 @@ public class DocumentTypeManagerTestCase {
 
         Field arrayarrayfloat = type.getField("arrayarrayfloat");
         ArrayDataType subType = (ArrayDataType) arrayarrayfloat.getDataType();
-        assertTrue(!arrayarrayfloat.isHeader());
         assertTrue(subType.getCode() == 4003);
         assertTrue(subType.getValueClass().equals(Array.class));
         assertTrue(subType.getNestedType().getCode() == 99);
@@ -218,14 +215,14 @@ public class DocumentTypeManagerTestCase {
         DocumentType customtypes = manager.getDocumentType(new DataTypeName("customtypes"));
 
         assertNull(banana.getField("newfield"));
-        assertEquals(new Field("arrayfloat", 9489, new ArrayDataType(DataType.FLOAT, 99), false), customtypes.getField("arrayfloat"));
+        assertEquals(new Field("arrayfloat", 9489, new ArrayDataType(DataType.FLOAT, 99)), customtypes.getField("arrayfloat"));
 
         DocumentTypeManagerConfigurer.configure(manager, "file:src/test/document/documentmanager.updated.cfg");
 
         banana = manager.getDocumentType(new DataTypeName("banana"));
         customtypes = manager.getDocumentType(new DataTypeName("customtypes"));
 
-        assertEquals(new Field("newfield", 12345, DataType.STRING, true), banana.getField("newfield"));
+        assertEquals(new Field("newfield", 12345, DataType.STRING), banana.getField("newfield"));
         assertNull(customtypes.getField("arrayfloat"));
     }
 
@@ -559,6 +556,35 @@ search annotationsimplicitstruct {
         assertTrue(fieldRefType.getTargetType() == targetDocType);
     }
 
+    @Test
+    public void imported_fields_are_empty_if_no_fields_provided_in_config() {
+        var manager = createConfiguredManager("file:src/test/document/documentmanager.singlereference.cfg");
+        var docType = manager.getDocumentType("type_with_ref");
+
+        assertNotNull(docType.getImportedFieldNames());
+        assertEquals(docType.getImportedFieldNames().size(), 0);
+        assertFalse(docType.hasImportedField("foo"));
+    }
+
+    @Test
+    public void imported_fields_are_populated_from_config() {
+        var manager = createConfiguredManager("file:src/test/document/documentmanager.importedfields.cfg");
+        var docType = manager.getDocumentType("type_with_ref");
+
+        var expectedFields = new HashSet<String>();
+        expectedFields.add("my_cool_imported_field");
+        expectedFields.add("my_awesome_imported_field");
+        assertEquals(docType.getImportedFieldNames(), expectedFields);
+
+        assertTrue(docType.hasImportedField("my_cool_imported_field"));
+        assertTrue(docType.hasImportedField("my_awesome_imported_field"));
+        assertFalse(docType.hasImportedField("a_missing_imported_field"));
+    }
+
+    // TODO test clone(). Also fieldSets not part of clone()..!
+
+    // TODO add imported field to equals()/hashCode() for DocumentType? fieldSets not part of this...
+
     // TODO test reference to own doc type
 
 }
diff --git a/document/src/test/java/com/yahoo/document/DocumentUpdateTestCase.java b/document/src/test/java/com/yahoo/document/DocumentUpdateTestCase.java
index c381a093b4e..9f05a4441b2 100644
--- a/document/src/test/java/com/yahoo/document/DocumentUpdateTestCase.java
+++ b/document/src/test/java/com/yahoo/document/DocumentUpdateTestCase.java
@@ -799,8 +799,8 @@ public class DocumentUpdateTestCase {
         public TensorUpdateSerializeFixture() {
             docMan = new DocumentTypeManager();
             docType = new DocumentType("test");
-            docType.addHeaderField("sparse_tensor", new TensorDataType(TensorType.fromSpec("tensor(x{})")));
-            docType.addHeaderField("dense_tensor", new TensorDataType(TensorType.fromSpec("tensor(x[4])")));
+            docType.addField("sparse_tensor", new TensorDataType(TensorType.fromSpec("tensor(x{})")));
+            docType.addField("dense_tensor", new TensorDataType(TensorType.fromSpec("tensor(x[4])")));
             docMan.registerDocumentType(docType);
         }
 
diff --git a/document/src/test/java/com/yahoo/document/IdIdStringTest.java b/document/src/test/java/com/yahoo/document/IdIdStringTest.java
index 88b5f38ec0c..7915f2c8b45 100644
--- a/document/src/test/java/com/yahoo/document/IdIdStringTest.java
+++ b/document/src/test/java/com/yahoo/document/IdIdStringTest.java
@@ -2,9 +2,11 @@
 package com.yahoo.document;
 
 import com.yahoo.document.idstring.IdIdString;
+import com.yahoo.document.idstring.IdString;
 import org.junit.Test;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
 /**
@@ -50,6 +52,7 @@ public class IdIdStringTest {
             new IdIdString("namespace", "type", "illegal=key", "foo");
             fail();
         } catch (IllegalArgumentException e) {
+            assertEquals("Illegal key 'illegal'", e.getMessage());
         }
     }
 
@@ -59,6 +62,35 @@ public class IdIdStringTest {
             new IdIdString("namespace", "type", "illegal-pair", "foo");
             fail();
         } catch (IllegalArgumentException e) {
+            assertEquals("Illegal key-value pair 'illegal-pair'", e.getMessage());
+        }
+    }
+
+    @Test
+    public void requireThatTooLongPreNamespaceSpecificThrowsWhileParsing() throws Exception {
+        StringBuilder builder = new StringBuilder("id:");
+        for (int i = 0; i < 0x10000; i++) {
+            builder.append('n');
+        }
+        builder.append(":type::namespacespecificpart_01");
+        try {
+            IdString.createIdString(builder.toString());
+            fail();
+        } catch (IllegalArgumentException e) {
+            assertEquals("Document id prior to the namespace specific part, 65545, is longer than 65534", e.getMessage().substring(0, 77));
+        }
+    }
+    @Test
+    public void requireThatTooLongPreNamespaceSpecificThrowsOnConstruction() {
+        StringBuilder builder = new StringBuilder();
+        for (int i = 0; i < 0x10000; i++) {
+            builder.append('n');
+        }
+        try {
+            new IdIdString(builder.toString(), "type", "", "namespacespecificpart_01");
+            fail();
+        } catch (IllegalArgumentException e) {
+            assertEquals("Length of namespace(65536) + doctype(4) + key/values(0), is longer than 65529", e.getMessage());
         }
     }
 
diff --git a/document/src/test/java/com/yahoo/document/IncompatibleFieldTypesTest.java b/document/src/test/java/com/yahoo/document/IncompatibleFieldTypesTest.java
index 286f7d72b24..7cf0adf6ed1 100644
--- a/document/src/test/java/com/yahoo/document/IncompatibleFieldTypesTest.java
+++ b/document/src/test/java/com/yahoo/document/IncompatibleFieldTypesTest.java
@@ -17,9 +17,9 @@ public class IncompatibleFieldTypesTest {
     public void setUp() {
         arrayOfStrings = new ArrayDataType(DataType.STRING);
         struct = new StructDataType("fancypants");
-        struct.addField(new Field("stringarray", arrayOfStrings, false));
+        struct.addField(new Field("stringarray", arrayOfStrings));
         DataType weightedSetOfStrings = DataType.getWeightedSet(DataType.STRING, false, false);
-        struct.addField(new Field("stringws", weightedSetOfStrings, false));
+        struct.addField(new Field("stringws", weightedSetOfStrings));
 
         root = struct.createFieldValue();
         root.setFieldValue("stringarray", arrayOfStrings.createFieldValue());
diff --git a/document/src/test/java/com/yahoo/document/datatypes/StructTestCase.java b/document/src/test/java/com/yahoo/document/datatypes/StructTestCase.java
index 4197938f0ee..295cfa37e89 100644
--- a/document/src/test/java/com/yahoo/document/datatypes/StructTestCase.java
+++ b/document/src/test/java/com/yahoo/document/datatypes/StructTestCase.java
@@ -20,14 +20,14 @@ public class StructTestCase {
     @Test
     public void testBasicStuff() throws Exception {
         StructDataType type = new StructDataType("teststr");
-        type.addField(new Field("int", 0, DataType.INT, true));
-        type.addField(new Field("flt", 1, DataType.FLOAT, true));
-        type.addField(new Field("str", 2, DataType.STRING, true));
-        type.addField(new Field("raw", 3, DataType.RAW, true));
-        type.addField(new Field("lng", 4, DataType.LONG, true));
-        type.addField(new Field("dbl", 5, DataType.DOUBLE, true));
-        type.addField(new Field("uri", 6, DataType.URI, true));
-        type.addField(new Field("byt", 8, DataType.BYTE, true));
+        type.addField(new Field("int", 0, DataType.INT));
+        type.addField(new Field("flt", 1, DataType.FLOAT));
+        type.addField(new Field("str", 2, DataType.STRING));
+        type.addField(new Field("raw", 3, DataType.RAW));
+        type.addField(new Field("lng", 4, DataType.LONG));
+        type.addField(new Field("dbl", 5, DataType.DOUBLE));
+        type.addField(new Field("uri", 6, DataType.URI));
+        type.addField(new Field("byt", 8, DataType.BYTE));
 
         Struct struct = new Struct(type);
         {
@@ -236,7 +236,7 @@ public class StructTestCase {
     @Test
     public void testSetUnknownType() {
         StructDataType type = new StructDataType("teststr");
-        type.addField(new Field("int", 0, DataType.INT, true));
+        type.addField(new Field("int", 0, DataType.INT));
 
         Struct struct = new Struct(type);
         try {
@@ -251,9 +251,9 @@ public class StructTestCase {
     public void testCompareToDoesNotMutateStateBug6394548() {
         StructDataType type = new StructDataType("test");
         // NOTE: non-increasing ID order!
-        type.addField(new Field("int", 2, DataType.INT, true));
-        type.addField(new Field("flt", 1, DataType.FLOAT, true));
-        type.addField(new Field("str", 0, DataType.STRING, true));
+        type.addField(new Field("int", 2, DataType.INT));
+        type.addField(new Field("flt", 1, DataType.FLOAT));
+        type.addField(new Field("str", 0, DataType.STRING));
 
         Struct a = new Struct(type);
         a.setFieldValue("int", new IntegerFieldValue(123));
diff --git a/document/src/test/java/com/yahoo/document/fieldset/FieldSetTestCase.java b/document/src/test/java/com/yahoo/document/fieldset/FieldSetTestCase.java
index c11c58ff729..404b069277b 100644
--- a/document/src/test/java/com/yahoo/document/fieldset/FieldSetTestCase.java
+++ b/document/src/test/java/com/yahoo/document/fieldset/FieldSetTestCase.java
@@ -98,13 +98,13 @@ public class FieldSetTestCase extends DocumentTestCaseBase {
         assertFalse(new DocIdOnly().contains(headerField));
 
         assertTrue(new HeaderFields().contains(headerField));
-        assertFalse(new HeaderFields().contains(bodyField));
+        assertTrue(new HeaderFields().contains(bodyField));
         assertTrue(new HeaderFields().contains(new DocIdOnly()));
         assertTrue(new HeaderFields().contains(new NoFields()));
 
-        assertContains("[body]", "testdoc:rawattr");
+        assertNotContains("[body]", "testdoc:rawattr");
         assertContains("[header]", "testdoc:intattr");
-        assertNotContains("[header]", "testdoc:rawattr");
+        assertContains("[header]", "testdoc:rawattr");
         assertContains("testdoc:rawattr,intattr", "testdoc:intattr");
         assertNotContains("testdoc:intattr", "testdoc:rawattr,intattr");
         assertContains("testdoc:intattr,rawattr", "testdoc:rawattr,intattr");
@@ -141,10 +141,10 @@ public class FieldSetTestCase extends DocumentTestCaseBase {
         Document doc = getTestDocument();
         doc.removeFieldValue("rawattr");
 
-        assertEquals("floatattr:3.56", doCopyFields(doc, "[body]"));
-        assertEquals("stringattr:tjohei,intattr:50,byteattr:30,floatattr:3.56", doCopyFields(doc, "[all]"));
-        assertEquals("stringattr:tjohei,intattr:50,byteattr:30", doCopyFields(doc, "[header]"));
-        assertEquals("byteattr:30,floatattr:3.56", doCopyFields(doc, "testdoc:floatattr,byteattr"));
+        assertEquals("", doCopyFields(doc, "[body]"));
+        assertEquals("floatattr:3.56,stringattr:tjohei,intattr:50,byteattr:30", doCopyFields(doc, "[header]"));
+        assertEquals("floatattr:3.56,stringattr:tjohei,intattr:50,byteattr:30", doCopyFields(doc, "[all]"));
+        assertEquals("floatattr:3.56,byteattr:30", doCopyFields(doc, "testdoc:floatattr,byteattr"));
     }
 
     String doStripFields(Document source, String fieldSet) {
@@ -159,10 +159,10 @@ public class FieldSetTestCase extends DocumentTestCaseBase {
         Document doc = getTestDocument();
         doc.removeFieldValue("rawattr");
 
-        assertEquals("floatattr:3.56", doStripFields(doc, "[body]"));
-        assertEquals("stringattr:tjohei,intattr:50,byteattr:30,floatattr:3.56", doStripFields(doc, "[all]"));
-        assertEquals("stringattr:tjohei,intattr:50,byteattr:30", doStripFields(doc, "[header]"));
-        assertEquals("byteattr:30,floatattr:3.56", doStripFields(doc, "testdoc:floatattr,byteattr"));
+        assertEquals("", doStripFields(doc, "[body]"));
+        assertEquals("floatattr:3.56,stringattr:tjohei,intattr:50,byteattr:30", doStripFields(doc, "[header]"));
+        assertEquals("floatattr:3.56,stringattr:tjohei,intattr:50,byteattr:30", doStripFields(doc, "[all]"));
+        assertEquals("floatattr:3.56,byteattr:30", doStripFields(doc, "testdoc:floatattr,byteattr"));
     }
 
     @Test
diff --git a/document/src/test/java/com/yahoo/document/select/DocumentSelectorTestCase.java b/document/src/test/java/com/yahoo/document/select/DocumentSelectorTestCase.java
index 0ff0bd81d90..2e08814bf43 100644
--- a/document/src/test/java/com/yahoo/document/select/DocumentSelectorTestCase.java
+++ b/document/src/test/java/com/yahoo/document/select/DocumentSelectorTestCase.java
@@ -8,10 +8,13 @@ import com.yahoo.document.select.parser.ParseException;
 import com.yahoo.document.select.parser.TokenMgrException;
 import com.yahoo.yolean.Exceptions;
 import org.junit.Before;
+import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.ExpectedException;
 
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 
@@ -26,20 +29,24 @@ import static org.junit.Assert.fail;
  */
 public class DocumentSelectorTestCase {
 
+    @Rule
+    public final ExpectedException exceptionRule = ExpectedException.none();
+
     private static DocumentTypeManager manager = new DocumentTypeManager();
 
     @Before
     public void setUp() {
-        DocumentType type = new DocumentType("test");
-        type.addHeaderField("hint", DataType.INT);
-        type.addHeaderField("hfloat", DataType.FLOAT);
-        type.addHeaderField("hstring", DataType.STRING);
+        var importedFields = new HashSet<>(List.of("my_imported_field"));
+        DocumentType type = new DocumentType("test", importedFields);
+        type.addField("hint", DataType.INT);
+        type.addField("hfloat", DataType.FLOAT);
+        type.addField("hstring", DataType.STRING);
         type.addField("content", DataType.STRING);
 
         StructDataType mystruct = new StructDataType("mystruct");
-        mystruct.addField(new Field("key", DataType.INT, false));
-        mystruct.addField(new Field("value", DataType.STRING, false));
-        type.addHeaderField("mystruct", mystruct);
+        mystruct.addField(new Field("key", DataType.INT));
+        mystruct.addField(new Field("value", DataType.STRING));
+        type.addField("mystruct", mystruct);
 
         ArrayDataType structarray = new ArrayDataType(mystruct);
         type.addField("structarray", structarray);
@@ -75,6 +82,7 @@ public class DocumentSelectorTestCase {
         assertParse("id.hash() > 0");
         assertParse("id.namespace.hash() > 0");
         assertParse("music.artist = \"*\"");
+        assertParse("music.artist = \"&\"");
         assertParse("music.artist.lowercase() = \"*\"");
         assertParse("music_.artist = \"*\"");
         assertParse("music_foo.artist = \"*\"");
@@ -332,6 +340,7 @@ public class DocumentSelectorTestCase {
         documents.add(createDocument("id:myspace:test:n=2345:mail2", 15, 1.0f, "bar", "baz"));
         documents.add(createDocument("id:myspace:test:g=mygroup:qux", 15, 1.0f, "quux", "corge"));
         documents.add(createDocument("id:myspace:test::missingint", null, 2.0f, null, "bar"));
+        documents.add(createDocument("id:myspace:test::ampersand", null, 2.0f, null, "&"));
 
         // Add some array/struct info to doc 1
         Struct sval = new Struct(documents.get(1).getDocument().getField("mystruct").getDataType());
@@ -476,6 +485,8 @@ public class DocumentSelectorTestCase {
         assertEquals(Result.FALSE, evaluate("test.hstring == test.content", documents.get(0)));
         assertEquals(Result.TRUE, evaluate("test.hstring == test.content", documents.get(2)));
         assertEquals(Result.TRUE, evaluate("test.hint + 1 > 13", documents.get(1)));
+        assertEquals(Result.FALSE, evaluate("test.content = \"&\"", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("test.content = \"&\"", documents.get(7)));
         // Case where field is not present (i.e. null) is defined for (in)equality comparisons, but
         // not for other relations.
         DocumentPut doc1234 = documents.get(6);
@@ -697,6 +708,44 @@ public class DocumentSelectorTestCase {
     }
 
     @Test
+    public void using_non_commutative_comparison_operator_with_field_value_is_well_defined() throws ParseException {
+        var documents = createDocs();
+        // Doc 0 contains 24 in `hint` field.
+        assertEquals(Result.FALSE, evaluate("25 <= test.hint", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("24 <= test.hint", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("25 > test.hint", documents.get(0)));
+        assertEquals(Result.FALSE, evaluate("24 > test.hint", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("24 >= test.hint", documents.get(0)));
+
+        assertEquals(Result.FALSE, evaluate("test.hint <= 23", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("test.hint <= 24", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("test.hint > 23", documents.get(0)));
+        assertEquals(Result.FALSE, evaluate("test.hint > 24", documents.get(0)));
+        assertEquals(Result.TRUE, evaluate("test.hint >= 24", documents.get(0)));
+    }
+
+    @Test
+    public void imported_field_references_are_treated_as_valid_field_with_missing_value() throws ParseException {
+        var documents = createDocs();
+        assertEquals(Result.TRUE, evaluate("test.my_imported_field == null", documents.get(0)));
+        assertEquals(Result.FALSE, evaluate("test.my_imported_field != null", documents.get(0)));
+        assertEquals(Result.FALSE, evaluate("test.my_imported_field", documents.get(0)));
+        // Only (in)equality operators are well defined for null values; everything else becomes Invalid.
+        assertEquals(Result.INVALID, evaluate("test.my_imported_field > 0", documents.get(0)));
+    }
+
+    @Test
+    public void imported_fields_only_supported_for_simple_expressions() throws ParseException {
+        exceptionRule.expect(IllegalArgumentException.class);
+        // TODO we should probably handle this case specially and give a better exception message
+        exceptionRule.expectMessage("Field 'my_imported_field' not found in type datatype test");
+
+        var documents = createDocs();
+        // Nested field access is NOT considered a simple expression.
+        evaluate("test.my_imported_field.foo", documents.get(0));
+    }
+
+    @Test
     public void testTicket1769674() {
         assertParseError("music.uri=\"junk",
                          "Lexical error at line -1, column 17.  Encountered: <EOF> after : \"\\\"junk\"");
@@ -829,7 +878,6 @@ public class DocumentSelectorTestCase {
         } catch (ParseException e) {
             fail("The expression '" + expressionString + "' should assertEquals ok.");
         } catch (RuntimeException e) {
-            System.err.println("Error was : " + e);
             assertTrue(e.getMessage().length() >= expectedError.length());
             assertEquals(expectedError, e.getMessage().substring(0, expectedError.length()));
         }
diff --git a/document/src/test/resources/predicates/false__java b/document/src/test/resources/predicates/false__java
index 00a71d5fe73..c9f426b3aaf 100644
--- a/document/src/test/resources/predicates/false__java
+++ b/document/src/test/resources/predicates/false__java
diff --git a/document/src/test/resources/predicates/foo_in_6_9__java b/document/src/test/resources/predicates/foo_in_6_9__java
index d18b937bead..9c1413f1b77 100644
--- a/document/src/test/resources/predicates/foo_in_6_9__java
+++ b/document/src/test/resources/predicates/foo_in_6_9__java
diff --git a/document/src/test/resources/predicates/foo_in_6_x__java b/document/src/test/resources/predicates/foo_in_6_x__java
index b8ae334882e..d53f4d03996 100644
--- a/document/src/test/resources/predicates/foo_in_6_x__java
+++ b/document/src/test/resources/predicates/foo_in_6_x__java
diff --git a/document/src/test/resources/predicates/foo_in_bar__java b/document/src/test/resources/predicates/foo_in_bar__java
index aeb10e2b5d7..3adde3ae2fe 100644
--- a/document/src/test/resources/predicates/foo_in_bar__java
+++ b/document/src/test/resources/predicates/foo_in_bar__java
diff --git a/document/src/test/resources/predicates/foo_in_bar_and_baz_in_cox__java b/document/src/test/resources/predicates/foo_in_bar_and_baz_in_cox__java
index 8a2705e6f62..8583f37da57 100644
--- a/document/src/test/resources/predicates/foo_in_bar_and_baz_in_cox__java
+++ b/document/src/test/resources/predicates/foo_in_bar_and_baz_in_cox__java
diff --git a/document/src/test/resources/predicates/foo_in_bar_baz__java b/document/src/test/resources/predicates/foo_in_bar_baz__java
index ea3314d9bd7..1db1a175c0c 100644
--- a/document/src/test/resources/predicates/foo_in_bar_baz__java
+++ b/document/src/test/resources/predicates/foo_in_bar_baz__java
diff --git a/document/src/test/resources/predicates/foo_in_bar_or_baz_in_cox__java b/document/src/test/resources/predicates/foo_in_bar_or_baz_in_cox__java
index 4d5474c24e4..3b1dbd541fa 100644
--- a/document/src/test/resources/predicates/foo_in_bar_or_baz_in_cox__java
+++ b/document/src/test/resources/predicates/foo_in_bar_or_baz_in_cox__java
diff --git a/document/src/test/resources/predicates/foo_in_x_9__java b/document/src/test/resources/predicates/foo_in_x_9__java
index 017a610a7d5..9a2bfc9af89 100644
--- a/document/src/test/resources/predicates/foo_in_x_9__java
+++ b/document/src/test/resources/predicates/foo_in_x_9__java
diff --git a/document/src/test/resources/predicates/foo_in_x__java b/document/src/test/resources/predicates/foo_in_x__java
index 6537cc6bdeb..8faf7762be7 100644
--- a/document/src/test/resources/predicates/foo_in_x__java
+++ b/document/src/test/resources/predicates/foo_in_x__java
diff --git a/document/src/test/resources/predicates/foo_in_x_x__java b/document/src/test/resources/predicates/foo_in_x_x__java
index 5060718417a..bf01a265651 100644
--- a/document/src/test/resources/predicates/foo_in_x_x__java
+++ b/document/src/test/resources/predicates/foo_in_x_x__java
diff --git a/document/src/test/resources/predicates/not_foo_in_bar__java b/document/src/test/resources/predicates/not_foo_in_bar__java
index b654de6d53e..a7cbacbaf35 100644
--- a/document/src/test/resources/predicates/not_foo_in_bar__java
+++ b/document/src/test/resources/predicates/not_foo_in_bar__java
diff --git a/document/src/test/resources/predicates/true__java b/document/src/test/resources/predicates/true__java
index 2b5da7409d5..87356a5ed44 100644
--- a/document/src/test/resources/predicates/true__java
+++ b/document/src/test/resources/predicates/true__java
diff --git a/document/src/test/resources/tensor/empty_tensor__java b/document/src/test/resources/tensor/empty_tensor__java
index 2c15c152558..cf878f0e689 100644
--- a/document/src/test/resources/tensor/empty_tensor__java
+++ b/document/src/test/resources/tensor/empty_tensor__java
diff --git a/document/src/test/resources/tensor/multi_cell_tensor__java b/document/src/test/resources/tensor/multi_cell_tensor__java
index d923fc10559..deb53463fb5 100644
--- a/document/src/test/resources/tensor/multi_cell_tensor__java
+++ b/document/src/test/resources/tensor/multi_cell_tensor__java
diff --git a/document/src/test/resources/tensor/non_existing_tensor__java b/document/src/test/resources/tensor/non_existing_tensor__java
index 08cbcac6dd3..7a1d95ff132 100644
--- a/document/src/test/resources/tensor/non_existing_tensor__java
+++ b/document/src/test/resources/tensor/non_existing_tensor__java
diff --git a/document/src/tests/data/serializejava-compressed.dat b/document/src/tests/data/serializejava-compressed.dat
index 0ac391422ff..e11bb3b53db 100644
--- a/document/src/tests/data/serializejava-compressed.dat
+++ b/document/src/tests/data/serializejava-compressed.dat
diff --git a/document/src/tests/data/serializejava.dat b/document/src/tests/data/serializejava.dat
index 10873c9d905..3fa21bdccd3 100644
--- a/document/src/tests/data/serializejava.dat
+++ b/document/src/tests/data/serializejava.dat
diff --git a/document/src/tests/data/serializejavawithannotations.dat b/document/src/tests/data/serializejavawithannotations.dat
index fe683d1580e..08854d03a29 100644
--- a/document/src/tests/data/serializejavawithannotations.dat
+++ b/document/src/tests/data/serializejavawithannotations.dat
diff --git a/document/src/tests/documentselectparsertest.cpp b/document/src/tests/documentselectparsertest.cpp
index 79b849c5ba9..9ac402f56ef 100644
--- a/document/src/tests/documentselectparsertest.cpp
+++ b/document/src/tests/documentselectparsertest.cpp
@@ -16,7 +16,9 @@
 #include <vespa/document/select/invalidconstant.h>
 #include <vespa/document/select/doctype.h>
 #include <vespa/document/select/compare.h>
+#include <vespa/document/select/operator.h>
 #include <vespa/document/select/parse_utils.h>
+#include <vespa/document/select/parser_limits.h>
 #include <vespa/vespalib/util/exceptions.h>
 #include <limits>
 #include <gtest/gtest.h>
@@ -32,6 +34,8 @@ protected:
     std::vector<Document::SP > _doc;
     std::vector<DocumentUpdate::SP > _update;
 
+    ~DocumentSelectParserTest();
+
     Document::SP createDoc(
             const std::string& doctype, const std::string& id, uint32_t hint,
             double hfloat, const std::string& hstr, const std::string& cstr,
@@ -56,16 +60,6 @@ protected:
     void SetUp() override;
     void createDocs();
 
-    void testOperators0();
-    void testOperators1();
-    void testOperators2();
-    void testOperators3();
-    void testOperators4();
-    void testOperators5();
-    void testOperators6();
-    void testOperators7();
-    void testOperators8();
-    void testOperators9();
     void testDocumentUpdates0();
     void testDocumentUpdates1();
     void testDocumentUpdates2();
@@ -73,6 +67,7 @@ protected:
     void testDocumentUpdates4();
 };
 
+DocumentSelectParserTest::~DocumentSelectParserTest() = default;
 
 namespace {
     std::shared_ptr<const DocumentTypeRepo> _repo;
@@ -92,6 +87,10 @@ void DocumentSelectParserTest::SetUp()
     builder.document(-1673092522, "usergroup",
                      Struct("usergroup.header"),
                      Struct("usergroup.body"));
+    builder.document(1234567, "with_imported",
+                     Struct("with_imported.header"),
+                     Struct("with_imported.body"))
+                     .imported_field("my_imported_field");
     _repo = std::make_unique<DocumentTypeRepo>(builder.config());
 
     _parser = std::make_unique<select::Parser>(*_repo, _bucketIdFactory);
@@ -103,7 +102,7 @@ Document::SP DocumentSelectParserTest::createDoc(
         uint64_t hlong)
 {
     const DocumentType* type = _repo->getDocumentType(doctype);
-    Document::SP doc(new Document(*type, DocumentId(id)));
+    auto doc = std::make_shared<Document>(*type, DocumentId(id));
     doc->setValue(doc->getField("headerval"), IntFieldValue(hint));
 
     if (hlong != 0) {
@@ -498,21 +497,7 @@ DocumentSelectParserTest::doParse(vespalib::stringref expr,
     EXPECT_EQ(select::ResultList(select::Result::result), \
               doParse(expr, (doc).getId())) << (std::string("Doc id: ") + expr);
 
-TEST_F(DocumentSelectParserTest, testOperators)
-{
-    testOperators0();
-    testOperators1();
-    testOperators2();
-    testOperators3();
-    testOperators4();
-    testOperators5();
-    testOperators6();
-    testOperators7();
-    testOperators8();
-    testOperators9();
-}
-
-void DocumentSelectParserTest::testOperators0()
+TEST_F(DocumentSelectParserTest, operators_0)
 {
     createDocs();
 
@@ -551,8 +536,17 @@ void DocumentSelectParserTest::testOperators0()
     PARSE("\"foo\" == 'foo'", *_doc[0], True);
     PARSE("\"bar\" = \"a\"", *_doc[0], False);
     PARSE("\"bar\" = \"*a*\"", *_doc[0], True);
+    PARSE("\"bar\" = \"*x*\"", *_doc[0], False);
+    PARSE("\"bar\" = \"ba*\"", *_doc[0], True);
+    PARSE("\"bar\" = \"a*\"", *_doc[0], False)
+    PARSE("\"bar\" = \"*ar\"", *_doc[0], True);
+    PARSE("\"bar\" = \"*a\"", *_doc[0], False);
     PARSE("\"bar\" = \"\"", *_doc[0], False);
     PARSE("\"\" = \"\"", *_doc[0], True);
+    PARSE("\"\" = \"*\"", *_doc[0], True);
+    PARSE("\"\" = \"****\"", *_doc[0], True);
+    PARSE("\"a\" = \"*?*\"", *_doc[0], True);
+    PARSE("\"a\" = \"*??*\"", *_doc[0], False);
     PARSE("\"bar\" =~ \"^a$\"", *_doc[0], False);
     PARSE("\"bar\" =~ \"a\"", *_doc[0], True);
     PARSE("\"bar\" =~ \"\"", *_doc[0], True);
@@ -561,7 +555,47 @@ void DocumentSelectParserTest::testOperators0()
     PARSE("30 = 30", *_doc[0], True);
 }
 
-void DocumentSelectParserTest::testOperators1()
+TEST_F(DocumentSelectParserTest, using_non_commutative_comparison_operator_with_field_value_is_well_defined) {
+    auto doc = createDoc("testdoctype1", "id:foo:testdoctype1::bar", 24, 0.0, "foo", "bar", 0);
+    // Document's `headerval` field has value of 24.
+    PARSE("25 <= testdoctype1.headerval", *doc, False);
+    PARSE("24 <= testdoctype1.headerval", *doc, True);
+    PARSE("25 > testdoctype1.headerval", *doc, True);
+    PARSE("24 > testdoctype1.headerval", *doc, False);
+    PARSE("24 >= testdoctype1.headerval", *doc, True);
+
+    PARSE("testdoctype1.headerval <= 23", *doc, False);
+    PARSE("testdoctype1.headerval <= 24", *doc, True);
+    PARSE("testdoctype1.headerval > 23", *doc, True);
+    PARSE("testdoctype1.headerval > 24", *doc, False);
+    PARSE("testdoctype1.headerval >= 24", *doc, True);
+}
+
+TEST_F(DocumentSelectParserTest, regex_matching_does_not_bind_anchors_to_newlines) {
+    createDocs();
+
+    PARSE("\"a\\nb\\nc\" =~ \"^b$\"", *_doc[0], False);
+    PARSE("\"a\\r\\nb\\r\\nc\" =~ \"^b$\"", *_doc[0], False);
+    // Same applies to implicit regex created from glob expression
+    PARSE("\"a\\nb\\nc\" = \"b\"", *_doc[0], False);
+}
+
+// With a recursive backtracking regex implementation like that found in (at the time of
+// writing) GCC's std::regex implementation, certain expressions on a sufficiently large
+// input will cause a stack overflow and send the whole thing spiraling into a flaming
+// vortex of doom. See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=86164 for context.
+//
+// Since crashing the process based on user input is considered bad karma for all the
+// obvious reasons, test that the underlying regex engine is not susceptible to such
+// crashes.
+TEST_F(DocumentSelectParserTest, regex_matching_is_not_susceptible_to_catastrophic_backtracking) {
+    std::string long_string(1024*50, 'A'); // -> hstringval field
+    auto doc = createDoc("testdoctype1", "id:foo:testdoctype1::bar", 24, 0.0, long_string, "bar", 0);
+    // This _will_ crash std::regex on GCC 8.3. Don't try this at home. Unless you want to.
+    PARSE(R"(testdoctype1.hstringval =~ ".*")", *doc, True);
+}
+
+TEST_F(DocumentSelectParserTest, operators_1)
 {
     createDocs();
 
@@ -608,7 +642,7 @@ void DocumentSelectParserTest::testOperators1()
     PARSE("testdoctype1.headerval = 10", *_doc[4], True);
 }
 
-void DocumentSelectParserTest::testOperators2()
+TEST_F(DocumentSelectParserTest, operators_2)
 {
     createDocs();
 
@@ -633,7 +667,7 @@ void DocumentSelectParserTest::testOperators2()
     PARSEI("id.group == \"xyzzy\"", *_doc[10], True);
 }
 
-void DocumentSelectParserTest::testOperators3()
+TEST_F(DocumentSelectParserTest, operators_3)
 {
     createDocs();
     {
@@ -666,7 +700,7 @@ void DocumentSelectParserTest::testOperators3()
     PARSEI("id == \"id:footype:testdoctype1:n=123456789:badger\"", *_doc[5], False);
 }
 
-void DocumentSelectParserTest::testOperators4()
+TEST_F(DocumentSelectParserTest, operators_4)
 {
     createDocs();
 
@@ -693,7 +727,7 @@ void DocumentSelectParserTest::testOperators4()
     PARSE("false or testdoctype1.content = 1", *_doc[0], Invalid);
 }
 
-void DocumentSelectParserTest::testOperators5()
+TEST_F(DocumentSelectParserTest, operators_5)
 {
     createDocs();
 
@@ -722,7 +756,7 @@ void DocumentSelectParserTest::testOperators5()
     PARSEI("-6 % 10 = -6", *_doc[0], True);
 }
 
-void DocumentSelectParserTest::testOperators6()
+TEST_F(DocumentSelectParserTest, operators_6)
 {
     createDocs();
 
@@ -766,7 +800,7 @@ void DocumentSelectParserTest::testOperators6()
     PARSE("testdoctype1.headerlongval<0", *_doc[7], True);
 }
 
-void DocumentSelectParserTest::testOperators7()
+TEST_F(DocumentSelectParserTest, operators_7)
 {
     createDocs();
 
@@ -803,7 +837,7 @@ void DocumentSelectParserTest::testOperators7()
     PARSE("testdoctype1.structarray[$x].key == 15 AND testdoctype1.structarray[$y].value == \"structval2\"", *_doc[1], True);
 }
 
-void DocumentSelectParserTest::testOperators8()
+TEST_F(DocumentSelectParserTest, operators_8)
 {
     createDocs();
 
@@ -836,7 +870,7 @@ void DocumentSelectParserTest::testOperators8()
     PARSE("testdoctype1.structarrmap{$x}[$y].key == 15 AND testdoctype1.structarrmap{$y}[$x].value == \"structval2\"", *_doc[1], False);
 }
 
-void DocumentSelectParserTest::testOperators9()
+TEST_F(DocumentSelectParserTest, operators_9)
 {
     createDocs();
 
@@ -1047,6 +1081,11 @@ void DocumentSelectParserTest::testDocumentUpdates0()
     PARSEI("\"foo\" == \"foo\"", *_update[0], True);
     PARSEI("\"bar\" = \"a\"", *_update[0], False);
     PARSEI("\"bar\" = \"*a*\"", *_update[0], True);
+    PARSEI("\"bar\" = \"**\"", *_update[0], True);
+    PARSEI("\"bar\" = \"***\"", *_update[0], True);
+    PARSEI("\"bar\" = \"****\"", *_update[0], True);
+    PARSEI("\"bar\" = \"???\"", *_update[0], True);
+    PARSEI("\"bar\" = \"????\"", *_update[0], False);
     PARSEI("\"bar\" = \"\"", *_update[0], False);
     PARSEI("\"\" = \"\"", *_update[0], True);
     PARSEI("\"bar\" =~ \"^a$\"", *_update[0], False);
@@ -1212,17 +1251,17 @@ TEST_F(DocumentSelectParserTest, testThatSimpleFieldValuesHaveCorrectFieldName)
 
 TEST_F(DocumentSelectParserTest, testThatComplexFieldValuesHaveCorrectFieldNames)
 {
-    EXPECT_EQ(
-        vespalib::string("headerval"),
-        parseFieldValue("testdoctype1.headerval{test}")->getRealFieldName());
+    EXPECT_EQ(vespalib::string("headerval"),
+              parseFieldValue("testdoctype1.headerval{test}")->getRealFieldName());
 
-    EXPECT_EQ(
-        vespalib::string("headerval"),
-        parseFieldValue("testdoctype1.headerval[42]")->getRealFieldName());
+    EXPECT_EQ(vespalib::string("headerval"),
+              parseFieldValue("testdoctype1.headerval[42]")->getRealFieldName());
 
-    EXPECT_EQ(
-        vespalib::string("headerval"),
-        parseFieldValue("testdoctype1.headerval.meow.meow{test}")->getRealFieldName());
+    EXPECT_EQ(vespalib::string("headerval"),
+              parseFieldValue("testdoctype1.headerval.meow.meow{test}")->getRealFieldName());
+
+    EXPECT_EQ(vespalib::string("headerval"),
+              parseFieldValue("testdoctype1.headerval .meow.meow{test}")->getRealFieldName());
 }
 
 namespace {
@@ -1524,4 +1563,108 @@ TEST_F(DocumentSelectParserTest, test_parse_utilities_handle_malformed_input)
     check_parse_double("1.79769e+309", true, std::numeric_limits<double>::infinity());
 }
 
+TEST_F(DocumentSelectParserTest, imported_field_references_are_treated_as_valid_field_with_missing_value) {
+    const DocumentType* type = _repo->getDocumentType("with_imported");
+    ASSERT_TRUE(type != nullptr);
+    Document doc(*type, DocumentId("id::with_imported::foo"));
+
+    PARSE("with_imported.my_imported_field == null", doc, True);
+    PARSE("with_imported.my_imported_field != null", doc, False);
+    PARSE("with_imported.my_imported_field", doc, False);
+    // Only (in)equality operators are well defined for null values; everything else becomes Invalid.
+    PARSE("with_imported.my_imported_field > 0", doc, Invalid);
+}
+
+TEST_F(DocumentSelectParserTest, imported_field_references_only_support_for_simple_expressions) {
+    const DocumentType* type = _repo->getDocumentType("with_imported");
+    ASSERT_TRUE(type != nullptr);
+    Document doc(*type, DocumentId("id::with_imported::foo"));
+
+    PARSE("with_imported.my_imported_field.foo", doc, Invalid);
+    PARSE("with_imported.my_imported_field[0]", doc, Invalid);
+    PARSE("with_imported.my_imported_field{foo}", doc, Invalid);
+}
+
+TEST_F(DocumentSelectParserTest, prefix_and_suffix_wildcard_globs_are_rewritten_to_optimized_form) {
+    using select::GlobOperator;
+    EXPECT_EQ(GlobOperator::convertToRegex("*foo"), "foo$");
+    EXPECT_EQ(GlobOperator::convertToRegex("foo*"), "^foo");
+    EXPECT_EQ(GlobOperator::convertToRegex("*foo*"), "foo");
+    EXPECT_EQ(GlobOperator::convertToRegex("*"), ""); // Matches any string.
+    EXPECT_EQ(GlobOperator::convertToRegex("**"), ""); // Still matches any string.
+}
+
+TEST_F(DocumentSelectParserTest, redundant_glob_wildcards_are_collapsed_into_minimal_form) {
+    using select::GlobOperator;
+    EXPECT_EQ(GlobOperator::convertToRegex("***"), ""); // Even still matches any string.
+    EXPECT_EQ(GlobOperator::convertToRegex("**foo**"), "foo");
+    EXPECT_EQ(GlobOperator::convertToRegex("foo***"), "^foo");
+    EXPECT_EQ(GlobOperator::convertToRegex("***foo"), "foo$");
+    EXPECT_EQ(GlobOperator::convertToRegex("foo**bar"), "^foo.*bar$");
+    EXPECT_EQ(GlobOperator::convertToRegex("**foo*bar**"), "foo.*bar");
+    EXPECT_EQ(GlobOperator::convertToRegex("**foo***bar**"), "foo.*bar");
+    EXPECT_EQ(GlobOperator::convertToRegex("*?*"), ".");
+    EXPECT_EQ(GlobOperator::convertToRegex("*?*?*?*"), "..*..*."); // Don't try this at home, kids!
+}
+
+TEST_F(DocumentSelectParserTest, recursion_depth_is_bounded_for_field_exprs) {
+    createDocs();
+    std::string expr = "testdoctype1";
+    for (size_t i = 0; i < 50000; ++i) {
+        expr += ".foo";
+    }
+    expr += ".hash() != 0";
+    verifyFailedParse(expr, "ParsingFailedException: expression is too deeply nested (max 1024 levels)");
+}
+
+TEST_F(DocumentSelectParserTest, recursion_depth_is_bounded_for_arithmetic_exprs) {
+    createDocs();
+    std::string expr = "1";
+    for (size_t i = 0; i < 50000; ++i) {
+        expr += "+1";
+    }
+    expr += " != 0";
+    verifyFailedParse(expr, "ParsingFailedException: expression is too deeply nested (max 1024 levels)");
+}
+
+TEST_F(DocumentSelectParserTest, recursion_depth_is_bounded_for_binary_logical_exprs) {
+    createDocs();
+    // Also throw in some comparisons to ensure they carry over the max depth.
+    std::string expr = "1 == 2";
+    std::string cmp_subexpr = "3 != 4";
+    for (size_t i = 0; i < 10000; ++i) {
+        expr += (i % 2 == 0 ? " and " : " or ") + cmp_subexpr;
+    }
+    verifyFailedParse(expr, "ParsingFailedException: expression is too deeply nested (max 1024 levels)");
+}
+
+TEST_F(DocumentSelectParserTest, recursion_depth_is_bounded_for_unary_logical_exprs) {
+    createDocs();
+    std::string expr;
+    for (size_t i = 0; i < 10000; ++i) {
+        expr += "not ";
+    }
+    expr += "true";
+    verifyFailedParse(expr, "ParsingFailedException: expression is too deeply nested (max 1024 levels)");
+}
+
+TEST_F(DocumentSelectParserTest, selection_has_upper_limit_on_input_size) {
+    createDocs();
+    std::string expr = ("testdoctype1.a_biii"
+                        + std::string(select::ParserLimits::MaxSelectionByteSize, 'i')
+                        + "iiig_identifier");
+    verifyFailedParse(expr, "ParsingFailedException: expression is too large to be "
+                            "parsed (max 1048576 bytes, got 1048610)");
+}
+
+TEST_F(DocumentSelectParserTest, lexing_does_not_have_superlinear_time_complexity) {
+    createDocs();
+    std::string expr = ("testdoctype1.hstringval == 'a_biii"
+                        + std::string(select::ParserLimits::MaxSelectionByteSize - 100, 'i')
+                        + "iiig string'");
+    // If the lexer is not compiled with the appropriate options, this will take a long time.
+    // A really, really long time.
+    PARSE(expr, *_doc[0], False);
+}
+
 } // document
diff --git a/document/src/tests/documenttestcase.cpp b/document/src/tests/documenttestcase.cpp
index 6993596392c..968470e9693 100644
--- a/document/src/tests/documenttestcase.cpp
+++ b/document/src/tests/documenttestcase.cpp
@@ -36,7 +36,7 @@ TEST(DocumentTest, testSizeOf)
     EXPECT_EQ(32u, sizeof(vespalib::GrowableByteBuffer));
     EXPECT_EQ(88ul, sizeof(IdString));
     EXPECT_EQ(104ul, sizeof(DocumentId));
-    EXPECT_EQ(232ul, sizeof(Document));
+    EXPECT_EQ(240ul, sizeof(Document));
     EXPECT_EQ(96ul, sizeof(StructFieldValue));
     EXPECT_EQ(16ul, sizeof(StructuredFieldValue));
     EXPECT_EQ(56ul, sizeof(SerializableArray));
@@ -585,7 +585,7 @@ TEST(DocumentTest, testReadSerializedFile)
 
     EXPECT_TRUE(buf2.empty());
     buf2.rp(0);
-    EXPECT_EQ(len - 13, buf2.size()); // Size is smaller as we are merging to one chunk.
+    EXPECT_EQ(len, buf2.size());
 
     doc2.setValue("stringfield", StringFieldValue("hei"));
 
diff --git a/document/src/tests/documentupdatetestcase.cpp b/document/src/tests/documentupdatetestcase.cpp
index 5543cb48ba4..18001c35da5 100644
--- a/document/src/tests/documentupdatetestcase.cpp
+++ b/document/src/tests/documentupdatetestcase.cpp
@@ -872,6 +872,13 @@ struct TensorUpdateFixture {
         EXPECT_EQ(actTensor, expTensor);
     }
 
+    void assertTensorNull() {
+        auto field = getTensor();
+        auto tensor_field = dynamic_cast<TensorFieldValue*>(field.get());
+        ASSERT_TRUE(tensor_field);
+        EXPECT_TRUE(tensor_field->getAsTensorPtr().get() == nullptr);
+    }
+
     void assertTensor(const TensorSpec &expSpec) {
         auto expTensor = makeTensor(expSpec);
         assertTensor(*expTensor);
@@ -886,6 +893,19 @@ struct TensorUpdateFixture {
         assertTensor(expTensor);
     }
 
+    void assertApplyUpdateNonExisting(const ValueUpdate &update,
+                                      const TensorSpec &expTensor) {
+        applyUpdate(update);
+        assertDocumentUpdated();
+        assertTensor(expTensor);
+    }
+
+    void assertApplyUpdateNonExisting(const ValueUpdate &update) {
+        applyUpdate(update);
+        assertDocumentUpdated();
+        assertTensorNull();
+    }
+
     template <typename ValueUpdateType>
     void assertRoundtripSerialize(const ValueUpdateType &valueUpdate) {
         testRoundtripSerialize(valueUpdate, tensorDataType);
@@ -933,6 +953,16 @@ TEST(DocumentUpdateTest, tensor_add_update_can_be_applied)
                                 .add({{"x", "c"}}, 7));
 }
 
+TEST(DocumentUpdateTest, tensor_add_update_can_be_applied_to_nonexisting_tensor)
+{
+    TensorUpdateFixture f;
+    f.assertApplyUpdateNonExisting(TensorAddUpdate(f.makeTensor(f.spec().add({{"x", "b"}}, 5)
+                                                                        .add({{"x", "c"}}, 7))),
+
+                        f.spec().add({{"x", "b"}}, 5)
+                                .add({{"x", "c"}}, 7));
+}
+
 TEST(DocumentUpdateTest, tensor_remove_update_can_be_applied)
 {
     TensorUpdateFixture f;
@@ -944,6 +974,12 @@ TEST(DocumentUpdateTest, tensor_remove_update_can_be_applied)
                         f.spec().add({{"x", "a"}}, 2));
 }
 
+TEST(DocumentUpdateTest, tensor_remove_update_can_be_applied_to_nonexisting_tensor)
+{
+    TensorUpdateFixture f;
+    f.assertApplyUpdateNonExisting(TensorRemoveUpdate(f.makeTensor(f.spec().add({{"x", "b"}}, 1))));
+}
+
 TEST(DocumentUpdateTest, tensor_modify_update_can_be_applied)
 {
     TensorUpdateFixture f;
@@ -970,6 +1006,13 @@ TEST(DocumentUpdateTest, tensor_modify_update_can_be_applied)
                                 .add({{"x", "b"}}, 15));
 }
 
+TEST(DocumentUpdateTest, tensor_modify_update_can_be_applied_to_nonexisting_tensor)
+{
+    TensorUpdateFixture f;
+    f.assertApplyUpdateNonExisting(TensorModifyUpdate(TensorModifyUpdate::Operation::ADD,
+                                                      f.makeTensor(f.spec().add({{"x", "b"}}, 5))));
+}
+
 TEST(DocumentUpdateTest, tensor_assign_update_can_be_roundtrip_serialized)
 {
     TensorUpdateFixture f;
diff --git a/document/src/tests/repo/documenttyperepo_test.cpp b/document/src/tests/repo/documenttyperepo_test.cpp
index b263ad75930..0bc80ebcd16 100644
--- a/document/src/tests/repo/documenttyperepo_test.cpp
+++ b/document/src/tests/repo/documenttyperepo_test.cpp
@@ -532,6 +532,47 @@ TEST("Reference fields are resolved to correct reference type") {
     EXPECT_EQUAL(*ref1_type, type->getFieldsType().getField("ref3").getDataType());
 }
 
+TEST("Config with no imported fields has empty imported fields set in DocumentType") {
+    DocumenttypesConfigBuilderHelper builder;
+    builder.document(doc_type_id, type_name,
+                     Struct(header_name), Struct(body_name));
+    DocumentTypeRepo repo(builder.config());
+    const auto *type = repo.getDocumentType(doc_type_id);
+    ASSERT_TRUE(type != nullptr);
+    EXPECT_TRUE(type->imported_field_names().empty());
+    EXPECT_FALSE(type->has_imported_field_name("foo"));
+}
+
+TEST("Configured imported field names are available in the DocumentType") {
+    const int type_2_id = doc_type_id + 1;
+    // Note: we cheat a bit by specifying imported field names in types that have no
+    // reference fields. Add to test if we add config read-time validation of this. :)
+    DocumenttypesConfigBuilderHelper builder;
+    // Type with one imported field
+    builder.document(doc_type_id, type_name,
+                     Struct(header_name), Struct(body_name))
+                     .imported_field("my_cool_field");
+    // Type with two imported fields
+    builder.document(type_2_id, type_name_2,
+                     Struct(header_name_2), Struct(body_name_2))
+                     .imported_field("my_awesome_field")
+                     .imported_field("my_swag_field");
+
+    DocumentTypeRepo repo(builder.config());
+    const auto* type = repo.getDocumentType(doc_type_id);
+    ASSERT_TRUE(type != nullptr);
+    EXPECT_EQUAL(1u, type->imported_field_names().size());
+    EXPECT_TRUE(type->has_imported_field_name("my_cool_field"));
+    EXPECT_FALSE(type->has_imported_field_name("my_awesome_field"));
+
+    type = repo.getDocumentType(type_2_id);
+    ASSERT_TRUE(type != nullptr);
+    EXPECT_EQUAL(2u, type->imported_field_names().size());
+    EXPECT_TRUE(type->has_imported_field_name("my_awesome_field"));
+    EXPECT_TRUE(type->has_imported_field_name("my_swag_field"));
+    EXPECT_FALSE(type->has_imported_field_name("my_cool_field"));
+}
+
 namespace {
 
 const TensorDataType &
diff --git a/document/src/vespa/document/base/documentid.h b/document/src/vespa/document/base/documentid.h
index 5dcda838623..9da28f54b82 100644
--- a/document/src/vespa/document/base/documentid.h
+++ b/document/src/vespa/document/base/documentid.h
@@ -65,7 +65,7 @@ public:
 
     const IdString& getScheme() const { return _id; }
     bool hasDocType() const { return _id.hasDocType(); }
-    vespalib::string getDocType() const { return _id.getDocType(); }
+    vespalib::stringref getDocType() const { return _id.getDocType(); }
 
     const GlobalId& getGlobalId() const {
         if (!_globalId.first) { calculateGlobalId(); }
diff --git a/document/src/vespa/document/config/documentmanager.def b/document/src/vespa/document/config/documentmanager.def
index 1961f67d83c..092a29d9293 100644
--- a/document/src/vespa/document/config/documentmanager.def
+++ b/document/src/vespa/document/config/documentmanager.def
@@ -93,6 +93,9 @@ datatype[].documenttype[].bodystruct int
 ## Field sets
 datatype[].documenttype[].fieldsets{}.fields[] string
 
+## Imported fields (specified outside the document block in the schema)
+datatype[].documenttype[].importedfield[].name string
+
 ## Cross-document reference with ID of target document type
 datatype[].referencetype[].target_type_id int
 
diff --git a/document/src/vespa/document/config/documenttypes.def b/document/src/vespa/document/config/documenttypes.def
index fb3fcd8b123..0f0a9e3e37c 100644
--- a/document/src/vespa/document/config/documenttypes.def
+++ b/document/src/vespa/document/config/documenttypes.def
@@ -104,3 +104,6 @@ documenttype[].referencetype[].id int
 
 ## Numeric ID of the document type instances of the reference point to.
 documenttype[].referencetype[].target_type_id int
+
+## Imported fields (specified outside the document block in the schema)
+documenttype[].importedfield[].name string
diff --git a/document/src/vespa/document/datatype/documenttype.cpp b/document/src/vespa/document/datatype/documenttype.cpp
index ffc250eacba..c2739dc1303 100644
--- a/document/src/vespa/document/datatype/documenttype.cpp
+++ b/document/src/vespa/document/datatype/documenttype.cpp
@@ -23,9 +23,10 @@ DocumentType::DocumentType() = default;
 DocumentType::DocumentType(stringref name, int32_t id)
     : StructuredDataType(name, id),
       _inheritedTypes(),
-      _ownedFields(new StructDataType(name + ".header")),
+      _ownedFields(std::make_shared<StructDataType>(name + ".header")),
       _fields(_ownedFields.get()),
-      _fieldSets()
+      _fieldSets(),
+      _imported_field_names()
 {
     if (name != "document") {
         _inheritedTypes.push_back(DataType::DOCUMENT);
@@ -36,7 +37,8 @@ DocumentType::DocumentType(stringref name, int32_t id, const StructDataType& fie
     : StructuredDataType(name, id),
       _inheritedTypes(),
       _fields(&fields),
-      _fieldSets()
+      _fieldSets(),
+      _imported_field_names()
 {
     if (name != "document") {
         _inheritedTypes.push_back(DataType::DOCUMENT);
@@ -46,12 +48,13 @@ DocumentType::DocumentType(stringref name, int32_t id, const StructDataType& fie
 DocumentType::DocumentType(stringref name)
     : StructuredDataType(name),
       _inheritedTypes(),
-      _ownedFields(new StructDataType(name + ".header")),
+      _ownedFields(std::make_shared<StructDataType>(name + ".header")),
       _fields(_ownedFields.get()),
-      _fieldSets()
+      _fieldSets(),
+      _imported_field_names()
 {
     if (name != "document") {
-        _inheritedTypes.push_back(DataType::DOCUMENT);
+        _inheritedTypes.emplace_back(DataType::DOCUMENT);
     }
 }
 
@@ -59,27 +62,28 @@ DocumentType::DocumentType(stringref name, const StructDataType& fields)
     : StructuredDataType(name),
       _inheritedTypes(),
       _fields(&fields),
-      _fieldSets()
+      _fieldSets(),
+      _imported_field_names()
 {
     if (name != "document") {
-        _inheritedTypes.push_back(DataType::DOCUMENT);
+        _inheritedTypes.emplace_back(DataType::DOCUMENT);
     }
 }
 
 DocumentType::~DocumentType() = default;
 
 DocumentType &
-DocumentType::addFieldSet(const vespalib::string & name, const FieldSet::Fields & fields)
+DocumentType::addFieldSet(const vespalib::string & name, FieldSet::Fields fields)
 {
-    _fieldSets[name] = FieldSet(name, fields);
+    _fieldSets[name] = FieldSet(name, std::move(fields));
     return *this;
 }
 
 const DocumentType::FieldSet *
 DocumentType::getFieldSet(const vespalib::string & name) const
 {
-    FieldSetMap::const_iterator it(_fieldSets.find(name));
-    return (it != _fieldSets.end()) ? & it->second : NULL;
+    auto it = _fieldSets.find(name);
+    return (it != _fieldSets.end()) ? & it->second : nullptr;
 }
 
 void
@@ -107,37 +111,34 @@ DocumentType::inherit(const DocumentType &docType) {
                 "Document type " + docType.toString() + " already inherits type "
                 + toString() + ". Cannot add cyclic dependencies.", VESPA_STRLOC);
     }
-        // If we already inherits this type, there is no point in adding it
-        // again.
+    // If we already inherits this type, there is no point in adding it again.
     if (isA(docType)) {
-            // If we already directly inherits it, complain
-        for (std::vector<const DocumentType *>::const_iterator
-                it = _inheritedTypes.begin(); it != _inheritedTypes.end(); ++it)
-        {
-            if (**it == docType) {
+        // If we already directly inherits it, complain
+        for (const auto* inherited : _inheritedTypes) {
+            if (*inherited == docType) {
                 throw IllegalArgumentException(
                         "DocumentType " + getName() + " already inherits "
                         "document type " + docType.getName(), VESPA_STRLOC);
             }
         }
-            // Indirectly already inheriting it is oki, as this can happen
-            // due to inherited documents inheriting the same type.
+        // Indirectly already inheriting it is oki, as this can happen
+        // due to inherited documents inheriting the same type.
         LOG(info, "Document type %s inherits document type %s from multiple "
                   "types.", getName().c_str(), docType.getName().c_str());
         return;
     }
-        // Add non-conflicting types.
+    // Add non-conflicting types.
     Field::Set fs = docType._fields->getFieldSet();
-    for (Field::Set::const_iterator it = fs.begin(); it != fs.end(); ++it) {
+    for (const auto* field : fs) {
         if (!_ownedFields.get()) {
             _ownedFields.reset(_fields->clone());
             _fields = _ownedFields.get();
         }
-        _ownedFields->addInheritedField(**it);
+        _ownedFields->addInheritedField(*field);
     }
     // If we inherit default document type Document.0, remove that if adding
     // another parent, as that has to also inherit Document
-    if (_inheritedTypes.size() == 1 && *_inheritedTypes[0] == *DataType::DOCUMENT) {
+    if ((_inheritedTypes.size() == 1) && (*_inheritedTypes[0] == *DataType::DOCUMENT)) {
         _inheritedTypes.clear();
     }
     _inheritedTypes.push_back(&docType);
@@ -168,8 +169,7 @@ DocumentType::print(std::ostream& out, bool verbose, const std::string& indent)
     out << ")";
     if (verbose) {
         if (!_inheritedTypes.empty()) {
-            std::vector<const DocumentType *>::const_iterator it(
-                    _inheritedTypes.begin());
+            auto it = _inheritedTypes.begin();
             out << "\n" << indent << "    : ";
             (*it)->print(out, false, "");
             while (++it != _inheritedTypes.end()) {
@@ -188,17 +188,18 @@ DocumentType::operator==(const DataType& other) const
 {
     if (&other == this) return true;
     if (!DataType::operator==(other)) return false;
-    const DocumentType* o(dynamic_cast<const DocumentType*>(&other));
-    if (o == 0) return false;
+    const auto* o(dynamic_cast<const DocumentType*>(&other));
+    if (o == nullptr) return false;
     if (*_fields != *o->_fields) return false;
     if (_inheritedTypes.size() != o->_inheritedTypes.size()) return false;
-    std::vector<const DocumentType *>::const_iterator it1(_inheritedTypes.begin());
-    std::vector<const DocumentType *>::const_iterator it2(o->_inheritedTypes.begin());
+    auto it1 = _inheritedTypes.begin();
+    auto it2 = o->_inheritedTypes.begin();
     while (it1 != _inheritedTypes.end()) {
         if (**it1 != **it2) return false;
         ++it1;
         ++it2;
     }
+    // TODO imported fields? like in the Java impl, field sets are not considered either... :I
     return true;
 }
 
@@ -228,6 +229,14 @@ DocumentType::getFieldSet() const
     return _fields->getFieldSet();
 }
 
+bool DocumentType::has_imported_field_name(const vespalib::string& name) const noexcept {
+    return (_imported_field_names.find(name) != _imported_field_names.end());
+}
+
+void DocumentType::add_imported_field_name(const vespalib::string& name) {
+    _imported_field_names.insert(name);
+}
+
 DocumentType *
 DocumentType::clone() const {
     return new DocumentType(*this);
diff --git a/document/src/vespa/document/datatype/documenttype.h b/document/src/vespa/document/datatype/documenttype.h
index 7fcf5dfa237..ed6e9e66ab5 100644
--- a/document/src/vespa/document/datatype/documenttype.h
+++ b/document/src/vespa/document/datatype/documenttype.h
@@ -12,9 +12,11 @@
 #pragma once
 
 #include <vespa/document/datatype/structdatatype.h>
-
+#include <vespa/vespalib/stllike/hash_set.h>
+#include <vespa/vespalib/stllike/string.h>
 #include <vector>
 #include <map>
+#include <set>
 
 namespace document {
 
@@ -25,12 +27,17 @@ class DocumentType : public StructuredDataType {
 public:
     class FieldSet {
     public:
-        typedef std::set<vespalib::string> Fields;
-        FieldSet() : _name(), _fields() {}
-        FieldSet(const vespalib::string & name) : _name(name), _fields() {}
-        FieldSet(const vespalib::string & name, const Fields & fields) : _name(name), _fields(fields) {}
-        const vespalib::string & getName() const { return _name; }
-        const Fields & getFields() const { return _fields; }
+        using Fields = std::set<vespalib::string>;
+        FieldSet() = default;
+        explicit FieldSet(const vespalib::string & name) : _name(name), _fields() {}
+        FieldSet(const vespalib::string & name, Fields fields) : _name(name), _fields(std::move(fields)) {}
+        FieldSet(const FieldSet&) = default;
+        FieldSet& operator=(const FieldSet&) = default;
+        FieldSet(FieldSet&&) noexcept = default;
+        FieldSet& operator=(FieldSet&&) noexcept = default;
+
+        const vespalib::string & getName() const noexcept { return _name; }
+        const Fields & getFields() const noexcept { return _fields; }
         FieldSet & add(vespalib::string & field) {
             _fields.insert(field);
             return *this;
@@ -39,28 +46,31 @@ public:
         vespalib::string _name;
         Fields           _fields;
     };
-    typedef std::map<vespalib::string, FieldSet> FieldSetMap;
+    using FieldSetMap = std::map<vespalib::string, FieldSet>;
+    using ImportedFieldNames = vespalib::hash_set<vespalib::string>;
+
     std::vector<const DocumentType *> _inheritedTypes;
-    StructDataType::SP _ownedFields;
-    const StructDataType* _fields;
-    FieldSetMap _fieldSets;
+    StructDataType::SP                _ownedFields;
+    const StructDataType*             _fields;
+    FieldSetMap                       _fieldSets;
+    ImportedFieldNames                _imported_field_names;
 
 public:
-    typedef std::unique_ptr<DocumentType> UP;
-    typedef std::shared_ptr<DocumentType> SP;
+    using UP = std::unique_ptr<DocumentType>;
+    using SP = std::shared_ptr<DocumentType>;
 
     DocumentType();
     DocumentType(vespalib::stringref name, int32_t id);
     DocumentType(vespalib::stringref name, int32_t id,
                  const StructDataType& fields);
 
-    DocumentType(vespalib::stringref name);
+    explicit DocumentType(vespalib::stringref name);
     DocumentType(vespalib::stringref name,
                  const StructDataType& fields);
 
-    ~DocumentType();
+    ~DocumentType() override;
 
-    const StructDataType& getFieldsType() const { return *_fields; }
+    const StructDataType& getFieldsType() const noexcept { return *_fields; }
 
     void addField(const Field&);
 
@@ -89,9 +99,16 @@ public:
     Field::Set getFieldSet() const override;
     DocumentType* clone() const override;
 
-    DocumentType & addFieldSet(const vespalib::string & name, const FieldSet::Fields & fields);
+    DocumentType & addFieldSet(const vespalib::string & name, FieldSet::Fields fields);
     const FieldSet * getFieldSet(const vespalib::string & name) const;
 
+    const ImportedFieldNames& imported_field_names() const noexcept {
+        return _imported_field_names;
+    }
+    bool has_imported_field_name(const vespalib::string& name) const noexcept;
+    // Ideally the type would be immutable, but this is how it's built today.
+    void add_imported_field_name(const vespalib::string& name);
+
     DECLARE_IDENTIFIABLE(DocumentType);
 };
 
diff --git a/document/src/vespa/document/datatype/structdatatype.cpp b/document/src/vespa/document/datatype/structdatatype.cpp
index 7c308202e3b..ed17d22da59 100644
--- a/document/src/vespa/document/datatype/structdatatype.cpp
+++ b/document/src/vespa/document/datatype/structdatatype.cpp
@@ -118,7 +118,7 @@ StructDataType::addInheritedField(const Field& field)
 FieldValue::UP
 StructDataType::createFieldValue() const
 {
-    return FieldValue::UP(new StructFieldValue(*this));
+    return std::make_unique<StructFieldValue>(*this);
 }
 
 const Field&
diff --git a/document/src/vespa/document/fieldvalue/document.cpp b/document/src/vespa/document/fieldvalue/document.cpp
index cc60234a093..fb79b516b90 100644
--- a/document/src/vespa/document/fieldvalue/document.cpp
+++ b/document/src/vespa/document/fieldvalue/document.cpp
@@ -11,6 +11,7 @@
 #include <vespa/document/util/serializableexceptions.h>
 #include <vespa/document/fieldset/fieldsets.h>
 #include <vespa/document/util/bytebuffer.h>
+#include <vespa/vespalib/data/databuffer.h>
 #include <vespa/vespalib/util/xmlstream.h>
 #include <vespa/vespalib/stllike/hash_map.hpp>
 #include <cassert>
@@ -71,6 +72,7 @@ Document::Document()
     : StructuredFieldValue(*DataType::DOCUMENT),
       _id(),
       _fields(getType().getFieldsType()),
+      _backingBuffer(),
       _lastModified(0)
 {
     _fields.setDocumentType(getType());
@@ -80,6 +82,7 @@ Document::Document(const Document& rhs)
     : StructuredFieldValue(rhs),
       _id(rhs._id),
       _fields(rhs._fields),
+      _backingBuffer(),
       _lastModified(rhs._lastModified)
 {}
 
@@ -87,6 +90,7 @@ Document::Document(const DataType &type, DocumentId documentId)
     : StructuredFieldValue(verifyDocumentType(&type)),
       _id(std::move(documentId)),
       _fields(getType().getFieldsType()),
+      _backingBuffer(),
       _lastModified(0)
 {
     _fields.setDocumentType(getType());
@@ -104,11 +108,29 @@ Document::Document(const DocumentTypeRepo& repo, vespalib::nbostream & is)
     : StructuredFieldValue(*DataType::DOCUMENT),
       _id(),
       _fields(static_cast<const DocumentType &>(getType()).getFieldsType()),
+      _backingBuffer(),
       _lastModified(0)
 {
     deserialize(repo, is);
 }
 
+Document::Document(const DocumentTypeRepo& repo, vespalib::DataBuffer && backingBuffer)
+    : StructuredFieldValue(*DataType::DOCUMENT),
+      _id(),
+      _fields(static_cast<const DocumentType &>(getType()).getFieldsType()),
+      _backingBuffer(),
+      _lastModified(0)
+{
+    if (backingBuffer.referencesExternalData()) {
+        vespalib::nbostream is(backingBuffer.getData(), backingBuffer.getDataLen());
+        deserialize(repo, is);
+    } else {
+        vespalib::nbostream_longlivedbuf is(backingBuffer.getData(), backingBuffer.getDataLen());
+        deserialize(repo, is);
+        _backingBuffer = std::make_unique<vespalib::DataBuffer>(std::move(backingBuffer));
+    }
+}
+
 Document::Document(Document &&) noexcept = default;
 Document::~Document() noexcept = default;
 
@@ -117,6 +139,7 @@ Document::operator =(Document &&rhs) noexcept {
     assert( ! _cache && ! rhs._cache);
     _id = std::move(rhs._id);
     _fields = std::move(rhs._fields);
+    _backingBuffer = std::move(rhs._backingBuffer);
     _lastModified = rhs._lastModified;
     StructuredFieldValue::operator=(std::move(rhs));
     return *this;
@@ -124,15 +147,16 @@ Document::operator =(Document &&rhs) noexcept {
 
 Document &
 Document::operator =(const Document &rhs) {
+    if (this == &rhs) return *this;
     assert( ! _cache && ! rhs._cache);
     _id = rhs._id;
     _fields = rhs._fields;
     _lastModified = rhs._lastModified;
     StructuredFieldValue::operator=(rhs);
+    _backingBuffer.reset();
     return *this;
 }
 
-
 const DocumentType&
 Document::getType() const {
     return static_cast<const DocumentType &>(StructuredFieldValue::getType());
diff --git a/document/src/vespa/document/fieldvalue/document.h b/document/src/vespa/document/fieldvalue/document.h
index b487aa067da..cb27ca5f338 100644
--- a/document/src/vespa/document/fieldvalue/document.h
+++ b/document/src/vespa/document/fieldvalue/document.h
@@ -19,6 +19,7 @@
 #include <vespa/document/base/documentid.h>
 #include <vespa/document/base/field.h>
 
+namespace vespalib { class DataBuffer; }
 namespace document {
 
 class TransactionGuard;
@@ -29,6 +30,7 @@ private:
     DocumentId _id;
     StructFieldValue _fields;
     std::unique_ptr<StructuredCache> _cache;
+    std::unique_ptr<vespalib::DataBuffer> _backingBuffer;
 
     // To avoid having to return another container object out of docblocks
     // the meta data has been added to document. This will not be serialized
@@ -50,6 +52,7 @@ public:
     Document & operator =(Document &&) noexcept;
     Document(const DataType &, DocumentId id);
     Document(const DocumentTypeRepo& repo, vespalib::nbostream& stream);
+    Document(const DocumentTypeRepo& repo, vespalib::DataBuffer && buffer);
     ~Document() noexcept override;
 
     void setRepo(const DocumentTypeRepo & repo);
diff --git a/document/src/vespa/document/fieldvalue/referencefieldvalue.cpp b/document/src/vespa/document/fieldvalue/referencefieldvalue.cpp
index 6c046c1787b..273038b1cf3 100644
--- a/document/src/vespa/document/fieldvalue/referencefieldvalue.cpp
+++ b/document/src/vespa/document/fieldvalue/referencefieldvalue.cpp
@@ -47,7 +47,7 @@ void ReferenceFieldValue::requireIdOfMatchingType(
                 make_string("Can't assign document ID '%s' (of type '%s') to "
                             "reference of document type '%s'",
                             id.toString().c_str(),
-                            id.getDocType().c_str(),
+                            vespalib::string(id.getDocType()).c_str(),
                             type.getName().c_str()),
                 VESPA_STRLOC);
     }
diff --git a/document/src/vespa/document/fieldvalue/serializablearray.cpp b/document/src/vespa/document/fieldvalue/serializablearray.cpp
index be5001c6f5b..84fe7ba34a6 100644
--- a/document/src/vespa/document/fieldvalue/serializablearray.cpp
+++ b/document/src/vespa/document/fieldvalue/serializablearray.cpp
@@ -26,22 +26,20 @@ public:
 
 }
 
-SerializableArray::SerializableArray() = default;
-
-SerializableArray::SerializableArray(EntryMap entries, ByteBuffer buffer,
-                                     CompressionConfig::Type comp_type, uint32_t uncompressed_length)
-    : _entries(std::move(entries)),
-      _uncompSerData(),
-      _unlikely()
+void
+SerializableArray::set(EntryMap entries, ByteBuffer buffer,
+                       CompressionConfig::Type comp_type, uint32_t uncompressed_length)
 {
-
+    _entries = std::move(entries);
     if (CompressionConfig::isCompressed(comp_type)) {
         _unlikely = std::make_unique<RarelyUsedBuffers>();
         _unlikely->_compSerData = std::move(buffer);
         _unlikely->_serializedCompression = comp_type;
         _unlikely->_uncompressedLength = uncompressed_length;
+        _uncompSerData = ByteBuffer();
     } else {
         _uncompSerData = std::move(buffer);
+        _unlikely.reset();
     }
 }
 
diff --git a/document/src/vespa/document/fieldvalue/serializablearray.h b/document/src/vespa/document/fieldvalue/serializablearray.h
index 80edc8a810c..11186593e76 100644
--- a/document/src/vespa/document/fieldvalue/serializablearray.h
+++ b/document/src/vespa/document/fieldvalue/serializablearray.h
@@ -84,15 +84,15 @@ public:
     using CompressionConfig = vespalib::compression::CompressionConfig;
     using CompressionInfo = vespalib::compression::CompressionInfo;
 
-    SerializableArray();
+    SerializableArray() = default;
     SerializableArray(const SerializableArray&);
     SerializableArray& operator=(const SerializableArray&);
     SerializableArray(SerializableArray &&) noexcept;
     SerializableArray& operator=(SerializableArray &&) noexcept;
-    SerializableArray(EntryMap entries, ByteBuffer buffer,
-                      CompressionConfig::Type comp_type, uint32_t uncompressed_length);
     ~SerializableArray();
 
+    void set(EntryMap entries, ByteBuffer buffer,
+             CompressionConfig::Type comp_type, uint32_t uncompressed_length);
     /**
      * Stores a value in the array.
      *
diff --git a/document/src/vespa/document/fieldvalue/structfieldvalue.cpp b/document/src/vespa/document/fieldvalue/structfieldvalue.cpp
index b448b2b33bc..bc2dd2a059a 100644
--- a/document/src/vespa/document/fieldvalue/structfieldvalue.cpp
+++ b/document/src/vespa/document/fieldvalue/structfieldvalue.cpp
@@ -67,7 +67,7 @@ StructFieldValue::lazyDeserialize(const FixedTypeRepo &repo,
     _doc_type = &repo.getDocumentType();
     _version = version;
 
-    _fields = SerializableArray(std::move(fm), std::move(buffer), comp_type, uncompressed_length);
+    _fields.set(std::move(fm), std::move(buffer), comp_type, uncompressed_length);
     _hasChanged = false;
 }
 
@@ -390,7 +390,7 @@ StructFieldValue::getIterator(const Field* toFind) const
 void
 StructFieldValue::setType(const DataType& type)
 {
-    clear();
+    reset();
     StructuredFieldValue::setType(type);
 }
 
diff --git a/document/src/vespa/document/fieldvalue/structuredfieldvalue.cpp b/document/src/vespa/document/fieldvalue/structuredfieldvalue.cpp
index 755186edd4c..2541a253c80 100644
--- a/document/src/vespa/document/fieldvalue/structuredfieldvalue.cpp
+++ b/document/src/vespa/document/fieldvalue/structuredfieldvalue.cpp
@@ -30,12 +30,6 @@ StructuredFieldValue::Iterator::Iterator(const StructuredFieldValue& owner, cons
 {
 }
 
-StructuredFieldValue::StructuredFieldValue(const DataType &type)
-    : FieldValue(),
-      _type(&type)
-{
-}
-
 void
 StructuredFieldValue::setFieldValue(const Field & field, const FieldValue & value)
 {
diff --git a/document/src/vespa/document/fieldvalue/structuredfieldvalue.h b/document/src/vespa/document/fieldvalue/structuredfieldvalue.h
index 55b964147be..54036566a26 100644
--- a/document/src/vespa/document/fieldvalue/structuredfieldvalue.h
+++ b/document/src/vespa/document/fieldvalue/structuredfieldvalue.h
@@ -44,7 +44,7 @@ class StructuredFieldValue : public FieldValue
     virtual StructuredCache * getCache() const { return nullptr; }
 
 protected:
-    VESPA_DLL_LOCAL StructuredFieldValue(const DataType &type);
+    StructuredFieldValue(const DataType &type) : FieldValue(), _type(&type) {}
 
     /** Called from Document when deserializing alters type. */
     virtual void setType(const DataType& type) { _type = &type; }
@@ -94,6 +94,11 @@ protected:
     virtual bool hasFieldValue(const Field&) const = 0;
     virtual void removeFieldValue(const Field&) = 0;
     virtual FieldValue::UP getFieldValue(const Field&) const = 0;
+    /**
+     * Fetches the value of the field and return true if present.
+     * The document, or the buffer the document was constructed from must live longer than the value.
+     * This restriction allows lightweight object representation and is significantly faster in many cases.
+     */
     virtual bool getFieldValue(const Field& field, FieldValue& value) const = 0;
     virtual void setFieldValue(const Field&, FieldValue::UP value) = 0;
     void setFieldValue(const Field & field, const FieldValue & value);
diff --git a/document/src/vespa/document/fieldvalue/tensorfieldvalue.cpp b/document/src/vespa/document/fieldvalue/tensorfieldvalue.cpp
index 9b318a39f0a..56d7b6ab078 100644
--- a/document/src/vespa/document/fieldvalue/tensorfieldvalue.cpp
+++ b/document/src/vespa/document/fieldvalue/tensorfieldvalue.cpp
@@ -104,6 +104,19 @@ TensorFieldValue::operator=(std::unique_ptr<Tensor> rhs)
 }
 
 
+void
+TensorFieldValue::make_empty_if_not_existing()
+{
+    if (!_tensor) {
+        TensorSpec empty_spec(_dataType.getTensorType().to_spec());
+        auto empty_value = Engine::ref().from_spec(empty_spec);
+        auto tensor_ptr = dynamic_cast<Tensor*>(empty_value.get());
+        assert(tensor_ptr != nullptr);
+        _tensor.reset(tensor_ptr);
+        empty_value.release();
+    }
+}
+
 
 void
 TensorFieldValue::accept(FieldValueVisitor &visitor)
diff --git a/document/src/vespa/document/fieldvalue/tensorfieldvalue.h b/document/src/vespa/document/fieldvalue/tensorfieldvalue.h
index 80b18be55a0..ea3f8dea9be 100644
--- a/document/src/vespa/document/fieldvalue/tensorfieldvalue.h
+++ b/document/src/vespa/document/fieldvalue/tensorfieldvalue.h
@@ -28,6 +28,8 @@ public:
     TensorFieldValue &operator=(const TensorFieldValue &rhs);
     TensorFieldValue &operator=(std::unique_ptr<vespalib::tensor::Tensor> rhs);
 
+    void make_empty_if_not_existing();
+
     virtual void accept(FieldValueVisitor &visitor) override;
     virtual void accept(ConstFieldValueVisitor &visitor) const override;
     virtual const DataType *getDataType() const override;
diff --git a/document/src/vespa/document/repo/configbuilder.h b/document/src/vespa/document/repo/configbuilder.h
index 15ee0da0c79..52ed8d878a2 100644
--- a/document/src/vespa/document/repo/configbuilder.h
+++ b/document/src/vespa/document/repo/configbuilder.h
@@ -38,9 +38,9 @@ struct TypeOrId {
 };
 
 struct Struct : DatatypeConfig {
-    Struct(const vespalib::string &name) {
+    explicit Struct(vespalib::string name) {
         type = Type::STRUCT;
-        sstruct.name = name;
+        sstruct.name = std::move(name);
     }
     Struct &setCompression(Sstruct::Compression::Type t, int32_t level,
                            int32_t threshold, int32_t min_size) {
@@ -63,7 +63,7 @@ struct Struct : DatatypeConfig {
 };
 
 struct Array : DatatypeConfig {
-    Array(TypeOrId nested_type) {
+    explicit Array(TypeOrId nested_type) {
         addNestedType(nested_type);
         type = Type::ARRAY;
         array.element.id = nested_type.id;
@@ -71,7 +71,7 @@ struct Array : DatatypeConfig {
 };
 
 struct Wset : DatatypeConfig {
-    Wset(TypeOrId nested_type) {
+    explicit Wset(TypeOrId nested_type) {
         addNestedType(nested_type);
         type = Type::WSET;
         wset.key.id = nested_type.id;
@@ -94,7 +94,7 @@ struct Map : DatatypeConfig {
 };
 
 struct AnnotationRef : DatatypeConfig {
-    AnnotationRef(int32_t annotation_type_id) {
+    explicit AnnotationRef(int32_t annotation_type_id) {
         type = Type::ANNOTATIONREF;
         annotationref.annotation.id = annotation_type_id;
     }
@@ -103,7 +103,7 @@ struct AnnotationRef : DatatypeConfig {
 struct DocTypeRep {
     DocumenttypesConfig::Documenttype &doc_type;
 
-    DocTypeRep(DocumenttypesConfig::Documenttype &type)
+    explicit DocTypeRep(DocumenttypesConfig::Documenttype &type)
         : doc_type(type) {}
     DocTypeRep &inherit(int32_t id) {
         doc_type.inherits.resize(doc_type.inherits.size() + 1);
@@ -127,6 +127,12 @@ struct DocTypeRep {
         doc_type.referencetype.back().targetTypeId = target_type_id;
         return *this;
     }
+
+    DocTypeRep& imported_field(vespalib::string field_name) {
+        doc_type.importedfield.resize(doc_type.importedfield.size() + 1);
+        doc_type.importedfield.back().name = std::move(field_name);
+        return *this;
+    }
 };
 
 class DocumenttypesConfigBuilderHelper {
diff --git a/document/src/vespa/document/repo/documenttyperepo.cpp b/document/src/vespa/document/repo/documenttyperepo.cpp
index da59f527115..58dbd16beb6 100644
--- a/document/src/vespa/document/repo/documenttyperepo.cpp
+++ b/document/src/vespa/document/repo/documenttyperepo.cpp
@@ -441,10 +441,10 @@ void addFieldSet(const DocumenttypesConfig::Documenttype::FieldsetsMap & fsv, Do
     for (const auto & entry : fsv) {
         const DocumenttypesConfig::Documenttype::Fieldsets & fs(entry.second);
         DocumentType::FieldSet::Fields fields;
-        for (size_t j(0); j < fs.fields.size(); j++) {
-            fields.insert(fs.fields[j]);
+        for (const auto& f : fs.fields) {
+            fields.insert(f);
         }
-        doc_type.addFieldSet(entry.first, fields);
+        doc_type.addFieldSet(entry.first, std::move(fields));
     }
 }
 
@@ -457,6 +457,14 @@ void addReferenceTypes(const vector<DocumenttypesConfig::Documenttype::Reference
     }
 }
 
+void add_imported_fields(const DocumenttypesConfig::Documenttype::ImportedfieldVector& imported_fields,
+                         DocumentType& doc_type)
+{
+    for (const auto& imported : imported_fields) {
+        doc_type.add_imported_field_name(imported.name);
+    }
+}
+
 void configureDataTypeRepo(const DocumenttypesConfig::Documenttype &doc_type, DocumentTypeMap &type_map) {
     DataTypeRepo *data_types = type_map[doc_type.id];
     inheritAnnotationTypes(doc_type.inherits, type_map, data_types->annotations);
@@ -467,6 +475,7 @@ void configureDataTypeRepo(const DocumenttypesConfig::Documenttype &doc_type, Do
     setAnnotationDataTypes(doc_type.annotationtype, data_types->annotations, data_types->repo);
     inheritDocumentTypes(doc_type.inherits, type_map, *data_types->doc_type);
     addFieldSet(doc_type.fieldsets, *data_types->doc_type);
+    add_imported_fields(doc_type.importedfield, *data_types->doc_type);
 }
 
 void addDataTypeRepo(DataTypeRepo::UP data_types, DocumentTypeMap &doc_types) {
diff --git a/document/src/vespa/document/select/CMakeLists.txt b/document/src/vespa/document/select/CMakeLists.txt
index 81e5d86675c..f210e8abdd7 100644
--- a/document/src/vespa/document/select/CMakeLists.txt
+++ b/document/src/vespa/document/select/CMakeLists.txt
@@ -36,6 +36,7 @@ vespa_add_library(document_select OBJECT
     parser.cpp
     parse_utils.cpp
     parsing_failed_exception.cpp
+    parser_limits.cpp
     ${BISON_DocSelParser_OUTPUTS}
     ${FLEX_DocSelLexer_OUTPUTS}
     AFTER
diff --git a/document/src/vespa/document/select/branch.cpp b/document/src/vespa/document/select/branch.cpp
index b3d5f97ccab..9104e2c5544 100644
--- a/document/src/vespa/document/select/branch.cpp
+++ b/document/src/vespa/document/select/branch.cpp
@@ -8,7 +8,7 @@
 namespace document::select {
 
 And::And(std::unique_ptr<Node> left, std::unique_ptr<Node> right, const char* name)
-    : Branch(name ? name : "and"),
+    : Branch(name ? name : "and", std::max(left->max_depth(), right->max_depth()) + 1),
       _left(std::move(left)),
       _right(std::move(right))
 {
@@ -54,7 +54,7 @@ And::trace(const Context& context, std::ostream& out) const
 }
 
 Or::Or(std::unique_ptr<Node> left, std::unique_ptr<Node> right, const char* name)
-    : Branch(name ? name : "or"),
+    : Branch(name ? name : "or", std::max(left->max_depth(), right->max_depth()) + 1),
       _left(std::move(left)),
       _right(std::move(right))
 {
@@ -100,7 +100,7 @@ Or::trace(const Context& context, std::ostream& out) const
 }
 
 Not::Not(std::unique_ptr<Node> child, const char* name)
-    : Branch(name ? name : "not"),
+    : Branch(name ? name : "not", child->max_depth() + 1),
       _child(std::move(child))
 {
     assert(_child.get());
diff --git a/document/src/vespa/document/select/branch.h b/document/src/vespa/document/select/branch.h
index 8637b41de89..77ed74030b5 100644
--- a/document/src/vespa/document/select/branch.h
+++ b/document/src/vespa/document/select/branch.h
@@ -19,7 +19,8 @@ namespace document::select {
 class Branch : public Node
 {
 public:
-    Branch(vespalib::stringref name) : Node(name) {}
+    explicit Branch(vespalib::stringref name) : Node(name) {}
+    Branch(vespalib::stringref name, uint32_t max_depth) : Node(name, max_depth) {}
 
     bool isLeafNode() const override { return false; }
 };
@@ -30,7 +31,7 @@ class And : public Branch
     std::unique_ptr<Node> _right;
 public:
     And(std::unique_ptr<Node> left, std::unique_ptr<Node> right,
-        const char* name = 0);
+        const char* name = nullptr);
 
     ResultList contains(const Context& context) const override {
         return (_left->contains(context) && _right->contains(context));
@@ -53,7 +54,7 @@ class Or : public Branch
     std::unique_ptr<Node> _right;
 public:
     Or(std::unique_ptr<Node> left, std::unique_ptr<Node> right,
-        const char* name = 0);
+       const char* name = nullptr);
 
     ResultList contains(const Context& context) const  override {
         return (_left->contains(context) || _right->contains(context));
@@ -74,7 +75,7 @@ class Not : public Branch
 {
     std::unique_ptr<Node> _child;
 public:
-    Not(std::unique_ptr<Node> child, const char* name = 0);
+    Not(std::unique_ptr<Node> child, const char* name = nullptr);
 
     ResultList contains(const Context& context) const override { return !_child->contains(context); }
     ResultList trace(const Context&, std::ostream& trace) const override;
diff --git a/document/src/vespa/document/select/compare.cpp b/document/src/vespa/document/select/compare.cpp
index 7db40929a64..caef1bdd250 100644
--- a/document/src/vespa/document/select/compare.cpp
+++ b/document/src/vespa/document/select/compare.cpp
@@ -15,7 +15,7 @@ Compare::Compare(std::unique_ptr<ValueNode> left,
                  const Operator& op,
                  std::unique_ptr<ValueNode> right,
                  const BucketIdFactory& bucketIdFactory)
-    : Node("Compare"),
+    : Node("Compare", std::max(left->max_depth(), right->max_depth()) + 1),
       _left(std::move(left)),
       _right(std::move(right)),
       _operator(op),
diff --git a/document/src/vespa/document/select/grammar/lexer.ll b/document/src/vespa/document/select/grammar/lexer.ll
index bd011c8ebf6..1222aac02a2 100644
--- a/document/src/vespa/document/select/grammar/lexer.ll
+++ b/document/src/vespa/document/select/grammar/lexer.ll
@@ -7,6 +7,13 @@
 %option noyywrap nounput
 %option yyclass="document::select::DocSelScanner"
 
+ /* Flex lexer must be compiled with batch mode (as opposed to interactive mode)
+  * or parsing of large tokens appears to trigger superlinear time complexity.
+  * Also use full, non-compressed lookup tables for maximum performance.
+  */
+%option batch
+%option full
+
  /* Used to track source locations, see https://github.com/bingmann/flex-bison-cpp-example/blob/master/src/scanner.ll */
 %{
 #define YY_USER_ACTION yyloc->columns(yyleng);
diff --git a/document/src/vespa/document/select/node.h b/document/src/vespa/document/select/node.h
index 48a64ae63f5..9a3b687d81c 100644
--- a/document/src/vespa/document/select/node.h
+++ b/document/src/vespa/document/select/node.h
@@ -12,6 +12,7 @@
 
 #include "resultlist.h"
 #include "context.h"
+#include "parser_limits.h"
 
 namespace document::select {
 
@@ -21,19 +22,33 @@ class Node : public Printable
 {
 protected:
     vespalib::string _name;
+    uint32_t _max_depth;
     bool _parentheses; // Set to true if parentheses was used around this part
                        // Set such that we can recreate original query in print.
 public:
     typedef std::unique_ptr<Node> UP;
     typedef std::shared_ptr<Node> SP;
 
-    Node(vespalib::stringref name) : _name(name), _parentheses(false) {}
-    ~Node() override {}
+    Node(vespalib::stringref name, uint32_t max_depth)
+        : _name(name), _max_depth(max_depth), _parentheses(false)
+    {
+        throw_parse_error_if_max_depth_exceeded();
+    }
 
-    void setParentheses() { _parentheses = true; }
+    explicit Node(vespalib::stringref name)
+        : _name(name), _max_depth(1), _parentheses(false)
+    {}
+    ~Node() override = default;
 
-    void clearParentheses() { _parentheses = false; }
+    // Depth is explicitly tracked to limit recursion to a sane maximum when building and
+    // processing ASTs, as the Bison framework does not have anything useful for us there.
+    // The AST is built from the leaves up towards the root, so we can cheaply track depth
+    // of subtrees in O(1) time per node by computing a node's own depth based on immediate
+    // children at node construction time.
+    [[nodiscard]] uint32_t max_depth() const noexcept { return _max_depth; }
 
+    void setParentheses() { _parentheses = true; }
+    void clearParentheses() { _parentheses = false; }
     bool hadParentheses() const { return _parentheses; }
 
     virtual ResultList contains(const Context&) const = 0;
@@ -43,6 +58,12 @@ public:
 
     virtual Node::UP clone() const = 0;
 protected:
+    void throw_parse_error_if_max_depth_exceeded() const {
+        if (_max_depth > ParserLimits::MaxRecursionDepth) {
+            throw_max_depth_exceeded_exception();
+        }
+    }
+
     Node::UP wrapParens(Node* node) const {
         Node::UP ret(node);
         if (_parentheses) {
diff --git a/document/src/vespa/document/select/operator.cpp b/document/src/vespa/document/select/operator.cpp
index eaa795549bf..f5cc681c906 100644
--- a/document/src/vespa/document/select/operator.cpp
+++ b/document/src/vespa/document/select/operator.cpp
@@ -1,9 +1,9 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include "operator.h"
-#include <regex>
 #include <vespa/vespalib/stllike/asciistream.h>
 #include <vespa/vespalib/stllike/hash_map.hpp>
+#include <vespa/vespalib/regex/regex.h>
 #include <cassert>
 #include <ostream>
 
@@ -96,23 +96,25 @@ RegexOperator::trace(const Value& a, const Value& b, std::ostream& out) const
 ResultList
 RegexOperator::compareImpl(const Value& a, const Value& b) const
 {
-    const StringValue* left(dynamic_cast<const StringValue*>(&a));
-    const StringValue* right(dynamic_cast<const StringValue*>(&b));
-    if (left == 0 || right == 0) return ResultList(Result::Invalid);
+    const auto* left(dynamic_cast<const StringValue*>(&a));
+    const auto* right(dynamic_cast<const StringValue*>(&b));
+    if (left == nullptr || right == nullptr) {
+        return ResultList(Result::Invalid);
+    }
     return match(left->getValue(), right->getValue());
 }
 
 ResultList
 RegexOperator::traceImpl(const Value& a, const Value& b, std::ostream& out) const
 {
-    const StringValue* left(dynamic_cast<const StringValue*>(&a));
-    const StringValue* right(dynamic_cast<const StringValue*>(&b));
-    if (left == 0) {
+    const auto* left(dynamic_cast<const StringValue*>(&a));
+    const auto* right(dynamic_cast<const StringValue*>(&b));
+    if (left == nullptr) {
         out << "Operator(" << getName() << ") - Left value not a string. "
             << "Returning invalid.\n";
         return ResultList(Result::Invalid);
     }
-    if (right == 0) {
+    if (right == nullptr) {
         out << "Operator(" << getName() << ") - Right value not a string. "
             << "Returning invalid.\n";
         return ResultList(Result::Invalid);
@@ -126,14 +128,12 @@ RegexOperator::traceImpl(const Value& a, const Value& b, std::ostream& out) cons
 ResultList
 RegexOperator::match(const vespalib::string& val, vespalib::stringref expr) const
 {
-        // Should we catch this in parsing?
-    if (expr.size() == 0) return ResultList(Result::True);
-    try {
-        std::regex expression(expr.data(), expr.size());
-        return ResultList(Result::get(std::regex_search(val.c_str(), val.c_str() + val.size(), expression)));
-    } catch (std::regex_error &) {
-        return ResultList(Result::False);
+    if (expr.empty()) {
+        return ResultList(Result::True); // Should we catch this in parsing?
     }
+    return ResultList(Result::get(
+            vespalib::Regex::partial_match(std::string_view(val.data(), val.size()),
+                                           std::string_view(expr.data(), expr.size()))));
 }
 
 const RegexOperator RegexOperator::REGEX("=~");
@@ -158,13 +158,15 @@ GlobOperator::trace(const Value& a, const Value& b, std::ostream& out) const
 ResultList
 GlobOperator::compareImpl(const Value& a, const Value& b) const
 {
-    const StringValue* right(dynamic_cast<const StringValue*>(&b));
-        // Fall back to operator== if it isn't string matching
-    if (right == 0) {
+    const auto* right(dynamic_cast<const StringValue*>(&b));
+    // Fall back to operator== if it isn't string matching
+    if (right == nullptr) {
         return FunctionOperator::EQ.compare(a, b);
     }
-    const StringValue* left(dynamic_cast<const StringValue*>(&a));
-    if (left == 0) return ResultList(Result::Invalid);
+    const auto* left(dynamic_cast<const StringValue*>(&a));
+    if (left == nullptr) {
+        return ResultList(Result::Invalid);
+    }
     vespalib::string regex(convertToRegex(right->getValue()));
     return match(left->getValue(), regex);
 }
@@ -172,15 +174,15 @@ GlobOperator::compareImpl(const Value& a, const Value& b) const
 ResultList
 GlobOperator::traceImpl(const Value& a, const Value& b, std::ostream& ost) const
 {
-    const StringValue* right(dynamic_cast<const StringValue*>(&b));
-        // Fall back to operator== if it isn't string matching
-    if (right == 0) {
+    const auto* right(dynamic_cast<const StringValue*>(&b));
+    // Fall back to operator== if it isn't string matching
+    if (right == nullptr) {
         ost << "Operator(" << getName() << ") - Right val not a string, "
             << "falling back to == behavior.\n";
         return FunctionOperator::EQ.trace(a, b, ost);
     }
-    const StringValue* left(dynamic_cast<const StringValue*>(&a));
-    if (left == 0) {
+    const auto* left(dynamic_cast<const StringValue*>(&a));
+    if (left == nullptr) {
         ost << "Operator(" << getName() << ") - Left value is not a string, "
             << "returning invalid.\n";
         return ResultList(Result::Invalid);
@@ -191,35 +193,64 @@ GlobOperator::traceImpl(const Value& a, const Value& b, std::ostream& ost) const
     return match(left->getValue(), regex);
 }
 
+namespace {
+
+// Returns the number of consecutive wildcard ('*') characters found from
+// _and including_ the character at `i`, i.e. the wildcard run length.
+size_t wildcard_run_length(size_t i, vespalib::stringref str) {
+    size_t n = 0;
+    for (; (i < str.size()) && (str[i] == '*'); ++n, ++i) {}
+    return n;
+}
+
+}
+
 vespalib::string
-GlobOperator::convertToRegex(vespalib::stringref globpattern) const
+GlobOperator::convertToRegex(vespalib::stringref globpattern)
 {
+    if (globpattern.empty()) {
+        return "^$"; // Empty glob can only match the empty string.
+    }
     vespalib::asciistream ost;
-    ost << '^';
-    for(uint32_t i=0, n=globpattern.size(); i<n; ++i) {
+    size_t i = 0;
+    if (globpattern[0] != '*') {
+        ost << '^';
+    } else {
+        i += wildcard_run_length(0, globpattern); // Skip entire prefix wildcard run.
+    }
+    const size_t n = globpattern.size();
+    for (; i < n; ++i) {
         switch(globpattern[i]) {
-            case '*': ost << ".*";
-                      break;
-            case '?': ost << ".";
-                      break;
-            case '^':
-            case '$':
-            case '|':
-            case '{':
-            case '}':
-            case '(':
-            case ')':
-            case '[':
-            case ']':
-            case '\\':
-            case '+':
-            case '.': ost << '\\' << globpattern[i];
-                      break;
-                // Are there other regex special chars we need to escape?
-            default: ost << globpattern[i];
+        case '*':
+            i += wildcard_run_length(i, globpattern) - 1; // -1 since we always inc by 1 anyway.
+            if (i != (n - 1)) { // Don't emit trailing wildcard.
+                ost << ".*";
+            }
+            break;
+        case '?':
+            ost << '.';
+            break;
+        case '^':
+        case '$':
+        case '|':
+        case '{':
+        case '}':
+        case '(':
+        case ')':
+        case '[':
+        case ']':
+        case '\\':
+        case '+':
+        case '.':
+            ost << '\\' << globpattern[i];
+            break;
+        // Are there other regex special chars we need to escape?
+        default: ost << globpattern[i];
         }
     }
-    ost << '$';
+    if (globpattern[n - 1] != '*') {
+        ost << '$';
+    }
     return ost.str();
 }
 
diff --git a/document/src/vespa/document/select/operator.h b/document/src/vespa/document/select/operator.h
index 7383b6cf545..ad73d284c63 100644
--- a/document/src/vespa/document/select/operator.h
+++ b/document/src/vespa/document/select/operator.h
@@ -88,7 +88,27 @@ public:
     // Delegates to Value::globCompare
     ResultList compare(const Value& a, const Value& b) const override;
     ResultList trace(const Value&, const Value&, std::ostream& trace) const override;
-    vespalib::string convertToRegex(vespalib::stringref globpattern) const;
+    /**
+     * Converts a standard glob expression into a regular expression string,
+     * supporting the following glob semantics:
+     *   '*' matches 0-n arbitrary characters
+     *   '?' matches exactly 1 arbitrary character
+     * This code simplifies the resulting regex as much as possible to help
+     * minimize the number of possible catastrophic backtracking cases that
+     * can be triggered by wildcard regexes.
+     *
+     * The following simplifications are currently performed:
+     *   - ''      -> /^$/   (empty string match)
+     *   '*'       -> //     (any string match)
+     *   - '*foo*' -> /foo/  (substring match)
+     *   - '*foo'  -> /foo$/ (suffix match)
+     *   - 'foo*'  -> /^foo/ (prefix match)
+     *   - collapsing runs of consecutive '*' wildcards into a single
+     *     wildcard. *** is identical to ** which is identical to * etc,
+     *     as all these match 0-n characters each. This also works with
+     *     simplification, i.e. '***foo***' -> /foo/ and '***' -> //
+     */
+    static vespalib::string convertToRegex(vespalib::stringref globpattern);
     static bool containsVariables(vespalib::stringref expression);
 
     static const GlobOperator GLOB;
diff --git a/document/src/vespa/document/select/parser.cpp b/document/src/vespa/document/select/parser.cpp
index 9f015409011..fadb46e5aa3 100644
--- a/document/src/vespa/document/select/parser.cpp
+++ b/document/src/vespa/document/select/parser.cpp
@@ -1,5 +1,6 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 #include "parser.h"
+#include "parser_limits.h"
 #include "scanner.h"
 #include <vespa/document/base/exceptions.h>
 #include <vespa/document/util/stringutil.h>
@@ -8,7 +9,20 @@
 
 namespace document::select {
 
+namespace {
+
+void verify_expression_not_too_large(const std::string& expr) {
+    if (expr.size() > ParserLimits::MaxSelectionByteSize) {
+        throw ParsingFailedException(vespalib::make_string(
+                "expression is too large to be parsed (max %zu bytes, got %zu)",
+                ParserLimits::MaxSelectionByteSize, expr.size()));
+    }
+}
+
+}
+
 std::unique_ptr<Node> Parser::parse(const std::string& str) const {
+    verify_expression_not_too_large(str);
     try {
         std::istringstream ss(str);
         DocSelScanner scanner(&ss);
diff --git a/document/src/vespa/document/select/parser_limits.cpp b/document/src/vespa/document/select/parser_limits.cpp
new file mode 100644
index 00000000000..13e494b376f
--- /dev/null
+++ b/document/src/vespa/document/select/parser_limits.cpp
@@ -0,0 +1,13 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include "parser_limits.h"
+#include "parsing_failed_exception.h"
+#include <vespa/vespalib/util/stringfmt.h>
+
+namespace document::select {
+
+void throw_max_depth_exceeded_exception() {
+    throw ParsingFailedException(vespalib::make_string(
+            "expression is too deeply nested (max %u levels)", ParserLimits::MaxRecursionDepth));
+}
+
+}
diff --git a/document/src/vespa/document/select/parser_limits.h b/document/src/vespa/document/select/parser_limits.h
new file mode 100644
index 00000000000..24c0a165611
--- /dev/null
+++ b/document/src/vespa/document/select/parser_limits.h
@@ -0,0 +1,19 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <cstdint>
+#include <cstddef>
+
+namespace document::select {
+
+// Any resource constraints set for parsing document selection expressions
+struct ParserLimits {
+    // Max depth allowed for nodes in the AST tree.
+    constexpr static uint32_t MaxRecursionDepth    = 1024;
+    // Max size of entire input document selection string, in bytes.
+    constexpr static size_t   MaxSelectionByteSize = 1024*1024;
+};
+
+void __attribute__((noinline)) throw_max_depth_exceeded_exception();
+
+}
diff --git a/document/src/vespa/document/select/valuenode.h b/document/src/vespa/document/select/valuenode.h
index 04ed8178b40..8dd535a736a 100644
--- a/document/src/vespa/document/select/valuenode.h
+++ b/document/src/vespa/document/select/valuenode.h
@@ -5,12 +5,13 @@
  *
  * @brief Node representing a value in the tree
  *
- * @author H�kon Humberset
+ * @author Håkon Humberset
  */
 
 #pragma once
 
 #include "value.h"
+#include "parser_limits.h"
 
 namespace document::select {
 
@@ -22,8 +23,19 @@ class ValueNode : public Printable
 public:
     using UP = std::unique_ptr<ValueNode>;
 
-    ValueNode() : _parentheses(false) {}
-    virtual ~ValueNode() {}
+    explicit ValueNode(uint32_t max_depth)
+        : _max_depth(max_depth), _parentheses(false)
+    {
+        throw_parse_error_if_max_depth_exceeded();
+    }
+    ValueNode() : _max_depth(1), _parentheses(false) {}
+    ~ValueNode() override = default;
+
+    // See comments for same function in node.h for a description on how and why
+    // we track this. Since Node and ValueNode live in completely separate type
+    // hierarchies, this particular bit of code duplication is unfortunate but
+    // incurs the least cognitive overhead.
+    [[nodiscard]] uint32_t max_depth() const noexcept { return _max_depth; }
 
     void setParentheses() { _parentheses = true; }
     void clearParentheses() { _parentheses = false; }
@@ -34,9 +46,17 @@ public:
     virtual ValueNode::UP clone() const = 0;
     virtual std::unique_ptr<Value> traceValue(const Context &context, std::ostream &out) const;
 private:
+    uint32_t _max_depth;
     bool _parentheses; // Set to true if parentheses was used around this part
                        // Set such that we can recreate original query in print.
+
 protected:
+    void throw_parse_error_if_max_depth_exceeded() const {
+        if (_max_depth > ParserLimits::MaxRecursionDepth) {
+            throw_max_depth_exceeded_exception();
+        }
+    }
+
     ValueNode::UP wrapParens(ValueNode* node) const {
         ValueNode::UP ret(node);
         if (_parentheses) {
diff --git a/document/src/vespa/document/select/valuenodes.cpp b/document/src/vespa/document/select/valuenodes.cpp
index 5c7820b76d7..026623cf83c 100644
--- a/document/src/vespa/document/select/valuenodes.cpp
+++ b/document/src/vespa/document/select/valuenodes.cpp
@@ -21,10 +21,6 @@ LOG_SETUP(".document.select.valuenode");
 namespace document::select {
 
 namespace {
-    static const std::regex FIELD_NAME_REGEX("^([_A-Za-z][_A-Za-z0-9]*).*");
-}
-
-namespace {
     bool documentTypeEqualsName(const DocumentType& type, vespalib::stringref name)
     {
         if (type.getName() == name) return true;
@@ -40,7 +36,7 @@ namespace {
 
 InvalidValueNode::InvalidValueNode(vespalib::stringref name)
     : _name(name)
-{ }
+{}
 
 
 void
@@ -194,15 +190,33 @@ FieldValueNode::FieldValueNode(const vespalib::string& doctype,
 
 FieldValueNode::~FieldValueNode() = default;
 
-vespalib::string
-FieldValueNode::extractFieldName(const std::string & fieldExpression) {
-    std::smatch match;
+namespace {
 
-    if (std::regex_match(fieldExpression, match, FIELD_NAME_REGEX) && match[1].matched) {
-        return vespalib::string(match[1].first, match[1].second);
+size_t first_ident_length_or_npos(const vespalib::string& expr) {
+    for (size_t i = 0; i < expr.size(); ++i) {
+        switch (expr[i]) {
+        case '.':
+        case '{':
+        case '[':
+        case ' ':
+        case '\n':
+        case '\t':
+            return i;
+        default:
+            continue;
+        }
     }
+    return vespalib::string::npos;
+}
 
-    throw ParsingFailedException("Fatal: could not extract field name from field expression '" + fieldExpression + "'");
+}
+
+// TODO remove this pile of fun in favor of actually parsed AST nodes...!
+vespalib::string
+FieldValueNode::extractFieldName(const vespalib::string & fieldExpression) {
+    // When we get here the actual contents of the field expression shall already
+    // have been structurally and syntactically verified by the parser.
+    return fieldExpression.substr(0, first_ident_length_or_npos(fieldExpression));
 }
 
 namespace {
@@ -340,10 +354,34 @@ FieldValueNode::initFieldPath(const DocumentType& type) const {
     }
 }
 
+namespace {
+
+bool looks_like_complex_field_path(const vespalib::string& expr) {
+    for (const char c : expr) {
+        switch (c) {
+        case '.':
+        case '[':
+        case '{':
+            return true;
+        default: continue;
+        }
+    }
+    return false;
+}
+
+bool is_simple_imported_field(const vespalib::string& expr, const DocumentType& doc_type) {
+    if (looks_like_complex_field_path(expr)) {
+        return false;
+    }
+    return (doc_type.has_imported_field_name(expr));
+}
+
+}
+
 std::unique_ptr<Value>
 FieldValueNode::getValue(const Context& context) const
 {
-    if (context._doc == NULL) {
+    if (context._doc == nullptr) {
         return std::make_unique<InvalidValue>();
     }
 
@@ -352,7 +390,17 @@ FieldValueNode::getValue(const Context& context) const
     if (!documentTypeEqualsName(doc.getType(), _doctype)) {
         return std::make_unique<InvalidValue>();
     }
-    try{
+    // Imported fields can only be meaningfully evaluated inside Proton, so we
+    // explicitly treat them as if they are valid fields with missing values. This
+    // will be treated the same as if it's a normal field by the selection operators.
+    // This avoids any awkward interaction with Invalid values or having to
+    // augment the FieldPath code with knowledge of imported fields.
+    // When a selection is running inside Proton, it will patch FieldValueNodes for
+    // imported fields, which removes this check entirely.
+    if (is_simple_imported_field(_fieldExpression, doc.getType())) {
+        return std::make_unique<NullValue>();
+    }
+    try {
         initFieldPath(doc.getType());
 
         IteratorHandler handler;
@@ -363,7 +411,7 @@ FieldValueNode::getValue(const Context& context) const
         } else {
             const std::vector<ArrayValue::VariableValue>& values = handler.getValues();
 
-            if (values.size() == 0) {
+            if (values.empty()) {
                 return std::make_unique<NullValue>();
             } else {
                 return std::make_unique<ArrayValue>(handler.getValues());
@@ -399,7 +447,7 @@ FieldValueNode::print(std::ostream& out, bool verbose,
 std::unique_ptr<Value>
 FieldValueNode::traceValue(const Context &context, std::ostream& out) const
 {
-    if (context._doc == NULL) {
+    if (context._doc == nullptr) {
         return defaultTrace(getValue(context), out);
     }
     const Document &doc(*context._doc);
@@ -408,7 +456,12 @@ FieldValueNode::traceValue(const Context &context, std::ostream& out) const
             << _doctype << " document, thus resolving invalid.\n";
         return std::make_unique<InvalidValue>();
     }
-    try{
+    if (is_simple_imported_field(_fieldExpression, doc.getType())) {
+        out << "Field '" << _fieldExpression << "' refers to an imported field; "
+            << "returning NullValue to treat this as an unset field value.\n";
+        return std::make_unique<NullValue>();
+    }
+    try {
         initFieldPath(doc.getType());
 
         IteratorHandler handler;
@@ -805,7 +858,8 @@ FunctionValueNode::print(std::ostream& out, bool verbose,
 ArithmeticValueNode::ArithmeticValueNode(
         std::unique_ptr<ValueNode> left, vespalib::stringref op,
         std::unique_ptr<ValueNode> right)
-    : _operator(),
+    : ValueNode(std::max(left->max_depth(), right->max_depth()) + 1),
+      _operator(),
       _left(std::move(left)),
       _right(std::move(right))
 {
diff --git a/document/src/vespa/document/select/valuenodes.h b/document/src/vespa/document/select/valuenodes.h
index 8009542c364..a7d5fa15f37 100644
--- a/document/src/vespa/document/select/valuenodes.h
+++ b/document/src/vespa/document/select/valuenodes.h
@@ -160,7 +160,7 @@ public:
     FieldValueNode & operator = (const FieldValueNode &) = delete;
     FieldValueNode(FieldValueNode &&) = default;
     FieldValueNode & operator = (FieldValueNode &&) = default;
-    ~FieldValueNode();
+    ~FieldValueNode() override;
 
     const vespalib::string& getDocType() const { return _doctype; }
     const vespalib::string& getRealFieldName() const { return _fieldName; }
@@ -175,7 +175,7 @@ public:
         return wrapParens(new FieldValueNode(_doctype, _fieldExpression));
     }
 
-    static vespalib::string extractFieldName(const std::string & fieldExpression);
+    static vespalib::string extractFieldName(const vespalib::string & fieldExpression);
 
 private:
 
@@ -192,13 +192,15 @@ class FieldExprNode final : public ValueNode {
 public:
     explicit FieldExprNode(const vespalib::string& doctype) : _left_expr(), _right_expr(doctype) {}
     FieldExprNode(std::unique_ptr<FieldExprNode> left_expr, vespalib::stringref right_expr)
-        : _left_expr(std::move(left_expr)), _right_expr(right_expr)
+        : ValueNode(left_expr->max_depth() + 1),
+          _left_expr(std::move(left_expr)),
+          _right_expr(right_expr)
     {}
     FieldExprNode(const FieldExprNode &) = delete;
     FieldExprNode & operator = (const FieldExprNode &) = delete;
     FieldExprNode(FieldExprNode &&) = default;
     FieldExprNode & operator = (FieldExprNode &&) = default;
-    ~FieldExprNode();
+    ~FieldExprNode() override;
 
     std::unique_ptr<FieldValueNode> convert_to_field_value() const;
     std::unique_ptr<FunctionValueNode> convert_to_function_call() const;
diff --git a/document/src/vespa/document/serialization/util.h b/document/src/vespa/document/serialization/util.h
index 05953f1de2b..0e0bfabdde5 100644
--- a/document/src/vespa/document/serialization/util.h
+++ b/document/src/vespa/document/serialization/util.h
@@ -2,6 +2,8 @@
 
 #pragma once
 
+#include <cstdint>
+
 namespace document {
 
 // Sets the value of a variable for the duration of this object's lifetime.
diff --git a/document/src/vespa/document/serialization/vespadocumentdeserializer.cpp b/document/src/vespa/document/serialization/vespadocumentdeserializer.cpp
index 039df466872..94644438f5c 100644
--- a/document/src/vespa/document/serialization/vespadocumentdeserializer.cpp
+++ b/document/src/vespa/document/serialization/vespadocumentdeserializer.cpp
@@ -74,15 +74,15 @@ void VespaDocumentDeserializer::readDocument(Document &value) {
     if (type) {
         Document::verifyIdAndType(value.getId(), type);
         value.setType(*type);
-        value.clear();
         value.setLastModified(0);
+    } else {
+        value.getFields().reset();
     }
     value.setRepo(_repo.getDocumentTypeRepo());
 
     FixedTypeRepo repo(_repo.getDocumentTypeRepo(), value.getType());
     VarScope<FixedTypeRepo> repo_scope(_repo, repo);
     uint32_t chunkCount = getChunkCount(content_code);
-    value.getFields().reset();
     for (uint32_t i = 0; i < chunkCount; ++i) {
         readStructNoReset(value.getFields());
     }
diff --git a/document/src/vespa/document/update/tensor_add_update.cpp b/document/src/vespa/document/update/tensor_add_update.cpp
index c35a8058133..2e5fa194c20 100644
--- a/document/src/vespa/document/update/tensor_add_update.cpp
+++ b/document/src/vespa/document/update/tensor_add_update.cpp
@@ -93,6 +93,7 @@ TensorAddUpdate::applyTo(FieldValue& value) const
 {
     if (value.inherits(TensorFieldValue::classId)) {
         TensorFieldValue &tensorFieldValue = static_cast<TensorFieldValue &>(value);
+        tensorFieldValue.make_empty_if_not_existing();
         auto &oldTensor = tensorFieldValue.getAsTensorPtr();
         auto newTensor = applyTo(*oldTensor);
         if (newTensor) {
diff --git a/document/src/vespa/document/update/tensor_modify_update.cpp b/document/src/vespa/document/update/tensor_modify_update.cpp
index 0d821de8922..dfc7479e5cd 100644
--- a/document/src/vespa/document/update/tensor_modify_update.cpp
+++ b/document/src/vespa/document/update/tensor_modify_update.cpp
@@ -174,9 +174,11 @@ TensorModifyUpdate::applyTo(FieldValue& value) const
     if (value.inherits(TensorFieldValue::classId)) {
         TensorFieldValue &tensorFieldValue = static_cast<TensorFieldValue &>(value);
         auto &oldTensor = tensorFieldValue.getAsTensorPtr();
-        auto newTensor = applyTo(*oldTensor);
-        if (newTensor) {
-            tensorFieldValue = std::move(newTensor);
+        if (oldTensor) {
+            auto newTensor = applyTo(*oldTensor);
+            if (newTensor) {
+                tensorFieldValue = std::move(newTensor);
+            }
         }
     } else {
         vespalib::string err = make_string("Unable to perform a tensor modify update on a '%s' field value",
diff --git a/document/src/vespa/document/update/tensor_remove_update.cpp b/document/src/vespa/document/update/tensor_remove_update.cpp
index f3bf7da7a0b..91b4c0a6ca3 100644
--- a/document/src/vespa/document/update/tensor_remove_update.cpp
+++ b/document/src/vespa/document/update/tensor_remove_update.cpp
@@ -120,9 +120,11 @@ TensorRemoveUpdate::applyTo(FieldValue &value) const
     if (value.inherits(TensorFieldValue::classId)) {
         TensorFieldValue &tensorFieldValue = static_cast<TensorFieldValue &>(value);
         auto &oldTensor = tensorFieldValue.getAsTensorPtr();
-        auto newTensor = applyTo(*oldTensor);
-        if (newTensor) {
-            tensorFieldValue = std::move(newTensor);
+        if (oldTensor) {
+            auto newTensor = applyTo(*oldTensor);
+            if (newTensor) {
+                tensorFieldValue = std::move(newTensor);
+            }
         }
     } else {
         vespalib::string err = make_string("Unable to perform a tensor remove update on a '%s' field value",
diff --git a/document/src/vespa/document/util/bytebuffer.cpp b/document/src/vespa/document/util/bytebuffer.cpp
index bc7e726cb06..644edb0664d 100644
--- a/document/src/vespa/document/util/bytebuffer.cpp
+++ b/document/src/vespa/document/util/bytebuffer.cpp
@@ -61,14 +61,6 @@ BufferOutOfBoundsException::BufferOutOfBoundsException(size_t pos, size_t len, c
 {
 }
 
-ByteBuffer::ByteBuffer(const char* buffer, uint32_t len) :
-      _buffer(const_cast<char *>(buffer)),
-      _len(len),
-      _pos(0),
-      _ownedBuffer()
-{
-}
-
 ByteBuffer::ByteBuffer(Alloc buffer, uint32_t len)
   : _buffer(static_cast<const char *>(buffer.get())),
     _len(len),
@@ -99,8 +91,6 @@ ByteBuffer::ByteBuffer(const ByteBuffer& rhs)
     }
 }
 
-ByteBuffer::~ByteBuffer() = default;
-
 ByteBuffer
 ByteBuffer::copyBuffer(const char* buffer, uint32_t len)
 {
diff --git a/document/src/vespa/document/util/bytebuffer.h b/document/src/vespa/document/util/bytebuffer.h
index 1eb359e261b..4c367c0f143 100644
--- a/document/src/vespa/document/util/bytebuffer.h
+++ b/document/src/vespa/document/util/bytebuffer.h
@@ -29,7 +29,7 @@ public:
     ByteBuffer& operator=(ByteBuffer &&) = default;
 
     ByteBuffer() : ByteBuffer(nullptr, 0) { }
-    ~ByteBuffer();
+    ~ByteBuffer() = default;
 
     /**
      * Create a buffer with the given content.
@@ -37,7 +37,12 @@ public:
      * @param buffer The buffer to represent.
      * @param len The length of the buffer
      */
-    ByteBuffer(const char* buffer, uint32_t len);
+    ByteBuffer(const char* buffer, uint32_t len)
+        : _buffer(const_cast<char *>(buffer)),
+          _len(len),
+          _pos(0),
+          _ownedBuffer()
+    { }
 
     /**
      * Create a buffer with the given content.
diff --git a/documentapi/src/main/java/com/yahoo/documentapi/DocumentOpVisitorResponse.java b/documentapi/src/main/java/com/yahoo/documentapi/DocumentOpVisitorResponse.java
index f5641b915f4..4316003acc6 100644
--- a/documentapi/src/main/java/com/yahoo/documentapi/DocumentOpVisitorResponse.java
+++ b/documentapi/src/main/java/com/yahoo/documentapi/DocumentOpVisitorResponse.java
@@ -10,6 +10,7 @@ import com.yahoo.document.DocumentOperation;
  * @author Arne H Juul
  */
 public class DocumentOpVisitorResponse extends VisitorResponse {
+
     private DocumentOperation op;
 
     /**
@@ -25,4 +26,5 @@ public class DocumentOpVisitorResponse extends VisitorResponse {
 
     /** @return the document operation */
     public DocumentOperation getDocumentOperation() { return op; }
+
 }
diff --git a/documentapi/src/main/java/com/yahoo/documentapi/SyncSession.java b/documentapi/src/main/java/com/yahoo/documentapi/SyncSession.java
index 6c4306b683c..cc0f6dc7cd5 100755
--- a/documentapi/src/main/java/com/yahoo/documentapi/SyncSession.java
+++ b/documentapi/src/main/java/com/yahoo/documentapi/SyncSession.java
@@ -23,15 +23,15 @@ public interface SyncSession extends Session {
      * Puts a document. When this method returns, the document is safely
      * received. This enables setting condition compared to using Document.
      *
-     * @param documentPut The DocumentPut operation
+     * @param documentPut the DocumentPut operation
      */
     void put(DocumentPut documentPut);
 
     /**
      * Puts a document. When this method returns, the document is safely received.
      *
-     * @param documentPut The DocumentPut operation
-     * @param priority The priority with which to perform this operation.
+     * @param documentPut the DocumentPut operation
+     * @param priority the priority with which to perform this operation
      */
     default void put(DocumentPut documentPut, DocumentProtocol.Priority priority) {
         put(documentPut);
@@ -40,11 +40,9 @@ public interface SyncSession extends Session {
     /**
      * Gets a document.
      *
-     * @param id The id of the document to get.
-     * @return The known document having this id, or null if there is no
-     *         document having this id.
-     * @throws UnsupportedOperationException Thrown if this access does not
-     *                                       support retrieving.
+     * @param id the id of the document to get.
+     * @return the known document having this id, or null if there is no document having this id
+     * @throws UnsupportedOperationException thrown if this access does not support retrieving
      */
     default Document get(DocumentId id) { return get(id, null); }
 
@@ -64,8 +62,8 @@ public interface SyncSession extends Session {
     /**
      * Gets a document with timeout.
      *
-     * @param id The id of the document to get
-     * @param timeout Timeout. If timeout is null, an unspecified default will be used
+     * @param id the id of the document to get
+     * @param timeout timeout. If timeout is null, an unspecified default will be used
      * @return the document with this id, or null if there is none
      * @throws UnsupportedOperationException thrown if this access does not support retrieving
      * @throws DocumentAccessException on any messagebus error, including timeout ({@link com.yahoo.messagebus.ErrorCode#TIMEOUT}).
@@ -75,53 +73,50 @@ public interface SyncSession extends Session {
     /**
      * Gets a document with timeout.
      *
-     * @param id       The id of the document to get.
-     * @param fieldSet A comma-separated list of fields to retrieve
-     * @param priority The priority with which to perform this operation.
-     * @param timeout Timeout. If timeout is null, an unspecified default will be used.
-     * @return The known document having this id, or null if there is no
-     *         document having this id.
-     * @throws UnsupportedOperationException Thrown if this access does not support retrieving.
-     * @throws DocumentAccessException on any messagebus error, including timeout ({@link com.yahoo.messagebus.ErrorCode#TIMEOUT}).
+     * @param id       the id of the document to get
+     * @param fieldSet a comma-separated list of fields to retrieve
+     * @param priority the priority with which to perform this operation
+     * @param timeout timeout. If timeout is null, an unspecified default will be used
+     * @return the known document having this id, or null if there is no document having this id
+     * @throws UnsupportedOperationException thrown if this access does not support retrieving
+     * @throws DocumentAccessException on any messagebus error, including timeout ({@link com.yahoo.messagebus.ErrorCode#TIMEOUT})
      */
     Document get(DocumentId id, String fieldSet, DocumentProtocol.Priority priority, Duration timeout);
 
     /**
-     * <p>Removes a document if it is present and condition is fulfilled.</p>
+     * Removes a document if it is present and condition is fulfilled.
+     *
      * @param documentRemove document to delete
-     * @return true if the document with this id was removed, false otherwise.
+     * @return true if the document with this id was removed, false otherwise
      */
     boolean remove(DocumentRemove documentRemove);
 
     /**
      * Removes a document if it is present.
      *
-     * @param documentRemove Document remove operation
-     * @param priority The priority with which to perform this operation.
-     * @return true If the document with this id was removed, false otherwise.
-     * @throws UnsupportedOperationException Thrown if this access does not
-     *                                       support removal.
+     * @param documentRemove document remove operation
+     * @param priority the priority with which to perform this operation
+     * @return true if the document with this id was removed, false otherwise.
+     * @throws UnsupportedOperationException thrown if this access does not support removal
      */
     boolean remove(DocumentRemove documentRemove, DocumentProtocol.Priority priority);
 
     /**
      * Updates a document.
      *
-     * @param update The updates to perform.
-     * @return True, if the document was found and updated.
-     * @throws UnsupportedOperationException Thrown if this access does not
-     *                                       support update.
+     * @param update the updates to perform
+     * @return true, if the document was found and updated
+     * @throws UnsupportedOperationException thrown if this access does not support update
      */
     boolean update(DocumentUpdate update);
 
     /**
      * Updates a document.
      *
-     * @param update   The updates to perform.
-     * @param priority The priority with which to perform this operation.
-     * @return True, if the document was found and updated.
-     * @throws UnsupportedOperationException Thrown if this access does not
-     *                                       support update.
+     * @param update   the updates to perform.
+     * @param priority the priority with which to perform this operation
+     * @return true, if the document was found and updated
+     * @throws UnsupportedOperationException thrown if this access does not support update
      */
     boolean update(DocumentUpdate update, DocumentProtocol.Priority priority);
 
diff --git a/documentapi/src/vespa/documentapi/messagebus/documentprotocol.cpp b/documentapi/src/vespa/documentapi/messagebus/documentprotocol.cpp
index a2e8219e916..01b8515400b 100644
--- a/documentapi/src/vespa/documentapi/messagebus/documentprotocol.cpp
+++ b/documentapi/src/vespa/documentapi/messagebus/documentprotocol.cpp
@@ -23,9 +23,9 @@ const mbus::string DocumentProtocol::NAME = "document";
 DocumentProtocol::DocumentProtocol(const LoadTypeSet& loadTypes,
                                    std::shared_ptr<const DocumentTypeRepo> repo,
                                    const string &configId) :
-    _routingPolicyRepository(new RoutingPolicyRepository()),
-    _routableRepository(new RoutableRepository(loadTypes)),
-    _repo(repo)
+    _routingPolicyRepository(std::make_unique<RoutingPolicyRepository>()),
+    _routableRepository(std::make_unique<RoutableRepository>(loadTypes)),
+    _repo(std::move(repo))
 {
     // Prepare config string for routing policy factories.
     string cfg = (configId.empty() ? "client" : configId);
@@ -148,10 +148,8 @@ DocumentProtocol &
 DocumentProtocol::putRoutableFactory(uint32_t type, IRoutableFactory::SP factory,
                                      const std::vector<vespalib::VersionSpecification> &versions)
 {
-    for (std::vector<vespalib::VersionSpecification>::const_iterator it = versions.begin();
-         it != versions.end(); ++it)
-    {
-        putRoutableFactory(type, factory, *it);
+    for (const auto & version : versions) {
+        putRoutableFactory(type, factory, version);
     }
     return *this;
 }
diff --git a/documentapi/src/vespa/documentapi/messagebus/documentprotocol.h b/documentapi/src/vespa/documentapi/messagebus/documentprotocol.h
index 5582c0ea153..eeae4553b3b 100644
--- a/documentapi/src/vespa/documentapi/messagebus/documentprotocol.h
+++ b/documentapi/src/vespa/documentapi/messagebus/documentprotocol.h
@@ -197,7 +197,7 @@ public:
     DocumentProtocol(const LoadTypeSet& loadTypes,
                      std::shared_ptr<const document::DocumentTypeRepo> repo,
                      const string &configId = "");
-    ~DocumentProtocol();
+    ~DocumentProtocol() override;
 
     /**
      * Adds a new routable factory to this protocol. This method is thread-safe, and may be invoked on a
diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/localservicepolicy.cpp b/documentapi/src/vespa/documentapi/messagebus/policies/localservicepolicy.cpp
index 42e1c07f3e8..7db941a12f2 100644
--- a/documentapi/src/vespa/documentapi/messagebus/policies/localservicepolicy.cpp
+++ b/documentapi/src/vespa/documentapi/messagebus/policies/localservicepolicy.cpp
@@ -53,8 +53,7 @@ LocalServicePolicy::getRecipient(mbus::RoutingContext &ctx)
     CacheEntry &entry = update(ctx);
     if (entry._recipients.empty()) {
         mbus::Hop hop = ctx.getRoute().getHop(0);
-        hop.setDirective(ctx.getDirectiveIndex(),
-                         mbus::IHopDirective::SP(new mbus::VerbatimDirective("*")));
+        hop.setDirective(ctx.getDirectiveIndex(), std::make_shared<mbus::VerbatimDirective>("*"));
         return hop;
     }
     if (++entry._offset >= entry._recipients.size()) {
diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/subsetservicepolicy.cpp b/documentapi/src/vespa/documentapi/messagebus/policies/subsetservicepolicy.cpp
index 7e2d54f318f..7e324468ae5 100644
--- a/documentapi/src/vespa/documentapi/messagebus/policies/subsetservicepolicy.cpp
+++ b/documentapi/src/vespa/documentapi/messagebus/policies/subsetservicepolicy.cpp
@@ -75,8 +75,7 @@ SubsetServicePolicy::getRecipient(mbus::RoutingContext &ctx)
     }
     if (!hop.hasDirectives()) {
         hop = ctx.getRoute().getHop(0);
-        hop.setDirective(ctx.getDirectiveIndex(),
-                         mbus::IHopDirective::SP(new mbus::VerbatimDirective("*")));
+        hop.setDirective(ctx.getDirectiveIndex(),std::make_shared<mbus::VerbatimDirective>("*"));
     }
     return hop;
 }
diff --git a/documentgen-test/etc/complex/book.sd b/documentgen-test/etc/complex/book.sd
index 872634bf53b..dd6ccafeab5 100644
--- a/documentgen-test/etc/complex/book.sd
+++ b/documentgen-test/etc/complex/book.sd
@@ -67,6 +67,10 @@ search book {
             attribute: tensor(x{})
         }
     }
+
+    import field ref.dummy as my_dummy {}
+    import field ref.foo as my_foo {}
+
     field sw1 type float {
     }
     field didinteger type array<int> {
diff --git a/documentgen-test/etc/complex/parent.sd b/documentgen-test/etc/complex/parent.sd
index 99a50fdf8ea..50b8e76cf5a 100644
--- a/documentgen-test/etc/complex/parent.sd
+++ b/documentgen-test/etc/complex/parent.sd
@@ -4,7 +4,10 @@
 search parent {
     document parent {
         field dummy type string {
-
+            indexing: attribute
+        }
+        field foo type string {
+            indexing: attribute
         }
     }
 }
diff --git a/documentgen-test/src/test/java/com/yahoo/vespa/config/DocumentGenPluginTest.java b/documentgen-test/src/test/java/com/yahoo/vespa/config/DocumentGenPluginTest.java
index 6339416d007..29bee2e9e3e 100644
--- a/documentgen-test/src/test/java/com/yahoo/vespa/config/DocumentGenPluginTest.java
+++ b/documentgen-test/src/test/java/com/yahoo/vespa/config/DocumentGenPluginTest.java
@@ -1028,5 +1028,14 @@ public class DocumentGenPluginTest {
         assertTrue(book.getDataType().fieldSetAll().contains(posZcurve));
         assertTrue(book.getDataType().getFields().contains(posZcurve));
     }
+
+    @Test
+    public void imported_fields_are_enumerated_in_document_type() {
+        var docType = getBook().getDataType();
+        assertEquals(2, docType.getImportedFieldNames().size());
+        assertTrue(docType.hasImportedField("my_dummy"));
+        assertTrue(docType.hasImportedField("my_foo"));
+        assertFalse(docType.hasImportedField("some_field_that_does_not_exist"));
+    }
     
 }
diff --git a/eval/CMakeLists.txt b/eval/CMakeLists.txt
index b1212650a3f..8eb198a9a0b 100644
--- a/eval/CMakeLists.txt
+++ b/eval/CMakeLists.txt
@@ -22,6 +22,7 @@ vespa_define_module(
     src/tests/eval/param_usage
     src/tests/eval/simple_tensor
     src/tests/eval/tensor_function
+    src/tests/eval/tensor_lambda
     src/tests/eval/tensor_spec
     src/tests/eval/value_cache
     src/tests/eval/value_type
@@ -33,6 +34,7 @@ vespa_define_module(
     src/tests/tensor/dense_generic_join
     src/tests/tensor/dense_inplace_join_function
     src/tests/tensor/dense_inplace_map_function
+    src/tests/tensor/dense_matmul_function    
     src/tests/tensor/dense_remove_dimension_optimizer
     src/tests/tensor/dense_replace_type_function
     src/tests/tensor/dense_tensor_create_function
diff --git a/eval/src/tests/ann/CMakeLists.txt b/eval/src/tests/ann/CMakeLists.txt
index 05256d19f00..0ba38994c01 100644
--- a/eval/src/tests/ann/CMakeLists.txt
+++ b/eval/src/tests/ann/CMakeLists.txt
@@ -9,3 +9,23 @@ vespa_add_executable(eval_sift_benchmark_app
     DEPENDS
     vespaeval
 )
+
+vespa_add_executable(eval_gist_benchmark_app
+    SOURCES
+    gist_benchmark.cpp
+    xp-annoy-nns.cpp
+    extended-hnsw.cpp
+    xp-lsh-nns.cpp
+    DEPENDS
+    vespaeval
+)
+
+vespa_add_executable(eval_remove_bm_app
+    SOURCES
+    remove-bm.cpp
+    xp-annoy-nns.cpp
+    xp-hnswlike-nns.cpp
+    xp-lsh-nns.cpp
+    DEPENDS
+    vespaeval
+)
diff --git a/eval/src/tests/ann/bruteforce-nns.h b/eval/src/tests/ann/bruteforce-nns.h
new file mode 100644
index 00000000000..ecac73c0d10
--- /dev/null
+++ b/eval/src/tests/ann/bruteforce-nns.h
@@ -0,0 +1,74 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+std::vector<TopK> bruteforceResults;
+
+double computeDistance(const PointVector &query, uint32_t docid) {
+    const PointVector &docvector = generatedDocs[docid];
+    return l2distCalc.l2sq_dist(query, docvector);
+}
+
+struct BfHitComparator {
+    bool operator() (const Hit &lhs, const Hit& rhs) const {
+        if (lhs.distance < rhs.distance) return false;
+        if (lhs.distance > rhs.distance) return true;
+        return (lhs.docid > rhs.docid);
+    }
+};
+
+class BfHitHeap {
+private:
+    size_t _size;
+    vespalib::PriorityQueue<Hit, BfHitComparator> _priQ;
+public:
+    explicit BfHitHeap(size_t maxSize) : _size(maxSize), _priQ() {
+        _priQ.reserve(maxSize);
+    }
+    ~BfHitHeap() {}
+    void maybe_use(const Hit &hit) {
+        if (_priQ.size() < _size) {
+            _priQ.push(hit);
+        } else if (hit.distance < _priQ.front().distance) {
+            _priQ.front() = hit;
+            _priQ.adjust();
+        }
+    }
+    std::vector<Hit> bestHits() {
+        std::vector<Hit> result;
+        size_t i = _priQ.size();
+        result.resize(i);
+        while (i-- > 0) {
+            result[i] = _priQ.front();
+            _priQ.pop_front();
+        }
+        return result;
+    }
+};
+
+TopK bruteforce_nns(const PointVector &query) {
+    TopK result;
+    BfHitHeap heap(result.K);
+    for (uint32_t docid = 0; docid < EFFECTIVE_DOCS; ++docid) {
+        const PointVector &docvector = generatedDocs[docid];
+        double d = l2distCalc.l2sq_dist(query, docvector);
+        Hit h(docid, d);
+        heap.maybe_use(h);
+    }
+    std::vector<Hit> best = heap.bestHits();
+    for (size_t i = 0; i < result.K; ++i) {
+        result.hits[i] = best[i];
+    }
+    return result;
+}
+
+void verifyBF(uint32_t qid) {
+    const PointVector &query = generatedQueries[qid];
+    TopK &result = bruteforceResults[qid];
+    double min_distance = result.hits[0].distance;
+    for (uint32_t i = 0; i < EFFECTIVE_DOCS; ++i) {
+        double dist = computeDistance(query, i);
+        if (dist < min_distance) {
+            fprintf(stderr, "WARN dist %.9g < mindist %.9g\n", dist, min_distance);
+        }
+        EXPECT_FALSE(dist+0.000001 < min_distance);
+    }
+}
diff --git a/eval/src/tests/ann/extended-hnsw.cpp b/eval/src/tests/ann/extended-hnsw.cpp
new file mode 100644
index 00000000000..fbc4bedec05
--- /dev/null
+++ b/eval/src/tests/ann/extended-hnsw.cpp
@@ -0,0 +1,636 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "hnsw-like.h"
+
+static size_t distcalls_simple;
+static size_t distcalls_search_layer;
+static size_t distcalls_other;
+static size_t distcalls_heuristic;
+static size_t distcalls_shrink;
+static size_t distcalls_refill;
+static size_t refill_needed_calls;
+static size_t shrink_needed_calls;
+static size_t disconnected_weak_links;
+static size_t disconnected_for_symmetry;
+static size_t select_n_full;
+static size_t select_n_partial;
+
+
+HnswLikeNns::HnswLikeNns(uint32_t numDims, const DocVectorAccess<float> &dva)
+    : NNS(numDims, dva),
+      _nodes(),
+      _entryId(0),
+      _entryLevel(-1),
+      _M(16),
+      _efConstruction(200),
+      _levelMultiplier(1.0 / log(1.0 * _M)),
+      _rndGen(),
+      _ops_counter(0)
+{
+}
+
+// simple greedy search
+HnswHit
+HnswLikeNns::search_layer_simple(Vector vector, HnswHit curPoint, uint32_t searchLevel) {
+    bool keepGoing = true;
+    while (keepGoing) {
+        keepGoing = false;
+        const LinkList& neighbors = getLinkList(curPoint.docid, searchLevel);
+        for (uint32_t n_id : neighbors) {
+            double dist = distance(vector, n_id);
+            ++distcalls_simple;
+            if (dist < curPoint.dist) {
+                curPoint = HnswHit(n_id, SqDist(dist));
+                keepGoing = true;
+            }
+        }
+    }
+    return curPoint;
+}
+
+
+bool
+HnswLikeNns::haveCloserDistance(HnswHit e, const LinkList &r) const {
+    for (uint32_t prevId : r) {
+        double dist = distance(e.docid, prevId);
+        ++distcalls_heuristic;
+        if (dist < e.dist) return true;
+    }
+    return false;
+}
+
+void
+HnswLikeNns::addDoc(uint32_t docid) {
+    Vector vector = _dva.get(docid);
+    for (uint32_t id = _nodes.size(); id <= docid; ++id) {
+        _nodes.emplace_back(id, 0, _M);
+    }
+    int level = randomLevel();
+    assert(_nodes[docid]._links.size() == 0);
+    _nodes[docid] = Node(docid, level+1, _M);
+    if (_entryLevel < 0) {
+        _entryId = docid;
+        _entryLevel = level;
+        track_ops();
+        return;
+    }
+    int searchLevel = _entryLevel;
+    VisitedSet &visited = _visitedSetPool.get(_nodes.size());
+    double entryDist = distance(vector, _entryId);
+    ++distcalls_other;
+    HnswHit entryPoint(_entryId, SqDist(entryDist));
+#undef MULTI_ENTRY_I
+#ifdef MULTI_ENTRY_I
+    FurthestPriQ w;
+    w.push(entryPoint);
+    while (searchLevel > level) {
+        search_layer(vector, w, visited, 5 * _M, searchLevel);
+        --searchLevel;
+    }
+#else
+    while (searchLevel > level) {
+        entryPoint = search_layer_simple(vector, entryPoint, searchLevel);
+        --searchLevel;
+    }
+    FurthestPriQ w;
+    w.push(entryPoint);
+#endif
+    searchLevel = std::min(level, _entryLevel);
+    while (searchLevel >= 0) {
+        search_layer(vector, w, visited, _efConstruction, searchLevel);
+        LinkList neighbors = select_neighbors(w.peek(), _M);
+        connect_new_node(docid, neighbors, searchLevel);
+        each_shrink_ifneeded(neighbors, searchLevel);
+        --searchLevel;
+    }
+    if (level > _entryLevel) {
+        _entryLevel = level;
+        _entryId = docid;
+    }
+    track_ops();
+}
+
+void
+HnswLikeNns::track_ops() {
+    _ops_counter++;
+    if ((_ops_counter % 10000) == 0) {
+        double div = _ops_counter;
+        fprintf(stderr, "add / remove ops: %zu\n", _ops_counter);
+        fprintf(stderr, "distance calls for layer: %zu is %.3f per op\n", distcalls_search_layer, distcalls_search_layer/ div);
+        fprintf(stderr, "distance calls for heuristic: %zu is %.3f per op\n", distcalls_heuristic, distcalls_heuristic / div);
+        fprintf(stderr, "distance calls for simple: %zu is %.3f per op\n", distcalls_simple, distcalls_simple / div);
+        fprintf(stderr, "distance calls for shrink: %zu is %.3f per op\n", distcalls_shrink, distcalls_shrink / div);
+        fprintf(stderr, "distance calls for refill: %zu is %.3f per op\n", distcalls_refill, distcalls_refill / div);
+        fprintf(stderr, "distance calls for other: %zu is %.3f per op\n", distcalls_other, distcalls_other / div);
+        fprintf(stderr, "refill needed calls: %zu is %.3f per op\n", refill_needed_calls, refill_needed_calls / div);
+        fprintf(stderr, "shrink needed calls: %zu is %.3f per op\n", shrink_needed_calls, shrink_needed_calls / div);
+        fprintf(stderr, "disconnected weak links: %zu is %.3f per op\n", disconnected_weak_links, disconnected_weak_links / div);
+        fprintf(stderr, "disconnected for symmetry: %zu is %.3f per op\n", disconnected_for_symmetry, disconnected_for_symmetry / div);
+        fprintf(stderr, "select neighbors: partial %zu vs full %zu\n", select_n_partial, select_n_full);
+    }
+}                    
+
+#ifdef SIMPLE_REFILL
+void
+HnswLikeNns::refill_ifneeded(uint32_t my_id, const LinkList &replacements, uint32_t level) {
+    LinkList &my_links = getLinkList(my_id, level);
+    if (my_links.size() * 2 < _M) {
+        const uint32_t maxLinks = (level > 0) ? _M : (2 * _M);
+        ++refill_needed_calls;
+        for (uint32_t repl_id : replacements) {
+            if (repl_id == my_id) continue;
+            if (my_links.has_link_to(repl_id)) continue;
+            LinkList &other_links = getLinkList(repl_id, level);
+            if (other_links.size() >= maxLinks) continue;
+            other_links.push_back(my_id);
+            my_links.push_back(repl_id);
+            if (my_links.size() >= _M) return;
+        }
+    }
+}
+#endif
+
+#define REFILL_ALL
+#ifdef REFILL_ALL
+void
+HnswLikeNns::refill_ifneeded(uint32_t my_id, const LinkList &replacements, uint32_t level) {
+    LinkList &my_links = getLinkList(my_id, level);
+    if (my_links.size() >= _M) return;
+    ++refill_needed_calls;
+    const uint32_t maxLinks = (level > 0) ? _M : (2 * _M);
+    NearestPriQ w;
+    for (uint32_t repl_id : replacements) {
+        if (repl_id == my_id) continue;
+        if (my_links.has_link_to(repl_id)) continue;
+        const LinkList &other_links = getLinkList(repl_id, level);
+        if (other_links.size() >= maxLinks) continue;
+        double dist = distance(my_id, repl_id);
+        ++distcalls_refill;
+        w.emplace(repl_id, SqDist(dist));
+    }
+    while (! w.empty()) {
+        HnswHit e = w.top();
+        w.pop();
+        if (haveCloserDistance(e, my_links)) continue;
+        LinkList &other_links = getLinkList(e.docid, level);
+        my_links.push_back(e.docid);
+        other_links.push_back(my_id);
+        if (my_links.size() == _M) break;
+    }
+}
+#endif
+
+#ifdef REFILL_ONE
+void
+HnswLikeNns::refill_ifneeded(uint32_t my_id, const LinkList &replacements, uint32_t level) {
+    LinkList &my_links = getLinkList(my_id, level);
+    if (my_links.size() >= _M) return;
+    ++refill_needed_calls;
+    NearestPriQ w;
+    for (uint32_t repl_id : replacements) {
+        if (repl_id == my_id) continue;
+        if (my_links.has_link_to(repl_id)) continue;
+        LinkList &other_links = getLinkList(repl_id, level);
+        if (other_links.size() >= _M) continue;
+        double dist = distance(my_id, repl_id);
+        ++distcalls_refill;
+        w.emplace(repl_id, SqDist(dist));
+    }
+    while (! w.empty()) {
+        HnswHit e = w.top();
+        w.pop();
+        if (haveCloserDistance(e, my_links)) continue;
+        LinkList &other_links = getLinkList(e.docid, level);
+        my_links.push_back(e.docid);
+        other_links.push_back(my_id);
+        return;
+    }
+}
+#endif
+
+void
+HnswLikeNns::shrink_links(uint32_t shrink_id, uint32_t maxLinks, uint32_t level) {
+    LinkList &links = getLinkList(shrink_id, level);
+    NearestList distances;
+    for (uint32_t n_id : links) {
+        double n_dist = distance(shrink_id, n_id);
+        ++distcalls_shrink;
+        distances.emplace_back(n_id, SqDist(n_dist));
+    }
+    LinkList lostLinks;
+    LinkList oldLinks = links;
+    links = remove_weakest(distances, maxLinks, lostLinks);
+#define KEEP_SYM
+#ifdef KEEP_SYM
+    for (uint32_t lost_id : lostLinks) {
+        ++disconnected_for_symmetry;
+        remove_link_from(lost_id, shrink_id, level);
+    }
+#define DO_REFILL_AFTER_KEEP_SYM
+#ifdef DO_REFILL_AFTER_KEEP_SYM
+    for (uint32_t lost_id : lostLinks) {
+        refill_ifneeded(lost_id, oldLinks, level);
+    }
+#endif
+#endif
+}
+
+
+void
+HnswLikeNns::removeDoc(uint32_t docid) {
+    Node &node = _nodes[docid];
+    bool need_new_entrypoint = (docid == _entryId);
+    for (int level = node._links.size(); level-- > 0; ) {
+        LinkList my_links;
+        my_links.swap(node._links[level]);
+        for (uint32_t n_id : my_links) {
+            if (need_new_entrypoint) {
+                _entryId = n_id;
+                _entryLevel = level;
+                need_new_entrypoint = false;
+            }
+            remove_link_from(n_id, docid, level);
+        }
+        while (! my_links.empty()) {
+            uint32_t n_id = my_links.back();
+            my_links.pop_back();
+            refill_ifneeded(n_id, my_links, level);
+        }
+    }
+    node = Node(docid, 0, _M);
+    if (need_new_entrypoint) {
+        _entryLevel = -1;
+        _entryId = 0;
+        for (uint32_t i = 0; i < _nodes.size(); ++i) {
+            if (_nodes[i]._links.size() > 0) {
+                _entryId = i;
+                _entryLevel = _nodes[i]._links.size() - 1;
+                break;
+            }
+        }
+    }
+    track_ops();
+}
+
+std::vector<NnsHit>
+HnswLikeNns::topK(uint32_t k, Vector vector, uint32_t search_k) {
+    std::vector<NnsHit> result;
+    if (_entryLevel < 0) return result;
+    double entryDist = distance(vector, _entryId);
+    ++distcalls_other;
+    HnswHit entryPoint(_entryId, SqDist(entryDist));
+    int searchLevel = _entryLevel;
+    VisitedSet &visited = _visitedSetPool.get(_nodes.size());
+#undef MULTI_ENTRY_S
+#ifdef MULTI_ENTRY_S
+    FurthestPriQ w;
+    w.push(entryPoint);
+    while (searchLevel > 0) {
+        search_layer(vector, w, visited, std::min(k, search_k), searchLevel);
+        --searchLevel;
+    }
+#else
+    while (searchLevel > 0) {
+        entryPoint = search_layer_simple(vector, entryPoint, searchLevel);
+        --searchLevel;
+    }
+    FurthestPriQ w;
+    w.push(entryPoint);
+#endif
+    search_layer(vector, w, visited, std::max(k, search_k), 0);
+    while (w.size() > k) {
+        w.pop();
+    }
+    NearestList tmp = w.steal();
+    std::sort(tmp.begin(), tmp.end(), LesserDist());
+    result.reserve(tmp.size());
+    for (const auto & hit : tmp) {
+        result.emplace_back(hit.docid, SqDist(hit.dist));
+    }
+    return result;
+}
+
+
+double
+HnswLikeNns::distance(Vector v, uint32_t b) const
+{
+    Vector w = _dva.get(b);
+    return l2distCalc.l2sq_dist(v, w);
+}
+
+std::vector<NnsHit> 
+HnswLikeNns::topKfilter(uint32_t k, Vector vector, uint32_t search_k, const BitVector &blacklist)
+{
+    std::vector<NnsHit> result;
+    if (_entryLevel < 0) return result;
+    double entryDist = distance(vector, _entryId);
+    ++distcalls_other;
+    HnswHit entryPoint(_entryId, SqDist(entryDist));
+    int searchLevel = _entryLevel;
+    VisitedSet &visited = _visitedSetPool.get(_nodes.size());
+#ifdef MULTI_ENTRY_S
+    FurthestPriQ w;
+    w.push(entryPoint);
+    while (searchLevel > 0) {
+        search_layer(vector, w, visited, std::min(k, search_k), searchLevel);
+        --searchLevel;
+    }
+#else
+    while (searchLevel > 0) {
+        entryPoint = search_layer_simple(vector, entryPoint, searchLevel);
+        --searchLevel;
+    }
+    FurthestPriQ w;
+    w.push(entryPoint);
+#endif
+    search_layer_with_filter(vector, w, visited, std::max(k, search_k), 0, blacklist);
+    NearestList tmp = w.steal();
+    std::sort(tmp.begin(), tmp.end(), LesserDist());
+    result.reserve(std::min((size_t)k, tmp.size()));
+    for (const auto & hit : tmp) {
+        if (blacklist.isSet(hit.docid)) continue;
+        result.emplace_back(hit.docid, SqDist(hit.dist));
+        if (result.size() == k) break;
+    }
+    return result;
+}
+
+void
+HnswLikeNns::each_shrink_ifneeded(const LinkList &neighbors, uint32_t level) {
+    uint32_t maxLinks = (level > 0) ? _M : (2 * _M);
+    for (uint32_t old_id : neighbors) {
+        LinkList &oldLinks = getLinkList(old_id, level);
+        if (oldLinks.size() > maxLinks) {
+            ++shrink_needed_calls;
+            shrink_links(old_id, maxLinks, level);
+        }
+    }
+}
+
+void
+HnswLikeNns::search_layer(Vector vector, FurthestPriQ &w,
+                          VisitedSet &visited,
+                          uint32_t ef, uint32_t searchLevel)
+{
+    NearestPriQ candidates;
+
+    for (const HnswHit & entry : w.peek()) {
+        candidates.push(entry);
+        visited.mark(entry.docid);
+    }
+    double limd = std::numeric_limits<double>::max();
+    while (! candidates.empty()) {
+        HnswHit cand = candidates.top();
+        if (cand.dist > limd) {
+            break;
+        }
+        candidates.pop();
+        for (uint32_t e_id : getLinkList(cand.docid, searchLevel)) {
+            if (visited.isMarked(e_id)) continue;
+            visited.mark(e_id);
+            double e_dist = distance(vector, e_id);
+            ++distcalls_search_layer;
+            if (e_dist < limd) {
+                candidates.emplace(e_id, SqDist(e_dist));
+                w.emplace(e_id, SqDist(e_dist));
+                if (w.size() > ef) {
+                    w.pop();
+                    limd = w.top().dist;
+                }
+            }
+        }
+    }
+    return;
+}
+
+void
+HnswLikeNns::search_layer_with_filter(Vector vector, FurthestPriQ &w,
+                                      VisitedSet &visited,
+                                      uint32_t ef, uint32_t searchLevel,
+                                      const BitVector &blacklist)
+{
+    NearestPriQ candidates;
+
+    for (const HnswHit & entry : w.peek()) {
+        candidates.push(entry);
+        visited.mark(entry.docid);
+        if (blacklist.isSet(entry.docid)) ++ef;
+    }
+    double limd = std::numeric_limits<double>::max();
+    while (! candidates.empty()) {
+        HnswHit cand = candidates.top();
+        if (cand.dist > limd) {
+            break;
+        }
+        candidates.pop();
+        for (uint32_t e_id : getLinkList(cand.docid, searchLevel)) {
+            if (visited.isMarked(e_id)) continue;
+            visited.mark(e_id);
+            double e_dist = distance(vector, e_id);
+            ++distcalls_search_layer;
+            if (e_dist < limd) {
+                candidates.emplace(e_id, SqDist(e_dist));
+                if (blacklist.isSet(e_id)) continue;
+                w.emplace(e_id, SqDist(e_dist));
+                if (w.size() > ef) {
+                    w.pop();
+                    limd = w.top().dist;
+                }
+            }
+        }
+    }
+}
+
+LinkList
+HnswLikeNns::remove_weakest(const NearestList &neighbors, uint32_t curMax, LinkList &lost) const
+{
+    LinkList result;
+    result.reserve(curMax+1);
+    NearestPriQ w;
+    for (const auto & entry : neighbors) {
+        w.push(entry);
+    }
+    while (! w.empty()) {
+        HnswHit e = w.top();
+        w.pop();
+        if (result.size() == curMax || haveCloserDistance(e, result)) {
+            lost.push_back(e.docid);
+        } else {
+            result.push_back(e.docid);
+        }
+    }
+    return result;
+}
+
+#define NO_BACKFILL
+#ifdef NO_BACKFILL
+LinkList
+HnswLikeNns::select_neighbors(const NearestList &neighbors, uint32_t curMax) const
+{
+    LinkList result;
+    result.reserve(curMax+1);
+    NearestPriQ w;
+    for (const auto & entry : neighbors) {
+        w.push(entry);
+    }
+    while (! w.empty()) {
+        HnswHit e = w.top();
+        w.pop();
+        if (haveCloserDistance(e, result)) {
+            continue;
+        }
+        result.push_back(e.docid);
+        if (result.size() == curMax) {
+            ++select_n_full;
+            return result;
+        }
+    }
+    ++select_n_partial;
+    return result;
+}
+#else
+LinkList
+HnswLikeNns::select_neighbors(const NearestList &neighbors, uint32_t curMax) const
+{
+    LinkList result;
+    result.reserve(curMax+1);
+    bool needFiltering = (neighbors.size() > curMax);
+    NearestPriQ w;
+    for (const auto & entry : neighbors) {
+        w.push(entry);
+    }
+    LinkList backfill;
+    while (! w.empty()) {
+        HnswHit e = w.top();
+        w.pop();
+        if (needFiltering && haveCloserDistance(e, result)) {
+            backfill.push_back(e.docid);
+            continue;
+        }
+        result.push_back(e.docid);
+        if (result.size() == curMax) return result;
+    }
+    if (result.size() * 4 < _M) {
+        for (uint32_t fill_id : backfill) {
+            result.push_back(fill_id);
+            if (result.size() * 2 >= _M) break;
+        }
+    }
+    return result;
+}
+#endif
+
+void
+HnswLikeNns::connect_new_node(uint32_t id, const LinkList &neighbors, uint32_t level) {
+    LinkList &newLinks = getLinkList(id, level);
+    for (uint32_t neigh_id : neighbors) {
+        LinkList &oldLinks = getLinkList(neigh_id, level);
+        newLinks.push_back(neigh_id);
+        oldLinks.push_back(id);
+    }
+#define DISCONNECT_OLD_WEAK_LINKS
+#ifdef DISCONNECT_OLD_WEAK_LINKS
+    for (uint32_t i = 1; i < neighbors.size(); ++i) {
+        uint32_t n_1 = neighbors[i];
+        LinkList &links_1 = getLinkList(n_1, level);
+        for (uint32_t j = 0; j < i; ++j) {
+            uint32_t n_2 = neighbors[j];
+            if (links_1.has_link_to(n_2)) {
+                ++disconnected_weak_links;
+                LinkList &links_2 = getLinkList(n_2, level);
+                links_1.remove_link(n_2);
+                links_2.remove_link(n_1);
+            }
+        }
+    }
+#endif
+}
+
+uint32_t
+HnswLikeNns::count_reachable() const {
+    VisitedSet visited(_nodes.size());
+    int level = _entryLevel;
+    LinkList curList;
+    curList.push_back(_entryId);
+    visited.mark(_entryId);
+    uint32_t idx = 0;
+    while (level >= 0) {
+        while (idx < curList.size()) {
+            uint32_t id = curList[idx++];
+            const LinkList &links = getLinkList(id, level);
+            for (uint32_t n_id : links) {
+                if (visited.isMarked(n_id)) continue;
+                visited.mark(n_id);
+                curList.push_back(n_id);
+            }
+        }
+        --level;
+        idx = 0;
+    }
+    return curList.size();
+}
+
+void
+HnswLikeNns::dumpStats() const {
+    std::vector<uint32_t> levelCounts;
+    levelCounts.resize(_entryLevel + 2);
+    std::vector<uint32_t> outLinkHist;
+    outLinkHist.resize(2 * _M + 2);
+    uint32_t symmetrics = 0;
+    uint32_t level1links = 0;
+    uint32_t both_l_links = 0;
+    fprintf(stderr, "stats for HnswLikeNns with %zu nodes, entry level = %d, entry id = %u\n",
+            _nodes.size(), _entryLevel, _entryId);
+    
+    for (uint32_t id = 0; id < _nodes.size(); ++id) {
+        const auto &node = _nodes[id];
+        uint32_t levels = node._links.size();
+        levelCounts[levels]++;
+        if (levels < 1) {
+            outLinkHist[0]++;
+            continue;
+        }
+        const LinkList &link_list = getLinkList(id, 0);
+        uint32_t numlinks = link_list.size();
+        outLinkHist[numlinks]++;
+        if (numlinks < 1) {
+            fprintf(stderr, "node with %u links: id %u\n", numlinks, id);
+        }
+        bool all_sym = true;
+        for (uint32_t n_id : link_list) {
+            const LinkList &neigh_list = getLinkList(n_id, 0);
+            if (! neigh_list.has_link_to(id)) {
+#ifdef KEEP_SYM
+                fprintf(stderr, "BAD: %u has link to neighbor %u, but backlink is missing\n", id, n_id);
+#endif
+                all_sym = false;
+            }
+        }
+        if (all_sym) ++symmetrics;
+        if (levels < 2) continue;
+        const LinkList &link_list_1 = getLinkList(id, 1);
+        for (uint32_t n_id : link_list_1) {
+            ++level1links;
+            if (link_list.has_link_to(n_id)) ++both_l_links;
+        }
+    }
+    for (uint32_t l = 0; l < levelCounts.size(); ++l) {
+        fprintf(stderr, "Nodes on %u levels: %u\n", l, levelCounts[l]);
+    }
+    fprintf(stderr, "reachable nodes %u / %zu\n",
+            count_reachable(), _nodes.size() - levelCounts[0]);
+    fprintf(stderr, "level 1 links overlapping on l0: %u / total: %u\n",
+            both_l_links, level1links);
+    for (uint32_t l = 0; l < outLinkHist.size(); ++l) {
+        if (outLinkHist[l] != 0) {
+            fprintf(stderr, "Nodes with %u outward links on L0: %u\n", l, outLinkHist[l]);
+        }
+    }
+    fprintf(stderr, "Symmetric in-out nodes: %u\n", symmetrics);
+}
+
+std::unique_ptr<NNS<float>>
+make_hnsw_nns(uint32_t numDims, const DocVectorAccess<float> &dva)
+{
+    return std::make_unique<HnswLikeNns>(numDims, dva);
+}
diff --git a/eval/src/tests/ann/find-with-nns.h b/eval/src/tests/ann/find-with-nns.h
new file mode 100644
index 00000000000..3481b403f86
--- /dev/null
+++ b/eval/src/tests/ann/find-with-nns.h
@@ -0,0 +1,12 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+TopK find_with_nns(uint32_t sk, NNS_API &nns, uint32_t qid) {
+    TopK result;
+    const PointVector &qv = generatedQueries[qid];
+    vespalib::ConstArrayRef<float> query(qv.v, NUM_DIMS);
+    auto rv = nns.topK(result.K, query, sk);
+    for (size_t i = 0; i < result.K; ++i) {
+        result.hits[i] = Hit(rv[i].docid, rv[i].sq.distance);
+    }
+    return result;
+}
diff --git a/eval/src/tests/ann/for-sift-top-k.h b/eval/src/tests/ann/for-sift-top-k.h
index ba91cb2aebc..8a659a507bc 100644
--- a/eval/src/tests/ann/for-sift-top-k.h
+++ b/eval/src/tests/ann/for-sift-top-k.h
@@ -6,7 +6,7 @@ struct TopK {
     static constexpr size_t K = 100;
     Hit hits[K];
 
-    size_t recall(const TopK &other) {
+    size_t recall(const TopK &other) const {
         size_t overlap = 0;
         size_t i = 0;
         size_t j = 0;
diff --git a/eval/src/tests/ann/gist_benchmark.cpp b/eval/src/tests/ann/gist_benchmark.cpp
new file mode 100644
index 00000000000..5a317e77e72
--- /dev/null
+++ b/eval/src/tests/ann/gist_benchmark.cpp
@@ -0,0 +1,143 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/vespalib/testkit/test_kit.h>
+#include <vespa/vespalib/util/priority_queue.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <stdio.h>
+#include <chrono>
+
+#define NUM_DIMS 960
+#define NUM_DOCS 200000
+#define EFFECTIVE_DOCS NUM_DOCS
+#define NUM_REACH 10000
+#define NUM_Q 1000
+
+#include "doc_vector_access.h"
+#include "nns.h"
+#include "for-sift-hit.h"
+#include "for-sift-top-k.h"
+#include "time-util.h"
+#include "point-vector.h"
+#include "read-vecs.h"
+#include "bruteforce-nns.h"
+
+using NNS_API = NNS<float>;
+
+TEST("require that brute force works") {
+    TimePoint bef = std::chrono::steady_clock::now();
+    fprintf(stderr, "generating %u brute force results\n", NUM_Q);
+    bruteforceResults.reserve(NUM_Q);
+    for (uint32_t cnt = 0; cnt < NUM_Q; ++cnt) {
+        const PointVector &query = generatedQueries[cnt];
+        bruteforceResults.emplace_back(bruteforce_nns(query));
+    }
+    TimePoint aft = std::chrono::steady_clock::now();
+    fprintf(stderr, "timing for brute force: %.3f ms = %.3f ms per query\n",
+            to_ms(aft - bef), to_ms(aft - bef)/NUM_Q);
+    for (int cnt = 0; cnt < NUM_Q; cnt = (cnt+1)*2) {
+        verifyBF(cnt);
+    }
+}
+
+#include "find-with-nns.h"
+#include "verify-top-k.h"
+
+void timing_nns(const char *name, NNS_API &nns, std::vector<uint32_t> sk_list) {
+    for (uint32_t search_k : sk_list) {
+        TimePoint bef = std::chrono::steady_clock::now();
+        for (int cnt = 0; cnt < NUM_Q; ++cnt) {
+            find_with_nns(search_k, nns, cnt);
+        }
+        TimePoint aft = std::chrono::steady_clock::now();
+        fprintf(stderr, "timing for %s search_k=%u: %.3f ms = %.3f ms/q\n",
+                name, search_k, to_ms(aft - bef), to_ms(aft - bef)/NUM_Q);
+    }
+}
+
+#include "quality-nns.h"
+
+template <typename FUNC>
+void bm_nns_simple(const char *name, FUNC creator, std::vector<uint32_t> sk_list) {
+    std::unique_ptr<NNS_API> nnsp = creator();
+    NNS_API &nns = *nnsp;
+    fprintf(stderr, "trying %s indexing...\n", name);
+    TimePoint bef = std::chrono::steady_clock::now();
+    for (uint32_t i = 0; i < NUM_DOCS; ++i) {
+        nns.addDoc(i);
+    }
+    TimePoint aft = std::chrono::steady_clock::now();
+    fprintf(stderr, "build %s index with %u docs: %.3f ms\n", name, NUM_DOCS, to_ms(aft - bef));
+    timing_nns(name, nns, sk_list);
+    fprintf(stderr, "Quality for %s [A] clean build with %u documents:\n", name, NUM_DOCS);
+    quality_nns(nns, sk_list);
+}
+
+template <typename FUNC>
+void benchmark_nns(const char *name, FUNC creator, std::vector<uint32_t> sk_list) {
+    bm_nns_simple(name, creator, sk_list);
+}
+
+#if 0
+TEST("require that Locality Sensitive Hashing mostly works") {
+    DocVectorAdapter adapter;
+    auto creator = [&adapter]() { return make_rplsh_nns(NUM_DIMS, adapter); };
+    benchmark_nns("RPLSH", creator, { 200, 1000 });
+}
+#endif
+
+#if 0
+TEST("require that Annoy via NNS api mostly works") {
+    DocVectorAdapter adapter;
+    auto creator = [&adapter]() { return make_annoy_nns(NUM_DIMS, adapter); };
+    benchmark_nns("Annoy", creator, { 8000, 10000 });
+}
+#endif
+
+#if 1
+TEST("require that HNSW via NNS api mostly works") {
+    DocVectorAdapter adapter;
+    auto creator = [&adapter]() { return make_hnsw_nns(NUM_DIMS, adapter); };
+    benchmark_nns("HNSW-like", creator, { 100, 150, 200 });
+}
+#endif
+
+#if 0
+TEST("require that HNSW wrapped api mostly works") {
+    DocVectorAdapter adapter;
+    auto creator = [&adapter]() { return make_hnsw_wrap(NUM_DIMS, adapter); };
+    benchmark_nns("HNSW-wrap", creator, { 100, 150, 200 });
+}
+#endif
+
+/**
+ * Before running the benchmark the ANN_GIST1M data set must be downloaded and extracted:
+ *   wget ftp://ftp.irisa.fr/local/texmex/corpus/gist.tar.gz
+ *   tar -xf gist.tar.gz
+ *
+ * The benchmark program will load the data set from $HOME/gist if no directory is specified.
+ *
+ * More information about the dataset is found here: http://corpus-texmex.irisa.fr/.
+ */
+int main(int argc, char **argv) {
+    TEST_MASTER.init(__FILE__);
+    std::string data_set = "gist";
+    std::string data_dir = ".";
+    if (argc > 2) {
+        data_set = argv[1];
+        data_dir = argv[2];
+    } else if (argc > 1) {
+        data_dir = argv[1];
+    } else {
+        char *home = getenv("HOME");
+        if (home) {
+            data_dir = home;
+            data_dir += "/" + data_set;
+        }
+    }
+    read_data(data_dir, data_set);
+    TEST_RUN_ALL();
+    return (TEST_MASTER.fini() ? 0 : 1);
+}
diff --git a/eval/src/tests/ann/hnsw-like.h b/eval/src/tests/ann/hnsw-like.h
new file mode 100644
index 00000000000..36064c69860
--- /dev/null
+++ b/eval/src/tests/ann/hnsw-like.h
@@ -0,0 +1,203 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <algorithm>
+#include <assert.h>
+#include <queue>
+#include <cinttypes>
+#include "std-random.h"
+#include "nns.h"
+
+struct LinkList : std::vector<uint32_t>
+{
+    bool has_link_to(uint32_t id) const {
+        auto iter = std::find(begin(), end(), id);
+        return (iter != end());
+    }
+    void remove_link(uint32_t id) {
+        uint32_t last = back();
+        for (iterator iter = begin(); iter != end(); ++iter) {
+            if (*iter == id) {
+                *iter = last;
+                pop_back();
+                return;
+            }
+        }
+        fprintf(stderr, "BAD missing link to remove: %u\n", id);
+        abort();
+    }
+};
+
+struct Node {
+    std::vector<LinkList> _links;
+    Node(uint32_t , uint32_t numLevels, uint32_t M)
+        : _links(numLevels)
+    {
+        for (uint32_t i = 0; i < _links.size(); ++i) {
+            _links[i].reserve((i == 0) ? (2 * M + 1) : (M+1));
+        }
+    }
+};
+
+struct VisitedSet
+{
+    using Mark = unsigned short;
+    Mark *ptr;
+    Mark curval;
+    size_t sz;
+    VisitedSet(const VisitedSet &) = delete;
+    VisitedSet& operator=(const VisitedSet &) = delete;
+    explicit VisitedSet(size_t size) {
+        ptr = (Mark *)malloc(size * sizeof(Mark));
+        curval = -1;
+        sz = size;
+        clear();
+    }
+    void clear() {
+        ++curval;
+        if (curval == 0) {
+            memset(ptr, 0, sz * sizeof(Mark));
+            ++curval;
+        }
+    }
+    ~VisitedSet() { free(ptr); }
+    void mark(size_t id) { ptr[id] = curval; }
+    bool isMarked(size_t id) const { return ptr[id] == curval; }
+};
+
+struct VisitedSetPool
+{
+    std::unique_ptr<VisitedSet> lastUsed;
+    VisitedSetPool() {
+        lastUsed = std::make_unique<VisitedSet>(250);
+    }
+    ~VisitedSetPool() {}
+    VisitedSet &get(size_t size) {
+        if (size > lastUsed->sz) {
+            lastUsed = std::make_unique<VisitedSet>(size*2);
+        } else {
+            lastUsed->clear();
+        }
+        return *lastUsed;
+    }
+};
+
+struct HnswHit {
+    double dist;
+    uint32_t docid;
+    HnswHit(uint32_t di, SqDist sq) : dist(sq.distance), docid(di) {}
+};
+
+struct GreaterDist {
+    bool operator() (const HnswHit &lhs, const HnswHit& rhs) const {
+        return (rhs.dist < lhs.dist);
+    }
+};
+struct LesserDist {
+    bool operator() (const HnswHit &lhs, const HnswHit& rhs) const {
+        return (lhs.dist < rhs.dist);
+    }
+};
+
+using NearestList = std::vector<HnswHit>;
+
+struct NearestPriQ : std::priority_queue<HnswHit, NearestList, GreaterDist>
+{
+};
+
+struct FurthestPriQ : std::priority_queue<HnswHit, NearestList, LesserDist>
+{
+    NearestList steal() {
+        NearestList result;
+        c.swap(result);
+        return result;
+    }
+    const NearestList& peek() const { return c; }
+};
+
+class HnswLikeNns : public NNS<float>
+{
+private:
+    std::vector<Node> _nodes;
+    uint32_t _entryId;
+    int _entryLevel;
+    uint32_t _M;
+    uint32_t _efConstruction;
+    double _levelMultiplier;
+    RndGen _rndGen;
+    VisitedSetPool _visitedSetPool;
+    size_t _ops_counter;
+
+    double distance(Vector v, uint32_t id) const;
+
+    double distance(uint32_t a, uint32_t b) const {
+        Vector v = _dva.get(a);
+        return distance(v, b);
+    }
+
+    int randomLevel() {
+        double unif = _rndGen.nextUniform();
+        double r = -log(1.0-unif) * _levelMultiplier;
+        return (int) r;
+    }
+
+    uint32_t count_reachable() const;
+    void dumpStats() const;
+
+public:
+    HnswLikeNns(uint32_t numDims, const DocVectorAccess<float> &dva);
+    ~HnswLikeNns() { dumpStats(); }
+
+    LinkList& getLinkList(uint32_t docid, uint32_t level) {
+        return _nodes[docid]._links[level];
+    }
+
+    const LinkList& getLinkList(uint32_t docid, uint32_t level) const {
+        return _nodes[docid]._links[level];
+    }
+
+    HnswHit search_layer_simple(Vector vector, HnswHit curPoint, uint32_t searchLevel);
+
+    void search_layer(Vector vector, FurthestPriQ &w,
+                      uint32_t ef, uint32_t searchLevel);
+    void search_layer(Vector vector, FurthestPriQ &w,
+                      VisitedSet &visited,
+                      uint32_t ef, uint32_t searchLevel);
+    void search_layer_with_filter(Vector vector, FurthestPriQ &w,
+                                  uint32_t ef, uint32_t searchLevel,
+                                  const BitVector &blacklist);
+    void search_layer_with_filter(Vector vector, FurthestPriQ &w,
+                                  VisitedSet &visited,
+                                  uint32_t ef, uint32_t searchLevel,
+                                  const BitVector &blacklist);
+
+    bool haveCloserDistance(HnswHit e, const LinkList &r) const;
+
+    LinkList select_neighbors(const NearestList &neighbors, uint32_t curMax) const;
+
+    LinkList remove_weakest(const NearestList &neighbors, uint32_t curMax, LinkList &removed) const;
+
+    void addDoc(uint32_t docid) override;
+
+    void track_ops();
+
+    void remove_link_from(uint32_t from_id, uint32_t remove_id, uint32_t level) {
+        LinkList &links = getLinkList(from_id, level);
+        links.remove_link(remove_id);
+    }
+
+    void refill_ifneeded(uint32_t my_id, const LinkList &replacements, uint32_t level);
+
+    void connect_new_node(uint32_t id, const LinkList &neighbors, uint32_t level);
+
+    void shrink_links(uint32_t shrink_id, uint32_t maxLinks, uint32_t level);
+
+    void each_shrink_ifneeded(const LinkList &neighbors, uint32_t level);
+
+    void removeDoc(uint32_t docid) override;
+
+    std::vector<NnsHit> topK(uint32_t k, Vector vector, uint32_t search_k) override;
+
+    std::vector<NnsHit> topKfilter(uint32_t k, Vector vector, uint32_t search_k, const BitVector &blacklist) override;
+};
diff --git a/eval/src/tests/ann/nns-l2.h b/eval/src/tests/ann/nns-l2.h
index dcad5f1bda6..82a95741200 100644
--- a/eval/src/tests/ann/nns-l2.h
+++ b/eval/src/tests/ann/nns-l2.h
@@ -1,7 +1,8 @@
 // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #pragma once
-#include <string.h>
+#include <cstring>
+#include <vespa/vespalib/util/arrayref.h>
 #include <vespa/vespalib/hwaccelrated/iaccelrated.h>
 
 template <typename T, size_t VLEN>
@@ -33,7 +34,7 @@ static double hw_l2_sq_dist(const T * af, const T * bf, size_t sz)
 
 template <typename FltType = float>
 struct L2DistCalc {
-    vespalib::hwaccelrated::IAccelrated::UP _hw;
+    const vespalib::hwaccelrated::IAccelrated & _hw;
 
     L2DistCalc() : _hw(vespalib::hwaccelrated::IAccelrated::getAccelrator()) {}
 
@@ -41,16 +42,16 @@ struct L2DistCalc {
     using ConstArr = vespalib::ConstArrayRef<FltType>;
 
     double product(const FltType *v1, const FltType *v2, size_t sz) {
-        return _hw->dotProduct(v1, v2, sz);
+        return _hw.dotProduct(v1, v2, sz);
     }
     double product(ConstArr v1, ConstArr v2) {
         const FltType *p1 = v1.begin();
         const FltType *p2 = v2.begin();
-        return _hw->dotProduct(p1, p2, v1.size());
+        return _hw.dotProduct(p1, p2, v1.size());
     }
     double l2sq(ConstArr vector) {
         const FltType *v = vector.begin();
-        return _hw->dotProduct(v, v, vector.size());
+        return _hw.dotProduct(v, v, vector.size());
     }
     double l2sq_dist(ConstArr v1, ConstArr v2, Arr tmp) {
         for (size_t i = 0; i < v1.size(); ++i) {
diff --git a/eval/src/tests/ann/nns.h b/eval/src/tests/ann/nns.h
index 79c1aac4379..ef3e4b5d69c 100644
--- a/eval/src/tests/ann/nns.h
+++ b/eval/src/tests/ann/nns.h
@@ -37,6 +37,31 @@ struct NnsHitComparatorLessDocid {
     }
 };
 
+class BitVector {
+private:
+    std::vector<uint64_t> _bits;
+public:
+    BitVector(size_t sz) : _bits((sz+63)/64) {}
+    BitVector& setBit(size_t idx) {
+        uint64_t mask = 1;
+        mask <<= (idx%64);
+        _bits[idx/64] |= mask;
+        return *this;
+    }
+    bool isSet(size_t idx) const {
+        uint64_t mask = 1;
+        mask <<= (idx%64);
+        uint64_t word = _bits[idx/64];
+        return (word & mask) != 0;
+    }
+    BitVector& clearBit(size_t idx) {
+        uint64_t mask = 1;
+        mask <<= (idx%64);
+        _bits[idx/64] &= ~mask;
+        return *this;
+    }
+};
+
 template <typename FltType = float>
 class NNS
 {
@@ -50,6 +75,7 @@ public:
 
     using Vector = vespalib::ConstArrayRef<FltType>;
     virtual std::vector<NnsHit> topK(uint32_t k, Vector vector, uint32_t search_k) = 0;
+    virtual std::vector<NnsHit> topKfilter(uint32_t k, Vector vector, uint32_t search_k, const BitVector &blacklist) = 0;
     virtual ~NNS() {}
 protected:
     uint32_t _numDims;
@@ -67,3 +93,7 @@ make_rplsh_nns(uint32_t numDims, const DocVectorAccess<float> &dva);
 extern
 std::unique_ptr<NNS<float>>
 make_hnsw_nns(uint32_t numDims, const DocVectorAccess<float> &dva);
+
+extern
+std::unique_ptr<NNS<float>>
+make_hnsw_wrap(uint32_t numDims, const DocVectorAccess<float> &dva);
diff --git a/eval/src/tests/ann/point-vector.h b/eval/src/tests/ann/point-vector.h
new file mode 100644
index 00000000000..eca60e11194
--- /dev/null
+++ b/eval/src/tests/ann/point-vector.h
@@ -0,0 +1,30 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+struct PointVector {
+    float v[NUM_DIMS];
+    using ConstArr = vespalib::ConstArrayRef<float>;
+    operator ConstArr() const { return ConstArr(v, NUM_DIMS); }
+};
+
+static PointVector *aligned_alloc(size_t num) {
+    size_t num_bytes = num * sizeof(PointVector);
+    double mega_bytes = num_bytes / (1024.0*1024.0);
+    fprintf(stderr, "allocate %.2f MB of vectors\n", mega_bytes);
+    char *mem = (char *)malloc(num_bytes + 512);
+    mem += 512;
+    size_t val = (size_t)mem;
+    size_t unalign = val % 512;
+    mem -= unalign;
+    return reinterpret_cast<PointVector *>(mem);
+}
+
+static PointVector *generatedQueries = aligned_alloc(NUM_Q);
+static PointVector *generatedDocs = aligned_alloc(NUM_DOCS);
+
+struct DocVectorAdapter : public DocVectorAccess<float>
+{
+    vespalib::ConstArrayRef<float> get(uint32_t docid) const override {
+        ASSERT_TRUE(docid < NUM_DOCS);
+        return generatedDocs[docid];
+    }
+};
diff --git a/eval/src/tests/ann/quality-nns.h b/eval/src/tests/ann/quality-nns.h
new file mode 100644
index 00000000000..9ac37f0ef04
--- /dev/null
+++ b/eval/src/tests/ann/quality-nns.h
@@ -0,0 +1,42 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+bool reach_with_nns_k(NNS_API &nns, uint32_t docid, uint32_t k) {
+    const PointVector &qv = generatedDocs[docid];
+    vespalib::ConstArrayRef<float> query(qv.v, NUM_DIMS);
+    auto rv = nns.topK(k, query, k);
+    if (rv.size() != k) {
+        fprintf(stderr, "Result/K=%u from query for %u is %zu hits\n",
+                k, docid, rv.size());
+        return false;
+    }
+    if (rv[0].docid != docid) {
+      if (rv[0].sq.distance != 0.0)
+        fprintf(stderr, "Expected/K=%u to find %u but got %u with sq distance %.3f\n",
+                k, docid, rv[0].docid, rv[0].sq.distance);
+    }
+    return (rv[0].docid == docid || rv[0].sq.distance == 0.0);
+}
+
+void quality_nns(NNS_API &nns, std::vector<uint32_t> sk_list) {
+    for (uint32_t search_k : sk_list) {
+        double sum_recall = 0;
+        for (int cnt = 0; cnt < NUM_Q; ++cnt) {
+            sum_recall += verify_nns_quality(search_k, nns, cnt);
+        }
+        fprintf(stderr, "Overall average recall: %.2f\n", sum_recall / NUM_Q);
+    }
+    for (uint32_t search_k : { 1, 10, 100, 1000 }) {
+        TimePoint bef = std::chrono::steady_clock::now();
+        uint32_t reached = 0;
+        for (uint32_t i = 0; i < NUM_REACH; ++i) {
+            uint32_t target = i * (NUM_DOCS / NUM_REACH);
+            if (reach_with_nns_k(nns, target, search_k)) ++reached;
+        }
+        fprintf(stderr, "Could reach %u of %u documents with k=%u\n",
+                reached, NUM_REACH, search_k);
+        TimePoint aft = std::chrono::steady_clock::now();
+        fprintf(stderr, "reach time k=%u: %.3f ms = %.3f ms/q\n",
+                search_k, to_ms(aft - bef), to_ms(aft - bef)/NUM_REACH);
+        if (reached == NUM_REACH) break;
+    }
+}
diff --git a/eval/src/tests/ann/read-vecs.h b/eval/src/tests/ann/read-vecs.h
new file mode 100644
index 00000000000..39c2a332710
--- /dev/null
+++ b/eval/src/tests/ann/read-vecs.h
@@ -0,0 +1,45 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+void read_queries(std::string fn) {
+    int fd = open(fn.c_str(), O_RDONLY);
+    ASSERT_TRUE(fd > 0);
+    int d;
+    size_t rv;
+    fprintf(stderr, "reading %u queries from %s\n", NUM_Q, fn.c_str());
+    for (uint32_t qid = 0; qid < NUM_Q; ++qid) {
+        rv = read(fd, &d, 4);
+        ASSERT_EQUAL(rv, 4u);
+        ASSERT_EQUAL(d, NUM_DIMS);
+        rv = read(fd, &generatedQueries[qid].v, NUM_DIMS*sizeof(float));
+        ASSERT_EQUAL(rv, sizeof(PointVector));
+    }
+    close(fd);
+}
+
+void read_docs(std::string fn) {
+    int fd = open(fn.c_str(), O_RDONLY);
+    ASSERT_TRUE(fd > 0);
+    int d;
+    size_t rv;
+    fprintf(stderr, "reading %u doc vectors from %s\n", NUM_DOCS, fn.c_str());
+    for (uint32_t docid = 0; docid < NUM_DOCS; ++docid) {
+        rv = read(fd, &d, 4);
+        ASSERT_EQUAL(rv, 4u);
+        ASSERT_EQUAL(d, NUM_DIMS);
+        rv = read(fd, &generatedDocs[docid].v, NUM_DIMS*sizeof(float));
+        ASSERT_EQUAL(rv, sizeof(PointVector));
+    }
+    close(fd);
+}
+
+void read_data(const std::string& dir, const std::string& data_set) {
+    fprintf(stderr, "read data set '%s' from directory '%s'\n", data_set.c_str(), dir.c_str());
+    TimePoint bef = std::chrono::steady_clock::now();
+    read_queries(dir + "/" + data_set + "_query.fvecs");
+    TimePoint aft = std::chrono::steady_clock::now();
+    fprintf(stderr, "read queries: %.3f ms\n", to_ms(aft - bef));
+    bef = std::chrono::steady_clock::now();
+    read_docs(dir + "/" + data_set + "_base.fvecs");
+    aft = std::chrono::steady_clock::now();
+    fprintf(stderr, "read docs: %.3f ms\n", to_ms(aft - bef));
+}
diff --git a/eval/src/tests/ann/remove-bm.cpp b/eval/src/tests/ann/remove-bm.cpp
new file mode 100644
index 00000000000..546c2cfd75e
--- /dev/null
+++ b/eval/src/tests/ann/remove-bm.cpp
@@ -0,0 +1,248 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/vespalib/testkit/test_kit.h>
+#include <vespa/vespalib/util/priority_queue.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <stdio.h>
+#include <chrono>
+
+#define NUM_DIMS 960
+#define NUM_DOCS 250000
+#define NUM_DOCS_REMOVE 50000
+#define EFFECTIVE_DOCS (NUM_DOCS - NUM_DOCS_REMOVE)
+#define NUM_REACH 10000
+#define NUM_Q 1000
+
+#include "doc_vector_access.h"
+#include "nns.h"
+#include "for-sift-hit.h"
+#include "for-sift-top-k.h"
+#include "time-util.h"
+#include "point-vector.h"
+#include "read-vecs.h"
+#include "bruteforce-nns.h"
+
+using NNS_API = NNS<float>;
+
+#if 1
+TEST("require that HNSW via NNS api remove all works") {
+    DocVectorAdapter adapter;
+    std::unique_ptr<NNS_API> nns = make_hnsw_nns(NUM_DIMS, adapter);
+    fprintf(stderr, "adding and removing all docs forward...\n");
+    for (uint32_t i = 0; i < 1000; ++i) {
+        nns->addDoc(i);
+    }
+    for (uint32_t i = 0; i < 1000; ++i) {
+        nns->removeDoc(i);
+    }
+    fprintf(stderr, "adding and removing all docs reverse...\n");
+    for (uint32_t i = 1000; i < 2000; ++i) {
+        nns->addDoc(i);
+    }
+    for (uint32_t i = 2000; i-- > 1000; ) {
+        nns->removeDoc(i);
+    }
+}
+#endif
+
+TEST("require that brute force works") {
+    TimePoint bef = std::chrono::steady_clock::now();
+    fprintf(stderr, "generating %u brute force results\n", NUM_Q);
+    bruteforceResults.reserve(NUM_Q);
+    for (uint32_t cnt = 0; cnt < NUM_Q; ++cnt) {
+        const PointVector &query = generatedQueries[cnt];
+        bruteforceResults.emplace_back(bruteforce_nns(query));
+    }
+    TimePoint aft = std::chrono::steady_clock::now();
+    fprintf(stderr, "timing for brute force: %.3f ms = %.3f ms per query\n",
+            to_ms(aft - bef), to_ms(aft - bef)/NUM_Q);
+    for (int cnt = 0; cnt < NUM_Q; cnt = (cnt+1)*2) {
+        verifyBF(cnt);
+    }
+}
+
+#include "find-with-nns.h"
+#include "verify-top-k.h"
+
+void timing_nns(const char *name, NNS_API &nns, std::vector<uint32_t> sk_list) {
+    for (uint32_t search_k : sk_list) {
+        TimePoint bef = std::chrono::steady_clock::now();
+        for (int cnt = 0; cnt < NUM_Q; ++cnt) {
+            find_with_nns(search_k, nns, cnt);
+        }
+        TimePoint aft = std::chrono::steady_clock::now();
+        fprintf(stderr, "timing for %s search_k=%u: %.3f ms = %.3f ms/q\n",
+                name, search_k, to_ms(aft - bef), to_ms(aft - bef)/NUM_Q);
+    }
+}
+
+#include "quality-nns.h"
+
+template <typename FUNC>
+void bm_nns_simple(const char *name, FUNC creator, std::vector<uint32_t> sk_list) {
+    std::unique_ptr<NNS_API> nnsp = creator();
+    NNS_API &nns = *nnsp;
+    fprintf(stderr, "trying %s indexing...\n", name);
+    TimePoint bef = std::chrono::steady_clock::now();
+    for (uint32_t i = 0; i < EFFECTIVE_DOCS; ++i) {
+        nns.addDoc(i);
+    }
+    TimePoint aft = std::chrono::steady_clock::now();
+    fprintf(stderr, "build %s index with %u docs: %.3f ms\n", name, EFFECTIVE_DOCS, to_ms(aft - bef));
+    timing_nns(name, nns, sk_list);
+    fprintf(stderr, "Quality for %s [A] clean build with %u documents:\n", name, EFFECTIVE_DOCS);
+    quality_nns(nns, sk_list);
+    bef = std::chrono::steady_clock::now();
+    for (uint32_t i = 0; i < NUM_DOCS_REMOVE; ++i) {
+        nns.addDoc(EFFECTIVE_DOCS + i);
+    }
+    for (uint32_t i = 0; i < NUM_DOCS_REMOVE; ++i) {
+        nns.removeDoc(EFFECTIVE_DOCS + i);
+    }
+    aft = std::chrono::steady_clock::now();
+    fprintf(stderr, "build %s index add then remove %u docs: %.3f ms\n",
+            name, NUM_DOCS_REMOVE, to_ms(aft - bef));
+    timing_nns(name, nns, sk_list);
+    fprintf(stderr, "Quality for %s [B] remove-damaged build with %u documents:\n", name, EFFECTIVE_DOCS);
+    quality_nns(nns, sk_list);
+}
+
+template <typename FUNC>
+void bm_nns_remove_old(const char *name, FUNC creator, std::vector<uint32_t> sk_list) {
+    std::unique_ptr<NNS_API> nnsp = creator();
+    NNS_API &nns = *nnsp;
+    TimePoint bef = std::chrono::steady_clock::now();
+    for (uint32_t i = 0; i < NUM_DOCS_REMOVE; ++i) {
+        nns.addDoc(EFFECTIVE_DOCS + i);
+    }
+    for (uint32_t i = 0; i < EFFECTIVE_DOCS; ++i) {
+        nns.addDoc(i);
+    }
+    for (uint32_t i = 0; i < NUM_DOCS_REMOVE; ++i) {
+        nns.removeDoc(EFFECTIVE_DOCS + i);
+    }
+    TimePoint aft = std::chrono::steady_clock::now();
+    fprintf(stderr, "build %s index with %u docs: %.3f ms\n", name, EFFECTIVE_DOCS, to_ms(aft - bef));
+    timing_nns(name, nns, sk_list);
+    fprintf(stderr, "Quality for %s [C] remove-oldest build with %u documents:\n", name, EFFECTIVE_DOCS);
+    quality_nns(nns, sk_list);
+}
+
+template <typename FUNC>
+void bm_nns_interleave(const char *name, FUNC creator, std::vector<uint32_t> sk_list) {
+    std::unique_ptr<NNS_API> nnsp = creator();
+    NNS_API &nns = *nnsp;
+    TimePoint bef = std::chrono::steady_clock::now();
+    for (uint32_t i = 0; i < NUM_DOCS_REMOVE; ++i) {
+        nns.addDoc(EFFECTIVE_DOCS + i);
+    }
+    for (uint32_t i = 0; i < EFFECTIVE_DOCS - NUM_DOCS_REMOVE; ++i) {
+        nns.addDoc(i);
+    }
+    for (uint32_t i = 0; i < NUM_DOCS_REMOVE; ++i) {
+        nns.removeDoc(EFFECTIVE_DOCS + i);
+        nns.addDoc(EFFECTIVE_DOCS - NUM_DOCS_REMOVE + i);
+    }
+    TimePoint aft = std::chrono::steady_clock::now();
+    fprintf(stderr, "build %s index with %u docs: %.3f ms\n", name, EFFECTIVE_DOCS, to_ms(aft - bef));
+    timing_nns(name, nns, sk_list);
+    fprintf(stderr, "Quality for %s [D] realistic build with %u documents:\n", name, EFFECTIVE_DOCS);
+    quality_nns(nns, sk_list);
+}
+
+template <typename FUNC>
+void bm_nns_remove_old_add_new(const char *name, FUNC creator, std::vector<uint32_t> sk_list) {
+    std::unique_ptr<NNS_API> nnsp = creator();
+    NNS_API &nns = *nnsp;
+    TimePoint bef = std::chrono::steady_clock::now();
+    for (uint32_t i = 0; i < NUM_DOCS_REMOVE; ++i) {
+        nns.addDoc(EFFECTIVE_DOCS + i);
+    }
+    for (uint32_t i = 0; i < EFFECTIVE_DOCS - NUM_DOCS_REMOVE; ++i) {
+        nns.addDoc(i);
+    }
+    for (uint32_t i = 0; i < NUM_DOCS_REMOVE; ++i) {
+        nns.removeDoc(EFFECTIVE_DOCS + i);
+    }
+    for (uint32_t i = 0; i < NUM_DOCS_REMOVE; ++i) {
+        nns.addDoc(EFFECTIVE_DOCS - NUM_DOCS_REMOVE + i);
+    }
+    TimePoint aft = std::chrono::steady_clock::now();
+    fprintf(stderr, "build %s index with %u docs: %.3f ms\n", name, EFFECTIVE_DOCS, to_ms(aft - bef));
+    timing_nns(name, nns, sk_list);
+    fprintf(stderr, "Quality for %s [E] remove old, add new build with %u documents:\n", name, EFFECTIVE_DOCS);
+    quality_nns(nns, sk_list);
+}
+
+template <typename FUNC>
+void benchmark_nns(const char *name, FUNC creator, std::vector<uint32_t> sk_list) {
+    bm_nns_simple(name, creator, sk_list);
+    bm_nns_remove_old(name, creator, sk_list);
+    bm_nns_interleave(name, creator, sk_list);
+    bm_nns_remove_old_add_new(name, creator, sk_list);
+}
+
+#if 0
+TEST("require that Locality Sensitive Hashing mostly works") {
+    DocVectorAdapter adapter;
+    auto creator = [&adapter]() { return make_rplsh_nns(NUM_DIMS, adapter); };
+    benchmark_nns("RPLSH", creator, { 200, 1000 });
+}
+#endif
+
+#if 0
+TEST("require that Annoy via NNS api mostly works") {
+    DocVectorAdapter adapter;
+    auto creator = [&adapter]() { return make_annoy_nns(NUM_DIMS, adapter); };
+    benchmark_nns("Annoy", creator, { 8000, 10000 });
+}
+#endif
+
+#if 1
+TEST("require that HNSW via NNS api mostly works") {
+    DocVectorAdapter adapter;
+    auto creator = [&adapter]() { return make_hnsw_nns(NUM_DIMS, adapter); };
+    benchmark_nns("HNSW-like", creator, { 100, 150, 200 });
+}
+#endif
+
+#if 0
+TEST("require that HNSW wrapped api mostly works") {
+    DocVectorAdapter adapter;
+    auto creator = [&adapter]() { return make_hnsw_wrap(NUM_DIMS, adapter); };
+    benchmark_nns("HNSW-wrap", creator, { 100, 150, 200 });
+}
+#endif
+
+/**
+ * Before running the benchmark the ANN_GIST1M data set must be downloaded and extracted:
+ *   wget ftp://ftp.irisa.fr/local/texmex/corpus/gist.tar.gz
+ *   tar -xf gist.tar.gz
+ *
+ * The benchmark program will load the data set from $HOME/gist if no directory is specified.
+ *
+ * More information about the dataset is found here: http://corpus-texmex.irisa.fr/.
+ */
+int main(int argc, char **argv) {
+    TEST_MASTER.init(__FILE__);
+    std::string data_set = "gist";
+    std::string data_dir = ".";
+    if (argc > 2) {
+        data_set = argv[1];
+        data_dir = argv[2];
+    } else if (argc > 1) {
+        data_dir = argv[1];
+    } else {
+        char *home = getenv("HOME");
+        if (home) {
+            data_dir = home;
+            data_dir += "/" + data_set;
+        }
+    }
+    read_data(data_dir, data_set);
+    TEST_RUN_ALL();
+    return (TEST_MASTER.fini() ? 0 : 1);
+}
diff --git a/eval/src/tests/ann/sift_benchmark.cpp b/eval/src/tests/ann/sift_benchmark.cpp
index 5ca505e5f1e..4bbe8f61ef1 100644
--- a/eval/src/tests/ann/sift_benchmark.cpp
+++ b/eval/src/tests/ann/sift_benchmark.cpp
@@ -8,174 +8,62 @@
 #include <unistd.h>
 #include <stdio.h>
 #include <chrono>
+#include <cstdlib>
 
 #define NUM_DIMS 128
 #define NUM_DOCS 1000000
+#define EFFECTIVE_DOCS NUM_DOCS
 #define NUM_Q 1000
+#define NUM_REACH 10000
 
 #include "doc_vector_access.h"
 #include "nns.h"
 #include "for-sift-hit.h"
 #include "for-sift-top-k.h"
+#include "std-random.h"
+#include "time-util.h"
+#include "point-vector.h"
+#include "read-vecs.h"
+#include "bruteforce-nns.h"
 
-std::vector<TopK> bruteforceResults;
-std::vector<float> tmp_v(NUM_DIMS);
-
-struct PointVector {
-    float v[NUM_DIMS];
-    using ConstArr = vespalib::ConstArrayRef<float>;
-    operator ConstArr() const { return ConstArr(v, NUM_DIMS); }
-};
-
-static PointVector *aligned_alloc(size_t num) {
-    char *mem = (char *)malloc(num * sizeof(PointVector) + 512);
-    mem += 512;
-    size_t val = (size_t)mem;
-    size_t unalign = val % 512;
-    mem -= unalign;
-    return reinterpret_cast<PointVector *>(mem);
-}
-
-static PointVector *generatedQueries = aligned_alloc(NUM_Q);
-static PointVector *generatedDocs = aligned_alloc(NUM_DOCS);
-
-struct DocVectorAdapter : public DocVectorAccess<float>
-{
-    vespalib::ConstArrayRef<float> get(uint32_t docid) const override {
-        ASSERT_TRUE(docid < NUM_DOCS);
-        return generatedDocs[docid];
-    }
-};
-
-double computeDistance(const PointVector &query, uint32_t docid) {
-    const PointVector &docvector = generatedDocs[docid];
-    return l2distCalc.l2sq_dist(query, docvector, tmp_v);
-}
-
-void read_queries(std::string fn) {
-    int fd = open(fn.c_str(), O_RDONLY);
-    ASSERT_TRUE(fd > 0);
-    int d;
-    size_t rv;
-    fprintf(stderr, "reading %u queries from %s\n", NUM_Q, fn.c_str());
-    for (uint32_t qid = 0; qid < NUM_Q; ++qid) {
-        rv = read(fd, &d, 4);
-        ASSERT_EQUAL(rv, 4u);
-        ASSERT_EQUAL(d, NUM_DIMS);
-        rv = read(fd, &generatedQueries[qid].v, NUM_DIMS*sizeof(float));
-        ASSERT_EQUAL(rv, sizeof(PointVector));
-    }
-    close(fd);
-}
-
-void read_docs(std::string fn) {
-    int fd = open(fn.c_str(), O_RDONLY);
-    ASSERT_TRUE(fd > 0);
-    int d;
-    size_t rv;
-    fprintf(stderr, "reading %u doc vectors from %s\n", NUM_DOCS, fn.c_str());
-    for (uint32_t docid = 0; docid < NUM_DOCS; ++docid) {
-        rv = read(fd, &d, 4);
-        ASSERT_EQUAL(rv, 4u);
-        ASSERT_EQUAL(d, NUM_DIMS);
-        rv = read(fd, &generatedDocs[docid].v, NUM_DIMS*sizeof(float));
-        ASSERT_EQUAL(rv, sizeof(PointVector));
-    }
-    close(fd);
-}
-
-using TimePoint = std::chrono::steady_clock::time_point;
-using Duration = std::chrono::steady_clock::duration;
-
-double to_ms(Duration elapsed) {
-    std::chrono::duration<double, std::milli> ms(elapsed);
-    return ms.count();
-}
-
-void read_data(const std::string& dir, const std::string& data_set) {
-    fprintf(stderr, "read data set '%s' from directory '%s'\n", data_set.c_str(), dir.c_str());
-    TimePoint bef = std::chrono::steady_clock::now();
-    read_queries(dir + "/" + data_set + "_query.fvecs");
-    TimePoint aft = std::chrono::steady_clock::now();
-    fprintf(stderr, "read queries: %.3f ms\n", to_ms(aft - bef));
-    bef = std::chrono::steady_clock::now();
-    read_docs(dir + "/" + data_set + "_base.fvecs");
-    aft = std::chrono::steady_clock::now();
-    fprintf(stderr, "read docs: %.3f ms\n", to_ms(aft - bef));
-}
-
-
-struct BfHitComparator {
-    bool operator() (const Hit &lhs, const Hit& rhs) const {
-        if (lhs.distance < rhs.distance) return false;
-        if (lhs.distance > rhs.distance) return true;
-        return (lhs.docid > rhs.docid);
-    }
-};
-
-class BfHitHeap {
-private:
-    size_t _size;
-    vespalib::PriorityQueue<Hit, BfHitComparator> _priQ;
-public:
-    explicit BfHitHeap(size_t maxSize) : _size(maxSize), _priQ() {
-        _priQ.reserve(maxSize);
-    }
-    ~BfHitHeap() {}
-    void maybe_use(const Hit &hit) {
-        if (_priQ.size() < _size) {
-            _priQ.push(hit);
-        } else if (hit.distance < _priQ.front().distance) {
-            _priQ.front() = hit;
-            _priQ.adjust();
-        }
-    }
-    std::vector<Hit> bestHits() {
-        std::vector<Hit> result;
-        size_t i = _priQ.size();
-        result.resize(i);
-        while (i-- > 0) {
-            result[i] = _priQ.front();
-            _priQ.pop_front();
-        }
-        return result;
-    }
-};
-
-TopK bruteforce_nns(const PointVector &query) {
+TopK bruteforce_nns_filter(const PointVector &query, const BitVector &blacklist) {
     TopK result;
     BfHitHeap heap(result.K);
     for (uint32_t docid = 0; docid < NUM_DOCS; ++docid) {
+        if (blacklist.isSet(docid)) continue;
         const PointVector &docvector = generatedDocs[docid];
-        double d = l2distCalc.l2sq_dist(query, docvector, tmp_v);
+        double d = l2distCalc.l2sq_dist(query, docvector);
         Hit h(docid, d);
         heap.maybe_use(h);
     }
     std::vector<Hit> best = heap.bestHits();
+    EXPECT_EQUAL(best.size(), result.K);
     for (size_t i = 0; i < result.K; ++i) {
         result.hits[i] = best[i];
     }
     return result;
 }
 
-void verifyBF(uint32_t qid) {
-    const PointVector &query = generatedQueries[qid];
-    TopK &result = bruteforceResults[qid];
-    double min_distance = result.hits[0].distance;
-    std::vector<double> all_c2;
-    for (uint32_t i = 0; i < NUM_DOCS; ++i) {
-        double dist = computeDistance(query, i);
-        if (dist < min_distance) {
-            fprintf(stderr, "WARN dist %.9g < mindist %.9g\n", dist, min_distance);
+void timing_bf_filter(int percent)
+{
+    BitVector blacklist(NUM_DOCS);
+    RndGen rnd;
+    for (uint32_t idx = 0; idx < NUM_DOCS; ++idx) {
+        if (rnd.nextUniform() < 0.01 * percent) {
+            blacklist.setBit(idx);
+        } else {
+            blacklist.clearBit(idx);
         }
-        EXPECT_FALSE(dist+0.000001 < min_distance);
-        if (qid == 6) all_c2.push_back(dist / min_distance);
     }
-    if (all_c2.size() != NUM_DOCS) return;
-    std::sort(all_c2.begin(), all_c2.end());
-    for (uint32_t idx : { 1, 3, 10, 30, 100, 300, 1000, 3000, NUM_DOCS/2, NUM_DOCS-1}) {
-        fprintf(stderr, "c2-factor[%u] = %.3f\n", idx, all_c2[idx]);
+    TimePoint bef = std::chrono::steady_clock::now();
+    for (int cnt = 0; cnt < NUM_Q; ++cnt) {
+        const PointVector &qv = generatedQueries[cnt];
+        auto res = bruteforce_nns_filter(qv, blacklist);
+        EXPECT_TRUE(res.hits[res.K - 1].distance > 0.0);
     }
+    TimePoint aft = std::chrono::steady_clock::now();
+    fprintf(stderr, "timing for bruteforce filter %d %%: %.3f ms = %.3f ms/q\n",
+            percent, to_ms(aft - bef), to_ms(aft - bef)/NUM_Q);
 }
 
 TEST("require that brute force works") {
@@ -191,52 +79,90 @@ TEST("require that brute force works") {
     for (int cnt = 0; cnt < NUM_Q; cnt = (cnt+1)*2) {
         verifyBF(cnt);
     }
+#if 1
+    for (uint32_t filter_percent : { 0, 1, 10, 50, 90, 95, 99 }) {
+        timing_bf_filter(filter_percent);
+    }
+#endif
 }
 
 using NNS_API = NNS<float>;
 
-TopK find_with_nns(uint32_t sk, NNS_API &nns, uint32_t qid) {
-    TopK result;
+size_t search_with_filter(uint32_t sk, NNS_API &nns, uint32_t qid,
+                          const BitVector &blacklist)
+{
     const PointVector &qv = generatedQueries[qid];
     vespalib::ConstArrayRef<float> query(qv.v, NUM_DIMS);
-    auto rv = nns.topK(result.K, query, sk);
-    for (size_t i = 0; i < result.K; ++i) {
-        result.hits[i] = Hit(rv[i].docid, rv[i].sq.distance);
+    auto rv = nns.topKfilter(100, query, sk, blacklist);
+    return rv.size();
+}
+
+#include "find-with-nns.h"
+#include "verify-top-k.h"
+
+void verify_with_filter(uint32_t sk, NNS_API &nns, uint32_t qid,
+                        const BitVector &blacklist)
+{
+    const PointVector &qv = generatedQueries[qid];
+    auto expected = bruteforce_nns_filter(qv, blacklist);
+    vespalib::ConstArrayRef<float> query(qv.v, NUM_DIMS);
+    auto rv = nns.topKfilter(expected.K, query, sk, blacklist);
+    TopK actual;
+    for (size_t i = 0; i < actual.K; ++i) {
+        actual.hits[i] = Hit(rv[i].docid, rv[i].sq.distance);
     }
-    return result;
+    verify_top_k(expected, actual, sk, qid);
 }
 
-void verify_nns_quality(uint32_t sk, NNS_API &nns, uint32_t qid) {
-    TopK perfect = bruteforceResults[qid];
-    TopK result = find_with_nns(sk, nns, qid);
-    int recall = perfect.recall(result);
-    EXPECT_TRUE(recall > 40);
-    double sum_error = 0.0;
-    double c_factor = 1.0;
-    for (size_t i = 0; i < result.K; ++i) {
-        double factor = (result.hits[i].distance / perfect.hits[i].distance);
-        if (factor < 0.99 || factor > 25) {
-            fprintf(stderr, "hit[%zu] got distance %.3f, expected %.3f\n",
-                    i, result.hits[i].distance, perfect.hits[i].distance);
+void timing_nns_filter(const char *name, NNS_API &nns,
+                       std::vector<uint32_t> sk_list, int percent)
+{
+    BitVector blacklist(NUM_DOCS);
+    RndGen rnd;
+    for (uint32_t idx = 0; idx < NUM_DOCS; ++idx) {
+        if (rnd.nextUniform() < 0.01 * percent) {
+            blacklist.setBit(idx);
+        } else {
+            blacklist.clearBit(idx);
+        }
+    }
+    for (uint32_t search_k : sk_list) {
+        TimePoint bef = std::chrono::steady_clock::now();
+        for (int cnt = 0; cnt < NUM_Q; ++cnt) {
+            uint32_t nh = search_with_filter(search_k, nns, cnt, blacklist);
+            EXPECT_EQUAL(nh, 100u);
+        }
+        TimePoint aft = std::chrono::steady_clock::now();
+        fprintf(stderr, "timing for %s filter %d %% search_k=%u: %.3f ms = %.3f ms/q\n",
+                name, percent, search_k, to_ms(aft - bef), to_ms(aft - bef)/NUM_Q);
+#if 0
+        fprintf(stderr, "Quality check for %s filter %d %%:\n", name, percent);
+        for (int cnt = 0; cnt < NUM_Q; ++cnt) {
+            verify_with_filter(search_k, nns, cnt, blacklist);
         }
-        sum_error += factor;
-        c_factor = std::max(c_factor, factor);
+#endif
     }
-    EXPECT_TRUE(c_factor < 1.5);
-    fprintf(stderr, "quality sk=%u: query %u: recall %d, c2-factor %.3f, avg c2: %.3f\n",
-            sk, qid, recall, c_factor, sum_error / result.K);
-    if (qid == 6) {
-        for (size_t i = 0; i < 10; ++i) {
-            fprintf(stderr, "topk[%zu] BF{%u %.3f} index{%u %.3f}\n",
-                    i,
-                    perfect.hits[i].docid, perfect.hits[i].distance,
-                    result.hits[i].docid, result.hits[i].distance);
+}
+
+void timing_nns(const char *name, NNS_API &nns, std::vector<uint32_t> sk_list) {
+    for (uint32_t search_k : sk_list) {
+        TimePoint bef = std::chrono::steady_clock::now();
+        for (int cnt = 0; cnt < NUM_Q; ++cnt) {
+            find_with_nns(search_k, nns, cnt);
         }
+        TimePoint aft = std::chrono::steady_clock::now();
+        fprintf(stderr, "timing for %s search_k=%u: %.3f ms = %.3f ms/q\n",
+                name, search_k, to_ms(aft - bef), to_ms(aft - bef)/NUM_Q);
     }
 }
 
-void benchmark_nns(const char *name, NNS_API &nns, std::vector<uint32_t> sk_list) {
+#include "quality-nns.h"
+
+template <typename FUNC>
+void benchmark_nns(const char *name, FUNC creator, std::vector<uint32_t> sk_list) {
     fprintf(stderr, "trying %s indexing...\n", name);
+    std::unique_ptr<NNS_API> nnsp = creator();
+    NNS_API &nns = *nnsp;
     TimePoint bef = std::chrono::steady_clock::now();
     for (uint32_t i = 0; i < NUM_DOCS; ++i) {
         nns.addDoc(i);
@@ -246,45 +172,46 @@ void benchmark_nns(const char *name, NNS_API &nns, std::vector<uint32_t> sk_list
     TimePoint aft = std::chrono::steady_clock::now();
     fprintf(stderr, "build %s index: %.3f ms\n", name, to_ms(aft - bef));
 
-    for (uint32_t search_k : sk_list) {
-        bef = std::chrono::steady_clock::now();
-        for (int cnt = 0; cnt < NUM_Q; ++cnt) {
-            find_with_nns(search_k, nns, cnt);
-        }
-        aft = std::chrono::steady_clock::now();
-        fprintf(stderr, "timing for %s search_k=%u: %.3f ms = %.3f ms/q\n",
-                name, search_k, to_ms(aft - bef), to_ms(aft - bef)/NUM_Q);
-        for (int cnt = 0; cnt < NUM_Q; ++cnt) {
-            verify_nns_quality(search_k, nns, cnt);
-        }
+    fprintf(stderr, "Timings for %s :\n", name);
+    timing_nns(name, nns, sk_list);
+    for (uint32_t filter_percent : { 0, 1, 10, 50, 90, 95, 99 }) {
+        timing_nns_filter(name, nns, sk_list, filter_percent);
     }
+    fprintf(stderr, "Quality for %s :\n", name);
+    quality_nns(nns, sk_list);
 }
 
-
-#if 1
+#if 0
 TEST("require that Locality Sensitive Hashing mostly works") {
     DocVectorAdapter adapter;
-    std::unique_ptr<NNS_API> nns = make_rplsh_nns(NUM_DIMS, adapter);
-    benchmark_nns("RPLSH", *nns, { 200, 1000 });
+    auto creator = [&adapter]() { return make_rplsh_nns(NUM_DIMS, adapter); };
+    benchmark_nns("RPLSH", creator, { 200, 1000 });
 }
 #endif
 
 #if 1
 TEST("require that Annoy via NNS api mostly works") {
     DocVectorAdapter adapter;
-    std::unique_ptr<NNS_API> nns = make_annoy_nns(NUM_DIMS, adapter);
-    benchmark_nns("Annoy", *nns, { 8000, 10000 });
+    auto creator = [&adapter]() { return make_annoy_nns(NUM_DIMS, adapter); };
+    benchmark_nns("Annoy", creator, { 8000, 10000 });
 }
 #endif
 
 #if 1
 TEST("require that HNSW via NNS api mostly works") {
     DocVectorAdapter adapter;
-    std::unique_ptr<NNS_API> nns = make_hnsw_nns(NUM_DIMS, adapter);
-    benchmark_nns("HNSW", *nns, { 100, 200 });
+    auto creator = [&adapter]() { return make_hnsw_nns(NUM_DIMS, adapter); };
+    benchmark_nns("HNSW-like", creator, { 100, 150, 200 });
 }
 #endif
 
+#if 0
+TEST("require that HNSW wrapped api mostly works") {
+    DocVectorAdapter adapter;
+    auto creator = [&adapter]() { return make_hnsw_wrap(NUM_DIMS, adapter); };
+    benchmark_nns("HNSW-wrap", creator, { 100, 150, 200 });
+}
+#endif
 
 /**
  * Before running the benchmark the ANN_SIFT1M data set must be downloaded and extracted:
diff --git a/eval/src/tests/ann/time-util.h b/eval/src/tests/ann/time-util.h
new file mode 100644
index 00000000000..2f5c2bdd583
--- /dev/null
+++ b/eval/src/tests/ann/time-util.h
@@ -0,0 +1,9 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+using TimePoint = std::chrono::steady_clock::time_point;
+using Duration = std::chrono::steady_clock::duration;
+
+double to_ms(Duration elapsed) {
+    std::chrono::duration<double, std::milli> ms(elapsed);
+    return ms.count();
+}
diff --git a/eval/src/tests/ann/verify-top-k.h b/eval/src/tests/ann/verify-top-k.h
new file mode 100644
index 00000000000..220c273d017
--- /dev/null
+++ b/eval/src/tests/ann/verify-top-k.h
@@ -0,0 +1,27 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+int verify_top_k(const TopK &perfect, const TopK &result, uint32_t sk, uint32_t qid) {
+    int recall = perfect.recall(result);
+    EXPECT_TRUE(recall > 40);
+    double sum_error = 0.0;
+    double c_factor = 1.0;
+    for (size_t i = 0; i < result.K; ++i) {
+        double factor = (result.hits[i].distance / perfect.hits[i].distance);
+        if (factor < 0.99 || factor > 25) {
+            fprintf(stderr, "hit[%zu] got distance %.3f, expected %.3f\n",
+                    i, result.hits[i].distance, perfect.hits[i].distance);
+        }
+        sum_error += factor;
+        c_factor = std::max(c_factor, factor);
+    }
+    EXPECT_TRUE(c_factor < 1.5);
+    fprintf(stderr, "quality sk=%u: query %u: recall %d  c2-factor %.3f  avg c2: %.3f\n",
+            sk, qid, recall, c_factor, sum_error / result.K);
+    return recall;
+}
+
+int verify_nns_quality(uint32_t sk, NNS_API &nns, uint32_t qid) {
+    TopK perfect = bruteforceResults[qid];
+    TopK result = find_with_nns(sk, nns, qid);
+    return verify_top_k(perfect, result, sk, qid);
+}
diff --git a/eval/src/tests/ann/xp-annoy-nns.cpp b/eval/src/tests/ann/xp-annoy-nns.cpp
index 45392084c80..213e583d95a 100644
--- a/eval/src/tests/ann/xp-annoy-nns.cpp
+++ b/eval/src/tests/ann/xp-annoy-nns.cpp
@@ -3,6 +3,7 @@
 #include "nns.h"
 #include "std-random.h"
 #include <assert.h>
+#include <cinttypes>
 #include <algorithm>
 #include <queue>
 #include <set>
@@ -11,11 +12,11 @@ using V = vespalib::ConstArrayRef<float>;
 class AnnoyLikeNns;
 struct Node;
 
-static uint64_t plane_dist_cnt = 0;
-static uint64_t w_cen_dist_cnt = 0;
-static uint64_t leaf_split_cnt = 0;
-static uint64_t find_top_k_cnt = 0;
-static uint64_t find_cand_cnt = 0;
+static size_t plane_dist_cnt = 0;
+static size_t w_cen_dist_cnt = 0;
+static size_t leaf_split_cnt = 0;
+static size_t find_top_k_cnt = 0;
+static size_t find_cand_cnt = 0;
 
 using QueueNode = std::pair<double, Node *>;
 using NodeQueue = std::priority_queue<QueueNode>;
@@ -26,6 +27,7 @@ struct Node {
     virtual Node *addDoc(uint32_t docid, V vector, AnnoyLikeNns &meta) = 0;
     virtual int remove(uint32_t docid, V vector) = 0;
     virtual void findCandidates(std::set<uint32_t> &cands, V vector, NodeQueue &queue, double minDist) const = 0;
+    virtual void filterCandidates(std::set<uint32_t> &cands, V vector, NodeQueue &queue, double minDist, const BitVector &blacklist) const = 0;
     virtual void stats(std::vector<uint32_t> &depths) = 0;
 };
 
@@ -37,6 +39,7 @@ struct LeafNode : public Node {
     Node *addDoc(uint32_t docid, V vector, AnnoyLikeNns &meta) override;
     int remove(uint32_t docid, V vector) override;
     void findCandidates(std::set<uint32_t> &cands, V vector, NodeQueue &queue, double minDist) const override;
+    void filterCandidates(std::set<uint32_t> &cands, V vector, NodeQueue &queue, double minDist, const BitVector &blacklist) const override;
 
     Node *split(AnnoyLikeNns &meta);
     virtual void stats(std::vector<uint32_t> &depths) override { depths.push_back(1); }
@@ -54,6 +57,7 @@ struct SplitNode : public Node {
     Node *addDoc(uint32_t docid, V vector, AnnoyLikeNns &meta) override;
     int remove(uint32_t docid, V vector) override;
     void findCandidates(std::set<uint32_t> &cands, V vector, NodeQueue &queue, double minDist) const override;
+    void filterCandidates(std::set<uint32_t> &cands, V vector, NodeQueue &queue, double minDist, const BitVector &blacklist) const override;
 
     double planeDistance(V vector) const;
     virtual void stats(std::vector<uint32_t> &depths) override {
@@ -105,6 +109,8 @@ public:
     }
     std::vector<NnsHit> topK(uint32_t k, Vector vector, uint32_t search_k) override;
 
+    std::vector<NnsHit> topKfilter(uint32_t k, Vector vector, uint32_t search_k, const BitVector &bitvector) override;
+
     V getVector(uint32_t docid) const { return _dva.get(docid); }
     double uniformRnd() { return _rndGen.nextUniform(); } 
     uint32_t dims() const { return _numDims; }
@@ -303,6 +309,16 @@ LeafNode::findCandidates(std::set<uint32_t> &cands, V, NodeQueue &, double) cons
     }
 }
 
+void
+LeafNode::filterCandidates(std::set<uint32_t> &cands, V, NodeQueue &, double, const BitVector &blacklist) const
+{
+    for (uint32_t d : docids) {
+        if (blacklist.isSet(d)) continue;
+        cands.insert(d);
+    }
+}
+
+
 SplitNode::~SplitNode()
 {
     delete leftChildren;
@@ -343,6 +359,15 @@ SplitNode::findCandidates(std::set<uint32_t> &, V vector, NodeQueue &queue, doub
     queue.push(std::make_pair(std::min(d, minDist), rightChildren));
 }
 
+void
+SplitNode::filterCandidates(std::set<uint32_t> &, V vector, NodeQueue &queue, double minDist, const BitVector &) const
+{
+    double d = planeDistance(vector);
+    // fprintf(stderr, "push 2 nodes dist %g\n", d);
+    queue.push(std::make_pair(std::min(-d, minDist), leftChildren));
+    queue.push(std::make_pair(std::min(d, minDist), rightChildren));
+}
+
 std::vector<NnsHit>
 AnnoyLikeNns::topK(uint32_t k, Vector vector, uint32_t search_k)
 {
@@ -386,6 +411,40 @@ AnnoyLikeNns::topK(uint32_t k, Vector vector, uint32_t search_k)
     return r;
 }
 
+std::vector<NnsHit>
+AnnoyLikeNns::topKfilter(uint32_t k, Vector vector, uint32_t search_k, const BitVector &blacklist)
+{
+    ++find_top_k_cnt;
+    std::vector<NnsHit> r;
+    r.reserve(k);
+    std::set<uint32_t> candidates;
+    NodeQueue queue;
+    for (Node *root : _roots) {
+        double dist = std::numeric_limits<double>::max();
+        queue.push(std::make_pair(dist, root));
+    }
+    while ((candidates.size() < std::max(k, search_k)) && (queue.size() > 0)) {
+        const QueueNode& top = queue.top();
+        double md = top.first;
+        // fprintf(stderr, "find candidates: node with min distance %g\n", md);
+        Node *n = top.second;
+        queue.pop();
+        n->filterCandidates(candidates, vector, queue, md, blacklist);
+        ++find_cand_cnt;
+    }
+    for (uint32_t docid : candidates) {
+        if (blacklist.isSet(docid)) continue;
+        double dist = l2distCalc.l2sq_dist(vector, _dva.get(docid));
+        NnsHit hit(docid, SqDist(dist));
+        r.push_back(hit);
+    }
+    std::sort(r.begin(), r.end(), NnsHitComparatorLessDistance());
+    while (r.size() > k) r.pop_back();
+    return r;
+}
+
+
+
 void
 AnnoyLikeNns::dumpStats() {
     fprintf(stderr, "stats for AnnoyLikeNns:\n");
diff --git a/eval/src/tests/ann/xp-hnsw-wrap.cpp b/eval/src/tests/ann/xp-hnsw-wrap.cpp
new file mode 100644
index 00000000000..45c7a974254
--- /dev/null
+++ b/eval/src/tests/ann/xp-hnsw-wrap.cpp
@@ -0,0 +1,84 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "nns.h"
+#include <iostream>
+#include "/git/hnswlib/hnswlib/hnswlib.h"
+
+class HnswWrapNns : public NNS<float>
+{
+private:
+    using Implementation = hnswlib::HierarchicalNSW<float>;
+    hnswlib::L2Space _l2space;
+    Implementation _hnsw;
+
+public:
+    HnswWrapNns(uint32_t numDims, const DocVectorAccess<float> &dva)
+        : NNS(numDims, dva),
+          _l2space(numDims),
+          _hnsw(&_l2space, 2500000, 16, 200)
+    {
+    }
+
+    ~HnswWrapNns() {}
+
+    void addDoc(uint32_t docid) override {
+        Vector vector = _dva.get(docid);
+        _hnsw.addPoint(vector.cbegin(), docid);
+    }
+
+    void removeDoc(uint32_t docid) override {
+        _hnsw.markDelete(docid);
+    }
+
+    std::vector<NnsHit> topK(uint32_t k, Vector vector, uint32_t search_k) override {
+        std::vector<NnsHit> reversed;
+        _hnsw.setEf(search_k);
+        auto priQ = _hnsw.searchKnn(vector.cbegin(), k);
+        while (! priQ.empty()) {
+            auto pair = priQ.top();
+            reversed.emplace_back(pair.second, SqDist(pair.first));
+            priQ.pop();
+        }
+        std::vector<NnsHit> result;
+        while (result.size() < k && !reversed.empty()) {
+            result.push_back(reversed.back());
+            reversed.pop_back();
+        }
+        return result;
+    }
+
+    std::vector<NnsHit> topKfilter(uint32_t k, Vector vector, uint32_t search_k, const BitVector &blacklist) override {
+        std::vector<NnsHit> reversed;
+        uint32_t adjusted_k = k+4;
+        uint32_t adjusted_sk = search_k+4;
+        for (int retry = 0; (retry < 5) && (reversed.size() < k); ++retry) {
+            reversed.clear();
+            _hnsw.setEf(adjusted_sk);
+            auto priQ = _hnsw.searchKnn(vector.cbegin(), adjusted_k);
+            while (! priQ.empty()) {
+                auto pair = priQ.top();
+                if (! blacklist.isSet(pair.second)) {
+                    reversed.emplace_back(pair.second, SqDist(pair.first));
+                }
+                priQ.pop();
+            }
+            double got = 1 + reversed.size();
+            double factor = 1.25 * k / got;
+            adjusted_k *= factor;
+            adjusted_sk *= factor;
+        }
+        std::vector<NnsHit> result;
+        while (result.size() < k && !reversed.empty()) {
+            result.push_back(reversed.back());
+            reversed.pop_back();
+        }
+        return result;
+    }
+};
+
+std::unique_ptr<NNS<float>>
+make_hnsw_wrap(uint32_t numDims, const DocVectorAccess<float> &dva)
+{
+    NNS<float> *p = new HnswWrapNns(numDims, dva);
+    return std::unique_ptr<NNS<float>>(p);
+}
diff --git a/eval/src/tests/ann/xp-hnswlike-nns.cpp b/eval/src/tests/ann/xp-hnswlike-nns.cpp
index ec831610a71..b7cae9f731c 100644
--- a/eval/src/tests/ann/xp-hnswlike-nns.cpp
+++ b/eval/src/tests/ann/xp-hnswlike-nns.cpp
@@ -1,305 +1,249 @@
 // Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
-#include <algorithm>
-#include <assert.h>
-#include <queue>
-#include <random>
-#include "nns.h"
-
-using LinkList = std::vector<uint32_t>;
-
-struct Node {
-    std::vector<LinkList> _links;
-    Node(uint32_t , uint32_t numLevels, uint32_t M)
-        : _links(numLevels)
-    {
-        for (uint32_t i = 0; i < _links.size(); ++i) {
-            _links[i].reserve((i == 0) ? (2 * M + 1) : (M+1));
-        }
-    }
-};
-
-struct VisitedSet
+#include "hnsw-like.h"
+
+/*
+ Todo:
+
+ measure effect of:
+ 1) removing leftover backlinks during "shrink" operation
+ 2) refilling to low-watermark after 1) happens
+ 3) refilling to mid-watermark after 1) happens
+ 4) adding then removing 20% extra documents
+ 5) removing 20% first-added documents
+ 6) removing first-added documents while inserting new ones
+
+ 7) auto-tune search_k to ensure >= 50% recall on 1000 Q with k=100
+ 8) auto-tune search_k to ensure avg 90% recall on 1000 Q with k=100
+ 9) auto-tune search_k to ensure >= 90% reachability of 10000 docids
+
+ 10) timings for SIFT, GIST, and DEEP data (100k, 200k, 300k, 500k, 700k, 1000k)
+ */
+
+static size_t distcalls_simple;
+static size_t distcalls_search_layer;
+static size_t distcalls_other;
+static size_t distcalls_heuristic;
+static size_t distcalls_shrink;
+static size_t distcalls_refill;
+static size_t refill_needed_calls;
+static size_t shrink_needed_calls;
+static size_t disconnected_weak_links;
+static size_t disconnected_for_symmetry;
+static size_t select_n_full;
+static size_t select_n_partial;
+
+
+
+HnswLikeNns::HnswLikeNns(uint32_t numDims, const DocVectorAccess<float> &dva)
+    : NNS(numDims, dva),
+      _nodes(),
+      _entryId(0),
+      _entryLevel(-1),
+      _M(16),
+      _efConstruction(200),
+      _levelMultiplier(1.0 / log(1.0 * _M)),
+      _rndGen(),
+      _ops_counter(0)
 {
-    using Mark = unsigned short;
-    Mark *ptr;
-    Mark curval;
-    size_t sz;
-    VisitedSet(const VisitedSet &) = delete;
-    VisitedSet& operator=(const VisitedSet &) = delete;
-    explicit VisitedSet(size_t size) {
-        ptr = (Mark *)malloc(size * sizeof(Mark));
-        curval = -1;
-        sz = size;
-    }
-    void clear() {
-        ++curval;
-        if (curval == 0) {
-            memset(ptr, 0, sz * sizeof(Mark));
-            ++curval;
-        }
-    }
-    ~VisitedSet() { free(ptr); }
-    void mark(size_t id) { ptr[id] = curval; }
-    bool isMarked(size_t id) const { return ptr[id] == curval; }
-};
+}
 
-struct VisitedSetPool
-{
-    std::unique_ptr<VisitedSet> lastUsed;
-    VisitedSetPool() {
-        lastUsed = std::make_unique<VisitedSet>(250);
-    }
-    ~VisitedSetPool() {}
-    VisitedSet &get(size_t size) {
-        if (size > lastUsed->sz) {
-            lastUsed = std::make_unique<VisitedSet>(size*2);
+// simple greedy search
+HnswHit
+HnswLikeNns::search_layer_simple(Vector vector, HnswHit curPoint, uint32_t searchLevel) {
+    bool keepGoing = true;
+    while (keepGoing) {
+        keepGoing = false;
+        const LinkList& neighbors = getLinkList(curPoint.docid, searchLevel);
+        for (uint32_t n_id : neighbors) {
+            double dist = distance(vector, n_id);
+            ++distcalls_simple;
+            if (dist < curPoint.dist) {
+                curPoint = HnswHit(n_id, SqDist(dist));
+                keepGoing = true;
+            }
         }
-        lastUsed->clear();
-        return *lastUsed;
     }
-};
-
-struct HnswHit {
-    float dist;
-    uint32_t docid;
-    HnswHit(uint32_t di, SqDist sq) : dist(sq.distance), docid(di) {}
-};
+    return curPoint;
+}
 
+bool
+HnswLikeNns::haveCloserDistance(HnswHit e, const LinkList &r) const {
+    for (uint32_t prevId : r) {
+        double dist = distance(e.docid, prevId);
+        ++distcalls_heuristic;
+        if (dist < e.dist) return true;
+    }
+    return false;
+}
 
-using QueueEntry = HnswHit;
-struct GreaterDist {
-    bool operator() (const QueueEntry &lhs, const QueueEntry& rhs) const {
-        return (rhs.dist < lhs.dist);
+void
+HnswLikeNns::addDoc(uint32_t docid) {
+    Vector vector = _dva.get(docid);
+    for (uint32_t id = _nodes.size(); id <= docid; ++id) {
+        _nodes.emplace_back(id, 0, _M);
     }
-};
-struct LesserDist {
-    bool operator() (const QueueEntry &lhs, const QueueEntry& rhs) const {
-        return (lhs.dist < rhs.dist);
+    int level = randomLevel();
+    assert(_nodes[docid]._links.size() == 0);
+    _nodes[docid] = Node(docid, level+1, _M);
+    if (_entryLevel < 0) {
+        _entryId = docid;
+        _entryLevel = level;
+        track_ops();
+        return;
     }
-};
-
-using NearestList = std::vector<QueueEntry>;
-
-struct NearestPriQ : std::priority_queue<QueueEntry, NearestList, GreaterDist>
-{
-};
-
-struct FurthestPriQ : std::priority_queue<QueueEntry, NearestList, LesserDist>
-{
-   NearestList steal() {
-       NearestList result;
-       c.swap(result);
-       return result;
-   }
-   const NearestList& peek() const { return c; }
-};
-
-class HnswLikeNns : public NNS<float>
-{
-private:
-    std::vector<Node> _nodes;
-    uint32_t _entryId;
-    int _entryLevel;
-    uint32_t _M;
-    uint32_t _efConstruction;
-    double _levelMultiplier;
-    std::default_random_engine _rndGen;
-    VisitedSetPool _visitedSetPool;
-
-    double distance(Vector v, uint32_t id) const;
-
-    double distance(uint32_t a, uint32_t b) const {
-        Vector v = _dva.get(a);
-        return distance(v, b);
+    int searchLevel = _entryLevel;
+    double entryDist = distance(vector, _entryId);
+    ++distcalls_other;
+    HnswHit entryPoint(_entryId, SqDist(entryDist));
+    while (searchLevel > level) {
+        entryPoint = search_layer_simple(vector, entryPoint, searchLevel);
+        --searchLevel;
     }
-
-    int randomLevel() {
-        std::uniform_real_distribution<double> distribution(0.0, 1.0);
-        double r = -log(distribution(_rndGen)) * _levelMultiplier;
-        return (int) r;
+    searchLevel = std::min(level, _entryLevel);
+    FurthestPriQ w;
+    w.push(entryPoint);
+    while (searchLevel >= 0) {
+        search_layer(vector, w, _efConstruction, searchLevel);
+        LinkList neighbors = select_neighbors(w.peek(), _M);
+        connect_new_node(docid, neighbors, searchLevel);
+        each_shrink_ifneeded(neighbors, searchLevel);
+        --searchLevel;
     }
-
-public:
-    HnswLikeNns(uint32_t numDims, const DocVectorAccess<float> &dva)
-        : NNS(numDims, dva),
-          _nodes(),
-          _entryId(0),
-          _entryLevel(-1),
-          _M(16),
-          _efConstruction(150),
-          _levelMultiplier(1.0 / log(1.0 * _M))
-    {
-        _nodes.reserve(1234567);
+    if (level > _entryLevel) {
+        _entryLevel = level;
+        _entryId = docid;
     }
+    track_ops();
+}
 
-    ~HnswLikeNns() {}
-
-    LinkList& getLinkList(uint32_t docid, uint32_t level) {
-        // assert(docid < _nodes.size());
-        // assert(level < _nodes[docid]._links.size());
-        return _nodes[docid]._links[level];
+void
+HnswLikeNns::track_ops() {
+    _ops_counter++;
+    if ((_ops_counter % 10000) == 0) {
+        double div = _ops_counter;
+        fprintf(stderr, "add / remove ops: %zu\n", _ops_counter);
+        fprintf(stderr, "distance calls for layer: %zu is %.3f per op\n", distcalls_search_layer, distcalls_search_layer/ div);
+        fprintf(stderr, "distance calls for heuristic: %zu is %.3f per op\n", distcalls_heuristic, distcalls_heuristic / div);
+        fprintf(stderr, "distance calls for simple: %zu is %.3f per op\n", distcalls_simple, distcalls_simple / div);
+        fprintf(stderr, "distance calls for shrink: %zu is %.3f per op\n", distcalls_shrink, distcalls_shrink / div);
+        fprintf(stderr, "distance calls for refill: %zu is %.3f per op\n", distcalls_refill, distcalls_refill / div);
+        fprintf(stderr, "distance calls for other: %zu is %.3f per op\n", distcalls_other, distcalls_other / div);
+        fprintf(stderr, "refill needed calls: %zu is %.3f per op\n", refill_needed_calls, refill_needed_calls / div);
+        fprintf(stderr, "shrink needed calls: %zu is %.3f per op\n", shrink_needed_calls, shrink_needed_calls / div);
+        fprintf(stderr, "disconnected weak links: %zu is %.3f per op\n", disconnected_weak_links, disconnected_weak_links / div);
+        fprintf(stderr, "disconnected for symmetry: %zu is %.3f per op\n", disconnected_for_symmetry, disconnected_for_symmetry / div);
+        fprintf(stderr, "select neighbors: partial %zu vs full %zu\n", select_n_partial, select_n_full);
     }
+}                    
 
-    // simple greedy search
-    QueueEntry search_layer_simple(Vector vector, QueueEntry curPoint, uint32_t searchLevel) {
-        bool keepGoing = true;
-        while (keepGoing) {
-            keepGoing = false;
-            const LinkList& neighbors = getLinkList(curPoint.docid, searchLevel);
-            for (uint32_t n_id : neighbors) {
-                double dist = distance(vector, n_id);
-                if (dist < curPoint.dist) {
-                     curPoint = QueueEntry(n_id, SqDist(dist));
-                     keepGoing = true;
-                }
-            }
+void
+HnswLikeNns::refill_ifneeded(uint32_t my_id, const LinkList &replacements, uint32_t level) {
+    LinkList &my_links = getLinkList(my_id, level);
+    if (my_links.size() < 8) {
+        ++refill_needed_calls;
+        for (uint32_t repl_id : replacements) {
+            if (repl_id == my_id) continue;
+            if (my_links.has_link_to(repl_id)) continue;
+            LinkList &other_links = getLinkList(repl_id, level);
+            if (other_links.size() + 1 >= _M) continue;
+            other_links.push_back(my_id);
+            my_links.push_back(repl_id);
+            if (my_links.size() >= _M) return;
         }
-        return curPoint;
     }
+}
 
-    void search_layer_foradd(Vector vector, FurthestPriQ &w,
-                             uint32_t ef, uint32_t searchLevel);
-
-    FurthestPriQ search_layer(Vector vector, NearestList entryPoints,
-                              uint32_t ef, uint32_t searchLevel) {
-        VisitedSet &visited = _visitedSetPool.get(_nodes.size());
-        NearestPriQ candidates;
-        FurthestPriQ w;
-        for (auto point : entryPoints) {
-            candidates.push(point);
-            w.push(point);
-            visited.mark(point.docid);
-        }
-        double limd = std::numeric_limits<double>::max();
-        while (! candidates.empty()) {
-            QueueEntry cand = candidates.top();
-            candidates.pop();
-            if (cand.dist > limd) {
-                break;
-            }
-            for (uint32_t e_id : getLinkList(cand.docid, searchLevel)) {
-                if (visited.isMarked(e_id)) continue;
-                visited.mark(e_id);
-                double e_dist = distance(vector, e_id);
-                if (e_dist < limd) {
-                    candidates.emplace(e_id, SqDist(e_dist));
-                    w.emplace(e_id, SqDist(e_dist));
-                    if (w.size() > ef) {
-                        w.pop();
-                        limd = w.top().dist;
-                    }
-                }
-            }
-        }
-        return w;
+void
+HnswLikeNns::shrink_links(uint32_t shrink_id, uint32_t maxLinks, uint32_t level) {
+    LinkList &links = getLinkList(shrink_id, level);
+    NearestList distances;
+    for (uint32_t n_id : links) {
+        double n_dist = distance(shrink_id, n_id);
+        ++distcalls_shrink;
+        distances.emplace_back(n_id, SqDist(n_dist));
     }
-
-    bool haveCloserDistance(QueueEntry e, const LinkList &r) const {
-        for (uint32_t prevId : r) {
-            double dist = distance(e.docid, prevId);
-            if (dist < e.dist) return true;
-        }
-        return false;
+    LinkList lostLinks;
+    LinkList oldLinks = links;
+    links = remove_weakest(distances, maxLinks, lostLinks);
+#define KEEP_SYM
+#ifdef KEEP_SYM
+    for (uint32_t lost_id : lostLinks) {
+        ++disconnected_for_symmetry;
+        remove_link_from(lost_id, shrink_id, level);
     }
+#define DO_REFILL_AFTER_KEEP_SYM
+#ifdef DO_REFILL_AFTER_KEEP_SYM
+    for (uint32_t lost_id : lostLinks) {
+        refill_ifneeded(lost_id, oldLinks, level);
+    }
+#endif
+#endif
+}
 
-    LinkList select_neighbors(NearestPriQ &&w, uint32_t curMax) const;
-
-    LinkList select_neighbors(const NearestList &neighbors, uint32_t curMax) {
-        if (neighbors.size() <= curMax) {
-            LinkList result;
-            result.reserve(curMax+1);
-            for (const auto & entry : neighbors) {
-                result.push_back(entry.docid);
+void
+HnswLikeNns::removeDoc(uint32_t docid) {
+    Node &node = _nodes[docid];
+    bool need_new_entrypoint = (docid == _entryId);
+    for (int level = node._links.size(); level-- > 0; ) {
+        LinkList my_links;
+        my_links.swap(node._links[level]);
+        for (uint32_t n_id : my_links) {
+            if (need_new_entrypoint) {
+                _entryId = n_id;
+                _entryLevel = level;
+                need_new_entrypoint = false;
             }
-            return result;
+            remove_link_from(n_id, docid, level);
         }
-        NearestPriQ w;
-        for (const QueueEntry & entry : neighbors) {
-            w.push(entry);
+        while (! my_links.empty()) {
+            uint32_t n_id = my_links.back();
+            my_links.pop_back();
+            refill_ifneeded(n_id, my_links, level);
         }
-        return select_neighbors(std::move(w), curMax);
     }
-
-    void addDoc(uint32_t docid) override {
-        Vector vector = _dva.get(docid);
-        for (uint32_t id = _nodes.size(); id <= docid; ++id) {
-            _nodes.emplace_back(id, 0, _M);
-        }
-        int level = randomLevel();
-        assert(_nodes[docid]._links.size() == 0);
-        _nodes[docid] = Node(docid, level+1, _M);
-        if (_entryLevel < 0) {
-            _entryId = docid;
-            _entryLevel = level;
-            return;
-        }
-        int searchLevel = _entryLevel;
-        double entryDist = distance(vector, _entryId);
-        QueueEntry entryPoint(_entryId, SqDist(entryDist));
-        while (searchLevel > level) {
-            entryPoint = search_layer_simple(vector, entryPoint, searchLevel);
-            --searchLevel;
-        }
-        searchLevel = std::min(level, _entryLevel);
-        FurthestPriQ w;
-        w.push(entryPoint);
-        while (searchLevel >= 0) {
-            search_layer_foradd(vector, w, _efConstruction, searchLevel);
-            uint32_t maxLinks = (searchLevel > 0) ? _M : (2 * _M);
-            LinkList neighbors = select_neighbors(w.peek(), maxLinks);
-            connect_new_node(docid, neighbors, searchLevel);
-            each_shrink_ifneeded(neighbors, searchLevel);
-            --searchLevel;
-        }
-        if (level > _entryLevel) {
-            _entryLevel = level;
-            _entryId = docid;
+    node = Node(docid, 0, _M);
+    if (need_new_entrypoint) {
+        _entryLevel = -1;
+        _entryId = 0;
+        for (uint32_t i = 0; i < _nodes.size(); ++i) {
+            if (_nodes[i]._links.size() > 0) {
+                _entryId = i;
+                _entryLevel = _nodes[i]._links.size() - 1;
+                break;
+            }
         }
     }
+    track_ops();
+}
 
-    void connect_new_node(uint32_t id, const LinkList &neighbors, uint32_t level);
-
-    void each_shrink_ifneeded(const LinkList &neighbors, uint32_t level);
-
-    void removeDoc(uint32_t ) override {
+std::vector<NnsHit>
+HnswLikeNns::topK(uint32_t k, Vector vector, uint32_t search_k) {
+    std::vector<NnsHit> result;
+    if (_entryLevel < 0) return result;
+    double entryDist = distance(vector, _entryId);
+    ++distcalls_other;
+    HnswHit entryPoint(_entryId, SqDist(entryDist));
+    int searchLevel = _entryLevel;
+    while (searchLevel > 0) {
+        entryPoint = search_layer_simple(vector, entryPoint, searchLevel);
+        --searchLevel;
     }
-
-    std::vector<NnsHit> topK(uint32_t k, Vector vector, uint32_t search_k) override {
-        std::vector<NnsHit> result;
-        if (_entryLevel < 0) return result;
-        double entryDist = distance(vector, _entryId);
-        QueueEntry entryPoint(_entryId, SqDist(entryDist));
-        int searchLevel = _entryLevel;
-        while (searchLevel > 0) {
-            entryPoint = search_layer_simple(vector, entryPoint, searchLevel);
-            --searchLevel;
-        }
-        NearestList entryPoints;
-        entryPoints.push_back(entryPoint);
-        FurthestPriQ w = search_layer(vector, entryPoints, std::max(k, search_k), 0);
-        if (w.size() < k) {
-            fprintf(stderr, "fewer than expected hits: k=%u, ks=%u, got=%zu\n",
-                    k, search_k, w.size());
-        }
-        while (w.size() > k) {
-            w.pop();
-        }
-        std::vector<QueueEntry> reversed;
-        reversed.reserve(w.size());
-        while (! w.empty()) {
-            reversed.push_back(w.top());
-            w.pop();
-        }
-        result.reserve(reversed.size());
-        while (! reversed.empty()) {
-            const QueueEntry &hit = reversed.back();
-            result.emplace_back(hit.docid, SqDist(hit.dist));
-            reversed.pop_back();
-        }
-        return result;
+    FurthestPriQ w;
+    w.push(entryPoint);
+    search_layer(vector, w, std::max(k, search_k), 0);
+    while (w.size() > k) {
+        w.pop();
+    }
+    NearestList tmp = w.steal();
+    std::sort(tmp.begin(), tmp.end(), LesserDist());
+    result.reserve(tmp.size());
+    for (const auto & hit : tmp) {
+        result.emplace_back(hit.docid, SqDist(hit.dist));
     }
-};
+    return result;
+}
+
 
 double
 HnswLikeNns::distance(Vector v, uint32_t b) const
@@ -308,87 +252,293 @@ HnswLikeNns::distance(Vector v, uint32_t b) const
     return l2distCalc.l2sq_dist(v, w);
 }
 
+std::vector<NnsHit> 
+HnswLikeNns::topKfilter(uint32_t k, Vector vector, uint32_t search_k, const BitVector &blacklist)
+{
+    std::vector<NnsHit> result;
+    if (_entryLevel < 0) return result;
+    double entryDist = distance(vector, _entryId);
+    ++distcalls_other;
+    HnswHit entryPoint(_entryId, SqDist(entryDist));
+    int searchLevel = _entryLevel;
+    while (searchLevel > 0) {
+        entryPoint = search_layer_simple(vector, entryPoint, searchLevel);
+        --searchLevel;
+    }
+    FurthestPriQ w;
+    w.push(entryPoint);
+    search_layer_with_filter(vector, w, std::max(k, search_k), 0, blacklist);
+    NearestList tmp = w.steal();
+    std::sort(tmp.begin(), tmp.end(), LesserDist());
+    result.reserve(std::min((size_t)k, tmp.size()));
+    for (const auto & hit : tmp) {
+        if (blacklist.isSet(hit.docid)) continue;
+        result.emplace_back(hit.docid, SqDist(hit.dist));
+        if (result.size() == k) break;
+    }
+    return result;
+}
+
 void
 HnswLikeNns::each_shrink_ifneeded(const LinkList &neighbors, uint32_t level) {
-        uint32_t maxLinks = (level > 0) ? _M : (2 * _M);
-        for (uint32_t old_id : neighbors) {
-            LinkList &oldLinks = getLinkList(old_id, level);
-            if (oldLinks.size() > maxLinks) {
-                NearestPriQ w;
-                for (uint32_t n_id : oldLinks) {
-                    double n_dist = distance(old_id, n_id);
-                    w.emplace(n_id, SqDist(n_dist));
+    uint32_t maxLinks = (level > 0) ? _M : (2 * _M);
+    for (uint32_t old_id : neighbors) {
+        LinkList &oldLinks = getLinkList(old_id, level);
+        if (oldLinks.size() > maxLinks) {
+            ++shrink_needed_calls;
+            shrink_links(old_id, maxLinks, level);
+        }
+    }
+}
+
+void
+HnswLikeNns::search_layer(Vector vector, FurthestPriQ &w,
+                          uint32_t ef, uint32_t searchLevel)
+{
+    NearestPriQ candidates;
+    VisitedSet &visited = _visitedSetPool.get(_nodes.size());
+
+    for (const HnswHit & entry : w.peek()) {
+        candidates.push(entry);
+        visited.mark(entry.docid);
+    }
+    double limd = std::numeric_limits<double>::max();
+    while (! candidates.empty()) {
+        HnswHit cand = candidates.top();
+        if (cand.dist > limd) {
+            break;
+        }
+        candidates.pop();
+        for (uint32_t e_id : getLinkList(cand.docid, searchLevel)) {
+            if (visited.isMarked(e_id)) continue;
+            visited.mark(e_id);
+            double e_dist = distance(vector, e_id);
+            ++distcalls_search_layer;
+            if (e_dist < limd) {
+                candidates.emplace(e_id, SqDist(e_dist));
+                w.emplace(e_id, SqDist(e_dist));
+                if (w.size() > ef) {
+                    w.pop();
+                    limd = w.top().dist;
                 }
-                oldLinks = select_neighbors(std::move(w), maxLinks);
             }
         }
+    }
+    return;
 }
 
 void
-HnswLikeNns::search_layer_foradd(Vector vector, FurthestPriQ &w,
-                                 uint32_t ef, uint32_t searchLevel)
+HnswLikeNns::search_layer_with_filter(Vector vector, FurthestPriQ &w,
+                                      uint32_t ef, uint32_t searchLevel,
+                                      const BitVector &blacklist)
 {
-        NearestPriQ candidates;
-        VisitedSet &visited = _visitedSetPool.get(_nodes.size());
+    NearestPriQ candidates;
+    VisitedSet &visited = _visitedSetPool.get(_nodes.size());
 
-        for (const QueueEntry& entry : w.peek()) {
-            candidates.push(entry);
-            visited.mark(entry.docid);
+    for (const HnswHit & entry : w.peek()) {
+        candidates.push(entry);
+        visited.mark(entry.docid);
+        if (blacklist.isSet(entry.docid)) ++ef;
+    }
+    double limd = std::numeric_limits<double>::max();
+    while (! candidates.empty()) {
+        HnswHit cand = candidates.top();
+        if (cand.dist > limd) {
+            break;
         }
-
-        double limd = std::numeric_limits<double>::max();
-        while (! candidates.empty()) {
-            QueueEntry cand = candidates.top();
-            candidates.pop();
-            if (cand.dist > limd) {
-                break;
-            }
-            for (uint32_t e_id : getLinkList(cand.docid, searchLevel)) {
-                if (visited.isMarked(e_id)) continue;
-                visited.mark(e_id);
-                double e_dist = distance(vector, e_id);
-                if (e_dist < limd) {
-                    candidates.emplace(e_id, SqDist(e_dist));
-                    w.emplace(e_id, SqDist(e_dist));
-                    if (w.size() > ef) {
-                        w.pop();
-                        limd = w.top().dist;
-                    }
+        candidates.pop();
+        for (uint32_t e_id : getLinkList(cand.docid, searchLevel)) {
+            if (visited.isMarked(e_id)) continue;
+            visited.mark(e_id);
+            double e_dist = distance(vector, e_id);
+            ++distcalls_search_layer;
+            if (e_dist < limd) {
+                candidates.emplace(e_id, SqDist(e_dist));
+                if (blacklist.isSet(e_id)) continue;
+                w.emplace(e_id, SqDist(e_dist));
+                if (w.size() > ef) {
+                    w.pop();
+                    limd = w.top().dist;
                 }
             }
         }
-        return;
+    }
 }
 
 LinkList
-HnswLikeNns::select_neighbors(NearestPriQ &&w, uint32_t curMax) const {
-        LinkList result;
-        result.reserve(curMax+1);
-        while (! w.empty()) {
-            QueueEntry e = w.top();
-            w.pop();
-            if (haveCloserDistance(e, result)) continue;
+HnswLikeNns::remove_weakest(const NearestList &neighbors, uint32_t curMax, LinkList &lost) const
+{
+    LinkList result;
+    result.reserve(curMax+1);
+    NearestPriQ w;
+    for (const auto & entry : neighbors) {
+        w.push(entry);
+    }
+    while (! w.empty()) {
+        HnswHit e = w.top();
+        w.pop();
+        if (result.size() == curMax || haveCloserDistance(e, result)) {
+            lost.push_back(e.docid);
+        } else {
             result.push_back(e.docid);
-            if (result.size() >= curMax) break;
         }
-        return result;
+    }
+    return result;
 }
 
+#define NO_BACKFILL
+#ifdef NO_BACKFILL
+LinkList
+HnswLikeNns::select_neighbors(const NearestList &neighbors, uint32_t curMax) const
+{
+    LinkList result;
+    result.reserve(curMax+1);
+    NearestPriQ w;
+    for (const auto & entry : neighbors) {
+        w.push(entry);
+    }
+    while (! w.empty()) {
+        HnswHit e = w.top();
+        w.pop();
+        if (haveCloserDistance(e, result)) {
+            continue;
+        }
+        result.push_back(e.docid);
+        if (result.size() == curMax) {
+            ++select_n_full;
+            return result;
+        }
+    }
+    ++select_n_partial;
+    return result;
+}
+#else
+LinkList
+HnswLikeNns::select_neighbors(const NearestList &neighbors, uint32_t curMax) const
+{
+    LinkList result;
+    result.reserve(curMax+1);
+    bool needFiltering = (neighbors.size() > curMax);
+    NearestPriQ w;
+    for (const auto & entry : neighbors) {
+        w.push(entry);
+    }
+    LinkList backfill;
+    while (! w.empty()) {
+        HnswHit e = w.top();
+        w.pop();
+        if (needFiltering && haveCloserDistance(e, result)) {
+            backfill.push_back(e.docid);
+            continue;
+        }
+        result.push_back(e.docid);
+        if (result.size() == curMax) return result;
+    }
+    if (result.size() * 4 < _M) {
+        for (uint32_t fill_id : backfill) {
+            result.push_back(fill_id);
+            if (result.size() * 2 >= _M) break;
+        }
+    }
+    return result;
+}
+#endif
+
 void
 HnswLikeNns::connect_new_node(uint32_t id, const LinkList &neighbors, uint32_t level) {
-        LinkList &newLinks = getLinkList(id, level);
-        for (uint32_t neigh_id : neighbors) {
-            LinkList &oldLinks = getLinkList(neigh_id, level);
-            newLinks.push_back(neigh_id);
-            oldLinks.push_back(id);
+    LinkList &newLinks = getLinkList(id, level);
+    for (uint32_t neigh_id : neighbors) {
+        LinkList &oldLinks = getLinkList(neigh_id, level);
+        newLinks.push_back(neigh_id);
+        oldLinks.push_back(id);
+    }
+}
+
+uint32_t
+HnswLikeNns::count_reachable() const {
+    VisitedSet visited(_nodes.size());
+    int level = _entryLevel;
+    LinkList curList;
+    curList.push_back(_entryId);
+    visited.mark(_entryId);
+    uint32_t idx = 0;
+    while (level >= 0) {
+        while (idx < curList.size()) {
+            uint32_t id = curList[idx++];
+            const LinkList &links = getLinkList(id, level);
+            for (uint32_t n_id : links) {
+                if (visited.isMarked(n_id)) continue;
+                visited.mark(n_id);
+                curList.push_back(n_id);
+            }
         }
+        --level;
+        idx = 0;
+    }
+    return curList.size();
 }
 
+void
+HnswLikeNns::dumpStats() const {
+    std::vector<uint32_t> levelCounts;
+    levelCounts.resize(_entryLevel + 2);
+    std::vector<uint32_t> outLinkHist;
+    outLinkHist.resize(2 * _M + 2);
+    uint32_t symmetrics = 0;
+    uint32_t level1links = 0;
+    uint32_t both_l_links = 0;
+    fprintf(stderr, "stats for HnswLikeNns with %zu nodes, entry level = %d, entry id = %u\n",
+            _nodes.size(), _entryLevel, _entryId);
+    
+    for (uint32_t id = 0; id < _nodes.size(); ++id) {
+        const auto &node = _nodes[id];
+        uint32_t levels = node._links.size();
+        levelCounts[levels]++;
+        if (levels < 1) {
+            outLinkHist[0]++;
+            continue;
+        }
+        const LinkList &link_list = getLinkList(id, 0);
+        uint32_t numlinks = link_list.size();
+        outLinkHist[numlinks]++;
+        if (numlinks < 1) {
+            fprintf(stderr, "node with %u links: id %u\n", numlinks, id);
+        }
+        bool all_sym = true;
+        for (uint32_t n_id : link_list) {
+            const LinkList &neigh_list = getLinkList(n_id, 0);
+            if (! neigh_list.has_link_to(id)) {
+#ifdef KEEP_SYM
+                fprintf(stderr, "BAD: %u has link to neighbor %u, but backlink is missing\n", id, n_id);
+#endif
+                all_sym = false;
+            }
+        }
+        if (all_sym) ++symmetrics;
+        if (levels < 2) continue;
+        const LinkList &link_list_1 = getLinkList(id, 1);
+        for (uint32_t n_id : link_list_1) {
+            ++level1links;
+            if (link_list.has_link_to(n_id)) ++both_l_links;
+        }
+    }
+    for (uint32_t l = 0; l < levelCounts.size(); ++l) {
+        fprintf(stderr, "Nodes on %u levels: %u\n", l, levelCounts[l]);
+    }
+    fprintf(stderr, "reachable nodes %u / %zu\n",
+            count_reachable(), _nodes.size() - levelCounts[0]);
+    fprintf(stderr, "level 1 links overlapping on l0: %u / total: %u\n",
+            both_l_links, level1links);
+    for (uint32_t l = 0; l < outLinkHist.size(); ++l) {
+        if (outLinkHist[l] != 0) {
+            fprintf(stderr, "Nodes with %u outward links on L0: %u\n", l, outLinkHist[l]);
+        }
+    }
+    fprintf(stderr, "Symmetric in-out nodes: %u\n", symmetrics);
+}
 
 std::unique_ptr<NNS<float>>
 make_hnsw_nns(uint32_t numDims, const DocVectorAccess<float> &dva)
 {
     return std::make_unique<HnswLikeNns>(numDims, dva);
 }
-
-
diff --git a/eval/src/tests/ann/xp-lsh-nns.cpp b/eval/src/tests/ann/xp-lsh-nns.cpp
index 0ea119a9c70..c028a07a9d7 100644
--- a/eval/src/tests/ann/xp-lsh-nns.cpp
+++ b/eval/src/tests/ann/xp-lsh-nns.cpp
@@ -118,6 +118,7 @@ public:
         }
     }
     std::vector<NnsHit> topK(uint32_t k, Vector vector, uint32_t search_k) override;
+    std::vector<NnsHit> topKfilter(uint32_t k, Vector vector, uint32_t search_k, const BitVector &bitvector) override;
 
     V getVector(uint32_t docid) const { return _dva.get(docid); }
     double uniformRnd() { return _rndGen.nextUniform(); } 
@@ -196,6 +197,45 @@ public:
 };
 
 std::vector<NnsHit>
+RpLshNns::topKfilter(uint32_t k, Vector vector, uint32_t search_k, const BitVector &blacklist)
+{
+    std::vector<NnsHit> result;
+    result.reserve(k);
+
+    std::vector<float> tmp(_numDims);
+    vespalib::ArrayRef<float> tmpArr(tmp);
+
+    LsMaskHash query_hash = mask_hash_from_pv(vector, _transformationMatrix);
+    LshHitHeap heap(std::max(k, search_k));
+    int limit_hash_dist = 99999;
+    int skipCnt = 0;
+    int fullCnt = 0;
+    int whdcCnt = 0;
+    size_t docidLimit = _generated_doc_hashes.size();
+    for (uint32_t docid = 0; docid < docidLimit; ++docid) {
+        if (blacklist.isSet(docid)) continue;
+        int hd = hash_dist(query_hash, _generated_doc_hashes[docid]);
+        if (hd <= limit_hash_dist) {
+            ++fullCnt;
+            double dist = l2distCalc.l2sq_dist(vector, _dva.get(docid), tmpArr);
+            LshHit h(docid, dist, hd);
+            if (heap.maybe_use(h)) {
+                ++whdcCnt;
+                limit_hash_dist = heap.limitHashDistance();
+            }
+        } else {
+            ++skipCnt;
+        }
+    }
+    std::vector<LshHit> best = heap.bestLshHits();
+    size_t numHits = std::min((size_t)k, best.size());
+    for (size_t i = 0; i < numHits; ++i) {
+        result.emplace_back(best[i].docid, SqDist(best[i].distance));
+    }
+    return result;
+}
+
+std::vector<NnsHit>
 RpLshNns::topK(uint32_t k, Vector vector, uint32_t search_k)
 {
     std::vector<NnsHit> result;
diff --git a/eval/src/tests/eval/function/function_test.cpp b/eval/src/tests/eval/function/function_test.cpp
index 5316217d549..598d9c251a6 100644
--- a/eval/src/tests/eval/function/function_test.cpp
+++ b/eval/src/tests/eval/function/function_test.cpp
@@ -813,8 +813,8 @@ TEST("require that tensor rename dimension lists must have equal size") {
 //-----------------------------------------------------------------------------
 
 TEST("require that tensor lambda can be parsed") {
-    EXPECT_EQUAL("tensor(x[3]):{{x:0}:0,{x:1}:1,{x:2}:2}", Function::parse({}, "tensor(x[3])(x)")->dump());
-    EXPECT_EQUAL("tensor(x[2],y[2]):{{x:0,y:0}:(0==0),{x:0,y:1}:(0==1),{x:1,y:0}:(1==0),{x:1,y:1}:(1==1)}",
+    EXPECT_EQUAL("tensor(x[3])(x)", Function::parse({}, "tensor(x[3])(x)")->dump());
+    EXPECT_EQUAL("tensor(x[2],y[2])(x==y)",
                  Function::parse({}, " tensor ( x [ 2 ] , y [ 2 ] ) ( x == y ) ")->dump());
 }
 
@@ -825,7 +825,7 @@ TEST("require that tensor lambda requires appropriate tensor type") {
 }
 
 TEST("require that tensor lambda can use non-dimension symbols") {
-    EXPECT_EQUAL("tensor(x[2]):{{x:0}:(0==a),{x:1}:(1==a)}",
+    EXPECT_EQUAL("tensor(x[2])(x==a)",
                  Function::parse({"a"}, "tensor(x[2])(x==a)")->dump());
 }
 
@@ -977,9 +977,9 @@ TEST("require that tensor peek can contain expressions") {
 }
 
 TEST("require that trivial tensor peek number expressions are converted to verbatim labels") {
-    TEST_DO(verify_parse("t{x:(5.7)}", "f(t)(t{x:\"6\"})"));
+    TEST_DO(verify_parse("t{x:(5.7)}", "f(t)(t{x:\"5\"})"));
     TEST_DO(verify_parse("t{x:(5.3)}", "f(t)(t{x:\"5\"})"));
-    TEST_DO(verify_parse("t{x:(-5.7)}", "f(t)(t{x:\"-6\"})"));
+    TEST_DO(verify_parse("t{x:(-5.7)}", "f(t)(t{x:\"-5\"})"));
     TEST_DO(verify_parse("t{x:(-5.3)}", "f(t)(t{x:\"-5\"})"));
 }
 
@@ -1002,11 +1002,8 @@ TEST("require that tensor peek empty label is not allowed") {
 //-----------------------------------------------------------------------------
 
 TEST("require that nested tensor lambda using tensor peek can be parsed") {
-    vespalib::string expect("tensor(x[2]):{{x:0}:tensor(y[2]):{{y:0}:((0+0)+a),{y:1}:((0+1)+a)}{y:\"0\"},"
-                            "{x:1}:tensor(y[2]):{{y:0}:((1+0)+a),{y:1}:((1+1)+a)}{y:\"1\"}}");
+    vespalib::string expect("tensor(x[2])(tensor(y[2])((x+y)+a){y:(x)})");
     EXPECT_EQUAL(Function::parse(expect)->dump(), expect);
-    auto fun = Function::parse("tensor(x[2])(tensor(y[2])(x+y+a){y:(x)})");
-    EXPECT_EQUAL(fun->dump(), expect);
 }
 
 //-----------------------------------------------------------------------------
diff --git a/eval/src/tests/eval/interpreted_function/interpreted_function_test.cpp b/eval/src/tests/eval/interpreted_function/interpreted_function_test.cpp
index d946d244d17..60700817266 100644
--- a/eval/src/tests/eval/interpreted_function/interpreted_function_test.cpp
+++ b/eval/src/tests/eval/interpreted_function/interpreted_function_test.cpp
@@ -83,7 +83,6 @@ struct MyEvalTest : test::EvalSpec::EvalTest {
                                ? NodeTypes(function, std::vector<ValueType>(params.params.size(), ValueType::double_type()))
                                : NodeTypes();
         InterpretedFunction ifun(engine, function, node_types);
-        ASSERT_EQUAL(ifun.num_params(), params.params.size());
         InterpretedFunction::Context ictx(ifun);
         const Value &result_value = ifun.eval(ictx, params);
         report_result(result_value.is_double(), result_value.as_double(), expected_result, description);
diff --git a/eval/src/tests/eval/node_types/node_types_test.cpp b/eval/src/tests/eval/node_types/node_types_test.cpp
index 07bca5b53a6..7912ec213bc 100644
--- a/eval/src/tests/eval/node_types/node_types_test.cpp
+++ b/eval/src/tests/eval/node_types/node_types_test.cpp
@@ -18,6 +18,14 @@ struct TypeSpecExtractor : public vespalib::eval::SymbolExtractor {
     }
 };
 
+void print_errors(const NodeTypes &types) {
+    if (!types.errors().empty()) {
+        for (const auto &msg: types.errors()) {
+            fprintf(stderr, "type error: %s\n", msg.c_str());
+        }
+    }
+}
+
 void verify(const vespalib::string &type_expr, const vespalib::string &type_spec) {
     auto function = Function::parse(type_expr, TypeSpecExtractor());
     if (!EXPECT_TRUE(!function->has_error())) {
@@ -29,6 +37,7 @@ void verify(const vespalib::string &type_expr, const vespalib::string &type_spec
         input_types.push_back(ValueType::from_spec(function->param_name(i)));
     }
     NodeTypes types(*function, input_types);
+    print_errors(types);
     ValueType expected_type = ValueType::from_spec(type_spec);
     ValueType actual_type = types.get_type(function->root());
     EXPECT_EQUAL(expected_type, actual_type);
@@ -82,6 +91,7 @@ TEST("require that reduce resolves correct type") {
     TEST_DO(verify("reduce(tensor(x{},y{},z{}),sum,x,z)", "tensor(y{})"));
     TEST_DO(verify("reduce(tensor(x{},y{},z{}),sum,y,z,x)", "double"));
     TEST_DO(verify("reduce(tensor(x{},y{},z{}),sum,w)", "error"));
+    TEST_DO(verify("reduce(tensor(x{},y{},z{}),sum,a,b,c)", "error"));
     TEST_DO(verify("reduce(tensor(x{}),sum,x)", "double"));
     TEST_DO(verify("reduce(tensor<float>(x{},y{},z{}),sum,x,z)", "tensor<float>(y{})"));
     TEST_DO(verify("reduce(tensor<float>(x{}),sum,x)", "double"));
@@ -217,7 +227,7 @@ TEST("require that merge resolves to the appropriate type") {
     TEST_DO(verify(strfmt(pattern, "tensor<float>(x[5])", "double"), "error"));
 }
 
-TEST("require that lambda tensor resolves correct type") {
+TEST("require that static tensor lambda resolves correct type") {
     TEST_DO(verify("tensor(x[5])(1.0)", "tensor(x[5])"));
     TEST_DO(verify("tensor(x[5],y[10])(1.0)", "tensor(x[5],y[10])"));
     TEST_DO(verify("tensor(x[5],y[10],z[15])(1.0)", "tensor(x[5],y[10],z[15])"));
@@ -236,6 +246,14 @@ TEST("require that tensor create resolves correct type") {
     TEST_DO(verify("tensor(x[3]):{{x:0}:double,{x:1}:error,{x:2}:double}", "error"));
 }
 
+TEST("require that dynamic tensor lambda resolves correct type") {
+    TEST_DO(verify("tensor(x[3])(error)", "error"));
+    TEST_DO(verify("tensor(x[3])(double)", "tensor(x[3])"));
+    TEST_DO(verify("tensor<float>(x[3])(double)", "tensor<float>(x[3])"));
+    TEST_DO(verify("tensor(x[3])(tensor(x[2]))", "error"));
+    TEST_DO(verify("tensor(x[3])(reduce(tensor(x[2])+tensor(x[4]),sum))", "error"));
+}
+
 TEST("require that tensor peek resolves correct type") {
     TEST_DO(verify("tensor(x[3]){x:1}", "double"));
     TEST_DO(verify("tensor(x[3]){x:double}", "error"));
@@ -292,4 +310,38 @@ TEST("require that empty type repo works as expected") {
     EXPECT_FALSE(types.all_types_are_double());
 }
 
+TEST("require that types for a subtree can be exported") {
+    auto function = Function::parse("(1+2)+3");
+    const auto &root = function->root();
+    ASSERT_EQUAL(root.num_children(), 2u);
+    const auto &n_1_2 = root.get_child(0);
+    const auto &n_3 = root.get_child(1);
+    ASSERT_EQUAL(n_1_2.num_children(), 2u);
+    const auto &n_1 = n_1_2.get_child(0);
+    const auto &n_2 = n_1_2.get_child(1);
+    NodeTypes all_types(*function, {});
+    NodeTypes some_types = all_types.export_types(n_1_2);
+    EXPECT_EQUAL(all_types.errors().size(), 0u);
+    EXPECT_EQUAL(some_types.errors().size(), 0u);
+    for (const auto node: {&root, &n_3}) {
+        EXPECT_TRUE(all_types.get_type(*node).is_double());
+        EXPECT_TRUE(some_types.get_type(*node).is_error());
+    }
+    for (const auto node: {&n_1_2, &n_1, &n_2}) {
+        EXPECT_TRUE(all_types.get_type(*node).is_double());
+        EXPECT_TRUE(some_types.get_type(*node).is_double());
+    }
+}
+
+TEST("require that export_types produces an error for missing types") {
+    auto fun1 = Function::parse("1+2");
+    auto fun2 = Function::parse("1+2");
+    NodeTypes fun1_types(*fun1, {});
+    NodeTypes bad_export = fun1_types.export_types(fun2->root());
+    EXPECT_EQUAL(bad_export.errors().size(), 1u);
+    print_errors(bad_export);
+    EXPECT_TRUE(fun1_types.get_type(fun1->root()).is_double());
+    EXPECT_TRUE(bad_export.get_type(fun2->root()).is_error());
+}
+
 TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/eval/src/tests/eval/tensor_lambda/CMakeLists.txt b/eval/src/tests/eval/tensor_lambda/CMakeLists.txt
new file mode 100644
index 00000000000..29cbbd936aa
--- /dev/null
+++ b/eval/src/tests/eval/tensor_lambda/CMakeLists.txt
@@ -0,0 +1,8 @@
+# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(eval_tensor_lambda_test_app TEST
+    SOURCES
+    tensor_lambda_test.cpp
+    DEPENDS
+    vespaeval
+)
+vespa_add_test(NAME eval_tensor_lambda_test_app COMMAND eval_tensor_lambda_test_app)
diff --git a/eval/src/tests/eval/tensor_lambda/tensor_lambda_test.cpp b/eval/src/tests/eval/tensor_lambda/tensor_lambda_test.cpp
new file mode 100644
index 00000000000..5b0f2cf0a7e
--- /dev/null
+++ b/eval/src/tests/eval/tensor_lambda/tensor_lambda_test.cpp
@@ -0,0 +1,100 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/vespalib/testkit/test_kit.h>
+#include <vespa/eval/eval/tensor_function.h>
+#include <vespa/eval/eval/simple_tensor.h>
+#include <vespa/eval/eval/simple_tensor_engine.h>
+#include <vespa/eval/tensor/default_tensor_engine.h>
+#include <vespa/eval/tensor/dense/dense_replace_type_function.h>
+#include <vespa/eval/tensor/dense/dense_fast_rename_optimizer.h>
+#include <vespa/eval/tensor/dense/dense_tensor.h>
+#include <vespa/eval/eval/test/tensor_model.hpp>
+#include <vespa/eval/eval/test/eval_fixture.h>
+#include <vespa/eval/eval/tensor_nodes.h>
+
+#include <vespa/vespalib/util/stringfmt.h>
+#include <vespa/vespalib/util/stash.h>
+
+using namespace vespalib;
+using namespace vespalib::eval;
+using namespace vespalib::eval::test;
+using namespace vespalib::tensor;
+using namespace vespalib::eval::tensor_function;
+
+const TensorEngine &prod_engine = DefaultTensorEngine::ref();
+
+EvalFixture::ParamRepo make_params() {
+    return EvalFixture::ParamRepo()
+        .add("a", spec(1))
+        .add("x3", spec({x(3)}, N()))
+        .add("x3f", spec(float_cells({x(3)}), N()));
+}
+EvalFixture::ParamRepo param_repo = make_params();
+
+void verify_dynamic(const vespalib::string &expr, const vespalib::string &expect) {
+    EvalFixture fixture(prod_engine, expr, param_repo, true);
+    EXPECT_EQUAL(fixture.result(), EvalFixture::ref(expr, param_repo));
+    EXPECT_EQUAL(fixture.result(), EvalFixture::ref(expect, param_repo));
+    auto info = fixture.find_all<Lambda>();
+    EXPECT_EQUAL(info.size(), 1u);
+}
+
+void verify_const(const vespalib::string &expr, const vespalib::string &expect) {
+    EvalFixture fixture(prod_engine, expr, param_repo, true);
+    EXPECT_EQUAL(fixture.result(), EvalFixture::ref(expr, param_repo));
+    EXPECT_EQUAL(fixture.result(), EvalFixture::ref(expect, param_repo));
+    auto info = fixture.find_all<ConstValue>();
+    EXPECT_EQUAL(info.size(), 1u);
+}
+
+TEST("require that simple constant tensor lambda works") {
+    TEST_DO(verify_const("tensor(x[3])(x+1)", "tensor(x[3]):[1,2,3]"));
+}
+
+TEST("require that simple dynamic tensor lambda works") {
+    TEST_DO(verify_dynamic("tensor(x[3])(x+a)", "tensor(x[3]):[1,2,3]"));
+}
+
+TEST("require that tensor lambda can be used for tensor slicing") {
+    TEST_DO(verify_dynamic("tensor(x[2])(x3{x:(x+a)})", "tensor(x[2]):[2,3]"));
+    TEST_DO(verify_dynamic("tensor(x[2])(a+x3{x:(x)})", "tensor(x[2]):[2,3]"));
+}
+
+TEST("require that tensor lambda can be used for tensor casting") {
+    TEST_DO(verify_dynamic("tensor(x[3])(x3f{x:(x)})", "tensor(x[3]):[1,2,3]"));
+    TEST_DO(verify_dynamic("tensor<float>(x[3])(x3{x:(x)})", "tensor<float>(x[3]):[1,2,3]"));
+}
+
+TEST("require that constant nested tensor lambda using tensor peek works") {
+    TEST_DO(verify_const("tensor(x[2])(tensor(y[2])((x+y)+1){y:(x)})", "tensor(x[2]):[1,3]"));
+}
+
+TEST("require that dynamic nested tensor lambda using tensor peek works") {
+    TEST_DO(verify_dynamic("tensor(x[2])(tensor(y[2])((x+y)+a){y:(x)})", "tensor(x[2]):[1,3]"));
+}
+
+TEST("require that non-double result from inner tensor lambda function fails type resolving") {
+    auto fun_a = Function::parse("tensor(x[2])(a)");
+    auto fun_b = Function::parse("tensor(x[2])(a{y:(x)})");
+    NodeTypes types_ad(*fun_a, {ValueType::from_spec("double")});
+    NodeTypes types_at(*fun_a, {ValueType::from_spec("tensor(y[2])")});
+    NodeTypes types_bd(*fun_b, {ValueType::from_spec("double")});
+    NodeTypes types_bt(*fun_b, {ValueType::from_spec("tensor(y[2])")});
+    EXPECT_EQUAL(types_ad.get_type(fun_a->root()).to_spec(), "tensor(x[2])");
+    EXPECT_EQUAL(types_at.get_type(fun_a->root()).to_spec(), "error");
+    EXPECT_EQUAL(types_bd.get_type(fun_b->root()).to_spec(), "error");
+    EXPECT_EQUAL(types_bt.get_type(fun_b->root()).to_spec(), "tensor(x[2])");
+}
+
+TEST("require that type resolving also include nodes in the inner tensor lambda function") {
+    auto fun = Function::parse("tensor(x[2])(a)");
+    NodeTypes types(*fun, {ValueType::from_spec("double")});
+    auto lambda = nodes::as<nodes::TensorLambda>(fun->root());
+    ASSERT_TRUE(lambda != nullptr);
+    EXPECT_EQUAL(types.get_type(*lambda).to_spec(), "tensor(x[2])");
+    auto symbol = nodes::as<nodes::Symbol>(lambda->lambda().root());
+    ASSERT_TRUE(symbol != nullptr);
+    EXPECT_EQUAL(types.get_type(*symbol).to_spec(), "double");
+}
+
+TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/eval/src/tests/tensor/dense_matmul_function/CMakeLists.txt b/eval/src/tests/tensor/dense_matmul_function/CMakeLists.txt
new file mode 100644
index 00000000000..7234e8b9e69
--- /dev/null
+++ b/eval/src/tests/tensor/dense_matmul_function/CMakeLists.txt
@@ -0,0 +1,8 @@
+# Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(eval_dense_matmul_function_test_app TEST
+    SOURCES
+    dense_matmul_function_test.cpp
+    DEPENDS
+    vespaeval
+)
+vespa_add_test(NAME eval_dense_matmul_function_test_app COMMAND eval_dense_matmul_function_test_app)
diff --git a/eval/src/tests/tensor/dense_matmul_function/dense_matmul_function_test.cpp b/eval/src/tests/tensor/dense_matmul_function/dense_matmul_function_test.cpp
new file mode 100644
index 00000000000..5d7c0be704e
--- /dev/null
+++ b/eval/src/tests/tensor/dense_matmul_function/dense_matmul_function_test.cpp
@@ -0,0 +1,147 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/vespalib/testkit/test_kit.h>
+#include <vespa/eval/eval/tensor_function.h>
+#include <vespa/eval/eval/operation.h>
+#include <vespa/eval/eval/simple_tensor.h>
+#include <vespa/eval/eval/simple_tensor_engine.h>
+#include <vespa/eval/tensor/default_tensor_engine.h>
+#include <vespa/eval/tensor/dense/dense_matmul_function.h>
+#include <vespa/eval/tensor/dense/dense_tensor.h>
+#include <vespa/eval/tensor/dense/dense_tensor_view.h>
+#include <vespa/eval/eval/test/tensor_model.hpp>
+#include <vespa/eval/eval/test/eval_fixture.h>
+
+#include <vespa/vespalib/util/stringfmt.h>
+#include <vespa/vespalib/util/stash.h>
+
+using namespace vespalib;
+using namespace vespalib::eval;
+using namespace vespalib::eval::test;
+using namespace vespalib::tensor;
+using namespace vespalib::eval::tensor_function;
+
+const TensorEngine &prod_engine = DefaultTensorEngine::ref();
+
+void add_matrix(EvalFixture::ParamRepo &repo, const char *d1, size_t s1, const char *d2, size_t s2) {
+    for (bool float_cells: {false, true}) {
+        auto name = make_string("%s%zu%s%zu%s", d1, s1, d2, s2, float_cells ? "f" : "");
+        auto type_str = make_string("tensor%s(%s[%zu],%s[%zu])", float_cells ? "<float>" : "", d1, s1, d2, s2);
+        TensorSpec matrix(type_str);
+        for (size_t i = 0; i < s1; ++i) {
+            for (size_t j = 0; j < s2; ++j) {
+                double value = (i + s1 + s2) * 3.0 + (j + s2) * 7.0;
+                matrix.add({{d1, i}, {d2, j}}, value);
+            }
+        }
+        repo.add(name, matrix);
+    }
+}
+
+EvalFixture::ParamRepo make_params() {
+    EvalFixture::ParamRepo repo;
+    add_matrix(repo, "a", 2, "d", 3); // inner/inner
+    add_matrix(repo, "a", 2, "b", 5); // inner/outer
+    add_matrix(repo, "b", 5, "c", 2); // outer/outer
+    add_matrix(repo, "a", 2, "c", 3); // not matching
+    //-----------------------------------------------
+    add_matrix(repo, "b", 5, "d", 3); // fixed param
+    return repo;
+}
+EvalFixture::ParamRepo param_repo = make_params();
+
+void verify_optimized(const vespalib::string &expr,
+                      size_t lhs_size, size_t common_size, size_t rhs_size,
+                      bool lhs_inner, bool rhs_inner)
+{
+    EvalFixture slow_fixture(prod_engine, expr, param_repo, false);
+    EvalFixture fixture(prod_engine, expr, param_repo, true);
+    EXPECT_EQUAL(fixture.result(), EvalFixture::ref(expr, param_repo));
+    EXPECT_EQUAL(fixture.result(), slow_fixture.result());
+    auto info = fixture.find_all<DenseMatMulFunction>();
+    ASSERT_EQUAL(info.size(), 1u);
+    EXPECT_TRUE(info[0]->result_is_mutable());
+    EXPECT_EQUAL(info[0]->lhs_size(), lhs_size);
+    EXPECT_EQUAL(info[0]->common_size(), common_size);
+    EXPECT_EQUAL(info[0]->rhs_size(), rhs_size);
+    EXPECT_EQUAL(info[0]->lhs_common_inner(), lhs_inner);
+    EXPECT_EQUAL(info[0]->rhs_common_inner(), rhs_inner);
+}
+
+void verify_not_optimized(const vespalib::string &expr) {
+    EvalFixture slow_fixture(prod_engine, expr, param_repo, false);
+    EvalFixture fixture(prod_engine, expr, param_repo, true);
+    EXPECT_EQUAL(fixture.result(), EvalFixture::ref(expr, param_repo));
+    EXPECT_EQUAL(fixture.result(), slow_fixture.result());
+    auto info = fixture.find_all<DenseMatMulFunction>();
+    EXPECT_TRUE(info.empty());
+}
+
+TEST("require that matmul can be optimized") {
+    TEST_DO(verify_optimized("reduce(a2d3*b5d3,sum,d)", 2, 3, 5, true, true));
+}
+
+TEST("require that matmul with lambda can be optimized") {
+    TEST_DO(verify_optimized("reduce(join(a2d3,b5d3,f(x,y)(x*y)),sum,d)", 2, 3, 5, true, true));
+    TEST_DO(verify_optimized("reduce(join(a2d3,b5d3,f(x,y)(y*x)),sum,d)", 2, 3, 5, true, true));
+}
+
+TEST("require that expressions similar to matmul are not optimized") {
+    TEST_DO(verify_not_optimized("reduce(a2d3*b5d3,sum,a)"));
+    TEST_DO(verify_not_optimized("reduce(a2d3*b5d3,sum,b)"));
+    TEST_DO(verify_not_optimized("reduce(a2d3*b5d3,prod,d)"));
+    TEST_DO(verify_not_optimized("reduce(a2d3*b5d3,sum)"));
+    TEST_DO(verify_not_optimized("reduce(join(a2d3,b5d3,f(x,y)(x+y)),sum,d)"));
+    TEST_DO(verify_not_optimized("reduce(join(a2d3,b5d3,f(x,y)(x*x)),sum,d)"));
+    TEST_DO(verify_not_optimized("reduce(join(a2d3,b5d3,f(x,y)(y*y)),sum,d)"));
+    TEST_DO(verify_not_optimized("reduce(join(a2d3,b5d3,f(x,y)(x*y*1)),sum,d)"));
+    TEST_DO(verify_not_optimized("reduce(a2c3*b5d3,sum,d)"));
+    TEST_DO(verify_not_optimized("reduce(a2c3*b5d3,sum,c)"));
+}
+
+TEST("require that xw product can be debug dumped") {
+    EvalFixture fixture(prod_engine, "reduce(a2d3*b5d3,sum,d)", param_repo, true);
+    auto info = fixture.find_all<DenseMatMulFunction>();
+    ASSERT_EQUAL(info.size(), 1u);
+    fprintf(stderr, "%s\n", info[0]->as_string().c_str());
+}
+
+vespalib::string make_expr(const vespalib::string &a, const vespalib::string &b, const vespalib::string &common,
+                           bool float_a, bool float_b)
+{
+    return make_string("reduce(%s%s*%s%s,sum,%s)", a.c_str(), float_a ? "f" : "", b.c_str(), float_b ? "f" : "", common.c_str());
+}
+
+void verify_optimized_multi(const vespalib::string &a, const vespalib::string &b, const vespalib::string &common,
+                            size_t lhs_size, size_t common_size, size_t rhs_size,
+                            bool lhs_inner, bool rhs_inner)
+{
+    for (bool float_a: {false, true}) {
+        for (bool float_b: {false, true}) {
+            {
+                auto expr = make_expr(a, b, common, float_a, float_b);
+                TEST_STATE(expr.c_str());
+                TEST_DO(verify_optimized(expr, lhs_size, common_size, rhs_size, lhs_inner, rhs_inner));
+            }
+            {
+                auto expr = make_expr(b, a, common, float_b, float_a);
+                TEST_STATE(expr.c_str());
+                TEST_DO(verify_optimized(expr, lhs_size, common_size, rhs_size, lhs_inner, rhs_inner));
+            }
+        }
+    }
+}
+
+TEST("require that matmul inner/inner works correctly") {
+    TEST_DO(verify_optimized_multi("a2d3", "b5d3", "d", 2, 3, 5, true, true));
+}
+
+TEST("require that matmul inner/outer works correctly") {
+    TEST_DO(verify_optimized_multi("a2b5", "b5d3", "b", 2, 5, 3, true, false));
+}
+
+TEST("require that matmul outer/outer works correctly") {
+    TEST_DO(verify_optimized_multi("b5c2", "b5d3", "b", 2, 5, 3, false, false));
+}
+
+TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/eval/src/tests/tensor/dense_replace_type_function/dense_replace_type_function_test.cpp b/eval/src/tests/tensor/dense_replace_type_function/dense_replace_type_function_test.cpp
index 0533b1c92b7..732fc9c3e69 100644
--- a/eval/src/tests/tensor/dense_replace_type_function/dense_replace_type_function_test.cpp
+++ b/eval/src/tests/tensor/dense_replace_type_function/dense_replace_type_function_test.cpp
@@ -23,7 +23,7 @@ struct ChildMock : Leaf {
     bool is_mutable;
     ChildMock(const ValueType &type) : Leaf(type), is_mutable(true) {}
     bool result_is_mutable() const override { return is_mutable; }
-    InterpretedFunction::Instruction compile_self(Stash &) const override { abort(); }
+    InterpretedFunction::Instruction compile_self(const TensorEngine &, Stash &) const override { abort(); }
 };
 
 struct Fixture {
@@ -43,7 +43,7 @@ struct Fixture {
     {
         my_fun.push_children(children);
         state.stack.push_back(*my_value);
-        my_fun.compile_self(state.stash).perform(state);
+        my_fun.compile_self(engine, state.stash).perform(state);
         ASSERT_EQUAL(children.size(), 1u);
         ASSERT_EQUAL(state.stack.size(), 1u);
         ASSERT_TRUE(!new_type.is_error());
diff --git a/eval/src/tests/tensor/dense_tensor_peek_function/dense_tensor_peek_function_test.cpp b/eval/src/tests/tensor/dense_tensor_peek_function/dense_tensor_peek_function_test.cpp
index a3afb710d1a..5bbdfeaf543 100644
--- a/eval/src/tests/tensor/dense_tensor_peek_function/dense_tensor_peek_function_test.cpp
+++ b/eval/src/tests/tensor/dense_tensor_peek_function/dense_tensor_peek_function_test.cpp
@@ -73,13 +73,13 @@ TEST("require that tensor peek is not optimized for mixed tensor") {
     TEST_DO(verify("xmy2{x:(a),y:(b)}", 0.0, 0, 1));
 }
 
-TEST("require that indexes are rounded to nearest integer") {
-    TEST_DO(verify("x3{x:(a-0.3)}", 2.0, 1, 0));
+TEST("require that indexes are truncated when converted to integers") {
+    TEST_DO(verify("x3{x:(a+0.7)}", 2.0, 1, 0));
     TEST_DO(verify("x3{x:(a+0.3)}", 2.0, 1, 0));
-    TEST_DO(verify("xm{x:(a-0.3)}", 1.0, 0, 1));
+    TEST_DO(verify("xm{x:(a+0.7)}", 1.0, 0, 1));
     TEST_DO(verify("xm{x:(a+0.3)}", 1.0, 0, 1));
+    TEST_DO(verify("xm{x:(-a-0.7)}", 4.0, 0, 1));
     TEST_DO(verify("xm{x:(-a-0.3)}", 4.0, 0, 1));
-    TEST_DO(verify("xm{x:(-a+0.3)}", 4.0, 0, 1));
 }
 
 TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/eval/src/tests/tensor/dense_xw_product_function/dense_xw_product_function_test.cpp b/eval/src/tests/tensor/dense_xw_product_function/dense_xw_product_function_test.cpp
index 426281686d7..0b924451907 100644
--- a/eval/src/tests/tensor/dense_xw_product_function/dense_xw_product_function_test.cpp
+++ b/eval/src/tests/tensor/dense_xw_product_function/dense_xw_product_function_test.cpp
@@ -1,8 +1,5 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
-#include <vespa/log/log.h>
-LOG_SETUP("dense_dot_product_function_test");
-
 #include <vespa/vespalib/testkit/test_kit.h>
 #include <vespa/eval/eval/tensor_function.h>
 #include <vespa/eval/eval/operation.h>
@@ -26,6 +23,12 @@ using namespace vespalib::eval::tensor_function;
 
 const TensorEngine &prod_engine = DefaultTensorEngine::ref();
 
+struct First {
+    bool value;
+    explicit First(bool value_in) : value(value_in) {}
+    operator bool() const { return value; }
+};
+
 struct MyVecSeq : Sequence {
     double operator[](size_t i) const override { return (3.0 + i) * 7.0; }
 };
@@ -34,65 +37,100 @@ struct MyMatSeq : Sequence {
     double operator[](size_t i) const override { return (5.0 + i) * 43.0; }
 };
 
+void add_vector(EvalFixture::ParamRepo &repo, const char *d1, size_t s1) {
+    auto name = make_string("%s%zu", d1, s1);
+    auto layout = Layout({{d1, s1}});
+    repo.add(name, spec(layout, MyVecSeq()));
+    repo.add(name + "f", spec(float_cells(layout), MyVecSeq()));
+}
+
+void add_matrix(EvalFixture::ParamRepo &repo, const char *d1, size_t s1, const char *d2, size_t s2) {
+    auto name = make_string("%s%zu%s%zu", d1, s1, d2, s2);
+    auto layout = Layout({{d1, s1}, {d2, s2}});
+    repo.add(name, spec(layout, MyMatSeq()));
+    repo.add(name + "f", spec(float_cells(layout), MyMatSeq()));
+}
+
 EvalFixture::ParamRepo make_params() {
-    return EvalFixture::ParamRepo()
-        .add("y1", spec({y(1)}, MyVecSeq()))
-        .add("y3", spec({y(3)}, MyVecSeq()))
-        .add("y3f", spec(float_cells({y(3)}), MyVecSeq()))
-        .add("y5", spec({y(5)}, MyVecSeq()))
-        .add("y16", spec({y(16)}, MyVecSeq()))
-        .add("x1y1", spec({x(1),y(1)}, MyMatSeq()))
-        .add("y1z1", spec({y(1),z(1)}, MyMatSeq()))
-        .add("x2y3", spec({x(2),y(3)}, MyMatSeq()))
-        .add("x2y3f", spec(float_cells({x(2),y(3)}), MyMatSeq()))
-        .add("y3z2f", spec(float_cells({y(3),z(2)}), MyMatSeq()))
-        .add("x2z3", spec({x(2),z(3)}, MyMatSeq()))
-        .add("y3z2", spec({y(3),z(2)}, MyMatSeq()))
-        .add("x8y5", spec({x(8),y(5)}, MyMatSeq()))
-        .add("y5z8", spec({y(5),z(8)}, MyMatSeq()))
-        .add("x5y16", spec({x(5),y(16)}, MyMatSeq()))
-        .add("y16z5", spec({y(16),z(5)}, MyMatSeq()));
+    EvalFixture::ParamRepo repo;
+    add_vector(repo, "y",  1);
+    add_vector(repo, "y",  3);
+    add_vector(repo, "y",  5);
+    add_vector(repo, "y", 16);
+    add_matrix(repo, "x",  1, "y",  1);
+    add_matrix(repo, "y",  1, "z",  1);
+    add_matrix(repo, "x",  2, "y",  3);
+    add_matrix(repo, "y",  3, "z",  2);
+    add_matrix(repo, "x",  2, "z",  3);
+    add_matrix(repo, "x",  8, "y",  5);
+    add_matrix(repo, "y",  5, "z",  8);
+    add_matrix(repo, "x",  5, "y", 16);
+    add_matrix(repo, "y", 16, "z",  5);
+    return repo;
 }
 EvalFixture::ParamRepo param_repo = make_params();
 
 void verify_optimized(const vespalib::string &expr, size_t vec_size, size_t res_size, bool happy) {
+    EvalFixture slow_fixture(prod_engine, expr, param_repo, false);
     EvalFixture fixture(prod_engine, expr, param_repo, true);
     EXPECT_EQUAL(fixture.result(), EvalFixture::ref(expr, param_repo));
+    EXPECT_EQUAL(fixture.result(), slow_fixture.result());
     auto info = fixture.find_all<DenseXWProductFunction>();
     ASSERT_EQUAL(info.size(), 1u);
     EXPECT_TRUE(info[0]->result_is_mutable());
-    EXPECT_EQUAL(info[0]->vectorSize(), vec_size);
-    EXPECT_EQUAL(info[0]->resultSize(), res_size);
-    EXPECT_EQUAL(info[0]->matrixHasCommonDimensionInnermost(), happy);
+    EXPECT_EQUAL(info[0]->vector_size(), vec_size);
+    EXPECT_EQUAL(info[0]->result_size(), res_size);
+    EXPECT_EQUAL(info[0]->common_inner(), happy);
+}
+
+vespalib::string make_expr(const vespalib::string &a, const vespalib::string &b, const vespalib::string &common,
+                           bool float_a, bool float_b)
+{
+    return make_string("reduce(%s%s*%s%s,sum,%s)", a.c_str(), float_a ? "f" : "", b.c_str(), float_b ? "f" : "", common.c_str());
+}
+
+void verify_optimized_multi(const vespalib::string &a, const vespalib::string &b, const vespalib::string &common,
+                            size_t vec_size, size_t res_size, bool happy, First first = First(true))
+{
+    for (bool float_a: {false, true}) {
+        for (bool float_b: {false, true}) {
+            auto expr = make_expr(a, b, common, float_a, float_b);
+            TEST_STATE(expr.c_str());
+            TEST_DO(verify_optimized(expr, vec_size, res_size, happy));
+        }
+    }
+    if (first) {
+        TEST_DO(verify_optimized_multi(b, a, common, vec_size, res_size, happy, First(false)));
+    }
 }
 
 void verify_not_optimized(const vespalib::string &expr) {
+    EvalFixture slow_fixture(prod_engine, expr, param_repo, false);
     EvalFixture fixture(prod_engine, expr, param_repo, true);
     EXPECT_EQUAL(fixture.result(), EvalFixture::ref(expr, param_repo));
+    EXPECT_EQUAL(fixture.result(), slow_fixture.result());
     auto info = fixture.find_all<DenseXWProductFunction>();
     EXPECT_TRUE(info.empty());
 }
 
 TEST("require that xw product gives same results as reference join/reduce") {
     // 1 -> 1 happy/unhappy
-    TEST_DO(verify_optimized("reduce(y1*x1y1,sum,y)", 1, 1, true));
-    TEST_DO(verify_optimized("reduce(y1*y1z1,sum,y)", 1, 1, false));
+    TEST_DO(verify_optimized_multi("y1", "x1y1", "y", 1, 1, true));
+    TEST_DO(verify_optimized_multi("y1", "y1z1", "y", 1, 1, false));
     // 3 -> 2 happy/unhappy
-    TEST_DO(verify_optimized("reduce(y3*x2y3,sum,y)", 3, 2, true));
-    TEST_DO(verify_optimized("reduce(y3*y3z2,sum,y)", 3, 2, false));
+    TEST_DO(verify_optimized_multi("y3", "x2y3", "y", 3, 2, true));
+    TEST_DO(verify_optimized_multi("y3", "y3z2", "y", 3, 2, false));
     // 5 -> 8 happy/unhappy
-    TEST_DO(verify_optimized("reduce(y5*x8y5,sum,y)", 5, 8, true));
-    TEST_DO(verify_optimized("reduce(y5*y5z8,sum,y)", 5, 8, false));
+    TEST_DO(verify_optimized_multi("y5", "x8y5", "y", 5, 8, true));
+    TEST_DO(verify_optimized_multi("y5", "y5z8", "y", 5, 8, false));
     // 16 -> 5 happy/unhappy
-    TEST_DO(verify_optimized("reduce(y16*x5y16,sum,y)", 16, 5, true));
-    TEST_DO(verify_optimized("reduce(y16*y16z5,sum,y)", 16, 5, false));
+    TEST_DO(verify_optimized_multi("y16", "x5y16", "y", 16, 5, true));
+    TEST_DO(verify_optimized_multi("y16", "y16z5", "y", 16, 5, false));
 }
 
 TEST("require that various variants of xw product can be optimized") {
-    TEST_DO(verify_optimized("reduce(y3*x2y3,sum,y)", 3, 2, true));
-    TEST_DO(verify_optimized("reduce(x2y3*y3,sum,y)", 3, 2, true));
     TEST_DO(verify_optimized("reduce(join(y3,x2y3,f(x,y)(x*y)),sum,y)", 3, 2, true));
-    TEST_DO(verify_optimized("reduce(join(x2y3,y3,f(x,y)(x*y)),sum,y)", 3, 2, true));
+    TEST_DO(verify_optimized("reduce(join(y3,x2y3,f(x,y)(y*x)),sum,y)", 3, 2, true));
 }
 
 TEST("require that expressions similar to xw product are not optimized") {
@@ -100,13 +138,9 @@ TEST("require that expressions similar to xw product are not optimized") {
     TEST_DO(verify_not_optimized("reduce(y3*x2y3,prod,y)"));
     TEST_DO(verify_not_optimized("reduce(y3*x2y3,sum)"));
     TEST_DO(verify_not_optimized("reduce(join(y3,x2y3,f(x,y)(x+y)),sum,y)"));
-    // TEST_DO(verify_not_optimized("reduce(join(y3,x2y3,f(x,y)(y*x)),sum,y)"));
     TEST_DO(verify_not_optimized("reduce(join(y3,x2y3,f(x,y)(x*x)),sum,y)"));
     TEST_DO(verify_not_optimized("reduce(join(y3,x2y3,f(x,y)(y*y)),sum,y)"));
     TEST_DO(verify_not_optimized("reduce(join(y3,x2y3,f(x,y)(y*x*1)),sum,y)"));
-}
-
-TEST("require that xw products with incompatible dimensions are not optimized") {
     TEST_DO(verify_not_optimized("reduce(y3*x2z3,sum,y)"));
     TEST_DO(verify_not_optimized("reduce(y3*x2z3,sum,z)"));
 }
@@ -119,16 +153,4 @@ TEST("require that xw product can be debug dumped") {
     fprintf(stderr, "%s\n", info[0]->as_string().c_str());
 }
 
-TEST("require that optimization works for float cells") {
-    TEST_DO(verify_optimized("reduce(y3f*x2y3,sum,y)", 3, 2, true));
-    TEST_DO(verify_optimized("reduce(y3*x2y3f,sum,y)", 3, 2, true));
-    TEST_DO(verify_optimized("reduce(y3f*x2y3f,sum,y)", 3, 2, true));
-}
-
-TEST("require that optimization works for float cells with inconvenient dimension nesting") {
-    TEST_DO(verify_optimized("reduce(y3f*y3z2,sum,y)", 3, 2, false));
-    TEST_DO(verify_optimized("reduce(y3*y3z2f,sum,y)", 3, 2, false));
-    TEST_DO(verify_optimized("reduce(y3f*y3z2f,sum,y)", 3, 2, false));
-}
-
 TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/eval/src/vespa/eval/CMakeLists.txt b/eval/src/vespa/eval/CMakeLists.txt
index 90972de7c80..c28643e605e 100644
--- a/eval/src/vespa/eval/CMakeLists.txt
+++ b/eval/src/vespa/eval/CMakeLists.txt
@@ -14,3 +14,6 @@ vespa_add_library(vespaeval
     DEPENDS
     ${VESPA_LLVM_LIB}
 )
+
+set(BLA_VENDOR OpenBLAS)
+vespa_add_target_package_dependency(vespaeval BLAS)
diff --git a/eval/src/vespa/eval/eval/basic_nodes.cpp b/eval/src/vespa/eval/eval/basic_nodes.cpp
index 1c46f2ee322..bc7202bebc1 100644
--- a/eval/src/vespa/eval/eval/basic_nodes.cpp
+++ b/eval/src/vespa/eval/eval/basic_nodes.cpp
@@ -20,18 +20,12 @@ struct Frame {
     const Node &next_child() { return node.get_child(child_idx++); }
 };
 
-struct NoParams : LazyParams {
-    const Value &resolve(size_t, Stash &) const override {
-        abort();
-    }
-};
-
 } // namespace vespalib::eval::nodes::<unnamed>
 
 double
 Node::get_const_value() const {
     assert(is_const());
-    InterpretedFunction function(SimpleTensorEngine::ref(), *this, 0, NodeTypes());
+    InterpretedFunction function(SimpleTensorEngine::ref(), *this, NodeTypes());
     NoParams no_params;
     InterpretedFunction::Context ctx(function);
     return function.eval(ctx, no_params).as_double();
diff --git a/eval/src/vespa/eval/eval/compile_tensor_function.cpp b/eval/src/vespa/eval/eval/compile_tensor_function.cpp
index ac36720895f..18ef59507d8 100644
--- a/eval/src/vespa/eval/eval/compile_tensor_function.cpp
+++ b/eval/src/vespa/eval/eval/compile_tensor_function.cpp
@@ -32,10 +32,11 @@ struct Frame {
 };
 
 struct ProgramCompiler {
+    const TensorEngine &engine;
     Stash &stash;
     std::vector<Frame> stack;
     std::vector<Instruction> prog;
-    ProgramCompiler(Stash &stash_in) : stash(stash_in), stack(), prog() {}
+    ProgramCompiler(const TensorEngine &engine_in, Stash &stash_in) : engine(engine_in), stash(stash_in), stack(), prog() {}
 
     void append(const std::vector<Instruction> &other_prog) {
         prog.insert(prog.end(), other_prog.begin(), other_prog.end());
@@ -43,9 +44,9 @@ struct ProgramCompiler {
 
     void open(const TensorFunction &node) {
         if (auto if_node = as<tensor_function::If>(node)) {
-            append(compile_tensor_function(if_node->cond(), stash));
-            auto true_prog = compile_tensor_function(if_node->true_child(), stash);
-            auto false_prog = compile_tensor_function(if_node->false_child(), stash);
+            append(compile_tensor_function(engine, if_node->cond(), stash));
+            auto true_prog = compile_tensor_function(engine, if_node->true_child(), stash);
+            auto false_prog = compile_tensor_function(engine, if_node->false_child(), stash);
             true_prog.emplace_back(op_skip, false_prog.size());
             prog.emplace_back(op_skip_if_false, true_prog.size());
             append(true_prog);
@@ -56,7 +57,7 @@ struct ProgramCompiler {
     }
 
     void close(const TensorFunction &node) {
-        prog.push_back(node.compile_self(stash));
+        prog.push_back(node.compile_self(engine, stash));
     }
 
     std::vector<Instruction> compile(const TensorFunction &function) {
@@ -75,8 +76,8 @@ struct ProgramCompiler {
 
 } // namespace vespalib::eval::<unnamed>
 
-std::vector<Instruction> compile_tensor_function(const TensorFunction &function, Stash &stash) {   
-    ProgramCompiler compiler(stash);
+std::vector<Instruction> compile_tensor_function(const TensorEngine &engine, const TensorFunction &function, Stash &stash) {
+    ProgramCompiler compiler(engine, stash);
     return compiler.compile(function);
 }
 
diff --git a/eval/src/vespa/eval/eval/compile_tensor_function.h b/eval/src/vespa/eval/eval/compile_tensor_function.h
index 013d228c2f9..63f00fde053 100644
--- a/eval/src/vespa/eval/eval/compile_tensor_function.h
+++ b/eval/src/vespa/eval/eval/compile_tensor_function.h
@@ -10,7 +10,8 @@ namespace vespalib { class Stash; }
 namespace vespalib::eval {
 
 struct TensorFunction;
+struct TensorEngine;
 
-std::vector<InterpretedFunction::Instruction> compile_tensor_function(const TensorFunction &function, Stash &stash);
+std::vector<InterpretedFunction::Instruction> compile_tensor_function(const TensorEngine &engine, const TensorFunction &function, Stash &stash);
 
 } // namespace vespalib::eval
diff --git a/eval/src/vespa/eval/eval/function.cpp b/eval/src/vespa/eval/eval/function.cpp
index 53107fefb32..5d4d597bbf8 100644
--- a/eval/src/vespa/eval/eval/function.cpp
+++ b/eval/src/vespa/eval/eval/function.cpp
@@ -31,17 +31,6 @@ bool has_duplicates(const std::vector<vespalib::string> &list) {
     return false;
 }
 
-bool step_labels(std::vector<size_t> &labels, const ValueType &type) {
-    for (size_t idx = labels.size(); idx-- > 0; ) {
-        if (++labels[idx] < type.dimensions()[idx].size) {
-            return true;
-        } else {
-            labels[idx] = 0;
-        }
-    }
-    return false;
-}
-
 //-----------------------------------------------------------------------------
 
 class Params {
@@ -89,6 +78,13 @@ struct ExplicitParams : Params {
 };
 
 struct ImplicitParams : Params {
+    ImplicitParams() = default;
+    explicit ImplicitParams(const std::vector<vespalib::string> &params_in) {
+        for (const auto &param: params_in) {
+            assert(lookup(param) == UNDEF);
+            lookup_add(param);
+        }
+    }
     bool implicit() const override { return true; }
     size_t resolve(const vespalib::string &token) const override {
         return const_cast<ImplicitParams*>(this)->lookup_add(token);
@@ -100,10 +96,8 @@ struct ImplicitParams : Params {
 struct ResolveContext {
     const Params &params;
     const SymbolExtractor *symbol_extractor;
-    const std::map<vespalib::string,size_t*> *aliases;
-    ResolveContext(const Params &params_in, const SymbolExtractor *symbol_extractor_in,
-                   const std::map<vespalib::string,size_t*> *aliases_in)
-        : params(params_in), symbol_extractor(symbol_extractor_in), aliases(aliases_in) {}
+    ResolveContext(const Params &params_in, const SymbolExtractor *symbol_extractor_in)
+        : params(params_in), symbol_extractor(symbol_extractor_in) {}
 };
 
 class ParseContext
@@ -127,7 +121,7 @@ public:
           _scratch(), _failure(),
           _expression_stack(), _operator_stack(),
           _operator_mark(0),
-          _resolve_stack({ResolveContext(params, symbol_extractor, nullptr)})
+          _resolve_stack({ResolveContext(params, symbol_extractor)})
     {
         if (_pos < _end) {
             _curr = *_pos;
@@ -151,12 +145,11 @@ public:
     }
 
     void push_resolve_context(const Params &params) {
-        _resolve_stack.emplace_back(params, nullptr, nullptr);
-    }
-
-    void push_resolve_context(const std::map<vespalib::string,size_t*> &aliases) {
-        assert(!_resolve_stack.empty());
-        _resolve_stack.emplace_back(resolver().params, resolver().symbol_extractor, &aliases);
+        if (params.implicit()) {
+            _resolve_stack.emplace_back(params, resolver().symbol_extractor);
+        } else {
+            _resolve_stack.emplace_back(params, nullptr);
+        }
     }
 
     void pop_resolve_context() {
@@ -165,21 +158,6 @@ public:
         assert(!_resolve_stack.empty());
     }
 
-    bool has_alias(const vespalib::string &ident) const {
-        if (auto aliases = resolver().aliases) {
-            return (aliases->find(ident) != aliases->end());
-        }
-        return false;
-    }
-
-    size_t get_alias_value(const vespalib::string &ident) const {
-        auto aliases = resolver().aliases;
-        assert(aliases);
-        auto pos = aliases->find(ident);
-        assert(pos != aliases->end());
-        return *(pos->second);
-    }
-
     void fail(const vespalib::string &msg) {
         if (_failure.empty()) {
             _failure = msg;
@@ -761,30 +739,25 @@ void parse_tensor_create(ParseContext &ctx, const ValueType &type,
 }
 
 void parse_tensor_lambda(ParseContext &ctx, const ValueType &type) {
+    ImplicitParams params(type.dimension_names());
+    ctx.push_resolve_context(params);
     ctx.skip_spaces();
     ctx.eat('(');
-    ParseContext::InputMark before_expr = ctx.get_input_mark();
-    std::vector<size_t> params(type.dimensions().size(), 0);
-    std::map<vespalib::string,size_t*> my_aliases;
-    if (auto parent_aliases = ctx.resolver().aliases) {
-        my_aliases = *parent_aliases;
-    }
-    for (size_t i = 0; i < params.size(); ++i) {
-        my_aliases.emplace(type.dimensions()[i].name, &params[i]);
-    }
-    ctx.push_resolve_context(my_aliases);
-    nodes::TensorCreate::Spec create_spec;
-    do {
-        ctx.restore_input_mark(before_expr);
-        TensorSpec::Address address;
-        for (size_t i = 0; i < params.size(); ++i) {
-            address.emplace(type.dimensions()[i].name, params[i]);
-        }
-        create_spec.emplace(std::move(address), get_expression(ctx));
-    } while (!ctx.failed() && step_labels(params, type));
+    Node_UP lambda_root = get_expression(ctx);
     ctx.eat(')');
+    ctx.skip_spaces();
     ctx.pop_resolve_context();
-    ctx.push_expression(std::make_unique<nodes::TensorCreate>(type, std::move(create_spec)));
+    auto param_names = params.extract();
+    std::vector<size_t> bindings;
+    for (size_t i = type.dimensions().size(); i < param_names.size(); ++i) {
+        size_t id = ctx.resolve_parameter(param_names[i]);
+        if (id == Params::UNDEF) {
+            return ctx.fail(make_string("unable to resolve: '%s'", param_names[i].c_str()));
+        }
+        bindings.push_back(id);
+    }
+    auto function = Function::create(std::move(lambda_root), std::move(param_names));
+    ctx.push_expression(std::make_unique<nodes::TensorLambda>(type, std::move(bindings), std::move(function)));
 }
 
 bool maybe_parse_tensor_generator(ParseContext &ctx) {
@@ -831,7 +804,7 @@ void parse_tensor_peek(ParseContext &ctx) {
         if (ctx.get() == '(') {
             auto expr = get_expression(ctx);
             if (auto num = nodes::as<nodes::Number>(*expr)) {
-                peek_spec.emplace(dim_name, make_string("%" PRId64, int64_t(round(num->value()))));
+                peek_spec.emplace(dim_name, make_string("%" PRId64, int64_t(num->value())));
             } else {
                 peek_spec.emplace(dim_name, std::move(expr));
             }
@@ -896,18 +869,14 @@ void parse_symbol_or_call(ParseContext &ctx) {
     bool was_tensor_generate = ((name == "tensor") && maybe_parse_tensor_generator(ctx));
     if (!was_tensor_generate && !maybe_parse_call(ctx, name)) {
         ctx.extract_symbol(name, before_name);
-        if (ctx.has_alias(name)) {
-            ctx.push_expression(Node_UP(new nodes::Number(ctx.get_alias_value(name))));
+        if (name.empty()) {
+            ctx.fail("missing value");
         } else {
-            if (name.empty()) {
-                ctx.fail("missing value");
+            size_t id = ctx.resolve_parameter(name);
+            if (id == Params::UNDEF) {
+                ctx.fail(make_string("unknown symbol: '%s'", name.c_str()));
             } else {
-                size_t id = ctx.resolve_parameter(name);
-                if (id == Params::UNDEF) {
-                    ctx.fail(make_string("unknown symbol: '%s'", name.c_str()));
-                } else {
-                    ctx.push_expression(Node_UP(new nodes::Symbol(id)));
-                }
+                ctx.push_expression(Node_UP(new nodes::Symbol(id)));
             }
         }
     }
diff --git a/eval/src/vespa/eval/eval/interpreted_function.cpp b/eval/src/vespa/eval/eval/interpreted_function.cpp
index 0a630a3e20a..121db4ffb6e 100644
--- a/eval/src/vespa/eval/eval/interpreted_function.cpp
+++ b/eval/src/vespa/eval/eval/interpreted_function.cpp
@@ -3,21 +3,16 @@
 #include "interpreted_function.h"
 #include "node_visitor.h"
 #include "node_traverser.h"
-#include "check_type.h"
-#include "tensor_spec.h"
-#include "operation.h"
 #include "tensor_nodes.h"
 #include "tensor_engine.h"
+#include "make_tensor_function.h"
+#include "compile_tensor_function.h"
 #include <vespa/vespalib/util/classname.h>
 #include <vespa/eval/eval/llvm/compile_cache.h>
 #include <vespa/vespalib/util/benchmark_timer.h>
 #include <set>
 
-#include "make_tensor_function.h"
-#include "compile_tensor_function.h"
-
-namespace vespalib {
-namespace eval {
+namespace vespalib::eval {
 
 namespace {
 
@@ -42,11 +37,12 @@ InterpretedFunction::State::State(const TensorEngine &engine_in)
       params(nullptr),
       stash(),
       stack(),
-      program_offset(0)
+      program_offset(0),
+      if_cnt(0)
 {
 }
 
-InterpretedFunction::State::~State() {}
+InterpretedFunction::State::~State() = default;
 
 void
 InterpretedFunction::State::init(const LazyParams &params_in) {
@@ -65,24 +61,22 @@ InterpretedFunction::Context::Context(const InterpretedFunction &ifun)
 InterpretedFunction::InterpretedFunction(const TensorEngine &engine, const TensorFunction &function)
     : _program(),
       _stash(),
-      _num_params(0),
       _tensor_engine(engine)
 {
-    _program = compile_tensor_function(function, _stash);
+    _program = compile_tensor_function(engine, function, _stash);
 }
 
-InterpretedFunction::InterpretedFunction(const TensorEngine &engine, const nodes::Node &root, size_t num_params_in, const NodeTypes &types)
+InterpretedFunction::InterpretedFunction(const TensorEngine &engine, const nodes::Node &root, const NodeTypes &types)
     : _program(),
       _stash(),
-      _num_params(num_params_in),
       _tensor_engine(engine)
 {
     const TensorFunction &plain_fun = make_tensor_function(engine, root, types, _stash);
     const TensorFunction &optimized = engine.optimize(plain_fun, _stash);
-    _program = compile_tensor_function(optimized, _stash);
+    _program = compile_tensor_function(engine, optimized, _stash);
 }
 
-InterpretedFunction::~InterpretedFunction() {}
+InterpretedFunction::~InterpretedFunction() = default;
 
 const Value &
 InterpretedFunction::eval(Context &ctx, const LazyParams &params) const
@@ -123,5 +117,4 @@ InterpretedFunction::detect_issues(const Function &function)
     return Function::Issues(std::move(checker.issues));
 }
 
-} // namespace vespalib::eval
-} // namespace vespalib
+}
diff --git a/eval/src/vespa/eval/eval/interpreted_function.h b/eval/src/vespa/eval/eval/interpreted_function.h
index e3e8d18b44f..fb67fcb0b74 100644
--- a/eval/src/vespa/eval/eval/interpreted_function.h
+++ b/eval/src/vespa/eval/eval/interpreted_function.h
@@ -7,8 +7,7 @@
 #include "lazy_params.h"
 #include <vespa/vespalib/util/stash.h>
 
-namespace vespalib {
-namespace eval {
+namespace vespalib::eval {
 
 namespace nodes { struct Node; }
 struct TensorEngine;
@@ -88,24 +87,21 @@ public:
 private:
     std::vector<Instruction> _program;
     Stash                    _stash;
-    size_t                   _num_params;
     const TensorEngine      &_tensor_engine;
 
 public:
     typedef std::unique_ptr<InterpretedFunction> UP;
     // for testing; use with care; the tensor function must be kept alive
     InterpretedFunction(const TensorEngine &engine, const TensorFunction &function);
-    InterpretedFunction(const TensorEngine &engine, const nodes::Node &root, size_t num_params_in, const NodeTypes &types);
+    InterpretedFunction(const TensorEngine &engine, const nodes::Node &root, const NodeTypes &types);
     InterpretedFunction(const TensorEngine &engine, const Function &function, const NodeTypes &types)
-        : InterpretedFunction(engine, function.root(), function.num_params(), types) {}
+        : InterpretedFunction(engine, function.root(), types) {}
     InterpretedFunction(InterpretedFunction &&rhs) = default;
     ~InterpretedFunction();
     size_t program_size() const { return _program.size(); }
-    size_t num_params() const { return _num_params; }
     const Value &eval(Context &ctx, const LazyParams &params) const;
     double estimate_cost_us(const std::vector<double> &params, double budget = 5.0) const;
     static Function::Issues detect_issues(const Function &function);
 };
 
-} // namespace vespalib::eval
-} // namespace vespalib
+}
diff --git a/eval/src/vespa/eval/eval/key_gen.cpp b/eval/src/vespa/eval/eval/key_gen.cpp
index 46137b5878c..cd31f92f96a 100644
--- a/eval/src/vespa/eval/eval/key_gen.cpp
+++ b/eval/src/vespa/eval/eval/key_gen.cpp
@@ -42,7 +42,8 @@ struct KeyGen : public NodeVisitor, public NodeTraverser {
     void visit(const TensorRename &) override { add_byte(14); } // dimensions should be part of key
     void visit(const TensorConcat &) override { add_byte(15); } // dimension should be part of key
     void visit(const TensorCreate &) override { add_byte(16); } // type/addr should be part of key
-    void visit(const TensorPeek   &) override { add_byte(17); } // addr should be part of key
+    void visit(const TensorLambda &) override { add_byte(17); } // type/lambda should be part of key
+    void visit(const TensorPeek   &) override { add_byte(18); } // addr should be part of key
     void visit(const Add          &) override { add_byte(20); }
     void visit(const Sub          &) override { add_byte(21); }
     void visit(const Mul          &) override { add_byte(22); }
diff --git a/eval/src/vespa/eval/eval/lazy_params.cpp b/eval/src/vespa/eval/eval/lazy_params.cpp
index aec8cf78059..2c00c4c312b 100644
--- a/eval/src/vespa/eval/eval/lazy_params.cpp
+++ b/eval/src/vespa/eval/eval/lazy_params.cpp
@@ -1,19 +1,16 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include "lazy_params.h"
-#include <assert.h>
+#include <vespa/vespalib/util/stash.h>
+#include <cassert>
 
 namespace vespalib::eval {
 
-LazyParams::~LazyParams()
-{
-}
+LazyParams::~LazyParams() = default;
 
 //-----------------------------------------------------------------------------
 
-SimpleObjectParams::~SimpleObjectParams()
-{
-}
+SimpleObjectParams::~SimpleObjectParams() = default;
 
 const Value &
 SimpleObjectParams::resolve(size_t idx, Stash &) const
@@ -24,9 +21,7 @@ SimpleObjectParams::resolve(size_t idx, Stash &) const
 
 //-----------------------------------------------------------------------------
 
-SimpleParams::~SimpleParams()
-{
-}
+SimpleParams::~SimpleParams() = default;
 
 const Value &
 SimpleParams::resolve(size_t idx, Stash &stash) const
diff --git a/eval/src/vespa/eval/eval/lazy_params.h b/eval/src/vespa/eval/eval/lazy_params.h
index b6b7753eff9..d75e216571b 100644
--- a/eval/src/vespa/eval/eval/lazy_params.h
+++ b/eval/src/vespa/eval/eval/lazy_params.h
@@ -46,5 +46,14 @@ struct SimpleParams : LazyParams {
     const Value &resolve(size_t idx, Stash &stash) const override;
 };
 
+/**
+ * Simple wrapper for cases where you have no parameters.
+ **/
+struct NoParams : LazyParams {
+    const Value &resolve(size_t, Stash &) const override {
+        abort();
+    }
+};
+
 } // namespace vespalib::eval
 } // namespace vespalib
diff --git a/eval/src/vespa/eval/eval/llvm/compile_cache.h b/eval/src/vespa/eval/eval/llvm/compile_cache.h
index 65cec9c0d48..aaadec772a5 100644
--- a/eval/src/vespa/eval/eval/llvm/compile_cache.h
+++ b/eval/src/vespa/eval/eval/llvm/compile_cache.h
@@ -5,6 +5,7 @@
 #include "compiled_function.h"
 #include <vespa/vespalib/util/executor.h>
 #include <condition_variable>
+#include <atomic>
 #include <mutex>
 
 namespace vespalib {
diff --git a/eval/src/vespa/eval/eval/llvm/compiled_function.cpp b/eval/src/vespa/eval/eval/llvm/compiled_function.cpp
index facfc502111..4101cf10e1f 100644
--- a/eval/src/vespa/eval/eval/llvm/compiled_function.cpp
+++ b/eval/src/vespa/eval/eval/llvm/compiled_function.cpp
@@ -133,6 +133,7 @@ CompiledFunction::detect_issues(const Function &function)
                                   nodes::TensorRename,
                                   nodes::TensorConcat,
                                   nodes::TensorCreate,
+                                  nodes::TensorLambda,
                                   nodes::TensorPeek>(node))
             {
                 issues.push_back(make_string("unsupported node type: %s",
diff --git a/eval/src/vespa/eval/eval/llvm/llvm_wrapper.cpp b/eval/src/vespa/eval/eval/llvm/llvm_wrapper.cpp
index 1d5515d7f4a..89f1789e97b 100644
--- a/eval/src/vespa/eval/eval/llvm/llvm_wrapper.cpp
+++ b/eval/src/vespa/eval/eval/llvm/llvm_wrapper.cpp
@@ -12,15 +12,18 @@
 #include <llvm/Analysis/Passes.h>
 #include <llvm/IR/DataLayout.h>
 #include <llvm/Transforms/Scalar.h>
-#if LLVM_VERSION_MAJOR == 9 && defined(__clang__)
+#if LLVM_VERSION_MAJOR >= 9 && defined(__clang__)
 // Avoid reference to undefined symbol llvm::cfg::Update<llvm::BasicBlock*>::dump() const
 #define NDEBUG
 #endif
 #include <llvm/LinkAllPasses.h>
-#if LLVM_VERSION_MAJOR == 9 && defined(__clang__)
+#if LLVM_VERSION_MAJOR >= 9 && defined(__clang__)
 #undef NDEBUG
 #endif
 #include <llvm/Transforms/IPO/PassManagerBuilder.h>
+#if LLVM_VERSION_MAJOR > 9
+#include <llvm/Support/ManagedStatic.h>
+#endif
 #include <vespa/eval/eval/check_type.h>
 #include <vespa/vespalib/stllike/hash_set.h>
 #include <vespa/vespalib/util/approx.h>
@@ -482,6 +485,9 @@ struct FunctionBuilder : public NodeVisitor, public NodeTraverser {
     void visit(const TensorCreate &node) override {
         make_error(node.num_children());
     }
+    void visit(const TensorLambda &node) override {
+        make_error(node.num_children());
+    }
     void visit(const TensorPeek &node) override {
         make_error(node.num_children());
     }
diff --git a/eval/src/vespa/eval/eval/make_tensor_function.cpp b/eval/src/vespa/eval/eval/make_tensor_function.cpp
index 849270b89a7..f503532c1f9 100644
--- a/eval/src/vespa/eval/eval/make_tensor_function.cpp
+++ b/eval/src/vespa/eval/eval/make_tensor_function.cpp
@@ -121,6 +121,17 @@ struct TensorFunctionBuilder : public NodeVisitor, public NodeTraverser {
         stack.push_back(tensor_function::create(node.type(), spec, stash));
     }
 
+    void make_lambda(const TensorLambda &node) {
+        if (node.bindings().empty()) {
+            NoParams no_bound_params;
+            InterpretedFunction my_fun(tensor_engine, node.lambda().root(), types);
+            TensorSpec spec = tensor_function::Lambda::create_spec_impl(node.type(), no_bound_params, node.bindings(), my_fun);
+            make_const(node, *stash.create<Value::UP>(tensor_engine.from_spec(spec)));
+        } else {
+            stack.push_back(tensor_function::lambda(node.type(), node.bindings(), node.lambda(), types.export_types(node.lambda().root()), stash));
+        }
+    }
+
     void make_peek(const TensorPeek &node) {
         assert(stack.size() >= node.num_children());
         const tensor_function::Node &param = stack[stack.size()-node.num_children()];
@@ -221,6 +232,9 @@ struct TensorFunctionBuilder : public NodeVisitor, public NodeTraverser {
     void visit(const TensorCreate &node) override {
         make_create(node);
     }
+    void visit(const TensorLambda &node) override {
+        make_lambda(node);
+    }
     void visit(const TensorPeek &node) override {
         make_peek(node);
     }
diff --git a/eval/src/vespa/eval/eval/node_types.cpp b/eval/src/vespa/eval/eval/node_types.cpp
index bf87628e301..468b9a58655 100644
--- a/eval/src/vespa/eval/eval/node_types.cpp
+++ b/eval/src/vespa/eval/eval/node_types.cpp
@@ -3,6 +3,10 @@
 #include "check_type.h"
 #include "node_traverser.h"
 #include "node_types.h"
+#include <vespa/vespalib/util/stringfmt.h>
+#include <vespa/vespalib/util/classname.h>
+
+using vespalib::make_string_short::fmt;
 
 namespace vespalib::eval {
 namespace nodes {
@@ -13,11 +17,13 @@ class State
 private:
     const std::vector<ValueType>      &_params;
     std::map<const Node *, ValueType> &_type_map;
+    std::vector<vespalib::string>     &_errors;
 
 public:
     State(const std::vector<ValueType> &params,
-          std::map<const Node *, ValueType> &type_map)
-        : _params(params), _type_map(type_map) {}
+          std::map<const Node *, ValueType> &type_map,
+          std::vector<vespalib::string> &errors)
+        : _params(params), _type_map(type_map), _errors(errors) {}
 
     const ValueType &param_type(size_t idx) {
         assert(idx < _params.size());
@@ -33,32 +39,69 @@ public:
         assert(pos != _type_map.end());
         return pos->second;
     }
+    void add_error(const vespalib::string &msg) {
+        _errors.push_back(msg);
+    }
 };
 
 struct TypeResolver : public NodeVisitor, public NodeTraverser {
     State state;
     TypeResolver(const std::vector<ValueType> &params_in,
-                 std::map<const Node *, ValueType> &type_map_out);
+                 std::map<const Node *, ValueType> &type_map_out,
+                 std::vector<vespalib::string> &errors_out);
     ~TypeResolver();
 
     const ValueType &param_type(size_t idx) {
         return state.param_type(idx);
     }
 
-    void bind(const ValueType &type, const Node &node) {
-        state.bind(type, node);
+    void fail(const Node &node, const vespalib::string &msg, bool child_types = true) {
+        auto str = fmt("%s: %s", vespalib::getClassName(node).c_str(), msg.c_str());
+        if (child_types) {
+            str += ", child types: [";
+            for (size_t i = 0; i < node.num_children(); ++i) {
+                if (i > 0) {
+                    str += ", ";
+                }
+                str += state.type(node.get_child(i)).to_spec();
+            }
+            str += "]";
+        }
+        state.add_error(str);
+        state.bind(ValueType::error_type(), node);
+    }
+
+    void bind(const ValueType &type, const Node &node, bool check_error = true) {
+        if (check_error && type.is_error()) {
+            fail(node, "type resolving failed");
+        } else {
+            state.bind(type, node);
+        }
     }
 
     const ValueType &type(const Node &node) {
         return state.type(node);
     }
 
+    void import_errors(const NodeTypes &types) {
+        for (const auto &err: types.errors()) {
+            state.add_error(fmt("[lambda]: %s", err.c_str()));
+        }
+    }
+
+    void import_types(const NodeTypes &types) {
+        types.each([&](const Node &node, const ValueType &type)
+                   {
+                       state.bind(type, node);
+                   });
+    }
+
     //-------------------------------------------------------------------------
 
     bool check_error(const Node &node) {
         for (size_t i = 0; i < node.num_children(); ++i) {
             if (type(node.get_child(i)).is_error()) {
-                bind(ValueType::error_type(), node);
+                bind(ValueType::error_type(), node, false);
                 return true;
             }
         }
@@ -80,7 +123,7 @@ struct TypeResolver : public NodeVisitor, public NodeTraverser {
         bind(ValueType::double_type(), node);
     }
     void visit(const Symbol &node) override {
-        bind(param_type(node.id()), node);
+        bind(param_type(node.id()), node, false);
     }
     void visit(const String &node) override {
         bind(ValueType::double_type(), node);
@@ -93,7 +136,7 @@ struct TypeResolver : public NodeVisitor, public NodeTraverser {
                                type(node.false_expr())), node);
     }
     void visit(const Error &node) override {
-        bind(ValueType::error_type(), node);
+        bind(ValueType::error_type(), node, false);
     }
     void visit(const TensorMap &node) override { resolve_op1(node); }
     void visit(const TensorJoin &node) override { resolve_op2(node); }
@@ -102,12 +145,33 @@ struct TypeResolver : public NodeVisitor, public NodeTraverser {
                               type(node.get_child(1))), node);
     }
     void visit(const TensorReduce &node) override {
-        const ValueType &child = type(node.get_child(0));
-        bind(child.reduce(node.dimensions()), node);
+        auto my_type = type(node.get_child(0)).reduce(node.dimensions());
+        if (my_type.is_error()) {
+            auto str = fmt("aggr: %s, dimensions: [",
+                           AggrNames::name_of(node.aggr())->c_str());
+            size_t i = 0;
+            for (const auto &dimension: node.dimensions()) {
+                if (i++ > 0) {
+                    str += ",";
+                }
+                str += dimension;
+            }
+            str += "]";
+            fail(node, str);
+        } else {
+            bind(my_type, node);
+        }
     }
     void visit(const TensorRename &node) override {
-        const ValueType &child = type(node.get_child(0));
-        bind(child.rename(node.from(), node.to()), node);
+        auto my_type = type(node.get_child(0)).rename(node.from(), node.to());
+        if (my_type.is_error()) {
+            auto str = fmt("%s -> %s",
+                           TensorRename::flatten(node.from()).c_str(),
+                           TensorRename::flatten(node.to()).c_str());
+            fail(node, str);
+        } else {
+            bind(my_type, node);
+        }
     }
     void visit(const TensorConcat &node) override {
         bind(ValueType::concat(type(node.get_child(0)),
@@ -116,11 +180,30 @@ struct TypeResolver : public NodeVisitor, public NodeTraverser {
     void visit(const TensorCreate &node) override {
         for (size_t i = 0; i < node.num_children(); ++i) {
             if (!type(node.get_child(i)).is_double()) {
-                return bind(ValueType::error_type(), node);
+                return fail(node, fmt("non-double child at index %zu", i), false);
             }
         }
         bind(node.type(), node);
     }
+    void visit(const TensorLambda &node) override {
+        std::vector<ValueType> arg_types;
+        for (const auto &dim: node.type().dimensions()) {
+            (void) dim;
+            arg_types.push_back(ValueType::double_type());
+        }
+        for (size_t binding: node.bindings()) {
+            arg_types.push_back(param_type(binding));
+        }
+        NodeTypes lambda_types(node.lambda(), arg_types);
+        const ValueType &lambda_type = lambda_types.get_type(node.lambda().root());
+        if (!lambda_type.is_double()) {
+            import_errors(lambda_types);
+            return fail(node, fmt("lambda function has non-double result type: %s",
+                                  lambda_type.to_spec().c_str()), false);
+        }
+        import_types(lambda_types);
+        bind(node.type(), node);
+    }
     void visit(const TensorPeek &node) override {
         const ValueType &param_type = type(node.param());
         std::vector<vespalib::string> dimensions;
@@ -128,20 +211,22 @@ struct TypeResolver : public NodeVisitor, public NodeTraverser {
             dimensions.push_back(dim.first);
             if (dim.second.is_expr()) {
                 if (!type(*dim.second.expr).is_double()) {
-                    return bind(ValueType::error_type(), node);
+                    return fail(node, fmt("non-double label expression for dimension %s", dim.first.c_str()));
                 }
             } else {
                 size_t dim_idx = param_type.dimension_index(dim.first);
                 if (dim_idx == ValueType::Dimension::npos) {
-                    return bind(ValueType::error_type(), node);
+                    return fail(node, fmt("dimension not in param: %s", dim.first.c_str()));
                 }
                 const auto &param_dim = param_type.dimensions()[dim_idx];
                 if (param_dim.is_indexed()) {
                     if (!is_number(dim.second.label)) {
-                        return bind(ValueType::error_type(), node);
+                        return fail(node, fmt("non-numeric label for dimension %s: '%s'",
+                                        dim.first.c_str(), dim.second.label.c_str()));
                     }
                     if (as_number(dim.second.label) >= param_dim.size) {
-                        return bind(ValueType::error_type(), node);
+                        return fail(node, fmt("out-of-bounds label for dimension %s: %s",
+                                        dim.first.c_str(), dim.second.label.c_str()));
                     }
                 }
             }
@@ -204,13 +289,34 @@ struct TypeResolver : public NodeVisitor, public NodeTraverser {
 };
 
 TypeResolver::TypeResolver(const std::vector<ValueType> &params_in,
-                           std::map<const Node *, ValueType> &type_map_out)
-    : state(params_in, type_map_out)
+                           std::map<const Node *, ValueType> &type_map_out,
+                           std::vector<vespalib::string> &errors_out)
+    : state(params_in, type_map_out, errors_out)
 {
 }
 
 TypeResolver::~TypeResolver() {}
 
+struct TypeExporter : public NodeTraverser {
+    const std::map<const Node *, ValueType> &parent_type_map;
+    std::map<const Node *, ValueType> &exported_type_map;
+    size_t missing_cnt;
+    TypeExporter(const std::map<const Node *, ValueType> &parent_type_map_in,
+                 std::map<const Node *, ValueType> &exported_type_map_out)
+        : parent_type_map(parent_type_map_in),
+          exported_type_map(exported_type_map_out),
+          missing_cnt(0) {}
+    bool open(const Node &) override { return true; }
+    void close(const Node &node) override {
+        auto pos = parent_type_map.find(&node);
+        if (pos != parent_type_map.end()) {
+            exported_type_map.emplace(&node, pos->second);
+        } else {
+            ++missing_cnt;
+        }
+    }
+};
+
 } // namespace vespalib::eval::nodes::<unnamed>
 } // namespace vespalib::eval::nodes
 
@@ -225,10 +331,24 @@ NodeTypes::NodeTypes(const Function &function, const std::vector<ValueType> &inp
       _type_map()
 {
     assert(input_types.size() == function.num_params());
-    nodes::TypeResolver resolver(input_types, _type_map);
+    nodes::TypeResolver resolver(input_types, _type_map, _errors);
     function.root().traverse(resolver);
 }
 
+NodeTypes::~NodeTypes() = default;
+
+NodeTypes
+NodeTypes::export_types(const nodes::Node &root) const
+{
+    NodeTypes exported_types;
+    nodes::TypeExporter exporter(_type_map, exported_types._type_map);
+    root.traverse(exporter);
+    if (exporter.missing_cnt > 0) {
+        exported_types._errors.push_back(fmt("[export]: %zu nodes had missing types", exporter.missing_cnt));
+    }
+    return exported_types;
+}
+
 const ValueType &
 NodeTypes::get_type(const nodes::Node &node) const
 {
diff --git a/eval/src/vespa/eval/eval/node_types.h b/eval/src/vespa/eval/eval/node_types.h
index a9ddf371c31..72332564409 100644
--- a/eval/src/vespa/eval/eval/node_types.h
+++ b/eval/src/vespa/eval/eval/node_types.h
@@ -23,22 +23,29 @@ class NodeTypes
 private:
     ValueType _not_found;
     std::map<const nodes::Node*,ValueType> _type_map;
+    std::vector<vespalib::string> _errors;
 public:
     NodeTypes();
+    NodeTypes(NodeTypes &&rhs) = default;
+    NodeTypes &operator=(NodeTypes &&rhs) = default;
     NodeTypes(const Function &function, const std::vector<ValueType> &input_types);
+    ~NodeTypes();
+    const std::vector<vespalib::string> &errors() const { return _errors; }
+    NodeTypes export_types(const nodes::Node &root) const;
     const ValueType &get_type(const nodes::Node &node) const;
-    template <typename P>
-    bool check_types(const P &pred) const {
+    template <typename F>
+    void each(F &&f) const {
         for (const auto &entry: _type_map) {
-            if (!pred(entry.second)) {
-                return false;
-            }
+            f(*entry.first, entry.second);
         }
-        return (_type_map.size() > 0);
     }
     bool all_types_are_double() const {
-        return check_types([](const ValueType &type)
-                           { return type.is_double(); });
+        bool all_double = true;
+        each([&all_double](const nodes::Node &, const ValueType &type)
+             {
+                 all_double &= type.is_double();
+             });
+        return (all_double && (_type_map.size() > 0));
     }
 };
 
diff --git a/eval/src/vespa/eval/eval/node_visitor.h b/eval/src/vespa/eval/eval/node_visitor.h
index 8f9722858b7..d3e066c8f53 100644
--- a/eval/src/vespa/eval/eval/node_visitor.h
+++ b/eval/src/vespa/eval/eval/node_visitor.h
@@ -36,6 +36,7 @@ struct NodeVisitor {
     virtual void visit(const nodes::TensorRename &) = 0;
     virtual void visit(const nodes::TensorConcat &) = 0;
     virtual void visit(const nodes::TensorCreate &) = 0;
+    virtual void visit(const nodes::TensorLambda &) = 0;
     virtual void visit(const nodes::TensorPeek   &) = 0;
 
     // operator nodes
@@ -106,6 +107,7 @@ struct EmptyNodeVisitor : NodeVisitor {
     void visit(const nodes::TensorRename &) override {}
     void visit(const nodes::TensorConcat &) override {}
     void visit(const nodes::TensorCreate &) override {}
+    void visit(const nodes::TensorLambda &) override {}
     void visit(const nodes::TensorPeek   &) override {}
     void visit(const nodes::Add          &) override {}
     void visit(const nodes::Sub          &) override {}
diff --git a/eval/src/vespa/eval/eval/tensor_function.cpp b/eval/src/vespa/eval/eval/tensor_function.cpp
index 45e8094570e..2656e240a5b 100644
--- a/eval/src/vespa/eval/eval/tensor_function.cpp
+++ b/eval/src/vespa/eval/eval/tensor_function.cpp
@@ -9,6 +9,7 @@
 #include "visit_stuff.h"
 #include "string_stuff.h"
 #include <vespa/vespalib/objects/objectdumper.h>
+#include <vespa/vespalib/objects/visit.hpp>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".eval.eval.tensor_function");
@@ -133,6 +134,13 @@ void op_tensor_create(State &state, uint64_t param) {
     state.pop_n_push(i, result);
 }
 
+void op_tensor_lambda(State &state, uint64_t param) {
+    const Lambda::Self &self = unwrap_param<Lambda::Self>(param);
+    TensorSpec spec = self.parent.create_spec(*state.params, self.fun);
+    const Value &result = *state.stash.create<Value::UP>(state.engine.from_spec(spec));
+    state.stack.emplace_back(result);
+}
+
 const Value &extract_single_value(const TensorSpec &spec, const TensorSpec::Address &addr, State &state) {
     auto pos = spec.cells().find(addr);
     if (pos == spec.cells().end()) {
@@ -174,7 +182,7 @@ void op_tensor_peek(State &state, uint64_t param) {
                            addr.emplace(pos->first, label);
                        },
                        [&](const TensorFunction::Child &) {
-                           double index = round(state.peek(child_cnt++).as_double());
+                           double index = state.peek(child_cnt++).as_double();
                            size_t dim_idx = self.param_type().dimension_index(pos->first);
                            assert(dim_idx != ValueType::Dimension::npos);
                            const auto &param_dim = self.param_type().dimensions()[dim_idx];
@@ -193,7 +201,7 @@ void op_tensor_peek(State &state, uint64_t param) {
     state.pop_n_push(child_cnt, result);
 }
 
-} // namespace vespalib::eval::tensor_function
+} // namespace vespalib::eval::tensor_function::<unnamed>
 
 //-----------------------------------------------------------------------------
 
@@ -235,7 +243,7 @@ Op2::visit_children(vespalib::ObjectVisitor &visitor) const
 //-----------------------------------------------------------------------------
 
 Instruction
-ConstValue::compile_self(Stash &) const
+ConstValue::compile_self(const TensorEngine &, Stash &) const
 {
     return Instruction(op_load_const, wrap_param<Value>(_value));
 }
@@ -254,7 +262,7 @@ ConstValue::visit_self(vespalib::ObjectVisitor &visitor) const
 //-----------------------------------------------------------------------------
 
 Instruction
-Inject::compile_self(Stash &) const
+Inject::compile_self(const TensorEngine &, Stash &) const
 {
     return Instruction::fetch_param(_param_idx);
 }
@@ -269,7 +277,7 @@ Inject::visit_self(vespalib::ObjectVisitor &visitor) const
 //-----------------------------------------------------------------------------
 
 Instruction
-Reduce::compile_self(Stash &stash) const
+Reduce::compile_self(const TensorEngine &, Stash &stash) const
 {
     ReduceParams &params = stash.create<ReduceParams>(_aggr, _dimensions);
     return Instruction(op_tensor_reduce, wrap_param<ReduceParams>(params));
@@ -286,7 +294,7 @@ Reduce::visit_self(vespalib::ObjectVisitor &visitor) const
 //-----------------------------------------------------------------------------
 
 Instruction
-Map::compile_self(Stash &) const
+Map::compile_self(const TensorEngine &, Stash &) const
 {
     if (result_type().is_double()) {
         return Instruction(op_double_map, to_param(_function));
@@ -304,7 +312,7 @@ Map::visit_self(vespalib::ObjectVisitor &visitor) const
 //-----------------------------------------------------------------------------
 
 Instruction
-Join::compile_self(Stash &) const
+Join::compile_self(const TensorEngine &, Stash &) const
 {
     if (result_type().is_double()) {
         if (_function == operation::Mul::f) {
@@ -328,7 +336,7 @@ Join::visit_self(vespalib::ObjectVisitor &visitor) const
 //-----------------------------------------------------------------------------
 
 Instruction
-Merge::compile_self(Stash &) const
+Merge::compile_self(const TensorEngine &, Stash &) const
 {
     return Instruction(op_tensor_merge, to_param(_function));
 }
@@ -343,7 +351,7 @@ Merge::visit_self(vespalib::ObjectVisitor &visitor) const
 //-----------------------------------------------------------------------------
 
 Instruction
-Concat::compile_self(Stash &) const
+Concat::compile_self(const TensorEngine &, Stash &) const
 {
     return Instruction(op_tensor_concat, wrap_param<vespalib::string>(_dimension));
 }
@@ -366,7 +374,7 @@ Create::push_children(std::vector<Child::CREF> &children) const
 }
 
 Instruction
-Create::compile_self(Stash &) const
+Create::compile_self(const TensorEngine &, Stash &) const
 {
     return Instruction(op_tensor_create, wrap_param<Create>(*this));
 }
@@ -381,6 +389,74 @@ Create::visit_children(vespalib::ObjectVisitor &visitor) const
 
 //-----------------------------------------------------------------------------
 
+namespace {
+
+bool step_labels(std::vector<size_t> &labels, const ValueType &type) {
+    for (size_t idx = labels.size(); idx-- > 0; ) {
+        if (++labels[idx] < type.dimensions()[idx].size) {
+            return true;
+        } else {
+            labels[idx] = 0;
+        }
+    }
+    return false;
+}
+
+struct ParamProxy : public LazyParams {
+    const std::vector<size_t> &labels;
+    const LazyParams          &params;
+    const std::vector<size_t> &bindings;
+    ParamProxy(const std::vector<size_t> &labels_in, const LazyParams &params_in, const std::vector<size_t> &bindings_in)
+        : labels(labels_in), params(params_in), bindings(bindings_in) {}
+    const Value &resolve(size_t idx, Stash &stash) const override {
+        if (idx < labels.size()) {
+            return stash.create<DoubleValue>(labels[idx]);
+        }
+        return params.resolve(bindings[idx - labels.size()], stash);
+    }
+};
+
+}
+
+TensorSpec
+Lambda::create_spec_impl(const ValueType &type, const LazyParams &params, const std::vector<size_t> &bind, const InterpretedFunction &fun)
+{
+    std::vector<size_t> labels(type.dimensions().size(), 0);
+    ParamProxy param_proxy(labels, params, bind);
+    InterpretedFunction::Context ctx(fun);
+    TensorSpec spec(type.to_spec());
+    do {
+        TensorSpec::Address address;
+        for (size_t i = 0; i < labels.size(); ++i) {
+            address.emplace(type.dimensions()[i].name, labels[i]);
+        }
+        spec.add(std::move(address), fun.eval(ctx, param_proxy).as_double());
+    } while (step_labels(labels, type));
+    return spec;
+}
+
+InterpretedFunction::Instruction
+Lambda::compile_self(const TensorEngine &engine, Stash &stash) const
+{
+    InterpretedFunction fun(engine, _lambda->root(), _lambda_types);
+    Self &self = stash.create<Self>(*this, std::move(fun));
+    return Instruction(op_tensor_lambda, wrap_param<Self>(self));
+}
+
+void
+Lambda::push_children(std::vector<Child::CREF> &) const
+{
+}
+
+void
+Lambda::visit_self(vespalib::ObjectVisitor &visitor) const
+{
+    Super::visit_self(visitor);
+    ::visit(visitor, "bindings", _bindings);
+}
+
+//-----------------------------------------------------------------------------
+
 void
 Peek::push_children(std::vector<Child::CREF> &children) const
 {
@@ -397,7 +473,7 @@ Peek::push_children(std::vector<Child::CREF> &children) const
 }
 
 Instruction
-Peek::compile_self(Stash &) const
+Peek::compile_self(const TensorEngine &, Stash &) const
 {
     return Instruction(op_tensor_peek, wrap_param<Peek>(*this));
 }
@@ -426,7 +502,7 @@ Peek::visit_children(vespalib::ObjectVisitor &visitor) const
 //-----------------------------------------------------------------------------
 
 Instruction
-Rename::compile_self(Stash &stash) const
+Rename::compile_self(const TensorEngine &, Stash &stash) const
 {
     RenameParams &params = stash.create<RenameParams>(_from, _to);
     return Instruction(op_tensor_rename, wrap_param<RenameParams>(params));
@@ -450,7 +526,7 @@ If::push_children(std::vector<Child::CREF> &children) const
 }
 
 Instruction
-If::compile_self(Stash &) const
+If::compile_self(const TensorEngine &, Stash &) const
 {
     // 'if' is handled directly by compile_tensor_function to enable
     // lazy-evaluation of true/false sub-expressions.
@@ -504,6 +580,10 @@ const Node &create(const ValueType &type, const std::map<TensorSpec::Address,Nod
     return stash.create<Create>(type, spec);
 }
 
+const Node &lambda(const ValueType &type, const std::vector<size_t> &bindings, const Function &function, NodeTypes node_types, Stash &stash) {
+    return stash.create<Lambda>(type, bindings, function, std::move(node_types));
+}
+
 const Node &peek(const Node &param, const std::map<vespalib::string, std::variant<TensorSpec::Label, Node::CREF>> &spec, Stash &stash) {
     std::vector<vespalib::string> dimensions;
     for (const auto &dim_spec: spec) {
diff --git a/eval/src/vespa/eval/eval/tensor_function.h b/eval/src/vespa/eval/eval/tensor_function.h
index 4b862e9ec6a..2cc70f50b15 100644
--- a/eval/src/vespa/eval/eval/tensor_function.h
+++ b/eval/src/vespa/eval/eval/tensor_function.h
@@ -2,20 +2,17 @@
 
 #pragma once
 
-#include <memory>
-#include <vector>
-#include <variant>
-#include <vespa/vespalib/stllike/asciistream.h>
-#include <vespa/vespalib/stllike/string.h>
-#include <vespa/vespalib/util/arrayref.h>
-#include <vespa/vespalib/util/overload.h>
 #include "tensor_spec.h"
 #include "lazy_params.h"
 #include "value_type.h"
 #include "value.h"
 #include "aggr.h"
-
 #include "interpreted_function.h"
+#include <vespa/vespalib/stllike/asciistream.h>
+#include <vespa/vespalib/stllike/string.h>
+#include <vespa/vespalib/util/arrayref.h>
+#include <vespa/vespalib/util/overload.h>
+#include <variant>
 
 namespace vespalib {
 
@@ -33,11 +30,13 @@ class Tensor;
  * with information about operation sequencing and intermediate
  * results. Each node in the tree describes a single tensor
  * operation. This is the intermediate representation of a tensor
- * function.
+ * function. Note that some nodes in the tree are already indirectly
+ * implementation-specific in that they are bound to a specific tensor
+ * engine (typically tensor constants and tensor lambdas).
  *
  * A tensor function will initially be created based on a Function
- * (expression AST) and associated type-resolving. In this tree, each
- * node will directly represent a single call to the tensor engine
+ * (expression AST) and associated type-resolving. In this tree, most
+ * nodes will directly represent a single call to the tensor engine
  * immediate API.
  *
  * The generic tree will then be optimized (in-place, bottom-up) where
@@ -102,9 +101,10 @@ struct TensorFunction
      * the value stack during execution.
      *
      * @return instruction representing the operation of this node
+     * @param engine the tensor engine used for evaluation
      * @param stash heterogeneous object store
      **/
-    virtual InterpretedFunction::Instruction compile_self(Stash &stash) const = 0;
+    virtual InterpretedFunction::Instruction compile_self(const TensorEngine &engine, Stash &stash) const = 0;
 
     // for debug dumping
     vespalib::string as_string() const;
@@ -186,7 +186,7 @@ public:
     ConstValue(const Value &value_in) : Leaf(value_in.type()), _value(value_in) {}
     const Value &value() const { return _value; }
     bool result_is_mutable() const override { return false; }
-    InterpretedFunction::Instruction compile_self(Stash &stash) const final override;
+    InterpretedFunction::Instruction compile_self(const TensorEngine &engine, Stash &stash) const final override;
     void visit_self(vespalib::ObjectVisitor &visitor) const override;
 };
 
@@ -202,7 +202,7 @@ public:
         : Leaf(result_type_in), _param_idx(param_idx_in) {}
     size_t param_idx() const { return _param_idx; }
     bool result_is_mutable() const override { return false; }
-    InterpretedFunction::Instruction compile_self(Stash &stash) const final override;
+    InterpretedFunction::Instruction compile_self(const TensorEngine &engine, Stash &stash) const final override;
     void visit_self(vespalib::ObjectVisitor &visitor) const override;
 };
 
@@ -223,7 +223,7 @@ public:
     Aggr aggr() const { return _aggr; }
     const std::vector<vespalib::string> &dimensions() const { return _dimensions; }
     bool result_is_mutable() const override { return true; }
-    InterpretedFunction::Instruction compile_self(Stash &stash) const final override;
+    InterpretedFunction::Instruction compile_self(const TensorEngine &engine, Stash &stash) const final override;
     void visit_self(vespalib::ObjectVisitor &visitor) const override;
 };
 
@@ -241,7 +241,7 @@ public:
         : Op1(result_type_in, child_in), _function(function_in) {}
     map_fun_t function() const { return _function; }
     bool result_is_mutable() const override { return true; }
-    InterpretedFunction::Instruction compile_self(Stash &stash) const override;
+    InterpretedFunction::Instruction compile_self(const TensorEngine &engine, Stash &stash) const override;
     void visit_self(vespalib::ObjectVisitor &visitor) const override;
 };
 
@@ -260,7 +260,7 @@ public:
         : Op2(result_type_in, lhs_in, rhs_in), _function(function_in) {}
     join_fun_t function() const { return _function; }
     bool result_is_mutable() const override { return true; }
-    InterpretedFunction::Instruction compile_self(Stash &stash) const override;
+    InterpretedFunction::Instruction compile_self(const TensorEngine &engine, Stash &stash) const override;
     void visit_self(vespalib::ObjectVisitor &visitor) const override;
 };
 
@@ -279,7 +279,7 @@ public:
         : Op2(result_type_in, lhs_in, rhs_in), _function(function_in) {}
     join_fun_t function() const { return _function; }
     bool result_is_mutable() const override { return true; }
-    InterpretedFunction::Instruction compile_self(Stash &stash) const override;
+    InterpretedFunction::Instruction compile_self(const TensorEngine &engine, Stash &stash) const override;
     void visit_self(vespalib::ObjectVisitor &visitor) const override;
 };
 
@@ -298,7 +298,7 @@ public:
         : Op2(result_type_in, lhs_in, rhs_in), _dimension(dimension_in) {}
     const vespalib::string &dimension() const { return _dimension; }
     bool result_is_mutable() const override { return true; }
-    InterpretedFunction::Instruction compile_self(Stash &stash) const final override;
+    InterpretedFunction::Instruction compile_self(const TensorEngine &engine, Stash &stash) const final override;
     void visit_self(vespalib::ObjectVisitor &visitor) const override;
 };
 
@@ -319,13 +319,42 @@ public:
     }
     const std::map<TensorSpec::Address, Child> &spec() const { return _spec; }
     bool result_is_mutable() const override { return true; }
-    InterpretedFunction::Instruction compile_self(Stash &stash) const final override;
+    InterpretedFunction::Instruction compile_self(const TensorEngine &engine, Stash &stash) const final override;
     void push_children(std::vector<Child::CREF> &children) const final override;
     void visit_children(vespalib::ObjectVisitor &visitor) const final override;
 };
 
 //-----------------------------------------------------------------------------
 
+class Lambda : public Node
+{
+    using Super = Node;
+public:
+    struct Self {
+        const Lambda &parent;
+        InterpretedFunction fun;
+        Self(const Lambda &parent_in, InterpretedFunction fun_in)
+            : parent(parent_in), fun(std::move(fun_in)) {}
+    };
+private:
+    std::vector<size_t> _bindings;
+    std::shared_ptr<Function const> _lambda;
+    NodeTypes _lambda_types;
+public:
+    Lambda(const ValueType &result_type_in, const std::vector<size_t> &bindings_in, const Function &lambda_in, NodeTypes lambda_types_in)
+        : Node(result_type_in), _bindings(bindings_in), _lambda(lambda_in.shared_from_this()), _lambda_types(std::move(lambda_types_in)) {}
+    static TensorSpec create_spec_impl(const ValueType &type, const LazyParams &params, const std::vector<size_t> &bind, const InterpretedFunction &fun);
+    TensorSpec create_spec(const LazyParams &params, const InterpretedFunction &fun) const {
+        return create_spec_impl(result_type(), params, _bindings, fun);
+    }
+    bool result_is_mutable() const override { return true; }
+    InterpretedFunction::Instruction compile_self(const TensorEngine &engine, Stash &stash) const final override;
+    void push_children(std::vector<Child::CREF> &children) const final override;
+    void visit_self(vespalib::ObjectVisitor &visitor) const override;
+};
+
+//-----------------------------------------------------------------------------
+
 class Peek : public Node
 {
     using Super = Node;
@@ -354,7 +383,7 @@ public:
     const std::map<vespalib::string, MyLabel> &spec() const { return _spec; }
     const ValueType &param_type() const { return _param.get().result_type(); }
     bool result_is_mutable() const override { return true; }
-    InterpretedFunction::Instruction compile_self(Stash &stash) const final override;
+    InterpretedFunction::Instruction compile_self(const TensorEngine &engine, Stash &stash) const final override;
     void push_children(std::vector<Child::CREF> &children) const final override;
     void visit_children(vespalib::ObjectVisitor &visitor) const final override;
 };
@@ -376,7 +405,7 @@ public:
     const std::vector<vespalib::string> &from() const { return _from; }
     const std::vector<vespalib::string> &to() const { return _to; }
     bool result_is_mutable() const override { return true; }
-    InterpretedFunction::Instruction compile_self(Stash &stash) const final override;
+    InterpretedFunction::Instruction compile_self(const TensorEngine &engine, Stash &stash) const final override;
     void visit_self(vespalib::ObjectVisitor &visitor) const override;
 };
 
@@ -402,7 +431,7 @@ public:
         return (true_child().result_is_mutable() &&
                 false_child().result_is_mutable());
     }
-    InterpretedFunction::Instruction compile_self(Stash &stash) const final override;
+    InterpretedFunction::Instruction compile_self(const TensorEngine &engine, Stash &stash) const final override;
     void visit_children(vespalib::ObjectVisitor &visitor) const final override;
 };
 
@@ -416,6 +445,7 @@ const Node &join(const Node &lhs, const Node &rhs, join_fun_t function, Stash &s
 const Node &merge(const Node &lhs, const Node &rhs, join_fun_t function, Stash &stash);
 const Node &concat(const Node &lhs, const Node &rhs, const vespalib::string &dimension, Stash &stash);
 const Node &create(const ValueType &type, const std::map<TensorSpec::Address, Node::CREF> &spec, Stash &stash);
+const Node &lambda(const ValueType &type, const std::vector<size_t> &bindings, const Function &function, NodeTypes node_types, Stash &stash);
 const Node &peek(const Node &param, const std::map<vespalib::string, std::variant<TensorSpec::Label, Node::CREF>> &spec, Stash &stash);
 const Node &rename(const Node &child, const std::vector<vespalib::string> &from, const std::vector<vespalib::string> &to, Stash &stash);
 const Node &if_node(const Node &cond, const Node &true_child, const Node &false_child, Stash &stash);
diff --git a/eval/src/vespa/eval/eval/tensor_nodes.cpp b/eval/src/vespa/eval/eval/tensor_nodes.cpp
index 82d108300dd..5cb064ad127 100644
--- a/eval/src/vespa/eval/eval/tensor_nodes.cpp
+++ b/eval/src/vespa/eval/eval/tensor_nodes.cpp
@@ -14,6 +14,7 @@ void TensorReduce::accept(NodeVisitor &visitor) const { visitor.visit(*this); }
 void TensorRename::accept(NodeVisitor &visitor) const { visitor.visit(*this); }
 void TensorConcat::accept(NodeVisitor &visitor) const { visitor.visit(*this); }
 void TensorCreate::accept(NodeVisitor &visitor) const { visitor.visit(*this); }
+void TensorLambda::accept(NodeVisitor &visitor) const { visitor.visit(*this); }
 void TensorPeek  ::accept(NodeVisitor &visitor) const { visitor.visit(*this); }
 
 } // namespace vespalib::eval::nodes
diff --git a/eval/src/vespa/eval/eval/tensor_nodes.h b/eval/src/vespa/eval/eval/tensor_nodes.h
index daba46c1fc5..07be7e77b71 100644
--- a/eval/src/vespa/eval/eval/tensor_nodes.h
+++ b/eval/src/vespa/eval/eval/tensor_nodes.h
@@ -267,6 +267,39 @@ public:
     }
 };
 
+class TensorLambda : public Node {
+private:
+    ValueType _type;
+    std::vector<size_t> _bindings;
+    std::shared_ptr<Function const> _lambda;
+public:
+    TensorLambda(ValueType type_in, std::vector<size_t> bindings, std::shared_ptr<Function const> lambda)
+        : _type(std::move(type_in)), _bindings(std::move(bindings)), _lambda(std::move(lambda))
+    {
+        assert(_type.is_dense());
+        assert(_lambda->num_params() == (_type.dimensions().size() + _bindings.size()));
+    }
+    const ValueType &type() const { return _type; }
+    const std::vector<size_t> &bindings() const { return _bindings; }
+    const Function &lambda() const { return *_lambda; }
+    vespalib::string dump(DumpContext &) const override {
+        vespalib::string str = _type.to_spec();
+        vespalib::string expr = _lambda->dump();
+        if (starts_with(expr, "(")) {
+            str += expr;
+        } else {
+            str += "(";
+            str += expr;
+            str += ")";
+        }
+        return str;
+    }
+    void accept(NodeVisitor &visitor) const override;
+    size_t num_children() const override { return 0; }
+    const Node &get_child(size_t) const override { abort(); }
+    void detach_children(NodeHandler &) override {}
+};
+
 class TensorPeek : public Node {
 public:
     struct MyLabel {
diff --git a/eval/src/vespa/eval/eval/tensor_spec.h b/eval/src/vespa/eval/eval/tensor_spec.h
index 25af4c7a93c..22aa47f5ddb 100644
--- a/eval/src/vespa/eval/eval/tensor_spec.h
+++ b/eval/src/vespa/eval/eval/tensor_spec.h
@@ -66,8 +66,8 @@ public:
     TensorSpec(const TensorSpec &);
     TensorSpec & operator = (const TensorSpec &);
     ~TensorSpec();
-    TensorSpec &add(const Address &address, double value) {
-        auto res = _cells.emplace(address, value);
+    TensorSpec &add(Address address, double value) {
+        auto res = _cells.emplace(std::move(address), value);
         if (!res.second) {
             res.first->second.value += value;
         }
diff --git a/eval/src/vespa/eval/eval/test/eval_fixture.cpp b/eval/src/vespa/eval/eval/test/eval_fixture.cpp
index e578b28da18..7c4d4caf854 100644
--- a/eval/src/vespa/eval/eval/test/eval_fixture.cpp
+++ b/eval/src/vespa/eval/eval/test/eval_fixture.cpp
@@ -10,6 +10,14 @@ using ParamRepo = EvalFixture::ParamRepo;
 
 namespace {
 
+std::shared_ptr<Function const> verify_function(std::shared_ptr<Function const> fun) {
+    if (fun->has_error()) {
+        fprintf(stderr, "eval_fixture: function parse failed: %s\n", fun->get_error().c_str());
+    }
+    ASSERT_TRUE(!fun->has_error());
+    return fun;
+}
+
 NodeTypes get_types(const Function &function, const ParamRepo &param_repo) {
     std::vector<ValueType> param_types;
     for (size_t i = 0; i < function.num_params(); ++i) {
@@ -83,6 +91,19 @@ std::vector<Value::CREF> get_refs(const std::vector<Value::UP> &values) {
 
 } // namespace vespalib::eval::test
 
+void
+EvalFixture::detect_param_tampering(const ParamRepo &param_repo, bool allow_mutable) const
+{
+    for (size_t i = 0; i < _function->num_params(); ++i) {
+        auto pos = param_repo.map.find(_function->param_name(i));
+        ASSERT_TRUE(pos != param_repo.map.end());
+        bool allow_tampering = allow_mutable && pos->second.is_mutable;
+        if (!allow_tampering) {
+            ASSERT_EQUAL(pos->second.value, _engine.to_spec(*_param_values[i]));
+        }
+    }
+}
+
 EvalFixture::EvalFixture(const TensorEngine &engine,
                          const vespalib::string &expr,
                          const ParamRepo &param_repo,
@@ -90,7 +111,7 @@ EvalFixture::EvalFixture(const TensorEngine &engine,
                          bool allow_mutable)
     : _engine(engine),
       _stash(),
-      _function(Function::parse(expr)),
+      _function(verify_function(Function::parse(expr))),
       _node_types(get_types(*_function, param_repo)),
       _mutable_set(get_mutable(*_function, param_repo)),
       _plain_tensor_function(make_tensor_function(_engine, _function->root(), _node_types, _stash)),
@@ -104,6 +125,7 @@ EvalFixture::EvalFixture(const TensorEngine &engine,
 {
     auto result_type = ValueType::from_spec(_result.type());
     ASSERT_TRUE(!result_type.is_error());
+    TEST_DO(detect_param_tampering(param_repo, allow_mutable));
 }
 
 const TensorSpec
diff --git a/eval/src/vespa/eval/eval/test/eval_fixture.h b/eval/src/vespa/eval/eval/test/eval_fixture.h
index 48f6a7e5d2e..1d39dc52cba 100644
--- a/eval/src/vespa/eval/eval/test/eval_fixture.h
+++ b/eval/src/vespa/eval/eval/test/eval_fixture.h
@@ -67,6 +67,8 @@ private:
         }
     }
 
+    void detect_param_tampering(const ParamRepo &param_repo, bool allow_mutable) const;
+
 public:
     EvalFixture(const TensorEngine &engine, const vespalib::string &expr, const ParamRepo &param_repo,
                 bool optimized = true, bool allow_mutable = false);
diff --git a/eval/src/vespa/eval/eval/value.h b/eval/src/vespa/eval/eval/value.h
index 6701173bcd3..cad76c93c5c 100644
--- a/eval/src/vespa/eval/eval/value.h
+++ b/eval/src/vespa/eval/eval/value.h
@@ -2,13 +2,11 @@
 
 #pragma once
 
-#include <vespa/vespalib/stllike/string.h>
-#include <memory>
-#include <vespa/vespalib/util/stash.h>
 #include "value_type.h"
+#include <vespa/vespalib/util/traits.h>
+#include <memory>
 
-namespace vespalib {
-namespace eval {
+namespace vespalib::eval {
 
 class Tensor;
 
@@ -40,7 +38,6 @@ public:
     static const ValueType &double_type() { return _type; }
 };
 
-} // namespace vespalib::eval
-} // namespace vespalib
+}
 
 VESPA_CAN_SKIP_DESTRUCTION(::vespalib::eval::DoubleValue);
diff --git a/eval/src/vespa/eval/tensor/default_tensor_engine.cpp b/eval/src/vespa/eval/tensor/default_tensor_engine.cpp
index b4449309812..b16241fe5e5 100644
--- a/eval/src/vespa/eval/tensor/default_tensor_engine.cpp
+++ b/eval/src/vespa/eval/tensor/default_tensor_engine.cpp
@@ -10,9 +10,11 @@
 #include "dense/typed_dense_tensor_builder.h"
 #include "dense/dense_dot_product_function.h"
 #include "dense/dense_xw_product_function.h"
+#include "dense/dense_matmul_function.h"
 #include "dense/dense_fast_rename_optimizer.h"
 #include "dense/dense_add_dimension_optimizer.h"
 #include "dense/dense_remove_dimension_optimizer.h"
+#include "dense/dense_lambda_peek_optimizer.h"
 #include "dense/dense_inplace_join_function.h"
 #include "dense/dense_inplace_map_function.h"
 #include "dense/vector_from_doubles_function.h"
@@ -266,9 +268,11 @@ DefaultTensorEngine::optimize(const TensorFunction &expr, Stash &stash) const
         const Child &child = nodes.back();
         child.set(VectorFromDoublesFunction::optimize(child.get(), stash));
         child.set(DenseTensorCreateFunction::optimize(child.get(), stash));
+        child.set(DenseLambdaPeekOptimizer::optimize(child.get(), stash));
         child.set(DenseTensorPeekFunction::optimize(child.get(), stash));
         child.set(DenseDotProductFunction::optimize(child.get(), stash));
         child.set(DenseXWProductFunction::optimize(child.get(), stash));
+        child.set(DenseMatMulFunction::optimize(child.get(), stash));
         child.set(DenseFastRenameOptimizer::optimize(child.get(), stash));
         child.set(DenseAddDimensionOptimizer::optimize(child.get(), stash));
         child.set(DenseRemoveDimensionOptimizer::optimize(child.get(), stash));
diff --git a/eval/src/vespa/eval/tensor/dense/CMakeLists.txt b/eval/src/vespa/eval/tensor/dense/CMakeLists.txt
index 2ad54d48ab3..9e4c9857bd1 100644
--- a/eval/src/vespa/eval/tensor/dense/CMakeLists.txt
+++ b/eval/src/vespa/eval/tensor/dense/CMakeLists.txt
@@ -7,12 +7,14 @@ vespa_add_library(eval_tensor_dense OBJECT
     dense_fast_rename_optimizer.cpp
     dense_inplace_join_function.cpp
     dense_inplace_map_function.cpp
+    dense_matmul_function.cpp
     dense_remove_dimension_optimizer.cpp
     dense_replace_type_function.cpp
     dense_tensor.cpp
     dense_tensor_address_mapper.cpp
     dense_tensor_cells_iterator.cpp
     dense_tensor_create_function.cpp
+    dense_lambda_peek_optimizer.cpp
     dense_tensor_modify.cpp
     dense_tensor_peek_function.cpp
     dense_tensor_reduce.cpp
diff --git a/eval/src/vespa/eval/tensor/dense/dense_add_dimension_optimizer.cpp b/eval/src/vespa/eval/tensor/dense/dense_add_dimension_optimizer.cpp
index a4331b6b251..2e2b94cc7a6 100644
--- a/eval/src/vespa/eval/tensor/dense/dense_add_dimension_optimizer.cpp
+++ b/eval/src/vespa/eval/tensor/dense/dense_add_dimension_optimizer.cpp
@@ -2,9 +2,7 @@
 
 #include "dense_add_dimension_optimizer.h"
 #include "dense_replace_type_function.h"
-#include <vespa/eval/eval/value_type.h>
 #include <vespa/eval/eval/operation.h>
-#include <vespa/eval/eval/value.h>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".eval.tensor.dense.add_dimension_optimizer");
diff --git a/eval/src/vespa/eval/tensor/dense/dense_dot_product_function.cpp b/eval/src/vespa/eval/tensor/dense/dense_dot_product_function.cpp
index 8bcaddba3b4..c9ff57e4a65 100644
--- a/eval/src/vespa/eval/tensor/dense/dense_dot_product_function.cpp
+++ b/eval/src/vespa/eval/tensor/dense/dense_dot_product_function.cpp
@@ -1,16 +1,17 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include "dense_dot_product_function.h"
-#include "dense_tensor.h"
 #include "dense_tensor_view.h"
 #include <vespa/eval/eval/operation.h>
 #include <vespa/eval/eval/value.h>
-#include <vespa/eval/tensor/tensor.h>
+
+#include <cblas.h>
 
 namespace vespalib::tensor {
 
 using eval::ValueType;
 using eval::TensorFunction;
+using eval::TensorEngine;
 using eval::as;
 using eval::Aggr;
 using namespace eval::tensor_function;
@@ -19,32 +20,29 @@ using namespace eval::operation;
 namespace {
 
 template <typename LCT, typename RCT>
-struct HWSupport {
-    static double call(hwaccelrated::IAccelrated *, const ConstArrayRef<LCT> &lhs, const ConstArrayRef<RCT> &rhs) {
-        double result = 0.0;
-        for (size_t i = 0; i < lhs.size(); ++i) {
-            result += (lhs[i] * rhs[i]);
-        }
-        return result;
+void my_dot_product_op(eval::InterpretedFunction::State &state, uint64_t) {
+    auto lhs_cells = DenseTensorView::typify_cells<LCT>(state.peek(1));
+    auto rhs_cells = DenseTensorView::typify_cells<RCT>(state.peek(0));
+    double result = 0.0;
+    const LCT *lhs = lhs_cells.cbegin();
+    const RCT *rhs = rhs_cells.cbegin();
+    for (size_t i = 0; i < lhs_cells.size(); ++i) {
+        result += ((*lhs++) * (*rhs++));
     }
-};
-template <> struct HWSupport<float, float> {
-    static double call(hwaccelrated::IAccelrated *hw, const ConstArrayRef<float> &lhs, const ConstArrayRef<float> &rhs) {
-        return hw->dotProduct(lhs.cbegin(), rhs.cbegin(), lhs.size());
-    }
-};
-template <> struct HWSupport<double, double> {
-    static double call(hwaccelrated::IAccelrated *hw, const ConstArrayRef<double> &lhs, const ConstArrayRef<double> &rhs) {
-        return hw->dotProduct(lhs.cbegin(), rhs.cbegin(), lhs.size());
-    }
-};
+    state.pop_pop_push(state.stash.create<eval::DoubleValue>(result));
+}
 
-template <typename LCT, typename RCT>
-void my_dot_product_op(eval::InterpretedFunction::State &state, uint64_t param) {
-    auto *hw = (hwaccelrated::IAccelrated *)(param);
-    auto lhs = DenseTensorView::typify_cells<LCT>(state.peek(1));
-    auto rhs = DenseTensorView::typify_cells<RCT>(state.peek(0));
-    double result = HWSupport<LCT,RCT>::call(hw, lhs, rhs);
+void my_cblas_double_dot_product_op(eval::InterpretedFunction::State &state, uint64_t) {
+    auto lhs_cells = DenseTensorView::typify_cells<double>(state.peek(1));
+    auto rhs_cells = DenseTensorView::typify_cells<double>(state.peek(0));
+    double result = cblas_ddot(lhs_cells.size(), lhs_cells.cbegin(), 1, rhs_cells.cbegin(), 1);
+    state.pop_pop_push(state.stash.create<eval::DoubleValue>(result));
+}
+
+void my_cblas_float_dot_product_op(eval::InterpretedFunction::State &state, uint64_t) {
+    auto lhs_cells = DenseTensorView::typify_cells<float>(state.peek(1));
+    auto rhs_cells = DenseTensorView::typify_cells<float>(state.peek(0));
+    double result = cblas_sdot(lhs_cells.size(), lhs_cells.cbegin(), 1, rhs_cells.cbegin(), 1);
     state.pop_pop_push(state.stash.create<eval::DoubleValue>(result));
 }
 
@@ -53,21 +51,31 @@ struct MyDotProductOp {
     static auto get_fun() { return my_dot_product_op<LCT,RCT>; }
 };
 
+eval::InterpretedFunction::op_function my_select(CellType lct, CellType rct) {
+    if (lct == rct) {
+        if (lct == ValueType::CellType::DOUBLE) {
+            return my_cblas_double_dot_product_op;
+        }
+        if (lct == ValueType::CellType::FLOAT) {
+            return my_cblas_float_dot_product_op;
+        }
+    }
+    return select_2<MyDotProductOp>(lct, rct);
+}
+
 } // namespace vespalib::tensor::<unnamed>
 
 DenseDotProductFunction::DenseDotProductFunction(const eval::TensorFunction &lhs_in,
                                                  const eval::TensorFunction &rhs_in)
-    : eval::tensor_function::Op2(eval::ValueType::double_type(), lhs_in, rhs_in),
-      _hwAccelerator(hwaccelrated::IAccelrated::getAccelrator())
+    : eval::tensor_function::Op2(eval::ValueType::double_type(), lhs_in, rhs_in)
 {
 }
 
 eval::InterpretedFunction::Instruction
-DenseDotProductFunction::compile_self(Stash &) const
+DenseDotProductFunction::compile_self(const TensorEngine &, Stash &) const
 {
-    auto op = select_2<MyDotProductOp>(lhs().result_type().cell_type(),
-                                       rhs().result_type().cell_type());
-    return eval::InterpretedFunction::Instruction(op, (uint64_t)(_hwAccelerator.get()));
+    auto op = my_select(lhs().result_type().cell_type(), rhs().result_type().cell_type());
+    return eval::InterpretedFunction::Instruction(op);
 }
 
 bool
diff --git a/eval/src/vespa/eval/tensor/dense/dense_dot_product_function.h b/eval/src/vespa/eval/tensor/dense/dense_dot_product_function.h
index d6181d33887..1ee6baff2a5 100644
--- a/eval/src/vespa/eval/tensor/dense/dense_dot_product_function.h
+++ b/eval/src/vespa/eval/tensor/dense/dense_dot_product_function.h
@@ -3,7 +3,6 @@
 #pragma once
 
 #include <vespa/eval/eval/tensor_function.h>
-#include <vespa/vespalib/hwaccelrated/iaccelrated.h>
 
 namespace vespalib::tensor {
 
@@ -13,12 +12,11 @@ namespace vespalib::tensor {
 class DenseDotProductFunction : public eval::tensor_function::Op2
 {
 private:
-    hwaccelrated::IAccelrated::UP _hwAccelerator;
     using ValueType = eval::ValueType;
 public:
     DenseDotProductFunction(const eval::TensorFunction &lhs_in,
                             const eval::TensorFunction &rhs_in);
-    eval::InterpretedFunction::Instruction compile_self(Stash &stash) const override;
+    eval::InterpretedFunction::Instruction compile_self(const eval::TensorEngine &engine, Stash &stash) const override;
     bool result_is_mutable() const override { return true; }
     static bool compatible_types(const ValueType &res, const ValueType &lhs, const ValueType &rhs);
     static const eval::TensorFunction &optimize(const eval::TensorFunction &expr, Stash &stash);
diff --git a/eval/src/vespa/eval/tensor/dense/dense_fast_rename_optimizer.cpp b/eval/src/vespa/eval/tensor/dense/dense_fast_rename_optimizer.cpp
index ac8442477e4..7c3dde5fa2e 100644
--- a/eval/src/vespa/eval/tensor/dense/dense_fast_rename_optimizer.cpp
+++ b/eval/src/vespa/eval/tensor/dense/dense_fast_rename_optimizer.cpp
@@ -2,10 +2,8 @@
 
 #include "dense_fast_rename_optimizer.h"
 #include "dense_replace_type_function.h"
-#include "dense_tensor.h"
 #include "dense_tensor_view.h"
 #include <vespa/eval/eval/value.h>
-#include <vespa/eval/tensor/tensor.h>
 
 namespace vespalib::tensor {
 
diff --git a/eval/src/vespa/eval/tensor/dense/dense_inplace_join_function.cpp b/eval/src/vespa/eval/tensor/dense/dense_inplace_join_function.cpp
index 0b5bba88d37..2107c7661f2 100644
--- a/eval/src/vespa/eval/tensor/dense/dense_inplace_join_function.cpp
+++ b/eval/src/vespa/eval/tensor/dense/dense_inplace_join_function.cpp
@@ -1,17 +1,16 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include "dense_inplace_join_function.h"
-#include "dense_tensor.h"
 #include "dense_tensor_view.h"
 #include <vespa/vespalib/objects/objectvisitor.h>
 #include <vespa/eval/eval/value.h>
-#include <vespa/eval/tensor/tensor.h>
 
 namespace vespalib::tensor {
 
 using eval::Value;
 using eval::ValueType;
 using eval::TensorFunction;
+using eval::TensorEngine;
 using eval::as;
 using namespace eval::tensor_function;
 
@@ -76,7 +75,7 @@ DenseInplaceJoinFunction::~DenseInplaceJoinFunction()
 }
 
 eval::InterpretedFunction::Instruction
-DenseInplaceJoinFunction::compile_self(Stash &) const
+DenseInplaceJoinFunction::compile_self(const TensorEngine &, Stash &) const
 {
     auto op = my_select(lhs().result_type().cell_type(),
                         rhs().result_type().cell_type(), _write_left);
diff --git a/eval/src/vespa/eval/tensor/dense/dense_inplace_join_function.h b/eval/src/vespa/eval/tensor/dense/dense_inplace_join_function.h
index 83acee6d31f..acd1a2d716b 100644
--- a/eval/src/vespa/eval/tensor/dense/dense_inplace_join_function.h
+++ b/eval/src/vespa/eval/tensor/dense/dense_inplace_join_function.h
@@ -25,7 +25,7 @@ public:
     ~DenseInplaceJoinFunction();
     bool write_left() const { return _write_left; }
     bool result_is_mutable() const override { return true; }
-    eval::InterpretedFunction::Instruction compile_self(Stash &stash) const override;
+    eval::InterpretedFunction::Instruction compile_self(const eval::TensorEngine &engine, Stash &stash) const override;
     void visit_self(vespalib::ObjectVisitor &visitor) const override;
     static const eval::TensorFunction &optimize(const eval::TensorFunction &expr, Stash &stash);
 };
diff --git a/eval/src/vespa/eval/tensor/dense/dense_inplace_map_function.cpp b/eval/src/vespa/eval/tensor/dense/dense_inplace_map_function.cpp
index c82cda34a28..62434073f8e 100644
--- a/eval/src/vespa/eval/tensor/dense/dense_inplace_map_function.cpp
+++ b/eval/src/vespa/eval/tensor/dense/dense_inplace_map_function.cpp
@@ -1,16 +1,15 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include "dense_inplace_map_function.h"
-#include "dense_tensor.h"
 #include "dense_tensor_view.h"
 #include <vespa/eval/eval/value.h>
-#include <vespa/eval/tensor/tensor.h>
 
 namespace vespalib::tensor {
 
 using eval::Value;
 using eval::ValueType;
 using eval::TensorFunction;
+using eval::TensorEngine;
 using eval::as;
 using namespace eval::tensor_function;
 
@@ -44,7 +43,7 @@ DenseInplaceMapFunction::~DenseInplaceMapFunction()
 }
 
 eval::InterpretedFunction::Instruction
-DenseInplaceMapFunction::compile_self(Stash &) const
+DenseInplaceMapFunction::compile_self(const TensorEngine &, Stash &) const
 {
     auto op = select_1<MyInplaceMapOp>(result_type().cell_type());
     return eval::InterpretedFunction::Instruction(op, (uint64_t)function());
diff --git a/eval/src/vespa/eval/tensor/dense/dense_inplace_map_function.h b/eval/src/vespa/eval/tensor/dense/dense_inplace_map_function.h
index acc4504176a..52122f4d8dc 100644
--- a/eval/src/vespa/eval/tensor/dense/dense_inplace_map_function.h
+++ b/eval/src/vespa/eval/tensor/dense/dense_inplace_map_function.h
@@ -18,7 +18,7 @@ public:
                             map_fun_t function_in);
     ~DenseInplaceMapFunction();
     bool result_is_mutable() const override { return true; }
-    eval::InterpretedFunction::Instruction compile_self(Stash &stash) const override;
+    eval::InterpretedFunction::Instruction compile_self(const eval::TensorEngine &engine, Stash &stash) const override;
     static const eval::TensorFunction &optimize(const eval::TensorFunction &expr, Stash &stash);
 };
 
diff --git a/eval/src/vespa/eval/tensor/dense/dense_lambda_peek_optimizer.cpp b/eval/src/vespa/eval/tensor/dense/dense_lambda_peek_optimizer.cpp
new file mode 100644
index 00000000000..14954a77834
--- /dev/null
+++ b/eval/src/vespa/eval/tensor/dense/dense_lambda_peek_optimizer.cpp
@@ -0,0 +1,13 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "dense_lambda_peek_optimizer.h"
+
+namespace vespalib::tensor {
+
+const eval::TensorFunction &
+DenseLambdaPeekOptimizer::optimize(const eval::TensorFunction &expr, Stash &)
+{
+    return expr;
+}
+
+}
diff --git a/eval/src/vespa/eval/tensor/dense/dense_lambda_peek_optimizer.h b/eval/src/vespa/eval/tensor/dense/dense_lambda_peek_optimizer.h
new file mode 100644
index 00000000000..7d2a0efde76
--- /dev/null
+++ b/eval/src/vespa/eval/tensor/dense/dense_lambda_peek_optimizer.h
@@ -0,0 +1,18 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/eval/eval/tensor_function.h>
+
+namespace vespalib::tensor {
+
+/**
+ * Tensor lambda optimizer for creating a new dense tensor based on
+ * peeking cells of a single existing tensor. This can represent a
+ * wide area of operations (reshape, gather, slice).
+ **/
+struct DenseLambdaPeekOptimizer {
+    static const eval::TensorFunction &optimize(const eval::TensorFunction &expr, Stash &stash);
+};
+
+}
diff --git a/eval/src/vespa/eval/tensor/dense/dense_matmul_function.cpp b/eval/src/vespa/eval/tensor/dense/dense_matmul_function.cpp
new file mode 100644
index 00000000000..695e0fddd08
--- /dev/null
+++ b/eval/src/vespa/eval/tensor/dense/dense_matmul_function.cpp
@@ -0,0 +1,234 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "dense_matmul_function.h"
+#include "dense_tensor_view.h"
+#include <vespa/vespalib/objects/objectvisitor.h>
+#include <vespa/eval/eval/value.h>
+#include <vespa/eval/eval/operation.h>
+#include <cassert>
+
+#include <cblas.h>
+
+namespace vespalib::tensor {
+
+using eval::ValueType;
+using eval::TensorFunction;
+using eval::TensorEngine;
+using eval::as;
+using eval::Aggr;
+using namespace eval::tensor_function;
+using namespace eval::operation;
+
+namespace {
+
+template <typename LCT, typename RCT, bool lhs_common_inner, bool rhs_common_inner>
+double my_dot_product(const LCT *lhs, const RCT *rhs, size_t lhs_size, size_t common_size, size_t rhs_size) {
+    double result = 0.0;
+    for (size_t i = 0; i < common_size; ++i) {
+        result += ((*lhs) * (*rhs));
+        lhs += (lhs_common_inner ? 1 : lhs_size);
+        rhs += (rhs_common_inner ? 1 : rhs_size);
+    }
+    return result;
+}
+
+template <typename LCT, typename RCT, bool lhs_common_inner, bool rhs_common_inner>
+void my_matmul_op(eval::InterpretedFunction::State &state, uint64_t param) {
+    const DenseMatMulFunction::Self &self = *((const DenseMatMulFunction::Self *)(param));
+    using OCT = typename eval::UnifyCellTypes<LCT,RCT>::type;
+    auto lhs_cells = DenseTensorView::typify_cells<LCT>(state.peek(1));
+    auto rhs_cells = DenseTensorView::typify_cells<RCT>(state.peek(0));
+    auto dst_cells = state.stash.create_array<OCT>(self.lhs_size * self.rhs_size);
+    OCT *dst = dst_cells.begin();
+    const LCT *lhs = lhs_cells.cbegin();
+    for (size_t i = 0; i < self.lhs_size; ++i) {
+        const RCT *rhs = rhs_cells.cbegin();
+        for (size_t j = 0; j < self.rhs_size; ++j) {
+            *dst++ = my_dot_product<LCT,RCT,lhs_common_inner,rhs_common_inner>(lhs, rhs, self.lhs_size, self.common_size, self.rhs_size);
+            rhs += (rhs_common_inner ? self.common_size : 1);
+        }
+        lhs += (lhs_common_inner ? self.common_size : 1);
+    }
+    state.pop_pop_push(state.stash.create<DenseTensorView>(self.result_type, TypedCells(dst_cells)));
+}
+
+template <bool lhs_common_inner, bool rhs_common_inner>
+void my_cblas_double_matmul_op(eval::InterpretedFunction::State &state, uint64_t param) {
+    const DenseMatMulFunction::Self &self = *((const DenseMatMulFunction::Self *)(param));
+    auto lhs_cells = DenseTensorView::typify_cells<double>(state.peek(1));
+    auto rhs_cells = DenseTensorView::typify_cells<double>(state.peek(0));
+    auto dst_cells = state.stash.create_array<double>(self.lhs_size * self.rhs_size);
+    cblas_dgemm(CblasRowMajor, lhs_common_inner ? CblasNoTrans : CblasTrans, rhs_common_inner ? CblasTrans : CblasNoTrans,
+                self.lhs_size, self.rhs_size, self.common_size, 1.0,
+                lhs_cells.cbegin(), lhs_common_inner ? self.common_size : self.lhs_size,
+                rhs_cells.cbegin(), rhs_common_inner ? self.common_size : self.rhs_size,
+                0.0, dst_cells.begin(), self.rhs_size);
+    state.pop_pop_push(state.stash.create<DenseTensorView>(self.result_type, TypedCells(dst_cells)));
+}
+
+template <bool lhs_common_inner, bool rhs_common_inner>
+void my_cblas_float_matmul_op(eval::InterpretedFunction::State &state, uint64_t param) {
+    const DenseMatMulFunction::Self &self = *((const DenseMatMulFunction::Self *)(param));
+    auto lhs_cells = DenseTensorView::typify_cells<float>(state.peek(1));
+    auto rhs_cells = DenseTensorView::typify_cells<float>(state.peek(0));
+    auto dst_cells = state.stash.create_array<float>(self.lhs_size * self.rhs_size);
+    cblas_sgemm(CblasRowMajor, lhs_common_inner ? CblasNoTrans : CblasTrans, rhs_common_inner ? CblasTrans : CblasNoTrans,
+                self.lhs_size, self.rhs_size, self.common_size, 1.0,
+                lhs_cells.cbegin(), lhs_common_inner ? self.common_size : self.lhs_size,
+                rhs_cells.cbegin(), rhs_common_inner ? self.common_size : self.rhs_size,
+                0.0, dst_cells.begin(), self.rhs_size);
+    state.pop_pop_push(state.stash.create<DenseTensorView>(self.result_type, TypedCells(dst_cells)));
+}
+
+template <bool lhs_common_inner, bool rhs_common_inner>
+struct MyMatMulOp {
+    template <typename LCT, typename RCT>
+    static auto get_fun() { return my_matmul_op<LCT,RCT,lhs_common_inner,rhs_common_inner>; }
+};
+
+template <bool lhs_common_inner, bool rhs_common_inner>
+eval::InterpretedFunction::op_function my_select3(CellType lct, CellType rct)
+{
+    if (lct == rct) {
+        if (lct == ValueType::CellType::DOUBLE) {
+            return my_cblas_double_matmul_op<lhs_common_inner,rhs_common_inner>;
+        }
+        if (lct == ValueType::CellType::FLOAT) {
+            return my_cblas_float_matmul_op<lhs_common_inner,rhs_common_inner>;
+        }
+    }
+    return select_2<MyMatMulOp<lhs_common_inner,rhs_common_inner>>(lct, rct);
+}
+
+template <bool lhs_common_inner>
+eval::InterpretedFunction::op_function my_select2(CellType lct, CellType rct,
+                                                  bool rhs_common_inner)
+{
+    if (rhs_common_inner) {
+        return my_select3<lhs_common_inner,true>(lct, rct);
+    } else {
+        return my_select3<lhs_common_inner,false>(lct, rct);
+    }
+}
+
+eval::InterpretedFunction::op_function my_select(CellType lct, CellType rct,
+                                                 bool lhs_common_inner, bool rhs_common_inner)
+{
+    if (lhs_common_inner) {
+        return my_select2<true>(lct, rct, rhs_common_inner);
+    } else {
+        return my_select2<false>(lct, rct, rhs_common_inner);
+    }
+}
+
+bool is_matrix(const ValueType &type) {
+    return (type.is_dense() && (type.dimensions().size() == 2));
+}
+
+bool is_matmul(const eval::ValueType &a, const eval::ValueType &b,
+               const vespalib::string &reduce_dim, const eval::ValueType &result_type)
+{
+    size_t npos = ValueType::Dimension::npos;
+    return (is_matrix(a) && is_matrix(b) && is_matrix(result_type) &&
+            (a.dimension_index(reduce_dim) != npos) &&
+            (b.dimension_index(reduce_dim) != npos));
+}
+
+const ValueType::Dimension &dim(const TensorFunction &expr, size_t idx) {
+    return expr.result_type().dimensions()[idx];
+}
+
+size_t inv(size_t idx) { return (1 - idx); }
+
+const TensorFunction &create_matmul(const TensorFunction &a, const TensorFunction &b,
+                                    const vespalib::string &reduce_dim, const ValueType &result_type, Stash &stash) {
+    size_t a_idx = a.result_type().dimension_index(reduce_dim);
+    size_t b_idx = b.result_type().dimension_index(reduce_dim);
+    assert(a_idx != ValueType::Dimension::npos);
+    assert(b_idx != ValueType::Dimension::npos);
+    assert(dim(a, a_idx).size == dim(b, b_idx).size);
+    bool a_common_inner = (a_idx == 1);
+    bool b_common_inner = (b_idx == 1);
+    size_t a_size = dim(a, inv(a_idx)).size;
+    size_t b_size = dim(b, inv(b_idx)).size;
+    size_t common_size = dim(a, a_idx).size;
+    bool a_is_lhs = (dim(a, inv(a_idx)).name < dim(b, inv(b_idx)).name);
+    if (a_is_lhs) {
+        return stash.create<DenseMatMulFunction>(result_type, a, b, a_size, common_size, b_size, a_common_inner, b_common_inner);
+    } else {
+        return stash.create<DenseMatMulFunction>(result_type, b, a, b_size, common_size, a_size, b_common_inner, a_common_inner);
+    }
+}
+
+} // namespace vespalib::tensor::<unnamed>
+
+DenseMatMulFunction::Self::Self(const eval::ValueType &result_type_in,
+                                size_t lhs_size_in,
+                                size_t common_size_in,
+                                size_t rhs_size_in)
+    : result_type(result_type_in),
+      lhs_size(lhs_size_in),
+      common_size(common_size_in),
+      rhs_size(rhs_size_in)
+{
+}
+
+DenseMatMulFunction::Self::~Self() = default;
+
+DenseMatMulFunction::DenseMatMulFunction(const eval::ValueType &result_type,
+                                         const eval::TensorFunction &lhs_in,
+                                         const eval::TensorFunction &rhs_in,
+                                         size_t lhs_size,
+                                         size_t common_size,
+                                         size_t rhs_size,
+                                         bool lhs_common_inner,
+                                         bool rhs_common_inner)
+    : Super(result_type, lhs_in, rhs_in),
+      _lhs_size(lhs_size),
+      _common_size(common_size),
+      _rhs_size(rhs_size),
+      _lhs_common_inner(lhs_common_inner),
+      _rhs_common_inner(rhs_common_inner)
+{
+}
+
+DenseMatMulFunction::~DenseMatMulFunction() = default;
+
+eval::InterpretedFunction::Instruction
+DenseMatMulFunction::compile_self(const TensorEngine &, Stash &stash) const
+{
+    Self &self = stash.create<Self>(result_type(), _lhs_size, _common_size, _rhs_size);
+    auto op = my_select(lhs().result_type().cell_type(), rhs().result_type().cell_type(),
+                        _lhs_common_inner, _rhs_common_inner);
+    return eval::InterpretedFunction::Instruction(op, (uint64_t)(&self));
+}
+
+void
+DenseMatMulFunction::visit_self(vespalib::ObjectVisitor &visitor) const
+{
+    Super::visit_self(visitor);
+    visitor.visitInt("lhs_size", _lhs_size);
+    visitor.visitInt("common_size", _common_size);
+    visitor.visitInt("rhs_size", _rhs_size);
+    visitor.visitBool("lhs_common_inner", _lhs_common_inner);
+    visitor.visitBool("rhs_common_inner", _rhs_common_inner);
+}
+
+const TensorFunction &
+DenseMatMulFunction::optimize(const eval::TensorFunction &expr, Stash &stash)
+{
+    auto reduce = as<Reduce>(expr);
+    if (reduce && (reduce->aggr() == Aggr::SUM) && (reduce->dimensions().size() == 1)) {
+        auto join = as<Join>(reduce->child());
+        if (join && (join->function() == Mul::f)) {
+            const TensorFunction &a = join->lhs();
+            const TensorFunction &b = join->rhs();
+            if (is_matmul(a.result_type(), b.result_type(), reduce->dimensions()[0], expr.result_type())) {
+                return create_matmul(a, b, reduce->dimensions()[0], expr.result_type(), stash);
+            }
+        }
+    }
+    return expr;
+}
+
+} // namespace vespalib::tensor
diff --git a/eval/src/vespa/eval/tensor/dense/dense_matmul_function.h b/eval/src/vespa/eval/tensor/dense/dense_matmul_function.h
new file mode 100644
index 00000000000..88d7b9f37e0
--- /dev/null
+++ b/eval/src/vespa/eval/tensor/dense/dense_matmul_function.h
@@ -0,0 +1,58 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/eval/eval/tensor_function.h>
+#include "dense_tensor_view.h"
+
+namespace vespalib::tensor {
+
+/**
+ * Tensor function for dense matrix multiplication.
+ **/
+class DenseMatMulFunction : public eval::tensor_function::Op2
+{
+    using Super = eval::tensor_function::Op2;
+public:
+    struct Self {
+        eval::ValueType result_type;
+        size_t lhs_size;
+        size_t common_size;
+        size_t rhs_size;
+        Self(const eval::ValueType &result_type_in,
+             size_t lhs_size_in, size_t common_size_in, size_t rhs_size_in);
+        ~Self();
+    };
+
+private:
+    size_t _lhs_size;
+    size_t _common_size;
+    size_t _rhs_size;
+    bool   _lhs_common_inner;
+    bool   _rhs_common_inner;
+
+public:
+    DenseMatMulFunction(const eval::ValueType &result_type,
+                        const eval::TensorFunction &lhs_in,
+                        const eval::TensorFunction &rhs_in,
+                        size_t lhs_size,
+                        size_t common_size,
+                        size_t rhs_size,
+                        bool lhs_common_inner,
+                        bool rhs_common_inner);
+    ~DenseMatMulFunction();
+
+    bool result_is_mutable() const override { return true; }
+
+    size_t lhs_size() const { return _lhs_size; }
+    size_t common_size() const { return _common_size; }
+    size_t rhs_size() const { return _rhs_size; }
+    bool lhs_common_inner() const { return _lhs_common_inner; }
+    bool rhs_common_inner() const { return _rhs_common_inner; }
+
+    eval::InterpretedFunction::Instruction compile_self(const eval::TensorEngine &engine, Stash &stash) const override;
+    void visit_self(vespalib::ObjectVisitor &visitor) const override;
+    static const eval::TensorFunction &optimize(const eval::TensorFunction &expr, Stash &stash);
+};
+
+} // namespace vespalib::tensor
diff --git a/eval/src/vespa/eval/tensor/dense/dense_replace_type_function.cpp b/eval/src/vespa/eval/tensor/dense/dense_replace_type_function.cpp
index b81b0f2c876..e2dc2241555 100644
--- a/eval/src/vespa/eval/tensor/dense/dense_replace_type_function.cpp
+++ b/eval/src/vespa/eval/tensor/dense/dense_replace_type_function.cpp
@@ -9,6 +9,7 @@ namespace vespalib::tensor {
 using eval::Value;
 using eval::ValueType;
 using eval::TensorFunction;
+using eval::TensorEngine;
 using eval::as;
 using namespace eval::tensor_function;
 
@@ -38,7 +39,7 @@ DenseReplaceTypeFunction::~DenseReplaceTypeFunction()
 }
 
 eval::InterpretedFunction::Instruction
-DenseReplaceTypeFunction::compile_self(Stash &) const
+DenseReplaceTypeFunction::compile_self(const TensorEngine &, Stash &) const
 {
     return eval::InterpretedFunction::Instruction(my_replace_type_op, (uint64_t)&(result_type()));
 }
diff --git a/eval/src/vespa/eval/tensor/dense/dense_replace_type_function.h b/eval/src/vespa/eval/tensor/dense/dense_replace_type_function.h
index 4ad1f4c1cee..22c3886022d 100644
--- a/eval/src/vespa/eval/tensor/dense/dense_replace_type_function.h
+++ b/eval/src/vespa/eval/tensor/dense/dense_replace_type_function.h
@@ -16,7 +16,7 @@ public:
     DenseReplaceTypeFunction(const eval::ValueType &result_type,
                              const eval::TensorFunction &child);
     ~DenseReplaceTypeFunction();
-    eval::InterpretedFunction::Instruction compile_self(Stash &stash) const override;
+    eval::InterpretedFunction::Instruction compile_self(const eval::TensorEngine &engine, Stash &stash) const override;
     bool result_is_mutable() const override { return child().result_is_mutable(); }
     static const DenseReplaceTypeFunction &create_compact(const eval::ValueType &result_type,
             const eval::TensorFunction &child,
diff --git a/eval/src/vespa/eval/tensor/dense/dense_tensor.cpp b/eval/src/vespa/eval/tensor/dense/dense_tensor.cpp
index 73cdfc12a38..7faba87dbd3 100644
--- a/eval/src/vespa/eval/tensor/dense/dense_tensor.cpp
+++ b/eval/src/vespa/eval/tensor/dense/dense_tensor.cpp
@@ -3,7 +3,6 @@
 #include "dense_tensor.h"
 #include <vespa/vespalib/util/stringfmt.h>
 #include <vespa/vespalib/util/exceptions.h>
-#include <vespa/eval/eval/operation.h>
 
 using vespalib::eval::TensorSpec;
 
diff --git a/eval/src/vespa/eval/tensor/dense/dense_tensor_address_mapper.cpp b/eval/src/vespa/eval/tensor/dense/dense_tensor_address_mapper.cpp
index c1c24d28b7f..f32af0d097b 100644
--- a/eval/src/vespa/eval/tensor/dense/dense_tensor_address_mapper.cpp
+++ b/eval/src/vespa/eval/tensor/dense/dense_tensor_address_mapper.cpp
@@ -2,7 +2,6 @@
 
 #include "dense_tensor_address_mapper.h"
 #include <vespa/eval/eval/value_type.h>
-#include <vespa/eval/tensor/types.h>
 #include <vespa/eval/tensor/tensor_address.h>
 #include <vespa/eval/tensor/tensor_address_element_iterator.h>
 
diff --git a/eval/src/vespa/eval/tensor/dense/dense_tensor_create_function.cpp b/eval/src/vespa/eval/tensor/dense/dense_tensor_create_function.cpp
index d590e870cfe..3533ab20175 100644
--- a/eval/src/vespa/eval/tensor/dense/dense_tensor_create_function.cpp
+++ b/eval/src/vespa/eval/tensor/dense/dense_tensor_create_function.cpp
@@ -2,7 +2,6 @@
 
 #include "dense_tensor_create_function.h"
 #include "dense_tensor_view.h"
-#include <vespa/eval/eval/operation.h>
 #include <vespa/eval/eval/value.h>
 #include <vespa/eval/tensor/tensor.h>
 
@@ -13,10 +12,10 @@ using eval::DoubleValue;
 using eval::ValueType;
 using eval::TensorSpec;
 using eval::TensorFunction;
+using eval::TensorEngine;
 using Child = eval::TensorFunction::Child;
 using eval::as;
 using namespace eval::tensor_function;
-using namespace eval::operation;
 
 namespace {
 
@@ -30,7 +29,7 @@ void my_tensor_create_op(eval::InterpretedFunction::State &state, uint64_t param
         state.stack.pop_back();
     }
     const Value &result = state.stash.create<DenseTensorView>(self->result_type, TypedCells(cells)); 
-    state.stack.push_back(result);
+    state.stack.emplace_back(result);
 }
 
 struct MyTensorCreateOp {
@@ -59,20 +58,18 @@ DenseTensorCreateFunction::DenseTensorCreateFunction(const ValueType &res_type,
 {
 }
 
-DenseTensorCreateFunction::~DenseTensorCreateFunction()
-{
-}
+DenseTensorCreateFunction::~DenseTensorCreateFunction() = default;
 
 void
 DenseTensorCreateFunction::push_children(std::vector<Child::CREF> &target) const
 {
     for (const Child &c : _children) {
-        target.push_back(c);
+        target.emplace_back(c);
     }
 }
 
 eval::InterpretedFunction::Instruction
-DenseTensorCreateFunction::compile_self(Stash &) const
+DenseTensorCreateFunction::compile_self(const TensorEngine &, Stash &) const
 {
     static_assert(sizeof(uint64_t) == sizeof(&_self));
     auto op = select_1<MyTensorCreateOp>(result_type().cell_type());
diff --git a/eval/src/vespa/eval/tensor/dense/dense_tensor_create_function.h b/eval/src/vespa/eval/tensor/dense/dense_tensor_create_function.h
index 6d262f48aa6..d471658fba0 100644
--- a/eval/src/vespa/eval/tensor/dense/dense_tensor_create_function.h
+++ b/eval/src/vespa/eval/tensor/dense/dense_tensor_create_function.h
@@ -25,7 +25,7 @@ public:
     ~DenseTensorCreateFunction();
     const eval::ValueType &result_type() const override { return _self.result_type; }
     void push_children(std::vector<Child::CREF> &children) const override;
-    eval::InterpretedFunction::Instruction compile_self(Stash &stash) const override;
+    eval::InterpretedFunction::Instruction compile_self(const eval::TensorEngine &engine, Stash &stash) const override;
     bool result_is_mutable() const override { return true; }
     static const eval::TensorFunction &optimize(const eval::TensorFunction &expr, Stash &stash);
 };
diff --git a/eval/src/vespa/eval/tensor/dense/dense_tensor_peek_function.cpp b/eval/src/vespa/eval/tensor/dense/dense_tensor_peek_function.cpp
index 2920ca26234..5cb1cbfd88f 100644
--- a/eval/src/vespa/eval/tensor/dense/dense_tensor_peek_function.cpp
+++ b/eval/src/vespa/eval/tensor/dense/dense_tensor_peek_function.cpp
@@ -2,8 +2,6 @@
 
 #include "dense_tensor_peek_function.h"
 #include "dense_tensor_view.h"
-#include <vespa/vespalib/util/overload.h>
-#include <vespa/eval/eval/operation.h>
 #include <vespa/eval/eval/value.h>
 #include <vespa/eval/tensor/tensor.h>
 
@@ -14,10 +12,10 @@ using eval::DoubleValue;
 using eval::ValueType;
 using eval::TensorSpec;
 using eval::TensorFunction;
+using eval::TensorEngine;
 using Child = eval::TensorFunction::Child;
 using eval::as;
 using namespace eval::tensor_function;
-using namespace eval::operation;
 
 namespace {
 
@@ -31,7 +29,7 @@ void my_tensor_peek_op(eval::InterpretedFunction::State &state, uint64_t param)
         if (dim.first >= 0) {
             idx += (dim.first * factor);
         } else {
-            size_t dim_idx(round(state.peek(0).as_double()));
+            size_t dim_idx = state.peek(0).as_double();
             state.stack.pop_back();
             valid &= (dim_idx < dim.second);
             idx += (dim_idx * factor);
@@ -41,7 +39,7 @@ void my_tensor_peek_op(eval::InterpretedFunction::State &state, uint64_t param)
     auto cells = DenseTensorView::typify_cells<CT>(state.peek(0));
     state.stack.pop_back();
     const Value &result = state.stash.create<DoubleValue>(valid ? cells[idx] : 0.0);
-    state.stack.push_back(result);
+    state.stack.emplace_back(result);
 }
 
 struct MyTensorPeekOp {
@@ -52,10 +50,10 @@ struct MyTensorPeekOp {
 } // namespace vespalib::tensor::<unnamed>
 
 DenseTensorPeekFunction::DenseTensorPeekFunction(std::vector<Child> children,
-                                                 const std::vector<std::pair<int64_t,size_t>> &spec)
+                                                 std::vector<std::pair<int64_t,size_t>> spec)
     : TensorFunction(),
       _children(std::move(children)),
-      _spec(spec)
+      _spec(std::move(spec))
 {
 }
 
@@ -65,12 +63,12 @@ void
 DenseTensorPeekFunction::push_children(std::vector<Child::CREF> &target) const
 {
     for (const Child &c: _children) {
-        target.push_back(c);
+        target.emplace_back(c);
     }
 }
 
 eval::InterpretedFunction::Instruction
-DenseTensorPeekFunction::compile_self(Stash &) const
+DenseTensorPeekFunction::compile_self(const TensorEngine &, Stash &) const
 {
     static_assert(sizeof(uint64_t) == sizeof(&_spec));
     auto op = select_1<MyTensorPeekOp>(_children[0].get().result_type().cell_type());
@@ -100,7 +98,7 @@ DenseTensorPeekFunction::optimize(const eval::TensorFunction &expr, Stash &stash
                                }
                            }, dim_spec->second);
             }
-            return stash.create<DenseTensorPeekFunction>(peek->copy_children(), spec);
+            return stash.create<DenseTensorPeekFunction>(peek->copy_children(), std::move(spec));
         }
     }
     return expr;
diff --git a/eval/src/vespa/eval/tensor/dense/dense_tensor_peek_function.h b/eval/src/vespa/eval/tensor/dense/dense_tensor_peek_function.h
index 4924a0de329..8ed672f95b0 100644
--- a/eval/src/vespa/eval/tensor/dense/dense_tensor_peek_function.h
+++ b/eval/src/vespa/eval/tensor/dense/dense_tensor_peek_function.h
@@ -22,11 +22,11 @@ private:
     // (note that child expression order is inverted by the stack)
     std::vector<std::pair<int64_t,size_t>> _spec;
 public:
-    DenseTensorPeekFunction(std::vector<Child> children, const std::vector<std::pair<int64_t,size_t>> &spec);
+    DenseTensorPeekFunction(std::vector<Child> children, std::vector<std::pair<int64_t,size_t>> spec);
     ~DenseTensorPeekFunction();
     const eval::ValueType &result_type() const override { return eval::DoubleValue::double_type(); }
     void push_children(std::vector<Child::CREF> &children) const override;
-    eval::InterpretedFunction::Instruction compile_self(Stash &stash) const override;
+    eval::InterpretedFunction::Instruction compile_self(const eval::TensorEngine &engine, Stash &stash) const override;
     bool result_is_mutable() const override { return true; }
     static const eval::TensorFunction &optimize(const eval::TensorFunction &expr, Stash &stash);
 };
diff --git a/eval/src/vespa/eval/tensor/dense/dense_tensor_reduce.cpp b/eval/src/vespa/eval/tensor/dense/dense_tensor_reduce.cpp
index 252be199208..d44fe88bb10 100644
--- a/eval/src/vespa/eval/tensor/dense/dense_tensor_reduce.cpp
+++ b/eval/src/vespa/eval/tensor/dense/dense_tensor_reduce.cpp
@@ -14,8 +14,7 @@ DimensionReducer::calcCellsSize(const eval::ValueType &type)
     return cellsSize;
 }
 
-DimensionReducer::DimensionReducer(const eval::ValueType &oldType,
-                 const string &dimensionToRemove)
+DimensionReducer::DimensionReducer(const eval::ValueType &oldType, const string &dimensionToRemove)
         : _type(oldType.reduce({ dimensionToRemove })),
           _innerDimSize(1),
           _sumDimSize(1),
diff --git a/eval/src/vespa/eval/tensor/dense/dense_tensor_view.cpp b/eval/src/vespa/eval/tensor/dense/dense_tensor_view.cpp
index e450aa49284..9cb0b7d2510 100644
--- a/eval/src/vespa/eval/tensor/dense/dense_tensor_view.cpp
+++ b/eval/src/vespa/eval/tensor/dense/dense_tensor_view.cpp
@@ -138,7 +138,7 @@ joinDenseTensors(const DenseTensorView &lhs, const Tensor &rhs,
                  vespalib::stringref operation,
                  Function &&func)
 {
-    const DenseTensorView *view = dynamic_cast<const DenseTensorView *>(&rhs);
+    auto view = dynamic_cast<const DenseTensorView *>(&rhs);
     if (view) {
         checkDimensions(lhs, *view, operation);
         return joinDenseTensors(lhs, *view, func);
@@ -215,7 +215,7 @@ DenseTensorView::apply(const CellFunction &func) const
 bool
 DenseTensorView::equals(const Tensor &arg) const
 {
-    const DenseTensorView *view = dynamic_cast<const DenseTensorView *>(&arg);
+    auto view = dynamic_cast<const DenseTensorView *>(&arg);
     if (view) {
         return *this == *view;
     }
diff --git a/eval/src/vespa/eval/tensor/dense/dense_xw_product_function.cpp b/eval/src/vespa/eval/tensor/dense/dense_xw_product_function.cpp
index 2db5b4e8f92..a0d63a1ce1e 100644
--- a/eval/src/vespa/eval/tensor/dense/dense_xw_product_function.cpp
+++ b/eval/src/vespa/eval/tensor/dense/dense_xw_product_function.cpp
@@ -1,19 +1,19 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include "dense_xw_product_function.h"
-#include "dense_tensor.h"
 #include "dense_tensor_view.h"
 #include <vespa/vespalib/objects/objectvisitor.h>
 #include <vespa/eval/eval/value.h>
 #include <vespa/eval/eval/operation.h>
-#include <vespa/eval/tensor/tensor.h>
-#include <vespa/vespalib/util/exceptions.h>
-#include <assert.h>
+#include <cassert>
+
+#include <cblas.h>
 
 namespace vespalib::tensor {
 
 using eval::ValueType;
 using eval::TensorFunction;
+using eval::TensorEngine;
 using eval::as;
 using eval::Aggr;
 using namespace eval::tensor_function;
@@ -21,76 +21,59 @@ using namespace eval::operation;
 
 namespace {
 
-template <typename LCT, typename RCT>
-struct HWSupport {
-    static double call(hwaccelrated::IAccelrated *, const LCT *lhs, const RCT *rhs, size_t len) {
-        double result = 0.0;
-        for (size_t i = 0; i < len; ++i) {
-            result += (lhs[i] * rhs[i]);
-        }
-        return result;
+template <typename LCT, typename RCT, bool common_inner>
+double my_dot_product(const LCT *lhs, const RCT *rhs, size_t vector_size, size_t result_size) {
+    double result = 0.0;
+    for (size_t i = 0; i < vector_size; ++i) {
+        result += ((*lhs) * (*rhs));
+        ++lhs;
+        rhs += (common_inner ? 1 : result_size);
     }
-};
-template <> struct HWSupport<float, float> {
-    static double call(hwaccelrated::IAccelrated *hw, const float *lhs, const float *rhs, size_t len) {
-        return hw->dotProduct(lhs, rhs, len);
-    }
-};
-template <> struct HWSupport<double, double> {
-    static double call(hwaccelrated::IAccelrated *hw, const double *lhs, const double *rhs, size_t len) {
-        return hw->dotProduct(lhs, rhs, len);
-    }
-};
-
-template <typename LCT, typename RCT, typename OCT>
-void multiDotProduct(const DenseXWProductFunction::Self &self,
-                     const ConstArrayRef<LCT> &vectorCells, const ConstArrayRef<RCT> &matrixCells, ArrayRef<OCT> &result)
-{
-    OCT *out = result.begin();
-    const RCT *matrixP = matrixCells.cbegin();
-    const LCT * const vectorP = vectorCells.cbegin();
-    for (size_t row = 0; row < self._resultSize; ++row) {
-        double cell = HWSupport<LCT,RCT>::call(self._hwAccelerator.get(), vectorP, matrixP, self._vectorSize);
-        *out++ = cell;
-        matrixP += self._vectorSize;
-    }
-    assert(out == result.end());
-    assert(matrixP == matrixCells.cend());
-}
-
-template <typename LCT, typename RCT, typename OCT>
-void transposedProduct(const DenseXWProductFunction::Self &self,
-                       const ConstArrayRef<LCT> &vectorCells, const ConstArrayRef<RCT> &matrixCells, ArrayRef<OCT> &result)
-{
-    OCT *out = result.begin();
-    const RCT * const matrixP = matrixCells.cbegin();
-    const LCT * const vectorP = vectorCells.cbegin();
-    for (size_t row = 0; row < self._resultSize; ++row) {
-        double cell = 0;
-        for (size_t col = 0; col < self._vectorSize; ++col) {
-            cell += matrixP[col*self._resultSize + row] * vectorP[col];
-        }
-        *out++ = cell;
-    }
-    assert(out == result.end());
+    return result;
 }
 
-template <typename LCT, typename RCT, bool commonDimensionInnermost>
+template <typename LCT, typename RCT, bool common_inner>
 void my_xw_product_op(eval::InterpretedFunction::State &state, uint64_t param) {
-    DenseXWProductFunction::Self *self = (DenseXWProductFunction::Self *)(param);
-
+    const DenseXWProductFunction::Self &self = *((const DenseXWProductFunction::Self *)(param));
     using OCT = typename eval::UnifyCellTypes<LCT,RCT>::type;
-    auto vectorCells = DenseTensorView::typify_cells<LCT>(state.peek(1));
-    auto matrixCells = DenseTensorView::typify_cells<RCT>(state.peek(0));
-    auto outputCells = state.stash.create_array<OCT>(self->_resultSize);
-
-    if (commonDimensionInnermost) {
-        multiDotProduct(*self, vectorCells, matrixCells, outputCells);
-    } else {
-        transposedProduct(*self, vectorCells, matrixCells, outputCells);
+    auto vector_cells = DenseTensorView::typify_cells<LCT>(state.peek(1));
+    auto matrix_cells = DenseTensorView::typify_cells<RCT>(state.peek(0));
+    auto dst_cells = state.stash.create_array<OCT>(self.result_size);
+    OCT *dst = dst_cells.begin();
+    const RCT *matrix = matrix_cells.cbegin();
+    for (size_t i = 0; i < self.result_size; ++i) {
+        *dst++ = my_dot_product<LCT,RCT,common_inner>(vector_cells.cbegin(), matrix, self.vector_size, self.result_size);
+        matrix += (common_inner ? self.vector_size : 1);
     }
+    state.pop_pop_push(state.stash.create<DenseTensorView>(self.result_type, TypedCells(dst_cells)));
+}
+
+template <bool common_inner>
+void my_cblas_double_xw_product_op(eval::InterpretedFunction::State &state, uint64_t param) {
+    const DenseXWProductFunction::Self &self = *((const DenseXWProductFunction::Self *)(param));
+    auto vector_cells = DenseTensorView::typify_cells<double>(state.peek(1));
+    auto matrix_cells = DenseTensorView::typify_cells<double>(state.peek(0));
+    auto dst_cells = state.stash.create_array<double>(self.result_size);
+    cblas_dgemv(CblasRowMajor, common_inner ? CblasNoTrans : CblasTrans,
+                common_inner ? self.result_size : self.vector_size,
+                common_inner ? self.vector_size : self.result_size,
+                1.0, matrix_cells.cbegin(), common_inner ? self.vector_size : self.result_size, vector_cells.cbegin(), 1,
+                0.0, dst_cells.begin(), 1);
+    state.pop_pop_push(state.stash.create<DenseTensorView>(self.result_type, TypedCells(dst_cells)));
+}
 
-    state.pop_pop_push(state.stash.create<DenseTensorView>(self->_resultType, TypedCells(outputCells)));
+template <bool common_inner>
+void my_cblas_float_xw_product_op(eval::InterpretedFunction::State &state, uint64_t param) {
+    const DenseXWProductFunction::Self &self = *((const DenseXWProductFunction::Self *)(param));
+    auto vector_cells = DenseTensorView::typify_cells<float>(state.peek(1));
+    auto matrix_cells = DenseTensorView::typify_cells<float>(state.peek(0));
+    auto dst_cells = state.stash.create_array<float>(self.result_size);
+    cblas_sgemv(CblasRowMajor, common_inner ? CblasNoTrans : CblasTrans,
+                common_inner ? self.result_size : self.vector_size,
+                common_inner ? self.vector_size : self.result_size,
+                1.0, matrix_cells.cbegin(), common_inner ? self.vector_size : self.result_size, vector_cells.cbegin(), 1,
+                0.0, dst_cells.begin(), 1);
+    state.pop_pop_push(state.stash.create<DenseTensorView>(self.result_type, TypedCells(dst_cells)));
 }
 
 template <bool common_inner>
@@ -99,11 +82,24 @@ struct MyXWProductOp {
     static auto get_fun() { return my_xw_product_op<LCT,RCT,common_inner>; }
 };
 
-eval::InterpretedFunction::op_function my_select(CellType lct, CellType rct, bool common_innermost) {
-    if (common_innermost) {
-        return select_2<MyXWProductOp<true> >(lct, rct);
+template <bool common_inner>
+eval::InterpretedFunction::op_function my_select2(CellType lct, CellType rct) {
+    if (lct == rct) {
+        if (lct == ValueType::CellType::DOUBLE) {
+            return my_cblas_double_xw_product_op<common_inner>;
+        }
+        if (lct == ValueType::CellType::FLOAT) {
+            return my_cblas_float_xw_product_op<common_inner>;
+        }
+    }
+    return select_2<MyXWProductOp<common_inner>>(lct, rct);
+}
+
+eval::InterpretedFunction::op_function my_select(CellType lct, CellType rct, bool common_inner) {
+    if (common_inner) {
+        return my_select2<true>(lct, rct);
     } else {
-        return select_2<MyXWProductOp<false> >(lct, rct);
+        return my_select2<false>(lct, rct);
     }
 }
 
@@ -129,42 +125,43 @@ bool isDenseXWProduct(const ValueType &res, const ValueType &vec, const ValueTyp
 }
 
 const TensorFunction &createDenseXWProduct(const ValueType &res, const TensorFunction &vec, const TensorFunction &mat, Stash &stash) {
-    bool common_is_inner = (mat.result_type().dimension_index(vec.result_type().dimensions()[0].name) == 1);
+    bool common_inner = (mat.result_type().dimension_index(vec.result_type().dimensions()[0].name) == 1);
     return stash.create<DenseXWProductFunction>(res, vec, mat,
                                                 vec.result_type().dimensions()[0].size,
                                                 res.dimensions()[0].size,
-                                                common_is_inner);
+                                                common_inner);
 }
 
 } // namespace vespalib::tensor::<unnamed>
 
-DenseXWProductFunction::Self::Self(const eval::ValueType &resultType,
-                                   size_t vectorSize,
-                                   size_t resultSize)
-    : _resultType(resultType),
-      _vectorSize(vectorSize),
-      _resultSize(resultSize),
-      _hwAccelerator(hwaccelrated::IAccelrated::getAccelrator())
-{}
+DenseXWProductFunction::Self::Self(const eval::ValueType &result_type_in,
+                                   size_t vector_size_in, size_t result_size_in)
+    : result_type(result_type_in),
+      vector_size(vector_size_in),
+      result_size(result_size_in)
+{
+}
+DenseXWProductFunction::Self::~Self() = default;
 
-DenseXWProductFunction::DenseXWProductFunction(const eval::ValueType &resultType,
+DenseXWProductFunction::DenseXWProductFunction(const eval::ValueType &result_type,
                                                const eval::TensorFunction &vector_in,
                                                const eval::TensorFunction &matrix_in,
-                                               size_t vectorSize,
-                                               size_t resultSize,
-                                               bool matrixHasCommonDimensionInnermost)
-    : eval::tensor_function::Op2(resultType, vector_in, matrix_in),
-      _vectorSize(vectorSize),
-      _resultSize(resultSize),
-      _commonDimensionInnermost(matrixHasCommonDimensionInnermost)
-{}
+                                               size_t vector_size,
+                                               size_t result_size,
+                                               bool common_inner)
+    : eval::tensor_function::Op2(result_type, vector_in, matrix_in),
+      _vector_size(vector_size),
+      _result_size(result_size),
+      _common_inner(common_inner)
+{
+}
 
 eval::InterpretedFunction::Instruction
-DenseXWProductFunction::compile_self(Stash &stash) const
+DenseXWProductFunction::compile_self(const TensorEngine &, Stash &stash) const
 {
-    Self &self = stash.create<Self>(result_type(), _vectorSize, _resultSize);
+    Self &self = stash.create<Self>(result_type(), _vector_size, _result_size);
     auto op = my_select(lhs().result_type().cell_type(),
-                        rhs().result_type().cell_type(), _commonDimensionInnermost);
+                        rhs().result_type().cell_type(), _common_inner);
     return eval::InterpretedFunction::Instruction(op, (uint64_t)(&self));
 }
 
@@ -172,9 +169,9 @@ void
 DenseXWProductFunction::visit_self(vespalib::ObjectVisitor &visitor) const
 {
     Super::visit_self(visitor);
-    visitor.visitInt("vector_size", _vectorSize);
-    visitor.visitInt("result_size", _resultSize);
-    visitor.visitBool("common_dimension_innermost", _commonDimensionInnermost);
+    visitor.visitInt("vector_size", _vector_size);
+    visitor.visitInt("result_size", _result_size);
+    visitor.visitBool("common_inner", _common_inner);
 }
 
 const TensorFunction &
diff --git a/eval/src/vespa/eval/tensor/dense/dense_xw_product_function.h b/eval/src/vespa/eval/tensor/dense/dense_xw_product_function.h
index f2f4d67c0f0..9f05222fff6 100644
--- a/eval/src/vespa/eval/tensor/dense/dense_xw_product_function.h
+++ b/eval/src/vespa/eval/tensor/dense/dense_xw_product_function.h
@@ -4,7 +4,6 @@
 
 #include <vespa/eval/eval/tensor_function.h>
 #include "dense_tensor_view.h"
-#include <vespa/vespalib/hwaccelrated/iaccelrated.h>
 
 namespace vespalib::tensor {
 
@@ -16,39 +15,36 @@ class DenseXWProductFunction : public eval::tensor_function::Op2
     using Super = eval::tensor_function::Op2;
 public:
     struct Self {
-        const eval::ValueType _resultType;
-        const size_t _vectorSize;
-        const size_t _resultSize;
-        hwaccelrated::IAccelrated::UP _hwAccelerator;
-        Self(const eval::ValueType &resultType,
-             size_t vectorSize,
-             size_t resultSize);
-        ~Self() {}
+        eval::ValueType result_type;
+        size_t vector_size;
+        size_t result_size;
+        Self(const eval::ValueType &result_type_in,
+             size_t vector_size_in, size_t result_size_in);
+        ~Self();
     };
 
 private:
-    const size_t _vectorSize;
-    const size_t _resultSize;
-    bool _commonDimensionInnermost;
+    size_t _vector_size;
+    size_t _result_size;
+    bool _common_inner;
 
 public:
-    DenseXWProductFunction(const eval::ValueType &resultType,
+    DenseXWProductFunction(const eval::ValueType &result_type,
                            const eval::TensorFunction &vector_in,
                            const eval::TensorFunction &matrix_in,
-                           size_t vectorSize,
-                           size_t resultSize,
-                           bool matrixHasCommonDimensionInnermost);
+                           size_t vector_size,
+                           size_t result_size,
+                           bool common_inner);
 
     ~DenseXWProductFunction() {}
 
     bool result_is_mutable() const override { return true; }
 
-    size_t vectorSize() const { return _vectorSize; }
-    size_t resultSize() const { return _resultSize; }
+    size_t vector_size() const { return _vector_size; }
+    size_t result_size() const { return _result_size; }
+    bool common_inner() const { return _common_inner; }
 
-    bool matrixHasCommonDimensionInnermost() const { return _commonDimensionInnermost; }
-
-    eval::InterpretedFunction::Instruction compile_self(Stash &stash) const override;
+    eval::InterpretedFunction::Instruction compile_self(const eval::TensorEngine &engine, Stash &stash) const override;
     void visit_self(vespalib::ObjectVisitor &visitor) const override;
     static const eval::TensorFunction &optimize(const eval::TensorFunction &expr, Stash &stash);
 };
diff --git a/eval/src/vespa/eval/tensor/dense/vector_from_doubles_function.cpp b/eval/src/vespa/eval/tensor/dense/vector_from_doubles_function.cpp
index 4694aea717d..7a4b5917f00 100644
--- a/eval/src/vespa/eval/tensor/dense/vector_from_doubles_function.cpp
+++ b/eval/src/vespa/eval/tensor/dense/vector_from_doubles_function.cpp
@@ -1,21 +1,18 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include "vector_from_doubles_function.h"
-#include "dense_tensor.h"
 #include "dense_tensor_view.h"
-#include <vespa/eval/eval/operation.h>
 #include <vespa/eval/eval/value.h>
-#include <vespa/eval/tensor/tensor.h>
 
 namespace vespalib::tensor {
 
 using eval::Value;
 using eval::ValueType;
 using eval::TensorFunction;
+using eval::TensorEngine;
 using Child = eval::TensorFunction::Child;
 using eval::as;
 using namespace eval::tensor_function;
-using namespace eval::operation;
 
 namespace {
 
@@ -38,7 +35,7 @@ void my_vector_from_doubles_op(eval::InterpretedFunction::State &state, uint64_t
     size_t numCells = self->resultSize;
     TypedCells cells = dispatch_0<CallVectorFromDoubles>(ct, state, numCells);
     const Value &result = state.stash.create<DenseTensorView>(self->resultType, cells);
-    state.stack.push_back(result);
+    state.stack.emplace_back(result);
 }
 
 size_t vector_size(const TensorFunction &child, const vespalib::string &dimension) {
@@ -55,7 +52,7 @@ size_t vector_size(const TensorFunction &child, const vespalib::string &dimensio
 
 void flatten_into(const TensorFunction &child, std::vector<Child> &vec) {
     if (child.result_type().is_double()) {
-        vec.push_back(child);
+        vec.emplace_back(child);
     } else {
         std::vector<Child::CREF> tmp;
         child.push_children(tmp);
@@ -83,20 +80,18 @@ VectorFromDoublesFunction::VectorFromDoublesFunction(std::vector<Child> children
 {
 }
 
-VectorFromDoublesFunction::~VectorFromDoublesFunction()
-{
-}
+VectorFromDoublesFunction::~VectorFromDoublesFunction() = default;
 
 void
 VectorFromDoublesFunction::push_children(std::vector<Child::CREF> &target) const
 {
     for (const Child &c : _children) {
-        target.push_back(c);
+        target.emplace_back(c);
     }
 }
 
 eval::InterpretedFunction::Instruction
-VectorFromDoublesFunction::compile_self(Stash &) const
+VectorFromDoublesFunction::compile_self(const TensorEngine &, Stash &) const
 {
     return eval::InterpretedFunction::Instruction(my_vector_from_doubles_op, (uint64_t)&_self);
 }
diff --git a/eval/src/vespa/eval/tensor/dense/vector_from_doubles_function.h b/eval/src/vespa/eval/tensor/dense/vector_from_doubles_function.h
index 378c9026f84..28346c4cb3b 100644
--- a/eval/src/vespa/eval/tensor/dense/vector_from_doubles_function.h
+++ b/eval/src/vespa/eval/tensor/dense/vector_from_doubles_function.h
@@ -30,7 +30,7 @@ public:
         return _self.resultType.dimensions()[0].name;
     }
     size_t size() const { return _self.resultSize; }
-    eval::InterpretedFunction::Instruction compile_self(Stash &stash) const override;
+    eval::InterpretedFunction::Instruction compile_self(const eval::TensorEngine &engine, Stash &stash) const override;
     bool result_is_mutable() const override { return true; }
     static const eval::TensorFunction &optimize(const eval::TensorFunction &expr, Stash &stash);
 };
diff --git a/eval/src/vespa/eval/tensor/sparse/CMakeLists.txt b/eval/src/vespa/eval/tensor/sparse/CMakeLists.txt
index 8e1d18a87b7..a25d2abb477 100644
--- a/eval/src/vespa/eval/tensor/sparse/CMakeLists.txt
+++ b/eval/src/vespa/eval/tensor/sparse/CMakeLists.txt
@@ -1,6 +1,7 @@
 # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 vespa_add_library(eval_tensor_sparse OBJECT
     SOURCES
+    direct_sparse_tensor_builder.cpp
     sparse_tensor.cpp
     sparse_tensor_add.cpp
     sparse_tensor_address_builder.cpp
diff --git a/eval/src/vespa/eval/tensor/sparse/direct_sparse_tensor_builder.cpp b/eval/src/vespa/eval/tensor/sparse/direct_sparse_tensor_builder.cpp
new file mode 100644
index 00000000000..c47521e702d
--- /dev/null
+++ b/eval/src/vespa/eval/tensor/sparse/direct_sparse_tensor_builder.cpp
@@ -0,0 +1,50 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "direct_sparse_tensor_builder.h"
+
+namespace vespalib::tensor {
+
+void
+DirectSparseTensorBuilder::copyCells(const Cells &cells_in)
+{
+    for (const auto &cell : cells_in) {
+        SparseTensorAddressRef oldRef = cell.first;
+        SparseTensorAddressRef newRef(oldRef, _stash);
+        _cells[newRef] = cell.second;
+    }
+}
+
+DirectSparseTensorBuilder::DirectSparseTensorBuilder()
+    : _stash(SparseTensor::STASH_CHUNK_SIZE),
+      _type(eval::ValueType::double_type()),
+      _cells()
+{
+}
+
+DirectSparseTensorBuilder::DirectSparseTensorBuilder(const eval::ValueType &type_in)
+    : _stash(SparseTensor::STASH_CHUNK_SIZE),
+      _type(type_in),
+      _cells()
+{
+}
+
+DirectSparseTensorBuilder::DirectSparseTensorBuilder(const eval::ValueType &type_in, const Cells &cells_in)
+    : _stash(SparseTensor::STASH_CHUNK_SIZE),
+      _type(type_in),
+      _cells()
+{
+    copyCells(cells_in);
+}
+
+DirectSparseTensorBuilder::~DirectSparseTensorBuilder() = default;
+
+Tensor::UP
+DirectSparseTensorBuilder::build() {
+    return std::make_unique<SparseTensor>(std::move(_type), std::move(_cells), std::move(_stash));
+}
+
+void DirectSparseTensorBuilder::reserve(uint32_t estimatedCells) {
+    _cells.resize(estimatedCells*2);
+}
+
+}
\ No newline at end of file
diff --git a/eval/src/vespa/eval/tensor/sparse/direct_sparse_tensor_builder.h b/eval/src/vespa/eval/tensor/sparse/direct_sparse_tensor_builder.h
index d54c8810a81..bcb22c0761d 100644
--- a/eval/src/vespa/eval/tensor/sparse/direct_sparse_tensor_builder.h
+++ b/eval/src/vespa/eval/tensor/sparse/direct_sparse_tensor_builder.h
@@ -25,43 +25,13 @@ private:
     Cells _cells;
 
 public:
-    void
-    copyCells(const Cells &cells_in)
-    {
-        for (const auto &cell : cells_in) {
-            SparseTensorAddressRef oldRef = cell.first;
-            SparseTensorAddressRef newRef(oldRef, _stash);
-            _cells[newRef] = cell.second;
-        }
-    }
-
-    DirectSparseTensorBuilder()
-        : _stash(SparseTensor::STASH_CHUNK_SIZE),
-          _type(eval::ValueType::double_type()),
-          _cells()
-    {
-    }
+    void copyCells(const Cells &cells_in);
+    DirectSparseTensorBuilder();
+    DirectSparseTensorBuilder(const eval::ValueType &type_in);
+    DirectSparseTensorBuilder(const eval::ValueType &type_in, const Cells &cells_in);
+    ~DirectSparseTensorBuilder();
 
-    DirectSparseTensorBuilder(const eval::ValueType &type_in)
-        : _stash(SparseTensor::STASH_CHUNK_SIZE),
-          _type(type_in),
-          _cells()
-    {
-    }
-
-    DirectSparseTensorBuilder(const eval::ValueType &type_in, const Cells &cells_in)
-        : _stash(SparseTensor::STASH_CHUNK_SIZE),
-          _type(type_in),
-          _cells()
-    {
-        copyCells(cells_in);
-    }
-
-    ~DirectSparseTensorBuilder() {};
-
-    Tensor::UP build() {
-        return std::make_unique<SparseTensor>(std::move(_type), std::move(_cells), std::move(_stash));
-    }
+    Tensor::UP build();
 
     template <class Function>
     void insertCell(SparseTensorAddressRef address, double value, Function &&func)
@@ -81,8 +51,7 @@ public:
     }
 
     template <class Function>
-    void insertCell(SparseTensorAddressBuilder &address, double value, Function &&func)
-    {
+    void insertCell(SparseTensorAddressBuilder &address, double value, Function &&func) {
         insertCell(address.getAddressRef(), value, func);
     }
 
@@ -93,7 +62,7 @@ public:
 
     eval::ValueType &fast_type() { return _type; }
     Cells &cells() { return _cells; }
-    void reserve(uint32_t estimatedCells) { _cells.resize(estimatedCells*2); }
+    void reserve(uint32_t estimatedCells);
 };
 
 }
diff --git a/eval/src/vespa/eval/tensor/sparse/sparse_tensor.cpp b/eval/src/vespa/eval/tensor/sparse/sparse_tensor.cpp
index 1fc93e8234f..d183c33f5cd 100644
--- a/eval/src/vespa/eval/tensor/sparse/sparse_tensor.cpp
+++ b/eval/src/vespa/eval/tensor/sparse/sparse_tensor.cpp
@@ -245,5 +245,4 @@ SparseTensor::remove(const CellValues &cellAddresses) const
 
 }
 
-VESPALIB_HASH_MAP_INSTANTIATE_H_E_M(vespalib::tensor::SparseTensorAddressRef, double, vespalib::hash<vespalib::tensor::SparseTensorAddressRef>,
-        std::equal_to<vespalib::tensor::SparseTensorAddressRef>, vespalib::hashtable_base::and_modulator);
+VESPALIB_HASH_MAP_INSTANTIATE(vespalib::tensor::SparseTensorAddressRef, double);
diff --git a/eval/src/vespa/eval/tensor/sparse/sparse_tensor.h b/eval/src/vespa/eval/tensor/sparse/sparse_tensor.h
index 880cd32c605..e5ea639b460 100644
--- a/eval/src/vespa/eval/tensor/sparse/sparse_tensor.h
+++ b/eval/src/vespa/eval/tensor/sparse/sparse_tensor.h
@@ -22,7 +22,7 @@ class SparseTensor : public Tensor
 {
 public:
     using Cells = hash_map<SparseTensorAddressRef, double, hash<SparseTensorAddressRef>,
-            std::equal_to<SparseTensorAddressRef>, hashtable_base::and_modulator>;
+                           std::equal_to<>, hashtable_base::and_modulator>;
 
     static constexpr size_t STASH_CHUNK_SIZE = 16384u;
 
diff --git a/fastos/src/vespa/fastos/CMakeLists.txt b/fastos/src/vespa/fastos/CMakeLists.txt
index 1437f5c55f3..f062432d967 100644
--- a/fastos/src/vespa/fastos/CMakeLists.txt
+++ b/fastos/src/vespa/fastos/CMakeLists.txt
@@ -4,6 +4,7 @@ vespa_add_library(fastos_objects OBJECT
     app.cpp
     backtrace.c
     file.cpp
+    file_rw_ops.cpp
     linux_file.cpp
     process.cpp
     thread.cpp
diff --git a/fastos/src/vespa/fastos/file.cpp b/fastos/src/vespa/fastos/file.cpp
index 56f6addd743..861dd9f4259 100644
--- a/fastos/src/vespa/fastos/file.cpp
+++ b/fastos/src/vespa/fastos/file.cpp
@@ -184,12 +184,23 @@ FastOS_FileInterface::DirectIOPadding(int64_t offset,
 
 
 void *
-FastOS_FileInterface::AllocateDirectIOBuffer(size_t byteSize, void *&realPtr)
+FastOS_FileInterface::allocateGenericDirectIOBuffer(size_t byteSize, void *&realPtr)
 {
     realPtr = malloc(byteSize);    // Default - use malloc allignment
     return realPtr;
 }
 
+size_t
+FastOS_FileInterface::getMaxDirectIOMemAlign()
+{
+    return 1u;
+}
+
+void *
+FastOS_FileInterface::AllocateDirectIOBuffer(size_t byteSize, void *&realPtr)
+{
+    return allocateGenericDirectIOBuffer(byteSize, realPtr);
+}
 
 void
 FastOS_FileInterface::enableMemoryMap(int mmapFlags)
diff --git a/fastos/src/vespa/fastos/file.h b/fastos/src/vespa/fastos/file.h
index 15c4dfa33cd..3d2194229e7 100644
--- a/fastos/src/vespa/fastos/file.h
+++ b/fastos/src/vespa/fastos/file.h
@@ -559,6 +559,24 @@ public:
      *                           This value is always set.
      * @return Alligned pointer value or nullptr if out of memory
      */
+    static void *allocateGenericDirectIOBuffer(size_t byteSize, void *&realPtr);
+
+    /**
+     * Get maximum memory alignment for directio buffers.
+     * @return maximum memory alignment for directio buffers.
+     */
+    static size_t getMaxDirectIOMemAlign();
+
+    /**
+     * Allocate a buffer properly alligned with regards to direct io
+     * access restrictions.
+     * @param  byteSize          Number of bytes to be allocated
+     * @param  realPtr           Reference where the actual pointer returned
+     *                           from malloc will be saved.  Use free() with
+     *                           this pointer to deallocate the buffer.
+     *                           This value is always set.
+     * @return Alligned pointer value or nullptr if out of memory
+     */
     virtual void *AllocateDirectIOBuffer(size_t byteSize, void *&realPtr);
 
     /**
diff --git a/fastos/src/vespa/fastos/file_rw_ops.cpp b/fastos/src/vespa/fastos/file_rw_ops.cpp
new file mode 100644
index 00000000000..f5d81eab1cf
--- /dev/null
+++ b/fastos/src/vespa/fastos/file_rw_ops.cpp
@@ -0,0 +1,13 @@
+// Copyright 2020 Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "file_rw_ops.h"
+#include <unistd.h>
+
+namespace fastos {
+
+File_RW_Ops::ReadFunc File_RW_Ops::_read = ::read;
+File_RW_Ops::WriteFunc File_RW_Ops::_write = ::write;
+File_RW_Ops::PreadFunc File_RW_Ops::_pread = ::pread;
+File_RW_Ops::PwriteFunc File_RW_Ops::_pwrite = ::pwrite;
+
+}
diff --git a/fastos/src/vespa/fastos/file_rw_ops.h b/fastos/src/vespa/fastos/file_rw_ops.h
new file mode 100644
index 00000000000..9328bdbf9b4
--- /dev/null
+++ b/fastos/src/vespa/fastos/file_rw_ops.h
@@ -0,0 +1,33 @@
+// Copyright 2020 Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <sys/types.h>
+
+
+namespace fastos {
+
+/*
+ * Class handling pointers to functions used by FastOS_File for read
+ * and writa access. Unit tests might modify pointers to inject errors.
+ */
+class File_RW_Ops
+{
+    using ReadFunc = ssize_t (*)(int fd, void* buf, size_t count);
+    using WriteFunc = ssize_t (*)(int fd, const void* buf, size_t count);
+    using PreadFunc = ssize_t (*)(int fd, void* buf, size_t count, off_t offset);
+    using PwriteFunc = ssize_t (*)(int fd, const void* buf, size_t count, off_t offset);
+
+public:
+    static ReadFunc _read;
+    static WriteFunc _write;
+    static PreadFunc _pread;
+    static PwriteFunc _pwrite;
+
+    static ssize_t read(int fd, void* buf, size_t count) { return _read(fd, buf, count); }
+    static ssize_t write(int fd, const void* buf, size_t count) { return _write(fd, buf, count); }
+    static ssize_t pread(int fd, void* buf, size_t count, off_t offset) { return _pread(fd, buf, count, offset); }
+    static ssize_t pwrite(int fd, const void* buf, size_t count, off_t offset) { return _pwrite(fd, buf, count, offset); }
+};
+
+}
diff --git a/fastos/src/vespa/fastos/linux_file.cpp b/fastos/src/vespa/fastos/linux_file.cpp
index b4acaaa6073..0f91a8ff41c 100644
--- a/fastos/src/vespa/fastos/linux_file.cpp
+++ b/fastos/src/vespa/fastos/linux_file.cpp
@@ -12,6 +12,12 @@
 #include <sstream>
 #include <unistd.h>
 #include <fcntl.h>
+#include "file_rw_ops.h"
+#include <cstdio>
+#include <cstring>
+#include <system_error>
+
+using fastos::File_RW_Ops;
 
 const size_t FastOS_Linux_File::_directIOFileAlign = 4096;
 const size_t FastOS_Linux_File::_directIOMemAlign = 4096;
@@ -31,7 +37,7 @@ FastOS_Linux_File::FastOS_Linux_File(const char *filename)
 ssize_t
 FastOS_Linux_File::readInternal(int fh, void *buffer, size_t length, int64_t readOffset)
 {
-    ssize_t readResult = ::pread(fh, buffer, length, readOffset);
+    ssize_t readResult = File_RW_Ops::pread(fh, buffer, length, readOffset);
     if (readResult < 0 && _failedHandler != nullptr) {
         int error = errno;
         const char *fileName = GetFileName();
@@ -45,7 +51,7 @@ FastOS_Linux_File::readInternal(int fh, void *buffer, size_t length, int64_t rea
 ssize_t
 FastOS_Linux_File::readInternal(int fh, void *buffer, size_t length)
 {
-    ssize_t readResult = ::read(fh, buffer, length);
+    ssize_t readResult = File_RW_Ops::read(fh, buffer, length);
     if (readResult < 0 && _failedHandler != nullptr) {
         int error = errno;
         int64_t readOffset = GetPosition();
@@ -60,7 +66,7 @@ FastOS_Linux_File::readInternal(int fh, void *buffer, size_t length)
 ssize_t
 FastOS_Linux_File::writeInternal(int fh, const void *buffer, size_t length, int64_t writeOffset)
 {
-    ssize_t writeRes = ::pwrite(fh, buffer, length, writeOffset);
+    ssize_t writeRes = File_RW_Ops::pwrite(fh, buffer, length, writeOffset);
     if (writeRes < 0 && _failedHandler != nullptr) {
         int error = errno;
         const char *fileName = GetFileName();
@@ -73,7 +79,7 @@ FastOS_Linux_File::writeInternal(int fh, const void *buffer, size_t length, int6
 ssize_t
 FastOS_Linux_File::writeInternal(int fh, const void *buffer, size_t length)
 {
-    ssize_t writeRes = ::write(fh, buffer, length);
+    ssize_t writeRes = File_RW_Ops::write(fh, buffer, length);
     if (writeRes < 0 && _failedHandler != nullptr) {
         int error = errno;
         int64_t writeOffset = GetPosition();
@@ -428,6 +434,27 @@ FastOS_Linux_File::InitializeClass()
     return FastOS_UNIX_File::InitializeClass();
 }
 
+int
+FastOS_Linux_File::count_open_files()
+{
+    static const char * const fd_dir_name = "/proc/self/fd";
+    int count = 0;
+    DIR *dp = opendir(fd_dir_name);
+    if (dp != nullptr) {
+        struct dirent *ptr;
+        while ((ptr = readdir(dp)) != nullptr) {
+            if ((strcmp(".", ptr->d_name) != 0) && (strcmp("..", ptr->d_name) != 0)) {
+                ++count;
+            }
+        }
+        closedir(dp);
+    } else {
+        std::error_code ec(errno, std::system_category());
+        fprintf(stderr, "could not scan directory %s: %s\n", fd_dir_name, ec.message().c_str());
+    }
+    return count;
+}
+
 #include <vespa/fastos/backtrace.h>
 
 void forceStaticLinkOf_backtrace()
diff --git a/fastos/src/vespa/fastos/linux_file.h b/fastos/src/vespa/fastos/linux_file.h
index e304b6f0ce8..06f54de6870 100644
--- a/fastos/src/vespa/fastos/linux_file.h
+++ b/fastos/src/vespa/fastos/linux_file.h
@@ -45,6 +45,7 @@ public:
     static bool InitializeClass();
     static size_t getMaxDirectIOMemAlign();
     static void *allocateGenericDirectIOBuffer(size_t byteSize, void *&realPtr);
+    static int count_open_files();
 private:
     ssize_t internalWrite2(const void *buffer, size_t len);
     ssize_t readUnalignedEnd(void *buffer, size_t length, int64_t readOffset);
diff --git a/fastos/src/vespa/fastos/unix_file.cpp b/fastos/src/vespa/fastos/unix_file.cpp
index 17a11cd68f3..ef52d34e49f 100644
--- a/fastos/src/vespa/fastos/unix_file.cpp
+++ b/fastos/src/vespa/fastos/unix_file.cpp
@@ -21,11 +21,25 @@
 #else
 #include <sys/mount.h>
 #endif
+#ifdef __APPLE__
+#include <libproc.h>
+#include <sys/proc_info.h>
+#endif
+#include "file_rw_ops.h"
+
+using fastos::File_RW_Ops;
 
 ssize_t
 FastOS_UNIX_File::Read(void *buffer, size_t len)
 {
-    ssize_t nRead = read(_filedes, buffer, len);
+    ssize_t nRead = File_RW_Ops::read(_filedes, buffer, len);
+    if (nRead < 0 && _failedHandler != nullptr) {
+        int error = errno;
+        int64_t readOffset = GetPosition();
+        const char *fileName = GetFileName();
+        _failedHandler("read", fileName, error, readOffset, len, nRead);
+        errno = error;
+    }
     return nRead;
 }
 
@@ -33,7 +47,14 @@ FastOS_UNIX_File::Read(void *buffer, size_t len)
 ssize_t
 FastOS_UNIX_File::Write2(const void *buffer, size_t len)
 {
-    ssize_t writeRes = write(_filedes, buffer, len);
+    ssize_t writeRes = File_RW_Ops::write(_filedes, buffer, len);
+    if (writeRes < 0 && _failedHandler != nullptr) {
+        int error = errno;
+        int64_t writeOffset = GetPosition();
+        const char *fileName = GetFileName();
+        _failedHandler("write", fileName, error, writeOffset, len, writeRes);
+        errno = error;
+    }
     return writeRes;
 }
 
@@ -57,7 +78,13 @@ void FastOS_UNIX_File::ReadBuf(void *buffer, size_t length,
 {
     ssize_t readResult;
 
-    readResult = pread(_filedes, buffer, length, readOffset);
+    readResult = File_RW_Ops::pread(_filedes, buffer, length, readOffset);
+    if (readResult < 0 && _failedHandler != nullptr) {
+        int error = errno;
+        const char *fileName = GetFileName();
+        _failedHandler("read", fileName, error, readOffset, length, readResult);
+        errno = error;
+    }
     if (static_cast<size_t>(readResult) != length) {
         std::string errorString = readResult != -1 ?
                                   std::string("short read") :
@@ -475,6 +502,17 @@ int64_t FastOS_UNIX_File::GetFreeDiskSpace (const char *path)
     return freeSpace;
 }
 
+int
+FastOS_UNIX_File::count_open_files()
+{
+#ifdef __APPLE__
+    int buffer_size = proc_pidinfo(getpid(), PROC_PIDLISTFDS, 0, nullptr, 0);
+    return buffer_size / sizeof(proc_fdinfo);
+#else
+    return 0;
+#endif
+}
+
 FastOS_UNIX_DirectoryScan::FastOS_UNIX_DirectoryScan(const char *searchPath)
     : FastOS_DirectoryScanInterface(searchPath),
       _statRun(false),
diff --git a/fastos/src/vespa/fastos/unix_file.h b/fastos/src/vespa/fastos/unix_file.h
index 3dffe1fc089..3c3ffedc171 100644
--- a/fastos/src/vespa/fastos/unix_file.h
+++ b/fastos/src/vespa/fastos/unix_file.h
@@ -96,6 +96,7 @@ public:
     static Error TranslateError(const int osError);
     static std::string getErrorString(const int osError);
     static int64_t GetFreeDiskSpace (const char *path);
+    static int count_open_files();
 };
 
 #include <dirent.h>
diff --git a/fbench/CMakeLists.txt b/fbench/CMakeLists.txt
index 5cc56786227..3da632d98a6 100644
--- a/fbench/CMakeLists.txt
+++ b/fbench/CMakeLists.txt
@@ -15,6 +15,7 @@ vespa_define_module(
 
     TESTS
     src/test
+    src/test/authority
 )
 
 vespa_install_script(util/resultfilter.pl vespa-fbench-result-filter.pl bin)
diff --git a/fbench/src/fbench/fbench.cpp b/fbench/src/fbench/fbench.cpp
index 91475ce2125..88d27a33bd7 100644
--- a/fbench/src/fbench/fbench.cpp
+++ b/fbench/src/fbench/fbench.cpp
@@ -3,10 +3,10 @@
 #include <httpclient/httpclient.h>
 #include <util/filereader.h>
 #include <util/clientstatus.h>
+#include <vespa/vespalib/crypto/crypto_exception.h>
 #include <vespa/vespalib/net/crypto_engine.h>
 #include <vespa/vespalib/net/tls/transport_security_options.h>
 #include <vespa/vespalib/net/tls/tls_crypto_engine.h>
-#include <vespa/vespalib/net/tls/crypto_exception.h>
 #include <vespa/vespalib/io/mapped_file_input.h>
 #include "client.h"
 #include "fbench.h"
@@ -86,17 +86,20 @@ FBench::init_crypto_engine(const std::string &ca_certs_file_name,
         return false;
     }
     bool load_failed = false;
-    vespalib::net::tls::TransportSecurityOptions
-        tls_opts(maybe_load(ca_certs_file_name, load_failed),
-                 maybe_load(cert_chain_file_name, load_failed),
-                 maybe_load(private_key_file_name, load_failed));
+    auto ts_builder = vespalib::net::tls::TransportSecurityOptions::Params().
+            ca_certs_pem(maybe_load(ca_certs_file_name, load_failed)).
+            cert_chain_pem(maybe_load(cert_chain_file_name, load_failed)).
+            private_key_pem(maybe_load(private_key_file_name, load_failed)).
+            authorized_peers(vespalib::net::tls::AuthorizedPeers::allow_all_authenticated()).
+            disable_hostname_validation(true); // TODO configurable or default false!
+    vespalib::net::tls::TransportSecurityOptions tls_opts(std::move(ts_builder));
     if (load_failed) {
         fprintf(stderr, "failed to load transport security options\n");
         return false;
     }
     try {
         _crypto_engine = std::make_shared<vespalib::TlsCryptoEngine>(tls_opts);
-    } catch (vespalib::net::tls::CryptoException &e) {
+    } catch (vespalib::crypto::CryptoException &e) {
         fprintf(stderr, "%s\n", e.what());
         return false;
     }
@@ -197,6 +200,28 @@ FBench::StopClients()
     printf("\nClients Joined.\n");
 }
 
+namespace {
+
+const char *
+approx(double latency, const ClientStatus & status) {
+    return (latency > (status._timetable.size() / status._timetableResolution - 1))
+           ? "ms (approx)"
+           : "ms";
+}
+
+std::string
+fmtPercentile(double percentile) {
+    char buf[32];
+    if (percentile <= 99.0) {
+        snprintf(buf, sizeof(buf), "%2d  ", int(percentile));
+    } else {
+        snprintf(buf, sizeof(buf), "%2.1f", percentile);
+    }
+    return buf;
+}
+
+}
+
 void
 FBench::PrintSummary()
 {
@@ -224,13 +249,6 @@ FBench::PrintSummary()
     actualRate = (status._realTime > 0) ?
                  realNumClients * 1000.0 * status._requestCnt / status._realTime : 0;
 
-    double p25 = status.GetPercentile(25);
-    double p50 = status.GetPercentile(50);
-    double p75 = status.GetPercentile(75);
-    double p90 = status.GetPercentile(90);
-    double p95 = status.GetPercentile(95);
-    double p99 = status.GetPercentile(99);
-
     if (_keepAlive) {
         printf("*** HTTP keep-alive statistics ***\n");
         printf("connection reuse count -- %" PRIu64 "\n", status._reuseCnt);
@@ -247,24 +265,13 @@ FBench::PrintSummary()
     printf("minimum response time:  %8.2f ms\n", status._minTime);
     printf("maximum response time:  %8.2f ms\n", status._maxTime);
     printf("average response time:  %8.2f ms\n", status.GetAverage());
-    if (p25 > status._timetable.size() / status._timetableResolution - 1)
-        printf("25 percentile:          %8.2f ms (approx)\n", p25);
-    else         printf("25 percentile:          %8.2f ms\n", p25);
-    if (p50 > status._timetable.size() / status._timetableResolution - 1)
-        printf("50 percentile:          %8.2f ms (approx)\n", p50);
-    else         printf("50 percentile:          %8.2f ms\n", p50);
-    if (p75 > status._timetable.size() / status._timetableResolution - 1)
-        printf("75 percentile:          %8.2f ms (approx)\n", p75);
-    else         printf("75 percentile:          %8.2f ms\n", p75);
-    if (p90 > status._timetable.size() / status._timetableResolution - 1)
-        printf("90 percentile:          %8.2f ms (approx)\n", p90);
-    else         printf("90 percentile:          %8.2f ms\n", p90);
-    if (p95 > status._timetable.size() / status._timetableResolution - 1)
-        printf("95 percentile:          %8.2f ms (approx)\n", p95);
-    else         printf("95 percentile:          %8.2f ms\n", p95);
-    if (p99 > status._timetable.size() / status._timetableResolution - 1)
-        printf("99 percentile:          %8.2f ms (approx)\n", p99);
-    else         printf("99 percentile:          %8.2f ms\n", p99);
+
+    for (double percentile : {25.0, 50.0, 75.0, 90.0, 95.0, 98.0, 99.0, 99.5, 99.6, 99.7, 99.8, 99.9}) {
+        double latency = status.GetPercentile(percentile);
+        printf("%s percentile:          %8.2f %s\n",
+               fmtPercentile(percentile).c_str(), latency, approx(latency, status));
+    }
+
     printf("actual query rate:      %8.2f Q/s\n", actualRate);
     printf("utilization:            %8.2f %%\n",
            (maxRate > 0) ? 100 * (actualRate / maxRate) : 0);
diff --git a/fbench/src/httpclient/CMakeLists.txt b/fbench/src/httpclient/CMakeLists.txt
index 5f3333128b3..a28f3666383 100644
--- a/fbench/src/httpclient/CMakeLists.txt
+++ b/fbench/src/httpclient/CMakeLists.txt
@@ -3,5 +3,6 @@ vespa_add_library(fbench_httpclient STATIC
     SOURCES
     httpclient.cpp
     DEPENDS
+    fbench_util
     fastos
 )
diff --git a/fbench/src/httpclient/httpclient.cpp b/fbench/src/httpclient/httpclient.cpp
index 002d2770dcd..2feed087b43 100644
--- a/fbench/src/httpclient/httpclient.cpp
+++ b/fbench/src/httpclient/httpclient.cpp
@@ -1,5 +1,7 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 #include "httpclient.h"
+#include <vespa/vespalib/net/socket_spec.h>
+#include <util/authority.h>
 #include <cassert>
 #include <cstring>
 
@@ -28,13 +30,13 @@ HTTPClient::HTTPClient(vespalib::CryptoEngine::SP engine, const char *hostname,
       _keepAlive(keepAlive),
       _headerBenchmarkdataCoverage(headerBenchmarkdataCoverage),
       _extraHeaders(extraHeaders),
-      _authority(authority),
+    _sni_spec(make_sni_spec(authority, hostname, port, _engine->use_tls_when_client())),
+    _host_header_value(make_host_header_value(_sni_spec, _engine->use_tls_when_client())),
     _reuseCount(0),
     _bufsize(10240),
     _buf(new char[_bufsize]),
     _bufused(0),
     _bufpos(0),
-    _headerinfo(),
     _isOpen(false),
     _httpVersion(0),
     _requestStatus(0),
@@ -50,11 +52,6 @@ HTTPClient::HTTPClient(vespalib::CryptoEngine::SP engine, const char *hostname,
     _dataDone(false),
     _reader(NULL)
 {
-    if (_authority == "") {
-        char tmp[1024];
-        snprintf(tmp, 1024, "%s:%d", hostname, port);
-        _authority = tmp;
-    }
 }
 
 bool
@@ -69,7 +66,7 @@ HTTPClient::connect_socket()
     if (!handle.valid()) {
         return false;
     }
-    _socket = vespalib::SyncCryptoSocket::create(*_engine, std::move(handle), false);
+    _socket = vespalib::SyncCryptoSocket::create_client(*_engine, std::move(handle), _sni_spec);
     return bool(_socket);
 }
 
@@ -113,8 +110,7 @@ HTTPClient::ReadLine(char *buf, size_t bufsize)
 bool
 HTTPClient::Connect(const char *url, bool usePost, const char *content, int cLen)
 {
-    char tmp[4096];
-    char *req = NULL;
+    std::unique_ptr<char[]> req;
     uint32_t req_max = 0;
     uint32_t url_len = strlen(url);
     uint32_t host_len = _hostname.size();
@@ -129,14 +125,8 @@ HTTPClient::Connect(const char *url, bool usePost, const char *content, int cLen
         headers += "X-Yahoo-Vespa-Benchmarkdata-Coverage: true\r\n";
     }
 
-    if (url_len + host_len + headers.length() + FIXED_REQ_MAX < sizeof(tmp)) {
-        req = tmp;
-        req_max = sizeof(tmp);
-    } else {
-        req_max = url_len + host_len + headers.length() + FIXED_REQ_MAX;
-        req = new char[req_max];
-        assert(req != NULL);
-    }
+    req_max = url_len + host_len + headers.length() + FIXED_REQ_MAX;
+    req = std::make_unique<char []>(req_max);
 
     if (!_keepAlive) {
         headers += "Connection: close\r\n";
@@ -145,35 +135,33 @@ HTTPClient::Connect(const char *url, bool usePost, const char *content, int cLen
 
     // create request
     if (usePost) {
-        snprintf(req, req_max,
+        snprintf(req.get(), req_max,
                  "POST %s HTTP/1.1\r\n"
                  "Host: %s\r\n"
                  "Content-Length: %d\r\n"
                  "%s"
                  "\r\n",
-                 url, _authority.c_str(), cLen, headers.c_str());
+                 url, _host_header_value.c_str(), cLen, headers.c_str());
     } else {
-        snprintf(req, req_max,
+        snprintf(req.get(), req_max,
                  "GET %s HTTP/1.1\r\n"
                  "Host: %s\r\n"
                  "%s"
                  "\r\n",
-                 url, _authority.c_str(), headers.c_str());
+                 url, _host_header_value.c_str(), headers.c_str());
     }
 
     // try to reuse connection if keep-alive is enabled
+    ssize_t reqLen = strlen(req.get());
     if (_keepAlive
         && _socket
-        && _socket->write(req, strlen(req)) == (ssize_t)strlen(req)
+        && _socket->write(req.get(), reqLen) == reqLen
         && (!usePost || _socket->write(content, cLen) == (ssize_t)cLen)
         && FillBuffer() > 0) {
 
         // DEBUG
         // printf("Socket Connection reused!\n");
         _reuseCount++;
-        if (req != tmp) {
-            delete [] req;
-        }
         return true;
     } else {
         _socket.reset();
@@ -182,25 +170,14 @@ HTTPClient::Connect(const char *url, bool usePost, const char *content, int cLen
 
     // try to open new connection to server
     if (connect_socket()
-        && _socket->write(req, strlen(req)) == (ssize_t)strlen(req)
+        && _socket->write(req.get(), reqLen) == reqLen
         && (!usePost || _socket->write(content, cLen) == (ssize_t)cLen))
     {
-
-        // DEBUG
-        // printf("New Socket connection!\n");
-        if (req != tmp) {
-            delete [] req;
-        }
         return true;
     } else {
         _socket.reset();
     }
 
-    // DEBUG
-    // printf("Connect FAILED!\n");
-    if (req != tmp) {
-        delete [] req;
-    }
     return false;
 }
 
@@ -217,11 +194,11 @@ HTTPClient::SplitString(char *input, int &argc, char **argv, int maxargs)
         }
     if (*(argv[argc]) != '\0')
         argc++;
-    return NULL;                    // COMPLETE
+    return nullptr;                    // COMPLETE
 }
 
 bool
-HTTPClient::ReadHTTPHeader()
+HTTPClient::ReadHTTPHeader(std::string & headerinfo)
 {
     int     lineLen;
     char    line[4096];
@@ -244,8 +221,7 @@ HTTPClient::ReadHTTPHeader()
     if (argc >= 2) {
         if (strncmp(argv[0], "HTTP/", 5) != 0)
             return false;
-        _httpVersion = (strncmp(argv[0], "HTTP/1.0", 8) == 0) ?
-                       0 : 1;
+        _httpVersion = (strncmp(argv[0], "HTTP/1.0", 8) == 0) ? 0 : 1;
         _requestStatus = atoi(argv[1]);
     } else {
         return false;
@@ -270,8 +246,8 @@ HTTPClient::ReadHTTPHeader()
             }
 
             // Make sure to have enough memory in _headerinfo
-            _headerinfo += benchmark_data;
-            _headerinfo += "\n";
+            headerinfo += benchmark_data;
+            headerinfo += "\n";
         }
 
         SplitString(line, argc, argv, 32);
@@ -322,7 +298,7 @@ HTTPClient::ReadChunkHeader()
     char c;
     int  i;
 
-    if (_chunkSeq++ > 0 && ReadLine(NULL, 0) != 0)
+    if (_chunkSeq++ > 0 && ReadLine(nullptr, 0) != 0)
         return false;                  // no CRLF(/LF) after data block
 
     assert(_chunkLeft == 0);
@@ -347,7 +323,7 @@ HTTPClient::ReadChunkHeader()
     // printf("CHUNK: Length: %d\n", _chunkLeft);
 
     if (_chunkLeft == 0) {
-        while ((lineLen = ReadLine(NULL, 0)) > 0);   // skip trailer
+        while ((lineLen = ReadLine(nullptr, 0)) > 0);   // skip trailer
         if (lineLen < 0)
             return false;                              // data error
         _dataDone = true;                            // got last chunk
@@ -356,7 +332,7 @@ HTTPClient::ReadChunkHeader()
 }
 
 bool
-HTTPClient::Open(const char *url, bool usePost, const char *content, int cLen)
+HTTPClient::Open(std::string & headerinfo, const char *url, bool usePost, const char *content, int cLen)
 {
     if (_isOpen)
         Close();
@@ -365,7 +341,7 @@ HTTPClient::Open(const char *url, bool usePost, const char *content, int cLen)
     _dataRead  = 0;
     _dataDone  = false;
     _isOpen    = Connect(url, usePost, content, cLen);
-    if(!_isOpen || !ReadHTTPHeader()) {
+    if(!_isOpen || !ReadHTTPHeader(headerinfo)) {
         Close();
         return false;
     }
@@ -534,24 +510,24 @@ HTTPClient::Fetch(const char *url, std::ostream *file,
     ssize_t readRes  = 0;
     ssize_t written  = 0;
 
-    if (!Open(url, usePost, content, contentLen)) {
+    std::string headerinfo;
+    if (!Open(headerinfo, url, usePost, content, contentLen)) {
         return FetchStatus(false, _requestStatus, _totalHitCount, 0);
     }
 
     // Write headerinfo
     if (file) {
-        file->write(_headerinfo.c_str(), _headerinfo.length());
+        file->write(headerinfo.c_str(), headerinfo.length());
         if (file->fail()) {
             Close();
             return FetchStatus(false, _requestStatus, _totalHitCount, 0);
         }
         file->write("\r\n", 2);
         // Reset header data.
-        _headerinfo = "";
     }
 
     while((readRes = Read(buf, buflen)) > 0) {
-        if(file != NULL) {
+        if(file != nullptr) {
             if (!file->write(buf, readRes)) {
                 Close();
                 return FetchStatus(false, _requestStatus, _totalHitCount, written);
diff --git a/fbench/src/httpclient/httpclient.h b/fbench/src/httpclient/httpclient.h
index 9c3ccd437d1..5b96daa2441 100644
--- a/fbench/src/httpclient/httpclient.h
+++ b/fbench/src/httpclient/httpclient.h
@@ -6,6 +6,7 @@
 #include <vespa/vespalib/net/sync_crypto_socket.h>
 #include <vespa/vespalib/net/crypto_engine.h>
 #include <vespa/vespalib/net/socket_address.h>
+#include <vespa/vespalib/net/socket_spec.h>
 
 /**
  * This class implements a HTTP client that may be used to fetch
@@ -94,22 +95,20 @@ protected:
     vespalib::SocketAddress        _address;
     vespalib::SyncCryptoSocket::UP _socket;
 
-  std::string      _hostname;
-  int              _port;
-  bool             _keepAlive;
-  bool             _headerBenchmarkdataCoverage;
-  std::string      _extraHeaders;
-  std::string      _authority;
-  uint64_t         _reuseCount;
+  const std::string    _hostname;
+  int                  _port;
+  bool                 _keepAlive;
+  bool                 _headerBenchmarkdataCoverage;
+  const std::string    _extraHeaders;
+  vespalib::SocketSpec _sni_spec;
+  std::string          _host_header_value;
+  uint64_t             _reuseCount;
 
   size_t           _bufsize;
   char            *_buf;
   ssize_t          _bufused;
   ssize_t          _bufpos;
 
-  std::string      _headerinfo;
-  unsigned int     _headerinfoPos;
-
   bool             _isOpen;
   unsigned int     _httpVersion;
   unsigned int     _requestStatus;
@@ -204,15 +203,14 @@ protected:
    * @param argv the argument array.
    * @param maxargs the size of 'argv'.
    **/
-  static char *SplitString(char *input, int &argc, char **argv,
-				       int maxargs);
+  static char *SplitString(char *input, int &argc, char **argv, int maxargs);
 
   /**
    * Read and parse the HTTP Header.
    *
    * @return success(true)/failure(fail)
    **/
-  bool ReadHTTPHeader();
+  bool ReadHTTPHeader(std::string & headerinfo);
 
   /**
    * Read and parse a chunk header. Only used with chunked encoding.
@@ -221,6 +219,43 @@ protected:
    **/
   bool ReadChunkHeader();
 
+  /**
+   * Connect to the given url and read the response HTTP header. Note
+   * that this method will fail if the host returns a status code
+   * other than 200. This is done in order to make the interface as
+   * simple as possible.
+   *
+   * @return success(true)/failure(false)
+   * @param url the url you want to connect to
+   * @param usePost whether to use POST in the request
+   * @param content if usePost is true, the content to post
+   * @param cLen length of content in bytes
+   **/
+  bool Open(std::string & headerinfo, const char *url, bool usePost = false, const char *content = 0, int cLen = 0);
+  
+  /**
+   * Close the connection to the url we are currently reading
+   * from. Will also close the physical connection if keepAlive is not
+   * enabled or if all the url content was not read. This is done
+   * because skipping will probably be more expencive than creating a
+   * new connection.
+   *
+   * @return success(true)/failure(false)
+   **/
+  bool Close();
+
+  /**
+   * Read data from the url we are currently connected to. This method
+   * should be called repeatedly until it returns 0 in order to
+   * completely read the URL content. If @ref Close is called before
+   * all URL content is read the physical connection will be closed
+   * even if keepAlive is enabled.
+   *
+   * @return bytes read or -1 on failure.
+   * @param buf where to store the incoming data.
+   * @param len length of buf.
+   **/
+  ssize_t Read(void *buf, size_t len);
 public:
 
   /**
@@ -253,44 +288,6 @@ public:
   }
 
   /**
-   * Connect to the given url and read the response HTTP header. Note
-   * that this method will fail if the host returns a status code
-   * other than 200. This is done in order to make the interface as
-   * simple as possible.
-   *
-   * @return success(true)/failure(false)
-   * @param url the url you want to connect to
-   * @param usePost whether to use POST in the request
-   * @param content if usePost is true, the content to post
-   * @param cLen length of content in bytes
-   **/
-  bool Open(const char *url, bool usePost = false, const char *content = 0, int cLen = 0);
-
-  /**
-   * Read data from the url we are currently connected to. This method
-   * should be called repeatedly until it returns 0 in order to
-   * completely read the URL content. If @ref Close is called before
-   * all URL content is read the physical connection will be closed
-   * even if keepAlive is enabled.
-   *
-   * @return bytes read or -1 on failure.
-   * @param buf where to store the incoming data.
-   * @param len length of buf.
-   **/
-  ssize_t Read(void *buf, size_t len);
-
-  /**
-   * Close the connection to the url we are currently reading
-   * from. Will also close the physical connection if keepAlive is not
-   * enabled or if all the url content was not read. This is done
-   * because skipping will probably be more expencive than creating a
-   * new connection.
-   *
-   * @return success(true)/failure(false)
-   **/
-  bool Close();
-
-  /**
    * Class that provides status about the executed fetch method.
   **/
   class FetchStatus final
diff --git a/fbench/src/test/authority/CMakeLists.txt b/fbench/src/test/authority/CMakeLists.txt
new file mode 100644
index 00000000000..00f804f43f6
--- /dev/null
+++ b/fbench/src/test/authority/CMakeLists.txt
@@ -0,0 +1,10 @@
+# Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(fbench_authority_test_app TEST
+    SOURCES
+    authority_test.cpp
+    DEPENDS
+    fbench_util
+    vespalib
+    gtest
+)
+vespa_add_test(NAME fbench_authority_test_app COMMAND fbench_authority_test_app)
diff --git a/fbench/src/test/authority/authority_test.cpp b/fbench/src/test/authority/authority_test.cpp
new file mode 100644
index 00000000000..de723a8730f
--- /dev/null
+++ b/fbench/src/test/authority/authority_test.cpp
@@ -0,0 +1,87 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <util/authority.h>
+#include <vespa/vespalib/gtest/gtest.h>
+
+using vespalib::SocketSpec;
+
+//-----------------------------------------------------------------------------
+
+TEST(MakeSNISpecTest, host_port_is_parsed_as_expected) {
+    EXPECT_EQ(make_sni_spec("my_host:123", "fallback", 456, false).host(), "my_host");
+    EXPECT_EQ(make_sni_spec("my_host:123", "fallback", 456, true).host(), "my_host");
+    EXPECT_EQ(make_sni_spec("my_host:123", "fallback", 456, false).port(), 123);
+    EXPECT_EQ(make_sni_spec("my_host:123", "fallback", 456, true).port(), 123);
+}
+
+TEST(MakeSNISpecTest, user_info_is_stripped) {
+    EXPECT_EQ(make_sni_spec("myuser:deprecated@my_host:123", "fallback", 456, false).host(), "my_host");
+    EXPECT_EQ(make_sni_spec("myuser:deprecated@my_host:123", "fallback", 456, true).host(), "my_host");
+    EXPECT_EQ(make_sni_spec("myuser:deprecated@my_host:123", "fallback", 456, false).port(), 123);
+    EXPECT_EQ(make_sni_spec("myuser:deprecated@my_host:123", "fallback", 456, true).port(), 123);
+}
+
+TEST(MakeSNISpecTest, port_can_be_skipped) {
+    EXPECT_EQ(make_sni_spec("my_host", "fallback", 456, false).host(), "my_host");
+    EXPECT_EQ(make_sni_spec("my_host", "fallback", 456, true).host(), "my_host");
+    EXPECT_EQ(make_sni_spec("my_host", "fallback", 456, false).port(), 80);
+    EXPECT_EQ(make_sni_spec("my_host", "fallback", 456, true).port(), 443);
+}
+
+TEST(MakeSNISpecTest, quoted_ip_addresses_work_as_expected) {
+    EXPECT_EQ(make_sni_spec("[::1]:123", "fallback", 456, false).host(), "::1");
+    EXPECT_EQ(make_sni_spec("[::1]:123", "fallback", 456, true).host(), "::1");
+    EXPECT_EQ(make_sni_spec("[::1]:123", "fallback", 456, false).port(), 123);
+    EXPECT_EQ(make_sni_spec("[::1]:123", "fallback", 456, true).port(), 123);
+    EXPECT_EQ(make_sni_spec("[::1]", "fallback", 456, false).host(), "::1");
+    EXPECT_EQ(make_sni_spec("[::1]", "fallback", 456, true).host(), "::1");
+    EXPECT_EQ(make_sni_spec("[::1]", "fallback", 456, false).port(), 80);
+    EXPECT_EQ(make_sni_spec("[::1]", "fallback", 456, true).port(), 443);
+}
+
+TEST(MakeSNISpecTest, supplied_host_port_is_used_as_fallback) {
+    EXPECT_EQ(make_sni_spec("", "fallback", 456, false).host(), "fallback");
+    EXPECT_EQ(make_sni_spec("", "fallback", 456, true).host(), "fallback");
+    EXPECT_EQ(make_sni_spec("", "fallback", 456, false).port(), 456);
+    EXPECT_EQ(make_sni_spec("", "fallback", 456, true).port(), 456);
+}
+
+//-----------------------------------------------------------------------------
+
+TEST(MakeHostHeaderValueTest, host_port_is_formatted_as_expected) {
+    auto my_spec = SocketSpec::from_host_port("myhost", 123);
+    EXPECT_EQ(make_host_header_value(my_spec, false), "myhost:123");
+    EXPECT_EQ(make_host_header_value(my_spec, true), "myhost:123");
+}
+
+TEST(MakeHostHeaderValueTest, inappropriate_spec_gives_empty_host_value) {
+    std::vector<SocketSpec> bad_specs = {
+        SocketSpec::invalid,
+        SocketSpec::from_port(123),
+        SocketSpec::from_name("foo"),
+        SocketSpec::from_path("bar")
+    };
+    for (const auto &spec: bad_specs) {
+        EXPECT_EQ(make_host_header_value(spec, false), "");
+        EXPECT_EQ(make_host_header_value(spec, true), "");
+    }
+}
+
+TEST(MakeHostHeaderValueTest, default_port_is_omitted) {
+    auto spec1 = SocketSpec::from_host_port("myhost", 80);
+    auto spec2 = SocketSpec::from_host_port("myhost", 443);
+    EXPECT_EQ(make_host_header_value(spec1, false), "myhost");
+    EXPECT_EQ(make_host_header_value(spec1, true), "myhost:80");
+    EXPECT_EQ(make_host_header_value(spec2, false), "myhost:443");
+    EXPECT_EQ(make_host_header_value(spec2, true), "myhost");
+}
+
+TEST(MakeHostHeaderValueTest, ipv6_addresses_are_quoted) {
+    auto my_spec = SocketSpec::from_host_port("::1", 123);
+    EXPECT_EQ(make_host_header_value(my_spec, false), "[::1]:123");
+    EXPECT_EQ(make_host_header_value(my_spec, true), "[::1]:123");
+}
+
+//-----------------------------------------------------------------------------
+
+GTEST_MAIN_RUN_ALL_TESTS()
diff --git a/fbench/src/util/CMakeLists.txt b/fbench/src/util/CMakeLists.txt
index 47cc46ffc8f..3cdff26ce16 100644
--- a/fbench/src/util/CMakeLists.txt
+++ b/fbench/src/util/CMakeLists.txt
@@ -1,8 +1,9 @@
 # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 vespa_add_library(fbench_util STATIC
     SOURCES
+    authority.cpp
+    clientstatus.cpp
     filereader.cpp
     timer.cpp
-    clientstatus.cpp
     DEPENDS
 )
diff --git a/fbench/src/util/authority.cpp b/fbench/src/util/authority.cpp
new file mode 100644
index 00000000000..6247c72d9b0
--- /dev/null
+++ b/fbench/src/util/authority.cpp
@@ -0,0 +1,42 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "authority.h"
+#include <vespa/vespalib/util/stringfmt.h>
+#include <cassert>
+
+namespace {
+
+int default_port(bool use_https) { return use_https ? 443 : 80; }
+
+}
+
+vespalib::SocketSpec make_sni_spec(const std::string &authority, const char *hostname, int port, bool use_https) {
+    if (authority.empty()) {
+        return vespalib::SocketSpec::from_host_port(hostname, port);
+    }
+    auto split = authority.rfind('@');
+    std::string spec_str = (split == std::string::npos) ? authority : authority.substr(split + 1);
+    auto a = spec_str.rfind(':');
+    auto b = spec_str.rfind(']');
+    bool has_port = (a != std::string::npos) && ((b == std::string::npos) || (a > b));
+    if (has_port) {
+        spec_str = "tcp/" + spec_str;
+    } else {
+        spec_str = vespalib::make_string("tcp/%s:%d", spec_str.c_str(), default_port(use_https));
+    }
+    // use SocketSpec parser to ensure ipv6 addresses are dequoted
+    return vespalib::SocketSpec(spec_str);
+}
+
+std::string make_host_header_value(const vespalib::SocketSpec &sni_spec, bool use_https) {
+    if (sni_spec.host().empty()) {
+        return "";
+    }
+    if (sni_spec.port() == default_port(use_https)) {
+        return sni_spec.host();
+    }
+    // use SocketSpec formatter to ensure ipv6 addresses are quoted
+    std::string spec_str = sni_spec.spec();
+    assert(spec_str.find("tcp/") == 0);
+    return spec_str.substr(4);
+}
diff --git a/fbench/src/util/authority.h b/fbench/src/util/authority.h
new file mode 100644
index 00000000000..49dab4a29fd
--- /dev/null
+++ b/fbench/src/util/authority.h
@@ -0,0 +1,30 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/vespalib/net/socket_spec.h>
+
+/**
+ * Assemble an SNI (Server Name Indication) spec that will be used
+ * when handshaking over TLS. The authority will be used if
+ * non-empty. Hostname/port will be used as fall-back. Note that the
+ * SNI spec will also be used to generate the Host header used in
+ * subsequent HTTP requests.
+ *
+ * @return sni spec
+ * @param authority user-provided authority
+ * @param hostname name of the host we are connecting to
+ * @param port which port we are connecting to
+ * @param use_https are we using https? (TLS)
+ **/
+vespalib::SocketSpec make_sni_spec(const std::string &authority, const char *hostname, int port, bool use_https);
+
+/**
+ * Use an SNI spec to generate a matching Host header to be used in
+ * HTTP requests. Note that default port numbers will be omitted.
+ *
+ * @return host header value
+ * @param sni_spec SNI spec
+ * @param use_https are we using https? (TLS)
+ **/
+std::string make_host_header_value(const vespalib::SocketSpec &sni_spec, bool use_https);
diff --git a/fbench/src/util/clientstatus.cpp b/fbench/src/util/clientstatus.cpp
index 9e03068e13c..6ef188da201 100644
--- a/fbench/src/util/clientstatus.cpp
+++ b/fbench/src/util/clientstatus.cpp
@@ -1,6 +1,6 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 #include "clientstatus.h"
-#include <string.h>
+#include <cstring>
 #include <cmath>
 
 
@@ -24,9 +24,7 @@ ClientStatus::ClientStatus()
 {
 }
 
-ClientStatus::~ClientStatus()
-{
-}
+ClientStatus::~ClientStatus() = default;
 
 void
 ClientStatus::SetError(const char *errorMsg)
diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/CompressedFileReference.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/CompressedFileReference.java
index a1ad5c8a200..ddab35ba14d 100644
--- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/CompressedFileReference.java
+++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/CompressedFileReference.java
@@ -75,7 +75,6 @@ public class CompressedFileReference {
         int entries = 0;
         ArchiveEntry entry;
         while ((entry = archiveInputStream.getNextEntry()) != null) {
-            log.log(LogLevel.DEBUG, "Unpacking " + entry.getName());
             File outFile = new File(outputFile, entry.getName());
             if (entry.isDirectory()) {
                 if (!(outFile.exists() && outFile.isDirectory())) {
diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java
index 24b3fcac3e3..5829ab37b77 100644
--- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java
+++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java
@@ -1,7 +1,6 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.filedistribution;
 
-import com.google.common.util.concurrent.SettableFuture;
 import com.yahoo.config.FileReference;
 import com.yahoo.log.LogLevel;
 import com.yahoo.vespa.config.ConnectionPool;
@@ -13,6 +12,7 @@ import java.time.Duration;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
+import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
@@ -57,21 +57,14 @@ public class FileDownloader {
         }
     }
 
-    private Future<Optional<File>> getFutureFile(FileReferenceDownload fileReferenceDownload) {
+    Future<Optional<File>> getFutureFile(FileReferenceDownload fileReferenceDownload) {
         FileReference fileReference = fileReferenceDownload.fileReference();
         Objects.requireNonNull(fileReference, "file reference cannot be null");
-        log.log(LogLevel.DEBUG, () -> "Checking if file reference '" + fileReference.value() + "' exists in '" +
-                downloadDirectory.getAbsolutePath() + "' ");
-        Optional<File> file = getFileFromFileSystem(fileReference, downloadDirectory);
-        if (file.isPresent()) {
-            SettableFuture<Optional<File>> future = SettableFuture.create();
-            future.set(file);
-            return future;
-        } else {
-            log.log(LogLevel.DEBUG, () -> "File reference '" + fileReference.value() + "' not found in " +
-                    downloadDirectory.getAbsolutePath() + ", starting download");
-            return download(fileReferenceDownload);
-        }
+
+        Optional<File> file = getFileFromFileSystem(fileReference);
+        return (file.isPresent())
+                ? CompletableFuture.completedFuture(file)
+                : download(fileReferenceDownload);
     }
 
     double downloadStatus(FileReference fileReference) {
@@ -86,9 +79,10 @@ public class FileDownloader {
         return downloadDirectory;
     }
 
-    private Optional<File> getFileFromFileSystem(FileReference fileReference, File directory) {
-        File[] files = new File(directory, fileReference.value()).listFiles();
-        if (directory.exists() && directory.isDirectory() && files != null && files.length > 0) {
+    // Files are moved atomically, so if file reference exists and is accessible we can use it
+    private Optional<File> getFileFromFileSystem(FileReference fileReference) {
+        File[] files = new File(downloadDirectory, fileReference.value()).listFiles();
+        if (downloadDirectory.exists() && downloadDirectory.isDirectory() && files != null && files.length > 0) {
             File file = files[0];
             if (!file.exists()) {
                 throw new RuntimeException("File reference '" + fileReference.value() + "' does not exist");
@@ -105,33 +99,25 @@ public class FileDownloader {
 
     private boolean alreadyDownloaded(FileReference fileReference) {
         try {
-            return (getFileFromFileSystem(fileReference, downloadDirectory).isPresent());
+            return (getFileFromFileSystem(fileReference).isPresent());
         } catch (RuntimeException e) {
             return false;
         }
     }
 
-    public boolean downloadIfNeeded(FileReferenceDownload fileReferenceDownload) {
-        if (!alreadyDownloaded(fileReferenceDownload.fileReference())) {
-            download(fileReferenceDownload);
-            return true;
-        } else {
-            log.log(LogLevel.DEBUG, () -> "Download not needed, " + fileReferenceDownload.fileReference() + " already downloaded" );
-            return false;
-        }
+    /** Start a download, don't wait for result */
+    public void downloadIfNeeded(FileReferenceDownload fileReferenceDownload) {
+        FileReference fileReference = fileReferenceDownload.fileReference();
+        if (alreadyDownloaded(fileReference)) return;
+
+        download(fileReferenceDownload);
     }
 
+    /** Download, the future returned will be complete()d by receiving method in {@link FileReceiver} */
     private synchronized Future<Optional<File>> download(FileReferenceDownload fileReferenceDownload) {
         FileReference fileReference = fileReferenceDownload.fileReference();
-        Future<Optional<File>> inProgress = fileReferenceDownloader.addDownloadListener(fileReference, () -> getFile(fileReferenceDownload));
-        if (inProgress != null) {
-            log.log(LogLevel.DEBUG, () -> "Already downloading '" + fileReference.value() + "'");
-            return inProgress;
-        }
-
-        Future<Optional<File>> future = queueForDownload(fileReferenceDownload);
-        log.log(LogLevel.DEBUG, () -> "Queued '" + fileReference.value() + "' for download with timeout " + timeout);
-        return future;
+        FileReferenceDownload inProgress = fileReferenceDownloader.getDownloadInProgress(fileReference);
+        return (inProgress == null) ? queueForDownload(fileReferenceDownload) : inProgress.future();
     }
 
     private Future<Optional<File>> queueForDownload(FileReferenceDownload fileReferenceDownload) {
diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java
index 4089f800913..ff9efc74929 100644
--- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java
+++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java
@@ -96,9 +96,9 @@ public class FileReceiver {
             try {
                 Files.write(inprogressFile.toPath(), part, StandardOpenOption.WRITE, StandardOpenOption.APPEND);
             } catch (IOException e) {
-                log.log(LogLevel.ERROR, "Failed writing to file(" + inprogressFile.toPath() + "): " + e.getMessage(), e);
+                log.log(LogLevel.ERROR, "Failed writing to file (" + inprogressFile.toPath() + "): " + e.getMessage(), e);
                 inprogressFile.delete();
-                throw new RuntimeException("Failed writing to file(" + inprogressFile.toPath() + "): ", e);
+                throw new RuntimeException("Failed writing to file (" + inprogressFile.toPath() + "): ", e);
             }
             currentFileSize += part.length;
             currentPartId++;
@@ -111,13 +111,9 @@ public class FileReceiver {
             }
             File file = new File(fileReferenceDir, fileName);
             try {
-                // Delete destination dir, in case a previous attempt at writing to disk failed and the directory
-                // exists, but has no or incomplete content
-                deleteFileOrDirectory(fileReferenceDir);
                 // Unpack if necessary
                 if (fileType == FileReferenceData.Type.compressed) {
                     File decompressedDir = Files.createTempDirectory(tmpDir.toPath(), "archive").toFile();
-                    log.log(LogLevel.DEBUG, () -> "Archived file, unpacking " + inprogressFile + " to " + decompressedDir);
                     CompressedFileReference.decompress(inprogressFile, decompressedDir);
                     moveFileToDestination(decompressedDir, fileReferenceDir);
                 } else {
@@ -259,7 +255,7 @@ public class FileReceiver {
             retval = 1;
         }
         double completeness = (double) session.currentFileSize / (double) session.fileSize;
-        log.log(LogLevel.DEBUG, () -> String.format("%.1f percent of '%s' downloaded", completeness * 100, reference.value()));
+        log.log(LogLevel.SPAM, () -> String.format("%.1f percent of '%s' downloaded", completeness * 100, reference.value()));
         downloader.setDownloadStatus(reference, completeness);
         req.returnValues().add(new Int32Value(retval));
     }
diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownload.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownload.java
index 4a9fadf1a61..fe501484faf 100644
--- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownload.java
+++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownload.java
@@ -2,16 +2,16 @@
 
 package com.yahoo.vespa.filedistribution;
 
-import com.google.common.util.concurrent.SettableFuture;
 import com.yahoo.config.FileReference;
 
 import java.io.File;
 import java.util.Optional;
+import java.util.concurrent.CompletableFuture;
 
 public class FileReferenceDownload {
 
     private final FileReference fileReference;
-    private final SettableFuture<Optional<File>> future;
+    private final CompletableFuture<Optional<File>> future;
     // If a config server wants to download from another config server (because it does not have the
     // file itself) we set this flag to true to avoid an eternal loop
     private final boolean downloadFromOtherSourceIfNotFound;
@@ -22,7 +22,7 @@ public class FileReferenceDownload {
 
     public FileReferenceDownload(FileReference fileReference, boolean downloadFromOtherSourceIfNotFound) {
         this.fileReference = fileReference;
-        this.future = SettableFuture.create();
+        this.future = new CompletableFuture<>();
         this.downloadFromOtherSourceIfNotFound = downloadFromOtherSourceIfNotFound;
     }
 
@@ -30,7 +30,7 @@ public class FileReferenceDownload {
         return fileReference;
     }
 
-    SettableFuture<Optional<File>> future() {
+    CompletableFuture<Optional<File>> future() {
         return future;
     }
 
diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java
index 66b86866c3e..f1d31156d79 100644
--- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java
+++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java
@@ -1,8 +1,6 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.filedistribution;
 
-import com.google.common.collect.ImmutableMap;
-import com.google.common.util.concurrent.ListenableFuture;
 import com.yahoo.concurrent.DaemonThreadFactory;
 import com.yahoo.config.FileReference;
 import com.yahoo.jrt.Int32Value;
@@ -14,6 +12,7 @@ import com.yahoo.vespa.config.ConnectionPool;
 
 import java.io.File;
 import java.time.Duration;
+import java.time.Instant;
 import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.Map;
@@ -37,9 +36,12 @@ public class FileReferenceDownloader {
     private final static Duration rpcTimeout = Duration.ofSeconds(10);
 
     private final ExecutorService downloadExecutor =
-            Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors(), new DaemonThreadFactory("filereference downloader"));
+            Executors.newFixedThreadPool(Math.max(8, Runtime.getRuntime().availableProcessors()),
+                                         new DaemonThreadFactory("filereference downloader"));
     private final ConnectionPool connectionPool;
+    /* Ongoing downloads */
     private final Map<FileReference, FileReferenceDownload> downloads = new LinkedHashMap<>();
+    /* Status for ongoing and finished downloads */
     private final Map<FileReference, Double> downloadStatus = new HashMap<>();  // between 0 and 1
     private final Duration downloadTimeout;
     private final Duration sleepBetweenRetries;
@@ -52,12 +54,12 @@ public class FileReferenceDownloader {
         new FileReceiver(connectionPool.getSupervisor(), this, downloadDirectory, tmpDirectory);
     }
 
-    private void startDownload(Duration timeout, FileReferenceDownload fileReferenceDownload) {
+    private void startDownload(FileReferenceDownload fileReferenceDownload) {
         FileReference fileReference = fileReferenceDownload.fileReference();
-        long end = System.currentTimeMillis() + timeout.toMillis();
+        Instant end = Instant.now().plus(downloadTimeout);
         boolean downloadStarted = false;
         int retryCount = 0;
-        while ((System.currentTimeMillis() < end) && !downloadStarted) {
+        do {
             try {
                 if (startDownloadRpc(fileReferenceDownload, retryCount)) {
                     downloadStarted = true;
@@ -67,10 +69,10 @@ public class FileReferenceDownloader {
                 }
             }
             catch (InterruptedException e) { /* ignored */}
-        }
+        } while (Instant.now().isBefore(end) && !downloadStarted);
 
         if ( !downloadStarted) {
-            fileReferenceDownload.future().setException(new RuntimeException("Failed getting file reference '" + fileReference.value() + "'"));
+            fileReferenceDownload.future().completeExceptionally(new RuntimeException("Failed getting file reference '" + fileReference.value() + "'"));
             synchronized (downloads) {
                 downloads.remove(fileReference);
             }
@@ -84,7 +86,7 @@ public class FileReferenceDownloader {
             downloads.put(fileReference, fileReferenceDownload);
             downloadStatus.put(fileReference, 0.0);
         }
-        downloadExecutor.submit(() -> startDownload(downloadTimeout, fileReferenceDownload));
+        downloadExecutor.submit(() -> startDownload(fileReferenceDownload));
     }
 
     void completedDownloading(FileReference fileReference, File file) {
@@ -93,7 +95,7 @@ public class FileReferenceDownloader {
             if (download != null) {
                 downloadStatus.put(fileReference, 1.0);
                 downloads.remove(fileReference);
-                download.future().set(Optional.of(file));
+                download.future().complete(Optional.of(file));
             } else {
                 log.log(LogLevel.DEBUG, () -> "Received '" + fileReference + "', which was not requested. Can be ignored if happening during upgrades/restarts");
             }
@@ -114,10 +116,10 @@ public class FileReferenceDownloader {
         request.parameters().add(new StringValue(fileReference));
         request.parameters().add(new Int32Value(fileReferenceDownload.downloadFromOtherSourceIfNotFound() ? 0 : 1));
 
-        execute(request, connection);
+        connection.invokeSync(request, (double) rpcTimeout.getSeconds());
         Level logLevel = (retryCount > 0 ? LogLevel.INFO : LogLevel.DEBUG);
         if (validateResponse(request)) {
-            log.log(logLevel, () -> "Request callback, OK. Req: " + request + "\nSpec: " + connection);
+            log.log(logLevel, () -> "Request callback, OK. Req: " + request + "\nSpec: " + connection + ", retry count " + retryCount);
             if (request.returnValues().get(0).asInt32() == 0) {
                 log.log(logLevel, () -> "Found file reference '" + fileReference + "' available at " + connection.getAddress());
                 return true;
@@ -127,8 +129,9 @@ public class FileReferenceDownloader {
                 return false;
             }
         } else {
-            log.log(logLevel, "Request failed. Req: " + request + "\nSpec: " + connection.getAddress() +
-                    ", error code: " + request.errorCode() + ", set error for connection and use another for next request");
+            log.log(logLevel, () -> "Request failed. Req: " + request + "\nSpec: " + connection.getAddress() +
+            ", error code: " + request.errorCode() + ", set error for spec, use another spec for next request" +
+            ", retry count " + retryCount);
             connectionPool.setError(connection, request.errorCode());
             return false;
         }
@@ -140,19 +143,10 @@ public class FileReferenceDownloader {
         }
     }
 
-    ListenableFuture<Optional<File>> addDownloadListener(FileReference fileReference, Runnable runnable) {
+    FileReferenceDownload getDownloadInProgress(FileReference fileReference) {
         synchronized (downloads) {
-            FileReferenceDownload download = downloads.get(fileReference);
-            if (download != null) {
-                download.future().addListener(runnable, downloadExecutor);
-                return download.future();
-            }
+            return downloads.get(fileReference);
         }
-        return null;
-    }
-
-    private void execute(Request request, Connection connection) {
-        connection.invokeSync(request, (double) rpcTimeout.getSeconds());
     }
 
     private boolean validateResponse(Request request) {
@@ -186,7 +180,7 @@ public class FileReferenceDownloader {
 
     Map<FileReference, Double> downloadStatus() {
         synchronized (downloads) {
-            return ImmutableMap.copyOf(downloadStatus);
+            return Map.copyOf(downloadStatus);
         }
     }
 
diff --git a/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java b/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java
index d1d12cb07b7..52d8507acea 100644
--- a/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java
+++ b/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java
@@ -27,6 +27,10 @@ import java.nio.file.Path;
 import java.time.Duration;
 import java.util.Arrays;
 import java.util.Optional;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
 
 import static com.yahoo.jrt.ErrorCode.CONNECTION;
 import static org.junit.Assert.assertEquals;
@@ -35,6 +39,7 @@ import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
 public class FileDownloaderTest {
+    private static final Duration sleepBetweenRetries = Duration.ofMillis(10);
 
     private MockConnection connection;
     private FileDownloader fileDownloader;
@@ -47,7 +52,7 @@ public class FileDownloaderTest {
             downloadDir = Files.createTempDirectory("filedistribution").toFile();
             tempDir = Files.createTempDirectory("download").toFile();
             connection = new MockConnection();
-            fileDownloader = new FileDownloader(connection, downloadDir, tempDir, Duration.ofSeconds(2), Duration.ofMillis(100));
+            fileDownloader = new FileDownloader(connection, downloadDir, tempDir, Duration.ofSeconds(1), sleepBetweenRetries);
         } catch (IOException e) {
             e.printStackTrace();
             fail(e.getMessage());
@@ -108,7 +113,7 @@ public class FileDownloaderTest {
 
             // Receives fileReference, should return and make it available to caller
             String filename = "abc.jar";
-            receiveFile(fileReference, filename, FileReferenceData.Type.file, "some other content");
+            receiveFile(fileDownloader, fileReference, filename, FileReferenceData.Type.file, "some other content");
             Optional<File> downloadedFile = fileDownloader.getFile(fileReference);
 
             assertTrue(downloadedFile.isPresent());
@@ -142,7 +147,7 @@ public class FileDownloaderTest {
 
             File tarFile = CompressedFileReference.compress(tempPath.toFile(), Arrays.asList(fooFile, barFile), new File(tempPath.toFile(), filename));
             byte[] tarredContent = IOUtils.readFileBytes(tarFile);
-            receiveFile(fileReference, filename, FileReferenceData.Type.compressed, tarredContent);
+            receiveFile(fileDownloader, fileReference, filename, FileReferenceData.Type.compressed, tarredContent);
             Optional<File> downloadedFile = fileDownloader.getFile(fileReference);
 
             assertTrue(downloadedFile.isPresent());
@@ -158,7 +163,7 @@ public class FileDownloaderTest {
 
     @Test
     public void getFileWhenConnectionError() throws IOException {
-        fileDownloader = new FileDownloader(connection, downloadDir, tempDir, Duration.ofSeconds(3), Duration.ofMillis(100));
+        fileDownloader = new FileDownloader(connection, downloadDir, tempDir, Duration.ofSeconds(1), sleepBetweenRetries);
         File downloadDir = fileDownloader.downloadDirectory();
 
         int timesToFail = 2;
@@ -175,7 +180,7 @@ public class FileDownloaderTest {
 
         // Receives fileReference, should return and make it available to caller
         String filename = "abc.jar";
-        receiveFile(fileReference, filename, FileReferenceData.Type.file, "some other content");
+        receiveFile(fileDownloader, fileReference, filename, FileReferenceData.Type.file, "some other content");
         Optional<File> downloadedFile = fileDownloader.getFile(fileReference);
         assertTrue(downloadedFile.isPresent());
         File downloadedFileFullPath = new File(fileReferenceFullPath, filename);
@@ -189,26 +194,68 @@ public class FileDownloaderTest {
     }
 
     @Test
+    public void getFileWhenDownloadInProgress() throws IOException, ExecutionException, InterruptedException {
+        ExecutorService executor = Executors.newFixedThreadPool(2);
+        String filename = "abc.jar";
+        fileDownloader = new FileDownloader(connection, downloadDir, tempDir, Duration.ofSeconds(3), sleepBetweenRetries);
+        File downloadDir = fileDownloader.downloadDirectory();
+
+        // Delay response so that we can make a second request while downloading the file from the first request
+        connection.setResponseHandler(new MockConnection.WaitResponseHandler(Duration.ofSeconds(1)));
+
+        FileReference fileReference = new FileReference("fileReference");
+        File fileReferenceFullPath = fileReferenceFullPath(downloadDir, fileReference);
+        FileReferenceDownload fileReferenceDownload = new FileReferenceDownload(fileReference);
+
+        Future<Future<Optional<File>>> future1 = executor.submit(() -> fileDownloader.getFutureFile(fileReferenceDownload));
+        do {
+            Thread.sleep(10);
+        } while (! fileDownloader.fileReferenceDownloader().isDownloading(fileReference));
+        assertTrue(fileDownloader.fileReferenceDownloader().isDownloading(fileReference));
+
+        // Request file while download is in progress
+        Future<Future<Optional<File>>> future2 = executor.submit(() -> fileDownloader.getFutureFile(fileReferenceDownload));
+
+        // Receive file, will complete downloading and futures
+        receiveFile(fileDownloader, fileReference, filename, FileReferenceData.Type.file, "some other content");
+
+        // Check that we got file correctly with first request
+        Optional<File> downloadedFile = future1.get().get();
+        assertTrue(downloadedFile.isPresent());
+        File downloadedFileFullPath = new File(fileReferenceFullPath, filename);
+        assertEquals(downloadedFileFullPath.getAbsolutePath(), downloadedFile.get().getAbsolutePath());
+        assertEquals("some other content", IOUtils.readFile(downloadedFile.get()));
+
+        // Check that request done while downloading works
+        downloadedFile = future2.get().get();
+        assertTrue(downloadedFile.isPresent());
+        executor.shutdownNow();
+    }
+
+    @Test
     public void setFilesToDownload() throws IOException {
         Duration timeout = Duration.ofMillis(200);
-        Duration sleepBetweenRetries = Duration.ofMillis(200);
         MockConnection connectionPool = new MockConnection();
         connectionPool.setResponseHandler(new MockConnection.WaitResponseHandler(timeout.plus(Duration.ofMillis(1000))));
         FileDownloader fileDownloader = new FileDownloader(connectionPool, downloadDir, tempDir, timeout, sleepBetweenRetries);
         FileReference foo = new FileReference("foo");
         // Should download since we do not have the file on disk
-        assertTrue(fileDownloader.downloadIfNeeded(new FileReferenceDownload(foo)));
+        fileDownloader.downloadIfNeeded(new FileReferenceDownload(foo));
+        assertTrue(fileDownloader.fileReferenceDownloader().isDownloading(foo));
+        assertFalse(fileDownloader.getFile(foo).isPresent());
         // Receive files to simulate download
         receiveFile();
         // Should not download, since file has already been downloaded
-        assertFalse(fileDownloader.downloadIfNeeded(new FileReferenceDownload(foo)));
+        fileDownloader.downloadIfNeeded(new FileReferenceDownload(foo));
+        // and file should be available
+        assertTrue(fileDownloader.getFile(foo).isPresent());
     }
 
     @Test
     public void receiveFile() throws IOException {
         FileReference foo = new FileReference("foo");
         String filename = "foo.jar";
-        receiveFile(foo, filename, FileReferenceData.Type.file, "content");
+        receiveFile(fileDownloader, foo, filename, FileReferenceData.Type.file, "content");
         File downloadedFile = new File(fileReferenceFullPath(downloadDir, foo), filename);
         assertEquals("content", IOUtils.readFile(downloadedFile));
     }
@@ -229,16 +276,19 @@ public class FileDownloaderTest {
         assertEquals(expectedDownloadStatus, downloadStatus, 0.0001);
     }
 
-    private void receiveFile(FileReference fileReference, String filename, FileReferenceData.Type type, String content) {
-        receiveFile(fileReference, filename, type, Utf8.toBytes(content));
+    private void receiveFile(FileDownloader fileDownloader, FileReference fileReference, String filename,
+                             FileReferenceData.Type type, String content) {
+        receiveFile(fileDownloader, fileReference, filename, type, Utf8.toBytes(content));
     }
 
-    private void receiveFile(FileReference fileReference, String filename, FileReferenceData.Type type, byte[] content) {
+    private void receiveFile(FileDownloader fileDownloader, FileReference fileReference, String filename,
+                             FileReferenceData.Type type, byte[] content) {
         XXHash64 hasher = XXHashFactory.fastestInstance().hash64();
         FileReceiver.Session session =
                 new FileReceiver.Session(downloadDir, tempDir, 1, fileReference, type, filename, content.length);
         session.addPart(0, content);
-        session.close(hasher.hash(ByteBuffer.wrap(content), 0));
+        File file = session.close(hasher.hash(ByteBuffer.wrap(content), 0));
+        fileDownloader.fileReferenceDownloader().completedDownloading(fileReference, file);
     }
 
     private static class MockConnection implements ConnectionPool, com.yahoo.vespa.config.Connection {
diff --git a/flags/src/main/java/com/yahoo/vespa/flags/FetchVector.java b/flags/src/main/java/com/yahoo/vespa/flags/FetchVector.java
index 866d4782a2a..5897a5ab58b 100644
--- a/flags/src/main/java/com/yahoo/vespa/flags/FetchVector.java
+++ b/flags/src/main/java/com/yahoo/vespa/flags/FetchVector.java
@@ -84,8 +84,12 @@ public class FetchVector {
 
     public boolean isEmpty() { return map.isEmpty(); }
 
-    /** Returns a new FetchVector, identical to {@code this} except for its value in {@code dimension}. */
+    /**
+     * Returns a new FetchVector, identical to {@code this} except for its value in {@code dimension}.
+     * Dimension is removed if the value is null.
+     */
     public FetchVector with(Dimension dimension, String value) {
+        if (value == null) return makeFetchVector(merged -> merged.remove(dimension));
         return makeFetchVector(merged -> merged.put(dimension, value));
     }
 
diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java
index 44cf55f1b78..40faefdd052 100644
--- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java
+++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java
@@ -55,6 +55,25 @@ public class Flags {
             "Whether to enable Nessus.", "Takes effect on next host admin tick",
             HOSTNAME);
 
+    public static final UnboundBooleanFlag ENABLE_FLEET_SSHD_CONFIG = defineFeatureFlag(
+            "enable-fleet-sshd-config", true,
+            "Whether fleet should manage the /etc/ssh/sshd_config file.",
+            "Takes effect on next host admin tick.",
+            HOSTNAME);
+
+    public static final UnboundBooleanFlag FLEET_CANARY = defineFeatureFlag(
+            "fleet-canary", false,
+            "Whether the host is a fleet canary.",
+            "Takes effect on next host admin tick.",
+            HOSTNAME);
+
+    public static final UnboundBooleanFlag USE_NEW_VESPA_RPMS = defineFeatureFlag(
+            "use-new-vespa-rpms", false,
+            "Whether to use the new vespa-rpms YUM repo when upgrading/downgrading.  The vespa-version " +
+            "when fetching the flag value is the wanted version of the host.",
+            "Takes effect when upgrading or downgrading host admin to a different version.",
+            HOSTNAME, NODE_TYPE, VESPA_VERSION);
+
     public static final UnboundListFlag<String> DISABLED_HOST_ADMIN_TASKS = defineListFlag(
             "disabled-host-admin-tasks", List.of(), String.class,
             "List of host-admin task names (as they appear in the log, e.g. root>main>UpgradeTask) that should be skipped",
@@ -62,7 +81,7 @@ public class Flags {
             HOSTNAME, NODE_TYPE);
 
     public static final UnboundStringFlag DOCKER_VERSION = defineStringFlag(
-            "docker-version", "1.13.1-91.git07f3374",
+            "docker-version", "1.13.1-102.git7f2769b",
             "The version of the docker to use of the format VERSION-REL: The YUM package to be installed will be " +
             "2:docker-VERSION-REL.el7.centos.x86_64 in AWS (and without '.centos' otherwise). " +
             "If docker-version is not of this format, it must be parseable by YumPackageName::fromString.",
@@ -83,23 +102,6 @@ public class Flags {
             "Takes effect on next node agent tick. Change is orchestrated, but does NOT require container restart",
             HOSTNAME, APPLICATION_ID);
 
-    public static final UnboundBooleanFlag INCLUDE_SIS_IN_TRUSTSTORE = defineFeatureFlag(
-            "include-sis-in-truststore", false,
-            "Whether to use the trust store backed by Athenz and (in public) Service Identity certificates in " +
-            "host-admin and/or Docker containers",
-            "Takes effect on restart of host-admin (for host-admin), and restart of Docker container.",
-            // For host-admin, HOSTNAME and NODE_TYPE is available
-            // For Docker containers, HOSTNAME and APPLICATION_ID is available
-            // WARNING: Having different sets of dimensions is DISCOURAGED in general, but needed for here since
-            // trust store for host-admin is determined before having access to application ID from node repo.
-            HOSTNAME, NODE_TYPE, APPLICATION_ID);
-
-    public static final UnboundStringFlag TLS_INSECURE_MIXED_MODE = defineStringFlag(
-            "tls-insecure-mixed-mode", "tls_client_mixed_server",
-            "TLS insecure mixed mode. Allowed values: ['plaintext_client_mixed_server', 'tls_client_mixed_server', 'tls_client_tls_server']",
-            "Takes effect on restart of Docker container",
-            NODE_TYPE, APPLICATION_ID, HOSTNAME);
-
     public static final UnboundStringFlag TLS_INSECURE_AUTHORIZATION_MODE = defineStringFlag(
             "tls-insecure-authorization-mode", "log_only",
             "TLS insecure authorization mode. Allowed values: ['disable', 'log_only', 'enforce']",
@@ -110,7 +112,7 @@ public class Flags {
             "use-adaptive-dispatch", false,
             "Should adaptive dispatch be used over round robin",
             "Takes effect at redeployment",
-            APPLICATION_ID);
+            ZONE_ID, APPLICATION_ID);
 
     public static final UnboundIntFlag REBOOT_INTERVAL_IN_DAYS = defineIntFlag(
             "reboot-interval-in-days", 30,
@@ -118,6 +120,13 @@ public class Flags {
             "scheduled evenly distributed in the 1x-2x range (and naturally guaranteed at the 2x boundary).",
             "Takes effect on next run of NodeRebooter");
 
+    public static final UnboundBooleanFlag RETIRE_WITH_PERMANENTLY_DOWN = defineFeatureFlag(
+            "retire-with-permanently-down", false,
+            "If enabled, retirement will end with setting the host status to PERMANENTLY_DOWN, " +
+            "instead of ALLOWED_TO_BE_DOWN (old behavior).",
+            "Takes effect on the next run of RetiredExpirer.",
+            HOSTNAME);
+
     public static final UnboundBooleanFlag ENABLE_DYNAMIC_PROVISIONING = defineFeatureFlag(
             "enable-dynamic-provisioning", false,
             "Provision a new docker host when we otherwise can't allocate a docker node",
@@ -132,9 +141,21 @@ public class Flags {
 
     public static final UnboundDoubleFlag DEFAULT_TERM_WISE_LIMIT = defineDoubleFlag(
             "default-term-wise-limit", 1.0,
-            "Node resource memory in Gb for admin cluster nodes",
+            "Default limit for when to apply termwise query evaluation",
             "Takes effect at redeployment",
-            APPLICATION_ID);
+            ZONE_ID, APPLICATION_ID);
+
+    public static final UnboundDoubleFlag DEFAULT_SOFT_START_SECONDS = defineDoubleFlag(
+            "default-soft-start-seconds", 0.0,
+            "Default number of seconds that a soft start shall use",
+            "Takes effect at redeployment",
+            ZONE_ID, APPLICATION_ID);
+
+    public static final UnboundDoubleFlag DEFAULT_TOP_K_PROBABILITY = defineDoubleFlag(
+            "default-top-k-probability", 1.0,
+            "Default probability that you will get the globally top K documents when merging many partitions.",
+            "Takes effect at redeployment",
+            ZONE_ID, APPLICATION_ID);
 
     public static final UnboundBooleanFlag HOST_HARDENING = defineFeatureFlag(
             "host-hardening", false,
@@ -142,6 +163,12 @@ public class Flags {
             "Takes effect on next tick or on host-admin restart (may vary where used).",
             HOSTNAME);
 
+    public static final UnboundBooleanFlag TCP_ABORT_ON_OVERFLOW = defineFeatureFlag(
+            "tcp-abort-on-overflow", false,
+            "Whether to set /proc/sys/net/ipv4/tcp_abort_on_overflow to 0 (false) or 1 (true)",
+            "Takes effect on next host-admin tick.",
+            HOSTNAME);
+
     public static final UnboundStringFlag ZOOKEEPER_SERVER_MAJOR_MINOR_VERSION = defineStringFlag(
             "zookeeper-server-version", "3.5",
             "The version of ZooKeeper server to use (major.minor, not full version)",
@@ -149,7 +176,7 @@ public class Flags {
             NODE_TYPE, APPLICATION_ID, HOSTNAME);
 
     public static final UnboundStringFlag TLS_FOR_ZOOKEEPER_QUORUM_COMMUNICATION = defineStringFlag(
-            "tls-for-zookeeper-quorum-communication", "OFF",
+            "tls-for-zookeeper-quorum-communication", "TLS_WITH_PORT_UNIFICATION",
             "How to setup TLS for ZooKeeper quorum communication. Valid values are OFF, PORT_UNIFICATION, TLS_WITH_PORT_UNIFICATION, TLS_ONLY",
             "Takes effect on restart of config server",
             NODE_TYPE, HOSTNAME);
@@ -167,52 +194,60 @@ public class Flags {
             NODE_TYPE, HOSTNAME);
 
     public static final UnboundBooleanFlag ENABLE_DISK_WRITE_TEST = defineFeatureFlag(
-            "enable-disk-write-test", false,
+            "enable-disk-write-test", true,
             "Regularly issue a small write to disk and fail the host if it is not successful",
             "Takes effect on next node agent tick (but does not clear existing failure reports)",
             HOSTNAME);
-  
-    public static final UnboundBooleanFlag DISABLE_CM3 = defineFeatureFlag(
-            "disable-cm3", false,
-            "Whether to disable CM3.", "Takes effect on next host admin tick",
-            HOSTNAME);
 
-    public static final UnboundBooleanFlag USE_4443_UPSTREAM = defineFeatureFlag(
-            "use-4443-upstream", false,
-            "Use port 4443 for nginx upstream",
-            "Takes effect when routing container asks for new config",
+    public static final UnboundBooleanFlag USE_REFRESHED_ENDPOINT_CERTIFICATE = defineFeatureFlag(
+            "use-refreshed-endpoint-certificate", false,
+            "Whether an application should start using a newer certificate/key pair if available",
+            "Takes effect on the next deployment of the application",
             APPLICATION_ID);
 
-    public static final UnboundBooleanFlag ENABLE_IN_PLACE_RESIZE = defineFeatureFlag(
-            "enable-in-place-resize", false,
-            "Whether nodes can be resized in-place when certain conditions are met",
-            "Takes effect on next deployment",
+    public static final UnboundBooleanFlag VALIDATE_ENDPOINT_CERTIFICATES = defineFeatureFlag(
+            "validate-endpoint-certificates", false,
+            "Whether endpoint certificates should be validated before use",
+            "Takes effect on the next deployment of the application");
+
+    public static final UnboundStringFlag ENDPOINT_CERTIFICATE_BACKFILL = defineStringFlag(
+            "endpoint-certificate-backfill", "disable",
+            "Whether the endpoint certificate maintainer should backfill missing certificate data from cameo",
+            "Takes effect on next scheduled run of maintainer - set to \"disable\", \"dryrun\" or \"enable\"");
+
+    public static final UnboundStringFlag DOCKER_IMAGE_REPO = defineStringFlag(
+            "docker-image-repo", "",
+            "Override default docker image repo. Docker image version will be Vespa version.",
+            "Takes effect on next deployment from controller",
+            ZONE_ID, APPLICATION_ID);
+
+    public static final UnboundBooleanFlag ENDPOINT_CERT_IN_SHARED_ROUTING = defineFeatureFlag(
+            "endpoint-cert-in-shared-routing", false,
+            "Whether to provision and use endpoint certs for apps in shared routing zones",
+            "Takes effect on next deployment of the application", APPLICATION_ID);
+
+    public static final UnboundBooleanFlag PHRASE_SEGMENTING = defineFeatureFlag(
+            "phrase-segmenting", false,
+            "Should 'implicit phrases' in queries we parsed to a phrase or and?",
+            "Takes effect on redeploy",
+            ZONE_ID, APPLICATION_ID);
+
+    public static final UnboundStringFlag PROXY_PROTOCOL = defineStringFlag(
+            "proxy-protocol", "https-only",
+            "Enable proxy protocol support on application containers. Allowed values: ['https-only', 'https+proxy-protocol', 'proxy-protocol-only']",
+            "Takes effect on internal redeploy",
             APPLICATION_ID);
 
-    public static final UnboundBooleanFlag USE_CONFIG_SERVER_FOR_TESTER_API_CALLS = defineFeatureFlag(
-            "use-config-server-for-tester-api-calls", false,
-            "Whether controller should send requests to tester API through config server (if false) or tester endpoint (if true)",
+    public static final UnboundBooleanFlag ALLOW_DIRECT_ROUTING = defineFeatureFlag(
+            "publish-direct-routing-endpoint", false,
+            "Whether an application should receive a directly routed endpoint in its endpoint list",
             "Takes effect immediately",
-            ZONE_ID);
-
-    public static final UnboundBooleanFlag INSTALL_L4_ROUTING_SUPPORT = defineFeatureFlag(
-            "install-l4-routing-support", false,
-            "Whether routing nodes should install package supporting L4 routing",
-            "Takes effect immediately",
-            ZONE_ID, HOSTNAME);
-
-    public static final UnboundBooleanFlag GENERATE_L4_ROUTING_CONFIG = defineFeatureFlag(
-            "generate-l4-routing-config", false,
-            "Whether routing nodes should generate L4 routing config",
-            "Takes effect immediately",
-            ZONE_ID, HOSTNAME);
-
-    public static final UnboundBooleanFlag GENERATE_ROUTING_CONFIG_FOR_TESTER_APPLICATIONS = defineFeatureFlag(
-            "generate-routing-config-for-tester-applications", true,
-            "Whether config server should generate routing config (lb-services) for tester applications",
-            "Takes effect immediately",
-            ZONE_ID);
+            APPLICATION_ID);
 
+    public static final UnboundBooleanFlag NGINX_UPSTREAM_PROXY_PROTOCOL = defineFeatureFlag(
+            "nginx-upstream-proxy-protocol", false,
+            "Whether the nginx should enable proxy-protocol for all upstreams",
+            "Takes effect immediately");
 
     /** WARNING: public for testing: All flags should be defined in {@link Flags}. */
     public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, String description,
diff --git a/flags/src/main/java/com/yahoo/vespa/flags/json/RelationalCondition.java b/flags/src/main/java/com/yahoo/vespa/flags/json/RelationalCondition.java
index afaf94b26b6..db2f0a3a197 100644
--- a/flags/src/main/java/com/yahoo/vespa/flags/json/RelationalCondition.java
+++ b/flags/src/main/java/com/yahoo/vespa/flags/json/RelationalCondition.java
@@ -5,7 +5,6 @@ import com.yahoo.component.Version;
 import com.yahoo.vespa.flags.FetchVector;
 import com.yahoo.vespa.flags.json.wire.WireCondition;
 
-import java.util.List;
 import java.util.function.Predicate;
 
 /**
diff --git a/flags/src/main/java/com/yahoo/vespa/flags/json/RelationalPredicate.java b/flags/src/main/java/com/yahoo/vespa/flags/json/RelationalPredicate.java
index c5ad195e0d2..72bc5627112 100644
--- a/flags/src/main/java/com/yahoo/vespa/flags/json/RelationalPredicate.java
+++ b/flags/src/main/java/com/yahoo/vespa/flags/json/RelationalPredicate.java
@@ -23,7 +23,7 @@ public class RelationalPredicate {
 
         for (var operator : operatorsByDecendingLength) {
             if (predicateString.startsWith(operator.toText())) {
-                String suffix = predicateString.substring(operator.toText().length());
+                String suffix = predicateString.substring(operator.toText().length()).trim();
                 return new RelationalPredicate(predicateString, operator, suffix);
             }
         }
diff --git a/flags/src/test/java/com/yahoo/vespa/flags/json/FlagDataTest.java b/flags/src/test/java/com/yahoo/vespa/flags/json/FlagDataTest.java
index 9eaa4ae4504..7f939d10bb3 100644
--- a/flags/src/test/java/com/yahoo/vespa/flags/json/FlagDataTest.java
+++ b/flags/src/test/java/com/yahoo/vespa/flags/json/FlagDataTest.java
@@ -64,6 +64,12 @@ public class FlagDataTest {
                 .with(FetchVector.Dimension.HOSTNAME, "host1")
                 .with(FetchVector.Dimension.APPLICATION_ID, "app3"));
 
+        // Verify unsetting a dimension with null works.
+        verify(Optional.of("true"), vector
+                .with(FetchVector.Dimension.HOSTNAME, "host1")
+                .with(FetchVector.Dimension.APPLICATION_ID, "app3")
+                .with(FetchVector.Dimension.APPLICATION_ID, null));
+
         // No rules apply if zone is overridden to an unknown zone
         verify(Optional.empty(), vector.with(FetchVector.Dimension.ZONE_ID, "unknown zone"));
     }
diff --git a/fnet/src/tests/connect/connect_test.cpp b/fnet/src/tests/connect/connect_test.cpp
index b70b3fa8b01..62000efb682 100644
--- a/fnet/src/tests/connect/connect_test.cpp
+++ b/fnet/src/tests/connect/connect_test.cpp
@@ -65,7 +65,13 @@ struct BlockingCryptoEngine : public CryptoEngine {
     Gate handshake_work_enter;
     Gate handshake_work_exit;
     Gate handshake_socket_deleted;
-    CryptoSocket::UP create_crypto_socket(SocketHandle socket, bool) override {
+    bool use_tls_when_client() const override { return false; }
+    bool always_use_tls_when_server() const override { return false; }
+    CryptoSocket::UP create_client_crypto_socket(SocketHandle socket, const SocketSpec &) override {
+        return std::make_unique<BlockingCryptoSocket>(std::move(socket),
+                handshake_work_enter, handshake_work_exit, handshake_socket_deleted);
+    }
+    CryptoSocket::UP create_server_crypto_socket(SocketHandle socket) override {
         return std::make_unique<BlockingCryptoSocket>(std::move(socket),
                 handshake_work_enter, handshake_work_exit, handshake_socket_deleted);
     }
diff --git a/fnet/src/tests/frt/values/values_test.cpp b/fnet/src/tests/frt/values/values_test.cpp
index 5bc6b0e2dce..3b36e8989c1 100644
--- a/fnet/src/tests/frt/values/values_test.cpp
+++ b/fnet/src/tests/frt/values/values_test.cpp
@@ -3,6 +3,7 @@
 #include <vespa/fnet/frt/values.h>
 #include <vespa/fnet/databuffer.h>
 #include <vespa/fnet/info.h>
+#include <vespa/vespalib/util/stash.h>
 
 using vespalib::Stash;
 
diff --git a/fnet/src/vespa/fnet/connection.cpp b/fnet/src/vespa/fnet/connection.cpp
index 74a7d19387c..c5afd627a5a 100644
--- a/fnet/src/vespa/fnet/connection.cpp
+++ b/fnet/src/vespa/fnet/connection.cpp
@@ -9,10 +9,13 @@
 #include "config.h"
 #include "transport_thread.h"
 #include "transport.h"
+#include <vespa/vespalib/net/socket_spec.h>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".fnet");
 
+std::atomic<uint64_t> FNET_Connection::_num_connections = 0;
+
 namespace {
 class SyncPacket : public FNET_DummyPacket {
 private:
@@ -470,7 +473,7 @@ FNET_Connection::FNET_Connection(FNET_TransportThread *owner,
       _streamer(streamer),
       _serverAdapter(serverAdapter),
       _adminChannel(nullptr),
-      _socket(owner->owner().create_crypto_socket(std::move(socket), true)),
+      _socket(owner->owner().create_server_crypto_socket(std::move(socket))),
       _resolve_handler(nullptr),
       _context(),
       _state(FNET_CONNECTING),
@@ -489,6 +492,7 @@ FNET_Connection::FNET_Connection(FNET_TransportThread *owner,
       _cleanup(nullptr)
 {
     assert(_socket && (_socket->get_fd() >= 0));
+    _num_connections.fetch_add(1, std::memory_order_relaxed);
 }
 
 
@@ -526,6 +530,7 @@ FNET_Connection::FNET_Connection(FNET_TransportThread *owner,
         _adminChannel = admin.get();
         _channels.Register(admin.release());
     }
+    _num_connections.fetch_add(1, std::memory_order_relaxed);
 }
 
 
@@ -536,6 +541,7 @@ FNET_Connection::~FNET_Connection()
         delete _adminChannel;
     }
     assert(_cleanup == nullptr);
+    _num_connections.fetch_sub(1, std::memory_order_relaxed);
 }
 
 
@@ -574,7 +580,7 @@ FNET_Connection::handle_add_event()
 {
     if (_resolve_handler) {
         auto tweak = [this](vespalib::SocketHandle &handle) { return Owner()->tune(handle); };
-        _socket = Owner()->owner().create_crypto_socket(_resolve_handler->address.connect(tweak), false);
+        _socket = Owner()->owner().create_client_crypto_socket(_resolve_handler->address.connect(tweak), vespalib::SocketSpec(GetSpec()));
         _ioc_socket_fd = _socket->get_fd();
         _resolve_handler.reset();
     }
diff --git a/fnet/src/vespa/fnet/connection.h b/fnet/src/vespa/fnet/connection.h
index 760b1b96d4d..c9c49c5151a 100644
--- a/fnet/src/vespa/fnet/connection.h
+++ b/fnet/src/vespa/fnet/connection.h
@@ -10,6 +10,7 @@
 #include <vespa/vespalib/net/socket_handle.h>
 #include <vespa/vespalib/net/async_resolver.h>
 #include <vespa/vespalib/net/crypto_socket.h>
+#include <atomic>
 
 class FNET_IPacketStreamer;
 class FNET_IServerAdapter;
@@ -111,6 +112,8 @@ private:
 
     FNET_IConnectionCleanupHandler *_cleanup;  // cleanup handler
 
+    static std::atomic<uint64_t> _num_connections; // total number of connections
+
     FNET_Connection(const FNET_Connection &);
     FNET_Connection &operator=(const FNET_Connection &);
 
@@ -517,5 +520,10 @@ public:
      */
     uint32_t getInputBufferSize() const { return _input.GetBufSize(); }
 
+    /**
+     * @return the total number of connection objects
+     **/
+    static uint64_t get_num_connections() {
+        return _num_connections.load(std::memory_order_relaxed);
+    }
 };
-
diff --git a/fnet/src/vespa/fnet/databuffer.cpp b/fnet/src/vespa/fnet/databuffer.cpp
index 74a8bc4e12c..d982f9609e0 100644
--- a/fnet/src/vespa/fnet/databuffer.cpp
+++ b/fnet/src/vespa/fnet/databuffer.cpp
@@ -1,6 +1,7 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include "databuffer.h"
+#include <cstdio>
 
 FNET_DataBuffer::FNET_DataBuffer(uint32_t len)
     : _bufstart(nullptr),
diff --git a/fnet/src/vespa/fnet/frt/rpcrequest.h b/fnet/src/vespa/fnet/frt/rpcrequest.h
index cc871e7ac0c..eaa34a46b7a 100644
--- a/fnet/src/vespa/fnet/frt/rpcrequest.h
+++ b/fnet/src/vespa/fnet/frt/rpcrequest.h
@@ -5,6 +5,7 @@
 #include "values.h"
 #include "error.h"
 #include <vespa/fnet/context.h>
+#include <vespa/vespalib/util/stash.h>
 #include <atomic>
 
 class FNETConnection;
diff --git a/fnet/src/vespa/fnet/frt/supervisor.cpp b/fnet/src/vespa/fnet/frt/supervisor.cpp
index f9cbfe2a662..a8cb7884534 100644
--- a/fnet/src/vespa/fnet/frt/supervisor.cpp
+++ b/fnet/src/vespa/fnet/frt/supervisor.cpp
@@ -409,7 +409,7 @@ FRT_Supervisor::SchedulerPtr::SchedulerPtr(FNET_TransportThread *transport_threa
 namespace fnet::frt {
 
 StandaloneFRT::StandaloneFRT()
-    : _threadPool(std::make_unique<FastOS_ThreadPool>(1024*60)),
+    : _threadPool(std::make_unique<FastOS_ThreadPool>(1024*128)),
       _transport(std::make_unique<FNET_Transport>()),
       _supervisor(std::make_unique<FRT_Supervisor>(_transport.get()))
 {
@@ -417,7 +417,7 @@ StandaloneFRT::StandaloneFRT()
 }
 
 StandaloneFRT::StandaloneFRT(vespalib::CryptoEngine::SP crypto)
-    : _threadPool(std::make_unique<FastOS_ThreadPool>(1024*60)),
+    : _threadPool(std::make_unique<FastOS_ThreadPool>(1024*128)),
       _transport(std::make_unique<FNET_Transport>(std::move(crypto), 1)),
       _supervisor(std::make_unique<FRT_Supervisor>(_transport.get()))
 {
diff --git a/fnet/src/vespa/fnet/frt/values.cpp b/fnet/src/vespa/fnet/frt/values.cpp
index a5f59df19b2..3b37aa9a1bc 100644
--- a/fnet/src/vespa/fnet/frt/values.cpp
+++ b/fnet/src/vespa/fnet/frt/values.cpp
@@ -3,6 +3,7 @@
 #include "values.h"
 #include <vespa/fnet/databuffer.h>
 #include <vespa/vespalib/util/stringfmt.h>
+#include <vespa/vespalib/util/stash.h>
 #include <cassert>
 
 static_assert(sizeof(uint8_t) == 1, "uint8_t must be 1 byte.");
@@ -81,7 +82,7 @@ FRT_Values::FRT_Values(Stash &stash)
       _stash(stash)
 { }
 
-FRT_Values::~FRT_Values() { }
+FRT_Values::~FRT_Values() = default;
 
 LocalBlob::LocalBlob(const char *data, uint32_t len) :
         _data(Alloc::alloc(len)),
@@ -294,7 +295,7 @@ FRT_Values::AddSharedData(FRT_ISharedBlob *blob) {
 }
 
 void
-FRT_Values::AddData(vespalib::alloc::Alloc buf, uint32_t len) {
+FRT_Values::AddData(vespalib::alloc::Alloc && buf, uint32_t len) {
     AddSharedData(&_stash.create<LocalBlob>(std::move(buf), len));
 }
 
diff --git a/fnet/src/vespa/fnet/frt/values.h b/fnet/src/vespa/fnet/frt/values.h
index e00aec8423c..2aa7551c423 100644
--- a/fnet/src/vespa/fnet/frt/values.h
+++ b/fnet/src/vespa/fnet/frt/values.h
@@ -3,9 +3,10 @@
 #pragma once
 
 #include "isharedblob.h"
-#include <vespa/vespalib/util/stash.h>
 #include <cstring>
 
+namespace vespalib { class Stash; }
+namespace vespalib::alloc { class Alloc; }
 namespace fnet {
     char * copyString(char *dst, const char *src, size_t len);
     char * copyData(char *dst, const void *src, size_t len);
@@ -216,7 +217,7 @@ public:
     char *AddString(uint32_t len);
     FRT_StringValue *AddStringArray(uint32_t len);
     void AddSharedData(FRT_ISharedBlob *blob);
-    void AddData(Alloc buf, uint32_t len);
+    void AddData(Alloc && buf, uint32_t len);
     void AddData(const char *buf, uint32_t len);
     char *AddData(uint32_t len);
     FRT_DataValue *AddDataArray(uint32_t len);
diff --git a/fnet/src/vespa/fnet/transport.cpp b/fnet/src/vespa/fnet/transport.cpp
index 28e645d9e03..d3b52969c8c 100644
--- a/fnet/src/vespa/fnet/transport.cpp
+++ b/fnet/src/vespa/fnet/transport.cpp
@@ -54,9 +54,15 @@ FNET_Transport::resolve_async(const vespalib::string &spec,
 }
 
 vespalib::CryptoSocket::UP
-FNET_Transport::create_crypto_socket(vespalib::SocketHandle socket, bool is_server)
+FNET_Transport::create_client_crypto_socket(vespalib::SocketHandle socket, const vespalib::SocketSpec &spec)
 {
-    return _crypto_engine->create_crypto_socket(std::move(socket), is_server);
+    return _crypto_engine->create_client_crypto_socket(std::move(socket), spec);
+}
+
+vespalib::CryptoSocket::UP
+FNET_Transport::create_server_crypto_socket(vespalib::SocketHandle socket)
+{
+    return _crypto_engine->create_server_crypto_socket(std::move(socket));
 }
 
 FNET_TransportThread *
diff --git a/fnet/src/vespa/fnet/transport.h b/fnet/src/vespa/fnet/transport.h
index 8d1ba48c1b0..02ef22c7fb6 100644
--- a/fnet/src/vespa/fnet/transport.h
+++ b/fnet/src/vespa/fnet/transport.h
@@ -79,17 +79,25 @@ public:
                        vespalib::AsyncResolver::ResultHandler::WP result_handler);
 
     /**
-     * Wrap a plain socket endpoint in a CryptoSocket. The
+     * Wrap a plain socket endpoint (client side) in a CryptoSocket. The
      * implementation will be determined by the CryptoEngine used by
      * this Transport.
      *
      * @return socket abstraction able to perform encryption and decryption
      * @param socket low-level socket
-     * @param is_server which end of the connection the socket
-     *                  represents. This is needed to support
-     *                  asymmetrical handshaking.
+     * @param spec who we are connecting to
      **/
-    vespalib::CryptoSocket::UP create_crypto_socket(vespalib::SocketHandle socket, bool is_server);
+    vespalib::CryptoSocket::UP create_client_crypto_socket(vespalib::SocketHandle socket, const vespalib::SocketSpec &spec);
+
+    /**
+     * Wrap a plain socket endpoint (server side) in a CryptoSocket. The
+     * implementation will be determined by the CryptoEngine used by
+     * this Transport.
+     *
+     * @return socket abstraction able to perform encryption and decryption
+     * @param socket low-level socket
+     **/
+    vespalib::CryptoSocket::UP create_server_crypto_socket(vespalib::SocketHandle socket);
 
     /**
      * Select one of the underlying transport threads. The selection
diff --git a/functions.cmake b/functions.cmake
index fc6fbc0b146..43dd2d30853 100644
--- a/functions.cmake
+++ b/functions.cmake
@@ -558,7 +558,7 @@ function(install_fat_java_artifact NAME)
 endfunction()
 
 function(install_absolute_symlink TARGET LINK)
-    install(CODE "execute_process(COMMAND ln -snf ${TARGET} \$ENV{DESTDIR}/${CMAKE_INSTALL_PREFIX}/${LINK})")
+    install(CODE "execute_process(COMMAND ln -snf ${TARGET} \$ENV{DESTDIR}${CMAKE_INSTALL_PREFIX}/${LINK})")
 endfunction(install_absolute_symlink)
 
 function(install_symlink TARGET LINK)
diff --git a/hosted-api/src/main/java/ai/vespa/hosted/api/ControllerHttpClient.java b/hosted-api/src/main/java/ai/vespa/hosted/api/ControllerHttpClient.java
index f1486ae7117..ee30f6fd471 100644
--- a/hosted-api/src/main/java/ai/vespa/hosted/api/ControllerHttpClient.java
+++ b/hosted-api/src/main/java/ai/vespa/hosted/api/ControllerHttpClient.java
@@ -17,6 +17,7 @@ import com.yahoo.slime.JsonFormat;
 import com.yahoo.slime.ObjectTraverser;
 import com.yahoo.slime.Slime;
 
+import javax.net.ssl.SSLContext;
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
@@ -37,6 +38,7 @@ import java.util.ArrayList;
 import java.util.List;
 import java.util.OptionalLong;
 import java.util.concurrent.Callable;
+import java.util.function.Consumer;
 import java.util.function.Supplier;
 import java.util.stream.Stream;
 
@@ -76,6 +78,11 @@ public abstract class ControllerHttpClient {
         return new SigningControllerHttpClient(endpoint, privateKeyFile, id);
     }
 
+    /** Creates an HTTP client against the given endpoint, which uses the given SSL context for authentication. */
+    public static ControllerHttpClient withSSLContext(URI endpoint, SSLContext sslContext) {
+        return new MutualTlsControllerHttpClient(endpoint, sslContext);
+    }
+
     /** Creates an HTTP client against the given endpoint, which uses the given private key and certificate identity. */
     public static ControllerHttpClient withKeyAndCertificate(URI endpoint, Path privateKeyFile, Path certificateFile) {
         var privateKey = unchecked(() -> KeyUtils.fromPemEncodedPrivateKey(Files.readString(privateKeyFile, UTF_8)));
@@ -125,7 +132,7 @@ public abstract class ControllerHttpClient {
 
     /** Returns the Vespa version to compile against, for a hosted Vespa application. This is its lowest runtime version. */
     public String compileVersion(ApplicationId id) {
-        return toInspector(send(request(HttpRequest.newBuilder(applicationPath(id.tenant(), id.application()))
+        return toInspector(send(request(HttpRequest.newBuilder(compileVersionPath(id.tenant(), id.application()))
                                                    .timeout(Duration.ofSeconds(20)),
                                         GET)))
                 .field("compileVersion").asString();
@@ -141,10 +148,36 @@ public abstract class ControllerHttpClient {
     /** Returns the sorted list of log entries after the given after from the deployment job of the given ids. */
     public DeploymentLog deploymentLog(ApplicationId id, ZoneId zone, long run, long after) {
         return toDeploymentLog(send(request(HttpRequest.newBuilder(runPath(id, zone, run, after))
-                                                       .timeout(Duration.ofSeconds(10)),
+                                                       .timeout(Duration.ofMinutes(2)),
                                             GET)));
     }
 
+    /** Follows the given deployment job until it is done, or this thread is interrupted, at which point the current status is returned. */
+    public DeploymentLog followDeploymentUntilDone(ApplicationId id, ZoneId zone, long run,
+                                                   Consumer<DeploymentLog.Entry> out) {
+        long last = -1;
+        DeploymentLog log = null;
+        while (true) {
+            DeploymentLog update = deploymentLog(id, zone, run, last);
+            for (DeploymentLog.Entry entry : update.entries())
+                out.accept(entry);
+            log = (log == null ? update : log.updatedWith(update));
+            last = log.last().orElse(last);
+
+            if ( ! log.isActive())
+                break;
+
+            try {
+                Thread.sleep(1000);
+            }
+            catch (InterruptedException e) {
+                Thread.currentThread().interrupt();
+                break;
+            }
+        }
+        return log;
+    }
+
     /** Returns the sorted list of log entries from the deployment job of the given ids. */
     public DeploymentLog deploymentLog(ApplicationId id, ZoneId zone, long run) {
         return deploymentLog(id, zone, run, -1);
@@ -179,6 +212,10 @@ public abstract class ControllerHttpClient {
         return concatenated(tenantPath(tenant), "application", application.value());
     }
 
+    private URI compileVersionPath(TenantName tenant, ApplicationName application) {
+        return concatenated(applicationPath(tenant, application), "compile-version");
+    }
+
     private URI instancePath(ApplicationId id) {
         return concatenated(applicationPath(id.tenant(), id.application()), "instance", id.instance().value());
     }
@@ -384,25 +421,29 @@ public abstract class ControllerHttpClient {
     /** Client that uses a given key / certificate identity to authenticate to the remote controller. */
     private static class MutualTlsControllerHttpClient extends ControllerHttpClient {
 
+        private MutualTlsControllerHttpClient(URI endpoint, SSLContext sslContext) {
+            super(endpoint, HttpClient.newBuilder().sslContext(sslContext));
+        }
+
         private MutualTlsControllerHttpClient(URI endpoint, PrivateKey privateKey, List<X509Certificate> certs) {
-            super(endpoint,
-                  HttpClient.newBuilder()
-                            .sslContext(new SslContextBuilder().withKeyStore(privateKey, certs).build()));
+            this(endpoint, new SslContextBuilder().withKeyStore(privateKey, certs).build());
         }
 
     }
 
+
     private static DeploymentLog.Status valueOf(String status) {
         switch (status) {
-            case "running":             return DeploymentLog.Status.running;
-            case "aborted":             return DeploymentLog.Status.aborted;
-            case "error":               return DeploymentLog.Status.error;
-            case "testFailure":         return DeploymentLog.Status.testFailure;
-            case "outOfCapacity":       return DeploymentLog.Status.outOfCapacity;
-            case "installationFailed":  return DeploymentLog.Status.installationFailed;
-            case "deploymentFailed":    return DeploymentLog.Status.deploymentFailed;
-            case "success":             return DeploymentLog.Status.success;
-            default:                    throw new IllegalArgumentException("Unexpected status '" + status + "'");
+            case "running":                    return DeploymentLog.Status.running;
+            case "aborted":                    return DeploymentLog.Status.aborted;
+            case "error":                      return DeploymentLog.Status.error;
+            case "testFailure":                return DeploymentLog.Status.testFailure;
+            case "outOfCapacity":              return DeploymentLog.Status.outOfCapacity;
+            case "installationFailed":         return DeploymentLog.Status.installationFailed;
+            case "deploymentFailed":           return DeploymentLog.Status.deploymentFailed;
+            case "endpointCertificateTimeout": return DeploymentLog.Status.endpointCertificateTimeout;
+            case "success":                    return DeploymentLog.Status.success;
+            default: throw new IllegalArgumentException("Unexpected status '" + status + "'");
         }
     }
 
diff --git a/hosted-api/src/main/java/ai/vespa/hosted/api/DeploymentLog.java b/hosted-api/src/main/java/ai/vespa/hosted/api/DeploymentLog.java
index 177c72107e0..90e973da49d 100644
--- a/hosted-api/src/main/java/ai/vespa/hosted/api/DeploymentLog.java
+++ b/hosted-api/src/main/java/ai/vespa/hosted/api/DeploymentLog.java
@@ -4,6 +4,7 @@ package ai.vespa.hosted.api;
 import java.time.Instant;
 import java.util.List;
 import java.util.OptionalLong;
+import java.util.stream.Stream;
 
 import static java.util.Comparator.comparing;
 import static java.util.stream.Collectors.toUnmodifiableList;
@@ -27,6 +28,14 @@ public class DeploymentLog {
         this.last = last;
     }
 
+    /** Returns this log updated with the content of the other. */
+    public DeploymentLog updatedWith(DeploymentLog other) {
+        return new DeploymentLog(Stream.concat(entries.stream(), other.entries.stream()).collect(toUnmodifiableList()),
+                                 other.active,
+                                 other.status,
+                                 other.last);
+    }
+
     public List<Entry> entries() {
         return entries;
     }
@@ -103,6 +112,7 @@ public class DeploymentLog {
         outOfCapacity,
         installationFailed,
         deploymentFailed,
+        endpointCertificateTimeout,
         success;
     }
 
diff --git a/hosted-api/src/main/java/ai/vespa/hosted/api/Properties.java b/hosted-api/src/main/java/ai/vespa/hosted/api/Properties.java
index fd640bcc235..22c32cfa9ec 100644
--- a/hosted-api/src/main/java/ai/vespa/hosted/api/Properties.java
+++ b/hosted-api/src/main/java/ai/vespa/hosted/api/Properties.java
@@ -52,7 +52,7 @@ public class Properties {
         return getNonBlankProperty("apiCertificateFile").map(Paths::get);
     }
 
-    /** Returns the actual private key as a string */
+    /** Returns the actual private key as a string. */
     public static Optional<String> apiKey() {
         return getNonBlankProperty("apiKey");
     }
diff --git a/http-utils/src/main/java/ai/vespa/util/http/retry/DelaySupplier.java b/http-utils/src/main/java/ai/vespa/util/http/retry/DelaySupplier.java
new file mode 100644
index 00000000000..a97024df95c
--- /dev/null
+++ b/http-utils/src/main/java/ai/vespa/util/http/retry/DelaySupplier.java
@@ -0,0 +1,44 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package ai.vespa.util.http.retry;
+
+import java.time.Duration;
+
+/**
+ * An abstraction that calculates the next delay based on the current retry count.
+ *
+ * @author bjorncs
+ */
+@FunctionalInterface
+interface DelaySupplier {
+    Duration getDelay(int executionCount);
+
+    class Fixed implements DelaySupplier {
+        private final Duration delay;
+
+        Fixed(Duration delay) {
+            this.delay = delay;
+        }
+
+        @Override
+        public Duration getDelay(int executionCount) { return delay; }
+    }
+
+    class Exponential implements DelaySupplier {
+        private final Duration startDelay;
+        private final Duration maxDelay;
+
+        Exponential(Duration startDelay, Duration maxDelay) {
+            this.startDelay = startDelay;
+            this.maxDelay = maxDelay;
+        }
+
+        @Override
+        public Duration getDelay(int executionCount) {
+            Duration nextDelay = startDelay;
+            for (int i = 1; i < executionCount; ++i) {
+                nextDelay = nextDelay.multipliedBy(2);
+            }
+            return maxDelay.compareTo(nextDelay) > 0 ? nextDelay : maxDelay;
+        }
+    }
+}
diff --git a/http-utils/src/main/java/ai/vespa/util/http/retry/DelayedHttpRequestRetryHandler.java b/http-utils/src/main/java/ai/vespa/util/http/retry/DelayedConnectionLevelRetryHandler.java
index 72bb171c4c7..dca1907c08b 100644
--- a/http-utils/src/main/java/ai/vespa/util/http/retry/DelayedHttpRequestRetryHandler.java
+++ b/http-utils/src/main/java/ai/vespa/util/http/retry/DelayedConnectionLevelRetryHandler.java
@@ -10,7 +10,6 @@ import org.apache.http.protocol.HttpContext;
 import java.io.IOException;
 import java.time.Duration;
 import java.util.List;
-import java.util.function.BiPredicate;
 import java.util.function.Predicate;
 import java.util.logging.Logger;
 
@@ -20,36 +19,23 @@ import java.util.logging.Logger;
  * @author bjorncs
  */
 @Contract(threading = ThreadingBehavior.IMMUTABLE)
-public class DelayedHttpRequestRetryHandler implements HttpRequestRetryHandler {
+public class DelayedConnectionLevelRetryHandler implements HttpRequestRetryHandler {
 
     private static final Logger log = Logger.getLogger(HttpRequestRetryHandler.class.getName());
 
-    @FunctionalInterface
-    public interface RetryConsumer {
-        void onRetry(IOException exception, Duration delay, int executionCount, HttpClientContext context);
-    }
-
-    @FunctionalInterface
-    public interface RetryFailedConsumer {
-        void onRetryFailed(IOException exception, int executionCount, HttpClientContext context);
-    }
-
-    @FunctionalInterface
-    public interface RetryPredicate extends BiPredicate<IOException, HttpClientContext> {}
-
     private final DelaySupplier delaySupplier;
     private final int maxRetries;
-    private final RetryPredicate predicate;
-    private final RetryConsumer retryConsumer;
-    private final RetryFailedConsumer retryFailedConsumer;
+    private final RetryPredicate<IOException> predicate;
+    private final RetryConsumer<IOException> retryConsumer;
+    private final RetryFailedConsumer<IOException> retryFailedConsumer;
     private final Sleeper sleeper;
 
-    private DelayedHttpRequestRetryHandler(
+    private DelayedConnectionLevelRetryHandler(
             DelaySupplier delaySupplier,
             int maxRetries,
-            RetryPredicate predicate,
-            RetryConsumer retryConsumer,
-            RetryFailedConsumer retryFailedConsumer,
+            RetryPredicate<IOException> predicate,
+            RetryConsumer<IOException> retryConsumer,
+            RetryFailedConsumer<IOException> retryFailedConsumer,
             Sleeper sleeper) {
         this.delaySupplier = delaySupplier;
         this.maxRetries = maxRetries;
@@ -84,10 +70,10 @@ public class DelayedHttpRequestRetryHandler implements HttpRequestRetryHandler {
 
         private final DelaySupplier delaySupplier;
         private final int maxRetries;
-        private RetryPredicate predicate = (ioException, ctx) -> true;
-        private RetryConsumer retryConsumer = (exception, delay, count, ctx) -> {};
-        private RetryFailedConsumer retryFailedConsumer = (exception, count, ctx) -> {};
-        private Sleeper sleeper = new DefaultSleeper();
+        private RetryPredicate<IOException> predicate = (ioException, ctx) -> true;
+        private RetryConsumer<IOException> retryConsumer = (exception, delay, count, ctx) -> {};
+        private RetryFailedConsumer<IOException> retryFailedConsumer = (exception, count, ctx) -> {};
+        private Sleeper sleeper = new Sleeper.Default();
 
         private Builder(DelaySupplier delaySupplier, int maxRetries) {
             this.delaySupplier = delaySupplier;
@@ -95,19 +81,11 @@ public class DelayedHttpRequestRetryHandler implements HttpRequestRetryHandler {
         }
 
         public static Builder withFixedDelay(Duration delay, int maxRetries) {
-            return new Builder(executionCount -> delay, maxRetries);
+            return new Builder(new DelaySupplier.Fixed(delay), maxRetries);
         }
 
         public static Builder withExponentialBackoff(Duration startDelay, Duration maxDelay, int maxRetries) {
-            return new Builder(
-                    executionCount -> {
-                        Duration nextDelay = startDelay;
-                        for (int i = 1; i < executionCount; ++i) {
-                            nextDelay = nextDelay.multipliedBy(2);
-                        }
-                        return maxDelay.compareTo(nextDelay) > 0 ? nextDelay : maxDelay;
-                    },
-                    maxRetries);
+            return new Builder(new DelaySupplier.Exponential(startDelay, maxDelay), maxRetries);
         }
 
         public Builder retryForExceptions(List<Class<? extends IOException>> exceptionTypes) {
@@ -120,17 +98,17 @@ public class DelayedHttpRequestRetryHandler implements HttpRequestRetryHandler {
             return this;
         }
 
-        public Builder retryFor(RetryPredicate predicate) {
+        public Builder retryFor(RetryPredicate<IOException> predicate) {
             this.predicate = predicate;
             return this;
         }
 
-        public Builder onRetry(RetryConsumer consumer) {
+        public Builder onRetry(RetryConsumer<IOException> consumer) {
             this.retryConsumer = consumer;
             return this;
         }
 
-        public Builder onRetryFailed(RetryFailedConsumer consumer) {
+        public Builder onRetryFailed(RetryFailedConsumer<IOException> consumer) {
             this.retryFailedConsumer = consumer;
             return this;
         }
@@ -141,29 +119,8 @@ public class DelayedHttpRequestRetryHandler implements HttpRequestRetryHandler {
             return this;
         }
 
-        public DelayedHttpRequestRetryHandler build() {
-            return new DelayedHttpRequestRetryHandler(delaySupplier, maxRetries, predicate, retryConsumer, retryFailedConsumer, sleeper);
+        public DelayedConnectionLevelRetryHandler build() {
+            return new DelayedConnectionLevelRetryHandler(delaySupplier, maxRetries, predicate, retryConsumer, retryFailedConsumer, sleeper);
         }
-
-        private static class DefaultSleeper implements Sleeper {
-            @Override
-            public void sleep(Duration duration) {
-                try {
-                    Thread.sleep(duration.toMillis());
-                } catch (InterruptedException e) {
-                    throw new RuntimeException(e);
-                }
-            }
-        }
-    }
-
-    // For unit testing
-    interface Sleeper {
-        void sleep(Duration duration);
-    }
-
-    @FunctionalInterface
-    private interface DelaySupplier {
-        Duration getDelay(int executionCount);
     }
 }
diff --git a/http-utils/src/main/java/ai/vespa/util/http/retry/DelayedResponseLevelRetryHandler.java b/http-utils/src/main/java/ai/vespa/util/http/retry/DelayedResponseLevelRetryHandler.java
new file mode 100644
index 00000000000..041b501809c
--- /dev/null
+++ b/http-utils/src/main/java/ai/vespa/util/http/retry/DelayedResponseLevelRetryHandler.java
@@ -0,0 +1,125 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package ai.vespa.util.http.retry;
+
+import org.apache.http.HttpResponse;
+import org.apache.http.annotation.Contract;
+import org.apache.http.annotation.ThreadingBehavior;
+import org.apache.http.client.ServiceUnavailableRetryStrategy;
+import org.apache.http.client.protocol.HttpClientContext;
+import org.apache.http.protocol.HttpContext;
+
+import java.time.Duration;
+import java.util.List;
+import java.util.function.Predicate;
+import java.util.logging.Logger;
+
+/**
+ * A {@link ServiceUnavailableRetryStrategy} that supports delayed retries on any response types.
+ *
+ * @author bjorncs
+ */
+@Contract(threading = ThreadingBehavior.IMMUTABLE)
+public class DelayedResponseLevelRetryHandler implements ServiceUnavailableRetryStrategy {
+
+    private static final Logger log = Logger.getLogger(DelayedResponseLevelRetryHandler.class.getName());
+
+    private final DelaySupplier delaySupplier;
+    private final int maxRetries;
+    private final RetryPredicate<HttpResponse> predicate;
+    private final RetryConsumer<HttpResponse> retryConsumer;
+    private final RetryFailedConsumer<HttpResponse> retryFailedConsumer;
+    private final ThreadLocal<Long> retryInterval = ThreadLocal.withInitial(() -> 0L);
+
+    private DelayedResponseLevelRetryHandler(
+            DelaySupplier delaySupplier,
+            int maxRetries,
+            RetryPredicate<HttpResponse> predicate,
+            RetryConsumer<HttpResponse> retryConsumer,
+            RetryFailedConsumer<HttpResponse> retryFailedConsumer) {
+
+        this.delaySupplier = delaySupplier;
+        this.maxRetries = maxRetries;
+        this.predicate = predicate;
+        this.retryConsumer = retryConsumer;
+        this.retryFailedConsumer = retryFailedConsumer;
+    }
+
+    @Override
+    public boolean retryRequest(HttpResponse response, int executionCount, HttpContext ctx) {
+        log.fine(() -> String.format("retryRequest(responseCode='%s', executionCount='%d', ctx='%s'",
+                                     response.getStatusLine().getStatusCode(), executionCount, ctx));
+        HttpClientContext clientCtx = HttpClientContext.adapt(ctx);
+        if (!predicate.test(response, clientCtx)) {
+            log.fine(() -> String.format("Not retrying for '%s'", ctx));
+            return false;
+        }
+        if (executionCount > maxRetries) {
+            log.fine(() -> String.format("Max retries exceeded for '%s'", ctx));
+            retryFailedConsumer.onRetryFailed(response, executionCount, clientCtx);
+            return false;
+        }
+        Duration delay = delaySupplier.getDelay(executionCount);
+        log.fine(() -> String.format("Retrying after %s for '%s'", delay, ctx));
+        retryInterval.set(delay.toMillis());
+        retryConsumer.onRetry(response, delay, executionCount, clientCtx);
+        return true;
+    }
+
+    @Override
+    public long getRetryInterval() {
+        // Calls to getRetryInterval are always guarded by a call to retryRequest (using the same thread).
+        // A thread local allows this retry handler to be thread safe and support dynamic retry intervals
+        return retryInterval.get();
+    }
+
+    public static class Builder {
+
+        private final DelaySupplier delaySupplier;
+        private final int maxRetries;
+        private RetryPredicate<HttpResponse> predicate = (response, ctx) -> true;
+        private RetryConsumer<HttpResponse> retryConsumer = (response, delay, count, ctx) -> {};
+        private RetryFailedConsumer<HttpResponse> retryFailedConsumer = (response, count, ctx) -> {};
+
+        private Builder(DelaySupplier delaySupplier, int maxRetries) {
+            this.delaySupplier = delaySupplier;
+            this.maxRetries = maxRetries;
+        }
+
+        public static Builder withFixedDelay(Duration delay, int maxRetries) {
+            return new Builder(new DelaySupplier.Fixed(delay), maxRetries);
+        }
+
+        public static Builder withExponentialBackoff(Duration startDelay, Duration maxDelay, int maxRetries) {
+            return new Builder(new DelaySupplier.Exponential(startDelay, maxDelay), maxRetries);
+        }
+
+        public Builder retryForStatusCodes(List<Integer> statusCodes) {
+            this.predicate = (response, ctx) -> statusCodes.contains(response.getStatusLine().getStatusCode());
+            return this;
+        }
+
+        public Builder retryForResponses(Predicate<HttpResponse> predicate) {
+            this.predicate = (response, ctx) -> predicate.test(response);
+            return this;
+        }
+
+        public Builder retryFor(RetryPredicate<HttpResponse> predicate) {
+            this.predicate = predicate;
+            return this;
+        }
+
+        public Builder onRetry(RetryConsumer<HttpResponse> consumer) {
+            this.retryConsumer = consumer;
+            return this;
+        }
+
+        public Builder onRetryFailed(RetryFailedConsumer<HttpResponse> consumer) {
+            this.retryFailedConsumer = consumer;
+            return this;
+        }
+
+        public DelayedResponseLevelRetryHandler build() {
+            return new DelayedResponseLevelRetryHandler(delaySupplier, maxRetries, predicate, retryConsumer, retryFailedConsumer);
+        }
+    }
+}
diff --git a/http-utils/src/main/java/ai/vespa/util/http/retry/RetryConsumer.java b/http-utils/src/main/java/ai/vespa/util/http/retry/RetryConsumer.java
new file mode 100644
index 00000000000..494be051673
--- /dev/null
+++ b/http-utils/src/main/java/ai/vespa/util/http/retry/RetryConsumer.java
@@ -0,0 +1,16 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package ai.vespa.util.http.retry;
+
+import org.apache.http.client.protocol.HttpClientContext;
+
+import java.time.Duration;
+
+/**
+ * Invoked before performing a delay and retry.
+ *
+ * @author bjorncs
+ */
+@FunctionalInterface
+public interface RetryConsumer<T> {
+    void onRetry(T data, Duration delay, int executionCount, HttpClientContext context);
+}
diff --git a/http-utils/src/main/java/ai/vespa/util/http/retry/RetryFailedConsumer.java b/http-utils/src/main/java/ai/vespa/util/http/retry/RetryFailedConsumer.java
new file mode 100644
index 00000000000..ed326ac1210
--- /dev/null
+++ b/http-utils/src/main/java/ai/vespa/util/http/retry/RetryFailedConsumer.java
@@ -0,0 +1,14 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package ai.vespa.util.http.retry;
+
+import org.apache.http.client.protocol.HttpClientContext;
+
+/**
+ * Invoked after the last retry has failed.
+ *
+ * @author bjorncs
+ */
+@FunctionalInterface
+public interface RetryFailedConsumer<T> {
+    void onRetryFailed(T response, int executionCount, HttpClientContext context);
+}
diff --git a/http-utils/src/main/java/ai/vespa/util/http/retry/RetryPredicate.java b/http-utils/src/main/java/ai/vespa/util/http/retry/RetryPredicate.java
new file mode 100644
index 00000000000..ccf62c9be9f
--- /dev/null
+++ b/http-utils/src/main/java/ai/vespa/util/http/retry/RetryPredicate.java
@@ -0,0 +1,13 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package ai.vespa.util.http.retry;
+
+import org.apache.http.client.protocol.HttpClientContext;
+
+import java.util.function.BiPredicate;
+
+/**
+ * A predicate that determines whether an operation should be retried.
+ *
+ * @author bjorncs
+ */
+public interface RetryPredicate<T> extends BiPredicate<T, HttpClientContext> {}
diff --git a/http-utils/src/main/java/ai/vespa/util/http/retry/Sleeper.java b/http-utils/src/main/java/ai/vespa/util/http/retry/Sleeper.java
new file mode 100644
index 00000000000..06a7359f307
--- /dev/null
+++ b/http-utils/src/main/java/ai/vespa/util/http/retry/Sleeper.java
@@ -0,0 +1,26 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package ai.vespa.util.http.retry;
+
+import java.time.Duration;
+
+/**
+ * An abstraction used for mocking {@link Thread#sleep(long)} in unit tests.
+ *
+ * @author bjorncs
+ */
+interface Sleeper {
+    void sleep(Duration duration);
+
+    class Default implements Sleeper {
+        @Override
+        public void sleep(Duration duration) {
+            try {
+                Thread.sleep(duration.toMillis());
+            } catch (InterruptedException e) {
+                Thread.currentThread().interrupt();
+                throw new RuntimeException(e);
+            }
+        }
+    }
+}
+
diff --git a/http-utils/src/test/java/ai/vespa/util/http/retry/DelayedHttpRequestRetryHandlerTest.java b/http-utils/src/test/java/ai/vespa/util/http/retry/DelayedConnectionLevelRetryHandlerTest.java
index 51a05f6b2a7..85adeae6d78 100644
--- a/http-utils/src/test/java/ai/vespa/util/http/retry/DelayedHttpRequestRetryHandlerTest.java
+++ b/http-utils/src/test/java/ai/vespa/util/http/retry/DelayedConnectionLevelRetryHandlerTest.java
@@ -1,9 +1,6 @@
 // Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package ai.vespa.util.http.retry;
 
-import ai.vespa.util.http.retry.DelayedHttpRequestRetryHandler.RetryConsumer;
-import ai.vespa.util.http.retry.DelayedHttpRequestRetryHandler.RetryFailedConsumer;
-import ai.vespa.util.http.retry.DelayedHttpRequestRetryHandler.Sleeper;
 import com.yahoo.vespa.jdk8compat.List;
 import org.apache.http.client.protocol.HttpClientContext;
 import org.junit.Test;
@@ -22,17 +19,18 @@ import static org.mockito.Mockito.verify;
 /**
  * @author bjorncs
  */
-public class DelayedHttpRequestRetryHandlerTest {
+public class DelayedConnectionLevelRetryHandlerTest {
 
+    @SuppressWarnings("unchecked")
     @Test
     public void retry_consumers_are_invoked() {
-        RetryConsumer retryConsumer = mock(RetryConsumer.class);
-        RetryFailedConsumer retryFailedConsumer = mock(RetryFailedConsumer.class);
+        RetryConsumer<IOException> retryConsumer = (RetryConsumer<IOException>) mock(RetryConsumer.class);
+        RetryFailedConsumer<IOException> retryFailedConsumer = (RetryFailedConsumer<IOException>) mock(RetryFailedConsumer.class);
 
         Duration delay = Duration.ofSeconds(10);
         int maxRetries = 5;
 
-        DelayedHttpRequestRetryHandler handler = DelayedHttpRequestRetryHandler.Builder
+        DelayedConnectionLevelRetryHandler handler = DelayedConnectionLevelRetryHandler.Builder
                 .withFixedDelay(delay, maxRetries)
                 .withSleeper(mock(Sleeper.class))
                 .onRetry(retryConsumer)
@@ -59,7 +57,7 @@ public class DelayedHttpRequestRetryHandlerTest {
         Duration delay = Duration.ofSeconds(2);
         int maxRetries = 2;
 
-        DelayedHttpRequestRetryHandler handler = DelayedHttpRequestRetryHandler.Builder
+        DelayedConnectionLevelRetryHandler handler = DelayedConnectionLevelRetryHandler.Builder
                 .withFixedDelay(delay, maxRetries)
                 .withSleeper(sleeper)
                 .build();
@@ -82,7 +80,7 @@ public class DelayedHttpRequestRetryHandlerTest {
         Duration maxDelay = Duration.ofSeconds(5);
         int maxRetries = 10;
 
-        DelayedHttpRequestRetryHandler handler = DelayedHttpRequestRetryHandler.Builder
+        DelayedConnectionLevelRetryHandler handler = DelayedConnectionLevelRetryHandler.Builder
                 .withExponentialBackoff(startDelay, maxDelay, maxRetries)
                 .withSleeper(sleeper)
                 .build();
@@ -105,7 +103,7 @@ public class DelayedHttpRequestRetryHandlerTest {
     public void retries_for_listed_exceptions_until_max_retries_exceeded() {
         int maxRetries = 2;
 
-        DelayedHttpRequestRetryHandler handler = DelayedHttpRequestRetryHandler.Builder
+        DelayedConnectionLevelRetryHandler handler = DelayedConnectionLevelRetryHandler.Builder
                 .withFixedDelay(Duration.ofSeconds(2), maxRetries)
                 .retryForExceptions(List.of(SSLException.class, ConnectException.class))
                 .withSleeper(mock(Sleeper.class))
@@ -122,7 +120,7 @@ public class DelayedHttpRequestRetryHandlerTest {
 
     @Test
     public void does_not_retry_for_non_listed_exception() {
-        DelayedHttpRequestRetryHandler handler = DelayedHttpRequestRetryHandler.Builder
+        DelayedConnectionLevelRetryHandler handler = DelayedConnectionLevelRetryHandler.Builder
                 .withFixedDelay(Duration.ofSeconds(2), 2)
                 .retryForExceptions(List.of(SSLException.class, ConnectException.class))
                 .withSleeper(mock(Sleeper.class))
diff --git a/http-utils/src/test/java/ai/vespa/util/http/retry/DelayedResponseLevelRetryHandlerTest.java b/http-utils/src/test/java/ai/vespa/util/http/retry/DelayedResponseLevelRetryHandlerTest.java
new file mode 100644
index 00000000000..7be1e143078
--- /dev/null
+++ b/http-utils/src/test/java/ai/vespa/util/http/retry/DelayedResponseLevelRetryHandlerTest.java
@@ -0,0 +1,129 @@
+package ai.vespa.util.http.retry;// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+import org.apache.http.HttpResponse;
+import org.apache.http.HttpStatus;
+import org.apache.http.HttpVersion;
+import org.apache.http.client.protocol.HttpClientContext;
+import org.apache.http.message.BasicHttpResponse;
+import org.apache.http.message.BasicStatusLine;
+import org.junit.Test;
+
+import java.time.Duration;
+import java.util.List;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+
+/**
+ * @author bjorncs
+ */
+public class DelayedResponseLevelRetryHandlerTest {
+
+    @Test
+    @SuppressWarnings("unchecked")
+    public void retry_consumers_are_invoked() {
+        RetryConsumer<HttpResponse> retryConsumer = mock(RetryConsumer.class);
+        RetryFailedConsumer<HttpResponse> retryFailedConsumer = mock(RetryFailedConsumer.class);
+
+        Duration delay = Duration.ofSeconds(10);
+        int maxRetries = 5;
+
+        DelayedResponseLevelRetryHandler handler = DelayedResponseLevelRetryHandler.Builder
+                .withFixedDelay(delay, maxRetries)
+                .onRetry(retryConsumer)
+                .onRetryFailed(retryFailedConsumer)
+                .build();
+
+        HttpResponse response = createResponse(HttpStatus.SC_SERVICE_UNAVAILABLE);
+        HttpClientContext ctx = new HttpClientContext();
+        int lastExecutionCount = maxRetries + 1;
+        for (int i = 1; i <= lastExecutionCount; i++) {
+            handler.retryRequest(response, i, ctx);
+        }
+
+        verify(retryFailedConsumer).onRetryFailed(response, lastExecutionCount, ctx);
+        for (int i = 1; i < lastExecutionCount; i++) {
+            verify(retryConsumer).onRetry(response, delay, i, ctx);
+        }
+    }
+
+    @Test
+    public void retry_with_fixed_delay_sleeps_for_expected_duration() {
+        Duration delay = Duration.ofSeconds(2);
+        int maxRetries = 2;
+
+        DelayedResponseLevelRetryHandler handler = DelayedResponseLevelRetryHandler.Builder
+                .withFixedDelay(delay, maxRetries)
+                .build();
+
+        HttpResponse response = createResponse(HttpStatus.SC_SERVICE_UNAVAILABLE);
+        HttpClientContext ctx = new HttpClientContext();
+        int lastExecutionCount = maxRetries + 1;
+        for (int i = 1; i <= lastExecutionCount; i++) {
+            handler.retryRequest(response, i, ctx);
+            assertThat(handler.getRetryInterval()).isEqualTo(delay.toMillis());
+        }
+    }
+
+    @Test
+    public void retry_with_fixed_backoff_sleeps_for_expected_durations() {
+        Duration startDelay = Duration.ofMillis(500);
+        Duration maxDelay = Duration.ofSeconds(5);
+        int maxRetries = 10;
+
+        DelayedResponseLevelRetryHandler handler = DelayedResponseLevelRetryHandler.Builder
+                .withExponentialBackoff(startDelay, maxDelay, maxRetries)
+                .build();
+
+        HttpResponse response = createResponse(HttpStatus.SC_SERVICE_UNAVAILABLE);
+        HttpClientContext ctx = new HttpClientContext();
+        int lastExecutionCount = maxRetries + 1;
+        List<Duration> expectedIntervals =
+                com.yahoo.vespa.jdk8compat.List.of(
+                        startDelay, Duration.ofSeconds(1), Duration.ofSeconds(2), Duration.ofSeconds(4),
+                        Duration.ofSeconds(5), Duration.ofSeconds(5), Duration.ofSeconds(5), Duration.ofSeconds(5),
+                        Duration.ofSeconds(5), Duration.ofSeconds(5), Duration.ofSeconds(5));
+        for (int i = 1; i <= lastExecutionCount; i++) {
+            handler.retryRequest(response, i, ctx);
+            assertThat(handler.getRetryInterval()).isEqualTo(expectedIntervals.get(i-1).toMillis());
+        }
+    }
+
+    @Test
+    public void retries_for_listed_exceptions_until_max_retries_exceeded() {
+        int maxRetries = 2;
+
+        DelayedResponseLevelRetryHandler handler = DelayedResponseLevelRetryHandler.Builder
+                .withFixedDelay(Duration.ofSeconds(2), maxRetries)
+                .retryForStatusCodes(com.yahoo.vespa.jdk8compat.List.of(HttpStatus.SC_SERVICE_UNAVAILABLE, HttpStatus.SC_BAD_GATEWAY))
+                .build();
+
+        HttpResponse response = createResponse(HttpStatus.SC_SERVICE_UNAVAILABLE);
+        HttpClientContext ctx = new HttpClientContext();
+        int lastExecutionCount = maxRetries + 1;
+        for (int i = 1; i < lastExecutionCount; i++) {
+            assertTrue(handler.retryRequest(response, i, ctx));
+        }
+        assertFalse(handler.retryRequest(response, lastExecutionCount, ctx));
+    }
+
+    @Test
+    public void does_not_retry_for_non_listed_exception() {
+        DelayedResponseLevelRetryHandler handler = DelayedResponseLevelRetryHandler.Builder
+                .withFixedDelay(Duration.ofSeconds(2), 2)
+                .retryForStatusCodes(com.yahoo.vespa.jdk8compat.List.of(HttpStatus.SC_SERVICE_UNAVAILABLE, HttpStatus.SC_BAD_GATEWAY))
+                .build();
+
+        HttpResponse response = createResponse(HttpStatus.SC_OK);
+        HttpClientContext ctx = new HttpClientContext();
+        assertFalse(handler.retryRequest(response, 1, ctx));
+    }
+
+    private static HttpResponse createResponse(int statusCode) {
+        return new BasicHttpResponse(new BasicStatusLine(HttpVersion.HTTP_1_1, statusCode, "reason phrase"));
+    }
+
+}
\ No newline at end of file
diff --git a/jaxrs_client_utils/pom.xml b/jaxrs_client_utils/pom.xml
index d32d4c5eccc..e199a8e2639 100644
--- a/jaxrs_client_utils/pom.xml
+++ b/jaxrs_client_utils/pom.xml
@@ -81,6 +81,16 @@
       <artifactId>mockito-core</artifactId>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>com.sun.activation</groupId>
+      <artifactId>javax.activation</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>javax.xml.bind</groupId>
+      <artifactId>jaxb-api</artifactId>
+      <scope>test</scope>
+    </dependency>
   </dependencies>
   <build>
     <plugins>
diff --git a/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/athenz/AthenzAuthorizationFilter.java b/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/athenz/AthenzAuthorizationFilter.java
index 74e0ee36959..53270f3b51e 100644
--- a/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/athenz/AthenzAuthorizationFilter.java
+++ b/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/athenz/AthenzAuthorizationFilter.java
@@ -2,26 +2,35 @@
 package com.yahoo.jdisc.http.filter.security.athenz;
 
 import com.google.inject.Inject;
-import com.yahoo.jdisc.Response;
+import com.yahoo.jdisc.Metric;
 import com.yahoo.jdisc.http.filter.DiscFilterRequest;
-import com.yahoo.jdisc.http.filter.security.athenz.AthenzAuthorizationFilterConfig.CredentialsToVerify;
 import com.yahoo.jdisc.http.filter.security.athenz.RequestResourceMapper.ResourceNameAndAction;
 import com.yahoo.jdisc.http.filter.security.base.JsonSecurityRequestFilterBase;
+import com.yahoo.log.LogLevel;
+import com.yahoo.vespa.athenz.api.AthenzAccessToken;
 import com.yahoo.vespa.athenz.api.AthenzIdentity;
 import com.yahoo.vespa.athenz.api.AthenzPrincipal;
-import com.yahoo.vespa.athenz.api.AthenzResourceName;
-import com.yahoo.vespa.athenz.api.AthenzRole;
 import com.yahoo.vespa.athenz.api.ZToken;
 import com.yahoo.vespa.athenz.tls.AthenzX509CertificateUtils;
+import com.yahoo.vespa.athenz.utils.AthenzIdentities;
 import com.yahoo.vespa.athenz.zpe.AuthorizationResult;
 import com.yahoo.vespa.athenz.zpe.DefaultZpe;
 import com.yahoo.vespa.athenz.zpe.Zpe;
 
 import java.security.cert.X509Certificate;
+import java.util.EnumSet;
+import java.util.List;
+import java.util.Map;
 import java.util.Optional;
-import java.util.function.Function;
+import java.util.logging.Logger;
+import java.util.stream.Collectors;
 
-import static java.util.Collections.singletonList;
+import static com.yahoo.jdisc.Response.Status.FORBIDDEN;
+import static com.yahoo.jdisc.Response.Status.UNAUTHORIZED;
+import static com.yahoo.jdisc.http.filter.security.athenz.AthenzAuthorizationFilterConfig.EnabledCredentials;
+import static com.yahoo.jdisc.http.filter.security.athenz.AthenzAuthorizationFilterConfig.EnabledCredentials.ACCESS_TOKEN;
+import static com.yahoo.jdisc.http.filter.security.athenz.AthenzAuthorizationFilterConfig.EnabledCredentials.ROLE_CERTIFICATE;
+import static com.yahoo.jdisc.http.filter.security.athenz.AthenzAuthorizationFilterConfig.EnabledCredentials.ROLE_TOKEN;
 
 /**
  * An Athenz security filter that uses a configured action and resource name to control access.
@@ -30,116 +39,193 @@ import static java.util.Collections.singletonList;
  */
 public class AthenzAuthorizationFilter extends JsonSecurityRequestFilterBase {
 
-    private final String headerName;
+    private static final String ATTRIBUTE_PREFIX = "jdisc-security-filters.athenz-authorization-filter";
+    public static final String RESULT_ATTRIBUTE = ATTRIBUTE_PREFIX + ".result";
+    public static final String MATCHED_ROLE_ATTRIBUTE = ATTRIBUTE_PREFIX + ".matched-role";
+    public static final String IDENTITY_NAME_ATTRIBUTE = ATTRIBUTE_PREFIX + ".identity-name";
+    public static final String MATCHED_CREDENTIAL_TYPE_ATTRIBUTE = ATTRIBUTE_PREFIX + ".credentials-type";
+    private static final String ACCEPTED_METRIC_NAME = "jdisc.http.filter.athenz.accepted_requests";
+    private static final String REJECTED_METRIC_NAME = "jdisc.http.filter.athenz.rejected_requests";
+
+    private static final Logger log = Logger.getLogger(AthenzAuthorizationFilter.class.getName());
+
+    private final String roleTokenHeaderName;
+    private final EnumSet<EnabledCredentials.Enum> enabledCredentials;
     private final Zpe zpe;
     private final RequestResourceMapper requestResourceMapper;
-    private final CredentialsToVerify.Enum credentialsToVerify;
+    private final Metric metric;
 
     @Inject
-    public AthenzAuthorizationFilter(AthenzAuthorizationFilterConfig config, RequestResourceMapper resourceMapper) {
-        this(config, resourceMapper, new DefaultZpe());
+    public AthenzAuthorizationFilter(AthenzAuthorizationFilterConfig config, RequestResourceMapper resourceMapper, Metric metric) {
+        this(config, resourceMapper, new DefaultZpe(), metric);
     }
 
-    AthenzAuthorizationFilter(AthenzAuthorizationFilterConfig config,
-                              RequestResourceMapper resourceMapper,
-                              Zpe zpe) {
-        this.headerName = config.roleTokenHeaderName();
-        this.credentialsToVerify = config.credentialsToVerify();
+    public AthenzAuthorizationFilter(AthenzAuthorizationFilterConfig config,
+                                     RequestResourceMapper resourceMapper,
+                                     Zpe zpe,
+                                     Metric metric) {
+        this.roleTokenHeaderName = config.roleTokenHeaderName();
+        List<EnabledCredentials.Enum> enabledCredentials = config.enabledCredentials();
+        this.enabledCredentials = enabledCredentials.isEmpty()
+                ? EnumSet.allOf(EnabledCredentials.Enum.class)
+                : EnumSet.copyOf(enabledCredentials);
         this.requestResourceMapper = resourceMapper;
         this.zpe = zpe;
+        this.metric = metric;
     }
 
     @Override
-    protected Optional<ErrorResponse> filter(DiscFilterRequest request) {
-        Optional<ResourceNameAndAction> resourceMapping =
-                requestResourceMapper.getResourceNameAndAction(request.getMethod(), request.getRequestURI(), request.getQueryString());
-        if (!resourceMapping.isPresent()) {
-            return Optional.empty();
-        }
-        Optional<X509Certificate> roleCertificate = getRoleCertificate(request);
-        Optional<ZToken> roleToken = getRoleToken(request, headerName);
-        switch (credentialsToVerify) {
-            case CERTIFICATE_ONLY: {
-                if (!roleCertificate.isPresent()) {
-                    return Optional.of(new ErrorResponse(Response.Status.UNAUTHORIZED, "Missing client certificate"));
-                }
-                return checkAccessAllowed(roleCertificate.get(), resourceMapping.get(), request);
-            }
-            case TOKEN_ONLY: {
-                if (!roleToken.isPresent()) {
-                    return Optional.of(new ErrorResponse(Response.Status.UNAUTHORIZED,
-                                                         String.format("Role token header '%s' is missing or does not have a value.", headerName)));
-                }
-                return checkAccessAllowed(roleToken.get(), resourceMapping.get(), request);
+    public Optional<ErrorResponse> filter(DiscFilterRequest request) {
+        try {
+            Optional<ResourceNameAndAction> resourceMapping =
+                    requestResourceMapper.getResourceNameAndAction(request.getMethod(), request.getRequestURI(), request.getQueryString());
+            log.log(LogLevel.DEBUG, () -> String.format("Resource mapping for '%s': %s", request, resourceMapping));
+            if (resourceMapping.isEmpty()) {
+                incrementAcceptedMetrics(request, false);
+                return Optional.empty();
             }
-            case ANY: {
-                if (!roleCertificate.isPresent() && !roleToken.isPresent()) {
-                    return Optional.of(new ErrorResponse(Response.Status.UNAUTHORIZED, "Both role token and role certificate is missing"));
-                }
-                if (roleCertificate.isPresent()) {
-                    return checkAccessAllowed(roleCertificate.get(), resourceMapping.get(), request);
-                } else {
-                    return checkAccessAllowed(roleToken.get(), resourceMapping.get(), request);
-                }
-            }
-            default: {
-                throw new IllegalStateException("Unexpected mode: " + credentialsToVerify);
+            Result result = checkAccessAllowed(request, resourceMapping.get());
+            AuthorizationResult.Type resultType = result.zpeResult.type();
+            setAttribute(request, RESULT_ATTRIBUTE, resultType.name());
+            if (resultType == AuthorizationResult.Type.ALLOW) {
+                populateRequestWithResult(request, result);
+                incrementAcceptedMetrics(request, true);
+                return Optional.empty();
             }
+            log.log(LogLevel.DEBUG, () -> String.format("Forbidden (403) for '%s': %s", request, resultType.name()));
+            incrementRejectedMetrics(request, FORBIDDEN, resultType.name());
+            return Optional.of(new ErrorResponse(FORBIDDEN, "Access forbidden: " + resultType.getDescription()));
+        } catch (IllegalArgumentException e) {
+            log.log(LogLevel.DEBUG, () -> String.format("Unauthorized (401) for '%s': %s", request, e.getMessage()));
+            incrementRejectedMetrics(request, UNAUTHORIZED, "Unauthorized");
+            return Optional.of(new ErrorResponse(UNAUTHORIZED, e.getMessage()));
         }
     }
 
-    private static Optional<X509Certificate> getRoleCertificate(DiscFilterRequest request) {
-        return Optional.of(request.getClientCertificateChain())
-                .filter(chain -> !chain.isEmpty())
-                .map(chain -> chain.get(0))
-                .filter(AthenzX509CertificateUtils::isAthenzRoleCertificate);
+    private Result checkAccessAllowed(DiscFilterRequest request, ResourceNameAndAction resourceAndAction) {
+        // Note: the ordering of the if-constructs determines the precedence of the credential types
+        if (enabledCredentials.contains(ACCESS_TOKEN)
+                && isAccessTokenPresent(request)
+                && isClientCertificatePresent(request)) {
+            return checkAccessWithAccessToken(request, resourceAndAction);
+        } else if (enabledCredentials.contains(ROLE_CERTIFICATE)
+                && isClientCertificatePresent(request)) {
+            return checkAccessWithRoleCertificate(request, resourceAndAction);
+        } else if (enabledCredentials.contains(ROLE_TOKEN)
+                && isRoleTokenPresent(request)) {
+            return checkAccessWithRoleToken(request, resourceAndAction);
+        } else {
+            throw new IllegalArgumentException(
+                    "Not authorized - request did not contain any of the allowed credentials: " + toPrettyString(enabledCredentials));
+        }
     }
 
-    private static Optional<ZToken> getRoleToken(DiscFilterRequest request, String headerName) {
-        return Optional.ofNullable(request.getHeader(headerName))
-                .filter(token -> !token.isEmpty())
-                .map(ZToken::new);
+    private Result checkAccessWithAccessToken(DiscFilterRequest request, ResourceNameAndAction resourceAndAction) {
+        AthenzAccessToken accessToken = getAccessToken(request);
+        X509Certificate identityCertificate = getClientCertificate(request);
+        var zpeResult = zpe.checkAccessAllowed(
+                accessToken, identityCertificate, resourceAndAction.resourceName(), resourceAndAction.action());
+        return new Result(ACCESS_TOKEN, AthenzIdentities.from(identityCertificate), zpeResult);
     }
 
-    private Optional<ErrorResponse> checkAccessAllowed(X509Certificate certificate,
-                                                       ResourceNameAndAction resourceNameAndAction,
-                                                       DiscFilterRequest request) {
-        return checkAccessAllowed(
-                certificate, resourceNameAndAction, request, zpe::checkAccessAllowed, AthenzAuthorizationFilter::createPrincipal);
+    private Result checkAccessWithRoleCertificate(DiscFilterRequest request, ResourceNameAndAction resourceAndAction) {
+        X509Certificate roleCertificate = getClientCertificate(request);
+        var zpeResult = zpe.checkAccessAllowed(roleCertificate, resourceAndAction.resourceName(), resourceAndAction.action());
+        AthenzIdentity identity = AthenzX509CertificateUtils.getIdentityFromRoleCertificate(roleCertificate);
+        return new Result(ROLE_CERTIFICATE, identity, zpeResult);
     }
 
-    private Optional<ErrorResponse> checkAccessAllowed(ZToken roleToken,
-                                                       ResourceNameAndAction resourceNameAndAction,
-                                                       DiscFilterRequest request) {
-        return checkAccessAllowed(
-                roleToken, resourceNameAndAction, request, zpe::checkAccessAllowed, AthenzAuthorizationFilter::createPrincipal);
+    private Result checkAccessWithRoleToken(DiscFilterRequest request, ResourceNameAndAction resourceAndAction) {
+        ZToken roleToken = getRoleToken(request);
+        var zpeResult = zpe.checkAccessAllowed(roleToken, resourceAndAction.resourceName(), resourceAndAction.action());
+        return new Result(ROLE_TOKEN, roleToken.getIdentity(), zpeResult);
     }
 
-    private static <C> Optional<ErrorResponse> checkAccessAllowed(C credentials,
-                                                                  ResourceNameAndAction resAndAction,
-                                                                  DiscFilterRequest request,
-                                                                  ZpeCheck<C> accessCheck,
-                                                                  Function<C, AthenzPrincipal> principalFactory) {
-        AuthorizationResult authorizationResult = accessCheck.checkAccess(credentials, resAndAction.resourceName(), resAndAction.action());
-        if (authorizationResult == AuthorizationResult.ALLOW) {
-            request.setUserPrincipal(principalFactory.apply(credentials));
-            return Optional.empty();
+    private static boolean isAccessTokenPresent(DiscFilterRequest request) {
+        return request.getHeader(AthenzAccessToken.HTTP_HEADER_NAME) != null;
+    }
+
+    private static boolean isClientCertificatePresent(DiscFilterRequest request) {
+        return !request.getClientCertificateChain().isEmpty();
+    }
+
+    private boolean isRoleTokenPresent(DiscFilterRequest request) {
+        return request.getHeader(roleTokenHeaderName) != null;
+    }
+
+    private static AthenzAccessToken getAccessToken(DiscFilterRequest request) {
+        return new AthenzAccessToken(request.getHeader(AthenzAccessToken.HTTP_HEADER_NAME));
+    }
+
+    private static X509Certificate getClientCertificate(DiscFilterRequest request) {
+        return request.getClientCertificateChain().get(0);
+    }
+
+    private ZToken getRoleToken(DiscFilterRequest request) {
+        return new ZToken(request.getHeader(roleTokenHeaderName));
+    }
+
+    private static String toPrettyString(EnumSet<EnabledCredentials.Enum> enabledCredentialSet) {
+        return enabledCredentialSet.stream()
+                .map(AthenzAuthorizationFilter::toPrettyString)
+                .collect(Collectors.joining(", ", "[", "]"));
+    }
+
+    private static String toPrettyString(EnabledCredentials.Enum enabledCredential) {
+        switch (enabledCredential) {
+            case ACCESS_TOKEN:
+                return "Athenz access token with X.509 identity certificate";
+            case ROLE_TOKEN:
+                return "Athenz role token (ZToken)";
+            case ROLE_CERTIFICATE:
+                return "Athenz X.509 role certificate";
+            default:
+                throw new IllegalArgumentException("Unknown credential type: " + enabledCredential);
         }
-        return Optional.of(new ErrorResponse(Response.Status.FORBIDDEN, "Access forbidden: " + authorizationResult.getDescription()));
     }
 
-    private static AthenzPrincipal createPrincipal(X509Certificate certificate) {
-        AthenzIdentity identity = AthenzX509CertificateUtils.getIdentityFromRoleCertificate(certificate);
-        AthenzRole role = AthenzX509CertificateUtils.getRolesFromRoleCertificate(certificate);
-        return new AthenzPrincipal(identity, singletonList(role));
+    private static void populateRequestWithResult(DiscFilterRequest request, Result result) {
+        request.setUserPrincipal(
+                new AthenzPrincipal(result.identity, result.zpeResult.matchedRole().map(List::of).orElse(List.of())));
+        result.zpeResult.matchedRole().ifPresent(role -> {
+            request.setUserRoles(new String[]{role.roleName()});
+            setAttribute(request, MATCHED_ROLE_ATTRIBUTE, role.roleName());
+        });
+        setAttribute(request, IDENTITY_NAME_ATTRIBUTE, result.identity.getFullName());
+        setAttribute(request, MATCHED_CREDENTIAL_TYPE_ATTRIBUTE, result.credentialType.name());
+    }
+
+    private static void setAttribute(DiscFilterRequest request, String name, String value) {
+        log.log(LogLevel.DEBUG, () -> String.format("Setting attribute on '%s': '%s' = '%s'", request, name, value));
+        request.setAttribute(name, value);
     }
 
-    private static AthenzPrincipal createPrincipal(ZToken roleToken) {
-        return new AthenzPrincipal(roleToken.getIdentity(), roleToken.getRoles());
+    private void incrementAcceptedMetrics(DiscFilterRequest request, boolean authzRequired) {
+        String hostHeader = request.getHeader("Host");
+        Metric.Context context = metric.createContext(Map.of(
+                "endpoint", hostHeader != null ? hostHeader : "",
+                "authz-required", Boolean.toString(authzRequired)));
+        metric.add(ACCEPTED_METRIC_NAME, 1L, context);
     }
 
-    @FunctionalInterface private interface ZpeCheck<C> {
-        AuthorizationResult checkAccess(C credentials, AthenzResourceName resourceName, String action);
+    private void incrementRejectedMetrics(DiscFilterRequest request, int statusCode, String zpeCode) {
+        String hostHeader = request.getHeader("Host");
+        Metric.Context context = metric.createContext(Map.of(
+                "endpoint", hostHeader != null ? hostHeader : "",
+                "status-code", Integer.toString(statusCode),
+                "zpe-status", zpeCode));
+        metric.add(REJECTED_METRIC_NAME, 1L, context);
     }
 
+    private static class Result {
+        final EnabledCredentials.Enum credentialType;
+        final AthenzIdentity identity;
+        final AuthorizationResult zpeResult;
+
+        Result(EnabledCredentials.Enum credentialType, AthenzIdentity identity, AuthorizationResult zpeResult) {
+            this.credentialType = credentialType;
+            this.identity = identity;
+            this.zpeResult = zpeResult;
+        }
+    }
 }
diff --git a/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/athenz/RequestResourceMapper.java b/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/athenz/RequestResourceMapper.java
index 77709975cba..0bf000efc00 100644
--- a/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/athenz/RequestResourceMapper.java
+++ b/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/athenz/RequestResourceMapper.java
@@ -33,5 +33,13 @@ public interface RequestResourceMapper {
         public String action() {
             return action;
         }
+
+        @Override
+        public String toString() {
+            return "ResourceNameAndAction{" +
+                    "resourceName=" + resourceName +
+                    ", action='" + action + '\'' +
+                    '}';
+        }
     }
 }
diff --git a/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/base/JsonSecurityRequestFilterBase.java b/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/base/JsonSecurityRequestFilterBase.java
index ec8a93019b0..33d344c4f2a 100644
--- a/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/base/JsonSecurityRequestFilterBase.java
+++ b/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/base/JsonSecurityRequestFilterBase.java
@@ -4,39 +4,45 @@ package com.yahoo.jdisc.http.filter.security.base;
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.yahoo.component.AbstractComponent;
 import com.yahoo.jdisc.Response;
 import com.yahoo.jdisc.handler.FastContentWriter;
 import com.yahoo.jdisc.handler.ResponseDispatch;
 import com.yahoo.jdisc.handler.ResponseHandler;
 import com.yahoo.jdisc.http.filter.DiscFilterRequest;
 import com.yahoo.jdisc.http.filter.SecurityRequestFilter;
+import com.yahoo.log.LogLevel;
 
 import java.io.UncheckedIOException;
 import java.util.Optional;
+import java.util.logging.Logger;
 
 /**
  * A base class for {@link SecurityRequestFilter} implementations that renders an error response as JSON.
  *
  * @author bjorncs
  */
-public abstract class JsonSecurityRequestFilterBase implements SecurityRequestFilter {
+public abstract class JsonSecurityRequestFilterBase extends AbstractComponent implements SecurityRequestFilter {
 
+    private static final Logger log = Logger.getLogger(JsonSecurityRequestFilterBase.class.getName());
     private static final ObjectMapper mapper = new ObjectMapper();
 
     @Override
     public final void filter(DiscFilterRequest request, ResponseHandler handler) {
         filter(request)
-                .ifPresent(errorResponse -> writeResponse(errorResponse, handler));
+                .ifPresent(errorResponse -> writeResponse(request, errorResponse, handler));
     }
 
     protected abstract Optional<ErrorResponse> filter(DiscFilterRequest request);
 
-    private void writeResponse(ErrorResponse error, ResponseHandler responseHandler) {
+    private void writeResponse(DiscFilterRequest request, ErrorResponse error, ResponseHandler responseHandler) {
         ObjectNode errorMessage = mapper.createObjectNode();
         errorMessage.put("code", error.errorCode);
         errorMessage.put("message", error.message);
         error.response.headers().put("Content-Type", "application/json"); // Note: Overwrites header if already exists
         error.response.headers().put("Cache-Control", "must-revalidate,no-cache,no-store");
+        log.log(LogLevel.DEBUG, () -> String.format("Error response for '%s': statusCode=%d, errorCode=%d, message='%s'",
+                                                    request, error.response.getStatus(), error.errorCode, error.message));
         try (FastContentWriter writer = ResponseDispatch.newInstance(error.response).connectFastWriter(responseHandler)) {
             writer.write(mapper.writerWithDefaultPrettyPrinter().writeValueAsString(errorMessage));
         } catch (JsonProcessingException e) {
diff --git a/jdisc-security-filters/src/main/resources/configdefinitions/athenz-authorization-filter.def b/jdisc-security-filters/src/main/resources/configdefinitions/athenz-authorization-filter.def
index c60b7a125f8..ab8c4a204df 100644
--- a/jdisc-security-filters/src/main/resources/configdefinitions/athenz-authorization-filter.def
+++ b/jdisc-security-filters/src/main/resources/configdefinitions/athenz-authorization-filter.def
@@ -2,7 +2,7 @@
 namespace=jdisc.http.filter.security.athenz
 
 # Which credentials to verify. Note: ANY will prioritize token over certificate if both are present.
-credentialsToVerify enum { CERTIFICATE_ONLY, TOKEN_ONLY, ANY } default=ANY
+enabledCredentials[] enum { ROLE_CERTIFICATE, ROLE_TOKEN, ACCESS_TOKEN }
 
-# Name of header which includes role token. Must be set if 'credentialsTypeRequired' is set to TOKEN_ONLY or ANY.
-roleTokenHeaderName string default=""
+# Name of role token http header
+roleTokenHeaderName string default="Athenz-Role-Token"
diff --git a/jdisc-security-filters/src/test/java/com/yahoo/jdisc/http/filter/security/athenz/AthenzAuthorizationFilterTest.java b/jdisc-security-filters/src/test/java/com/yahoo/jdisc/http/filter/security/athenz/AthenzAuthorizationFilterTest.java
index b81b26d458b..768c1068dfc 100644
--- a/jdisc-security-filters/src/test/java/com/yahoo/jdisc/http/filter/security/athenz/AthenzAuthorizationFilterTest.java
+++ b/jdisc-security-filters/src/test/java/com/yahoo/jdisc/http/filter/security/athenz/AthenzAuthorizationFilterTest.java
@@ -1,25 +1,54 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.jdisc.http.filter.security.athenz;
 
-import com.yahoo.container.jdisc.RequestHandlerTestDriver;
+import com.auth0.jwt.JWT;
+import com.auth0.jwt.algorithms.Algorithm;
+import com.yahoo.container.jdisc.RequestHandlerTestDriver.MockResponseHandler;
+import com.yahoo.docproc.jdisc.metric.NullMetric;
+import com.yahoo.jdisc.Metric;
 import com.yahoo.jdisc.Response;
 import com.yahoo.jdisc.http.filter.DiscFilterRequest;
+import com.yahoo.jdisc.http.filter.security.athenz.AthenzAuthorizationFilterConfig.EnabledCredentials;
+import com.yahoo.security.KeyAlgorithm;
+import com.yahoo.security.KeyUtils;
+import com.yahoo.security.SubjectAlternativeName;
+import com.yahoo.security.X509CertificateBuilder;
+import com.yahoo.vespa.athenz.api.AthenzAccessToken;
+import com.yahoo.vespa.athenz.api.AthenzIdentity;
 import com.yahoo.vespa.athenz.api.AthenzResourceName;
+import com.yahoo.vespa.athenz.api.AthenzRole;
 import com.yahoo.vespa.athenz.api.ZToken;
+import com.yahoo.vespa.athenz.utils.AthenzIdentities;
 import com.yahoo.vespa.athenz.zpe.AuthorizationResult;
 import com.yahoo.vespa.athenz.zpe.Zpe;
 import org.junit.Test;
 import org.mockito.Mockito;
 
+import javax.security.auth.x500.X500Principal;
+import java.math.BigInteger;
+import java.security.KeyPair;
 import java.security.cert.X509Certificate;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
 
-import static com.yahoo.jdisc.http.filter.security.athenz.AthenzAuthorizationFilterConfig.CredentialsToVerify.Enum.ANY;
-import static java.util.Collections.emptyList;
+import static com.yahoo.jdisc.http.filter.security.athenz.AthenzAuthorizationFilter.MATCHED_CREDENTIAL_TYPE_ATTRIBUTE;
+import static com.yahoo.jdisc.http.filter.security.athenz.AthenzAuthorizationFilter.MATCHED_ROLE_ATTRIBUTE;
+import static com.yahoo.jdisc.http.filter.security.athenz.AthenzAuthorizationFilter.RESULT_ATTRIBUTE;
+import static com.yahoo.security.SignatureAlgorithm.SHA256_WITH_ECDSA;
+import static com.yahoo.security.SubjectAlternativeName.Type.RFC822_NAME;
+import static com.yahoo.vespa.athenz.zpe.AuthorizationResult.Type;
 import static org.hamcrest.CoreMatchers.containsString;
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.CoreMatchers.hasItem;
+import static org.hamcrest.CoreMatchers.notNullValue;
+import static org.hamcrest.CoreMatchers.nullValue;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertThat;
+import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
 /**
@@ -28,77 +57,243 @@ import static org.mockito.Mockito.when;
 public class AthenzAuthorizationFilterTest {
 
     private static final AthenzResourceName RESOURCE_NAME = new AthenzResourceName("domain", "my-resource-name");
+    private static final ZToken ROLE_TOKEN = new ZToken("v=Z1;d=domain;r=my-role;p=my-domain.my-service");
+    private static final AthenzAccessToken ACCESS_TOKEN = new AthenzAccessToken(JWT.create().sign(Algorithm.none()));
+    private static final AthenzIdentity IDENTITY = AthenzIdentities.from("user.john");
+    private static final AthenzRole ROLE = new AthenzRole("my.domain", "my-role");
+    private static final X509Certificate IDENTITY_CERTIFICATE = createDummyIdentityCertificate(IDENTITY);
+    private static final X509Certificate ROLE_CERTIFICATE = createDummyRoleCertificate(ROLE, IDENTITY);
     private static final String ACTION = "update";
     private static final String HEADER_NAME = "Athenz-Role-Token";
-    private static final AthenzAuthorizationFilterConfig CONFIG = createConfig();
+    private static final String ACCEPTED_METRIC_NAME = "jdisc.http.filter.athenz.accepted_requests";
+    private static final String REJECTED_METRIC_NAME = "jdisc.http.filter.athenz.rejected_requests";
 
-    private static AthenzAuthorizationFilterConfig createConfig() {
-        return new AthenzAuthorizationFilterConfig(
-                new AthenzAuthorizationFilterConfig.Builder()
-                        .roleTokenHeaderName(HEADER_NAME)
-                        .credentialsToVerify(ANY));
+    @Test
+    public void accepts_request_with_access_token() {
+        AthenzAuthorizationFilter filter = createFilter(new AllowingZpe(), List.of());
+
+        MockResponseHandler responseHandler = new MockResponseHandler();
+        DiscFilterRequest request = createRequest(null, ACCESS_TOKEN, IDENTITY_CERTIFICATE);
+        filter.filter(request, responseHandler);
+
+        assertAuthorizationResult(request, Type.ALLOW);
+        assertRequestNotFiltered(responseHandler);
+        assertMatchedCredentialType(request, EnabledCredentials.ACCESS_TOKEN);
+        assertMatchedRole(request, ROLE);
     }
 
     @Test
-    public void accepts_valid_requests() {
-        AthenzAuthorizationFilter filter =
-                new AthenzAuthorizationFilter(
-                        CONFIG, new StaticRequestResourceMapper(RESOURCE_NAME, ACTION), new AllowingZpe());
+    public void accepts_request_with_role_certificate() {
+        AthenzAuthorizationFilter filter = createFilter(new AllowingZpe(), List.of());
+
+        MockResponseHandler responseHandler = new MockResponseHandler();
+        DiscFilterRequest request = createRequest(null, null, ROLE_CERTIFICATE);
+        filter.filter(request, responseHandler);
+
+        assertAuthorizationResult(request, Type.ALLOW);
+        assertRequestNotFiltered(responseHandler);
+        assertMatchedCredentialType(request, EnabledCredentials.ROLE_CERTIFICATE);
+        assertMatchedRole(request, ROLE);
+    }
+
+    @Test
+    public void accepts_request_with_role_token() {
+        AthenzAuthorizationFilter filter = createFilter(new AllowingZpe(), List.of());
 
-        RequestHandlerTestDriver.MockResponseHandler responseHandler = new RequestHandlerTestDriver.MockResponseHandler();
-        filter.filter(createRequest(), responseHandler);
+        MockResponseHandler responseHandler = new MockResponseHandler();
+        DiscFilterRequest request = createRequest(ROLE_TOKEN, null, null);
+        filter.filter(request, responseHandler);
 
-        assertNull(responseHandler.getResponse());
+        assertAuthorizationResult(request, Type.ALLOW);
+        assertRequestNotFiltered(responseHandler);
+        assertMatchedCredentialType(request, EnabledCredentials.ROLE_TOKEN);
+        assertMatchedRole(request, ROLE);
     }
 
     @Test
-    public void returns_error_on_forbidden_requests() {
+    public void returns_unauthorized_for_request_with_disabled_credential_type() {
         AthenzAuthorizationFilter filter =
-                new AthenzAuthorizationFilter(
-                        CONFIG, new StaticRequestResourceMapper(RESOURCE_NAME, ACTION), new DenyingZpe());
+                createFilter(new AllowingZpe(), List.of(EnabledCredentials.ROLE_CERTIFICATE, EnabledCredentials.ACCESS_TOKEN));
 
-        RequestHandlerTestDriver.MockResponseHandler responseHandler = new RequestHandlerTestDriver.MockResponseHandler();
-        filter.filter(createRequest(), responseHandler);
+        MockResponseHandler responseHandler = new MockResponseHandler();
+        DiscFilterRequest request = createRequest(ROLE_TOKEN, null, null);
+        filter.filter(request, responseHandler);
 
-        Response response = responseHandler.getResponse();
-        assertNotNull(response);
-        assertEquals(403, response.getStatus());
-        String content = responseHandler.readAll();
-        assertThat(content, containsString(AuthorizationResult.DENY.getDescription()));
+        assertStatusCode(responseHandler, 401);
+        assertErrorMessage(responseHandler, "Not authorized - request did not contain any of the allowed credentials: " +
+                "[Athenz X.509 role certificate, Athenz access token with X.509 identity certificate]");
+    }
+
+    @Test
+    public void returns_forbidden_for_credentials_rejected_by_zpe() {
+        AthenzAuthorizationFilter filter = createFilter(new DenyingZpe(), List.of());
+
+        MockResponseHandler responseHandler = new MockResponseHandler();
+        DiscFilterRequest request = createRequest(ROLE_TOKEN, null, null);
+        filter.filter(request, responseHandler);
+
+        assertStatusCode(responseHandler, 403);
+        assertAuthorizationResult(request, Type.DENY);
     }
 
-    private static DiscFilterRequest createRequest() {
+    @Test
+    public void reports_metrics_for_rejected_requests() {
+        MetricMock metric = new MetricMock();
+        AthenzAuthorizationFilter filter = createFilter(new DenyingZpe(), List.of(), metric);
+        MockResponseHandler responseHandler = new MockResponseHandler();
+        DiscFilterRequest request = createRequest(null, ACCESS_TOKEN, IDENTITY_CERTIFICATE);
+        filter.filter(request, responseHandler);
+
+        assertMetrics(metric, REJECTED_METRIC_NAME, Map.of("zpe-status", "DENY", "status-code", "403"));
+    }
+
+    @Test
+    public void reports_metrics_for_accepted_requests() {
+        MetricMock metric = new MetricMock();
+        AthenzAuthorizationFilter filter = createFilter(new AllowingZpe(), List.of(EnabledCredentials.ACCESS_TOKEN), metric);
+        MockResponseHandler responseHandler = new MockResponseHandler();
+        DiscFilterRequest request = createRequest(null, ACCESS_TOKEN, IDENTITY_CERTIFICATE);
+        filter.filter(request, responseHandler);
+
+        assertMetrics(metric, ACCEPTED_METRIC_NAME, Map.of("authz-required", "true"));
+    }
+
+    private void assertMetrics(MetricMock metric, String metricName, Map<String, String> dimensions) {
+        assertThat(metric.addInvocations.keySet(), hasItem(metricName));
+        SimpleMetricContext metricContext = metric.addInvocations.get(metricName);
+        assertNotNull("Metric not found " + metricName, metricName);
+        for (Map.Entry<String, String> entry : dimensions.entrySet()) {
+            String dimensionName = entry.getKey();
+            String expected = entry.getValue();
+            assertThat(metricContext.dimensions.keySet(), hasItem(dimensionName));
+            assertEquals(expected, metricContext.dimensions.get(dimensionName));
+        }
+    }
+
+    private static X509Certificate createDummyIdentityCertificate(AthenzIdentity identity) {
+        KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.EC, 256);
+        X500Principal x500Name = new X500Principal("CN="+ identity.getFullName());
+        Instant now = Instant.now();
+        return X509CertificateBuilder
+                .fromKeypair(keyPair, x500Name, now, now.plus(Duration.ofDays(30)), SHA256_WITH_ECDSA, BigInteger.ONE)
+                .build();
+    }
+
+    private static X509Certificate createDummyRoleCertificate(AthenzRole role, AthenzIdentity identity) {
+        KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.EC, 256);
+        X500Principal x500Name = new X500Principal("CN="+ role.domain().getName() + ":role." + role.roleName());
+        Instant now = Instant.now();
+        return X509CertificateBuilder
+                .fromKeypair(keyPair, x500Name, now, now.plus(Duration.ofDays(30)), SHA256_WITH_ECDSA, BigInteger.ONE)
+                .addSubjectAlternativeName(new SubjectAlternativeName(RFC822_NAME, identity.getFullName() + "@my.domain.my-identity-provider"))
+                .build();
+    }
+
+    private static DiscFilterRequest createRequest(ZToken roleToken, AthenzAccessToken accessToken, X509Certificate clientCert) {
         DiscFilterRequest request = Mockito.mock(DiscFilterRequest.class);
-        when(request.getHeader(HEADER_NAME)).thenReturn("v=Z1;d=domain;r=my-role;p=my-domain.my-service");
+        when(request.getHeader(HEADER_NAME)).thenReturn(roleToken != null ? roleToken.getRawToken() : null);
+        when(request.getHeader(AthenzAccessToken.HTTP_HEADER_NAME)).thenReturn(accessToken != null ? "Bearer " + accessToken.value() : null);
         when(request.getMethod()).thenReturn("GET");
         when(request.getRequestURI()).thenReturn("/my/path");
         when(request.getQueryString()).thenReturn(null);
-        when(request.getClientCertificateChain()).thenReturn(emptyList());
+        when(request.getClientCertificateChain()).thenReturn(clientCert != null ? List.of(clientCert) : List.of());
         return request;
     }
 
-    static class AllowingZpe implements Zpe {
+    private static AthenzAuthorizationFilter createFilter(Zpe zpe, List<EnabledCredentials.Enum> enabledCredentials) {
+        return createFilter(zpe, enabledCredentials, new NullMetric());
+    }
+
+    private static AthenzAuthorizationFilter createFilter(Zpe zpe, List<EnabledCredentials.Enum> enabledCredentials, Metric metric) {
+        return new AthenzAuthorizationFilter(
+                new AthenzAuthorizationFilterConfig(
+                        new AthenzAuthorizationFilterConfig.Builder()
+                                .roleTokenHeaderName(HEADER_NAME)
+                                .enabledCredentials(enabledCredentials)),
+                new StaticRequestResourceMapper(RESOURCE_NAME, ACTION),
+                zpe,
+                metric);
+    }
+
+    private static void assertAuthorizationResult(DiscFilterRequest request, Type expectedResult) {
+        verify(request).setAttribute(RESULT_ATTRIBUTE, expectedResult.name());
+    }
+
+    private static void assertStatusCode(MockResponseHandler responseHandler, int statusCode) {
+        Response response = responseHandler.getResponse();
+        assertThat(response, notNullValue());
+        assertThat(response.getStatus(), equalTo(statusCode));
+    }
+
+    private static void assertMatchedCredentialType(DiscFilterRequest request, EnabledCredentials.Enum expectedType) {
+        verify(request).setAttribute(MATCHED_CREDENTIAL_TYPE_ATTRIBUTE, expectedType.name());
+    }
+
+    private static void assertRequestNotFiltered(MockResponseHandler responseHandler) {
+        assertThat(responseHandler.getResponse(), nullValue());
+    }
+
+    private static void assertMatchedRole(DiscFilterRequest request, AthenzRole role) {
+        verify(request).setAttribute(MATCHED_ROLE_ATTRIBUTE, role.roleName());
+    }
+
+    private static void assertErrorMessage(MockResponseHandler responseHandler, String errorMessage) {
+        Response response = responseHandler.getResponse();
+        assertThat(response, notNullValue());
+        String content = responseHandler.readAll();
+        assertThat(content, containsString(errorMessage));
+    }
+
+    private static class AllowingZpe implements Zpe {
+
         @Override
         public AuthorizationResult checkAccessAllowed(ZToken roleToken, AthenzResourceName resourceName, String action) {
-            return AuthorizationResult.ALLOW;
+            return new AuthorizationResult(Type.ALLOW, ROLE);
         }
 
         @Override
         public AuthorizationResult checkAccessAllowed(X509Certificate roleCertificate, AthenzResourceName resourceName, String action) {
-            return AuthorizationResult.ALLOW;
+            return new AuthorizationResult(Type.ALLOW, ROLE);
+        }
+
+        @Override
+        public AuthorizationResult checkAccessAllowed(AthenzAccessToken accessToken, X509Certificate identityCertificate, AthenzResourceName resourceName, String action) {
+            return new AuthorizationResult(Type.ALLOW, ROLE);
         }
     }
 
-    static class DenyingZpe implements Zpe {
+    private static class DenyingZpe implements Zpe {
         @Override
         public AuthorizationResult checkAccessAllowed(ZToken roleToken, AthenzResourceName resourceName, String action) {
-            return AuthorizationResult.DENY;
+            return new AuthorizationResult(Type.DENY);
         }
 
         @Override
         public AuthorizationResult checkAccessAllowed(X509Certificate roleCertificate, AthenzResourceName resourceName, String action) {
-            return AuthorizationResult.DENY;
+            return new AuthorizationResult(Type.DENY);
+        }
+
+        @Override
+        public AuthorizationResult checkAccessAllowed(AthenzAccessToken accessToken, X509Certificate identityCertificate, AthenzResourceName resourceName, String action) {
+            return new AuthorizationResult(Type.DENY);
+        }
+    }
+
+    private static class MetricMock implements Metric {
+        final ConcurrentHashMap<String, SimpleMetricContext> addInvocations = new ConcurrentHashMap<>();
+
+        @Override public void add(String key, Number val, Context ctx) {
+            addInvocations.put(key, (SimpleMetricContext)ctx);
         }
+        @Override public void set(String key, Number val, Context ctx) {}
+        @Override public Context createContext(Map<String, ?> properties) { return new SimpleMetricContext(properties); }
+    }
+
+    private static class SimpleMetricContext implements Metric.Context {
+        final Map<String, String> dimensions;
+
+        @SuppressWarnings("unchecked")
+        SimpleMetricContext(Map<String, ?> dimensions) { this.dimensions = (Map<String, String>)dimensions; }
     }
 
 }
diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/Metric.java b/jdisc_core/src/main/java/com/yahoo/jdisc/Metric.java
index d6206dcf966..0e9db8fb1ea 100644
--- a/jdisc_core/src/main/java/com/yahoo/jdisc/Metric.java
+++ b/jdisc_core/src/main/java/com/yahoo/jdisc/Metric.java
@@ -22,7 +22,7 @@ import java.util.Map;
 public interface Metric {
 
     /**
-     * <p>Set a metric value. This is typically used with histogram-type metrics.</p>
+     * Set a metric value. This is typically used with histogram-type metrics.
      *
      * @param key The name of the metric to modify.
      * @param val The value to assign to the named metric.
@@ -31,28 +31,28 @@ public interface Metric {
     void set(String key, Number val, Context ctx);
 
     /**
-     * <p>Add to a metric value. This is typically used with counter-type metrics.</p>
+     * Add to a metric value. This is typically used with counter-type metrics.
      *
-     * @param key The name of the metric to modify.
-     * @param val The value to add to the named metric.
-     * @param ctx The context to further describe this entry.
+     * @param key the name of the metric to modify
+     * @param val the value to add to the named metric
+     * @param ctx the context to further describe this entry
      */
     void add(String key, Number val, Context ctx);
 
     /**
-     * <p>Creates a {@link MetricConsumer}-specific {@link Context} object that encapsulates the given properties. The
+     * Creates a {@link MetricConsumer}-specific {@link Context} object that encapsulates the given properties. The
      * returned Context object should be passed along every future call to {@link #set(String, Number, Context)} and
-     * {@link #add(String, Number, Context)} where the properties match those given here.</p>
+     * {@link #add(String, Number, Context)} where the properties match those given here.
      *
-     * @param properties The properties to incorporate in the context.
-     * @return The created context.
+     * @param properties the properties to incorporate in the context
+     * @return the created context
      */
     Context createContext(Map<String, ?> properties);
 
     /**
-     * <p>Declares the interface for the arbitrary context object to pass to both the {@link
+     * Declares the interface for the arbitrary context object to pass to both the {@link
      * #set(String, Number, Context)} and {@link #add(String, Number, Context)} methods. This is intentionally empty so
-     * that implementations can vary.</p>
+     * that implementations can vary.
      */
     interface Context {
 
diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/Timer.java b/jdisc_core/src/main/java/com/yahoo/jdisc/Timer.java
index a111425fbc0..4db30636137 100644
--- a/jdisc_core/src/main/java/com/yahoo/jdisc/Timer.java
+++ b/jdisc_core/src/main/java/com/yahoo/jdisc/Timer.java
@@ -4,7 +4,10 @@ package com.yahoo.jdisc;
 import com.google.inject.ImplementedBy;
 import com.yahoo.jdisc.core.SystemTimer;
 
+import java.time.Clock;
 import java.time.Instant;
+import java.time.ZoneId;
+import java.time.ZoneOffset;
 
 /**
  * <p>This class provides access to the current time in milliseconds, as viewed by the {@link Container}. Inject an
@@ -29,11 +32,33 @@ public interface Timer {
      */
     long currentTimeMillis();
 
-    /**
-     * Convenience method for getting an java.util.Instance from currentTimeMillis().
-     */
+    /** Convenience method for getting an java.util.Instance from currentTimeMillis(). */
     default Instant currentTime() {
         return Instant.ofEpochMilli(currentTimeMillis());
     }
 
+    /** Return a UTC Clock backed by this timer. */
+    default Clock toUtcClock() {
+        return new ClockAdapter(this, ZoneOffset.UTC);
+    }
+
+    /** Create a Timer backed by the given Clock. */
+    static Timer fromClock(Clock clock) {
+        return clock::millis;
+    }
+
+    class ClockAdapter extends Clock {
+        private final Timer timer;
+        private final ZoneId zoneId;
+
+        private ClockAdapter(Timer timer, ZoneId zoneId) {
+            this.timer = timer;
+            this.zoneId = zoneId;
+        }
+
+        @Override public ZoneId getZone() { return zoneId; }
+        @Override public Clock withZone(ZoneId zone) { return new ClockAdapter(timer, zoneId); }
+        @Override public Instant instant() { return timer.currentTime(); }
+    }
+
 }
diff --git a/jdisc_http_service/abi-spec.json b/jdisc_http_service/abi-spec.json
index 1615cf7e686..c5a0a676a70 100644
--- a/jdisc_http_service/abi-spec.json
+++ b/jdisc_http_service/abi-spec.json
@@ -41,6 +41,10 @@
       "public com.yahoo.jdisc.http.ConnectorConfig$Builder ssl(com.yahoo.jdisc.http.ConnectorConfig$Ssl$Builder)",
       "public com.yahoo.jdisc.http.ConnectorConfig$Builder tlsClientAuthEnforcer(com.yahoo.jdisc.http.ConnectorConfig$TlsClientAuthEnforcer$Builder)",
       "public com.yahoo.jdisc.http.ConnectorConfig$Builder healthCheckProxy(com.yahoo.jdisc.http.ConnectorConfig$HealthCheckProxy$Builder)",
+      "public com.yahoo.jdisc.http.ConnectorConfig$Builder proxyProtocol(com.yahoo.jdisc.http.ConnectorConfig$ProxyProtocol$Builder)",
+      "public com.yahoo.jdisc.http.ConnectorConfig$Builder secureRedirect(com.yahoo.jdisc.http.ConnectorConfig$SecureRedirect$Builder)",
+      "public com.yahoo.jdisc.http.ConnectorConfig$Builder maxRequestsPerConnection(int)",
+      "public com.yahoo.jdisc.http.ConnectorConfig$Builder maxConnectionLife(double)",
       "public final boolean dispatchGetConfig(com.yahoo.config.ConfigInstance$Producer)",
       "public final java.lang.String getDefMd5()",
       "public final java.lang.String getDefName()",
@@ -51,7 +55,9 @@
       "public com.yahoo.jdisc.http.ConnectorConfig$Throttling$Builder throttling",
       "public com.yahoo.jdisc.http.ConnectorConfig$Ssl$Builder ssl",
       "public com.yahoo.jdisc.http.ConnectorConfig$TlsClientAuthEnforcer$Builder tlsClientAuthEnforcer",
-      "public com.yahoo.jdisc.http.ConnectorConfig$HealthCheckProxy$Builder healthCheckProxy"
+      "public com.yahoo.jdisc.http.ConnectorConfig$HealthCheckProxy$Builder healthCheckProxy",
+      "public com.yahoo.jdisc.http.ConnectorConfig$ProxyProtocol$Builder proxyProtocol",
+      "public com.yahoo.jdisc.http.ConnectorConfig$SecureRedirect$Builder secureRedirect"
     ]
   },
   "com.yahoo.jdisc.http.ConnectorConfig$HealthCheckProxy$Builder": {
@@ -100,6 +106,68 @@
     ],
     "fields": []
   },
+  "com.yahoo.jdisc.http.ConnectorConfig$ProxyProtocol$Builder": {
+    "superClass": "java.lang.Object",
+    "interfaces": [
+      "com.yahoo.config.ConfigBuilder"
+    ],
+    "attributes": [
+      "public"
+    ],
+    "methods": [
+      "public void <init>()",
+      "public void <init>(com.yahoo.jdisc.http.ConnectorConfig$ProxyProtocol)",
+      "public com.yahoo.jdisc.http.ConnectorConfig$ProxyProtocol$Builder enabled(boolean)",
+      "public com.yahoo.jdisc.http.ConnectorConfig$ProxyProtocol$Builder mixedMode(boolean)",
+      "public com.yahoo.jdisc.http.ConnectorConfig$ProxyProtocol build()"
+    ],
+    "fields": []
+  },
+  "com.yahoo.jdisc.http.ConnectorConfig$ProxyProtocol": {
+    "superClass": "com.yahoo.config.InnerNode",
+    "interfaces": [],
+    "attributes": [
+      "public",
+      "final"
+    ],
+    "methods": [
+      "public void <init>(com.yahoo.jdisc.http.ConnectorConfig$ProxyProtocol$Builder)",
+      "public boolean enabled()",
+      "public boolean mixedMode()"
+    ],
+    "fields": []
+  },
+  "com.yahoo.jdisc.http.ConnectorConfig$SecureRedirect$Builder": {
+    "superClass": "java.lang.Object",
+    "interfaces": [
+      "com.yahoo.config.ConfigBuilder"
+    ],
+    "attributes": [
+      "public"
+    ],
+    "methods": [
+      "public void <init>()",
+      "public void <init>(com.yahoo.jdisc.http.ConnectorConfig$SecureRedirect)",
+      "public com.yahoo.jdisc.http.ConnectorConfig$SecureRedirect$Builder enabled(boolean)",
+      "public com.yahoo.jdisc.http.ConnectorConfig$SecureRedirect$Builder port(int)",
+      "public com.yahoo.jdisc.http.ConnectorConfig$SecureRedirect build()"
+    ],
+    "fields": []
+  },
+  "com.yahoo.jdisc.http.ConnectorConfig$SecureRedirect": {
+    "superClass": "com.yahoo.config.InnerNode",
+    "interfaces": [],
+    "attributes": [
+      "public",
+      "final"
+    ],
+    "methods": [
+      "public void <init>(com.yahoo.jdisc.http.ConnectorConfig$SecureRedirect$Builder)",
+      "public boolean enabled()",
+      "public int port()"
+    ],
+    "fields": []
+  },
   "com.yahoo.jdisc.http.ConnectorConfig$Ssl$Builder": {
     "superClass": "java.lang.Object",
     "interfaces": [
@@ -119,9 +187,16 @@
       "public com.yahoo.jdisc.http.ConnectorConfig$Ssl$Builder caCertificateFile(java.lang.String)",
       "public com.yahoo.jdisc.http.ConnectorConfig$Ssl$Builder caCertificate(java.lang.String)",
       "public com.yahoo.jdisc.http.ConnectorConfig$Ssl$Builder clientAuth(com.yahoo.jdisc.http.ConnectorConfig$Ssl$ClientAuth$Enum)",
+      "public com.yahoo.jdisc.http.ConnectorConfig$Ssl$Builder enabledCipherSuites(java.lang.String)",
+      "public com.yahoo.jdisc.http.ConnectorConfig$Ssl$Builder enabledCipherSuites(java.util.Collection)",
+      "public com.yahoo.jdisc.http.ConnectorConfig$Ssl$Builder enabledProtocols(java.lang.String)",
+      "public com.yahoo.jdisc.http.ConnectorConfig$Ssl$Builder enabledProtocols(java.util.Collection)",
       "public com.yahoo.jdisc.http.ConnectorConfig$Ssl build()"
     ],
-    "fields": []
+    "fields": [
+      "public java.util.List enabledCipherSuites",
+      "public java.util.List enabledProtocols"
+    ]
   },
   "com.yahoo.jdisc.http.ConnectorConfig$Ssl$ClientAuth$Enum": {
     "superClass": "java.lang.Enum",
@@ -174,7 +249,11 @@
       "public java.lang.String certificate()",
       "public java.lang.String caCertificateFile()",
       "public java.lang.String caCertificate()",
-      "public com.yahoo.jdisc.http.ConnectorConfig$Ssl$ClientAuth$Enum clientAuth()"
+      "public com.yahoo.jdisc.http.ConnectorConfig$Ssl$ClientAuth$Enum clientAuth()",
+      "public java.util.List enabledCipherSuites()",
+      "public java.lang.String enabledCipherSuites(int)",
+      "public java.util.List enabledProtocols()",
+      "public java.lang.String enabledProtocols(int)"
     ],
     "fields": []
   },
@@ -278,7 +357,11 @@
       "public com.yahoo.jdisc.http.ConnectorConfig$Throttling throttling()",
       "public com.yahoo.jdisc.http.ConnectorConfig$Ssl ssl()",
       "public com.yahoo.jdisc.http.ConnectorConfig$TlsClientAuthEnforcer tlsClientAuthEnforcer()",
-      "public com.yahoo.jdisc.http.ConnectorConfig$HealthCheckProxy healthCheckProxy()"
+      "public com.yahoo.jdisc.http.ConnectorConfig$HealthCheckProxy healthCheckProxy()",
+      "public com.yahoo.jdisc.http.ConnectorConfig$ProxyProtocol proxyProtocol()",
+      "public com.yahoo.jdisc.http.ConnectorConfig$SecureRedirect secureRedirect()",
+      "public int maxRequestsPerConnection()",
+      "public double maxConnectionLife()"
     ],
     "fields": [
       "public static final java.lang.String CONFIG_DEF_MD5",
diff --git a/jdisc_http_service/pom.xml b/jdisc_http_service/pom.xml
index c5555d5b690..bd8c77bc9cc 100644
--- a/jdisc_http_service/pom.xml
+++ b/jdisc_http_service/pom.xml
@@ -86,6 +86,12 @@
       <scope>test</scope>
     </dependency>
     <dependency>
+      <groupId>org.eclipse.jetty</groupId>
+      <artifactId>jetty-client</artifactId>
+      <version>${jetty.version}</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
       <groupId>org.cthul</groupId>
       <artifactId>cthul-matchers</artifactId>
       <scope>test</scope>
diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/ConnectorFactory.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/ConnectorFactory.java
index 01d6fa02d6e..94c08212706 100644
--- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/ConnectorFactory.java
+++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/ConnectorFactory.java
@@ -5,12 +5,13 @@ import com.google.inject.Inject;
 import com.yahoo.jdisc.Metric;
 import com.yahoo.jdisc.http.ConnectorConfig;
 import com.yahoo.jdisc.http.ssl.SslContextFactoryProvider;
+import com.yahoo.security.tls.MixedMode;
 import com.yahoo.security.tls.TransportSecurityUtils;
-import org.eclipse.jetty.http.HttpVersion;
 import org.eclipse.jetty.server.ConnectionFactory;
+import org.eclipse.jetty.server.DetectorConnectionFactory;
 import org.eclipse.jetty.server.HttpConfiguration;
 import org.eclipse.jetty.server.HttpConnectionFactory;
-import org.eclipse.jetty.server.OptionalSslConnectionFactory;
+import org.eclipse.jetty.server.ProxyConnectionFactory;
 import org.eclipse.jetty.server.SecureRequestCustomizer;
 import org.eclipse.jetty.server.Server;
 import org.eclipse.jetty.server.ServerConnector;
@@ -32,17 +33,35 @@ public class ConnectorFactory {
     @Inject
     public ConnectorFactory(ConnectorConfig connectorConfig,
                             SslContextFactoryProvider sslContextFactoryProvider) {
+        runtimeConnectorConfigValidation(connectorConfig);
         this.connectorConfig = connectorConfig;
         this.sslContextFactoryProvider = sslContextFactoryProvider;
     }
 
+    // Perform extra connector config validation that can only be performed at runtime,
+    // e.g. due to TLS configuration through environment variables.
+    private static void runtimeConnectorConfigValidation(ConnectorConfig config) {
+        validateProxyProtocolConfiguration(config);
+    }
+
+    private static void validateProxyProtocolConfiguration(ConnectorConfig config) {
+        ConnectorConfig.ProxyProtocol proxyProtocolConfig = config.proxyProtocol();
+        if (proxyProtocolConfig.enabled()) {
+            boolean sslEnabled = config.ssl().enabled() || TransportSecurityUtils.isTransportSecurityEnabled();
+            boolean tlsMixedModeEnabled = TransportSecurityUtils.getInsecureMixedMode() != MixedMode.DISABLED;
+            if (!sslEnabled || tlsMixedModeEnabled) {
+                throw new IllegalArgumentException("Proxy protocol can only be enabled if connector is effectively HTTPS only");
+            }
+        }
+    }
+
     public ConnectorConfig getConnectorConfig() {
         return connectorConfig;
     }
 
-    public ServerConnector createConnector(final Metric metric, final Server server, final ServerSocketChannel ch) {
+    public ServerConnector createConnector(final Metric metric, final Server server) {
         ServerConnector connector = new JDiscServerConnector(
-                connectorConfig, metric, server, ch, createConnectionFactories().toArray(ConnectionFactory[]::new));
+                connectorConfig, metric, server, createConnectionFactories(metric).toArray(ConnectionFactory[]::new));
         connector.setPort(connectorConfig.listenPort());
         connector.setName(connectorConfig.name());
         connector.setAcceptQueueSize(connectorConfig.acceptQueueSize());
@@ -51,25 +70,38 @@ public class ConnectorFactory {
         return connector;
     }
 
-    private List<ConnectionFactory> createConnectionFactories() {
-        HttpConnectionFactory httpConnectionFactory = newHttpConnectionFactory();
-        if (connectorConfig.healthCheckProxy().enable()) {
-            return List.of(httpConnectionFactory);
+    private List<ConnectionFactory> createConnectionFactories(Metric metric) {
+        HttpConnectionFactory httpFactory = newHttpConnectionFactory();
+        if (connectorConfig.healthCheckProxy().enable() || connectorConfig.secureRedirect().enabled()) {
+            return List.of(httpFactory);
         } else if (connectorConfig.ssl().enabled()) {
-            return List.of(newSslConnectionFactory(), httpConnectionFactory);
+            return connectionFactoriesForHttps(metric, httpFactory);
         } else if (TransportSecurityUtils.isTransportSecurityEnabled()) {
-            SslConnectionFactory sslConnectionsFactory = newSslConnectionFactory();
             switch (TransportSecurityUtils.getInsecureMixedMode()) {
                 case TLS_CLIENT_MIXED_SERVER:
                 case PLAINTEXT_CLIENT_MIXED_SERVER:
-                    return List.of(newOptionalSslConnectionFactory(sslConnectionsFactory), sslConnectionsFactory, httpConnectionFactory);
+                    return List.of(new DetectorConnectionFactory(newSslConnectionFactory(metric, httpFactory)), httpFactory);
                 case DISABLED:
-                    return List.of(sslConnectionsFactory, httpConnectionFactory);
+                    return connectionFactoriesForHttps(metric, httpFactory);
                 default:
                     throw new IllegalStateException();
             }
         } else {
-            return List.of(httpConnectionFactory);
+            return List.of(httpFactory);
+        }
+    }
+
+    private List<ConnectionFactory> connectionFactoriesForHttps(Metric metric, HttpConnectionFactory httpFactory) {
+        ConnectorConfig.ProxyProtocol proxyProtocolConfig = connectorConfig.proxyProtocol();
+        SslConnectionFactory sslFactory = newSslConnectionFactory(metric, httpFactory);
+        if (proxyProtocolConfig.enabled()) {
+            if (proxyProtocolConfig.mixedMode()) {
+                return List.of(new DetectorConnectionFactory(sslFactory, new ProxyConnectionFactory(sslFactory.getProtocol())), sslFactory, httpFactory);
+            } else {
+                return List.of(new ProxyConnectionFactory(), sslFactory, httpFactory);
+            }
+        } else {
+            return List.of(sslFactory, httpFactory);
         }
     }
 
@@ -88,13 +120,11 @@ public class ConnectorFactory {
         return new HttpConnectionFactory(httpConfig);
     }
 
-    private SslConnectionFactory newSslConnectionFactory() {
-        SslContextFactory factory = sslContextFactoryProvider.getInstance(connectorConfig.name(), connectorConfig.listenPort());
-        return new SslConnectionFactory(factory, HttpVersion.HTTP_1_1.asString());
-    }
-
-    private OptionalSslConnectionFactory newOptionalSslConnectionFactory(SslConnectionFactory sslConnectionsFactory) {
-        return new OptionalSslConnectionFactory(sslConnectionsFactory, HttpVersion.HTTP_1_1.asString());
+    private SslConnectionFactory newSslConnectionFactory(Metric metric, HttpConnectionFactory httpFactory) {
+        SslContextFactory ctxFactory = sslContextFactoryProvider.getInstance(connectorConfig.name(), connectorConfig.listenPort());
+        SslConnectionFactory connectionFactory = new SslConnectionFactory(ctxFactory, httpFactory.getProtocol());
+        connectionFactory.addBean(new SslHandshakeFailedListener(metric, connectorConfig.name(), connectorConfig.listenPort()));
+        return connectionFactory;
     }
 
 }
diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/HealthCheckProxyHandler.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/HealthCheckProxyHandler.java
index ffff63a424e..9dc3380baac 100644
--- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/HealthCheckProxyHandler.java
+++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/HealthCheckProxyHandler.java
@@ -5,22 +5,29 @@ import com.yahoo.jdisc.Response;
 import com.yahoo.jdisc.http.ConnectorConfig;
 import org.apache.http.Header;
 import org.apache.http.HttpEntity;
+import org.apache.http.client.config.RequestConfig;
 import org.apache.http.client.methods.CloseableHttpResponse;
 import org.apache.http.client.methods.HttpGet;
 import org.apache.http.conn.ssl.NoopHostnameVerifier;
-import org.apache.http.impl.NoConnectionReuseStrategy;
+import org.apache.http.conn.ssl.TrustAllStrategy;
 import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.http.impl.client.HttpClientBuilder;
+import org.apache.http.ssl.SSLContexts;
+import org.eclipse.jetty.server.DetectorConnectionFactory;
 import org.eclipse.jetty.server.Request;
 import org.eclipse.jetty.server.SslConnectionFactory;
 import org.eclipse.jetty.server.handler.HandlerWrapper;
 import org.eclipse.jetty.util.ssl.SslContextFactory;
 
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.SSLException;
 import javax.servlet.ServletException;
 import javax.servlet.ServletOutputStream;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.time.Duration;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -53,6 +60,9 @@ class HealthCheckProxyHandler extends HandlerWrapper {
             ConnectorConfig.HealthCheckProxy proxyConfig = connector.connectorConfig().healthCheckProxy();
             if (proxyConfig.enable()) {
                 mapping.put(connector.listenPort(), createProxyTarget(proxyConfig.port(), connectors));
+                log.info(String.format("Port %1$d is configured as a health check proxy for port %2$d. " +
+                                               "HTTP requests to '%3$s' on %1$d are proxied as HTTPS to %2$d.",
+                                       connector.listenPort(), proxyConfig.port(), HEALTH_CHECK_PATH));
             }
         }
         return mapping;
@@ -63,9 +73,11 @@ class HealthCheckProxyHandler extends HandlerWrapper {
                 .filter(connector -> connector.listenPort() == targetPort)
                 .findAny()
                 .orElseThrow(() -> new IllegalArgumentException("Could not find any connector with listen port " + targetPort));
-        SslContextFactory sslContextFactory =
+        SslContextFactory.Server sslContextFactory =
                 Optional.ofNullable(targetConnector.getConnectionFactory(SslConnectionFactory.class))
-                        .map(SslConnectionFactory::getSslContextFactory)
+                        .or(() -> Optional.ofNullable(targetConnector.getConnectionFactory(DetectorConnectionFactory.class))
+                                .map(detectorConnFactory -> detectorConnFactory.getBean(SslConnectionFactory.class)))
+                        .map(connFactory -> (SslContextFactory.Server) connFactory.getSslContextFactory())
                         .orElseThrow(() -> new IllegalArgumentException("Health check proxy can only target https port"));
         return new ProxyTarget(targetPort, sslContextFactory);
     }
@@ -111,18 +123,23 @@ class HealthCheckProxyHandler extends HandlerWrapper {
 
     private static class ProxyTarget implements AutoCloseable {
         final int port;
-        final SslContextFactory sslContextFactory;
+        final SslContextFactory.Server sslContextFactory;
         volatile CloseableHttpClient client;
 
-        ProxyTarget(int port, SslContextFactory sslContextFactory) {
+        ProxyTarget(int port, SslContextFactory.Server sslContextFactory) {
             this.port = port;
             this.sslContextFactory = sslContextFactory;
         }
 
         CloseableHttpResponse requestStatusHtml() throws IOException {
-            HttpGet request = new HttpGet("https://localhost:" + port + HEALTH_CHECK_PATH);
-            request.setHeader("Connection", "Close");
-            return client().execute(request);
+            try {
+                HttpGet request = new HttpGet("https://localhost:" + port + HEALTH_CHECK_PATH);
+                return client().execute(request);
+            } catch (SSLException e) {
+                log.log(Level.SEVERE, "SSL connection failed. Closing existing client, a new client will be created on next request", e);
+                close();
+                throw e;
+            }
         }
 
         // Client construction must be delayed to ensure that the SslContextFactory is started before calling getSslContext().
@@ -132,11 +149,17 @@ class HealthCheckProxyHandler extends HandlerWrapper {
                     if (client == null) {
                         client = HttpClientBuilder.create()
                                 .disableAutomaticRetries()
-                                .setConnectionReuseStrategy(NoConnectionReuseStrategy.INSTANCE)
-                                .setSSLContext(sslContextFactory.getSslContext())
+                                .setMaxConnPerRoute(4)
+                                .setSSLContext(getSslContext(sslContextFactory))
                                 .setSSLHostnameVerifier(NoopHostnameVerifier.INSTANCE)
                                 .setUserTokenHandler(context -> null) // https://stackoverflow.com/a/42112034/1615280
                                 .setUserAgent("health-check-proxy-client")
+                                .setDefaultRequestConfig(
+                                        RequestConfig.custom()
+                                                .setConnectTimeout((int) Duration.ofSeconds(4).toMillis())
+                                                .setConnectionRequestTimeout((int) Duration.ofSeconds(4).toMillis())
+                                                .setSocketTimeout((int) Duration.ofSeconds(8).toMillis())
+                                                .build())
                                 .build();
                     }
                 }
@@ -144,11 +167,31 @@ class HealthCheckProxyHandler extends HandlerWrapper {
             return client;
         }
 
+        private SSLContext getSslContext(SslContextFactory.Server sslContextFactory) {
+            if (sslContextFactory.getNeedClientAuth()) {
+                log.info(String.format("Port %d requires client certificate. HTTPS client will use the target server connector's ssl context.", port));
+                // A client certificate is only required if the server connector's ssl context factory is configured with "need-auth".
+                // We use the server's ssl context (truststore + keystore) if a client certificate is required.
+                // This will only work if the server certificate's CA is in the truststore.
+                return sslContextFactory.getSslContext();
+            } else {
+                log.info(String.format(
+                        "Port %d does not require a client certificate. HTTPS client will use a custom ssl context accepting all certificates.", port));
+                // No client certificate required. The client is configured with a trust manager that accepts all certificates.
+                try {
+                    return SSLContexts.custom().loadTrustMaterial(new TrustAllStrategy()).build();
+                } catch (GeneralSecurityException e) {
+                    throw new RuntimeException(e);
+                }
+            }
+        }
+
         @Override
         public void close() throws IOException {
             synchronized (this) {
                 if (client != null) {
                     client.close();
+                    client = null;
                 }
             }
         }
diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpRequestDispatch.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpRequestDispatch.java
index 71dcb7d0682..b9d686c1d6b 100644
--- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpRequestDispatch.java
+++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpRequestDispatch.java
@@ -10,6 +10,7 @@ import com.yahoo.jdisc.handler.BindingNotFoundException;
 import com.yahoo.jdisc.handler.ContentChannel;
 import com.yahoo.jdisc.handler.OverloadException;
 import com.yahoo.jdisc.handler.RequestHandler;
+import com.yahoo.jdisc.http.ConnectorConfig;
 import com.yahoo.jdisc.http.HttpHeaders;
 import com.yahoo.jdisc.http.HttpRequest;
 import org.eclipse.jetty.io.EofException;
@@ -22,6 +23,7 @@ import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
+import java.time.Instant;
 import java.util.Arrays;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.CompletionException;
@@ -34,6 +36,7 @@ import java.util.logging.Logger;
 import static com.yahoo.jdisc.http.HttpHeaders.Values.APPLICATION_X_WWW_FORM_URLENCODED;
 import static com.yahoo.jdisc.http.core.HttpServletRequestUtils.getConnection;
 import static com.yahoo.jdisc.http.server.jetty.Exceptions.throwUnchecked;
+import static com.yahoo.jdisc.http.server.jetty.JDiscHttpServlet.getConnector;
 
 /**
  * @author Simon Thoresen Hult
@@ -64,14 +67,13 @@ class HttpRequestDispatch {
 
         this.jettyRequest = (Request) servletRequest;
         this.metricReporter = new MetricReporter(jDiscContext.metric, metricContext, jettyRequest.getTimeStamp());
-        honourMaxKeepAliveRequests();
         this.servletResponseController = new ServletResponseController(
                 servletRequest,
                 servletResponse,
                 jDiscContext.janitor,
                 metricReporter,
                 jDiscContext.developerMode());
-
+        markConnectionAsNonPersistentIfThresholdReached(servletRequest);
         this.async = servletRequest.startAsync();
         async.setTimeout(0);
         metricReporter.uriLength(jettyRequest.getOriginalURI().length());
@@ -102,15 +104,6 @@ class HttpRequestDispatch {
         }
     }
 
-    private void honourMaxKeepAliveRequests() {
-        if (jDiscContext.serverConfig.maxKeepAliveRequests() > 0) {
-            HttpConnection connection = getConnection(jettyRequest);
-            if (connection.getMessagesIn() >= jDiscContext.serverConfig.maxKeepAliveRequests()) {
-                connection.getGenerator().setPersistent(false);
-            }
-        }
-    }
-
     private BiConsumer<Void, Throwable> completeRequestCallback;
     {
         AtomicBoolean completeRequestCalled = new AtomicBoolean(false);
@@ -151,6 +144,25 @@ class HttpRequestDispatch {
         };
     }
 
+    private static void markConnectionAsNonPersistentIfThresholdReached(HttpServletRequest request) {
+        ConnectorConfig connectorConfig = getConnector(request).connectorConfig();
+        int maxRequestsPerConnection = connectorConfig.maxRequestsPerConnection();
+        if (maxRequestsPerConnection > 0) {
+            HttpConnection connection = getConnection(request);
+            if (connection.getMessagesIn() >= maxRequestsPerConnection) {
+                connection.getGenerator().setPersistent(false);
+            }
+        }
+        double maxConnectionLifeInSeconds = connectorConfig.maxConnectionLife();
+        if (maxConnectionLifeInSeconds > 0) {
+            HttpConnection connection = getConnection(request);
+            Instant expireAt = Instant.ofEpochMilli((long)(connection.getCreatedTimeStamp() + maxConnectionLifeInSeconds * 1000));
+            if (Instant.now().isAfter(expireAt)) {
+                connection.getGenerator().setPersistent(false);
+            }
+        }
+    }
+
     @SafeVarargs
     @SuppressWarnings("varargs")
     private static boolean isErrorOfType(Throwable throwable, Class<? extends Throwable>... handledTypes) {
diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/JDiscHttpServlet.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/JDiscHttpServlet.java
index cf66af31a79..a6b2deb4681 100644
--- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/JDiscHttpServlet.java
+++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/JDiscHttpServlet.java
@@ -4,6 +4,7 @@ package com.yahoo.jdisc.http.server.jetty;
 import com.yahoo.container.logging.AccessLogEntry;
 import com.yahoo.jdisc.Metric;
 import com.yahoo.jdisc.handler.OverloadException;
+import com.yahoo.jdisc.http.HttpRequest.Method;
 
 import javax.servlet.ServletException;
 import javax.servlet.annotation.WebServlet;
@@ -11,12 +12,12 @@ import javax.servlet.http.HttpServlet;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 import java.io.IOException;
-import java.util.Arrays;
 import java.util.Enumeration;
-import java.util.HashSet;
 import java.util.Set;
 import java.util.logging.Level;
 import java.util.logging.Logger;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 import static com.yahoo.jdisc.http.core.HttpServletRequestUtils.getConnection;
 
@@ -78,8 +79,6 @@ class JDiscHttpServlet extends HttpServlet {
         dispatchHttpRequest(request, response);
     }
 
-    private static final Set<String> JETTY_UNSUPPORTED_METHODS = new HashSet<>(Arrays.asList("PATCH"));
-
     /**
      * Override to set connector attribute before the request becomes an upgrade request in the web socket case.
      * (After the upgrade, the HttpConnection is no longer available.)
@@ -93,13 +92,25 @@ class JDiscHttpServlet extends HttpServlet {
         context.metric.add(JettyHttpServer.Metrics.NUM_REQUESTS, 1, metricContext);
         context.metric.add(JettyHttpServer.Metrics.JDISC_HTTP_REQUESTS, 1, metricContext);
 
-        if (JETTY_UNSUPPORTED_METHODS.contains(request.getMethod().toUpperCase())) {
+
+        Set<String> servletSupportedMethods =
+                Stream.of(Method.OPTIONS, Method.GET, Method.HEAD, Method.POST, Method.PUT, Method.DELETE, Method.TRACE)
+                        .map(Method::name)
+                        .collect(Collectors.toSet());
+        String method = request.getMethod().toUpperCase();
+        if (servletSupportedMethods.contains(method)) {
+            super.service(request, response);
+        } else if (method.equals(Method.PATCH.name())) {
+            // PATCH method is not handled by the Servlet spec
             dispatchHttpRequest(request, response);
         } else {
-            super.service(request, response);
+            // Divergence from HTTP / Servlet spec: JDisc returns 405 for both unknown and known (but unsupported) methods.
+            response.sendError(HttpServletResponse.SC_METHOD_NOT_ALLOWED);
         }
     }
 
+
+
     static JDiscServerConnector getConnector(HttpServletRequest request) {
         return (JDiscServerConnector)getConnection(request).getConnector();
     }
diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/JDiscServerConnector.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/JDiscServerConnector.java
index 5fa43a15912..51bcb892591 100644
--- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/JDiscServerConnector.java
+++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/JDiscServerConnector.java
@@ -10,39 +10,30 @@ import org.eclipse.jetty.server.ServerConnector;
 
 import javax.servlet.ServletRequest;
 import javax.servlet.http.HttpServletRequest;
-import java.io.IOException;
-import java.lang.reflect.Field;
 import java.net.Socket;
 import java.net.SocketException;
-import java.nio.channels.ServerSocketChannel;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Objects;
 import java.util.concurrent.ConcurrentHashMap;
-import java.util.logging.Level;
-import java.util.logging.Logger;
 
 /**
  * @author bjorncs
  */
 class JDiscServerConnector extends ServerConnector {
     public static final String REQUEST_ATTRIBUTE = JDiscServerConnector.class.getName();
-    private final static Logger log = Logger.getLogger(JDiscServerConnector.class.getName());
     private final Metric.Context metricCtx;
     private final Map<RequestDimensions, Metric.Context> requestMetricContextCache = new ConcurrentHashMap<>();
     private final ServerConnectionStatistics statistics;
     private final ConnectorConfig config;
     private final boolean tcpKeepAlive;
     private final boolean tcpNoDelay;
-    private final ServerSocketChannel channelOpenedByActivator;
     private final Metric metric;
     private final String connectorName;
     private final int listenPort;
 
-    JDiscServerConnector(ConnectorConfig config, Metric metric, Server server,
-                         ServerSocketChannel channelOpenedByActivator, ConnectionFactory... factories) {
+    JDiscServerConnector(ConnectorConfig config, Metric metric, Server server, ConnectionFactory... factories) {
         super(server, factories);
-        this.channelOpenedByActivator = channelOpenedByActivator;
         this.config = config;
         this.tcpKeepAlive = config.tcpKeepAliveEnabled();
         this.tcpNoDelay = config.tcpNoDelay();
@@ -69,54 +60,6 @@ class JDiscServerConnector extends ServerConnector {
         }
     }
 
-    @Override
-    public void open() throws IOException {
-        if (channelOpenedByActivator == null) {
-            log.log(Level.INFO, "No channel set by activator, opening channel ourselves.");
-            try {
-                super.open();
-            } catch (RuntimeException e) {
-                log.log(Level.SEVERE, "failed org.eclipse.jetty.server.Server open() with port " + getPort());
-                throw e;
-            }
-            return;
-        }
-        log.log(Level.INFO, "Using channel set by activator: " + channelOpenedByActivator);
-
-        channelOpenedByActivator.socket().setReuseAddress(getReuseAddress());
-        int localPort = channelOpenedByActivator.socket().getLocalPort();
-        try {
-            uglySetLocalPort(localPort);
-        } catch (NoSuchFieldException | IllegalAccessException e) {
-            throw new RuntimeException("Could not set local port.", e);
-        }
-        if (localPort <= 0) {
-            throw new IOException("Server channel not bound");
-        }
-        addBean(channelOpenedByActivator);
-        channelOpenedByActivator.configureBlocking(true);
-        addBean(channelOpenedByActivator);
-
-        try {
-            uglySetChannel(channelOpenedByActivator);
-        } catch (NoSuchFieldException | IllegalAccessException e) {
-            throw new RuntimeException("Could not set server channel.", e);
-        }
-    }
-
-    private void uglySetLocalPort(int localPort) throws NoSuchFieldException, IllegalAccessException {
-        Field localPortField = ServerConnector.class.getDeclaredField("_localPort");
-        localPortField.setAccessible(true);
-        localPortField.set(this, localPort);
-    }
-
-    private void uglySetChannel(ServerSocketChannel channelOpenedByActivator) throws NoSuchFieldException,
-            IllegalAccessException {
-        Field acceptChannelField = ServerConnector.class.getDeclaredField("_acceptChannel");
-        acceptChannelField.setAccessible(true);
-        acceptChannelField.set(this, channelOpenedByActivator);
-    }
-
     public ServerConnectionStatistics getStatistics() {
         return statistics;
     }
diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/JettyHttpServer.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/JettyHttpServer.java
index 140feb75026..04db58f6d07 100644
--- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/JettyHttpServer.java
+++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/JettyHttpServer.java
@@ -8,7 +8,6 @@ import com.yahoo.component.ComponentId;
 import com.yahoo.component.provider.ComponentRegistry;
 import com.yahoo.container.logging.AccessLog;
 import com.yahoo.jdisc.Metric;
-import com.yahoo.jdisc.application.OsgiFramework;
 import com.yahoo.jdisc.http.ConnectorConfig;
 import com.yahoo.jdisc.http.ServerConfig;
 import com.yahoo.jdisc.http.ServletPathsConfig;
@@ -35,9 +34,6 @@ import org.eclipse.jetty.servlet.ServletHolder;
 import org.eclipse.jetty.util.log.JavaUtilLog;
 import org.eclipse.jetty.util.log.Log;
 import org.eclipse.jetty.util.thread.QueuedThreadPool;
-import org.osgi.framework.BundleContext;
-import org.osgi.framework.InvalidSyntaxException;
-import org.osgi.framework.ServiceReference;
 
 import javax.management.remote.JMXServiceURL;
 import javax.servlet.DispatcherType;
@@ -45,10 +41,8 @@ import java.io.IOException;
 import java.lang.management.ManagementFactory;
 import java.net.BindException;
 import java.net.MalformedURLException;
-import java.nio.channels.ServerSocketChannel;
 import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.Collection;
 import java.util.EnumSet;
 import java.util.HashMap;
 import java.util.List;
@@ -114,6 +108,13 @@ public class JettyHttpServer extends AbstractServerProvider {
 
         String URI_LENGTH = "jdisc.http.request.uri_length";
         String CONTENT_SIZE = "jdisc.http.request.content_size";
+
+        String SSL_HANDSHAKE_FAILURE_MISSING_CLIENT_CERT = "jdisc.http.ssl.handshake.failure.missing_client_cert";
+        String SSL_HANDSHAKE_FAILURE_EXPIRED_CLIENT_CERT = "jdisc.http.ssl.handshake.failure.expired_client_cert";
+        String SSL_HANDSHAKE_FAILURE_INVALID_CLIENT_CERT = "jdisc.http.ssl.handshake.failure.invalid_client_cert";
+        String SSL_HANDSHAKE_FAILURE_INCOMPATIBLE_PROTOCOLS = "jdisc.http.ssl.handshake.failure.incompatible_protocols";
+        String SSL_HANDSHAKE_FAILURE_INCOMPATIBLE_CIPHERS = "jdisc.http.ssl.handshake.failure.incompatible_ciphers";
+        String SSL_HANDSHAKE_FAILURE_UNKNOWN = "jdisc.http.ssl.handshake.failure.unknown";
     }
 
     private final static Logger log = Logger.getLogger(JettyHttpServer.class.getName());
@@ -134,7 +135,6 @@ public class JettyHttpServer extends AbstractServerProvider {
             final FilterBindings filterBindings,
             final ComponentRegistry<ConnectorFactory> connectorFactories,
             final ComponentRegistry<ServletHolder> servletHolders,
-            final OsgiFramework osgiFramework,
             final FilterInvoker filterInvoker,
             final AccessLog accessLog) {
         super(container);
@@ -150,12 +150,9 @@ public class JettyHttpServer extends AbstractServerProvider {
         setupJmx(server, serverConfig);
         ((QueuedThreadPool)server.getThreadPool()).setMaxThreads(serverConfig.maxWorkerThreads());
 
-        List<ConnectorConfig> connectorConfigs = new ArrayList<>();
         for (ConnectorFactory connectorFactory : connectorFactories.allComponents()) {
             ConnectorConfig connectorConfig = connectorFactory.getConnectorConfig();
-            connectorConfigs.add(connectorConfig);
-            ServerSocketChannel preBoundChannel = getChannelFromServiceLayer(connectorConfig.listenPort(), osgiFramework.bundleContext());
-            server.addConnector(connectorFactory.createConnector(metric, server, preBoundChannel));
+            server.addConnector(connectorFactory.createConnector(metric, server));
             listenedPorts.add(connectorConfig.listenPort());
         }
 
@@ -245,10 +242,13 @@ public class JettyHttpServer extends AbstractServerProvider {
 
         servletContextHandler.addServlet(jdiscServlet, "/*");
 
+        List<ConnectorConfig> connectorConfigs = connectors.stream().map(JDiscServerConnector::connectorConfig).collect(toList());
+        var secureRedirectHandler = new SecuredRedirectHandler(connectorConfigs);
+        secureRedirectHandler.setHandler(servletContextHandler);
+
         var proxyHandler = new HealthCheckProxyHandler(connectors);
-        proxyHandler.setHandler(servletContextHandler);
+        proxyHandler.setHandler(secureRedirectHandler);
 
-        List<ConnectorConfig> connectorConfigs = connectors.stream().map(JDiscServerConnector::connectorConfig).collect(toList());
         var authEnforcer = new TlsClientAuthenticationEnforcer(connectorConfigs);
         authEnforcer.setHandler(proxyHandler);
 
@@ -281,25 +281,6 @@ public class JettyHttpServer extends AbstractServerProvider {
         return ports.stream().map(Object::toString).collect(Collectors.joining(":"));
     }
 
-    private ServerSocketChannel getChannelFromServiceLayer(int listenPort, BundleContext bundleContext) {
-        log.log(Level.FINE, "Retrieving channel for port " + listenPort + " from " + bundleContext.getClass().getName());
-        Collection<ServiceReference<ServerSocketChannel>> refs;
-        final String filter = "(port=" + listenPort + ")";
-        try {
-            refs = bundleContext.getServiceReferences(ServerSocketChannel.class, filter);
-        } catch (InvalidSyntaxException e) {
-            throw new IllegalStateException("OSGi framework rejected filter " + filter, e);
-        }
-        if (refs.isEmpty()) {
-            return null;
-        }
-        if (refs.size() != 1) {
-            throw new IllegalStateException("Got more than one service reference for " + ServerSocketChannel.class + " port " + listenPort + ".");
-        }
-        ServiceReference<ServerSocketChannel> ref = refs.iterator().next();
-        return bundleContext.getService(ref);
-    }
-
     private static ExecutorService newJanitor(ThreadFactory factory) {
         int threadPoolSize = Runtime.getRuntime().availableProcessors();
         log.info("Creating janitor executor with " + threadPoolSize + " threads");
diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/SecuredRedirectHandler.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/SecuredRedirectHandler.java
new file mode 100644
index 00000000000..7798b5e6ae3
--- /dev/null
+++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/SecuredRedirectHandler.java
@@ -0,0 +1,56 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.jdisc.http.server.jetty;
+
+import com.yahoo.jdisc.http.ConnectorConfig;
+import org.eclipse.jetty.server.Request;
+import org.eclipse.jetty.server.handler.HandlerWrapper;
+import org.eclipse.jetty.util.URIUtil;
+
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * A secure redirect handler inspired by {@link org.eclipse.jetty.server.handler.SecuredRedirectHandler}.
+ *
+ * @author bjorncs
+ */
+class SecuredRedirectHandler extends HandlerWrapper {
+
+    private static final String HEALTH_CHECK_PATH = "/status.html";
+
+    private final Map<Integer, Integer> redirectMap;
+
+    SecuredRedirectHandler(List<ConnectorConfig> connectorConfigs) {
+        this.redirectMap = createRedirectMap(connectorConfigs);
+    }
+
+    @Override
+    public void handle(String target, Request request, HttpServletRequest servletRequest, HttpServletResponse servletResponse) throws IOException, ServletException {
+        int localPort = servletRequest.getLocalPort();
+        if (!redirectMap.containsKey(localPort)) {
+            _handler.handle(target, request, servletRequest, servletResponse);
+            return;
+        }
+        servletResponse.setContentLength(0);
+        if (!servletRequest.getRequestURI().equals(HEALTH_CHECK_PATH)) {
+            servletResponse.sendRedirect(
+                    URIUtil.newURI("https", request.getServerName(), redirectMap.get(localPort), request.getRequestURI(), request.getQueryString()));
+        }
+        request.setHandled(true);
+    }
+
+    private static Map<Integer, Integer> createRedirectMap(List<ConnectorConfig> connectorConfigs) {
+        var redirectMap = new HashMap<Integer, Integer>();
+        for (ConnectorConfig connectorConfig : connectorConfigs) {
+            if (connectorConfig.secureRedirect().enabled()) {
+                redirectMap.put(connectorConfig.listenPort(), connectorConfig.secureRedirect().port());
+            }
+        }
+        return redirectMap;
+    }
+}
diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/SslHandshakeFailedListener.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/SslHandshakeFailedListener.java
new file mode 100644
index 00000000000..75df82036a2
--- /dev/null
+++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/SslHandshakeFailedListener.java
@@ -0,0 +1,88 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.jdisc.http.server.jetty;
+
+import com.yahoo.jdisc.Metric;
+import com.yahoo.jdisc.http.server.jetty.JettyHttpServer.Metrics;
+import org.eclipse.jetty.io.ssl.SslHandshakeListener;
+
+import javax.net.ssl.SSLHandshakeException;
+import java.util.Map;
+import java.util.Optional;
+import java.util.function.Predicate;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import java.util.regex.Pattern;
+
+/**
+ * A {@link SslHandshakeListener} that reports metrics for SSL handshake failures.
+ *
+ * @author bjorncs
+ */
+class SslHandshakeFailedListener implements SslHandshakeListener {
+
+    private final static Logger log = Logger.getLogger(SslHandshakeFailedListener.class.getName());
+
+    private final Metric metric;
+    private final String connectorName;
+    private final int listenPort;
+
+    SslHandshakeFailedListener(Metric metric, String connectorName, int listenPort) {
+        this.metric = metric;
+        this.connectorName = connectorName;
+        this.listenPort = listenPort;
+    }
+
+    @Override
+    public void handshakeFailed(Event event, Throwable throwable) {
+        log.log(Level.FINE, throwable, () -> "Ssl handshake failed: " + throwable.getMessage());
+        String metricName = SslHandshakeFailure.fromSslHandshakeException((SSLHandshakeException) throwable)
+                .map(SslHandshakeFailure::metricName)
+                .orElse(Metrics.SSL_HANDSHAKE_FAILURE_UNKNOWN);
+        metric.add(metricName, 1L, metric.createContext(createDimensions()));
+    }
+
+    private Map<String, Object> createDimensions() {
+        return Map.of(Metrics.NAME_DIMENSION, connectorName, Metrics.PORT_DIMENSION, listenPort);
+    }
+
+    private enum SslHandshakeFailure {
+        INCOMPATIBLE_PROTOCOLS(
+                Metrics.SSL_HANDSHAKE_FAILURE_INCOMPATIBLE_PROTOCOLS,
+                "(Client requested protocol \\S+? is not enabled or supported in server context" +
+                        "|The client supported protocol versions \\[\\S+?\\] are not accepted by server preferences \\[\\S+?\\])"),
+        INCOMPATIBLE_CIPHERS(
+                Metrics.SSL_HANDSHAKE_FAILURE_INCOMPATIBLE_CIPHERS,
+                "no cipher suites in common"),
+        MISSING_CLIENT_CERT(
+                Metrics.SSL_HANDSHAKE_FAILURE_MISSING_CLIENT_CERT,
+                "Empty server certificate chain"),
+        EXPIRED_CLIENT_CERTIFICATE(
+                Metrics.SSL_HANDSHAKE_FAILURE_EXPIRED_CLIENT_CERT,
+                // Note: this pattern will match certificates with too late notBefore as well
+                "PKIX path validation failed: java.security.cert.CertPathValidatorException: validity check failed"),
+        INVALID_CLIENT_CERT(
+                Metrics.SSL_HANDSHAKE_FAILURE_INVALID_CLIENT_CERT,
+                "PKIX path (building|validation) failed: .+");
+
+        private final String metricName;
+        private final Predicate<String> messageMatcher;
+
+        SslHandshakeFailure(String metricName, String messagePattern) {
+            this.metricName = metricName;
+            this.messageMatcher = Pattern.compile(messagePattern).asMatchPredicate();
+        }
+
+        String metricName() { return metricName; }
+
+        static Optional<SslHandshakeFailure> fromSslHandshakeException(SSLHandshakeException exception) {
+            String message = exception.getMessage();
+            if (message == null || message.isBlank()) return Optional.empty();
+            for (SslHandshakeFailure failure : values()) {
+                if (failure.messageMatcher.test(message)) {
+                    return Optional.of(failure);
+                }
+            }
+            return Optional.empty();
+        }
+    }
+}
diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/ConfiguredSslContextFactoryProvider.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/ConfiguredSslContextFactoryProvider.java
index 48a7c246500..90848f1dfd4 100644
--- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/ConfiguredSslContextFactoryProvider.java
+++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/ConfiguredSslContextFactoryProvider.java
@@ -2,14 +2,16 @@
 package com.yahoo.jdisc.http.ssl.impl;
 
 import com.yahoo.jdisc.http.ConnectorConfig;
+import com.yahoo.jdisc.http.ConnectorConfig.Ssl.ClientAuth;
 import com.yahoo.jdisc.http.ssl.SslContextFactoryProvider;
 import com.yahoo.security.KeyUtils;
+import com.yahoo.security.SslContextBuilder;
 import com.yahoo.security.X509CertificateUtils;
-import com.yahoo.security.tls.DefaultTlsContext;
-import com.yahoo.security.tls.PeerAuthentication;
+import com.yahoo.security.tls.AutoReloadingX509KeyManager;
 import com.yahoo.security.tls.TlsContext;
 import org.eclipse.jetty.util.ssl.SslContextFactory;
 
+import javax.net.ssl.SSLContext;
 import java.io.IOException;
 import java.io.UncheckedIOException;
 import java.nio.charset.StandardCharsets;
@@ -17,16 +19,21 @@ import java.nio.file.Files;
 import java.nio.file.Paths;
 import java.security.PrivateKey;
 import java.security.cert.X509Certificate;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Optional;
 
+import static com.yahoo.jdisc.http.ssl.impl.SslContextFactoryUtils.setEnabledCipherSuites;
+import static com.yahoo.jdisc.http.ssl.impl.SslContextFactoryUtils.setEnabledProtocols;
+
 /**
  * An implementation of {@link SslContextFactoryProvider} that uses the {@link ConnectorConfig} to construct a {@link SslContextFactory}.
  *
  * @author bjorncs
  */
-public class ConfiguredSslContextFactoryProvider extends TlsContextBasedProvider {
+public class ConfiguredSslContextFactoryProvider implements SslContextFactoryProvider {
 
+    private volatile AutoReloadingX509KeyManager keyManager;
     private final ConnectorConfig connectorConfig;
 
     public ConfiguredSslContextFactoryProvider(ConnectorConfig connectorConfig) {
@@ -35,17 +42,50 @@ public class ConfiguredSslContextFactoryProvider extends TlsContextBasedProvider
     }
 
     @Override
-    protected TlsContext getTlsContext(String containerId, int port) {
+    public SslContextFactory getInstance(String containerId, int port) {
         ConnectorConfig.Ssl sslConfig = connectorConfig.ssl();
         if (!sslConfig.enabled()) throw new IllegalStateException();
 
-        PrivateKey privateKey = KeyUtils.fromPemEncodedPrivateKey(getPrivateKey(sslConfig));
-        List<X509Certificate> certificates = X509CertificateUtils.certificateListFromPem(getCertificate(sslConfig));
+        SslContextBuilder builder = new SslContextBuilder();
+        if (sslConfig.certificateFile().isBlank() || sslConfig.privateKeyFile().isBlank()) {
+            PrivateKey privateKey = KeyUtils.fromPemEncodedPrivateKey(getPrivateKey(sslConfig));
+            List<X509Certificate> certificates = X509CertificateUtils.certificateListFromPem(getCertificate(sslConfig));
+            builder.withKeyStore(privateKey, certificates);
+        } else {
+            keyManager = AutoReloadingX509KeyManager.fromPemFiles(Paths.get(sslConfig.privateKeyFile()), Paths.get(sslConfig.certificateFile()));
+            builder.withKeyManager(keyManager);
+        }
         List<X509Certificate> caCertificates = getCaCertificates(sslConfig)
                 .map(X509CertificateUtils::certificateListFromPem)
                 .orElse(List.of());
-        PeerAuthentication peerAuthentication = toPeerAuthentication(sslConfig.clientAuth());
-        return new DefaultTlsContext(certificates, privateKey, caCertificates, null, null, peerAuthentication);
+        builder.withTrustStore(caCertificates);
+
+        SSLContext sslContext = builder.build();
+
+        SslContextFactory.Server factory = new SslContextFactory.Server();
+        factory.setSslContext(sslContext);
+
+        factory.setNeedClientAuth(sslConfig.clientAuth() == ClientAuth.Enum.NEED_AUTH);
+        factory.setWantClientAuth(sslConfig.clientAuth() == ClientAuth.Enum.WANT_AUTH);
+
+        List<String> protocols = !sslConfig.enabledProtocols().isEmpty()
+                ? sslConfig.enabledProtocols()
+                : new ArrayList<>(TlsContext.getAllowedProtocols(sslContext));
+        setEnabledProtocols(factory, sslContext, protocols);
+
+        List<String> ciphers = !sslConfig.enabledCipherSuites().isEmpty()
+                ? sslConfig.enabledCipherSuites()
+                : new ArrayList<>(TlsContext.getAllowedCipherSuites(sslContext));
+        setEnabledCipherSuites(factory, sslContext, ciphers);
+
+        return factory;
+    }
+
+    @Override
+    public void close() {
+        if (keyManager != null) {
+            keyManager.close();
+        }
     }
 
     private static void validateConfig(ConnectorConfig.Ssl config) {
@@ -64,19 +104,6 @@ public class ConfiguredSslContextFactoryProvider extends TlsContextBasedProvider
             throw new IllegalArgumentException("Specified neither private key or private key file.");
     }
 
-    private static PeerAuthentication toPeerAuthentication(ConnectorConfig.Ssl.ClientAuth.Enum clientAuth) {
-        switch (clientAuth) {
-            case DISABLED:
-                return PeerAuthentication.DISABLED;
-            case NEED_AUTH:
-                return PeerAuthentication.NEED;
-            case WANT_AUTH:
-                return PeerAuthentication.WANT;
-            default:
-                throw new IllegalArgumentException("Unknown client auth: " + clientAuth);
-        }
-    }
-
     private static boolean hasBoth(String a, String b) { return !a.isBlank() && !b.isBlank(); }
     private static boolean hasNeither(String a, String b) { return a.isBlank() && b.isBlank(); }
 
diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/SslContextFactoryUtils.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/SslContextFactoryUtils.java
new file mode 100644
index 00000000000..a0172668cbb
--- /dev/null
+++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/SslContextFactoryUtils.java
@@ -0,0 +1,32 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.jdisc.http.ssl.impl;
+
+import org.eclipse.jetty.util.ssl.SslContextFactory;
+
+import javax.net.ssl.SSLContext;
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * @author bjorncs
+ */
+class SslContextFactoryUtils {
+
+    static void setEnabledCipherSuites(SslContextFactory factory, SSLContext sslContext, List<String> enabledCiphers) {
+        String[] supportedCiphers = sslContext.getSupportedSSLParameters().getCipherSuites();
+        factory.setIncludeCipherSuites(enabledCiphers.toArray(String[]::new));
+        factory.setExcludeCipherSuites(createExclusionList(enabledCiphers, supportedCiphers));
+    }
+
+    static void setEnabledProtocols(SslContextFactory factory, SSLContext sslContext, List<String> enabledProtocols) {
+        String[] supportedProtocols = sslContext.getSupportedSSLParameters().getProtocols();
+        factory.setIncludeProtocols(enabledProtocols.toArray(String[]::new));
+        factory.setExcludeProtocols(createExclusionList(enabledProtocols, supportedProtocols));
+    }
+
+    private static String[] createExclusionList(List<String> enabledValues, String[] supportedValues) {
+        return Arrays.stream(supportedValues)
+                .filter(supportedValue -> !enabledValues.contains(supportedValue))
+                .toArray(String[]::new);
+    }
+}
diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/TlsContextBasedProvider.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/TlsContextBasedProvider.java
index e8ae13e48be..93d4f1dca3f 100644
--- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/TlsContextBasedProvider.java
+++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/TlsContextBasedProvider.java
@@ -8,7 +8,10 @@ import org.eclipse.jetty.util.ssl.SslContextFactory;
 
 import javax.net.ssl.SSLContext;
 import javax.net.ssl.SSLParameters;
-import java.util.Arrays;
+import java.util.List;
+
+import static com.yahoo.jdisc.http.ssl.impl.SslContextFactoryUtils.setEnabledCipherSuites;
+import static com.yahoo.jdisc.http.ssl.impl.SslContextFactoryUtils.setEnabledProtocols;
 
 /**
  * A {@link SslContextFactoryProvider} that creates {@link SslContextFactory} instances from {@link TlsContext} instances.
@@ -31,24 +34,9 @@ public abstract class TlsContextBasedProvider extends AbstractComponent implemen
         sslContextFactory.setNeedClientAuth(parameters.getNeedClientAuth());
         sslContextFactory.setWantClientAuth(parameters.getWantClientAuth());
 
-        String[] enabledProtocols = parameters.getProtocols();
-        sslContextFactory.setIncludeProtocols(enabledProtocols);
-        String[] supportedProtocols = sslContext.getSupportedSSLParameters().getProtocols();
-        sslContextFactory.setExcludeProtocols(createExclusionList(enabledProtocols, supportedProtocols));
+        setEnabledProtocols(sslContextFactory, sslContext, List.of(parameters.getProtocols()));
+        setEnabledCipherSuites(sslContextFactory, sslContext, List.of(parameters.getCipherSuites()));
 
-        String[] enabledCiphers = parameters.getCipherSuites();
-        String[] supportedCiphers = sslContext.getSupportedSSLParameters().getCipherSuites();
-        sslContextFactory.setIncludeCipherSuites(enabledCiphers);
-        sslContextFactory.setExcludeCipherSuites(createExclusionList(enabledCiphers, supportedCiphers));
         return sslContextFactory;
     }
-
-    private static String[] createExclusionList(String[] enabledValues, String[] supportedValues) {
-        return Arrays.stream(supportedValues)
-                .filter(supportedValue ->
-                                Arrays.stream(enabledValues)
-                                        .noneMatch(enabledValue -> enabledValue.equals(supportedValue)))
-                .toArray(String[]::new);
-    }
-
 }
diff --git a/jdisc_http_service/src/main/resources/configdefinitions/jdisc.http.connector.def b/jdisc_http_service/src/main/resources/configdefinitions/jdisc.http.connector.def
index 1122b1db3a9..fa7ed6657d9 100644
--- a/jdisc_http_service/src/main/resources/configdefinitions/jdisc.http.connector.def
+++ b/jdisc_http_service/src/main/resources/configdefinitions/jdisc.http.connector.def
@@ -81,6 +81,12 @@ ssl.caCertificate                   string default=""
 # Client authentication mode. See SSLEngine.getNeedClientAuth()/getWantClientAuth() for details.
 ssl.clientAuth                      enum { DISABLED, WANT_AUTH, NEED_AUTH }  default=DISABLED
 
+# List of enabled cipher suites. JDisc will use Vespa default if empty.
+ssl.enabledCipherSuites[]           string
+
+# List of enabled TLS protocol versions. JDisc will use Vespa default if empty.
+ssl.enabledProtocols[]              string
+
 # Enforce TLS client authentication for https requests at the http layer.
 # Intended to be used with connectors with optional client authentication enabled.
 # 401 status code is returned for requests from non-authenticated clients.
@@ -94,3 +100,21 @@ healthCheckProxy.enable        bool    default=false
 
 # Which port to proxy
 healthCheckProxy.port          int     default=8080
+
+# Enable PROXY protocol V1/V2 support (only for https connectors).
+proxyProtocol.enabled          bool    default=false
+
+# Allow https in parallel with proxy protocol
+proxyProtocol.mixedMode        bool    default=false
+
+# Redirect all requests to https port
+secureRedirect.enabled         bool    default=false
+
+# Target port for redirect
+secureRedirect.port            int     default=443
+
+# Maximum number of request per connection before server marks connections as non-persistent. Set to '0' to disable.
+maxRequestsPerConnection       int     default=0
+
+# Maximum number of seconds a connection can live before it's marked as non-persistent. Set to '0' to disable.
+maxConnectionLife              double  default=0.0
diff --git a/jdisc_http_service/src/main/resources/configdefinitions/jdisc.http.server.def b/jdisc_http_service/src/main/resources/configdefinitions/jdisc.http.server.def
index 0836a080e1f..33f82963243 100644
--- a/jdisc_http_service/src/main/resources/configdefinitions/jdisc.http.server.def
+++ b/jdisc_http_service/src/main/resources/configdefinitions/jdisc.http.server.def
@@ -7,13 +7,16 @@ developerMode                         bool   default=false
 # The gzip compression level to use, if compression is enabled in a request.
 responseCompressionLevel              int    default=6
 
-# Whether to enable HTTP keep-alive for requests that support this.
+# DEPRECATED - Ignored, no longer in use.
 httpKeepAliveEnabled                  bool   default=true
+# TODO Vespa 8 Remove httpKeepAliveEnabled
 
 # Maximum number of request per http connection before server will hangup.
 # Naming taken from apache http server.
 # 0 means never hangup.
+# DEPRECATED - Ignored, no longer in use. Use similar parameter in connector config instead.
 maxKeepAliveRequests                  int    default=0
+# TODO Vespa 8 Remove maxKeepAliveRequests
 
 # Whether the request body of POSTed forms should be removed (form parameters are available as request parameters).
 removeRawPostBodyForWwwUrlEncodedPost bool   default=false
diff --git a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/ConnectorFactoryTest.java b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/ConnectorFactoryTest.java
index 4e38d6dc7fb..07bffffdbbd 100644
--- a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/ConnectorFactoryTest.java
+++ b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/ConnectorFactoryTest.java
@@ -9,12 +9,9 @@ import org.eclipse.jetty.server.Server;
 import org.eclipse.jetty.server.handler.AbstractHandler;
 import org.junit.Test;
 
-import javax.servlet.ServletException;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.nio.channels.ServerSocketChannel;
 import java.util.Map;
 
 import static org.hamcrest.CoreMatchers.equalTo;
@@ -25,41 +22,13 @@ import static org.hamcrest.CoreMatchers.equalTo;
 public class ConnectorFactoryTest {
 
     @Test
-    public void requireThatNoPreBoundChannelWorks() throws Exception {
+    public void requireThatServerCanBindChannel() throws Exception {
         Server server = new Server();
         try {
             ConnectorConfig config = new ConnectorConfig(new ConnectorConfig.Builder());
             ConnectorFactory factory = createConnectorFactory(config);
             JDiscServerConnector connector =
-                    (JDiscServerConnector)factory.createConnector(new DummyMetric(), server, null);
-            server.addConnector(connector);
-            server.setHandler(new HelloWorldHandler());
-            server.start();
-
-            SimpleHttpClient client = new SimpleHttpClient(null, connector.getLocalPort(), false);
-            SimpleHttpClient.RequestExecutor ex = client.newGet("/blaasdfnb");
-            SimpleHttpClient.ResponseValidator val = ex.execute();
-            val.expectContent(equalTo("Hello world"));
-        } finally {
-            try {
-                server.stop();
-            } catch (Exception e) {
-                //ignore
-            }
-        }
-    }
-
-    @Test
-    public void requireThatPreBoundChannelWorks() throws Exception {
-        Server server = new Server();
-        try {
-            ServerSocketChannel serverChannel = ServerSocketChannel.open();
-            serverChannel.socket().bind(new InetSocketAddress(0));
-
-            ConnectorConfig config = new ConnectorConfig(new ConnectorConfig.Builder());
-            ConnectorFactory factory = createConnectorFactory(config);
-            JDiscServerConnector connector =
-                    (JDiscServerConnector) factory.createConnector(new DummyMetric(), server, serverChannel);
+                    (JDiscServerConnector)factory.createConnector(new DummyMetric(), server);
             server.addConnector(connector);
             server.setHandler(new HelloWorldHandler());
             server.start();
@@ -83,7 +52,7 @@ public class ConnectorFactoryTest {
 
     private static class HelloWorldHandler extends AbstractHandler {
         @Override
-        public void handle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException {
+        public void handle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException {
             response.getWriter().write("Hello world");
             response.getWriter().flush();
             response.getWriter().close();
diff --git a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpServerTest.java b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpServerTest.java
index 31ecf3ca2fc..dfa4fe37ef3 100644
--- a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpServerTest.java
+++ b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpServerTest.java
@@ -5,10 +5,12 @@ import com.google.inject.AbstractModule;
 import com.google.inject.Module;
 import com.yahoo.container.logging.AccessLog;
 import com.yahoo.container.logging.AccessLogEntry;
+import com.yahoo.jdisc.Metric;
 import com.yahoo.jdisc.References;
 import com.yahoo.jdisc.Request;
 import com.yahoo.jdisc.Response;
 import com.yahoo.jdisc.application.BindingSetSelector;
+import com.yahoo.jdisc.application.MetricConsumer;
 import com.yahoo.jdisc.handler.AbstractRequestHandler;
 import com.yahoo.jdisc.handler.CompletionHandler;
 import com.yahoo.jdisc.handler.ContentChannel;
@@ -21,19 +23,31 @@ import com.yahoo.jdisc.http.Cookie;
 import com.yahoo.jdisc.http.HttpRequest;
 import com.yahoo.jdisc.http.HttpResponse;
 import com.yahoo.jdisc.http.ServerConfig;
+import com.yahoo.jdisc.http.server.jetty.JettyHttpServer.Metrics;
+import com.yahoo.jdisc.http.server.jetty.TestDrivers.TlsClientAuth;
 import com.yahoo.jdisc.service.BindingSetNotFoundException;
 import com.yahoo.security.KeyUtils;
+import com.yahoo.security.Pkcs10Csr;
+import com.yahoo.security.Pkcs10CsrBuilder;
 import com.yahoo.security.SslContextBuilder;
 import com.yahoo.security.X509CertificateBuilder;
 import com.yahoo.security.X509CertificateUtils;
+import org.apache.http.conn.ssl.NoopHostnameVerifier;
 import org.apache.http.entity.ContentType;
 import org.apache.http.entity.mime.FormBodyPart;
 import org.apache.http.entity.mime.content.StringBody;
+import org.eclipse.jetty.client.HttpClient;
+import org.eclipse.jetty.client.ProxyProtocolClientConnectionFactory.V1;
+import org.eclipse.jetty.client.ProxyProtocolClientConnectionFactory.V2;
+import org.eclipse.jetty.client.api.ContentResponse;
+import org.eclipse.jetty.util.ssl.SslContextFactory;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.TemporaryFolder;
 
 import javax.net.ssl.SSLContext;
+import javax.net.ssl.SSLException;
+import javax.net.ssl.SSLHandshakeException;
 import javax.security.auth.x500.X500Principal;
 import java.io.IOException;
 import java.math.BigInteger;
@@ -44,6 +58,7 @@ import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.security.KeyPair;
+import java.security.PrivateKey;
 import java.security.cert.X509Certificate;
 import java.time.Instant;
 import java.time.temporal.ChronoUnit;
@@ -53,7 +68,12 @@ import java.util.Comparator;
 import java.util.List;
 import java.util.Map;
 import java.util.TreeMap;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.logging.Level;
+import java.util.logging.Logger;
 import java.util.regex.Pattern;
 
 import static com.yahoo.jdisc.Response.Status.GATEWAY_TIMEOUT;
@@ -81,15 +101,23 @@ import static org.hamcrest.CoreMatchers.not;
 import static org.hamcrest.CoreMatchers.startsWith;
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasSize;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+import static org.mockito.ArgumentMatchers.anyMap;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
 /**
  * @author Oyvind Bakksjo
  * @author Simon Thoresen Hult
+ * @author bjorncs
  */
 public class HttpServerTest {
 
+    private static final Logger log = Logger.getLogger(HttpServerTest.class.getName());
+
     @Rule
     public TemporaryFolder tmpFolder = new TemporaryFolder();
 
@@ -164,7 +192,7 @@ public class HttpServerTest {
 
     private static class AccessLogMock extends AccessLog {
 
-        final List<AccessLogEntry> logEntries = new ArrayList<>();
+        final List<AccessLogEntry> logEntries = new CopyOnWriteArrayList<>();
 
         AccessLogMock() { super(null); }
 
@@ -459,8 +487,8 @@ public class HttpServerTest {
     public void requireThatConnectionIsClosedAfterXRequests() throws Exception {
         final int MAX_KEEPALIVE_REQUESTS = 100;
         final TestDriver driver = TestDrivers.newConfiguredInstance(new EchoRequestHandler(),
-                new ServerConfig.Builder().maxKeepAliveRequests(MAX_KEEPALIVE_REQUESTS),
-                new ConnectorConfig.Builder());
+                new ServerConfig.Builder(),
+                new ConnectorConfig.Builder().maxRequestsPerConnection(MAX_KEEPALIVE_REQUESTS));
         for (int i = 0; i < MAX_KEEPALIVE_REQUESTS - 1; i++) {
             driver.client().get("/status.html")
                     .expectStatusCode(is(OK))
@@ -478,7 +506,7 @@ public class HttpServerTest {
         Path certificateFile = tmpFolder.newFile().toPath();
         generatePrivateKeyAndCertificate(privateKeyFile, certificateFile);
 
-        final TestDriver driver = TestDrivers.newInstanceWithSsl(new EchoRequestHandler(), certificateFile, privateKeyFile);
+        final TestDriver driver = TestDrivers.newInstanceWithSsl(new EchoRequestHandler(), certificateFile, privateKeyFile, TlsClientAuth.WANT);
         driver.client().get("/status.html")
               .expectStatusCode(is(OK));
         assertThat(driver.close(), is(true));
@@ -489,7 +517,7 @@ public class HttpServerTest {
         Path privateKeyFile = tmpFolder.newFile().toPath();
         Path certificateFile = tmpFolder.newFile().toPath();
         generatePrivateKeyAndCertificate(privateKeyFile, certificateFile);
-        TestDriver driver = TestDrivers.newInstanceWithSsl(new EchoRequestHandler(), certificateFile, privateKeyFile);
+        TestDriver driver = TestDrivers.newInstanceWithSsl(new EchoRequestHandler(), certificateFile, privateKeyFile, TlsClientAuth.WANT);
 
         SSLContext trustStoreOnlyCtx = new SslContextBuilder()
                 .withTrustStore(certificateFile)
@@ -507,7 +535,7 @@ public class HttpServerTest {
         Path privateKeyFile = tmpFolder.newFile().toPath();
         Path certificateFile = tmpFolder.newFile().toPath();
         generatePrivateKeyAndCertificate(privateKeyFile, certificateFile);
-        TestDriver driver = TestDrivers.newInstanceWithSsl(new EchoRequestHandler(), certificateFile, privateKeyFile);
+        TestDriver driver = TestDrivers.newInstanceWithSsl(new EchoRequestHandler(), certificateFile, privateKeyFile, TlsClientAuth.WANT);
 
         SSLContext trustStoreOnlyCtx = new SslContextBuilder()
                 .withTrustStore(certificateFile)
@@ -559,6 +587,222 @@ public class HttpServerTest {
         assertThat(driver.close(), is(true));
     }
 
+    @Test
+    public void requireThatMetricIsIncrementedWhenClientIsMissingCertificateOnHandshake() throws IOException {
+        Path privateKeyFile = tmpFolder.newFile().toPath();
+        Path certificateFile = tmpFolder.newFile().toPath();
+        generatePrivateKeyAndCertificate(privateKeyFile, certificateFile);
+        var metricConsumer = new MetricConsumerMock();
+        TestDriver driver = createSslTestDriver(certificateFile, privateKeyFile, metricConsumer);
+
+        SSLContext clientCtx = new SslContextBuilder()
+                .withTrustStore(certificateFile)
+                .build();
+        assertHttpsRequestTriggersSslHandshakeException(
+                driver, clientCtx, null, null, "Received fatal alert: bad_certificate");
+        verify(metricConsumer.mockitoMock())
+                .add(Metrics.SSL_HANDSHAKE_FAILURE_MISSING_CLIENT_CERT, 1L, MetricConsumerMock.STATIC_CONTEXT);
+        assertThat(driver.close(), is(true));
+    }
+
+    @Test
+    public void requireThatMetricIsIncrementedWhenClientUsesIncompatibleTlsVersion() throws IOException {
+        Path privateKeyFile = tmpFolder.newFile().toPath();
+        Path certificateFile = tmpFolder.newFile().toPath();
+        generatePrivateKeyAndCertificate(privateKeyFile, certificateFile);
+        var metricConsumer = new MetricConsumerMock();
+        TestDriver driver = createSslTestDriver(certificateFile, privateKeyFile, metricConsumer);
+
+        SSLContext clientCtx = new SslContextBuilder()
+                .withTrustStore(certificateFile)
+                .withKeyStore(privateKeyFile, certificateFile)
+                .build();
+
+        assertHttpsRequestTriggersSslHandshakeException(
+                driver, clientCtx, "TLSv1.3", null, "Received fatal alert: protocol_version");
+        verify(metricConsumer.mockitoMock())
+                .add(Metrics.SSL_HANDSHAKE_FAILURE_INCOMPATIBLE_PROTOCOLS, 1L, MetricConsumerMock.STATIC_CONTEXT);
+        assertThat(driver.close(), is(true));
+    }
+
+    @Test
+    public void requireThatMetricIsIncrementedWhenClientUsesIncompatibleCiphers() throws IOException {
+        Path privateKeyFile = tmpFolder.newFile().toPath();
+        Path certificateFile = tmpFolder.newFile().toPath();
+        generatePrivateKeyAndCertificate(privateKeyFile, certificateFile);
+        var metricConsumer = new MetricConsumerMock();
+        TestDriver driver = createSslTestDriver(certificateFile, privateKeyFile, metricConsumer);
+
+        SSLContext clientCtx = new SslContextBuilder()
+                .withTrustStore(certificateFile)
+                .withKeyStore(privateKeyFile, certificateFile)
+                .build();
+
+        assertHttpsRequestTriggersSslHandshakeException(
+                driver, clientCtx, null, "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA", "Received fatal alert: handshake_failure");
+        verify(metricConsumer.mockitoMock())
+                .add(Metrics.SSL_HANDSHAKE_FAILURE_INCOMPATIBLE_CIPHERS, 1L, MetricConsumerMock.STATIC_CONTEXT);
+        assertThat(driver.close(), is(true));
+    }
+
+    @Test
+    public void requireThatMetricIsIncrementedWhenClientUsesInvalidCertificateInHandshake() throws IOException {
+        Path serverPrivateKeyFile = tmpFolder.newFile().toPath();
+        Path serverCertificateFile = tmpFolder.newFile().toPath();
+        generatePrivateKeyAndCertificate(serverPrivateKeyFile, serverCertificateFile);
+        var metricConsumer = new MetricConsumerMock();
+        TestDriver driver = createSslTestDriver(serverCertificateFile, serverPrivateKeyFile, metricConsumer);
+
+        Path clientPrivateKeyFile = tmpFolder.newFile().toPath();
+        Path clientCertificateFile = tmpFolder.newFile().toPath();
+        generatePrivateKeyAndCertificate(clientPrivateKeyFile, clientCertificateFile);
+
+        SSLContext clientCtx = new SslContextBuilder()
+                .withKeyStore(clientPrivateKeyFile, clientCertificateFile)
+                .withTrustStore(serverCertificateFile)
+                .build();
+
+        assertHttpsRequestTriggersSslHandshakeException(
+                driver, clientCtx, null, null, "Received fatal alert: certificate_unknown");
+        verify(metricConsumer.mockitoMock())
+                .add(Metrics.SSL_HANDSHAKE_FAILURE_INVALID_CLIENT_CERT, 1L, MetricConsumerMock.STATIC_CONTEXT);
+        assertThat(driver.close(), is(true));
+    }
+
+    @Test
+    public void requireThatMetricIsIncrementedWhenClientUsesExpiredCertificateInHandshake() throws IOException {
+        Path rootPrivateKeyFile = tmpFolder.newFile().toPath();
+        Path rootCertificateFile = tmpFolder.newFile().toPath();
+        Path privateKeyFile = tmpFolder.newFile().toPath();
+        Path certificateFile = tmpFolder.newFile().toPath();
+        Instant notAfter = Instant.now().minus(100, ChronoUnit.DAYS);
+        generatePrivateKeyAndCertificate(rootPrivateKeyFile, rootCertificateFile, privateKeyFile, certificateFile, notAfter);
+        var metricConsumer = new MetricConsumerMock();
+        TestDriver driver = createSslTestDriver(rootCertificateFile, rootPrivateKeyFile, metricConsumer);
+
+        SSLContext clientCtx = new SslContextBuilder()
+                .withTrustStore(rootCertificateFile)
+                .withKeyStore(privateKeyFile, certificateFile)
+                .build();
+
+        assertHttpsRequestTriggersSslHandshakeException(
+                driver, clientCtx, null, null, "Received fatal alert: certificate_unknown");
+        verify(metricConsumer.mockitoMock())
+                .add(Metrics.SSL_HANDSHAKE_FAILURE_EXPIRED_CLIENT_CERT, 1L, MetricConsumerMock.STATIC_CONTEXT);
+        assertThat(driver.close(), is(true));
+    }
+
+    @Test
+    public void requireThatProxyProtocolIsAcceptedAndActualRemoteAddressStoredInAccessLog() throws Exception {
+        Path privateKeyFile = tmpFolder.newFile().toPath();
+        Path certificateFile = tmpFolder.newFile().toPath();
+        generatePrivateKeyAndCertificate(privateKeyFile, certificateFile);
+        AccessLogMock accessLogMock = new AccessLogMock();
+        TestDriver driver = createSslWithProxyProtocolTestDriver(certificateFile, privateKeyFile, accessLogMock, /*mixedMode*/false);
+        HttpClient client = createJettyHttpClient(certificateFile);
+
+        String proxiedRemoteAddress = "192.168.0.100";
+        int proxiedRemotePort = 12345;
+        sendJettyClientRequest(driver, client, new V1.Tag(proxiedRemoteAddress, proxiedRemotePort));
+        sendJettyClientRequest(driver, client, new V2.Tag(proxiedRemoteAddress, proxiedRemotePort));
+        client.stop();
+        assertThat(driver.close(), is(true));
+
+        assertThat(accessLogMock.logEntries, hasSize(2));
+        assertLogEntryHasRemote(accessLogMock.logEntries.get(0), proxiedRemoteAddress, proxiedRemotePort);
+        assertLogEntryHasRemote(accessLogMock.logEntries.get(1), proxiedRemoteAddress, proxiedRemotePort);
+    }
+
+    @Test
+    public void requireThatConnectorWithProxyProtocolMixedEnabledAcceptsBothProxyProtocolAndHttps() throws Exception {
+        Path privateKeyFile = tmpFolder.newFile().toPath();
+        Path certificateFile = tmpFolder.newFile().toPath();
+        generatePrivateKeyAndCertificate(privateKeyFile, certificateFile);
+        AccessLogMock accessLogMock = new AccessLogMock();
+        TestDriver driver = createSslWithProxyProtocolTestDriver(certificateFile, privateKeyFile, accessLogMock, /*mixedMode*/true);
+        HttpClient client = createJettyHttpClient(certificateFile);
+
+        String proxiedRemoteAddress = "192.168.0.100";
+        sendJettyClientRequest(driver, client, null);
+        sendJettyClientRequest(driver, client, new V2.Tag(proxiedRemoteAddress, 12345));
+        client.stop();
+        assertThat(driver.close(), is(true));
+
+        assertThat(accessLogMock.logEntries, hasSize(2));
+        assertLogEntryHasRemote(accessLogMock.logEntries.get(0), "127.0.0.1", 0);
+        assertLogEntryHasRemote(accessLogMock.logEntries.get(1), proxiedRemoteAddress, 0);
+    }
+
+    private void sendJettyClientRequest(TestDriver testDriver, HttpClient client, Object tag)
+            throws InterruptedException, ExecutionException, TimeoutException {
+        ContentResponse response = client.newRequest(URI.create("https://localhost:" + testDriver.server().getListenPort() + "/"))
+                .tag(tag)
+                .send();
+        assertEquals(200, response.getStatus());
+    }
+
+    // Using Jetty's http client as Apache httpclient does not support the proxy-protocol v1/v2.
+    private static HttpClient createJettyHttpClient(Path certificateFile) throws Exception {
+        SslContextFactory.Client clientSslCtxFactory = new SslContextFactory.Client();
+        clientSslCtxFactory.setHostnameVerifier(NoopHostnameVerifier.INSTANCE);
+        clientSslCtxFactory.setSslContext(new SslContextBuilder().withTrustStore(certificateFile).build());
+
+        HttpClient client = new HttpClient(clientSslCtxFactory);
+        client.start();
+        return client;
+    }
+
+    private static void assertLogEntryHasRemote(AccessLogEntry entry, String expectedAddress, int expectedPort) {
+        assertEquals(expectedAddress, entry.getPeerAddress());
+        if (expectedPort > 0) {
+            assertEquals(expectedPort, entry.getPeerPort());
+        }
+    }
+
+    private static TestDriver createSslWithProxyProtocolTestDriver(
+            Path certificateFile, Path privateKeyFile, AccessLogMock accessLogMock, boolean mixedMode) throws IOException {
+        ConnectorConfig.Builder connectorConfig = new ConnectorConfig.Builder()
+                .proxyProtocol(new ConnectorConfig.ProxyProtocol.Builder()
+                                       .enabled(true)
+                                       .mixedMode(mixedMode))
+                .ssl(new ConnectorConfig.Ssl.Builder()
+                             .enabled(true)
+                             .privateKeyFile(privateKeyFile.toString())
+                             .certificateFile(certificateFile.toString())
+                             .caCertificateFile(certificateFile.toString()));
+        return TestDrivers.newConfiguredInstance(
+                new EchoRequestHandler(),
+                new ServerConfig.Builder(),
+                connectorConfig,
+                binder -> binder.bind(AccessLog.class).toInstance(accessLogMock));
+    }
+
+    private static TestDriver createSslTestDriver(
+            Path serverCertificateFile, Path serverPrivateKeyFile, MetricConsumerMock metricConsumer) throws IOException {
+        return TestDrivers.newInstanceWithSsl(
+                new EchoRequestHandler(), serverCertificateFile, serverPrivateKeyFile, TlsClientAuth.NEED, metricConsumer.asGuiceModule());
+    }
+
+    private static void assertHttpsRequestTriggersSslHandshakeException(
+            TestDriver testDriver,
+            SSLContext sslContext,
+            String protocolOverride,
+            String cipherOverride,
+            String expectedExceptionSubstring) throws IOException {
+        List<String> protocols = protocolOverride != null ? List.of(protocolOverride) : null;
+        List<String> ciphers = cipherOverride != null ? List.of(cipherOverride) : null;
+        try (var client = new SimpleHttpClient(sslContext, protocols, ciphers, testDriver.server().getListenPort(), false)) {
+            client.get("/status.html");
+            fail("SSLHandshakeException expected");
+        } catch (SSLHandshakeException e) {
+            assertThat(e.getMessage(), containsString(expectedExceptionSubstring));
+        } catch (SSLException e) {
+            // This exception is thrown if Apache httpclient's write thread detects the handshake failure before the read thread.
+            log.log(Level.WARNING, "Client failed to get a proper TLS handshake response: " + e.getMessage(), e);
+            assertThat(e.getMessage(), containsString("readHandshakeRecord")); // Only ignore this specific ssl exception
+        }
+    }
+
     private static void generatePrivateKeyAndCertificate(Path privateKeyFile, Path certificateFile) throws IOException {
         KeyPair keyPair = KeyUtils.generateKeypair(RSA, 2048);
         Files.writeString(privateKeyFile, KeyUtils.toPem(keyPair.getPrivate()));
@@ -570,6 +814,21 @@ public class HttpServerTest {
         Files.writeString(certificateFile, X509CertificateUtils.toPem(certificate));
     }
 
+    private static void generatePrivateKeyAndCertificate(Path rootPrivateKeyFile, Path rootCertificateFile,
+                                                         Path privateKeyFile, Path certificateFile, Instant notAfter) throws IOException {
+        generatePrivateKeyAndCertificate(rootPrivateKeyFile, rootCertificateFile);
+        X509Certificate rootCertificate = X509CertificateUtils.fromPem(Files.readString(rootCertificateFile));
+        PrivateKey privateKey = KeyUtils.fromPemEncodedPrivateKey(Files.readString(rootPrivateKeyFile));
+
+        KeyPair keyPair = KeyUtils.generateKeypair(RSA, 2048);
+        Files.writeString(privateKeyFile, KeyUtils.toPem(keyPair.getPrivate()));
+        Pkcs10Csr csr = Pkcs10CsrBuilder.fromKeypair(new X500Principal("CN=myclient"), keyPair, SHA256_WITH_RSA).build();
+        X509Certificate certificate = X509CertificateBuilder
+                .fromCsr(csr, rootCertificate.getSubjectX500Principal(), Instant.EPOCH, notAfter, privateKey, SHA256_WITH_RSA, BigInteger.ONE)
+                .build();
+        Files.writeString(certificateFile, X509CertificateUtils.toPem(certificate));
+    }
+
     private static RequestHandler mockRequestHandler() {
         final RequestHandler mockRequestHandler = mock(RequestHandler.class);
         when(mockRequestHandler.refer()).thenReturn(References.NOOP_REFERENCE);
@@ -749,4 +1008,18 @@ public class HttpServerTest {
             return lhs.getName().compareTo(rhs.getName());
         }
     }
+
+    private static class MetricConsumerMock {
+        static final Metric.Context STATIC_CONTEXT = new Metric.Context() {};
+
+        private final MetricConsumer mockitoMock = mock(MetricConsumer.class);
+
+        MetricConsumerMock() {
+            when(mockitoMock.createContext(anyMap())).thenReturn(STATIC_CONTEXT);
+        }
+
+        MetricConsumer mockitoMock() { return mockitoMock; }
+        Module asGuiceModule() { return binder -> binder.bind(MetricConsumer.class).toInstance(mockitoMock); }
+    }
+
 }
diff --git a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/JDiscHttpServletTest.java b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/JDiscHttpServletTest.java
index 7a645337967..230f59cbb34 100644
--- a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/JDiscHttpServletTest.java
+++ b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/JDiscHttpServletTest.java
@@ -14,11 +14,14 @@ import org.apache.http.client.methods.HttpOptions;
 import org.apache.http.client.methods.HttpPatch;
 import org.apache.http.client.methods.HttpPost;
 import org.apache.http.client.methods.HttpPut;
+import org.apache.http.client.methods.HttpRequestBase;
 import org.apache.http.client.methods.HttpTrace;
 import org.junit.Test;
 
+import java.io.IOException;
 import java.net.URI;
 
+import static com.yahoo.jdisc.Response.Status.METHOD_NOT_ALLOWED;
 import static com.yahoo.jdisc.Response.Status.OK;
 import static org.hamcrest.CoreMatchers.is;
 import static org.hamcrest.MatcherAssert.assertThat;
@@ -51,6 +54,15 @@ public class JDiscHttpServletTest {
         assertThat(driver.close(), is(true));
     }
 
+    @Test
+    public void requireThatServerResponds405ToUnknownMethods() throws IOException {
+        TestDriver driver = TestDrivers.newInstance(newEchoHandler());
+        final URI uri = driver.client().newUri("/status.html");
+        driver.client().execute(new UnknownMethodHttpRequest(uri))
+                .expectStatusCode(is(METHOD_NOT_ALLOWED));
+        assertThat(driver.close(), is(true));
+    }
+
     private static RequestHandler newEchoHandler() {
         return new AbstractRequestHandler() {
 
@@ -60,4 +72,9 @@ public class JDiscHttpServletTest {
             }
         };
     }
+
+    private static class UnknownMethodHttpRequest extends HttpRequestBase {
+        UnknownMethodHttpRequest(URI uri) { setURI(uri); }
+        @Override public String getMethod() { return "UNKNOWN_METHOD"; }
+    }
 }
diff --git a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/SimpleHttpClient.java b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/SimpleHttpClient.java
index b0f570317d6..8035734a76c 100644
--- a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/SimpleHttpClient.java
+++ b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/SimpleHttpClient.java
@@ -1,12 +1,11 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.jdisc.http.server.jetty;
 
-import com.yahoo.jdisc.http.HttpHeaders;
 import org.apache.http.Header;
 import org.apache.http.HttpEntity;
 import org.apache.http.HttpResponse;
-import org.apache.http.client.HttpClient;
 import org.apache.http.client.entity.GzipCompressingEntity;
+import org.apache.http.client.methods.CloseableHttpResponse;
 import org.apache.http.client.methods.HttpGet;
 import org.apache.http.client.methods.HttpPost;
 import org.apache.http.client.methods.HttpUriRequest;
@@ -19,6 +18,7 @@ import org.apache.http.entity.ByteArrayEntity;
 import org.apache.http.entity.StringEntity;
 import org.apache.http.entity.mime.FormBodyPart;
 import org.apache.http.entity.mime.MultipartEntityBuilder;
+import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.http.impl.client.HttpClientBuilder;
 import org.apache.http.impl.conn.BasicHttpClientConnectionManager;
 import org.apache.http.util.EntityUtils;
@@ -26,16 +26,11 @@ import org.hamcrest.Matcher;
 import org.hamcrest.MatcherAssert;
 
 import javax.net.ssl.SSLContext;
-import java.io.ByteArrayOutputStream;
-import java.io.EOFException;
 import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStreamWriter;
-import java.net.Socket;
 import java.net.URI;
 import java.nio.charset.StandardCharsets;
 import java.util.Arrays;
-import java.util.regex.Pattern;
+import java.util.List;
 
 import static org.hamcrest.CoreMatchers.is;
 import static org.hamcrest.CoreMatchers.not;
@@ -46,14 +41,20 @@ import static org.hamcrest.MatcherAssert.assertThat;
  * A simple http client for testing
  *
  * @author Simon Thoresen Hult
+ * @author bjorncs
  */
-public class SimpleHttpClient {
+public class SimpleHttpClient implements AutoCloseable {
 
-    private final HttpClient delegate;
+    private final CloseableHttpClient delegate;
     private final String scheme;
     private final int listenPort;
 
-    public SimpleHttpClient(final SSLContext sslContext, final int listenPort, final boolean useCompression) {
+    public SimpleHttpClient(SSLContext sslContext, int listenPort, boolean useCompression) {
+        this(sslContext, null, null, listenPort, useCompression);
+    }
+
+    public SimpleHttpClient(SSLContext sslContext, List<String> enabledProtocols, List<String> enabledCiphers,
+                            int listenPort, boolean useCompression) {
         HttpClientBuilder builder = HttpClientBuilder.create();
         if (!useCompression) {
             builder.disableContentCompression();
@@ -61,6 +62,8 @@ public class SimpleHttpClient {
         if (sslContext != null) {
             SSLConnectionSocketFactory sslConnectionFactory = new SSLConnectionSocketFactory(
                     sslContext,
+                    toArray(enabledProtocols),
+                    toArray(enabledCiphers),
                     new DefaultHostnameVerifier());
             builder.setSSLSocketFactory(sslConnectionFactory);
 
@@ -76,6 +79,10 @@ public class SimpleHttpClient {
         this.listenPort = listenPort;
     }
 
+    private static String[] toArray(List<String> list) {
+        return list != null ? list.toArray(new String[0]) : null;
+    }
+
     public URI newUri(final String path) {
         return URI.create(scheme + "://localhost:" + listenPort + path);
     }
@@ -100,40 +107,9 @@ public class SimpleHttpClient {
         return newGet(path).execute();
     }
 
-    public String raw(final String request) throws IOException {
-        final Socket socket = new Socket("localhost", listenPort);
-        final OutputStreamWriter out = new OutputStreamWriter(socket.getOutputStream(), StandardCharsets.UTF_8);
-        out.write(request);
-        out.flush();
-
-        final ByteArrayOutputStream buf = new ByteArrayOutputStream();
-        final InputStream in = socket.getInputStream();
-        final int[] TERMINATOR = { '\r', '\n', '\r', '\n' };
-        for (int pos = 0; pos < TERMINATOR.length; ++pos) {
-            final int b = in.read();
-            if (b < 0) {
-                throw new EOFException();
-            }
-            if (b != TERMINATOR[pos]) {
-                pos = -1;
-            }
-            buf.write(b);
-        }
-        final String response = buf.toString(StandardCharsets.UTF_8.name());
-        final java.util.regex.Matcher matcher = Pattern.compile(HttpHeaders.Names.CONTENT_LENGTH + ": (.+)\r\n").matcher(response);
-        if (matcher.find()) {
-            final int len = Integer.valueOf(matcher.group(1));
-            for (int i = 0; i < len; ++i) {
-                final int b = in.read();
-                if (b < 0) {
-                    throw new EOFException();
-                }
-                buf.write(b);
-            }
-        }
-
-        socket.close();
-        return buf.toString(StandardCharsets.UTF_8.name());
+    @Override
+    public void close() throws IOException {
+        delegate.close();
     }
 
     public class RequestExecutor {
@@ -177,7 +153,9 @@ public class SimpleHttpClient {
             if (entity != null) {
                 ((HttpPost)request).setEntity(entity);
             }
-            return new ResponseValidator(delegate.execute(request));
+            try (CloseableHttpResponse response = delegate.execute(request)){
+                return new ResponseValidator(response);
+            }
         }
     }
 
@@ -218,9 +196,5 @@ public class SimpleHttpClient {
             return this;
         }
 
-        public ResponseValidator expectTrailer(final String trailerName, final Matcher<String> matcher) {
-            // TODO: check trailer, not header
-            return expectHeader(trailerName, matcher);
-        }
     }
 }
diff --git a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/TestDrivers.java b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/TestDrivers.java
index e0933ac485e..4908da2ba75 100644
--- a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/TestDrivers.java
+++ b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/server/jetty/TestDrivers.java
@@ -20,6 +20,7 @@ import java.nio.file.Path;
 
 /**
  * @author Simon Thoresen Hult
+ * @author bjorncs
  */
 public class TestDrivers {
 
@@ -45,9 +46,12 @@ public class TestDrivers {
                 ));
     }
 
+    public enum TlsClientAuth { NEED, WANT }
+
     public static TestDriver newInstanceWithSsl(final RequestHandler requestHandler,
                                                 Path certificateFile,
                                                 Path privateKeyFile,
+                                                TlsClientAuth tlsClientAuth,
                                                 final Module... guiceModules) throws IOException {
         return TestDriver.newInstance(
                 JettyHttpServer.class,
@@ -61,7 +65,9 @@ public class TestDrivers {
                                                 .pathWhitelist("/status.html"))
                                 .ssl(new ConnectorConfig.Ssl.Builder()
                                              .enabled(true)
-                                             .clientAuth(ConnectorConfig.Ssl.ClientAuth.Enum.WANT_AUTH)
+                                             .clientAuth(tlsClientAuth == TlsClientAuth.NEED
+                                                                 ? ConnectorConfig.Ssl.ClientAuth.Enum.NEED_AUTH
+                                                                 : ConnectorConfig.Ssl.ClientAuth.Enum.WANT_AUTH)
                                              .privateKeyFile(privateKeyFile.toString())
                                              .certificateFile(certificateFile.toString())
                                              .caCertificateFile(certificateFile.toString())),
diff --git a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/ssl/impl/TlsContextBasedProviderTest.java b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/ssl/impl/TlsContextBasedProviderTest.java
index 88db5c99de9..eb292199ea2 100644
--- a/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/ssl/impl/TlsContextBasedProviderTest.java
+++ b/jdisc_http_service/src/test/java/com/yahoo/jdisc/http/ssl/impl/TlsContextBasedProviderTest.java
@@ -5,6 +5,7 @@ import com.yahoo.security.KeyUtils;
 import com.yahoo.security.X509CertificateBuilder;
 import com.yahoo.security.tls.AuthorizationMode;
 import com.yahoo.security.tls.DefaultTlsContext;
+import com.yahoo.security.tls.HostnameVerification;
 import com.yahoo.security.tls.PeerAuthentication;
 import com.yahoo.security.tls.TlsContext;
 import com.yahoo.security.tls.policy.AuthorizedPeers;
@@ -51,7 +52,7 @@ public class TlsContextBasedProviderTest {
                         BigInteger.ONE)
                 .build();
         return new DefaultTlsContext(
-                List.of(certificate), keyPair.getPrivate(), List.of(certificate), new AuthorizedPeers(Set.of()), AuthorizationMode.ENFORCE, PeerAuthentication.NEED);
+                List.of(certificate), keyPair.getPrivate(), List.of(certificate), new AuthorizedPeers(Set.of()), AuthorizationMode.ENFORCE, PeerAuthentication.NEED, HostnameVerification.ENABLED);
     }
 
     private static class SimpleTlsContextBasedProvider extends TlsContextBasedProvider {
diff --git a/jrt/src/com/yahoo/jrt/Acceptor.java b/jrt/src/com/yahoo/jrt/Acceptor.java
index 9e9dafcbcb5..aed22ac090c 100644
--- a/jrt/src/com/yahoo/jrt/Acceptor.java
+++ b/jrt/src/com/yahoo/jrt/Acceptor.java
@@ -101,7 +101,7 @@ public class Acceptor {
         while (serverChannel.isOpen()) {
             try {
                 TransportThread tt = parent.selectThread();
-                tt.addConnection(new Connection(tt, owner, serverChannel.accept()));
+                tt.addConnection(new Connection(tt, owner, serverChannel.accept(), parent.getTcpNoDelay()));
                 tt.sync();
             } catch (ClosedChannelException ignore) {
             } catch (Exception e) {
diff --git a/jrt/src/com/yahoo/jrt/Connection.java b/jrt/src/com/yahoo/jrt/Connection.java
index d4e1a15b957..5a4478cf91e 100644
--- a/jrt/src/com/yahoo/jrt/Connection.java
+++ b/jrt/src/com/yahoo/jrt/Connection.java
@@ -36,6 +36,7 @@ class Connection extends Target {
     private final Buffer output  = new Buffer(WRITE_SIZE * 2);
     private int maxInputSize  = 64*1024;
     private int maxOutputSize = 64*1024;
+    private final boolean tcpNoDelay;
     private final Map<Integer, ReplyHandler> replyMap = new HashMap<>();
     private final Map<TargetWatcher, TargetWatcher> watchers = new IdentityHashMap<>();
     private int activeReqs = 0;
@@ -89,21 +90,23 @@ class Connection extends Target {
     }
 
     public Connection(TransportThread parent, Supervisor owner,
-                      SocketChannel channel) {
+                      SocketChannel channel, boolean tcpNoDelay) {
 
         this.parent = parent;
         this.owner = owner;
-        this.socket = parent.transport().createCryptoSocket(channel, true);
+        this.socket = parent.transport().createServerCryptoSocket(channel);
         this.spec = null;
+        this.tcpNoDelay = tcpNoDelay;
         server = true;
         owner.sessionInit(this);
     }
 
-    public Connection(TransportThread parent, Supervisor owner, Spec spec, Object context) {
+    public Connection(TransportThread parent, Supervisor owner, Spec spec, Object context, boolean tcpNoDelay) {
         super(context);
         this.parent = parent;
         this.owner = owner;
         this.spec = spec;
+        this.tcpNoDelay = tcpNoDelay;
         server = false;
         owner.sessionInit(this);
     }
@@ -171,7 +174,7 @@ class Connection extends Target {
             return this;
         }
         try {
-            socket = parent.transport().createCryptoSocket(SocketChannel.open(spec.resolveAddress()), false);
+            socket = parent.transport().createClientCryptoSocket(SocketChannel.open(spec.resolveAddress()), spec);
         } catch (Exception e) {
             setLostReason(e);
         }
@@ -184,7 +187,7 @@ class Connection extends Target {
         }
         try {
             socket.channel().configureBlocking(false);
-            socket.channel().socket().setTcpNoDelay(true);
+            socket.channel().socket().setTcpNoDelay(tcpNoDelay);
             selectionKey = socket.channel().register(selector,
                     SelectionKey.OP_READ | SelectionKey.OP_WRITE,
                     this);
diff --git a/jrt/src/com/yahoo/jrt/Connector.java b/jrt/src/com/yahoo/jrt/Connector.java
index 4c83a2884bd..10f2b3742f2 100644
--- a/jrt/src/com/yahoo/jrt/Connector.java
+++ b/jrt/src/com/yahoo/jrt/Connector.java
@@ -1,76 +1,67 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.jrt;
 
+import com.yahoo.concurrent.CachedThreadPoolWithFallback;
 
-class Connector {
+import java.util.concurrent.Executor;
+import java.util.concurrent.RejectedExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
 
-    private class Run implements Runnable {
-        public void run() {
-            try {
-                Connector.this.run();
-            } catch (Throwable problem) {
-                parent.handleFailure(problem, Connector.this);
-            }
-        }
-    }
+class Connector implements AutoCloseable {
 
-    private Thread      thread = new Thread(new Run(), "<jrt-connector>");
-    private Transport   parent;
-    private ThreadQueue connectQueue = new ThreadQueue();
-    private boolean     done = false;
-    private boolean     exit = false;
+    private static final Object globalLock = new Object();
+    private static CachedThreadPoolWithFallback globalExecutor = null;
+    private static long usages = 0;
 
-    public Connector(Transport parent) {
-        this.parent = parent;
-        thread.setDaemon(true);
-        thread.start();
-    }
-
-    public void connectLater(Connection c) {
-        if ( ! connectQueue.enqueue(c)) {
-            c.transportThread().addConnection(c);
+    private static CachedThreadPoolWithFallback acquire() {
+        synchronized (globalLock) {
+            if (globalExecutor == null) {
+                globalExecutor = new CachedThreadPoolWithFallback("jrt.connector", 1, 64, 1L, TimeUnit.SECONDS);
+            }
+            usages++;
+            return globalExecutor;
         }
     }
 
-    private void run() {
-        try {
-            while (true) {
-                Connection conn = (Connection) connectQueue.dequeue();
-                conn.transportThread().addConnection(conn.connect());
-            }
-        } catch (EndOfQueueException e) {}
-        synchronized (this) {
-            done = true;
-            notifyAll();
-            while (!exit) {
-                try { wait(); } catch (InterruptedException x) {}
+    private static void release(CachedThreadPoolWithFallback executor) {
+        synchronized (globalLock) {
+            assert executor == globalExecutor;
+            usages--;
+            if (usages == 0) {
+                globalExecutor.close();
+                globalExecutor = null;
             }
         }
     }
 
-    public Connector shutdown() {
-        connectQueue.close();
-        return this;
-    }
+    private final AtomicReference<CachedThreadPoolWithFallback> executor;
 
-    public synchronized void waitDone() {
-        while (!done) {
-            try { wait(); } catch (InterruptedException x) {}
-        }
+    Connector() {
+        executor = new AtomicReference<>(acquire());
     }
 
-    public synchronized Connector exit() {
-        exit = true;
-        notifyAll();
-        return this;
+    private void connect(Connection conn) {
+        conn.transportThread().addConnection(conn.connect());
     }
 
-    public void join() {
-        while (true) {
+    public void connectLater(Connection conn) {
+        Executor executor = this.executor.get();
+        if (executor != null) {
             try {
-                thread.join();
+                executor.execute(() -> connect(conn));
                 return;
-            } catch (InterruptedException e) {}
+            } catch (RejectedExecutionException ignored) {
+            }
+        }
+        conn.transportThread().addConnection(conn);
+    }
+
+    @Override
+    public void close() {
+        CachedThreadPoolWithFallback toShutdown = executor.getAndSet(null);
+        if (toShutdown != null) {
+            release(toShutdown);
         }
     }
 }
diff --git a/jrt/src/com/yahoo/jrt/CryptoEngine.java b/jrt/src/com/yahoo/jrt/CryptoEngine.java
index 8812264a3f1..6d1955d7f66 100644
--- a/jrt/src/com/yahoo/jrt/CryptoEngine.java
+++ b/jrt/src/com/yahoo/jrt/CryptoEngine.java
@@ -18,7 +18,8 @@ import java.nio.channels.SocketChannel;
  * encryption.
  **/
 public interface CryptoEngine extends AutoCloseable {
-    CryptoSocket createCryptoSocket(SocketChannel channel, boolean isServer);
+    CryptoSocket createClientCryptoSocket(SocketChannel channel, Spec spec);
+    CryptoSocket createServerCryptoSocket(SocketChannel channel);
     static CryptoEngine createDefault() {
         if (!TransportSecurityUtils.isTransportSecurityEnabled()) {
             return new NullCryptoEngine();
diff --git a/jrt/src/com/yahoo/jrt/MaybeTlsCryptoEngine.java b/jrt/src/com/yahoo/jrt/MaybeTlsCryptoEngine.java
index 801f2075c4e..18549df6f2c 100644
--- a/jrt/src/com/yahoo/jrt/MaybeTlsCryptoEngine.java
+++ b/jrt/src/com/yahoo/jrt/MaybeTlsCryptoEngine.java
@@ -21,17 +21,20 @@ public class MaybeTlsCryptoEngine implements CryptoEngine {
     }
 
     @Override
-    public CryptoSocket createCryptoSocket(SocketChannel channel, boolean isServer) {
-        if (isServer) {
-            return new MaybeTlsCryptoSocket(channel, tlsEngine, isServer);
-        } else if (useTlsWhenClient) {
-            return tlsEngine.createCryptoSocket(channel, false);
+    public CryptoSocket createClientCryptoSocket(SocketChannel channel, Spec spec) {
+        if (useTlsWhenClient) {
+            return tlsEngine.createClientCryptoSocket(channel, spec);
         } else {
-            return new NullCryptoSocket(channel, isServer);
+            return new NullCryptoSocket(channel, false);
         }
     }
 
     @Override
+    public CryptoSocket createServerCryptoSocket(SocketChannel channel) {
+        return new MaybeTlsCryptoSocket(channel, tlsEngine);
+    }
+
+    @Override
     public String toString() { return "MaybeTlsCryptoEngine(useTlsWhenClient:" + useTlsWhenClient + ")"; }
 
     @Override
diff --git a/jrt/src/com/yahoo/jrt/MaybeTlsCryptoSocket.java b/jrt/src/com/yahoo/jrt/MaybeTlsCryptoSocket.java
index 5c4510665e7..60b7f342c9c 100644
--- a/jrt/src/com/yahoo/jrt/MaybeTlsCryptoSocket.java
+++ b/jrt/src/com/yahoo/jrt/MaybeTlsCryptoSocket.java
@@ -61,8 +61,8 @@ public class MaybeTlsCryptoSocket implements CryptoSocket {
         private TlsCryptoEngine factory;
         private Buffer buffer;
 
-        MyCryptoSocket(SocketChannel channel, TlsCryptoEngine factory, boolean isServer) {
-            super(channel, isServer);
+        MyCryptoSocket(SocketChannel channel, TlsCryptoEngine factory) {
+            super(channel, true);
             this.factory = factory;
             this.buffer = new Buffer(4096);
         }
@@ -81,7 +81,7 @@ public class MaybeTlsCryptoSocket implements CryptoSocket {
                     data[i] = src.get(i);
                 }
                 if (looksLikeTlsToMe(data)) {
-                    TlsCryptoSocket tlsSocket = factory.createCryptoSocket(channel(), true);
+                    TlsCryptoSocket tlsSocket = factory.createServerCryptoSocket(channel());
                     tlsSocket.injectReadData(buffer);
                     socket = tlsSocket;
                     return socket.handshake();
@@ -117,8 +117,8 @@ public class MaybeTlsCryptoSocket implements CryptoSocket {
         }
     }
 
-    public MaybeTlsCryptoSocket(SocketChannel channel, TlsCryptoEngine factory, boolean isServer) {
-        this.socket = new MyCryptoSocket(channel, factory, isServer);
+    public MaybeTlsCryptoSocket(SocketChannel channel, TlsCryptoEngine factory) {
+        this.socket = new MyCryptoSocket(channel, factory);
     }
 
     @Override public SocketChannel channel() { return socket.channel(); }
diff --git a/jrt/src/com/yahoo/jrt/NullCryptoEngine.java b/jrt/src/com/yahoo/jrt/NullCryptoEngine.java
index b5a53accf92..b97ec17a5dc 100644
--- a/jrt/src/com/yahoo/jrt/NullCryptoEngine.java
+++ b/jrt/src/com/yahoo/jrt/NullCryptoEngine.java
@@ -9,7 +9,10 @@ import java.nio.channels.SocketChannel;
  * CryptoEngine implementation that performs no encryption.
  **/
 public class NullCryptoEngine implements CryptoEngine {
-    @Override public CryptoSocket createCryptoSocket(SocketChannel channel, boolean isServer) {
-        return new NullCryptoSocket(channel, isServer);
+    @Override public CryptoSocket createClientCryptoSocket(SocketChannel channel, Spec spec) {
+        return new NullCryptoSocket(channel, false);
+    }
+    @Override public CryptoSocket createServerCryptoSocket(SocketChannel channel) {
+        return new NullCryptoSocket(channel, true);
     }
 }
diff --git a/jrt/src/com/yahoo/jrt/Supervisor.java b/jrt/src/com/yahoo/jrt/Supervisor.java
index 09360c2da7b..d4168e97743 100644
--- a/jrt/src/com/yahoo/jrt/Supervisor.java
+++ b/jrt/src/com/yahoo/jrt/Supervisor.java
@@ -1,7 +1,6 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.jrt;
 
-
 import java.util.HashMap;
 import java.util.concurrent.atomic.AtomicReference;
 
@@ -103,19 +102,6 @@ public class Supervisor {
     }
 
     /**
-     * Remove a method from the set of methods held by this Supervisor
-     *
-     * @param methodName name of the method to remove
-     **/
-    public void removeMethod(String methodName) {
-        synchronized (methodMapLock) {
-            HashMap<String, Method> newMap = new HashMap<>(methodMap());
-            newMap.remove(methodName);
-            methodMap.setRelease(newMap);
-        }
-    }
-
-    /**
      * Remove a method from the set of methods held by this
      * Supervisor. Use this if you know exactly which method to remove
      * and not only the name.
diff --git a/jrt/src/com/yahoo/jrt/TlsCryptoEngine.java b/jrt/src/com/yahoo/jrt/TlsCryptoEngine.java
index 84fbb7d4f01..a140e87713c 100644
--- a/jrt/src/com/yahoo/jrt/TlsCryptoEngine.java
+++ b/jrt/src/com/yahoo/jrt/TlsCryptoEngine.java
@@ -20,9 +20,17 @@ public class TlsCryptoEngine implements CryptoEngine {
     }
 
     @Override
-    public TlsCryptoSocket createCryptoSocket(SocketChannel channel, boolean isServer)  {
+    public TlsCryptoSocket createClientCryptoSocket(SocketChannel channel, Spec spec)  {
+        String peerHost = spec.host() != null ? spec.host() : "localhost"; // Use localhost for wildcard address
+        SSLEngine sslEngine = tlsContext.createSslEngine(peerHost, spec.port());
+        sslEngine.setUseClientMode(true);
+        return new TlsCryptoSocket(channel, sslEngine);
+    }
+
+    @Override
+    public TlsCryptoSocket createServerCryptoSocket(SocketChannel channel)  {
         SSLEngine sslEngine = tlsContext.createSslEngine();
-        sslEngine.setUseClientMode(!isServer);
+        sslEngine.setUseClientMode(false);
         return new TlsCryptoSocket(channel, sslEngine);
     }
 
diff --git a/jrt/src/com/yahoo/jrt/Transport.java b/jrt/src/com/yahoo/jrt/Transport.java
index f4eb1acd096..02a6e3e05f7 100644
--- a/jrt/src/com/yahoo/jrt/Transport.java
+++ b/jrt/src/com/yahoo/jrt/Transport.java
@@ -4,7 +4,6 @@ package com.yahoo.jrt;
 
 import java.nio.channels.SocketChannel;
 import java.util.ArrayList;
-import java.util.Iterator;
 import java.util.Random;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.logging.Level;
@@ -19,16 +18,17 @@ import java.util.logging.Logger;
  **/
 public class Transport {
 
-    private static Logger log = Logger.getLogger(Transport.class.getName());
+    private static final Logger log = Logger.getLogger(Transport.class.getName());
 
     private final FatalErrorHandler fatalHandler; // NB: this must be set first
     private final CryptoEngine      cryptoEngine;
     private final Connector         connector;
     private final Worker            worker;
     private final AtomicInteger     runCnt;
+    private final boolean tcpNoDelay;
 
     private final TransportMetrics metrics = TransportMetrics.getInstance();
-    private final ArrayList<TransportThread> threads = new ArrayList<TransportThread>();
+    private final ArrayList<TransportThread> threads = new ArrayList<>();
     private final Random rnd = new Random();
 
     /**
@@ -41,22 +41,21 @@ public class Transport {
      * @param cryptoEngine crypto engine to use
      * @param numThreads number of {@link TransportThread}s.
      **/
-    public Transport(FatalErrorHandler fatalHandler, CryptoEngine cryptoEngine, int numThreads) {
-        synchronized (this) {
-            this.fatalHandler = fatalHandler; // NB: this must be set first
-        }
+    public Transport(FatalErrorHandler fatalHandler, CryptoEngine cryptoEngine, int numThreads, boolean tcpNoDelay) {
+        this.fatalHandler = fatalHandler; // NB: this must be set first
         this.cryptoEngine = cryptoEngine;
-        connector = new Connector(this);
+        this.tcpNoDelay = tcpNoDelay;
+        connector = new Connector();
         worker = new Worker(this);
         runCnt = new AtomicInteger(numThreads);
         for (int i = 0; i < numThreads; ++i) {
             threads.add(new TransportThread(this));
         }
     }
-    public Transport(CryptoEngine cryptoEngine, int numThreads) { this(null, cryptoEngine, numThreads); }
-    public Transport(FatalErrorHandler fatalHandler, int numThreads) { this(fatalHandler, CryptoEngine.createDefault(), numThreads); }
-    public Transport(int numThreads) { this(null, CryptoEngine.createDefault(), numThreads); }
-    public Transport() { this(null, CryptoEngine.createDefault(), 1); }
+    public Transport(CryptoEngine cryptoEngine, int numThreads) { this(null, cryptoEngine, numThreads, true); }
+    public Transport(int numThreads) { this(null, CryptoEngine.createDefault(), numThreads, true); }
+    public Transport(int numThreads, boolean tcpNoDelay) { this(null, CryptoEngine.createDefault(), numThreads, tcpNoDelay); }
+    public Transport() { this(null, CryptoEngine.createDefault(), 1, true); }
 
     /**
      * Select a random transport thread
@@ -67,15 +66,29 @@ public class Transport {
         return threads.get(rnd.nextInt(threads.size()));
     }
 
+    boolean getTcpNoDelay() { return tcpNoDelay; }
+
+    /**
+     * Use the underlying CryptoEngine to create a CryptoSocket for
+     * the client side of a connection.
+     *
+     * @return CryptoSocket handling appropriate encryption
+     * @param channel low-level socket channel to be wrapped by the CryptoSocket
+     * @param spec who we are connecting to, for hostname validation
+     **/
+    CryptoSocket createClientCryptoSocket(SocketChannel channel, Spec spec) {
+        return cryptoEngine.createClientCryptoSocket(channel, spec);
+    }
+
     /**
-     * Use the underlying CryptoEngine to create a CryptoSocket.
+     * Use the underlying CryptoEngine to create a CryptoSocket for
+     * the server side of a connection.
      *
      * @return CryptoSocket handling appropriate encryption
      * @param channel low-level socket channel to be wrapped by the CryptoSocket
-     * @param isServer flag indicating which end of the connection we are
      **/
-    CryptoSocket createCryptoSocket(SocketChannel channel, boolean isServer) {
-        return cryptoEngine.createCryptoSocket(channel, isServer);
+    CryptoSocket createServerCryptoSocket(SocketChannel channel) {
+        return cryptoEngine.createServerCryptoSocket(channel);
     }
 
     /**
@@ -119,7 +132,7 @@ public class Transport {
      * @param context application context for the new connection
      */
     Connection connect(Supervisor owner, Spec spec, Object context) {
-        Connection conn = new Connection(selectThread(), owner, spec, context);
+        Connection conn = new Connection(selectThread(), owner, spec, context, getTcpNoDelay());
         connector.connectLater(conn);
         return conn;
     }
@@ -162,7 +175,7 @@ public class Transport {
      * @return this object, to enable chaining with join
      **/
     public Transport shutdown() {
-        connector.shutdown().waitDone();
+        connector.close();
         for (TransportThread thread: threads) {
             thread.shutdown();
         }
@@ -181,7 +194,6 @@ public class Transport {
     void notifyDone(TransportThread self) {
         if (runCnt.decrementAndGet() == 0) {
             worker.shutdown().join();
-            connector.exit().join();
             try { cryptoEngine.close(); } catch (Exception e) {}
         }
     }
diff --git a/jrt/src/com/yahoo/jrt/XorCryptoEngine.java b/jrt/src/com/yahoo/jrt/XorCryptoEngine.java
index 4ba6d00faa4..d720ca4dc26 100644
--- a/jrt/src/com/yahoo/jrt/XorCryptoEngine.java
+++ b/jrt/src/com/yahoo/jrt/XorCryptoEngine.java
@@ -11,7 +11,10 @@ import java.nio.channels.SocketChannel;
  * from TLS.
  **/
 public class XorCryptoEngine implements CryptoEngine {
-    @Override public CryptoSocket createCryptoSocket(SocketChannel channel, boolean isServer) {
-        return new XorCryptoSocket(channel, isServer);
+    @Override public CryptoSocket createClientCryptoSocket(SocketChannel channel, Spec spec) {
+        return new XorCryptoSocket(channel, false);
+    }
+    @Override public CryptoSocket createServerCryptoSocket(SocketChannel channel) {
+        return new XorCryptoSocket(channel, true);
     }
 }
diff --git a/jrt/tests/com/yahoo/jrt/CryptoUtils.java b/jrt/tests/com/yahoo/jrt/CryptoUtils.java
index e7e4eea568d..95ea581cb90 100644
--- a/jrt/tests/com/yahoo/jrt/CryptoUtils.java
+++ b/jrt/tests/com/yahoo/jrt/CryptoUtils.java
@@ -5,6 +5,7 @@ import com.yahoo.security.KeyUtils;
 import com.yahoo.security.X509CertificateBuilder;
 import com.yahoo.security.tls.AuthorizationMode;
 import com.yahoo.security.tls.DefaultTlsContext;
+import com.yahoo.security.tls.HostnameVerification;
 import com.yahoo.security.tls.PeerAuthentication;
 import com.yahoo.security.tls.TlsContext;
 import com.yahoo.security.tls.policy.AuthorizedPeers;
@@ -35,21 +36,23 @@ class CryptoUtils {
     static final KeyPair keyPair = KeyUtils.generateKeypair(EC);
 
     static final X509Certificate certificate = X509CertificateBuilder
-            .fromKeypair(keyPair, new X500Principal("CN=dummy"), EPOCH, Instant.now().plus(1, DAYS), SHA256_WITH_ECDSA, generateRandomSerialNumber())
+            .fromKeypair(keyPair, new X500Principal("CN=localhost"), EPOCH, Instant.now().plus(1, DAYS), SHA256_WITH_ECDSA, generateRandomSerialNumber())
             .build();
 
     static final AuthorizedPeers authorizedPeers = new AuthorizedPeers(
             singleton(
                     new PeerPolicy(
-                            "dummy-policy",
+                            "localhost-policy",
                             singleton(
-                                    new Role("dummy-role")),
+                                    new Role("localhost-role")),
                             singletonList(
                                     new RequiredPeerCredential(
-                                            Field.CN, new HostGlobPattern("dummy"))))));
+                                            Field.CN, new HostGlobPattern("localhost"))))));
 
     static TlsContext createTestTlsContext() {
-        return new DefaultTlsContext(singletonList(certificate), keyPair.getPrivate(), singletonList(certificate), authorizedPeers, AuthorizationMode.ENFORCE, PeerAuthentication.NEED);
+        return new DefaultTlsContext(
+                singletonList(certificate), keyPair.getPrivate(), singletonList(certificate), authorizedPeers,
+                AuthorizationMode.ENFORCE, PeerAuthentication.NEED, HostnameVerification.ENABLED);
     }
 
 }
diff --git a/juniper/src/vespa/juniper/expcache.h b/juniper/src/vespa/juniper/expcache.h
index 6939785914a..beaa491148c 100644
--- a/juniper/src/vespa/juniper/expcache.h
+++ b/juniper/src/vespa/juniper/expcache.h
@@ -2,6 +2,7 @@
 #pragma once
 
 #include "simplemap.h"
+#include <cstdint>
 
 class MatchObject;
 
diff --git a/logd/src/logd/rpc_forwarder.cpp b/logd/src/logd/rpc_forwarder.cpp
index cf774504bf3..ffc43bce0bb 100644
--- a/logd/src/logd/rpc_forwarder.cpp
+++ b/logd/src/logd/rpc_forwarder.cpp
@@ -57,19 +57,15 @@ RpcForwarder::RpcForwarder(Metrics& metrics, const ForwardMap& forward_filter, F
       _connection_spec(make_string("tcp/%s:%d", hostname.c_str(), rpc_port)),
       _rpc_timeout_secs(rpc_timeout_secs),
       _max_messages_per_request(max_messages_per_request),
-      _target(),
+      _target(supervisor.GetTarget(_connection_spec.c_str())),
       _messages(),
       _bad_lines(0),
       _forward_filter(forward_filter)
 {
-    _target = supervisor.GetTarget(_connection_spec.c_str());
     ping_logserver();
 }
 
-RpcForwarder::~RpcForwarder()
-{
-    _target->SubRef();
-}
+RpcForwarder::~RpcForwarder() = default;
 
 namespace {
 
diff --git a/logd/src/logd/rpc_forwarder.h b/logd/src/logd/rpc_forwarder.h
index 404041156bf..37729db088f 100644
--- a/logd/src/logd/rpc_forwarder.h
+++ b/logd/src/logd/rpc_forwarder.h
@@ -6,12 +6,20 @@
 #include "proto_converter.h"
 #include <vespa/log/log_message.h>
 #include <vespa/fnet/frt/frt.h>
+#include <memory>
 #include <vector>
 
 namespace logdemon {
 
 struct Metrics;
 
+struct RpcTargetSubRef {
+    void operator()(FRT_Target* target) const noexcept {
+        target->SubRef();
+    }
+};
+using RpcTargetGuard = std::unique_ptr<FRT_Target, RpcTargetSubRef>;
+
 /**
  * Implementation of the Forwarder interface that uses RPC to send protobuf encoded log messages to the logserver.
  */
@@ -21,7 +29,7 @@ private:
     vespalib::string _connection_spec;
     double _rpc_timeout_secs;
     size_t _max_messages_per_request;
-    FRT_Target* _target;
+    RpcTargetGuard _target;
     std::vector<ns_log::LogMessage> _messages;
     int _bad_lines;
     ForwardMap _forward_filter;
diff --git a/logd/src/logd/watcher.cpp b/logd/src/logd/watcher.cpp
index e481d64f721..a91de132061 100644
--- a/logd/src/logd/watcher.cpp
+++ b/logd/src/logd/watcher.cpp
@@ -116,6 +116,8 @@ void
 Watcher::watchfile()
 {
     struct donecache already;
+    char newfn[FILENAME_MAX];
+    int spamfill_counter = 0;
 
     char *target = getenv("VESPA_LOG_TARGET");
     if (target == nullptr || strncmp(target, "file:", 5) != 0) {
@@ -214,17 +216,30 @@ Watcher::watchfile()
 
         if (rotate) {
             vespalib::duration rotTime = rotTimer.elapsed();
-            if (rotTime > 59s || (sb.st_size == offset && rotTime > 4s)) {
-                removeOldLogs(filename);
+            off_t overflow_size = (1.1 * _confsubscriber.getRotateSize());
+            if ((rotTime > 59s) ||
+                (sb.st_size == offset && rotTime > 4s) ||
+                (sb.st_size > overflow_size && rotTime > 2s))
+            {
                 if (sb.st_size != offset) {
                     LOG(warning, "logfile rotation incomplete after %2.3f s (dropping %" PRIu64 " bytes)",
                         vespalib::to_s(rotTime), static_cast<uint64_t>(sb.st_size - offset));
                 } else {
                     LOG(debug, "logfile rotation complete after %2.3f s", vespalib::to_s(rotTime));
                 }
+                if (((now - created) < (rotTime + 180s)) && (sb.st_size > overflow_size)) {
+                    ++spamfill_counter;
+                } else {
+                    spamfill_counter = 0;
+                }
                 created = now;
                 rotate = false;
                 close(_wfd);
+                if (spamfill_counter > 2) {
+                    LOG(warning, "logfile spamming %d times, aggressively removing %s", spamfill_counter, newfn);
+                    unlink(newfn);
+                }
+                removeOldLogs(filename);
                 goto again;
             }
         } else if (stat(filename, &sb) != 0
@@ -240,7 +255,6 @@ Watcher::watchfile()
             rotTimer = vespalib::Timer();
             LOG(debug, "preparing to rotate logfile, old logfile size %d, age %2.3f seconds",
                 (int)offset, vespalib::to_s(now-created));
-            char newfn[FILENAME_MAX];
             int l = strlen(filename);
             strcpy(newfn, filename);
             time_t seconds = vespalib::count_s(now.time_since_epoch());
diff --git a/logforwarder/src/apps/vespa-logforwarder-start/cf-handler.cpp b/logforwarder/src/apps/vespa-logforwarder-start/cf-handler.cpp
index addf9125508..e37a6fb2dcb 100644
--- a/logforwarder/src/apps/vespa-logforwarder-start/cf-handler.cpp
+++ b/logforwarder/src/apps/vespa-logforwarder-start/cf-handler.cpp
@@ -46,10 +46,11 @@ bool fixDir(const vespalib::string &path) {
 }
 
 vespalib::string
-cfFilePath(const vespalib::string &parent) {
+cfFilePath(const vespalib::string &parent, const vespalib::string &filename) {
     vespalib::string path = parent + "/etc/system/local";
     fixDir(path);
-    path += "/deploymentclient.conf";
+    path += "/";
+    path += filename;
     return path;
 }
 
@@ -61,7 +62,7 @@ CfHandler::doConfigure()
     std::unique_ptr<LogforwarderConfig> cfg(_handle->getConfig());
     const LogforwarderConfig& config(*cfg);
 
-    vespalib::string path = cfFilePath(config.splunkHome);
+    vespalib::string path = cfFilePath(config.splunkHome, "deploymentclient.conf");
     vespalib::string tmpPath = path + ".new";
     FILE *fp = fopen(tmpPath.c_str(), "w");
     if (fp == NULL) return;
@@ -76,6 +77,24 @@ CfHandler::doConfigure()
     fclose(fp);
     rename(tmpPath.c_str(), path.c_str());
 
+    if (getenv("VESPA_HOSTNAME") != NULL &&
+        getenv("VESPA_TENANT") != NULL &&
+        getenv("VESPA_APPLICATION")!= NULL &&
+        getenv("VESPA_INSTANCE") != NULL &&
+        getenv("VESPA_ENVIRONMENT") != NULL &&
+        getenv("VESPA_REGION") != NULL)
+        {
+        path = cfFilePath(config.splunkHome, "inputs.conf");
+        tmpPath = path + ".new";
+        fp = fopen(tmpPath.c_str(), "w");
+        if (fp != NULL) {
+            fprintf(fp, "[default]\n");
+            fprintf(fp, "host = %s\n", getenv("VESPA_HOSTNAME"));
+            fprintf(fp, "_meta = vespa_tenant::%s vespa_app::%s.%s vespa_zone::%s.%s\n", getenv("VESPA_TENANT"), getenv("VESPA_APPLICATION"), getenv("VESPA_INSTANCE"), getenv("VESPA_ENVIRONMENT"), getenv("VESPA_REGION"));
+            fclose(fp);
+            rename(tmpPath.c_str(), path.c_str());
+        }
+    }
     if (config.clientName.size() == 0 ||
         config.deploymentServer.size() == 0)
     {
diff --git a/logserver/bin/logserver-start.sh b/logserver/bin/logserver-start.sh
index 9f55e218140..fe4b5ebbb64 100755
--- a/logserver/bin/logserver-start.sh
+++ b/logserver/bin/logserver-start.sh
@@ -78,7 +78,7 @@ ROOT=${VESPA_HOME%/}
 export ROOT
 cd $ROOT || { echo "Cannot cd to $ROOT" 1>&2; exit 1; }
 
-addopts="-server -Xms32m -Xmx256m -XX:MaxDirectMemorySize=76m -XX:MaxJavaStackTraceDepth=1000000"
+addopts="-server -Xms32m -Xmx256m -XX:CompressedClassSpaceSize=32m -XX:MaxDirectMemorySize=32m -XX:ThreadStackSize=256 -XX:MaxJavaStackTraceDepth=1000 -XX:ActiveProcessorCount=2 -Djava.io.tmpdir=${VESPA_HOME}/tmp"
 
 oomopt="-XX:+ExitOnOutOfMemoryError"
 
diff --git a/messagebus/abi-spec.json b/messagebus/abi-spec.json
index 9c445aea6fc..159c82971cd 100644
--- a/messagebus/abi-spec.json
+++ b/messagebus/abi-spec.json
@@ -120,6 +120,7 @@
       "public void processReply(com.yahoo.messagebus.Reply)",
       "public com.yahoo.messagebus.DynamicThrottlePolicy setEfficiencyThreshold(double)",
       "public com.yahoo.messagebus.DynamicThrottlePolicy setWindowSizeIncrement(double)",
+      "public com.yahoo.messagebus.DynamicThrottlePolicy setWindowSizeDecrementFactor(double)",
       "public com.yahoo.messagebus.DynamicThrottlePolicy setWindowSizeBackOff(double)",
       "public com.yahoo.messagebus.DynamicThrottlePolicy setResizeRate(double)",
       "public com.yahoo.messagebus.DynamicThrottlePolicy setWeight(double)",
diff --git a/messagebus/src/main/java/com/yahoo/messagebus/DynamicThrottlePolicy.java b/messagebus/src/main/java/com/yahoo/messagebus/DynamicThrottlePolicy.java
index 525d7ae8867..76d3bc07d2b 100644
--- a/messagebus/src/main/java/com/yahoo/messagebus/DynamicThrottlePolicy.java
+++ b/messagebus/src/main/java/com/yahoo/messagebus/DynamicThrottlePolicy.java
@@ -27,6 +27,7 @@ public class DynamicThrottlePolicy extends StaticThrottlePolicy {
     private double windowSizeIncrement = 20;
     private double windowSize = windowSizeIncrement;
     private double minWindowSize = windowSizeIncrement;
+    private double decrementFactor = 2.0;
     private double maxWindowSize = Integer.MAX_VALUE;
     private double windowSizeBackOff = 0.9;
     private double weight = 1.0;
@@ -93,15 +94,15 @@ public class DynamicThrottlePolicy extends StaticThrottlePolicy {
         numSent = 0;
         numOk = 0;
 
-        if (log.isLoggable(LogLevel.DEBUG)) {
-            log.log(LogLevel.DEBUG, "windowSize " + windowSize + " throughput " + throughput);
-        }
 
         if (maxThroughput > 0 && throughput > maxThroughput * 0.95) {
             // No need to increase window when we're this close to max.
         } else if (throughput > localMaxThroughput * 1.01) {
             localMaxThroughput = throughput;
             windowSize += weight*windowSizeIncrement;
+            if (log.isLoggable(LogLevel.DEBUG)) {
+                log.log(LogLevel.DEBUG, "windowSize " + windowSize + " throughput " + throughput + " local max " + localMaxThroughput);
+            }
         } else {
             // scale up/down throughput for comparing to window size
             double period = 1;
@@ -113,11 +114,14 @@ public class DynamicThrottlePolicy extends StaticThrottlePolicy {
             }
             double efficiency = throughput*period/windowSize;
             if (efficiency < efficiencyThreshold) {
-                windowSize = Math.min(windowSize * windowSizeBackOff, windowSize - 2* windowSizeIncrement);
+                windowSize = Math.min(windowSize * windowSizeBackOff, windowSize - decrementFactor * windowSizeIncrement);
                 localMaxThroughput = 0;
             } else {
                 windowSize += weight*windowSizeIncrement;
             }
+            if (log.isLoggable(LogLevel.DEBUG)) {
+                log.log(LogLevel.DEBUG, "windowSize " + windowSize + " throughput " + throughput + " local max " + localMaxThroughput + " efficiency " + efficiency);
+            }
         }
         windowSize = Math.max(minWindowSize, windowSize);
         windowSize = Math.min(maxWindowSize, windowSize);
@@ -157,6 +161,17 @@ public class DynamicThrottlePolicy extends StaticThrottlePolicy {
     }
 
     /**
+     * Sets the relative stepsize when decreasing window size.
+     *
+     * @param decrementFactor the step size to set
+     * @return this, to allow chaining
+     */
+    public DynamicThrottlePolicy setWindowSizeDecrementFactor(double decrementFactor) {
+        this.decrementFactor = decrementFactor;
+        return this;
+    }
+
+    /**
      * Sets the factor of window size to back off to when the algorithm determines that efficiency is not increasing.
      * A value of 1 means that there is no back off from the local maxima, and means that the algorithm will fail to
      * reduce window size to something lower than a previous maxima. This value is capped to the [0, 1] range.
diff --git a/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCNetwork.java b/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCNetwork.java
index adf889a7b6f..554977d7eb1 100644
--- a/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCNetwork.java
+++ b/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCNetwork.java
@@ -67,10 +67,15 @@ public class RPCNetwork implements Network, MethodHandler {
             new ThreadPoolExecutor(getNumThreads(), getNumThreads(), 0L, TimeUnit.SECONDS,
                                    new SynchronousQueue<>(false),
                                    ThreadFactoryFactory.getDaemonThreadFactory("mbus.net"), new ThreadPoolExecutor.CallerRunsPolicy());
+
     private static int getNumThreads() {
         return Math.max(2, Runtime.getRuntime().availableProcessors()/2);
     }
 
+    private static boolean shouldEnableTcpNodelay(RPCNetworkParams.Optimization optimization) {
+        return optimization == RPCNetworkParams.Optimization.LATENCY;
+    }
+
     /**
      * Create an RPCNetwork. The servicePrefix is combined with session names to create service names. If the service
      * prefix is 'a/b' and the session name is 'c', the resulting service name that identifies the session on the
@@ -82,7 +87,7 @@ public class RPCNetwork implements Network, MethodHandler {
     public RPCNetwork(RPCNetworkParams params, SlobrokConfigSubscriber slobrokConfig) {
         this.slobroksConfig = slobrokConfig;
         identity = params.getIdentity();
-        orb = new Supervisor(new Transport(2));
+        orb = new Supervisor(new Transport(params.getNumNetworkThreads(), shouldEnableTcpNodelay(params.getOptimization())));
         orb.setMaxInputBufferSize(params.getMaxInputBufferSize());
         orb.setMaxOutputBufferSize(params.getMaxOutputBufferSize());
         targetPool = new RPCTargetPool(params.getConnectionExpireSecs(), params.getNumTargetsPerSpec());
@@ -473,7 +478,7 @@ public class RPCNetwork implements Network, MethodHandler {
             synchronized (this) {
                 if (version == null) {
                     hasError = true;
-                } else if (version.compareTo(this.version) < 0) {
+                } else if (version.isBefore(this.version)) {
                     this.version = version;
                 }
                 if (--pending == 0) {
diff --git a/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCNetworkParams.java b/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCNetworkParams.java
index d6d7603f54a..e77cddd8b06 100755
--- a/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCNetworkParams.java
+++ b/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCNetworkParams.java
@@ -20,6 +20,9 @@ public class RPCNetworkParams {
     private int maxOutputBufferSize = 256 * 1024;
     private double connectionExpireSecs = 30;
     private int numTargetsPerSpec = 1;
+    private int numNetworkThreads = 2;
+    public enum Optimization {LATENCY, THROUGHPUT}
+    Optimization optimization = Optimization.LATENCY;
 
     /**
      * Constructs a new instance of this class with reasonable default values.
@@ -42,6 +45,8 @@ public class RPCNetworkParams {
         maxInputBufferSize = params.maxInputBufferSize;
         maxOutputBufferSize = params.maxOutputBufferSize;
         numTargetsPerSpec = params.numTargetsPerSpec;
+        numNetworkThreads = params.numNetworkThreads;
+        optimization = params.optimization;
     }
 
     /**
@@ -152,6 +157,22 @@ public class RPCNetworkParams {
         return numTargetsPerSpec;
     }
 
+    public RPCNetworkParams setNumNetworkThreads(int numNetworkThreads) {
+        this.numNetworkThreads = numNetworkThreads;
+        return this;
+    }
+    int getNumNetworkThreads() {
+        return numNetworkThreads;
+    }
+
+    public RPCNetworkParams setOptimization(Optimization optimization) {
+        this.optimization = optimization;
+        return this;
+    }
+    Optimization getOptimization() {
+        return optimization;
+    }
+
     /**
      * Returns the maximum input buffer size allowed for the underlying FNET connection.
      *
diff --git a/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCSendV1.java b/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCSendV1.java
index 952bcdcfe04..ccded0e8d1b 100755
--- a/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCSendV1.java
+++ b/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCSendV1.java
@@ -64,7 +64,7 @@ public class RPCSendV1 extends RPCSend {
                                     long timeRemaining, byte[] payload, int traceLevel) {
         Request req = new Request(METHOD_NAME);
         Values v = req.parameters();
-        v.add(new StringValue(version.toString()));
+        v.add(new StringValue(version.toUtf8()));
         v.add(new StringValue(route.toString()));
         v.add(new StringValue(address.getSessionName()));
         v.add(new Int8Value(msg.getRetryEnabled() ? (byte)1 : (byte)0));
@@ -140,7 +140,7 @@ public class RPCSendV1 extends RPCSend {
             eMessages[i] = error.getMessage();
             eServices[i] = error.getService() != null ? error.getService() : "";
         }
-        ret.add(new StringValue(version.toString()));
+        ret.add(new StringValue(version.toUtf8()));
         ret.add(new DoubleValue(reply.getRetryDelay()));
         ret.add(new Int32Array(eCodes));
         ret.add(new StringArray(eMessages));
diff --git a/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCSendV2.java b/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCSendV2.java
index c6de8c8628f..bb243651447 100644
--- a/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCSendV2.java
+++ b/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCSendV2.java
@@ -21,7 +21,6 @@ import com.yahoo.slime.BinaryFormat;
 import com.yahoo.slime.Cursor;
 import com.yahoo.slime.Inspector;
 import com.yahoo.slime.Slime;
-import com.yahoo.text.Utf8;
 import com.yahoo.text.Utf8Array;
 
 /**
@@ -57,21 +56,21 @@ public class RPCSendV2 extends RPCSend {
                 .returnDesc(5, "body_payload", "Slime encoded body payload.");
         return method;
     }
-    private static final String VERSION_F = new String("version");
-    private static final String ROUTE_F = new String("route");
-    private static final String SESSION_F = new String("session");
-    private static final String PROTOCOL_F = new String("prot");
-    private static final String TRACELEVEL_F = new String("tracelevel");
-    private static final String TRACE_F = new String("trace");
-    private static final String USERETRY_F = new String("useretry");
-    private static final String RETRY_F = new String("retry");
-    private static final String RETRYDELAY_F = new String("retrydelay");
-    private static final String TIMEREMAINING_F = new String("timeleft");
-    private static final String ERRORS_F = new String("errors");
-    private static final String SERVICE_F = new String("service");
-    private static final String CODE_F = new String("code");
-    private static final String BLOB_F = new String("msg");
-    private static final String MSG_F = new String("msg");
+    private static final String VERSION_F = "version";
+    private static final String ROUTE_F = "route";
+    private static final String SESSION_F = "session";
+    private static final String PROTOCOL_F = "prot";
+    private static final String TRACELEVEL_F = "tracelevel";
+    private static final String TRACE_F = "trace";
+    private static final String USERETRY_F = "useretry";
+    private static final String RETRY_F = "retry";
+    private static final String RETRYDELAY_F = "retrydelay";
+    private static final String TIMEREMAINING_F = "timeleft";
+    private static final String ERRORS_F = "errors";
+    private static final String SERVICE_F = "service";
+    private static final String CODE_F = "code";
+    private static final String BLOB_F = "msg";
+    private static final String MSG_F = "msg";
 
     @Override
     protected Request encodeRequest(Version version, Route route, RPCServiceAddress address, Message msg,
@@ -88,7 +87,7 @@ public class RPCSendV2 extends RPCSend {
         Slime slime = new Slime();
         Cursor root = slime.setObject();
 
-        root.setString(VERSION_F, version.toString());
+        root.setString(VERSION_F, version.toUtf8().getBytes());
         root.setString(ROUTE_F, route.toString());
         root.setString(SESSION_F, address.getSessionName());
         root.setString(PROTOCOL_F, msg.getProtocol().toString());
@@ -115,7 +114,7 @@ public class RPCSendV2 extends RPCSend {
         Slime slime = BinaryFormat.decode(slimeBytes);
         Inspector root = slime.get();
 
-        Version version = new Version(root.field(VERSION_F).asString());
+        Version version = new Version(new Utf8Array(root.field(VERSION_F).asUtf8()));
         byte[] payload = root.field(BLOB_F).asData();
 
         // Make sure that the owner understands the protocol.
@@ -156,13 +155,13 @@ public class RPCSendV2 extends RPCSend {
         Slime slime = BinaryFormat.decode(slimeBytes);
         Inspector root = slime.get();
         Params p = new Params();
-        p.version = new Version(root.field(VERSION_F).asString());
+        p.version = new Version(new Utf8Array(root.field(VERSION_F).asUtf8()));
         p.route = root.field(ROUTE_F).asString();
         p.session = root.field(SESSION_F).asString();
         p.retryEnabled = root.field(USERETRY_F).asBool();
         p.retry = (int)root.field(RETRY_F).asLong();
         p.timeRemaining = root.field(TIMEREMAINING_F).asLong();
-        p.protocolName = new Utf8Array(Utf8.toBytes(root.field(PROTOCOL_F).asString()));
+        p.protocolName = new Utf8Array(root.field(PROTOCOL_F).asUtf8());
         p.payload = root.field(BLOB_F).asData();
         p.traceLevel = (int)root.field(TRACELEVEL_F).asLong();
         return p;
@@ -177,9 +176,9 @@ public class RPCSendV2 extends RPCSend {
         Slime slime = new Slime();
         Cursor root = slime.setObject();
 
-        root.setString(VERSION_F, version.toString());
+        root.setString(VERSION_F, version.toUtf8().getBytes());
         root.setDouble(RETRYDELAY_F, reply.getRetryDelay());
-        root.setString(PROTOCOL_F, reply.getProtocol().toString());
+        root.setString(PROTOCOL_F, reply.getProtocol().getBytes());
         root.setData(BLOB_F, payload);
         if (reply.getTrace().getLevel() > 0) {
             root.setString(TRACE_F, reply.getTrace().getRoot().encode());
diff --git a/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCTarget.java b/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCTarget.java
index fe248c6b8df..53c0c855116 100755
--- a/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCTarget.java
+++ b/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCTarget.java
@@ -8,6 +8,7 @@ import com.yahoo.jrt.Spec;
 import com.yahoo.jrt.Supervisor;
 import com.yahoo.jrt.Target;
 import com.yahoo.log.LogLevel;
+import com.yahoo.text.Utf8Array;
 
 import java.util.LinkedList;
 import java.util.List;
@@ -133,7 +134,7 @@ public class RPCTarget implements RequestWaiter {
         synchronized (this) {
             targetInvoked = false;
             if (req.checkReturnTypes("s")) {
-                String str = req.returnValues().get(0).asString();
+                Utf8Array str = req.returnValues().get(0).asUtf8Array();
                 try {
                     version = new Version(str);
                     if (shouldLog) {
diff --git a/messagebus/src/tests/CMakeLists.txt b/messagebus/src/tests/CMakeLists.txt
index cb2a403f55d..e05f732d8b4 100644
--- a/messagebus/src/tests/CMakeLists.txt
+++ b/messagebus/src/tests/CMakeLists.txt
@@ -9,7 +9,6 @@ add_subdirectory(context)
 add_subdirectory(emptyreply)
 add_subdirectory(error)
 add_subdirectory(identity)
-add_subdirectory(loadbalance)
 add_subdirectory(messagebus)
 add_subdirectory(messageordering)
 add_subdirectory(messenger)
diff --git a/messagebus/src/tests/error/error.cpp b/messagebus/src/tests/error/error.cpp
index 1d8a489a5ed..244efe0bf99 100644
--- a/messagebus/src/tests/error/error.cpp
+++ b/messagebus/src/tests/error/error.cpp
@@ -18,8 +18,6 @@
 
 using namespace mbus;
 
-TEST_SETUP(Test);
-
 RoutingSpec getRouting() {
     return RoutingSpec()
         .addTable(RoutingTableSpec("Simple")
@@ -28,10 +26,7 @@ RoutingSpec getRouting() {
                   .addRoute(RouteSpec("test").addHop("pxy").addHop("dst")));
 }
 
-int
-Test::Main()
-{
-    TEST_INIT("error_test");
+TEST("error_test") {
 
     Slobrok     slobrok;
     TestServer  srcNet(Identity("test/src"), getRouting(), slobrok);
@@ -51,30 +46,31 @@ Test::Main()
     ASSERT_TRUE(pxyNet.waitSlobrok("test/dst/session"));
 
     for (int i = 0; i < 5; i++) {
-        ASSERT_TRUE(ss->send(SimpleMessage::UP(new SimpleMessage("test message")), "test").isAccepted());
+        ASSERT_TRUE(ss->send(std::make_unique<SimpleMessage>("test message"), "test").isAccepted());
         Message::UP msg = pxy.getMessage();
-        ASSERT_TRUE(msg.get() != 0);
+        ASSERT_TRUE(msg);
         is->forward(std::move(msg));
 
         msg = dst.getMessage();
-        ASSERT_TRUE(msg.get() != 0);
-        Reply::UP reply(new EmptyReply());
+        ASSERT_TRUE(msg);
+        Reply::UP reply = std::make_unique<EmptyReply>();
         msg->swapState(*reply);
         reply->addError(Error(ErrorCode::APP_FATAL_ERROR, "fatality"));
         ds->reply(std::move(reply));
 
         reply = pxy.getReply();
-        ASSERT_TRUE(reply.get() != 0);
+        ASSERT_TRUE(reply);
         ASSERT_EQUAL(reply->getNumErrors(), 1u);
         EXPECT_EQUAL(reply->getError(0).getService(), "test/dst/session");
         reply->addError(Error(ErrorCode::APP_FATAL_ERROR, "fatality"));
         is->forward(std::move(reply));
 
         reply = src.getReply();
-        ASSERT_TRUE(reply.get() != 0);
+        ASSERT_TRUE(reply);
         ASSERT_EQUAL(reply->getNumErrors(), 2u);
         EXPECT_EQUAL(reply->getError(0).getService(), "test/dst/session");
         EXPECT_EQUAL(reply->getError(1).getService(), "test/pxy/session");
     }
-    TEST_DONE();
 }
+
+TEST_MAIN() { TEST_RUN_ALL(); }
\ No newline at end of file
diff --git a/messagebus/src/tests/loadbalance/.gitignore b/messagebus/src/tests/loadbalance/.gitignore
deleted file mode 100644
index d1cbb5977f1..00000000000
--- a/messagebus/src/tests/loadbalance/.gitignore
+++ /dev/null
@@ -1,4 +0,0 @@
-.depend
-Makefile
-loadbalance_test
-messagebus_loadbalance_test_app
diff --git a/messagebus/src/tests/loadbalance/CMakeLists.txt b/messagebus/src/tests/loadbalance/CMakeLists.txt
deleted file mode 100644
index e249a8284a6..00000000000
--- a/messagebus/src/tests/loadbalance/CMakeLists.txt
+++ /dev/null
@@ -1,9 +0,0 @@
-# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-vespa_add_executable(messagebus_loadbalance_test_app TEST
-    SOURCES
-    loadbalance.cpp
-    DEPENDS
-    messagebus_messagebus-test
-    messagebus
-)
-vespa_add_test(NAME messagebus_loadbalance_test_app COMMAND messagebus_loadbalance_test_app)
diff --git a/messagebus/src/tests/loadbalance/loadbalance.cpp b/messagebus/src/tests/loadbalance/loadbalance.cpp
deleted file mode 100644
index 05ea6d78871..00000000000
--- a/messagebus/src/tests/loadbalance/loadbalance.cpp
+++ /dev/null
@@ -1,88 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#include <vespa/vespalib/testkit/testapp.h>
-#include <vespa/messagebus/destinationsession.h>
-#include <vespa/messagebus/intermediatesession.h>
-#include <vespa/messagebus/messagebus.h>
-#include <vespa/messagebus/routablequeue.h>
-#include <vespa/messagebus/sourcesession.h>
-#include <vespa/messagebus/sourcesessionparams.h>
-#include <vespa/messagebus/testlib/receptor.h>
-#include <vespa/messagebus/routing/routingspec.h>
-#include <vespa/messagebus/testlib/simplemessage.h>
-#include <vespa/messagebus/testlib/simplereply.h>
-#include <vespa/messagebus/testlib/simpleprotocol.h>
-#include <vespa/messagebus/testlib/slobrok.h>
-#include <vespa/messagebus/testlib/testserver.h>
-
-using namespace mbus;
-using namespace std::chrono_literals;
-
-struct Handler : public IMessageHandler
-{
-    DestinationSession::UP session;
-    uint32_t               cnt;
-
-    Handler(MessageBus &mb) : session(), cnt(0) {
-        session = mb.createDestinationSession("session", true, *this);
-    }
-    ~Handler() {
-        session.reset();
-    }
-    void handleMessage(Message::UP msg) override {
-        ++cnt;
-        session->acknowledge(std::move(msg));
-    }
-};
-
-RoutingSpec getRouting() {
-    return RoutingSpec()
-        .addTable(RoutingTableSpec("Simple")
-                  .addHop(HopSpec("dst", "test/*/session"))
-                  .addRoute(RouteSpec("test").addHop("dst")));
-}
-
-TEST_SETUP(Test);
-
-int
-Test::Main()
-{
-    TEST_INIT("loadbalance_test");
-
-    Slobrok     slobrok;
-    TestServer  src(Identity(""), getRouting(), slobrok);
-    TestServer  dst1(Identity("test/dst1"), getRouting(), slobrok);
-    TestServer  dst2(Identity("test/dst2"), getRouting(), slobrok);
-    TestServer  dst3(Identity("test/dst3"), getRouting(), slobrok);
-
-    Handler h1(dst1.mb);
-    Handler h2(dst2.mb);
-    Handler h3(dst3.mb);
-
-    ASSERT_TRUE(src.waitSlobrok("test/dst1/session"));
-    ASSERT_TRUE(src.waitSlobrok("test/dst2/session"));
-    ASSERT_TRUE(src.waitSlobrok("test/dst3/session"));
-
-    RoutableQueue queue;
-    SourceSessionParams params;
-    params.setTimeout(30s);
-    params.setThrottlePolicy(IThrottlePolicy::SP());
-    SourceSession::UP ss = src.mb.createSourceSession(queue, params);
-
-    uint32_t msgCnt = 90;
-    ASSERT_TRUE(msgCnt % 3 == 0);
-    for (uint32_t i = 0; i < msgCnt; ++i) {
-        ss->send(Message::UP(new SimpleMessage("test")), "test");
-    }
-    for (uint32_t i = 0; i < 1000; ++i) {
-        if (queue.size() == msgCnt) {
-            break;
-        }
-        std::this_thread::sleep_for(10ms);
-    }
-    EXPECT_TRUE(queue.size() == msgCnt);
-    EXPECT_TRUE(h1.cnt == msgCnt / 3);
-    EXPECT_TRUE(h2.cnt == msgCnt / 3);
-    EXPECT_TRUE(h3.cnt == msgCnt / 3);
-    TEST_DONE();
-}
diff --git a/messagebus/src/tests/messagebus/messagebus.cpp b/messagebus/src/tests/messagebus/messagebus.cpp
index 86c7bf91f2a..367bfc997d0 100644
--- a/messagebus/src/tests/messagebus/messagebus.cpp
+++ b/messagebus/src/tests/messagebus/messagebus.cpp
@@ -112,13 +112,10 @@ public:
     Test();
     ~Test();
     int Main() override;
-    void testSendToAny();
     void testSendToCol();
-    void testSendToAnyThenCol();
     void testDirectHop();
     void testDirectRoute();
     void testRoutingPolicyCache();
-    void debugTrace();
 
 private:
     void setup();
@@ -131,21 +128,18 @@ private:
 
 TEST_APPHOOK(Test);
 
-Test::Test() {}
-Test::~Test() {}
+Test::Test() = default;
+Test::~Test() = default;
 
 int
 Test::Main()
 {
     TEST_INIT("messagebus_test");
 
-    testSendToAny();          TEST_FLUSH();
     testSendToCol();          TEST_FLUSH();
-    testSendToAnyThenCol();   TEST_FLUSH();
     testDirectHop();          TEST_FLUSH();
     testDirectRoute();        TEST_FLUSH();
     testRoutingPolicyCache(); TEST_FLUSH();
-    debugTrace();             TEST_FLUSH();
 
     TEST_DONE();
 }
@@ -206,38 +200,6 @@ void Test::teardown()
 }
 
 void
-Test::testSendToAny()
-{
-    setup();
-    for (uint32_t i = 0; i < 300; ++i) {
-        Message::UP msg(new SimpleMessage("test"));
-        EXPECT_TRUE(client->session->send(std::move(msg), "DocProc").isAccepted());
-    }
-    EXPECT_TRUE(dp0->waitQueueSize(100));
-    EXPECT_TRUE(dp1->waitQueueSize(100));
-    EXPECT_TRUE(dp2->waitQueueSize(100));
-    for (uint32_t i = 0; i < dpVec.size(); ++i) {
-        DocProc *p = dpVec[i];
-        while (p->queue.size() > 0) {
-            Routable::UP msg = p->queue.dequeue();
-            ASSERT_TRUE(msg);
-            Reply::UP reply(new EmptyReply());
-            msg->swapState(*reply);
-            reply->addError(Error(ErrorCode::FATAL_ERROR, ""));
-            p->session->forward(std::move(reply));
-        }
-    }
-    EXPECT_TRUE(client->waitQueueSize(300));
-    while (client->queue.size() > 0) {
-        Routable::UP reply = client->queue.dequeue();
-        ASSERT_TRUE(reply);
-        ASSERT_TRUE(reply->isReply());
-        EXPECT_TRUE(static_cast<Reply&>(*reply).getNumErrors() == 1);
-    }
-    teardown();
-}
-
-void
 Test::testSendToCol()
 {
     setup();
@@ -282,83 +244,6 @@ Test::testSendToCol()
 }
 
 void
-Test::testSendToAnyThenCol()
-{
-    setup();
-    ASSERT_TRUE(SimpleMessage("msg").getHash() % 2 == 0);
-    for (uint32_t i = 0; i < 150; ++i) {
-        Message::UP msg(new SimpleMessage("msg"));
-        EXPECT_TRUE(client->session->send(std::move(msg), "Index").isAccepted());
-    }
-    EXPECT_TRUE(dp0->waitQueueSize(50));
-    EXPECT_TRUE(dp1->waitQueueSize(50));
-    EXPECT_TRUE(dp2->waitQueueSize(50));
-    for (uint32_t i = 0; i < dpVec.size(); ++i) {
-        DocProc *p = dpVec[i];
-        while (p->queue.size() > 0) {
-            Routable::UP r = p->queue.dequeue();
-            ASSERT_TRUE(r);
-            p->session->forward(std::move(r));
-        }
-    }
-    EXPECT_TRUE(search00->waitQueueSize(150));
-    EXPECT_TRUE(search01->waitQueueSize(0));
-    EXPECT_TRUE(search10->waitQueueSize(150));
-    EXPECT_TRUE(search11->waitQueueSize(0));
-    ASSERT_TRUE(SimpleMessage("msh").getHash() % 2 == 1);
-    for (uint32_t i = 0; i < 150; ++i) {
-        Message::UP msg(new SimpleMessage("msh"));
-        ASSERT_TRUE(client->session->send(std::move(msg), "Index").isAccepted());
-    }
-    EXPECT_TRUE(dp0->waitQueueSize(50));
-    EXPECT_TRUE(dp1->waitQueueSize(50));
-    EXPECT_TRUE(dp2->waitQueueSize(50));
-    for (uint32_t i = 0; i < dpVec.size(); ++i) {
-        DocProc *p = dpVec[i];
-        while (p->queue.size() > 0) {
-            Routable::UP r = p->queue.dequeue();
-            ASSERT_TRUE(r);
-            p->session->forward(std::move(r));
-        }
-    }
-    EXPECT_TRUE(search00->waitQueueSize(150));
-    EXPECT_TRUE(search01->waitQueueSize(150));
-    EXPECT_TRUE(search10->waitQueueSize(150));
-    EXPECT_TRUE(search11->waitQueueSize(150));
-    for (uint32_t i = 0; i < searchVec.size(); ++i) {
-        Search *s = searchVec[i];
-        while (s->queue.size() > 0) {
-            Routable::UP msg = s->queue.dequeue();
-            ASSERT_TRUE(msg);
-            Reply::UP reply(new EmptyReply());
-            msg->swapState(*reply);
-            s->session->reply(std::move(reply));
-        }
-    }
-    EXPECT_TRUE(dp0->waitQueueSize(100));
-    EXPECT_TRUE(dp1->waitQueueSize(100));
-    EXPECT_TRUE(dp2->waitQueueSize(100));
-    for (uint32_t i = 0; i < dpVec.size(); ++i) {
-        DocProc *p = dpVec[i];
-        while (p->queue.size() > 0) {
-            Routable::UP r = p->queue.dequeue();
-            ASSERT_TRUE(r);
-            p->session->forward(std::move(r));
-        }
-    }
-    client->waitQueueSize(300);
-    std::this_thread::sleep_for(100ms);
-    client->waitQueueSize(300);
-    while (client->queue.size() > 0) {
-        Routable::UP reply = client->queue.dequeue();
-        ASSERT_TRUE(reply);
-        ASSERT_TRUE(reply->isReply());
-        EXPECT_TRUE(static_cast<Reply&>(*reply).getNumErrors() == 0);
-    }
-    teardown();
-}
-
-void
 Test::testDirectHop()
 {
     setup();
@@ -468,65 +353,3 @@ Test::testRoutingPolicyCache()
 
     teardown();
 }
-
-void
-Test::debugTrace()
-{
-    setup();
-    ASSERT_TRUE(SimpleMessage("msg").getHash() % 2 == 0);
-    for (uint32_t i = 0; i < 3; ++i) {
-        Message::UP msg(new SimpleMessage("msg"));
-        msg->getTrace().setLevel(4 + i);
-        EXPECT_TRUE(client->session->send(std::move(msg), "Index").isAccepted());
-    }
-    EXPECT_TRUE(dp0->waitQueueSize(1));
-    EXPECT_TRUE(dp1->waitQueueSize(1));
-    EXPECT_TRUE(dp2->waitQueueSize(1));
-    for (uint32_t i = 0; i < dpVec.size(); ++i) {
-        DocProc *p = dpVec[i];
-        while (p->queue.size() > 0) {
-            Routable::UP r = p->queue.dequeue();
-            ASSERT_TRUE(r);
-            p->session->forward(std::move(r));
-        }
-    }
-    EXPECT_TRUE(search00->waitQueueSize(3));
-    EXPECT_TRUE(search01->waitQueueSize(0));
-    EXPECT_TRUE(search10->waitQueueSize(3));
-    EXPECT_TRUE(search11->waitQueueSize(0));
-    for (uint32_t i = 0; i < searchVec.size(); ++i) {
-        Search *s = searchVec[i];
-        while (s->queue.size() > 0) {
-            Routable::UP msg = s->queue.dequeue();
-            ASSERT_TRUE(msg);
-            Reply::UP reply(new EmptyReply());
-            msg->swapState(*reply);
-            s->session->reply(std::move(reply));
-        }
-    }
-    EXPECT_TRUE(dp0->waitQueueSize(1));
-    EXPECT_TRUE(dp1->waitQueueSize(1));
-    EXPECT_TRUE(dp2->waitQueueSize(1));
-    for (uint32_t i = 0; i < dpVec.size(); ++i) {
-        DocProc *p = dpVec[i];
-        while (p->queue.size() > 0) {
-            Routable::UP r = p->queue.dequeue();
-            ASSERT_TRUE(r);
-            p->session->forward(std::move(r));
-        }
-    }
-    client->waitQueueSize(3);
-    Routable::UP reply = client->queue.dequeue();
-    fprintf(stderr, "\nTRACE DUMP(level=%d):\n%s\n\n",
-            reply->getTrace().getLevel(),
-            reply->getTrace().toString().c_str());
-    reply = client->queue.dequeue();
-    fprintf(stderr, "\nTRACE DUMP(level=%d):\n%s\n\n",
-            reply->getTrace().getLevel(),
-            reply->getTrace().toString().c_str());
-    reply = client->queue.dequeue();
-    fprintf(stderr, "\nTRACE DUMP(level=%d):\n%s\n\n",
-            reply->getTrace().getLevel(),
-            reply->getTrace().toString().c_str());
-    teardown();
-}
diff --git a/messagebus/src/tests/routeparser/routeparser.cpp b/messagebus/src/tests/routeparser/routeparser.cpp
index eb72b03e76a..12054c6c494 100644
--- a/messagebus/src/tests/routeparser/routeparser.cpp
+++ b/messagebus/src/tests/routeparser/routeparser.cpp
@@ -29,11 +29,11 @@ public:
 private:
     bool testError(const Route &route, const string &msg);
     bool testError(const Hop &hop, const string &msg);
-    bool testErrorDirective(IHopDirective::SP dir, const string &msg);
-    bool testPolicyDirective(IHopDirective::SP dir, const string &name, const string &param);
-    bool testRouteDirective(IHopDirective::SP dir, const string &name);
-    bool testTcpDirective(IHopDirective::SP dir, const string &host, uint32_t port, const string &session);
-    bool testVerbatimDirective(IHopDirective::SP dir, const string &image);
+    bool testErrorDirective(const IHopDirective & dir, const string &msg);
+    bool testPolicyDirective(const IHopDirective & dir, const string &name, const string &param);
+    bool testRouteDirective(const IHopDirective & dir, const string &name);
+    bool testTcpDirective(const IHopDirective & dir, const string &host, uint32_t port, const string &session);
+    bool testVerbatimDirective(const IHopDirective & dir, const string &image);
 };
 
 TEST_APPHOOK(Test);
@@ -77,84 +77,69 @@ Test::testError(const Hop &hop, const string &msg)
 }
 
 bool
-Test::testErrorDirective(IHopDirective::SP dir, const string &msg)
+Test::testErrorDirective(const IHopDirective & dir, const string &msg)
 {
-    if (!EXPECT_TRUE(dir.get() != NULL)) {
+    if (!EXPECT_EQUAL(IHopDirective::TYPE_ERROR, dir.getType())) {
         return false;
     }
-    if (!EXPECT_EQUAL(IHopDirective::TYPE_ERROR, dir->getType())) {
-        return false;
-    }
-    if (!EXPECT_EQUAL(msg, static_cast<const ErrorDirective&>(*dir).getMessage())) {
+    if (!EXPECT_EQUAL(msg, static_cast<const ErrorDirective&>(dir).getMessage())) {
         return false;
     }
     return true;
 }
 
 bool
-Test::testPolicyDirective(IHopDirective::SP dir, const string &name, const string &param)
+Test::testPolicyDirective(const IHopDirective & dir, const string &name, const string &param)
 {
-    if (!EXPECT_TRUE(dir.get() != NULL)) {
+    if (!EXPECT_EQUAL(IHopDirective::TYPE_POLICY, dir.getType())) {
         return false;
     }
-    if (!EXPECT_EQUAL(IHopDirective::TYPE_POLICY, dir->getType())) {
+    if (!EXPECT_EQUAL(name, static_cast<const PolicyDirective&>(dir).getName())) {
         return false;
     }
-    if (!EXPECT_EQUAL(name, static_cast<const PolicyDirective&>(*dir).getName())) {
-        return false;
-    }
-    if (!EXPECT_EQUAL(param, static_cast<const PolicyDirective&>(*dir).getParam())) {
+    if (!EXPECT_EQUAL(param, static_cast<const PolicyDirective&>(dir).getParam())) {
         return false;
     }
     return true;
 }
 
 bool
-Test::testRouteDirective(IHopDirective::SP dir, const string &name)
+Test::testRouteDirective(const IHopDirective & dir, const string &name)
 {
-    if (!EXPECT_TRUE(dir.get() != NULL)) {
-        return false;
-    }
-    if (!EXPECT_EQUAL(IHopDirective::TYPE_ROUTE, dir->getType())) {
+    if (!EXPECT_EQUAL(IHopDirective::TYPE_ROUTE, dir.getType())) {
         return false;
     }
-    if (!EXPECT_EQUAL(name, static_cast<const RouteDirective&>(*dir).getName())) {
+    if (!EXPECT_EQUAL(name, static_cast<const RouteDirective&>(dir).getName())) {
         return false;
     }
     return true;
 }
 
 bool
-Test::testTcpDirective(IHopDirective::SP dir, const string &host, uint32_t port, const string &session)
+Test::testTcpDirective(const IHopDirective & dir, const string &host, uint32_t port, const string &session)
 {
-    if (!EXPECT_TRUE(dir.get() != NULL)) {
-        return false;
-    }
-    if (!EXPECT_EQUAL(IHopDirective::TYPE_TCP, dir->getType())) {
+    if (!EXPECT_EQUAL(IHopDirective::TYPE_TCP, dir.getType())) {
         return false;
     }
-    if (!EXPECT_EQUAL(host, static_cast<const TcpDirective&>(*dir).getHost())) {
+    if (!EXPECT_EQUAL(host, static_cast<const TcpDirective&>(dir).getHost())) {
         return false;
     }
-    if (!EXPECT_EQUAL(port, static_cast<const TcpDirective&>(*dir).getPort())) {
+    if (!EXPECT_EQUAL(port, static_cast<const TcpDirective&>(dir).getPort())) {
         return false;
     }
-    if (!EXPECT_EQUAL(session, static_cast<const TcpDirective&>(*dir).getSession())) {
+    if (!EXPECT_EQUAL(session, static_cast<const TcpDirective&>(dir).getSession())) {
         return false;
     }
     return true;
 }
 
 bool
-Test::testVerbatimDirective(IHopDirective::SP dir, const string &image)
+Test::testVerbatimDirective(const IHopDirective & dir, const string &image)
 {
-    if (!EXPECT_TRUE(dir.get() != NULL)) {
-        return false;
-    }
-    if (!EXPECT_EQUAL(IHopDirective::TYPE_VERBATIM, dir->getType())) {
+    if (!EXPECT_EQUAL(IHopDirective::TYPE_VERBATIM, dir.getType())) {
         return false;
     }
-    if (!EXPECT_EQUAL(image, static_cast<const VerbatimDirective&>(*dir).getImage())) {
+    if (!EXPECT_EQUAL(image, static_cast<const VerbatimDirective&>(dir).getImage())) {
         return false;
     }
     return true;
diff --git a/messagebus/src/tests/serviceaddress/serviceaddress.cpp b/messagebus/src/tests/serviceaddress/serviceaddress.cpp
index 441da5a80ac..c5d07bc437a 100644
--- a/messagebus/src/tests/serviceaddress/serviceaddress.cpp
+++ b/messagebus/src/tests/serviceaddress/serviceaddress.cpp
@@ -7,34 +7,49 @@
 
 using namespace mbus;
 
-class Test : public vespalib::TestApp {
-public:
-    int Main() override;
-    void testAddrServiceAddress();
-    void testNameServiceAddress();
-
-private:
-    bool waitSlobrok(RPCNetwork &network, const string &pattern, size_t num);
-    bool testAddress(RPCNetwork& network, const string &pattern,
-                     const string &expectedSpec, const string &expectedSession);
-    bool testNullAddress(RPCNetwork &network, const string &pattern);
-};
-
-int
-Test::Main()
+bool
+waitSlobrok(RPCNetwork &network, const string &pattern, size_t num)
 {
-    TEST_INIT("serviceaddress_test");
-
-    testAddrServiceAddress(); TEST_FLUSH();
-    testNameServiceAddress(); TEST_FLUSH();
+    for (int i = 0; i < 1000; i++) {
+        slobrok::api::IMirrorAPI::SpecList res = network.getMirror().lookup(pattern);
+        if (res.size() == num) {
+            return true;
+        }
+        std::this_thread::sleep_for(10ms);
+    }
+    return false;
+}
 
-    TEST_DONE();
+bool
+testNullAddress(RPCNetwork &network, const string &pattern)
+{
+    RPCService service(network.getMirror(), pattern);
+    RPCServiceAddress::UP obj = service.make_address();
+    if ( ! EXPECT_FALSE(obj)) {
+        return false;
+    }
+    return true;
 }
 
-TEST_APPHOOK(Test);
+bool
+testAddress(RPCNetwork &network, const string &pattern,
+            const string &expectedSpec, const string &expectedSession)
+{
+    RPCService service(network.getMirror(), pattern);
+    RPCServiceAddress::UP obj = service.make_address();
+    if (!EXPECT_TRUE(obj)) {
+        return false;
+    }
+    if (!EXPECT_EQUAL(expectedSpec, obj->getConnectionSpec())) {
+        return false;
+    }
+    if (!EXPECT_EQUAL(expectedSession, obj->getSessionName())) {
+        return false;
+    }
+    return true;
+}
 
-void
-Test::testAddrServiceAddress()
+TEST("testAddrServiceAddress")
 {
     Slobrok slobrok;
     RPCNetwork network(RPCNetworkParams(slobrok.config())
@@ -55,8 +70,7 @@ Test::testAddrServiceAddress()
     network.shutdown();
 }
 
-void
-Test::testNameServiceAddress()
+TEST("testNameServiceAddress")
 {
     Slobrok slobrok;
     RPCNetwork network(RPCNetworkParams(slobrok.config())
@@ -74,45 +88,4 @@ Test::testNameServiceAddress()
     network.shutdown();
 }
 
-bool
-Test::waitSlobrok(RPCNetwork &network, const string &pattern, size_t num)
-{
-    for (int i = 0; i < 1000; i++) {
-        slobrok::api::IMirrorAPI::SpecList res = network.getMirror().lookup(pattern);
-        if (res.size() == num) {
-            return true;
-        }
-        std::this_thread::sleep_for(10ms);
-    }
-    return false;
-}
-
-bool
-Test::testNullAddress(RPCNetwork &network, const string &pattern)
-{
-    RPCService service(network.getMirror(), pattern);
-    RPCServiceAddress::UP obj = service.resolve();
-    if (!EXPECT_TRUE(obj.get() == NULL)) {
-        return false;
-    }
-    return true;
-}
-
-bool
-Test::testAddress(RPCNetwork &network, const string &pattern,
-                    const string &expectedSpec, const string &expectedSession)
-{
-    RPCService service(network.getMirror(), pattern);
-    RPCServiceAddress::UP obj = service.resolve();
-    if (!EXPECT_TRUE(obj.get() != NULL)) {
-        return false;
-    }
-    if (!EXPECT_EQUAL(expectedSpec, obj->getConnectionSpec())) {
-        return false;
-    }
-    if (!EXPECT_EQUAL(expectedSession, obj->getSessionName())) {
-        return false;
-    }
-    return true;
-}
-
+TEST_MAIN() { TEST_RUN_ALL(); }
\ No newline at end of file
diff --git a/messagebus/src/tests/servicepool/servicepool.cpp b/messagebus/src/tests/servicepool/servicepool.cpp
index 1334831c30c..58fc76f6b50 100644
--- a/messagebus/src/tests/servicepool/servicepool.cpp
+++ b/messagebus/src/tests/servicepool/servicepool.cpp
@@ -4,67 +4,59 @@
 #include <vespa/messagebus/network/rpcnetworkparams.h>
 #include <vespa/messagebus/network/rpcservicepool.h>
 #include <vespa/messagebus/testlib/slobrok.h>
+#include <vespa/messagebus/testlib/testserver.h>
 #include <vespa/vespalib/testkit/testapp.h>
 
 using namespace mbus;
 
-class Test : public vespalib::TestApp {
-private:
-    void testMaxSize();
-
-public:
-    int Main() override {
-        TEST_INIT("servicepool_test");
-
-        testMaxSize(); TEST_FLUSH();
-
-        TEST_DONE();
-    }
-};
-
-TEST_APPHOOK(Test);
-
-void
-Test::testMaxSize()
+TEST("testMaxSize")
 {
     Slobrok slobrok;
-    RPCNetwork net(RPCNetworkParams(slobrok.config()));
-    RPCServicePool pool(net, 2);
-    net.start();
-
-    pool.resolve("foo");
+    TestServer me(Identity("me"), RoutingSpec(), slobrok);
+    RPCNetwork & net = me.net;
+    net.registerSession("foo");
+    net.registerSession("bar");
+    net.registerSession("baz");
+    me.waitSlobrok("me/foo");
+    me.waitSlobrok("me/bar");
+    me.waitSlobrok("me/baz");
+    RPCServicePool pool(net.getMirror(), 2);
+
+    RPCServiceAddress::UP addr = pool.resolve("me/foo");
     EXPECT_EQUAL(1u, pool.getSize());
-    EXPECT_TRUE(pool.hasService("foo"));
-    EXPECT_TRUE(!pool.hasService("bar"));
-    EXPECT_TRUE(!pool.hasService("baz"));
+    EXPECT_TRUE(pool.hasService("me/foo"));
+    EXPECT_TRUE(!pool.hasService("me/bar"));
+    EXPECT_TRUE(!pool.hasService("me/baz"));
 
-    pool.resolve("foo");
+    addr = pool.resolve("me/foo");
     EXPECT_EQUAL(1u, pool.getSize());
-    EXPECT_TRUE(pool.hasService("foo"));
-    EXPECT_TRUE(!pool.hasService("bar"));
-    EXPECT_TRUE(!pool.hasService("baz"));
+    EXPECT_TRUE(pool.hasService("me/foo"));
+    EXPECT_TRUE(!pool.hasService("me/bar"));
+    EXPECT_TRUE(!pool.hasService("me/baz"));
 
-    pool.resolve("bar");
+    addr = pool.resolve("me/bar");
     EXPECT_EQUAL(2u, pool.getSize());
-    EXPECT_TRUE(pool.hasService("foo"));
-    EXPECT_TRUE(pool.hasService("bar"));
-    EXPECT_TRUE(!pool.hasService("baz"));
+    EXPECT_TRUE(pool.hasService("me/foo"));
+    EXPECT_TRUE(pool.hasService("me/bar"));
+    EXPECT_TRUE(!pool.hasService("me/baz"));
 
-    pool.resolve("baz");
+    addr = pool.resolve("me/baz");
     EXPECT_EQUAL(2u, pool.getSize());
-    EXPECT_TRUE(!pool.hasService("foo"));
-    EXPECT_TRUE(pool.hasService("bar"));
-    EXPECT_TRUE(pool.hasService("baz"));
+    EXPECT_TRUE(!pool.hasService("me/foo"));
+    EXPECT_TRUE(pool.hasService("me/bar"));
+    EXPECT_TRUE(pool.hasService("me/baz"));
 
-    pool.resolve("bar");
+    addr = pool.resolve("me/bar");
     EXPECT_EQUAL(2u, pool.getSize());
-    EXPECT_TRUE(!pool.hasService("foo"));
-    EXPECT_TRUE(pool.hasService("bar"));
-    EXPECT_TRUE(pool.hasService("baz"));
+    EXPECT_TRUE(!pool.hasService("me/foo"));
+    EXPECT_TRUE(pool.hasService("me/bar"));
+    EXPECT_TRUE(pool.hasService("me/baz"));
 
-    pool.resolve("foo");
+    addr = pool.resolve("me/foo");
     EXPECT_EQUAL(2u, pool.getSize());
-    EXPECT_TRUE(pool.hasService("foo"));
-    EXPECT_TRUE(pool.hasService("bar"));
-    EXPECT_TRUE(!pool.hasService("baz"));
+    EXPECT_TRUE(pool.hasService("me/foo"));
+    EXPECT_TRUE(pool.hasService("me/bar"));
+    EXPECT_TRUE(!pool.hasService("me/baz"));
 }
+
+TEST_MAIN() { TEST_RUN_ALL(); }
\ No newline at end of file
diff --git a/messagebus/src/tests/shutdown/shutdown.cpp b/messagebus/src/tests/shutdown/shutdown.cpp
index e415622707f..07d9f0fae5d 100644
--- a/messagebus/src/tests/shutdown/shutdown.cpp
+++ b/messagebus/src/tests/shutdown/shutdown.cpp
@@ -12,30 +12,9 @@
 
 using namespace mbus;
 
-class Test : public vespalib::TestApp {
-private:
-    void requireThatListenFailedIsExceptionSafe();
-    void requireThatShutdownOnSourceWithPendingIsSafe();
-    void requireThatShutdownOnIntermediateWithPendingIsSafe();
-
-public:
-    int Main() override {
-        TEST_INIT("shutdown_test");
-
-        requireThatListenFailedIsExceptionSafe();             TEST_FLUSH();
-        requireThatShutdownOnSourceWithPendingIsSafe();       TEST_FLUSH();
-        requireThatShutdownOnIntermediateWithPendingIsSafe(); TEST_FLUSH();
-
-        TEST_DONE();
-    }
-};
-
 static const duration TIMEOUT = 120s;
 
-TEST_APPHOOK(Test);
-
-void
-Test::requireThatListenFailedIsExceptionSafe()
+TEST("requireThatListenFailedIsExceptionSafe")
 {
     fnet::frt::StandaloneFRT orb;
     ASSERT_TRUE(orb.supervisor().Listen(0));
@@ -51,8 +30,7 @@ Test::requireThatListenFailedIsExceptionSafe()
     }
 }
 
-void
-Test::requireThatShutdownOnSourceWithPendingIsSafe()
+TEST("requireThatShutdownOnSourceWithPendingIsSafe")
 {
     Slobrok slobrok;
     TestServer dstServer(MessageBusParams()
@@ -87,8 +65,7 @@ Test::requireThatShutdownOnSourceWithPendingIsSafe()
     }
 }
 
-void
-Test::requireThatShutdownOnIntermediateWithPendingIsSafe()
+TEST("requireThatShutdownOnIntermediateWithPendingIsSafe")
 {
     Slobrok slobrok;
     TestServer dstServer(MessageBusParams()
@@ -114,7 +91,7 @@ Test::requireThatShutdownOnIntermediateWithPendingIsSafe()
     ASSERT_TRUE(srcServer.waitSlobrok("dst/session", 1));
 
     for (uint32_t i = 0; i < 10; ++i) {
-        Message::UP msg(new SimpleMessage("msg"));
+        Message::UP msg = std::make_unique<SimpleMessage>("msg");
         {
             TestServer itrServer(MessageBusParams()
                     .setRetryPolicy(std::make_shared<RetryTransientErrorsPolicy>())
@@ -141,3 +118,5 @@ Test::requireThatShutdownOnIntermediateWithPendingIsSafe()
         dstServer.mb.sync();
     }
 }
+
+TEST_MAIN() { TEST_RUN_ALL(); }
\ No newline at end of file
diff --git a/messagebus/src/tests/targetpool/targetpool.cpp b/messagebus/src/tests/targetpool/targetpool.cpp
index 0e0e566f2be..9259f992d6c 100644
--- a/messagebus/src/tests/targetpool/targetpool.cpp
+++ b/messagebus/src/tests/targetpool/targetpool.cpp
@@ -22,12 +22,7 @@ public:
     }
 };
 
-TEST_SETUP(Test);
-
-int
-Test::Main()
-{
-    TEST_INIT("targetpool_test");
+TEST("targetpool_test") {
 
     // Necessary setup to be able to resolve targets.
     Slobrok slobrok;
@@ -46,9 +41,9 @@ Test::Main()
 
     // Assert that all connections expire.
     RPCTarget::SP target;
-    ASSERT_TRUE((target = pool.getTarget(orb, adr1)).get() != NULL); target.reset();
-    ASSERT_TRUE((target = pool.getTarget(orb, adr2)).get() != NULL); target.reset();
-    ASSERT_TRUE((target = pool.getTarget(orb, adr3)).get() != NULL); target.reset();
+    ASSERT_TRUE((target = pool.getTarget(orb, adr1))); target.reset();
+    ASSERT_TRUE((target = pool.getTarget(orb, adr2))); target.reset();
+    ASSERT_TRUE((target = pool.getTarget(orb, adr3))); target.reset();
     EXPECT_EQUAL(3u, pool.size());
     for (uint32_t i = 0; i < 10; ++i) {
         pool.flushTargets(false);
@@ -59,19 +54,19 @@ Test::Main()
     EXPECT_EQUAL(0u, pool.size());
 
     // Assert that only idle connections expire.
-    ASSERT_TRUE((target = pool.getTarget(orb, adr1)).get() != NULL); target.reset();
-    ASSERT_TRUE((target = pool.getTarget(orb, adr2)).get() != NULL); target.reset();
-    ASSERT_TRUE((target = pool.getTarget(orb, adr3)).get() != NULL); target.reset();
+    ASSERT_TRUE((target = pool.getTarget(orb, adr1))); target.reset();
+    ASSERT_TRUE((target = pool.getTarget(orb, adr2))); target.reset();
+    ASSERT_TRUE((target = pool.getTarget(orb, adr3))); target.reset();
     EXPECT_EQUAL(3u, pool.size());
     timer.millis += 444;
     pool.flushTargets(false);
     EXPECT_EQUAL(3u, pool.size());
-    ASSERT_TRUE((target = pool.getTarget(orb, adr2)).get() != NULL); target.reset();
-    ASSERT_TRUE((target = pool.getTarget(orb, adr3)).get() != NULL); target.reset();
+    ASSERT_TRUE((target = pool.getTarget(orb, adr2))); target.reset();
+    ASSERT_TRUE((target = pool.getTarget(orb, adr3))); target.reset();
     timer.millis += 444;
     pool.flushTargets(false);
     EXPECT_EQUAL(2u, pool.size());
-    ASSERT_TRUE((target = pool.getTarget(orb, adr3)).get() != NULL); target.reset();
+    ASSERT_TRUE((target = pool.getTarget(orb, adr3))); target.reset();
     timer.millis += 444;
     pool.flushTargets(false);
     EXPECT_EQUAL(1u, pool.size());
@@ -80,7 +75,7 @@ Test::Main()
     EXPECT_EQUAL(0u, pool.size());
 
     // Assert that connections never expire while they are referenced.
-    ASSERT_TRUE((target = pool.getTarget(orb, adr1)).get() != NULL);
+    ASSERT_TRUE((target = pool.getTarget(orb, adr1)));
     EXPECT_EQUAL(1u, pool.size());
     for (int i = 0; i < 10; ++i) {
         timer.millis += 999;
@@ -91,6 +86,6 @@ Test::Main()
     timer.millis += 999;
     pool.flushTargets(false);
     EXPECT_EQUAL(0u, pool.size());
-
-    TEST_DONE();
 }
+
+TEST_MAIN() { TEST_RUN_ALL(); }
\ No newline at end of file
diff --git a/messagebus/src/vespa/messagebus/callstack.cpp b/messagebus/src/vespa/messagebus/callstack.cpp
index b7179e14cad..ab22f1ace34 100644
--- a/messagebus/src/vespa/messagebus/callstack.cpp
+++ b/messagebus/src/vespa/messagebus/callstack.cpp
@@ -20,7 +20,7 @@ CallStack::discard()
     }
 }
 
-CallStack::~CallStack() { }
+CallStack::~CallStack() = default;
 
 IReplyHandler &
 CallStack::pop(Reply &reply)
diff --git a/messagebus/src/vespa/messagebus/messagebus.cpp b/messagebus/src/vespa/messagebus/messagebus.cpp
index d277063a273..fee9504007b 100644
--- a/messagebus/src/vespa/messagebus/messagebus.cpp
+++ b/messagebus/src/vespa/messagebus/messagebus.cpp
@@ -99,7 +99,7 @@ MessageBus::MessageBus(INetwork &net, ProtocolSet protocols) :
     MessageBusParams params;
     while (!protocols.empty()) {
         IProtocol::SP protocol = protocols.extract();
-        if (protocol.get() != nullptr) {
+        if (protocol) {
             params.addProtocol(protocol);
         }
     }
@@ -132,8 +132,7 @@ MessageBus::~MessageBus()
     bool done = false;
     while (!done) {
         vespalib::Gate gate;
-        Messenger::ITask::UP task(new ShutdownTask(_network, *_msn, done, gate));
-        _msn->enqueue(std::move(task));
+        _msn->enqueue(std::make_unique<ShutdownTask>(_network, *_msn, done, gate));
         gate.await();
     }
 }
@@ -157,11 +156,10 @@ MessageBus::setup(const MessageBusParams &params)
 
     // Start messenger.
     IRetryPolicy::SP retryPolicy = params.getRetryPolicy();
-    if (retryPolicy.get() != nullptr) {
-        _resender.reset(new Resender(retryPolicy));
+    if (retryPolicy) {
+        _resender = std::make_unique<Resender>(retryPolicy);
 
-        Messenger::ITask::UP task(new ResenderTask(*_resender));
-        _msn->addRecurrentTask(std::move(task));
+        _msn->addRecurrentTask(std::make_unique<ResenderTask>(*_resender));
     }
     if (!_msn->start()) {
         throw vespalib::NetworkSetupFailureException("Failed to start messenger.");
@@ -273,7 +271,7 @@ MessageBus::sync()
 void
 MessageBus::handleMessage(Message::UP msg)
 {
-    if (_resender.get() != nullptr && msg->hasBucketSequence()) {
+    if (_resender && msg->hasBucketSequence()) {
         deliverError(std::move(msg), ErrorCode::SEQUENCE_ERROR,
                      "Bucket sequences not supported when resender is enabled.");
         return;
@@ -292,8 +290,7 @@ MessageBus::setupRouting(const RoutingSpec &spec)
             LOG(info, "Protocol '%s' is not supported, ignoring routing table.", cfg.getProtocol().c_str());
             continue;
         }
-        RoutingTable::SP rt(new RoutingTable(cfg));
-        rtm[cfg.getProtocol()] = rt;
+        rtm[cfg.getProtocol()] = std::make_shared<RoutingTable>(cfg);
     }
     {
         LockGuard guard(_lock);
@@ -383,7 +380,7 @@ MessageBus::deliverMessage(Message::UP msg, const string &session)
 void
 MessageBus::deliverError(Message::UP msg, uint32_t errCode, const string &errMsg)
 {
-    Reply::UP reply(new EmptyReply());
+    auto reply = std::make_unique<EmptyReply>();
     reply->swapState(*msg);
     reply->addError(Error(errCode, errMsg));
 
diff --git a/messagebus/src/vespa/messagebus/messenger.cpp b/messagebus/src/vespa/messagebus/messenger.cpp
index 63df7f2f482..5313c4adcbb 100644
--- a/messagebus/src/vespa/messagebus/messenger.cpp
+++ b/messagebus/src/vespa/messagebus/messenger.cpp
@@ -33,7 +33,7 @@ public:
     }
 
     ~MessageTask() {
-        if (_msg.get() != nullptr) {
+        if (_msg) {
             _msg->discard();
         }
     }
@@ -43,7 +43,7 @@ public:
     }
 
     uint8_t priority() const override {
-        if (_msg.get() != nullptr) {
+        if (_msg) {
             return _msg->priority();
         }
 
@@ -65,7 +65,7 @@ public:
     }
 
     ~ReplyTask() {
-        if (_reply.get() != nullptr) {
+        if (_reply) {
             _reply->discard();
         }
     }
@@ -75,7 +75,7 @@ public:
     }
 
     uint8_t priority() const override {
-        if (_reply.get() != nullptr) {
+        if (_reply) {
             return _reply->priority();
         }
 
@@ -206,7 +206,7 @@ Messenger::Run(FastOS_ThreadInterface *thread, void *arg)
                 _queue.pop();
             }
         }
-        if (task.get() != nullptr) {
+        if (task) {
             try {
                 task->run();
             } catch (const std::exception &e) {
@@ -223,16 +223,14 @@ Messenger::Run(FastOS_ThreadInterface *thread, void *arg)
 void
 Messenger::addRecurrentTask(ITask::UP task)
 {
-    ITask::UP add(new AddRecurrentTask(_children, std::move(task)));
-    enqueue(std::move(add));
+    enqueue(std::make_unique<AddRecurrentTask>(_children, std::move(task)));
 }
 
 void
 Messenger::discardRecurrentTasks()
 {
     vespalib::Gate gate;
-    ITask::UP task(new DiscardRecurrentTasks(gate, _children));
-    enqueue(std::move(task));
+    enqueue(std::make_unique<DiscardRecurrentTasks>(gate, _children));
     gate.await();
 }
 
@@ -248,13 +246,13 @@ Messenger::start()
 void
 Messenger::deliverMessage(Message::UP msg, IMessageHandler &handler)
 {
-    enqueue(ITask::UP(new MessageTask(std::move(msg), handler)));
+    enqueue(std::make_unique<MessageTask>(std::move(msg), handler));
 }
 
 void
 Messenger::deliverReply(Reply::UP reply, IReplyHandler &handler)
 {
-    enqueue(ITask::UP(new ReplyTask(std::move(reply), handler)));
+    enqueue(std::make_unique<ReplyTask>(std::move(reply), handler));
 }
 
 void
@@ -273,7 +271,7 @@ void
 Messenger::sync()
 {
     vespalib::Gate gate;
-    enqueue(ITask::UP(new SyncTask(gate)));
+    enqueue(std::make_unique<SyncTask>(gate));
     gate.await();
 }
 
diff --git a/messagebus/src/vespa/messagebus/network/rpcnetwork.cpp b/messagebus/src/vespa/messagebus/network/rpcnetwork.cpp
index c6f61b383bc..de3be2ffa01 100644
--- a/messagebus/src/vespa/messagebus/network/rpcnetwork.cpp
+++ b/messagebus/src/vespa/messagebus/network/rpcnetwork.cpp
@@ -26,6 +26,8 @@ LOG_SETUP(".rpcnetwork");
 using vespalib::make_string;
 using namespace std::chrono_literals;
 
+namespace mbus {
+
 namespace {
 
 /**
@@ -45,7 +47,9 @@ public:
         _gate() {
         ScheduleNow();
     }
-    ~SyncTask() override = default;
+    ~SyncTask() override {
+        Kill();
+    }
 
     void await() {
         _gate.await();
@@ -56,9 +60,25 @@ public:
     }
 };
 
-} // namespace <unnamed>
+struct TargetPoolTask : public FNET_Task {
+    RPCTargetPool &_pool;
 
-namespace mbus {
+    TargetPoolTask(FNET_Scheduler &scheduler, RPCTargetPool &pool)
+        : FNET_Task(&scheduler),
+        _pool(pool)
+    {
+        ScheduleNow();
+    }
+    ~TargetPoolTask() override {
+        Kill();
+    }
+    void PerformTask() override {
+        _pool.flushTargets(false);
+        Schedule(1.0);
+    }
+};
+
+}
 
 RPCNetwork::SendContext::SendContext(RPCNetwork &net, const Message &msg,
                                      const std::vector<RoutingNode*> &recipients)
@@ -92,20 +112,6 @@ RPCNetwork::SendContext::handleVersion(const vespalib::Version *version)
     }
 }
 
-RPCNetwork::TargetPoolTask::TargetPoolTask(FNET_Scheduler &scheduler, RPCTargetPool &pool)
-    : FNET_Task(&scheduler),
-      _pool(pool)
-{
-    ScheduleNow();
-}
-
-void
-RPCNetwork::TargetPoolTask::PerformTask()
-{
-    _pool.flushTargets(false);
-    Schedule(1.0);
-}
-
 RPCNetwork::RPCNetwork(const RPCNetworkParams &params) :
     _owner(nullptr),
     _ident(params.getIdentity()),
@@ -113,13 +119,13 @@ RPCNetwork::RPCNetwork(const RPCNetworkParams &params) :
     _transport(std::make_unique<FNET_Transport>()),
     _orb(std::make_unique<FRT_Supervisor>(_transport.get())),
     _scheduler(*_transport->GetScheduler()),
-    _targetPool(std::make_unique<RPCTargetPool>(params.getConnectionExpireSecs())),
-    _targetPoolTask(_scheduler, *_targetPool),
-    _servicePool(std::make_unique<RPCServicePool>(*this, 4096)),
     _slobrokCfgFactory(std::make_unique<slobrok::ConfiguratorFactory>(params.getSlobrokConfig())),
     _mirror(std::make_unique<slobrok::api::MirrorAPI>(*_orb, *_slobrokCfgFactory)),
     _regAPI(std::make_unique<slobrok::api::RegisterAPI>(*_orb, *_slobrokCfgFactory)),
     _requestedPort(params.getListenPort()),
+    _targetPool(std::make_unique<RPCTargetPool>(params.getConnectionExpireSecs())),
+    _targetPoolTask(std::make_unique<TargetPoolTask>(_scheduler, *_targetPool)),
+    _servicePool(std::make_unique<RPCServicePool>(*_mirror, 4096)),
     _executor(std::make_unique<vespalib::ThreadStackExecutor>(params.getNumThreads(), 65536)),
     _sendV1(std::make_unique<RPCSendV1>()),
     _sendV2(std::make_unique<RPCSendV2>()),
@@ -130,6 +136,7 @@ RPCNetwork::RPCNetwork(const RPCNetworkParams &params) :
 {
     _transport->SetMaxInputBufferSize(params.getMaxInputBufferSize());
     _transport->SetMaxOutputBufferSize(params.getMaxOutputBufferSize());
+    _transport->SetTCPNoDelay(params.getTcpNoDelay());
 }
 
 RPCNetwork::~RPCNetwork()
@@ -306,8 +313,8 @@ RPCNetwork::resolveServiceAddress(RoutingNode &recipient, const string &serviceN
                      make_string("Failed to connect to service '%s' from host '%s'.",
                                  serviceName.c_str(), getIdentity().getHostname().c_str()));
     }
-    ret->setTarget(target); // free by freeServiceAddress()
-    recipient.setServiceAddress(IServiceAddress::UP(ret.release()));
+    ret->setTarget(std::move(target)); // free by freeServiceAddress()
+    recipient.setServiceAddress(std::move(ret));
     return Error();
 }
 
diff --git a/messagebus/src/vespa/messagebus/network/rpcnetwork.h b/messagebus/src/vespa/messagebus/network/rpcnetwork.h
index a6c2724929d..a8eb514387c 100644
--- a/messagebus/src/vespa/messagebus/network/rpcnetwork.h
+++ b/messagebus/src/vespa/messagebus/network/rpcnetwork.h
@@ -50,13 +50,6 @@ private:
         void handleVersion(const vespalib::Version *version) override;
     };
 
-    struct TargetPoolTask : public FNET_Task {
-        RPCTargetPool &_pool;
-
-        TargetPoolTask(FNET_Scheduler &scheduler, RPCTargetPool &pool);
-        void PerformTask() override;
-    };
-
     using SendAdapterMap = std::map<vespalib::Version, RPCSendAdapter*>;
 
     INetworkOwner                                  *_owner;
@@ -65,13 +58,13 @@ private:
     std::unique_ptr<FNET_Transport>                 _transport;
     std::unique_ptr<FRT_Supervisor>                 _orb;
     FNET_Scheduler                                 &_scheduler;
-    std::unique_ptr<RPCTargetPool>                  _targetPool;
-    TargetPoolTask                                  _targetPoolTask;
-    std::unique_ptr<RPCServicePool>                 _servicePool;
     std::unique_ptr<slobrok::ConfiguratorFactory>   _slobrokCfgFactory;
     std::unique_ptr<slobrok::api::IMirrorAPI>       _mirror;
     std::unique_ptr<slobrok::api::RegisterAPI>      _regAPI;
     int                                             _requestedPort;
+    std::unique_ptr<RPCTargetPool>                  _targetPool;
+    std::unique_ptr<FNET_Task>                      _targetPoolTask;
+    std::unique_ptr<RPCServicePool>                 _servicePool;
     std::unique_ptr<vespalib::ThreadStackExecutor>  _executor;
     std::unique_ptr<RPCSendAdapter>                 _sendV1;
     std::unique_ptr<RPCSendAdapter>                 _sendV2;
diff --git a/messagebus/src/vespa/messagebus/network/rpcnetworkparams.cpp b/messagebus/src/vespa/messagebus/network/rpcnetworkparams.cpp
index bd87e4dbbe2..5bf277a8ee6 100644
--- a/messagebus/src/vespa/messagebus/network/rpcnetworkparams.cpp
+++ b/messagebus/src/vespa/messagebus/network/rpcnetworkparams.cpp
@@ -15,6 +15,7 @@ RPCNetworkParams::RPCNetworkParams(config::ConfigUri configUri) :
     _maxInputBufferSize(256*1024),
     _maxOutputBufferSize(256*1024),
     _numThreads(4),
+    _tcpNoDelay(true),
     _dispatchOnEncode(true),
     _dispatchOnDecode(false),
     _connectionExpireSecs(600),
diff --git a/messagebus/src/vespa/messagebus/network/rpcnetworkparams.h b/messagebus/src/vespa/messagebus/network/rpcnetworkparams.h
index ba530257030..140f81c611c 100644
--- a/messagebus/src/vespa/messagebus/network/rpcnetworkparams.h
+++ b/messagebus/src/vespa/messagebus/network/rpcnetworkparams.h
@@ -20,6 +20,7 @@ private:
     uint32_t          _maxInputBufferSize;
     uint32_t          _maxOutputBufferSize;
     uint32_t          _numThreads;
+    bool              _tcpNoDelay;
     bool              _dispatchOnEncode;
     bool              _dispatchOnDecode;
     double            _connectionExpireSecs;
@@ -106,6 +107,13 @@ public:
 
     uint32_t getNumThreads() const { return _numThreads; }
 
+    RPCNetworkParams &setTcpNoDelay(bool tcpNoDelay) {
+        _tcpNoDelay = tcpNoDelay;
+        return *this;
+    }
+
+    bool getTcpNoDelay() const { return _tcpNoDelay; }
+
     /**
      * Returns the number of seconds before an idle network connection expires.
      *
diff --git a/messagebus/src/vespa/messagebus/network/rpcsendv2.cpp b/messagebus/src/vespa/messagebus/network/rpcsendv2.cpp
index 3b0c10500b9..6afa1528092 100644
--- a/messagebus/src/vespa/messagebus/network/rpcsendv2.cpp
+++ b/messagebus/src/vespa/messagebus/network/rpcsendv2.cpp
@@ -207,7 +207,7 @@ RPCSendV2::createReply(const FRT_Values & ret, const string & serviceName,
         reply = decode(root[PROTOCOL_F].asString().make_stringref(), version, BlobRef(payload.data, payload.size), error);
     }
     if ( ! reply ) {
-        reply.reset(new EmptyReply());
+        reply = std::make_unique<EmptyReply>();
     }
     reply->setRetryDelay(root[RETRYDELAY_F].asDouble());
     Inspector & errors = root[ERRORS_F];
@@ -217,7 +217,10 @@ RPCSendV2::createReply(const FRT_Values & ret, const string & serviceName,
         reply->addError(Error(e[CODE_F].asLong(), e[MSG_F].asString().make_string(),
                               (service.size > 0) ? service.make_string() : serviceName));
     }
-    rootTrace.addChild(TraceNode::decode(root[TRACE_F].asString().make_string()));
+    Inspector & trace = root[TRACE_F];
+    if (trace.valid() && (trace.asString().size > 0)) {
+        rootTrace.addChild(TraceNode::decode(trace.asString().make_string()));
+    }
     return reply;
 }
 
diff --git a/messagebus/src/vespa/messagebus/network/rpcservice.cpp b/messagebus/src/vespa/messagebus/network/rpcservice.cpp
index 6e7c73b38ee..ecf40973187 100644
--- a/messagebus/src/vespa/messagebus/network/rpcservice.cpp
+++ b/messagebus/src/vespa/messagebus/network/rpcservice.cpp
@@ -5,43 +5,38 @@
 
 namespace mbus {
 
-RPCService::RPCService(const Mirror &mirror,
-                       const string &pattern) :
-    _mirror(mirror),
-    _pattern(pattern),
-    _addressIdx(random()),
-    _addressGen(0),
-    _addressList()
-{ }
-
-RPCService::~RPCService() {}
-
-RPCServiceAddress::UP
-RPCService::resolve()
+RPCService::RPCService(const Mirror &mirror, const string &pattern) :
+    _serviceName(),
+    _connectionSpec()
 {
-    if (_pattern.find("tcp/") == 0) {
-        size_t pos = _pattern.find_last_of('/');
-        if (pos != string::npos && pos < _pattern.size() - 1) {
-            RPCServiceAddress::UP ret(new RPCServiceAddress(
-                            _pattern,
-                            _pattern.substr(0, pos)));
-            if (!ret->isMalformed()) {
-                return ret;
+    if (pattern.find("tcp/") == 0) {
+        size_t pos = pattern.find_last_of('/');
+        if (pos != string::npos && pos < pattern.size() - 1) {
+            RPCServiceAddress test(pattern, pattern.substr(0, pos));
+            if ( ! test.isMalformed()) {
+                _serviceName = pattern;
+                _connectionSpec = pattern.substr(0, pos);
             }
         }
     } else {
-        if (_addressGen != _mirror.updates()) {
-            _addressGen = _mirror.updates();
-            _addressList = _mirror.lookup(_pattern);
-        }
-        if (!_addressList.empty()) {
-            _addressIdx = (_addressIdx + 1) % _addressList.size();
-            const AddressList::value_type &entry = _addressList[_addressIdx];
-            return RPCServiceAddress::UP(new RPCServiceAddress(
-                            entry.first,
-                            entry.second));
+        Mirror::SpecList addressList = mirror.lookup(pattern);
+        if (!addressList.empty()) {
+            assert(addressList.size() == 1); //TODO URGENT remove assert after a few factory runs.
+            const auto &entry = addressList.front();
+            _serviceName = entry.first;
+            _connectionSpec = entry.second;
         }
     }
+}
+
+RPCService::~RPCService() = default;
+
+RPCServiceAddress::UP
+RPCService::make_address()
+{
+    if ( !_serviceName.empty()) {
+        return std::make_unique<RPCServiceAddress>(_serviceName, _connectionSpec);
+    }
     return RPCServiceAddress::UP();
 }
 
diff --git a/messagebus/src/vespa/messagebus/network/rpcservice.h b/messagebus/src/vespa/messagebus/network/rpcservice.h
index 18c847b0298..13792163693 100644
--- a/messagebus/src/vespa/messagebus/network/rpcservice.h
+++ b/messagebus/src/vespa/messagebus/network/rpcservice.h
@@ -16,13 +16,9 @@ class RPCNetwork;
 class RPCService {
 private:
     typedef slobrok::api::IMirrorAPI  Mirror;
-    typedef Mirror::SpecList          AddressList;
 
-    const Mirror &_mirror;
-    string        _pattern;
-    uint32_t      _addressIdx;
-    uint32_t      _addressGen;
-    AddressList   _addressList;
+    string        _serviceName;
+    string        _connectionSpec;
 
 public:
     using UP = std::unique_ptr<RPCService>;
@@ -44,15 +40,9 @@ public:
      *
      * @return A concrete service address.
      */
-    RPCServiceAddress::UP resolve();
+    RPCServiceAddress::UP make_address();
 
-    /**
-     * Returns the pattern used when querying for the naming server for
-     * addresses. This is given at construtor time.
-     *
-     * @return The service pattern.
-     */
-    const string &getPattern() const { return _pattern; }
+    bool isValid() const { return ! _connectionSpec.empty(); }
 };
 
 } // namespace mbus
diff --git a/messagebus/src/vespa/messagebus/network/rpcserviceaddress.cpp b/messagebus/src/vespa/messagebus/network/rpcserviceaddress.cpp
index eac33195caa..e76832f0620 100644
--- a/messagebus/src/vespa/messagebus/network/rpcserviceaddress.cpp
+++ b/messagebus/src/vespa/messagebus/network/rpcserviceaddress.cpp
@@ -16,7 +16,7 @@ RPCServiceAddress::RPCServiceAddress(const string &serviceName,
     }
 }
 
-RPCServiceAddress::~RPCServiceAddress() {}
+RPCServiceAddress::~RPCServiceAddress() = default;
 
 bool
 RPCServiceAddress::isMalformed()
diff --git a/messagebus/src/vespa/messagebus/network/rpcserviceaddress.h b/messagebus/src/vespa/messagebus/network/rpcserviceaddress.h
index 36dde19bd18..77d64517c40 100644
--- a/messagebus/src/vespa/messagebus/network/rpcserviceaddress.h
+++ b/messagebus/src/vespa/messagebus/network/rpcserviceaddress.h
@@ -32,9 +32,8 @@ public:
      * @param serviceName    The full service name of the address.
      * @param connectionSpec The connection specification.
      */
-    RPCServiceAddress(const string &serviceName,
-                      const string &connectionSpec);
-    ~RPCServiceAddress();
+    RPCServiceAddress(const string &serviceName, const string &connectionSpec);
+    ~RPCServiceAddress() override;
 
     /**
      * Returns whether or not this service address is malformed.
@@ -69,7 +68,7 @@ public:
      *
      * @param target The target to set.
      */
-    void setTarget(RPCTarget::SP target) { _target = target; }
+    void setTarget(RPCTarget::SP target) { _target = std::move(target); }
 
     /**
      * Returns the RPC target to be used when communicating with the remove service. Make sure that {@link
@@ -84,7 +83,7 @@ public:
      *
      * @return True if target is set.
      */
-    bool hasTarget() const { return _target.get() != nullptr; }
+    bool hasTarget() const { return bool(_target); }
 };
 
 } // namespace mbus
diff --git a/messagebus/src/vespa/messagebus/network/rpcservicepool.cpp b/messagebus/src/vespa/messagebus/network/rpcservicepool.cpp
index b306cf29cf9..358698570d2 100644
--- a/messagebus/src/vespa/messagebus/network/rpcservicepool.cpp
+++ b/messagebus/src/vespa/messagebus/network/rpcservicepool.cpp
@@ -6,41 +6,68 @@
 
 namespace mbus {
 
-RPCServicePool::RPCServicePool(RPCNetwork &net, uint32_t maxSize) :
-    _net(net),
-    _lru(maxSize)
+RPCServicePool::RPCServicePool(const slobrok::api::IMirrorAPI & mirror, uint32_t maxSize) :
+    _mirror(mirror),
+    _lock(),
+    _lru(std::make_unique<ServiceCache>(maxSize)),
+    _updateGen(0),
+    _maxSize(maxSize)
 {
-    _lru.reserve(maxSize);
+    _lru->reserve(maxSize);
     assert(maxSize > 0);
 }
 
-RPCServicePool::~RPCServicePool()
-{
-}
+RPCServicePool::~RPCServicePool() = default;
 
 RPCServiceAddress::UP
 RPCServicePool::resolve(const string &pattern)
 {
-    if (_lru.hasKey(pattern)) {
-        return _lru[pattern]->resolve();
+    std::shared_ptr<RPCService> service;
+    {
+        LockGuard guard(_lock);
+        handleMirrorUpdates(guard);
+        std::shared_ptr<RPCService> *found = _lru->findAndRef(pattern);
+        if (found) {
+            service = *found;
+        }
+    }
+
+    if (service) {
+        return service->make_address();
     } else {
-        RPCService::UP service(new RPCService(_net.getMirror(), pattern));
-        auto result = service->resolve();
-        _lru[pattern] = std::move(service);
+        service = std::make_shared<RPCService>(_mirror, pattern);
+        auto result = service->make_address();
+        if (service->isValid()) {
+            LockGuard guard(_lock);
+            (*_lru)[pattern] = std::move(service);
+        }
         return result;
     }
+
+}
+
+void
+RPCServicePool::handleMirrorUpdates(const LockGuard &) {
+    uint32_t currentgen = _mirror.updates();
+    if (_updateGen != currentgen) {
+        auto lru = std::make_unique<ServiceCache>(_maxSize);
+        _lru.swap(lru);
+        _updateGen = currentgen;
+    }
 }
 
 uint32_t
 RPCServicePool::getSize() const
 {
-    return _lru.size();
+    LockGuard guard(_lock);
+    return _lru->size();
 }
 
 bool
 RPCServicePool::hasService(const string &pattern) const
 {
-    return _lru.hasKey(pattern);
+    LockGuard guard(_lock);
+    return _lru->hasKey(pattern);
 }
 
 } // namespace mbus
diff --git a/messagebus/src/vespa/messagebus/network/rpcservicepool.h b/messagebus/src/vespa/messagebus/network/rpcservicepool.h
index 2614363838c..212c975a38c 100644
--- a/messagebus/src/vespa/messagebus/network/rpcservicepool.h
+++ b/messagebus/src/vespa/messagebus/network/rpcservicepool.h
@@ -13,12 +13,6 @@ class RPCNetwork;
  * the rpc network.
  */
 class RPCServicePool {
-private:
-    typedef vespalib::lrucache_map< vespalib::LruParam<string, RPCService::UP> > ServiceCache;
-
-    RPCNetwork    &_net;
-    ServiceCache   _lru;
-
 public:
     RPCServicePool(const RPCServicePool &) = delete;
     RPCServicePool & operator = (const RPCServicePool &) = delete;
@@ -28,7 +22,7 @@ public:
      * @param net     The underlying RPC network.
      * @param maxSize The max number of services to cache.
      */
-    RPCServicePool(RPCNetwork &net, uint32_t maxSize);
+    RPCServicePool(const slobrok::api::IMirrorAPI & mirror, uint32_t maxSize);
 
     /**
      * Destructor. Frees any allocated resources.
@@ -61,6 +55,17 @@ public:
      * @return True if a corresponding service is in the pool.
      */
     bool hasService(const string &pattern) const;
+private:
+    using ServiceCache = vespalib::lrucache_map< vespalib::LruParam<string, std::shared_ptr<RPCService> >>;
+    using LockGuard = std::lock_guard<std::mutex>;
+
+    void handleMirrorUpdates(const LockGuard & guard);
+
+    const slobrok::api::IMirrorAPI & _mirror;
+    mutable std::mutex               _lock;
+    std::unique_ptr<ServiceCache>    _lru;
+    uint32_t                         _updateGen;
+    uint32_t                         _maxSize;
 };
 
 } // namespace mbus
diff --git a/messagebus/src/vespa/messagebus/network/rpctarget.cpp b/messagebus/src/vespa/messagebus/network/rpctarget.cpp
index 63470b6b707..ea21010e21c 100644
--- a/messagebus/src/vespa/messagebus/network/rpctarget.cpp
+++ b/messagebus/src/vespa/messagebus/network/rpctarget.cpp
@@ -25,12 +25,14 @@ RPCTarget::~RPCTarget()
 void
 RPCTarget::resolveVersion(duration timeout, RPCTarget::IVersionHandler &handler)
 {
-    bool hasVersion = false;
     bool shouldInvoke = false;
-    {
+    ResolveState state = _state.load(std::memory_order_acquire);
+    bool hasVersion = (state == VERSION_RESOLVED);
+    if ( ! hasVersion ) {
         vespalib::MonitorGuard guard(_lock);
-        if (_state == VERSION_RESOLVED || _state == PROCESSING_HANDLERS) {
-            while (_state == PROCESSING_HANDLERS) {
+        state = _state.load(std::memory_order_relaxed);
+        if (state == VERSION_RESOLVED || state == PROCESSING_HANDLERS) {
+            while (_state.load(std::memory_order::memory_order_relaxed) == PROCESSING_HANDLERS) {
                 guard.wait();
             }
             hasVersion = true;
@@ -54,11 +56,11 @@ RPCTarget::resolveVersion(duration timeout, RPCTarget::IVersionHandler &handler)
 bool
 RPCTarget::isValid() const
 {
-    vespalib::MonitorGuard guard(_lock);
     if (_target.IsValid()) {
         return true;
     }
-    if (_state == TARGET_INVOKED || _state == PROCESSING_HANDLERS) {
+    ResolveState state = _state.load(std::memory_order_relaxed);
+    if (state == TARGET_INVOKED || state == PROCESSING_HANDLERS) {
         return true; // keep alive until RequestDone() is called
     }
     return false;
diff --git a/messagebus/src/vespa/messagebus/network/rpctarget.h b/messagebus/src/vespa/messagebus/network/rpctarget.h
index b6488f25cb7..d927292f26d 100644
--- a/messagebus/src/vespa/messagebus/network/rpctarget.h
+++ b/messagebus/src/vespa/messagebus/network/rpctarget.h
@@ -50,13 +50,13 @@ private:
     };
     typedef std::unique_ptr<vespalib::Version> Version_UP;
 
-    vespalib::Monitor _lock;
-    FRT_Supervisor   &_orb;
-    string            _name;
-    FRT_Target       &_target;
-    ResolveState      _state;
-    Version_UP        _version;
-    HandlerList       _versionHandlers;
+    vespalib::Monitor          _lock;
+    FRT_Supervisor            &_orb;
+    string                     _name;
+    FRT_Target                &_target;
+    std::atomic<ResolveState>  _state;
+    Version_UP                 _version;
+    HandlerList                _versionHandlers;
 
 public:
     /**
diff --git a/messagebus/src/vespa/messagebus/network/rpctargetpool.cpp b/messagebus/src/vespa/messagebus/network/rpctargetpool.cpp
index 7fcc214faa7..b42ac47e54d 100644
--- a/messagebus/src/vespa/messagebus/network/rpctargetpool.cpp
+++ b/messagebus/src/vespa/messagebus/network/rpctargetpool.cpp
@@ -32,7 +32,7 @@ void
 RPCTargetPool::flushTargets(bool force)
 {
     uint64_t currentTime = _timer->getMilliTime();
-    vespalib::LockGuard guard(_lock);
+    LockGuard guard(_lock);
     TargetMap::iterator it = _targets.begin();
     while (it != _targets.end()) {
         Entry &entry = it->second;
@@ -56,26 +56,27 @@ RPCTargetPool::flushTargets(bool force)
 size_t
 RPCTargetPool::size()
 {
-    vespalib::LockGuard guard(_lock);
+    LockGuard guard(_lock);
     return _targets.size();
 }
 
 RPCTarget::SP
 RPCTargetPool::getTarget(FRT_Supervisor &orb, const RPCServiceAddress &address)
 {
-    vespalib::LockGuard guard(_lock);
-    string spec = address.getConnectionSpec();
-    TargetMap::iterator it = _targets.find(spec);
+    const string & spec = address.getConnectionSpec();
+    uint64_t currentTime = _timer->getMilliTime();
+    LockGuard guard(_lock);
+    auto it = _targets.find(spec);
     if (it != _targets.end()) {
         Entry &entry = it->second;
         if (entry._target->isValid()) {
-            entry._lastUse = _timer->getMilliTime();
+            entry._lastUse = currentTime;
             return entry._target;
         }
         _targets.erase(it);
     }
-    RPCTarget::SP ret(new RPCTarget(spec, orb));
-    _targets.insert(TargetMap::value_type(spec, Entry(ret, _timer->getMilliTime())));
+    auto ret = std::make_shared<RPCTarget>(spec, orb);
+    _targets.insert(TargetMap::value_type(spec, Entry(ret, currentTime)));
     return ret;
 }
 
diff --git a/messagebus/src/vespa/messagebus/network/rpctargetpool.h b/messagebus/src/vespa/messagebus/network/rpctargetpool.h
index 5f858f66993..d47fd977356 100644
--- a/messagebus/src/vespa/messagebus/network/rpctargetpool.h
+++ b/messagebus/src/vespa/messagebus/network/rpctargetpool.h
@@ -28,9 +28,10 @@ private:
 
         Entry(RPCTarget::SP target, uint64_t lastUse);
     };
-    typedef std::map<string, Entry> TargetMap;
+    using TargetMap = std::map<string, Entry>;
+    using LockGuard = std::lock_guard<std::mutex>;
 
-    vespalib::Lock _lock;
+    std::mutex     _lock;
     TargetMap      _targets;
     ITimer::UP     _timer;
     uint64_t       _expireMillis;
diff --git a/messagebus/src/vespa/messagebus/routable.cpp b/messagebus/src/vespa/messagebus/routable.cpp
index 6c62d7e75ca..801029c5bc7 100644
--- a/messagebus/src/vespa/messagebus/routable.cpp
+++ b/messagebus/src/vespa/messagebus/routable.cpp
@@ -11,7 +11,7 @@ Routable::Routable() :
     _trace()
 { }
 
-Routable::~Routable() { }
+Routable::~Routable() = default;
 
 void
 Routable::discard()
diff --git a/messagebus/src/vespa/messagebus/routing/hop.cpp b/messagebus/src/vespa/messagebus/routing/hop.cpp
index 232ae9f840a..76f58d57d7b 100644
--- a/messagebus/src/vespa/messagebus/routing/hop.cpp
+++ b/messagebus/src/vespa/messagebus/routing/hop.cpp
@@ -25,41 +25,19 @@ Hop::Hop(std::vector<IHopDirective::SP> selector, bool ignoreResult) :
 
 Hop::Hop(const Hop &) = default;
 Hop & Hop::operator = (const Hop &) = default;
-Hop::~Hop() { }
+Hop::~Hop() = default;
 
 Hop &
 Hop::addDirective(IHopDirective::SP dir)
 {
-    _selector.push_back(dir);
+    _selector.emplace_back(std::move(dir));
     return *this;
 }
 
 Hop &
 Hop::setDirective(uint32_t i, IHopDirective::SP dir)
 {
-    _selector[i] = dir;
-    return *this;
-}
-
-IHopDirective::SP
-Hop::removeDirective(uint32_t i)
-{
-    IHopDirective::SP ret = _selector[i];
-    _selector.erase(_selector.begin() + i);
-    return ret;
-}
-
-Hop &
-Hop::clearDirectives()
-{
-    _selector.clear();
-    return *this;
-}
-
-Hop &
-Hop::setIgnoreResult(bool ignoreResult)
-{
-    _ignoreResult = ignoreResult;
+    _selector[i] = std::move(dir);
     return *this;
 }
 
@@ -76,7 +54,7 @@ Hop::matches(const Hop &hop) const
         return false;
     }
     for (uint32_t i = 0; i < hop.getNumDirectives(); ++i) {
-        if (!_selector[i]->matches(*hop.getDirective(i))) {
+        if (!_selector[i]->matches(hop.getDirective(i))) {
             return false;
         }
     }
diff --git a/messagebus/src/vespa/messagebus/routing/hop.h b/messagebus/src/vespa/messagebus/routing/hop.h
index 6e7ab94caa1..94140bdd953 100644
--- a/messagebus/src/vespa/messagebus/routing/hop.h
+++ b/messagebus/src/vespa/messagebus/routing/hop.h
@@ -77,7 +77,8 @@ public:
      * @param i The index of the directive to return.
      * @return The item.
      */
-    IHopDirective::SP getDirective(uint32_t i) const { return _selector[i]; }
+    const IHopDirective & getDirective(uint32_t i) const { return *_selector[i]; }
+    IHopDirective::SP getDirectiveSP(uint32_t i) const { return _selector[i]; }
 
     /**
      * Sets the directive at a given index.
@@ -89,21 +90,6 @@ public:
     Hop &setDirective(uint32_t i, IHopDirective::SP dir);
 
     /**
-     * Removes the directive at the given index.
-     *
-     * @param i The index of the directive to remove.
-     * @return The removed directive.
-     */
-    IHopDirective::SP removeDirective(uint32_t i);
-
-    /**
-     * Clears all directives from this hop.
-     *
-     * @return This, to allow chaining.
-     */
-    Hop &clearDirectives();
-
-    /**
      * Returns the service name referenced by this hop. This is the concatenation of all selector primitives,
      * but with no ignore-result prefix.
      *
@@ -124,7 +110,10 @@ public:
      * @param ignoreResult Whether or not to ignore the result.
      * @return This, to allow chaining.
      */
-    Hop &setIgnoreResult(bool ignoreResult);
+    Hop &setIgnoreResult(bool ignoreResult) {
+        _ignoreResult = ignoreResult;
+        return *this;
+    }
 
     /**
      * Parses the given string as a single hop. The {@link this#toString()} method is compatible with this parser.
diff --git a/messagebus/src/vespa/messagebus/routing/hopblueprint.cpp b/messagebus/src/vespa/messagebus/routing/hopblueprint.cpp
index 0bd9aaa6c50..b9d6780fd63 100644
--- a/messagebus/src/vespa/messagebus/routing/hopblueprint.cpp
+++ b/messagebus/src/vespa/messagebus/routing/hopblueprint.cpp
@@ -11,26 +11,17 @@ HopBlueprint::HopBlueprint(const HopSpec &spec) :
 {
     Hop hop = Hop::parse(spec.getSelector());
     for (uint32_t i = 0; i < hop.getNumDirectives(); ++i) {
-        _selector.push_back(hop.getDirective(i));
+        _selector.emplace_back(hop.getDirectiveSP(i));
     }
     std::vector<string> lst;
     for (uint32_t i = 0; i < spec.getNumRecipients(); ++i) {
-        lst.push_back(spec.getRecipient(i));
+        lst.emplace_back(spec.getRecipient(i));
     }
-    for (std::vector<string>::iterator it = lst.begin();
-         it != lst.end(); ++it)
-    {
-        _recipients.push_back(Hop::parse(*it));
+    for (const string & recipient : lst) {
+        _recipients.emplace_back(Hop::parse(recipient));
     }
 }
 
-HopBlueprint &
-HopBlueprint::setIgnoreResult(bool ignoreResult)
-{
-    _ignoreResult = ignoreResult;
-    return *this;
-}
-
 string
 HopBlueprint::toString() const
 {
diff --git a/messagebus/src/vespa/messagebus/routing/hopblueprint.h b/messagebus/src/vespa/messagebus/routing/hopblueprint.h
index 1ae8def398d..21b449651f1 100644
--- a/messagebus/src/vespa/messagebus/routing/hopblueprint.h
+++ b/messagebus/src/vespa/messagebus/routing/hopblueprint.h
@@ -84,14 +84,6 @@ public:
     bool getIgnoreResult() const { return _ignoreResult; }
 
     /**
-     * Sets whether or not to ignore the result when routing through this hop.
-     *
-     * @param ignoreResult Whether or not to ignore the result.
-     * @return This, to allow chaining.
-     */
-    HopBlueprint &setIgnoreResult(bool ignoreResult);
-
-    /**
      * Returns a string representation of this.
      *
      * @return The string.
diff --git a/messagebus/src/vespa/messagebus/routing/routeparser.cpp b/messagebus/src/vespa/messagebus/routing/routeparser.cpp
index 88d5d6e4a0f..c9c10b38014 100644
--- a/messagebus/src/vespa/messagebus/routing/routeparser.cpp
+++ b/messagebus/src/vespa/messagebus/routing/routeparser.cpp
@@ -85,8 +85,8 @@ RouteParser::createHop(stringref str)
     }
     if (len > 4 && str.substr(0, 4) == "tcp/") {
         IHopDirective::SP tcp = createTcpDirective(str.substr(4));
-        if (tcp.get() != nullptr) {
-            return Hop().addDirective(tcp);
+        if (tcp) {
+            return Hop().addDirective(std::move(tcp));
         }
     }
     if (len > 6 && str.substr(0, 6) == "route:") {
@@ -128,7 +128,7 @@ RouteParser::createRoute(stringref str)
             if (from < at - 1) {
                 Hop hop = createHop(str.substr(from, at - from));
                 if (hop.hasDirectives() &&
-                    hop.getDirective(0)->getType() == IHopDirective::TYPE_ERROR)
+                    hop.getDirective(0).getType() == IHopDirective::TYPE_ERROR)
                 {
                     return std::move(Route().addHop(std::move(hop)));
                 }
diff --git a/messagebus/src/vespa/messagebus/routing/routingcontext.cpp b/messagebus/src/vespa/messagebus/routing/routingcontext.cpp
index 12aaa160eab..3da0ad700a3 100644
--- a/messagebus/src/vespa/messagebus/routing/routingcontext.cpp
+++ b/messagebus/src/vespa/messagebus/routing/routingcontext.cpp
@@ -44,14 +44,13 @@ RoutingContext::getMatchedRecipients(std::vector<Route> &ret) const
     std::set<string> done;
     const std::vector<Route> &recipients = _node.getRecipients();
     const Hop &hop = getHop();
-    for (std::vector<Route>::const_iterator it = recipients.begin();
-         it != recipients.end(); ++it)
+    for (const Route & recipient : recipients)
     {
-        if (it->hasHops() && hop.matches(it->getHop(0))) {
-            IHopDirective::SP dir = it->getHop(0).getDirective(_directive);
+        if (recipient.hasHops() && hop.matches(recipient.getHop(0))) {
+            IHopDirective::SP dir = recipient.getHop(0).getDirectiveSP(_directive);
             string key = dir->toString();
             if (done.find(key) == done.end()) {
-                Route add = *it;
+                Route add = recipient;
                 add.setHop(0, hop);
                 add.getHop(0).setDirective(_directive, std::move(dir));
                 ret.push_back(std::move(add));
@@ -95,7 +94,7 @@ RoutingContext::getDirectiveIndex() const
 const PolicyDirective &
 RoutingContext::getDirective() const
 {
-    return static_cast<const PolicyDirective&>(*getHop().getDirective(_directive));
+    return static_cast<const PolicyDirective&>(getHop().getDirective(_directive));
 }
 
 string
diff --git a/messagebus/src/vespa/messagebus/routing/routingnode.cpp b/messagebus/src/vespa/messagebus/routing/routingnode.cpp
index b5b99ea43c2..a47abe185cc 100644
--- a/messagebus/src/vespa/messagebus/routing/routingnode.cpp
+++ b/messagebus/src/vespa/messagebus/routing/routingnode.cpp
@@ -67,10 +67,8 @@ RoutingNode::~RoutingNode()
 void
 RoutingNode::clearChildren()
 {
-    for (std::vector<RoutingNode*>::iterator it = _children.begin();
-         it != _children.end(); ++it)
-    {
-        delete *it;
+    for (auto * child : _children) {
+        delete child;
     }
     _children.clear();
 }
@@ -101,15 +99,12 @@ RoutingNode::prepareForRetry()
 {
     _shouldRetry = false;
     _reply.reset();
-    if (_routingContext.get() != nullptr && _routingContext->getSelectOnRetry()) {
+    if (_routingContext && _routingContext->getSelectOnRetry()) {
         clearChildren();
     } else if (!_children.empty()) {
         bool retryingSome = false;
-        for (std::vector<RoutingNode*>::iterator it = _children.begin();
-             it != _children.end(); ++it)
-        {
-            RoutingNode *child= *it;
-            if (child->_shouldRetry || child->_reply.get() == nullptr) {
+        for (auto * child : _children) {
+            if (child->_shouldRetry || ! child->_reply) {
                 child->prepareForRetry();
                 retryingSome = true;
             }
@@ -159,7 +154,7 @@ RoutingNode::setError(uint32_t code, const string &msg)
 void
 RoutingNode::setError(const Error &err)
 {
-    Reply::UP reply(new EmptyReply());
+    auto reply = std::make_unique<EmptyReply>();
     reply->getTrace().setLevel(_trace.getLevel());
     reply->addError(err);
     setReply(std::move(reply));
@@ -188,8 +183,10 @@ RoutingNode::setReply(Reply::UP reply)
 {
     if (reply) {
         _shouldRetry = _resender != nullptr && _resender->shouldRetry(*reply);
-        _trace.getRoot().addChild(std::move(reply->getTrace().getRoot()));
-        reply->getTrace().clear();
+        if ( ! reply->getTrace().getRoot().isEmpty()) {
+            _trace.getRoot().addChild(std::move(reply->getTrace().getRoot()));
+            reply->getTrace().clear();
+        }
     }
     _reply = std::move(reply);
 }
@@ -211,16 +208,14 @@ RoutingNode::notifyAbort(const string &msg)
         mystack.pop();
         if (!node->_isActive) {
             // reply not pending
-        } else if (node->_reply.get() != nullptr) {
+        } else if (node->_reply) {
             node->notifyParent();
         } else if (node->_children.empty()) {
             node->setError(ErrorCode::SEND_ABORTED, msg);
             node->notifyParent();
         } else {
-            for (std::vector<RoutingNode*>::iterator it = node->_children.begin();
-                 it != node->_children.end(); ++it)
-            {
-                mystack.push(*it);
+            for (auto * child : node->_children) {
+                mystack.push(child);
             }
         }
     }
@@ -240,14 +235,12 @@ RoutingNode::notifyTransmit()
                 if (node->hasReply()) {
                     node->notifyParent();
                 } else {
-                    assert(node->_serviceAddress.get() != nullptr);
+                    assert(node->_serviceAddress);
                     sendTo.push_back(node);
                 }
             } else {
-                for (std::vector<RoutingNode*>::iterator it = node->_children.begin();
-                     it != node->_children.end(); ++it)
-                {
-                    mystack.push(*it);
+                for (auto * child : node->_children) {
+                    mystack.push(child);
                 }
             }
         }
@@ -275,10 +268,8 @@ RoutingNode::notifyMerge()
     // manipulating the trace in case tracing is disabled.
     if (_trace.getLevel() > 0) {
         TraceNode tail;
-        for (std::vector<RoutingNode*>::iterator it = _children.begin();
-             it != _children.end(); ++it)
-        {
-            TraceNode &root = (*it)->_trace.getRoot();
+        for (auto * child : _children) {
+            TraceNode &root = child->_trace.getRoot();
             tail.addChild(root);
             root.clear();
         }
@@ -296,7 +287,7 @@ RoutingNode::notifyMerge()
         setError(ErrorCode::POLICY_ERROR, make_string("Policy '%s' threw an exception; %s",
                                                       dir.getName().c_str(), e.what()));
     }
-    if (_reply.get() == nullptr) {
+    if ( ! _reply) {
         setError(ErrorCode::APP_FATAL_ERROR, make_string("Routing policy '%s' failed to merge replies.",
                                                          dir.getName().c_str()));
     }
@@ -315,12 +306,12 @@ RoutingNode::hasUnconsumedErrors()
     while (!mystack.empty()) {
         RoutingNode *node = mystack.top();
         mystack.pop();
-        if (node->_reply.get() != nullptr) {
+        if (node->_reply) {
             for (uint32_t i = 0; i < node->_reply->getNumErrors(); ++i) {
                 int errorCode = node->_reply->getError(i).getCode();
                 RoutingNode *it = node;
                 while (it != nullptr) {
-                    if (it->_routingContext.get() != nullptr &&
+                    if (it->_routingContext &&
                         it->_routingContext->isConsumableError(errorCode))
                     {
                         errorCode = ErrorCode::NONE;
@@ -337,10 +328,8 @@ RoutingNode::hasUnconsumedErrors()
                 }
             }
         } else {
-            for (std::vector<RoutingNode*>::iterator it = node->_children.begin();
-                 it != node->_children.end(); ++it)
-            {
-                mystack.push(*it);
+            for (auto * child : node->_children) {
+                mystack.push(child);
             }
         }
     }
@@ -374,17 +363,17 @@ RoutingNode::resolve(uint32_t depth)
         if (executePolicySelect()) {
             return resolveChildren(depth + 1);
         }
-        return _reply.get() != nullptr;
+        return bool(_reply);
     }
     _net.allocServiceAddress(*this);
-    return _serviceAddress.get() != nullptr || _reply.get() != nullptr;
+    return _serviceAddress || _reply;
 }
 
 bool
 RoutingNode::lookupHop()
 {
     RoutingTable::SP table = _mbus.getRoutingTable(_msg.getProtocol());
-    if (table.get() != nullptr) {
+    if (table) {
         string name = _route.getHop(0).getServiceName();
         if (table->hasHop(name)) {
             const HopBlueprint *hop = table->getHop(name);
@@ -402,8 +391,9 @@ RoutingNode::lookupRoute()
 {
     RoutingTable::SP table = _mbus.getRoutingTable(_msg.getProtocol());
     Hop &hop = _route.getHop(0);
-    if (hop.getDirective(0)->getType() == IHopDirective::TYPE_ROUTE) {
-        RouteDirective &dir = static_cast<RouteDirective&>(*hop.getDirective(0));
+    const RouteDirective &dir = static_cast<const RouteDirective&>(hop.getDirective(0));
+    if (dir.getType() == IHopDirective::TYPE_ROUTE) {
+
         if (!table || !table->hasRoute(dir.getName())) {
             setError(ErrorCode::ILLEGAL_ROUTE, make_string("Route '%s' does not exist.", dir.getName().c_str()));
             return false;
@@ -443,10 +433,10 @@ RoutingNode::findErrorDirective()
 {
     Hop &hop = _route.getHop(0);
     for (uint32_t i = 0; i < hop.getNumDirectives(); ++i) {
-        IHopDirective::SP dir = hop.getDirective(i);
-        if (dir->getType() == IHopDirective::TYPE_ERROR) {
+        const IHopDirective & dir = hop.getDirective(i);
+        if (dir.getType() == IHopDirective::TYPE_ERROR) {
             setError(ErrorCode::ILLEGAL_ROUTE,
-                     static_cast<ErrorDirective&>(*dir).getMessage());
+                     static_cast<const ErrorDirective&>(dir).getMessage());
             return true;
         }
     }
@@ -458,9 +448,8 @@ RoutingNode::findPolicyDirective()
 {
     Hop &hop = _route.getHop(0);
     for (uint32_t i = 0; i < hop.getNumDirectives(); ++i) {
-        IHopDirective::SP dir = hop.getDirective(i);
-        if (dir->getType() == IHopDirective::TYPE_POLICY) {
-            _routingContext.reset(new RoutingContext(*this, i));
+        if (hop.getDirective(i).getType() == IHopDirective::TYPE_POLICY) {
+            _routingContext = std::make_unique<RoutingContext>(*this, i);
             return true;
         }
     }
@@ -472,7 +461,7 @@ RoutingNode::executePolicySelect()
 {
     const PolicyDirective &dir = _routingContext->getDirective();
     _policy = _mbus.getRoutingPolicy(_msg.getProtocol(), dir.getName(), dir.getParam());
-    if (_policy.get() == nullptr) {
+    if ( ! _policy) {
         setError(ErrorCode::UNKNOWN_POLICY, make_string(
                 "Protocol '%s' could not create routing policy '%s' with parameter '%s'.",
                 _msg.getProtocol().c_str(), dir.getName().c_str(), dir.getParam().c_str()));
@@ -487,7 +476,7 @@ RoutingNode::executePolicySelect()
         return false;
     }
     if (_children.empty()) {
-        if (_reply.get() == nullptr) {
+        if ( ! _reply) {
             setError(ErrorCode::NO_SERVICES_FOR_ROUTE,
                      make_string("Policy '%s' selected no recipients for route '%s'.",
                                  dir.getName().c_str(), _route.toString().c_str()));
@@ -497,10 +486,7 @@ RoutingNode::executePolicySelect()
         }
         return false;
     }
-    for (std::vector<RoutingNode*>::iterator it = _children.begin();
-         it != _children.end(); ++it)
-    {
-        RoutingNode *child = *it;
+    for (auto * child : _children) {
         Hop &hop = child->_route.getHop(0);
         child->_trace.trace(TraceLevel::SPLIT_MERGE,
                             make_string("Component '%s' selected by policy '%s'.",
@@ -514,13 +500,10 @@ RoutingNode::resolveChildren(uint32_t childDepth)
 {
     int numActiveChildren = 0;
     bool ret = true;
-    for (std::vector<RoutingNode*>::iterator it = _children.begin();
-         it != _children.end(); ++it)
-    {
-        RoutingNode *child = *it;
+    for (auto * child : _children) {
         child->_trace.trace(TraceLevel::SPLIT_MERGE,
                             make_string("Resolving '%s'.", child->_route.toString().c_str()));
-        child->_isActive = (child->_reply.get() == nullptr);
+        child->_isActive = ! child->_reply;
         if (child->_isActive) {
             ++numActiveChildren;
             if (!child->resolve(childDepth)) {
@@ -560,10 +543,10 @@ RoutingNode::tryIgnoreResult()
     if (!shouldIgnoreResult()) {
         return false;
     }
-    if (_reply.get() == nullptr || !_reply->hasErrors()) {
+    if ( ! _reply || !_reply->hasErrors()) {
         return false;
     }
-    setReply(Reply::UP(new EmptyReply()));
+    setReply(std::make_unique<EmptyReply>());
     _trace.trace(TraceLevel::SPLIT_MERGE, "Ignoring errors in reply.");
     return true;
 }
diff --git a/messagebus/src/vespa/messagebus/trace.h b/messagebus/src/vespa/messagebus/trace.h
index a5f02ea2fa4..5a26cb68e46 100644
--- a/messagebus/src/vespa/messagebus/trace.h
+++ b/messagebus/src/vespa/messagebus/trace.h
@@ -3,11 +3,11 @@
 #pragma once
 
 #include <vespa/vespalib/trace/trace.h>
-#include <vespa/messagebus/tracenode.h>
 
 namespace mbus {
 
-    typedef vespalib::Trace Trace;
+    using Trace = vespalib::Trace;
+    using TraceNode = vespalib::TraceNode;
 
 #define MBUS_TRACE2(ttrace, level, note, addTime) \
     VESPALIB_TRACE2(ttrace, level, note, addTime)
diff --git a/messagebus/src/vespa/messagebus/tracenode.h b/messagebus/src/vespa/messagebus/tracenode.h
deleted file mode 100644
index f582a70a151..00000000000
--- a/messagebus/src/vespa/messagebus/tracenode.h
+++ /dev/null
@@ -1,11 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-#pragma once
-
-#include <vespa/vespalib/trace/tracenode.h>
-
-namespace mbus {
-
-    using TraceNode = vespalib::TraceNode;
-
-} // namespace mbus
-
diff --git a/messagebus_test/src/tests/error/cpp-client.cpp b/messagebus_test/src/tests/error/cpp-client.cpp
index abc7967bfe5..6296253d3df 100644
--- a/messagebus_test/src/tests/error/cpp-client.cpp
+++ b/messagebus_test/src/tests/error/cpp-client.cpp
@@ -7,7 +7,6 @@
 #include <vespa/messagebus/rpcmessagebus.h>
 #include <vespa/messagebus/network/rpcnetworkparams.h>
 #include <vespa/messagebus/testlib/receptor.h>
-#include <vespa/vespalib/util/time.h>
 #include <thread>
 #include <vespa/fastos/app.h>
 
@@ -34,11 +33,11 @@ App::Main()
 
     SourceSession::UP ss = mb.getMessageBus().createSourceSession(src, SourceSessionParams().setTimeout(300s));
     for (int i = 0; i < 10; ++i) {
-        msg.reset(new SimpleMessage("test"));
+        msg = std::make_unique<SimpleMessage>("test");
         msg->getTrace().setLevel(9);
         ss->send(std::move(msg), "test");
         reply = src.getReply(600s); // 10 minutes timeout
-        if (reply.get() == 0) {
+        if ( ! reply) {
             fprintf(stderr, "CPP-CLIENT: no reply\n");
         } else {
             fprintf(stderr, "CPP-CLIENT:\n%s\n",
@@ -49,7 +48,7 @@ App::Main()
         }
         std::this_thread::sleep_for(1s);
     }
-    if (reply.get() == 0) {
+    if ( ! reply) {
         fprintf(stderr, "CPP-CLIENT: no reply\n");
         return 1;
     }
diff --git a/messagebus_test/src/tests/error/error.cpp b/messagebus_test/src/tests/error/error.cpp
index e5749db452b..87eb391fc86 100644
--- a/messagebus_test/src/tests/error/error.cpp
+++ b/messagebus_test/src/tests/error/error.cpp
@@ -1,48 +1,47 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-#include <vespa/log/log.h>
-LOG_SETUP("error_test");
+
 #include <vespa/vespalib/testkit/testapp.h>
 #include <vespa/vespalib/testkit/test_kit.h>
 #include <vespa/messagebus/testlib/slobrok.h>
 #include <vespa/vespalib/util/stringfmt.h>
 
+#include <vespa/log/log.h>
+LOG_SETUP("error_test");
+
 using namespace mbus;
 using vespalib::make_string;
 
-TEST_SETUP(Test);
 
-int
-Test::Main()
-{
-    TEST_INIT("error_test");
+TEST("error_test") {
     Slobrok slobrok;
     const std::string routing_template = TEST_PATH("routing-template.cfg");
     const std::string ctl_script = TEST_PATH("ctl.sh");
     
     { // Make slobrok config
-        EXPECT_TRUE(system("echo slobrok[1] > slobrok.cfg") == 0);
-        EXPECT_TRUE(system(make_string("echo 'slobrok[0].connectionspec tcp/localhost:%d' "
-                                      ">> slobrok.cfg", slobrok.port()).c_str()) == 0);
+        EXPECT_EQUAL(0, system("echo slobrok[1] > slobrok.cfg"));
+        EXPECT_EQUAL(0, system(make_string("echo 'slobrok[0].connectionspec tcp/localhost:%d' "
+                                      ">> slobrok.cfg", slobrok.port()).c_str()));
     }
     { // CPP SERVER
         { // Make routing config
-            EXPECT_TRUE(system(("cat " + routing_template + " | sed 's#session#cpp/session#' > routing.cfg").c_str()) == 0);
+            EXPECT_EQUAL(0, system(("cat " + routing_template + " | sed 's#session#cpp/session#' > routing.cfg").c_str()));
         }
         fprintf(stderr, "STARTING CPP-SERVER\n");
-        EXPECT_TRUE(system((ctl_script + " start server cpp").c_str()) == 0);
-        EXPECT_TRUE(system("./messagebus_test_cpp-client-error_app") == 0);
-        EXPECT_TRUE(system("../../binref/runjava JavaClient") == 0);
-        EXPECT_TRUE(system((ctl_script + " stop server cpp").c_str()) == 0);
+        EXPECT_EQUAL(0, system((ctl_script + " start server cpp").c_str()));
+        EXPECT_EQUAL(0, system("./messagebus_test_cpp-client-error_app"));
+        EXPECT_EQUAL(0, system("../../binref/runjava JavaClient"));
+        EXPECT_EQUAL(0, system((ctl_script + " stop server cpp").c_str()));
     }
     { // JAVA SERVER
         { // Make routing config
-            EXPECT_TRUE(system(("cat " + routing_template + " | sed 's#session#java/session#' > routing.cfg").c_str()) == 0);
+            EXPECT_EQUAL(0, system(("cat " + routing_template + " | sed 's#session#java/session#' > routing.cfg").c_str()));
         }
         fprintf(stderr, "STARTING JAVA-SERVER\n");
-        EXPECT_TRUE(system((ctl_script + " start server java").c_str()) == 0);
-        EXPECT_TRUE(system("./messagebus_test_cpp-client-error_app") == 0);
-        EXPECT_TRUE(system("../../binref/runjava JavaClient") == 0);
-        EXPECT_TRUE(system((ctl_script + " stop server java").c_str()) == 0);
+        EXPECT_EQUAL(0, system((ctl_script + " start server java").c_str()));
+        EXPECT_EQUAL(0, system("./messagebus_test_cpp-client-error_app"));
+        EXPECT_EQUAL(0, system("../../binref/runjava JavaClient"));
+        EXPECT_EQUAL(0, system((ctl_script + " stop server java").c_str()));
     }
-    TEST_DONE();
 }
+
+TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/metrics-proxy/CMakeLists.txt b/metrics-proxy/CMakeLists.txt
index 41fedb8e8c4..63520385bf5 100644
--- a/metrics-proxy/CMakeLists.txt
+++ b/metrics-proxy/CMakeLists.txt
@@ -1,10 +1,15 @@
 # Copyright 2019 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 install_fat_java_artifact(metrics-proxy)
 
+vespa_install_script(src/main/sh/start-telegraf.sh libexec/vespa)
+vespa_install_script(src/main/sh/stop-telegraf.sh libexec/vespa)
+
 install_config_definition(src/main/resources/configdefinitions/application-dimensions.def ai.vespa.metricsproxy.metric.dimensions.application-dimensions.def)
 install_config_definition(src/main/resources/configdefinitions/consumers.def ai.vespa.metricsproxy.core.consumers.def)
 install_config_definition(src/main/resources/configdefinitions/monitoring.def ai.vespa.metricsproxy.core.monitoring.def)
 install_config_definition(src/main/resources/configdefinitions/metrics-nodes.def ai.vespa.metricsproxy.http.application.metrics-nodes.def)
 install_config_definition(src/main/resources/configdefinitions/node-dimensions.def ai.vespa.metricsproxy.metric.dimensions.node-dimensions.def)
+install_config_definition(src/main/resources/configdefinitions/node-info.def ai.vespa.metricsproxy.http.metrics.node-info.def)
 install_config_definition(src/main/resources/configdefinitions/rpc-connector.def ai.vespa.metricsproxy.rpc.rpc-connector.def)
 install_config_definition(src/main/resources/configdefinitions/vespa-services.def ai.vespa.metricsproxy.service.vespa-services.def)
+install_config_definition(src/main/resources/configdefinitions/telegraf.def ai.vespa.metricsproxy.telegraf.telegraf.def)
diff --git a/metrics-proxy/pom.xml b/metrics-proxy/pom.xml
index f72ad75c6af..355f420c2a4 100644
--- a/metrics-proxy/pom.xml
+++ b/metrics-proxy/pom.xml
@@ -132,6 +132,10 @@
       <groupId>org.apache.httpcomponents</groupId>
       <artifactId>httpclient</artifactId>
     </dependency>
+    <dependency>
+      <groupId>org.apache.velocity</groupId>
+      <artifactId>velocity</artifactId>
+    </dependency>
 
     <!-- test scope -->
 
diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsManager.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsManager.java
index 4c4015220bc..53a05ef88f0 100644
--- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsManager.java
+++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsManager.java
@@ -79,6 +79,16 @@ public class MetricsManager {
      * @return Metrics for all matching services.
      */
     public List<MetricsPacket> getMetrics(List<VespaService> services, Instant startTime) {
+        return getMetricsAsBuilders(services, startTime).stream()
+                .map(MetricsPacket.Builder::build)
+                .collect(Collectors.toList());
+    }
+
+    /**
+     * Returns the metrics for the given services, in mutable state for further processing.
+     * NOTE: Use {@link #getMetrics(List, Instant)} instead, unless further processing of the metrics is necessary.
+     */
+    public List<MetricsPacket.Builder> getMetricsAsBuilders(List<VespaService> services, Instant startTime) {
         if (services.isEmpty()) return Collections.emptyList();
 
         log.log(DEBUG, () -> "Updating services prior to fetching metrics, number of services= " + services.size());
@@ -99,7 +109,6 @@ public class MetricsManager {
                 .map(builder -> builder.putDimensionsIfAbsent(getGlobalDimensions()))
                 .map(builder -> builder.putDimensionsIfAbsent(extraDimensions))
                 .map(builder -> adjustTimestamp(builder, startTime))
-                .map(MetricsPacket.Builder::build)
                 .collect(Collectors.toList());
     }
 
diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/VespaMetrics.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/VespaMetrics.java
index c9d7618b9d7..c04dca465a1 100644
--- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/VespaMetrics.java
+++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/VespaMetrics.java
@@ -35,6 +35,7 @@ import static com.google.common.base.Strings.isNullOrEmpty;
  * @author gjoranv
  */
 public class VespaMetrics {
+
     private static final Logger log = Logger.getLogger(VespaMetrics.class.getPackage().getName());
 
     public static final ConsumerId VESPA_CONSUMER_ID = toConsumerId("Vespa");
diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/ValuesFetcher.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/ValuesFetcher.java
index 768c1beebef..51bdae1aab3 100644
--- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/ValuesFetcher.java
+++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/ValuesFetcher.java
@@ -21,6 +21,7 @@ import static ai.vespa.metricsproxy.metric.model.ConsumerId.toConsumerId;
  * @author gjoranv
  */
 public class ValuesFetcher {
+
     private static final Logger log = Logger.getLogger(ValuesFetcher.class.getName());
 
     public static final ConsumerId DEFAULT_PUBLIC_CONSUMER_ID = toConsumerId("default");
@@ -46,6 +47,16 @@ public class ValuesFetcher {
                 .collect(Collectors.toList());
     }
 
+    public List<MetricsPacket.Builder> fetchMetricsAsBuilders(String requestedConsumer) throws JsonRenderingException {
+        ConsumerId consumer = getConsumerOrDefault(requestedConsumer, metricsConsumers);
+
+        return metricsManager.getMetricsAsBuilders(vespaServices.getVespaServices(), Instant.now())
+                .stream()
+                .filter(builder -> builder.hasConsumer(consumer))
+                .collect(Collectors.toList());
+    }
+
+
     public List<MetricsPacket> fetchAllMetrics() throws JsonRenderingException {
         return metricsManager.getMetrics(vespaServices.getVespaServices(), Instant.now());
     }
diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsHandler.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsHandler.java
index af37590a09d..d9303e80dcd 100644
--- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsHandler.java
+++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsHandler.java
@@ -36,7 +36,7 @@ import static java.util.stream.Collectors.toList;
 public class ApplicationMetricsHandler extends HttpHandlerBase {
 
     public static final String V1_PATH = "/applicationmetrics/v1";
-    static final String VALUES_PATH = V1_PATH + "/values";
+    public static final String VALUES_PATH = V1_PATH + "/values";
 
     private static final int MAX_DIMENSIONS = 10;
 
diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsRetriever.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsRetriever.java
index c51970ce3ae..d20d4b0cb3c 100644
--- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsRetriever.java
+++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsRetriever.java
@@ -6,6 +6,7 @@ import ai.vespa.metricsproxy.metric.model.MetricsPacket;
 import ai.vespa.util.http.VespaHttpClientBuilder;
 import com.google.inject.Inject;
 import com.yahoo.component.AbstractComponent;
+import com.yahoo.log.LogLevel;
 import org.apache.http.client.HttpClient;
 import org.apache.http.client.config.RequestConfig;
 import org.apache.http.impl.client.CloseableHttpClient;
@@ -33,6 +34,7 @@ import static java.util.stream.Collectors.toMap;
  * @author gjoranv
  */
 public class ApplicationMetricsRetriever extends AbstractComponent {
+
     private static final Logger log = Logger.getLogger(ApplicationMetricsRetriever.class.getName());
 
     private static final int PARALLELISM = 20;
@@ -44,11 +46,11 @@ public class ApplicationMetricsRetriever extends AbstractComponent {
 
     private final HttpClient httpClient = createHttpClient();
     private final List<NodeMetricsClient> clients;
+    private final ForkJoinPool forkJoinPool = new ForkJoinPool(PARALLELISM);
 
     // Non-final for testing
     private Duration taskTimeout;
 
-    private ForkJoinPool forkJoinPool = new ForkJoinPool(PARALLELISM);
 
     @Inject
     public ApplicationMetricsRetriever(MetricsNodesConfig nodesConfig) {
@@ -67,7 +69,7 @@ public class ApplicationMetricsRetriever extends AbstractComponent {
     }
 
     public Map<Node, List<MetricsPacket.Builder>> getMetrics(ConsumerId consumer) {
-        log.info(() -> "Retrieving metrics from " + clients.size() + " nodes.");
+        log.log(LogLevel.DEBUG, () -> "Retrieving metrics from " + clients.size() + " nodes.");
         var forkJoinTask = forkJoinPool.submit(() -> clients.parallelStream()
                 .map(client -> getNodeMetrics(client, consumer))
                 .collect(toMap(Map.Entry::getKey, Map.Entry::getValue)));
@@ -75,13 +77,11 @@ public class ApplicationMetricsRetriever extends AbstractComponent {
         try {
             var metricsByNode = forkJoinTask.get(taskTimeout.toMillis(), TimeUnit.MILLISECONDS);
 
-            log.info(() -> "Finished retrieving metrics from " + clients.size() + " nodes.");
+            log.log(LogLevel.DEBUG, () -> "Finished retrieving metrics from " + clients.size() + " nodes.");
             return metricsByNode;
 
         } catch (Exception e) {
             // Since the task is a ForkJoinTask, we don't need special handling of InterruptedException
-            forkJoinPool.shutdownNow();
-            forkJoinPool = new ForkJoinPool(PARALLELISM);
             throw new ApplicationMetricsException("Failed retrieving metrics.", e);
         }
     }
diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/Node.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/Node.java
index c8a8e65be5d..c439a037774 100644
--- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/Node.java
+++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/Node.java
@@ -25,7 +25,7 @@ public class Node {
     }
 
     public Node(String role, String hostname, int port, String path) {
-        Objects.requireNonNull(role, "Null configId is not allowed");
+        Objects.requireNonNull(role, "Null role is not allowed");
         Objects.requireNonNull(hostname, "Null hostname is not allowed");
         Objects.requireNonNull(path, "Null path is not allowed");
 
diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/NodeMetricsClient.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/NodeMetricsClient.java
index f2ee326029a..145eef3f745 100644
--- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/NodeMetricsClient.java
+++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/NodeMetricsClient.java
@@ -13,9 +13,11 @@ import java.io.IOException;
 import java.time.Clock;
 import java.time.Duration;
 import java.time.Instant;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.logging.Logger;
 
 import static com.yahoo.log.LogLevel.DEBUG;
@@ -32,6 +34,7 @@ import static java.util.Collections.emptyList;
  * @author gjoranv
  */
 public class NodeMetricsClient {
+
     private static final Logger log = Logger.getLogger(NodeMetricsClient.class.getName());
 
     static final Duration METRICS_TTL = Duration.ofSeconds(30);
@@ -40,7 +43,7 @@ public class NodeMetricsClient {
     private final HttpClient httpClient;
     private final Clock clock;
 
-    private final Map<ConsumerId, Snapshot> snapshots = new HashMap<>();
+    private final Map<ConsumerId, Snapshot> snapshots = new ConcurrentHashMap<>();
     private long snapshotsRetrieved = 0;
 
     NodeMetricsClient(HttpClient httpClient, Node node, Clock clock) {
@@ -80,7 +83,6 @@ public class NodeMetricsClient {
         return snapshotsRetrieved;
     }
 
-
     /**
      * Convenience class for storing a metrics snapshot with its timestamp.
      */
diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/PublicDimensionsProcessor.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/PublicDimensionsProcessor.java
index 395ec0bea4f..4d1d57644b5 100644
--- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/PublicDimensionsProcessor.java
+++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/PublicDimensionsProcessor.java
@@ -21,7 +21,7 @@ public class PublicDimensionsProcessor implements MetricsProcessor {
     private final int maxDimensions;
     private Set<DimensionId> publicDimensions = getPublicDimensions();
 
-    PublicDimensionsProcessor(int maxDimensions) {
+    public PublicDimensionsProcessor(int maxDimensions) {
         int numCommonDimensions = PublicDimensions.commonDimensions.size();
         if (numCommonDimensions > maxDimensions) {
             throw new IllegalArgumentException(String.format(
diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/metrics/MetricsV2Handler.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/metrics/MetricsV2Handler.java
new file mode 100644
index 00000000000..71d7857e48a
--- /dev/null
+++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/metrics/MetricsV2Handler.java
@@ -0,0 +1,92 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package ai.vespa.metricsproxy.http.metrics;
+
+import ai.vespa.metricsproxy.core.MetricsConsumers;
+import ai.vespa.metricsproxy.core.MetricsManager;
+import ai.vespa.metricsproxy.http.ValuesFetcher;
+import ai.vespa.metricsproxy.http.application.ClusterIdDimensionProcessor;
+import ai.vespa.metricsproxy.http.application.Node;
+import ai.vespa.metricsproxy.http.application.PublicDimensionsProcessor;
+import ai.vespa.metricsproxy.http.application.ServiceIdDimensionProcessor;
+import ai.vespa.metricsproxy.metric.model.MetricsPacket;
+import ai.vespa.metricsproxy.metric.model.processing.MetricsProcessor;
+import ai.vespa.metricsproxy.service.VespaServices;
+import com.google.inject.Inject;
+import com.yahoo.container.handler.metrics.ErrorResponse;
+import com.yahoo.container.handler.metrics.HttpHandlerBase;
+import com.yahoo.container.handler.metrics.JsonResponse;
+import com.yahoo.container.jdisc.HttpResponse;
+import com.yahoo.restapi.Path;
+
+import java.net.URI;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.concurrent.Executor;
+import java.util.logging.Level;
+
+import static ai.vespa.metricsproxy.metric.model.json.GenericJsonUtil.toGenericApplicationModel;
+import static ai.vespa.metricsproxy.metric.model.processing.MetricsProcessor.applyProcessors;
+import static com.yahoo.jdisc.Response.Status.INTERNAL_SERVER_ERROR;
+import static com.yahoo.jdisc.Response.Status.OK;
+import static java.util.Collections.singletonMap;
+import static java.util.stream.Collectors.toList;
+
+/**
+ * Http handler for the metrics/v2 rest api.
+ *
+ * @author gjoranv
+ */
+public class MetricsV2Handler  extends HttpHandlerBase {
+
+    public static final String V2_PATH = "/metrics/v2";
+    public static final String VALUES_PATH = V2_PATH + "/values";
+    private static final int MAX_DIMENSIONS = 10;
+
+    private final ValuesFetcher valuesFetcher;
+    private final NodeInfoConfig nodeInfoConfig;
+
+    @Inject
+    public MetricsV2Handler(Executor executor,
+                            MetricsManager metricsManager,
+                            VespaServices vespaServices,
+                            MetricsConsumers metricsConsumers,
+                            NodeInfoConfig nodeInfoConfig) {
+        super(executor);
+        this.nodeInfoConfig = nodeInfoConfig;
+        valuesFetcher = new ValuesFetcher(metricsManager, vespaServices, metricsConsumers);
+    }
+
+    @Override
+    public Optional<HttpResponse> doHandle(URI requestUri, Path apiPath, String consumer) {
+        if (apiPath.matches(V2_PATH)) return Optional.of(resourceListResponse(requestUri, List.of(VALUES_PATH)));
+        if (apiPath.matches(VALUES_PATH)) return Optional.of(valuesResponse(consumer));
+        return Optional.empty();
+    }
+
+    private JsonResponse valuesResponse(String consumer) {
+        try {
+            List<MetricsPacket.Builder> builders = valuesFetcher.fetchMetricsAsBuilders(consumer);
+            List<MetricsPacket> metrics = processAndBuild(builders,
+                                                          new ServiceIdDimensionProcessor(),
+                                                          new ClusterIdDimensionProcessor(),
+                                                          new PublicDimensionsProcessor(MAX_DIMENSIONS));
+
+            Node localNode = new Node(nodeInfoConfig.role(), nodeInfoConfig.hostname(), 0, "");
+            Map<Node, List<MetricsPacket>> metricsByNode = singletonMap(localNode, metrics);
+            return new JsonResponse(OK, toGenericApplicationModel(metricsByNode).serialize());
+        } catch (Exception e) {
+            log.log(Level.WARNING, "Got exception when rendering metrics:", e);
+            return new ErrorResponse(INTERNAL_SERVER_ERROR, e.getMessage());
+        }
+    }
+
+    private static List<MetricsPacket> processAndBuild(List<MetricsPacket.Builder> builders,
+                                                       MetricsProcessor... processors) {
+        return builders.stream()
+                .map(builder -> applyProcessors(builder, processors))
+                .map(MetricsPacket.Builder::build)
+                .collect(toList());
+    }
+
+}
diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/dimensions/PublicDimensions.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/dimensions/PublicDimensions.java
index 465b419be09..8d525310d6f 100644
--- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/dimensions/PublicDimensions.java
+++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/dimensions/PublicDimensions.java
@@ -22,7 +22,7 @@ public final class PublicDimensions {
     // Node-specific.
     public static final String INTERNAL_CLUSTER_TYPE = "clustertype";
     public static final String INTERNAL_CLUSTER_ID = "clusterid";
-    public static final String CLUSTER_ID = INTERNAL_CLUSTER_ID;
+    public static final String CLUSTER_ID = "clusterId";
 
     // Internal name (instance) is confusing, so renamed to 'serviceId' for public use.
     // This is added by the metrics-proxy.
diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/ConsumerId.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/ConsumerId.java
index 62de9649bb0..795d1005b10 100644
--- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/ConsumerId.java
+++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/ConsumerId.java
@@ -7,6 +7,7 @@ import java.util.Objects;
  * @author gjoranv
  */
 public class ConsumerId {
+
     public final String id;
     private ConsumerId(String id) { this.id = id; }
 
diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/MetricsPacket.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/MetricsPacket.java
index 8ecf57237ef..8d5a1f50918 100644
--- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/MetricsPacket.java
+++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/model/MetricsPacket.java
@@ -179,6 +179,10 @@ public class MetricsPacket {
             return this;
         }
 
+        public boolean hasConsumer(ConsumerId id) {
+            return consumers.contains(id);
+        }
+
         public MetricsPacket build() {
             return new MetricsPacket(statusCode, statusMessage, timestamp, service, metrics, dimensions, consumers);
         }
diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/ConfigSentinelClient.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/ConfigSentinelClient.java
index 09fb66bffab..b5bd6c073cd 100644
--- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/ConfigSentinelClient.java
+++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/ConfigSentinelClient.java
@@ -2,11 +2,18 @@
 package ai.vespa.metricsproxy.service;
 
 import com.google.inject.Inject;
+import com.yahoo.component.AbstractComponent;
 import com.yahoo.log.LogLevel;
 
+import com.yahoo.jrt.ErrorCode;
+import com.yahoo.jrt.Request;
+import com.yahoo.jrt.Spec;
+import com.yahoo.jrt.Supervisor;
+import com.yahoo.jrt.Target;
+import com.yahoo.jrt.Transport;
+
 import java.io.BufferedReader;
-import java.io.InputStream;
-import java.io.InputStreamReader;
+import java.io.StringReader;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.logging.Logger;
@@ -14,14 +21,19 @@ import java.util.logging.Logger;
 /**
  * Connects to the config sentinel and gets information like pid for the services on the node
  */
-public class ConfigSentinelClient {
+public class ConfigSentinelClient extends AbstractComponent {
     private final static Logger log = Logger.getLogger(ConfigSentinelClient.class.getName());
 
-    private final CmdClient client;
+    private final Supervisor supervisor = new Supervisor(new Transport());
 
     @Inject
     public ConfigSentinelClient() {
-        this.client = new CmdClient();
+    }
+
+    @Override
+    public void deconstruct() {
+        supervisor.transport().shutdown().join();
+        super.deconstruct();
     }
 
     /**
@@ -61,11 +73,8 @@ public class ConfigSentinelClient {
      * @throws Exception if something went wrong
      */
     protected synchronized void setStatus(List<VespaService> services) throws Exception {
-        InputStream in;
-        client.connect();
-
-        in = client.getInputStream();
-        BufferedReader reader = new BufferedReader(new InputStreamReader(in));
+        String in = sentinelLs();
+        BufferedReader reader = new BufferedReader(new StringReader(in));
         String line;
         List<VespaService> updatedServices = new ArrayList<>();
         while ((line = reader.readLine()) != null) {
@@ -90,7 +99,6 @@ public class ConfigSentinelClient {
 
         //Close streams
         reader.close();
-        client.disconnect();
     }
 
     static VespaService parseServiceString(String line, List<VespaService> services) {
@@ -144,28 +152,28 @@ public class ConfigSentinelClient {
         return service;
     }
 
-    static class CmdClient {
-        Process proc;
-        // NOTE: hostname/port not used yet
-        void connect() {
-            String[] args = new String[]{"vespa-sentinel-cmd", "list"};
-            try {
-                proc = Runtime.getRuntime().exec(args);
-            } catch (Exception e) {
-                log.log(LogLevel.WARNING, "could not run vespa-sentinel-cmd: "+e);
-                proc = null;
-            }
-        }
-        void disconnect() {
-            if (proc.isAlive()) {
-                proc.destroy();
+    String sentinelLs() {
+        String servicelist = "";
+        int rpcPort = 19097;
+        Spec spec = new Spec("localhost", rpcPort);
+        Target connection = supervisor.connect(spec);
+        try {
+            if (connection.isValid()) {
+                Request req = new Request("sentinel.ls");
+                connection.invokeSync(req, 5.0);
+                if (req.errorCode() == ErrorCode.NONE &&
+                    req.checkReturnTypes("s"))
+                {
+                    servicelist = req.returnValues().get(0).asString();
+                } else {
+                    log.log(LogLevel.WARNING, "Bad answer to RPC request: " + req.errorMessage());
+                }
+            } else {
+                log.log(LogLevel.WARNING, "Could not connect to sentinel at: "+spec);
             }
-            proc = null;
-        }
-        InputStream getInputStream() {
-            return (proc != null)
-                    ? proc.getInputStream()
-                    : new java.io.ByteArrayInputStream(new byte[0]);
+            return servicelist;
+        } finally {
+            connection.close();
         }
     }
 }
diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/CpuJiffies.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/CpuJiffies.java
index 481068f0df2..e07a67770bc 100644
--- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/CpuJiffies.java
+++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/CpuJiffies.java
@@ -2,6 +2,7 @@
 package ai.vespa.metricsproxy.service;
 
 class CpuJiffies {
+
     private int cpuId;
     private long jiffies;
 
@@ -37,4 +38,5 @@ class CpuJiffies {
     public long getTotalJiffies() {
         return jiffies;
     }
+
 }
diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/HttpMetricFetcher.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/HttpMetricFetcher.java
index 922a2a15ffd..9068be81b65 100644
--- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/HttpMetricFetcher.java
+++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/HttpMetricFetcher.java
@@ -20,6 +20,7 @@ import java.util.logging.Logger;
  * @author bjorncs
  */
 public abstract class HttpMetricFetcher {
+
     private final static Logger log = Logger.getLogger(HttpMetricFetcher.class.getPackage().getName());
     public final static String STATE_PATH = "/state/v1/";
     // The call to apache will do 3 retries. As long as we check the services in series, we can't have this too high.
@@ -31,8 +32,8 @@ public abstract class HttpMetricFetcher {
 
 
     /**
-     * @param service The service to fetch metrics from
-     * @param port    The port to use
+     * @param service the service to fetch metrics from
+     * @param port    the port to use
      */
     HttpMetricFetcher(VespaService service, int port, String path) {
         this.service = service;
@@ -86,4 +87,5 @@ public abstract class HttpMetricFetcher {
                                                  .build())
                 .build();
     }
+
 }
diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/SystemPoller.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/SystemPoller.java
index 379e5296bb8..c8fbc83eb59 100644
--- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/SystemPoller.java
+++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/SystemPoller.java
@@ -24,6 +24,7 @@ import java.util.logging.Logger;
  * @author Eirik Nygaard
  */
 public class SystemPoller {
+
     final private static Logger log = Logger.getLogger(SystemPoller.class.getName());
 
     private final int pollingIntervalSecs;
diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/telegraf/Telegraf.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/telegraf/Telegraf.java
new file mode 100644
index 00000000000..9600dfca042
--- /dev/null
+++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/telegraf/Telegraf.java
@@ -0,0 +1,103 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package ai.vespa.metricsproxy.telegraf;
+
+import com.google.inject.Inject;
+import com.yahoo.component.AbstractComponent;
+import com.yahoo.log.LogLevel;
+import com.yahoo.system.execution.ProcessExecutor;
+import com.yahoo.system.execution.ProcessResult;
+import org.apache.velocity.VelocityContext;
+import org.apache.velocity.app.VelocityEngine;
+
+import java.io.FileWriter;
+import java.io.InputStreamReader;
+import java.io.Reader;
+import java.io.Writer;
+import java.util.logging.Logger;
+
+import static com.yahoo.vespa.defaults.Defaults.getDefaults;
+import static com.yahoo.yolean.Exceptions.uncheck;
+
+/**
+ * @author olaa
+ */
+public class Telegraf extends AbstractComponent {
+
+    // These paths must coincide with the paths in the start/stop-telegraf shell scripts.
+    private static final String TELEGRAF_CONFIG_PATH = getDefaults().underVespaHome("conf/telegraf/telegraf.conf");
+    private static final String TELEGRAF_LOG_FILE_PATH = getDefaults().underVespaHome("logs/telegraf/telegraf.log");
+
+    private static final String START_TELEGRAF_SCRIPT = getDefaults().underVespaHome("libexec/vespa/start-telegraf.sh");
+    private static final String STOP_TELEGRAF_SCRIPT = getDefaults().underVespaHome("libexec/vespa/stop-telegraf.sh");
+
+    private static final String TELEGRAF_CONFIG_TEMPLATE_PATH = "templates/telegraf.conf.vm";
+
+    private final TelegrafRegistry telegrafRegistry;
+
+    private static final Logger logger = Logger.getLogger(Telegraf.class.getName());
+
+    @Inject
+    public Telegraf(TelegrafRegistry telegrafRegistry, TelegrafConfig telegrafConfig) {
+        this.telegrafRegistry = telegrafRegistry;
+        telegrafRegistry.addInstance(this);
+        writeConfig(telegrafConfig, uncheck(() -> new FileWriter(TELEGRAF_CONFIG_PATH)), TELEGRAF_LOG_FILE_PATH);
+        restartTelegraf();
+    }
+
+    protected static void writeConfig(TelegrafConfig telegrafConfig, Writer writer, String logFilePath) {
+        VelocityContext context = new VelocityContext();
+        context.put("logFilePath", logFilePath);
+        context.put("intervalSeconds", telegrafConfig.intervalSeconds());
+        context.put("cloudwatchPlugins", telegrafConfig.cloudWatch());
+        context.put("protocol", telegrafConfig.isHostedVespa() ? "https" : "http");
+        // TODO: Add node cert if hosted
+
+        VelocityEngine velocityEngine = new VelocityEngine();
+        velocityEngine.init();
+        velocityEngine.evaluate(context, writer, "TelegrafConfigWriter", getTemplateReader());
+        uncheck(writer::close);
+    }
+
+    private void restartTelegraf() {
+        executeCommand(STOP_TELEGRAF_SCRIPT);
+        executeCommand(START_TELEGRAF_SCRIPT);
+    }
+
+    private void stopTelegraf() {
+        executeCommand(STOP_TELEGRAF_SCRIPT);
+    }
+
+    private void executeCommand(String command) {
+        logger.info(String.format("Running command: %s", command));
+        ProcessExecutor processExecutor = new ProcessExecutor
+                .Builder(10)
+                .successExitCodes(0)
+                .build();
+        ProcessResult processResult = uncheck(() -> processExecutor.execute(command))
+                .orElseThrow(() -> new RuntimeException("Timed out running command: " + command));
+
+        logger.log(LogLevel.DEBUG, () -> String.format("Exit code: %d\nstdOut: %s\nstdErr: %s",
+                                                        processResult.exitCode,
+                                                        processResult.stdOut,
+                                                        processResult.stdErr));
+
+        if (!processResult.stdErr.isBlank())
+            logger.warning(String.format("stdErr not empty: %s", processResult.stdErr));
+    }
+
+    @SuppressWarnings("ConstantConditions")
+    private static Reader getTemplateReader() {
+        return new InputStreamReader(Telegraf.class.getClassLoader()
+                                        .getResourceAsStream(TELEGRAF_CONFIG_TEMPLATE_PATH)
+        );
+
+    }
+
+    @Override
+    public void deconstruct() {
+        telegrafRegistry.removeInstance(this);
+        if (telegrafRegistry.isEmpty()) {
+            stopTelegraf();
+        }
+    }
+}
diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/telegraf/TelegrafRegistry.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/telegraf/TelegrafRegistry.java
new file mode 100644
index 00000000000..23da51ea082
--- /dev/null
+++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/telegraf/TelegrafRegistry.java
@@ -0,0 +1,33 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package ai.vespa.metricsproxy.telegraf;
+
+import com.yahoo.log.LogLevel;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.logging.Logger;
+
+/**
+ * @author olaa
+ */
+public class TelegrafRegistry {
+
+    private static final List<Telegraf> telegrafInstances = Collections.synchronizedList(new ArrayList<>());
+
+    private static final Logger logger = Logger.getLogger(TelegrafRegistry.class.getName());
+
+    public void addInstance(Telegraf telegraf) {
+        logger.log(LogLevel.DEBUG, () -> "Adding Telegraf instance to registry: " + telegraf.hashCode());
+        telegrafInstances.add(telegraf);
+    }
+
+    public void removeInstance(Telegraf telegraf) {
+        logger.log(LogLevel.DEBUG, () -> "Removing Telegraf instance from registry: " + telegraf.hashCode());
+        telegrafInstances.remove(telegraf);
+    }
+
+    public boolean isEmpty() {
+        return telegrafInstances.isEmpty();
+    }
+}
diff --git a/metrics-proxy/src/main/resources/configdefinitions/node-info.def b/metrics-proxy/src/main/resources/configdefinitions/node-info.def
new file mode 100644
index 00000000000..e66433a96d0
--- /dev/null
+++ b/metrics-proxy/src/main/resources/configdefinitions/node-info.def
@@ -0,0 +1,5 @@
+# Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package=ai.vespa.metricsproxy.http.metrics
+
+role string
+hostname string
diff --git a/metrics-proxy/src/main/resources/configdefinitions/telegraf.def b/metrics-proxy/src/main/resources/configdefinitions/telegraf.def
new file mode 100644
index 00000000000..ba58f6e6817
--- /dev/null
+++ b/metrics-proxy/src/main/resources/configdefinitions/telegraf.def
@@ -0,0 +1,22 @@
+# Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package=ai.vespa.metricsproxy.telegraf
+
+isHostedVespa bool default=false
+
+# Metrics pull/push interval
+intervalSeconds int default=60
+
+
+# The Vespa metrics consumer to get metrics for
+cloudWatch[].consumer string
+
+cloudWatch[].region string default="us-east-1"
+cloudWatch[].namespace string
+
+# Only valid and required for hosted Vespa
+cloudWatch[].accessKeyName string default=""
+cloudWatch[].secretKeyName string default=""
+
+# Only valid and optional for self-hosted Vespa
+cloudWatch[].file string default=""
+cloudWatch[].profile string default="default"
diff --git a/metrics-proxy/src/main/resources/templates/telegraf.conf.vm b/metrics-proxy/src/main/resources/templates/telegraf.conf.vm
new file mode 100644
index 00000000000..5a5f2d5f712
--- /dev/null
+++ b/metrics-proxy/src/main/resources/templates/telegraf.conf.vm
@@ -0,0 +1,44 @@
+# Configuration for telegraf agent
+[agent]
+  interval = "${intervalSeconds}s"
+  round_interval = true
+  metric_batch_size = 1000
+  metric_buffer_limit = 10000
+  collection_jitter = "0s"
+  flush_interval = "${intervalSeconds}s"
+  flush_jitter = "0s"
+  precision = ""
+  logtarget = "file"
+  logfile = "$logFilePath"
+  logfile_rotation_interval = "1d"
+  logfile_rotation_max_size = "20MB"
+  logfile_rotation_max_archives = 5
+
+#foreach( $cloudwatch in $cloudwatchPlugins )
+# Configuration for AWS CloudWatch output.
+[[outputs.cloudwatch]]
+  region = "$cloudwatch.region()"
+  namespace = "$cloudwatch.namespace()"
+#if( $cloudwatch.accessKeyName() != "" )
+  access_key = "$cloudwatch.accessKeyName()"
+  secret_key = "$cloudwatch.secretKeyName()"
+#elseif( $cloudwatch.profile() != "" )
+  profile = "$cloudwatch.profile()"
+#end
+  tagexclude = ["vespa_consumer"]
+  [outputs.cloudwatch.tagpass]
+    vespa_consumer = ["$cloudwatch.consumer()"]
+
+# Configuration for Vespa input plugin
+[[inputs.vespa]]
+  url = "${protocol}://localhost:19092/metrics/v2/values?consumer=$cloudwatch.consumer()"
+  [inputs.vespa.tags]
+    vespa_consumer = "$cloudwatch.consumer()"
+#* TODO: Add node cert if hosted
+#if( $isHosted )
+  tls_cert = "${VESPA_CERTIFICATE_PATH}"
+  tls_key = "${VESPA_KEY_PATH}"
+  insecure_skip_verify = true
+#end
+*###
+#end
\ No newline at end of file
diff --git a/metrics-proxy/src/main/sh/start-telegraf.sh b/metrics-proxy/src/main/sh/start-telegraf.sh
new file mode 100644
index 00000000000..c6e5f2d7f18
--- /dev/null
+++ b/metrics-proxy/src/main/sh/start-telegraf.sh
@@ -0,0 +1,101 @@
+#!/bin/sh
+# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+# BEGIN environment bootstrap section
+# Do not edit between here and END as this section should stay identical in all scripts
+
+findpath () {
+    myname=${0}
+    mypath=${myname%/*}
+    myname=${myname##*/}
+    empty_if_start_slash=${mypath%%/*}
+    if [ "${empty_if_start_slash}" ]; then
+        mypath=$(pwd)/${mypath}
+    fi
+    if [ "$mypath" ] && [ -d "$mypath" ]; then
+        return
+    fi
+    mypath=$(pwd)
+    if [ -f "${mypath}/${myname}" ]; then
+        return
+    fi
+    echo "FATAL: Could not figure out the path where $myname lives from $0"
+    exit 1
+}
+
+COMMON_ENV=libexec/vespa/common-env.sh
+
+source_common_env () {
+    if [ "$VESPA_HOME" ] && [ -d "$VESPA_HOME" ]; then
+        export VESPA_HOME
+        common_env=$VESPA_HOME/$COMMON_ENV
+        if [ -f "$common_env" ]; then
+            . $common_env
+            return
+        fi
+    fi
+    return 1
+}
+
+findroot () {
+    source_common_env && return
+    if [ "$VESPA_HOME" ]; then
+        echo "FATAL: bad VESPA_HOME value '$VESPA_HOME'"
+        exit 1
+    fi
+    if [ "$ROOT" ] && [ -d "$ROOT" ]; then
+        VESPA_HOME="$ROOT"
+        source_common_env && return
+    fi
+    findpath
+    while [ "$mypath" ]; do
+        VESPA_HOME=${mypath}
+        source_common_env && return
+        mypath=${mypath%/*}
+    done
+    echo "FATAL: missing VESPA_HOME environment variable"
+    echo "Could not locate $COMMON_ENV anywhere"
+    exit 1
+}
+
+findhost () {
+    if [ "${VESPA_HOSTNAME}" = "" ]; then
+        VESPA_HOSTNAME=$(vespa-detect-hostname || hostname -f || hostname || echo "localhost") || exit 1
+    fi
+    validate="${VESPA_HOME}/bin/vespa-validate-hostname"
+    if [ -f "$validate" ]; then
+        "$validate" "${VESPA_HOSTNAME}" || exit 1
+    fi
+    export VESPA_HOSTNAME
+}
+
+findroot
+findhost
+
+# END environment bootstrap section
+
+fixddir () {
+    if ! [ -d $1 ]; then
+       echo "Creating data directory $1"
+       mkdir -p $1 || exit 1
+    fi
+    if [ "${VESPA_USER}" ] && [ "${VESPA_UNPRIVILEGED}" != yes ]; then
+       chown ${VESPA_USER} $1
+    fi
+    chmod 755 $1
+}
+
+# Note: these directories must coincide with the paths defined in the Telegraf Java component
+conf_dir=${VESPA_HOME}/conf/telegraf
+log_dir=${VESPA_HOME}/logs/telegraf
+fixddir ${conf_dir}
+fixddir ${log_dir}
+
+configfile=${conf_dir}/telegraf.conf
+pidfile="${VESPA_HOME}/var/run/telegraf.pid"
+
+TELEGRAF_CMD=/opt/vespa-deps/bin/telegraf
+
+vespa-run-as-vespa-user vespa-runserver -s telegraf -r 30 -p $pidfile -- \
+${TELEGRAF_CMD} --config ${configfile}
+
diff --git a/metrics-proxy/src/main/sh/stop-telegraf.sh b/metrics-proxy/src/main/sh/stop-telegraf.sh
new file mode 100644
index 00000000000..c4f96fbec33
--- /dev/null
+++ b/metrics-proxy/src/main/sh/stop-telegraf.sh
@@ -0,0 +1,78 @@
+#!/bin/sh
+# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+# BEGIN environment bootstrap section
+# Do not edit between here and END as this section should stay identical in all scripts
+
+findpath () {
+    myname=${0}
+    mypath=${myname%/*}
+    myname=${myname##*/}
+    empty_if_start_slash=${mypath%%/*}
+    if [ "${empty_if_start_slash}" ]; then
+        mypath=$(pwd)/${mypath}
+    fi
+    if [ "$mypath" ] && [ -d "$mypath" ]; then
+        return
+    fi
+    mypath=$(pwd)
+    if [ -f "${mypath}/${myname}" ]; then
+        return
+    fi
+    echo "FATAL: Could not figure out the path where $myname lives from $0"
+    exit 1
+}
+
+COMMON_ENV=libexec/vespa/common-env.sh
+
+source_common_env () {
+    if [ "$VESPA_HOME" ] && [ -d "$VESPA_HOME" ]; then
+        export VESPA_HOME
+        common_env=$VESPA_HOME/$COMMON_ENV
+        if [ -f "$common_env" ]; then
+            . $common_env
+            return
+        fi
+    fi
+    return 1
+}
+
+findroot () {
+    source_common_env && return
+    if [ "$VESPA_HOME" ]; then
+        echo "FATAL: bad VESPA_HOME value '$VESPA_HOME'"
+        exit 1
+    fi
+    if [ "$ROOT" ] && [ -d "$ROOT" ]; then
+        VESPA_HOME="$ROOT"
+        source_common_env && return
+    fi
+    findpath
+    while [ "$mypath" ]; do
+        VESPA_HOME=${mypath}
+        source_common_env && return
+        mypath=${mypath%/*}
+    done
+    echo "FATAL: missing VESPA_HOME environment variable"
+    echo "Could not locate $COMMON_ENV anywhere"
+    exit 1
+}
+
+findhost () {
+    if [ "${VESPA_HOSTNAME}" = "" ]; then
+        VESPA_HOSTNAME=$(vespa-detect-hostname || hostname -f || hostname || echo "localhost") || exit 1
+    fi
+    validate="${VESPA_HOME}/bin/vespa-validate-hostname"
+    if [ -f "$validate" ]; then
+        "$validate" "${VESPA_HOSTNAME}" || exit 1
+    fi
+    export VESPA_HOSTNAME
+}
+
+findroot
+findhost
+
+# END environment bootstrap section
+
+pidfile="${VESPA_HOME}/var/run/telegraf.pid"
+vespa-run-as-vespa-user vespa-runserver -s telegraf -p $pidfile -S
diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/HttpHandlerTestBase.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/HttpHandlerTestBase.java
index 77c3a719cd9..d776368687d 100644
--- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/HttpHandlerTestBase.java
+++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/HttpHandlerTestBase.java
@@ -24,6 +24,7 @@ import java.util.List;
 
 import static ai.vespa.metricsproxy.http.ValuesFetcher.DEFAULT_PUBLIC_CONSUMER_ID;
 import static ai.vespa.metricsproxy.metric.ExternalMetrics.VESPA_NODE_SERVICE_ID;
+import static ai.vespa.metricsproxy.metric.dimensions.PublicDimensions.REASON;
 import static ai.vespa.metricsproxy.service.DummyService.METRIC_1;
 
 /**
@@ -61,11 +62,12 @@ public class HttpHandlerTestBase {
     }
 
     protected static MetricsConsumers getMetricsConsumers() {
+        // Must use a whitelisted dimension to avoid it being removed for the MetricsV2Handler
         var defaultConsumerDimension = new ConsumersConfig.Consumer.Metric.Dimension.Builder()
-                .key("consumer-dim").value("default-val");
+                .key(REASON).value("default-val");
 
         var customConsumerDimension = new ConsumersConfig.Consumer.Metric.Dimension.Builder()
-                .key("consumer-dim").value("custom-val");
+                .key(REASON).value("custom-val");
 
         return new MetricsConsumers(new ConsumersConfig.Builder()
                                             .consumer(new ConsumersConfig.Consumer.Builder()
diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/metrics/MetricsHandlerTestBase.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/metrics/MetricsHandlerTestBase.java
new file mode 100644
index 00000000000..1c5ce695155
--- /dev/null
+++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/metrics/MetricsHandlerTestBase.java
@@ -0,0 +1,196 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package ai.vespa.metricsproxy.http.metrics;
+
+import ai.vespa.metricsproxy.http.HttpHandlerTestBase;
+import ai.vespa.metricsproxy.metric.model.json.GenericJsonModel;
+import ai.vespa.metricsproxy.metric.model.json.GenericMetrics;
+import ai.vespa.metricsproxy.metric.model.json.GenericService;
+import ai.vespa.metricsproxy.service.DownService;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.json.JSONArray;
+import org.json.JSONObject;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import java.io.IOException;
+
+import static ai.vespa.metricsproxy.metric.dimensions.PublicDimensions.INTERNAL_SERVICE_ID;
+import static ai.vespa.metricsproxy.metric.dimensions.PublicDimensions.REASON;
+import static ai.vespa.metricsproxy.metric.dimensions.PublicDimensions.SERVICE_ID;
+import static ai.vespa.metricsproxy.metric.model.StatusCode.DOWN;
+import static ai.vespa.metricsproxy.metric.model.json.JacksonUtil.createObjectMapper;
+import static ai.vespa.metricsproxy.service.DummyService.METRIC_1;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+/**
+ * @author gjoranv
+ */
+public abstract class MetricsHandlerTestBase<MODEL> extends HttpHandlerTestBase {
+
+    static String rootUri;
+    static String valuesUri;
+
+    Class<MODEL> modelClass;
+
+    abstract GenericJsonModel getGenericJsonModel(MODEL model);
+
+    private MODEL getResponseAsJsonModel(String consumer) {
+        String response = testDriver.sendRequest(valuesUri + "?consumer=" + consumer).readAll();
+        try {
+            return createObjectMapper().readValue(response, modelClass);
+        } catch (IOException e) {
+            fail("Failed to create json model: " + e.getMessage());
+            throw new RuntimeException(e);
+        }
+    }
+
+    private GenericJsonModel getResponseAsGenericJsonModel(String consumer) {
+        return getGenericJsonModel(getResponseAsJsonModel(consumer));
+    }
+
+    @Test
+    public void invalid_path_yields_error_response() throws Exception {
+        String response = testDriver.sendRequest(rootUri + "/invalid").readAll();
+        JSONObject root = new JSONObject(response);
+        assertTrue(root.has("error"));
+    }
+
+    @Test
+    public void root_response_contains_values_uri() throws Exception {
+        String response = testDriver.sendRequest(rootUri).readAll();
+        JSONObject root = new JSONObject(response);
+        assertTrue(root.has("resources"));
+
+        JSONArray resources = root.getJSONArray("resources");
+        assertEquals(1, resources.length());
+
+        JSONObject valuesUrl = resources.getJSONObject(0);
+        assertEquals(valuesUri, valuesUrl.getString("url"));
+    }
+
+    @Ignore
+    @Test
+    public void visually_inspect_values_response() throws Exception {
+        String response = testDriver.sendRequest(valuesUri).readAll();
+        ObjectMapper mapper = createObjectMapper();
+        var jsonModel = mapper.readValue(response, modelClass);
+        System.out.println(mapper.writerWithDefaultPrettyPrinter().writeValueAsString(jsonModel));
+    }
+
+    @Test
+    public void no_explicit_consumer_gives_the_default_consumer() {
+        String responseDefaultConsumer = testDriver.sendRequest(valuesUri + "?consumer=default").readAll();
+        String responseNoConsumer = testDriver.sendRequest(valuesUri).readAll();
+        assertEqualsExceptTimestamps(responseDefaultConsumer, responseNoConsumer);
+    }
+
+    @Test
+    public void unknown_consumer_gives_the_default_consumer() {
+        String response = testDriver.sendRequest(valuesUri).readAll();
+        String responseUnknownConsumer = testDriver.sendRequest(valuesUri + "?consumer=not_defined").readAll();
+        assertEqualsExceptTimestamps(response, responseUnknownConsumer);
+    }
+
+    private void assertEqualsExceptTimestamps(String s1, String s2) {
+        assertEquals(replaceTimestamps(s1), replaceTimestamps(s2));
+    }
+
+    private String replaceTimestamps(String s) {
+        return s.replaceAll("timestamp\":\\d+,", "timestamp\":1,");
+    }
+
+    @Test
+    public void response_contains_node_metrics() {
+        GenericJsonModel jsonModel = getResponseAsGenericJsonModel(DEFAULT_CONSUMER);
+
+        assertNotNull(jsonModel.node);
+        assertEquals(1, jsonModel.node.metrics.size());
+        assertEquals(12.345, jsonModel.node.metrics.get(0).values.get(CPU_METRIC), 0.0001d);
+    }
+
+    @Test
+    public void response_contains_service_metrics() {
+        GenericJsonModel jsonModel = getResponseAsGenericJsonModel(DEFAULT_CONSUMER);
+
+        assertEquals(2, jsonModel.services.size());
+        GenericService dummyService = jsonModel.services.get(0);
+        assertEquals(2, dummyService.metrics.size());
+
+        GenericMetrics dummy0Metrics = getMetricsForService("dummy0", dummyService);
+        assertEquals(1L, dummy0Metrics.values.get(METRIC_1).longValue());
+        assertEquals("default-val", dummy0Metrics.dimensions.get(REASON));
+
+        GenericMetrics dummy1Metrics = getMetricsForService("dummy1", dummyService);
+        assertEquals(6L, dummy1Metrics.values.get(METRIC_1).longValue());
+        assertEquals("default-val", dummy1Metrics.dimensions.get(REASON));
+    }
+
+    @Test
+    public void custom_consumer_gets_only_its_whitelisted_metrics() {
+        GenericJsonModel jsonModel = getResponseAsGenericJsonModel(CUSTOM_CONSUMER);
+
+        assertNotNull(jsonModel.node);
+        // TODO: see comment in ExternalMetrics.setExtraMetrics
+        // assertEquals(0, jsonModel.node.metrics.size());
+
+        assertEquals(2, jsonModel.services.size());
+        GenericService dummyService = jsonModel.services.get(0);
+        assertEquals(2, dummyService.metrics.size());
+
+        GenericMetrics dummy0Metrics = getMetricsForService("dummy0", dummyService);
+        assertEquals("custom-val", dummy0Metrics.dimensions.get(REASON));
+
+        GenericMetrics dummy1Metrics = getMetricsForService("dummy1", dummyService);
+        assertEquals("custom-val", dummy1Metrics.dimensions.get(REASON));
+    }
+
+    private static GenericMetrics getMetricsForService(String serviceInstance, GenericService service) {
+        for (var metrics : service.metrics) {
+            if (getServiceIdDimension(metrics).equals(serviceInstance))
+                return metrics;
+        }
+        fail("Could not find metrics for service instance " + serviceInstance);
+        throw new RuntimeException();
+    }
+
+    @Test
+    public void all_timestamps_are_equal_and_non_zero() {
+        GenericJsonModel jsonModel = getResponseAsGenericJsonModel(DEFAULT_CONSUMER);
+
+        Long nodeTimestamp = jsonModel.node.timestamp;
+        assertNotEquals(0L, (long) nodeTimestamp);
+        for (var service : jsonModel.services)
+            assertEquals(nodeTimestamp, service.timestamp);
+    }
+
+    @Test
+    public void all_consumers_get_health_from_service_that_is_down() {
+        assertDownServiceHealth(DEFAULT_CONSUMER);
+        assertDownServiceHealth(CUSTOM_CONSUMER);
+    }
+
+    private void assertDownServiceHealth(String consumer) {
+        GenericJsonModel jsonModel = getResponseAsGenericJsonModel(consumer);
+
+        GenericService downService = jsonModel.services.get(1);
+        assertEquals(DOWN.status, downService.status.code);
+        assertEquals("No response", downService.status.description);
+
+        // Service should output metric dimensions, even without metrics, because they contain important info about the service.
+        assertEquals(1, downService.metrics.size());
+        assertEquals(0, downService.metrics.get(0).values.size());
+        assertFalse(downService.metrics.get(0).dimensions.isEmpty());
+        assertEquals(DownService.NAME, getServiceIdDimension(downService.metrics.get(0)));
+    }
+
+    private static String getServiceIdDimension(GenericMetrics metrics) {
+        var instanceDimension = metrics.dimensions.get(INTERNAL_SERVICE_ID);
+        return instanceDimension != null ? instanceDimension : metrics.dimensions.get(SERVICE_ID);
+    }
+
+}
diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/metrics/MetricsV1HandlerTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/metrics/MetricsV1HandlerTest.java
index 22f61114622..fe823466f7b 100644
--- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/metrics/MetricsV1HandlerTest.java
+++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/metrics/MetricsV1HandlerTest.java
@@ -1,46 +1,30 @@
 // Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package ai.vespa.metricsproxy.http.metrics;
 
-import ai.vespa.metricsproxy.http.HttpHandlerTestBase;
 import ai.vespa.metricsproxy.metric.model.json.GenericJsonModel;
-import ai.vespa.metricsproxy.metric.model.json.GenericMetrics;
-import ai.vespa.metricsproxy.metric.model.json.GenericService;
-import ai.vespa.metricsproxy.service.DownService;
-import com.fasterxml.jackson.databind.ObjectMapper;
 import com.yahoo.container.jdisc.RequestHandlerTestDriver;
-import org.json.JSONArray;
-import org.json.JSONObject;
+import org.junit.Before;
 import org.junit.BeforeClass;
-import org.junit.Ignore;
-import org.junit.Test;
 
-import java.io.IOException;
 import java.util.concurrent.Executors;
 
-import static ai.vespa.metricsproxy.core.VespaMetrics.INSTANCE_DIMENSION_ID;
 import static ai.vespa.metricsproxy.http.metrics.MetricsV1Handler.V1_PATH;
 import static ai.vespa.metricsproxy.http.metrics.MetricsV1Handler.VALUES_PATH;
-import static ai.vespa.metricsproxy.metric.model.StatusCode.DOWN;
-import static ai.vespa.metricsproxy.metric.model.json.JacksonUtil.createObjectMapper;
-import static ai.vespa.metricsproxy.service.DummyService.METRIC_1;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
 
 /**
  * @author gjoranv
  */
 @SuppressWarnings("UnstableApiUsage")
-public class MetricsV1HandlerTest extends HttpHandlerTestBase {
+public class MetricsV1HandlerTest extends MetricsHandlerTestBase<GenericJsonModel> {
 
     private static final String V1_URI = URI_BASE + V1_PATH;
     private static final String VALUES_URI = URI_BASE + VALUES_PATH;
 
+
     @BeforeClass
     public static void setup() {
+        rootUri = V1_URI;
+        valuesUri = VALUES_URI;
         var handler = new MetricsV1Handler(Executors.newSingleThreadExecutor(),
                                                         getMetricsManager(),
                                                         vespaServices,
@@ -48,149 +32,14 @@ public class MetricsV1HandlerTest extends HttpHandlerTestBase {
         testDriver = new RequestHandlerTestDriver(handler);
     }
 
-    private GenericJsonModel getResponseAsJsonModel(String consumer) {
-        String response = testDriver.sendRequest(VALUES_URI + "?consumer=" + consumer).readAll();
-        try {
-            return createObjectMapper().readValue(response, GenericJsonModel.class);
-        } catch (IOException e) {
-            fail("Failed to create json model: " + e.getMessage());
-            throw new RuntimeException(e);
-        }
-    }
-
-    @Test
-    public void v1_response_contains_values_uri() throws Exception {
-        String response = testDriver.sendRequest(V1_URI).readAll();
-        JSONObject root = new JSONObject(response);
-        assertTrue(root.has("resources"));
-
-        JSONArray resources = root.getJSONArray("resources");
-        assertEquals(1, resources.length());
-
-        JSONObject valuesUrl = resources.getJSONObject(0);
-        assertEquals(VALUES_URI, valuesUrl.getString("url"));
-    }
-
-    @Ignore
-    @Test
-    public void visually_inspect_values_response() throws Exception {
-        String response = testDriver.sendRequest(VALUES_URI).readAll();
-        ObjectMapper mapper = createObjectMapper();
-        var jsonModel = mapper.readValue(response, GenericJsonModel.class);
-        System.out.println(mapper.writerWithDefaultPrettyPrinter().writeValueAsString(jsonModel));
-    }
-
-    @Test
-    public void no_explicit_consumer_gives_the_default_consumer() {
-        String responseDefaultConsumer = testDriver.sendRequest(VALUES_URI + "?consumer=default").readAll();
-        String responseNoConsumer = testDriver.sendRequest(VALUES_URI).readAll();
-        assertEqualsExceptTimestamps(responseDefaultConsumer, responseNoConsumer);
-    }
-
-    @Test
-    public void unknown_consumer_gives_the_default_consumer() {
-        String response = testDriver.sendRequest(VALUES_URI).readAll();
-        String responseUnknownConsumer = testDriver.sendRequest(VALUES_URI + "?consumer=not_defined").readAll();
-        assertEqualsExceptTimestamps(response, responseUnknownConsumer);
-    }
-
-    private void assertEqualsExceptTimestamps(String s1, String s2) {
-        assertEquals(replaceTimestamps(s1), replaceTimestamps(s2));
-    }
-
-    @Test
-    public void response_contains_node_metrics() {
-        GenericJsonModel jsonModel = getResponseAsJsonModel(DEFAULT_CONSUMER);
-
-        assertNotNull(jsonModel.node);
-        assertEquals(1, jsonModel.node.metrics.size());
-        assertEquals(12.345, jsonModel.node.metrics.get(0).values.get(CPU_METRIC), 0.0001d);
-    }
-
-    @Test
-    public void response_contains_service_metrics() {
-        GenericJsonModel jsonModel = getResponseAsJsonModel(DEFAULT_CONSUMER);
-
-        assertEquals(2, jsonModel.services.size());
-        GenericService dummyService = jsonModel.services.get(0);
-        assertEquals(2, dummyService.metrics.size());
-
-        GenericMetrics dummy0Metrics = getMetricsForInstance("dummy0", dummyService);
-        assertEquals(1L, dummy0Metrics.values.get(METRIC_1).longValue());
-        assertEquals("default-val", dummy0Metrics.dimensions.get("consumer-dim"));
-
-        GenericMetrics dummy1Metrics = getMetricsForInstance("dummy1", dummyService);
-        assertEquals(6L, dummy1Metrics.values.get(METRIC_1).longValue());
-        assertEquals("default-val", dummy1Metrics.dimensions.get("consumer-dim"));
-    }
-
-    @Test
-    public void all_consumers_get_health_from_service_that_is_down() {
-        assertDownServiceHealth(DEFAULT_CONSUMER);
-        assertDownServiceHealth(CUSTOM_CONSUMER);
-    }
-
-    @Test
-    public void all_timestamps_are_equal_and_non_zero() {
-        GenericJsonModel jsonModel = getResponseAsJsonModel(DEFAULT_CONSUMER);
-
-        Long nodeTimestamp = jsonModel.node.timestamp;
-        assertNotEquals(0L, (long) nodeTimestamp);
-        for (var service : jsonModel.services)
-            assertEquals(nodeTimestamp, service.timestamp);
-    }
-
-    @Test
-    public void custom_consumer_gets_only_its_whitelisted_metrics() {
-        GenericJsonModel jsonModel = getResponseAsJsonModel(CUSTOM_CONSUMER);
-
-        assertNotNull(jsonModel.node);
-        // TODO: see comment in ExternalMetrics.setExtraMetrics
-        // assertEquals(0, jsonModel.node.metrics.size());
-
-        assertEquals(2, jsonModel.services.size());
-        GenericService dummyService = jsonModel.services.get(0);
-        assertEquals(2, dummyService.metrics.size());
-
-        GenericMetrics dummy0Metrics = getMetricsForInstance("dummy0", dummyService);
-        assertEquals("custom-val", dummy0Metrics.dimensions.get("consumer-dim"));
-
-        GenericMetrics dummy1Metrics = getMetricsForInstance("dummy1", dummyService);
-        assertEquals("custom-val", dummy1Metrics.dimensions.get("consumer-dim"));
-    }
-
-    @Test
-    public void invalid_path_yields_error_response() throws Exception {
-        String response = testDriver.sendRequest(V1_URI + "/invalid").readAll();
-        JSONObject root = new JSONObject(response);
-        assertTrue(root.has("error"));
-    }
-
-    private void assertDownServiceHealth(String consumer) {
-        GenericJsonModel jsonModel = getResponseAsJsonModel(consumer);
-
-        GenericService downService = jsonModel.services.get(1);
-        assertEquals(DOWN.status, downService.status.code);
-        assertEquals("No response", downService.status.description);
-
-        // Service should output metric dimensions, even without metrics, because they contain important info about the service.
-        assertEquals(1, downService.metrics.size());
-        assertEquals(0, downService.metrics.get(0).values.size());
-        assertFalse(downService.metrics.get(0).dimensions.isEmpty());
-        assertEquals(DownService.NAME, downService.metrics.get(0).dimensions.get(INSTANCE_DIMENSION_ID.id));
-    }
-
-    private String replaceTimestamps(String s) {
-        return s.replaceAll("timestamp\":\\d+,", "timestamp\":1,");
+    @Before
+    public void initModelClass() {
+        modelClass = GenericJsonModel.class;
     }
 
-    private static GenericMetrics getMetricsForInstance(String instance, GenericService service) {
-        for (var metrics : service.metrics) {
-            if (metrics.dimensions.get(INSTANCE_DIMENSION_ID.id).equals(instance))
-                return metrics;
-        }
-        fail("Could not find metrics for service instance " + instance);
-        throw new RuntimeException();
+    @Override
+    GenericJsonModel getGenericJsonModel(GenericJsonModel genericJsonModel) {
+        return genericJsonModel;
     }
 
 }
diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/metrics/MetricsV2HandlerTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/metrics/MetricsV2HandlerTest.java
new file mode 100644
index 00000000000..27ee6be4be3
--- /dev/null
+++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/metrics/MetricsV2HandlerTest.java
@@ -0,0 +1,53 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package ai.vespa.metricsproxy.http.metrics;
+
+import ai.vespa.metricsproxy.metric.model.json.GenericApplicationModel;
+import ai.vespa.metricsproxy.metric.model.json.GenericJsonModel;
+import com.yahoo.container.jdisc.RequestHandlerTestDriver;
+import org.junit.Before;
+import org.junit.BeforeClass;
+
+import java.util.concurrent.Executors;
+
+import static ai.vespa.metricsproxy.http.metrics.MetricsV2Handler.V2_PATH;
+import static ai.vespa.metricsproxy.http.metrics.MetricsV2Handler.VALUES_PATH;
+
+/**
+ * @author gjoranv
+ */
+@SuppressWarnings("UnstableApiUsage")
+public class MetricsV2HandlerTest extends MetricsHandlerTestBase<GenericApplicationModel> {
+
+    private static final String V2_URI = URI_BASE + V2_PATH;
+    private static final String VALUES_URI = URI_BASE + VALUES_PATH;
+
+
+    @BeforeClass
+    public static void setup() {
+        rootUri = V2_URI;
+        valuesUri = VALUES_URI;
+        var handler = new MetricsV2Handler(Executors.newSingleThreadExecutor(),
+                                           getMetricsManager(),
+                                           vespaServices,
+                                           getMetricsConsumers(),
+                                           nodeInfoConfig());
+        testDriver = new RequestHandlerTestDriver(handler);
+    }
+
+    @Before
+    public void initModelClass() {
+        modelClass = GenericApplicationModel.class;
+    }
+
+    @Override
+    GenericJsonModel getGenericJsonModel(GenericApplicationModel genericApplicationModel) {
+        return genericApplicationModel.nodes.get(0);
+    }
+
+    private static NodeInfoConfig nodeInfoConfig() {
+        return new NodeInfoConfig.Builder()
+                .role("my-role")
+                .hostname("my-hostname")
+                .build();
+    }
+}
diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/prometheus/PrometheusHandlerTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/prometheus/PrometheusHandlerTest.java
index a85f0425b4b..a224c4090b3 100644
--- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/prometheus/PrometheusHandlerTest.java
+++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/prometheus/PrometheusHandlerTest.java
@@ -12,6 +12,7 @@ import org.junit.Test;
 
 import java.util.concurrent.Executors;
 
+import static ai.vespa.metricsproxy.metric.dimensions.PublicDimensions.REASON;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 
@@ -83,7 +84,7 @@ public class PrometheusHandlerTest extends HttpHandlerTestBase {
     @Test
     public void service_metrics_have_configured_dimensions() {
         String dummy0 = getLine(valuesResponse, DummyService.NAME + "0");
-        assertTrue(dummy0.contains("consumer_dim=\"default-val\""));
+        assertTrue(dummy0.contains(REASON + "=\"default-val\""));
     }
 
     @Test
diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/MetricsPacketTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/MetricsPacketTest.java
index 9b37a805245..78c80689299 100644
--- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/MetricsPacketTest.java
+++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/metric/model/MetricsPacketTest.java
@@ -14,6 +14,7 @@ import java.util.Map;
 import static ai.vespa.metricsproxy.metric.model.ConsumerId.toConsumerId;
 import static ai.vespa.metricsproxy.metric.model.MetricId.toMetricId;
 import static ai.vespa.metricsproxy.metric.model.ServiceId.toServiceId;
+import static java.util.Collections.singleton;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
@@ -45,13 +46,23 @@ public class MetricsPacketTest {
         MetricsPacket packet = new MetricsPacket.Builder(toServiceId("foo"))
                 .statusCode(0)
                 .statusMessage("")
-                .addConsumers(Collections.singleton(DUPLICATE_CONSUMER))
-                .addConsumers(Collections.singleton(DUPLICATE_CONSUMER))
+                .addConsumers(singleton(DUPLICATE_CONSUMER))
+                .addConsumers(singleton(DUPLICATE_CONSUMER))
                 .build();
         assertEquals(1, packet.consumers().size());
     }
 
     @Test
+    public void builder_allows_inspecting_consumers() {
+        var consumer = toConsumerId("my-consumer");
+        var builder = new MetricsPacket.Builder(toServiceId("foo"))
+                .statusCode(0)
+                .statusMessage("")
+                .addConsumers(singleton(consumer));
+        assertTrue(builder.hasConsumer(consumer));
+    }
+
+    @Test
     public void builder_can_retain_subset_of_metrics() {
         MetricsPacket packet = new MetricsPacket.Builder(toServiceId("foo"))
                 .putMetrics(ImmutableList.of(
diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/ConfigSentinelClientTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/ConfigSentinelClientTest.java
index 8c6c5ffc3f8..810596d6d0b 100644
--- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/ConfigSentinelClientTest.java
+++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/ConfigSentinelClientTest.java
@@ -26,38 +26,39 @@ public class ConfigSentinelClientTest {
         services.add(qrserver);
         services.add(docproc);
 
-        MockConfigSentinelClient client = new MockConfigSentinelClient(configsentinel);
-        client.updateServiceStatuses(services);
+        try (MockConfigSentinelClient client = new MockConfigSentinelClient(configsentinel)) {
+            client.updateServiceStatuses(services);
 
-        assertThat(qrserver.getPid(), is(6520));
-        assertThat(qrserver.getState(), is("RUNNING"));
-        assertThat(qrserver.isAlive(), is(true));
-        assertThat(searchnode4.getPid(), is(6534));
-        assertThat(searchnode4.getState(), is("RUNNING"));
-        assertThat(searchnode4.isAlive(), is(true));
+            assertThat(qrserver.getPid(), is(6520));
+            assertThat(qrserver.getState(), is("RUNNING"));
+            assertThat(qrserver.isAlive(), is(true));
+            assertThat(searchnode4.getPid(), is(6534));
+            assertThat(searchnode4.getState(), is("RUNNING"));
+            assertThat(searchnode4.isAlive(), is(true));
 
-        assertThat(docproc.getPid(), is(-1));
-        assertThat(docproc.getState(), is("FINISHED"));
-        assertThat(docproc.isAlive(), is(false));
+            assertThat(docproc.getPid(), is(-1));
+            assertThat(docproc.getState(), is("FINISHED"));
+            assertThat(docproc.isAlive(), is(false));
 
 
-        configsentinel.reConfigure();
+            configsentinel.reConfigure();
 
-        client.ping(docproc);
-        assertThat(docproc.getPid(), is(100));
-        assertThat(docproc.getState(), is("RUNNING"));
-        assertThat(docproc.isAlive(), is(true));
+            client.ping(docproc);
+            assertThat(docproc.getPid(), is(100));
+            assertThat(docproc.getState(), is("RUNNING"));
+            assertThat(docproc.isAlive(), is(true));
 
-        //qrserver has yet not been checked
-        assertThat(qrserver.isAlive(), is(true));
+            //qrserver has yet not been checked
+            assertThat(qrserver.isAlive(), is(true));
 
-        client.updateServiceStatuses(services);
+            client.updateServiceStatuses(services);
 
-        assertThat(docproc.getPid(), is(100));
-        assertThat(docproc.getState(), is("RUNNING"));
-        assertThat(docproc.isAlive(), is(true));
-        //qrserver is no longer running on this node - so should be false
-        assertThat(qrserver.isAlive(), is(false));
+            assertThat(docproc.getPid(), is(100));
+            assertThat(docproc.getState(), is("RUNNING"));
+            assertThat(docproc.isAlive(), is(true));
+            //qrserver is no longer running on this node - so should be false
+            assertThat(qrserver.isAlive(), is(false));
+        }
     }
 
     @Test
@@ -88,14 +89,16 @@ public class ConfigSentinelClientTest {
         services.add(containerClusterController);
         services.add(notPresent);
 
-        MockConfigSentinelClient client = new MockConfigSentinelClient(configsentinel);
-        client.updateServiceStatuses(services);
-        assertThat(container.isAlive(),is(true));
-        assertThat(container.getPid(),is(14338));
-        assertThat(container.getState(),is("RUNNING"));
+        try (MockConfigSentinelClient client = new MockConfigSentinelClient(configsentinel)) {
+            client.updateServiceStatuses(services);
+            assertThat(container.isAlive(),is(true));
+            assertThat(container.getPid(),is(14338));
+            assertThat(container.getState(),is("RUNNING"));
 
-        assertThat(containerClusterController.isAlive(),is(true));
-        assertThat(containerClusterController.getPid(),is(25020));
-        assertThat(containerClusterController.getState(),is("RUNNING"));
+            assertThat(containerClusterController.isAlive(),is(true));
+            assertThat(containerClusterController.getPid(),is(25020));
+            assertThat(containerClusterController.getState(),is("RUNNING"));
+        }
     }
+
 }
diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/MetricsFetcherTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/MetricsFetcherTest.java
index be57b24d92f..57185d55131 100644
--- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/MetricsFetcherTest.java
+++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/MetricsFetcherTest.java
@@ -9,9 +9,9 @@ import static org.hamcrest.CoreMatchers.is;
 import static org.junit.Assert.assertThat;
 
 /**
- * @author Unknowm
  */
 public class MetricsFetcherTest {
+
     private static int port = 9;  //port number is not used in this test
 
     @Test
diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/MockConfigSentinelClient.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/MockConfigSentinelClient.java
index 6b439093682..55702b4dc45 100644
--- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/MockConfigSentinelClient.java
+++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/MockConfigSentinelClient.java
@@ -1,10 +1,6 @@
 // Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package ai.vespa.metricsproxy.service;
 
-import com.yahoo.log.LogLevel;
-
-import java.util.ArrayList;
-import java.util.List;
 import java.util.logging.Logger;
 
 /**
@@ -12,7 +8,7 @@ import java.util.logging.Logger;
  *
  * @author hmusum
  */
-public class MockConfigSentinelClient extends ConfigSentinelClient {
+public class MockConfigSentinelClient extends ConfigSentinelClient implements AutoCloseable {
     private final ConfigSentinelDummy configSentinel;
     private final static Logger log = Logger.getLogger(MockConfigSentinelClient.class.getPackage().getName());
 
@@ -22,27 +18,12 @@ public class MockConfigSentinelClient extends ConfigSentinelClient {
     }
 
     @Override
-    protected synchronized void setStatus(List<VespaService> services) throws Exception {
-        List<VespaService> updatedServices = new ArrayList<>();
-        String[] lines = configSentinel.getServiceList().split("\n");
-        for (String line : lines) {
-            if (line.equals("")) {
-                break;
-            }
-
-            VespaService s = parseServiceString(line, services);
-            if (s != null) {
-                updatedServices.add(s);
-            }
-        }
+    String sentinelLs() {
+        return configSentinel.getServiceList();
+    }
 
-        //Check if there are services that were not found in
-        //from the sentinel
-        for (VespaService s : services) {
-            if (!updatedServices.contains(s)) {
-                log.log(LogLevel.DEBUG, "Service " + s + " is no longer found with sentinel - setting alive = false");
-                s.setAlive(false);
-            }
-        }
+    @Override
+    public void close() {
+        super.deconstruct();
     }
 }
diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/telegraf/TelegrafTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/telegraf/TelegrafTest.java
new file mode 100644
index 00000000000..b4f2b4ab10c
--- /dev/null
+++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/telegraf/TelegrafTest.java
@@ -0,0 +1,44 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package ai.vespa.metricsproxy.telegraf;
+
+import ai.vespa.metricsproxy.TestUtil;
+import org.junit.Test;
+
+import java.io.StringWriter;
+
+import static org.junit.Assert.*;
+
+/**
+ * @author olaa
+ */
+public class TelegrafTest {
+
+    @Test
+    public void test_writing_correct_telegraf_plugin_config() {
+        TelegrafConfig telegrafConfig = new TelegrafConfig.Builder()
+                .cloudWatch(
+                        new TelegrafConfig.CloudWatch.Builder()
+                                .accessKeyName("accessKey1")
+                                .namespace("namespace1")
+                                .secretKeyName("secretKey1")
+                                .region("us-east-1")
+                                .consumer("consumer1")
+                )
+                .cloudWatch(
+                        new TelegrafConfig.CloudWatch.Builder()
+                                .namespace("namespace2")
+                                .profile("awsprofile")
+                                .region("us-east-2")
+                                .consumer("consumer2")
+                )
+                .intervalSeconds(300)
+                .isHostedVespa(true)
+                .build();
+        StringWriter stringWriter = new StringWriter();
+        String logFilePath = "/path/to/logs/telegraf/telegraf.log";
+        Telegraf.writeConfig(telegrafConfig, stringWriter, logFilePath);
+        String expectedConfig = TestUtil.getFileContents( "telegraf-config-with-two-cloudwatch-plugins.txt");
+        assertEquals(expectedConfig, stringWriter.toString());
+    }
+
+}
diff --git a/metrics-proxy/src/test/resources/telegraf-config-with-two-cloudwatch-plugins.txt b/metrics-proxy/src/test/resources/telegraf-config-with-two-cloudwatch-plugins.txt
new file mode 100644
index 00000000000..3194b290b78
--- /dev/null
+++ b/metrics-proxy/src/test/resources/telegraf-config-with-two-cloudwatch-plugins.txt
@@ -0,0 +1,46 @@
+# Configuration for telegraf agent
+[agent]
+  interval = "300s"
+  round_interval = true
+  metric_batch_size = 1000
+  metric_buffer_limit = 10000
+  collection_jitter = "0s"
+  flush_interval = "300s"
+  flush_jitter = "0s"
+  precision = ""
+  logtarget = "file"
+  logfile = "/path/to/logs/telegraf/telegraf.log"
+  logfile_rotation_interval = "1d"
+  logfile_rotation_max_size = "20MB"
+  logfile_rotation_max_archives = 5
+
+# Configuration for AWS CloudWatch output.
+[[outputs.cloudwatch]]
+  region = "us-east-1"
+  namespace = "namespace1"
+  access_key = "accessKey1"
+  secret_key = "secretKey1"
+  tagexclude = ["vespa_consumer"]
+  [outputs.cloudwatch.tagpass]
+    vespa_consumer = ["consumer1"]
+
+# Configuration for Vespa input plugin
+[[inputs.vespa]]
+  url = "https://localhost:19092/metrics/v2/values?consumer=consumer1"
+  [inputs.vespa.tags]
+    vespa_consumer = "consumer1"
+# Configuration for AWS CloudWatch output.
+[[outputs.cloudwatch]]
+  region = "us-east-2"
+  namespace = "namespace2"
+  profile = "awsprofile"
+  tagexclude = ["vespa_consumer"]
+  [outputs.cloudwatch.tagpass]
+    vespa_consumer = ["consumer2"]
+
+# Configuration for Vespa input plugin
+[[inputs.vespa]]
+  url = "https://localhost:19092/metrics/v2/values?consumer=consumer2"
+  [inputs.vespa.tags]
+    vespa_consumer = "consumer2"
+
diff --git a/metrics/CMakeLists.txt b/metrics/CMakeLists.txt
index 6cf1eadd6f7..6f854fed7c6 100644
--- a/metrics/CMakeLists.txt
+++ b/metrics/CMakeLists.txt
@@ -9,6 +9,7 @@ vespa_define_module(
 
     LIBS
     src/vespa/metrics
+    src/vespa/metrics/common
 
     TESTS
     src/tests
diff --git a/metrics/src/vespa/metrics/CMakeLists.txt b/metrics/src/vespa/metrics/CMakeLists.txt
index 96156dc84b0..0d7eeba3601 100644
--- a/metrics/src/vespa/metrics/CMakeLists.txt
+++ b/metrics/src/vespa/metrics/CMakeLists.txt
@@ -20,6 +20,7 @@ vespa_add_library(metrics
     valuemetric.cpp
     valuemetricvalues.cpp
     xmlwriter.cpp
+    $<TARGET_OBJECTS:metrics_common>
 
     INSTALL lib64
     DEPENDS
diff --git a/metrics/src/vespa/metrics/common/CMakeLists.txt b/metrics/src/vespa/metrics/common/CMakeLists.txt
new file mode 100644
index 00000000000..50183655dad
--- /dev/null
+++ b/metrics/src/vespa/metrics/common/CMakeLists.txt
@@ -0,0 +1,6 @@
+# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_library(metrics_common OBJECT
+    SOURCES
+    memory_usage_metrics.cpp
+    DEPENDS
+)
diff --git a/metrics/src/vespa/metrics/common/memory_usage_metrics.cpp b/metrics/src/vespa/metrics/common/memory_usage_metrics.cpp
new file mode 100644
index 00000000000..0c38e567749
--- /dev/null
+++ b/metrics/src/vespa/metrics/common/memory_usage_metrics.cpp
@@ -0,0 +1,28 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "memory_usage_metrics.h"
+#include <vespa/vespalib/util/memoryusage.h>
+
+namespace metrics {
+
+MemoryUsageMetrics::MemoryUsageMetrics(metrics::MetricSet* parent)
+    : MetricSet("memory_usage", {}, "The memory usage for a given component", parent),
+      _allocated_bytes("allocated_bytes", {}, "The number of allocated bytes", this),
+      _used_bytes("used_bytes", {}, "The number of used bytes (<= allocated_bytes)", this),
+      _dead_bytes("dead_bytes", {}, "The number of dead bytes (<= used_bytes)", this),
+      _on_hold_bytes("onhold_bytes", {}, "The number of bytes on hold", this)
+{
+}
+
+MemoryUsageMetrics::~MemoryUsageMetrics() = default;
+
+void
+MemoryUsageMetrics::update(const vespalib::MemoryUsage& usage)
+{
+    _allocated_bytes.set(usage.allocatedBytes());
+    _used_bytes.set(usage.usedBytes());
+    _dead_bytes.set(usage.deadBytes());
+    _on_hold_bytes.set(usage.allocatedBytesOnHold());
+}
+
+}
diff --git a/metrics/src/vespa/metrics/common/memory_usage_metrics.h b/metrics/src/vespa/metrics/common/memory_usage_metrics.h
new file mode 100644
index 00000000000..7030db8c163
--- /dev/null
+++ b/metrics/src/vespa/metrics/common/memory_usage_metrics.h
@@ -0,0 +1,26 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/metrics/metrics.h>
+
+namespace vespalib { class MemoryUsage; }
+
+namespace metrics {
+
+/**
+ * Metric set for memory usage metrics.
+ */
+class MemoryUsageMetrics : public metrics::MetricSet {
+    metrics::LongValueMetric _allocated_bytes;
+    metrics::LongValueMetric _used_bytes;
+    metrics::LongValueMetric _dead_bytes;
+    metrics::LongValueMetric _on_hold_bytes;
+
+public:
+    explicit MemoryUsageMetrics(metrics::MetricSet* parent);
+    ~MemoryUsageMetrics() override;
+    void update(const vespalib::MemoryUsage& usage);
+};
+
+} // namespace metrics
diff --git a/model-evaluation/abi-spec.json b/model-evaluation/abi-spec.json
index 5a75c8b31ea..d465464de7f 100644
--- a/model-evaluation/abi-spec.json
+++ b/model-evaluation/abi-spec.json
@@ -8,6 +8,7 @@
     "methods": [
       "public ai.vespa.models.evaluation.FunctionEvaluator bind(java.lang.String, com.yahoo.tensor.Tensor)",
       "public ai.vespa.models.evaluation.FunctionEvaluator bind(java.lang.String, double)",
+      "public ai.vespa.models.evaluation.FunctionEvaluator bind(java.lang.String, java.lang.String)",
       "public ai.vespa.models.evaluation.FunctionEvaluator setMissingValue(com.yahoo.tensor.Tensor)",
       "public ai.vespa.models.evaluation.FunctionEvaluator setMissingValue(double)",
       "public com.yahoo.tensor.Tensor evaluate()",
diff --git a/model-evaluation/src/main/java/ai/vespa/models/evaluation/FunctionEvaluator.java b/model-evaluation/src/main/java/ai/vespa/models/evaluation/FunctionEvaluator.java
index 994f6dd9b64..e373a54bcd1 100644
--- a/model-evaluation/src/main/java/ai/vespa/models/evaluation/FunctionEvaluator.java
+++ b/model-evaluation/src/main/java/ai/vespa/models/evaluation/FunctionEvaluator.java
@@ -2,6 +2,7 @@
 package ai.vespa.models.evaluation;
 
 import com.yahoo.searchlib.rankingexpression.ExpressionFunction;
+import com.yahoo.searchlib.rankingexpression.evaluation.StringValue;
 import com.yahoo.searchlib.rankingexpression.evaluation.TensorValue;
 import com.yahoo.tensor.Tensor;
 import com.yahoo.tensor.TensorType;
@@ -61,6 +62,21 @@ public class FunctionEvaluator {
     }
 
     /**
+     * Binds the given variable referred in this expression to the given value.
+     * String values are not yet supported in tensors.
+     *
+     * @param name the variable to bind
+     * @param value the value this becomes bound to
+     * @return this for chaining
+     */
+    public FunctionEvaluator bind(String name, String value) {
+        if (evaluated)
+            throw new IllegalStateException("Cannot bind a new value in a used evaluator");
+        context.put(name, new StringValue(value));
+        return this;
+    }
+
+    /**
      * Sets the default value to use for variables which are not bound
      *
      * @param value the default value
diff --git a/model-evaluation/src/main/java/ai/vespa/models/handler/ModelsEvaluationHandler.java b/model-evaluation/src/main/java/ai/vespa/models/handler/ModelsEvaluationHandler.java
index 5c353fcdf35..de23a8c6526 100644
--- a/model-evaluation/src/main/java/ai/vespa/models/handler/ModelsEvaluationHandler.java
+++ b/model-evaluation/src/main/java/ai/vespa/models/handler/ModelsEvaluationHandler.java
@@ -77,8 +77,14 @@ public class ModelsEvaluationHandler extends ThreadedHttpRequestHandler {
         property(request, missingValueKey).ifPresent(missingValue -> evaluator.setMissingValue(Tensor.from(missingValue)));
 
         for (Map.Entry<String, TensorType> argument : evaluator.function().argumentTypes().entrySet()) {
-            property(request, argument.getKey()).ifPresent(value -> evaluator.bind(argument.getKey(),
-                                                                                   Tensor.from(argument.getValue(), value)));
+            Optional<String> value = property(request, argument.getKey());
+            if (value.isPresent()) {
+                try {
+                    evaluator.bind(argument.getKey(), Tensor.from(argument.getValue(), value.get()));
+                } catch (IllegalArgumentException e) {
+                    evaluator.bind(argument.getKey(), value.get());  // since we don't yet support tensors with string values
+                }
+            }
         }
         Tensor result = evaluator.evaluate();
         return new Response(200, JsonFormat.encode(result));
diff --git a/model-evaluation/src/test/java/ai/vespa/models/evaluation/MlModelsImportingTest.java b/model-evaluation/src/test/java/ai/vespa/models/evaluation/MlModelsImportingTest.java
index 9320ac3fad8..0d13b7d4660 100644
--- a/model-evaluation/src/test/java/ai/vespa/models/evaluation/MlModelsImportingTest.java
+++ b/model-evaluation/src/test/java/ai/vespa/models/evaluation/MlModelsImportingTest.java
@@ -25,7 +25,7 @@ public class MlModelsImportingTest {
     public void testImportingModels() {
         ModelTester tester = new ModelTester("src/test/resources/config/models/");
 
-        assertEquals(4, tester.models().size());
+        assertEquals(5, tester.models().size());
 
         // TODO: When we get type information in Models, replace the evaluator.context().names() check below by that
         {
@@ -47,7 +47,24 @@ public class MlModelsImportingTest {
         }
 
         {
+            Model lightgbm = tester.models().get("lightgbm_regression");
 
+            // Function
+            assertEquals(1, lightgbm.functions().size());
+            ExpressionFunction function = tester.assertFunction("lightgbm_regression",
+                    "(optimized sum of condition trees of size 480 bytes)",
+                    lightgbm);
+            assertEquals("tensor()", function.returnType().get().toString());
+            assertEquals("categorical_1, categorical_2, numerical_1, numerical_2", commaSeparated(function.arguments()));
+            function.arguments().forEach(arg -> assertEquals(TensorType.empty, function.argumentTypes().get(arg)));
+
+            // Evaluator
+            FunctionEvaluator evaluator = lightgbm.evaluatorOf();
+            assertEquals("categorical_1, categorical_2, numerical_1, numerical_2", evaluator.context().names().stream().sorted().collect(Collectors.joining(", ")));
+            assertEquals(1.91300868202, evaluator.evaluate().sum().asDouble(), delta);
+        }
+
+        {
             Model onnxMnistSoftmax = tester.models().get("mnist_softmax");
 
             // Function
diff --git a/model-evaluation/src/test/java/ai/vespa/models/handler/ModelsEvaluationHandlerTest.java b/model-evaluation/src/test/java/ai/vespa/models/handler/ModelsEvaluationHandlerTest.java
index 629c82f410a..c9e49d3be02 100644
--- a/model-evaluation/src/test/java/ai/vespa/models/handler/ModelsEvaluationHandlerTest.java
+++ b/model-evaluation/src/test/java/ai/vespa/models/handler/ModelsEvaluationHandlerTest.java
@@ -56,7 +56,7 @@ public class ModelsEvaluationHandlerTest {
     public void testListModels() {
         String url = "http://localhost/model-evaluation/v1";
         String expected =
-                "{\"mnist_softmax\":\"http://localhost/model-evaluation/v1/mnist_softmax\",\"mnist_saved\":\"http://localhost/model-evaluation/v1/mnist_saved\",\"mnist_softmax_saved\":\"http://localhost/model-evaluation/v1/mnist_softmax_saved\",\"xgboost_2_2\":\"http://localhost/model-evaluation/v1/xgboost_2_2\"}";
+                "{\"mnist_softmax\":\"http://localhost/model-evaluation/v1/mnist_softmax\",\"mnist_saved\":\"http://localhost/model-evaluation/v1/mnist_saved\",\"mnist_softmax_saved\":\"http://localhost/model-evaluation/v1/mnist_softmax_saved\",\"xgboost_2_2\":\"http://localhost/model-evaluation/v1/xgboost_2_2\",\"lightgbm_regression\":\"http://localhost/model-evaluation/v1/lightgbm_regression\"}";
         assertResponse(url, 200, expected);
     }
 
@@ -94,6 +94,39 @@ public class ModelsEvaluationHandlerTest {
     }
 
     @Test
+    public void testLightGBMEvaluationWithoutBindings() {
+        String url = "http://localhost/model-evaluation/v1/lightgbm_regression/eval";
+        String expected = "{\"cells\":[{\"address\":{},\"value\":1.9130086820218188}]}";
+        assertResponse(url, 200, expected);
+    }
+
+    @Test
+    public void testLightGBMEvaluationWithBindings() {
+        Map<String, String> properties = new HashMap<>();
+        properties.put("numerical_1", "0.1");
+        properties.put("numerical_2", "0.2");
+        properties.put("categorical_1", "a");
+        properties.put("categorical_2", "i");
+        properties.put("non-existing-binding", "-1");
+        String url = "http://localhost/model-evaluation/v1/lightgbm_regression/eval";
+        String expected = "{\"cells\":[{\"address\":{},\"value\":2.054697758469921}]}";
+        assertResponse(url, properties, 200, expected);
+    }
+
+    @Test
+    public void testLightGBMEvaluationWithMissingValue() {
+        Map<String, String> properties = new HashMap<>();
+        properties.put("missing-value", "-1.0");
+        properties.put("numerical_2", "0.5");
+        properties.put("categorical_1", "b");
+        properties.put("categorical_2", "j");
+        properties.put("non-existing-binding", "-1");
+        String url = "http://localhost/model-evaluation/v1/lightgbm_regression/eval";
+        String expected = "{\"cells\":[{\"address\":{},\"value\":2.0745534018208094}]}";
+        assertResponse(url, properties, 200, expected);
+    }
+
+    @Test
     public void testMnistSoftmaxDetails() {
         String url = "http://localhost:8080/model-evaluation/v1/mnist_softmax";
         String expected = "{\"model\":\"mnist_softmax\",\"functions\":[{\"function\":\"default.add\",\"info\":\"http://localhost:8080/model-evaluation/v1/mnist_softmax/default.add\",\"eval\":\"http://localhost:8080/model-evaluation/v1/mnist_softmax/default.add/eval\",\"arguments\":[{\"name\":\"Placeholder\",\"type\":\"tensor(d0[],d1[784])\"}]}]}";
diff --git a/model-evaluation/src/test/resources/config/models/rank-profiles.cfg b/model-evaluation/src/test/resources/config/models/rank-profiles.cfg
index c25c5ba555b..385115b7cd4 100644
--- a/model-evaluation/src/test/resources/config/models/rank-profiles.cfg
+++ b/model-evaluation/src/test/resources/config/models/rank-profiles.cfg
@@ -26,3 +26,6 @@ rankprofile[3].fef.property[3].name "rankingExpression(serving_default.y).input.
 rankprofile[3].fef.property[3].value "tensor(d0[],d1[784])"
 rankprofile[3].fef.property[4].name "rankingExpression(serving_default.y).type"
 rankprofile[3].fef.property[4].value "tensor(d1[10])"
+rankprofile[4].name "lightgbm_regression"
+rankprofile[4].fef.property[0].name "rankingExpression(lightgbm_regression).rankingScript"
+rankprofile[4].fef.property[0].value "if (!(numerical_2 >= 0.46643291586559305), 2.1594397038037663, if (categorical_2 in ["k", "l", "m"], 2.235297305276056, 2.1792953471546546)) + if (categorical_1 in ["d", "e"], 0.03070842919354316, if (!(numerical_1 >= 0.5102250691730842), -0.04439151147520909, 0.005117411709368601)) + if (!(numerical_2 >= 0.668665477622446), if (!(numerical_2 >= 0.008118820676863816), -0.15361238490967524, -0.01192330846157292), 0.03499044894987518) + if (!(numerical_1 >= 0.5201391072644542), -0.02141000620783247, if (categorical_1 in ["a", "b"], -0.004121485787596721, 0.04534090904886873)) + if (categorical_2 in ["k", "l", "m"], if (!(numerical_2 >= 0.27283279016959255), -0.01924803254356527, 0.03643772842347651), -0.02701711918923075)"
diff --git a/model-integration/src/main/config/model-integration.xml b/model-integration/src/main/config/model-integration.xml
index 90ec7d7275e..34f5f0ce31a 100644
--- a/model-integration/src/main/config/model-integration.xml
+++ b/model-integration/src/main/config/model-integration.xml
@@ -1,11 +1,12 @@
 <!-- Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
 <!-- Component which can import some ml model.
      This is included into the config server services.xml to enable it to translate
-     model pseudofeatures in ranking expressions during config mddel building.
+     model pseudo features in ranking expressions during config model building.
      It is provided as separate bundles instead of being included in the config model
      because some of these (TensorFlow) includes
      JNI code, and so can only exist in one instance in the server. -->
 <component id="ai.vespa.rankingexpression.importer.onnx.OnnxImporter" bundle="model-integration" />
 <component id="ai.vespa.rankingexpression.importer.tensorflow.TensorFlowImporter" bundle="model-integration" />
 <component id="ai.vespa.rankingexpression.importer.xgboost.XGBoostImporter" bundle="model-integration" />
+<component id="ai.vespa.rankingexpression.importer.lightgbm.LightGBMImporter" bundle="model-integration" />
 <component id="ai.vespa.rankingexpression.importer.vespa.VespaImporter" bundle="model-integration" />
diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/DimensionRenamer.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/DimensionRenamer.java
index d22a8067bd4..c7f320ed3b4 100644
--- a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/DimensionRenamer.java
+++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/DimensionRenamer.java
@@ -224,6 +224,11 @@ public class DimensionRenamer {
         /** Returns whether this is an opposite of another constraint */
         boolean isOpposite() { return opposite; }
 
+        public static Constraint equal() { return new EqualConstraint(false, false); }
+        public static Constraint notEqual() { return new NotEqualConstraint(false, false); }
+        public static Constraint lessThan() { return new LessThanConstraint(false, false); }
+        public static Constraint greaterThan() { return new GreaterThanConstraint(false, false); }
+
         public static Constraint equal(boolean soft) { return new EqualConstraint(soft, false); }
         public static Constraint notEqual(boolean soft) { return new NotEqualConstraint(soft, false); }
         public static Constraint lessThan(boolean soft) { return new LessThanConstraint(soft, false); }
diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/lightgbm/LightGBMImporter.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/lightgbm/LightGBMImporter.java
new file mode 100644
index 00000000000..76caa652ad2
--- /dev/null
+++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/lightgbm/LightGBMImporter.java
@@ -0,0 +1,54 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package ai.vespa.rankingexpression.importer.lightgbm;
+
+import ai.vespa.rankingexpression.importer.ImportedModel;
+import ai.vespa.rankingexpression.importer.ModelImporter;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.yahoo.searchlib.rankingexpression.RankingExpression;
+import com.yahoo.searchlib.rankingexpression.parser.ParseException;
+
+import java.io.File;
+import java.io.IOException;
+
+/**
+ * Converts a LightGBM model into a ranking expression.
+ *
+ * @author lesters
+ */
+public class LightGBMImporter extends ModelImporter {
+
+    @Override
+    public boolean canImport(String modelPath) {
+        File modelFile = new File(modelPath);
+        if ( ! modelFile.isFile()) return false;
+        return modelFile.toString().endsWith(".json") && probe(modelFile);
+    }
+
+    /**
+     * Returns true if the give file looks like a LightGBM json file.
+     * Currently, we just check if the json has an element called "tree_info"
+     */
+    private boolean probe(File modelFile) {
+        try {
+            return new ObjectMapper().readTree(modelFile).has("tree_info");
+        } catch (IOException e) {
+            throw new IllegalArgumentException("Could not read '" + modelFile + "'", e);
+        }
+    }
+
+    @Override
+    public ImportedModel importModel(String modelName, String modelPath) {
+        try {
+            ImportedModel model = new ImportedModel(modelName, modelPath);
+            LightGBMParser parser = new LightGBMParser(modelPath);
+            RankingExpression expression = new RankingExpression(parser.toRankingExpression());
+            model.expression(modelName, expression);
+            return model;
+        } catch (IOException e) {
+            throw new IllegalArgumentException("Could not import LightGBM model from '" + modelPath + "'", e);
+        } catch (ParseException e) {
+            throw new IllegalArgumentException("Could not parse ranking expression resulting from '" + modelPath + "'", e);
+        }
+    }
+
+}
diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/lightgbm/LightGBMNode.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/lightgbm/LightGBMNode.java
new file mode 100644
index 00000000000..dc76ed8cb6f
--- /dev/null
+++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/lightgbm/LightGBMNode.java
@@ -0,0 +1,67 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package ai.vespa.rankingexpression.importer.lightgbm;
+
+/**
+ * @author lesters
+ */
+public class LightGBMNode {
+
+    // split nodes
+    private int split_feature;
+    private String threshold;  // double for numerical, string for categorical
+    private String decision_type;
+    private boolean default_left;
+    private String missing_type;
+    private int internal_count;
+    private LightGBMNode left_child;
+    private LightGBMNode right_child;
+
+    // leaf nodes
+    private double leaf_value;
+    private int leaf_count;
+
+    public int getSplit_feature() {
+        return split_feature;
+    }
+
+    public String getThreshold() {
+        return threshold;
+    }
+
+    public String getDecision_type() {
+        return decision_type;
+    }
+
+    public boolean isDefault_left() {
+        return default_left;
+    }
+
+    public String getMissing_type() {
+        return missing_type;
+    }
+
+    public int getInternal_count() {
+        return internal_count;
+    }
+
+    public LightGBMNode getLeft_child() {
+        return left_child;
+    }
+
+    public LightGBMNode getRight_child() {
+        return right_child;
+    }
+
+    public double getLeaf_value() {
+        return leaf_value;
+    }
+
+    public int getLeaf_count() {
+        return leaf_count;
+    }
+
+    public boolean isLeaf() {
+        return left_child == null && right_child == null;
+    }
+
+}
\ No newline at end of file
diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/lightgbm/LightGBMParser.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/lightgbm/LightGBMParser.java
new file mode 100644
index 00000000000..996343674ce
--- /dev/null
+++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/lightgbm/LightGBMParser.java
@@ -0,0 +1,146 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package ai.vespa.rankingexpression.importer.lightgbm;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.stream.Collectors;
+
+/**
+ * @author lesters
+ */
+class LightGBMParser {
+
+    private final String objective;
+    private final List<LightGBMNode> nodes;
+    private final List<String> featureNames;
+    private final Map<Integer, List<String>> categoryValues;  // pr feature index
+
+    LightGBMParser(String filePath) throws JsonProcessingException, IOException {
+        ObjectMapper mapper = new ObjectMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+        JsonNode root = mapper.readTree(new File(filePath));
+
+        objective = root.get("objective").asText("regression");
+        featureNames = parseFeatureNames(root);
+        nodes = parseTrees(mapper, root);
+        categoryValues = parseCategoryValues(root);
+    }
+
+    private List<String> parseFeatureNames(JsonNode root) {
+        List<String> features = new ArrayList<>();
+        for (JsonNode name : root.get("feature_names")) {
+            features.add(name.textValue());
+        }
+        return features;
+    }
+
+    private List<LightGBMNode> parseTrees(ObjectMapper mapper, JsonNode root) throws JsonProcessingException {
+        List<LightGBMNode> nodes = new ArrayList<>();
+        for (JsonNode treeNode : root.get("tree_info")) {
+            nodes.add(mapper.treeToValue(treeNode.get("tree_structure"), LightGBMNode.class));
+        }
+        return nodes;
+    }
+
+    private Map<Integer, List<String>> parseCategoryValues(JsonNode root) {
+        Map<Integer, List<String>> categoryValues = new HashMap<>();
+
+        // Since the JSON format does not explicitly tell which features are
+        // categorical, we traverse the decision tree looking for categorical
+        // decisions and use that to determine which categorical features.
+        Set<Integer> categoricalFeatures = new TreeSet<>();
+        nodes.forEach(node -> findCategoricalFeatures(node, categoricalFeatures));
+
+        // Again, the LightGBM JSON format does not explicitly tell which
+        // categorical values map to each categorical feature. The assumption
+        // here is that the order they appear in the "pandas_categorical"
+        // structure is the same order as the "feature_names".
+        var pandasFeatureIterator = root.get("pandas_categorical").iterator();
+        var categoricalFeatureIterator = categoricalFeatures.iterator();
+        while (pandasFeatureIterator.hasNext() && categoricalFeatureIterator.hasNext()) {
+            List<String> values = new ArrayList<>();
+            pandasFeatureIterator.next().forEach(value -> values.add(value.textValue()));
+            categoryValues.put(categoricalFeatureIterator.next(), values);
+        }
+
+        return categoryValues;
+    }
+
+    private void findCategoricalFeatures(LightGBMNode node, Set<Integer> categoricalFeatures) {
+        if (node == null || node.isLeaf()) {
+            return;
+        }
+        if (node.getDecision_type().equals("==")) {
+            categoricalFeatures.add(node.getSplit_feature());
+        }
+        findCategoricalFeatures(node.getLeft_child(), categoricalFeatures);
+        findCategoricalFeatures(node.getRight_child(), categoricalFeatures);
+    }
+
+    String toRankingExpression() {
+        return applyObjective(nodes.stream().map(this::nodeToRankingExpression).collect(Collectors.joining(" + \n")));
+    }
+
+    // See https://lightgbm.readthedocs.io/en/latest/Parameters.html#objective
+    private String applyObjective(String expression) {
+        if (objective.startsWith("binary") || objective.equals("cross_entropy")) {
+            return "sigmoid(" + expression + ")";
+        }
+        if (objective.equals("poisson") || objective.equals("gamma") || objective.equals("tweedie")) {
+            return "exp(" + expression + ")";
+        }
+        return expression;  // else: use expression directly
+    }
+
+    private String nodeToRankingExpression(LightGBMNode node) {
+        if (node.isLeaf()) {
+            return Double.toString(node.getLeaf_value());
+        } else {
+            String condition;
+            String feature = featureNames.get(node.getSplit_feature());
+            if (node.getDecision_type().equals("==")) {
+                String values = transformCategoryIndexesToValues(node);
+                if (node.isDefault_left()) {  // means go left (true) when isNan
+                    condition = "isNan(" + feature + ") || (" + feature + " in [ " + values + "])";
+                } else {
+                    condition = feature + " in [" + values + "]";
+                }
+            } else {  // assumption: all other decision types are <=
+                double value = Double.parseDouble(node.getThreshold());
+                if (node.isDefault_left()) {
+                    condition = "!(" + feature + " >= " + value + ")";
+                } else {
+                    condition = feature + " < " + value;
+                }
+            }
+            String left = nodeToRankingExpression(node.getLeft_child());
+            String right = nodeToRankingExpression(node.getRight_child());
+            return "if (" + condition + ", " + left + ", " + right + ")";
+        }
+    }
+
+    private String transformCategoryIndexesToValues(LightGBMNode node) {
+        return Arrays.stream(node.getThreshold().split("\\|\\|"))
+                .map(index -> "\"" + transformCategoryIndexToValue(node.getSplit_feature(), index) + "\"")
+                .collect(Collectors.joining(","));
+    }
+
+    private String transformCategoryIndexToValue(int featureIndex, String valueIndex) {
+        if ( ! categoryValues.containsKey(featureIndex) ) {
+            return valueIndex;  // We don't have a pandas categorical lookup table
+        }
+        return categoryValues.get(featureIndex).get(Integer.parseInt(valueIndex));
+    }
+
+}
\ No newline at end of file
diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/lightgbm/package-info.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/lightgbm/package-info.java
new file mode 100644
index 00000000000..b29145ee21b
--- /dev/null
+++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/lightgbm/package-info.java
@@ -0,0 +1,5 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package ai.vespa.rankingexpression.importer.lightgbm;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/onnx/AttributeConverter.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/onnx/AttributeConverter.java
index 8caa158e5be..b272d4c6750 100644
--- a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/onnx/AttributeConverter.java
+++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/onnx/AttributeConverter.java
@@ -5,6 +5,7 @@ import ai.vespa.rankingexpression.importer.OrderedTensorType;
 import ai.vespa.rankingexpression.importer.operations.IntermediateOperation;
 import com.yahoo.searchlib.rankingexpression.evaluation.DoubleValue;
 import com.yahoo.searchlib.rankingexpression.evaluation.StringValue;
+import com.yahoo.searchlib.rankingexpression.evaluation.TensorValue;
 import com.yahoo.searchlib.rankingexpression.evaluation.Value;
 import onnx.Onnx;
 
@@ -37,6 +38,7 @@ class AttributeConverter implements IntermediateOperation.AttributeMap {
                     case INT: return Optional.of(DoubleValue.frozen(attr.getI()));
                     case FLOAT: return Optional.of(DoubleValue.frozen(attr.getF()));
                     case STRING: return Optional.of(StringValue.frozen(attr.getS().toString()));
+                    case TENSOR: return Optional.of(new TensorValue(TensorConverter.toVespaTensor(attr.getT(), TypeConverter.typeFrom(attr.getT()))));
                     default:
                         return Optional.empty();
                 }
diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/onnx/GraphImporter.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/onnx/GraphImporter.java
index d42338deaf8..ffc64c38f16 100644
--- a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/onnx/GraphImporter.java
+++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/onnx/GraphImporter.java
@@ -2,13 +2,18 @@
 
 package ai.vespa.rankingexpression.importer.onnx;
 
+import ai.vespa.rankingexpression.importer.operations.ExpandDims;
+import ai.vespa.rankingexpression.importer.operations.Gather;
+import ai.vespa.rankingexpression.importer.operations.OnnxCast;
 import ai.vespa.rankingexpression.importer.operations.Gemm;
 import ai.vespa.rankingexpression.importer.operations.ConcatReduce;
 import ai.vespa.rankingexpression.importer.operations.OnnxConcat;
 import ai.vespa.rankingexpression.importer.operations.Reduce;
 import ai.vespa.rankingexpression.importer.operations.Select;
+import ai.vespa.rankingexpression.importer.operations.Slice;
 import ai.vespa.rankingexpression.importer.operations.Softmax;
 import ai.vespa.rankingexpression.importer.operations.Squeeze;
+import ai.vespa.rankingexpression.importer.operations.Unsqueeze;
 import com.yahoo.searchlib.rankingexpression.evaluation.DoubleValue;
 import com.yahoo.searchlib.rankingexpression.evaluation.TensorValue;
 import ai.vespa.rankingexpression.importer.IntermediateGraph;
@@ -67,6 +72,7 @@ class GraphImporter {
             case "add":         return new Join(modelName, nodeName, inputs, ScalarFunctions.add());
             case "asin":        return new Map(modelName, nodeName, inputs, ScalarFunctions.asin());
             case "atan":        return new Map(modelName, nodeName, inputs, ScalarFunctions.atan());
+            case "cast":        return new OnnxCast(modelName, nodeName, inputs, attributes);
             case "ceil":        return new Map(modelName, nodeName, inputs, ScalarFunctions.ceil());
             case "concat":      return new OnnxConcat(modelName, nodeName, inputs, attributes);
             case "cos":         return new Map(modelName, nodeName, inputs, ScalarFunctions.cos());
@@ -75,6 +81,7 @@ class GraphImporter {
             case "equal":       return new Join(modelName, nodeName, inputs, ScalarFunctions.equal());
             case "exp":         return new Map(modelName, nodeName, inputs, ScalarFunctions.exp());
             case "floor":       return new Map(modelName, nodeName, inputs, ScalarFunctions.floor());
+            case "gather":      return new Gather(modelName, nodeName, inputs, attributes);
             case "gemm":        return new Gemm(modelName, nodeName, inputs, attributes);
             case "greater":     return new Join(modelName, nodeName, inputs, ScalarFunctions.greater());
             case "identity":    return new Identity(modelName, nodeName, inputs);
@@ -105,6 +112,7 @@ class GraphImporter {
             case "shape":       return new Shape(modelName, nodeName, inputs);
             case "sigmoid":     return new Map(modelName, nodeName, inputs, ScalarFunctions.sigmoid());
             case "sin":         return new Map(modelName, nodeName, inputs, ScalarFunctions.sin());
+            case "slice":       return new Slice(modelName, nodeName, inputs, attributes);
             case "softmax":     return new Softmax(modelName, nodeName, inputs, attributes);
             case "sub":         return new Join(modelName, nodeName, inputs, ScalarFunctions.subtract());
             case "squeeze":     return new Squeeze(modelName, nodeName, inputs, attributes);
@@ -113,6 +121,7 @@ class GraphImporter {
             case "where":       return new Select(modelName, nodeName, inputs);
             case "tan":         return new Map(modelName, nodeName, inputs, ScalarFunctions.tan());
             case "tanh":        return new Map(modelName, nodeName, inputs, ScalarFunctions.tanh());
+            case "unsqueeze":   return new Unsqueeze(modelName, nodeName, inputs, attributes);
         }
 
         IntermediateOperation op = new NoOp(modelName, nodeName, inputs);
diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/onnx/TensorConverter.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/onnx/TensorConverter.java
index 69d18d0ffcb..f8c7dc15857 100644
--- a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/onnx/TensorConverter.java
+++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/onnx/TensorConverter.java
@@ -37,12 +37,14 @@ class TensorConverter {
                 case BOOL: return new RawBoolValues(tensorProto);
                 case FLOAT: return new RawFloatValues(tensorProto);
                 case DOUBLE: return new RawDoubleValues(tensorProto);
+                case INT32: return new RawIntValues(tensorProto);
                 case INT64: return new RawLongValues(tensorProto);
             }
         } else {
             switch (tensorProto.getDataType()) {
                 case FLOAT: return new FloatValues(tensorProto);
                 case DOUBLE: return new DoubleValues(tensorProto);
+                case INT32: return new IntValues(tensorProto);
                 case INT64: return new LongValues(tensorProto);
             }
         }
@@ -96,6 +98,17 @@ class TensorConverter {
         @Override int size() { return size; }
     }
 
+    private static class RawIntValues extends RawValues {
+        private final IntBuffer values;
+        private final int size;
+        RawIntValues(Onnx.TensorProto tensorProto) {
+            values = bytes(tensorProto).asIntBuffer();
+            size = values.remaining();
+        }
+        @Override double get(int i) { return values.get(i); }
+        @Override int size() { return size; }
+    }
+
     private static class RawLongValues extends RawValues {
         private final LongBuffer values;
         private final int size;
@@ -125,6 +138,15 @@ class TensorConverter {
         @Override int size() { return tensorProto.getDoubleDataCount(); }
     }
 
+    private static class IntValues extends Values {
+        private final Onnx.TensorProto tensorProto;
+        IntValues(Onnx.TensorProto tensorProto) {
+            this.tensorProto = tensorProto;
+        }
+        @Override double get(int i) { return tensorProto.getInt32Data(i); }
+        @Override int size() { return tensorProto.getInt32DataCount(); }
+    }
+
     private static class LongValues extends Values {
         private final Onnx.TensorProto tensorProto;
         LongValues(Onnx.TensorProto tensorProto) {
diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/ExpandDims.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/ExpandDims.java
index 3487d889338..e02f29a63f9 100644
--- a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/ExpandDims.java
+++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/ExpandDims.java
@@ -40,7 +40,7 @@ public class ExpandDims extends IntermediateOperation {
         OrderedTensorType inputType = inputs.get(0).type().get();
         int dimensionToInsert = (int)axis.asDouble();
         if (dimensionToInsert < 0) {
-            dimensionToInsert = inputType.dimensions().size() - dimensionToInsert;
+            dimensionToInsert = inputType.dimensions().size() + dimensionToInsert;
         }
 
         OrderedTensorType.Builder typeBuilder = new OrderedTensorType.Builder(resultValueType());
diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Gather.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Gather.java
new file mode 100644
index 00000000000..2a34ae53d5e
--- /dev/null
+++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Gather.java
@@ -0,0 +1,170 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package ai.vespa.rankingexpression.importer.operations;
+
+import ai.vespa.rankingexpression.importer.DimensionRenamer;
+import ai.vespa.rankingexpression.importer.OrderedTensorType;
+import com.yahoo.searchlib.rankingexpression.Reference;
+import com.yahoo.searchlib.rankingexpression.evaluation.DoubleValue;
+import com.yahoo.searchlib.rankingexpression.rule.ArithmeticNode;
+import com.yahoo.searchlib.rankingexpression.rule.ArithmeticOperator;
+import com.yahoo.searchlib.rankingexpression.rule.ConstantNode;
+import com.yahoo.searchlib.rankingexpression.rule.EmbracedNode;
+import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode;
+import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode;
+import com.yahoo.searchlib.rankingexpression.rule.TensorFunctionNode;
+import com.yahoo.tensor.TensorType;
+import com.yahoo.tensor.functions.Generate;
+import com.yahoo.tensor.functions.Slice;
+import com.yahoo.tensor.functions.TensorFunction;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+
+import static com.yahoo.searchlib.rankingexpression.rule.TensorFunctionNode.wrapScalar;
+
+/*
+ * Onnx gather is the same as Numpy take.
+ */
+public class Gather extends IntermediateOperation {
+
+    private final AttributeMap attributeMap;
+
+    private int axis;
+
+    public Gather(String modelName, String nodeName, List<IntermediateOperation> inputs, AttributeMap attributeMap) {
+        super(modelName, nodeName, inputs);
+        this.attributeMap = attributeMap;
+    }
+
+    @Override
+    protected OrderedTensorType lazyGetType() {
+        if ( ! allInputTypesPresent(2)) return null;
+
+        OrderedTensorType dataType = inputs.get(0).type().get();
+        OrderedTensorType indicesType = inputs.get(1).type().get();
+
+        axis = (int) attributeMap.get("axis").orElse(DoubleValue.zero).asDouble();
+        if (axis < 0)
+            axis = dataType.rank() + axis;
+
+        OrderedTensorType.Builder typeBuilder = new OrderedTensorType.Builder(resultValueType());
+        for (int i = 0; i < axis; ++i) {
+            addDimension(i, dataType.dimensions().get(i).size().orElse(-1L), typeBuilder);
+        }
+        for (int i = 0; i < indicesType.rank(); ++i) {
+            addDimension(i + axis, indicesType.dimensions().get(i).size().orElse(-1L), typeBuilder);
+        }
+        for (int i = axis + 1; i < dataType.rank(); ++i) {
+            addDimension(i + indicesType.rank(), dataType.dimensions().get(i).size().orElse(-1L), typeBuilder);
+        }
+
+        inputs.get(0).exportAsRankingFunction = true;
+        inputs.get(1).exportAsRankingFunction = true;
+
+        return typeBuilder.build();
+    }
+
+    private void addDimension(int dimensionIndex, long size, OrderedTensorType.Builder typeBuilder) {
+        String name = String.format("%s_%d", vespaName(), dimensionIndex);
+        typeBuilder.add(TensorType.Dimension.indexed(name, size));
+    }
+
+    @Override
+    protected TensorFunction lazyGetFunction() {
+        if ( ! allInputFunctionsPresent(2)) return null;
+
+        IntermediateOperation data = inputs.get(0);
+        IntermediateOperation indices = inputs.get(1);
+        OrderedTensorType dataType = data.type().get();
+        OrderedTensorType indicesType = indices.type().get();
+
+        String dataFunctionName = data.rankingExpressionFunctionName();
+        String indicesFunctionName = indices.rankingExpressionFunctionName();
+
+        List<Slice.DimensionValue<Reference>> dataSliceDimensions = new ArrayList<>();
+        for (int i = 0; i < axis; ++i) {
+            addSliceDimension(dataSliceDimensions, dataType.dimensions().get(i).name(), i);
+        }
+
+        List<Slice.DimensionValue<Reference>> indicesSliceDimensions = new ArrayList<>();
+        for (int i = 0; i < indicesType.rank(); ++i) {
+            addSliceDimension(indicesSliceDimensions, indicesType.dimensions().get(i).name(), axis + i);
+        }
+        ExpressionNode sliceExpression = createSliceExpression(indicesSliceDimensions, indicesFunctionName);
+        ExpressionNode indexExpression = createIndexExpression(dataType, sliceExpression);
+        addSliceDimension(dataSliceDimensions, dataType.dimensions().get(axis).name(), indexExpression);
+
+        for (int i = axis + 1; i < dataType.rank(); ++i) {
+            addSliceDimension(dataSliceDimensions, dataType.dimensions().get(i).name(), i + indicesType.rank() - 1);
+        }
+
+        sliceExpression = createSliceExpression(dataSliceDimensions, dataFunctionName);
+        return Generate.bound(type.type(), wrapScalar(sliceExpression));
+    }
+
+    private ExpressionNode createSliceExpression(List<Slice.DimensionValue<Reference>> dimensionValues, String referenceName) {
+        TensorFunction<Reference> inputIndices = new TensorFunctionNode.ExpressionTensorFunction(new ReferenceNode(referenceName));
+        Slice<Reference> sliceIndices = new Slice<>(inputIndices, dimensionValues);
+        return new TensorFunctionNode(sliceIndices);
+    }
+
+    /** to support negative indexing */
+    private ExpressionNode createIndexExpression(OrderedTensorType dataType, ExpressionNode slice) {
+        ExpressionNode axisSize = new ConstantNode(new DoubleValue(dataType.dimensions().get(axis).size().get()));
+        ExpressionNode plus = new EmbracedNode(new ArithmeticNode(slice, ArithmeticOperator.PLUS, axisSize));
+        ExpressionNode mod = new ArithmeticNode(plus, ArithmeticOperator.MODULO, axisSize);
+        return mod;
+    }
+
+    private void addSliceDimension(List<Slice.DimensionValue<Reference>> dimensionValues, String dimensionName, ExpressionNode expr) {
+        dimensionValues.add(new Slice.DimensionValue<>(Optional.of(dimensionName), wrapScalar(new EmbracedNode(expr))));
+    }
+
+    private void addSliceDimension(List<Slice.DimensionValue<Reference>> dimensionValues, String dimensionName, int dimensionIndex) {
+        String outputDimensionName = type.dimensions().get(dimensionIndex).name();
+        addSliceDimension(dimensionValues, dimensionName, new ReferenceNode(outputDimensionName));
+    }
+
+    @Override
+    public void addDimensionNameConstraints(DimensionRenamer renamer) {
+        if ( ! allInputTypesPresent(2)) return;
+
+        for (int i = 0; i < type.dimensions().size(); i++) {
+            renamer.addDimension(type.dimensions().get(i).name());
+            for (int j = i + 1; j < type.dimensions().size(); j++) {
+                renamer.addConstraint(type.dimensions().get(i).name(), type.dimensions().get(j).name(),
+                        DimensionRenamer.Constraint.lessThan(), this);
+            }
+        }
+
+        OrderedTensorType dataType = inputs.get(0).type().get();
+        OrderedTensorType indicesType = inputs.get(1).type().get();
+
+        for (int i = 0; i < axis; ++i) {
+            renamer.addConstraint(type.dimensions().get(i).name(),
+                                  dataType.dimensions().get(i).name(),
+                                  DimensionRenamer.Constraint.equal(), this);
+        }
+        for (int i = 0; i < indicesType.rank(); ++i) {
+            renamer.addConstraint(type.dimensions().get(i + axis).name(),
+                                  indicesType.dimensions().get(i).name(),
+                                  DimensionRenamer.Constraint.equal(), this);
+        }
+        for (int i = axis + 1; i < dataType.rank(); ++i) {
+            renamer.addConstraint(type.dimensions().get(i + indicesType.rank() - 1).name(),
+                                  dataType.dimensions().get(i).name(),
+                                  DimensionRenamer.Constraint.equal(), this);
+        }
+
+    }
+
+    @Override
+    public Gather withInputs(List<IntermediateOperation> inputs) {
+        return new Gather(modelName(), name(), inputs, attributeMap);
+    }
+
+    @Override
+    public String operationName() { return "Gather"; }
+
+}
diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/IntermediateOperation.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/IntermediateOperation.java
index 724b5c6b3ac..2aa8b2a0d48 100644
--- a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/IntermediateOperation.java
+++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/IntermediateOperation.java
@@ -45,6 +45,7 @@ public abstract class IntermediateOperation {
     protected OrderedTensorType type;
     protected TensorFunction function;
     protected TensorFunction rankingExpressionFunction = null;
+    protected boolean exportAsRankingFunction = false;
 
     private final List<String> importWarnings = new ArrayList<>();
     private Value constantValue = null;
@@ -78,7 +79,7 @@ public abstract class IntermediateOperation {
             if (isConstant()) {
                 ExpressionNode constant = new ReferenceNode(Reference.simple("constant", vespaName()));
                 function = new TensorFunctionNode.ExpressionTensorFunction(constant);
-            } else if (outputs.size() > 1) {
+            } else if (outputs.size() > 1 || exportAsRankingFunction) {
                 rankingExpressionFunction = lazyGetFunction();
                 function = new VariableTensor(rankingExpressionFunctionName(), type.type());
             } else {
@@ -137,7 +138,7 @@ public abstract class IntermediateOperation {
             return Optional.of(constantValue);
         }
         if (constantValueFunction != null) {
-            return Optional.of(constantValueFunction.apply(type));
+            return Optional.of(constantValueFunction.apply(type().orElse(null)));
         }
         return Optional.empty();
     }
@@ -188,7 +189,7 @@ public abstract class IntermediateOperation {
             throw new IllegalArgumentException("Attempted to evaluate non-constant operation as a constant.");
         }
         Value val = evaluateAsConstant(new MapContext(DoubleValue.NaN));
-        if ( ! val.asTensor().type().equals(type.type()) ) {
+        if (type != null && ! val.asTensor().type().equals(type.type()) ) {
             throw new IllegalArgumentException("Constant evaluation in " + name + " resulted in wrong type. " +
                     "Expected: " + type.type() + " Got: " + val.asTensor().type());
         }
@@ -211,6 +212,9 @@ public abstract class IntermediateOperation {
                 result = new TensorValue(lazyGetFunction().evaluate(context));
             }
             context.put(constantName, result);
+            if (outputs.size() > 1 || exportAsRankingFunction) {
+                context.put(rankingExpressionFunctionName(), result);
+            }
         }
         return result;
     }
diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/OnnxCast.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/OnnxCast.java
new file mode 100644
index 00000000000..d15ac1b69f7
--- /dev/null
+++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/OnnxCast.java
@@ -0,0 +1,82 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package ai.vespa.rankingexpression.importer.operations;
+
+import ai.vespa.rankingexpression.importer.OrderedTensorType;
+import com.yahoo.tensor.functions.TensorFunction;
+import onnx.Onnx.TensorProto.DataType;
+
+import java.util.List;
+import java.util.function.DoubleUnaryOperator;
+
+public class OnnxCast extends IntermediateOperation {
+
+    private final AttributeMap attributeMap;
+    private final DataType toType;
+
+    public OnnxCast(String modelName, String nodeName, List<IntermediateOperation> inputs, AttributeMap attributeMap) {
+        super(modelName, nodeName, inputs);
+        this.attributeMap = attributeMap;
+        if (attributeMap.get("to").isEmpty()) {
+            throw new IllegalArgumentException("OnnxCast in " + name + ": Required attribute 'to' is missing.");
+        }
+        toType = DataType.forNumber((int) attributeMap.get("to").get().asDouble());
+    }
+
+    @Override
+    protected OrderedTensorType lazyGetType() {
+        if (!allInputTypesPresent(1))
+            return null;
+        return inputs.get(0).type().orElse(null);
+    }
+
+    @Override
+    protected TensorFunction lazyGetFunction() {
+        if ( ! allInputFunctionsPresent(1))
+            return null;
+        TensorFunction input = inputs.get(0).function().get();
+        switch (toType) {
+            case BOOL:
+                return new com.yahoo.tensor.functions.Map(input, new AsBool());
+            case INT8:
+            case INT16:
+            case INT32:
+            case INT64:
+            case UINT8:
+            case UINT16:
+            case UINT32:
+            case UINT64:
+                return new com.yahoo.tensor.functions.Map(input, new AsInt());
+            case FLOAT:
+            case DOUBLE:
+            case FLOAT16:
+                return input;
+            case STRING:
+                throw new IllegalArgumentException("OnnxCast in " + name + ": Casting to string is not implemented.");
+            default:
+                throw new IllegalArgumentException("OnnxCast in " + name + ": Unknown or undefined cast: " + toType.name());
+        }
+    }
+
+    @Override
+    public OnnxCast withInputs(List<IntermediateOperation> inputs) {
+        return new OnnxCast(modelName(), name(), inputs, attributeMap);
+    }
+
+    @Override
+    public String operationName() { return "Cast"; }
+
+    private static class AsBool implements DoubleUnaryOperator {
+        @Override
+        public double applyAsDouble(double operand) { return operand != 0.0 ? 1 : 0; }
+        @Override
+        public String toString() { return "f(a)(a!=0)"; }
+    }
+
+    private static class AsInt implements DoubleUnaryOperator {
+        @Override
+        public double applyAsDouble(double operand) { return operand < 0 ? Math.ceil(operand) : Math.floor(operand); }
+        @Override
+        public String toString() { return "f(a)(if (a < 0, ceil(a), floor(a)))"; }
+    }
+
+}
diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Slice.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Slice.java
new file mode 100644
index 00000000000..e5463291ef8
--- /dev/null
+++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Slice.java
@@ -0,0 +1,203 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package ai.vespa.rankingexpression.importer.operations;
+
+import ai.vespa.rankingexpression.importer.DimensionRenamer;
+import ai.vespa.rankingexpression.importer.OrderedTensorType;
+import com.yahoo.searchlib.rankingexpression.Reference;
+import com.yahoo.searchlib.rankingexpression.evaluation.DoubleValue;
+import com.yahoo.searchlib.rankingexpression.evaluation.Value;
+import com.yahoo.searchlib.rankingexpression.rule.ArithmeticNode;
+import com.yahoo.searchlib.rankingexpression.rule.ArithmeticOperator;
+import com.yahoo.searchlib.rankingexpression.rule.ConstantNode;
+import com.yahoo.searchlib.rankingexpression.rule.EmbracedNode;
+import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode;
+import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode;
+import com.yahoo.searchlib.rankingexpression.rule.TensorFunctionNode;
+import com.yahoo.tensor.TensorType;
+import com.yahoo.tensor.functions.Generate;
+import com.yahoo.tensor.functions.TensorFunction;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Optional;
+
+import static com.yahoo.searchlib.rankingexpression.rule.TensorFunctionNode.wrapScalar;
+
+/**
+ * Onnx slice operation.
+ *
+ * Opset 1 to 9 accepts starts, ends, and axes tensors as attributes
+ *
+ * Opset 10 and up accepts starts, ends, axes, and steps tensors as inputs. Here we assume these are
+ * constants, otherwise we can't import this model because that would mean we
+ * would not know the resulting tensor type until run-time, and that is currently
+ * not supported in Vespa.
+ */
+public class Slice extends IntermediateOperation {
+
+    private final AttributeMap attributes;
+
+    private int[] starts;
+    private int[] ends;
+    private int[] steps;
+
+    public Slice(String modelName, String nodeName, List<IntermediateOperation> inputs, AttributeMap attributes) {
+        super(modelName, nodeName, inputs);
+        this.attributes = attributes;
+    }
+
+    @Override
+    protected OrderedTensorType lazyGetType() {
+        if (inputs.size() < 1 || inputs.get(0).type().isEmpty()) {
+            return null;
+        }
+        OrderedTensorType dataType = inputs.get(0).type().get();
+
+        // required as we use tensor create
+        inputs.get(0).exportAsRankingFunction = true;
+
+        // Todo: only supports opsets 1-9, for >= get these from inputs
+        int[] startsInput = attributeListAsArray("starts", 0);
+        int[] endsInput = attributeListAsArray("ends", 0);
+        int[] stepsInput = new int[dataType.rank()]; Arrays.fill(stepsInput, 1);  // Todo: get from input when opset >= 10
+
+        int[] axes;
+        if (attributes.getList("axes").isPresent()) {
+            axes = attributeListAsArray("axes", 0);
+        } else {
+            // infer axes: default is [0, 1, ..., len('starts')-1]
+            axes = new int[startsInput.length];
+            for (int i = 0; i < startsInput.length; ++i) {
+                axes[i] = i;
+            }
+        }
+
+        if (startsInput.length != endsInput.length) {
+            throw new IllegalArgumentException("Slice in " + name + ": 'starts' and 'ends' indexes are not of the same size.");
+        }
+        if (startsInput.length != axes.length) {
+            throw new IllegalArgumentException("Slice in " + name + ": 'axes' and 'starts' are not of same size.");
+        }
+
+        int[] dimensionSizes = new int [dataType.rank()];
+        for (int i = 0; i < dataType.rank(); ++i) {
+            dimensionSizes[i] = dataType.dimensions().get(i).size().get().intValue();
+        }
+
+        starts = new int[dataType.rank()]; Arrays.fill(starts, 0);
+        ends = new int[dataType.rank()];
+        steps = new int[dataType.rank()]; Arrays.fill(steps, 1);
+
+        for (int i = 0; i < axes.length; ++i) {
+            int axis = axes[i];
+            int start = startsInput[i];
+            int end = endsInput[i];
+            int step = stepsInput[i];
+
+            axis = Math.min(axis, dataType.rank() - 1);
+            axis = axis < 0 ? axis + dataType.rank() : axis;
+
+            start = Math.min(start, dimensionSizes[axis]);
+            start = start < 0 ? start + dimensionSizes[axis] : start;
+
+            end = Math.min(end, dimensionSizes[axis]);
+            end = end < 0 ? end + dimensionSizes[axis] : end;
+
+            // Todo: check negative values for step size
+
+            starts[axis] = start;
+            steps[axis] = step;
+
+            if (step == 0) {
+                throw new IllegalArgumentException("Slice in " + name + ": illegal step size of 0.");
+            }
+            if ((end - start) < 1) {
+                throw new IllegalArgumentException("Slice in " + name + ": illegal start (" + start + ") and end (" + end + ") index.");
+            }
+            dimensionSizes[axis] = (end - start) / step;
+        }
+
+        OrderedTensorType.Builder typeBuilder = new OrderedTensorType.Builder(resultValueType());
+        for (int i = 0; i < dataType.rank(); ++i) {
+            addDimension(i, dimensionSizes[i], typeBuilder);
+        }
+        return typeBuilder.build();
+    }
+
+    private int[] attributeListAsArray(String name, int defaultValue) {
+        if (attributes.getList(name).isEmpty()) {
+            throw new IllegalArgumentException("Slice in " + name + ": Required attribute '" + name + "' is missing.");
+        }
+        List<Value> list = attributes.getList(name).get();
+        int[] result = new int[list.size()]; Arrays.fill(result, defaultValue);
+        for (int i = 0; i < list.size(); ++i) {
+            result[i] = (int)list.get(i).asDouble();
+        }
+        return result;
+    }
+
+    private void addDimension(int dimensionIndex, long size, OrderedTensorType.Builder typeBuilder) {
+        String name = String.format("%s_%d", vespaName(), dimensionIndex);
+        typeBuilder.add(TensorType.Dimension.indexed(name, size));
+    }
+
+    @Override
+    protected TensorFunction lazyGetFunction() {
+        if (inputs.size() < 1 || inputs.get(0).function().isEmpty()) {
+            return null;
+        }
+
+        IntermediateOperation data = inputs.get(0);
+        OrderedTensorType dataType = data.type().get();
+        String dataFunctionName = data.rankingExpressionFunctionName();
+
+        List<com.yahoo.tensor.functions.Slice.DimensionValue<Reference>> dimensionValues = new ArrayList<>();
+
+        for (int axis = 0; axis < dataType.rank(); ++axis) {
+            int start = starts[axis];
+            int step = steps[axis];
+
+            String inputDimensionName = dataType.dimensions().get(axis).name();
+            String outputDimensionName = type.dimensions().get(axis).name();
+
+            ExpressionNode stepSize = new ConstantNode(new DoubleValue(step));
+            ExpressionNode startIndex = new ConstantNode(new DoubleValue(start));
+
+            // step * (d0 + start)
+            ExpressionNode reference = new ReferenceNode(outputDimensionName);
+            ExpressionNode plus = new EmbracedNode(new ArithmeticNode(reference, ArithmeticOperator.PLUS, startIndex));
+            ExpressionNode mul = new ArithmeticNode(stepSize, ArithmeticOperator.MULTIPLY, plus);
+
+            dimensionValues.add(new com.yahoo.tensor.functions.Slice.DimensionValue<>(Optional.of(inputDimensionName), wrapScalar(new EmbracedNode(mul))));
+        }
+
+        TensorFunction<Reference> inputIndices = new TensorFunctionNode.ExpressionTensorFunction(new ReferenceNode(dataFunctionName));
+        com.yahoo.tensor.functions.Slice<Reference> sliceIndices = new com.yahoo.tensor.functions.Slice<>(inputIndices, dimensionValues);
+        ExpressionNode sliceExpression = new TensorFunctionNode(sliceIndices);
+
+        TensorFunction generate = Generate.bound(type.type(), wrapScalar(sliceExpression));
+        return generate;
+    }
+
+    @Override
+    public void addDimensionNameConstraints(DimensionRenamer renamer) {
+        // Todo: what to do?
+        for (int i = 0; i < type.dimensions().size(); i++) {
+            renamer.addDimension(type.dimensions().get(i).name());
+            for (int j = i + 1; j < type.dimensions().size(); j++) {
+                renamer.addConstraint(type.dimensions().get(i).name(), type.dimensions().get(j).name(),
+                        DimensionRenamer.Constraint.lessThan(), this);
+            }
+        }
+    }
+
+    @Override
+    public Slice withInputs(List<IntermediateOperation> inputs) {
+        return new Slice(modelName(), name(), inputs, attributes);
+    }
+
+    @Override
+    public String operationName() { return "Slice"; }
+
+}
diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Unsqueeze.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Unsqueeze.java
new file mode 100644
index 00000000000..0df09c21530
--- /dev/null
+++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Unsqueeze.java
@@ -0,0 +1,109 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package ai.vespa.rankingexpression.importer.operations;
+
+import ai.vespa.rankingexpression.importer.DimensionRenamer;
+import ai.vespa.rankingexpression.importer.OrderedTensorType;
+import com.yahoo.searchlib.rankingexpression.evaluation.DoubleValue;
+import com.yahoo.searchlib.rankingexpression.rule.ConstantNode;
+import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode;
+import com.yahoo.searchlib.rankingexpression.rule.GeneratorLambdaFunctionNode;
+import com.yahoo.tensor.TensorType;
+import com.yahoo.tensor.functions.Generate;
+import com.yahoo.tensor.functions.ScalarFunctions;
+import com.yahoo.tensor.functions.TensorFunction;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+public class Unsqueeze extends IntermediateOperation {
+
+    private final AttributeMap attributeMap;
+    private List<String> expandDimensions;
+
+    public Unsqueeze(String modelName, String nodeName, List<IntermediateOperation> inputs, AttributeMap attributeMap) {
+        super(modelName, nodeName, inputs);
+        this.attributeMap = attributeMap;
+        if (attributeMap.getList("axes").isEmpty()) {
+            throw new IllegalArgumentException("Unsqueeze in " + name + ": Required attribute 'axes' is missing.");
+        }
+    }
+
+    @Override
+    protected OrderedTensorType lazyGetType() {
+        if ( ! allInputTypesPresent(1)) return null;
+
+        OrderedTensorType inputType = inputs.get(0).type().get();
+        Set<Integer> dimensionsToInsert = attributeMap.getList("axes").get().stream().
+                map(d -> (int)d.asDouble()).collect(Collectors.toSet());
+
+        // handle negative dimension indexes
+        int rank = inputType.rank() + dimensionsToInsert.size();
+        dimensionsToInsert = dimensionsToInsert.stream().map(d -> d < 0 ? rank + d : d).collect(Collectors.toSet());
+
+        expandDimensions = new ArrayList<>();
+        OrderedTensorType.Builder typeBuilder = new OrderedTensorType.Builder(resultValueType());
+        int inputDimensionIndex = 0;
+        for (int expandedDimensionIndex = 0; expandedDimensionIndex < rank; ++expandedDimensionIndex) {
+            if (dimensionsToInsert.contains(expandedDimensionIndex)) {
+                addDimension(expandedDimensionIndex, typeBuilder);
+            } else {
+                typeBuilder.add(inputType.dimensions().get(inputDimensionIndex));
+                inputDimensionIndex++;
+            }
+        }
+        return typeBuilder.build();
+    }
+
+    private void addDimension(int dimensionIndex, OrderedTensorType.Builder typeBuilder) {
+        String name = String.format("%s_%d", vespaName(), dimensionIndex);
+        expandDimensions.add(name);
+        typeBuilder.add(TensorType.Dimension.indexed(name, 1L));
+    }
+
+    @Override
+    protected TensorFunction lazyGetFunction() {
+        if ( ! allInputFunctionsPresent(1)) return null;
+
+        // multiply with a generated tensor created from the expanded dimensions
+        TensorType.Builder typeBuilder = new TensorType.Builder(resultValueType());
+        for (String name : expandDimensions) {
+            typeBuilder.indexed(name, 1);
+        }
+        TensorType generatedType = typeBuilder.build();
+        ExpressionNode generatedExpression = new ConstantNode(new DoubleValue(1));
+        Generate generatedFunction = new Generate(generatedType,
+                new GeneratorLambdaFunctionNode(generatedType, generatedExpression).asLongListToDoubleOperator());
+        return new com.yahoo.tensor.functions.Join(inputs().get(0).function().get(), generatedFunction, ScalarFunctions.multiply());
+    }
+
+    @Override
+    public void addDimensionNameConstraints(DimensionRenamer renamer) {
+        addConstraintsFrom(type, renamer);
+    }
+
+    @Override
+    public void renameDimensions(DimensionRenamer renamer) {
+        super.renameDimensions(renamer);
+        List<String> renamedDimensions = new ArrayList<>(expandDimensions.size());
+        for (String name : expandDimensions) {
+            Optional<String> newName = renamer.dimensionNameOf(name);
+            if (newName.isEmpty()) {
+                return;  // presumably, already renamed
+            }
+            renamedDimensions.add(newName.get());
+        }
+        expandDimensions = renamedDimensions;
+    }
+
+    @Override
+    public Unsqueeze withInputs(List<IntermediateOperation> inputs) {
+        return new Unsqueeze(modelName(), name(), inputs, attributeMap);
+    }
+
+    @Override
+    public String operationName() { return "Unsqueeze"; }
+
+}
diff --git a/model-integration/src/test/java/ai/vespa/rankingexpression/importer/lightgbm/LightGBMImportEvaluationTestCase.java b/model-integration/src/test/java/ai/vespa/rankingexpression/importer/lightgbm/LightGBMImportEvaluationTestCase.java
new file mode 100644
index 00000000000..d2ef4b984ff
--- /dev/null
+++ b/model-integration/src/test/java/ai/vespa/rankingexpression/importer/lightgbm/LightGBMImportEvaluationTestCase.java
@@ -0,0 +1,49 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package ai.vespa.rankingexpression.importer.lightgbm;
+
+import com.yahoo.searchlib.rankingexpression.RankingExpression;
+import com.yahoo.searchlib.rankingexpression.evaluation.ArrayContext;
+import com.yahoo.searchlib.rankingexpression.evaluation.ContextIndex;
+import com.yahoo.searchlib.rankingexpression.evaluation.DoubleValue;
+import com.yahoo.searchlib.rankingexpression.evaluation.ExpressionOptimizer;
+import com.yahoo.searchlib.rankingexpression.evaluation.gbdtoptimization.GBDTForestNode;
+import org.junit.Test;
+
+import static org.junit.Assert.assertTrue;
+
+/**
+ * @author lesters
+ */
+public class LightGBMImportEvaluationTestCase extends LightGBMTestBase {
+
+    @Test
+    public void testRegression() {
+        RankingExpression expression = importModel("src/test/models/lightgbm/regression.json");
+        ArrayContext context = new ArrayContext(expression, true, DoubleValue.NaN);
+
+        assertEvaluation(1.91300868, expression, features(context));
+        assertEvaluation(2.05469776, expression, features(context).add("numerical_1", 0.1).add("numerical_2", 0.2).add("categorical_1", "a").add("categorical_2", "i"));
+        assertEvaluation(2.0745534,  expression, features(context).add("numerical_2", 0.5).add("categorical_1", "b").add("categorical_2", "j"));
+        assertEvaluation(2.3571838,  expression, features(context).add("numerical_1", 0.7).add("numerical_2", 0.8).add("categorical_2", "m"));
+
+        ExpressionOptimizer optimizer = new ExpressionOptimizer();
+        optimizer.optimize(expression, (ContextIndex)context);
+        assertTrue(expression.getRoot() instanceof GBDTForestNode);
+
+        assertEvaluation(1.91300868, expression, features(context));
+        assertEvaluation(2.05469776, expression, features(context).add("numerical_1", 0.1).add("numerical_2", 0.2).add("categorical_1", "a").add("categorical_2", "i"));
+        assertEvaluation(2.0745534,  expression, features(context).add("numerical_2", 0.5).add("categorical_1", "b").add("categorical_2", "j"));
+        assertEvaluation(2.3571838,  expression, features(context).add("numerical_1", 0.7).add("numerical_2", 0.8).add("categorical_2", "m"));
+    }
+
+    @Test
+    public void testClassification() {
+        RankingExpression expression = importModel("src/test/models/lightgbm/classification.json");
+        ArrayContext context = new ArrayContext(expression, DoubleValue.NaN);
+        assertEvaluation(0.37464997, expression, features(context));
+        assertEvaluation(0.37464997, expression, features(context).add("numerical_1", 0.1).add("numerical_2", 0.2).add("categorical_1", "a").add("categorical_2", "i"));
+        assertEvaluation(0.38730827, expression, features(context).add("numerical_2", 0.5).add("categorical_1", "b").add("categorical_2", "j"));
+        assertEvaluation(0.5647872,  expression, features(context).add("numerical_1", 0.7).add("numerical_2", 0.8).add("categorical_2", "m"));
+    }
+
+}
diff --git a/model-integration/src/test/java/ai/vespa/rankingexpression/importer/lightgbm/LightGBMTestBase.java b/model-integration/src/test/java/ai/vespa/rankingexpression/importer/lightgbm/LightGBMTestBase.java
new file mode 100644
index 00000000000..80c2ce68394
--- /dev/null
+++ b/model-integration/src/test/java/ai/vespa/rankingexpression/importer/lightgbm/LightGBMTestBase.java
@@ -0,0 +1,42 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package ai.vespa.rankingexpression.importer.lightgbm;
+
+import com.yahoo.searchlib.rankingexpression.RankingExpression;
+import com.yahoo.searchlib.rankingexpression.evaluation.ArrayContext;
+import com.yahoo.searchlib.rankingexpression.evaluation.StringValue;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author lesters
+ */
+class LightGBMTestBase {
+
+    RankingExpression importModel(String path) {
+        return new LightGBMImporter().importModel("lightgbm", path).expressions().get("lightgbm");
+    }
+
+    void assertEvaluation(double expected, RankingExpression expr, TestFeatures features) {
+        assertEquals(expected, expr.evaluate(features.context).asDouble(), 1e-6);
+    }
+
+    TestFeatures features(ArrayContext context) {
+        return new TestFeatures(context.clone());
+    }
+
+    static class TestFeatures {
+        private final ArrayContext context;
+        TestFeatures(ArrayContext context) {
+            this.context = context;
+        }
+        TestFeatures add(String name, double value) {
+            context.put(name, value);
+            return this;
+        }
+        TestFeatures add(String name, String value) {
+            context.put(name, new StringValue(value));
+            return this;
+        }
+    }
+
+}
diff --git a/model-integration/src/test/java/ai/vespa/rankingexpression/importer/onnx/OnnxOperationsTestCase.java b/model-integration/src/test/java/ai/vespa/rankingexpression/importer/onnx/OnnxOperationsTestCase.java
index 6954abe5157..94c5577357b 100644
--- a/model-integration/src/test/java/ai/vespa/rankingexpression/importer/onnx/OnnxOperationsTestCase.java
+++ b/model-integration/src/test/java/ai/vespa/rankingexpression/importer/onnx/OnnxOperationsTestCase.java
@@ -17,6 +17,7 @@ import com.yahoo.tensor.functions.ConstantTensor;
 import com.yahoo.tensor.functions.Rename;
 import com.yahoo.tensor.functions.TensorFunction;
 import onnx.Onnx;
+import org.junit.Ignore;
 import org.junit.Test;
 
 import java.util.ArrayList;
@@ -26,7 +27,9 @@ import static ai.vespa.rankingexpression.importer.onnx.GraphImporter.*;
 import static onnx.Onnx.AttributeProto.AttributeType.FLOAT;
 import static onnx.Onnx.AttributeProto.AttributeType.INT;
 import static onnx.Onnx.AttributeProto.AttributeType.INTS;
+import static onnx.Onnx.AttributeProto.AttributeType.TENSOR;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
 
 /**
  * Unit tests for ONNX operators. The number on the test reflects the minimum
@@ -294,6 +297,27 @@ public class OnnxOperationsTestCase {
     }
 
     @Test
+    public void testUnsqueeze1() throws ParseException {
+        Tensor x = evaluate("tensor(d0[2]):[1, 2]");
+        assertEval("unsqueeze", x, evaluate("tensor(d0[1],d1[2]):[1, 2]"), createAttribute("axes", new int[] {0}));
+        assertEval("unsqueeze", x, evaluate("tensor(d0[2],d1[1]):[1, 2]"), createAttribute("axes", new int[] {1}));
+        assertEval("unsqueeze", x, evaluate("tensor(d0[2],d1[1]):[1, 2]"), createAttribute("axes", new int[] {-1}));
+        assertEval("unsqueeze", x, evaluate("tensor(d0[1],d1[2]):[1, 2]"), createAttribute("axes", new int[] {-2}));
+        assertEval("unsqueeze", x, evaluate("tensor(d0[1],d1[2]):[1, 2]"), createAttribute("axes", new int[] {0,0}));
+        assertEval("unsqueeze", x, evaluate("tensor(d0[1],d1[2],d2[1]):[1, 2]"), createAttribute("axes", new int[] {0,2}));
+        assertEval("unsqueeze", x, evaluate("tensor(d0[1],d1[2],d2[1]):[1, 2]"), createAttribute("axes", new int[] {2,0}));
+
+        x = evaluate("tensor(d0[2],d1[3]):[1,2,3,4,5,6]");
+        assertEval("unsqueeze", x, evaluate("tensor(d0[1],d1[1],d2[2],d3[3]):[1,2,3,4,5,6]"), createAttribute("axes", new int[] {0,1}));
+        assertEval("unsqueeze", x, evaluate("tensor(d0[1],d1[2],d2[1],d3[3]):[1,2,3,4,5,6]"), createAttribute("axes", new int[] {0,2}));
+        assertEval("unsqueeze", x, evaluate("tensor(d0[1],d1[2],d2[3],d3[1]):[1,2,3,4,5,6]"), createAttribute("axes", new int[] {0,3}));
+        assertEval("unsqueeze", x, evaluate("tensor(d0[2],d1[1],d2[1],d3[3]):[1,2,3,4,5,6]"), createAttribute("axes", new int[] {1,2}));
+        assertEval("unsqueeze", x, evaluate("tensor(d0[2],d1[3],d2[1],d3[1]):[1,2,3,4,5,6]"), createAttribute("axes", new int[] {2,3}));
+        assertEval("unsqueeze", x, evaluate("tensor(d0[1],d1[2],d2[1],d3[3],d4[1]):[1,2,3,4,5,6]"), createAttribute("axes", new int[] {0,2,4}));
+        assertEval("unsqueeze", x, evaluate("tensor(d0[1],d1[2],d2[1],d3[3],d4[1]):[1,2,3,4,5,6]"), createAttribute("axes", new int[] {4,2,0}));
+    }
+
+    @Test
     public void testWhere9() throws ParseException {
         Tensor x = evaluate("tensor(d0[2],d1[2]):[1, 2, 3, 4]");
         Tensor y = evaluate("tensor(d0[2],d1[2]):[5, 6, 7, 8]");
@@ -308,6 +332,109 @@ public class OnnxOperationsTestCase {
         assertEval("where", evaluate("tensor(d0[1],d1[1]):[1]"), x, y, x);
     }
 
+    @Test
+    public void testCast1() throws ParseException {
+        Tensor x = evaluate("tensor(d0[4]):[-1.9, 0.0, 1.1, 2.9]");
+        assertEval("cast", x, evaluate("tensor(d0[4]):[1,0,1,1]"), createAttribute("to", 9));  // boolean
+        assertEval("cast", x, evaluate("tensor(d0[4]):[-1,0,1,2]"), createAttribute("to", 6));  // int32
+        assertEval("cast", x, evaluate("tensor(d0[4]):[-1,0,1,2]"), createAttribute("to", 12)); // uint32
+        assertEval("cast", x, evaluate("tensor(d0[4]):[-1.9,0,1.1,2.9]"), createAttribute("to", 1));  // float
+        try {
+            assertEval("cast", x, evaluate("tensor(d0[4]):[1,0,1,1]"), createAttribute("to", 8));  // string
+            fail();
+        } catch (IllegalArgumentException e) {
+            assertEquals(e.getMessage(), "OnnxCast in cast: Casting to string is not implemented.");
+        }
+    }
+
+    @Test
+    public void testGather1() throws ParseException {
+        // 1 dim input, 1 dim indices
+        Tensor x = evaluate("tensor(d0[6]):[1,2,3,4,5,6]");
+        Tensor y = evaluate("tensor(d0[3]):[0,2,4]");
+        assertEval("gather", x, y, evaluate("tensor(d0[3]):[1,3,5]"));
+
+        // 2 dim input, 1 dim indices - axis 0
+        x = evaluate("tensor(d0[3],d1[2]):[1, 2, 3, 4, 5, 6]");
+        y = evaluate("tensor(d0[4]):[2, 1, 0, 2]");
+        assertEval("gather", x, y, evaluate("tensor(d0[4],d1[2]):[5, 6, 3, 4, 1, 2, 5, 6]"));
+
+        // 1 dim input, 2 dim indices - axis 0
+        x = evaluate("tensor(d0[6]):[1, 2, 3, 4, 5, 6]");
+        y = evaluate("tensor(d0[2],d1[2]):[0, 1, 3, 5]");
+        assertEval("gather", x, y, evaluate("tensor(d0[2],d1[2]):[1, 2, 4, 6]"));
+
+        // 2 dim input, 2 dim indices - axis 0
+        x = evaluate("tensor(d0[3],d1[2]):[1, 2, 3, 4, 5, 6]");
+        y = evaluate("tensor(d0[2],d1[2]):[0, 1, 1, 2]");
+        assertEval("gather", x, y, evaluate("tensor(d0[2],d1[2],d2[2]):[1, 2, 3, 4, 3, 4, 5, 6]"), createAttribute("axis", -2));
+
+        // 2 dim input, 1 dim indices - axis 1
+        x = evaluate("tensor(d0[3],d1[2]):[1, 2, 3, 4, 5, 6]");
+        y = evaluate("tensor(d0[4]):[0, 1, 0, 1]");
+        assertEval("gather", x, y, evaluate("tensor(d0[3],d1[4]):[1,2,1,2,3,4,3,4,5,6,5,6]"), createAttribute("axis", 1));
+
+        // 2 dim input, 2 dim indices - axis 1
+        x = evaluate("tensor(d0[3],d1[3]):[1, 2, 3, 4, 5, 6, 7, 8, 9]");
+        y = evaluate("tensor(d0[1],d1[2]):[0, 2]");
+        assertEval("gather", x, y, evaluate("tensor(d0[3],d1[1],d2[2]):[1,3,4,6,7,9]"), createAttribute("axis", 1));
+
+        // 1 dim input, 1 dim indices - negative indices
+        x = evaluate("tensor(d0[6]):[1,2,3,4,5,6]");
+        y = evaluate("tensor(d0[3]):[0,-2,-4]");
+        assertEval("gather", x, y, evaluate("tensor(d0[3]):[1,5,3]"));
+    }
+
+    @Test
+    public void testSlice1() throws ParseException {
+        Tensor x = evaluate("tensor(d0[2],d1[4]):[ [1,2,3,4],[5,6,7,8] ]");
+        AttributeConverter attributes = createAttributes().
+                attr("starts", new int[] {1, 0}).
+                attr("ends", new int[] {2, 3}).
+                attr("axes", new int[] {0, 1}).build();
+        assertEval("slice", x, evaluate("tensor(d0[1],d1[3]):[ [5,6,7] ]"), attributes);
+
+        attributes = createAttributes().
+                attr("starts", new int[] {0, 1}).
+                attr("ends", new int[] {-1, 1000}).build();
+        assertEval("slice", x, evaluate("tensor(d0[1],d1[3]):[ [2,3,4] ]"), attributes);
+
+        attributes = createAttributes().
+                attr("starts", new int[] {0, 1}).
+                attr("ends", new int[] {3, 2}).
+                attr("axes", new int[] {1, 0}).build();  // axes are switched
+        assertEval("slice", x, evaluate("tensor(d0[1],d1[3]):[ [5,6,7] ]"), attributes);
+
+        attributes = createAttributes().
+                attr("starts", new int[] {1, 0}).
+                attr("ends", new int[] {2, 3}).
+                attr("axes", new int[] {0, -1}).build();  // negative axes
+        assertEval("slice", x, evaluate("tensor(d0[1],d1[3]):[ [5,6,7] ]"), attributes);
+
+        attributes = createAttributes().
+                attr("starts", new int[] {1}).
+                attr("ends", new int[] {2}).
+                attr("axes", new int[] {0}).build();  // axis 1 is not specified
+        assertEval("slice", x, evaluate("tensor(d0[1],d1[4]):[ [5,6,7,8] ]"), attributes);
+
+        attributes = createAttributes().
+                attr("starts", new int[] {0}).
+                attr("ends", new int[] {1}).build();
+        assertEval("slice", x, evaluate("tensor(d0[1],d1[4]):[ [1,2,3,4] ]"), attributes);
+    }
+
+    @Ignore
+    @Test
+    public void testSlice10() throws ParseException {
+        Tensor x = evaluate("tensor(d0[2],d1[4]):[ [1,2,3,4],[5,6,7,8] ]");
+        Tensor starts = evaluate("tensor(d0[2]):[1,0]");
+        Tensor ends = evaluate("tensor(d0[2]):[2,3]");
+        Tensor axes = evaluate("tensor(d0[2]):[0,1]");
+        Tensor steps = evaluate("tensor(d0[2]):[1,2]");
+        assertEval("slice", x, starts, ends, axes, steps, evaluate("tensor(d0[1],d1[2]):[ [5,7] ]"));
+
+    }
+
     private Tensor evaluate(String expr) throws ParseException {
         return evaluate(expr, null, null, null);
     }
@@ -334,28 +461,40 @@ public class OnnxOperationsTestCase {
     }
 
     private void assertEval(String opName, Tensor x, Tensor expected) {
-        assertEval(opName, x, null, null, expected, null);
+        assertEval(opName, x, null, null, null, null, expected, null);
     }
 
     private void assertEval(String opName, Tensor x, Tensor expected, AttributeConverter attr) {
-        assertEval(opName, x, null, null, expected, attr);
+        assertEval(opName, x, null, null, null, null, expected, attr);
     }
 
     private void assertEval(String opName, Tensor x, Tensor y, Tensor expected, AttributeConverter attr) {
-        assertEval(opName, x, y, null, expected, attr);
+        assertEval(opName, x, y, null, null, null, expected, attr);
     }
 
     private void assertEval(String opName, Tensor x, Tensor y, Tensor expected) {
-        assertEval(opName, x, y, null, expected, null);
+        assertEval(opName, x, y, null, null, null, expected, null);
     }
 
     private void assertEval(String opName, Tensor x, Tensor y, Tensor z, Tensor expected) {
-        assertEval(opName, x, y, z, expected, null);
+        assertEval(opName, x, y, z, null, null, expected, null);
     }
 
     private void assertEval(String opName, Tensor x, Tensor y, Tensor z, Tensor expected, AttributeConverter attr) {
+        assertEval(opName, x, y, z, null, null, expected, attr);
+    }
+
+    private void assertEval(String opName, Tensor x, Tensor y, Tensor z, Tensor q, Tensor expected) {
+        assertEval(opName, x, y, z, q, null, expected, null);
+    }
+
+    private void assertEval(String opName, Tensor x, Tensor y, Tensor z, Tensor q, Tensor r, Tensor expected) {
+        assertEval(opName, x, y, z, q, r, expected, null);
+    }
+
+    private void assertEval(String opName, Tensor x, Tensor y, Tensor z, Tensor q, Tensor r, Tensor expected, AttributeConverter attr) {
         Context context = new MapContext(DoubleValue.NaN);
-        List<IntermediateOperation> inputs = createInputs(context, x, y, z);
+        List<IntermediateOperation> inputs = createInputs(context, x, y, z, q, r);
         IntermediateOperation op = mapOperation(opName, inputs, modelName, opName, attr != null ? attr : createAttributes().build());
         optimizeAndRename(opName, op);
         Tensor result = evaluate(op);
@@ -363,11 +502,13 @@ public class OnnxOperationsTestCase {
         assertEquals(expected.type(), result.type());
     }
 
-    private List<IntermediateOperation> createInputs(Context context, Tensor x, Tensor y, Tensor z) {
+    private List<IntermediateOperation> createInputs(Context context, Tensor x, Tensor y, Tensor z, Tensor q, Tensor r) {
         List<IntermediateOperation> inputs = new ArrayList<>();
         addInput(inputs, context, x, "x");
         addInput(inputs, context, y, "y");
         addInput(inputs, context, z, "z");
+        addInput(inputs, context, q, "q");
+        addInput(inputs, context, r, "r");
         return inputs;
     }
 
@@ -451,6 +592,16 @@ public class OnnxOperationsTestCase {
             return this;
         }
 
+        Attributes attr(String name, Tensor tensor) {
+            Onnx.TensorProto.Builder builder = Onnx.TensorProto.newBuilder();
+            builder.setDataType(Onnx.TensorProto.DataType.DOUBLE);;
+            tensor.type().dimensions().forEach(d -> builder.addDims(d.size().get()));
+            tensor.valueIterator().forEachRemaining(builder::addDoubleData);
+            Onnx.TensorProto val = builder.build();
+            nodeBuilder.addAttribute(Onnx.AttributeProto.newBuilder().setName(name).setType(TENSOR).setT(val).build());
+            return this;
+        }
+
         AttributeConverter build() {
             return AttributeConverter.convert(nodeBuilder.build());
         }
diff --git a/model-integration/src/test/java/ai/vespa/rankingexpression/importer/onnx/SimpleImportTestCase.java b/model-integration/src/test/java/ai/vespa/rankingexpression/importer/onnx/SimpleImportTestCase.java
index d1dea730da5..9631bddd93d 100644
--- a/model-integration/src/test/java/ai/vespa/rankingexpression/importer/onnx/SimpleImportTestCase.java
+++ b/model-integration/src/test/java/ai/vespa/rankingexpression/importer/onnx/SimpleImportTestCase.java
@@ -3,8 +3,13 @@
 package ai.vespa.rankingexpression.importer.onnx;
 
 import ai.vespa.rankingexpression.importer.ImportedModel;
+import com.yahoo.searchlib.rankingexpression.RankingExpression;
+import com.yahoo.searchlib.rankingexpression.evaluation.Context;
 import com.yahoo.searchlib.rankingexpression.evaluation.MapContext;
 import com.yahoo.searchlib.rankingexpression.evaluation.TensorValue;
+import com.yahoo.searchlib.rankingexpression.rule.CompositeNode;
+import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode;
+import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode;
 import com.yahoo.tensor.Tensor;
 import com.yahoo.tensor.TensorType;
 import org.junit.Test;
@@ -21,21 +26,48 @@ public class SimpleImportTestCase {
         ImportedModel model = new OnnxImporter().importModel("test", "src/test/models/onnx/simple/simple.onnx");
 
         MapContext context = new MapContext();
-        context.put("query_tensor", new TensorValue(Tensor.Builder.of(TensorType.fromSpec("tensor(d0[1],d1[4])")).
-                cell(0.1, 0, 0).
-                cell(0.2, 0, 1).
-                cell(0.3, 0, 2).
-                cell(0.4, 0, 3).build()));
-        context.put("attribute_tensor", new TensorValue(Tensor.Builder.of(TensorType.fromSpec("tensor(d0[4],d1[1])")).
-                cell(0.1, 0, 0).
-                cell(0.2, 1, 0).
-                cell(0.3, 2, 0).
-                cell(0.4, 3, 0).build()));
-        context.put("bias_tensor", new TensorValue(Tensor.Builder.of(TensorType.fromSpec("tensor(d0[1],d1[1])")).
-                cell(1.0, 0, 0).build()));
+        context.put("query_tensor", new TensorValue(Tensor.from("tensor(d0[1],d1[4]):[0.1, 0.2, 0.3, 0.4]")));
+        context.put("attribute_tensor", new TensorValue(Tensor.from("tensor(d0[4],d1[1]):[0.1, 0.2, 0.3, 0.4]")));
+        context.put("bias_tensor", new TensorValue(Tensor.from("tensor(d0[1],d1[1]):[1.0]")));
 
         Tensor result = model.expressions().get("output").evaluate(context).asTensor();
         assertEquals(result, Tensor.from("tensor(d0[1],d1[1]):{{d0:0,d1:0}:1.3}"));
     }
 
+    @Test
+    public void testGather() {
+        ImportedModel model = new OnnxImporter().importModel("test", "src/test/models/onnx/simple/gather.onnx");
+
+        MapContext context = new MapContext();
+        context.put("data", new TensorValue(Tensor.from("tensor(d0[3],d1[2]):[1, 2, 3, 4, 5, 6]")));
+        context.put("indices", new TensorValue(Tensor.from("tensor(d0[2],d1[2]):[0, 1, 1, 2]")));
+
+        model.functions().forEach((k, v) -> evaluateFunction(context, model, k));
+
+        Tensor result = model.expressions().get("y").evaluate(context).asTensor();
+        assertEquals(result, Tensor.from("tensor(d0[2],d1[2],d2[2]):[1, 2, 3, 4, 3, 4, 5, 6]"));
+    }
+
+    private void evaluateFunction(Context context, ImportedModel model, String functionName) {
+        if (!context.names().contains(functionName)) {
+            RankingExpression e = RankingExpression.from(model.functions().get(functionName));
+            evaluateFunctionDependencies(context, model, e.getRoot());
+            context.put(functionName, new TensorValue(e.evaluate(context).asTensor()));
+        }
+    }
+
+    private void evaluateFunctionDependencies(Context context, ImportedModel model, ExpressionNode node) {
+        if (node instanceof ReferenceNode) {
+            String name = node.toString();
+            if (model.functions().containsKey(name)) {
+                evaluateFunction(context, model, name);
+            }
+        }
+        else if (node instanceof CompositeNode) {
+            for (ExpressionNode child : ((CompositeNode)node).children()) {
+                evaluateFunctionDependencies(context, model, child);
+            }
+        }
+    }
+
 }
diff --git a/model-integration/src/test/models/lightgbm/classification.json b/model-integration/src/test/models/lightgbm/classification.json
new file mode 100644
index 00000000000..1087446519d
--- /dev/null
+++ b/model-integration/src/test/models/lightgbm/classification.json
@@ -0,0 +1,275 @@
+{
+  "name": "tree",
+  "version": "v3",
+  "num_class": 1,
+  "num_tree_per_iteration": 1,
+  "label_index": 0,
+  "max_feature_idx": 3,
+  "average_output": false,
+  "objective": "binary sigmoid:1",
+  "feature_names": [
+    "numerical_1",
+    "numerical_2",
+    "categorical_1",
+    "categorical_2"
+  ],
+  "monotone_constraints": [],
+  "tree_info": [
+    {
+      "tree_index": 0,
+      "num_leaves": 3,
+      "num_cat": 2,
+      "shrinkage": 1,
+      "tree_structure": {
+        "split_index": 0,
+        "split_feature": 3,
+        "split_gain": 13080.099609375,
+        "threshold": "2||3||4",
+        "decision_type": "==",
+        "default_left": false,
+        "missing_type": "NaN",
+        "internal_value": 0,
+        "internal_weight": 0,
+        "internal_count": 100000,
+        "left_child": {
+          "split_index": 1,
+          "split_feature": 2,
+          "split_gain": 8303.599609375,
+          "threshold": "2||3||4",
+          "decision_type": "==",
+          "default_left": false,
+          "missing_type": "NaN",
+          "internal_value": 0.598248,
+          "internal_weight": 14841.2,
+          "internal_count": 59371,
+          "left_child": {
+            "leaf_index": 0,
+            "leaf_value": 0.10149588882231209,
+            "leaf_weight": 8812.104370772839,
+            "leaf_count": 35252
+          },
+          "right_child": {
+            "leaf_index": 2,
+            "leaf_value": -0.05076009488472203,
+            "leaf_weight": 6029.137221112847,
+            "leaf_count": 24119
+          }
+        },
+        "right_child": {
+          "leaf_index": 1,
+          "leaf_value": -0.1075553310531564,
+          "leaf_weight": 10156.217760130763,
+          "leaf_count": 40629
+        }
+      }
+    },
+    {
+      "tree_index": 1,
+      "num_leaves": 3,
+      "num_cat": 0,
+      "shrinkage": 0.1,
+      "tree_structure": {
+        "split_index": 0,
+        "split_feature": 1,
+        "split_gain": 12144.5,
+        "threshold": 0.4932456977560694,
+        "decision_type": "<=",
+        "default_left": true,
+        "missing_type": "NaN",
+        "internal_value": 0,
+        "internal_weight": 0,
+        "internal_count": 100000,
+        "left_child": {
+          "leaf_index": 0,
+          "leaf_value": -0.07039230856418545,
+          "leaf_weight": 12362.572675153613,
+          "leaf_count": 49561
+        },
+        "right_child": {
+          "split_index": 1,
+          "split_feature": 0,
+          "split_gain": 6445.509765625,
+          "threshold": 0.4026061210695467,
+          "decision_type": "<=",
+          "default_left": true,
+          "missing_type": "NaN",
+          "internal_value": 0.691647,
+          "internal_weight": 12581.6,
+          "internal_count": 50439,
+          "left_child": {
+            "leaf_index": 1,
+            "leaf_value": -0.016713933964828474,
+            "leaf_weight": 5157.183633238077,
+            "leaf_count": 20675
+          },
+          "right_child": {
+            "leaf_index": 2,
+            "leaf_value": 0.12881836794307533,
+            "leaf_weight": 7424.385220557451,
+            "leaf_count": 29764
+          }
+        }
+      }
+    },
+    {
+      "tree_index": 2,
+      "num_leaves": 3,
+      "num_cat": 2,
+      "shrinkage": 0.1,
+      "tree_structure": {
+        "split_index": 0,
+        "split_feature": 2,
+        "split_gain": 11470.099609375,
+        "threshold": "3||4",
+        "decision_type": "==",
+        "default_left": false,
+        "missing_type": "NaN",
+        "internal_value": 0,
+        "internal_weight": 0,
+        "internal_count": 100000,
+        "left_child": {
+          "leaf_index": 0,
+          "leaf_value": 0.0837843210726433,
+          "leaf_weight": 9858.360527098179,
+          "leaf_count": 39612
+        },
+        "right_child": {
+          "split_index": 1,
+          "split_feature": 3,
+          "split_gain": 8077.8701171875,
+          "threshold": "3||4",
+          "decision_type": "==",
+          "default_left": false,
+          "missing_type": "NaN",
+          "internal_value": -0.549408,
+          "internal_weight": 15039.7,
+          "internal_count": 60388,
+          "left_child": {
+            "leaf_index": 1,
+            "leaf_value": 0.035561394754096094,
+            "leaf_weight": 5955.117423638701,
+            "leaf_count": 23896
+          },
+          "right_child": {
+            "leaf_index": 2,
+            "leaf_value": -0.11424082565448186,
+            "leaf_weight": 9084.538012728095,
+            "leaf_count": 36492
+          }
+        }
+      }
+    },
+    {
+      "tree_index": 3,
+      "num_leaves": 3,
+      "num_cat": 0,
+      "shrinkage": 0.1,
+      "tree_structure": {
+        "split_index": 0,
+        "split_feature": 0,
+        "split_gain": 11022.599609375,
+        "threshold": 0.5135386524711826,
+        "decision_type": "<=",
+        "default_left": true,
+        "missing_type": "NaN",
+        "internal_value": 0,
+        "internal_weight": 0,
+        "internal_count": 100000,
+        "left_child": {
+          "split_index": 1,
+          "split_feature": 1,
+          "split_gain": 5789.919921875,
+          "threshold": 0.6237474076885036,
+          "decision_type": "<=",
+          "default_left": true,
+          "missing_type": "NaN",
+          "internal_value": -0.641438,
+          "internal_weight": 12881.9,
+          "internal_count": 51907,
+          "left_child": {
+            "leaf_index": 0,
+            "leaf_value": -0.11613056205533928,
+            "leaf_weight": 8044.6355674266815,
+            "leaf_count": 32426
+          },
+          "right_child": {
+            "leaf_index": 2,
+            "leaf_value": 0.022313103333779363,
+            "leaf_weight": 4837.266924858093,
+            "leaf_count": 19481
+          }
+        },
+        "right_child": {
+          "leaf_index": 1,
+          "leaf_value": 0.06927713686880098,
+          "leaf_weight": 11923.512641906738,
+          "leaf_count": 48093
+        }
+      }
+    },
+    {
+      "tree_index": 4,
+      "num_leaves": 3,
+      "num_cat": 2,
+      "shrinkage": 0.1,
+      "tree_structure": {
+        "split_index": 0,
+        "split_feature": 3,
+        "split_gain": 9828.9501953125,
+        "threshold": "3||4",
+        "decision_type": "==",
+        "default_left": false,
+        "missing_type": "NaN",
+        "internal_value": 0,
+        "internal_weight": 0,
+        "internal_count": 100000,
+        "left_child": {
+          "leaf_index": 0,
+          "leaf_value": 0.07771712562582928,
+          "leaf_weight": 9804.427681803703,
+          "leaf_count": 39586
+        },
+        "right_child": {
+          "split_index": 1,
+          "split_feature": 2,
+          "split_gain": 6332.2900390625,
+          "threshold": "3||4",
+          "decision_type": "==",
+          "default_left": false,
+          "missing_type": "NaN",
+          "internal_value": -0.51112,
+          "internal_weight": 14922.7,
+          "internal_count": 60414,
+          "left_child": {
+            "leaf_index": 1,
+            "leaf_value": 0.029062142260340918,
+            "leaf_weight": 5933.120021238923,
+            "leaf_count": 23922
+          },
+          "right_child": {
+            "leaf_index": 2,
+            "leaf_value": -0.10400033924773491,
+            "leaf_weight": 8989.602796778083,
+            "leaf_count": 36492
+          }
+        }
+      }
+    }
+  ],
+  "pandas_categorical": [
+    [
+      "a",
+      "b",
+      "c",
+      "d",
+      "e"
+    ],
+    [
+      "i",
+      "j",
+      "k",
+      "l",
+      "m"
+    ]
+  ]
+}
\ No newline at end of file
diff --git a/model-integration/src/test/models/lightgbm/regression.json b/model-integration/src/test/models/lightgbm/regression.json
new file mode 100644
index 00000000000..cf0488ecd8b
--- /dev/null
+++ b/model-integration/src/test/models/lightgbm/regression.json
@@ -0,0 +1,275 @@
+{
+  "name": "tree",
+  "version": "v3",
+  "num_class": 1,
+  "num_tree_per_iteration": 1,
+  "label_index": 0,
+  "max_feature_idx": 3,
+  "average_output": false,
+  "objective": "regression",
+  "feature_names": [
+    "numerical_1",
+    "numerical_2",
+    "categorical_1",
+    "categorical_2"
+  ],
+  "monotone_constraints": [],
+  "tree_info": [
+    {
+      "tree_index": 0,
+      "num_leaves": 3,
+      "num_cat": 1,
+      "shrinkage": 1,
+      "tree_structure": {
+        "split_index": 0,
+        "split_feature": 1,
+        "split_gain": 68.5353012084961,
+        "threshold": 0.46643291586559305,
+        "decision_type": "<=",
+        "default_left": true,
+        "missing_type": "NaN",
+        "internal_value": 0,
+        "internal_weight": 0,
+        "internal_count": 1000,
+        "left_child": {
+          "leaf_index": 0,
+          "leaf_value": 2.1594397038037663,
+          "leaf_weight": 469,
+          "leaf_count": 469
+        },
+        "right_child": {
+          "split_index": 1,
+          "split_feature": 3,
+          "split_gain": 41.27640151977539,
+          "threshold": "2||3||4",
+          "decision_type": "==",
+          "default_left": false,
+          "missing_type": "NaN",
+          "internal_value": 0.246035,
+          "internal_weight": 531,
+          "internal_count": 531,
+          "left_child": {
+            "leaf_index": 1,
+            "leaf_value": 2.235297305276056,
+            "leaf_weight": 302,
+            "leaf_count": 302
+          },
+          "right_child": {
+            "leaf_index": 2,
+            "leaf_value": 2.1792953471546546,
+            "leaf_weight": 229,
+            "leaf_count": 229
+          }
+        }
+      }
+    },
+    {
+      "tree_index": 1,
+      "num_leaves": 3,
+      "num_cat": 1,
+      "shrinkage": 0.1,
+      "tree_structure": {
+        "split_index": 0,
+        "split_feature": 2,
+        "split_gain": 64.22250366210938,
+        "threshold": "3||4",
+        "decision_type": "==",
+        "default_left": false,
+        "missing_type": "NaN",
+        "internal_value": 0,
+        "internal_weight": 0,
+        "internal_count": 1000,
+        "left_child": {
+          "leaf_index": 0,
+          "leaf_value": 0.03070842919354316,
+          "leaf_weight": 399,
+          "leaf_count": 399
+        },
+        "right_child": {
+          "split_index": 1,
+          "split_feature": 0,
+          "split_gain": 36.74250030517578,
+          "threshold": 0.5102250691730842,
+          "decision_type": "<=",
+          "default_left": true,
+          "missing_type": "NaN",
+          "internal_value": -0.204906,
+          "internal_weight": 601,
+          "internal_count": 601,
+          "left_child": {
+            "leaf_index": 1,
+            "leaf_value": -0.04439151147520909,
+            "leaf_weight": 315,
+            "leaf_count": 315
+          },
+          "right_child": {
+            "leaf_index": 2,
+            "leaf_value": 0.005117411709368601,
+            "leaf_weight": 286,
+            "leaf_count": 286
+          }
+        }
+      }
+    },
+    {
+      "tree_index": 2,
+      "num_leaves": 3,
+      "num_cat": 0,
+      "shrinkage": 0.1,
+      "tree_structure": {
+        "split_index": 0,
+        "split_feature": 1,
+        "split_gain": 57.1327018737793,
+        "threshold": 0.668665477622446,
+        "decision_type": "<=",
+        "default_left": true,
+        "missing_type": "NaN",
+        "internal_value": 0,
+        "internal_weight": 0,
+        "internal_count": 1000,
+        "left_child": {
+          "split_index": 1,
+          "split_feature": 1,
+          "split_gain": 40.859100341796875,
+          "threshold": 0.008118820676863816,
+          "decision_type": "<=",
+          "default_left": true,
+          "missing_type": "NaN",
+          "internal_value": -0.162926,
+          "internal_weight": 681,
+          "internal_count": 681,
+          "left_child": {
+            "leaf_index": 0,
+            "leaf_value": -0.15361238490967524,
+            "leaf_weight": 21,
+            "leaf_count": 21
+          },
+          "right_child": {
+            "leaf_index": 2,
+            "leaf_value": -0.01192330846157292,
+            "leaf_weight": 660,
+            "leaf_count": 660
+          }
+        },
+        "right_child": {
+          "leaf_index": 1,
+          "leaf_value": 0.03499044894987518,
+          "leaf_weight": 319,
+          "leaf_count": 319
+        }
+      }
+    },
+    {
+      "tree_index": 3,
+      "num_leaves": 3,
+      "num_cat": 1,
+      "shrinkage": 0.1,
+      "tree_structure": {
+        "split_index": 0,
+        "split_feature": 0,
+        "split_gain": 54.77090072631836,
+        "threshold": 0.5201391072644542,
+        "decision_type": "<=",
+        "default_left": true,
+        "missing_type": "NaN",
+        "internal_value": 0,
+        "internal_weight": 0,
+        "internal_count": 1000,
+        "left_child": {
+          "leaf_index": 0,
+          "leaf_value": -0.02141000620783247,
+          "leaf_weight": 543,
+          "leaf_count": 543
+        },
+        "right_child": {
+          "split_index": 1,
+          "split_feature": 2,
+          "split_gain": 27.200700759887695,
+          "threshold": "0||1",
+          "decision_type": "==",
+          "default_left": false,
+          "missing_type": "NaN",
+          "internal_value": 0.255704,
+          "internal_weight": 457,
+          "internal_count": 457,
+          "left_child": {
+            "leaf_index": 1,
+            "leaf_value": -0.004121485787596721,
+            "leaf_weight": 191,
+            "leaf_count": 191
+          },
+          "right_child": {
+            "leaf_index": 2,
+            "leaf_value": 0.04534090904886873,
+            "leaf_weight": 266,
+            "leaf_count": 266
+          }
+        }
+      }
+    },
+    {
+      "tree_index": 4,
+      "num_leaves": 3,
+      "num_cat": 1,
+      "shrinkage": 0.1,
+      "tree_structure": {
+        "split_index": 0,
+        "split_feature": 3,
+        "split_gain": 51.84349822998047,
+        "threshold": "2||3||4",
+        "decision_type": "==",
+        "default_left": false,
+        "missing_type": "NaN",
+        "internal_value": 0,
+        "internal_weight": 0,
+        "internal_count": 1000,
+        "left_child": {
+          "split_index": 1,
+          "split_feature": 1,
+          "split_gain": 39.352699279785156,
+          "threshold": 0.27283279016959255,
+          "decision_type": "<=",
+          "default_left": true,
+          "missing_type": "NaN",
+          "internal_value": 0.188414,
+          "internal_weight": 593,
+          "internal_count": 593,
+          "left_child": {
+            "leaf_index": 0,
+            "leaf_value": -0.01924803254356527,
+            "leaf_weight": 184,
+            "leaf_count": 184
+          },
+          "right_child": {
+            "leaf_index": 2,
+            "leaf_value": 0.03643772842347651,
+            "leaf_weight": 409,
+            "leaf_count": 409
+          }
+        },
+        "right_child": {
+          "leaf_index": 1,
+          "leaf_value": -0.02701711918923075,
+          "leaf_weight": 407,
+          "leaf_count": 407
+        }
+      }
+    }
+  ],
+  "pandas_categorical": [
+    [
+      "a",
+      "b",
+      "c",
+      "d",
+      "e"
+    ],
+    [
+      "i",
+      "j",
+      "k",
+      "l",
+      "m"
+    ]
+  ]
+}
\ No newline at end of file
diff --git a/model-integration/src/test/models/lightgbm/train_lightgbm_classification.py b/model-integration/src/test/models/lightgbm/train_lightgbm_classification.py
new file mode 100755
index 00000000000..ac00437d192
--- /dev/null
+++ b/model-integration/src/test/models/lightgbm/train_lightgbm_classification.py
@@ -0,0 +1,54 @@
+#! /usr/bin/env python3
+# coding: utf-8
+
+import json
+import random
+
+import lightgbm as lgb
+import numpy as np
+import pandas as pd
+
+
+def category_value(arr):
+    values = { np.NaN: 0, "a":1, "b":2, "c":3, "d":4, "e":5, "i":1, "j":2, "k":3, "l":4, "m":5 }
+    return np.array([ 0.21 * values[i] for i in arr ])
+
+# Create training set
+num_examples = 100000
+missing_prob = 0.01
+features = pd.DataFrame({
+                "numerical_1":   np.random.random(num_examples),
+                "numerical_2":   np.random.random(num_examples),
+                "categorical_1": pd.Series(np.random.permutation(["a", "b", "c", "d", "e"] * int(num_examples/5)), dtype="category"),
+                "categorical_2": pd.Series(np.random.permutation(["i", "j", "k", "l", "m"] * int(num_examples/5)), dtype="category"),
+           })
+
+# randomly insert missing values
+for i in range(int(num_examples * len(features.columns) * missing_prob)):
+    features.loc[random.randint(0, num_examples-1), features.columns[random.randint(0, len(features.columns)-1)]] = None
+
+# create targets (with 0.0 as default for missing values)
+target = features["numerical_1"] + features["numerical_2"] + category_value(features["categorical_1"]) + category_value(features["categorical_2"])
+target = (target > 2.24) * 1.0
+lgb_train = lgb.Dataset(features, target)
+
+# Train model
+params = {
+    'objective': 'binary',
+    'metric': 'binary_logloss',
+    'num_leaves': 3,
+}
+model = lgb.train(params, lgb_train, num_boost_round=5)
+
+# Save model
+with open("classification.json", "w") as f:
+    json.dump(model.dump_model(), f, indent=2)
+
+# Predict (for comparison with Vespa evaluation)
+predict_features = pd.DataFrame({
+    "numerical_1":   pd.Series([  None, 0.1, None,  0.7]),
+    "numerical_2":   pd.Series([np.NaN, 0.2,  0.5,  0.8]),
+    "categorical_1": pd.Series([  None, "a",  "b", None], dtype="category"),
+    "categorical_2": pd.Series([  None, "i",  "j",  "m"], dtype="category"),
+    })
+print(model.predict(predict_features))
diff --git a/model-integration/src/test/models/lightgbm/train_lightgbm_regression.py b/model-integration/src/test/models/lightgbm/train_lightgbm_regression.py
new file mode 100755
index 00000000000..3e74e38da35
--- /dev/null
+++ b/model-integration/src/test/models/lightgbm/train_lightgbm_regression.py
@@ -0,0 +1,53 @@
+#! /usr/bin/env python3
+# coding: utf-8
+
+import json
+import random
+
+import lightgbm as lgb
+import numpy as np
+import pandas as pd
+
+
+def category_value(arr):
+    values = { np.NaN: 0, "a":1, "b":2, "c":3, "d":4, "e":5, "i":1, "j":2, "k":3, "l":4, "m":5 }
+    return np.array([ 0.21 * values[i] for i in arr ])
+
+# Create training set
+num_examples = 100000
+missing_prob = 0.01
+features = pd.DataFrame({
+                "numerical_1":   np.random.random(num_examples),
+                "numerical_2":   np.random.random(num_examples),
+                "categorical_1": pd.Series(np.random.permutation(["a", "b", "c", "d", "e"] * int(num_examples/5)), dtype="category"),
+                "categorical_2": pd.Series(np.random.permutation(["i", "j", "k", "l", "m"] * int(num_examples/5)), dtype="category"),
+           })
+
+# randomly insert missing values
+for i in range(int(num_examples * len(features.columns) * missing_prob)):
+    features.loc[random.randint(0, num_examples-1), features.columns[random.randint(0, len(features.columns)-1)]] = None
+
+# create targets (with 0.0 as default for missing values)
+target = features["numerical_1"] + features["numerical_2"] + category_value(features["categorical_1"]) + category_value(features["categorical_2"])
+lgb_train = lgb.Dataset(features, target)
+
+# Train model
+params = {
+    'objective': 'mse',
+    'metric': {'l2', 'l1'},
+    'num_leaves': 3,
+}
+model = lgb.train(params, lgb_train, num_boost_round=2)
+
+# Save model
+with open("regression.json", "w") as f:
+    json.dump(model.dump_model(), f, indent=2)
+
+# Predict (for comparison with Vespa evaluation)
+predict_features = pd.DataFrame({
+    "numerical_1":   pd.Series([  None, 0.1, None,  0.7]),
+    "numerical_2":   pd.Series([np.NaN, 0.2,  0.5,  0.8]),
+    "categorical_1": pd.Series([  None, "a",  "b", None], dtype="category"),
+    "categorical_2": pd.Series([  None, "i",  "j",  "m"], dtype="category"),
+    })
+print(model.predict(predict_features))
diff --git a/model-integration/src/test/models/onnx/simple/gather.onnx b/model-integration/src/test/models/onnx/simple/gather.onnx
new file mode 100644
index 00000000000..62451ad953d
--- /dev/null
+++ b/model-integration/src/test/models/onnx/simple/gather.onnx
diff --git a/model-integration/src/test/models/onnx/simple/gather.py b/model-integration/src/test/models/onnx/simple/gather.py
new file mode 100755
index 00000000000..63a2103fd86
--- /dev/null
+++ b/model-integration/src/test/models/onnx/simple/gather.py
@@ -0,0 +1,23 @@
+# Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+import onnx
+import numpy as np
+from onnx import helper, TensorProto
+
+data_type = helper.make_tensor_value_info('data', TensorProto.FLOAT, [3,2])
+indices_type = helper.make_tensor_value_info('indices', TensorProto.FLOAT, [2,2])
+output_type = helper.make_tensor_value_info('y', TensorProto.FLOAT, [2,2,2])
+
+node = onnx.helper.make_node(
+    'Gather',
+    inputs=['data', 'indices'],
+    outputs=['y'],
+    axis=0,
+)
+graph_def = onnx.helper.make_graph(
+    nodes = [node],
+    name = 'gather_test',
+    inputs = [data_type, indices_type],
+    outputs = [output_type]
+)
+model_def = helper.make_model(graph_def, producer_name='gather.py')
+onnx.save(model_def, 'gather.onnx')
diff --git a/node-admin/CMakeLists.txt b/node-admin/CMakeLists.txt
index 03bf09121c3..8dbb32c7de1 100644
--- a/node-admin/CMakeLists.txt
+++ b/node-admin/CMakeLists.txt
@@ -1,2 +1,6 @@
 # Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 install(DIRECTORY DESTINATION logs/vespa/node-admin)
+install(FILES target/node-admin-jar-with-dependencies.jar DESTINATION conf/node-admin-app/components)
+install_symlink(lib/jars/flags-jar-with-dependencies.jar conf/node-admin-app/components/flags-jar-with-dependencies.jar)
+install(FILES src/main/application/services.xml DESTINATION conf/node-admin-app)
+install(PROGRAMS src/main/sh/node-admin.sh DESTINATION libexec/vespa)
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/AddNode.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/AddNode.java
index c1642c71e64..d1b70450226 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/AddNode.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/AddNode.java
@@ -3,7 +3,6 @@ package com.yahoo.vespa.hosted.node.admin.configserver.noderepository;
 
 import com.yahoo.config.provision.NodeResources;
 import com.yahoo.config.provision.NodeType;
-import com.yahoo.config.provision.TenantName;
 import com.yahoo.config.provision.host.FlavorOverrides;
 
 import java.util.Objects;
@@ -20,30 +19,27 @@ public class AddNode {
     public final Optional<String> nodeFlavor;
     public final Optional<FlavorOverrides> flavorOverrides;
     public final Optional<NodeResources> nodeResources;
-    public final Optional<TenantName> reservedTo;
     public final NodeType nodeType;
     public final Set<String> ipAddresses;
     public final Set<String> additionalIpAddresses;
 
-    public static AddNode forHost(String hostname, String nodeFlavor, Optional<FlavorOverrides> flavorOverrides, Optional<TenantName> reservedTo, NodeType nodeType, Set<String> ipAddresses, Set<String> additionalIpAddresses) {
-        return new AddNode(hostname, Optional.empty(), Optional.of(nodeFlavor), flavorOverrides, Optional.empty(), reservedTo, nodeType, ipAddresses, additionalIpAddresses);
+    public static AddNode forHost(String hostname, String nodeFlavor, Optional<FlavorOverrides> flavorOverrides, NodeType nodeType, Set<String> ipAddresses, Set<String> additionalIpAddresses) {
+        return new AddNode(hostname, Optional.empty(), Optional.of(nodeFlavor), flavorOverrides, Optional.empty(), nodeType, ipAddresses, additionalIpAddresses);
     }
 
     public static AddNode forNode(String hostname, String parentHostname, NodeResources nodeResources, NodeType nodeType, Set<String> ipAddresses) {
-        return new AddNode(hostname, Optional.of(parentHostname), Optional.empty(), Optional.empty(), Optional.of(nodeResources), Optional.empty(), nodeType, ipAddresses, Set.of());
+        return new AddNode(hostname, Optional.of(parentHostname), Optional.empty(), Optional.empty(), Optional.of(nodeResources), nodeType, ipAddresses, Set.of());
     }
 
     private AddNode(String hostname, Optional<String> parentHostname,
                     Optional<String> nodeFlavor, Optional<FlavorOverrides> flavorOverrides,
                     Optional<NodeResources> nodeResources,
-                    Optional<TenantName> reservedTo,
                     NodeType nodeType, Set<String> ipAddresses, Set<String> additionalIpAddresses) {
         this.hostname = hostname;
         this.parentHostname = parentHostname;
         this.nodeFlavor = nodeFlavor;
         this.flavorOverrides = flavorOverrides;
         this.nodeResources = nodeResources;
-        this.reservedTo = reservedTo;
         this.nodeType = nodeType;
         this.ipAddresses = ipAddresses;
         this.additionalIpAddresses = additionalIpAddresses;
@@ -57,7 +53,6 @@ public class AddNode {
         return Objects.equals(hostname, addNode.hostname) &&
                 Objects.equals(parentHostname, addNode.parentHostname) &&
                 Objects.equals(nodeFlavor, addNode.nodeFlavor) &&
-                Objects.equals(reservedTo, addNode.reservedTo) &&
                 nodeType == addNode.nodeType &&
                 Objects.equals(ipAddresses, addNode.ipAddresses) &&
                 Objects.equals(additionalIpAddresses, addNode.additionalIpAddresses);
@@ -65,7 +60,7 @@ public class AddNode {
 
     @Override
     public int hashCode() {
-        return Objects.hash(hostname, parentHostname, nodeFlavor, reservedTo, nodeType, ipAddresses, additionalIpAddresses);
+        return Objects.hash(hostname, parentHostname, nodeFlavor, nodeType, ipAddresses, additionalIpAddresses);
     }
 
     @Override
@@ -74,7 +69,6 @@ public class AddNode {
                 "hostname='" + hostname + '\'' +
                 ", parentHostname=" + parentHostname +
                 ", nodeFlavor='" + nodeFlavor + '\'' +
-               (reservedTo.isPresent() ? ", reservedTo='" + reservedTo.get().value() + "'" : "") +
                 ", nodeType=" + nodeType +
                 ", ipAddresses=" + ipAddresses +
                 ", additionalIpAddresses=" + additionalIpAddresses +
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeAttributes.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeAttributes.java
index a6c876117b9..94431f8ac57 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeAttributes.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeAttributes.java
@@ -4,7 +4,6 @@ package com.yahoo.vespa.hosted.node.admin.configserver.noderepository;
 import com.fasterxml.jackson.databind.JsonNode;
 import com.yahoo.component.Version;
 import com.yahoo.config.provision.DockerImage;
-import com.yahoo.config.provision.TenantName;
 
 import java.time.Instant;
 import java.util.Map;
@@ -29,8 +28,6 @@ public class NodeAttributes {
     private Optional<Version> vespaVersion = Optional.empty();
     private Optional<Version> currentOsVersion = Optional.empty();
     private Optional<Instant> currentFirmwareCheck = Optional.empty();
-    private Optional<Boolean> wantToDeprovision = Optional.empty();
-    private Optional<TenantName> reservedTo = Optional.empty();
     /** The list of reports to patch. A null value is used to remove the report. */
     private Map<String, JsonNode> reports = new TreeMap<>();
 
@@ -70,12 +67,6 @@ public class NodeAttributes {
         return this;
     }
 
-
-    public NodeAttributes withWantToDeprovision(boolean wantToDeprovision) {
-        this.wantToDeprovision = Optional.of(wantToDeprovision);
-        return this;
-    }
-
     public NodeAttributes withReports(Map<String, JsonNode> nodeReports) {
         this.reports = new TreeMap<>(nodeReports);
         return this;
@@ -115,22 +106,14 @@ public class NodeAttributes {
         return currentFirmwareCheck;
     }
 
-    public Optional<Boolean> getWantToDeprovision() {
-        return wantToDeprovision;
-    }
-
     public Map<String, JsonNode> getReports() {
         return reports;
     }
 
-    public Optional<TenantName> getReservedTo() {
-        return reservedTo;
-    }
-
     @Override
     public int hashCode() {
         return Objects.hash(restartGeneration, rebootGeneration, dockerImage, vespaVersion, currentOsVersion,
-                            currentFirmwareCheck, wantToDeprovision, reports, reservedTo);
+                            currentFirmwareCheck, reports);
     }
 
     public boolean isEmpty() {
@@ -150,9 +133,7 @@ public class NodeAttributes {
                 && Objects.equals(vespaVersion, other.vespaVersion)
                 && Objects.equals(currentOsVersion, other.currentOsVersion)
                 && Objects.equals(currentFirmwareCheck, other.currentFirmwareCheck)
-                && Objects.equals(reports, other.reports)
-                && Objects.equals(reservedTo, other.reservedTo)
-                && Objects.equals(wantToDeprovision, other.wantToDeprovision);
+                && Objects.equals(reports, other.reports);
     }
 
     @Override
@@ -164,9 +145,7 @@ public class NodeAttributes {
                         vespaVersion.map(ver -> "vespaVersion=" + ver.toFullString()),
                         currentOsVersion.map(ver -> "currentOsVersion=" + ver.toFullString()),
                         currentFirmwareCheck.map(at -> "currentFirmwareCheck=" + at),
-                        Optional.ofNullable(reports.isEmpty() ? null : "reports=" + reports),
-                        Optional.ofNullable(reservedTo.isEmpty() ? null : "reservedTo=" + reservedTo),
-                        wantToDeprovision.map(depr -> "wantToDeprovision=" + depr))
+                        Optional.ofNullable(reports.isEmpty() ? null : "reports=" + reports))
                 .filter(Optional::isPresent)
                 .map(Optional::get)
                 .collect(Collectors.joining(", ", "{", "}"));
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeSpec.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeSpec.java
index 0e8caf728f1..85c35369a22 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeSpec.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeSpec.java
@@ -9,6 +9,7 @@ import com.yahoo.config.provision.NodeResources;
 import com.yahoo.config.provision.NodeType;
 
 import java.time.Instant;
+import java.util.EnumSet;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
@@ -48,7 +49,6 @@ public class NodeSpec {
     private final Optional<String> modelName;
 
     private final Optional<Boolean> allowedToBeDown;
-    private final Optional<Boolean> wantToDeprovision;
     private final Optional<ApplicationId> owner;
     private final Optional<NodeMembership> membership;
 
@@ -73,7 +73,6 @@ public class NodeSpec {
             Optional<Version> wantedOsVersion,
             Optional<Version> currentOsVersion,
             Optional<Boolean> allowedToBeDown,
-            Optional<Boolean> wantToDeprovision,
             Optional<ApplicationId> owner,
             Optional<NodeMembership> membership,
             Optional<Long> wantedRestartGeneration,
@@ -89,10 +88,12 @@ public class NodeSpec {
             NodeReports reports,
             Optional<String> parentHostname) {
         if (state == NodeState.active) {
-            Objects.requireNonNull(wantedVespaVersion, "Unknown vespa version for active node");
-            Objects.requireNonNull(wantedDockerImage, "Unknown docker image for active node");
-            Objects.requireNonNull(wantedRestartGeneration, "Unknown restartGeneration for active node");
-            Objects.requireNonNull(currentRestartGeneration, "Unknown currentRestartGeneration for active node");
+            requireOptional(owner, "owner");
+            requireOptional(membership, "membership");
+            requireOptional(wantedVespaVersion, "wantedVespaVersion");
+            requireOptional(wantedDockerImage, "wantedDockerImage");
+            requireOptional(wantedRestartGeneration, "restartGeneration");
+            requireOptional(currentRestartGeneration, "currentRestartGeneration");
         }
 
         this.hostname = Objects.requireNonNull(hostname);
@@ -108,7 +109,6 @@ public class NodeSpec {
         this.wantedOsVersion = Objects.requireNonNull(wantedOsVersion);
         this.currentOsVersion = Objects.requireNonNull(currentOsVersion);
         this.allowedToBeDown = Objects.requireNonNull(allowedToBeDown);
-        this.wantToDeprovision = Objects.requireNonNull(wantToDeprovision);
         this.owner = Objects.requireNonNull(owner);
         this.membership = Objects.requireNonNull(membership);
         this.wantedRestartGeneration = wantedRestartGeneration;
@@ -200,10 +200,6 @@ public class NodeSpec {
         return allowedToBeDown;
     }
 
-    public Optional<Boolean> wantToDeprovision() {
-        return wantToDeprovision;
-    }
-
     public Optional<ApplicationId> owner() {
         return owner;
     }
@@ -269,7 +265,6 @@ public class NodeSpec {
                 Objects.equals(wantedOsVersion, that.wantedOsVersion) &&
                 Objects.equals(currentOsVersion, that.currentOsVersion) &&
                 Objects.equals(allowedToBeDown, that.allowedToBeDown) &&
-                Objects.equals(wantToDeprovision, that.wantToDeprovision) &&
                 Objects.equals(owner, that.owner) &&
                 Objects.equals(membership, that.membership) &&
                 Objects.equals(wantedRestartGeneration, that.wantedRestartGeneration) &&
@@ -300,7 +295,6 @@ public class NodeSpec {
                 wantedOsVersion,
                 currentOsVersion,
                 allowedToBeDown,
-                wantToDeprovision,
                 owner,
                 membership,
                 wantedRestartGeneration,
@@ -331,7 +325,6 @@ public class NodeSpec {
                 + " wantedOsVersion=" + wantedOsVersion
                 + " currentOsVersion=" + currentOsVersion
                 + " allowedToBeDown=" + allowedToBeDown
-                + " wantToDeprovision=" + wantToDeprovision
                 + " owner=" + owner
                 + " membership=" + membership
                 + " wantedRestartGeneration=" + wantedRestartGeneration
@@ -361,7 +354,6 @@ public class NodeSpec {
         private Optional<Version> wantedOsVersion = Optional.empty();
         private Optional<Version> currentOsVersion = Optional.empty();
         private Optional<Boolean> allowedToBeDown = Optional.empty();
-        private Optional<Boolean> wantToDeprovision = Optional.empty();
         private Optional<ApplicationId> owner = Optional.empty();
         private Optional<NodeMembership> membership = Optional.empty();
         private Optional<Long> wantedRestartGeneration = Optional.empty();
@@ -398,7 +390,6 @@ public class NodeSpec {
             node.wantedOsVersion.ifPresent(this::wantedOsVersion);
             node.currentOsVersion.ifPresent(this::currentOsVersion);
             node.allowedToBeDown.ifPresent(this::allowedToBeDown);
-            node.wantToDeprovision.ifPresent(this::wantToDeprovision);
             node.owner.ifPresent(this::owner);
             node.membership.ifPresent(this::membership);
             node.wantedRestartGeneration.ifPresent(this::wantedRestartGeneration);
@@ -468,11 +459,6 @@ public class NodeSpec {
             return this;
         }
 
-        public Builder wantToDeprovision(boolean wantToDeprovision) {
-            this.wantToDeprovision = Optional.of(wantToDeprovision);
-            return this;
-        }
-
         public Builder owner(ApplicationId owner) {
             this.owner = Optional.of(owner);
             return this;
@@ -573,7 +559,6 @@ public class NodeSpec {
             attributes.getCurrentOsVersion().ifPresent(this::currentOsVersion);
             attributes.getRebootGeneration().ifPresent(this::currentRebootGeneration);
             attributes.getRestartGeneration().ifPresent(this::currentRestartGeneration);
-            attributes.getWantToDeprovision().ifPresent(this::wantToDeprovision);
             NodeReports.fromMap(attributes.getReports());
 
             return this;
@@ -623,10 +608,6 @@ public class NodeSpec {
             return allowedToBeDown;
         }
 
-        public Optional<Boolean> wantToDeprovision() {
-            return wantToDeprovision;
-        }
-
         public Optional<ApplicationId> owner() {
             return owner;
         }
@@ -673,7 +654,7 @@ public class NodeSpec {
 
         public NodeSpec build() {
             return new NodeSpec(hostname, wantedDockerImage, currentDockerImage, state, type, flavor, cpuCores,
-                    wantedVespaVersion, currentVespaVersion, wantedOsVersion, currentOsVersion, allowedToBeDown, wantToDeprovision,
+                    wantedVespaVersion, currentVespaVersion, wantedOsVersion, currentOsVersion, allowedToBeDown,
                     owner, membership,
                     wantedRestartGeneration, currentRestartGeneration,
                     wantedRebootGeneration, currentRebootGeneration,
@@ -682,5 +663,39 @@ public class NodeSpec {
                     reports, parentHostname);
         }
 
+
+        public static Builder testSpec(String hostname) {
+            return testSpec(hostname, NodeState.active);
+        }
+
+        /**
+         * Creates a NodeSpec.Builder that has the given hostname, in a given state, and some
+         * reasonable values for the remaining required NodeSpec fields.
+         */
+        public static Builder testSpec(String hostname, NodeState state) {
+            Builder builder = new Builder()
+                    .hostname(hostname)
+                    .state(state)
+                    .type(NodeType.tenant)
+                    .flavor("d-2-8-50")
+                    .resources(new NodeResources(2, 8, 50, 10));
+
+            // Set the required allocated fields
+            if (EnumSet.of(NodeState.active, NodeState.inactive, NodeState.reserved).contains(state)) {
+                builder .owner(ApplicationId.defaultId())
+                        .membership(new NodeMembership("container", "my-id", "group", 0, false))
+                        .wantedVespaVersion(Version.fromString("7.1.1"))
+                        .wantedDockerImage(DockerImage.fromString("docker.domain.tld/repo/image:7.1.1"))
+                        .currentRestartGeneration(0)
+                        .wantedRestartGeneration(0);
+            }
+
+            return builder;
+        }
+    }
+
+    private static void requireOptional(Optional<?> optional, String name) {
+        if (optional == null || optional.isEmpty())
+            throw new IllegalArgumentException(name + " must be set, was " + optional);
     }
 }
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeState.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeState.java
index e1765efef31..e13785fdab9 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeState.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeState.java
@@ -9,5 +9,5 @@ package com.yahoo.vespa.hosted.node.admin.configserver.noderepository;
  * @author freva
  */
 public enum NodeState {
-    provisioned, ready, reserved, active, inactive, dirty, failed, parked
+    provisioned, ready, reserved, active, inactive, dirty, failed, parked, deprovisioned
 }
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java
index afe092fa325..ea737773289 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java
@@ -8,7 +8,6 @@ import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.DockerImage;
 import com.yahoo.config.provision.NodeResources;
 import com.yahoo.config.provision.NodeType;
-import com.yahoo.config.provision.TenantName;
 import com.yahoo.config.provision.host.FlavorOverrides;
 import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerApi;
 import com.yahoo.vespa.hosted.node.admin.configserver.HttpException;
@@ -164,7 +163,6 @@ public class RealNodeRepository implements NodeRepository {
                 Optional.ofNullable(node.wantedOsVersion).map(Version::fromString),
                 Optional.ofNullable(node.currentOsVersion).map(Version::fromString),
                 Optional.ofNullable(node.allowedToBeDown),
-                Optional.ofNullable(node.wantToDeprovision),
                 Optional.ofNullable(node.owner).map(o -> ApplicationId.from(o.tenant, o.application, o.instance)),
                 membership,
                 Optional.ofNullable(node.restartGeneration),
@@ -244,7 +242,6 @@ public class RealNodeRepository implements NodeRepository {
             node.resources.diskSpeed = toString(resources.diskSpeed());
             node.resources.storageType = toString(resources.storageType());
         });
-        node.reservedTo = addNode.reservedTo.map(TenantName::value).orElse(null);
         node.type = addNode.nodeType.name();
         node.ipAddresses = addNode.ipAddresses;
         node.additionalIpAddresses = addNode.additionalIpAddresses;
@@ -259,8 +256,6 @@ public class RealNodeRepository implements NodeRepository {
         node.vespaVersion = nodeAttributes.getVespaVersion().map(Version::toFullString).orElse(null);
         node.currentOsVersion = nodeAttributes.getCurrentOsVersion().map(Version::toFullString).orElse(null);
         node.currentFirmwareCheck = nodeAttributes.getCurrentFirmwareCheck().map(Instant::toEpochMilli).orElse(null);
-        node.wantToDeprovision = nodeAttributes.getWantToDeprovision().orElse(null);
-        node.reservedTo = nodeAttributes.getReservedTo().map(TenantName::value).orElse(null);
 
         Map<String, JsonNode> reports = nodeAttributes.getReports();
         node.reports = reports == null || reports.isEmpty() ? null : new TreeMap<>(reports);
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java
index c790e73037e..f5bb46ef62a 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java
@@ -66,6 +66,9 @@ public class DockerOperationsImpl implements DockerOperations {
                 .withHostName(context.node().hostname())
                 .withResources(containerResources)
                 .withManagedBy(MANAGER_NAME)
+                // The inet6 option is needed to prefer AAAA records with gethostbyname(3), used by (at least) a yca package
+                // TODO: Try to remove this
+                .withDnsOption("inet6")
                 .withUlimit("nofile", 262_144, 262_144)
                 // The nproc aka RLIMIT_NPROC resource limit works as follows:
                 //  - A process has a (soft) nproc limit, either inherited by the parent or changed with setrlimit(2).
@@ -81,9 +84,12 @@ public class DockerOperationsImpl implements DockerOperations {
                 .withAddCapability("SYS_ADMIN")  // Needed for perf
                 .withAddCapability("SYS_NICE");  // Needed for set_mempolicy to work
 
-        if (context.node().membership().map(NodeMembership::clusterType).map("content"::equalsIgnoreCase).orElse(false)) {
-            command.withSecurityOpts("seccomp=unconfined");
-        }
+        // Proxy and controller require new privileges to bind port 443
+        if (context.nodeType() != NodeType.proxy && context.nodeType() != NodeType.controller)
+            command.withSecurityOpt("no-new-privileges");
+
+        if (context.node().membership().map(NodeMembership::clusterType).map("content"::equalsIgnoreCase).orElse(false))
+            command.withSecurityOpt("seccomp=unconfined");
 
         DockerNetworking networking = context.dockerNetworking();
         command.withNetworkMode(networking.getDockerNetworkMode());
@@ -289,7 +295,9 @@ public class DockerOperationsImpl implements DockerOperations {
         } else if (context.nodeType() == NodeType.tenant)
             paths.add(varLibSia);
 
-        paths.forEach(path -> command.withVolume(context.pathOnHostFromPathInNode(path), path));
+        paths.forEach(path -> command.withVolume(
+                context.pathOnHostFromPathInNode(path),
+                context.rewritePathInNodeForWantedDockerImage(path)));
 
 
         // Shared paths
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java
index 2969611e729..1684c0dd2a1 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java
@@ -96,10 +96,9 @@ public class StorageMaintainer {
     }
 
 
-    /** Deletes old log files for vespa, nginx, logstash, etc. */
+    /** Deletes old log files for vespa and nginx */
     public void removeOldFilesFromNode(NodeAgentContext context) {
         Path[] logPaths = {
-                context.pathInNodeUnderVespaHome("logs/daemontools_y"),
                 context.pathInNodeUnderVespaHome("logs/nginx"),
                 context.pathInNodeUnderVespaHome("logs/vespa")
         };
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java
index 9b0a35d4b96..6aa5a8d9fc1 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java
@@ -2,6 +2,11 @@
 package com.yahoo.vespa.hosted.node.admin.maintenance.coredump;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.vespa.hosted.dockerapi.metrics.Dimensions;
+import com.yahoo.vespa.hosted.dockerapi.metrics.Metrics;
+import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeMembership;
+import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec;
 import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext;
 import com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder;
 import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath;
@@ -51,6 +56,7 @@ public class CoredumpHandler {
     private final Path doneCoredumpsPath;
     private final String operatorGroupName;
     private final Supplier<String> coredumpIdSupplier;
+    private final Metrics metrics;
 
     /**
      * @param crashPathInContainer path inside the container where core dump are dumped
@@ -58,19 +64,20 @@ public class CoredumpHandler {
      * @param operatorGroupName name of the group that will be set as the owner of the processed coredump
      */
     public CoredumpHandler(Terminal terminal, CoreCollector coreCollector, CoredumpReporter coredumpReporter,
-                           Path crashPathInContainer, Path doneCoredumpsPath, String operatorGroupName) {
+                           Path crashPathInContainer, Path doneCoredumpsPath, String operatorGroupName, Metrics metrics) {
         this(terminal, coreCollector, coredumpReporter, crashPathInContainer, doneCoredumpsPath,
-                operatorGroupName, () -> UUID.randomUUID().toString());
+                operatorGroupName, metrics, () -> UUID.randomUUID().toString());
     }
 
     CoredumpHandler(Terminal terminal, CoreCollector coreCollector, CoredumpReporter coredumpReporter,
-                           Path crashPathInContainer, Path doneCoredumpsPath, String operatorGroupName, Supplier<String> coredumpIdSupplier) {
+                           Path crashPathInContainer, Path doneCoredumpsPath, String operatorGroupName, Metrics metrics, Supplier<String> coredumpIdSupplier) {
         this.terminal = terminal;
         this.coreCollector = coreCollector;
         this.coredumpReporter = coredumpReporter;
         this.crashPatchInContainer = crashPathInContainer;
         this.doneCoredumpsPath = doneCoredumpsPath;
         this.operatorGroupName = operatorGroupName;
+        this.metrics = metrics;
         this.coredumpIdSupplier = coredumpIdSupplier;
     }
 
@@ -85,6 +92,8 @@ public class CoredumpHandler {
                 .maxDepth(1)
                 .deleteRecursively(context);
 
+        updateMetrics(context, containerCrashPathOnHost);
+
         // Check if we have already started to process a core dump or we can enqueue a new core one
         getCoredumpToProcess(containerCrashPathOnHost, containerProcessingPathOnHost)
                 .ifPresent(path -> processAndReportSingleCoredump(context, path, nodeAttributesSupplier));
@@ -180,8 +189,9 @@ public class CoredumpHandler {
         new UnixPath(compressedCoreFile).setGroup(operatorGroupName).setPermissions("rw-r-----");
         Files.delete(coreFile);
 
-        Path newCoredumpDirectory = doneCoredumpsPath.resolve(coredumpDirectory.getFileName());
-        Files.move(coredumpDirectory, newCoredumpDirectory);
+        Path newCoredumpDirectory = doneCoredumpsPath.resolve(context.containerName().asString());
+        uncheck(() -> Files.createDirectories(newCoredumpDirectory));
+        Files.move(coredumpDirectory, newCoredumpDirectory.resolve(coredumpDirectory.getFileName()));
     }
 
     Path findCoredumpFileInProcessingDirectory(Path coredumpProccessingDirectory) {
@@ -194,4 +204,57 @@ public class CoredumpHandler {
                 .orElseThrow(() -> new IllegalStateException(
                         "No coredump file found in processing directory " + coredumpProccessingDirectory));
     }
+
+    void updateMetrics(NodeAgentContext context, Path containerCrashPathOnHost) {
+        Dimensions dimensions = generateDimensions(context);
+
+        // Unprocessed coredumps
+        int numberOfUnprocessedCoredumps = FileFinder.files(containerCrashPathOnHost)
+                .match(nameStartsWith(".").negate())
+                .match(nameMatches(HS_ERR_PATTERN).negate())
+                .maxDepth(1)
+                .list().size();
+
+        metrics.declareGauge(Metrics.APPLICATION_NODE, "coredumps.enqueued", dimensions, Metrics.DimensionType.PRETAGGED).sample(numberOfUnprocessedCoredumps);
+
+        // Processed coredumps
+        Path processedCoredumpsPath = doneCoredumpsPath.resolve(context.containerName().asString());
+        int numberOfProcessedCoredumps = FileFinder.directories(processedCoredumpsPath)
+                .maxDepth(1)
+                .list().size();
+
+        metrics.declareGauge(Metrics.APPLICATION_NODE, "coredumps.processed", dimensions, Metrics.DimensionType.PRETAGGED).sample(numberOfProcessedCoredumps);
+    }
+
+    private Dimensions generateDimensions(NodeAgentContext context) {
+        NodeSpec node = context.node();
+        Dimensions.Builder dimensionsBuilder = new Dimensions.Builder()
+                .add("host", node.hostname())
+                .add("flavor", node.flavor())
+                .add("state", node.state().toString())
+                .add("zone", context.zone().getId().value());
+
+        node.owner().ifPresent(owner ->
+            dimensionsBuilder
+                    .add("tenantName", owner.tenant().value())
+                    .add("applicationName", owner.application().value())
+                    .add("instanceName", owner.instance().value())
+                    .add("app", String.join(".", owner.application().value(), owner.instance().value()))
+                    .add("applicationId", owner.toFullString())
+        );
+
+        node.membership().ifPresent(membership ->
+            dimensionsBuilder
+                    .add("clustertype", membership.clusterType())
+                    .add("clusterid", membership.clusterId())
+        );
+
+        node.parentHostname().ifPresent(parent -> dimensionsBuilder.add("parentHostname", parent));
+        node.allowedToBeDown().ifPresent(allowed ->
+                dimensionsBuilder.add("orchestratorState", allowed ? "ALLOWED_TO_BE_DOWN" : "NO_REMARKS"));
+        node.currentVespaVersion().ifPresent(vespaVersion -> dimensionsBuilder.add("vespaVersion", vespaVersion.toFullString()));
+
+        return dimensionsBuilder.build();
+    }
+
 }
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdater.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdater.java
index cb209d710c8..e6d6f9463d3 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdater.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdater.java
@@ -114,6 +114,10 @@ public class NodeAdminStateUpdater {
             // To avoid node agents stalling for too long, we'll force unfrozen ticks now.
             adjustNodeAgentsToRunFromNodeRepository();
             nodeAdmin.setFrozen(false);
+
+            NodeState currentNodeState = nodeRepository.getNode(hostHostname).state();
+            if (currentNodeState == NodeState.active) orchestrator.resume(hostHostname);
+
             throw new ConvergenceException("Timed out trying to freeze all nodes: will force an unfrozen tick");
         }
 
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContext.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContext.java
index 821765fea20..2820fb2fa70 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContext.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContext.java
@@ -12,6 +12,7 @@ import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec;
 import com.yahoo.vespa.hosted.node.admin.docker.DockerNetworking;
 
 import java.nio.file.Path;
+import java.nio.file.Paths;
 
 public interface NodeAgentContext extends TaskContext {
 
@@ -82,4 +83,19 @@ public interface NodeAgentContext extends TaskContext {
 
     /** @see #pathInNodeUnderVespaHome(Path) */
     Path pathInNodeUnderVespaHome(String relativePath);
+
+    /**
+     * Rewrite the given path in node to a path required by the image.
+     * WARNING: This method should only be used when starting the docker container, e.g. writing container data or
+     * configuring mounts.
+     * TODO: Remove when everyone has migrated of vespa/ci image
+     */
+    default Path rewritePathInNodeForWantedDockerImage(Path path) {
+        if (!node().wantedDockerImage().get().repository().endsWith("/vespa/ci")) return path;
+
+        Path originalVespaHome = pathInNodeUnderVespaHome("");
+        if (!path.startsWith(originalVespaHome)) return path;
+
+        return Paths.get("/home/y").resolve(originalVespaHome.relativize(path));
+    }
 }
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImpl.java
index 02e01264a57..4585d72d6de 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImpl.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImpl.java
@@ -2,7 +2,6 @@
 package com.yahoo.vespa.hosted.node.admin.nodeagent;
 
 import com.yahoo.config.provision.CloudName;
-import com.yahoo.config.provision.NodeType;
 import com.yahoo.config.provision.SystemName;
 import com.yahoo.config.provision.zone.ZoneApi;
 import com.yahoo.config.provision.zone.ZoneId;
@@ -11,7 +10,6 @@ import com.yahoo.vespa.athenz.api.AthenzService;
 import com.yahoo.vespa.hosted.dockerapi.ContainerName;
 import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.Acl;
 import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec;
-import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeState;
 import com.yahoo.vespa.hosted.node.admin.docker.DockerNetworking;
 
 import java.nio.file.FileSystem;
@@ -187,7 +185,7 @@ public class NodeAgentContextImpl implements NodeAgentContext {
 
     /** For testing only! */
     public static class Builder {
-        private NodeSpec.Builder nodeSpecBuilder = new NodeSpec.Builder();
+        private NodeSpec.Builder nodeSpecBuilder;
         private Acl acl;
         private AthenzIdentity identity;
         private DockerNetworking dockerNetworking;
@@ -209,11 +207,7 @@ public class NodeAgentContextImpl implements NodeAgentContext {
          * if you want to control the entire NodeSpec.
          */
         public Builder(String hostname) {
-            this.nodeSpecBuilder
-                    .hostname(hostname)
-                    .state(NodeState.active)
-                    .type(NodeType.tenant)
-                    .flavor("d-2-8-50");
+            this.nodeSpecBuilder = NodeSpec.Builder.testSpec(hostname);
         }
 
         public Builder nodeSpecBuilder(Function<NodeSpec.Builder, NodeSpec.Builder> nodeSpecBuilderModifier) {
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java
index 68178418e62..9c8b0ec2b86 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java
@@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.node.admin.nodeagent;
 
 import com.yahoo.config.provision.DockerImage;
 import com.yahoo.config.provision.Environment;
+import com.yahoo.config.provision.NodeType;
 import com.yahoo.config.provision.zone.ZoneApi;
 import com.yahoo.log.LogLevel;
 import com.yahoo.vespa.flags.DoubleFlag;
@@ -26,6 +27,9 @@ import com.yahoo.vespa.hosted.node.admin.maintenance.identity.CredentialsMaintai
 import com.yahoo.vespa.hosted.node.admin.nodeadmin.ConvergenceException;
 
 import java.nio.file.Path;
+import java.time.Clock;
+import java.time.Duration;
+import java.time.Instant;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Objects;
@@ -47,11 +51,11 @@ public class NodeAgentImpl implements NodeAgent {
     // This is used as a definition of 1 GB when comparing flavor specs in node-repo
     private static final long BYTES_IN_GB = 1_000_000_000L;
 
-    private static final Logger logger = Logger.getLogger(NodeAgentImpl.class.getName());
+    // Container is started with uncapped CPU and is kept that way until the first successful health check + this duration
+    // Subtract 1 second to avoid warmup coming in lockstep with tick time and always end up using an extra tick when there are just a few ms left
+    private static final Duration DEFAULT_WARM_UP_DURATION = Duration.ofSeconds(90).minus(Duration.ofSeconds(1));
 
-    private final AtomicBoolean terminated = new AtomicBoolean(false);
-    private boolean hasResumedNode = false;
-    private boolean hasStartedServices = true;
+    private static final Logger logger = Logger.getLogger(NodeAgentImpl.class.getName());
 
     private final NodeAgentContextSupplier contextSupplier;
     private final NodeRepository nodeRepository;
@@ -61,12 +65,20 @@ public class NodeAgentImpl implements NodeAgent {
     private final Optional<CredentialsMaintainer> credentialsMaintainer;
     private final Optional<AclMaintainer> aclMaintainer;
     private final Optional<HealthChecker> healthChecker;
+    private final Clock clock;
+    private final Duration warmUpDuration;
     private final DoubleFlag containerCpuCap;
 
     private Thread loopThread;
     private ContainerState containerState = UNKNOWN;
     private NodeSpec lastNode;
 
+    private final AtomicBoolean terminated = new AtomicBoolean(false);
+    private boolean hasResumedNode = false;
+    private boolean hasStartedServices = true;
+    private Optional<Instant> firstSuccessfulHealthCheckInstant = Optional.empty();
+    private boolean suspendedInOrchestrator = false;
+
     private int numberOfUnhandledException = 0;
     private long currentRebootGeneration = 0;
     private Optional<Long> currentRestartGeneration = Optional.empty();
@@ -87,16 +99,18 @@ public class NodeAgentImpl implements NodeAgent {
 
 
     // Created in NodeAdminImpl
-    public NodeAgentImpl(
-            final NodeAgentContextSupplier contextSupplier,
-            final NodeRepository nodeRepository,
-            final Orchestrator orchestrator,
-            final DockerOperations dockerOperations,
-            final StorageMaintainer storageMaintainer,
-            final FlagSource flagSource,
-            final Optional<CredentialsMaintainer> credentialsMaintainer,
-            final Optional<AclMaintainer> aclMaintainer,
-            final Optional<HealthChecker> healthChecker) {
+    public NodeAgentImpl(NodeAgentContextSupplier contextSupplier, NodeRepository nodeRepository,
+                         Orchestrator orchestrator, DockerOperations dockerOperations, StorageMaintainer storageMaintainer,
+                         FlagSource flagSource, Optional<CredentialsMaintainer> credentialsMaintainer,
+                         Optional<AclMaintainer> aclMaintainer, Optional<HealthChecker> healthChecker, Clock clock) {
+        this(contextSupplier, nodeRepository, orchestrator, dockerOperations, storageMaintainer, flagSource, credentialsMaintainer,
+             aclMaintainer, healthChecker, clock, DEFAULT_WARM_UP_DURATION);
+    }
+
+    public NodeAgentImpl(NodeAgentContextSupplier contextSupplier, NodeRepository nodeRepository,
+                        Orchestrator orchestrator, DockerOperations dockerOperations, StorageMaintainer storageMaintainer,
+                        FlagSource flagSource, Optional<CredentialsMaintainer> credentialsMaintainer,
+                        Optional<AclMaintainer> aclMaintainer, Optional<HealthChecker> healthChecker, Clock clock, Duration warmUpDuration) {
         this.contextSupplier = contextSupplier;
         this.nodeRepository = nodeRepository;
         this.orchestrator = orchestrator;
@@ -105,6 +119,8 @@ public class NodeAgentImpl implements NodeAgent {
         this.credentialsMaintainer = credentialsMaintainer;
         this.aclMaintainer = aclMaintainer;
         this.healthChecker = healthChecker;
+        this.clock = clock;
+        this.warmUpDuration = warmUpDuration;
         this.containerCpuCap = Flags.CONTAINER_CPU_CAP.bindTo(flagSource);
     }
 
@@ -194,9 +210,11 @@ public class NodeAgentImpl implements NodeAgent {
         }
     }
 
-    private void startContainer(NodeAgentContext context) {
+    private Container startContainer(NodeAgentContext context) {
         ContainerData containerData = createContainerData(context);
-        dockerOperations.createContainer(context, containerData, getContainerResources(context));
+        ContainerResources wantedResources = context.nodeType() != NodeType.tenant || warmUpDuration(context.zone()).isNegative() ?
+                getContainerResources(context) : getContainerResources(context).withUnlimitedCpus();
+        dockerOperations.createContainer(context, containerData, wantedResources);
         dockerOperations.startContainer(context);
 
         currentRebootGeneration = context.node().wantedRebootGeneration();
@@ -204,6 +222,8 @@ public class NodeAgentImpl implements NodeAgent {
         hasStartedServices = true; // Automatically started with the container
         hasResumedNode = false;
         context.log(logger, "Container successfully started, new containerState is " + containerState);
+        return dockerOperations.getContainer(context).orElseThrow(() ->
+                new ConvergenceException("Did not find container that was just started"));
     }
 
     private Optional<Container> removeContainerIfNeededUpdateContainerState(
@@ -250,6 +270,7 @@ public class NodeAgentImpl implements NodeAgent {
         if (containerState == ABSENT) return;
         try {
             hasStartedServices = hasResumedNode = false;
+            firstSuccessfulHealthCheckInstant = Optional.empty();
             dockerOperations.stopServices(context);
         } catch (ContainerNotFoundException e) {
             containerState = ABSENT;
@@ -333,9 +354,15 @@ public class NodeAgentImpl implements NodeAgent {
     }
 
 
-    private void updateContainerIfNeeded(NodeAgentContext context, Container existingContainer) {
+    private Container updateContainerIfNeeded(NodeAgentContext context, Container existingContainer) {
         ContainerResources wantedContainerResources = getContainerResources(context);
-        if (wantedContainerResources.equalsCpu(existingContainer.resources)) return;
+
+        if (healthChecker.isPresent() && firstSuccessfulHealthCheckInstant
+                .map(clock.instant().minus(warmUpDuration(context.zone()))::isBefore)
+                .orElse(true))
+            return existingContainer;
+
+        if (wantedContainerResources.equalsCpu(existingContainer.resources)) return existingContainer;
         context.log(logger, "Container should be running with different CPU allocation, wanted: %s, current: %s",
                 wantedContainerResources.toStringCpu(), existingContainer.resources.toStringCpu());
 
@@ -343,6 +370,8 @@ public class NodeAgentImpl implements NodeAgent {
 
         // Only update CPU resources
         dockerOperations.updateContainer(context, wantedContainerResources.withMemoryBytes(existingContainer.resources.memoryBytes()));
+        return dockerOperations.getContainer(context).orElseThrow(() ->
+                new ConvergenceException("Did not find container that was just updated"));
     }
 
     private ContainerResources getContainerResources(NodeAgentContext context) {
@@ -370,6 +399,7 @@ public class NodeAgentImpl implements NodeAgent {
     public void converge(NodeAgentContext context) {
         try {
             doConverge(context);
+            context.log(logger, LogLevel.INFO, "Converged");
         } catch (ConvergenceException e) {
             context.log(logger, e.getMessage());
         } catch (ContainerNotFoundException e) {
@@ -384,7 +414,7 @@ public class NodeAgentImpl implements NodeAgent {
         }
     }
 
-    // Public for testing
+    // Non-private for testing
     void doConverge(NodeAgentContext context) {
         NodeSpec node = context.node();
         Optional<Container> container = getContainer(context);
@@ -430,16 +460,25 @@ public class NodeAgentImpl implements NodeAgent {
                 credentialsMaintainer.ifPresent(maintainer -> maintainer.converge(context));
                 if (container.isEmpty()) {
                     containerState = STARTING;
-                    startContainer(context);
+                    container = Optional.of(startContainer(context));
                     containerState = UNKNOWN;
                 } else {
-                    updateContainerIfNeeded(context, container.get());
+                    container = Optional.of(updateContainerIfNeeded(context, container.get()));
                 }
 
                 aclMaintainer.ifPresent(maintainer -> maintainer.converge(context));
                 startServicesIfNeeded(context);
                 resumeNodeIfNeeded(context);
-                healthChecker.ifPresent(checker -> checker.verifyHealth(context));
+                if (healthChecker.isPresent()) {
+                    healthChecker.get().verifyHealth(context);
+                    if (firstSuccessfulHealthCheckInstant.isEmpty())
+                        firstSuccessfulHealthCheckInstant = Optional.of(clock.instant());
+
+                    Duration timeLeft = Duration.between(clock.instant(), firstSuccessfulHealthCheckInstant.get().plus(warmUpDuration(context.zone())));
+                    if (!container.get().resources.equalsCpu(getContainerResources(context)))
+                        throw new ConvergenceException("Refusing to resume until warm up period ends (" +
+                                (timeLeft.isNegative() ? "next tick" : "in " + timeLeft) + ")");
+                }
 
                 // Because it's more important to stop a bad release from rolling out in prod,
                 // we put the resume call last. So if we fail after updating the node repo attributes
@@ -452,8 +491,11 @@ public class NodeAgentImpl implements NodeAgent {
                 //  - Slobrok and internal orchestrator state is used to determine whether
                 //    to allow upgrade (suspend).
                 updateNodeRepoWithCurrentAttributes(context);
-                context.log(logger, "Call resume against Orchestrator");
-                orchestrator.resume(context.hostname().value());
+                if (suspendedInOrchestrator || node.allowedToBeDown().orElse(false)) {
+                    context.log(logger, "Call resume against Orchestrator");
+                    orchestrator.resume(context.hostname().value());
+                    suspendedInOrchestrator = false;
+                }
                 break;
             case provisioned:
                 nodeRepository.setNodeState(context.hostname().value(), NodeState.dirty);
@@ -526,6 +568,7 @@ public class NodeAgentImpl implements NodeAgent {
         context.log(logger, "Ask Orchestrator for permission to suspend node");
         try {
             orchestrator.suspend(context.hostname().value());
+            suspendedInOrchestrator = true;
         } catch (OrchestratorException e) {
             // Ensure the ACLs are up to date: The reason we're unable to suspend may be because some other
             // node is unable to resume because the ACL rules of SOME Docker container is wrong...
@@ -562,4 +605,10 @@ public class NodeAgentImpl implements NodeAgent {
     protected Optional<CredentialsMaintainer> credentialsMaintainer() {
         return credentialsMaintainer;
     }
+
+    private Duration warmUpDuration(ZoneApi zone) {
+        return zone.getSystemName().isCd() || zone.getEnvironment().isTest()
+                ? Duration.ofSeconds(-1)
+                : warmUpDuration;
+    }
 }
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/DefaultEnvWriter.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/DefaultEnvWriter.java
index 83ac3eeeaf4..22cabdc8ed9 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/DefaultEnvWriter.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/DefaultEnvWriter.java
@@ -1,6 +1,8 @@
 // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.node.admin.task.util;
 
+import com.yahoo.vespa.hosted.node.admin.component.TaskContext;
+
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
@@ -10,6 +12,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.TreeSet;
+import java.util.logging.Logger;
 
 import static com.yahoo.vespa.hosted.node.admin.task.util.file.IOExceptionUtil.ifExists;
 import static com.yahoo.yolean.Exceptions.uncheck;
@@ -23,6 +26,8 @@ import static java.util.stream.Collectors.joining;
  */
 public class DefaultEnvWriter {
 
+    private static final Logger logger = Logger.getLogger(DefaultEnvWriter.class.getName());
+
     private final Map<String, Operation> operations = new LinkedHashMap<>();
 
     public DefaultEnvWriter addOverride(String name, String value) {
@@ -50,12 +55,13 @@ public class DefaultEnvWriter {
      *
      * @return true if the file was modified
      */
-    public boolean updateFile(Path defaultEnvFile) {
+    public boolean updateFile(TaskContext context, Path defaultEnvFile) {
         List<String> currentDefaultEnvLines = ifExists(() -> Files.readAllLines(defaultEnvFile)).orElse(List.of());
         List<String> newDefaultEnvLines = generateContent(currentDefaultEnvLines);
         if (currentDefaultEnvLines.equals(newDefaultEnvLines)) {
             return false;
         } else {
+            context.log(logger, "Updating " + defaultEnvFile.toString());
             Path tempFile = Paths.get(defaultEnvFile.toString() + ".tmp");
             uncheck(() -> Files.write(tempFile, newDefaultEnvLines));
             uncheck(() -> Files.move(tempFile, defaultEnvFile, ATOMIC_MOVE));
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPath.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPath.java
index 268e0a5ccfd..07e73eb0ee7 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPath.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPath.java
@@ -11,6 +11,7 @@ import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.nio.file.StandardCopyOption;
 import java.nio.file.attribute.FileAttribute;
+import java.nio.file.attribute.FileTime;
 import java.nio.file.attribute.GroupPrincipal;
 import java.nio.file.attribute.PosixFileAttributeView;
 import java.nio.file.attribute.PosixFileAttributes;
@@ -37,22 +38,33 @@ import static com.yahoo.yolean.Exceptions.uncheck;
 public class UnixPath {
     private final Path path;
 
-    public UnixPath(Path path) {
-        this.path = path;
-    }
+    public UnixPath(Path path) { this.path = path; }
+    public UnixPath(String path) { this(Paths.get(path)); }
 
-    public UnixPath(String path) {
-        this(Paths.get(path));
-    }
+    public Path toPath() { return path; }
+    public UnixPath resolve(String relativeOrAbsolutePath) { return new UnixPath(path.resolve(relativeOrAbsolutePath)); }
+
+    public UnixPath getParent() {
+        Path parentPath = path.getParent();
+        if (parentPath == null) {
+            throw new IllegalStateException("Path has no parent directory: '" + path + "'");
+        }
 
-    public Path toPath() {
-        return path;
+        return new UnixPath(parentPath);
     }
 
-    public boolean exists() {
-        return Files.exists(path);
+    public String getFilename() {
+        Path filename = path.getFileName();
+        if (filename == null) {
+            // E.g. "/".
+            throw new IllegalStateException("Path has no filename: '" + path.toString() + "'");
+        }
+
+        return filename.toString();
     }
 
+    public boolean exists() { return Files.exists(path); }
+
     public String readUtf8File() {
         return new String(readBytes(), StandardCharsets.UTF_8);
     }
@@ -91,6 +103,18 @@ public class UnixPath {
         return this;
     }
 
+    public UnixPath atomicWriteUt8(String content) {
+        return atomicWriteBytes(content.getBytes(StandardCharsets.UTF_8));
+    }
+
+    /** Write a file to the same dir as this, and then atomically move it to this' path. */
+    public UnixPath atomicWriteBytes(byte[] content) {
+        UnixPath temporaryPath = getParent().resolve(getFilename() + ".10Ia2f4N5");
+        temporaryPath.writeBytes(content);
+        temporaryPath.atomicMove(path);
+        return this;
+    }
+
     public String getPermissions() {
         return getAttributes().permissions();
     }
@@ -135,6 +159,11 @@ public class UnixPath {
         return getAttributes().lastModifiedTime();
     }
 
+    public UnixPath updateLastModifiedTime() {
+        uncheck(() -> Files.setLastModifiedTime(path, FileTime.from(Instant.now())));
+        return this;
+    }
+
     public FileAttributes getAttributes() {
         PosixFileAttributes attributes = uncheck(() ->
                 Files.getFileAttributeView(path, PosixFileAttributeView.class).readAttributes());
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/systemd/SystemCtlTester.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/systemd/SystemCtlTester.java
index c7264c2fe4d..82906253f24 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/systemd/SystemCtlTester.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/systemd/SystemCtlTester.java
@@ -3,8 +3,7 @@ package com.yahoo.vespa.hosted.node.admin.task.util.systemd;
 
 import com.yahoo.vespa.hosted.node.admin.task.util.process.TestTerminal;
 
-import java.util.HashSet;
-import java.util.Set;
+import java.util.function.Consumer;
 
 /**
  * A {@link SystemCtl} tester that simplifies testing interaction with systemd units.
@@ -13,8 +12,6 @@ import java.util.Set;
  */
 public class SystemCtlTester extends SystemCtl {
 
-    private final Set<String> runningUnits = new HashSet<>();
-
     private final TestTerminal terminal;
 
     public SystemCtlTester(TestTerminal terminal) {
@@ -22,93 +19,64 @@ public class SystemCtlTester extends SystemCtl {
         this.terminal = terminal;
     }
 
-    /** Create expectation for given unit */
-    public Expectation expect(String unit) {
-        return new Expectation(unit, this);
+    public Expectation expectServiceExists(String unit) {
+        return new Expectation(wantedReturn ->
+                expectCommand("systemctl list-unit-files " + unit + ".service 2>&1", 0, (wantedReturn ? 1 : 0) + " unit files listed."));
     }
 
-    private void startUnit(String unit) {
-        runningUnits.add(unit);
+    public Expectation expectIsActive(String unit) {
+        return new Expectation(wantedReturn -> {
+            expectCommand("systemctl --quiet is-active " + unit + ".service 2>&1", wantedReturn ? 0 : 1, "");
+        });
     }
 
-    private void expectCommand(String command, int exitCode, String output) {
-        terminal.expectCommand((useSudo() ? "sudo " : "") + command, exitCode, output);
-    }
+    public Expectation expectEnable(String unit) { return forChangeEnabledState(unit, true); }
+    public Expectation expectDisable(String unit) { return forChangeEnabledState(unit, false); }
+    public Expectation expectStart(String unit) { return forChangeRunningState(unit, true); }
+    public Expectation expectStop(String unit) { return forChangeRunningState(unit, false); }
 
-    public static class Expectation {
-
-        private final String unit;
-        private final SystemCtlTester systemCtl;
-
-        public Expectation(String unit, SystemCtlTester systemCtl) {
-            this.unit = unit;
-            this.systemCtl = systemCtl;
-        }
-
-        /** Create expectation for given unit */
-        public Expectation expect(String name) {
-            return systemCtl.expect(name);
-        }
-
-        /** Expect that this will be started */
-        public Expectation toStart() {
-            return toStart(true);
-        }
-
-        /** Expect that this is already started */
-        public Expectation isStarted() {
-            return toStart(false);
-        }
+    public SystemCtlTester expectRestart(String unit) {
+        expectCommand("systemctl restart " + unit + " 2>&1", 0, "");
+        return this;
+    }
 
-        /** Expect that given unit will be restarted */
-        public Expectation toRestart() {
-            systemCtl.expectCommand("systemctl restart " + unit + " 2>&1", 0, "");
-            systemCtl.startUnit(unit);
-            return this;
-        }
+    public SystemCtlTester expectDaemonReload() {
+        expectCommand("systemctl daemon-reload 2>&1", 0, "");
+        return this;
+    }
 
-        /** Expect that this will be stopped */
-        public Expectation toStop() {
-            systemCtl.expectCommand("systemctl stop " + unit + " 2>&1", 0, "");
-            systemCtl.runningUnits.remove(unit);
-            return this;
-        }
 
-        /** Expect query for state of this */
-        public Expectation toQueryState() {
-            systemCtl.expectCommand("systemctl --quiet is-active " + unit + ".service 2>&1",
-                                    systemCtl.runningUnits.contains(unit) ? 0 : 1, "");
-            return this;
-        }
+    private void expectCommand(String command, int exitCode, String output) {
+        terminal.expectCommand((useSudo() ? "sudo " : "") + command, exitCode, output);
+    }
 
-        /** Expect that this will be enabled */
-        public Expectation toEnable() {
-            return toEnable(true);
-        }
+    private Expectation forChangeEnabledState(String unit, boolean enable) {
+        return new Expectation(wantedReturn -> {
+            expectCommand("systemctl --quiet is-enabled " + unit + " 2>&1", enable != wantedReturn ? 0 : 1, "");
+            if (wantedReturn)
+                expectCommand("systemctl " + (enable ? "enable" : "disable") + " " + unit + " 2>&1", 0, "");
+        });
+    }
 
-        /** Expect that given unit is already enabled */
-        public Expectation isEnabled() {
-            return toEnable(false);
-        }
+    private Expectation forChangeRunningState(String unit, boolean start) {
+        return new Expectation(wantedReturn -> {
+            expectCommand("systemctl show " + unit + " 2>&1", 0, "ActiveState=" + (start != wantedReturn ? "active" : "inactive"));
+            if (wantedReturn)
+                expectCommand("systemctl " + (start ? "start" : "stop") + " " + unit + " 2>&1", 0, "");
+        });
+    }
 
-        private Expectation toStart(boolean start) {
-            systemCtl.expectCommand("systemctl show " + unit + " 2>&1", 0,
-                                    "ActiveState=" + (start ? "inactive" : "active"));
-            if (start) {
-                systemCtl.expectCommand("systemctl start " + unit + " 2>&1", 0, "");
-                systemCtl.startUnit(unit);
-            }
-            return this;
+    public class Expectation {
+        private final Consumer<Boolean> converger;
+        public Expectation(Consumer<Boolean> converger) {
+            this.converger = converger;
         }
 
-        private Expectation toEnable(boolean enable) {
-            systemCtl.expectCommand("systemctl --quiet is-enabled " + unit + " 2>&1", enable ? 1 : 0, "");
-            if (enable) {
-                systemCtl.expectCommand("systemctl enable " + unit + " 2>&1", 0, "");
-            }
-            return this;
+        /** Mock the return value of the converge(TaskContext) method for this operation (true iff system was modified) */
+        public SystemCtlTester andReturn(boolean value) {
+            converger.accept(value);
+            return SystemCtlTester.this;
         }
-
     }
 
 }
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/Yum.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/Yum.java
index 85a7c065a86..27054f6ed01 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/Yum.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/Yum.java
@@ -2,33 +2,28 @@
 package com.yahoo.vespa.hosted.node.admin.task.util.yum;
 
 import com.yahoo.vespa.hosted.node.admin.component.TaskContext;
-import com.yahoo.vespa.hosted.node.admin.task.util.process.CommandLine;
 import com.yahoo.vespa.hosted.node.admin.task.util.process.CommandResult;
 import com.yahoo.vespa.hosted.node.admin.task.util.process.Terminal;
 
-import java.util.ArrayList;
 import java.util.List;
 import java.util.Optional;
 import java.util.function.Function;
-import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 import java.util.stream.Stream;
 
+import static com.yahoo.vespa.hosted.node.admin.task.util.yum.YumCommand.GenericYumCommand;
+import static com.yahoo.vespa.hosted.node.admin.task.util.yum.YumCommand.InstallFixedYumCommand;
+
 /**
  * @author hakonhall
  */
 public class Yum {
     // Note: "(?dm)" makes newline be \n (only), and enables multiline mode where ^$ match lines with find()
-    private static final Pattern CHECKING_FOR_UPDATE_PATTERN =
-            Pattern.compile("(?dm)^Package matching [^ ]+ already installed\\. Checking for update\\.$");
-    private static final Pattern NOTHING_TO_DO_PATTERN = Pattern.compile("(?dm)^Nothing to do$");
-    private static final Pattern INSTALL_NOOP_PATTERN = NOTHING_TO_DO_PATTERN;
+    private static final Pattern INSTALL_NOOP_PATTERN = Pattern.compile("(?dm)^Nothing to do$");
     private static final Pattern UPGRADE_NOOP_PATTERN = Pattern.compile("(?dm)^No packages marked for update$");
     private static final Pattern REMOVE_NOOP_PATTERN = Pattern.compile("(?dm)^No Packages marked for removal$");
-    private static final Pattern UNKNOWN_PACKAGE_PATTERN = Pattern.compile(
-            "(?dm)^No package ([^ ]+) available\\.$");
 
 
     // WARNING: These must be in the same order as the supplier below
@@ -64,109 +59,49 @@ public class Yum {
         return Optional.of(builder.build());
     }
 
-    /**
-     * Lock and install, or if necessary downgrade, a package to a given version.
-     *
-     * @return false only if the package was already locked and installed at the given version (no-op)
-     */
-    public boolean installFixedVersion(TaskContext context, YumPackageName yumPackage) {
-        String targetVersionLockName = yumPackage.toVersionLockName();
-
-        boolean alreadyLocked = terminal
-                .newCommandLine(context)
-                .add("yum", "--quiet", "versionlock", "list")
-                .executeSilently()
-                .getOutputLinesStream()
-                .map(YumPackageName::parseString)
-                .filter(Optional::isPresent) // removes garbage first lines, even with --quiet
-                .map(Optional::get)
-                .anyMatch(packageName -> {
-                    // Ignore lines for other packages
-                    if (packageName.getName().equals(yumPackage.getName())) {
-                        // If existing lock doesn't exactly match the full package name,
-                        // it means it's locked to another version and we must remove that lock.
-                        String versionLockName = packageName.toVersionLockName();
-                        if (versionLockName.equals(targetVersionLockName)) {
-                            return true;
-                        } else {
-                            terminal.newCommandLine(context)
-                                    .add("yum", "versionlock", "delete", versionLockName)
-                                    .execute();
-                        }
-                    }
-
-                    return false;
-                });
-
-        boolean modified = false;
-
-        if (!alreadyLocked) {
-            terminal.newCommandLine(context)
-                    .add("yum", "versionlock", "add", targetVersionLockName)
-                    .execute();
-            modified = true;
-        }
-
-        // The following 3 things may happen with yum install:
-        //  1. The package is installed or upgraded to the target version, in case we'd return
-        //     true from converge()
-        //  2. The package is already installed at target version, in case
-        //     "Nothing to do" is printed in the last line and we may return false from converge()
-        //  3. The package is already installed but at a later version than the target version,
-        //     in case the last 2 lines of the output is:
-        //       - "Package matching yakl-client-0.10-654.el7.x86_64 already installed. Checking for update."
-        //       - "Nothing to do"
-        //     And in case we need to downgrade and return true from converge()
-
-        CommandLine commandLine = terminal
-                .newCommandLine(context)
-                .add("yum", "install", "--assumeyes", yumPackage.toName());
-
-        String output = commandLine.executeSilently().getUntrimmedOutput();
-
-        if (NOTHING_TO_DO_PATTERN.matcher(output).find()) {
-            if (CHECKING_FOR_UPDATE_PATTERN.matcher(output).find()) {
-                // case 3.
-                terminal.newCommandLine(context)
-                        .add("yum", "downgrade", "--assumeyes", yumPackage.toName())
-                        .execute();
-                modified = true;
-            } else {
-                // case 2.
-            }
-        } else {
-            // case 1.
-            commandLine.recordSilentExecutionAsSystemModification();
-            modified = true;
-        }
-
-        return modified;
+    /** Lock and install, or if necessary downgrade, a package to a given version. */
+    public InstallFixedYumCommand installFixedVersion(YumPackageName yumPackage) {
+        return new InstallFixedYumCommand(terminal, yumPackage);
     }
 
     public GenericYumCommand install(YumPackageName... packages) {
-        return newYumCommand("install", packages, INSTALL_NOOP_PATTERN);
+        return new GenericYumCommand(terminal, "install", List.of(packages), INSTALL_NOOP_PATTERN);
     }
 
     public GenericYumCommand install(String package1, String... packages) {
         return install(toYumPackageNameArray(package1, packages));
     }
 
+    public GenericYumCommand install(List<String> packages) {
+        return install(packages.stream().map(YumPackageName::fromString).toArray(YumPackageName[]::new));
+    }
+
+
     public GenericYumCommand upgrade(YumPackageName... packages) {
-        return newYumCommand("upgrade", packages, UPGRADE_NOOP_PATTERN);
+        return new GenericYumCommand(terminal, "upgrade", List.of(packages), UPGRADE_NOOP_PATTERN);
     }
 
     public GenericYumCommand upgrade(String package1, String... packages) {
         return upgrade(toYumPackageNameArray(package1, packages));
     }
 
+    public GenericYumCommand upgrade(List<String> packages) {
+        return upgrade(packages.stream().map(YumPackageName::fromString).toArray(YumPackageName[]::new));
+    }
+
+
     public GenericYumCommand remove(YumPackageName... packages) {
-        return newYumCommand("remove", packages, REMOVE_NOOP_PATTERN);
+        return new GenericYumCommand(terminal, "remove", List.of(packages), REMOVE_NOOP_PATTERN);
     }
 
     public GenericYumCommand remove(String package1, String... packages) {
         return remove(toYumPackageNameArray(package1, packages));
     }
 
+    public GenericYumCommand remove(List<String> packages) {
+        return remove(packages.stream().map(YumPackageName::fromString).toArray(YumPackageName[]::new));
+    }
+
     static YumPackageName[] toYumPackageNameArray(String package1, String... packages) {
         YumPackageName[] array = new YumPackageName[1 + packages.length];
         array[0] = YumPackageName.fromString(package1);
@@ -175,71 +110,4 @@ public class Yum {
         }
         return array;
     }
-
-    private GenericYumCommand newYumCommand(String yumCommand,
-                                            YumPackageName[] packages,
-                                            Pattern noopPattern) {
-        return new GenericYumCommand(
-                terminal,
-                yumCommand,
-                List.of(packages),
-                noopPattern);
-    }
-
-    public static class GenericYumCommand {
-        private final Terminal terminal;
-        private final String yumCommand;
-        private final List<YumPackageName> packages;
-        private final Pattern commandOutputNoopPattern;
-
-        private final List<String> enabledRepo = new ArrayList<>();
-
-        private GenericYumCommand(Terminal terminal,
-                                  String yumCommand,
-                                  List<YumPackageName> packages,
-                                  Pattern commandOutputNoopPattern) {
-            this.terminal = terminal;
-            this.yumCommand = yumCommand;
-            this.packages = packages;
-            this.commandOutputNoopPattern = commandOutputNoopPattern;
-
-            if (packages.isEmpty() && ! "upgrade".equals(yumCommand)) {
-                throw new IllegalArgumentException("No packages specified");
-            }
-        }
-
-        public GenericYumCommand enableRepos(String... repos) {
-            enabledRepo.addAll(List.of(repos));
-            return this;
-        }
-
-        public boolean converge(TaskContext context) {
-            CommandLine commandLine = terminal.newCommandLine(context);
-            commandLine.add("yum", yumCommand, "--assumeyes");
-            enabledRepo.forEach(repo -> commandLine.add("--enablerepo=" + repo));
-            commandLine.add(packages.stream().map(YumPackageName::toName).collect(Collectors.toList()));
-
-            // There's no way to figure out whether a yum command would have been a no-op.
-            // Therefore, run the command and parse the output to decide.
-            boolean modifiedSystem = commandLine
-                    .executeSilently()
-                    .mapOutput(this::mapOutput);
-
-            if (modifiedSystem) {
-                commandLine.recordSilentExecutionAsSystemModification();
-            }
-
-            return modifiedSystem;
-        }
-
-        private boolean mapOutput(String output) {
-            Matcher unknownPackageMatcher = UNKNOWN_PACKAGE_PATTERN.matcher(output);
-            if (unknownPackageMatcher.find()) {
-                throw new IllegalArgumentException("Unknown package: " + unknownPackageMatcher.group(1));
-            }
-
-            return !commandOutputNoopPattern.matcher(output).find();
-        }
-    }
-
 }
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumCommand.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumCommand.java
new file mode 100644
index 00000000000..7b543304e6e
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumCommand.java
@@ -0,0 +1,208 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.yum;
+
+import com.yahoo.vespa.hosted.node.admin.component.TaskContext;
+import com.yahoo.vespa.hosted.node.admin.task.util.process.CommandLine;
+import com.yahoo.vespa.hosted.node.admin.task.util.process.Terminal;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
+/**
+ * @author freva
+ */
+public abstract class YumCommand<T extends YumCommand<T>> {
+
+    private List<String> enabledRepos = List.of();
+
+    /** Enables the given repos for this command */
+    public T enableRepo(String... repo) {
+        enabledRepos = List.of(repo);
+        return getThis();
+    }
+
+    protected abstract T getThis(); // Hack to get around unchecked cast warning
+
+    protected void addParametersToCommandLine(CommandLine commandLine) {
+        commandLine.add("--assumeyes");
+        enabledRepos.forEach(repo -> commandLine.add("--enablerepo=" + repo));
+    }
+
+    public abstract boolean converge(TaskContext context);
+
+
+    public static class GenericYumCommand extends YumCommand<GenericYumCommand> {
+        private static final Pattern UNKNOWN_PACKAGE_PATTERN = Pattern.compile("(?dm)^No package ([^ ]+) available\\.$");
+
+        private final Terminal terminal;
+        private final String yumCommand;
+        private final Pattern commandOutputNoopPattern;
+        private final List<YumPackageName> packages;
+        private final List<String> options = new ArrayList<>();
+
+        GenericYumCommand(Terminal terminal, String yumCommand, List<YumPackageName> packages, Pattern commandOutputNoopPattern) {
+            this.terminal = terminal;
+            this.yumCommand = yumCommand;
+            this.packages = packages;
+            this.commandOutputNoopPattern = commandOutputNoopPattern;
+
+            switch (yumCommand) {
+                case "install": {
+                    if (packages.size() > 1) options.add("skip_missing_names_on_install=False");
+                    break;
+                }
+                case "upgrade": {
+                    if (packages.size() > 1) options.add("skip_missing_names_on_update=False");
+                    break;
+                }
+                case "remove": break;
+                default: throw new IllegalArgumentException("Unknown yum command: " + yumCommand);
+            }
+
+            if (packages.isEmpty() && ! "upgrade".equals(yumCommand)) {
+                throw new IllegalArgumentException("No packages specified");
+            }
+        }
+
+        @Override
+        protected void addParametersToCommandLine(CommandLine commandLine) {
+            super.addParametersToCommandLine(commandLine);
+            options.forEach(option -> commandLine.add("--setopt", option));
+        }
+
+        @Override
+        public boolean converge(TaskContext context) {
+            if (packages.isEmpty() && ! "upgrade".equals(yumCommand)) {
+                throw new IllegalArgumentException("No packages specified");
+            }
+
+            CommandLine commandLine = terminal.newCommandLine(context);
+            commandLine.add("yum", yumCommand);
+            addParametersToCommandLine(commandLine);
+            commandLine.add(packages.stream().map(YumPackageName::toName).collect(Collectors.toList()));
+
+            // There's no way to figure out whether a yum command would have been a no-op.
+            // Therefore, run the command and parse the output to decide.
+            boolean modifiedSystem = commandLine
+                    .executeSilently()
+                    .mapOutput(this::mapOutput);
+
+            if (modifiedSystem) {
+                commandLine.recordSilentExecutionAsSystemModification();
+            }
+
+            return modifiedSystem;
+        }
+
+        private boolean mapOutput(String output) {
+            Matcher unknownPackageMatcher = UNKNOWN_PACKAGE_PATTERN.matcher(output);
+            if (unknownPackageMatcher.find()) {
+                throw new IllegalArgumentException("Unknown package: " + unknownPackageMatcher.group(1));
+            }
+
+            return !commandOutputNoopPattern.matcher(output).find();
+        }
+
+        protected GenericYumCommand getThis() { return this; }
+    }
+
+
+    public static class InstallFixedYumCommand extends YumCommand<InstallFixedYumCommand> {
+        // Note: "(?dm)" makes newline be \n (only), and enables multiline mode where ^$ match lines with find()
+        private static final Pattern CHECKING_FOR_UPDATE_PATTERN =
+                Pattern.compile("(?dm)^Package matching [^ ]+ already installed\\. Checking for update\\.$");
+        private static final Pattern NOTHING_TO_DO_PATTERN = Pattern.compile("(?dm)^Nothing to do$");
+
+        private final Terminal terminal;
+        private final YumPackageName yumPackage;
+
+        InstallFixedYumCommand(Terminal terminal, YumPackageName yumPackage) {
+            this.terminal = terminal;
+            this.yumPackage = yumPackage;
+        }
+
+        @Override
+        public boolean converge(TaskContext context) {
+            String targetVersionLockName = yumPackage.toVersionLockName();
+
+            boolean alreadyLocked = terminal
+                    .newCommandLine(context)
+                    .add("yum", "--quiet", "versionlock", "list")
+                    .executeSilently()
+                    .getOutputLinesStream()
+                    .map(YumPackageName::parseString)
+                    .filter(Optional::isPresent) // removes garbage first lines, even with --quiet
+                    .map(Optional::get)
+                    .anyMatch(packageName -> {
+                        // Ignore lines for other packages
+                        if (packageName.getName().equals(yumPackage.getName())) {
+                            // If existing lock doesn't exactly match the full package name,
+                            // it means it's locked to another version and we must remove that lock.
+                            String versionLockName = packageName.toVersionLockName();
+                            if (versionLockName.equals(targetVersionLockName)) {
+                                return true;
+                            } else {
+                                terminal.newCommandLine(context)
+                                        .add("yum", "versionlock", "delete", versionLockName)
+                                        .execute();
+                            }
+                        }
+
+                        return false;
+                    });
+
+            boolean modified = false;
+
+            if (!alreadyLocked) {
+                CommandLine commandLine = terminal.newCommandLine(context).add("yum", "versionlock", "add");
+                // If the targetVersionLockName refers to a package in a by-default-disabled repo,
+                // we must enable the repo unless targetVersionLockName is already installed.
+                // The other versionlock commands (list, delete) does not require --enablerepo.
+                addParametersToCommandLine(commandLine);
+                commandLine.add(targetVersionLockName).execute();
+                modified = true;
+            }
+
+            // The following 3 things may happen with yum install:
+            //  1. The package is installed or upgraded to the target version, in case we'd return
+            //     true from converge()
+            //  2. The package is already installed at target version, in case
+            //     "Nothing to do" is printed in the last line and we may return false from converge()
+            //  3. The package is already installed but at a later version than the target version,
+            //     in case the last 2 lines of the output is:
+            //       - "Package matching yakl-client-0.10-654.el7.x86_64 already installed. Checking for update."
+            //       - "Nothing to do"
+            //     And in case we need to downgrade and return true from converge()
+
+            var installCommand = terminal.newCommandLine(context).add("yum", "install");
+            addParametersToCommandLine(installCommand);
+            installCommand.add(yumPackage.toName());
+
+            String output = installCommand.executeSilently().getUntrimmedOutput();
+
+            if (NOTHING_TO_DO_PATTERN.matcher(output).find()) {
+                if (CHECKING_FOR_UPDATE_PATTERN.matcher(output).find()) {
+                    // case 3.
+                    var upgradeCommand = terminal.newCommandLine(context).add("yum", "downgrade");
+                    addParametersToCommandLine(upgradeCommand);
+                    upgradeCommand.add(yumPackage.toName()).execute();
+                    modified = true;
+                } else {
+                    // case 2.
+                }
+            } else {
+                // case 1.
+                installCommand.recordSilentExecutionAsSystemModification();
+                modified = true;
+            }
+
+            return modified;
+        }
+
+        protected InstallFixedYumCommand getThis() { return this; }
+    }
+}
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumPackageName.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumPackageName.java
index 54c8719bceb..3f5c3025850 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumPackageName.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumPackageName.java
@@ -72,6 +72,15 @@ public class YumPackageName {
             architecture = packageName.architecture;
         }
 
+        /**
+         * Set the epoch of the YUM package.
+         *
+         * <p>WARNING: Should only be invoked if the YUM package actually has an epoch. Typically
+         * YUM packages doesn't have one explicitly set, and in case "0" will be used with
+         * {@link #toVersionLockName()} (otherwise it fails), but it will be absent from an
+         * install with {@link #toName()} (otherwise it fails). This typically means that
+         * you should set this only if the epoch is != "0".</p>
+         */
         public Builder setEpoch(String epoch) { this.epoch = Optional.of(epoch); return this; }
         public Builder setName(String name) { this.name = name; return this; }
         public Builder setVersion(String version) { this.version = Optional.of(version); return this; }
@@ -235,7 +244,7 @@ public class YumPackageName {
      */
     public String toVersionLockName() {
         return String.format("%s:%s-%s-%s.%s",
-                epoch.orElseThrow(() -> new IllegalStateException("Epoch is missing for YUM package " + name)),
+                epoch.orElse("0"),
                 name,
                 version.orElseThrow(() -> new IllegalStateException("Version is missing for YUM package " + name)),
                 release.orElseThrow(() -> new IllegalStateException("Release is missing for YUM package " + name)),
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTester.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTester.java
new file mode 100644
index 00000000000..c5b46473218
--- /dev/null
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTester.java
@@ -0,0 +1,124 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.yum;
+
+import com.yahoo.vespa.hosted.node.admin.task.util.process.TestChildProcess2;
+import com.yahoo.vespa.hosted.node.admin.task.util.process.TestTerminal;
+
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+/**
+ * A {@link Yum} tester that simplifies testing interaction with yum.
+ *
+ * @author freva
+ */
+public class YumTester extends Yum {
+
+    private final TestTerminal terminal;
+
+    public YumTester(TestTerminal terminal) {
+        super(terminal);
+        this.terminal = terminal;
+    }
+
+    public GenericYumCommandExpectation expectInstall(String... packages) {
+        return new GenericYumCommandExpectation("install", packages);
+    }
+
+    public GenericYumCommandExpectation expectUpdate(String... packages) {
+        return new GenericYumCommandExpectation("upgrade", packages);
+    }
+
+    public GenericYumCommandExpectation expectRemove(String... packages) {
+        return new GenericYumCommandExpectation("remove", packages);
+    }
+
+    public InstallFixedCommandExpectation expectInstallFixedVersion(String yumPackage) {
+        return new InstallFixedCommandExpectation(yumPackage);
+    }
+
+    public QueryInstalledExpectation expectQueryInstalled(String packageName) {
+        return new QueryInstalledExpectation(packageName);
+    }
+
+
+    public class GenericYumCommandExpectation {
+        private final String command;
+        protected final List<YumPackageName> packages;
+        private List<String> enableRepos = List.of();
+
+        private GenericYumCommandExpectation(String command, String... packages) {
+            this.command = command;
+            this.packages = Stream.of(packages).map(YumPackageName::fromString).collect(Collectors.toList());
+        }
+
+        public GenericYumCommandExpectation withEnableRepo(String... repo) {
+            this.enableRepos = List.of(repo);
+            return this;
+        }
+
+        /** Mock the return value of the converge(TaskContext) method for this operation (true iff system was modified) */
+        public YumTester andReturn(boolean value) {
+            if (value) return execute("Success");
+            switch (command) {
+                case "install": return execute("Nothing to do");
+                case "upgrade": return execute("No packages marked for update");
+                case "remove": return execute("No Packages marked for removal");
+                default: throw new IllegalArgumentException("Unknown command: " + command);
+            }
+        }
+
+        private YumTester execute(String output) {
+            StringBuilder cmd = new StringBuilder();
+            cmd.append("yum ").append(command).append(" --assumeyes");
+            enableRepos.forEach(repo -> cmd.append(" --enablerepo=").append(repo));
+            if ("install".equals(command) && packages.size() > 1)
+                cmd.append(" --setopt skip_missing_names_on_install=False");
+            if ("upgrade".equals(command) && packages.size() > 1)
+                cmd.append(" --setopt skip_missing_names_on_update=False");
+            packages.forEach(pkg -> cmd.append(" ").append(pkg.toName()));
+            cmd.append(" 2>&1");
+
+            terminal.expectCommand(cmd.toString(), 0, output);
+            return YumTester.this;
+        }
+    }
+
+    public class InstallFixedCommandExpectation extends GenericYumCommandExpectation {
+        private InstallFixedCommandExpectation(String yumPackage) {
+            super("install", yumPackage);
+        }
+
+        @Override
+        public YumTester andReturn(boolean value) {
+            // Pretend package is already correctly version locked to simplify expectations
+            terminal.expectCommand("yum --quiet versionlock list 2>&1", 0, packages.get(0).toVersionLockName());
+            return super.andReturn(value);
+        }
+    }
+
+    public class QueryInstalledExpectation {
+        private final String packageName;
+
+        public QueryInstalledExpectation(String packageName) {
+            this.packageName = packageName;
+        }
+
+        /** Package name to return or null if package is not installed */
+        public YumTester andReturn(YumPackageName yumPackage) {
+            TestChildProcess2 process = new TestChildProcess2(
+                    yumPackage == null ? 1 : 0,
+                    yumPackage == null ? "not installed" : String.join("\n",
+                            yumPackage.getName(),
+                            yumPackage.getEpoch().orElse("(none)"),
+                            yumPackage.getVersion().orElseThrow(() -> new IllegalArgumentException("Version must be set")),
+                            yumPackage.getRelease().orElseThrow(() -> new IllegalArgumentException("Release must be set")),
+                            yumPackage.getArchitecture().orElse("(none)")));
+
+            terminal.expectCommand("rpm -q " + packageName + " --queryformat \"%{NAME}\\\\n%{EPOCH}\\\\n%{VERSION}\\\\n%{RELEASE}\\\\n%{ARCH}\" 2>&1", process);
+            return YumTester.this;
+        }
+    }
+
+}
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeStateTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeStateTest.java
index 4a71d3bdbe8..75ab493b3b8 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeStateTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeStateTest.java
@@ -22,4 +22,5 @@ public class NodeStateTest {
 
         assertEquals(nodeAdminStates, nodeRepositoryStates);
     }
+
 }
\ No newline at end of file
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java
index eadcd997ae8..4e0fd95384c 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java
@@ -164,7 +164,6 @@ public class RealNodeRepositoryTest {
         AddNode host = AddNode.forHost("host123.domain.tld",
                                        "default",
                                        Optional.of(FlavorOverrides.ofDisk(123)),
-                                       Optional.empty(),
                                        NodeType.confighost,
                                        Set.of("::1"), Set.of("::2", "::3"));
 
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerFailTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerFailTest.java
index 7776bf1f224..0ed52cae5b8 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerFailTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerFailTest.java
@@ -2,10 +2,8 @@
 package com.yahoo.vespa.hosted.node.admin.integrationTests;
 
 import com.yahoo.config.provision.DockerImage;
-import com.yahoo.config.provision.NodeType;
 import com.yahoo.vespa.hosted.dockerapi.ContainerName;
 import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec;
-import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeState;
 import org.junit.Test;
 
 import static org.mockito.ArgumentMatchers.any;
@@ -24,18 +22,10 @@ public class DockerFailTest {
             final DockerImage dockerImage = DockerImage.fromString("dockerImage");
             final ContainerName containerName = new ContainerName("host1");
             final String hostname = "host1.test.yahoo.com";
-            tester.addChildNodeRepositoryNode(new NodeSpec.Builder()
-                    .hostname(hostname)
+            tester.addChildNodeRepositoryNode(NodeSpec.Builder
+                    .testSpec(hostname)
                     .wantedDockerImage(dockerImage)
                     .currentDockerImage(dockerImage)
-                    .state(NodeState.active)
-                    .type(NodeType.tenant)
-                    .flavor("docker")
-                    .wantedRestartGeneration(1L)
-                    .currentRestartGeneration(1L)
-                    .vcpu(1)
-                    .memoryGb(1)
-                    .diskGb(1)
                     .build());
 
             tester.inOrder(tester.docker).createContainerCommand(eq(dockerImage), eq(containerName));
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerMock.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerMock.java
index e2ad9e3de97..1484e40afe9 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerMock.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerMock.java
@@ -176,7 +176,12 @@ public class DockerMock implements Docker {
         }
 
         @Override
-        public CreateContainerCommand withSecurityOpts(String securityOpt) {
+        public CreateContainerCommand withSecurityOpt(String securityOpt) {
+            return this;
+        }
+
+        @Override
+        public CreateContainerCommand withDnsOption(String dnsOption) {
             return this;
         }
 
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java
index 653a6f7f091..9b509829ba2 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java
@@ -7,7 +7,6 @@ import com.yahoo.vespa.flags.InMemoryFlagSource;
 import com.yahoo.vespa.hosted.dockerapi.Docker;
 import com.yahoo.vespa.hosted.dockerapi.metrics.Metrics;
 import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec;
-import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeState;
 import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.Orchestrator;
 import com.yahoo.vespa.hosted.node.admin.docker.DockerOperations;
 import com.yahoo.vespa.hosted.node.admin.docker.DockerOperationsImpl;
@@ -76,28 +75,22 @@ public class DockerTester implements AutoCloseable {
 
         TerminalImpl terminal = new TerminalImpl(command -> new TestChildProcess2(0, ""));
 
-        NodeSpec hostSpec = new NodeSpec.Builder()
-                .hostname(HOST_HOSTNAME.value())
-                .state(NodeState.active)
-                .type(NodeType.host)
-                .flavor("default")
-                .wantedRestartGeneration(1L)
-                .currentRestartGeneration(1L)
-                .build();
+        NodeSpec hostSpec = NodeSpec.Builder.testSpec(HOST_HOSTNAME.value()).type(NodeType.host).build();
         nodeRepository.updateNodeRepositoryNode(hostSpec);
 
+        Clock clock = Clock.systemUTC();
+        Metrics metrics = new Metrics();
         FileSystem fileSystem = TestFileSystem.create();
         DockerOperations dockerOperations = new DockerOperationsImpl(docker, terminal, ipAddresses, flagSource);
 
-        Metrics metrics = new Metrics();
         NodeAgentFactory nodeAgentFactory = (contextSupplier, nodeContext) -> new NodeAgentImpl(
                 contextSupplier, nodeRepository, orchestrator, dockerOperations, storageMaintainer, flagSource,
-                Optional.empty(), Optional.empty(), Optional.empty());
-        nodeAdmin = new NodeAdminImpl(nodeAgentFactory, metrics, Clock.systemUTC(), Duration.ofMillis(10), Duration.ZERO);
+                Optional.empty(), Optional.empty(), Optional.empty(), clock, Duration.ofSeconds(-1));
+        nodeAdmin = new NodeAdminImpl(nodeAgentFactory, metrics, clock, Duration.ofMillis(10), Duration.ZERO);
         NodeAgentContextFactory nodeAgentContextFactory = (nodeSpec, acl) ->
                 new NodeAgentContextImpl.Builder(nodeSpec).acl(acl).pathToContainerStorageFromFileSystem(fileSystem).build();
         nodeAdminStateUpdater = new NodeAdminStateUpdater(nodeAgentContextFactory, nodeRepository, orchestrator,
-                nodeAdmin, HOST_HOSTNAME, Clock.systemUTC());
+                nodeAdmin, HOST_HOSTNAME, clock);
 
         loopThread = new Thread(() -> {
             nodeAdminStateUpdater.start();
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/MultiDockerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/MultiDockerTest.java
index a09c6a9e907..e5156e68810 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/MultiDockerTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/MultiDockerTest.java
@@ -2,7 +2,6 @@
 package com.yahoo.vespa.hosted.node.admin.integrationTests;
 
 import com.yahoo.config.provision.DockerImage;
-import com.yahoo.config.provision.NodeType;
 import com.yahoo.vespa.hosted.dockerapi.ContainerName;
 import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeAttributes;
 import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec;
@@ -25,13 +24,7 @@ public class MultiDockerTest {
             NodeSpec nodeSpec2 = addAndWaitForNode(
                     tester, "host2.test.yahoo.com", DockerImage.fromString("image2"));
 
-            tester.addChildNodeRepositoryNode(
-                    new NodeSpec.Builder(nodeSpec2)
-                            .state(NodeState.dirty)
-                            .vcpu(1)
-                            .memoryGb(1)
-                            .diskGb(1)
-                            .build());
+            tester.addChildNodeRepositoryNode(NodeSpec.Builder.testSpec(nodeSpec2.hostname(), NodeState.dirty).build());
 
             tester.inOrder(tester.docker).deleteContainer(eq(new ContainerName("host2")));
             tester.inOrder(tester.storageMaintainer).archiveNodeStorage(
@@ -43,19 +36,7 @@ public class MultiDockerTest {
     }
 
     private NodeSpec addAndWaitForNode(DockerTester tester, String hostName, DockerImage dockerImage) {
-        NodeSpec nodeSpec = new NodeSpec.Builder()
-                .hostname(hostName)
-                .wantedDockerImage(dockerImage)
-                .state(NodeState.active)
-                .type(NodeType.tenant)
-                .flavor("docker")
-                .wantedRestartGeneration(1L)
-                .currentRestartGeneration(1L)
-                .vcpu(2)
-                .memoryGb(4)
-                .diskGb(1)
-                .build();
-
+        NodeSpec nodeSpec = NodeSpec.Builder.testSpec(hostName).wantedDockerImage(dockerImage).build();
         tester.addChildNodeRepositoryNode(nodeSpec);
 
         ContainerName containerName = ContainerName.fromHostname(hostName);
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RebootTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RebootTest.java
index d1778982043..30f23de15bf 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RebootTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RebootTest.java
@@ -1,12 +1,9 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.node.admin.integrationTests;
 
-import com.yahoo.component.Version;
 import com.yahoo.config.provision.DockerImage;
-import com.yahoo.config.provision.NodeType;
 import com.yahoo.vespa.hosted.dockerapi.ContainerName;
 import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec;
-import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeState;
 import com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminStateUpdater;
 import org.junit.Test;
 
@@ -31,7 +28,7 @@ public class RebootTest {
     @Test
     public void test() {
         try (DockerTester tester = new DockerTester()) {
-            tester.addChildNodeRepositoryNode(createNodeRepositoryNode());
+            tester.addChildNodeRepositoryNode(NodeSpec.Builder.testSpec(hostname).wantedDockerImage(dockerImage).build());
 
             tester.inOrder(tester.docker).createContainerCommand(eq(dockerImage), eq(new ContainerName("host1")));
 
@@ -46,17 +43,4 @@ public class RebootTest {
             assertTrue(tester.nodeAdmin.setFrozen(true));
         }
     }
-
-    private NodeSpec createNodeRepositoryNode() {
-        return new NodeSpec.Builder()
-                .hostname(hostname)
-                .wantedDockerImage(dockerImage)
-                .state(NodeState.active)
-                .type(NodeType.tenant)
-                .flavor("docker")
-                .currentVespaVersion(Version.fromString("6.50.0"))
-                .wantedRestartGeneration(1L)
-                .currentRestartGeneration(1L)
-                .build();
-    }
 }
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RestartTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RestartTest.java
index bfc54cac045..dd727d102e0 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RestartTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RestartTest.java
@@ -2,11 +2,9 @@
 package com.yahoo.vespa.hosted.node.admin.integrationTests;
 
 import com.yahoo.config.provision.DockerImage;
-import com.yahoo.config.provision.NodeType;
 import com.yahoo.vespa.hosted.dockerapi.ContainerName;
 import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeAttributes;
 import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec;
-import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeState;
 import org.junit.Test;
 
 import static com.yahoo.vespa.hosted.node.admin.integrationTests.DockerTester.NODE_PROGRAM;
@@ -26,15 +24,7 @@ public class RestartTest {
             String hostname = "host1.test.yahoo.com";
             DockerImage dockerImage = DockerImage.fromString("dockerImage:1.2.3");
 
-            tester.addChildNodeRepositoryNode(new NodeSpec.Builder()
-                    .hostname(hostname)
-                    .state(NodeState.active)
-                    .wantedDockerImage(dockerImage)
-                    .type(NodeType.tenant)
-                    .flavor("docker")
-                    .wantedRestartGeneration(1)
-                    .currentRestartGeneration(1)
-                    .build());
+            tester.addChildNodeRepositoryNode(NodeSpec.Builder.testSpec(hostname).wantedDockerImage(dockerImage).build());
 
             tester.inOrder(tester.docker).createContainerCommand(eq(dockerImage), eq(new ContainerName("host1")));
             tester.inOrder(tester.nodeRepository).updateNodeAttributes(
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java
index 3d9e3c08276..60a4462e9e9 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java
@@ -1,6 +1,8 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.node.admin.maintenance.coredump;
 
+import com.yahoo.vespa.hosted.dockerapi.metrics.DimensionMetrics;
+import com.yahoo.vespa.hosted.dockerapi.metrics.Metrics;
 import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext;
 import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImpl;
 import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath;
@@ -46,16 +48,17 @@ public class CoredumpHandlerTest {
     private final Path donePath = fileSystem.getPath("/home/docker/dumps");
     private final NodeAgentContext context = new NodeAgentContextImpl.Builder("container-123.domain.tld")
             .fileSystem(fileSystem).build();
-    private final Path crashPathInContainer = Paths.get("/var/crash");
+    private final Path crashPathInContainer = fileSystem.getPath("/var/crash");
     private final Path doneCoredumpsPath = fileSystem.getPath("/home/docker/dumps");
 
     private final TestTerminal terminal = new TestTerminal();
     private final CoreCollector coreCollector = mock(CoreCollector.class);
     private final CoredumpReporter coredumpReporter = mock(CoredumpReporter.class);
+    private final Metrics metrics = new Metrics();
     @SuppressWarnings("unchecked")
     private final Supplier<String> coredumpIdSupplier = mock(Supplier.class);
     private final CoredumpHandler coredumpHandler = new CoredumpHandler(terminal, coreCollector, coredumpReporter,
-            crashPathInContainer, doneCoredumpsPath, "users", coredumpIdSupplier);
+            crashPathInContainer, doneCoredumpsPath, "users", metrics, coredumpIdSupplier);
 
 
     @Test
@@ -206,13 +209,30 @@ public class CoredumpHandlerTest {
         verify(coreCollector, never()).collect(any(), any());
         verify(coredumpReporter, times(1)).reportCoredump(eq("id-123"), eq("metadata"));
         assertFalse(Files.exists(coredumpDirectory));
-        assertFolderContents(doneCoredumpsPath, "id-123");
-        assertFolderContents(doneCoredumpsPath.resolve("id-123"), "metadata.json", "dump_bash.core.431.lz4");
+        assertFolderContents(doneCoredumpsPath.resolve("container-123"), "id-123");
+        assertFolderContents(doneCoredumpsPath.resolve("container-123").resolve("id-123"), "metadata.json", "dump_bash.core.431.lz4");
+    }
+
+    @Test
+    public void report_enqueued_and_processed_metrics() throws IOException {
+        Files.createFile(crashPathInContainer.resolve("dump-1"));
+        Files.createFile(crashPathInContainer.resolve("dump-2"));
+        Files.createFile(crashPathInContainer.resolve("hs_err_pid2.log"));
+        new UnixPath(doneCoredumpsPath.resolve("container-123").resolve("dump-3-folder").resolve("dump-3"))
+                .createParents()
+                .createNewFile();
+
+        coredumpHandler.updateMetrics(context, crashPathInContainer);
+        List<DimensionMetrics> updatedMetrics = metrics.getMetricsByType(Metrics.DimensionType.PRETAGGED);
+        assertEquals(1, updatedMetrics.size());
+        Map<String, Number> values = updatedMetrics.get(0).getMetrics();
+        assertEquals(2, values.get("coredumps.enqueued").intValue());
+        assertEquals(1, values.get("coredumps.processed").intValue());
     }
 
     @Before
     public void setup() throws IOException {
-        Files.createDirectories(donePath);
+        Files.createDirectories(crashPathInContainer);
     }
 
     @After
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterTest.java
index 9a25e4188f9..36530ff014c 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdaterTest.java
@@ -2,7 +2,6 @@
 package com.yahoo.vespa.hosted.node.admin.nodeadmin;
 
 import com.yahoo.config.provision.HostName;
-import com.yahoo.config.provision.NodeType;
 import com.yahoo.test.ManualClock;
 import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.Acl;
 import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeRepository;
@@ -258,28 +257,12 @@ public class NodeAdminStateUpdaterTest {
 
     private void mockNodeRepo(NodeState hostState, int numberOfNodes) {
         List<NodeSpec> containersToRun = IntStream.range(1, numberOfNodes + 1)
-                .mapToObj(i -> new NodeSpec.Builder()
-                        .hostname("host" + i + ".yahoo.com")
-                        .state(NodeState.active)
-                        .type(NodeType.tenant)
-                        .flavor("docker")
-                        .vcpu(1)
-                        .memoryGb(1)
-                        .diskGb(1)
-                        .build())
+                .mapToObj(i -> NodeSpec.Builder.testSpec("host" + i + ".yahoo.com").build())
                 .collect(Collectors.toList());
 
         when(nodeRepository.getNodes(eq(hostHostname.value()))).thenReturn(containersToRun);
-
-        when(nodeRepository.getNode(eq(hostHostname.value()))).thenReturn(new NodeSpec.Builder()
-                .hostname(hostHostname.value())
-                .state(hostState)
-                .type(NodeType.tenant)
-                .flavor("default")
-                .vcpu(1)
-                .memoryGb(1)
-                .diskGb(1)
-                .build());
+        when(nodeRepository.getNode(eq(hostHostname.value()))).thenReturn(
+                NodeSpec.Builder.testSpec(hostHostname.value(), hostState).build());
     }
 
     private void mockAcl(Acl acl, int... nodeIds) {
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImplTest.java
index e84d8345815..250c005566b 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImplTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImplTest.java
@@ -1,10 +1,13 @@
 // Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.node.admin.nodeagent;
 
+import com.yahoo.config.provision.DockerImage;
 import com.yahoo.vespa.test.file.TestFileSystem;
 import org.junit.Test;
 
 import java.nio.file.FileSystem;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 
 import static org.junit.Assert.assertEquals;
 
@@ -69,4 +72,23 @@ public class NodeAgentContextImplTest {
     public void path_under_vespa_home_must_be_relative() {
         context.pathInNodeUnderVespaHome("/home");
     }
+
+    @Test
+    public void rewrites_vespa_home_mount_point() {
+        assertRewrite("docker.tld/vespa/ci:1.2.3", "/var/log", "/var/log");
+        assertRewrite("docker.tld/vespa/ci:1.2.3", "/home/y/log", "/home/y/log");
+        assertRewrite("docker.tld/vespa/ci:1.2.3", "/opt/vespa/log", "/home/y/log");
+
+        assertRewrite("docker.tld/vespa/hosted:1.2.3", "/var/log", "/var/log");
+        assertRewrite("docker.tld/vespa/hosted:1.2.3", "/home/y/log", "/home/y/log");
+        assertRewrite("docker.tld/vespa/hosted:1.2.3", "/opt/vespa/log", "/opt/vespa/log");
+    }
+
+    private static void assertRewrite(String dockerImage, String path, String expected) {
+        NodeAgentContext context = new NodeAgentContextImpl.Builder("node123")
+                .nodeSpecBuilder(ns -> ns.wantedDockerImage(DockerImage.fromString(dockerImage)))
+                .build();
+        Path actual = context.rewritePathInNodeForWantedDockerImage(Paths.get(path));
+        assertEquals(Paths.get(expected), actual);
+    }
 }
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java
index 9e1f84f07a1..b895300d7b1 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java
@@ -6,6 +6,7 @@ import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.DockerImage;
 import com.yahoo.config.provision.NodeResources;
 import com.yahoo.config.provision.NodeType;
+import com.yahoo.test.ManualClock;
 import com.yahoo.vespa.flags.Flags;
 import com.yahoo.vespa.flags.InMemoryFlagSource;
 import com.yahoo.vespa.hosted.dockerapi.Container;
@@ -26,12 +27,16 @@ import com.yahoo.vespa.hosted.node.admin.nodeadmin.ConvergenceException;
 import org.junit.Test;
 import org.mockito.InOrder;
 
+import java.time.Duration;
+import java.time.Instant;
 import java.util.Optional;
 
+import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.fail;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.eq;
 import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.doNothing;
 import static org.mockito.Mockito.doThrow;
 import static org.mockito.Mockito.inOrder;
 import static org.mockito.Mockito.mock;
@@ -47,13 +52,7 @@ import static org.mockito.Mockito.when;
 public class NodeAgentImplTest {
     private static final NodeResources resources = new NodeResources(2, 16, 250, 1, NodeResources.DiskSpeed.fast, NodeResources.StorageType.local);
     private static final Version vespaVersion = Version.fromString("1.2.3");
-
-    private final String hostName = "host1.test.yahoo.com";
-    private final NodeSpec.Builder nodeBuilder = new NodeSpec.Builder()
-            .hostname(hostName)
-            .type(NodeType.tenant)
-            .flavor("docker")
-            .resources(resources);
+    private static final String hostName = "host1.test.yahoo.com";
 
     private final NodeAgentContextSupplier contextSupplier = mock(NodeAgentContextSupplier.class);
     private final DockerImage dockerImage = DockerImage.fromString("dockerImage");
@@ -65,15 +64,15 @@ public class NodeAgentImplTest {
     private final HealthChecker healthChecker = mock(HealthChecker.class);
     private final CredentialsMaintainer credentialsMaintainer = mock(CredentialsMaintainer.class);
     private final InMemoryFlagSource flagSource = new InMemoryFlagSource();
+    private final ManualClock clock = new ManualClock(Instant.now());
 
 
     @Test
     public void upToDateContainerIsUntouched() {
-        final NodeSpec node = nodeBuilder
-                .state(NodeState.active)
+        final NodeSpec node = nodeBuilder(NodeState.active)
                 .wantedDockerImage(dockerImage).currentDockerImage(dockerImage)
                 .wantedVespaVersion(vespaVersion).currentVespaVersion(vespaVersion)
-                .wantedRestartGeneration(1).currentRestartGeneration(1)
+                .allowedToBeDown(false)
                 .build();
 
         NodeAgentContext context = createContext(node);
@@ -92,16 +91,14 @@ public class NodeAgentImplTest {
         // TODO: Verify this isn't run unless 1st time
         inOrder.verify(dockerOperations, never()).startServices(eq(context));
         inOrder.verify(dockerOperations, times(1)).resumeNode(eq(context));
-        inOrder.verify(orchestrator).resume(hostName);
+        inOrder.verify(orchestrator, never()).resume(hostName);
     }
 
     @Test
     public void verifyRemoveOldFilesIfDiskFull() {
-        final NodeSpec node = nodeBuilder
-                .state(NodeState.active)
+        final NodeSpec node = nodeBuilder(NodeState.active)
                 .wantedDockerImage(dockerImage).currentDockerImage(dockerImage)
                 .wantedVespaVersion(vespaVersion).currentVespaVersion(vespaVersion)
-                .wantedRestartGeneration(1).currentRestartGeneration(1)
                 .build();
 
         NodeAgentContext context = createContext(node);
@@ -117,11 +114,9 @@ public class NodeAgentImplTest {
     @Test
     public void startsAfterStoppingServices() {
         final InOrder inOrder = inOrder(dockerOperations);
-        final NodeSpec node = nodeBuilder
-                .state(NodeState.active)
+        final NodeSpec node = nodeBuilder(NodeState.active)
                 .wantedDockerImage(dockerImage).currentDockerImage(dockerImage)
                 .wantedVespaVersion(vespaVersion).currentVespaVersion(vespaVersion)
-                .wantedRestartGeneration(1).currentRestartGeneration(1)
                 .build();
 
         NodeAgentContext context = createContext(node);
@@ -153,11 +148,9 @@ public class NodeAgentImplTest {
 
     @Test
     public void absentContainerCausesStart() {
-        final NodeSpec node = nodeBuilder
-                .state(NodeState.active)
+        final NodeSpec node = nodeBuilder(NodeState.active)
                 .wantedDockerImage(dockerImage)
                 .wantedVespaVersion(vespaVersion)
-                .wantedRestartGeneration(1).currentRestartGeneration(1)
                 .build();
 
         NodeAgentContext context = createContext(node);
@@ -182,17 +175,15 @@ public class NodeAgentImplTest {
         inOrder.verify(healthChecker, times(1)).verifyHealth(eq(context));
         inOrder.verify(nodeRepository).updateNodeAttributes(
                 hostName, new NodeAttributes().withDockerImage(dockerImage).withVespaVersion(dockerImage.tagAsVersion()));
-        inOrder.verify(orchestrator).resume(hostName);
+        inOrder.verify(orchestrator, never()).resume(hostName);
     }
 
     @Test
     public void containerIsNotStoppedIfNewImageMustBePulled() {
         final DockerImage newDockerImage = DockerImage.fromString("new-image");
-        final NodeSpec node = nodeBuilder
-                .state(NodeState.active)
+        final NodeSpec node = nodeBuilder(NodeState.active)
                 .wantedDockerImage(newDockerImage).currentDockerImage(dockerImage)
                 .wantedVespaVersion(vespaVersion).currentVespaVersion(vespaVersion)
-                .wantedRestartGeneration(1).currentRestartGeneration(1)
                 .build();
 
         NodeAgentContext context = createContext(node);
@@ -214,11 +205,10 @@ public class NodeAgentImplTest {
 
     @Test
     public void containerIsUpdatedIfCpuChanged() {
-        NodeSpec.Builder specBuilder = nodeBuilder
-                .state(NodeState.active)
+        NodeSpec.Builder specBuilder = nodeBuilder(NodeState.active)
                 .wantedDockerImage(dockerImage).currentDockerImage(dockerImage)
                 .wantedVespaVersion(vespaVersion).currentVespaVersion(vespaVersion)
-                .wantedRestartGeneration(1).currentRestartGeneration(1);
+                .allowedToBeDown(false);
 
         NodeAgentContext firstContext = createContext(specBuilder.build());
         NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true);
@@ -226,17 +216,20 @@ public class NodeAgentImplTest {
         when(dockerOperations.pullImageAsyncIfNeeded(any())).thenReturn(true);
         when(storageMaintainer.getDiskUsageFor(any())).thenReturn(Optional.of(201326592000L));
 
+        InOrder inOrder = inOrder(orchestrator, dockerOperations);
+
         nodeAgent.doConverge(firstContext);
+        inOrder.verify(orchestrator, never()).resume(any(String.class));
+
         NodeAgentContext secondContext = createContext(specBuilder.diskGb(200).build());
         nodeAgent.doConverge(secondContext);
+        inOrder.verify(orchestrator, never()).resume(any(String.class));
+
         NodeAgentContext thirdContext = new NodeAgentContextImpl.Builder(specBuilder.vcpu(5).build()).cpuSpeedUp(1.25).build();
         nodeAgent.doConverge(thirdContext);
         ContainerResources resourcesAfterThird = ContainerResources.from(0, 4, 16);
         mockGetContainer(dockerImage, resourcesAfterThird, true);
 
-        InOrder inOrder = inOrder(orchestrator, dockerOperations);
-        inOrder.verify(orchestrator).resume(any(String.class));
-        inOrder.verify(orchestrator).resume(any(String.class));
         inOrder.verify(orchestrator).suspend(any(String.class));
         inOrder.verify(dockerOperations).updateContainer(eq(thirdContext), eq(resourcesAfterThird));
         inOrder.verify(dockerOperations, never()).removeContainer(any(), any());
@@ -248,7 +241,7 @@ public class NodeAgentImplTest {
         inOrder.verify(orchestrator, never()).suspend(any(String.class));
         inOrder.verify(dockerOperations, never()).updateContainer(eq(thirdContext), any());
         inOrder.verify(dockerOperations, never()).removeContainer(any(), any());
-        inOrder.verify(orchestrator).resume(any(String.class));
+        inOrder.verify(orchestrator, never()).resume(any(String.class));
 
         // Set the feature flag
         flagSource.withDoubleFlag(Flags.CONTAINER_CPU_CAP.id(), 2.3);
@@ -260,8 +253,7 @@ public class NodeAgentImplTest {
 
     @Test
     public void containerIsRecreatedIfMemoryChanged() {
-        NodeSpec.Builder specBuilder = nodeBuilder
-                .state(NodeState.active)
+        NodeSpec.Builder specBuilder = nodeBuilder(NodeState.active)
                 .wantedDockerImage(dockerImage).currentDockerImage(dockerImage)
                 .wantedVespaVersion(vespaVersion).currentVespaVersion(vespaVersion)
                 .wantedRestartGeneration(2).currentRestartGeneration(1);
@@ -293,8 +285,7 @@ public class NodeAgentImplTest {
 
     @Test
     public void noRestartIfOrchestratorSuspendFails() {
-        final NodeSpec node = nodeBuilder
-                .state(NodeState.active)
+        final NodeSpec node = nodeBuilder(NodeState.active)
                 .wantedDockerImage(dockerImage).currentDockerImage(dockerImage)
                 .wantedVespaVersion(vespaVersion).currentVespaVersion(vespaVersion)
                 .wantedRestartGeneration(2).currentRestartGeneration(1)
@@ -321,11 +312,9 @@ public class NodeAgentImplTest {
     @Test
     public void recreatesContainerIfRebootWanted() {
         final long wantedRebootGeneration = 2;
-        final NodeSpec node = nodeBuilder
-                .state(NodeState.active)
+        final NodeSpec node = nodeBuilder(NodeState.active)
                 .wantedDockerImage(dockerImage).currentDockerImage(dockerImage)
                 .wantedVespaVersion(vespaVersion).currentVespaVersion(vespaVersion)
-                .wantedRestartGeneration(1).currentRestartGeneration(1)
                 .wantedRebootGeneration(wantedRebootGeneration).currentRebootGeneration(1)
                 .build();
 
@@ -362,8 +351,7 @@ public class NodeAgentImplTest {
 
     @Test
     public void failedNodeRunningContainerShouldStillBeRunning() {
-        final NodeSpec node = nodeBuilder
-                .state(NodeState.failed)
+        final NodeSpec node = nodeBuilder(NodeState.failed)
                 .wantedDockerImage(dockerImage).currentDockerImage(dockerImage)
                 .wantedVespaVersion(vespaVersion).currentVespaVersion(vespaVersion)
                 .build();
@@ -382,9 +370,7 @@ public class NodeAgentImplTest {
 
     @Test
     public void readyNodeLeadsToNoAction() {
-        final NodeSpec node = nodeBuilder
-                .state(NodeState.ready)
-                .build();
+        final NodeSpec node = nodeBuilder(NodeState.ready).build();
 
         NodeAgentContext context = createContext(node);
         NodeAgentImpl nodeAgent = makeNodeAgent(null,false);
@@ -406,8 +392,7 @@ public class NodeAgentImplTest {
 
     @Test
     public void inactiveNodeRunningContainerShouldStillBeRunning() {
-        final NodeSpec node = nodeBuilder
-                .state(NodeState.inactive)
+        final NodeSpec node = nodeBuilder(NodeState.inactive)
                 .wantedDockerImage(dockerImage).currentDockerImage(dockerImage)
                 .wantedVespaVersion(vespaVersion).currentVespaVersion(vespaVersion)
                 .build();
@@ -428,8 +413,7 @@ public class NodeAgentImplTest {
 
     @Test
     public void reservedNodeDoesNotUpdateNodeRepoWithVersion() {
-        final NodeSpec node = nodeBuilder
-                .state(NodeState.reserved)
+        final NodeSpec node = nodeBuilder(NodeState.reserved)
                 .wantedDockerImage(dockerImage)
                 .wantedVespaVersion(vespaVersion)
                 .build();
@@ -445,14 +429,12 @@ public class NodeAgentImplTest {
     }
 
     private void nodeRunningContainerIsTakenDownAndCleanedAndRecycled(NodeState nodeState, Optional<Long> wantedRestartGeneration) {
-        wantedRestartGeneration.ifPresent(restartGeneration -> nodeBuilder
+        NodeSpec.Builder builder = nodeBuilder(nodeState)
+                .wantedDockerImage(dockerImage).currentDockerImage(dockerImage);
+        wantedRestartGeneration.ifPresent(restartGeneration -> builder
                 .wantedRestartGeneration(restartGeneration).currentRestartGeneration(restartGeneration));
 
-        final NodeSpec node = nodeBuilder
-                .state(nodeState)
-                .wantedDockerImage(dockerImage).currentDockerImage(dockerImage)
-                .build();
-
+        NodeSpec node = builder.build();
         NodeAgentContext context = createContext(node);
         NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true);
 
@@ -490,9 +472,8 @@ public class NodeAgentImplTest {
 
     @Test
     public void provisionedNodeIsMarkedAsDirty() {
-        final NodeSpec node = nodeBuilder
+        final NodeSpec node = nodeBuilder(NodeState.provisioned)
                 .wantedDockerImage(dockerImage)
-                .state(NodeState.provisioned)
                 .build();
 
         NodeAgentContext context = createContext(node);
@@ -505,8 +486,7 @@ public class NodeAgentImplTest {
 
     @Test
     public void testRestartDeadContainerAfterNodeAdminRestart() {
-        final NodeSpec node = nodeBuilder
-                .state(NodeState.active)
+        final NodeSpec node = nodeBuilder(NodeState.active)
                 .currentDockerImage(dockerImage).wantedDockerImage(dockerImage)
                 .currentVespaVersion(vespaVersion)
                 .build();
@@ -526,11 +506,11 @@ public class NodeAgentImplTest {
 
     @Test
     public void resumeProgramRunsUntilSuccess() {
-        final NodeSpec node = nodeBuilder
-                .state(NodeState.active)
+        final NodeSpec node = nodeBuilder(NodeState.active)
                 .wantedDockerImage(dockerImage).currentDockerImage(dockerImage)
                 .currentVespaVersion(vespaVersion)
                 .wantedRestartGeneration(1).currentRestartGeneration(1)
+                .allowedToBeDown(true)
                 .build();
 
         NodeAgentContext context = createContext(node);
@@ -563,8 +543,7 @@ public class NodeAgentImplTest {
 
     @Test
     public void start_container_subtask_failure_leads_to_container_restart() {
-        final NodeSpec node = nodeBuilder
-                .state(NodeState.active)
+        final NodeSpec node = nodeBuilder(NodeState.active)
                 .wantedDockerImage(dockerImage)
                 .wantedVespaVersion(vespaVersion)
                 .wantedRestartGeneration(1).currentRestartGeneration(1)
@@ -600,11 +579,11 @@ public class NodeAgentImplTest {
 
     @Test
     public void testRunningConfigServer() {
-        final NodeSpec node = nodeBuilder
-                .state(NodeState.active)
+        final NodeSpec node = nodeBuilder(NodeState.active)
                 .type(NodeType.config)
                 .wantedDockerImage(dockerImage)
                 .wantedVespaVersion(vespaVersion)
+                .allowedToBeDown(true)
                 .build();
 
         NodeAgentContext context = createContext(node);
@@ -641,6 +620,84 @@ public class NodeAgentImplTest {
         verifyThatContainerIsStopped(NodeState.inactive, Optional.of(ApplicationId.defaultId()));
     }
 
+    @Test
+    public void initial_cpu_cap_test() {
+        NodeSpec.Builder specBuilder = nodeBuilder(NodeState.active)
+                .wantedDockerImage(dockerImage).currentDockerImage(dockerImage)
+                .wantedVespaVersion(vespaVersion).currentVespaVersion(vespaVersion);
+
+        NodeAgentContext context = createContext(specBuilder.build());
+        NodeAgentImpl nodeAgent = makeNodeAgent(null, false, Duration.ofSeconds(30));
+
+        when(storageMaintainer.getDiskUsageFor(any())).thenReturn(Optional.of(201326592000L));
+
+        InOrder inOrder = inOrder(orchestrator, dockerOperations);
+
+        ConvergenceException healthCheckException = new ConvergenceException("Not yet up");
+        doThrow(healthCheckException).when(healthChecker).verifyHealth(any());
+        for (int i = 0; i < 3; i++) {
+            try {
+                nodeAgent.doConverge(context);
+                fail("Expected to fail with health check exception");
+            } catch (ConvergenceException e) {
+                assertEquals(healthCheckException, e);
+            }
+            clock.advance(Duration.ofSeconds(30));
+        }
+
+        doNothing().when(healthChecker).verifyHealth(any());
+        try {
+            nodeAgent.doConverge(context);
+            fail("Expected to fail due to warm up period not yet done");
+        } catch (ConvergenceException e) {
+            assertEquals("Refusing to resume until warm up period ends (in PT30S)", e.getMessage());
+        }
+        inOrder.verify(orchestrator, never()).resume(any());
+        inOrder.verify(orchestrator, never()).suspend(any());
+        inOrder.verify(dockerOperations, never()).updateContainer(any(), any());
+
+
+        clock.advance(Duration.ofSeconds(31));
+        nodeAgent.doConverge(context);
+
+        inOrder.verify(orchestrator).suspend(hostName);
+        inOrder.verify(dockerOperations).updateContainer(eq(context), eq(ContainerResources.from(0, 2, 16)));
+        inOrder.verify(dockerOperations, never()).removeContainer(any(), any());
+        inOrder.verify(dockerOperations, never()).startContainer(any());
+        inOrder.verify(orchestrator).resume(any(String.class));
+
+        // No changes
+        nodeAgent.converge(context);
+        inOrder.verify(orchestrator, never()).suspend(any(String.class));
+        inOrder.verify(dockerOperations, never()).updateContainer(eq(context), any());
+        inOrder.verify(dockerOperations, never()).removeContainer(any(), any());
+        inOrder.verify(orchestrator, never()).resume(any(String.class));
+    }
+
+    @Test
+    public void resumes_normally_if_container_is_already_capped_on_start() {
+        NodeSpec.Builder specBuilder = nodeBuilder(NodeState.active)
+                .wantedDockerImage(dockerImage).currentDockerImage(dockerImage)
+                .wantedVespaVersion(vespaVersion).currentVespaVersion(vespaVersion)
+                .wantedRestartGeneration(1).currentRestartGeneration(1);
+
+        NodeAgentContext context = createContext(specBuilder.build());
+        NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true, Duration.ofSeconds(30));
+        mockGetContainer(dockerImage, ContainerResources.from(0, 2, 16), true);
+
+        when(storageMaintainer.getDiskUsageFor(any())).thenReturn(Optional.of(201326592000L));
+
+        InOrder inOrder = inOrder(orchestrator, dockerOperations);
+
+        nodeAgent.doConverge(context);
+
+        nodeAgent.converge(context);
+        inOrder.verify(orchestrator, never()).suspend(any(String.class));
+        inOrder.verify(dockerOperations, never()).updateContainer(eq(context), any());
+        inOrder.verify(dockerOperations, never()).removeContainer(any(), any());
+        inOrder.verify(orchestrator, never()).resume(any(String.class));
+    }
+
     private void verifyThatContainerIsStopped(NodeState nodeState, Optional<ApplicationId> owner) {
         NodeSpec.Builder nodeBuilder = new NodeSpec.Builder()
                 .resources(resources)
@@ -672,11 +729,28 @@ public class NodeAgentImplTest {
     }
 
     private NodeAgentImpl makeNodeAgent(DockerImage dockerImage, boolean isRunning) {
+        return makeNodeAgent(dockerImage, isRunning, Duration.ofSeconds(-1));
+    }
+
+    private NodeAgentImpl makeNodeAgent(DockerImage dockerImage, boolean isRunning, Duration warmUpDuration) {
         mockGetContainer(dockerImage, isRunning);
+        doAnswer(invoc -> {
+            NodeAgentContext context = invoc.getArgument(0, NodeAgentContext.class);
+            ContainerResources resources = invoc.getArgument(2, ContainerResources.class);
+            mockGetContainer(context.node().wantedDockerImage().get(), resources, true);
+            return null;
+        }).when(dockerOperations).createContainer(any(), any(), any());
+
+        doAnswer(invoc -> {
+            NodeAgentContext context = invoc.getArgument(0, NodeAgentContext.class);
+            ContainerResources resources = invoc.getArgument(1, ContainerResources.class);
+            mockGetContainer(context.node().wantedDockerImage().get(), resources, true);
+            return null;
+        }).when(dockerOperations).updateContainer(any(), any());
 
         return new NodeAgentImpl(contextSupplier, nodeRepository, orchestrator, dockerOperations,
                 storageMaintainer, flagSource, Optional.of(credentialsMaintainer), Optional.of(aclMaintainer),
-                Optional.of(healthChecker));
+                Optional.of(healthChecker), clock, warmUpDuration);
     }
 
     private void mockGetContainer(DockerImage dockerImage, boolean isRunning) {
@@ -703,4 +777,8 @@ public class NodeAgentImplTest {
     private NodeAgentContext createContext(NodeSpec nodeSpec) {
         return new NodeAgentContextImpl.Builder(nodeSpec).build();
     }
+
+    private NodeSpec.Builder nodeBuilder(NodeState state) {
+        return NodeSpec.Builder.testSpec(hostName, state).resources(resources);
+    }
 }
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/DefaultEnvWriterTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/DefaultEnvWriterTest.java
index 5d687c06e94..a2457266560 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/DefaultEnvWriterTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/DefaultEnvWriterTest.java
@@ -1,6 +1,7 @@
 // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.node.admin.task.util;
 
+import com.yahoo.vespa.hosted.node.admin.component.TaskContext;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.TemporaryFolder;
@@ -9,11 +10,16 @@ import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.util.logging.Logger;
 
 import static java.nio.file.StandardCopyOption.REPLACE_EXISTING;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
 
 /**
  * @author bjorncs
@@ -23,8 +29,10 @@ public class DefaultEnvWriterTest {
     @Rule
     public TemporaryFolder temporaryFolder = new TemporaryFolder();
 
-    private static final Path EXAMPLE_FILE = Paths.get(DefaultEnvWriterTest.class.getResource("/default-env-example.txt").getFile());
-    private static final Path EXPECTED_RESULT_FILE = Paths.get(DefaultEnvWriterTest.class.getResource("/default-env-rewritten.txt").getFile());
+    private static final Path EXAMPLE_FILE = Paths.get("src/test/resources/default-env-example.txt");
+    private static final Path EXPECTED_RESULT_FILE = Paths.get("src/test/resources/default-env-rewritten.txt");
+
+    private final TaskContext context = mock(TaskContext.class);
 
     @Test
     public void default_env_is_correctly_rewritten() throws IOException {
@@ -36,14 +44,16 @@ public class DefaultEnvWriterTest {
         writer.addFallback("VESPA_CONFIGSERVER", "new-fallback-configserver");
         writer.addOverride("VESPA_TLS_CONFIG_FILE", "/override/path/to/config.file");
 
-        boolean modified = writer.updateFile(tempFile);
+        boolean modified = writer.updateFile(context, tempFile);
 
         assertTrue(modified);
         assertEquals(Files.readString(EXPECTED_RESULT_FILE), Files.readString(tempFile));
+        verify(context, times(1)).log(any(Logger.class), any(String.class));
 
-        modified = writer.updateFile(tempFile);
+        modified = writer.updateFile(context, tempFile);
         assertFalse(modified);
         assertEquals(Files.readString(EXPECTED_RESULT_FILE), Files.readString(tempFile));
+        verify(context, times(1)).log(any(Logger.class), any(String.class));
     }
 
     @Test
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPathTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPathTest.java
index 3b839f7f446..3159689c22e 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPathTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPathTest.java
@@ -3,6 +3,7 @@
 package com.yahoo.vespa.hosted.node.admin.task.util.file;
 
 import com.yahoo.vespa.test.file.TestFileSystem;
+import org.junit.ComparisonFailure;
 import org.junit.Test;
 
 import java.nio.file.FileSystem;
@@ -13,6 +14,7 @@ import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
 
 /**
  * @author hakonhall
@@ -124,4 +126,44 @@ public class UnixPathTest {
         assertFalse(dir1 + " deleted recursively", Files.exists(dir1));
     }
 
+    @Test
+    public void atomicWrite() {
+        var path = new UnixPath(fs.getPath("/dir/foo"));
+        path.createParents();
+        path.writeUtf8File("bar");
+        path.atomicWriteUt8("bar v2");
+        assertEquals("bar v2", path.readUtf8File());
+    }
+
+    @Test
+    public void testParentAndFilename() {
+        var absolutePath = new UnixPath("/foo/bar");
+        assertEquals("/foo", absolutePath.getParent().toString());
+        assertEquals("bar", absolutePath.getFilename());
+
+        var pathWithoutSlash = new UnixPath("foo");
+        assertRuntimeException(IllegalStateException.class, "Path has no parent directory: 'foo'", () -> pathWithoutSlash.getParent());
+        assertEquals("foo", pathWithoutSlash.getFilename());
+
+        var pathWithSlash = new UnixPath("/foo");
+        assertEquals("/", pathWithSlash.getParent().toString());
+        assertEquals("foo", pathWithSlash.getFilename());
+
+        assertRuntimeException(IllegalStateException.class, "Path has no parent directory: '/'", () -> new UnixPath("/").getParent());
+        assertRuntimeException(IllegalStateException.class, "Path has no filename: '/'", () -> new UnixPath("/").getFilename());
+    }
+
+    private <T extends RuntimeException> void assertRuntimeException(Class<T> baseClass, String message, Runnable runnable) {
+        try {
+            runnable.run();
+            fail("No exception was thrown");
+        } catch (RuntimeException e) {
+            if (!baseClass.isInstance(e)) {
+                throw new ComparisonFailure("Exception class mismatch", baseClass.getName(), e.getClass().getName());
+            }
+
+            assertEquals(message, e.getMessage());
+        }
+    }
+
 }
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/systemd/SystemCtlTesterTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/systemd/SystemCtlTesterTest.java
new file mode 100644
index 00000000000..1c3a6f6a412
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/systemd/SystemCtlTesterTest.java
@@ -0,0 +1,52 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.systemd;
+
+import com.yahoo.vespa.hosted.node.admin.component.TestTaskContext;
+import com.yahoo.vespa.hosted.node.admin.task.util.process.TestTerminal;
+import org.junit.Test;
+
+import java.util.List;
+import java.util.function.Function;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author freva
+ */
+public class SystemCtlTesterTest {
+
+    private static final String unit = "my-unit";
+    private final TestTerminal terminal = new TestTerminal();
+    private final SystemCtlTester systemCtl = new SystemCtlTester(terminal);
+    private final TestTaskContext context = new TestTaskContext();
+
+    @Test
+    public void return_expectations() {
+        assertSystemCtlMethod(sct -> sct.expectEnable(unit), sc -> sc.enable(unit).converge(context));
+        assertSystemCtlMethod(sct -> sct.expectDisable(unit), sc -> sc.disable(unit).converge(context));
+        assertSystemCtlMethod(sct -> sct.expectStart(unit), sc -> sc.start(unit).converge(context));
+        assertSystemCtlMethod(sct -> sct.expectStop(unit), sc -> sc.stop(unit).converge(context));
+        assertSystemCtlMethod(sct -> sct.expectServiceExists(unit), sc -> sc.serviceExists(context, unit));
+        assertSystemCtlMethod(sct -> sct.expectIsActive(unit), sc -> sc.isActive(context, unit));
+    }
+
+    @Test
+    public void void_tests() {
+        systemCtl.expectRestart(unit);
+        systemCtl.restart(unit).converge(context);
+        terminal.verifyAllCommandsExecuted();
+
+        systemCtl.expectDaemonReload();
+        systemCtl.daemonReload(context);
+        terminal.verifyAllCommandsExecuted();
+    }
+
+    private void assertSystemCtlMethod(Function<SystemCtlTester, SystemCtlTester.Expectation> systemCtlTesterExpectationFunction,
+                                       Function<SystemCtl, Boolean> systemCtlFunction) {
+        List.of(true, false).forEach(wantedReturnValue -> {
+            systemCtlTesterExpectationFunction.apply(systemCtl).andReturn(wantedReturnValue);
+            assertEquals(wantedReturnValue, systemCtlFunction.apply(systemCtl));
+            terminal.verifyAllCommandsExecuted();
+        });
+    }
+}
\ No newline at end of file
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumPackageNameTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumPackageNameTest.java
index 01664f5c22b..64e2997d486 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumPackageNameTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumPackageNameTest.java
@@ -56,7 +56,7 @@ public class YumPackageNameTest {
                 "1.el7",
                 null,
                 "docker-engine-selinux-1.12.6-1.el7",
-                null);
+                "0:docker-engine-selinux-1.12.6-1.el7.*");
 
         // name-ver-rel.arch
         verifyPackageName("docker-engine-selinux-1.12.6-1.el7.x86_64",
@@ -66,7 +66,7 @@ public class YumPackageNameTest {
                 "1.el7",
                 "x86_64",
                 "docker-engine-selinux-1.12.6-1.el7.x86_64",
-                null);
+                "0:docker-engine-selinux-1.12.6-1.el7.*");
 
         // name-epoch:ver-rel.arch
         verifyPackageName(
@@ -112,7 +112,7 @@ public class YumPackageNameTest {
                 yumPackageName.toVersionLockName();
                 fail();
             } catch (IllegalStateException e) {
-                assertThat(e.getMessage(), containsStringIgnoringCase("epoch is missing"));
+                assertThat(e.getMessage(), containsStringIgnoringCase("Version is missing "));
             }
         } else {
             assertEquals(toVersionName, yumPackageName.toVersionLockName());
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTest.java
index c7e2885a907..dc9c036b96b 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTest.java
@@ -9,7 +9,6 @@ import org.junit.Test;
 
 import java.util.Optional;
 
-import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotNull;
@@ -74,41 +73,22 @@ public class YumTest {
     }
 
     @Test
-    public void testArrayConversion() {
-        YumPackageName[] expected = new YumPackageName[] { new YumPackageName.Builder("1").build() };
-        assertArrayEquals(expected, Yum.toYumPackageNameArray("1"));
-
-        YumPackageName[] expected2 = new YumPackageName[] {
-                new YumPackageName.Builder("1").build(),
-                new YumPackageName.Builder("2").build()
-        };
-        assertArrayEquals(expected2, Yum.toYumPackageNameArray("1", "2"));
-
-        YumPackageName[] expected3 = new YumPackageName[] {
-                new YumPackageName.Builder("1").build(),
-                new YumPackageName.Builder("2").build(),
-                new YumPackageName.Builder("3").build()
-        };
-        assertArrayEquals(expected3, Yum.toYumPackageNameArray("1", "2", "3"));
-    }
-
-    @Test
     public void testAlreadyInstalled() {
         terminal.expectCommand(
-                "yum install --assumeyes --enablerepo=repo1 --enablerepo=repo2 package-1 package-2 2>&1",
+                "yum install --assumeyes --enablerepo=repo1 --enablerepo=repo2 --setopt skip_missing_names_on_install=False package-1 package-2 2>&1",
                 0,
                 "foobar\nNothing to do\n");
 
         assertFalse(yum
                 .install("package-1", "package-2")
-                .enableRepos("repo1", "repo2")
+                .enableRepo("repo1", "repo2")
                 .converge(taskContext));
     }
 
     @Test
     public void testAlreadyUpgraded() {
         terminal.expectCommand(
-                "yum upgrade --assumeyes package-1 package-2 2>&1",
+                "yum upgrade --assumeyes --setopt skip_missing_names_on_update=False package-1 package-2 2>&1",
                 0,
                 "foobar\nNo packages marked for update\n");
 
@@ -132,7 +112,7 @@ public class YumTest {
     @Test
     public void testInstall() {
         terminal.expectCommand(
-                "yum install --assumeyes package-1 package-2 2>&1",
+                "yum install --assumeyes --setopt skip_missing_names_on_install=False package-1 package-2 2>&1",
                 0,
                 "installing, installing");
 
@@ -144,13 +124,13 @@ public class YumTest {
     @Test
     public void testInstallWithEnablerepo() {
         terminal.expectCommand(
-                "yum install --assumeyes --enablerepo=repo-name package-1 package-2 2>&1",
+                "yum install --assumeyes --enablerepo=repo-name --setopt skip_missing_names_on_install=False package-1 package-2 2>&1",
                 0,
                 "installing, installing");
 
         assertTrue(yum
                 .install("package-1", "package-2")
-                .enableRepos("repo-name")
+                .enableRepo("repo-name")
                 .converge(taskContext));
     }
 
@@ -160,14 +140,13 @@ public class YumTest {
                 0,
                 "Repository chef_rpms-release is listed more than once in the configuration\n" +
                         "0:chef-12.21.1-1.el7.*\n");
-        terminal.expectCommand("yum versionlock add \"0:package-1-0.10-654.el7.*\" 2>&1");
+        terminal.expectCommand("yum versionlock add --assumeyes \"0:package-1-0.10-654.el7.*\" 2>&1");
         terminal.expectCommand(
                 "yum install --assumeyes 0:package-1-0.10-654.el7.x86_64 2>&1",
                 0,
                 "installing");
 
-        assertTrue(yum.installFixedVersion(taskContext,
-                YumPackageName.fromString("0:package-1-0.10-654.el7.x86_64")));
+        assertTrue(yum.installFixedVersion(YumPackageName.fromString("0:package-1-0.10-654.el7.x86_64")).converge(taskContext));
     }
 
     @Test
@@ -180,15 +159,18 @@ public class YumTest {
 
         terminal.expectCommand("yum versionlock delete \"0:package-1-0.1-8.el7.*\" 2>&1");
 
-        terminal.expectCommand("yum versionlock add \"0:package-1-0.10-654.el7.*\" 2>&1");
+        terminal.expectCommand("yum versionlock add --assumeyes --enablerepo=somerepo \"0:package-1-0.10-654.el7.*\" 2>&1");
 
         terminal.expectCommand(
-                "yum install --assumeyes 0:package-1-0.10-654.el7 2>&1",
+                "yum install --assumeyes --enablerepo=somerepo 0:package-1-0.10-654.el7 2>&1",
                 0,
                 "Nothing to do\n");
 
 
-        assertTrue(yum.installFixedVersion(taskContext, YumPackageName.fromString("0:package-1-0.10-654.el7")));
+        assertTrue(yum
+                .installFixedVersion(YumPackageName.fromString("0:package-1-0.10-654.el7"))
+                .enableRepo("somerepo")
+                .converge(taskContext));
     }
 
     @Test
@@ -203,7 +185,7 @@ public class YumTest {
                 0,
                 "Nothing to do\n");
 
-        assertFalse(yum.installFixedVersion(taskContext, YumPackageName.fromString("0:package-1-0.10-654.el7")));
+        assertFalse(yum.installFixedVersion(YumPackageName.fromString("0:package-1-0.10-654.el7")).converge(taskContext));
     }
 
     @Test
@@ -222,18 +204,19 @@ public class YumTest {
 
         terminal.expectCommand("yum downgrade --assumeyes 0:package-1-0.10-654.el7 2>&1");
 
-        assertTrue(yum.installFixedVersion(taskContext, YumPackageName.fromString("0:package-1-0.10-654.el7")));
+        assertTrue(yum.installFixedVersion(YumPackageName.fromString("0:package-1-0.10-654.el7")).converge(taskContext));
     }
 
     @Test(expected = ChildProcessFailureException.class)
     public void testFailedInstall() {
         terminal.expectCommand(
-                "yum install --assumeyes --enablerepo=repo-name package-1 package-2 2>&1",
+                "yum install --assumeyes --enablerepo=repo-name --setopt skip_missing_names_on_install=False package-1 package-2 2>&1",
                 1,
                 "error");
 
-        yum.install("package-1", "package-2")
-                .enableRepos("repo-name")
+        yum
+                .install("package-1", "package-2")
+                .enableRepo("repo-name")
                 .converge(taskContext);
         fail();
     }
@@ -241,7 +224,7 @@ public class YumTest {
     @Test
     public void testUnknownPackages() {
         terminal.expectCommand(
-                "yum install --assumeyes package-1 package-2 package-3 2>&1",
+                "yum install --assumeyes --setopt skip_missing_names_on_install=False package-1 package-2 package-3 2>&1",
                 0,
                 "Loaded plugins: fastestmirror, langpacks\n" +
                         "Loading mirror speeds from cached hostfile\n" +
@@ -249,10 +232,9 @@ public class YumTest {
                         "No package package-2 available.\n" +
                         "Nothing to do\n");
 
-        Yum.GenericYumCommand install = yum.install("package-1", "package-2", "package-3");
-
+        var command = yum.install("package-1", "package-2", "package-3");
         try {
-            install.converge(taskContext);
+            command.converge(taskContext);
             fail();
         } catch (Exception e) {
             assertNotNull(e.getCause());
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTesterTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTesterTest.java
new file mode 100644
index 00000000000..ef380046b75
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/yum/YumTesterTest.java
@@ -0,0 +1,61 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.task.util.yum;
+
+import com.yahoo.vespa.hosted.node.admin.component.TestTaskContext;
+import com.yahoo.vespa.hosted.node.admin.task.util.process.TestTerminal;
+import org.junit.Test;
+
+import java.util.List;
+import java.util.Optional;
+import java.util.function.Function;
+import java.util.stream.Stream;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author freva
+ */
+public class YumTesterTest {
+
+    private static final String[] packages = {"pkg1", "pkg2"};
+    private static final String[] repos = {"repo1", "repo2"};
+    private static final YumPackageName minimalPackage = YumPackageName.fromString("my-pkg-1.13.1-0.el7");
+    private static final YumPackageName fullPackage = YumPackageName.fromString("2:my-pkg-1.13.1-0.el7.x86_64");
+
+    private final TestTerminal terminal = new TestTerminal();
+    private final YumTester yum = new YumTester(terminal);
+    private final TestTaskContext context = new TestTaskContext();
+
+    @Test
+    public void generic_yum_methods() {
+        assertYumMethod(yum -> yum.expectInstall(packages).withEnableRepo(repos),
+                yum -> yum.install(List.of(packages)).enableRepo(repos).converge(context));
+
+        assertYumMethod(yum -> yum.expectUpdate(packages).withEnableRepo(repos),
+                yum -> yum.upgrade(List.of(packages)).enableRepo(repos).converge(context));
+
+        assertYumMethod(yum -> yum.expectRemove(packages).withEnableRepo(repos),
+                yum -> yum.remove(List.of(packages)).enableRepo(repos).converge(context));
+
+        assertYumMethod(yum -> yum.expectInstallFixedVersion(minimalPackage.toName()).withEnableRepo(repos),
+                yum -> yum.installFixedVersion(minimalPackage).enableRepo(repos).converge(context));
+    }
+
+    @Test
+    public void expect_query_installed() {
+        Stream.of(minimalPackage, fullPackage, null).forEach(pkg -> {
+            yum.expectQueryInstalled(packages[0]).andReturn(pkg);
+            assertEquals(Optional.ofNullable(pkg), yum.queryInstalled(context, packages[0]));
+            terminal.verifyAllCommandsExecuted();
+        });
+    }
+
+    private void assertYumMethod(Function<YumTester, YumTester.GenericYumCommandExpectation> yumTesterExpectationFunction,
+                                       Function<Yum, Boolean> yumFunction) {
+        List.of(true, false).forEach(wantedReturnValue -> {
+            yumTesterExpectationFunction.apply(yum).andReturn(wantedReturnValue);
+            assertEquals(wantedReturnValue, yumFunction.apply(yum));
+            terminal.verifyAllCommandsExecuted();
+        });
+    }
+}
\ No newline at end of file
diff --git a/node-admin/vespa-node-admin.spec b/node-admin/vespa-node-admin.spec
index cf8ddf51553..b48e5b1980d 100644
--- a/node-admin/vespa-node-admin.spec
+++ b/node-admin/vespa-node-admin.spec
@@ -9,7 +9,7 @@
 Name:           vespa-node-admin
 Version:        %version
 Release:        1%{?dist}
-BuildArch:      noarch
+BuildArch:      x86_64
 Summary:        Vespa Node Admin
 Group:          Applications/Databases
 License:        Commercial
diff --git a/node-repository/pom.xml b/node-repository/pom.xml
index 86a56c14c52..75bff479f42 100644
--- a/node-repository/pom.xml
+++ b/node-repository/pom.xml
@@ -89,6 +89,12 @@
             <groupId>org.apache.httpcomponents</groupId>
             <artifactId>httpclient</artifactId>
         </dependency>
+        <dependency>
+            <groupId>com.yahoo.vespa</groupId>
+            <artifactId>http-utils</artifactId>
+            <version>${project.version}</version>
+            <scope>compile</scope>
+        </dependency>
 
         <!-- test -->
         <dependency>
diff --git a/node-repository/src/main/config/node-repository.xml b/node-repository/src/main/config/node-repository.xml
index 274be6d572a..186f052a274 100644
--- a/node-repository/src/main/config/node-repository.xml
+++ b/node-repository/src/main/config/node-repository.xml
@@ -1,6 +1,8 @@
 <!-- services.xml snippet for the node repository. Included in config server services.xml if the package is installed-->
 <!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
 <component id="com.yahoo.vespa.hosted.provision.provisioning.InfraDeployerImpl" bundle="node-repository"/>
+<component id="com.yahoo.vespa.hosted.provision.autoscale.NodeMetricsFetcher" bundle="node-repository"/>
+<component id="com.yahoo.vespa.hosted.provision.autoscale.NodeMetricsDb" bundle="node-repository"/>
 <component id="com.yahoo.vespa.hosted.provision.provisioning.NodeRepositoryProvisioner" bundle="node-repository" />
 <component id="NodeRepository" class="com.yahoo.vespa.hosted.provision.NodeRepository" bundle="node-repository"/>
 <component id="com.yahoo.vespa.hosted.provision.maintenance.NodeRepositoryMaintenance" bundle="node-repository"/>
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java
index 7c0e0e7868b..d42efd7ba29 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java
@@ -83,7 +83,7 @@ public final class Node {
         this.reservedTo = Objects.requireNonNull(reservedTo, "reservedTo cannot be null");
 
         if (state == State.active)
-            requireNonEmpty(ipConfig.primary(), "An active node must have at least one valid IP address");
+            requireNonEmpty(ipConfig.primary(), "Active node " + hostname + " must have at least one valid IP address");
 
         if (parentHostname.isPresent()) {
             if (!ipConfig.pool().asSet().isEmpty()) throw new IllegalArgumentException("A child node cannot have an IP address pool");
@@ -375,8 +375,6 @@ public final class Node {
                        .deviation();
     }
 
-
-
     @Override
     public boolean equals(Object o) {
         if (this == o) return true;
@@ -392,15 +390,15 @@ public final class Node {
 
     @Override
     public String toString() {
-        return state + " node " +
+        return state +
+               ( parentHostname.isPresent() ? " child node " : " host " ) +
                hostname +
-               (allocation.map(allocation1 -> " " + allocation1).orElse("")) +
-               (parentHostname.map(parent -> " [on: " + parent + "]").orElse(""));
+               ( allocation.isPresent() ? " " + allocation.get() : "");
     }
 
     public enum State {
 
-        /** This node has been requested (from OpenStack) but is not yet ready for use */
+        /** This host has been requested (from OpenStack) but is not yet ready for use */
         provisioned,
 
         /** This node is free and ready for use */
@@ -426,12 +424,20 @@ public final class Node {
          * This state follows the same rules as failed except that it will never be automatically moved out of
          * this state.
          */
-        parked;
+        parked,
+
+        /** This host has previously been in use but is now removed. */
+        deprovisioned;
 
         /** Returns whether this is a state where the node is assigned to an application */
         public boolean isAllocated() {
-            return this == reserved || this == active || this == inactive || this == failed || this == parked;
+            return allocatedStates().contains(this);
         }
+
+        public static Set<State> allocatedStates() {
+            return Set.of(reserved, active, inactive, failed, parked);
+        }
+
     }
 
     /** The mean and mean deviation (squared difference) of a bunch of numbers */
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeList.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeList.java
index 21bde5900eb..93e5b160524 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeList.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeList.java
@@ -50,6 +50,11 @@ public class NodeList implements Iterable<Node> {
         return filter(node -> node.allocation().get().membership().retired());
     }
 
+    /** Returns the subset of nodes which are removable */
+    public NodeList removable() {
+        return filter(node -> node.allocation().get().isRemovable());
+    }
+
     /** Returns the subset of nodes having exactly the given resources */
     public NodeList resources(NodeResources resources) { return filter(node -> node.flavor().resources().equals(resources)); }
 
@@ -75,6 +80,11 @@ public class NodeList implements Iterable<Node> {
                               !node.status().vespaVersion().get().equals(node.allocation().get().membership().cluster().vespaVersion()));
     }
 
+    /** Returns the subset of nodes that are currently changing their OS version to given version */
+    public NodeList changingOsVersionTo(Version version) {
+        return filter(node -> node.status().osVersion().changingTo(version));
+    }
+
     /** Returns the subset of nodes that are currently changing their OS version */
     public NodeList changingOsVersion() {
         return filter(node -> node.status().osVersion().changing());
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java
index 7d875434e1c..40bccf22434 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java
@@ -1,4 +1,4 @@
-// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.provision;
 
 import com.google.inject.Inject;
@@ -16,7 +16,10 @@ import com.yahoo.config.provisioning.NodeRepositoryConfig;
 import com.yahoo.transaction.Mutex;
 import com.yahoo.transaction.NestedTransaction;
 import com.yahoo.vespa.curator.Curator;
+import com.yahoo.vespa.hosted.provision.Node.State;
+import com.yahoo.vespa.hosted.provision.applications.Applications;
 import com.yahoo.vespa.hosted.provision.lb.LoadBalancer;
+import com.yahoo.vespa.hosted.provision.lb.LoadBalancerId;
 import com.yahoo.vespa.hosted.provision.lb.LoadBalancerInstance;
 import com.yahoo.vespa.hosted.provision.lb.LoadBalancerList;
 import com.yahoo.vespa.hosted.provision.maintenance.InfrastructureVersions;
@@ -50,6 +53,7 @@ import java.util.Optional;
 import java.util.Set;
 import java.util.TreeSet;
 import java.util.function.BiFunction;
+import java.util.function.Predicate;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
@@ -75,10 +79,10 @@ import java.util.stream.Stream;
  * @author bratseth
  */
 // Node state transitions:
-// 1) (new) - > provisioned -> (dirty ->) ready -> reserved -> active -> inactive -> dirty -> ready
+// 1) (new) | deprovisioned - > provisioned -> (dirty ->) ready -> reserved -> active -> inactive -> dirty -> ready
 // 2) inactive -> reserved | parked
 // 3) reserved -> dirty
-// 3) * -> failed | parked -> dirty | active | (removed)
+// 3) * -> failed | parked -> dirty | active | deprovisioned
 // Nodes have an application assigned when in states reserved, active and inactive.
 // Nodes might have an application assigned in dirty.
 public class NodeRepository extends AbstractComponent {
@@ -93,6 +97,7 @@ public class NodeRepository extends AbstractComponent {
     private final FirmwareChecks firmwareChecks;
     private final DockerImages dockerImages;
     private final JobControl jobControl;
+    private final Applications applications;
 
     /**
      * Creates a node repository from a zookeeper provider.
@@ -119,17 +124,21 @@ public class NodeRepository extends AbstractComponent {
         this.firmwareChecks = new FirmwareChecks(db, clock);
         this.dockerImages = new DockerImages(db, dockerImage);
         this.jobControl = new JobControl(db);
+        this.applications = new Applications();
 
         // read and write all nodes to make sure they are stored in the latest version of the serialized format
-        for (Node.State state : Node.State.values())
+        for (State state : State.values())
+            // TODO(mpolden): Add per-node locking. In its current state this may collide with other callers making
+            //                node state changes. Example: A redeployment on another config server which moves a node
+            //                to another state while this is constructed.
             db.writeTo(state, db.getNodes(state), Agent.system, Optional.empty());
     }
 
     /** Returns the curator database client used by this */
     public CuratorDatabaseClient database() { return db; }
 
-    /** Returns the Docker image to use for nodes in this */
-    public DockerImage dockerImage(NodeType nodeType) { return dockerImages.dockerImageFor(nodeType); }
+    /** Returns the Docker image to use for given node */
+    public DockerImage dockerImage(Node node) { return dockerImages.dockerImageFor(node.type()); }
 
     /** @return The name resolver used to resolve hostname and ip addresses */
     public NameResolver nameResolver() { return nameResolver; }
@@ -149,6 +158,9 @@ public class NodeRepository extends AbstractComponent {
     /** Returns the status of maintenance jobs managed by this. */
     public JobControl jobControl() { return jobControl; }
 
+    /** Returns this node repo's view of the applications deployed to it */
+    public Applications applications() { return applications; }
+
     // ---------------- Query API ----------------------------------------------------------------
 
     /**
@@ -158,7 +170,7 @@ public class NodeRepository extends AbstractComponent {
      * @param inState the states the node may be in. If no states are given, it will be returned from any state
      * @return the node, or empty if it was not found in any of the given states
      */
-    public Optional<Node> getNode(String hostname, Node.State ... inState) {
+    public Optional<Node> getNode(String hostname, State ... inState) {
         return db.getNode(hostname, inState);
     }
 
@@ -168,7 +180,7 @@ public class NodeRepository extends AbstractComponent {
      * @param inState the states to return nodes from. If no states are given, all nodes of the given type are returned
      * @return the node, or empty if it was not found in any of the given states
      */
-    public List<Node> getNodes(Node.State ... inState) {
+    public List<Node> getNodes(State ... inState) {
         return new ArrayList<>(db.getNodes(inState));
     }
     /**
@@ -178,7 +190,7 @@ public class NodeRepository extends AbstractComponent {
      * @param inState the states to return nodes from. If no states are given, all nodes of the given type are returned
      * @return the node, or empty if it was not found in any of the given states
      */
-    public List<Node> getNodes(NodeType type, Node.State ... inState) {
+    public List<Node> getNodes(NodeType type, State ... inState) {
         return db.getNodes(inState).stream().filter(node -> node.type().equals(type)).collect(Collectors.toList());
     }
 
@@ -187,6 +199,11 @@ public class NodeRepository extends AbstractComponent {
         return NodeList.copyOf(getNodes());
     }
 
+    /** Returns a filterable list of all nodes of an application */
+    public NodeList list(ApplicationId application) {
+        return NodeList.copyOf(getNodes(application));
+    }
+
     /** Returns a locked list of all nodes in this repository */
     public LockedNodeList list(Mutex lock) {
         return new LockedNodeList(getNodes(), lock);
@@ -194,17 +211,26 @@ public class NodeRepository extends AbstractComponent {
 
     /** Returns a filterable list of all load balancers in this repository */
     public LoadBalancerList loadBalancers() {
-        return LoadBalancerList.copyOf(database().readLoadBalancers().values());
+        return loadBalancers((ignored) -> true);
+    }
+
+    /** Returns a filterable list of load balancers belonging to given application */
+    public LoadBalancerList loadBalancers(ApplicationId application) {
+        return loadBalancers((id) -> id.application().equals(application));
     }
 
-    public List<Node> getNodes(ApplicationId id, Node.State ... inState) { return db.getNodes(id, inState); }
-    public List<Node> getInactive() { return db.getNodes(Node.State.inactive); }
-    public List<Node> getFailed() { return db.getNodes(Node.State.failed); }
+    private LoadBalancerList loadBalancers(Predicate<LoadBalancerId> predicate) {
+        return LoadBalancerList.copyOf(db.readLoadBalancers(predicate).values());
+    }
+
+    public List<Node> getNodes(ApplicationId id, State ... inState) { return db.getNodes(id, inState); }
+    public List<Node> getInactive() { return db.getNodes(State.inactive); }
+    public List<Node> getFailed() { return db.getNodes(State.failed); }
 
     /**
      * Returns the ACL for the node (trusted nodes, networks and ports)
      */
-    private NodeAcl getNodeAcl(Node node, NodeList candidates, LoadBalancerList loadBalancers) {
+    private NodeAcl getNodeAcl(Node node, NodeList candidates) {
         Set<Node> trustedNodes = new TreeSet<>(Comparator.comparing(Node::hostname));
         Set<Integer> trustedPorts = new LinkedHashSet<>();
         Set<String> trustedNetworks = new LinkedHashSet<>();
@@ -221,10 +247,10 @@ public class NodeRepository extends AbstractComponent {
         candidates.parentOf(node).ifPresent(trustedNodes::add);
         node.allocation().ifPresent(allocation -> {
             trustedNodes.addAll(candidates.owner(allocation.owner()).asList());
-            loadBalancers.owner(allocation.owner()).asList().stream()
-                         .map(LoadBalancer::instance)
-                         .map(LoadBalancerInstance::networks)
-                         .forEach(trustedNetworks::addAll);
+            loadBalancers(allocation.owner()).asList().stream()
+                                             .map(LoadBalancer::instance)
+                                             .map(LoadBalancerInstance::networks)
+                                             .forEach(trustedNetworks::addAll);
         });
 
         switch (node.type()) {
@@ -240,7 +266,7 @@ public class NodeRepository extends AbstractComponent {
                 node.allocation().ifPresent(allocation ->
                         trustedNodes.addAll(candidates.parentsOf(candidates.owner(allocation.owner()).asList()).asList()));
 
-                if (node.state() == Node.State.ready) {
+                if (node.state() == State.ready) {
                     // Tenant nodes in state ready, trust:
                     // - All tenant nodes in zone. When a ready node is allocated to a an application there's a brief
                     //   window where current ACLs have not yet been applied on the node. To avoid service disruption
@@ -271,14 +297,14 @@ public class NodeRepository extends AbstractComponent {
                 // Controllers:
                 // - port 4443 (HTTPS + Athenz) from the world
                 // - port 443 (HTTPS + Okta) from the world
+                // - port 80 (HTTP) from the world - for redirect to HTTPS/443 only
                 trustedPorts.add(4443);
                 trustedPorts.add(443);
+                trustedPorts.add(80);
                 break;
 
             default:
-                throw new IllegalArgumentException(
-                        String.format("Don't know how to create ACL for node [hostname=%s type=%s]",
-                                      node.hostname(), node.type()));
+                illegal("Don't know how to create ACL for " + node + " of type " + node.type());
         }
 
         return new NodeAcl(node, trustedNodes, trustedNetworks, trustedPorts);
@@ -293,13 +319,12 @@ public class NodeRepository extends AbstractComponent {
      */
     public List<NodeAcl> getNodeAcls(Node node, boolean children) {
         NodeList candidates = list();
-        LoadBalancerList loadBalancers = loadBalancers();
         if (children) {
             return candidates.childrenOf(node).asList().stream()
-                             .map(childNode -> getNodeAcl(childNode, candidates, loadBalancers))
+                             .map(childNode -> getNodeAcl(childNode, candidates))
                              .collect(Collectors.collectingAndThen(Collectors.toList(), Collections::unmodifiableList));
         }
-        return Collections.singletonList(getNodeAcl(node, candidates, loadBalancers));
+        return Collections.singletonList(getNodeAcl(node, candidates));
     }
 
     public NodeFlavors getAvailableFlavors() {
@@ -311,9 +336,8 @@ public class NodeRepository extends AbstractComponent {
     /** Creates a new node object, without adding it to the node repo. If no IP address is given, it will be resolved */
     public Node createNode(String openStackId, String hostname, IP.Config ipConfig, Optional<String> parentHostname,
                            Flavor flavor, Optional<TenantName> reservedTo, NodeType type) {
-        if (ipConfig.primary().isEmpty()) { // TODO: Remove this. Only test code hits this path
+        if (ipConfig.primary().isEmpty()) // TODO: Remove this. Only test code hits this path
             ipConfig = ipConfig.with(nameResolver.getAllByNameOrThrow(hostname));
-        }
         return Node.create(openStackId, ipConfig, hostname, parentHostname, Optional.empty(), flavor, reservedTo, type);
     }
 
@@ -324,38 +348,55 @@ public class NodeRepository extends AbstractComponent {
     /** Adds a list of newly created docker container nodes to the node repository as <i>reserved</i> nodes */
     public List<Node> addDockerNodes(LockedNodeList nodes) {
         for (Node node : nodes) {
-            if (!node.flavor().getType().equals(Flavor.Type.DOCKER_CONTAINER)) {
-                throw new IllegalArgumentException("Cannot add " + node.hostname() + ": This is not a docker node");
-            }
-            if (!node.allocation().isPresent()) {
-                throw new IllegalArgumentException("Cannot add " + node.hostname() + ": Docker containers needs to be allocated");
-            }
+            if ( ! node.flavor().getType().equals(Flavor.Type.DOCKER_CONTAINER))
+                illegal("Cannot add " + node + ": This is not a docker node");
+            if ( ! node.allocation().isPresent())
+                illegal("Cannot add " + node + ": Docker containers needs to be allocated");
             Optional<Node> existing = getNode(node.hostname());
             if (existing.isPresent())
-                throw new IllegalArgumentException("Cannot add " + node.hostname() + ": A node with this name already exists (" +
-                                                   existing.get() + ", " + existing.get().history() + "). Node to be added: " +
-                                                   node + ", " + node.history());
+                illegal("Cannot add " + node + ": A node with this name already exists (" +
+                        existing.get() + ", " + existing.get().history() + "). Node to be added: " +
+                        node + ", " + node.history());
         }
-        return db.addNodesInState(nodes.asList(), Node.State.reserved);
+        return db.addNodesInState(nodes.asList(), State.reserved);
     }
 
-    /** Adds a list of (newly created) nodes to the node repository as <i>provisioned</i> nodes */
-    public List<Node> addNodes(List<Node> nodes) {
+    /**
+     * Adds a list of (newly created) nodes to the node repository as <i>provisioned</i> nodes.
+     * If any of the nodes already exists in the deprovisioned state, the new node will be merged
+     * with the history of that node.
+     */
+    public List<Node> addNodes(List<Node> nodes, Agent agent) {
         try (Mutex lock = lockUnallocated()) {
+            List<Node> nodesToAdd =  new ArrayList<>();
+            List<Node> nodesToRemove = new ArrayList<>();
             for (int i = 0; i < nodes.size(); i++) {
                 var node = nodes.get(i);
-                var message = "Cannot add " + node.hostname() + ": A node with this name already exists";
-
-                // Check for existing node
-                if (getNode(node.hostname()).isPresent()) throw new IllegalArgumentException(message);
 
-                // Check for duplicates in given list
+                // Check for duplicates
                 for (int j = 0; j < i; j++) {
-                    var other = nodes.get(j);
-                    if (node.equals(other)) throw new IllegalArgumentException(message);
+                    if (node.equals(nodes.get(j)))
+                        illegal("Cannot add nodes: " + node + " is duplicated in the argument list");
+                }
+
+                Optional<Node> existing = getNode(node.hostname());
+                if (existing.isPresent()) {
+                    if (existing.get().state() != State.deprovisioned)
+                        illegal("Cannot add " + node + ": A node with this name already exists");
+                    node = node.with(existing.get().history());
+                    node = node.with(existing.get().reports());
+                    node = node.with(node.status().withFailCount(existing.get().status().failCount()));
+                    if (existing.get().status().firmwareVerifiedAt().isPresent())
+                        node = node.with(node.status().withFirmwareVerifiedAt(existing.get().status().firmwareVerifiedAt().get()));
+                    nodesToRemove.add(existing.get());
                 }
+
+                nodesToAdd.add(node);
             }
-            return db.addNodesInState(IP.Config.verify(nodes, list(lock)), Node.State.provisioned);
+            List<Node> resultingNodes = new ArrayList<>();
+            resultingNodes.addAll(db.addNodesInState(IP.Config.verify(nodesToAdd, list(lock)), State.provisioned));
+            db.removeNodes(nodesToRemove);
+            return resultingNodes;
         }
     }
 
@@ -364,13 +405,13 @@ public class NodeRepository extends AbstractComponent {
         try (Mutex lock = lockUnallocated()) {
             List<Node> nodesWithResetFields = nodes.stream()
                     .map(node -> {
-                        if (node.state() != Node.State.provisioned && node.state() != Node.State.dirty)
-                            throw new IllegalArgumentException("Can not set " + node + " ready. It is not provisioned or dirty.");
+                        if (node.state() != State.provisioned && node.state() != State.dirty)
+                            illegal("Can not set " + node + " ready. It is not provisioned or dirty.");
                         return node.with(node.status().withWantToRetire(false).withWantToDeprovision(false));
                     })
                     .collect(Collectors.toList());
 
-            return db.writeTo(Node.State.ready, nodesWithResetFields, agent, Optional.of(reason));
+            return db.writeTo(State.ready, nodesWithResetFields, agent, Optional.of(reason));
         }
     }
 
@@ -378,18 +419,18 @@ public class NodeRepository extends AbstractComponent {
         Node nodeToReady = getNode(hostname).orElseThrow(() ->
                 new NoSuchNodeException("Could not move " + hostname + " to ready: Node not found"));
 
-        if (nodeToReady.state() == Node.State.ready) return nodeToReady;
+        if (nodeToReady.state() == State.ready) return nodeToReady;
         return setReady(Collections.singletonList(nodeToReady), agent, reason).get(0);
     }
 
     /** Reserve nodes. This method does <b>not</b> lock the node repository */
     public List<Node> reserve(List<Node> nodes) { 
-        return db.writeTo(Node.State.reserved, nodes, Agent.application, Optional.empty()); 
+        return db.writeTo(State.reserved, nodes, Agent.application, Optional.empty());
     }
 
     /** Activate nodes. This method does <b>not</b> lock the node repository */
     public List<Node> activate(List<Node> nodes, NestedTransaction transaction) {
-        return db.writeTo(Node.State.active, nodes, Agent.application, Optional.empty(), transaction);
+        return db.writeTo(State.active, nodes, Agent.application, Optional.empty(), transaction);
     }
 
     /**
@@ -409,7 +450,7 @@ public class NodeRepository extends AbstractComponent {
 
     public void deactivate(ApplicationId application, NestedTransaction transaction) {
         try (Mutex lock = lock(application)) {
-            deactivate(db.getNodes(application, Node.State.reserved, Node.State.active), transaction);
+            deactivate(db.getNodes(application, State.reserved, State.active), transaction);
         }
     }
 
@@ -419,7 +460,7 @@ public class NodeRepository extends AbstractComponent {
      * This method does <b>not</b> lock
      */
     public List<Node> deactivate(List<Node> nodes, NestedTransaction transaction) {
-        return db.writeTo(Node.State.inactive, nodes, Agent.application, Optional.empty(), transaction);
+        return db.writeTo(State.inactive, nodes, Agent.application, Optional.empty(), transaction);
     }
 
     /** Move nodes to the dirty state */
@@ -434,7 +475,7 @@ public class NodeRepository extends AbstractComponent {
      * @throws IllegalArgumentException if the node has hardware failure
      */
     public Node setDirty(Node node, Agent agent, String reason) {
-        return db.writeTo(Node.State.dirty, node, agent, Optional.of(reason));
+        return db.writeTo(State.dirty, node, agent, Optional.of(reason));
     }
 
     public List<Node> dirtyRecursively(String hostname, Agent agent, String reason) {
@@ -445,23 +486,20 @@ public class NodeRepository extends AbstractComponent {
                 (nodeToDirty.type().isDockerHost() ?
                         Stream.concat(list().childrenOf(hostname).asList().stream(), Stream.of(nodeToDirty)) :
                         Stream.of(nodeToDirty))
-                .filter(node -> node.state() != Node.State.dirty)
+                .filter(node -> node.state() != State.dirty)
                 .collect(Collectors.toList());
 
         List<String> hostnamesNotAllowedToDirty = nodesToDirty.stream()
-                .filter(node -> node.state() != Node.State.provisioned)
-                .filter(node -> node.state() != Node.State.failed)
-                .filter(node -> node.state() != Node.State.parked)
+                .filter(node -> node.state() != State.provisioned)
+                .filter(node -> node.state() != State.failed)
+                .filter(node -> node.state() != State.parked)
                 .map(Node::hostname)
                 .collect(Collectors.toList());
-        if (!hostnamesNotAllowedToDirty.isEmpty()) {
-            throw new IllegalArgumentException("Could not deallocate " + hostname + ": " +
-                    String.join(", ", hostnamesNotAllowedToDirty) + " must be in either provisioned, failed or parked state");
-        }
+        if ( ! hostnamesNotAllowedToDirty.isEmpty())
+            illegal("Could not deallocate " + nodeToDirty + ": " +
+                    hostnamesNotAllowedToDirty + " are not in states [provisioned, failed, parked]");
 
-        return nodesToDirty.stream()
-                .map(node -> setDirty(node, agent, reason))
-                .collect(Collectors.toList());
+        return nodesToDirty.stream().map(node -> setDirty(node, agent, reason)).collect(Collectors.toList());
     }
 
     /**
@@ -471,7 +509,7 @@ public class NodeRepository extends AbstractComponent {
      * @throws NoSuchNodeException if the node is not found
      */
     public Node fail(String hostname, Agent agent, String reason) {
-        return move(hostname, true, Node.State.failed, agent, Optional.of(reason));
+        return move(hostname, true, State.failed, agent, Optional.of(reason));
     }
 
     /**
@@ -480,7 +518,7 @@ public class NodeRepository extends AbstractComponent {
      * @return List of all the failed nodes in their new state
      */
     public List<Node> failRecursively(String hostname, Agent agent, String reason) {
-        return moveRecursively(hostname, Node.State.failed, agent, Optional.of(reason));
+        return moveRecursively(hostname, State.failed, agent, Optional.of(reason));
     }
 
     /**
@@ -490,7 +528,7 @@ public class NodeRepository extends AbstractComponent {
      * @throws NoSuchNodeException if the node is not found
      */
     public Node park(String hostname, boolean keepAllocation, Agent agent, String reason) {
-        return move(hostname, keepAllocation, Node.State.parked, agent, Optional.of(reason));
+        return move(hostname, keepAllocation, State.parked, agent, Optional.of(reason));
     }
 
     /**
@@ -499,7 +537,7 @@ public class NodeRepository extends AbstractComponent {
      * @return List of all the parked nodes in their new state
      */
     public List<Node> parkRecursively(String hostname, Agent agent, String reason) {
-        return moveRecursively(hostname, Node.State.parked, agent, Optional.of(reason));
+        return moveRecursively(hostname, State.parked, agent, Optional.of(reason));
     }
 
     /**
@@ -509,10 +547,10 @@ public class NodeRepository extends AbstractComponent {
      * @throws NoSuchNodeException if the node is not found
      */
     public Node reactivate(String hostname, Agent agent, String reason) {
-        return move(hostname, true, Node.State.active, agent, Optional.of(reason));
+        return move(hostname, true, State.active, agent, Optional.of(reason));
     }
 
-    private List<Node> moveRecursively(String hostname, Node.State toState, Agent agent, Optional<String> reason) {
+    private List<Node> moveRecursively(String hostname, State toState, Agent agent, Optional<String> reason) {
         List<Node> moved = list().childrenOf(hostname).asList().stream()
                                          .map(child -> move(child, toState, agent, reason))
                                          .collect(Collectors.toList());
@@ -521,7 +559,7 @@ public class NodeRepository extends AbstractComponent {
         return moved;
     }
 
-    private Node move(String hostname, boolean keepAllocation, Node.State toState, Agent agent, Optional<String> reason) {
+    private Node move(String hostname, boolean keepAllocation, State toState, Agent agent, Optional<String> reason) {
         Node node = getNode(hostname).orElseThrow(() ->
                 new NoSuchNodeException("Could not move " + hostname + " to " + toState + ": Node not found"));
 
@@ -532,17 +570,16 @@ public class NodeRepository extends AbstractComponent {
         return move(node, toState, agent, reason);
     }
 
-    private Node move(Node node, Node.State toState, Agent agent, Optional<String> reason) {
+    private Node move(Node node, State toState, Agent agent, Optional<String> reason) {
         if (toState == Node.State.active && ! node.allocation().isPresent())
-            throw new IllegalArgumentException("Could not set " + node.hostname() + " active. It has no allocation.");
+            illegal("Could not set " + node + " active. It has no allocation.");
 
         try (Mutex lock = lock(node)) {
-            if (toState == Node.State.active) {
-                for (Node currentActive : getNodes(node.allocation().get().owner(), Node.State.active)) {
+            if (toState == State.active) {
+                for (Node currentActive : getNodes(node.allocation().get().owner(), State.active)) {
                     if (node.allocation().get().membership().cluster().equals(currentActive.allocation().get().membership().cluster())
                         && node.allocation().get().membership().index() == currentActive.allocation().get().membership().index())
-                        throw new IllegalArgumentException("Could not move " + node + " to active:" +
-                                                           "It has the same cluster and index as an existing node");
+                        illegal("Could not set " + node + " active: Same cluster and index as " + currentActive);
                 }
             }
             return db.writeTo(toState, node, agent, reason);
@@ -556,20 +593,17 @@ public class NodeRepository extends AbstractComponent {
     public Node markNodeAvailableForNewAllocation(String hostname, Agent agent, String reason) {
         Node node = getNode(hostname).orElseThrow(() -> new NotFoundException("No node with hostname '" + hostname + "'"));
         if (node.flavor().getType() == Flavor.Type.DOCKER_CONTAINER && node.type() == NodeType.tenant) {
-            if (node.state() != Node.State.dirty) {
-                throw new IllegalArgumentException(
-                        "Cannot make " + hostname + " available for new allocation, must be in state dirty, but was in " + node.state());
-            }
+            if (node.state() != State.dirty)
+                illegal("Cannot make " + node  + " available for new allocation as it is not in state [dirty]");
             return removeRecursively(node, true).get(0);
         }
 
-        if (node.state() == Node.State.ready) return node;
+        if (node.state() == State.ready) return node;
 
         Node parentHost = node.parentHostname().flatMap(this::getNode).orElse(node);
         List<String> failureReasons = NodeFailer.reasonsToFailParentHost(parentHost);
-        if (!failureReasons.isEmpty()) {
-            throw new IllegalArgumentException("Node " + hostname + " cannot be readied because it has hard failures: " + failureReasons);
-        }
+        if ( ! failureReasons.isEmpty())
+            illegal(node + " cannot be readied because it has hard failures: " + failureReasons);
 
         return setReady(Collections.singletonList(node), agent, reason).get(0);
     }
@@ -577,7 +611,7 @@ public class NodeRepository extends AbstractComponent {
     /**
      * Removes all the nodes that are children of hostname before finally removing the hostname itself.
      *
-     * @return List of all the nodes that have been removed
+     * @return a List of all the nodes that have been removed or (for hosts) deprovisioned
      */
     public List<Node> removeRecursively(String hostname) {
         Node node = getNode(hostname).orElseThrow(() -> new NotFoundException("No node with hostname '" + hostname + "'"));
@@ -586,60 +620,57 @@ public class NodeRepository extends AbstractComponent {
 
     public List<Node> removeRecursively(Node node, boolean force) {
         try (Mutex lock = lockUnallocated()) {
-            List<Node> removed = new ArrayList<>();
-
-             if (node.type().isDockerHost()) {
-                 list().childrenOf(node).asList().stream()
-                       .filter(child -> force || canRemove(child, true))
-                       .forEach(removed::add);
-             }
-
-            if (force || canRemove(node, false)) removed.add(node);
-            db.removeNodes(removed);
-
-            return removed;
-        } catch (RuntimeException e) {
-            throw new IllegalArgumentException("Failed to delete " + node.hostname(), e);
+            requireRemovable(node, false, force);
+
+            if (node.type() == NodeType.host) {
+                List<Node> children = list().childrenOf(node).asList();
+                children.forEach(child -> requireRemovable(child, true, force));
+                db.removeNodes(children);
+                List<Node> removed = new ArrayList<>(children);
+                if (zone.cloud().value().equals("aws"))
+                    db.removeNodes(List.of(node));
+                else {
+                    node = node.with(IP.Config.EMPTY);
+                    move(node, State.deprovisioned, Agent.system, Optional.empty());
+                }
+                removed.add(node);
+                return removed;
+            }
+            else {
+                db.removeNodes(List.of(node));
+                return List.of(node);
+            }
         }
     }
 
     /**
-     * Returns whether given node can be removed. Removal is allowed if:
-     *  Tenant node: node is unallocated
-     *  Non-Docker-container node: iff in state provisioned|failed|parked
-     *  Docker-container-node:
-     *    If only removing the container node: node in state ready
-     *    If also removing the parent node: child is in state provisioned|failed|parked|dirty|ready
+     * Throws if the given node cannot be removed. Removal is allowed if:
+     *  - Tenant node: node is unallocated
+     *  - Non-Docker-container node: iff in state provisioned|failed|parked
+     *  - Docker-container-node:
+     *      If only removing the container node: node in state ready
+     *      If also removing the parent node: child is in state provisioned|failed|parked|dirty|ready
      */
-    private boolean canRemove(Node node, boolean deletingAsChild) {
-        if (node.type() == NodeType.tenant && node.allocation().isPresent()) {
-            throw new IllegalArgumentException("Node is currently allocated and cannot be removed: " +
-                                               node.allocation().get());
-        }
-        if (node.flavor().getType() == Flavor.Type.DOCKER_CONTAINER && !deletingAsChild) {
-            if (node.state() != Node.State.ready) {
-                throw new IllegalArgumentException(
-                        String.format("Docker container %s can only be removed when in ready state", node.hostname()));
-            }
+    private void requireRemovable(Node node, boolean removingAsChild, boolean force) {
+        if (force) return;
 
-        } else if (node.flavor().getType() == Flavor.Type.DOCKER_CONTAINER) {
-            Set<Node.State> legalStates = EnumSet.of(Node.State.provisioned, Node.State.failed, Node.State.parked,
-                                                     Node.State.dirty, Node.State.ready);
-
-            if (! legalStates.contains(node.state())) {
-                throw new IllegalArgumentException(String.format("Child node %s can only be removed from following states: %s",
-                        node.hostname(), legalStates.stream().map(Node.State::name).collect(Collectors.joining(", "))));
-            }
-        } else {
-            Set<Node.State> legalStates = EnumSet.of(Node.State.provisioned, Node.State.failed, Node.State.parked);
+        if (node.type() == NodeType.tenant && node.allocation().isPresent())
+            illegal(node + " is currently allocated and cannot be removed");
 
-            if (! legalStates.contains(node.state())) {
-                throw new IllegalArgumentException(String.format("Node %s can only be removed from following states: %s",
-                        node.hostname(), legalStates.stream().map(Node.State::name).collect(Collectors.joining(", "))));
-            }
+        if (node.flavor().getType() == Flavor.Type.DOCKER_CONTAINER && !removingAsChild) {
+            if (node.state() != State.ready)
+                illegal(node + " can not be removed as it is not in the state [ready]");
+        }
+        else if (node.flavor().getType() == Flavor.Type.DOCKER_CONTAINER) { // removing a child node
+            Set<State> legalStates = EnumSet.of(State.provisioned, State.failed, State.parked, State.dirty, State.ready);
+            if ( ! legalStates.contains(node.state()))
+                illegal(node + " can not be removed as it is not in the states " + legalStates);
+        }
+        else { // a host
+            Set<State> legalStates = EnumSet.of(State.provisioned, State.failed, State.parked);
+            if (! legalStates.contains(node.state()))
+                illegal(node + " can not be removed as it is not in the states " + legalStates);
         }
-
-        return true;
     }
 
     /**
@@ -648,7 +679,9 @@ public class NodeRepository extends AbstractComponent {
      * @return the nodes in their new state.
      */
     public List<Node> restart(NodeFilter filter) {
-        return performOn(StateFilter.from(Node.State.active, filter), (node, lock) -> write(node.withRestart(node.allocation().get().restartGeneration().withIncreasedWanted()), lock));
+        return performOn(StateFilter.from(State.active, filter),
+                         (node, lock) -> write(node.withRestart(node.allocation().get().restartGeneration().withIncreasedWanted()),
+                                               lock));
     }
 
     /**
@@ -726,6 +759,17 @@ public class NodeRepository extends AbstractComponent {
         return resultingNodes;
     }
 
+    public boolean canAllocateTenantNodeTo(Node host) {
+        if (!host.type().canRun(NodeType.tenant)) return false;
+
+        // Do not allocate to hosts we want to retire or are currently retiring
+        if (host.status().wantToRetire() || host.allocation().map(alloc -> alloc.membership().retired()).orElse(false))
+            return false;
+
+        if (!zone.cloud().value().equals("aws")) return host.state() == State.active;
+        else return EnumSet.of(State.active, State.ready, State.provisioned).contains(host.state());
+    }
+
     /** Returns the time keeper of this system */
     public Clock clock() { return clock; }
 
@@ -733,6 +777,7 @@ public class NodeRepository extends AbstractComponent {
     public Zone zone() { return zone; }
 
     /** Create a lock which provides exclusive rights to making changes to the given application */
+    // TODO(mpolden): Make this delegate to CuratorDatabaseClient#lockConfig instead
     public Mutex lock(ApplicationId application) { return db.lock(application); }
 
     /** Create a lock with a timeout which provides exclusive rights to making changes to the given application */
@@ -746,4 +791,8 @@ public class NodeRepository extends AbstractComponent {
         return node.allocation().isPresent() ? lock(node.allocation().get().owner()) : lockUnallocated();
     }
 
+    private void illegal(String message) {
+        throw new IllegalArgumentException(message);
+    }
+
 }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Application.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Application.java
new file mode 100644
index 00000000000..e56e426b499
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Application.java
@@ -0,0 +1,59 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.applications;
+
+import com.yahoo.config.provision.ClusterResources;
+import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.transaction.Mutex;
+
+import java.util.Map;
+import java.util.HashMap;
+import java.util.Optional;
+
+/**
+ * The node repository's view of an application deployment.
+ *
+ * This is immutable, and must be locked with the application lock on read-modify-write.
+ *
+ * @author bratseth
+ */
+public class Application {
+
+    private final Map<ClusterSpec.Id, Cluster> clusters;
+
+    public Application() {
+        this(Map.of());
+    }
+
+    private Application(Map<ClusterSpec.Id, Cluster> clusters) {
+        this.clusters = Map.copyOf(clusters);
+    }
+
+    /** Returns the cluster with the given id or null if none */
+    public Cluster cluster(ClusterSpec.Id id) { return clusters.get(id); }
+
+    public Application with(ClusterSpec.Id id, Cluster cluster) {
+        Map<ClusterSpec.Id, Cluster> clusters = new HashMap<>(this.clusters);
+        clusters.put(id, cluster);
+        return new Application(clusters);
+    }
+
+    /**
+     * Returns an application with the given cluster having the min and max resource limits of the given cluster.
+     * If the cluster has a target which is not inside the new limits, the target is removed.
+     */
+    public Application withClusterLimits(ClusterSpec.Id id, ClusterResources min, ClusterResources max) {
+        Cluster cluster = clusters.get(id);
+        return with(id, new Cluster(min, max, cluster == null ? Optional.empty() : cluster.targetResources()));
+    }
+
+    /**
+     * Returns an application with the given target for the given cluster,
+     * if it exists and the target is within the bounds
+     */
+    public Application withClusterTarget(ClusterSpec.Id id, ClusterResources target) {
+        Cluster cluster = clusters.get(id);
+        if (cluster == null) return this;
+        return with(id, cluster.withTarget(target));
+    }
+
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Applications.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Applications.java
new file mode 100644
index 00000000000..879fcc5f6cb
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Applications.java
@@ -0,0 +1,29 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.applications;
+
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.transaction.Mutex;
+
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * An (in-memory, for now) repository of the node repo's view of applications.
+ *
+ * This is multithread safe.
+ *
+ * @author bratseth
+ */
+public class Applications {
+
+    private final ConcurrentHashMap<ApplicationId, Application> applications = new ConcurrentHashMap<>();
+
+    /** Returns the application with the given id, or null if it does not exist and should not be created */
+    public Application get(ApplicationId applicationId, boolean create) {
+        return applications.computeIfAbsent(applicationId, id -> create ? new Application() : null);
+    }
+
+    public void set(ApplicationId id, Application application, Mutex applicationLock) {
+        applications.put(id, application);
+    }
+
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java
new file mode 100644
index 00000000000..6ff7f41be8f
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java
@@ -0,0 +1,66 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.applications;
+
+import com.yahoo.config.provision.ClusterResources;
+import com.yahoo.config.provision.NodeResources;
+
+import java.util.Objects;
+import java.util.Optional;
+
+/**
+ * The node repo's view of a cluster in an application deployment.
+ *
+ * This is immutable, and must be locked with the application lock on read-modify-write.
+ *
+ * @author bratseth
+ */
+public class Cluster {
+
+    private final ClusterResources min, max;
+    private final Optional<ClusterResources> target;
+
+    Cluster(ClusterResources minResources, ClusterResources maxResources, Optional<ClusterResources> targetResources) {
+        this.min = Objects.requireNonNull(minResources);
+        this.max = Objects.requireNonNull(maxResources);
+        Objects.requireNonNull(targetResources);
+
+        if (targetResources.isPresent() && ! targetResources.get().isWithin(minResources, maxResources))
+            this.target = Optional.empty();
+        else
+            this.target = targetResources;
+    }
+
+    /** Returns the configured minimal resources in this cluster */
+    public ClusterResources minResources() { return min; }
+
+    /** Returns the configured maximal resources in this cluster */
+    public ClusterResources maxResources() { return max; }
+
+    /**
+     * Returns the computed resources (between min and max, inclusive) this cluster should
+     * have allocated at the moment, or empty if the system currently have no opinion on this.
+     */
+    public Optional<ClusterResources> targetResources() { return target; }
+
+    public Cluster withTarget(ClusterResources target) {
+        return new Cluster(min, max, Optional.of(target));
+    }
+
+    public Cluster withoutTarget() {
+        return new Cluster(min, max, Optional.empty());
+    }
+
+    public NodeResources capAtLimits(NodeResources resources) {
+        resources = resources.withVcpu(between(min.nodeResources().vcpu(), max.nodeResources().vcpu(), resources.vcpu()));
+        resources = resources.withMemoryGb(between(min.nodeResources().memoryGb(), max.nodeResources().memoryGb(), resources.memoryGb()));
+        resources = resources.withDiskGb(between(min.nodeResources().diskGb(), max.nodeResources().diskGb(), resources.diskGb()));
+        return resources;
+    }
+
+    private double between(double min, double max, double value) {
+        value = Math.max(min, value);
+        value = Math.min(max, value);
+        return value;
+    }
+
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableClusterResources.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableClusterResources.java
new file mode 100644
index 00000000000..5ca09ddf51c
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableClusterResources.java
@@ -0,0 +1,124 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.autoscale;
+
+import com.yahoo.config.provision.ClusterResources;
+import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.config.provision.Flavor;
+import com.yahoo.config.provision.NodeResources;
+import com.yahoo.vespa.hosted.provision.Node;
+import com.yahoo.vespa.hosted.provision.provisioning.HostResourcesCalculator;
+
+import java.util.List;
+
+/**
+ * @author bratseth
+ */
+public class AllocatableClusterResources {
+
+    // We only depend on the ratios between these values
+    private static final double cpuUnitCost = 12.0;
+    private static final double memoryUnitCost = 1.2;
+    private static final double diskUnitCost = 0.045;
+
+    /** The node count in the cluster */
+    private final int nodes;
+
+    /** The number of node groups in the cluster */
+    private final int groups;
+
+    private final NodeResources realResources;
+    private final NodeResources advertisedResources;
+
+    private final ClusterSpec.Type clusterType;
+
+    private final double fulfilment;
+
+    public AllocatableClusterResources(List<Node> nodes, HostResourcesCalculator calculator) {
+        this.advertisedResources = nodes.get(0).flavor().resources();
+        this.realResources = calculator.realResourcesOf(nodes.get(0));
+        this.nodes = nodes.size();
+        this.groups = (int)nodes.stream().map(node -> node.allocation().get().membership().cluster().group()).distinct().count();
+        this.clusterType = nodes.get(0).allocation().get().membership().cluster().type();
+        this.fulfilment = 1;
+    }
+
+    public AllocatableClusterResources(ClusterResources realResources,
+                                       NodeResources advertisedResources,
+                                       NodeResources idealResources,
+                                       ClusterSpec.Type clusterType) {
+        this.realResources = realResources.nodeResources();
+        this.advertisedResources = advertisedResources;
+        this.nodes = realResources.nodes();
+        this.groups = realResources.groups();
+        this.clusterType = clusterType;
+        this.fulfilment = fulfilment(realResources.nodeResources(), idealResources);
+    }
+
+    public AllocatableClusterResources(ClusterResources realResources,
+                                       Flavor flavor,
+                                       NodeResources idealResources,
+                                       ClusterSpec.Type clusterType,
+                                       HostResourcesCalculator calculator) {
+        this.realResources = realResources.nodeResources();
+        this.advertisedResources = calculator.advertisedResourcesOf(flavor);
+        this.nodes = realResources.nodes();
+        this.groups = realResources.groups();
+        this.clusterType = clusterType;
+        this.fulfilment = fulfilment(realResources.nodeResources(), idealResources);
+    }
+
+    /**
+     * Returns the resources which will actually be available per node in this cluster with this allocation.
+     * These should be used for reasoning about allocation to meet measured demand.
+     */
+    public NodeResources realResources() { return realResources; }
+
+    /**
+     * Returns the resources advertised by the cloud provider, which are the basis for charging
+     * and which must be used in resource allocation requests
+     */
+    public NodeResources advertisedResources() { return advertisedResources; }
+
+    public ClusterResources toAdvertisedClusterResources() {
+        return new ClusterResources(nodes, groups, advertisedResources);
+    }
+
+    public int nodes() { return nodes; }
+    public int groups() { return groups; }
+    public ClusterSpec.Type clusterType() { return clusterType; }
+
+    public double cost() { return nodes * costOf(advertisedResources); }
+
+    /**
+     * Returns the fraction measuring how well the real resources fulfils the ideal: 1 means completely fulfiled,
+     * 0 means we have zero real resources.
+     * The real may be short of the ideal due to resource limits imposed by the system or application.
+     */
+    public double fulfilment() { return fulfilment; }
+
+    private static double costOf(NodeResources resources) {
+        return resources.vcpu() * cpuUnitCost +
+               resources.memoryGb() * memoryUnitCost +
+               resources.diskGb() * diskUnitCost;
+    }
+
+    private static double fulfilment(NodeResources realResources, NodeResources idealResources) {
+        double vcpuFulfilment     = Math.min(1, realResources.vcpu()     / idealResources.vcpu());
+        double memoryGbFulfilment = Math.min(1, realResources.memoryGb() / idealResources.memoryGb());
+        double diskGbFulfilment   = Math.min(1, realResources.diskGb()   / idealResources.diskGb());
+        return (vcpuFulfilment + memoryGbFulfilment + diskGbFulfilment) / 3;
+    }
+
+    public boolean preferableTo(AllocatableClusterResources other) {
+        if (this.fulfilment > other.fulfilment) return true; // we always want to fulfil as much as possible
+        return this.cost() < other.cost(); // otherwise, prefer lower cost
+    }
+
+    @Override
+    public String toString() {
+        return nodes + " nodes with " + realResources() +
+               " at cost $" + cost() +
+               (fulfilment < 1.0 ? " (fulfilment " + fulfilment + ")" : "");
+    }
+
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java
new file mode 100644
index 00000000000..447d3494fbc
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java
@@ -0,0 +1,193 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.autoscale;
+
+import com.yahoo.config.provision.CloudName;
+import com.yahoo.config.provision.ClusterResources;
+import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.config.provision.Flavor;
+import com.yahoo.config.provision.NodeResources;
+import com.yahoo.config.provision.host.FlavorOverrides;
+import com.yahoo.vespa.hosted.provision.Node;
+import com.yahoo.vespa.hosted.provision.NodeRepository;
+import com.yahoo.vespa.hosted.provision.applications.Cluster;
+import com.yahoo.vespa.hosted.provision.provisioning.HostResourcesCalculator;
+import com.yahoo.vespa.hosted.provision.provisioning.NodeResourceLimits;
+
+import java.time.Duration;
+import java.util.List;
+import java.util.Optional;
+import java.util.stream.Collectors;
+
+/**
+ * The autoscaler makes decisions about the flavor and node count that should be allocated to a cluster
+ * based on observed behavior.
+ *
+ * @author bratseth
+ */
+public class Autoscaler {
+
+    /*
+     TODO:
+     - Scale group size
+     - Consider taking spikes/variance into account
+     - Measure observed regulation lag (startup+redistribution) and take it into account when deciding regulation observation window
+     - Test AutoscalingMaintainer
+     - Scale by performance not just load+cost
+     */
+
+    private static final int minimumMeasurements = 500; // TODO: Per node instead? Also say something about interval?
+
+    /** What cost difference factor is worth a reallocation? */
+    private static final double costDifferenceWorthReallocation = 0.1;
+    /** What difference factor for a resource is worth a reallocation? */
+    private static final double resourceDifferenceWorthReallocation = 0.1;
+
+    private final HostResourcesCalculator resourcesCalculator;
+    private final NodeMetricsDb metricsDb;
+    private final NodeRepository nodeRepository;
+    private final NodeResourceLimits nodeResourceLimits;
+
+    public Autoscaler(HostResourcesCalculator resourcesCalculator,
+                      NodeMetricsDb metricsDb,
+                      NodeRepository nodeRepository) {
+        this.resourcesCalculator = resourcesCalculator;
+        this.metricsDb = metricsDb;
+        this.nodeRepository = nodeRepository;
+        this.nodeResourceLimits = new NodeResourceLimits(nodeRepository.zone());
+    }
+
+    /**
+     * Autoscale a cluster
+     *
+     * @param clusterNodes the list of all the active nodes in a cluster
+     * @return a new suggested allocation for this cluster, or empty if it should not be rescaled at this time
+     */
+    public Optional<AllocatableClusterResources> autoscale(Cluster cluster, List<Node> clusterNodes) {
+        if (unstable(clusterNodes)) return Optional.empty();
+
+        ClusterSpec.Type clusterType = clusterNodes.get(0).allocation().get().membership().cluster().type();
+        AllocatableClusterResources currentAllocation = new AllocatableClusterResources(clusterNodes, resourcesCalculator);
+        Optional<Double> cpuLoad    = averageLoad(Resource.cpu, clusterNodes, clusterType);
+        Optional<Double> memoryLoad = averageLoad(Resource.memory, clusterNodes, clusterType);
+        Optional<Double> diskLoad   = averageLoad(Resource.disk, clusterNodes, clusterType);
+        if (cpuLoad.isEmpty() || memoryLoad.isEmpty() || diskLoad.isEmpty()) return Optional.empty();
+
+        Optional<AllocatableClusterResources> bestAllocation = findBestAllocation(cpuLoad.get(),
+                                                                                  memoryLoad.get(),
+                                                                                  diskLoad.get(),
+                                                                                  currentAllocation,
+                                                                                  cluster);
+        if (bestAllocation.isEmpty()) return Optional.empty();
+        if (similar(bestAllocation.get(), currentAllocation)) return Optional.empty();
+        return bestAllocation;
+    }
+
+    private Optional<AllocatableClusterResources> findBestAllocation(double cpuLoad, double memoryLoad, double diskLoad,
+                                                                     AllocatableClusterResources currentAllocation,
+                                                                     Cluster cluster) {
+        Optional<AllocatableClusterResources> bestAllocation = Optional.empty();
+        for (ResourceIterator i = new ResourceIterator(cpuLoad, memoryLoad, diskLoad, currentAllocation, cluster); i.hasNext(); ) {
+            Optional<AllocatableClusterResources> allocatableResources = toAllocatableResources(i.next(),
+                                                                                                currentAllocation.clusterType(),
+                                                                                                cluster);
+            if (allocatableResources.isEmpty()) continue;
+            if (bestAllocation.isEmpty() || allocatableResources.get().preferableTo(bestAllocation.get()))
+                bestAllocation = allocatableResources;
+        }
+        return bestAllocation;
+    }
+
+    /** Returns true if both total real resources and total cost are similar */
+    private boolean similar(AllocatableClusterResources a, AllocatableClusterResources b) {
+        return similar(a.cost(), b.cost(), costDifferenceWorthReallocation) &&
+               similar(a.realResources().vcpu() * a.nodes(),
+                       b.realResources().vcpu() * b.nodes(), resourceDifferenceWorthReallocation) &&
+               similar(a.realResources().memoryGb() * a.nodes(),
+                       b.realResources().memoryGb() * b.nodes(), resourceDifferenceWorthReallocation) &&
+               similar(a.realResources().diskGb() * a.nodes(),
+                       b.realResources().diskGb() * b.nodes(),
+                       resourceDifferenceWorthReallocation);
+    }
+
+    private boolean similar(double r1, double r2, double threshold) {
+        return Math.abs(r1 - r2) / r1 < threshold;
+    }
+
+    /**
+     * Returns the smallest allocatable node resources larger than the given node resources,
+     * or empty if none available.
+     */
+    private Optional<AllocatableClusterResources> toAllocatableResources(ClusterResources resources,
+                                                                         ClusterSpec.Type clusterType,
+                                                                         Cluster cluster) {
+        NodeResources nodeResources = resources.nodeResources();
+        if ( ! cluster.minResources().equals(cluster.maxResources())) // enforce application limits unless suggest mode
+            nodeResources = cluster.capAtLimits(nodeResources);
+        nodeResources = nodeResourceLimits.enlargeToLegal(nodeResources, clusterType); // enforce system limits
+
+        if (allowsHostSharing(nodeRepository.zone().cloud())) {
+            // return the requested resources, or empty if they cannot fit on existing hosts
+            for (Flavor flavor : nodeRepository.getAvailableFlavors().getFlavors()) {
+                if (flavor.resources().satisfies(nodeResources))
+                    return Optional.of(new AllocatableClusterResources(resources.with(nodeResources),
+                                                                       nodeResources,
+                                                                       resources.nodeResources(),
+                                                                       clusterType));
+            }
+            return Optional.empty();
+        }
+        else {
+            // return the cheapest flavor satisfying the target resources, if any
+            Optional<AllocatableClusterResources> best = Optional.empty();
+            for (Flavor flavor : nodeRepository.getAvailableFlavors().getFlavors()) {
+                if ( ! flavor.resources().satisfies(nodeResources)) continue;
+
+                if (flavor.resources().storageType() == NodeResources.StorageType.remote)
+                    flavor = flavor.with(FlavorOverrides.ofDisk(nodeResources.diskGb()));
+                var candidate = new AllocatableClusterResources(resources.with(flavor.resources()),
+                                                                flavor,
+                                                                resources.nodeResources(),
+                                                                clusterType,
+                                                                resourcesCalculator);
+
+                if (best.isEmpty() || candidate.cost() <= best.get().cost())
+                    best = Optional.of(candidate);
+            }
+            return best;
+        }
+    }
+
+    /**
+     * Returns the average load of this resource in the measurement window,
+     * or empty if we are not in a position to make decisions from these measurements at this time.
+     */
+    private Optional<Double> averageLoad(Resource resource, List<Node> clusterNodes, ClusterSpec.Type clusterType) {
+        NodeMetricsDb.Window window = metricsDb.getWindow(nodeRepository.clock().instant().minus(scalingWindow(clusterType)),
+                                                          resource,
+                                                          clusterNodes.stream().map(Node::hostname).collect(Collectors.toList()));
+
+        if (window.measurementCount() < minimumMeasurements) return Optional.empty();
+        if (window.hostnames() != clusterNodes.size()) return Optional.empty(); // Regulate only when all nodes are measured
+
+        return Optional.of(window.average());
+    }
+
+    /** The duration of the window we need to consider to make a scaling decision */
+    private Duration scalingWindow(ClusterSpec.Type clusterType) {
+        if (clusterType.isContent()) return Duration.ofHours(12); // Ideally we should use observed redistribution time
+        return Duration.ofHours(12); // TODO: Measure much more often to get this down to minutes. And, ideally we should take node startup time into account
+    }
+
+    // TODO: Put this in zone config instead?
+    private boolean allowsHostSharing(CloudName cloudName) {
+        if (cloudName.value().equals("aws")) return false;
+        return true;
+    }
+
+    public static boolean unstable(List<Node> nodes) {
+        return nodes.stream().anyMatch(node -> node.status().wantToRetire() ||
+                                                      node.allocation().get().membership().retired() ||
+                                                      node.allocation().get().isRemovable());
+    }
+
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsResponse.java
new file mode 100644
index 00000000000..87551a5bd5f
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsResponse.java
@@ -0,0 +1,77 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.autoscale;
+
+import com.yahoo.slime.ArrayTraverser;
+import com.yahoo.slime.Inspector;
+import com.yahoo.slime.ObjectTraverser;
+import com.yahoo.slime.Slime;
+import com.yahoo.slime.SlimeUtils;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Consumes a response from the metrics/v2 API and populates the fields of this with the resulting values
+ *
+ * @author bratseth
+ */
+public class MetricsResponse {
+
+    private final List<NodeMetrics.MetricValue> metricValues = new ArrayList<>();
+
+    public MetricsResponse(byte[] response) {
+        this(SlimeUtils.jsonToSlime(response));
+    }
+
+    public MetricsResponse(String response) {
+        this(SlimeUtils.jsonToSlime(response));
+    }
+
+    public List<NodeMetrics.MetricValue> metrics() { return metricValues; }
+
+    private MetricsResponse(Slime response) {
+        Inspector root = response.get();
+        Inspector nodes = root.field("nodes");
+        nodes.traverse((ArrayTraverser)(__, node) -> consumeNode(node));
+    }
+
+    private void consumeNode(Inspector node) {
+        String hostname = node.field("hostname").asString();
+        consumeNodeMetrics(hostname, node.field("node"));
+        consumeServiceMetrics(hostname, node.field("services"));
+    }
+
+    private void consumeNodeMetrics(String hostname, Inspector node) {
+        long timestampSecond = node.field("timestamp").asLong();
+        Map<String, Double> values = consumeMetrics(node.field("metrics"));
+        for (Resource resource : Resource.values())
+            addMetricIfPresent(hostname, resource, timestampSecond, values);
+    }
+
+    private void addMetricIfPresent(String hostname, Resource resource, long timestampSecond, Map<String, Double> values) {
+        if (values.containsKey(resource.metricName()))
+            metricValues.add(new NodeMetrics.MetricValue(hostname,
+                                                         resource.metricName(),
+                                                         timestampSecond,
+                                                         values.get(resource.metricName())));
+    }
+
+    private void consumeServiceMetrics(String hostname, Inspector node) {
+        String name = node.field("name").asString();
+        long timestamp = node.field("timestamp").asLong();
+        Map<String, Double> values = consumeMetrics(node.field("metrics"));
+    }
+
+    private Map<String, Double> consumeMetrics(Inspector metrics) {
+        Map<String, Double> values = new HashMap<>();
+        metrics.traverse((ArrayTraverser) (__, item) -> consumeMetricsItem(item, values));
+        return values;
+    }
+
+    private void consumeMetricsItem(Inspector item, Map<String, Double> values) {
+        item.field("values").traverse((ObjectTraverser)(name, value) -> values.put(name, value.asDouble()));
+    }
+
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetrics.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetrics.java
new file mode 100644
index 00000000000..b0d73833bc6
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetrics.java
@@ -0,0 +1,49 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.autoscale;
+
+import com.yahoo.config.provision.ApplicationId;
+
+import java.time.Instant;
+import java.util.Collection;
+
+/**
+ * Interface to retrieve metrics on (tenant) nodes.
+ *
+ * @author bratseth
+ */
+public interface NodeMetrics {
+
+    /**
+     * Fetches metrics for an application. This call may be expensive.
+     *
+     * @param application the application to fetch metrics from
+     */
+    Collection<MetricValue> fetchMetrics(ApplicationId application);
+
+    final class MetricValue {
+
+        private final String hostname;
+        private final String name;
+        private long timestampSecond;
+        private final double value;
+
+        public MetricValue(String hostname, String name, long timestampSecond, double value) {
+            this.hostname = hostname;
+            this.name = name;
+            this.timestampSecond = timestampSecond;
+            this.value = value;
+        }
+
+        public String hostname() { return hostname; }
+        public String name() { return name; }
+        public long timestampSecond() { return timestampSecond; }
+        public double value() { return value; }
+
+        @Override
+        public String toString() {
+            return "metric value " + name + ": " + value + " at " + Instant.ofEpochSecond(timestampSecond) + " for " + hostname;
+        }
+
+    }
+
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsDb.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsDb.java
new file mode 100644
index 00000000000..75bc73df5b0
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsDb.java
@@ -0,0 +1,176 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.autoscale;
+
+import java.time.Clock;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.logging.Logger;
+import java.util.stream.Collectors;
+
+/**
+ * An in-memory time-series "database" of node metrics.
+ * Thread model: One writer, many readers.
+ *
+ * @author bratseth
+ */
+public class NodeMetricsDb {
+
+    private Logger log = Logger.getLogger(NodeMetricsDb.class.getName());
+    private static final Duration dbWindow = Duration.ofHours(24);
+
+    /** Measurements by key. Each list of measurements is sorted by increasing timestamp */
+    private Map<MeasurementKey, List<Measurement>> db = new HashMap<>();
+
+    /** Lock all access for now since we modify lists inside a map */
+    private final Object lock = new Object();
+
+    /** Add a measurement to this */
+    public void add(Collection<NodeMetrics.MetricValue> metricValues) {
+        synchronized (lock) {
+            for (var value : metricValues) {
+                Resource resource =  Resource.fromMetric(value.name());
+                List<Measurement> measurements = db.computeIfAbsent(new MeasurementKey(value.hostname(), resource),
+                                                                    (__) -> new ArrayList<>());
+                measurements.add(new Measurement(value.timestampSecond() * 1000,
+                                                 (float)resource.valueFromMetric(value.value())));
+            }
+        }
+    }
+
+    /** Must be called intermittently (as long as add is called) to gc old measurements */
+    public void gc(Clock clock) {
+        synchronized (lock) {
+            // TODO: We may need to do something more complicated to avoid spending too much memory to
+            // lower the measurement interval (see NodeRepositoryMaintenance)
+            // Each measurement is Object + long + float = 16 + 8 + 4 = 28 bytes
+            // 24 hours with 1k nodes and 3 resources and 1 measurement/sec is about 10Gb
+
+            long oldestTimestamp = clock.instant().minus(dbWindow).toEpochMilli();
+            for (Iterator<List<Measurement>> i = db.values().iterator(); i.hasNext(); ) {
+                List<Measurement> measurements = i.next();
+                while (!measurements.isEmpty() && measurements.get(0).timestamp < oldestTimestamp)
+                    measurements.remove(0);
+
+                if (measurements.isEmpty())
+                    i.remove();
+            }
+        }
+    }
+
+    /** Returns a window within which we can ask for specific information from this db */
+    public Window getWindow(Instant startTime, Resource resource, List<String> hostnames) {
+        return new Window(startTime, resource, hostnames);
+    }
+
+    public class Window {
+
+        private final long startTime;
+        private List<MeasurementKey> keys;
+
+        private Window(Instant startTime, Resource resource, List<String> hostnames) {
+            this.startTime = startTime.toEpochMilli();
+            keys = hostnames.stream().map(hostname -> new MeasurementKey(hostname, resource)).collect(Collectors.toList());
+        }
+
+        public int measurementCount() {
+            synchronized (lock) {
+                return (int) keys.stream()
+                                 .flatMap(key -> db.getOrDefault(key, List.of()).stream())
+                                 .filter(measurement -> measurement.timestamp >= startTime)
+                                 .count();
+            }
+        }
+
+        /** Returns the count of hostnames which have measurements in this window */
+        public int hostnames() {
+            synchronized (lock) {
+                int count = 0;
+                for (MeasurementKey key : keys) {
+                    List<Measurement> measurements = db.get(key);
+                    if (measurements == null || measurements.isEmpty()) continue;
+
+                    if (measurements.get(measurements.size() - 1).timestamp >= startTime)
+                        count++;
+                }
+                return count;
+            }
+        }
+
+        public double average() {
+            synchronized (lock) {
+                double sum = 0;
+                int count = 0;
+                for (MeasurementKey key : keys) {
+                    List<Measurement> measurements = db.get(key);
+                    if (measurements == null) continue;
+
+                    int index = measurements.size() - 1;
+                    while (index >= 0 && measurements.get(index).timestamp >= startTime) {
+                        sum += measurements.get(index).value;
+                        count++;
+
+                        index--;
+                    }
+                }
+                return sum / count;
+            }
+        }
+
+    }
+
+    private static class MeasurementKey {
+
+        private final String hostname;
+        private final Resource resource;
+
+        public MeasurementKey(String hostname, Resource resource) {
+            this.hostname = hostname;
+            this.resource = resource;
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(hostname, resource);
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if ( ! (o instanceof MeasurementKey)) return false;
+            MeasurementKey other = (MeasurementKey)o;
+            if ( ! this.hostname.equals(other.hostname)) return false;
+            if ( ! this.resource.equals(other.resource)) return false;
+            return true;
+        }
+
+        @Override
+        public String toString() { return "measurements of " + resource + " for " + hostname; }
+
+    }
+
+    private static class Measurement {
+
+        /** The time of this measurement in epoch millis */
+        private final long timestamp;
+
+        /** The measured value */
+        private final float value;
+
+        public Measurement(long timestamp, float value) {
+            this.timestamp = timestamp;
+            this.value = value;
+        }
+
+        @Override
+        public String toString() { return "measurement at " + timestamp + ": " + value; }
+
+    }
+
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsFetcher.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsFetcher.java
new file mode 100644
index 00000000000..232fee1df6a
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsFetcher.java
@@ -0,0 +1,120 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.autoscale;
+
+import ai.vespa.util.http.VespaHttpClientBuilder;
+import com.google.inject.Inject;
+import com.yahoo.component.AbstractComponent;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.vespa.applicationmodel.HostName;
+import com.yahoo.vespa.hosted.provision.Node;
+import com.yahoo.vespa.hosted.provision.NodeList;
+import com.yahoo.vespa.hosted.provision.NodeRepository;
+import com.yahoo.vespa.orchestrator.HostNameNotFoundException;
+import com.yahoo.vespa.orchestrator.Orchestrator;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.impl.client.BasicResponseHandler;
+import org.apache.http.impl.client.CloseableHttpClient;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Optional;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * Fetches node metrics over the metrics/v2 API
+ *
+ * @author bratseth
+ */
+public class NodeMetricsFetcher extends AbstractComponent implements NodeMetrics {
+
+    private static final Logger log = Logger.getLogger(NodeMetricsFetcher.class.getName());
+
+    private static final String apiPath = "/metrics/v2/values";
+
+    private final NodeRepository nodeRepository;
+    private final Orchestrator orchestrator;
+    private final HttpClient httpClient;
+
+    @Inject
+    @SuppressWarnings("unused")
+    public NodeMetricsFetcher(NodeRepository nodeRepository, Orchestrator orchestrator) {
+        this(nodeRepository, orchestrator, new ApacheHttpClient());
+    }
+
+    NodeMetricsFetcher(NodeRepository nodeRepository, Orchestrator orchestrator, HttpClient httpClient) {
+        this.nodeRepository = nodeRepository;
+        this.orchestrator = orchestrator;
+        this.httpClient = httpClient;
+    }
+
+    @Override
+    public Collection<MetricValue> fetchMetrics(ApplicationId application) {
+        NodeList applicationNodes = nodeRepository.list(application).state(Node.State.active);
+
+        // Do not try to draw conclusions from utilization while unstable
+        if (Autoscaler.unstable(applicationNodes.asList())) return Collections.emptyList();
+
+        Optional<Node> metricsV2Container = applicationNodes.container()
+                                                            .filter(node -> expectedUp(node))
+                                                            .stream()
+                                                            .findFirst();
+        if (metricsV2Container.isEmpty()) return Collections.emptyList();
+        String url = "http://" + metricsV2Container.get().hostname() + ":" + 4080 + apiPath + "?consumer=default";
+        String response = httpClient.get(url);
+        return new MetricsResponse(response).metrics();
+    }
+
+    @Override
+    public void deconstruct() {
+        httpClient.close();
+    }
+
+    private boolean expectedUp(Node node) {
+        try {
+            return ! orchestrator.getNodeStatus(new HostName(node.hostname())).isSuspended();
+        }
+        catch (HostNameNotFoundException e) {
+            return false;
+        }
+    }
+
+    /** The simplest possible http client interface */
+    public interface HttpClient {
+
+        String get(String url);
+        void close();
+
+    }
+
+    /** Implements the HttpClient interface by delegating to an Apache HTTP client */
+    public static class ApacheHttpClient implements HttpClient {
+
+        private final CloseableHttpClient httpClient = VespaHttpClientBuilder.createWithBasicConnectionManager().build();
+
+        @Override
+        public String get(String url) {
+            try {
+                return httpClient.execute(new HttpGet(url), new BasicResponseHandler());
+            }
+            catch (IOException e) {
+                throw new UncheckedIOException("Could not get " + url, e);
+            }
+        }
+
+        @Override
+        public void close() {
+            try {
+                httpClient.close();
+            }
+            catch (IOException e) {
+                log.log(Level.WARNING, "Exception deconstructing", e);
+            }
+        }
+
+
+    }
+
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Resource.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Resource.java
new file mode 100644
index 00000000000..3d5ce8881e0
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Resource.java
@@ -0,0 +1,52 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.autoscale;
+
+import com.yahoo.config.provision.NodeResources;
+
+/**
+ * A resource subject to autoscaling
+ *
+ * @author bratseth
+ */
+public enum Resource {
+
+    /** Cpu utilization ratio */
+    cpu {
+        public String metricName() { return "cpu.util"; }
+        double idealAverageLoad() { return 0.2; }
+        double valueFrom(NodeResources resources) { return resources.vcpu(); }
+        double valueFromMetric(double metricValue) { return metricValue / 100; } // % to ratio
+    },
+
+    /** Memory utilization ratio */
+    memory {
+        public String metricName() { return "mem_total.util"; }
+        double idealAverageLoad() { return 0.7; }
+        double valueFrom(NodeResources resources) { return resources.memoryGb(); }
+        double valueFromMetric(double metricValue) { return metricValue / 100; } // % to ratio
+    },
+
+    /** Disk utilization ratio */
+    disk {
+        public String metricName() { return "disk.util"; }
+        double idealAverageLoad() { return 0.6; }
+        double valueFrom(NodeResources resources) { return resources.diskGb(); }
+        double valueFromMetric(double metricValue) { return metricValue / 100; } // % to ratio
+    };
+
+    public abstract String metricName();
+
+    /** The load we should have of this resource on average, when one node in the cluster is down */
+    abstract double idealAverageLoad();
+
+    abstract double valueFrom(NodeResources resources);
+
+    abstract double valueFromMetric(double metricValue);
+
+    public static Resource fromMetric(String metricName) {
+        for (Resource resource : values())
+            if (resource.metricName().equals(metricName)) return resource;
+        throw new IllegalArgumentException("Metric '" + metricName + "' does not map to a resource");
+    }
+
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ResourceIterator.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ResourceIterator.java
new file mode 100644
index 00000000000..b7d5995884e
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ResourceIterator.java
@@ -0,0 +1,157 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.autoscale;
+
+import com.yahoo.config.provision.ClusterResources;
+import com.yahoo.config.provision.NodeResources;
+import com.yahoo.vespa.hosted.provision.applications.Cluster;
+
+/**
+ * Provides iteration over possible cluster resource allocations given a target total load
+ * and current groups/nodes allocation.
+ */
+public class ResourceIterator {
+
+    // Configured min and max nodes for suggestions for apps which have not activated autoscaling
+    private static final int minimumNodes = 3; // Since this is with redundancy it cannot be lower than 2
+    private static final int maximumNodes = 150;
+
+    // When a query is issued on a node the cost is the sum of a fixed cost component and a cost component
+    // proportional to document count. We must account for this when comparing configurations with more or fewer nodes.
+    // TODO: Measure this, and only take it into account with queries
+    private static final double fixedCpuCostFraction = 0.1;
+
+    // Prescribed state
+    private final Cluster cluster;
+
+    // Observed state
+    private final AllocatableClusterResources allocation;
+    private final double cpuLoad;
+    private final double memoryLoad;
+    private final double diskLoad;
+    private final int groupSize;
+
+    // Derived from the observed state
+    private final int nodeIncrement;
+    private final boolean singleGroupMode;
+
+    // Iterator state
+    private int currentNodes;
+
+    public ResourceIterator(double cpuLoad, double memoryLoad, double diskLoad,
+                            AllocatableClusterResources currentAllocation,
+                            Cluster cluster) {
+        this.cpuLoad = cpuLoad;
+        this.memoryLoad = memoryLoad;
+        this.diskLoad = diskLoad;
+
+        // ceil: If the division does not produce a whole number we assume some node is missing
+        groupSize = (int)Math.ceil((double)currentAllocation.nodes() / currentAllocation.groups());
+        allocation = currentAllocation;
+
+        this.cluster = cluster;
+
+        // What number of nodes is it effective to add or remove at the time from this cluster?
+        // This is the group size, since we (for now) assume the group size is decided by someone wiser than us
+        // and we decide the number of groups.
+        // The exception is when we only have one group, where we can add and remove single nodes in it.
+        singleGroupMode = currentAllocation.groups() == 1;
+        nodeIncrement = singleGroupMode ? 1 : groupSize;
+
+        // Step down to the right starting point
+        currentNodes = currentAllocation.nodes();
+        while (currentNodes - nodeIncrement >= minNodes()
+               && ( singleGroupMode || currentNodes - nodeIncrement > groupSize)) // group level redundancy
+            currentNodes -= nodeIncrement;
+    }
+
+    /** If autoscaling is not enabled (meaning max and min resources are the same), we want to suggest */
+    private boolean suggestMode() {
+        return cluster.minResources().equals(cluster.maxResources());
+    }
+
+    public ClusterResources next() {
+        ClusterResources next = resourcesWith(currentNodes);
+        currentNodes += nodeIncrement;
+        return next;
+    }
+
+    public boolean hasNext() {
+        return currentNodes <= maxNodes();
+    }
+
+    private int minNodes() {
+        if (suggestMode()) return minimumNodes;
+        if (singleGroupMode) return cluster.minResources().nodes();
+        return Math.max(cluster.minResources().nodes(), cluster.minResources().groups() * groupSize );
+    }
+
+    private int maxNodes() {
+        if (suggestMode()) return maximumNodes;
+        if (singleGroupMode) return cluster.maxResources().nodes();
+        return Math.min(cluster.maxResources().nodes(), cluster.maxResources().groups() * groupSize );
+    }
+
+    private ClusterResources resourcesWith(int nodes) {
+        int nodesWithRedundancy = nodes - (singleGroupMode ? 1 : groupSize);
+        return new ClusterResources(nodes,
+                                    singleGroupMode ? 1 : nodes / groupSize,
+                                    nodeResourcesWith(nodesWithRedundancy));
+    }
+
+    /**
+     * For the observed load this instance is initialized with, returns the resources needed per node to be at
+     * ideal load given a target node count
+     */
+    private NodeResources nodeResourcesWith(int nodeCount) {
+        // Cpu: Scales with cluster size (TODO: Only reads, writes scales with group size)
+        // Memory and disk: Scales with group size
+
+        double cpu, memory, disk;
+        if (singleGroupMode) {
+            // The fixed cost portion of cpu does not scale with changes to the node count
+            // TODO: Only for the portion of cpu consumed by queries
+            double totalCpu = clusterUsage(Resource.cpu, cpuLoad);
+            cpu = fixedCpuCostFraction       * totalCpu / groupSize / Resource.cpu.idealAverageLoad() +
+                  (1 - fixedCpuCostFraction) * totalCpu / nodeCount / Resource.cpu.idealAverageLoad();
+            if (allocation.clusterType().isContent()) { // load scales with node share of content
+                memory = groupUsage(Resource.memory, memoryLoad) / nodeCount / Resource.memory.idealAverageLoad();
+                disk = groupUsage(Resource.disk, diskLoad) / nodeCount / Resource.disk.idealAverageLoad();
+            }
+            else {
+                memory = nodeUsage(Resource.memory, memoryLoad) / Resource.memory.idealAverageLoad();
+                disk = nodeUsage(Resource.disk, diskLoad) / Resource.disk.idealAverageLoad();
+            }
+        }
+        else {
+            cpu = clusterUsage(Resource.cpu, cpuLoad) / nodeCount / Resource.cpu.idealAverageLoad();
+            if (allocation.clusterType().isContent()) { // load scales with node share of content
+                memory = groupUsage(Resource.memory, memoryLoad) / groupSize / Resource.memory.idealAverageLoad();
+                disk = groupUsage(Resource.disk, diskLoad) / groupSize / Resource.disk.idealAverageLoad();
+            }
+            else {
+                memory = nodeUsage(Resource.memory, memoryLoad) / Resource.memory.idealAverageLoad();
+                disk = nodeUsage(Resource.disk, diskLoad) / Resource.disk.idealAverageLoad();
+            }
+        }
+
+        // Combine the scaled resource values computed here
+        // and the currently combined values of non-scaled resources
+        return new NodeResources(cpu, memory, disk,
+                                 cluster.minResources().nodeResources().bandwidthGbps(),
+                                 cluster.minResources().nodeResources().diskSpeed(),
+                                 cluster.minResources().nodeResources().storageType());
+    }
+
+    private double clusterUsage(Resource resource, double load) {
+        return nodeUsage(resource, load) * allocation.nodes();
+    }
+
+    private double groupUsage(Resource resource, double load) {
+        return nodeUsage(resource, load) * groupSize;
+    }
+
+    private double nodeUsage(Resource resource, double load) {
+        return load * resource.valueFrom(allocation.realResources());
+    }
+
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerList.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerList.java
index 014d3df8d9a..bad16bf7d12 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerList.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerList.java
@@ -1,9 +1,6 @@
-// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.provision.lb;
 
-import com.yahoo.config.provision.ApplicationId;
-
-import java.time.Instant;
 import java.util.Collection;
 import java.util.Iterator;
 import java.util.List;
@@ -24,21 +21,11 @@ public class LoadBalancerList implements Iterable<LoadBalancer> {
         this.loadBalancers = List.copyOf(Objects.requireNonNull(loadBalancers, "loadBalancers must be non-null"));
     }
 
-    /** Returns the subset of load balancers owned by given application */
-    public LoadBalancerList owner(ApplicationId application) {
-        return of(loadBalancers.stream().filter(lb -> lb.id().application().equals(application)));
-    }
-
     /** Returns the subset of load balancers that are in given state */
     public LoadBalancerList in(LoadBalancer.State state) {
         return of(loadBalancers.stream().filter(lb -> lb.state() == state));
     }
 
-    /** Returns the subset of load balancers that last changed before given instant */
-    public LoadBalancerList changedBefore(Instant instant) {
-        return of(loadBalancers.stream().filter(lb -> lb.changedAt().isBefore(instant)));
-    }
-
     public List<LoadBalancer> asList() {
         return loadBalancers;
     }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java
index e9e09781e31..44dd023677c 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java
@@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.provision.maintenance;
 import com.yahoo.concurrent.DaemonThreadFactory;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.Deployer;
+import com.yahoo.jdisc.Metric;
 import com.yahoo.log.LogLevel;
 import com.yahoo.vespa.hosted.provision.NodeRepository;
 
@@ -22,6 +23,7 @@ import java.util.concurrent.TimeUnit;
 public abstract class ApplicationMaintainer extends Maintainer {
 
     private final Deployer deployer;
+    private final Metric metric;
     private final CopyOnWriteArrayList<ApplicationId> pendingDeployments = new CopyOnWriteArrayList<>();
 
     // Use a fixed thread pool to avoid overload on config servers. Resource usage when deploying varies
@@ -32,9 +34,10 @@ public abstract class ApplicationMaintainer extends Maintainer {
                                                                                  new LinkedBlockingQueue<>(),
                                                                                  new DaemonThreadFactory("node repo application maintainer"));
 
-    protected ApplicationMaintainer(Deployer deployer, NodeRepository nodeRepository, Duration interval) {
+    protected ApplicationMaintainer(Deployer deployer, Metric metric, NodeRepository nodeRepository, Duration interval) {
         super(nodeRepository, interval);
         this.deployer = deployer;
+        this.metric = metric;
     }
 
     @Override
@@ -73,7 +76,7 @@ public abstract class ApplicationMaintainer extends Maintainer {
 
     /** Redeploy this application. A lock will be taken for the duration of the deployment activation */
     protected final void deployWithLock(ApplicationId application) {
-        try (MaintenanceDeployment deployment = new MaintenanceDeployment(application, deployer, nodeRepository())) {
+        try (MaintenanceDeployment deployment = new MaintenanceDeployment(application, deployer, metric, nodeRepository())) {
             if ( ! deployment.isValid()) return; // this will be done at another config server
             if ( ! canDeployNow(application)) return; // redeployment is no longer needed
             deployment.activate();
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java
new file mode 100644
index 00000000000..0d7d10663c5
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java
@@ -0,0 +1,123 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.maintenance;
+
+import com.yahoo.collections.Pair;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.config.provision.Deployer;
+import com.yahoo.config.provision.NodeResources;
+import com.yahoo.jdisc.Metric;
+import com.yahoo.vespa.hosted.provision.Node;
+import com.yahoo.vespa.hosted.provision.NodeRepository;
+import com.yahoo.vespa.hosted.provision.applications.Application;
+import com.yahoo.vespa.hosted.provision.applications.Cluster;
+import com.yahoo.vespa.hosted.provision.autoscale.AllocatableClusterResources;
+import com.yahoo.vespa.hosted.provision.autoscale.Autoscaler;
+import com.yahoo.vespa.hosted.provision.autoscale.NodeMetricsDb;
+import com.yahoo.vespa.hosted.provision.provisioning.HostResourcesCalculator;
+
+import java.time.Duration;
+import java.time.Instant;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.stream.Collectors;
+
+/**
+ * Maintainer making automatic scaling decisions
+ *
+ * @author bratseth
+ */
+public class AutoscalingMaintainer extends Maintainer {
+
+    private final Autoscaler autoscaler;
+    private final Deployer deployer;
+    private final Metric metric;
+    private final Map<Pair<ApplicationId, ClusterSpec.Id>, Instant> lastLogged = new HashMap<>();
+
+    public AutoscalingMaintainer(NodeRepository nodeRepository,
+                                 HostResourcesCalculator hostResourcesCalculator,
+                                 NodeMetricsDb metricsDb,
+                                 Deployer deployer,
+                                 Metric metric,
+                                 Duration interval) {
+        super(nodeRepository, interval);
+        this.autoscaler = new Autoscaler(hostResourcesCalculator, metricsDb, nodeRepository);
+        this.metric = metric;
+        this.deployer = deployer;
+    }
+
+    @Override
+    protected void maintain() {
+        if ( ! nodeRepository().zone().environment().isProduction()) return;
+
+        activeNodesByApplication().forEach((applicationId, nodes) -> autoscale(applicationId, nodes));
+    }
+
+    private void autoscale(ApplicationId application, List<Node> applicationNodes) {
+        try (MaintenanceDeployment deployment = new MaintenanceDeployment(application, deployer, metric, nodeRepository())) {
+            if ( ! deployment.isValid()) return; // Another config server will consider this application
+            nodesByCluster(applicationNodes).forEach((clusterId, clusterNodes) -> autoscale(application, clusterId, clusterNodes, deployment));
+        }
+    }
+
+    private void autoscale(ApplicationId applicationId,
+                           ClusterSpec.Id clusterId,
+                           List<Node> clusterNodes,
+                           MaintenanceDeployment deployment) {
+        Application application = nodeRepository().applications().get(applicationId, true);
+        Cluster cluster = application.cluster(clusterId);
+        if (cluster == null) return; // no information on limits for this cluster
+        Optional<AllocatableClusterResources> target = autoscaler.autoscale(cluster, clusterNodes);
+        if (target.isEmpty()) return; // current resources are fine
+
+        if (cluster.minResources().equals(cluster.maxResources())) { // autoscaling is deactivated
+            logAutoscaling("Scaling suggestion for ", target.get(), applicationId, clusterId, clusterNodes);
+        }
+        else {
+            logAutoscaling("Autoscaling ", target.get(), applicationId, clusterId, clusterNodes);
+            autoscaleTo(target.get(), applicationId, clusterId, application, deployment);
+        }
+    }
+
+    private void autoscaleTo(AllocatableClusterResources target,
+                             ApplicationId applicationId,
+                             ClusterSpec.Id clusterId,
+                             Application application,
+                             MaintenanceDeployment deployment) {
+        nodeRepository().applications().set(applicationId,
+                                            application.withClusterTarget(clusterId, target.toAdvertisedClusterResources()),
+                                            deployment.applicationLock().get());
+        deployment.activate();
+    }
+
+    private void logAutoscaling(String prefix,
+                                AllocatableClusterResources target,
+                                ApplicationId application,
+                                ClusterSpec.Id clusterId,
+                                List<Node> clusterNodes) {
+        Instant lastLogTime = lastLogged.get(new Pair<>(application, clusterId));
+        if (lastLogTime != null && lastLogTime.isAfter(nodeRepository().clock().instant().minus(Duration.ofHours(1)))) return;
+
+        int currentGroups = (int)clusterNodes.stream().map(node -> node.allocation().get().membership().cluster().group()).distinct().count();
+        ClusterSpec.Type clusterType = clusterNodes.get(0).allocation().get().membership().cluster().type();
+        log.info(prefix + application + " " + clusterType + " " + clusterId + ":" +
+                 "\nfrom " + toString(clusterNodes.size(), currentGroups, clusterNodes.get(0).flavor().resources()) +
+                 "\nto   " + toString(target.nodes(), target.groups(), target.advertisedResources()));
+        lastLogged.put(new Pair<>(application, clusterId), nodeRepository().clock().instant());
+    }
+
+    private String toString(int nodes, int groups, NodeResources resources) {
+        return String.format(nodes + (groups > 1 ? " (in " + groups + " groups)" : "") +
+                             " * [vcpu: %0$.1f, memory: %1$.1f Gb, disk %2$.1f Gb]" +
+                             " (total: [vcpu: %3$.1f, memory: %4$.1f Gb, disk: %5$.1f Gb])",
+                             resources.vcpu(), resources.memoryGb(), resources.diskGb(),
+                             nodes * resources.vcpu(), nodes * resources.memoryGb(), nodes * resources.diskGb());
+    }
+
+    private Map<ClusterSpec.Id, List<Node>> nodesByCluster(List<Node> applicationNodes) {
+        return applicationNodes.stream().collect(Collectors.groupingBy(n -> n.allocation().get().membership().cluster().id()));
+    }
+
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java
index bb6d53d3304..906e6921f99 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java
@@ -17,7 +17,7 @@ import com.yahoo.vespa.hosted.provision.NodeRepository;
 import com.yahoo.vespa.hosted.provision.node.Agent;
 import com.yahoo.vespa.hosted.provision.provisioning.FatalProvisioningException;
 import com.yahoo.vespa.hosted.provision.provisioning.HostProvisioner;
-import com.yahoo.vespa.hosted.provision.provisioning.NodePrioritizer;
+import com.yahoo.vespa.hosted.provision.provisioning.HostResourcesCalculator;
 import com.yahoo.vespa.hosted.provision.provisioning.NodeResourceComparator;
 import com.yahoo.vespa.hosted.provision.provisioning.ProvisionedHost;
 import com.yahoo.yolean.Exceptions;
@@ -44,13 +44,15 @@ public class DynamicProvisioningMaintainer extends Maintainer {
     private static final ApplicationId preprovisionAppId = ApplicationId.from("hosted-vespa", "tenant-host", "preprovision");
 
     private final HostProvisioner hostProvisioner;
+    private final HostResourcesCalculator hostResourcesCalculator;
     private final BooleanFlag dynamicProvisioningEnabled;
     private final ListFlag<PreprovisionCapacity> preprovisionCapacityFlag;
 
-    DynamicProvisioningMaintainer(NodeRepository nodeRepository, Duration interval,
-                                  HostProvisioner hostProvisioner, FlagSource flagSource) {
+    DynamicProvisioningMaintainer(NodeRepository nodeRepository, Duration interval, HostProvisioner hostProvisioner,
+                                  HostResourcesCalculator hostResourcesCalculator, FlagSource flagSource) {
         super(nodeRepository, interval);
         this.hostProvisioner = hostProvisioner;
+        this.hostResourcesCalculator = hostResourcesCalculator;
         this.dynamicProvisioningEnabled = Flags.ENABLE_DYNAMIC_PROVISIONING.bindTo(flagSource);
         this.preprovisionCapacityFlag = Flags.PREPROVISION_CAPACITY.bindTo(flagSource);
     }
@@ -68,17 +70,14 @@ public class DynamicProvisioningMaintainer extends Maintainer {
     }
 
     void updateProvisioningNodes(NodeList nodes, Mutex lock) {
-        Map<String, Node> provisionedHostsByHostname = nodes.state(Node.State.provisioned).nodeType(NodeType.host)
-                .asList().stream()
-                .collect(Collectors.toMap(Node::hostname, Function.identity()));
-
-        Map<Node, Set<Node>> nodesByProvisionedParent = nodes.asList().stream()
-                .filter(node -> node.parentHostname().map(provisionedHostsByHostname::containsKey).orElse(false))
+        Map<String, Set<Node>> nodesByProvisionedParentHostname = nodes.nodeType(NodeType.tenant).asList().stream()
+                .filter(node -> node.parentHostname().isPresent())
                 .collect(Collectors.groupingBy(
-                        node -> provisionedHostsByHostname.get(node.parentHostname().get()),
+                        node -> node.parentHostname().get(),
                         Collectors.toSet()));
 
-        nodesByProvisionedParent.forEach((host, children) -> {
+        nodes.state(Node.State.provisioned).nodeType(NodeType.host).forEach(host -> {
+            Set<Node> children = nodesByProvisionedParentHostname.getOrDefault(host.hostname(), Set.of());
             try {
                 List<Node> updatedNodes = hostProvisioner.provision(host, children);
                 nodeRepository().write(updatedNodes, lock);
@@ -111,8 +110,8 @@ public class DynamicProvisioningMaintainer extends Maintainer {
         for (Iterator<NodeResources> it = preProvisionCapacity.iterator(); it.hasNext() && !removableHosts.isEmpty();) {
             NodeResources resources = it.next();
             removableHosts.stream()
-                    .filter(host -> NodePrioritizer.ALLOCATABLE_HOST_STATES.contains(host.state()))
-                    .filter(host -> host.flavor().resources().satisfies(resources))
+                    .filter(nodeRepository()::canAllocateTenantNodeTo)
+                    .filter(host -> hostResourcesCalculator.advertisedResourcesOf(host.flavor()).satisfies(resources))
                     .min(Comparator.comparingInt(n -> n.flavor().cost()))
                     .ifPresent(host -> {
                         removableHosts.remove(host);
@@ -127,7 +126,7 @@ public class DynamicProvisioningMaintainer extends Maintainer {
                         nodeRepository().database().getProvisionIndexes(1), resources, preprovisionAppId).stream()
                         .map(ProvisionedHost::generateHost)
                         .collect(Collectors.toList());
-                nodeRepository().addNodes(hosts);
+                nodeRepository().addNodes(hosts, Agent.DynamicProvisioningMaintainer);
             } catch (OutOfCapacityException | IllegalArgumentException | IllegalStateException e) {
                 log.log(Level.WARNING, "Failed to pre-provision " + resources + ":" + e.getMessage());
             } catch (RuntimeException e) {
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Expirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Expirer.java
index e2664cb14cd..ca576be8a69 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Expirer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Expirer.java
@@ -33,7 +33,7 @@ public abstract class Expirer extends Maintainer {
 
     Expirer(Node.State fromState, History.Event.Type eventType, NodeRepository nodeRepository,
             Clock clock, Duration expiryTime) {
-        super(nodeRepository, min(Duration.ofMinutes(25), expiryTime));
+        super(nodeRepository, min(Duration.ofMinutes(10), expiryTime));
         this.fromState = fromState;
         this.eventType = eventType;
         this.clock = clock;
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java
index 264df716fa8..af37aa5becf 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java
@@ -101,7 +101,7 @@ public class FailedExpirer extends Maintainer {
         for (Node candidate : nodes) {
             if (NodeFailer.hasHardwareIssue(candidate, nodeRepository)) {
                 List<String> unparkedChildren = !candidate.type().isDockerHost() ? Collections.emptyList() :
-                        nodeRepository.list().childrenOf(candidate).asList().stream()
+                                                nodeRepository.list().childrenOf(candidate).asList().stream()
                                       .filter(node -> node.state() != Node.State.parked)
                                       .map(Node::hostname)
                                       .collect(Collectors.toList());
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisioner.java
index 3392569d1f2..4d04409aaf0 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisioner.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisioner.java
@@ -25,15 +25,20 @@ public class InfrastructureProvisioner extends Maintainer {
         this.infraDeployer = infraDeployer;
     }
 
+    public void maintainButThrowOnException() {
+        try {
+            infraDeployer.activateAllSupportedInfraApplications(true);
+        } catch (RuntimeException e) {
+            logger.log(LogLevel.INFO, "Failed to deploy supported infrastructure applications, " +
+                    "will sleep 30s before propagating failure, to allow inspection of zk",
+                    e.getMessage());
+            try { Thread.sleep(30_000); } catch (InterruptedException ignored) { }
+            throw e;
+        }
+    }
+
     @Override
     protected void maintain() {
-        infraDeployer.getSupportedInfraDeployments().forEach((application, deployment) -> {
-            try {
-                deployment.activate();
-            } catch (RuntimeException e) {
-                logger.log(LogLevel.INFO, "Failed to activate " + application, e);
-                // loop around to activate the next application
-            }
-        });
+        infraDeployer.activateAllSupportedInfraApplications(false);
     }
 }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/JobControl.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/JobControl.java
index 6596d2abb1d..2d641ef57ab 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/JobControl.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/JobControl.java
@@ -1,41 +1,43 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.provision.maintenance;
 
-import com.yahoo.vespa.hosted.provision.persistence.CuratorDatabaseClient;
 import com.yahoo.vespa.curator.Lock;
+import com.yahoo.vespa.hosted.provision.persistence.CuratorDatabaseClient;
 
-import java.util.HashSet;
+import java.util.Collections;
+import java.util.Map;
 import java.util.Set;
-import java.util.concurrent.ConcurrentSkipListSet;
+import java.util.concurrent.ConcurrentSkipListMap;
 
 /**
  * Provides status and control over running maintenance jobs.
- * This is multithread safe.
+ *
+ * This is multi-thread safe.
  * 
  * @author bratseth
  */
 public class JobControl {
 
     /** This is not stored in ZooKeeper as all nodes start all jobs */
-    private final Set<String> startedJobs = new ConcurrentSkipListSet<>();
+    private final Map<String, Maintainer> startedJobs = new ConcurrentSkipListMap<>();
 
     /** Used to store deactivation in ZooKeeper to make changes take effect on all nodes */
     private final CuratorDatabaseClient db;
-    
+
     public JobControl(CuratorDatabaseClient db) {
         this.db = db;
     }
     
     /** Notifies this that a job was started */
-    public void started(String jobSimpleClassName) {
-        startedJobs.add(jobSimpleClassName);
+    public void started(String jobSimpleClassName, Maintainer maintainer) {
+        startedJobs.put(jobSimpleClassName, maintainer);
     }
 
     /**
      * Returns a snapshot of the set of jobs started on this system (whether deactivated or not).
      * Each job is represented by its simple (omitting package) class name.
      */
-    public Set<String> jobs() { return new HashSet<>(startedJobs); }
+    public Set<String> jobs() { return Collections.unmodifiableSet(startedJobs.keySet()); }
 
     /** Returns a snapshot containing the currently inactive jobs in this */
     public Set<String> inactiveJobs() { return db.readInactiveJobs(); }
@@ -56,5 +58,12 @@ public class JobControl {
             db.writeInactiveJobs(inactiveJobs);
         }
     }
+
+    /** Run given job (inactive or not) immediately */
+    public void run(String jobSimpleClassName) {
+        var job = startedJobs.get(jobSimpleClassName);
+        if (job == null) throw new IllegalArgumentException("No such job '" + jobSimpleClassName + "'");
+        job.runWithLock();
+    }
     
 }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java
index bcb0c901f14..7beb717ea8b 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java
@@ -15,6 +15,8 @@ import java.util.ArrayList;
 import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Objects;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Consumer;
 import java.util.stream.Collectors;
 
 /**
@@ -51,37 +53,31 @@ public class LoadBalancerExpirer extends Maintainer {
 
     /** Move reserved load balancer that have expired to inactive */
     private void expireReserved() {
-        try (var lock = db.lockLoadBalancers()) {
-            var now = nodeRepository().clock().instant();
-            var expirationTime = now.minus(reservedExpiry);
-            var expired = nodeRepository().loadBalancers()
-                                          .in(State.reserved)
-                                          .changedBefore(expirationTime);
-            expired.forEach(lb -> db.writeLoadBalancer(lb.with(State.inactive, now)));
-        }
+        var now = nodeRepository().clock().instant();
+        withLoadBalancersIn(State.reserved, lb -> {
+            var gracePeriod = now.minus(reservedExpiry);
+            if (!lb.changedAt().isBefore(gracePeriod)) return; // Should not move to inactive yet
+            db.writeLoadBalancer(lb.with(State.inactive, now));
+        });
     }
 
     /** Deprovision inactive load balancers that have expired */
     private void removeInactive() {
         var failed = new ArrayList<LoadBalancerId>();
-        Exception lastException = null;
-        try (var lock = db.lockLoadBalancers()) {
-            var now = nodeRepository().clock().instant();
-            var expirationTime = now.minus(inactiveExpiry);
-            var expired = nodeRepository().loadBalancers()
-                                          .in(State.inactive)
-                                          .changedBefore(expirationTime);
-            for (var lb : expired) {
-                if (!allocatedNodes(lb.id()).isEmpty()) continue; // Defer removal if there are still nodes allocated to application
-                try {
-                    service.remove(lb.id().application(), lb.id().cluster());
-                    db.removeLoadBalancer(lb.id());
-                } catch (Exception e) {
-                    failed.add(lb.id());
-                    lastException = e;
-                }
+        var lastException = new AtomicReference<Exception>();
+        var now = nodeRepository().clock().instant();
+        withLoadBalancersIn(State.inactive, lb -> {
+            var gracePeriod = now.minus(inactiveExpiry);
+            if (!lb.changedAt().isBefore(gracePeriod)) return; // Should not be removed yet
+            if (!allocatedNodes(lb.id()).isEmpty()) return;    // Still has nodes, do not remove
+            try {
+                service.remove(lb.id().application(), lb.id().cluster());
+                db.removeLoadBalancer(lb.id());
+            } catch (Exception e){
+                failed.add(lb.id());
+                lastException.set(e);
             }
-        }
+        });
         if (!failed.isEmpty()) {
             log.log(LogLevel.WARNING, String.format("Failed to remove %d load balancers: %s, retrying in %s",
                                                     failed.size(),
@@ -89,30 +85,27 @@ public class LoadBalancerExpirer extends Maintainer {
                                                           .map(LoadBalancerId::serializedForm)
                                                           .collect(Collectors.joining(", ")),
                                                     interval()),
-                    lastException);
+                    lastException.get());
         }
     }
 
     /** Remove reals from inactive load balancers */
     private void pruneReals() {
         var failed = new ArrayList<LoadBalancerId>();
-        Exception lastException = null;
-        try (var lock = db.lockLoadBalancers()) {
-            var deactivated = nodeRepository().loadBalancers().in(State.inactive);
-            for (var lb : deactivated) {
-                var allocatedNodes = allocatedNodes(lb.id()).stream().map(Node::hostname).collect(Collectors.toSet());
-                var reals = new LinkedHashSet<>(lb.instance().reals());
-                // Remove any real no longer allocated to this application
-                reals.removeIf(real -> !allocatedNodes.contains(real.hostname().value()));
-                try {
-                    service.create(lb.id().application(), lb.id().cluster(), reals, true);
-                    db.writeLoadBalancer(lb.with(lb.instance().withReals(reals)));
-                } catch (Exception e) {
-                    failed.add(lb.id());
-                    lastException = e;
-                }
+        var lastException = new AtomicReference<Exception>();
+        withLoadBalancersIn(State.inactive, lb -> {
+            var allocatedNodes = allocatedNodes(lb.id()).stream().map(Node::hostname).collect(Collectors.toSet());
+            var reals = new LinkedHashSet<>(lb.instance().reals());
+            // Remove any real no longer allocated to this application
+            reals.removeIf(real -> !allocatedNodes.contains(real.hostname().value()));
+            try {
+                service.create(lb.id().application(), lb.id().cluster(), reals, true);
+                db.writeLoadBalancer(lb.with(lb.instance().withReals(reals)));
+            } catch (Exception e) {
+                failed.add(lb.id());
+                lastException.set(e);
             }
-        }
+        });
         if (!failed.isEmpty()) {
             log.log(LogLevel.WARNING, String.format("Failed to remove reals from %d load balancers: %s, retrying in %s",
                                                     failed.size(),
@@ -120,7 +113,21 @@ public class LoadBalancerExpirer extends Maintainer {
                                                           .map(LoadBalancerId::serializedForm)
                                                           .collect(Collectors.joining(", ")),
                                                     interval()),
-                    lastException);
+                    lastException.get());
+        }
+    }
+
+    /** Apply operation to all load balancers that exist in given state, while holding lock */
+    private void withLoadBalancersIn(LoadBalancer.State state, Consumer<LoadBalancer> operation) {
+        for (var id : db.readLoadBalancerIds()) {
+            try (var lock = db.lockConfig(id.application())) {
+                try (var legacyLock = db.lockLoadBalancers(id.application())) {
+                    var loadBalancer = db.readLoadBalancer(id);
+                    if (loadBalancer.isEmpty()) continue;              // Load balancer was removed during loop
+                    if (loadBalancer.get().state() != state) continue; // Wrong state
+                    operation.accept(loadBalancer.get());
+                }
+            }
         }
     }
 
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Maintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Maintainer.java
index 0d5a8587902..17c73282b4b 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Maintainer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Maintainer.java
@@ -1,18 +1,24 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.provision.maintenance;
 
+import com.google.common.util.concurrent.UncheckedTimeoutException;
 import com.yahoo.component.AbstractComponent;
+import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.HostName;
+import com.yahoo.config.provision.NodeType;
+import com.yahoo.vespa.hosted.provision.Node;
 import com.yahoo.vespa.hosted.provision.NodeRepository;
 
 import java.time.Duration;
 import java.time.Instant;
 import java.util.List;
+import java.util.Map;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.ScheduledThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 import java.util.logging.Level;
 import java.util.logging.Logger;
+import java.util.stream.Collectors;
 
 /**
  * A maintainer is some job which runs at a fixed rate to perform some maintenance task on the node repo.
@@ -36,9 +42,9 @@ public abstract class Maintainer extends AbstractComponent implements Runnable {
 
         HostName hostname = HostName.from(com.yahoo.net.HostName.getLocalhost());
         long delay = staggeredDelay(nodeRepository.database().cluster(), hostname, nodeRepository.clock().instant(), interval);
-        service = new ScheduledThreadPoolExecutor(1);
+        service = new ScheduledThreadPoolExecutor(1, r -> new Thread(r, name() + "-worker"));
         service.scheduleAtFixedRate(this, delay, interval.toMillis(), TimeUnit.MILLISECONDS);
-        jobControl.started(name());
+        jobControl.started(name(), this);
     }
 
     /** Returns the node repository */
@@ -54,8 +60,11 @@ public abstract class Maintainer extends AbstractComponent implements Runnable {
     @Override
     public void run() {
         try {
-            if (jobControl.isActive(name()))
-                maintain();
+            if (jobControl.isActive(name())) {
+                runWithLock();
+            }
+        } catch (UncheckedTimeoutException ignored) {
+            // Another config server or operator is running this job
         } catch (Throwable e) {
             log.log(Level.WARNING, this + " failed. Will retry in " + interval.toMinutes() + " minutes", e);
         }
@@ -63,25 +72,48 @@ public abstract class Maintainer extends AbstractComponent implements Runnable {
 
     @Override
     public void deconstruct() {
-        this.service.shutdown();
+        var timeout = Duration.ofSeconds(30);
+        service.shutdown();
+        try {
+            if (!service.awaitTermination(timeout.toMillis(), TimeUnit.MILLISECONDS)) {
+                log.log(Level.WARNING, "Maintainer " + name() + " failed to shutdown " +
+                                       "within " + timeout);
+            }
+        } catch (InterruptedException e) {
+            throw new RuntimeException(e);
+        }
     }
 
     /** Returns the simple name of this job */
     @Override
     public final String toString() { return name(); }
 
+    /** Run this while holding the job lock */
+    public void runWithLock() {
+        try (var lock = nodeRepository.database().lockMaintenanceJob(name())) {
+            maintain();
+        }
+    }
+
     /** Called once each time this maintenance job should run */
     protected abstract void maintain();
     
     private String name() { return this.getClass().getSimpleName(); }
 
+    /** A utility to group active tenant nodes by application */
+    protected Map<ApplicationId, List<Node>> activeNodesByApplication() {
+        return nodeRepository().list().nodeType(NodeType.tenant).state(Node.State.active).asList()
+                               .stream()
+                               .filter(node -> ! node.allocation().get().owner().instance().isTester())
+                               .collect(Collectors.groupingBy(node -> node.allocation().get().owner()));
+    }
+
     static long staggeredDelay(List<HostName> cluster, HostName host, Instant now, Duration interval) {
         if ( ! cluster.contains(host))
             return interval.toMillis();
 
         long offset = cluster.indexOf(host) * interval.toMillis() / cluster.size();
-        long timeUntilNextRun = Math.floorMod(offset - now.toEpochMilli(), interval.toMillis());
-        return timeUntilNextRun + interval.toMillis() / cluster.size();
+        return Math.floorMod(offset - now.toEpochMilli(), interval.toMillis());
     }
 
 }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceDeployment.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceDeployment.java
index d25ef969c6b..8e1cd801a15 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceDeployment.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceDeployment.java
@@ -6,6 +6,7 @@ import com.yahoo.config.provision.ApplicationLockException;
 import com.yahoo.config.provision.Deployer;
 import com.yahoo.config.provision.Deployment;
 import com.yahoo.config.provision.TransientException;
+import com.yahoo.jdisc.Metric;
 import com.yahoo.log.LogLevel;
 import com.yahoo.transaction.Mutex;
 import com.yahoo.vespa.hosted.provision.Node;
@@ -14,6 +15,7 @@ import com.yahoo.yolean.Exceptions;
 
 import java.io.Closeable;
 import java.time.Duration;
+import java.util.Map;
 import java.util.Optional;
 import java.util.logging.Logger;
 
@@ -28,15 +30,26 @@ class MaintenanceDeployment implements Closeable {
     private static final Logger log = Logger.getLogger(MaintenanceDeployment.class.getName());
 
     private final ApplicationId application;
+    private final Metric metric;
     private final Optional<Mutex> lock;
     private final Optional<Deployment> deployment;
 
     private boolean closed = false;
 
-    public MaintenanceDeployment(ApplicationId application, Deployer deployer, NodeRepository nodeRepository) {
+    public MaintenanceDeployment(ApplicationId application,
+                                 Deployer deployer,
+                                 Metric metric,
+                                 NodeRepository nodeRepository) {
         this.application = application;
-        lock = tryLock(application, nodeRepository);
-        deployment = tryDeployment(lock, application, deployer, nodeRepository);
+        this.metric = metric;
+        Optional<Mutex> lock = tryLock(application, nodeRepository);
+        try {
+            deployment = tryDeployment(lock, application, deployer, nodeRepository);
+            this.lock = lock;
+            lock = Optional.empty();
+        } finally {
+            lock.ifPresent(Mutex::close);
+        }
     }
 
     /** Return whether this is - as yet - functional and can be used to carry out the deployment */
@@ -44,6 +57,16 @@ class MaintenanceDeployment implements Closeable {
         return deployment.isPresent();
     }
 
+    /**
+     * Returns the application lock held by this, or empty if it is not held.
+     *
+     * @throws IllegalStateException id this is called when closed
+     */
+    public Optional<Mutex> applicationLock() {
+        if (closed) throw new IllegalStateException(this + " is closed");
+        return lock;
+    }
+
     public boolean prepare() {
         return doStep(() -> deployment.get().prepare());
     }
@@ -53,16 +76,18 @@ class MaintenanceDeployment implements Closeable {
     }
 
     private boolean doStep(Runnable action) {
-        if (closed) throw new IllegalStateException("Deployment of '" + application + "' is closed");
+        if (closed) throw new IllegalStateException(this + "' is closed");
         if ( ! isValid()) return false;
         try {
             action.run();
             return true;
         } catch (TransientException e) {
+            metric.add("maintenanceDeployment.transientFailure", 1, metric.createContext(Map.of()));
             log.log(LogLevel.INFO, "Failed to maintenance deploy " + application + " with a transient error: " +
                                    Exceptions.toMessageString(e));
             return false;
         } catch (RuntimeException e) {
+            metric.add("maintenanceDeployment.failure", 1, metric.createContext(Map.of()));
             log.log(LogLevel.WARNING, "Exception on maintenance deploy of " + application, e);
             return false;
         }
@@ -93,4 +118,9 @@ class MaintenanceDeployment implements Closeable {
         closed = true;
     }
 
+    @Override
+    public String toString() {
+        return "deployment of " + application;
+    }
+
 }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java
index da0db1f1896..098d706bf05 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java
@@ -6,6 +6,7 @@ import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.NodeResources;
 import com.yahoo.config.provision.NodeType;
 import com.yahoo.jdisc.Metric;
+import com.yahoo.vespa.applicationmodel.ApplicationInstance;
 import com.yahoo.vespa.applicationmodel.HostName;
 import com.yahoo.vespa.applicationmodel.ServiceInstance;
 import com.yahoo.vespa.applicationmodel.ServiceStatus;
@@ -15,19 +16,20 @@ import com.yahoo.vespa.hosted.provision.NodeRepository;
 import com.yahoo.vespa.hosted.provision.node.Allocation;
 import com.yahoo.vespa.hosted.provision.node.History;
 import com.yahoo.vespa.orchestrator.Orchestrator;
-import com.yahoo.vespa.orchestrator.status.HostStatus;
+import com.yahoo.vespa.service.monitor.ServiceModel;
 import com.yahoo.vespa.service.monitor.ServiceMonitor;
 
+import java.time.Clock;
 import java.time.Duration;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
-import java.util.function.Function;
 import java.util.function.Supplier;
 import java.util.stream.Collectors;
 
 import static com.yahoo.config.provision.NodeResources.DiskSpeed.any;
+import static com.yahoo.vespa.hosted.provision.Node.State.active;
 
 /**
  * @author oyving
@@ -35,31 +37,33 @@ import static com.yahoo.config.provision.NodeResources.DiskSpeed.any;
 public class MetricsReporter extends Maintainer {
 
     private final Metric metric;
-    private final Function<HostName, Optional<HostStatus>> orchestrator;
+    private final Orchestrator orchestrator;
     private final ServiceMonitor serviceMonitor;
     private final Map<Map<String, String>, Metric.Context> contextMap = new HashMap<>();
     private final Supplier<Integer> pendingRedeploymentsSupplier;
+    private final Clock clock;
 
     MetricsReporter(NodeRepository nodeRepository,
                     Metric metric,
                     Orchestrator orchestrator,
                     ServiceMonitor serviceMonitor,
                     Supplier<Integer> pendingRedeploymentsSupplier,
-                    Duration interval) {
+                    Duration interval,
+                    Clock clock) {
         super(nodeRepository, interval);
         this.metric = metric;
-        this.orchestrator = orchestrator.getNodeStatuses();
+        this.orchestrator = orchestrator;
         this.serviceMonitor = serviceMonitor;
         this.pendingRedeploymentsSupplier = pendingRedeploymentsSupplier;
+        this.clock = clock;
     }
 
     @Override
     public void maintain() {
         NodeList nodes = nodeRepository().list();
-        Map<HostName, List<ServiceInstance>> servicesByHost =
-                serviceMonitor.getServiceModelSnapshot().getServiceInstancesByHostName();
+        ServiceModel serviceModel = serviceMonitor.getServiceModelSnapshot();
 
-        nodes.forEach(node -> updateNodeMetrics(node, servicesByHost));
+        nodes.forEach(node -> updateNodeMetrics(node, serviceModel));
         updateStateMetrics(nodes);
         updateMaintenanceMetrics();
         updateDockerMetrics(nodes);
@@ -70,7 +74,7 @@ public class MetricsReporter extends Maintainer {
         metric.set("hostedVespa.pendingRedeployments", pendingRedeploymentsSupplier.get(), null);
     }
 
-    private void updateNodeMetrics(Node node, Map<HostName, List<ServiceInstance>> servicesByHost) {
+    private void updateNodeMetrics(Node node, ServiceModel serviceModel) {
         Metric.Context context;
 
         Optional<Allocation> allocation = node.allocation();
@@ -125,13 +129,23 @@ public class MetricsReporter extends Maintainer {
         metric.set("wantToDeprovision", node.status().wantToDeprovision() ? 1 : 0, context);
         metric.set("failReport", NodeFailer.reasonsToFailParentHost(node).isEmpty() ? 0 : 1, context);
 
-        orchestrator.apply(new HostName(node.hostname()))
-                    .map(status -> status == HostStatus.ALLOWED_TO_BE_DOWN ? 1 : 0)
-                    .ifPresent(allowedToBeDown -> metric.set("allowedToBeDown", allowedToBeDown, context));
+        HostName hostname = new HostName(node.hostname());
+
+        serviceModel.getApplication(hostname)
+                .map(ApplicationInstance::reference)
+                .map(reference -> orchestrator.getHostInfo(reference, hostname))
+                .ifPresent(info -> {
+                    int suspended = info.status().isSuspended() ? 1 : 0;
+                    metric.set("suspended", suspended, context);
+                    metric.set("allowedToBeDown", suspended, context); // remove summer 2020.
+                    long suspendedSeconds = info.suspendedSince()
+                            .map(suspendedSince -> Duration.between(suspendedSince, clock.instant()).getSeconds())
+                            .orElse(0L);
+                    metric.set("suspendedSeconds", suspendedSeconds, context);
+                });
 
         long numberOfServices;
-        HostName hostName = new HostName(node.hostname());
-        List<ServiceInstance> services = servicesByHost.get(hostName);
+        List<ServiceInstance> services = serviceModel.getServiceInstancesByHostName().get(hostname);
         if (services == null) {
             numberOfServices = 0;
         } else {
@@ -236,16 +250,16 @@ public class MetricsReporter extends Maintainer {
     }
 
     private static NodeResources getCapacityTotal(NodeList nodes) {
-        return nodes.nodeType(NodeType.host).asList().stream()
+        return nodes.nodeType(NodeType.host).state(active).asList().stream()
                 .map(host -> host.flavor().resources())
-                .map(resources -> resources.justNumbers())
+                .map(NodeResources::justNumbers)
                 .reduce(new NodeResources(0, 0, 0, 0, any), NodeResources::add);
     }
 
     private static NodeResources getFreeCapacityTotal(NodeList nodes) {
-        return nodes.nodeType(NodeType.host).asList().stream()
+        return nodes.nodeType(NodeType.host).state(active).asList().stream()
                 .map(n -> freeCapacityOf(nodes, n))
-                .map(resources -> resources.justNumbers())
+                .map(NodeResources::justNumbers)
                 .reduce(new NodeResources(0, 0, 0, 0, any), NodeResources::add);
     }
 
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java
index b80446b06da..b0b8f7d8d2c 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java
@@ -21,7 +21,6 @@ import com.yahoo.vespa.orchestrator.ApplicationIdNotFoundException;
 import com.yahoo.vespa.orchestrator.HostNameNotFoundException;
 import com.yahoo.vespa.orchestrator.Orchestrator;
 import com.yahoo.vespa.orchestrator.status.ApplicationInstanceStatus;
-import com.yahoo.vespa.orchestrator.status.HostStatus;
 import com.yahoo.vespa.service.monitor.ServiceMonitor;
 import com.yahoo.yolean.Exceptions;
 
@@ -265,7 +264,7 @@ public class NodeFailer extends Maintainer {
 
     private boolean nodeSuspended(Node node) {
         try {
-            return orchestrator.getNodeStatus(new HostName(node.hostname())) == HostStatus.ALLOWED_TO_BE_DOWN;
+            return orchestrator.getNodeStatus(new HostName(node.hostname())).isSuspended();
         } catch (HostNameNotFoundException e) {
             // Treat it as not suspended
             return false;
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainer.java
new file mode 100644
index 00000000000..adc1911a169
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainer.java
@@ -0,0 +1,51 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.maintenance;
+
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.vespa.hosted.provision.autoscale.NodeMetrics;
+import com.yahoo.vespa.hosted.provision.NodeRepository;
+import com.yahoo.vespa.hosted.provision.autoscale.NodeMetricsDb;
+import com.yahoo.yolean.Exceptions;
+
+import java.time.Duration;
+import java.util.logging.Level;
+
+/**
+ * Maintainer which keeps the node metric db up to date by periodically fetching metrics from all
+ * active nodes.
+ *
+ * @author bratseth
+ */
+public class NodeMetricsDbMaintainer extends Maintainer {
+
+    private static final int maxWarningsPerInvocation = 2;
+
+    private final NodeMetrics nodeMetrics;
+    private final NodeMetricsDb nodeMetricsDb;
+
+    public NodeMetricsDbMaintainer(NodeRepository nodeRepository,
+                                   NodeMetrics nodeMetrics,
+                                   NodeMetricsDb nodeMetricsDb,
+                                   Duration interval) {
+        super(nodeRepository, interval);
+        this.nodeMetrics = nodeMetrics;
+        this.nodeMetricsDb = nodeMetricsDb;
+    }
+
+    @Override
+    protected void maintain() {
+        int warnings = 0;
+        for (ApplicationId application : activeNodesByApplication().keySet()) {
+            try {
+                nodeMetricsDb.add(nodeMetrics.fetchMetrics(application));
+            }
+            catch (Exception e) {
+                // TODO: Don't warn if this only happens occasionally
+                if (warnings++ < maxWarningsPerInvocation)
+                    log.log(Level.WARNING, "Could not update metrics for " + application + ": " + Exceptions.toMessageString(e));
+            }
+        }
+        nodeMetricsDb.gc(nodeRepository().clock());
+    }
+
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java
index 10aff833584..054c273dc99 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java
@@ -11,6 +11,8 @@ import com.yahoo.config.provision.Zone;
 import com.yahoo.jdisc.Metric;
 import com.yahoo.vespa.flags.FlagSource;
 import com.yahoo.vespa.hosted.provision.NodeRepository;
+import com.yahoo.vespa.hosted.provision.autoscale.NodeMetrics;
+import com.yahoo.vespa.hosted.provision.autoscale.NodeMetricsDb;
 import com.yahoo.vespa.hosted.provision.provisioning.ProvisionServiceProvider;
 import com.yahoo.vespa.orchestrator.Orchestrator;
 import com.yahoo.vespa.service.monitor.ServiceMonitor;
@@ -48,46 +50,51 @@ public class NodeRepositoryMaintenance extends AbstractComponent {
     private final CapacityReportMaintainer capacityReportMaintainer;
     private final OsUpgradeActivator osUpgradeActivator;
     private final Rebalancer rebalancer;
+    private final NodeMetricsDbMaintainer nodeMetricsDbMaintainer;
+    private final AutoscalingMaintainer autoscalingMaintainer;
 
     @SuppressWarnings("unused")
     @Inject
     public NodeRepositoryMaintenance(NodeRepository nodeRepository, Deployer deployer, InfraDeployer infraDeployer,
                                      HostLivenessTracker hostLivenessTracker, ServiceMonitor serviceMonitor,
                                      Zone zone, Orchestrator orchestrator, Metric metric,
-                                     ProvisionServiceProvider provisionServiceProvider,
-                                     FlagSource flagSource) {
+                                     ProvisionServiceProvider provisionServiceProvider, FlagSource flagSource,
+                                     NodeMetrics nodeMetrics, NodeMetricsDb nodeMetricsDb) {
         this(nodeRepository, deployer, infraDeployer, hostLivenessTracker, serviceMonitor, zone, Clock.systemUTC(),
-             orchestrator, metric, provisionServiceProvider, flagSource);
+             orchestrator, metric, provisionServiceProvider, flagSource, nodeMetrics, nodeMetricsDb);
     }
 
     public NodeRepositoryMaintenance(NodeRepository nodeRepository, Deployer deployer, InfraDeployer infraDeployer,
                                      HostLivenessTracker hostLivenessTracker, ServiceMonitor serviceMonitor,
                                      Zone zone, Clock clock, Orchestrator orchestrator, Metric metric,
-                                     ProvisionServiceProvider provisionServiceProvider, FlagSource flagSource) {
+                                     ProvisionServiceProvider provisionServiceProvider, FlagSource flagSource,
+                                     NodeMetrics nodeMetrics, NodeMetricsDb nodeMetricsDb) {
         DefaultTimes defaults = new DefaultTimes(zone);
 
         nodeFailer = new NodeFailer(deployer, hostLivenessTracker, serviceMonitor, nodeRepository, defaults.failGrace, clock, orchestrator, throttlePolicyFromEnv().orElse(defaults.throttlePolicy), metric);
-        periodicApplicationMaintainer = new PeriodicApplicationMaintainer(deployer, nodeRepository, defaults.redeployMaintainerInterval, defaults.periodicRedeployInterval);
-        operatorChangeApplicationMaintainer = new OperatorChangeApplicationMaintainer(deployer, nodeRepository, defaults.operatorChangeRedeployInterval);
+        periodicApplicationMaintainer = new PeriodicApplicationMaintainer(deployer, metric, nodeRepository, defaults.redeployMaintainerInterval, defaults.periodicRedeployInterval);
+        operatorChangeApplicationMaintainer = new OperatorChangeApplicationMaintainer(deployer, metric, nodeRepository, defaults.operatorChangeRedeployInterval);
         reservationExpirer = new ReservationExpirer(nodeRepository, clock, defaults.reservationExpiry);
-        retiredExpirer = new RetiredExpirer(nodeRepository, orchestrator, deployer, clock, defaults.retiredInterval, defaults.retiredExpiry);
+        retiredExpirer = new RetiredExpirer(nodeRepository, orchestrator, deployer, metric, clock, defaults.retiredInterval, defaults.retiredExpiry);
         inactiveExpirer = new InactiveExpirer(nodeRepository, clock, defaults.inactiveExpiry);
         failedExpirer = new FailedExpirer(nodeRepository, zone, clock, defaults.failedExpirerInterval);
         dirtyExpirer = new DirtyExpirer(nodeRepository, clock, defaults.dirtyExpiry);
         provisionedExpirer = new ProvisionedExpirer(nodeRepository, clock, defaults.provisionedExpiry);
         nodeRebooter = new NodeRebooter(nodeRepository, clock, flagSource);
-        metricsReporter = new MetricsReporter(nodeRepository, metric, orchestrator, serviceMonitor, periodicApplicationMaintainer::pendingDeployments, defaults.metricsInterval);
+        metricsReporter = new MetricsReporter(nodeRepository, metric, orchestrator, serviceMonitor, periodicApplicationMaintainer::pendingDeployments, defaults.metricsInterval, clock);
         infrastructureProvisioner = new InfrastructureProvisioner(nodeRepository, infraDeployer, defaults.infrastructureProvisionInterval);
         loadBalancerExpirer = provisionServiceProvider.getLoadBalancerService().map(lbService ->
                 new LoadBalancerExpirer(nodeRepository, defaults.loadBalancerExpirerInterval, lbService));
         dynamicProvisioningMaintainer = provisionServiceProvider.getHostProvisioner().map(hostProvisioner ->
-                new DynamicProvisioningMaintainer(nodeRepository, defaults.dynamicProvisionerInterval, hostProvisioner, flagSource));
+                new DynamicProvisioningMaintainer(nodeRepository, defaults.dynamicProvisionerInterval, hostProvisioner, provisionServiceProvider.getHostResourcesCalculator(), flagSource));
         capacityReportMaintainer = new CapacityReportMaintainer(nodeRepository, metric, defaults.capacityReportInterval);
         osUpgradeActivator = new OsUpgradeActivator(nodeRepository, defaults.osUpgradeActivatorInterval);
         rebalancer = new Rebalancer(deployer, nodeRepository, provisionServiceProvider.getHostResourcesCalculator(), provisionServiceProvider.getHostProvisioner(), metric, clock, defaults.rebalancerInterval);
+        nodeMetricsDbMaintainer = new NodeMetricsDbMaintainer(nodeRepository, nodeMetrics, nodeMetricsDb, defaults.nodeMetricsCollectionInterval);
+        autoscalingMaintainer = new AutoscalingMaintainer(nodeRepository, provisionServiceProvider.getHostResourcesCalculator(), nodeMetricsDb, deployer, metric, defaults.autoscalingInterval);
 
         // The DuperModel is filled with infrastructure applications by the infrastructure provisioner, so explicitly run that now
-        infrastructureProvisioner.maintain();
+        infrastructureProvisioner.maintainButThrowOnException();
     }
 
     @Override
@@ -109,6 +116,8 @@ public class NodeRepositoryMaintenance extends AbstractComponent {
         dynamicProvisioningMaintainer.ifPresent(Maintainer::deconstruct);
         osUpgradeActivator.deconstruct();
         rebalancer.deconstruct();
+        nodeMetricsDbMaintainer.deconstruct();
+        autoscalingMaintainer.deconstruct();
     }
 
     private static Optional<NodeFailer.ThrottlePolicy> throttlePolicyFromEnv() {
@@ -149,6 +158,8 @@ public class NodeRepositoryMaintenance extends AbstractComponent {
         private final Duration dynamicProvisionerInterval;
         private final Duration osUpgradeActivatorInterval;
         private final Duration rebalancerInterval;
+        private final Duration nodeMetricsCollectionInterval;
+        private final Duration autoscalingInterval;
 
         private final NodeFailer.ThrottlePolicy throttlePolicy;
 
@@ -165,10 +176,12 @@ public class NodeRepositoryMaintenance extends AbstractComponent {
             infrastructureProvisionInterval = Duration.ofMinutes(1);
             throttlePolicy = NodeFailer.ThrottlePolicy.hosted;
             loadBalancerExpirerInterval = Duration.ofMinutes(5);
-            reservationExpiry = Duration.ofMinutes(20); // Need to be long enough for deployment to be finished for all config model versions
+            reservationExpiry = Duration.ofMinutes(15); // Need to be long enough for deployment to be finished for all config model versions
             dynamicProvisionerInterval = Duration.ofMinutes(5);
             osUpgradeActivatorInterval = zone.system().isCd() ? Duration.ofSeconds(30) : Duration.ofMinutes(5);
             rebalancerInterval = Duration.ofMinutes(40);
+            nodeMetricsCollectionInterval = Duration.ofMinutes(1);
+            autoscalingInterval = Duration.ofMinutes(5);
 
             if (zone.environment().equals(Environment.prod) && ! zone.system().isCd()) {
                 inactiveExpiry = Duration.ofHours(4); // enough time for the application owner to discover and redeploy
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainer.java
index e7406bf3478..9f829c095f4 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainer.java
@@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.provision.maintenance;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.Deployer;
 import com.yahoo.config.provision.NodeType;
+import com.yahoo.jdisc.Metric;
 import com.yahoo.vespa.hosted.provision.Node;
 import com.yahoo.vespa.hosted.provision.NodeRepository;
 import com.yahoo.vespa.hosted.provision.node.Agent;
@@ -31,8 +32,8 @@ import java.util.stream.Collectors;
  */
 public class OperatorChangeApplicationMaintainer extends ApplicationMaintainer {
     
-    OperatorChangeApplicationMaintainer(Deployer deployer, NodeRepository nodeRepository, Duration interval) {
-        super(deployer, nodeRepository, interval);
+    OperatorChangeApplicationMaintainer(Deployer deployer, Metric metric, NodeRepository nodeRepository, Duration interval) {
+        super(deployer, metric, nodeRepository, interval);
     }
 
     @Override
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainer.java
index 6ab85e76ba2..d06d24872e1 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainer.java
@@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.provision.maintenance;
 
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.Deployer;
+import com.yahoo.jdisc.Metric;
 import com.yahoo.vespa.hosted.provision.Node;
 import com.yahoo.vespa.hosted.provision.NodeRepository;
 
@@ -29,9 +30,9 @@ public class PeriodicApplicationMaintainer extends ApplicationMaintainer {
     private final Clock clock;
     private final Instant start;
 
-    PeriodicApplicationMaintainer(Deployer deployer, NodeRepository nodeRepository,
+    PeriodicApplicationMaintainer(Deployer deployer, Metric metric, NodeRepository nodeRepository,
                                   Duration interval, Duration minTimeBetweenRedeployments) {
-        super(deployer, nodeRepository, interval);
+        super(deployer, metric, nodeRepository, interval);
         this.minTimeBetweenRedeployments = minTimeBetweenRedeployments;
         this.clock = nodeRepository.clock();
         this.start = clock.instant();
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Rebalancer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Rebalancer.java
index 675e3400722..6e8af120b3b 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Rebalancer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Rebalancer.java
@@ -14,7 +14,6 @@ import com.yahoo.vespa.hosted.provision.node.Agent;
 import com.yahoo.vespa.hosted.provision.provisioning.DockerHostCapacity;
 import com.yahoo.vespa.hosted.provision.provisioning.HostProvisioner;
 import com.yahoo.vespa.hosted.provision.provisioning.HostResourcesCalculator;
-import com.yahoo.vespa.hosted.provision.provisioning.NodePrioritizer;
 
 import java.time.Clock;
 import java.time.Duration;
@@ -92,7 +91,8 @@ public class Rebalancer extends Maintainer {
         for (Node node : allNodes.nodeType(NodeType.tenant).state(Node.State.active)) {
             if (node.parentHostname().isEmpty()) continue;
             if (node.allocation().get().owner().instance().isTester()) continue;
-            for (Node toHost : allNodes.nodeType(NodeType.host).state(NodePrioritizer.ALLOCATABLE_HOST_STATES)) {
+            if (node.allocation().get().owner().application().value().equals("lsbe-dictionaries")) continue; // TODO: Remove
+            for (Node toHost : allNodes.filter(nodeRepository()::canAllocateTenantNodeTo)) {
                 if (toHost.hostname().equals(node.parentHostname().get())) continue;
                 if ( ! capacity.freeCapacityOf(toHost).satisfies(node.flavor().resources())) continue;
 
@@ -128,7 +128,7 @@ public class Rebalancer extends Maintainer {
      */
     private boolean deployTo(Move move) {
         ApplicationId application = move.node.allocation().get().owner();
-        try (MaintenanceDeployment deployment = new MaintenanceDeployment(application, deployer, nodeRepository())) {
+        try (MaintenanceDeployment deployment = new MaintenanceDeployment(application, deployer, metric, nodeRepository())) {
             if ( ! deployment.isValid()) return false;
 
             boolean couldMarkRetiredNow = markWantToRetire(move.node, true);
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirer.java
index 1d31917b3e1..3c01c8bc23c 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirer.java
@@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.provision.maintenance;
 import com.google.common.util.concurrent.UncheckedTimeoutException;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.Deployer;
+import com.yahoo.jdisc.Metric;
 import com.yahoo.vespa.applicationmodel.HostName;
 import com.yahoo.vespa.hosted.provision.Node;
 import com.yahoo.vespa.hosted.provision.NodeRepository;
@@ -26,6 +27,7 @@ import java.util.stream.Collectors;
 public class RetiredExpirer extends Maintainer {
 
     private final Deployer deployer;
+    private final Metric metric;
     private final Orchestrator orchestrator;
     private final Duration retiredExpiry;
     private final Clock clock;
@@ -33,11 +35,13 @@ public class RetiredExpirer extends Maintainer {
     public RetiredExpirer(NodeRepository nodeRepository,
                           Orchestrator orchestrator,
                           Deployer deployer,
+                          Metric metric,
                           Clock clock,
                           Duration maintenanceInterval,
                           Duration retiredExpiry) {
         super(nodeRepository, maintenanceInterval);
         this.deployer = deployer;
+        this.metric = metric;
         this.orchestrator = orchestrator;
         this.retiredExpiry = retiredExpiry;
         this.clock = clock;
@@ -56,7 +60,7 @@ public class RetiredExpirer extends Maintainer {
             ApplicationId application = entry.getKey();
             List<Node> retiredNodes = entry.getValue();
 
-            try (MaintenanceDeployment deployment = new MaintenanceDeployment(application, deployer, nodeRepository())) {
+            try (MaintenanceDeployment deployment = new MaintenanceDeployment(application, deployer, metric, nodeRepository())) {
                 if ( ! deployment.isValid()) continue; // this will be done at another config server
 
                 List<Node> nodesToRemove = retiredNodes.stream().filter(this::canRemove).collect(Collectors.toList());
@@ -103,7 +107,7 @@ public class RetiredExpirer extends Maintainer {
             log.info("Node " + node + " has been granted permission to be removed");
             return true;
         } catch (UncheckedTimeoutException e) {
-            log.info("Timed out trying to aquire permission to remove " + node.hostname() + ": " + e.getMessage());
+            log.info("Timed out trying to acquire permission to remove " + node.hostname() + ": " + e.getMessage());
             return false;
         } catch (OrchestrationException e) {
             log.info("Did not get permission to remove retired " + node + ": " + e.getMessage());
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Agent.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Agent.java
index 7522e411e42..18a8fe7be6a 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Agent.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Agent.java
@@ -2,13 +2,13 @@
 package com.yahoo.vespa.hosted.provision.node;
 
 /**
- * The enum of kinds of actions making changes to the system.
+ * The enum of agents making changes to the system.
  * 
  * @author bratseth
  */
 public enum Agent {
     operator, // A hosted Vespa operator. Some logic recognizes these events.
-    application, // An application package change depoyment
+    application, // An application package change deployment
     system, // An unspecified system agent
     // Specific system agents:
     NodeFailer,
@@ -17,5 +17,6 @@ public enum Agent {
     FailedExpirer,
     InactiveExpirer,
     ProvisionedExpirer,
-    ReservationExpirer
+    ReservationExpirer,
+    DynamicProvisioningMaintainer
 }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/History.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/History.java
index ef6f531cc89..959071c83c4 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/History.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/History.java
@@ -80,15 +80,16 @@ public class History {
         // If the event is a re-reservation, allow the new one to override the older one.
         if (from == to && from != Node.State.reserved) return this;
         switch (to) {
-            case provisioned: return this.with(new Event(Event.Type.provisioned, agent, at));
-            case ready:       return this.withoutApplicationEvents().with(new Event(Event.Type.readied, agent, at));
-            case active:      return this.with(new Event(Event.Type.activated, agent, at));
-            case inactive:    return this.with(new Event(Event.Type.deactivated, agent, at));
-            case reserved:    return this.with(new Event(Event.Type.reserved, agent, at));
-            case failed:      return this.with(new Event(Event.Type.failed, agent, at));
-            case dirty:       return this.with(new Event(Event.Type.deallocated, agent, at));
-            case parked:      return this.with(new Event(Event.Type.parked, agent, at));
-            default:          return this;
+            case provisioned:   return this.with(new Event(Event.Type.provisioned, agent, at));
+            case deprovisioned: return this.with(new Event(Event.Type.deprovisioned, agent, at));
+            case ready:         return this.withoutApplicationEvents().with(new Event(Event.Type.readied, agent, at));
+            case active:        return this.with(new Event(Event.Type.activated, agent, at));
+            case inactive:      return this.with(new Event(Event.Type.deactivated, agent, at));
+            case reserved:      return this.with(new Event(Event.Type.reserved, agent, at));
+            case failed:        return this.with(new Event(Event.Type.failed, agent, at));
+            case dirty:         return this.with(new Event(Event.Type.deallocated, agent, at));
+            case parked:        return this.with(new Event(Event.Type.parked, agent, at));
+            default:            return this;
         }
     }
     
@@ -128,7 +129,7 @@ public class History {
 
         public enum Type { 
             // State move events
-            provisioned(false), readied, reserved, activated, deactivated, deallocated, parked,
+            provisioned(false), deprovisioned(false), readied, reserved, activated, deactivated, deallocated, parked,
             // The node was scheduled for retirement
             wantToRetire,
             // The active node was retired
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/OsVersion.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/OsVersion.java
index b06bbbb54b5..0622862f5ab 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/OsVersion.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/OsVersion.java
@@ -33,6 +33,11 @@ public class OsVersion {
         return wanted;
     }
 
+    /** Returns whether this node is currently changing its version to the given version */
+    public boolean changingTo(Version version) {
+        return changing() && wanted.get().equals(version);
+    }
+
     /** Returns whether this node is currently changing its version */
     public boolean changing() {
         return wanted.isPresent() && !current.equals(wanted);
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Report.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Report.java
index af32530a156..37141d8f25b 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Report.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Report.java
@@ -4,7 +4,7 @@ package com.yahoo.vespa.hosted.provision.node;
 import com.yahoo.slime.Cursor;
 import com.yahoo.slime.Inspector;
 import com.yahoo.slime.Slime;
-import com.yahoo.vespa.config.SlimeUtils;
+import com.yahoo.slime.SlimeUtils;
 
 import java.time.Instant;
 import java.util.Arrays;
@@ -17,10 +17,13 @@ import java.util.Arrays;
  * @author hakonhall
  */
 public class Report {
+
     /** The time the report was created, in milliseconds since Epoch. */
     public static final String CREATED_FIELD = "createdMillis";
+
     /** The type of the report. */
     public static final String TYPE_FIELD = "type";
+
     /** The description of the report. */
     public static final String DESCRIPTION_FIELD = "description";
 
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Reports.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Reports.java
index fd6094ae111..8680c569d44 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Reports.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Reports.java
@@ -17,6 +17,7 @@ import java.util.TreeMap;
  */
 // @Immutable
 public class Reports {
+
     private final Map<String, Report> reports;
 
     public Reports() { this(Collections.emptyMap()); }
@@ -71,6 +72,6 @@ public class Reports {
             return this;
         }
 
-        public Reports build() { return new Reports(Collections.unmodifiableMap(reportMap)); }
+        public Reports build() { return new Reports(reportMap); }
     }
 }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Status.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Status.java
index 15f3c481fe3..6b52bd68e73 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Status.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Status.java
@@ -64,7 +64,7 @@ public class Status {
 
     public Status withDecreasedFailCount() { return new Status(reboot, vespaVersion, dockerImage, failCount - 1, wantToRetire, wantToDeprovision, osVersion, firmwareVerifiedAt); }
 
-    public Status setFailCount(Integer value) { return new Status(reboot, vespaVersion, dockerImage, value, wantToRetire, wantToDeprovision, osVersion, firmwareVerifiedAt); }
+    public Status withFailCount(int value) { return new Status(reboot, vespaVersion, dockerImage, value, wantToRetire, wantToDeprovision, osVersion, firmwareVerifiedAt); }
 
     /** Returns how many times this node has been moved to the failed state. */
     public int failCount() { return failCount; }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsVersions.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsVersions.java
index 9b21d2aceba..e10ff3d24cd 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsVersions.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsVersions.java
@@ -120,8 +120,8 @@ public class OsVersions {
     /** Trigger upgrade of nodes of given type*/
     private void upgrade(NodeType type, Version version) {
         var nodes = nodeRepository.list().nodeType(type);
-        var numberToUpgrade = Math.max(0, maxActiveUpgrades - nodes.changingOsVersion().size());
-        var nodesToUpgrade = nodes.not().changingOsVersion()
+        var numberToUpgrade = Math.max(0, maxActiveUpgrades - nodes.changingOsVersionTo(version).size());
+        var nodesToUpgrade = nodes.not().changingOsVersionTo(version)
                                   .not().onOsVersion(version)
                                   .byIncreasingOsVersion()
                                   .first(numberToUpgrade);
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabase.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabase.java
index e12cd3e53b9..08f2cfec40f 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabase.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabase.java
@@ -38,18 +38,12 @@ public class CuratorDatabase {
     /** A partial cache of the Curator database, which is only valid if generations match */
     private final AtomicReference<Cache> cache = new AtomicReference<>();
 
-    /** Whether we should return data from the cache or always read fro ZooKeeper */
+    /** Whether we should return data from the cache or always read from ZooKeeper */
     private final boolean useCache;
 
     private final Object cacheCreationLock = new Object();
 
     /**
-     * All keys, to allow reentrancy.
-     * This will grow forever with the number of applications seen, but this should be too slow to be a problem.
-     */
-    private final ConcurrentHashMap<Path, Lock> locks = new ConcurrentHashMap<>();
-
-    /**
      * Creates a curator database
      *
      * @param curator the curator instance
@@ -72,11 +66,8 @@ public class CuratorDatabase {
     }
 
     /** Create a reentrant lock */
-    // Locks are not cached in the in-memory state
     public Lock lock(Path path, Duration timeout) {
-        Lock lock = locks.computeIfAbsent(path, (pathArg) -> new Lock(pathArg.getAbsolute(), curator));
-        lock.acquire(timeout);
-        return lock;
+        return curator.lock(path, timeout);
     }
 
     // --------- Write operations ------------------------------------------------------------------------------
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java
index ce0bcc2e337..6036cc2366f 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java
@@ -1,4 +1,4 @@
-// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.provision.persistence;
 
 import com.google.common.util.concurrent.UncheckedTimeoutException;
@@ -38,6 +38,7 @@ import java.util.Optional;
 import java.util.Set;
 import java.util.TreeMap;
 import java.util.function.Function;
+import java.util.function.Predicate;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 import java.util.stream.Collectors;
@@ -61,6 +62,7 @@ public class CuratorDatabaseClient {
 
     private static final Path root = Path.fromString("/provision/v1");
     private static final Path lockRoot = root.append("locks");
+    private static final Path configLockRoot = Path.fromString("/config/v2/locks/");
     private static final Path loadBalancersRoot = root.append("loadBalancers");
     private static final Duration defaultLockTimeout = Duration.ofMinutes(2);
 
@@ -311,7 +313,7 @@ public class CuratorDatabaseClient {
         return root.append(toDir(nodeState)).append(nodeName);
     }
 
-    /** Creates an returns the path to the lock for this application */
+    /** Creates and returns the path to the lock for this application */
     private Path lockPath(ApplicationId application) {
         Path lockPath =
                 lockRoot
@@ -322,6 +324,14 @@ public class CuratorDatabaseClient {
         return lockPath;
     }
 
+    /** Creates and returns the path to the config server lock for this application */
+    private Path configLockPath(ApplicationId application) {
+        // This must match the lock path used by com.yahoo.vespa.config.server.application.TenantApplications
+        Path lockPath = configLockRoot.append(application.tenant().value()).append(application.serializedForm());
+        curatorDatabase.create(lockPath);
+        return lockPath;
+    }
+
     private String toDir(Node.State state) {
         switch (state) {
             case active: return "allocated"; // legacy name
@@ -332,6 +342,7 @@ public class CuratorDatabaseClient {
             case provisioned: return "provisioned";
             case ready: return "ready";
             case reserved: return "reserved";
+            case deprovisioned: return "deprovisioned";
             default: throw new RuntimeException("Node state " + state + " does not map to a directory name");
         }
     }
@@ -356,6 +367,28 @@ public class CuratorDatabaseClient {
         }
     }
 
+    /**
+     * Acquires the single cluster-global, re-entrant config lock for given application. Note that this is the same lock
+     * that configserver itself takes when modifying applications.
+     *
+     * This lock must be taken when writes to paths owned by this class may happen on both the configserver and
+     * node-repository side. This behaviour is obviously wrong, but since we pass a NestedTransaction across the
+     * configserver and node-repository boundary, the ownership semantics of the transaction (and its operations)
+     * becomes unclear.
+     *
+     * Example of when to use: The config server creates a new transaction and passes the transaction to
+     * {@link com.yahoo.vespa.hosted.provision.provisioning.NodeRepositoryProvisioner}, which appends operations to the
+     * transaction. The config server then commits (writes) the transaction which may include operations that modify
+     * data in paths owned by this class.
+     */
+    public Lock lockConfig(ApplicationId application) {
+        try {
+            return lock(configLockPath(application), defaultLockTimeout);
+        } catch (UncheckedTimeoutException e) {
+            throw new ApplicationLockException(e);
+        }
+    }
+
     private Lock lock(Path path, Duration timeout) {
         return curatorDatabase.lock(path, timeout);
     }
@@ -364,8 +397,11 @@ public class CuratorDatabaseClient {
         return curatorDatabase.getData(path).filter(data -> data.length > 0).map(mapper);
     }
 
-
     // Maintenance jobs
+    public Lock lockMaintenanceJob(String jobName) {
+        return lock(lockRoot.append("maintenanceJobLocks").append(jobName), defaultLockTimeout);
+    }
+
     public Set<String> readInactiveJobs() {
         try {
             return read(inactiveJobsPath(), stringSetSerializer::fromJson).orElseGet(HashSet::new);
@@ -483,18 +519,16 @@ public class CuratorDatabaseClient {
 
     // Load balancers
     public List<LoadBalancerId> readLoadBalancerIds() {
-        return curatorDatabase.getChildren(loadBalancersRoot).stream()
-                              .map(LoadBalancerId::fromSerializedForm)
-                              .collect(Collectors.toUnmodifiableList());
+        return readLoadBalancerIds((ignored) -> true);
     }
 
-    public Map<LoadBalancerId, LoadBalancer> readLoadBalancers() {
-        return readLoadBalancerIds().stream()
-                                    .map(this::readLoadBalancer)
-                                    .filter(Optional::isPresent)
-                                    .map(Optional::get)
-                                    .collect(collectingAndThen(toMap(LoadBalancer::id, Function.identity()),
-                                                           Collections::unmodifiableMap));
+    public Map<LoadBalancerId, LoadBalancer> readLoadBalancers(Predicate<LoadBalancerId> filter) {
+        return readLoadBalancerIds(filter).stream()
+                                          .map(this::readLoadBalancer)
+                                          .filter(Optional::isPresent)
+                                          .map(Optional::get)
+                                          .collect(collectingAndThen(toMap(LoadBalancer::id, Function.identity()),
+                                                                     Collections::unmodifiableMap));
     }
 
     public Optional<LoadBalancer> readLoadBalancer(LoadBalancerId id) {
@@ -522,14 +556,22 @@ public class CuratorDatabaseClient {
         transaction.commit();
     }
 
-    public Lock lockLoadBalancers() {
-        return lock(lockRoot.append("loadBalancersLock"), defaultLockTimeout);
+    // TODO(mpolden): Remove this and usages after April 2020
+    public Lock lockLoadBalancers(ApplicationId application) {
+        return lock(lockRoot.append("loadBalancersLock2").append(application.serializedForm()), defaultLockTimeout);
     }
 
     private Path loadBalancerPath(LoadBalancerId id) {
         return loadBalancersRoot.append(id.serializedForm());
     }
 
+    private List<LoadBalancerId> readLoadBalancerIds(Predicate<LoadBalancerId> predicate) {
+        return curatorDatabase.getChildren(loadBalancersRoot).stream()
+                              .map(LoadBalancerId::fromSerializedForm)
+                              .filter(predicate)
+                              .collect(Collectors.toUnmodifiableList());
+    }
+
     private Transaction.Operation createOrSet(Path path, byte[] data) {
         if (curatorDatabase.exists(path)) {
             return CuratorOperations.setData(path.getAbsolute(), data);
@@ -547,4 +589,5 @@ public class CuratorDatabaseClient {
                 .mapToObj(i -> firstProvisionIndex + i)
                 .collect(Collectors.toList());
     }
+
 }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializer.java
index ae4c93621e5..66172521d4c 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializer.java
@@ -6,7 +6,7 @@ import com.yahoo.slime.ArrayTraverser;
 import com.yahoo.slime.Cursor;
 import com.yahoo.slime.Inspector;
 import com.yahoo.slime.Slime;
-import com.yahoo.vespa.config.SlimeUtils;
+import com.yahoo.slime.SlimeUtils;
 import com.yahoo.vespa.hosted.provision.lb.DnsZone;
 import com.yahoo.vespa.hosted.provision.lb.LoadBalancer;
 import com.yahoo.vespa.hosted.provision.lb.LoadBalancerId;
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java
index b6a276e5e57..94f4dab1245 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java
@@ -20,7 +20,7 @@ import com.yahoo.slime.Cursor;
 import com.yahoo.slime.Inspector;
 import com.yahoo.slime.Slime;
 import com.yahoo.slime.Type;
-import com.yahoo.vespa.config.SlimeUtils;
+import com.yahoo.slime.SlimeUtils;
 import com.yahoo.vespa.hosted.provision.Node;
 import com.yahoo.vespa.hosted.provision.node.Agent;
 import com.yahoo.vespa.hosted.provision.node.Allocation;
@@ -103,6 +103,7 @@ public class NodeSerializer {
     private static final String removableKey = "removable";
     // Saved as part of allocation instead of serviceId, since serviceId serialized form is not easily extendable.
     private static final String wantedVespaVersionKey = "wantedVespaVersion";
+    private static final String wantedDockerImageRepoKey = "wantedDockerImageRepo";
 
     // History event fields
     private static final String historyEventTypeKey = "type";
@@ -186,6 +187,7 @@ public class NodeSerializer {
         object.setLong(currentRestartGenerationKey, allocation.restartGeneration().current());
         object.setBool(removableKey, allocation.isRemovable());
         object.setString(wantedVespaVersionKey, allocation.membership().cluster().vespaVersion().toString());
+        allocation.membership().cluster().dockerImageRepo().ifPresent(repo -> object.setString(wantedDockerImageRepoKey, repo));
         allocation.networkPorts().ifPresent(ports -> NetworkPortsSerializer.toSlime(ports, object.setArray(networkPortsKey)));
     }
 
@@ -306,7 +308,8 @@ public class NodeSerializer {
 
     private ClusterMembership clusterMembershipFromSlime(Inspector object) {
         return ClusterMembership.from(object.field(serviceIdKey).asString(), 
-                                      versionFromSlime(object.field(wantedVespaVersionKey)).get());
+                                      versionFromSlime(object.field(wantedVespaVersionKey)).get(),
+                                      dockerImageRepoFromSlime(object.field(wantedDockerImageRepoKey)));
     }
 
     private Optional<Version> versionFromSlime(Inspector object) {
@@ -314,6 +317,11 @@ public class NodeSerializer {
         return Optional.of(Version.fromString(object.asString()));
     }
 
+    private Optional<String> dockerImageRepoFromSlime(Inspector object) {
+        if ( ! object.valid() || object.asString().isEmpty()) return Optional.empty();
+        return Optional.of(object.asString());
+    }
+
     private Optional<DockerImage> dockerImageFromSlime(Inspector object) {
         if ( ! object.valid()) return Optional.empty();
         return Optional.of(DockerImage.fromString(object.asString()));
@@ -358,6 +366,7 @@ public class NodeSerializer {
     private History.Event.Type eventTypeFromString(String eventTypeString) {
         switch (eventTypeString) {
             case "provisioned" : return History.Event.Type.provisioned;
+            case "deprovisioned" : return History.Event.Type.deprovisioned;
             case "readied" : return History.Event.Type.readied;
             case "reserved" : return History.Event.Type.reserved;
             case "activated" : return History.Event.Type.activated;
@@ -379,6 +388,7 @@ public class NodeSerializer {
     private String toString(History.Event.Type nodeEventType) {
         switch (nodeEventType) {
             case provisioned : return "provisioned";
+            case deprovisioned : return "deprovisioned";
             case readied : return "readied";
             case reserved : return "reserved";
             case activated : return "activated";
@@ -409,6 +419,7 @@ public class NodeSerializer {
             case "InactiveExpirer" : return Agent.InactiveExpirer;
             case "ProvisionedExpirer" : return Agent.ProvisionedExpirer;
             case "ReservationExpirer" : return Agent.ReservationExpirer;
+            case "DynamicProvisioningMaintainer" : return Agent.DynamicProvisioningMaintainer;
         }
         throw new IllegalArgumentException("Unknown node event agent '" + eventAgentField.asString() + "'");
     }
@@ -424,6 +435,7 @@ public class NodeSerializer {
             case InactiveExpirer : return "InactiveExpirer";
             case ProvisionedExpirer : return "ProvisionedExpirer";
             case ReservationExpirer : return "ReservationExpirer";
+            case DynamicProvisioningMaintainer : return "DynamicProvisioningMaintainer";
         }
         throw new IllegalArgumentException("Serialized form of '" + agent + "' not defined");
     }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeTypeDockerImagesSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeTypeDockerImagesSerializer.java
index 37dc8a8a1ad..6615dff24e5 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeTypeDockerImagesSerializer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeTypeDockerImagesSerializer.java
@@ -7,7 +7,7 @@ import com.yahoo.slime.Cursor;
 import com.yahoo.slime.Inspector;
 import com.yahoo.slime.ObjectTraverser;
 import com.yahoo.slime.Slime;
-import com.yahoo.vespa.config.SlimeUtils;
+import com.yahoo.slime.SlimeUtils;
 
 import java.io.IOException;
 import java.io.UncheckedIOException;
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeTypeVersionsSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeTypeVersionsSerializer.java
index dfa79a4fd9a..e27cdcd1842 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeTypeVersionsSerializer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeTypeVersionsSerializer.java
@@ -7,7 +7,7 @@ import com.yahoo.slime.Cursor;
 import com.yahoo.slime.Inspector;
 import com.yahoo.slime.ObjectTraverser;
 import com.yahoo.slime.Slime;
-import com.yahoo.vespa.config.SlimeUtils;
+import com.yahoo.slime.SlimeUtils;
 
 import java.io.IOException;
 import java.io.UncheckedIOException;
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/OsVersionsSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/OsVersionsSerializer.java
index 14c8c7fa18e..915b1acedaa 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/OsVersionsSerializer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/OsVersionsSerializer.java
@@ -5,7 +5,7 @@ import com.yahoo.component.Version;
 import com.yahoo.config.provision.NodeType;
 import com.yahoo.slime.ObjectTraverser;
 import com.yahoo.slime.Slime;
-import com.yahoo.vespa.config.SlimeUtils;
+import com.yahoo.slime.SlimeUtils;
 import com.yahoo.vespa.hosted.provision.node.OsVersion;
 
 import java.io.IOException;
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/StringSetSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/StringSetSerializer.java
index 1839f7187fa..1149b15a2b0 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/StringSetSerializer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/StringSetSerializer.java
@@ -5,7 +5,7 @@ import com.yahoo.slime.ArrayTraverser;
 import com.yahoo.slime.Cursor;
 import com.yahoo.slime.Inspector;
 import com.yahoo.slime.Slime;
-import com.yahoo.vespa.config.SlimeUtils;
+import com.yahoo.slime.SlimeUtils;
 
 import java.io.IOException;
 import java.util.HashSet;
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java
index 7c5ff35878b..648bf52f455 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java
@@ -19,65 +19,59 @@ import java.util.Locale;
 public class CapacityPolicies {
 
     private final Zone zone;
+
+    private final NodeResourceLimits nodeResourceLimits;
+
     /* Deployments must match 1-to-1 the advertised resources of a physical host */
     private final boolean isUsingAdvertisedResources;
 
     public CapacityPolicies(Zone zone) {
         this.zone = zone;
+        this.nodeResourceLimits = new NodeResourceLimits(zone);
         this.isUsingAdvertisedResources = zone.cloud().value().equals("aws");
     }
 
-    public int decideSize(Capacity capacity, ClusterSpec.Type clusterType, ApplicationId application) {
-        int requestedNodes = capacity.nodeCount();
-
+    public int decideSize(int requested, Capacity capacity, ClusterSpec cluster, ApplicationId application) {
         if (application.instance().isTester()) return 1;
 
-        ensureRedundancy(requestedNodes, clusterType, capacity.canFail());
-
-        if (capacity.isRequired()) return requestedNodes;
-
+        ensureRedundancy(requested, cluster, capacity.canFail());
+        if (capacity.isRequired()) return requested;
         switch(zone.environment()) {
             case dev : case test : return 1;
-            case perf : return Math.min(capacity.nodeCount(), 3);
-            case staging: return requestedNodes <= 1 ? requestedNodes : Math.max(2, requestedNodes / 10);
-            case prod : return requestedNodes;
+            case perf : return Math.min(requested, 3);
+            case staging: return requested <= 1 ? requested : Math.max(2, requested / 10);
+            case prod : return requested;
             default : throw new IllegalArgumentException("Unsupported environment " + zone.environment());
         }
     }
 
-    public NodeResources decideNodeResources(Capacity capacity, ClusterSpec cluster) {
-        NodeResources resources = capacity.nodeResources().orElse(defaultNodeResources(cluster.type()));
-        ensureSufficientResources(resources, cluster);
+    public NodeResources decideNodeResources(NodeResources requested, Capacity capacity, ClusterSpec cluster) {
+        if (requested == NodeResources.unspecified)
+            requested = defaultNodeResources(cluster.type());
+        ensureSufficientResources(requested, cluster);
 
-        if (capacity.isRequired()) return resources;
+        if (capacity.isRequired()) return requested;
 
         // Allow slow storage in zones which are not performance sensitive
         if (zone.system().isCd() || zone.environment() == Environment.dev || zone.environment() == Environment.test)
-            resources = resources.with(NodeResources.DiskSpeed.any).with(NodeResources.StorageType.any);
+            requested = requested.with(NodeResources.DiskSpeed.any).with(NodeResources.StorageType.any);
 
         // Dev does not cap the cpu of containers since usage is spotty: Allocate just a small amount exclusively
         // Do not cap in AWS as hosts are allocated on demand and 1-to-1, so the node can use the entire host
         if (zone.environment() == Environment.dev && !zone.region().value().contains("aws-"))
-            resources = resources.withVcpu(0.1);
+            requested = requested.withVcpu(0.1);
 
-        return resources;
+        return requested;
     }
 
     private void ensureSufficientResources(NodeResources resources, ClusterSpec cluster) {
-        double minMemoryGb = minMemoryGb(cluster.type());
+        double minMemoryGb = nodeResourceLimits.minMemoryGb(cluster.type());
         if (resources.memoryGb() >= minMemoryGb) return;
-
         throw new IllegalArgumentException(String.format(Locale.ENGLISH,
                 "Must specify at least %.2f Gb of memory for %s cluster '%s', was: %.2f Gb",
                 minMemoryGb, cluster.type().name(), cluster.id().value(), resources.memoryGb()));
     }
 
-    private int minMemoryGb(ClusterSpec.Type clusterType) {
-        if (zone.system() == SystemName.dev) return 1; // Allow small containers in dev system
-        if (clusterType == ClusterSpec.Type.admin) return 2;
-        return 4;
-    }
-
     private NodeResources defaultNodeResources(ClusterSpec.Type clusterType) {
         if (clusterType == ClusterSpec.Type.admin) {
             if (zone.system() == SystemName.dev) {
@@ -105,15 +99,14 @@ public class CapacityPolicies {
     /**
      * Throw if the node count is 1 for container and content clusters and we're in a production zone
      *
-     * @return the argument node count
      * @throws IllegalArgumentException if only one node is requested and we can fail
      */
-    private void ensureRedundancy(int nodeCount, ClusterSpec.Type clusterType, boolean canFail) {
+    private void ensureRedundancy(int nodeCount, ClusterSpec cluster, boolean canFail) {
         if (canFail &&
             nodeCount == 1 &&
-            requiresRedundancy(clusterType) &&
+            requiresRedundancy(cluster.type()) &&
             zone.environment().isProduction())
-            throw new IllegalArgumentException("Deployments to prod require at least 2 nodes per cluster for redundancy");
+            throw new IllegalArgumentException("Deployments to prod require at least 2 nodes per cluster for redundancy. Not fulfilled for " + cluster);
     }
 
     private static boolean requiresRedundancy(ClusterSpec.Type clusterType) {
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacity.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacity.java
index a609103ac89..672be25c5be 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacity.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacity.java
@@ -26,12 +26,6 @@ public class DockerHostCapacity {
         this.hostResourcesCalculator = Objects.requireNonNull(hostResourcesCalculator, "hostResourcesCalculator must be non-null");
     }
 
-    /** Returns the allocation skew of this host */
-    public double skew(Node host) {
-        NodeResources free = freeCapacityOf(host, false);
-        return Node.skew(host.flavor().resources(), free);
-    }
-
     int compareWithoutInactive(Node hostA, Node hostB) {
         int result = compare(freeCapacityOf(hostB, true), freeCapacityOf(hostA, true));
         if (result != 0) return result;
@@ -72,7 +66,7 @@ public class DockerHostCapacity {
     NodeResources freeCapacityOf(Node host, boolean excludeInactive) {
         // Only hosts have free capacity
         if (!host.type().canRun(NodeType.tenant)) return new NodeResources(0, 0, 0, 0);
-        NodeResources hostResources = hostResourcesCalculator.availableCapacityOf(host.flavor().resources());
+        NodeResources hostResources = hostResourcesCalculator.advertisedResourcesOf(host.flavor());
 
         return allNodes.childrenOf(host).asList().stream()
                 .filter(node -> !(excludeInactive && isInactiveOrRetired(node)))
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerImages.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerImages.java
index 9a06f2a980a..9edcfd6c697 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerImages.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/DockerImages.java
@@ -16,7 +16,7 @@ import java.util.concurrent.TimeUnit;
 import java.util.logging.Logger;
 
 /**
- * Multithread safe class to get and set docker images for given host types.
+ * Multithread safe class to get and set docker images for given node types.
  *
  * @author freva
  */
@@ -49,7 +49,7 @@ public class DockerImages {
 
     private void createCache() {
         this.dockerImages = Suppliers.memoizeWithExpiration(() -> Collections.unmodifiableMap(db.readDockerImages()),
-                cacheTtl.toMillis(), TimeUnit.MILLISECONDS);
+                                                            cacheTtl.toMillis(), TimeUnit.MILLISECONDS);
     }
 
     /** Returns the current docker images for each node type */
@@ -57,9 +57,11 @@ public class DockerImages {
         return dockerImages.get();
     }
 
-    /** Returns the current docker image for given node type, or default */
+    /** Returns the current docker image for given node type, or the type for corresponding child nodes
+     * if it is a Docker host, or default */
     public DockerImage dockerImageFor(NodeType type) {
-        return getDockerImages().getOrDefault(type, defaultImage);
+        NodeType typeToUseForLookup = type.isDockerHost() ? type.childNodeType() : type;
+        return getDockerImages().getOrDefault(typeToUseForLookup, defaultImage);
     }
 
     /** Set the docker image for nodes of given type */
@@ -69,8 +71,8 @@ public class DockerImages {
         }
         try (Lock lock = db.lockDockerImages()) {
             Map<NodeType, DockerImage> dockerImages = db.readDockerImages();
-
-            dockerImage.ifPresentOrElse(image -> dockerImages.put(nodeType, image), () -> dockerImages.remove(nodeType));
+            dockerImage.ifPresentOrElse(image -> dockerImages.put(nodeType, image),
+                                        () -> dockerImages.remove(nodeType));
             db.writeDockerImages(dockerImages);
             createCache(); // Throw away current cache
             log.info("Set docker image for " + nodeType + " nodes to " + dockerImage.map(DockerImage::asString).orElse(null));
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/EmptyProvisionServiceProvider.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/EmptyProvisionServiceProvider.java
index 05915b82bae..9566213bc91 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/EmptyProvisionServiceProvider.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/EmptyProvisionServiceProvider.java
@@ -1,7 +1,9 @@
 // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.provision.provisioning;
 
+import com.yahoo.config.provision.Flavor;
 import com.yahoo.config.provision.NodeResources;
+import com.yahoo.vespa.hosted.provision.Node;
 import com.yahoo.vespa.hosted.provision.lb.LoadBalancerService;
 
 import java.util.Optional;
@@ -10,6 +12,7 @@ import java.util.Optional;
  * @author freva
  */
 public class EmptyProvisionServiceProvider implements ProvisionServiceProvider {
+
     private final HostResourcesCalculator hostResourcesCalculator = new NoopHostResourcesCalculator();
 
     @Override
@@ -30,8 +33,14 @@ public class EmptyProvisionServiceProvider implements ProvisionServiceProvider {
     public static class NoopHostResourcesCalculator implements HostResourcesCalculator {
 
         @Override
-        public NodeResources availableCapacityOf(NodeResources hostResources) {
-            return hostResources;
+        public NodeResources realResourcesOf(Node node) {
+            return node.flavor().resources();
         }
+
+        @Override
+        public NodeResources advertisedResourcesOf(Flavor flavor) {
+            return flavor.resources();
+        }
+
     }
 }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java
index 6686160982a..8143076a3b2 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java
@@ -15,6 +15,7 @@ import com.yahoo.vespa.flags.custom.PreprovisionCapacity;
 import com.yahoo.vespa.hosted.provision.LockedNodeList;
 import com.yahoo.vespa.hosted.provision.Node;
 import com.yahoo.vespa.hosted.provision.NodeRepository;
+import com.yahoo.vespa.hosted.provision.node.Agent;
 
 import java.util.List;
 import java.util.Optional;
@@ -31,7 +32,6 @@ public class GroupPreparer {
     private final Optional<HostProvisioner> hostProvisioner;
     private final HostResourcesCalculator hostResourcesCalculator;
     private final BooleanFlag dynamicProvisioningEnabledFlag;
-    private final BooleanFlag enableInPlaceResize;
     private final ListFlag<PreprovisionCapacity> preprovisionCapacityFlag;
 
     public GroupPreparer(NodeRepository nodeRepository, Optional<HostProvisioner> hostProvisioner,
@@ -40,7 +40,6 @@ public class GroupPreparer {
         this.hostProvisioner = hostProvisioner;
         this.hostResourcesCalculator = hostResourcesCalculator;
         this.dynamicProvisioningEnabledFlag = Flags.ENABLE_DYNAMIC_PROVISIONING.bindTo(flagSource);
-        this.enableInPlaceResize = Flags.ENABLE_IN_PLACE_RESIZE.bindTo(flagSource);
         this.preprovisionCapacityFlag = Flags.PREPROVISION_CAPACITY.bindTo(flagSource);
     }
 
@@ -65,11 +64,7 @@ public class GroupPreparer {
         boolean dynamicProvisioningEnabled = hostProvisioner.isPresent() && dynamicProvisioningEnabledFlag
                 .with(FetchVector.Dimension.APPLICATION_ID, application.serializedForm())
                 .value();
-        // Do not in-place resize in dynamically provisioned zones
-        boolean inPlaceResizeEnabled = enableInPlaceResize
-                .with(FetchVector.Dimension.APPLICATION_ID, application.serializedForm())
-                .value() && !dynamicProvisioningEnabled;
-
+        boolean allocateFully = dynamicProvisioningEnabled && preprovisionCapacityFlag.value().isEmpty();
         try (Mutex lock = nodeRepository.lock(application)) {
 
             // Lock ready pool to ensure that the same nodes are not simultaneously allocated by others
@@ -79,12 +74,12 @@ public class GroupPreparer {
                 LockedNodeList nodeList = nodeRepository.list(allocationLock);
                 NodePrioritizer prioritizer = new NodePrioritizer(nodeList, application, cluster, requestedNodes,
                                                                   spareCount, wantedGroups, nodeRepository.nameResolver(),
-                                                                  hostResourcesCalculator, inPlaceResizeEnabled);
+                                                                  hostResourcesCalculator, allocateFully);
 
                 prioritizer.addApplicationNodes();
                 prioritizer.addSurplusNodes(surplusActiveNodes);
                 prioritizer.addReadyNodes();
-                prioritizer.addNewDockerNodes(dynamicProvisioningEnabled && preprovisionCapacityFlag.value().isEmpty());
+                prioritizer.addNewDockerNodes(nodeRepository::canAllocateTenantNodeTo);
 
                 // Allocate from the prioritized list
                 NodeAllocation allocation = new NodeAllocation(nodeList, application, cluster, requestedNodes,
@@ -104,7 +99,7 @@ public class GroupPreparer {
                     List<Node> hosts = provisionedHosts.stream()
                                                        .map(ProvisionedHost::generateHost)
                                                        .collect(Collectors.toList());
-                    nodeRepository.addNodes(hosts);
+                    nodeRepository.addNodes(hosts, Agent.application);
 
                     // Offer the nodes on the newly provisioned hosts, this should be enough to cover the deficit
                     List<PrioritizableNode> nodes = provisionedHosts.stream()
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java
index 394549e4141..0423f762f2b 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java
@@ -18,7 +18,7 @@ public interface HostProvisioner {
     /**
      * Schedule provisioning of a given number of hosts.
      *
-     * @param provisionIndexes List of unique provision indexes which will be used to generate the node hostnames
+     * @param provisionIndexes list of unique provision indexes which will be used to generate the node hostnames
      *                         on the form of <code>[prefix][index].[domain]</code>
      * @param resources the resources needed per node
      * @param applicationId id of the application that will own the provisioned host
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostResourcesCalculator.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostResourcesCalculator.java
index c5808a53837..b26351062e6 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostResourcesCalculator.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostResourcesCalculator.java
@@ -1,14 +1,24 @@
 // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.provision.provisioning;
 
+import com.yahoo.config.provision.Flavor;
 import com.yahoo.config.provision.NodeResources;
+import com.yahoo.vespa.hosted.provision.Node;
 
 /**
+ * Some cloud providers advertise that a certain amount of resources are available in a flavor
+ * but then actually provide somewhat less. This service provides the mapping between real and advertised
+ * resources for all clouds.
+ *
  * @author freva
+ * @author bratseth
  */
 public interface HostResourcesCalculator {
 
-    /** Calculates the resources that are reserved for host level processes and returns the remainder. */
-    NodeResources availableCapacityOf(NodeResources hostResources);
+    /** Nodes use advertised resources. This returns the real resources for the node. */
+    NodeResources realResourcesOf(Node node);
+
+    /** Flavors use real resources. This returns the advertised resources of the flavor. */
+    NodeResources advertisedResourcesOf(Flavor flavor);
 
 }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImpl.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImpl.java
index a56ace07c82..fb83385920c 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImpl.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImpl.java
@@ -20,7 +20,6 @@ import com.yahoo.vespa.service.monitor.DuperModelInfraApi;
 import com.yahoo.vespa.service.monitor.InfraApplicationApi;
 
 import java.util.List;
-import java.util.Map;
 import java.util.Optional;
 import java.util.logging.Logger;
 import java.util.stream.Collectors;
@@ -51,9 +50,22 @@ public class InfraDeployerImpl implements InfraDeployer {
     }
 
     @Override
-    public Map<ApplicationId, Deployment> getSupportedInfraDeployments() {
-        return duperModel.getSupportedInfraApplications().stream()
-                .collect(Collectors.toMap(InfraApplicationApi::getApplicationId, InfraDeployment::new));
+    public void activateAllSupportedInfraApplications(boolean propagateException) {
+        duperModel.getSupportedInfraApplications().forEach(api -> {
+            var application = api.getApplicationId();
+            var deployment = new InfraDeployment(api);
+            try {
+                deployment.activate();
+            } catch (RuntimeException e) {
+                logger.log(LogLevel.INFO, "Failed to activate " + application, e);
+                if (propagateException) {
+                    throw e;
+                }
+                // loop around to activate the next application
+            }
+        });
+
+        duperModel.infraApplicationsIsNowComplete();
     }
 
     private class InfraDeployment implements Deployment {
@@ -74,12 +86,10 @@ public class InfraDeployerImpl implements InfraDeployer {
             try (Mutex lock = nodeRepository.lock(application.getApplicationId())) {
                 NodeType nodeType = application.getCapacity().type();
                 Version targetVersion = infrastructureVersions.getTargetVersionFor(nodeType);
-                hostSpecs = provisioner.prepare(
-                        application.getApplicationId(),
-                        application.getClusterSpecWithVersion(targetVersion),
-                        application.getCapacity(),
-                        1, // groups
-                        logger::log);
+                hostSpecs = provisioner.prepare(application.getApplicationId(),
+                                                application.getClusterSpecWithVersion(targetVersion),
+                                                application.getCapacity(),
+                                                logger::log);
 
                 prepared = true;
             }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java
index f0caa358cb8..b694a7c3cd4 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java
@@ -1,4 +1,4 @@
-// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.provision.provisioning;
 
 import com.yahoo.config.provision.ApplicationId;
@@ -51,10 +51,12 @@ public class LoadBalancerProvisioner {
         this.db = nodeRepository.database();
         this.service = service;
         // Read and write all load balancers to make sure they are stored in the latest version of the serialization format
-        try (var lock = db.lockLoadBalancers()) {
-            for (var id : db.readLoadBalancerIds()) {
-                var loadBalancer = db.readLoadBalancer(id);
-                loadBalancer.ifPresent(db::writeLoadBalancer);
+        for (var id : db.readLoadBalancerIds()) {
+            try (var lock = db.lockConfig(id.application())) {
+                try (var legacyLock = db.lockLoadBalancers(id.application())) {
+                    var loadBalancer = db.readLoadBalancer(id);
+                    loadBalancer.ifPresent(db::writeLoadBalancer);
+                }
             }
         }
     }
@@ -72,8 +74,11 @@ public class LoadBalancerProvisioner {
     public void prepare(ApplicationId application, ClusterSpec cluster, NodeSpec requestedNodes) {
         if (requestedNodes.type() != NodeType.tenant) return; // Nothing to provision for this node type
         if (!cluster.type().isContainer()) return; // Nothing to provision for this cluster type
-        try (var loadBalancersLock = db.lockLoadBalancers()) {
-            provision(application, cluster.id(), false, loadBalancersLock);
+        if (application.instance().isTester()) return; // Do not provision for tester instances
+        try (var lock = db.lockConfig(application)) {
+            try (var legacyLock = db.lockLoadBalancers(application)) {
+                provision(application, effectiveId(cluster), false, lock);
+            }
         }
     }
 
@@ -89,15 +94,17 @@ public class LoadBalancerProvisioner {
      */
     public void activate(ApplicationId application, Set<ClusterSpec> clusters,
                          @SuppressWarnings("unused") Mutex applicationLock, NestedTransaction transaction) {
-        try (var loadBalancersLock = db.lockLoadBalancers()) {
-            var containerClusters = containerClusterOf(clusters);
-            for (var clusterId : containerClusters) {
-                // Provision again to ensure that load balancer instance is re-configured with correct nodes
-                provision(application, clusterId, true, loadBalancersLock);
+        try (var lock = db.lockConfig(application)) {
+            try (var legacyLock = db.lockLoadBalancers(application)) {
+                var containerClusters = containerClustersOf(clusters);
+                for (var clusterId : containerClusters) {
+                    // Provision again to ensure that load balancer instance is re-configured with correct nodes
+                    provision(application, clusterId, true, legacyLock);
+                }
+                // Deactivate any surplus load balancers, i.e. load balancers for clusters that have been removed
+                var surplusLoadBalancers = surplusLoadBalancersOf(application, containerClusters);
+                deactivate(surplusLoadBalancers, transaction);
             }
-            // Deactivate any surplus load balancers, i.e. load balancers for clusters that have been removed
-            var surplusLoadBalancers = surplusLoadBalancersOf(application, containerClusters);
-            deactivate(surplusLoadBalancers, transaction);
         }
     }
 
@@ -107,16 +114,17 @@ public class LoadBalancerProvisioner {
      */
     public void deactivate(ApplicationId application, NestedTransaction transaction) {
         try (var applicationLock = nodeRepository.lock(application)) {
-            try (Mutex loadBalancersLock = db.lockLoadBalancers()) {
-                deactivate(nodeRepository.loadBalancers().owner(application).asList(), transaction);
+            try (var lock = db.lockConfig(application)) {
+                try (var legacyLock = db.lockLoadBalancers(application)) {
+                    deactivate(nodeRepository.loadBalancers(application).asList(), transaction);
+                }
             }
         }
     }
 
     /** Returns load balancers of given application that are no longer referenced by given clusters */
     private List<LoadBalancer> surplusLoadBalancersOf(ApplicationId application, Set<ClusterSpec.Id> activeClusters) {
-        var activeLoadBalancersByCluster = nodeRepository.loadBalancers()
-                                                         .owner(application)
+        var activeLoadBalancersByCluster = nodeRepository.loadBalancers(application)
                                                          .in(LoadBalancer.State.active)
                                                          .asList()
                                                          .stream()
@@ -183,7 +191,7 @@ public class LoadBalancerProvisioner {
                        .owner(application)
                        .filter(node -> node.state().isAllocated())
                        .container()
-                       .filter(node -> node.allocation().get().membership().cluster().id().equals(clusterId))
+                       .filter(node -> effectiveId(node.allocation().get().membership().cluster()).equals(clusterId))
                        .asList();
     }
 
@@ -202,11 +210,16 @@ public class LoadBalancerProvisioner {
         return reachable;
     }
 
-    private static Set<ClusterSpec.Id> containerClusterOf(Set<ClusterSpec> clusters) {
+    /** Returns the container cluster IDs of the given clusters */
+    private static Set<ClusterSpec.Id> containerClustersOf(Set<ClusterSpec> clusters) {
         return clusters.stream()
                        .filter(c -> c.type().isContainer())
-                       .map(ClusterSpec::id)
+                       .map(LoadBalancerProvisioner::effectiveId)
                        .collect(Collectors.toUnmodifiableSet());
     }
 
+    private static ClusterSpec.Id effectiveId(ClusterSpec cluster) {
+        return cluster.combinedId().orElse(cluster.id());
+    }
+
 }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java
index ebd6a01e61f..652fd5c6861 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java
@@ -55,6 +55,9 @@ class NodeAllocation {
     /** The number of nodes in the accepted nodes which are of the requested flavor */
     private int acceptedOfRequestedFlavor = 0;
 
+    /** The number of nodes in the accepted nodes which are of the requested flavor and not already retired */
+    private int acceptedNonretiredOfRequestedFlavor = 0;
+
     /** The number of nodes rejected because of clashing parentHostname */
     private int rejectedWithClashingParentHost = 0;
 
@@ -100,7 +103,6 @@ class NodeAllocation {
         List<Node> accepted = new ArrayList<>();
         for (PrioritizableNode node : nodesPrioritized) {
             Node offered = node.node;
-
             if (offered.allocation().isPresent()) {
                 ClusterMembership membership = offered.allocation().get().membership();
                 if ( ! offered.allocation().get().owner().equals(application)) continue; // wrong application
@@ -116,7 +118,7 @@ class NodeAllocation {
                     if (offered.status().wantToRetire()) wantToRetireNode = true;
                     if (requestedNodes.isExclusive() && ! hostsOnly(application.tenant(), offered.parentHostname()))
                         wantToRetireNode = true;
-                    if (( ! saturated() && hasCompatibleFlavor(node)) || acceptToRetire(node))
+                    if (( ! saturatedByNonretired() && hasCompatibleFlavor(node)) || acceptToRetire(node))
                         accepted.add(acceptNode(node, wantToRetireNode, node.isResizable));
                 }
                 else {
@@ -214,6 +216,7 @@ class NodeAllocation {
     private boolean acceptToRetire(PrioritizableNode node) {
         if (node.node.state() != Node.State.active) return false;
         if (! node.node.allocation().get().membership().cluster().group().equals(cluster.group())) return false;
+        if (node.node.allocation().get().membership().retired()) return true; // don't second-guess if already retired
 
         return cluster.type().isContent() ||
                (cluster.type() == ClusterSpec.Type.container && !hasCompatibleFlavor(node));
@@ -230,17 +233,15 @@ class NodeAllocation {
             node = node.with(node.allocation().get().withRequestedResources(requestedNodes.resources().orElse(node.flavor().resources())));
 
         if (! wantToRetire) {
-            if (resize) {
-                NodeResources hostResources = allNodes.parentOf(node).get().flavor().resources();
-                node = node.with(new Flavor(requestedNodes.resources().get()
-                        .with(hostResources.diskSpeed())
-                        .with(hostResources.storageType())));
-            }
+            if (resize && ! ( node.allocation().isPresent() && node.allocation().get().membership().retired()))
+                node = resize(node);
 
             if (node.state() != Node.State.active) // reactivated node - make sure its not retired
                 node = node.unretire();
 
             acceptedOfRequestedFlavor++;
+            if ( ! (node.allocation().isPresent() && node.allocation().get().membership().retired()))
+                acceptedNonretiredOfRequestedFlavor++;
         } else {
             ++wasRetiredJustNow;
             // Retire nodes which are of an unwanted flavor, retired flavor or have an overlapping parent host
@@ -257,6 +258,13 @@ class NodeAllocation {
         return node;
     }
 
+    private Node resize(Node node) {
+        NodeResources hostResources = allNodes.parentOf(node).get().flavor().resources();
+        return node.with(new Flavor(requestedNodes.resources().get()
+                                                  .with(hostResources.diskSpeed())
+                                                  .with(hostResources.storageType())));
+    }
+
     private Node setCluster(ClusterSpec cluster, Node node) {
         ClusterMembership membership = node.allocation().get().membership().with(cluster);
         return node.with(node.allocation().get().with(membership));
@@ -267,6 +275,11 @@ class NodeAllocation {
         return requestedNodes.saturatedBy(acceptedOfRequestedFlavor);
     }
 
+    /** Returns true if no more nodes are needed in this list to not make changes to the retired set */
+    private boolean saturatedByNonretired() {
+        return requestedNodes.saturatedBy(acceptedNonretiredOfRequestedFlavor);
+    }
+
     /** Returns true if the content of this list is sufficient to meet the request */
     boolean fulfilled() {
         return requestedNodes.fulfilledBy(acceptedOfRequestedFlavor);
@@ -321,8 +334,10 @@ class NodeAllocation {
             }
         }
         else if (deltaRetiredCount < 0) { // unretire until deltaRetiredCount is 0
-            for (PrioritizableNode node : byIncreasingIndex(nodes)) {
-                if ( node.node.allocation().get().membership().retired() && hasCompatibleFlavor(node)) {
+            for (PrioritizableNode node : byUnretiringPriority(nodes)) {
+                if ( node.node.allocation().get().membership().retired() && hasCompatibleFlavor(node) ) {
+                    if (node.isResizable)
+                        node.node = resize(node.node);
                     node.node = node.node.unretire();
                     if (++deltaRetiredCount == 0) break;
                 }
@@ -333,7 +348,7 @@ class NodeAllocation {
             // Set whether the node is exclusive
             Allocation allocation = node.node.allocation().get();
             node.node = node.node.with(allocation.with(allocation.membership()
-                           .with(allocation.membership().cluster().exclusive(requestedNodes.isExclusive()))));
+                                .with(allocation.membership().cluster().exclusive(requestedNodes.isExclusive()))));
         }
 
         return nodes.stream().map(n -> n.node).collect(Collectors.toList());
@@ -364,8 +379,12 @@ class NodeAllocation {
         return nodes.stream().sorted(nodeIndexComparator().reversed()).collect(Collectors.toList());
     }
 
-    private List<PrioritizableNode> byIncreasingIndex(Set<PrioritizableNode> nodes) {
-        return nodes.stream().sorted(nodeIndexComparator()).collect(Collectors.toList());
+    /** Prefer to unretire nodes we don't want to retire, and otherwise those with lower index */
+    private List<PrioritizableNode> byUnretiringPriority(Set<PrioritizableNode> nodes) {
+        return nodes.stream()
+                    .sorted(Comparator.comparing((PrioritizableNode n) -> n.node.status().wantToRetire())
+                                      .thenComparing(n -> n.node.allocation().get().membership().index()))
+                    .collect(Collectors.toList());
     }
 
     private Comparator<PrioritizableNode> nodeIndexComparator() {
@@ -390,4 +409,5 @@ class NodeAllocation {
             return count;
         }
     }
+
 }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java
index a5fd37640d8..524b524f8c4 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java
@@ -18,6 +18,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
+import java.util.function.Predicate;
 import java.util.logging.Logger;
 import java.util.stream.Collectors;
 
@@ -31,9 +32,6 @@ import java.util.stream.Collectors;
  */
 public class NodePrioritizer {
 
-    /** Node states in which host can get new nodes allocated in, ordered by preference (ascending) */
-    public static final List<Node.State> ALLOCATABLE_HOST_STATES =
-            List.of(Node.State.provisioned, Node.State.ready, Node.State.active);
     private final static Logger log = Logger.getLogger(NodePrioritizer.class.getName());
 
     private final Map<Node, PrioritizableNode> nodes = new HashMap<>();
@@ -46,13 +44,14 @@ public class NodePrioritizer {
     private final boolean isDocker;
     private final boolean isAllocatingForReplacement;
     private final boolean isTopologyChange;
-    private final boolean inPlaceResizeEnabled;
+    /** If set, a host can only have nodes by single tenant and does not allow in-place resizing.  */
+    private final boolean allocateFully;
     private final int currentClusterSize;
     private final Set<Node> spareHosts;
 
     NodePrioritizer(LockedNodeList allNodes, ApplicationId application, ClusterSpec clusterSpec, NodeSpec nodeSpec,
                     int spares, int wantedGroups, NameResolver nameResolver, HostResourcesCalculator hostResourcesCalculator,
-                    boolean inPlaceResizeEnabled) {
+                    boolean allocateFully) {
         this.allNodes = allNodes;
         this.capacity = new DockerHostCapacity(allNodes, hostResourcesCalculator);
         this.requestedNodes = nodeSpec;
@@ -60,7 +59,7 @@ public class NodePrioritizer {
         this.application = application;
         this.nameResolver = nameResolver;
         this.spareHosts = findSpareHosts(allNodes, capacity, spares);
-        this.inPlaceResizeEnabled = inPlaceResizeEnabled;
+        this.allocateFully = allocateFully;
 
         NodeList nodesInCluster = allNodes.owner(application).type(clusterSpec.type()).cluster(clusterSpec.id());
         NodeList nonRetiredNodesInCluster = nodesInCluster.not().retired();
@@ -117,20 +116,15 @@ public class NodePrioritizer {
         }
     }
 
-    /**
-     * Add a node on each docker host with enough capacity for the requested flavor
-     *
-     * @param exclusively whether the ready docker nodes should only be added on hosts that
-     *                    already have nodes allocated to this tenant
-     */
-    void addNewDockerNodes(boolean exclusively) {
+    /** Add a node on each docker host with enough capacity for the requested flavor  */
+    void addNewDockerNodes(Predicate<Node> canAllocateTenantNodeTo) {
         if ( ! isDocker) return;
 
         LockedNodeList candidates = allNodes
-                .filter(node -> node.type() != NodeType.host || ALLOCATABLE_HOST_STATES.contains(node.state()))
+                .filter(node -> node.type() != NodeType.host || canAllocateTenantNodeTo.test(node))
                 .filter(node -> node.reservedTo().isEmpty() || node.reservedTo().get().equals(application.tenant()));
 
-        if (exclusively) {
+        if (allocateFully) {
             Set<String> candidateHostnames = candidates.asList().stream()
                                                        .filter(node -> node.type() == NodeType.tenant)
                                                        .filter(node -> node.allocation()
@@ -149,9 +143,6 @@ public class NodePrioritizer {
         NodeResources wantedResources = resources(requestedNodes);
 
         for (Node host : candidates) {
-            if (!host.type().canRun(requestedNodes.type())) continue;
-            if (host.status().wantToRetire()) continue;
-
             boolean hostHasCapacityForWantedFlavor = capacity.hasCapacity(host, wantedResources);
             boolean conflictingCluster = allNodes.childrenOf(host).owner(application).asList().stream()
                                                  .anyMatch(child -> child.allocation().get().membership().cluster().id().equals(clusterSpec.id()));
@@ -218,7 +209,7 @@ public class NodePrioritizer {
             builder.parent(parent).freeParentCapacity(parentCapacity);
 
             if (!isNewNode)
-                builder.resizable(inPlaceResizeEnabled && requestedNodes.canResize(
+                builder.resizable(!allocateFully && requestedNodes.canResize(
                         node.flavor().resources(), parentCapacity, isTopologyChange, currentClusterSize));
 
             if (spareHosts.contains(parent))
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java
index 07cf297a314..6802196f80d 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java
@@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.provision.provisioning;
 import com.google.inject.Inject;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.Capacity;
+import com.yahoo.config.provision.ClusterResources;
 import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.config.provision.Environment;
 import com.yahoo.config.provision.HostFilter;
@@ -14,10 +15,13 @@ import com.yahoo.config.provision.ProvisionLogger;
 import com.yahoo.config.provision.Provisioner;
 import com.yahoo.config.provision.Zone;
 import com.yahoo.log.LogLevel;
+import com.yahoo.transaction.Mutex;
 import com.yahoo.transaction.NestedTransaction;
 import com.yahoo.vespa.flags.FlagSource;
 import com.yahoo.vespa.hosted.provision.Node;
+import com.yahoo.vespa.hosted.provision.NodeList;
 import com.yahoo.vespa.hosted.provision.NodeRepository;
+import com.yahoo.vespa.hosted.provision.applications.Application;
 import com.yahoo.vespa.hosted.provision.node.Allocation;
 import com.yahoo.vespa.hosted.provision.node.filter.ApplicationFilter;
 import com.yahoo.vespa.hosted.provision.node.filter.NodeHostFilter;
@@ -69,46 +73,53 @@ public class NodeRepositoryProvisioner implements Provisioner {
         this.activator = new Activator(nodeRepository, loadBalancerProvisioner);
     }
 
+
     /**
      * Returns a list of nodes in the prepared or active state, matching the given constraints.
      * The nodes are ordered by increasing index number.
      */
+    @Deprecated // TODO: Remove after April 2020
     @Override
-    public List<HostSpec> prepare(ApplicationId application, ClusterSpec cluster, Capacity requestedCapacity, 
+    public List<HostSpec> prepare(ApplicationId application, ClusterSpec cluster, Capacity requestedCapacity,
                                   int wantedGroups, ProvisionLogger logger) {
-        if (cluster.group().isPresent()) throw new IllegalArgumentException("Node requests cannot specify a group");
-        if (requestedCapacity.nodeCount() > 0 && requestedCapacity.nodeCount() % wantedGroups != 0)
-            throw new IllegalArgumentException("Requested " + requestedCapacity.nodeCount() + " nodes in " + wantedGroups + " groups, " +
-                                               "which doesn't allow the nodes to be divided evenly into groups");
+        return prepare(application, cluster, requestedCapacity.withGroups(wantedGroups), logger);
+    }
 
+    /**
+     * Returns a list of nodes in the prepared or active state, matching the given constraints.
+     * The nodes are ordered by increasing index number.
+     */
+    @Override
+    public List<HostSpec> prepare(ApplicationId application, ClusterSpec cluster, Capacity requested,
+                                  ProvisionLogger logger) {
         log.log(zone.system().isCd() ? Level.INFO : LogLevel.DEBUG,
-                () -> "Received deploy prepare request for " + requestedCapacity + " in " +
-                      wantedGroups + " groups for application " + application + ", cluster " + cluster);
-
-        int effectiveGroups;
-        NodeSpec requestedNodes;
-        Optional<NodeResources> resources = requestedCapacity.nodeResources();
-        if ( requestedCapacity.type() == NodeType.tenant) {
-            int nodeCount = capacityPolicies.decideSize(requestedCapacity, cluster.type(), application);
-            if (zone.environment().isManuallyDeployed() && nodeCount < requestedCapacity.nodeCount())
-                logger.log(Level.INFO, "Requested " + requestedCapacity.nodeCount() + " nodes for " + cluster +
-                                       ", downscaling to " + nodeCount + " nodes in " + zone.environment());
-            resources = Optional.of(capacityPolicies.decideNodeResources(requestedCapacity, cluster));
-            boolean exclusive = capacityPolicies.decideExclusivity(cluster.isExclusive());
-            effectiveGroups = Math.min(wantedGroups, nodeCount); // cannot have more groups than nodes
-            requestedNodes = NodeSpec.from(nodeCount, resources.get(), exclusive, requestedCapacity.canFail());
+                () -> "Received deploy prepare request for " + requested +
+                      " for application " + application + ", cluster " + cluster);
+
+        if (cluster.group().isPresent()) throw new IllegalArgumentException("Node requests cannot specify a group");
 
-            if ( ! hasQuota(application, nodeCount))
-                throw new IllegalArgumentException(requestedCapacity + " requested for " + cluster +
-                                                   (requestedCapacity.nodeCount() != nodeCount ? " resolved to " + nodeCount + " nodes" : "") +
-                                                   " exceeds your quota. Resolve this at https://cloud.vespa.ai/quota");
+        if ( ! hasQuota(application, requested.maxResources().nodes()))
+            throw new IllegalArgumentException(requested + " requested for " + cluster +
+                                               ". Max value exceeds your quota. Resolve this at https://cloud.vespa.ai/quota");
+
+        int groups;
+        NodeResources resources;
+        NodeSpec nodeSpec;
+        if ( requested.type() == NodeType.tenant) {
+            ClusterResources target = decideTargetResources(application, cluster.id(), requested);
+            int nodeCount = capacityPolicies.decideSize(target.nodes(), requested, cluster, application);
+            resources = capacityPolicies.decideNodeResources(target.nodeResources(), requested, cluster);
+            boolean exclusive = capacityPolicies.decideExclusivity(cluster.isExclusive());
+            groups = Math.min(target.groups(), nodeCount); // cannot have more groups than nodes
+            nodeSpec = NodeSpec.from(nodeCount, resources, exclusive, requested.canFail());
+            logIfDownscaled(target.nodes(), nodeCount, cluster, logger);
         }
         else {
-            requestedNodes = NodeSpec.from(requestedCapacity.type());
-            effectiveGroups = 1; // type request with multiple groups is not supported
+            groups = 1; // type request with multiple groups is not supported
+            resources = requested.minResources().nodeResources();
+            nodeSpec = NodeSpec.from(requested.type());
         }
-
-        return asSortedHosts(preparer.prepare(application, cluster, requestedNodes, effectiveGroups), resources);
+        return asSortedHosts(preparer.prepare(application, cluster, nodeSpec, groups), resources);
     }
 
     @Override
@@ -128,6 +139,50 @@ public class NodeRepositoryProvisioner implements Provisioner {
         loadBalancerProvisioner.ifPresent(lbProvisioner -> lbProvisioner.deactivate(application, transaction));
     }
 
+    /**
+     * Returns the target cluster resources, a value between the min and max in the requested capacity,
+     * and updates the application store with the received min and max,
+     */
+    private ClusterResources decideTargetResources(ApplicationId applicationId, ClusterSpec.Id clusterId, Capacity requested) {
+        try (Mutex lock = nodeRepository.lock(applicationId)) {
+            Application application = nodeRepository.applications().get(applicationId, true);
+            application = application.withClusterLimits(clusterId, requested.minResources(), requested.maxResources());
+            nodeRepository.applications().set(applicationId, application, lock);
+            return application.cluster(clusterId).targetResources()
+                    .orElseGet(() -> currentResources(applicationId, clusterId, requested)
+                    .orElse(requested.minResources()));
+        }
+    }
+
+    /** Returns the current resources of this cluster, if it's already deployed and inside the requested limits */
+    private Optional<ClusterResources> currentResources(ApplicationId applicationId,
+                                                        ClusterSpec.Id clusterId,
+                                                        Capacity requested) {
+        List<Node> nodes = NodeList.copyOf(nodeRepository.getNodes(applicationId, Node.State.active))
+                                   .cluster(clusterId)
+                                   .not().retired()
+                                   .not().removable()
+                                   .asList();
+        if (nodes.isEmpty()) return Optional.empty();
+        long groups = nodes.stream().map(node -> node.allocation().get().membership().cluster().group()).distinct().count();
+
+        // To allow non-numeric settings to be updated without resetting to the min target, we need to use
+        // the non-numeric settings of the current min limit with the current numeric settings
+        NodeResources nodeResources = nodes.get(0).allocation().get().requestedResources()
+                                           .with(requested.minResources().nodeResources().diskSpeed())
+                                           .with(requested.maxResources().nodeResources().storageType());
+        var currentResources = new ClusterResources(nodes.size(), (int)groups, nodeResources);
+        if ( ! currentResources.isWithin(requested.minResources(), requested.maxResources())) return Optional.empty();
+
+        return Optional.of(currentResources);
+    }
+
+    private void logIfDownscaled(int targetNodes, int actualNodes, ClusterSpec cluster, ProvisionLogger logger) {
+        if (zone.environment().isManuallyDeployed() && actualNodes < targetNodes)
+            logger.log(Level.INFO, "Requested " + targetNodes + " nodes for " + cluster +
+                                   ", downscaling to " + actualNodes + " nodes in " + zone.environment());
+    }
+
     private boolean hasQuota(ApplicationId application, int requestedNodes) {
         if ( ! this.zone.system().isPublic()) return true; // no quota management
 
@@ -136,7 +191,7 @@ public class NodeRepositoryProvisioner implements Provisioner {
         return requestedNodes <= 5;
     }
 
-    private List<HostSpec> asSortedHosts(List<Node> nodes, Optional<NodeResources> requestedResources) {
+    private List<HostSpec> asSortedHosts(List<Node> nodes, NodeResources requestedResources) {
         nodes.sort(Comparator.comparingInt(node -> node.allocation().get().membership().index()));
         List<HostSpec> hosts = new ArrayList<>(nodes.size());
         for (Node node : nodes) {
@@ -148,7 +203,8 @@ public class NodeRepositoryProvisioner implements Provisioner {
                                    Optional.of(nodeAllocation.membership()),
                                    node.status().vespaVersion(),
                                    nodeAllocation.networkPorts(),
-                                   requestedResources));
+                                   requestedResources == NodeResources.unspecified ? Optional.empty() : Optional.of(requestedResources),
+                                   node.status().dockerImage()));
             if (nodeAllocation.networkPorts().isPresent()) {
                 log.log(LogLevel.DEBUG, () -> "Prepared node " + node.hostname() + " has port allocations");
             }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceLimits.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceLimits.java
new file mode 100644
index 00000000000..ca04bf66ce3
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceLimits.java
@@ -0,0 +1,32 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.provisioning;
+
+import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.config.provision.NodeResources;
+import com.yahoo.config.provision.SystemName;
+import com.yahoo.config.provision.Zone;
+
+/**
+ * Defines the resource limits for nodes in various zones
+ *
+ * @author bratseth
+ */
+public class NodeResourceLimits {
+
+    private final Zone zone;
+
+    public NodeResourceLimits(Zone zone) {
+        this.zone = zone;
+    }
+
+    public int minMemoryGb(ClusterSpec.Type clusterType) {
+        if (zone.system() == SystemName.dev) return 1; // Allow small containers in dev system
+        if (clusterType == ClusterSpec.Type.admin) return 2;
+        return 4;
+    }
+
+    public NodeResources enlargeToLegal(NodeResources resources, ClusterSpec.Type clusterType) {
+        return resources.withMemoryGb(Math.max(minMemoryGb(clusterType), resources.memoryGb()));
+    }
+
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java
index 72be68a7ee3..91c15cdb61b 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java
@@ -121,8 +121,7 @@ class Preparer {
      */
     private int findHighestIndex(ApplicationId application, ClusterSpec cluster) {
         int highestIndex = -1;
-        for (Node node : nodeRepository.getNodes(application,
-                                                 Node.State.active, Node.State.inactive, Node.State.parked, Node.State.failed)) {
+        for (Node node : nodeRepository.getNodes(application, Node.State.allocatedStates().toArray(new Node.State[0]))) {
             ClusterSpec nodeCluster = node.allocation().get().membership().cluster();
             if ( ! nodeCluster.id().equals(cluster.id())) continue;
             if ( ! nodeCluster.type().equals(cluster.type())) continue;
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/PrioritizableNode.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/PrioritizableNode.java
index 6183bffe5ba..8cfcbcb3797 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/PrioritizableNode.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/PrioritizableNode.java
@@ -4,10 +4,9 @@ package com.yahoo.vespa.hosted.provision.provisioning;
 import com.yahoo.config.provision.NodeResources;
 import com.yahoo.vespa.hosted.provision.Node;
 
+import java.util.List;
 import java.util.Optional;
 
-import static com.yahoo.vespa.hosted.provision.provisioning.NodePrioritizer.ALLOCATABLE_HOST_STATES;
-
 /**
  * A node with additional information required to prioritize it for allocation.
  *
@@ -15,6 +14,10 @@ import static com.yahoo.vespa.hosted.provision.provisioning.NodePrioritizer.ALLO
  */
 class PrioritizableNode implements Comparable<PrioritizableNode> {
 
+    /** List of host states ordered by preference (ascending) */
+    private static final List<Node.State> HOST_STATE_PRIORITY =
+            List.of(Node.State.provisioned, Node.State.ready, Node.State.active);
+
     private static final NodeResources zeroResources =
             new NodeResources(0, 0, 0, 0, NodeResources.DiskSpeed.any, NodeResources.StorageType.any);
 
@@ -111,8 +114,8 @@ class PrioritizableNode implements Comparable<PrioritizableNode> {
         if (other.node.flavor().cost() < this.node.flavor().cost()) return 1;
 
         // Choose nodes where host is in more desirable state
-        int thisHostStatePri = this.parent.map(host -> ALLOCATABLE_HOST_STATES.indexOf(host.state())).orElse(-2);
-        int otherHostStatePri = other.parent.map(host -> ALLOCATABLE_HOST_STATES.indexOf(host.state())).orElse(-2);
+        int thisHostStatePri = this.parent.map(host -> HOST_STATE_PRIORITY.indexOf(host.state())).orElse(-2);
+        int otherHostStatePri = other.parent.map(host -> HOST_STATE_PRIORITY.indexOf(host.state())).orElse(-2);
         if (thisHostStatePri != otherHostStatePri) return otherHostStatePri - thisHostStatePri;
 
         // All else equal choose hostname alphabetically
@@ -142,6 +145,18 @@ class PrioritizableNode implements Comparable<PrioritizableNode> {
         return node.id();
     }
 
+    @Override
+    public int hashCode() {
+        return node.hashCode();
+    }
+
+    @Override
+    public boolean equals(Object other) {
+        if (other == this) return true;
+        if ( ! (other instanceof PrioritizableNode)) return false;
+        return this.node.equals(((PrioritizableNode)other).node);
+    }
+
     static class Builder {
 
         public final Node node;
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/JobsResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/JobsResponse.java
index f3d8f42f3b7..4dfdef742d6 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/JobsResponse.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/JobsResponse.java
@@ -26,7 +26,7 @@ public class JobsResponse extends HttpResponse {
         Slime slime = new Slime();
         Cursor root = slime.setObject();
         Cursor jobArray = root.setArray("jobs");
-        for (String jobName : new TreeSet<>(jobControl.jobs()))
+        for (String jobName : jobControl.jobs())
             jobArray.addObject().setString("name", jobName);
 
         Cursor inactiveArray = root.setArray("inactive");
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/LoadBalancersResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/LoadBalancersResponse.java
index 9f8f4a804d1..3147d4caded 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/LoadBalancersResponse.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/LoadBalancersResponse.java
@@ -1,4 +1,4 @@
-// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.provision.restapi.v2;
 
 import com.yahoo.config.provision.ApplicationId;
@@ -37,10 +37,14 @@ public class LoadBalancersResponse extends HttpResponse {
     }
 
     private List<LoadBalancer> loadBalancers() {
-        LoadBalancerList loadBalancers = nodeRepository.loadBalancers();
-        return application().map(loadBalancers::owner)
-                            .map(LoadBalancerList::asList)
-                            .orElseGet(loadBalancers::asList);
+        LoadBalancerList loadBalancers;
+        var application = application();
+        if (application.isPresent()) {
+            loadBalancers = nodeRepository.loadBalancers(application.get());
+        } else {
+            loadBalancers = nodeRepository.loadBalancers();
+        }
+        return loadBalancers.asList();
     }
 
     @Override
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodePatcher.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodePatcher.java
index 09cb5dad0a9..12b245ede7f 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodePatcher.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodePatcher.java
@@ -6,12 +6,13 @@ import com.yahoo.config.provision.DockerImage;
 import com.yahoo.config.provision.Flavor;
 import com.yahoo.config.provision.NodeFlavors;
 import com.yahoo.config.provision.NodeResources;
+import com.yahoo.config.provision.NodeType;
 import com.yahoo.config.provision.TenantName;
 import com.yahoo.io.IOUtils;
 import com.yahoo.slime.Inspector;
 import com.yahoo.slime.ObjectTraverser;
 import com.yahoo.slime.Type;
-import com.yahoo.vespa.config.SlimeUtils;
+import com.yahoo.slime.SlimeUtils;
 import com.yahoo.vespa.hosted.provision.LockedNodeList;
 import com.yahoo.vespa.hosted.provision.Node;
 import com.yahoo.vespa.hosted.provision.node.Agent;
@@ -30,12 +31,13 @@ import java.util.List;
 import java.util.Optional;
 import java.util.Set;
 import java.util.TreeSet;
+import java.util.function.Supplier;
 import java.util.stream.Collectors;
 
 import static com.yahoo.config.provision.NodeResources.DiskSpeed.fast;
 import static com.yahoo.config.provision.NodeResources.DiskSpeed.slow;
-import static com.yahoo.config.provision.NodeResources.StorageType.remote;
 import static com.yahoo.config.provision.NodeResources.StorageType.local;
+import static com.yahoo.config.provision.NodeResources.StorageType.remote;
 
 /**
  * A class which can take a partial JSON node/v2 node JSON structure and apply it to a node object.
@@ -46,21 +48,17 @@ import static com.yahoo.config.provision.NodeResources.StorageType.local;
 public class NodePatcher {
 
     private static final String WANT_TO_RETIRE = "wantToRetire";
-    private static final String WANT_TO_DEPROVISION = "wantToDeprovision";
 
     private final NodeFlavors nodeFlavors;
     private final Inspector inspector;
-    private final LockedNodeList nodes;
+    private final Supplier<LockedNodeList> nodes;
     private final Clock clock;
 
     private Node node;
-    private List<Node> children;
-    private boolean childrenModified = false;
 
-    public NodePatcher(NodeFlavors nodeFlavors, InputStream json, Node node, LockedNodeList nodes, Clock clock) {
+    public NodePatcher(NodeFlavors nodeFlavors, InputStream json, Node node, Supplier<LockedNodeList> nodes, Clock clock) {
         this.nodeFlavors = nodeFlavors;
         this.node = node;
-        this.children = node.type().isDockerHost() ? nodes.childrenOf(node).asList() : List.of();
         this.nodes = nodes;
         this.clock = clock;
         try {
@@ -76,6 +74,7 @@ public class NodePatcher {
      * children that must be updated in a consistent manner.
      */
     public List<Node> apply() {
+        List<Node> patchedNodes = new ArrayList<>();
         inspector.traverse((String name, Inspector value) -> {
             try {
                 node = applyField(node, name, value);
@@ -84,23 +83,20 @@ public class NodePatcher {
             }
 
             try {
-                children = applyFieldRecursive(children, name, value);
-                childrenModified = true;
+                patchedNodes.addAll(applyFieldRecursive(name, value));
             } catch (IllegalArgumentException e) {
                 // Non recursive field, ignore
             }
         } );
+        patchedNodes.add(node);
 
-        List<Node> nodes = childrenModified ? new ArrayList<>(children) : new ArrayList<>();
-        nodes.add(node);
-
-        return nodes;
+        return patchedNodes;
     }
 
-    private List<Node> applyFieldRecursive(List<Node> childNodes, String name, Inspector value) {
+    private List<Node> applyFieldRecursive(String name, Inspector value) {
         switch (name) {
             case WANT_TO_RETIRE:
-            case WANT_TO_DEPROVISION:
+                List<Node> childNodes = node.type().isDockerHost() ? nodes.get().childrenOf(node).asList() : List.of();
                 return childNodes.stream()
                         .map(child -> applyField(child, name, value))
                         .collect(Collectors.toList());
@@ -128,18 +124,20 @@ public class NodePatcher {
             case "currentFirmwareCheck":
                 return node.withFirmwareVerifiedAt(Instant.ofEpochMilli(asLong(value)));
             case "failCount" :
-                return node.with(node.status().setFailCount(asLong(value).intValue()));
+                return node.with(node.status().withFailCount(asLong(value).intValue()));
             case "flavor" :
                 return node.with(nodeFlavors.getFlavorOrThrow(asString(value)));
             case "parentHostname" :
                 return node.withParentHostname(asString(value));
             case "ipAddresses" :
-                return IP.Config.verify(node.with(node.ipConfig().with(asStringSet(value))), nodes);
+                return IP.Config.verify(node.with(node.ipConfig().with(asStringSet(value))), nodes.get());
             case "additionalIpAddresses" :
-                return IP.Config.verify(node.with(node.ipConfig().with(IP.Pool.of(asStringSet(value)))), nodes);
+                return IP.Config.verify(node.with(node.ipConfig().with(IP.Pool.of(asStringSet(value)))), nodes.get());
             case WANT_TO_RETIRE :
                 return node.withWantToRetire(asBoolean(value), Agent.operator, clock.instant());
-            case WANT_TO_DEPROVISION :
+            case "wantToDeprovision" :
+                if (node.type() != NodeType.host && asBoolean(value))
+                    throw new IllegalArgumentException("wantToDeprovision can only be set for hosts");
                 return node.with(node.status().withWantToDeprovision(asBoolean(value)));
             case "reports" :
                 return nodeWithPatchedReports(node, value);
@@ -172,22 +170,42 @@ public class NodePatcher {
     }
 
     private Node nodeWithPatchedReports(Node node, Inspector reportsInspector) {
+        Node patchedNode;
         // "reports": null clears the reports
-        if (reportsInspector.type() == Type.NIX) return node.with(new Reports());
+        if (reportsInspector.type() == Type.NIX) {
+            patchedNode = node.with(new Reports());
+        } else {
+            var reportsBuilder = new Reports.Builder(node.reports());
+            reportsInspector.traverse((ObjectTraverser) (reportId, reportInspector) -> {
+                if (reportInspector.type() == Type.NIX) {
+                    // ... "reports": { "reportId": null } clears the report "reportId"
+                    reportsBuilder.clearReport(reportId);
+                } else {
+                    // ... "reports": { "reportId": {...} } overrides the whole report "reportId"
+                    reportsBuilder.setReport(Report.fromSlime(reportId, reportInspector));
+                }
+            });
+            patchedNode = node.with(reportsBuilder.build());
+        }
 
-        var reportsBuilder = new Reports.Builder(node.reports());
+        boolean hadHardFailReports = node.reports().getReports().stream()
+                .anyMatch(r -> r.getType() == Report.Type.HARD_FAIL);
+        boolean hasHardFailReports = patchedNode.reports().getReports().stream()
+                .anyMatch(r -> r.getType() == Report.Type.HARD_FAIL);
 
-        reportsInspector.traverse((ObjectTraverser) (reportId, reportInspector) -> {
-            if (reportInspector.type() == Type.NIX) {
-                // ... "reports": { "reportId": null } clears the report "reportId"
-                reportsBuilder.clearReport(reportId);
-            } else {
-                // ... "reports": { "reportId": {...} } overrides the whole report "reportId"
-                reportsBuilder.setReport(Report.fromSlime(reportId, reportInspector));
-            }
-        });
+        // If this patch resulted in going from not having HARD_FAIL report to having one, or vice versa
+        if (hadHardFailReports != hasHardFailReports) {
+            // Do not automatically change wantToDeprovision when
+            // 1. Transitioning to having a HARD_FAIL report and being in state failed:
+            //    To allow operators manually unset before the host is parked and deleted.
+            // 2. When in parked state: Deletion is imminent, possibly already underway
+            if ((hasHardFailReports && node.state() == Node.State.failed) || node.state() == Node.State.parked)
+                return patchedNode;
+
+            patchedNode = patchedNode.with(patchedNode.status().withWantToDeprovision(hasHardFailReports));
+        }
 
-        return node.with(reportsBuilder.build());
+        return patchedNode;
     }
 
     private Set<String> asStringSet(Inspector field) {
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodeSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodeSerializer.java
index 5ca2667ad5d..492994c2f0e 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodeSerializer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodeSerializer.java
@@ -22,6 +22,7 @@ public class NodeSerializer {
             case "provisioned": return Node.State.provisioned;
             case "ready": return Node.State.ready;
             case "reserved": return Node.State.reserved;
+            case "deprovisioned": return Node.State.deprovisioned;
             default: throw new IllegalArgumentException("Unknown node state '" + state + "'");
         }
     }
@@ -36,6 +37,7 @@ public class NodeSerializer {
             case provisioned: return "provisioned";
             case ready: return "ready";
             case reserved: return "reserved";
+            case deprovisioned: return "deprovisioned";
             default: throw new IllegalArgumentException("Unknown node state '" + state + "'");
         }
     }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java
index c64d1a9b5ff..809e4200e7e 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java
@@ -20,8 +20,8 @@ import com.yahoo.restapi.ResourceResponse;
 import com.yahoo.slime.ArrayTraverser;
 import com.yahoo.slime.Inspector;
 import com.yahoo.slime.Slime;
+import com.yahoo.slime.SlimeUtils;
 import com.yahoo.slime.Type;
-import com.yahoo.vespa.config.SlimeUtils;
 import com.yahoo.vespa.hosted.provision.NoSuchNodeException;
 import com.yahoo.vespa.hosted.provision.Node;
 import com.yahoo.vespa.hosted.provision.NodeRepository;
@@ -51,7 +51,7 @@ import java.util.function.Function;
 import java.util.logging.Level;
 import java.util.stream.Collectors;
 
-import static com.yahoo.vespa.config.SlimeUtils.optionalString;
+import static com.yahoo.slime.SlimeUtils.optionalString;
 
 /**
  * The implementation of the /nodes/v2 API.
@@ -145,11 +145,12 @@ public class NodesApiHandler extends LoggingRequestHandler {
     private HttpResponse handlePATCH(HttpRequest request) {
         String path = request.getUri().getPath();
         if (path.startsWith("/nodes/v2/node/")) {
+            // TODO: Node is fetched outside of lock, might change after getting lock
             Node node = nodeFromRequest(request);
             try (var lock = nodeRepository.lock(node)) {
-                var patchedNode = new NodePatcher(nodeFlavors, request.getData(), node, nodeRepository.list(lock),
-                                                  nodeRepository.clock()).apply();
-                nodeRepository.write(patchedNode, lock);
+                var patchedNodes = new NodePatcher(nodeFlavors, request.getData(), node, () -> nodeRepository.list(lock),
+                                                   nodeRepository.clock()).apply();
+                nodeRepository.write(patchedNodes, lock);
             }
             return new MessageResponse("Updated " + node.hostname());
         }
@@ -175,11 +176,17 @@ public class NodesApiHandler extends LoggingRequestHandler {
             return new MessageResponse("Added " + addedNodes + " nodes to the provisioned state");
         }
         if (path.matches("/nodes/v2/maintenance/inactive/{job}")) return setJobActive(path.get("job"), false);
+        if (path.matches("/nodes/v2/maintenance/run/{job}")) return runJob(path.get("job"));
         if (path.matches("/nodes/v2/upgrade/firmware")) return requestFirmwareCheckResponse();
 
         throw new NotFoundException("Nothing at path '" + request.getUri().getPath() + "'");
     }
 
+    private HttpResponse runJob(String job) {
+        nodeRepository.jobControl().run(job);
+        return new MessageResponse("Executed job '" + job + "'");
+    }
+
     private HttpResponse handleDELETE(HttpRequest request) {
         Path path = new Path(request.getUri());
         if (path.matches("/nodes/v2/node/{hostname}")) {
@@ -201,7 +208,7 @@ public class NodesApiHandler extends LoggingRequestHandler {
 
     public int addNodes(InputStream jsonStream) {
         List<Node> nodes = createNodesFromSlime(toSlime(jsonStream).get());
-        return nodeRepository.addNodes(nodes).size();
+        return nodeRepository.addNodes(nodes, Agent.operator).size();
     }
 
     private Slime toSlime(InputStream jsonStream) {
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java
index 0410bbd8064..a4412a502aa 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java
@@ -16,11 +16,10 @@ import com.yahoo.slime.Slime;
 import com.yahoo.vespa.applicationmodel.HostName;
 import com.yahoo.vespa.hosted.provision.Node;
 import com.yahoo.vespa.hosted.provision.NodeRepository;
-import com.yahoo.vespa.hosted.provision.node.Agent;
 import com.yahoo.vespa.hosted.provision.node.History;
 import com.yahoo.vespa.hosted.provision.node.filter.NodeFilter;
 import com.yahoo.vespa.orchestrator.Orchestrator;
-import com.yahoo.vespa.orchestrator.status.HostStatus;
+import com.yahoo.vespa.orchestrator.status.HostInfo;
 
 import java.io.IOException;
 import java.io.OutputStream;
@@ -46,7 +45,7 @@ class NodesResponse extends HttpResponse {
 
     private final NodeFilter filter;
     private final boolean recursive;
-    private final Function<HostName, Optional<HostStatus>> orchestrator;
+    private final Function<HostName, Optional<HostInfo>> orchestrator;
     private final NodeRepository nodeRepository;
     private final Slime slime;
     private final NodeSerializer serializer = new NodeSerializer();
@@ -58,7 +57,7 @@ class NodesResponse extends HttpResponse {
         this.nodeParentUrl = toNodeParentUrl(request);
         filter = NodesApiHandler.toNodeFilter(request);
         this.recursive = request.getBooleanProperty("recursive");
-        this.orchestrator = orchestrator.getNodeStatuses();
+        this.orchestrator = orchestrator.getHostResolver();
         this.nodeRepository = nodeRepository;
 
         slime = new Slime();
@@ -158,13 +157,16 @@ class NodesResponse extends HttpResponse {
             toSlime(allocation.membership(), object.setObject("membership"));
             object.setLong("restartGeneration", allocation.restartGeneration().wanted());
             object.setLong("currentRestartGeneration", allocation.restartGeneration().current());
-            object.setString("wantedDockerImage", dockerImageFor(node.type()).withTag(allocation.membership().cluster().vespaVersion()).asString());
+            object.setString("wantedDockerImage", allocation.membership().cluster().dockerImage()
+                    .orElseGet(() -> nodeRepository.dockerImage(node).withTag(allocation.membership().cluster().vespaVersion()).asString()));
             object.setString("wantedVespaVersion", allocation.membership().cluster().vespaVersion().toFullString());
             toSlime(allocation.requestedResources(), object.setObject("requestedResources"));
             allocation.networkPorts().ifPresent(ports -> NetworkPortsSerializer.toSlime(ports, object.setArray("networkPorts")));
             orchestrator.apply(new HostName(node.hostname()))
-                        .map(status -> status == HostStatus.ALLOWED_TO_BE_DOWN)
-                        .ifPresent(allowedToBeDown -> object.setBool("allowedToBeDown", allowedToBeDown));
+                        .ifPresent(info -> {
+                            object.setBool("allowedToBeDown", info.status().isSuspended());
+                            info.suspendedSince().ifPresent(since -> object.setLong("suspendedSinceMillis", since.toEpochMilli()));
+                        });
         });
         object.setLong("rebootGeneration", node.status().reboot().wanted());
         object.setLong("currentRebootGeneration", node.status().reboot().current());
@@ -217,20 +219,14 @@ class NodesResponse extends HttpResponse {
         object.setString("storageType", serializer.toString(resources.storageType()));
     }
 
-    // Hack: For non-docker noder, return current docker image as default prefix + current Vespa version
+    // Hack: For non-docker nodes, return current docker image as default prefix + current Vespa version
     // TODO: Remove current + wanted docker image from response for non-docker types
     private Optional<DockerImage> currentDockerImage(Node node) {
         return node.status().dockerImage()
-                .or(() -> Optional.of(node)
-                        .filter(n -> n.flavor().getType() != Flavor.Type.DOCKER_CONTAINER)
-                        .flatMap(n -> n.status().vespaVersion()
-                                .map(version -> dockerImageFor(n.type()).withTag(version))));
-    }
-
-    // Docker hosts are not running in an image, but return the image of the node type running on it anyway,
-    // this allows the docker host to pre-download the (likely) image its node will run
-    private DockerImage dockerImageFor(NodeType nodeType) {
-        return nodeRepository.dockerImage(nodeType.isDockerHost() ? nodeType.childNodeType() : nodeType);
+                   .or(() -> Optional.of(node)
+                                     .filter(n -> n.flavor().getType() != Flavor.Type.DOCKER_CONTAINER)
+                                     .flatMap(n -> n.status().vespaVersion()
+                                                    .map(version -> nodeRepository.dockerImage(n).withTag(version))));
     }
 
     private void ipAddressesToSlime(Set<String> ipAddresses, Cursor array) {
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java
index 888b8835e49..d26accd7a84 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java
@@ -13,7 +13,8 @@ public class ContainerConfig {
         return "<container version='1.0'>\n" +
                "  <config name=\"container.handler.threadpool\">\n" +
                "    <maxthreads>20</maxthreads>\n" +
-               "  </config>  \n" +
+               "  </config>\n" +
+               "  <accesslog type='disabled'/>\n" +
                "  <component id='com.yahoo.test.ManualClock'/>\n" +
                "  <component id='com.yahoo.vespa.curator.mock.MockCurator'/>\n" +
                "  <component id='com.yahoo.vespa.hosted.provision.testutils.OrchestratorMock'/>\n" +
@@ -24,6 +25,8 @@ public class ContainerConfig {
                "  <component id='com.yahoo.vespa.hosted.provision.testutils.ServiceMonitorStub'/>\n" +
                "  <component id='com.yahoo.vespa.hosted.provision.testutils.MockDuperModel'/>\n" +
                "  <component id='com.yahoo.vespa.hosted.provision.testutils.MockNodeFlavors'/>\n" +
+               "  <component id='com.yahoo.vespa.hosted.provision.autoscale.NodeMetricsDb'/>\n" +
+               "  <component id='com.yahoo.vespa.hosted.provision.testutils.MockNodeMetrics'/>\n" +
                "  <component id='com.yahoo.vespa.hosted.provision.testutils.MockNodeRepository'/>\n" +
                "  <component id='com.yahoo.vespa.hosted.provision.testutils.MockProvisionServiceProvider'/>\n" +
                "  <component id='com.yahoo.vespa.hosted.provision.maintenance.NodeRepositoryMaintenance'/>\n" +
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDeployer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDeployer.java
index 7ab42093bab..5ec5c2c08e8 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDeployer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDeployer.java
@@ -146,8 +146,8 @@ public class MockDeployer implements Deployer {
             this.clusterContexts = clusterContexts;
         }
 
-        public ApplicationContext(ApplicationId id, ClusterSpec cluster, Capacity capacity, int groups) {
-            this(id, List.of(new ClusterContext(id, cluster, capacity, groups)));
+        public ApplicationContext(ApplicationId id, ClusterSpec cluster, Capacity capacity) {
+            this(id, List.of(new ClusterContext(id, cluster, capacity)));
         }
 
         public ApplicationId id() { return id; }
@@ -169,13 +169,11 @@ public class MockDeployer implements Deployer {
         private final ApplicationId id;
         private final ClusterSpec cluster;
         private final Capacity capacity;
-        private final int groups;
 
-        public ClusterContext(ApplicationId id, ClusterSpec cluster, Capacity capacity, int groups) {
+        public ClusterContext(ApplicationId id, ClusterSpec cluster, Capacity capacity) {
             this.id = id;
             this.cluster = cluster;
             this.capacity = capacity;
-            this.groups = groups;
         }
 
         public ApplicationId id() { return id; }
@@ -183,7 +181,7 @@ public class MockDeployer implements Deployer {
         public ClusterSpec cluster() { return cluster; }
 
         private List<HostSpec> prepare(NodeRepositoryProvisioner provisioner) {
-            return provisioner.prepare(id, cluster, capacity, groups, null);
+            return provisioner.prepare(id, cluster, capacity, null);
         }
 
     }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDuperModel.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDuperModel.java
index 62e17ab63ad..e7ebf049e51 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDuperModel.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDuperModel.java
@@ -18,6 +18,7 @@ import java.util.concurrent.ConcurrentHashMap;
  * @author hakonhall
  */
 public class MockDuperModel implements DuperModelInfraApi {
+
     private final Map<ApplicationId, InfraApplicationApi> supportedInfraApps = new HashMap<>();
     private final ConcurrentHashMap<ApplicationId, List<HostName>> activeApps = new ConcurrentHashMap<>();
 
@@ -49,4 +50,8 @@ public class MockDuperModel implements DuperModelInfraApi {
     public void infraApplicationRemoved(ApplicationId applicationId) {
         activeApps.remove(applicationId);
     }
+
+    @Override
+    public void infraApplicationsIsNowComplete() {
+    }
 }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockInfraDeployer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockInfraDeployer.java
index e7bf76986ca..742a863fb38 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockInfraDeployer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockInfraDeployer.java
@@ -5,7 +5,6 @@ import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.Deployment;
 import com.yahoo.config.provision.InfraDeployer;
 
-import java.util.Map;
 import java.util.Optional;
 
 public class MockInfraDeployer implements InfraDeployer {
@@ -15,7 +14,5 @@ public class MockInfraDeployer implements InfraDeployer {
     }
 
     @Override
-    public Map<ApplicationId, Deployment> getSupportedInfraDeployments() {
-        return Map.of();
-    }
+    public void activateAllSupportedInfraApplications(boolean propagateException) { }
 }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeMetrics.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeMetrics.java
new file mode 100644
index 00000000000..d5397aa421c
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeMetrics.java
@@ -0,0 +1,20 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.testutils;
+
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.vespa.hosted.provision.autoscale.NodeMetrics;
+
+import java.util.ArrayList;
+import java.util.Collection;
+
+/**
+ * @author bratseth
+ */
+public class MockNodeMetrics implements NodeMetrics {
+
+    @Override
+    public Collection<MetricValue> fetchMetrics(ApplicationId application) {
+        return new ArrayList<>();
+    }
+
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java
index 1817470a63b..8ff17a8e5a3 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java
@@ -5,6 +5,7 @@ import com.yahoo.component.Version;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.ApplicationName;
 import com.yahoo.config.provision.Capacity;
+import com.yahoo.config.provision.ClusterResources;
 import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.config.provision.DockerImage;
 import com.yahoo.config.provision.Flavor;
@@ -128,7 +129,7 @@ public class MockNodeRepository extends NodeRepository {
                              flavors.getFlavorOrThrow("default"), Optional.empty(), NodeType.config));
 
         // Ready all nodes, except 7 and 55
-        nodes = addNodes(nodes);
+        nodes = addNodes(nodes, Agent.system);
         nodes.remove(node7);
         nodes.remove(node55);
         nodes = setDirty(nodes, Agent.system, getClass().getSimpleName());
@@ -138,46 +139,31 @@ public class MockNodeRepository extends NodeRepository {
         dirtyRecursively(node55.hostname(), Agent.system, getClass().getSimpleName());
 
         ApplicationId zoneApp = ApplicationId.from(TenantName.from("zoneapp"), ApplicationName.from("zoneapp"), InstanceName.from("zoneapp"));
-        ClusterSpec zoneCluster = ClusterSpec.request(ClusterSpec.Type.container,
-                                                      ClusterSpec.Id.from("node-admin"),
-                                                      Version.fromString("6.42"),
-                                                      false);
-        activate(provisioner.prepare(zoneApp, zoneCluster, Capacity.fromRequiredNodeType(NodeType.host), 1, null), zoneApp, provisioner);
+        ClusterSpec zoneCluster = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("node-admin")).vespaVersion("6.42").build();
+        activate(provisioner.prepare(zoneApp, zoneCluster, Capacity.fromRequiredNodeType(NodeType.host), null), zoneApp, provisioner);
 
         ApplicationId app1 = ApplicationId.from(TenantName.from("tenant1"), ApplicationName.from("application1"), InstanceName.from("instance1"));
-        ClusterSpec cluster1 = ClusterSpec.request(ClusterSpec.Type.container,
-                                                   ClusterSpec.Id.from("id1"),
-                                                   Version.fromString("6.42"),
-                                                   false);
-        provisioner.prepare(app1, cluster1, Capacity.fromCount(2, new NodeResources(2, 8, 50, 1)), 1, null);
+        ClusterSpec cluster1 = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("id1")).vespaVersion("6.42").build();
+        provisioner.prepare(app1, cluster1, Capacity.from(new ClusterResources(2, 1, new NodeResources(2, 8, 50, 1))), null);
 
         ApplicationId app2 = ApplicationId.from(TenantName.from("tenant2"), ApplicationName.from("application2"), InstanceName.from("instance2"));
-        ClusterSpec cluster2 = ClusterSpec.request(ClusterSpec.Type.content,
-                                                   ClusterSpec.Id.from("id2"),
-                                                   Version.fromString("6.42"),
-                                                   false);
-        activate(provisioner.prepare(app2, cluster2, Capacity.fromCount(2, new NodeResources(2, 8, 50, 1)), 1, null), app2, provisioner);
+        ClusterSpec cluster2 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("id2")).vespaVersion("6.42").build();
+        activate(provisioner.prepare(app2, cluster2, Capacity.from(new ClusterResources(2, 1, new NodeResources(2, 8, 50, 1))), null), app2, provisioner);
 
         ApplicationId app3 = ApplicationId.from(TenantName.from("tenant3"), ApplicationName.from("application3"), InstanceName.from("instance3"));
-        ClusterSpec cluster3 = ClusterSpec.request(ClusterSpec.Type.content,
-                                                   ClusterSpec.Id.from("id3"),
-                                                   Version.fromString("6.42"),
-                                                   false);
-        activate(provisioner.prepare(app3, cluster3, Capacity.fromCount(2, new NodeResources(1, 4, 100, 1), false, true), 1, null), app3, provisioner);
+        ClusterSpec cluster3 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("id3")).vespaVersion("6.42").build();
+        activate(provisioner.prepare(app3, cluster3, Capacity.from(new ClusterResources(2, 1, new NodeResources(1, 4, 100, 1)), false, true), null), app3, provisioner);
 
         List<Node> largeNodes = new ArrayList<>();
         largeNodes.add(createNode("node13", "host13.yahoo.com", ipConfig(13), Optional.empty(),
                                   new Flavor(new NodeResources(10, 48, 500, 1, fast, local)), Optional.empty(), NodeType.tenant));
         largeNodes.add(createNode("node14", "host14.yahoo.com", ipConfig(14), Optional.empty(),
                                   new Flavor(new NodeResources(10, 48, 500, 1, fast, local)), Optional.empty(), NodeType.tenant));
-        addNodes(largeNodes);
+        addNodes(largeNodes, Agent.system);
         setReady(largeNodes, Agent.system, getClass().getSimpleName());
         ApplicationId app4 = ApplicationId.from(TenantName.from("tenant4"), ApplicationName.from("application4"), InstanceName.from("instance4"));
-        ClusterSpec cluster4 = ClusterSpec.request(ClusterSpec.Type.container,
-                                                   ClusterSpec.Id.from("id4"),
-                                                   Version.fromString("6.42"),
-                                                   false);
-        activate(provisioner.prepare(app4, cluster4, Capacity.fromCount(2, new NodeResources(10, 48, 500, 1), false, true), 1, null), app4, provisioner);
+        ClusterSpec cluster4 = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("id4")).vespaVersion("6.42").build();
+        activate(provisioner.prepare(app4, cluster4, Capacity.from(new ClusterResources(2, 1, new NodeResources(10, 48, 500, 1)), false, true), null), app4, provisioner);
     }
 
     private void activate(List<HostSpec> hosts, ApplicationId application, NodeRepositoryProvisioner provisioner) {
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockProvisioner.java
index 3915ae41d6e..045d0cad1ad 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockProvisioner.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockProvisioner.java
@@ -20,11 +20,17 @@ import java.util.List;
 public class MockProvisioner implements Provisioner {
 
     @Override
+    @Deprecated // TODO: Remove after April 2020
     public List<HostSpec> prepare(ApplicationId applicationId, ClusterSpec cluster, Capacity capacity, int groups, ProvisionLogger logger) {
         return Collections.emptyList();
     }
 
     @Override
+    public List<HostSpec> prepare(ApplicationId applicationId, ClusterSpec cluster, Capacity capacity, ProvisionLogger logger) {
+        return Collections.emptyList();
+    }
+
+    @Override
     public void activate(NestedTransaction transaction, ApplicationId application, Collection<HostSpec> hosts) {
 
     }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/OrchestratorMock.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/OrchestratorMock.java
index 96ec0349fb2..ff9cb98783c 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/OrchestratorMock.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/OrchestratorMock.java
@@ -1,16 +1,22 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.provision.testutils;
 
+import com.yahoo.component.AbstractComponent;
 import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference;
 import com.yahoo.vespa.applicationmodel.HostName;
 import com.yahoo.vespa.orchestrator.Host;
 import com.yahoo.vespa.orchestrator.Orchestrator;
 import com.yahoo.vespa.orchestrator.status.ApplicationInstanceStatus;
+import com.yahoo.vespa.orchestrator.status.HostInfo;
 import com.yahoo.vespa.orchestrator.status.HostStatus;
 
+import java.time.Instant;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
 import java.util.function.Function;
@@ -18,9 +24,9 @@ import java.util.function.Function;
 /**
  * @author bratseth
  */
-public class OrchestratorMock implements Orchestrator {
+public class OrchestratorMock extends AbstractComponent implements Orchestrator {
 
-    private final Set<HostName> suspendedHosts = new HashSet<>();
+    private final Map<HostName, HostInfo> suspendedHosts = new HashMap<>();
     private final Set<ApplicationId> suspendedApplications = new HashSet<>();
 
     @Override
@@ -30,12 +36,19 @@ public class OrchestratorMock implements Orchestrator {
 
     @Override
     public HostStatus getNodeStatus(HostName hostName) {
-        return suspendedHosts.contains(hostName) ? HostStatus.ALLOWED_TO_BE_DOWN : HostStatus.NO_REMARKS;
+        HostInfo hostInfo = suspendedHosts.get(hostName);
+        return hostInfo == null ? HostStatus.NO_REMARKS : hostInfo.status();
     }
 
     @Override
-    public Function<HostName, Optional<HostStatus>> getNodeStatuses() {
-        return hostName -> Optional.of(getNodeStatus(hostName));
+    public HostInfo getHostInfo(ApplicationInstanceReference reference, HostName hostname) {
+        HostInfo hostInfo = suspendedHosts.get(hostname);
+        return hostInfo == null ? HostInfo.createNoRemarks() : hostInfo;
+    }
+
+    @Override
+    public Function<HostName, Optional<HostInfo>> getHostResolver() {
+        return hostName -> Optional.of(suspendedHosts.getOrDefault(hostName, HostInfo.createNoRemarks()));
     }
 
     @Override
@@ -48,7 +61,7 @@ public class OrchestratorMock implements Orchestrator {
 
     @Override
     public void suspend(HostName hostName) {
-        suspendedHosts.add(hostName);
+        suspendedHosts.put(hostName, HostInfo.createSuspended(HostStatus.ALLOWED_TO_BE_DOWN, Instant.EPOCH));
     }
 
     @Override
@@ -73,7 +86,7 @@ public class OrchestratorMock implements Orchestrator {
     }
 
     @Override
-    public void acquirePermissionToRemove(HostName hostName) {}
+    public void acquirePermissionToRemove(HostName hostName) { }
 
     @Override
     public void suspendAll(HostName parentHostname, List<HostName> hostNames) {
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTest.java
index c9671aeafbe..7bb80fe2a21 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTest.java
@@ -2,13 +2,18 @@
 package com.yahoo.vespa.hosted.provision;
 
 import com.yahoo.config.provision.NodeType;
+import com.yahoo.test.ManualClock;
 import com.yahoo.vespa.hosted.provision.node.Agent;
+import com.yahoo.vespa.hosted.provision.node.History;
+import com.yahoo.vespa.hosted.provision.node.IP;
 import com.yahoo.vespa.hosted.provision.node.Report;
 import com.yahoo.vespa.hosted.provision.node.Reports;
 import org.junit.Test;
 
+import java.time.Duration;
 import java.time.Instant;
 import java.util.Arrays;
+import java.util.EnumSet;
 import java.util.HashSet;
 import java.util.Set;
 import java.util.function.Predicate;
@@ -18,6 +23,7 @@ import static org.hamcrest.core.StringContains.containsString;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
 /**
@@ -118,6 +124,10 @@ public class NodeRepositoryTest {
         tester.addNode("node20", "node20", "host2", "docker", NodeType.tenant);
         assertEquals(6, tester.nodeRepository().getNodes().size());
 
+        Node node = tester.nodeRepository().getNode("host1").get();
+        IP.Config cfg = new IP.Config(Set.of("127.0.0.1"), Set.of());
+        node = node.with(cfg);
+
         tester.setNodeState("node11", Node.State.active);
 
         try {
@@ -130,16 +140,56 @@ public class NodeRepositoryTest {
 
         // Should be OK to delete host2 as both host2 and its only child, node20, are in state provisioned
         tester.nodeRepository().removeRecursively("host2");
-        assertEquals(4, tester.nodeRepository().getNodes().size());
+        assertEquals(5, tester.nodeRepository().getNodes().size());
+        assertEquals(Node.State.deprovisioned, tester.nodeRepository().getNode("host2").get().state());
 
         // Now node10 is in provisioned, set node11 to failed and node12 to ready, and it should be OK to delete host1
         tester.nodeRepository().fail("node11", Agent.system, getClass().getSimpleName());
         tester.nodeRepository().setReady("node12", Agent.system, getClass().getSimpleName());
         tester.nodeRepository().removeRecursively("node12"); // Remove one of the children first instead
-        assertEquals(3, tester.nodeRepository().getNodes().size());
+        assertEquals(4, tester.nodeRepository().getNodes().size());
+        tester.nodeRepository().removeRecursively("host1");
+        assertEquals(Node.State.deprovisioned, tester.nodeRepository().getNode("host1").get().state());
+        assertEquals(IP.Config.EMPTY.primary(), tester.nodeRepository().getNode("host1").get().ipConfig().primary());
+    }
 
+    @Test
+    public void relevant_information_from_deprovisioned_hosts_are_merged_into_readded_host() {
+        NodeRepositoryTester tester = new NodeRepositoryTester();
+        Instant testStart = tester.nodeRepository().clock().instant();
+
+        tester.clock().advance(Duration.ofSeconds(1));
+        tester.addNode("id1", "host1", "default", NodeType.host);
+        tester.addNode("id2", "host2", "default", NodeType.host);
+        assertFalse(tester.nodeRepository().getNode("host1").get().history().hasEventAfter(History.Event.Type.deprovisioned, testStart));
+
+        // Set host 1 properties and deprovision it
+        Node host1 = tester.nodeRepository().getNode("host1").get();
+        host1 = host1.withWantToRetire(true, Agent.system, tester.nodeRepository().clock().instant());
+        host1 = host1.with(host1.status().withWantToDeprovision(true));
+        host1 = host1.withFirmwareVerifiedAt(tester.clock().instant());
+        host1 = host1.with(host1.status().withIncreasedFailCount());
+        host1 = host1.with(host1.reports().withReport(Report.basicReport("id", Report.Type.HARD_FAIL, tester.clock().instant(), "Test report")));
+        tester.nodeRepository().write(host1, tester.nodeRepository().lock(host1));
         tester.nodeRepository().removeRecursively("host1");
-        assertEquals(0, tester.nodeRepository().getNodes().size());
+
+        // Host 1 is deprovisioned and unwanted properties are cleared
+        host1 = tester.nodeRepository().getNode("host1").get();
+        assertEquals(Node.State.deprovisioned, host1.state());
+        assertTrue(host1.history().hasEventAfter(History.Event.Type.deprovisioned, testStart));
+
+        // Adding it again preserves some information from the deprovisioned host and removes it
+        tester.addNode("id2", "host1", "default", NodeType.host);
+        host1 = tester.nodeRepository().getNode("host1").get();
+        assertEquals("This is the newly added node", "id2", host1.id());
+        assertFalse("The old 'host1' is removed",
+                    tester.nodeRepository().getNode("host1", Node.State.deprovisioned).isPresent());
+        assertFalse("Not transferred from deprovisioned host", host1.status().wantToRetire());
+        assertFalse("Not transferred from deprovisioned host", host1.status().wantToDeprovision());
+        assertTrue("Transferred from deprovisioned host", host1.history().hasEventAfter(History.Event.Type.deprovisioned, testStart));
+        assertTrue("Transferred from deprovisioned host", host1.status().firmwareVerifiedAt().isPresent());
+        assertEquals("Transferred from deprovisioned host", 1, host1.status().failCount());
+        assertEquals("Transferred from deprovisioned host", 1, host1.reports().getReports().size());
     }
 
     @Test
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTester.java
index f0f523b9b9b..ddbbb8e4482 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTester.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTester.java
@@ -25,10 +25,9 @@ public class NodeRepositoryTester {
 
     private final NodeFlavors nodeFlavors;
     private final NodeRepository nodeRepository;
-    private final Clock clock;
+    private final ManualClock clock;
     private final MockCurator curator;
-    
-    
+
     public NodeRepositoryTester() {
         nodeFlavors = new NodeFlavors(createConfig());
         clock = new ManualClock();
@@ -50,13 +49,13 @@ public class NodeRepositoryTester {
     public Node addNode(String id, String hostname, String flavor, NodeType type) {
         Node node = nodeRepository.createNode(id, hostname, Optional.empty(), 
                                               nodeFlavors.getFlavorOrThrow(flavor), type);
-        return nodeRepository.addNodes(Collections.singletonList(node)).get(0);
+        return nodeRepository.addNodes(Collections.singletonList(node), Agent.system).get(0);
     }
 
     public Node addNode(String id, String hostname, String parentHostname, String flavor, NodeType type) {
         Node node = nodeRepository.createNode(id, hostname, Optional.of(parentHostname),
-                nodeFlavors.getFlavorOrThrow(flavor), type);
-        return nodeRepository.addNodes(Collections.singletonList(node)).get(0);
+                                              nodeFlavors.getFlavorOrThrow(flavor), type);
+        return nodeRepository.addNodes(Collections.singletonList(node), Agent.system).get(0);
     }
 
     /**
@@ -77,4 +76,6 @@ public class NodeRepositoryTester {
         return b.build();
     }
 
+    public ManualClock clock() { return clock; }
+
 }
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingIntegrationTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingIntegrationTest.java
new file mode 100644
index 00000000000..f02acdc1fca
--- /dev/null
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingIntegrationTest.java
@@ -0,0 +1,133 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.autoscale;
+
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.ClusterResources;
+import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.config.provision.Flavor;
+import com.yahoo.config.provision.HostSpec;
+import com.yahoo.config.provision.NodeResources;
+import com.yahoo.test.ManualClock;
+import com.yahoo.vespa.hosted.provision.Node;
+import com.yahoo.vespa.hosted.provision.applications.Application;
+import com.yahoo.vespa.hosted.provision.applications.Cluster;
+import com.yahoo.vespa.hosted.provision.provisioning.HostResourcesCalculator;
+import com.yahoo.vespa.hosted.provision.testutils.OrchestratorMock;
+import org.junit.Test;
+
+import java.time.Duration;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class AutoscalingIntegrationTest {
+
+    @Test
+    public void testComponentIntegration() {
+        NodeResources nodes = new NodeResources(1, 10, 100, 1);
+        NodeResources hosts = new NodeResources(3, 20, 200, 1);
+
+        AutoscalingTester tester = new AutoscalingTester(hosts);
+        NodeMetricsFetcher fetcher = new NodeMetricsFetcher(tester.nodeRepository(),
+                                                            new OrchestratorMock(),
+                                                            new MockHttpClient(tester.clock()));
+        Autoscaler autoscaler = new Autoscaler(new MockHostResourcesCalculator(), tester.nodeMetricsDb(), tester.nodeRepository());
+
+        ApplicationId application1 = tester.applicationId("test1");
+        ClusterSpec cluster1 = tester.clusterSpec(ClusterSpec.Type.container, "test");
+        Set<String> hostnames = tester.deploy(application1, cluster1, 2, 1, nodes)
+                                      .stream().map(HostSpec::hostname)
+                                      .collect(Collectors.toSet());
+        // The metrics response (below) hardcodes these hostnames:
+        assertEquals(Set.of("node-1-of-host-1.yahoo.com", "node-1-of-host-10.yahoo.com"), hostnames);
+
+        for (int i = 0; i < 1000; i++) {
+            tester.clock().advance(Duration.ofSeconds(10));
+            tester.nodeMetricsDb().add(fetcher.fetchMetrics(application1));
+            tester.clock().advance(Duration.ofSeconds(10));
+            tester.nodeMetricsDb().gc(tester.clock());
+        }
+
+        ClusterResources min = new ClusterResources(2, 1, nodes);
+        ClusterResources max = new ClusterResources(2, 1, nodes);
+
+        Application application = tester.nodeRepository().applications().get(application1, true).withClusterLimits(cluster1.id(), min, max);
+        tester.nodeRepository().applications().set(application1, application, tester.nodeRepository().lock(application1));
+        var scaledResources = autoscaler.autoscale(application.cluster(cluster1.id()),
+                                                   tester.nodeRepository().getNodes(application1));
+        assertTrue(scaledResources.isPresent());
+    }
+
+    private static class MockHttpClient implements NodeMetricsFetcher.HttpClient {
+
+        private final ManualClock clock;
+
+        public MockHttpClient(ManualClock clock) {
+            this.clock = clock;
+        }
+
+        final String cannedResponse =
+                "{\n" +
+                "  \"nodes\": [\n" +
+                "    {\n" +
+                "      \"hostname\": \"node-1-of-host-1.yahoo.com\",\n" +
+                "      \"role\": \"role0\",\n" +
+                "      \"node\": {\n" +
+                "        \"timestamp\": [now],\n" +
+                "        \"metrics\": [\n" +
+                "          {\n" +
+                "            \"values\": {\n" +
+                "              \"cpu.util\": 16.2,\n" +
+                "              \"mem_total.util\": 23.1,\n" +
+                "              \"disk.util\": 82\n" +
+                "            },\n" +
+                "            \"dimensions\": {\n" +
+                "              \"state\": \"active\"\n" +
+                "            }\n" +
+                "          }\n" +
+                "        ]\n" +
+                "      }\n" +
+                "    },\n" +
+                "    {\n" +
+                "      \"hostname\": \"node-1-of-host-10.yahoo.com\",\n" +
+                "      \"role\": \"role1\",\n" +
+                "      \"node\": {\n" +
+                "        \"timestamp\": [now],\n" +
+                "        \"metrics\": [\n" +
+                "          {\n" +
+                "            \"values\": {\n" +
+                "              \"cpu.util\": 20,\n" +
+                "              \"mem_total.util\": 23.1,\n" +
+                "              \"disk.util\": 40\n" +
+                "            },\n" +
+                "            \"dimensions\": {\n" +
+                "              \"state\": \"active\"\n" +
+                "            }\n" +
+                "          }\n" +
+                "        ]\n" +
+                "      }\n" +
+                "    }\n" +
+                "  ]\n" +
+                "}\n";
+
+        @Override
+        public String get(String url) { return cannedResponse.replace("[now]", String.valueOf(clock.millis())); }
+
+        @Override
+        public void close() { }
+
+    }
+
+    private static class MockHostResourcesCalculator implements HostResourcesCalculator {
+
+        @Override
+        public NodeResources realResourcesOf(Node node) { return node.flavor().resources(); }
+
+        @Override
+        public NodeResources advertisedResourcesOf(Flavor flavor) { return flavor.resources(); }
+
+    }
+
+}
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java
new file mode 100644
index 00000000000..497a2a31ce5
--- /dev/null
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java
@@ -0,0 +1,296 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.autoscale;
+
+import com.google.common.collect.Sets;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.CloudName;
+import com.yahoo.config.provision.ClusterResources;
+import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.config.provision.Environment;
+import com.yahoo.config.provision.Flavor;
+import com.yahoo.config.provision.NodeResources;
+import com.yahoo.config.provision.RegionName;
+import com.yahoo.config.provision.SystemName;
+import com.yahoo.config.provision.Zone;
+import com.yahoo.io.IOUtils;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * @author bratseth
+ */
+public class AutoscalingTest {
+
+    @Test
+    public void testAutoscalingSingleContentGroup() {
+        NodeResources hostResources = new NodeResources(3, 100, 100, 1);
+        ClusterResources min = new ClusterResources( 2, 1,
+                                                     new NodeResources(1, 1, 1, 1, NodeResources.DiskSpeed.any));
+        ClusterResources max = new ClusterResources(20, 1,
+                                                    new NodeResources(100, 1000, 1000, 1, NodeResources.DiskSpeed.any));
+        AutoscalingTester tester = new AutoscalingTester(hostResources);
+
+        ApplicationId application1 = tester.applicationId("application1");
+        ClusterSpec cluster1 = tester.clusterSpec(ClusterSpec.Type.content, "cluster1");
+
+        // deploy
+        tester.deploy(application1, cluster1, 5, 1, hostResources);
+
+        assertTrue("No measurements -> No change", tester.autoscale(application1, cluster1.id(), min, max).isEmpty());
+
+        tester.addMeasurements(Resource.cpu, 0.25f, 1f, 60, application1);
+        assertTrue("Too few measurements -> No change", tester.autoscale(application1, cluster1.id(), min, max).isEmpty());
+
+        tester.addMeasurements(Resource.cpu, 0.25f, 1f, 60, application1);
+        AllocatableClusterResources scaledResources = tester.assertResources("Scaling up since resource usage is too high",
+                                                                             15, 1, 1.3,  28.6, 28.6,
+                                                                             tester.autoscale(application1, cluster1.id(), min, max));
+
+        tester.deploy(application1, cluster1, scaledResources);
+        assertTrue("Cluster in flux -> No further change", tester.autoscale(application1, cluster1.id(), min, max).isEmpty());
+
+        tester.deactivateRetired(application1, cluster1, scaledResources);
+        tester.addMeasurements(Resource.cpu, 0.8f, 1f, 3, application1);
+        assertTrue("Load change is large, but insufficient measurements for new config -> No change",
+                   tester.autoscale(application1, cluster1.id(), min, max).isEmpty());
+
+        tester.addMeasurements(Resource.cpu,  0.19f, 1f, 100, application1);
+        assertEquals("Load change is small -> No change", Optional.empty(), tester.autoscale(application1, cluster1.id(), min, max));
+
+        tester.addMeasurements(Resource.cpu,  0.1f, 1f, 120, application1);
+        tester.assertResources("Scaling down to minimum since usage has gone down significantly",
+                               14, 1, 1.0, 30.8, 30.8,
+                               tester.autoscale(application1, cluster1.id(), min, max));
+    }
+
+    @Test
+    public void testAutoscalingHandlesDiskSettingChanges() {
+        NodeResources hostResources = new NodeResources(3, 100, 100, 1, NodeResources.DiskSpeed.slow);
+        AutoscalingTester tester = new AutoscalingTester(hostResources);
+
+        ApplicationId application1 = tester.applicationId("application1");
+        ClusterSpec cluster1 = tester.clusterSpec(ClusterSpec.Type.content, "cluster1");
+
+        // deploy with slow
+        tester.deploy(application1, cluster1, 5, 1, hostResources);
+        tester.nodeRepository().getNodes(application1).stream()
+              .allMatch(n -> n.allocation().get().requestedResources().diskSpeed() == NodeResources.DiskSpeed.slow);
+
+        tester.addMeasurements(Resource.cpu, 0.25f, 1f, 120, application1);
+        // Changing min and max from slow to any
+        ClusterResources min = new ClusterResources( 2, 1,
+                                                     new NodeResources(1, 1, 1, 1, NodeResources.DiskSpeed.any));
+        ClusterResources max = new ClusterResources(20, 1,
+                                                    new NodeResources(100, 1000, 1000, 1, NodeResources.DiskSpeed.any));
+        AllocatableClusterResources scaledResources = tester.assertResources("Scaling up since resource usage is too high",
+                                                                             15, 1, 1.3,  28.6, 28.6,
+                                                                             tester.autoscale(application1, cluster1.id(), min, max));
+        assertEquals("Disk speed from min/max is used",
+                     NodeResources.DiskSpeed.any, scaledResources.realResources().diskSpeed());
+        tester.deploy(application1, cluster1, scaledResources);
+        tester.nodeRepository().getNodes(application1).stream()
+              .allMatch(n -> n.allocation().get().requestedResources().diskSpeed() == NodeResources.DiskSpeed.any);
+    }
+
+    /** We prefer fewer nodes for container clusters as (we assume) they all use the same disk and memory */
+    @Test
+    public void testAutoscalingSingleContainerGroup() {
+        NodeResources resources = new NodeResources(3, 100, 100, 1);
+        ClusterResources min = new ClusterResources( 2, 1, new NodeResources(1, 1, 1, 1));
+        ClusterResources max = new ClusterResources(20, 1, new NodeResources(100, 1000, 1000, 1));
+        AutoscalingTester tester = new AutoscalingTester(resources);
+
+        ApplicationId application1 = tester.applicationId("application1");
+        ClusterSpec cluster1 = tester.clusterSpec(ClusterSpec.Type.container, "cluster1");
+
+        // deploy
+        tester.deploy(application1, cluster1, 5, 1, resources);
+
+        tester.addMeasurements(Resource.cpu, 0.25f, 1f, 120, application1);
+        AllocatableClusterResources scaledResources = tester.assertResources("Scaling up since cpu usage is too high",
+                                                                             7, 1, 2.6,  80.0, 80.0,
+                                                                             tester.autoscale(application1, cluster1.id(), min, max));
+
+        tester.deploy(application1, cluster1, scaledResources);
+        tester.deactivateRetired(application1, cluster1, scaledResources);
+
+        tester.addMeasurements(Resource.cpu,  0.1f, 1f, 120, application1);
+        tester.assertResources("Scaling down since cpu usage has gone down",
+                               4, 1, 2.4, 68.6, 68.6,
+                               tester.autoscale(application1, cluster1.id(), min, max));
+    }
+
+    @Test
+    public void testAutoscalingRespectsUpperLimit() {
+        NodeResources hostResources = new NodeResources(3, 100, 100, 1);
+        ClusterResources min = new ClusterResources( 2, 1, new NodeResources(1, 1, 1, 1));
+        ClusterResources max = new ClusterResources( 6, 1, new NodeResources(2.4, 78, 79, 1));
+        AutoscalingTester tester = new AutoscalingTester(hostResources);
+
+        ApplicationId application1 = tester.applicationId("application1");
+        ClusterSpec cluster1 = tester.clusterSpec(ClusterSpec.Type.container, "cluster1");
+
+        // deploy
+        tester.deploy(application1, cluster1, 5, 1,
+                      new NodeResources(1.9, 70, 70, 1));
+        tester.addMeasurements(Resource.cpu,    0.25f, 120, application1);
+        tester.addMeasurements(Resource.memory, 0.95f, 120, application1);
+        tester.addMeasurements(Resource.disk,   0.95f, 120, application1);
+        tester.assertResources("Scaling up to limit since resource usage is too high",
+                               6, 1, 2.4,  78.0, 79.0,
+                               tester.autoscale(application1, cluster1.id(), min, max));
+    }
+
+    @Test
+    public void testAutoscalingRespectsLowerLimit() {
+        NodeResources resources = new NodeResources(3, 100, 100, 1);
+        ClusterResources min = new ClusterResources( 4, 1, new NodeResources(1.8, 7.4, 8.5, 1));
+        ClusterResources max = new ClusterResources( 6, 1, new NodeResources(2.4, 78, 79, 1));
+        AutoscalingTester tester = new AutoscalingTester(resources);
+
+        ApplicationId application1 = tester.applicationId("application1");
+        ClusterSpec cluster1 = tester.clusterSpec(ClusterSpec.Type.container, "cluster1");
+
+        // deploy
+        tester.deploy(application1, cluster1, 5, 1, resources);
+        tester.addMeasurements(Resource.cpu,    0.05f, 120, application1);
+        tester.addMeasurements(Resource.memory, 0.05f, 120, application1);
+        tester.addMeasurements(Resource.disk,   0.05f, 120, application1);
+        tester.assertResources("Scaling down to limit since resource usage is low",
+                               4, 1, 1.8,  7.4, 8.5,
+                               tester.autoscale(application1, cluster1.id(), min, max));
+    }
+
+    @Test
+    public void testAutoscalingRespectsGroupLimit() {
+        NodeResources hostResources = new NodeResources(30.0, 100, 100, 1);
+        ClusterResources min = new ClusterResources( 2, 2, new NodeResources(1, 1, 1, 1));
+        ClusterResources max = new ClusterResources(18, 6, new NodeResources(100, 1000, 1000, 1));
+        AutoscalingTester tester = new AutoscalingTester(hostResources);
+
+        ApplicationId application1 = tester.applicationId("application1");
+        ClusterSpec cluster1 = tester.clusterSpec(ClusterSpec.Type.container, "cluster1");
+
+        // deploy
+        tester.deploy(application1, cluster1, 5, 5, new NodeResources(3.0, 10, 10, 1));
+        tester.addMeasurements(Resource.cpu,  0.3f, 1f, 240, application1);
+        tester.assertResources("Scaling up since resource usage is too high",
+                               6, 6, 3.6,  8.0, 8.0,
+                               tester.autoscale(application1, cluster1.id(), min, max));
+    }
+
+    /** This condition ensures we get recommendation suggestions when deactivated */
+    @Test
+    public void testAutoscalingLimitsAreIgnoredIfMinEqualsMax() {
+        NodeResources resources = new NodeResources(3, 100, 100, 1);
+        ClusterResources min = new ClusterResources( 2, 1, new NodeResources(1, 1, 1, 1));
+        ClusterResources max = min;
+        AutoscalingTester tester = new AutoscalingTester(resources);
+
+        ApplicationId application1 = tester.applicationId("application1");
+        ClusterSpec cluster1 = tester.clusterSpec(ClusterSpec.Type.container, "cluster1");
+
+        // deploy
+        tester.deploy(application1, cluster1, 5, 1, resources);
+        tester.addMeasurements(Resource.cpu,  0.25f, 1f, 120, application1);
+        tester.assertResources("Scaling up since resource usage is too high",
+                               7, 1, 2.6,  80.0, 80.0,
+                               tester.autoscale(application1, cluster1.id(), min, max));
+    }
+
+    @Test
+    public void testAutoscalingGroupSize1() {
+        NodeResources resources = new NodeResources(3, 100, 100, 1);
+        ClusterResources min = new ClusterResources( 2, 2, new NodeResources(1, 1, 1, 1));
+        ClusterResources max = new ClusterResources(20, 20, new NodeResources(100, 1000, 1000, 1));
+        AutoscalingTester tester = new AutoscalingTester(resources);
+
+        ApplicationId application1 = tester.applicationId("application1");
+        ClusterSpec cluster1 = tester.clusterSpec(ClusterSpec.Type.container, "cluster1");
+
+        // deploy
+        tester.deploy(application1, cluster1, 5, 5, resources);
+        tester.addMeasurements(Resource.cpu,  0.25f, 1f, 120, application1);
+        tester.assertResources("Scaling up since resource usage is too high",
+                               7, 7, 2.5,  80.0, 80.0,
+                               tester.autoscale(application1, cluster1.id(), min, max));
+    }
+
+    @Test
+    public void testAutoscalingGroupSize3() {
+        NodeResources resources = new NodeResources(3, 100, 100, 1);
+        ClusterResources min = new ClusterResources( 3, 1, new NodeResources(1, 1, 1, 1));
+        ClusterResources max = new ClusterResources(21, 7, new NodeResources(100, 1000, 1000, 1));
+        AutoscalingTester tester = new AutoscalingTester(resources);
+
+        ApplicationId application1 = tester.applicationId("application1");
+        ClusterSpec cluster1 = tester.clusterSpec(ClusterSpec.Type.container, "cluster1");
+
+        // deploy
+        tester.deploy(application1, cluster1, 6, 2, resources);
+        tester.addMeasurements(Resource.cpu,  0.22f, 1f, 120, application1);
+        tester.assertResources("Scaling up since resource usage is too high",
+                               9, 3, 2.7,  83.3, 83.3,
+                               tester.autoscale(application1, cluster1.id(), min, max));
+    }
+
+    @Test
+    public void testAutoscalingAvoidsIllegalConfigurations() {
+        NodeResources resources = new NodeResources(3, 100, 100, 1);
+        ClusterResources min = new ClusterResources( 2, 1, new NodeResources(1, 1, 1, 1));
+        ClusterResources max = new ClusterResources(20, 1, new NodeResources(100, 1000, 1000, 1));
+        AutoscalingTester tester = new AutoscalingTester(resources);
+
+        ApplicationId application1 = tester.applicationId("application1");
+        ClusterSpec cluster1 = tester.clusterSpec(ClusterSpec.Type.content, "cluster1");
+
+        // deploy
+        tester.deploy(application1, cluster1, 6, 1, resources);
+        tester.addMeasurements(Resource.memory,  0.02f, 1f, 120, application1);
+        tester.assertResources("Scaling down",
+                               6, 1, 3.0, 4.0, 100.0,
+                               tester.autoscale(application1, cluster1.id(), min, max));
+    }
+
+    @Test
+    public void testAutoscalingAws() {
+        ClusterResources min = new ClusterResources( 2, 1, new NodeResources(1, 1, 1, 1));
+        ClusterResources max = new ClusterResources(20, 1, new NodeResources(100, 1000, 1000, 1));
+        List<Flavor> flavors = new ArrayList<>();
+        flavors.add(new Flavor("aws-xlarge", new NodeResources(3, 200, 100, 1, NodeResources.DiskSpeed.fast, NodeResources.StorageType.remote)));
+        flavors.add(new Flavor("aws-large",  new NodeResources(3, 150, 100, 1, NodeResources.DiskSpeed.fast, NodeResources.StorageType.remote)));
+        flavors.add(new Flavor("aws-medium", new NodeResources(3, 100, 100, 1, NodeResources.DiskSpeed.fast, NodeResources.StorageType.remote)));
+        flavors.add(new Flavor("aws-small",  new NodeResources(3,  80, 100, 1, NodeResources.DiskSpeed.fast, NodeResources.StorageType.remote)));
+        AutoscalingTester tester = new AutoscalingTester(new Zone(CloudName.from("aws"), SystemName.main,
+                                                                  Environment.prod, RegionName.from("us-east")),
+                                                         flavors);
+
+        ApplicationId application1 = tester.applicationId("application1");
+        ClusterSpec cluster1 = tester.clusterSpec(ClusterSpec.Type.content, "cluster1");
+
+        // deploy (Why 83 Gb memory? See AutoscalingTester.MockHostResourcesCalculator
+        tester.deploy(application1, cluster1, 5, 1, new NodeResources(3, 103, 100, 1));
+
+        tester.addMeasurements(Resource.memory, 0.9f, 0.6f, 120, application1);
+        AllocatableClusterResources scaledResources = tester.assertResources("Scaling up since resource usage is too high.",
+                                                                  8, 1, 3,  83, 34.3,
+                                                                  tester.autoscale(application1, cluster1.id(), min, max));
+
+        tester.deploy(application1, cluster1, scaledResources);
+        tester.deactivateRetired(application1, cluster1, scaledResources);
+
+        tester.addMeasurements(Resource.memory, 0.3f, 0.6f, 1000, application1);
+        tester.assertResources("Scaling down since resource usage has gone down",
+                               5, 1, 3, 83, 36,
+                               tester.autoscale(application1, cluster1.id(), min, max));
+    }
+
+}
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java
new file mode 100644
index 00000000000..aed262b6c96
--- /dev/null
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java
@@ -0,0 +1,285 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.autoscale;
+
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.Capacity;
+import com.yahoo.config.provision.ClusterResources;
+import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.config.provision.Environment;
+import com.yahoo.config.provision.Flavor;
+import com.yahoo.config.provision.HostSpec;
+import com.yahoo.config.provision.NodeResources;
+import com.yahoo.config.provision.NodeType;
+import com.yahoo.config.provision.RegionName;
+import com.yahoo.config.provision.Zone;
+import com.yahoo.config.provisioning.FlavorsConfig;
+import com.yahoo.test.ManualClock;
+import com.yahoo.transaction.Mutex;
+import com.yahoo.vespa.flags.FlagSource;
+import com.yahoo.vespa.flags.Flags;
+import com.yahoo.vespa.flags.InMemoryFlagSource;
+import com.yahoo.vespa.hosted.provision.Node;
+import com.yahoo.vespa.hosted.provision.NodeRepository;
+import com.yahoo.vespa.hosted.provision.applications.Application;
+import com.yahoo.vespa.hosted.provision.node.Agent;
+import com.yahoo.vespa.hosted.provision.node.IP;
+import com.yahoo.vespa.hosted.provision.provisioning.FatalProvisioningException;
+import com.yahoo.vespa.hosted.provision.provisioning.HostProvisioner;
+import com.yahoo.vespa.hosted.provision.provisioning.HostResourcesCalculator;
+import com.yahoo.vespa.hosted.provision.provisioning.ProvisionedHost;
+import com.yahoo.vespa.hosted.provision.provisioning.ProvisioningTester;
+
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+class AutoscalingTester {
+
+    private final ProvisioningTester provisioningTester;
+    private final Autoscaler autoscaler;
+    private final NodeMetricsDb db;
+    private final MockHostResourcesCalculator hostResourcesCalculator;
+
+    /** Creates an autoscaling tester with a single host type ready */
+    public AutoscalingTester(NodeResources hostResources) {
+        this(new Zone(Environment.prod, RegionName.from("us-east")), null, null, asConfig(hostResources));
+        provisioningTester.makeReadyNodes(20, "hostFlavor", NodeType.host, 8); // "hostFlavor" generated by asConfig
+        provisioningTester.deployZoneApp();
+    }
+
+    public AutoscalingTester(Zone zone, List<Flavor> flavors) {
+        this(zone,
+             flavors,
+             new InMemoryFlagSource().withBooleanFlag(Flags.ENABLE_DYNAMIC_PROVISIONING.id(), true),
+             asConfig(flavors));
+    }
+
+    private AutoscalingTester(Zone zone, List<Flavor> flavors, FlagSource flagSource, FlavorsConfig flavorsConfig) {
+        provisioningTester = new ProvisioningTester.Builder().zone(zone)
+                                                             .flavorsConfig(flavorsConfig)
+                                                             .hostProvisioner(new MockHostProvisioner(flavors))
+                                                             .flagSource(flagSource)
+                                                             .build();
+
+        hostResourcesCalculator = new MockHostResourcesCalculator(zone);
+        db = new NodeMetricsDb();
+        autoscaler = new Autoscaler(hostResourcesCalculator, db, nodeRepository());
+    }
+
+    public ApplicationId applicationId(String applicationName) {
+        return ApplicationId.from("tenant1", applicationName, "instance1");
+    }
+
+    public ClusterSpec clusterSpec(ClusterSpec.Type type, String clusterId) {
+        return ClusterSpec.request(type, ClusterSpec.Id.from(clusterId)).vespaVersion("7").build();
+    }
+
+    public void deploy(ApplicationId application, ClusterSpec cluster, AllocatableClusterResources resources) {
+        deploy(application, cluster, resources.nodes(), resources.groups(), resources.advertisedResources());
+    }
+
+    public List<HostSpec> deploy(ApplicationId application, ClusterSpec cluster, int nodes, int groups, NodeResources resources) {
+        List<HostSpec> hosts = provisioningTester.prepare(application, cluster, Capacity.from(new ClusterResources(nodes, groups, resources)));
+        for (HostSpec host : hosts)
+            makeReady(host.hostname());
+        provisioningTester.deployZoneApp();
+        provisioningTester.activate(application, hosts);
+        return hosts;
+    }
+
+    public void makeReady(String hostname) {
+        Node node = nodeRepository().getNode(hostname).get();
+        nodeRepository().write(node.with(new IP.Config(Set.of("::" + 0 + ":0"), Set.of())), nodeRepository().lock(node));
+        Node host = nodeRepository().getNode(node.parentHostname().get()).get();
+        host = host.with(new IP.Config(Set.of("::" + 0 + ":0"), Set.of("::" + 0 + ":2")));
+        if (host.state() == Node.State.provisioned)
+            nodeRepository().setReady(List.of(host), Agent.system, getClass().getSimpleName());
+    }
+
+    public void deactivateRetired(ApplicationId application, ClusterSpec cluster, AllocatableClusterResources resources) {
+        try (Mutex lock = nodeRepository().lock(application)){
+            for (Node node : nodeRepository().getNodes(application, Node.State.active)) {
+                if (node.allocation().get().membership().retired())
+                    nodeRepository().write(node.with(node.allocation().get().removable()), lock);
+            }
+        }
+        deploy(application, cluster, resources);
+    }
+
+    /**
+     * Adds measurements with the given resource value and ideal values for the other resources,
+     * scaled to take one node redundancy into account.
+     * (I.e we adjust to measure a bit lower load than "naively" wanted to offset for the autoscaler
+     * wanting to see the ideal load with one node missing.)
+     *
+     * @param resource the resource we are explicitly setting the value of
+     * @param otherResourcesLoad the load factor relative to ideal to use for other resources
+     * @param count the number of measurements
+     * @param applicationId the application we're adding measurements for all nodes of
+     */
+    public void addMeasurements(Resource resource, float value, float otherResourcesLoad,
+                                int count, ApplicationId applicationId) {
+        List<Node> nodes = nodeRepository().getNodes(applicationId, Node.State.active);
+        float oneExtraNodeFactor = (float)(nodes.size() - 1.0) / (nodes.size());
+        for (int i = 0; i < count; i++) {
+            clock().advance(Duration.ofMinutes(1));
+            for (Node node : nodes) {
+                for (Resource r : Resource.values()) {
+                    float effectiveValue = (r == resource ? value : (float) r.idealAverageLoad() * otherResourcesLoad)
+                                           * oneExtraNodeFactor;
+                    db.add(List.of(new NodeMetrics.MetricValue(node.hostname(),
+                                                               r.metricName(),
+                                                               clock().instant().toEpochMilli(),
+                                                               effectiveValue * 100))); // the metrics are in %
+                }
+            }
+        }
+    }
+
+    public void addMeasurements(Resource resource, float value, int count, ApplicationId applicationId) {
+        List<Node> nodes = nodeRepository().getNodes(applicationId, Node.State.active);
+        for (int i = 0; i < count; i++) {
+            clock().advance(Duration.ofMinutes(1));
+            for (Node node : nodes) {
+                db.add(List.of(new NodeMetrics.MetricValue(node.hostname(),
+                                                           resource.metricName(),
+                                                           clock().instant().toEpochMilli(),
+                                                           value * 100))); // the metrics are in %
+            }
+        }
+    }
+
+    public Optional<AllocatableClusterResources> autoscale(ApplicationId applicationId, ClusterSpec.Id clusterId,
+                                                           ClusterResources min, ClusterResources max) {
+        Application application = nodeRepository().applications().get(applicationId, true).withClusterLimits(clusterId, min, max);
+        nodeRepository().applications().set(applicationId, application, nodeRepository().lock(applicationId));
+        return autoscaler.autoscale(application.cluster(clusterId),
+                                    nodeRepository().getNodes(applicationId, Node.State.active));
+    }
+
+    public AllocatableClusterResources assertResources(String message,
+                                            int nodeCount, int groupCount,
+                                            double approxCpu, double approxMemory, double approxDisk,
+                                            Optional<AllocatableClusterResources> actualResources) {
+        double delta = 0.0000000001;
+        assertTrue(message, actualResources.isPresent());
+        assertEquals("Node count:  " + message, nodeCount, actualResources.get().nodes());
+        assertEquals("Group count: " + message, groupCount, actualResources.get().groups());
+        assertEquals("Cpu: "    + message, approxCpu, Math.round(actualResources.get().advertisedResources().vcpu() * 10) / 10.0, delta);
+        assertEquals("Memory: " + message, approxMemory, Math.round(actualResources.get().advertisedResources().memoryGb() * 10) / 10.0, delta);
+        assertEquals("Disk: "   + message, approxDisk, Math.round(actualResources.get().advertisedResources().diskGb() * 10) / 10.0, delta);
+        return actualResources.get();
+    }
+
+    public ManualClock clock() {
+        return provisioningTester.clock();
+    }
+
+    public NodeRepository nodeRepository() {
+        return provisioningTester.nodeRepository();
+    }
+
+    public NodeMetricsDb nodeMetricsDb() { return db; }
+
+    private static FlavorsConfig asConfig(NodeResources hostResources) {
+        FlavorsConfig.Builder b = new FlavorsConfig.Builder();
+        b.flavor(asFlavorConfig("hostFlavor", hostResources));
+        return b.build();
+    }
+
+    private static FlavorsConfig asConfig(List<Flavor> flavors) {
+        FlavorsConfig.Builder b = new FlavorsConfig.Builder();
+        for (Flavor flavor : flavors)
+            b.flavor(asFlavorConfig(flavor.name(), flavor.resources()));
+        return b.build();
+    }
+
+    private static FlavorsConfig.Flavor.Builder asFlavorConfig(String flavorName, NodeResources resources) {
+        FlavorsConfig.Flavor.Builder flavor = new FlavorsConfig.Flavor.Builder();
+        flavor.name(flavorName);
+        flavor.minCpuCores(resources.vcpu());
+        flavor.minMainMemoryAvailableGb(resources.memoryGb());
+        flavor.minDiskAvailableGb(resources.diskGb());
+        flavor.bandwidth(resources.bandwidthGbps() * 1000);
+        flavor.fastDisk(resources.diskSpeed().compatibleWith(NodeResources.DiskSpeed.fast));
+        flavor.remoteStorage(resources.storageType().compatibleWith(NodeResources.StorageType.remote));
+        return flavor;
+    }
+
+    private static class MockHostResourcesCalculator implements HostResourcesCalculator {
+
+        private final Zone zone;
+
+        public MockHostResourcesCalculator(Zone zone) {
+            this.zone = zone;
+        }
+
+        @Override
+        public NodeResources realResourcesOf(Node node) {
+            if (zone.cloud().value().equals("aws"))
+                return node.flavor().resources().withMemoryGb(node.flavor().resources().memoryGb() - 3);
+            else
+                return node.flavor().resources();
+        }
+
+        @Override
+        public NodeResources advertisedResourcesOf(Flavor flavor) {
+            if (zone.cloud().value().equals("aws"))
+                return flavor.resources().withMemoryGb(flavor.resources().memoryGb() + 3);
+            else
+                return flavor.resources();
+        }
+
+    }
+
+    private class MockHostProvisioner implements HostProvisioner {
+
+        private final List<Flavor> hostFlavors;
+
+        public MockHostProvisioner(List<Flavor> hostFlavors) {
+            this.hostFlavors = hostFlavors;
+        }
+
+        @Override
+        public List<ProvisionedHost> provisionHosts(List<Integer> provisionIndexes, NodeResources resources, ApplicationId applicationId) {
+            Flavor hostFlavor = hostFlavors.stream().filter(f -> matches(f, resources)).findAny()
+                                           .orElseThrow(() -> new RuntimeException("No flavor matching " + resources + ". Flavors: " + hostFlavors));
+
+            List<ProvisionedHost> hosts = new ArrayList<>();
+            for (int index : provisionIndexes) {
+                hosts.add(new ProvisionedHost("host" + index,
+                                              "hostname" + index,
+                                              hostFlavor,
+                                              "nodename" + index,
+                                              resources));
+            }
+            return hosts;
+        }
+
+        @Override
+        public List<Node> provision(Node host, Set<Node> children) throws FatalProvisioningException {
+            throw new RuntimeException("Not implemented");
+        }
+
+        @Override
+        public void deprovision(Node host) {
+            throw new RuntimeException("Not implemented");
+        }
+
+        private boolean matches(Flavor flavor, NodeResources resources) {
+            NodeResources flavorResources = hostResourcesCalculator.advertisedResourcesOf(flavor);
+            if (flavorResources.storageType() == NodeResources.StorageType.remote
+                && resources.diskGb() <= flavorResources.diskGb())
+                flavorResources = flavorResources.withDiskGb(resources.diskGb());
+
+            return flavorResources.justNumbers().equals(resources.justNumbers());
+        }
+
+    }
+
+}
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsDbTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsDbTest.java
new file mode 100644
index 00000000000..af87c008260
--- /dev/null
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsDbTest.java
@@ -0,0 +1,33 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.autoscale;
+
+import com.yahoo.test.ManualClock;
+import org.junit.Test;
+
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+
+public class NodeMetricsDbTest {
+
+    @Test
+    public void testNodeMetricsDb() {
+        ManualClock clock = new ManualClock();
+        NodeMetricsDb db = new NodeMetricsDb();
+        List<NodeMetrics.MetricValue> values = new ArrayList<>();
+        for (int i = 0; i < 40; i++) {
+            values.add(new NodeMetrics.MetricValue("host0", "cpu.util", clock.instant().getEpochSecond(), 0.9f));
+            clock.advance(Duration.ofHours(1));
+        }
+        db.add(values);
+
+        assertEquals(29, db.getWindow(clock.instant().minus(Duration.ofHours(30)), Resource.cpu,    List.of("host0")).measurementCount());
+        assertEquals( 0, db.getWindow(clock.instant().minus(Duration.ofHours(30)), Resource.memory, List.of("host0")).measurementCount());
+        db.gc(clock);
+        assertEquals(23, db.getWindow(clock.instant().minus(Duration.ofHours(30)), Resource.cpu,    List.of("host0")).measurementCount());
+        assertEquals( 0, db.getWindow(clock.instant().minus(Duration.ofHours(30)), Resource.memory, List.of("host0")).measurementCount());
+    }
+
+}
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsFetcherTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsFetcherTest.java
new file mode 100644
index 00000000000..6bf52218302
--- /dev/null
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/NodeMetricsFetcherTest.java
@@ -0,0 +1,156 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.autoscale;
+
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.Capacity;
+import com.yahoo.config.provision.ClusterResources;
+import com.yahoo.config.provision.NodeResources;
+import com.yahoo.vdslib.state.NodeState;
+import com.yahoo.vespa.hosted.provision.Node;
+import com.yahoo.vespa.hosted.provision.provisioning.ProvisioningTester;
+import com.yahoo.vespa.hosted.provision.testutils.OrchestratorMock;
+import com.yahoo.vespa.applicationmodel.HostName;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class NodeMetricsFetcherTest {
+
+    @Test
+    public void testMetricsFetch() {
+        NodeResources resources = new NodeResources(1, 10, 100, 1);
+        ProvisioningTester tester = new ProvisioningTester.Builder().build();
+        OrchestratorMock orchestrator = new OrchestratorMock();
+        MockHttpClient httpClient = new MockHttpClient();
+        NodeMetricsFetcher fetcher = new NodeMetricsFetcher(tester.nodeRepository(), orchestrator, httpClient);
+
+        tester.makeReadyNodes(4, resources); // Creates (in order) host-1.yahoo.com, host-2.yahoo.com, host-3.yahoo.com, host-4.yahoo.com
+        tester.deployZoneApp();
+
+        ApplicationId application1 = tester.makeApplicationId();
+        ApplicationId application2 = tester.makeApplicationId();
+        tester.deploy(application1, Capacity.from(new ClusterResources(2, 1, resources))); // host-1.yahoo.com, host-2.yahoo.com
+        tester.deploy(application2, Capacity.from(new ClusterResources(2, 1, resources))); // host-4.yahoo.com, host-3.yahoo.com
+
+        orchestrator.suspend(new HostName("host-4.yahoo.com"));
+
+        {
+            httpClient.cannedResponse = cannedResponseForApplication1;
+            List<NodeMetrics.MetricValue> values = new ArrayList<>(fetcher.fetchMetrics(application1));
+            assertEquals("http://host-1.yahoo.com:4080/metrics/v2/values?consumer=default",
+                         httpClient.requestsReceived.get(0));
+            assertEquals(5, values.size());
+            assertEquals("metric value cpu.util: 16.2 at 1970-01-01T00:20:34Z for host-1.yahoo.com", values.get(0).toString());
+            assertEquals("metric value mem_total.util: 23.1 at 1970-01-01T00:20:34Z for host-1.yahoo.com", values.get(1).toString());
+            assertEquals("metric value disk.util: 82.0 at 1970-01-01T00:20:34Z for host-1.yahoo.com", values.get(2).toString());
+            assertEquals("metric value cpu.util: 20.0 at 1970-01-01T00:20:00Z for host-2.yahoo.com", values.get(3).toString());
+            assertEquals("metric value disk.util: 40.0 at 1970-01-01T00:20:00Z for host-2.yahoo.com", values.get(4).toString());
+        }
+
+        {
+            httpClient.cannedResponse = cannedResponseForApplication2;
+            List<NodeMetrics.MetricValue> values = new ArrayList<>(fetcher.fetchMetrics(application2));
+            assertEquals("http://host-3.yahoo.com:4080/metrics/v2/values?consumer=default",
+                         httpClient.requestsReceived.get(1));
+            assertEquals(3, values.size());
+            assertEquals("metric value cpu.util: 10.0 at 1970-01-01T00:21:40Z for host-3.yahoo.com", values.get(0).toString());
+            assertEquals("metric value mem_total.util: 15.0 at 1970-01-01T00:21:40Z for host-3.yahoo.com", values.get(1).toString());
+            assertEquals("metric value disk.util: 20.0 at 1970-01-01T00:21:40Z for host-3.yahoo.com", values.get(2).toString());
+        }
+
+        {
+            tester.nodeRepository().write(tester.nodeRepository().getNodes(application1, Node.State.active).get(0).retire(tester.clock().instant()),
+                                          tester.nodeRepository().lock(application1));
+            assertTrue("No metrics fetching while unstable", fetcher.fetchMetrics(application1).isEmpty());
+        }
+    }
+
+    private static class MockHttpClient implements NodeMetricsFetcher.HttpClient {
+
+        List<String> requestsReceived = new ArrayList<>();
+
+        String cannedResponse = null;
+        @Override
+        public String get(String url) {
+            requestsReceived.add(url);
+            return cannedResponse;
+        }
+
+        @Override
+        public void close() { }
+
+    }
+
+    final String cannedResponseForApplication1 =
+            "{\n" +
+            "  \"nodes\": [\n" +
+            "    {\n" +
+            "      \"hostname\": \"host-1.yahoo.com\",\n" +
+            "      \"role\": \"role0\",\n" +
+            "      \"node\": {\n" +
+            "        \"timestamp\": 1234,\n" +
+            "        \"metrics\": [\n" +
+            "          {\n" +
+            "            \"values\": {\n" +
+            "              \"cpu.util\": 16.2,\n" +
+            "              \"mem_total.util\": 23.1,\n" +
+            "              \"disk.util\": 82\n" +
+            "            },\n" +
+            "            \"dimensions\": {\n" +
+            "              \"state\": \"active\"\n" +
+            "            }\n" +
+            "          }\n" +
+            "        ]\n" +
+            "      }\n" +
+            "    },\n" +
+            "    {\n" +
+            "      \"hostname\": \"host-2.yahoo.com\",\n" +
+            "      \"role\": \"role1\",\n" +
+            "      \"node\": {\n" +
+            "        \"timestamp\": 1200,\n" +
+            "        \"metrics\": [\n" +
+            "          {\n" +
+            "            \"values\": {\n" +
+            "              \"cpu.util\": 20,\n" +
+            "              \"disk.util\": 40\n" +
+            "            },\n" +
+            "            \"dimensions\": {\n" +
+            "              \"state\": \"active\"\n" +
+            "            }\n" +
+            "          }\n" +
+            "        ]\n" +
+            "      }\n" +
+            "    }\n" +
+            "  ]\n" +
+            "}\n";
+
+    final String cannedResponseForApplication2 =
+            "{\n" +
+            "  \"nodes\": [\n" +
+            "    {\n" +
+            "      \"hostname\": \"host-3.yahoo.com\",\n" +
+            "      \"role\": \"role0\",\n" +
+            "      \"node\": {\n" +
+            "        \"timestamp\": 1300,\n" +
+            "        \"metrics\": [\n" +
+            "          {\n" +
+            "            \"values\": {\n" +
+            "              \"cpu.util\": 10,\n" +
+            "              \"mem_total.util\": 15,\n" +
+            "              \"disk.util\": 20\n" +
+            "            },\n" +
+            "            \"dimensions\": {\n" +
+            "              \"state\": \"active\"\n" +
+            "            }\n" +
+            "          }\n" +
+            "        ]\n" +
+            "      }\n" +
+            "    }\n" +
+            "  ]\n" +
+            "}\n";
+
+}
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTest.java
new file mode 100644
index 00000000000..8f8f8d0f38b
--- /dev/null
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTest.java
@@ -0,0 +1,114 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.maintenance;
+
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.Capacity;
+import com.yahoo.config.provision.ClusterResources;
+import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.config.provision.Environment;
+import com.yahoo.config.provision.Flavor;
+import com.yahoo.config.provision.NodeResources;
+import com.yahoo.config.provision.NodeType;
+import com.yahoo.config.provision.RegionName;
+import com.yahoo.config.provision.Zone;
+import com.yahoo.config.provisioning.FlavorsConfig;
+import com.yahoo.vespa.hosted.provision.Node;
+import com.yahoo.vespa.hosted.provision.NodeRepository;
+import com.yahoo.vespa.hosted.provision.autoscale.NodeMetrics;
+import com.yahoo.vespa.hosted.provision.autoscale.NodeMetricsDb;
+import com.yahoo.vespa.hosted.provision.autoscale.Resource;
+import com.yahoo.vespa.hosted.provision.provisioning.FlavorConfigBuilder;
+import com.yahoo.vespa.hosted.provision.provisioning.ProvisioningTester;
+import com.yahoo.vespa.hosted.provision.testutils.MockDeployer;
+import org.junit.Test;
+
+import java.time.Duration;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Tests the autoscaling maintainer integration.
+ * The specific recommendations of the autoscaler are not tested here.
+ *
+ * @author bratseth
+ */
+public class AutoscalingMaintainerTest {
+
+    @Test
+    public void testAutoscalingMaintainer() {
+        ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east3"))).flavorsConfig(flavorsConfig()).build();
+
+        ApplicationId app1 = tester.makeApplicationId("app1");
+        ClusterSpec cluster1 = tester.clusterSpec();
+
+        ApplicationId app2 = tester.makeApplicationId("app2");
+        ClusterSpec cluster2 = tester.clusterSpec();
+
+        NodeResources lowResources = new NodeResources(4, 4, 10, 0.1);
+        NodeResources highResources = new NodeResources(6.5, 9, 20, 0.1);
+
+        Map<ApplicationId, MockDeployer.ApplicationContext> apps = Map.of(
+                app1, new MockDeployer.ApplicationContext(app1, cluster1, Capacity.from(new ClusterResources(2, 1, lowResources))),
+                app2, new MockDeployer.ApplicationContext(app2, cluster2, Capacity.from(new ClusterResources(2, 1, highResources))));
+        MockDeployer deployer = new MockDeployer(tester.provisioner(), tester.clock(), apps);
+
+        NodeMetricsDb nodeMetricsDb = new NodeMetricsDb();
+        AutoscalingMaintainer maintainer = new AutoscalingMaintainer(tester.nodeRepository(),
+                                                                     tester.identityHostResourcesCalculator(),
+                                                                     nodeMetricsDb,
+                                                                     deployer,
+                                                                     new TestMetric(),
+                                                                     Duration.ofMinutes(1));
+        maintainer.maintain(); // noop
+        assertTrue(deployer.lastDeployTime(app1).isEmpty());
+        assertTrue(deployer.lastDeployTime(app2).isEmpty());
+
+        tester.makeReadyNodes(20, "flt", NodeType.host, 8);
+        tester.deployZoneApp();
+
+        tester.deploy(app1, cluster1, Capacity.from(new ClusterResources(5, 1, new NodeResources(4, 4, 10, 0.1)),
+                                                    new ClusterResources(5, 1, new NodeResources(4, 4, 10, 0.1)),
+                                                    false, true));
+        tester.deploy(app2, cluster2, Capacity.from(new ClusterResources(5, 1, new NodeResources(4, 4, 10, 0.1)),
+                                                    new ClusterResources(10, 1, new NodeResources(6.5, 9, 20, 0.1)),
+                                                    false, true));
+
+        maintainer.maintain(); // noop
+        assertTrue(deployer.lastDeployTime(app1).isEmpty());
+        assertTrue(deployer.lastDeployTime(app2).isEmpty());
+
+        addMeasurements(Resource.cpu,    0.9f, 500, app1, tester.nodeRepository(), nodeMetricsDb);
+        addMeasurements(Resource.memory, 0.9f, 500, app1, tester.nodeRepository(), nodeMetricsDb);
+        addMeasurements(Resource.disk,   0.9f, 500, app1, tester.nodeRepository(), nodeMetricsDb);
+        addMeasurements(Resource.cpu,    0.9f, 500, app2, tester.nodeRepository(), nodeMetricsDb);
+        addMeasurements(Resource.memory, 0.9f, 500, app2, tester.nodeRepository(), nodeMetricsDb);
+        addMeasurements(Resource.disk,   0.9f, 500, app2, tester.nodeRepository(), nodeMetricsDb);
+
+        maintainer.maintain();
+        assertTrue(deployer.lastDeployTime(app1).isEmpty()); // since autoscaling is off
+        assertTrue(deployer.lastDeployTime(app2).isPresent());
+    }
+
+    public void addMeasurements(Resource resource, float value, int count, ApplicationId applicationId,
+                                NodeRepository nodeRepository, NodeMetricsDb db) {
+        List<Node> nodes = nodeRepository.getNodes(applicationId, Node.State.active);
+        for (int i = 0; i < count; i++) {
+            for (Node node : nodes)
+                db.add(List.of(new NodeMetrics.MetricValue(node.hostname(),
+                                                           resource.metricName(),
+                                                           nodeRepository.clock().instant().toEpochMilli(),
+                                                          value * 100))); // the metrics are in %
+        }
+    }
+
+    private FlavorsConfig flavorsConfig() {
+        FlavorConfigBuilder b = new FlavorConfigBuilder();
+        b.addFlavor("flt", 30, 30, 40, 3, Flavor.Type.BARE_METAL);
+        b.addFlavor("cpu", 40, 20, 40, 3, Flavor.Type.BARE_METAL);
+        b.addFlavor("mem", 20, 40, 40, 3, Flavor.Type.BARE_METAL);
+        return b.build();
+    }
+
+}
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTest.java
index 9d79af804ce..5813585554d 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTest.java
@@ -3,7 +3,6 @@ package com.yahoo.vespa.hosted.provision.maintenance;
 
 import com.yahoo.config.provision.NodeResources;
 import com.yahoo.config.provision.NodeType;
-import org.junit.Before;
 import org.junit.Test;
 
 import java.io.IOException;
@@ -20,19 +19,13 @@ import static org.junit.Assert.fail;
  * @author mgimle
  */
 public class CapacityCheckerTest {
-    private CapacityCheckerTester tester;
-
-    @Before
-    public void setup() {
-        tester = new CapacityCheckerTester();
-    }
 
     @Test
     public void testWithRealData() throws IOException {
+        CapacityCheckerTester tester = new CapacityCheckerTester();
         String path = "./src/test/resources/zookeeper_dump.json";
 
-        tester.cleanRepository();
-        tester.restoreNodeRepositoryFromJsonFile(Paths.get(path));
+        tester.populateNodeRepositoryFromJsonFile(Paths.get(path));
         var failurePath = tester.capacityChecker.worstCaseHostLossLeadingToFailure();
         assertTrue(failurePath.isPresent());
         assertTrue(tester.nodeRepository.getNodes(NodeType.host).containsAll(failurePath.get().hostsCausingFailure));
@@ -40,6 +33,7 @@ public class CapacityCheckerTest {
 
     @Test
     public void testOvercommittedHosts() {
+        CapacityCheckerTester tester = new CapacityCheckerTester();
         tester.createNodes(7, 4,
                10, new NodeResources(-1, 10, 100, 1), 10,
                 0, new NodeResources(1, 10, 100, 1), 10);
@@ -49,37 +43,50 @@ public class CapacityCheckerTest {
 
     @Test
     public void testEdgeCaseFailurePaths() {
-        tester.createNodes(1, 1,
-                0, new NodeResources(1, 10, 100, 1), 10,
-                0, new NodeResources(1, 10, 100, 1), 10);
-        var failurePath = tester.capacityChecker.worstCaseHostLossLeadingToFailure();
-        assertFalse("Computing worst case host loss with no hosts should return an empty optional.", failurePath.isPresent());
+        {
+            CapacityCheckerTester tester = new CapacityCheckerTester();
+            tester.createNodes(1, 1,
+                               0, new NodeResources(1, 10, 100, 1), 10,
+                               0, new NodeResources(1, 10, 100, 1), 10);
+            var failurePath = tester.capacityChecker.worstCaseHostLossLeadingToFailure();
+            assertFalse("Computing worst case host loss with no hosts should return an empty optional.", failurePath.isPresent());
+        }
 
         // Odd edge case that should never be able to occur in prod
-        tester.createNodes(1, 10,
-                10, new NodeResources(10, 1000, 10000, 1), 100,
-                1, new NodeResources(10, 1000, 10000, 1), 100);
-        failurePath = tester.capacityChecker.worstCaseHostLossLeadingToFailure();
-        assertTrue(failurePath.isPresent());
-        assertTrue("Computing worst case host loss if all hosts have to be removed should result in an non-empty failureReason with empty nodes.",
-                failurePath.get().failureReason.tenant.isEmpty() && failurePath.get().failureReason.host.isEmpty());
-        assertEquals(tester.nodeRepository.getNodes(NodeType.host).size(), failurePath.get().hostsCausingFailure.size());
-
-        tester.createNodes(3, 30,
-                10, new NodeResources(0, 0, 10000, 1), 1000,
-                0, new NodeResources(0, 0, 0, 0), 0);
-        failurePath = tester.capacityChecker.worstCaseHostLossLeadingToFailure();
-        assertTrue(failurePath.isPresent());
-        if (failurePath.get().failureReason.tenant.isPresent()) {
-            var failureReasons = failurePath.get().failureReason.allocationFailures;
-            assertEquals("When there are multiple lacking resources, all failures are multipleReasonFailures",
-                    failureReasons.size(), failureReasons.multipleReasonFailures().size());
-            assertEquals(0, failureReasons.singularReasonFailures().size());
-        } else fail();
+        {
+            CapacityCheckerTester tester = new CapacityCheckerTester();
+            tester.createNodes(1, 10,
+                               10, new NodeResources(10, 1000, 10000, 1), 100,
+                               1, new NodeResources(10, 1000, 10000, 1), 100);
+            var failurePath = tester.capacityChecker.worstCaseHostLossLeadingToFailure();
+            assertTrue(failurePath.isPresent());
+            assertTrue("Computing worst case host loss if all hosts have to be removed should result in an non-empty failureReason with empty nodes.",
+                       failurePath.get().failureReason.tenant.isEmpty() && failurePath.get().failureReason.host.isEmpty());
+            assertEquals(tester.nodeRepository.getNodes(NodeType.host).size(), failurePath.get().hostsCausingFailure.size());
+        }
+
+        {
+            CapacityCheckerTester tester = new CapacityCheckerTester();
+            tester.createNodes(3, 30,
+                               10, new NodeResources(0, 0, 10000, 1), 1000,
+                               0, new NodeResources(0, 0, 0, 0), 0);
+            var failurePath = tester.capacityChecker.worstCaseHostLossLeadingToFailure();
+            assertTrue(failurePath.isPresent());
+            if (failurePath.get().failureReason.tenant.isPresent()) {
+                var failureReasons = failurePath.get().failureReason.allocationFailures;
+                assertEquals("When there are multiple lacking resources, all failures are multipleReasonFailures",
+                             failureReasons.size(), failureReasons.multipleReasonFailures().size());
+                assertEquals(0, failureReasons.singularReasonFailures().size());
+            }
+            else {
+                fail();
+            }
+        }
     }
 
     @Test
     public void testIpFailurePaths() {
+        CapacityCheckerTester tester = new CapacityCheckerTester();
         tester.createNodes(1, 10,
                 10, new NodeResources(10, 1000, 10000, 1), 1,
                 10, new NodeResources(10, 1000, 10000, 1), 1);
@@ -95,79 +102,111 @@ public class CapacityCheckerTest {
 
     @Test
     public void testNodeResourceFailurePaths() {
-        tester.createNodes(1, 10,
-                10, new NodeResources(1, 100, 1000, 1), 100,
-                10, new NodeResources(0, 100, 1000, 1), 100);
-        var failurePath = tester.capacityChecker.worstCaseHostLossLeadingToFailure();
-        assertTrue(failurePath.isPresent());
-        if (failurePath.get().failureReason.tenant.isPresent()) {
-            var failureReasons = failurePath.get().failureReason.allocationFailures;
-            assertEquals("All failures should be due to hosts lacking cpu cores.",
-                    failureReasons.singularReasonFailures().insufficientVcpu(), failureReasons.size());
-        } else fail();
-
-        tester.createNodes(1, 10,
-                10, new NodeResources(10, 1, 1000, 1), 100,
-                10, new NodeResources(10, 0, 1000, 1), 100);
-        failurePath = tester.capacityChecker.worstCaseHostLossLeadingToFailure();
-        assertTrue(failurePath.isPresent());
-        if (failurePath.get().failureReason.tenant.isPresent()) {
-            var failureReasons = failurePath.get().failureReason.allocationFailures;
-            assertEquals("All failures should be due to hosts lacking memory.",
-                    failureReasons.singularReasonFailures().insufficientMemoryGb(), failureReasons.size());
-        } else fail();
-
-        tester.createNodes(1, 10,
-                10, new NodeResources(10, 100, 10, 1), 100,
-                10, new NodeResources(10, 100, 0, 1), 100);
-        failurePath = tester.capacityChecker.worstCaseHostLossLeadingToFailure();
-        assertTrue(failurePath.isPresent());
-        if (failurePath.get().failureReason.tenant.isPresent()) {
-            var failureReasons = failurePath.get().failureReason.allocationFailures;
-            assertEquals("All failures should be due to hosts lacking disk space.",
-                    failureReasons.singularReasonFailures().insufficientDiskGb(), failureReasons.size());
-        } else fail();
-
-        int emptyHostsWithSlowDisk = 10;
-        tester.createNodes(1, 10, List.of(new NodeResources(1, 10, 100, 1)),
-                10, new NodeResources(0, 0, 0, 0), 100,
-                10, new NodeResources(10, 1000, 10000, 1, NodeResources.DiskSpeed.slow), 100);
-        failurePath = tester.capacityChecker.worstCaseHostLossLeadingToFailure();
-        assertTrue(failurePath.isPresent());
-        if (failurePath.get().failureReason.tenant.isPresent()) {
-            var failureReasons = failurePath.get().failureReason.allocationFailures;
-            assertEquals("All empty hosts should be invalid due to having incompatible disk speed.",
-                    failureReasons.singularReasonFailures().incompatibleDiskSpeed(), emptyHostsWithSlowDisk);
-        } else fail();
-
+        {
+            CapacityCheckerTester tester = new CapacityCheckerTester();
+            tester.createNodes(1, 10,
+                               10, new NodeResources(1, 100, 1000, 1), 100,
+                               10, new NodeResources(0, 100, 1000, 1), 100);
+            var failurePath = tester.capacityChecker.worstCaseHostLossLeadingToFailure();
+            assertTrue(failurePath.isPresent());
+            if (failurePath.get().failureReason.tenant.isPresent()) {
+                var failureReasons = failurePath.get().failureReason.allocationFailures;
+                assertEquals("All failures should be due to hosts lacking cpu cores.",
+                             failureReasons.singularReasonFailures().insufficientVcpu(), failureReasons.size());
+            } else {
+                fail();
+            }
+        }
+
+        {
+            CapacityCheckerTester tester = new CapacityCheckerTester();
+            tester.createNodes(1, 10,
+                               10, new NodeResources(10, 1, 1000, 1), 100,
+                               10, new NodeResources(10, 0, 1000, 1), 100);
+            var failurePath = tester.capacityChecker.worstCaseHostLossLeadingToFailure();
+            assertTrue(failurePath.isPresent());
+            if (failurePath.get().failureReason.tenant.isPresent()) {
+                var failureReasons = failurePath.get().failureReason.allocationFailures;
+                assertEquals("All failures should be due to hosts lacking memory.",
+                             failureReasons.singularReasonFailures().insufficientMemoryGb(), failureReasons.size());
+            }
+            else {
+                fail();
+            }
+        }
+
+        {
+            CapacityCheckerTester tester = new CapacityCheckerTester();
+            tester.createNodes(1, 10,
+                               10, new NodeResources(10, 100, 10, 1), 100,
+                               10, new NodeResources(10, 100, 0, 1), 100);
+            var failurePath = tester.capacityChecker.worstCaseHostLossLeadingToFailure();
+            assertTrue(failurePath.isPresent());
+            if (failurePath.get().failureReason.tenant.isPresent()) {
+                var failureReasons = failurePath.get().failureReason.allocationFailures;
+                assertEquals("All failures should be due to hosts lacking disk space.",
+                             failureReasons.singularReasonFailures().insufficientDiskGb(), failureReasons.size());
+            } else {
+                fail();
+            }
+        }
+
+        {
+            CapacityCheckerTester tester = new CapacityCheckerTester();
+            int emptyHostsWithSlowDisk = 10;
+            tester.createNodes(1, 10, List.of(new NodeResources(1, 10, 100, 1)),
+                               10, new NodeResources(0, 0, 0, 0), 100,
+                               10, new NodeResources(10, 1000, 10000, 1, NodeResources.DiskSpeed.slow), 100);
+            var failurePath = tester.capacityChecker.worstCaseHostLossLeadingToFailure();
+            assertTrue(failurePath.isPresent());
+            if (failurePath.get().failureReason.tenant.isPresent()) {
+                var failureReasons = failurePath.get().failureReason.allocationFailures;
+                assertEquals("All empty hosts should be invalid due to having incompatible disk speed.",
+                             failureReasons.singularReasonFailures().incompatibleDiskSpeed(), emptyHostsWithSlowDisk);
+            } else {
+                fail();
+            }
+        }
     }
 
-
     @Test
     public void testParentHostPolicyIntegrityFailurePaths() {
-        tester.createNodes(1, 1,
-                10, new NodeResources(1, 100, 1000, 1), 100,
-                10, new NodeResources(10, 1000, 10000, 1), 100);
-        var failurePath = tester.capacityChecker.worstCaseHostLossLeadingToFailure();
-        assertTrue(failurePath.isPresent());
-        if (failurePath.get().failureReason.tenant.isPresent()) {
-            var failureReasons = failurePath.get().failureReason.allocationFailures;
-            assertEquals("With only one type of tenant, all failures should be due to violation of the parent host policy.",
-                    failureReasons.singularReasonFailures().violatesParentHostPolicy(), failureReasons.size());
-        } else fail();
-
-        tester.createNodes(1, 2,
-                10, new NodeResources(10, 100, 1000, 1), 1,
-                0, new NodeResources(0, 0, 0, 0), 0);
-        failurePath = tester.capacityChecker.worstCaseHostLossLeadingToFailure();
-        assertTrue(failurePath.isPresent());
-        if (failurePath.get().failureReason.tenant.isPresent()) {
-            var failureReasons = failurePath.get().failureReason.allocationFailures;
-            assertNotEquals("Fewer distinct children than hosts should result in some parent host policy violations.",
-                    failureReasons.size(), failureReasons.singularReasonFailures().violatesParentHostPolicy());
-            assertNotEquals(0, failureReasons.singularReasonFailures().violatesParentHostPolicy());
-        } else fail();
+        {
+            CapacityCheckerTester tester = new CapacityCheckerTester();
+            tester.createNodes(1, 1,
+                               10, new NodeResources(1, 100, 1000, 1), 100,
+                               10, new NodeResources(10, 1000, 10000, 1), 100);
+            var failurePath = tester.capacityChecker.worstCaseHostLossLeadingToFailure();
+            assertTrue(failurePath.isPresent());
+            if (failurePath.get().failureReason.tenant.isPresent()) {
+                var failureReasons = failurePath.get().failureReason.allocationFailures;
+                assertEquals("With only one type of tenant, all failures should be due to violation of the parent host policy.",
+                             failureReasons.singularReasonFailures().violatesParentHostPolicy(), failureReasons.size());
+            }
+            else {
+                fail();
+            }
+        }
+
+        {
+            CapacityCheckerTester tester = new CapacityCheckerTester();
+            tester.createNodes(1, 2,
+                               10, new NodeResources(10, 100, 1000, 1), 1,
+                               0, new NodeResources(0, 0, 0, 0), 0);
+            var failurePath = tester.capacityChecker.worstCaseHostLossLeadingToFailure();
+            assertTrue(failurePath.isPresent());
+            if (failurePath.get().failureReason.tenant.isPresent()) {
+                var failureReasons = failurePath.get().failureReason.allocationFailures;
+                assertNotEquals("Fewer distinct children than hosts should result in some parent host policy violations.",
+                                failureReasons.size(), failureReasons.singularReasonFailures().violatesParentHostPolicy());
+                assertNotEquals(0, failureReasons.singularReasonFailures().violatesParentHostPolicy());
+            }
+            else {
+                fail();
+            }
+        }
     }
+
 }
 
 
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTester.java
index afac44856c9..0bc4d2c65a1 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTester.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTester.java
@@ -24,6 +24,7 @@ import com.yahoo.vespa.curator.Curator;
 import com.yahoo.vespa.curator.mock.MockCurator;
 import com.yahoo.vespa.hosted.provision.Node;
 import com.yahoo.vespa.hosted.provision.NodeRepository;
+import com.yahoo.vespa.hosted.provision.node.Agent;
 import com.yahoo.vespa.hosted.provision.node.IP;
 import com.yahoo.vespa.hosted.provision.provisioning.FlavorConfigBuilder;
 import com.yahoo.vespa.hosted.provision.testutils.MockNameResolver;
@@ -173,14 +174,13 @@ public class CapacityCheckerTester {
     void createNodes(int childrenPerHost, int numDistinctChildren, List<NodeResources> childResources,
                      int numHosts, NodeResources hostExcessCapacity, int hostExcessIps,
                      int numEmptyHosts, NodeResources emptyHostExcessCapacity, int emptyHostExcessIps) {
-        cleanRepository();
         List<NodeModel> possibleChildren = createDistinctChildren(numDistinctChildren, childResources);
 
         List<Node> nodes = new ArrayList<>();
         nodes.addAll(createHostsWithChildren(childrenPerHost, possibleChildren, numHosts, hostExcessCapacity, hostExcessIps));
         nodes.addAll(createEmptyHosts(numHosts, numEmptyHosts, emptyHostExcessCapacity, emptyHostExcessIps));
 
-        nodeRepository.addNodes(nodes);
+        nodeRepository.addNodes(nodes, Agent.system);
         updateCapacityChecker();
     }
 
@@ -252,7 +252,8 @@ public class CapacityCheckerTester {
         if (nodeModel.membership != null && nodeModel.owner != null) {
             membership = ClusterMembership.from(
                     nodeModel.membership.toString(),
-                    Version.fromString(nodeModel.wantedVespaVersion));
+                    Version.fromString(nodeModel.wantedVespaVersion),
+                    Optional.empty());
             owner = ApplicationId.from(nodeModel.owner.tenant, nodeModel.owner.application, nodeModel.owner.instance);
         }
 
@@ -273,7 +274,7 @@ public class CapacityCheckerTester {
         }
     }
 
-    public void restoreNodeRepositoryFromJsonFile(Path path) throws IOException {
+    public void populateNodeRepositoryFromJsonFile(Path path) throws IOException {
         byte[] jsonData = Files.readAllBytes(path);
         ObjectMapper om = new ObjectMapper();
 
@@ -287,15 +288,8 @@ public class CapacityCheckerTester {
             nodes.add(createNodeFromModel(nmod));
         }
 
-        nodeRepository.addNodes(nodes);
+        nodeRepository.addNodes(nodes, Agent.system);
         updateCapacityChecker();
     }
 
-    void cleanRepository() {
-        nodeRepository.getNodes(NodeType.host).forEach(n -> nodeRepository.removeRecursively(n, true));
-        nodeRepository.getNodes().forEach(n -> nodeRepository.removeRecursively(n, true));
-        if (nodeRepository.getNodes().size() != 0) {
-            throw new IllegalStateException("Cleaning repository didn't remove all nodes! [" + nodeRepository.getNodes().size() + "]");
-        }
-    }
 }
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java
index c7a1486a1a4..ebec07fe5dc 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java
@@ -3,12 +3,16 @@ package com.yahoo.vespa.hosted.provision.maintenance;
 
 import com.yahoo.component.Version;
 import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.CloudName;
 import com.yahoo.config.provision.ClusterMembership;
 import com.yahoo.config.provision.DockerImage;
+import com.yahoo.config.provision.Environment;
 import com.yahoo.config.provision.Flavor;
 import com.yahoo.config.provision.NodeFlavors;
 import com.yahoo.config.provision.NodeResources;
 import com.yahoo.config.provision.NodeType;
+import com.yahoo.config.provision.RegionName;
+import com.yahoo.config.provision.SystemName;
 import com.yahoo.config.provision.Zone;
 import com.yahoo.test.ManualClock;
 import com.yahoo.vespa.curator.mock.MockCurator;
@@ -26,9 +30,11 @@ import com.yahoo.vespa.hosted.provision.node.Status;
 import com.yahoo.vespa.hosted.provision.provisioning.FatalProvisioningException;
 import com.yahoo.vespa.hosted.provision.provisioning.FlavorConfigBuilder;
 import com.yahoo.vespa.hosted.provision.provisioning.HostProvisioner;
+import com.yahoo.vespa.hosted.provision.provisioning.HostResourcesCalculator;
 import com.yahoo.vespa.hosted.provision.testutils.MockNameResolver;
 import org.hamcrest.BaseMatcher;
 import org.hamcrest.Description;
+import org.junit.Before;
 import org.junit.Test;
 
 import java.time.Duration;
@@ -45,10 +51,10 @@ import static com.yahoo.vespa.hosted.provision.maintenance.DynamicProvisioningMa
 import static com.yahoo.vespa.hosted.provision.maintenance.DynamicProvisioningMaintainerTest.HostProvisionerTester.tenantApp;
 import static com.yahoo.vespa.hosted.provision.maintenance.DynamicProvisioningMaintainerTest.HostProvisionerTester.tenantHostApp;
 import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.doAnswer;
 import static org.mockito.Mockito.doThrow;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.times;
@@ -64,18 +70,23 @@ public class DynamicProvisioningMaintainerTest {
 
     private final HostProvisionerTester tester = new HostProvisionerTester();
     private final HostProvisioner hostProvisioner = mock(HostProvisioner.class);
+    private final HostResourcesCalculator hostResourcesCalculator = mock(HostResourcesCalculator.class);
     private final InMemoryFlagSource flagSource = new InMemoryFlagSource()
             .withBooleanFlag(Flags.ENABLE_DYNAMIC_PROVISIONING.id(), true)
             .withListFlag(Flags.PREPROVISION_CAPACITY.id(), List.of(), PreprovisionCapacity.class);
     private final DynamicProvisioningMaintainer maintainer = new DynamicProvisioningMaintainer(
-            tester.nodeRepository, Duration.ofDays(1), hostProvisioner, flagSource);
+            tester.nodeRepository, Duration.ofDays(1), hostProvisioner, hostResourcesCalculator, flagSource);
 
     @Test
     public void delegates_to_host_provisioner_and_writes_back_result() {
         addNodes();
+        Node host3 = tester.nodeRepository.getNode("host3").orElseThrow();
         Node host4 = tester.nodeRepository.getNode("host4").orElseThrow();
         Node host41 = tester.nodeRepository.getNode("host4-1").orElseThrow();
-        assertTrue(Stream.of(host4, host41).map(Node::ipAddresses).allMatch(Set::isEmpty));
+        assertTrue(Stream.of(host3, host4, host41).map(Node::ipAddresses).allMatch(Set::isEmpty));
+
+        Node host3new = host3.with(host3.ipConfig().with(Set.of("::5")));
+        when(hostProvisioner.provision(eq(host3), eq(Set.of()))).thenReturn(List.of(host3new));
 
         Node host4new = host4.with(host4.ipConfig().with(Set.of("::2")));
         Node host41new = host41.with(host4.ipConfig().with(Set.of("::4", "10.0.0.1")));
@@ -83,8 +94,10 @@ public class DynamicProvisioningMaintainerTest {
 
         maintainer.updateProvisioningNodes(tester.nodeRepository.list(), () -> {});
         verify(hostProvisioner).provision(eq(host4), eq(Set.of(host41)));
+        verify(hostProvisioner).provision(eq(host3), eq(Set.of()));
         verifyNoMoreInteractions(hostProvisioner);
 
+        assertEquals(Optional.of(host3new), tester.nodeRepository.getNode("host3"));
         assertEquals(Optional.of(host4new), tester.nodeRepository.getNode("host4"));
         assertEquals(Optional.of(host41new), tester.nodeRepository.getNode("host4-1"));
     }
@@ -111,8 +124,8 @@ public class DynamicProvisioningMaintainerTest {
         verify(hostProvisioner).deprovision(argThatLambda(node -> node.hostname().equals("host2")));
         verify(hostProvisioner).deprovision(argThatLambda(node -> node.hostname().equals("host3")));
         verifyNoMoreInteractions(hostProvisioner);
-        assertFalse(tester.nodeRepository.getNode("host2").isPresent());
-        assertFalse(tester.nodeRepository.getNode("host3").isPresent());
+        assertTrue(tester.nodeRepository.getNode("host2").isEmpty());
+        assertTrue(tester.nodeRepository.getNode("host3").isEmpty());
     }
 
     @Test
@@ -127,11 +140,14 @@ public class DynamicProvisioningMaintainerTest {
 
     @Test
     public void provision_deficit_and_deprovision_excess() {
-        flagSource.withListFlag(Flags.PREPROVISION_CAPACITY.id(), List.of(new PreprovisionCapacity(1, 3, 2, 1), new PreprovisionCapacity(2, 3, 2, 2)), PreprovisionCapacity.class);
+        flagSource.withListFlag(Flags.PREPROVISION_CAPACITY.id(),
+                                List.of(new PreprovisionCapacity(2, 4, 8, 1),
+                                        new PreprovisionCapacity(2, 3, 2, 2)),
+                                PreprovisionCapacity.class);
         addNodes();
 
         maintainer.convergeToCapacity(tester.nodeRepository.list());
-        assertFalse(tester.nodeRepository.getNode("host2").isPresent());
+        assertTrue(tester.nodeRepository.getNode("host2").isEmpty());
         assertTrue(tester.nodeRepository.getNode("host3").isPresent());
         verify(hostProvisioner).deprovision(argThatLambda(node -> node.hostname().equals("host2")));
         verify(hostProvisioner, times(2)).provisionHosts(argThatLambda(list -> list.size() == 1), eq(new NodeResources(2, 3, 2, 1)), any());
@@ -150,6 +166,15 @@ public class DynamicProvisioningMaintainerTest {
         verifyNoMoreInteractions(hostProvisioner);
     }
 
+    @Before
+    public void setup() {
+        doAnswer(invocation ->  {
+            Flavor flavor = invocation.getArgument(0, Flavor.class);
+            if ("default".equals(flavor.name())) return new NodeResources(2, 4, 8, 1);
+            return invocation.getArguments()[1];
+        }).when(hostResourcesCalculator).advertisedResourcesOf(any());
+    }
+
     public void addNodes() {
         List.of(createNode("host1", Optional.empty(), NodeType.host, Node.State.active, Optional.of(tenantHostApp)),
                 createNode("host1-1", Optional.of("host1"), NodeType.tenant, Node.State.reserved, Optional.of(tenantApp)),
@@ -157,6 +182,7 @@ public class DynamicProvisioningMaintainerTest {
 
                 createNode("host2", Optional.empty(), NodeType.host, Node.State.failed, Optional.of(tenantApp)),
                 createNode("host2-1", Optional.of("host2"), NodeType.tenant, Node.State.failed, Optional.empty()),
+
                 createNode("host3", Optional.empty(), NodeType.host, Node.State.provisioned, Optional.empty()),
 
                 createNode("host4", Optional.empty(), NodeType.host, Node.State.provisioned, Optional.empty()),
@@ -185,8 +211,10 @@ public class DynamicProvisioningMaintainerTest {
         static final ApplicationId proxyApp = ApplicationId.from("vespa", "proxy", "default");
 
         private final ManualClock clock = new ManualClock();
+        private final Zone zone = new Zone(CloudName.from("aws"), SystemName.defaultSystem(), Environment.defaultEnvironment(), RegionName.defaultName());
         private final NodeRepository nodeRepository = new NodeRepository(
-                nodeFlavors, new MockCurator(), clock, Zone.defaultZone(), new MockNameResolver().mockAnyLookup(), DockerImage.fromString("docker-image"), true);
+                nodeFlavors, new MockCurator(), clock, zone, new MockNameResolver().mockAnyLookup(),
+                DockerImage.fromString("docker-image"), true);
 
         Node addNode(String hostname, Optional<String> parentHostname, NodeType nodeType, Node.State state, Optional<ApplicationId> application) {
             Node node = createNode(hostname, parentHostname, nodeType, state, application);
@@ -198,7 +226,7 @@ public class DynamicProvisioningMaintainerTest {
             Optional<Allocation> allocation = application
                     .map(app -> new Allocation(
                             app,
-                            ClusterMembership.from("container/default/0/0", Version.fromString("7.3")),
+                            ClusterMembership.from("container/default/0/0", Version.fromString("7.3"), Optional.empty()),
                             flavor.resources(),
                             Generation.initial(),
                             false));
@@ -207,4 +235,4 @@ public class DynamicProvisioningMaintainerTest {
                     state, allocation, History.empty(), nodeType, new Reports(), Optional.empty(), Optional.empty());
         }
     }
-}
\ No newline at end of file
+}
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java
index 8509722b016..17521261e1b 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirerTest.java
@@ -1,10 +1,10 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.provision.maintenance;
 
-import com.yahoo.component.Version;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.ApplicationName;
 import com.yahoo.config.provision.Capacity;
+import com.yahoo.config.provision.ClusterResources;
 import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.config.provision.DockerImage;
 import com.yahoo.config.provision.Environment;
@@ -54,10 +54,7 @@ public class FailedExpirerTest {
     private static final ApplicationId tenantHostApplicationId = ApplicationId.from("vespa", "zone-app", "default");
 
     private static final ClusterSpec tenantHostApplicationClusterSpec =
-            ClusterSpec.request(ClusterSpec.Type.container,
-                                ClusterSpec.Id.from("node-admin"),
-                                Version.fromString("6.42"),
-                                false);
+            ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("node-admin")).vespaVersion("6.42").build();
 
     private static final Capacity tenantHostApplicationCapacity = Capacity.fromRequiredNodeType(NodeType.host);
 
@@ -143,9 +140,9 @@ public class FailedExpirerTest {
                 .withNode(NodeType.proxy, FailureScenario.defaultFlavor, "proxy2")
                 .withNode(NodeType.proxy, FailureScenario.defaultFlavor, "proxy3")
                 .setReady("proxy1", "proxy2", "proxy3")
-                .allocate( ApplicationId.from("vespa", "zone-app", "default"),
-                        ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("routing"), Version.fromString("6.42"), false),
-                        Capacity.fromRequiredNodeType(NodeType.proxy))
+                .allocate(ApplicationId.from("vespa", "zone-app", "default"),
+                          ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("routing")).vespaVersion("6.42").build(),
+                          Capacity.fromRequiredNodeType(NodeType.proxy))
                 .failNode(1, "proxy1");
 
         for (int i = 0; i < 10; i++) {
@@ -275,10 +272,12 @@ public class FailedExpirerTest {
         }
 
         public FailureScenario withNode(NodeType type, NodeResources flavor, String hostname, String parentHostname) {
-            nodeRepository.addNodes(List.of(
-                    nodeRepository.createNode(UUID.randomUUID().toString(), hostname,
-                                              Optional.ofNullable(parentHostname), new Flavor(flavor), type)
-            ));
+            nodeRepository.addNodes(List.of(nodeRepository.createNode(UUID.randomUUID().toString(),
+                                                                      hostname,
+                                                                      Optional.ofNullable(parentHostname),
+                                                                      new Flavor(flavor),
+                                                                      type)),
+                                    Agent.system);
             return this;
         }
 
@@ -293,7 +292,7 @@ public class FailedExpirerTest {
         public FailureScenario failNode(int times, String... hostname) {
             Stream.of(hostname).forEach(h -> {
                 Node node = get(h);
-                nodeRepository.write(node.with(node.status().setFailCount(times)), () -> {});
+                nodeRepository.write(node.with(node.status().withFailCount(times)), () -> {});
                 nodeRepository.fail(h, Agent.system, "Failed by unit test");
             });
             return this;
@@ -323,17 +322,13 @@ public class FailedExpirerTest {
         }
 
         public FailureScenario allocate(ClusterSpec.Type clusterType, NodeResources flavor, String... hostname) {
-            ClusterSpec clusterSpec = ClusterSpec.request(clusterType,
-                                                          ClusterSpec.Id.from("test"),
-                                                          Version.fromString("6.42"),
-                                                          false
-            );
-            Capacity capacity = Capacity.fromCount(hostname.length, Optional.of(flavor), false, true);
+            ClusterSpec clusterSpec = ClusterSpec.request(clusterType, ClusterSpec.Id.from("test")).vespaVersion("6.42").build();
+            Capacity capacity = Capacity.from(new ClusterResources(hostname.length, 1, flavor), false, true);
             return allocate(applicationId, clusterSpec, capacity);
         }
 
         public FailureScenario allocate(ApplicationId applicationId, ClusterSpec clusterSpec, Capacity capacity) {
-            List<HostSpec> preparedNodes = provisioner.prepare(applicationId, clusterSpec, capacity, 1, null);
+            List<HostSpec> preparedNodes = provisioner.prepare(applicationId, clusterSpec, capacity, null);
             NestedTransaction transaction = new NestedTransaction().add(new CuratorTransaction(curator));
             provisioner.activate(transaction, applicationId, Set.copyOf(preparedNodes));
             transaction.commit();
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveAndFailedExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveAndFailedExpirerTest.java
index f9c2ef90bbd..4fcd5793b8a 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveAndFailedExpirerTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/InactiveAndFailedExpirerTest.java
@@ -1,10 +1,10 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.provision.maintenance;
 
-import com.yahoo.component.Version;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.ApplicationName;
 import com.yahoo.config.provision.Capacity;
+import com.yahoo.config.provision.ClusterResources;
 import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.config.provision.Environment;
 import com.yahoo.config.provision.HostSpec;
@@ -19,6 +19,7 @@ import com.yahoo.vespa.hosted.provision.node.Agent;
 import com.yahoo.vespa.hosted.provision.node.History;
 import com.yahoo.vespa.hosted.provision.provisioning.ProvisioningTester;
 import com.yahoo.vespa.hosted.provision.testutils.MockDeployer;
+import com.yahoo.vespa.hosted.provision.testutils.MockNodeMetrics;
 import com.yahoo.vespa.orchestrator.OrchestrationException;
 import com.yahoo.vespa.orchestrator.Orchestrator;
 import org.junit.Test;
@@ -54,8 +55,8 @@ public class InactiveAndFailedExpirerTest {
         List<Node> nodes = tester.makeReadyNodes(2, nodeResources);
 
         // Allocate then deallocate 2 nodes
-        ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test"), Version.fromString("6.42"), false);
-        List<HostSpec> preparedNodes = tester.prepare(applicationId, cluster, Capacity.fromCount(2, nodeResources), 1);
+        ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test")).vespaVersion("6.42").build();
+        List<HostSpec> preparedNodes = tester.prepare(applicationId, cluster, Capacity.from(new ClusterResources(2, 1, nodeResources)));
         tester.activate(applicationId, new HashSet<>(preparedNodes));
         assertEquals(2, tester.getNodes(applicationId, Node.State.active).size());
         tester.deactivate(applicationId);
@@ -92,11 +93,8 @@ public class InactiveAndFailedExpirerTest {
         List<Node> nodes = tester.makeReadyNodes(2, nodeResources);
 
         // Allocate and deallocate a single node
-        ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content,
-                                                  ClusterSpec.Id.from("test"),
-                                                  Version.fromString("6.42"),
-                                                  false);
-        List<HostSpec> preparedNodes = tester.prepare(applicationId, cluster, Capacity.fromCount(2, nodeResources), 1);
+        ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test")).vespaVersion("6.42").build();
+        List<HostSpec> preparedNodes = tester.prepare(applicationId, cluster, Capacity.from(new ClusterResources(2, 1, nodeResources)));
         tester.activate(applicationId, new HashSet<>(preparedNodes));
         assertEquals(2, tester.getNodes(applicationId, Node.State.active).size());
         tester.deactivate(applicationId);
@@ -121,16 +119,14 @@ public class InactiveAndFailedExpirerTest {
     @Test
     public void node_that_wants_to_retire_is_moved_to_parked() throws OrchestrationException {
         ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build();
-        ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test"),
-                                                  Version.fromString("6.42"), false);
+        ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test")).vespaVersion("6.42").build();
         tester.makeReadyNodes(5, nodeResources);
 
         // Allocate two nodes
         {
             List<HostSpec> hostSpecs = tester.prepare(applicationId,
                                                       cluster,
-                                                      Capacity.fromCount(2, nodeResources),
-                                                      1);
+                                                      Capacity.from(new ClusterResources(2, 1, nodeResources)));
             tester.activate(applicationId, new HashSet<>(hostSpecs));
             assertEquals(2, tester.getNodes(applicationId, Node.State.active).size());
         }
@@ -139,7 +135,7 @@ public class InactiveAndFailedExpirerTest {
         {
             Node toRetire = tester.getNodes(applicationId, Node.State.active).asList().get(0);
             tester.patchNode(toRetire.withWantToRetire(true, Agent.operator, tester.clock().instant()));
-            List<HostSpec> hostSpecs = tester.prepare(applicationId, cluster, Capacity.fromCount(2, nodeResources), 1);
+            List<HostSpec> hostSpecs = tester.prepare(applicationId, cluster, Capacity.from(new ClusterResources(2, 1, nodeResources)));
             tester.activate(applicationId, new HashSet<>(hostSpecs));
         }
 
@@ -151,15 +147,13 @@ public class InactiveAndFailedExpirerTest {
                 Collections.singletonMap(
                         applicationId,
                         new MockDeployer.ApplicationContext(applicationId, cluster,
-                                                            Capacity.fromCount(2,
-                                                                               nodeResources,
-                                                                               false, true),
-                                                            1)
+                                                            Capacity.from(new ClusterResources(2, 1, nodeResources),
+                                                                          false, true))
                 )
         );
         Orchestrator orchestrator = mock(Orchestrator.class);
         doThrow(new RuntimeException()).when(orchestrator).acquirePermissionToRemove(any());
-        new RetiredExpirer(tester.nodeRepository(), tester.orchestrator(), deployer, tester.clock(), Duration.ofDays(30),
+        new RetiredExpirer(tester.nodeRepository(), tester.orchestrator(), deployer, new TestMetric(), tester.clock(), Duration.ofDays(30),
                            Duration.ofMinutes(10)).run();
         assertEquals(1, tester.nodeRepository().getNodes(Node.State.inactive).size());
 
@@ -178,8 +172,8 @@ public class InactiveAndFailedExpirerTest {
         // Allocate then deallocate a node
         ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build();
         tester.makeReadyNodes(1, nodeResources);
-        ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test"), Version.fromString("6.42"), false);
-        List<HostSpec> preparedNodes = tester.prepare(testerId, cluster, Capacity.fromCount(2, nodeResources), 1);
+        ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test")).vespaVersion("6.42").build();
+        List<HostSpec> preparedNodes = tester.prepare(testerId, cluster, Capacity.from(new ClusterResources(2, 1, nodeResources)));
         tester.activate(testerId, new HashSet<>(preparedNodes));
         assertEquals(1, tester.getNodes(testerId, Node.State.active).size());
         tester.deactivate(testerId);
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/JobControlTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/JobControlTest.java
index 729e7f4cd94..396fcd67034 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/JobControlTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/JobControlTest.java
@@ -20,14 +20,16 @@ public class JobControlTest {
     public void testJobControl() {
         NodeRepositoryTester tester = new NodeRepositoryTester();
         JobControl jobControl = new JobControl(tester.nodeRepository().database());
-        
+
+        MockMaintainer maintainer1 = new MockMaintainer(tester.nodeRepository());
+        MockMaintainer maintainer2 = new MockMaintainer(tester.nodeRepository());
         assertTrue(jobControl.jobs().isEmpty());
 
         String job1 = "Job1";
         String job2 = "Job2";
         
-        jobControl.started(job1);
-        jobControl.started(job2);
+        jobControl.started(job1, maintainer1);
+        jobControl.started(job2, maintainer2);
         assertEquals(2, jobControl.jobs().size());
         assertTrue(jobControl.jobs().contains(job1));
         assertTrue(jobControl.jobs().contains(job2));
@@ -50,6 +52,18 @@ public class JobControlTest {
         jobControl.setActive(job2, true);
         assertTrue(jobControl.isActive(job1));
         assertTrue(jobControl.isActive(job2));
+
+        // Run jobs on-demand
+        jobControl.run(job1);
+        jobControl.run(job1);
+        assertEquals(2, maintainer1.maintenanceInvocations);
+        jobControl.run(job2);
+        assertEquals(1, maintainer2.maintenanceInvocations);
+
+        // Running jobs on-demand ignores inactive flag
+        jobControl.setActive(job1, false);
+        jobControl.run(job1);
+        assertEquals(3, maintainer1.maintenanceInvocations);
     }
     
     @Test
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirerTest.java
index 12b48fd7a35..e278cdc992a 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirerTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirerTest.java
@@ -1,4 +1,4 @@
-// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.provision.maintenance;
 
 import com.yahoo.component.Vtag;
@@ -32,14 +32,14 @@ import static org.junit.Assert.assertTrue;
  */
 public class LoadBalancerExpirerTest {
 
-    private ProvisioningTester tester = new ProvisioningTester.Builder().build();
+    private final ProvisioningTester tester = new ProvisioningTester.Builder().build();
 
     @Test
     public void expire_inactive() {
         LoadBalancerExpirer expirer = new LoadBalancerExpirer(tester.nodeRepository(),
                                                               Duration.ofDays(1),
                                                               tester.loadBalancerService());
-        Supplier<Map<LoadBalancerId, LoadBalancer>> loadBalancers = () -> tester.nodeRepository().database().readLoadBalancers();
+        Supplier<Map<LoadBalancerId, LoadBalancer>> loadBalancers = () -> tester.nodeRepository().database().readLoadBalancers((ignored) -> true);
 
         // Deploy two applications with a total of three load balancers
         ClusterSpec.Id cluster1 = ClusterSpec.Id.from("qrs");
@@ -67,14 +67,15 @@ public class LoadBalancerExpirerTest {
         // Expirer prunes reals before expiration time of load balancer itself
         expirer.maintain();
         assertEquals(Set.of(), tester.loadBalancerService().instances().get(lb1).reals());
-        assertEquals(Set.of(), tester.nodeRepository().loadBalancers().owner(lb1.application()).asList().get(0).instance().reals());
+        assertEquals(Set.of(), loadBalancers.get().get(lb1).instance().reals());
 
         // Expirer defers removal of load balancer until expiration time passes
         expirer.maintain();
+        assertSame(LoadBalancer.State.inactive, loadBalancers.get().get(lb1).state());
         assertTrue("Inactive load balancer not removed", tester.loadBalancerService().instances().containsKey(lb1));
 
         // Expirer removes load balancers once expiration time passes
-        tester.clock().advance(Duration.ofHours(1));
+        tester.clock().advance(Duration.ofHours(1).plus(Duration.ofSeconds(1)));
         expirer.maintain();
         assertFalse("Inactive load balancer removed", tester.loadBalancerService().instances().containsKey(lb1));
 
@@ -85,7 +86,7 @@ public class LoadBalancerExpirerTest {
         // A single cluster is removed
         deployApplication(app2, cluster1);
         expirer.maintain();
-        assertEquals(LoadBalancer.State.inactive, loadBalancers.get().get(lb3).state());
+        assertSame(LoadBalancer.State.inactive, loadBalancers.get().get(lb3).state());
 
         // Expirer defers removal while nodes are still allocated to cluster
         expirer.maintain();
@@ -93,7 +94,7 @@ public class LoadBalancerExpirerTest {
         dirtyNodesOf(app2, cluster2);
 
         // Expirer removes load balancer for removed cluster
-        tester.clock().advance(Duration.ofHours(1));
+        tester.clock().advance(Duration.ofHours(1).plus(Duration.ofSeconds(1)));
         expirer.maintain();
         assertFalse("Inactive load balancer removed", tester.loadBalancerService().instances().containsKey(lb3));
     }
@@ -103,7 +104,7 @@ public class LoadBalancerExpirerTest {
         LoadBalancerExpirer expirer = new LoadBalancerExpirer(tester.nodeRepository(),
                                                               Duration.ofDays(1),
                                                               tester.loadBalancerService());
-        Supplier<Map<LoadBalancerId, LoadBalancer>> loadBalancers = () -> tester.nodeRepository().database().readLoadBalancers();
+        Supplier<Map<LoadBalancerId, LoadBalancer>> loadBalancers = () -> tester.nodeRepository().database().readLoadBalancers((ignored) -> true);
 
 
         // Prepare application
@@ -121,7 +122,7 @@ public class LoadBalancerExpirerTest {
 
         // Application never activates and nodes are dirtied. Expirer moves load balancer to inactive after timeout
         dirtyNodesOf(app, cluster);
-        tester.clock().advance(Duration.ofHours(1));
+        tester.clock().advance(Duration.ofHours(1).plus(Duration.ofSeconds(1)));
         expirer.maintain();
         assertSame(LoadBalancer.State.inactive, loadBalancers.get().get(lb).state());
 
@@ -130,7 +131,7 @@ public class LoadBalancerExpirerTest {
         assertSame(LoadBalancer.State.inactive, loadBalancers.get().get(lb).state());
 
         // Expirer removes inactive load balancer
-        tester.clock().advance(Duration.ofHours(1));
+        tester.clock().advance(Duration.ofHours(1).plus(Duration.ofSeconds(1)));
         expirer.maintain();
         assertFalse("Inactive load balancer removed", loadBalancers.get().containsKey(lb));
     }
@@ -157,8 +158,7 @@ public class LoadBalancerExpirerTest {
         tester.makeReadyNodes(10, "d-1-4-10");
         List<HostSpec> hosts = new ArrayList<>();
         for (var cluster : clusters) {
-            hosts.addAll(tester.prepare(application, ClusterSpec.request(ClusterSpec.Type.container, cluster,
-                                                                         Vtag.currentVersion, false),
+            hosts.addAll(tester.prepare(application, ClusterSpec.request(ClusterSpec.Type.container, cluster).vespaVersion(Vtag.currentVersion).build(),
                                         2, 1,
                                         new NodeResources(1, 4, 10, 0.3)));
         }
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MaintainerTest.java
index c19b54a8cab..7ce64093491 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MaintainerTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MaintainerTest.java
@@ -19,15 +19,22 @@ public class MaintainerTest {
     @Test
     public void staggering() {
         List<HostName> cluster = Arrays.asList(HostName.from("cfg1"), HostName.from("cfg2"), HostName.from("cfg3"));
-        Instant now = Instant.ofEpochMilli(1001);
         Duration interval = Duration.ofMillis(300);
-        assertEquals(299, Maintainer.staggeredDelay(cluster, HostName.from("cfg1"), now, interval));
-        assertEquals(399, Maintainer.staggeredDelay(cluster, HostName.from("cfg2"), now, interval));
-        assertEquals(199, Maintainer.staggeredDelay(cluster, HostName.from("cfg3"), now, interval));
-        now = Instant.ofEpochMilli(1101);
+        Instant now = Instant.ofEpochMilli(1000);
+        assertEquals(200, Maintainer.staggeredDelay(cluster, HostName.from("cfg1"), now, interval));
+        assertEquals(  0, Maintainer.staggeredDelay(cluster, HostName.from("cfg2"), now, interval));
+        assertEquals(100, Maintainer.staggeredDelay(cluster, HostName.from("cfg3"), now, interval));
+
+        now = Instant.ofEpochMilli(1001);
         assertEquals(199, Maintainer.staggeredDelay(cluster, HostName.from("cfg1"), now, interval));
         assertEquals(299, Maintainer.staggeredDelay(cluster, HostName.from("cfg2"), now, interval));
-        assertEquals(399, Maintainer.staggeredDelay(cluster, HostName.from("cfg3"), now, interval));
+        assertEquals( 99, Maintainer.staggeredDelay(cluster, HostName.from("cfg3"), now, interval));
+
+        now = Instant.ofEpochMilli(1101);
+        assertEquals( 99, Maintainer.staggeredDelay(cluster, HostName.from("cfg1"), now, interval));
+        assertEquals(199, Maintainer.staggeredDelay(cluster, HostName.from("cfg2"), now, interval));
+        assertEquals(299, Maintainer.staggeredDelay(cluster, HostName.from("cfg3"), now, interval));
+
         assertEquals(300, Maintainer.staggeredDelay(cluster, HostName.from("cfg0"), now, interval));
     }
 }
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceTester.java
index a4b66d3cf9e..664809dc3ab 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceTester.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceTester.java
@@ -3,16 +3,20 @@ package com.yahoo.vespa.hosted.provision.maintenance;
 
 import com.yahoo.config.provision.DockerImage;
 import com.yahoo.config.provision.Environment;
+import com.yahoo.config.provision.Flavor;
 import com.yahoo.config.provision.NodeFlavors;
+import com.yahoo.config.provision.NodeResources;
 import com.yahoo.config.provision.NodeType;
 import com.yahoo.config.provision.RegionName;
 import com.yahoo.config.provision.Zone;
 import com.yahoo.test.ManualClock;
 import com.yahoo.vespa.curator.mock.MockCurator;
+import com.yahoo.vespa.flags.InMemoryFlagSource;
 import com.yahoo.vespa.hosted.provision.Node;
 import com.yahoo.vespa.hosted.provision.NodeRepository;
 import com.yahoo.vespa.hosted.provision.node.Agent;
 import com.yahoo.vespa.hosted.provision.provisioning.FlavorConfigBuilder;
+import com.yahoo.vespa.hosted.provision.provisioning.HostResourcesCalculator;
 import com.yahoo.vespa.hosted.provision.testutils.MockNameResolver;
 
 import java.time.Instant;
@@ -47,7 +51,7 @@ public class MaintenanceTester {
         List<Node> nodes = new ArrayList<>(count);
         for (int i = 0; i < count; i++)
             nodes.add(nodeRepository.createNode("node" + i, "host" + i, Optional.empty(), nodeFlavors.getFlavorOrThrow("default"), NodeType.tenant));
-        nodes = nodeRepository.addNodes(nodes);
+        nodes = nodeRepository.addNodes(nodes, Agent.system);
         nodes = nodeRepository.setDirty(nodes, Agent.system, getClass().getSimpleName());
         nodes = simulateInitialReboot(nodes);
         nodeRepository.setReady(nodes, Agent.system, getClass().getSimpleName());
@@ -57,7 +61,7 @@ public class MaintenanceTester {
         List<Node> nodes = new ArrayList<>(count);
         for (int i = 0; i < count; i++)
             nodes.add(nodeRepository.createNode("hostNode" + i, "realHost" + i, Optional.empty(), nodeFlavors.getFlavorOrThrow("default"), NodeType.host));
-        nodes = nodeRepository.addNodes(nodes);
+        nodes = nodeRepository.addNodes(nodes, Agent.system);
         nodes = nodeRepository.setDirty(nodes, Agent.system, getClass().getSimpleName());
         nodes = simulateInitialReboot(nodes);
         nodeRepository.setReady(nodes, Agent.system, getClass().getSimpleName());
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java
index 2a8783748ef..665dd74176d 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java
@@ -10,6 +10,10 @@ import com.yahoo.config.provision.NodeResources;
 import com.yahoo.config.provision.NodeType;
 import com.yahoo.config.provision.Zone;
 import com.yahoo.jdisc.Metric;
+import com.yahoo.test.ManualClock;
+import com.yahoo.transaction.NestedTransaction;
+import com.yahoo.vespa.applicationmodel.ApplicationInstance;
+import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference;
 import com.yahoo.vespa.curator.Curator;
 import com.yahoo.vespa.curator.mock.MockCurator;
 import com.yahoo.vespa.hosted.provision.LockedNodeList;
@@ -22,14 +26,16 @@ import com.yahoo.vespa.hosted.provision.node.IP;
 import com.yahoo.vespa.hosted.provision.provisioning.FlavorConfigBuilder;
 import com.yahoo.vespa.hosted.provision.testutils.MockNameResolver;
 import com.yahoo.vespa.orchestrator.Orchestrator;
+import com.yahoo.vespa.orchestrator.status.HostInfo;
 import com.yahoo.vespa.orchestrator.status.HostStatus;
 import com.yahoo.vespa.service.monitor.ServiceModel;
 import com.yahoo.vespa.service.monitor.ServiceMonitor;
+import org.junit.Before;
 import org.junit.Test;
 
 import java.time.Clock;
 import java.time.Duration;
-import java.util.ArrayList;
+import java.time.Instant;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -37,6 +43,8 @@ import java.util.Optional;
 import java.util.Set;
 
 import static org.junit.Assert.assertEquals;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
@@ -46,6 +54,24 @@ import static org.mockito.Mockito.when;
  */
 public class MetricsReporterTest {
 
+    private final ServiceMonitor serviceMonitor = mock(ServiceMonitor.class);
+    private final ApplicationInstanceReference reference = mock(ApplicationInstanceReference.class);
+
+    @Before
+    public void setUp() {
+        // On the serviceModel returned by serviceMonitor.getServiceModelSnapshot(),
+        // 2 methods should be used by MetricsReporter:
+        //  - getServiceInstancesByHostName() -> empty Map
+        //  - getApplication() which is mapped to a dummy ApplicationInstanceReference and
+        //    used for lookup.
+        ServiceModel serviceModel = mock(ServiceModel.class);
+        when(serviceMonitor.getServiceModelSnapshot()).thenReturn(serviceModel);
+        when(serviceModel.getServiceInstancesByHostName()).thenReturn(Map.of());
+        ApplicationInstance applicationInstance = mock(ApplicationInstance.class);
+        when(serviceModel.getApplication(any())).thenReturn(Optional.of(applicationInstance));
+        when(applicationInstance.reference()).thenReturn(reference);
+    }
+
     @Test
     public void test_registered_metric() {
         NodeFlavors nodeFlavors = FlavorConfigBuilder.createDummies("default");
@@ -55,9 +81,9 @@ public class MetricsReporterTest {
                                                            DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"),
                                                            true);
         Node node = nodeRepository.createNode("openStackId", "hostname", Optional.empty(), nodeFlavors.getFlavorOrThrow("default"), NodeType.tenant);
-        nodeRepository.addNodes(List.of(node));
+        nodeRepository.addNodes(List.of(node), Agent.system);
         Node hostNode = nodeRepository.createNode("openStackId2", "parent", Optional.empty(), nodeFlavors.getFlavorOrThrow("default"), NodeType.proxy);
-        nodeRepository.addNodes(List.of(hostNode));
+        nodeRepository.addNodes(List.of(hostNode), Agent.system);
 
         Map<String, Number> expectedMetrics = new HashMap<>();
         expectedMetrics.put("hostedVespa.provisionedHosts", 1);
@@ -68,6 +94,7 @@ public class MetricsReporterTest {
         expectedMetrics.put("hostedVespa.inactiveHosts", 0);
         expectedMetrics.put("hostedVespa.dirtyHosts", 0);
         expectedMetrics.put("hostedVespa.failedHosts", 0);
+        expectedMetrics.put("hostedVespa.deprovisionedHosts", 0);
         expectedMetrics.put("hostedVespa.pendingRedeployments", 42);
         expectedMetrics.put("hostedVespa.docker.totalCapacityDisk", 0.0);
         expectedMetrics.put("hostedVespa.docker.totalCapacityMem", 0.0);
@@ -82,15 +109,16 @@ public class MetricsReporterTest {
         expectedMetrics.put("wantToRetire", 0);
         expectedMetrics.put("wantToDeprovision", 0);
         expectedMetrics.put("failReport", 0);
-        expectedMetrics.put("allowedToBeDown", 0);
+        expectedMetrics.put("allowedToBeDown", 1);
+        expectedMetrics.put("suspended", 1);
+        expectedMetrics.put("suspendedSeconds", 123L);
         expectedMetrics.put("numberOfServices", 0L);
 
+        ManualClock clock = new ManualClock(Instant.ofEpochSecond(124));
+
         Orchestrator orchestrator = mock(Orchestrator.class);
-        ServiceMonitor serviceMonitor = mock(ServiceMonitor.class);
-        when(orchestrator.getNodeStatuses()).thenReturn(hostName -> Optional.of(HostStatus.NO_REMARKS));
-        ServiceModel serviceModel = mock(ServiceModel.class);
-        when(serviceMonitor.getServiceModelSnapshot()).thenReturn(serviceModel);
-        when(serviceModel.getServiceInstancesByHostName()).thenReturn(Map.of());
+        when(orchestrator.getHostInfo(eq(reference), any())).thenReturn(
+                HostInfo.createSuspended(HostStatus.ALLOWED_TO_BE_DOWN, Instant.ofEpochSecond(1)));
 
         TestMetric metric = new TestMetric();
         MetricsReporter metricsReporter = new MetricsReporter(
@@ -99,8 +127,8 @@ public class MetricsReporterTest {
                 orchestrator,
                 serviceMonitor,
                 () -> 42,
-                Duration.ofMinutes(1)
-        );
+                Duration.ofMinutes(1),
+                clock);
         metricsReporter.maintain();
 
         assertEquals(expectedMetrics, metric.values);
@@ -120,7 +148,7 @@ public class MetricsReporterTest {
 
         Node dockerHost = Node.create("openStackId1", new IP.Config(Set.of("::1"), ipAddressPool), "dockerHost",
                                       Optional.empty(), Optional.empty(), nodeFlavors.getFlavorOrThrow("host"), Optional.empty(), NodeType.host);
-        nodeRepository.addNodes(List.of(dockerHost));
+        nodeRepository.addNodes(List.of(dockerHost), Agent.system);
         nodeRepository.dirtyRecursively("dockerHost", Agent.system, getClass().getSimpleName());
         nodeRepository.setReady("dockerHost", Agent.system, getClass().getSimpleName());
 
@@ -134,22 +162,23 @@ public class MetricsReporterTest {
         container2 = container2.with(allocation(Optional.of("app2"), container2).get());
         nodeRepository.addDockerNodes(new LockedNodeList(List.of(container2), nodeRepository.lockUnallocated()));
 
+        NestedTransaction transaction = new NestedTransaction();
+        nodeRepository.activate(nodeRepository.getNodes(NodeType.host), transaction);
+        transaction.commit();
+
         Orchestrator orchestrator = mock(Orchestrator.class);
-        ServiceMonitor serviceMonitor = mock(ServiceMonitor.class);
-        when(orchestrator.getNodeStatuses()).thenReturn(hostName -> Optional.of(HostStatus.NO_REMARKS));
-        ServiceModel serviceModel = mock(ServiceModel.class);
-        when(serviceMonitor.getServiceModelSnapshot()).thenReturn(serviceModel);
-        when(serviceModel.getServiceInstancesByHostName()).thenReturn(Map.of());
+        when(orchestrator.getHostInfo(eq(reference), any())).thenReturn(HostInfo.createNoRemarks());
 
         TestMetric metric = new TestMetric();
+        ManualClock clock = new ManualClock();
         MetricsReporter metricsReporter = new MetricsReporter(
                 nodeRepository,
                 metric,
                 orchestrator,
                 serviceMonitor,
                 () -> 42,
-                Duration.ofMinutes(1)
-        );
+                Duration.ofMinutes(1),
+                clock);
         metricsReporter.maintain();
 
         assertEquals(0, metric.values.get("hostedVespa.readyHosts")); // Only tenants counts
@@ -184,7 +213,7 @@ public class MetricsReporterTest {
     private Optional<Allocation> allocation(Optional<String> tenant, Node owner) {
         if (tenant.isPresent()) {
             Allocation allocation = new Allocation(app(tenant.get()),
-                                                   ClusterMembership.from("container/id1/0/3", new Version()),
+                                                   ClusterMembership.from("container/id1/0/3", new Version(), Optional.empty()),
                                                    owner.flavor().resources(),
                                                    Generation.initial(),
                                                    false);
@@ -193,70 +222,4 @@ public class MetricsReporterTest {
         return Optional.empty();
     }
 
-    public static class TestMetric implements Metric {
-
-        public Map<String, Number> values = new HashMap<>();
-        public Map<String, List<Context>> context = new HashMap<>();
-
-        @Override
-        public void set(String key, Number val, Context ctx) {
-            values.put(key, val);
-            if (ctx != null) {
-                //Create one context pr value added - copy the context to not have side effects
-                TestContext kontekst = (TestContext)createContext(((TestContext) ctx).properties);
-                if (!context.containsKey(key)) {
-                    context.put(key, new ArrayList<>());
-                }
-                kontekst.setValue(val);
-                context.get(key).add(kontekst);
-            }
-        }
-
-        @Override
-        public void add(String key, Number val, Context ctx) {
-            values.put(key, val);
-            if (ctx != null) {
-                //Create one context pr value added - copy the context to not have side effects
-                TestContext copy = (TestContext) createContext(((TestContext) ctx).properties);
-                if (!context.containsKey(key)) {
-                    context.put(key, new ArrayList<>());
-                }
-                copy.setValue(val);
-                context.get(key).add(copy);
-            }
-        }
-
-        @Override
-        public Context createContext(Map<String, ?> properties) {
-            return new TestContext(properties);
-        }
-
-        double sumDoubleValues(String key, Context sumContext) {
-            double sum = 0.0;
-            for(Context c : context.get(key)) {
-                TestContext tc = (TestContext) c;
-                if (tc.value instanceof Double && tc.properties.equals(((TestContext) sumContext).properties)) {
-                    sum += (double) tc.value;
-                }
-            }
-            return sum;
-        }
-
-        /**
-         * Context where the propertymap is not shared - but unique to each value.
-         */
-        private static class TestContext implements Context{
-            Number value;
-            Map<String, ?> properties;
-
-            public TestContext(Map<String, ?> properties) {
-                this.properties = properties;
-            }
-
-            public void setValue(Number value) {
-                this.value = value;
-            }
-        }
-    }
-
 }
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java
index 5872a78e1e2..ab97de80418 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java
@@ -1,9 +1,9 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.provision.maintenance;
 
-import com.yahoo.component.Version;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.Capacity;
+import com.yahoo.config.provision.ClusterResources;
 import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.config.provision.DockerImage;
 import com.yahoo.config.provision.Environment;
@@ -65,7 +65,7 @@ public class NodeFailTester {
     public NodeFailer failer;
     public ServiceMonitorStub serviceMonitor;
     public MockDeployer deployer;
-    public MetricsReporterTest.TestMetric metric;
+    public TestMetric metric;
     private final TestHostLivenessTracker hostLivenessTracker;
     private final Orchestrator orchestrator;
     private final NodeRepositoryProvisioner provisioner;
@@ -75,7 +75,7 @@ public class NodeFailTester {
         clock = new ManualClock();
         curator = new MockCurator();
         nodeRepository = new NodeRepository(nodeFlavors, curator, clock, zone, new MockNameResolver().mockAnyLookup(),
-                DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), true);
+                                            DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), true);
         provisioner = new NodeRepositoryProvisioner(nodeRepository, zone, new MockProvisionServiceProvider(), new InMemoryFlagSource());
         hostLivenessTracker = new TestHostLivenessTracker(clock);
         orchestrator = new OrchestratorMock();
@@ -88,22 +88,22 @@ public class NodeFailTester {
         tester.createHostNodes(3);
 
         // Create applications
-        ClusterSpec clusterApp1 = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test"), Version.fromString("6.42"), false);
-        ClusterSpec clusterApp2 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test"), Version.fromString("6.42"), false);
-        Capacity capacity1 = Capacity.fromCount(5, nodeResources, false, true);
-        Capacity capacity2 = Capacity.fromCount(7, nodeResources, false, true);
+        ClusterSpec clusterApp1 = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test")).vespaVersion("6.42").build();
+        ClusterSpec clusterApp2 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test")).vespaVersion("6.42").build();
+        Capacity capacity1 = Capacity.from(new ClusterResources(5, 1, nodeResources), false, true);
+        Capacity capacity2 = Capacity.from(new ClusterResources(7, 1, nodeResources), false, true);
 
         tester.activate(app1, clusterApp1, capacity1);
         tester.activate(app2, clusterApp2, capacity2);
-        assertEquals(capacity1.nodeCount(), tester.nodeRepository.getNodes(app1, Node.State.active).size());
-        assertEquals(capacity2.nodeCount(), tester.nodeRepository.getNodes(app2, Node.State.active).size());
+        assertEquals(capacity1.minResources().nodes(), tester.nodeRepository.getNodes(app1, Node.State.active).size());
+        assertEquals(capacity2.minResources().nodes(), tester.nodeRepository.getNodes(app2, Node.State.active).size());
 
         Map<ApplicationId, MockDeployer.ApplicationContext> apps = Map.of(
-                app1, new MockDeployer.ApplicationContext(app1, clusterApp1, capacity1, 1),
-                app2, new MockDeployer.ApplicationContext(app2, clusterApp2, capacity2, 1));
+                app1, new MockDeployer.ApplicationContext(app1, clusterApp1, capacity1),
+                app2, new MockDeployer.ApplicationContext(app2, clusterApp2, capacity2));
         tester.deployer = new MockDeployer(tester.provisioner, tester.clock(), apps);
         tester.serviceMonitor = new ServiceMonitorStub(apps, tester.nodeRepository);
-        tester.metric = new MetricsReporterTest.TestMetric();
+        tester.metric = new TestMetric();
         tester.failer = tester.createFailer();
         return tester;
     }
@@ -119,27 +119,27 @@ public class NodeFailTester {
         }
 
         // Create applications
-        ClusterSpec clusterNodeAdminApp = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("node-admin"), Version.fromString("6.42"), false);
-        ClusterSpec clusterApp1 = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test"), Version.fromString("6.75.0"), false);
-        ClusterSpec clusterApp2 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test"), Version.fromString("6.75.0"), false);
+        ClusterSpec clusterNodeAdminApp = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("node-admin")).vespaVersion("6.42").build();
+        ClusterSpec clusterApp1 = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test")).vespaVersion("6.75.0").build();
+        ClusterSpec clusterApp2 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test")).vespaVersion("6.75.0").build();
         Capacity allHosts = Capacity.fromRequiredNodeType(NodeType.host);
-        Capacity capacity1 = Capacity.fromCount(3, new NodeResources(1, 4, 10, 0.3), false, true);
-        Capacity capacity2 = Capacity.fromCount(5, new NodeResources(1, 4, 10, 0.3), false, true);
+        Capacity capacity1 = Capacity.from(new ClusterResources(3, 1, new NodeResources(1, 4, 10, 0.3)), false, true);
+        Capacity capacity2 = Capacity.from(new ClusterResources(5, 1, new NodeResources(1, 4, 10, 0.3)), false, true);
         tester.activate(tenantHostApp, clusterNodeAdminApp, allHosts);
         tester.activate(app1, clusterApp1, capacity1);
         tester.activate(app2, clusterApp2, capacity2);
         assertEquals(Set.of(tester.nodeRepository.getNodes(NodeType.host)),
                 Set.of(tester.nodeRepository.getNodes(tenantHostApp, Node.State.active)));
-        assertEquals(capacity1.nodeCount(), tester.nodeRepository.getNodes(app1, Node.State.active).size());
-        assertEquals(capacity2.nodeCount(), tester.nodeRepository.getNodes(app2, Node.State.active).size());
+        assertEquals(capacity1.minResources().nodes(), tester.nodeRepository.getNodes(app1, Node.State.active).size());
+        assertEquals(capacity2.minResources().nodes(), tester.nodeRepository.getNodes(app2, Node.State.active).size());
 
         Map<ApplicationId, MockDeployer.ApplicationContext> apps = Map.of(
-                tenantHostApp, new MockDeployer.ApplicationContext(tenantHostApp, clusterNodeAdminApp, allHosts, 1),
-                app1, new MockDeployer.ApplicationContext(app1, clusterApp1, capacity1, 1),
-                app2, new MockDeployer.ApplicationContext(app2, clusterApp2, capacity2, 1));
+                tenantHostApp, new MockDeployer.ApplicationContext(tenantHostApp, clusterNodeAdminApp, allHosts),
+                app1, new MockDeployer.ApplicationContext(app1, clusterApp1, capacity1),
+                app2, new MockDeployer.ApplicationContext(app2, clusterApp2, capacity2));
         tester.deployer = new MockDeployer(tester.provisioner, tester.clock(), apps);
         tester.serviceMonitor = new ServiceMonitorStub(apps, tester.nodeRepository);
-        tester.metric = new MetricsReporterTest.TestMetric();
+        tester.metric = new TestMetric();
         tester.failer = tester.createFailer();
         return tester;
     }
@@ -150,18 +150,15 @@ public class NodeFailTester {
 
         // Create application
         Capacity allNodes = Capacity.fromRequiredNodeType(nodeType);
-        ClusterSpec clusterApp1 = ClusterSpec.request(ClusterSpec.Type.container,
-                                                      ClusterSpec.Id.from("test"),
-                                                      Version.fromString("6.42"),
-                                                      false);
+        ClusterSpec clusterApp1 = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test")).vespaVersion("6.42").build();
         tester.activate(app1, clusterApp1, allNodes);
         assertEquals(count, tester.nodeRepository.getNodes(nodeType, Node.State.active).size());
 
         Map<ApplicationId, MockDeployer.ApplicationContext> apps = Map.of(
-                app1, new MockDeployer.ApplicationContext(app1, clusterApp1, allNodes, 1));
+                app1, new MockDeployer.ApplicationContext(app1, clusterApp1, allNodes));
         tester.deployer = new MockDeployer(tester.provisioner, tester.clock(), apps);
         tester.serviceMonitor = new ServiceMonitorStub(apps, tester.nodeRepository);
-        tester.metric = new MetricsReporterTest.TestMetric();
+        tester.metric = new TestMetric();
         tester.failer = tester.createFailer();
         return tester;
     }
@@ -170,7 +167,7 @@ public class NodeFailTester {
         NodeFailTester tester = new NodeFailTester();
         tester.deployer = new MockDeployer(tester.provisioner, tester.clock(), Map.of());
         tester.serviceMonitor = new ServiceMonitorStub(Map.of(), tester.nodeRepository);
-        tester.metric = new MetricsReporterTest.TestMetric();
+        tester.metric = new TestMetric();
         tester.failer = tester.createFailer();
         return tester;
     }
@@ -241,7 +238,7 @@ public class NodeFailTester {
         for (int i = startIndex; i < startIndex + count; i++)
             nodes.add(nodeRepository.createNode("node" + i, "host" + i, parentHostname, flavor, nodeType));
 
-        nodes = nodeRepository.addNodes(nodes);
+        nodes = nodeRepository.addNodes(nodes, Agent.system);
         nodes = nodeRepository.setDirty(nodes, Agent.system, getClass().getSimpleName());
         return nodeRepository.setReady(nodes, Agent.system, getClass().getSimpleName());
     }
@@ -250,13 +247,13 @@ public class NodeFailTester {
         List<Node> nodes = new ArrayList<>(count);
         for (int i = 0; i < count; i++)
             nodes.add(nodeRepository.createNode("parent" + i, "parent" + i, Optional.empty(), nodeFlavors.getFlavorOrThrow("default"), NodeType.host));
-        nodes = nodeRepository.addNodes(nodes);
+        nodes = nodeRepository.addNodes(nodes, Agent.system);
         nodes = nodeRepository.setDirty(nodes, Agent.system, getClass().getSimpleName());
         return nodeRepository.setReady(nodes, Agent.system, getClass().getSimpleName());
     }
 
     private void activate(ApplicationId applicationId, ClusterSpec cluster, Capacity capacity) {
-        List<HostSpec> hosts = provisioner.prepare(applicationId, cluster, capacity, 1, null);
+        List<HostSpec> hosts = provisioner.prepare(applicationId, cluster, capacity, null);
         NestedTransaction transaction = new NestedTransaction().add(new CuratorTransaction(curator));
         provisioner.activate(transaction, applicationId, hosts);
         transaction.commit();
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainerTest.java
index 50c00c730bb..eb2a1d4db68 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainerTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/OperatorChangeApplicationMaintainerTest.java
@@ -1,10 +1,10 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.provision.maintenance;
 
-import com.yahoo.component.Version;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.ApplicationName;
 import com.yahoo.config.provision.Capacity;
+import com.yahoo.config.provision.ClusterResources;
 import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.config.provision.DockerImage;
 import com.yahoo.config.provision.Environment;
@@ -66,7 +66,10 @@ public class OperatorChangeApplicationMaintainerTest {
         // Create applications
         fixture.activate();
         assertEquals("Initial applications are deployed", 3, fixture.deployer.redeployments);
-        OperatorChangeApplicationMaintainer maintainer = new OperatorChangeApplicationMaintainer(fixture.deployer, nodeRepository, Duration.ofMinutes(1));
+        OperatorChangeApplicationMaintainer maintainer = new OperatorChangeApplicationMaintainer(fixture.deployer,
+                                                                                                 new TestMetric(),
+                                                                                                 nodeRepository,
+                                                                                                 Duration.ofMinutes(1));
         
         clock.advance(Duration.ofMinutes(2));
         maintainer.maintain();
@@ -102,7 +105,7 @@ public class OperatorChangeApplicationMaintainerTest {
         List<Node> nodes = new ArrayList<>(count);
         for (int i = 0; i < count; i++)
             nodes.add(nodeRepository.createNode("node" + i, "host" + i, Optional.empty(), flavor, NodeType.tenant));
-        nodes = nodeRepository.addNodes(nodes);
+        nodes = nodeRepository.addNodes(nodes, Agent.system);
         nodes = nodeRepository.setDirty(nodes, Agent.system, getClass().getSimpleName());
         nodeRepository.setReady(nodes, Agent.system, getClass().getSimpleName());
     }
@@ -111,7 +114,7 @@ public class OperatorChangeApplicationMaintainerTest {
         List<Node> nodes = new ArrayList<>(count);
         for (int i = 0; i < count; i++)
             nodes.add(nodeRepository.createNode("hostNode" + i, "realHost" + i, Optional.empty(), nodeFlavors.getFlavorOrThrow("default"), NodeType.host));
-        nodes = nodeRepository.addNodes(nodes);
+        nodes = nodeRepository.addNodes(nodes, Agent.system);
         nodes = nodeRepository.setDirty(nodes, Agent.system, getClass().getSimpleName());
         nodeRepository.setReady(nodes, Agent.system, getClass().getSimpleName());
     }
@@ -120,7 +123,7 @@ public class OperatorChangeApplicationMaintainerTest {
         List<Node> nodes = new ArrayList<>(count);
         for (int i = 0; i < count; i++)
             nodes.add(nodeRepository.createNode("proxyNode" + i, "proxyHost" + i, Optional.empty(), nodeFlavors.getFlavorOrThrow("default"), NodeType.proxy));
-        nodes = nodeRepository.addNodes(nodes);
+        nodes = nodeRepository.addNodes(nodes, Agent.system);
         nodes = nodeRepository.setDirty(nodes, Agent.system, getClass().getSimpleName());
         nodeRepository.setReady(nodes, Agent.system, getClass().getSimpleName());
     }
@@ -134,9 +137,9 @@ public class OperatorChangeApplicationMaintainerTest {
         final ApplicationId app1 = ApplicationId.from(TenantName.from("foo1"), ApplicationName.from("bar"), InstanceName.from("fuz"));
         final ApplicationId app2 = ApplicationId.from(TenantName.from("foo2"), ApplicationName.from("bar"), InstanceName.from("fuz"));
         final ApplicationId app3 = ApplicationId.from(TenantName.from("vespa-hosted"), ApplicationName.from("routing"), InstanceName.from("default"));
-        final ClusterSpec clusterApp1 = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test"), Version.fromString("6.42"), false);
-        final ClusterSpec clusterApp2 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test"), Version.fromString("6.42"), false);
-        final ClusterSpec clusterApp3 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("routing"), Version.fromString("6.42"), false);
+        final ClusterSpec clusterApp1 = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test")).vespaVersion("6.42").build();
+        final ClusterSpec clusterApp2 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test")).vespaVersion("6.42").build();
+        final ClusterSpec clusterApp3 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("routing")).vespaVersion("6.42").build();
         final int wantedNodesApp1 = 5;
         final int wantedNodesApp2 = 7;
         final int wantedNodesApp3 = 2;
@@ -149,9 +152,9 @@ public class OperatorChangeApplicationMaintainerTest {
                                                                                   new InMemoryFlagSource());
 
             Map<ApplicationId, MockDeployer.ApplicationContext> apps = Map.of(
-                    app1, new MockDeployer.ApplicationContext(app1, clusterApp1, Capacity.fromCount(wantedNodesApp1, nodeResources), 1),
-                    app2, new MockDeployer.ApplicationContext(app2, clusterApp2, Capacity.fromCount(wantedNodesApp2, nodeResources), 1),
-                    app3, new MockDeployer.ApplicationContext(app3, clusterApp3, Capacity.fromRequiredNodeType(NodeType.proxy), 0)) ;
+                    app1, new MockDeployer.ApplicationContext(app1, clusterApp1, Capacity.from(new ClusterResources(wantedNodesApp1, 1, nodeResources))),
+                    app2, new MockDeployer.ApplicationContext(app2, clusterApp2, Capacity.from(new ClusterResources(wantedNodesApp2, 1, nodeResources))),
+                    app3, new MockDeployer.ApplicationContext(app3, clusterApp3, Capacity.fromRequiredNodeType(NodeType.proxy))) ;
             this.deployer = new MockDeployer(provisioner, nodeRepository.clock(), apps);
         }
 
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java
index b1c3b23016c..63a22cc3029 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/PeriodicApplicationMaintainerTest.java
@@ -1,10 +1,10 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.provision.maintenance;
 
-import com.yahoo.component.Version;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.ApplicationName;
 import com.yahoo.config.provision.Capacity;
+import com.yahoo.config.provision.ClusterResources;
 import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.config.provision.Deployer;
 import com.yahoo.config.provision.DockerImage;
@@ -222,7 +222,7 @@ public class PeriodicApplicationMaintainerTest {
         List<Node> nodes = new ArrayList<>(count);
         for (int i = 0; i < count; i++)
             nodes.add(nodeRepository.createNode("node" + i, "host" + i, Optional.empty(), flavor, NodeType.tenant));
-        nodes = nodeRepository.addNodes(nodes);
+        nodes = nodeRepository.addNodes(nodes, Agent.system);
         nodes = nodeRepository.setDirty(nodes, Agent.system, getClass().getSimpleName());
         nodeRepository.setReady(nodes, Agent.system, getClass().getSimpleName());
     }
@@ -231,7 +231,7 @@ public class PeriodicApplicationMaintainerTest {
         List<Node> nodes = new ArrayList<>(count);
         for (int i = 0; i < count; i++)
             nodes.add(nodeRepository.createNode("hostNode" + i, "realHost" + i, Optional.empty(), nodeFlavors.getFlavorOrThrow("default"), NodeType.host));
-        nodes = nodeRepository.addNodes(nodes);
+        nodes = nodeRepository.addNodes(nodes, Agent.system);
         nodes = nodeRepository.setDirty(nodes, Agent.system, getClass().getSimpleName());
         nodeRepository.setReady(nodes, Agent.system, getClass().getSimpleName());
     }
@@ -244,8 +244,8 @@ public class PeriodicApplicationMaintainerTest {
         final NodeResources nodeResources = new NodeResources(2, 8, 50, 1);
         final ApplicationId app1 = ApplicationId.from(TenantName.from("foo1"), ApplicationName.from("bar"), InstanceName.from("fuz"));
         final ApplicationId app2 = ApplicationId.from(TenantName.from("foo2"), ApplicationName.from("bar"), InstanceName.from("fuz"));
-        final ClusterSpec clusterApp1 = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test"), Version.fromString("6.42"), false);
-        final ClusterSpec clusterApp2 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test"), Version.fromString("6.42"), false);
+        final ClusterSpec clusterApp1 = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test")).vespaVersion("6.42").build();
+        final ClusterSpec clusterApp2 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test")).vespaVersion("6.42").build();
         final int wantedNodesApp1 = 5;
         final int wantedNodesApp2 = 7;
 
@@ -259,8 +259,8 @@ public class PeriodicApplicationMaintainerTest {
                                                                                   new InMemoryFlagSource());
 
             Map<ApplicationId, MockDeployer.ApplicationContext> apps = Map.of(
-                    app1, new MockDeployer.ApplicationContext(app1, clusterApp1, Capacity.fromCount(wantedNodesApp1, nodeResources), 1),
-                    app2, new MockDeployer.ApplicationContext(app2, clusterApp2, Capacity.fromCount(wantedNodesApp2, nodeResources), 1));
+                    app1, new MockDeployer.ApplicationContext(app1, clusterApp1, Capacity.from(new ClusterResources(wantedNodesApp1, 1, nodeResources))),
+                    app2, new MockDeployer.ApplicationContext(app2, clusterApp2, Capacity.from(new ClusterResources(wantedNodesApp2, 1, nodeResources))));
             this.deployer = new MockDeployer(provisioner, nodeRepository.clock(), apps);
             this.maintainer = new TestablePeriodicApplicationMaintainer(deployer, nodeRepository, Duration.ofDays(1), // Long duration to prevent scheduled runs during test
                                                                         Duration.ofMinutes(30));
@@ -303,7 +303,7 @@ public class PeriodicApplicationMaintainerTest {
 
         TestablePeriodicApplicationMaintainer(Deployer deployer, NodeRepository nodeRepository, Duration interval,
                                               Duration minTimeBetweenRedeployments) {
-            super(deployer, nodeRepository, interval, minTimeBetweenRedeployments);
+            super(deployer, new TestMetric(), nodeRepository, interval, minTimeBetweenRedeployments);
         }
 
         @Override
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RebalancerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RebalancerTest.java
index d1a330a3bd6..d25ae234f35 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RebalancerTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RebalancerTest.java
@@ -1,9 +1,9 @@
 // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.provision.maintenance;
 
-import com.yahoo.component.Version;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.Capacity;
+import com.yahoo.config.provision.ClusterResources;
 import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.config.provision.Environment;
 import com.yahoo.config.provision.Flavor;
@@ -17,7 +17,6 @@ import com.yahoo.transaction.NestedTransaction;
 import com.yahoo.vespa.hosted.provision.Node;
 import com.yahoo.vespa.hosted.provision.node.Agent;
 import com.yahoo.vespa.hosted.provision.provisioning.FlavorConfigBuilder;
-import com.yahoo.vespa.hosted.provision.provisioning.HostResourcesCalculator;
 import com.yahoo.vespa.hosted.provision.provisioning.ProvisioningTester;
 import com.yahoo.vespa.hosted.provision.testutils.MockDeployer;
 import org.junit.Test;
@@ -46,16 +45,16 @@ public class RebalancerTest {
         NodeResources memResources = new NodeResources(4, 9, 1, 0.1);
 
         ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.perf, RegionName.from("us-east"))).flavorsConfig(flavorsConfig()).build();
-        MetricsReporterTest.TestMetric metric = new MetricsReporterTest.TestMetric();
+        TestMetric metric = new TestMetric();
 
         Map<ApplicationId, MockDeployer.ApplicationContext> apps = Map.of(
-                cpuApp, new MockDeployer.ApplicationContext(cpuApp, clusterSpec("c"), Capacity.fromCount(1, cpuResources), 1),
-                memApp, new MockDeployer.ApplicationContext(memApp, clusterSpec("c"), Capacity.fromCount(1, memResources), 1));
+                cpuApp, new MockDeployer.ApplicationContext(cpuApp, clusterSpec("c"), Capacity.from(new ClusterResources(1, 1, cpuResources))),
+                memApp, new MockDeployer.ApplicationContext(memApp, clusterSpec("c"), Capacity.from(new ClusterResources(1, 1, memResources))));
         MockDeployer deployer = new MockDeployer(tester.provisioner(), tester.clock(), apps);
 
         Rebalancer rebalancer = new Rebalancer(deployer,
                                                tester.nodeRepository(),
-                                               new IdentityHostResourcesCalculator(),
+                                               tester.identityHostResourcesCalculator(),
                                                Optional.empty(),
                                                metric,
                                                tester.clock(),
@@ -129,7 +128,7 @@ public class RebalancerTest {
     }
 
     private ClusterSpec clusterSpec(String clusterId) {
-        return ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from(clusterId), Version.fromString("6.42"), false);
+        return ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from(clusterId)).vespaVersion("6.42").build();
     }
 
     private ApplicationId makeApplicationId(String tenant, String appName) {
@@ -149,13 +148,4 @@ public class RebalancerTest {
         return b.build();
     }
 
-    private static class IdentityHostResourcesCalculator implements HostResourcesCalculator {
-
-        @Override
-        public NodeResources availableCapacityOf(NodeResources hostResources) {
-            return hostResources;
-        }
-
-    }
-
 }
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirerTest.java
index 11ee6637720..0fd967cad1b 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirerTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ReservationExpirerTest.java
@@ -1,9 +1,9 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.provision.maintenance;
 
-import com.yahoo.component.Version;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.Capacity;
+import com.yahoo.config.provision.ClusterResources;
 import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.config.provision.DockerImage;
 import com.yahoo.config.provision.Flavor;
@@ -54,15 +54,15 @@ public class ReservationExpirerTest {
         nodes.add(nodeRepository.createNode(UUID.randomUUID().toString(), UUID.randomUUID().toString(), Optional.empty(), new Flavor(new NodeResources(2, 8, 50, 1)), NodeType.tenant));
         nodes.add(nodeRepository.createNode(UUID.randomUUID().toString(), UUID.randomUUID().toString(), Optional.empty(), new Flavor(new NodeResources(2, 8, 50, 1)), NodeType.tenant));
         nodes.add(nodeRepository.createNode(UUID.randomUUID().toString(), UUID.randomUUID().toString(), Optional.empty(), flavors.getFlavorOrThrow("default"), NodeType.host));
-        nodes = nodeRepository.addNodes(nodes);
+        nodes = nodeRepository.addNodes(nodes, Agent.system);
         nodes = nodeRepository.setDirty(nodes, Agent.system, getClass().getSimpleName());
 
         // Reserve 2 nodes
         assertEquals(2, nodeRepository.getNodes(NodeType.tenant, Node.State.dirty).size());
         nodeRepository.setReady(nodes, Agent.system, getClass().getSimpleName());
         ApplicationId applicationId = new ApplicationId.Builder().tenant("foo").applicationName("bar").instanceName("fuz").build();
-        ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test"), Version.fromString("6.42"), false);
-        provisioner.prepare(applicationId, cluster, Capacity.fromCount(2, new NodeResources(2, 8, 50, 1)), 1, null);
+        ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test")).vespaVersion("6.42").build();
+        provisioner.prepare(applicationId, cluster, Capacity.from(new ClusterResources(2, 1, new NodeResources(2, 8, 50, 1))), null);
         assertEquals(2, nodeRepository.getNodes(NodeType.tenant, Node.State.reserved).size());
 
         // Reservation times out
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java
index 67af2df36e7..7ece8cba65e 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java
@@ -1,10 +1,10 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.provision.maintenance;
 
-import com.yahoo.component.Version;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.ApplicationName;
 import com.yahoo.config.provision.Capacity;
+import com.yahoo.config.provision.ClusterResources;
 import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.config.provision.Deployer;
 import com.yahoo.config.provision.DockerImage;
@@ -64,8 +64,8 @@ public class RetiredExpirerTest {
     private final Zone zone = new Zone(Environment.prod, RegionName.from("us-east"));
     private final NodeFlavors nodeFlavors = FlavorConfigBuilder.createDummies("default");
     private final NodeRepository nodeRepository = new NodeRepository(nodeFlavors, curator, clock, zone,
-            new MockNameResolver().mockAnyLookup(),
-            DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), true);
+                                                                     new MockNameResolver().mockAnyLookup(),
+                                                                     DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), true);
     private final NodeRepositoryProvisioner provisioner = new NodeRepositoryProvisioner(nodeRepository, zone, new MockProvisionServiceProvider(), new InMemoryFlagSource());
     private final Orchestrator orchestrator = mock(Orchestrator.class);
 
@@ -86,7 +86,7 @@ public class RetiredExpirerTest {
 
         // Allocate content cluster of sizes 7 -> 2 -> 3:
         // Should end up with 3 nodes in the cluster (one previously retired), and 4 retired
-        ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test"), Version.fromString("6.42"), false);
+        ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test")).vespaVersion("6.42").build();
         int wantedNodes;
         activate(applicationId, cluster, wantedNodes=7, 1, provisioner);
         activate(applicationId, cluster, wantedNodes=2, 1, provisioner);
@@ -99,7 +99,9 @@ public class RetiredExpirerTest {
         MockDeployer deployer =
             new MockDeployer(provisioner,
                              clock,
-                             Collections.singletonMap(applicationId, new MockDeployer.ApplicationContext(applicationId, cluster, Capacity.fromCount(wantedNodes, nodeResources), 1)));
+                             Collections.singletonMap(applicationId, new MockDeployer.ApplicationContext(applicationId,
+                                                                                                         cluster,
+                                                                                                         Capacity.from(new ClusterResources(wantedNodes, 1, nodeResources)))));
         createRetiredExpirer(deployer).run();
         assertEquals(3, nodeRepository.getNodes(applicationId, Node.State.active).size());
         assertEquals(4, nodeRepository.getNodes(applicationId, Node.State.inactive).size());
@@ -117,7 +119,7 @@ public class RetiredExpirerTest {
 
         ApplicationId applicationId = ApplicationId.from(TenantName.from("foo"), ApplicationName.from("bar"), InstanceName.from("fuz"));
 
-        ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test"), Version.fromString("6.42"), false);
+        ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test")).vespaVersion("6.42").build();
         activate(applicationId, cluster, 8, 8, provisioner);
         activate(applicationId, cluster, 2, 2, provisioner);
         assertEquals(8, nodeRepository.getNodes(applicationId, Node.State.active).size());
@@ -128,7 +130,9 @@ public class RetiredExpirerTest {
         MockDeployer deployer =
             new MockDeployer(provisioner,
                              clock,
-                             Collections.singletonMap(applicationId, new MockDeployer.ApplicationContext(applicationId, cluster, Capacity.fromCount(2, nodeResources), 1)));
+                             Collections.singletonMap(applicationId, new MockDeployer.ApplicationContext(applicationId,
+                                                                                                         cluster,
+                                                                                                         Capacity.from(new ClusterResources(2, 1, nodeResources)))));
         createRetiredExpirer(deployer).run();
         assertEquals(2, nodeRepository.getNodes(applicationId, Node.State.active).size());
         assertEquals(6, nodeRepository.getNodes(applicationId, Node.State.inactive).size());
@@ -148,7 +152,7 @@ public class RetiredExpirerTest {
 
         // Allocate content cluster of sizes 7 -> 2 -> 3:
         // Should end up with 3 nodes in the cluster (one previously retired), and 4 retired
-        ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test"), Version.fromString("6.42"), false);
+        ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test")).vespaVersion("6.42").build();
         int wantedNodes;
         activate(applicationId, cluster, wantedNodes=7, 1, provisioner);
         activate(applicationId, cluster, wantedNodes=2, 1, provisioner);
@@ -162,7 +166,9 @@ public class RetiredExpirerTest {
                                  clock,
                                  Collections.singletonMap(
                                      applicationId,
-                                     new MockDeployer.ApplicationContext(applicationId, cluster, Capacity.fromCount(wantedNodes, nodeResources), 1)));
+                                     new MockDeployer.ApplicationContext(applicationId,
+                                                                         cluster,
+                                                                         Capacity.from(new ClusterResources(wantedNodes, 1, nodeResources)))));
 
         // Allow the 1st and 3rd retired nodes permission to inactivate
         doNothing()
@@ -198,7 +204,7 @@ public class RetiredExpirerTest {
     }
 
     private void activate(ApplicationId applicationId, ClusterSpec cluster, int nodes, int groups, NodeRepositoryProvisioner provisioner) {
-        List<HostSpec> hosts = provisioner.prepare(applicationId, cluster, Capacity.fromCount(nodes, nodeResources), groups, null);
+        List<HostSpec> hosts = provisioner.prepare(applicationId, cluster, Capacity.from(new ClusterResources(nodes, groups, nodeResources)), null);
         NestedTransaction transaction = new NestedTransaction().add(new CuratorTransaction(curator));
         provisioner.activate(transaction, applicationId, hosts);
         transaction.commit();
@@ -216,7 +222,7 @@ public class RetiredExpirerTest {
         List<Node> nodes = new ArrayList<>(count);
         for (int i = 0; i < count; i++)
             nodes.add(nodeRepository.createNode("node" + i, "node" + i, Optional.empty(), flavor, NodeType.tenant));
-        nodes = nodeRepository.addNodes(nodes);
+        nodes = nodeRepository.addNodes(nodes, Agent.system);
         nodes = nodeRepository.setDirty(nodes, Agent.system, getClass().getSimpleName());
         nodeRepository.setReady(nodes, Agent.system, getClass().getSimpleName());
     }
@@ -225,7 +231,7 @@ public class RetiredExpirerTest {
         List<Node> nodes = new ArrayList<>(count);
         for (int i = 0; i < count; i++)
             nodes.add(nodeRepository.createNode("parent" + i, "parent" + i, Optional.empty(), nodeFlavors.getFlavorOrThrow("default"), NodeType.host));
-        nodes = nodeRepository.addNodes(nodes);
+        nodes = nodeRepository.addNodes(nodes, Agent.system);
         nodes = nodeRepository.setDirty(nodes, Agent.system, getClass().getSimpleName());
         nodeRepository.setReady(nodes, Agent.system, getClass().getSimpleName());
     }
@@ -235,6 +241,7 @@ public class RetiredExpirerTest {
                 nodeRepository,
                 orchestrator,
                 deployer,
+                new TestMetric(),
                 clock,
                 Duration.ofDays(30), /* Maintenance interval, use large value so it never runs by itself */
                 RETIRED_EXPIRATION);
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/TestMetric.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/TestMetric.java
new file mode 100644
index 00000000000..c98216f9d14
--- /dev/null
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/TestMetric.java
@@ -0,0 +1,75 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.maintenance;
+
+import com.yahoo.jdisc.Metric;
+
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+public class TestMetric implements Metric {
+
+    public Map<String, Number> values = new LinkedHashMap<>();
+    public Map<String, List<Context>> context = new LinkedHashMap<>();
+
+    @Override
+    public void set(String key, Number val, Context ctx) {
+        values.put(key, val);
+        if (ctx != null) {
+            //Create one context pr value added - copy the context to not have side effects
+            TestContext kontekst = (TestContext)createContext(((TestContext) ctx).properties);
+            if (!context.containsKey(key)) {
+                context.put(key, new ArrayList<>());
+            }
+            kontekst.setValue(val);
+            context.get(key).add(kontekst);
+        }
+    }
+
+    @Override
+    public void add(String key, Number val, Context ctx) {
+        values.put(key, val);
+        if (ctx != null) {
+            //Create one context pr value added - copy the context to not have side effects
+            TestContext copy = (TestContext) createContext(((TestContext) ctx).properties);
+            if (!context.containsKey(key)) {
+                context.put(key, new ArrayList<>());
+            }
+            copy.setValue(val);
+            context.get(key).add(copy);
+        }
+    }
+
+    @Override
+    public Context createContext(Map<String, ?> properties) {
+        return new TestContext(properties);
+    }
+
+    double sumDoubleValues(String key, Context sumContext) {
+        double sum = 0.0;
+        for(Context c : context.get(key)) {
+            TestContext tc = (TestContext) c;
+            if (tc.value instanceof Double && tc.properties.equals(((TestContext) sumContext).properties)) {
+                sum += (double) tc.value;
+            }
+        }
+        return sum;
+    }
+
+    /**
+     * Context where the propertymap is not shared - but unique to each value.
+     */
+    private static class TestContext implements Context{
+        Number value;
+        Map<String, ?> properties;
+
+        public TestContext(Map<String, ?> properties) {
+            this.properties = properties;
+        }
+
+        public void setValue(Number value) {
+            this.value = value;
+        }
+    }
+}
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/os/OsVersionsTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/os/OsVersionsTest.java
index 477319a8bbb..5e859cd3d25 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/os/OsVersionsTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/os/OsVersionsTest.java
@@ -5,13 +5,17 @@ import com.yahoo.component.Version;
 import com.yahoo.config.provision.NodeType;
 import com.yahoo.vespa.hosted.provision.Node;
 import com.yahoo.vespa.hosted.provision.NodeList;
+import com.yahoo.vespa.hosted.provision.node.OsVersion;
+import com.yahoo.vespa.hosted.provision.node.Status;
 import com.yahoo.vespa.hosted.provision.provisioning.ProvisioningTester;
 import org.junit.Test;
 
 import java.util.Comparator;
 import java.util.List;
 import java.util.Optional;
+import java.util.function.Function;
 import java.util.function.Supplier;
+import java.util.function.UnaryOperator;
 import java.util.stream.Collectors;
 
 import static org.junit.Assert.assertEquals;
@@ -94,9 +98,7 @@ public class OsVersionsTest {
             var nodesUpgrading = nodes.changingOsVersion();
             assertEquals("Target is changed for a subset of nodes", maxActiveUpgrades, nodesUpgrading.size());
             assertEquals("Wanted version is set for nodes upgrading", version1,
-                         nodesUpgrading.stream()
-                                       .map(node -> node.status().osVersion().wanted().get())
-                                       .min(Comparator.naturalOrder()).get());
+                         minVersion(nodesUpgrading, OsVersion::wanted));
             var nodesOnLowestVersion = nodes.asList().stream()
                                             .sorted(Comparator.comparing(node -> node.status().osVersion().current().orElse(Version.emptyVersion)))
                                             .collect(Collectors.toList())
@@ -108,18 +110,52 @@ public class OsVersionsTest {
 
         // Activating again after all nodes have upgraded does nothing
         versions.setActive(NodeType.host, true);
-        assertEquals("All nodes upgraded", version1,
-                     hostNodes.get().stream()
-                              .map(n -> n.status().osVersion().current().get())
-                              .min(Comparator.naturalOrder()).get());
+        assertEquals("All nodes upgraded", version1, minVersion(hostNodes.get(), OsVersion::current));
+    }
+
+    @Test
+    public void test_newer_upgrade_aborts_upgrade_to_stale_version() {
+        var versions = new OsVersions(tester.nodeRepository(), Integer.MAX_VALUE);
+        tester.makeReadyNodes(10, "default", NodeType.host);
+        Supplier<NodeList> hostNodes = () -> tester.nodeRepository().list().nodeType(NodeType.host);
+
+        // Some nodes are targeting an older version
+        var version1 = Version.fromString("7.1");
+        setWantedVersion(hostNodes.get().asList().subList(0, 5), version1);
+
+        // Trigger upgrade to next version
+        var version2 = Version.fromString("7.2");
+        versions.setTarget(NodeType.host, version2, false);
+        versions.setActive(NodeType.host, true);
+
+        // Wanted version is changed to newest target for all nodes
+        assertEquals(version2, minVersion(hostNodes.get(), OsVersion::wanted));
+    }
+
+    private Version minVersion(NodeList nodes, Function<OsVersion, Optional<Version>> versionField) {
+        return nodes.asList().stream()
+                    .map(Node::status)
+                    .map(Status::osVersion)
+                    .map(versionField)
+                    .flatMap(Optional::stream)
+                    .min(Comparator.naturalOrder())
+                    .orElse(Version.emptyVersion);
+
+    }
+
+    private void setWantedVersion(List<Node> nodes, Version wantedVersion) {
+        writeNode(nodes, node -> node.with(node.status().withOsVersion(node.status().osVersion().withWanted(Optional.of(wantedVersion)))));
     }
 
     private void setCurrentVersion(List<Node> nodes, Version currentVersion) {
+        writeNode(nodes, node -> node.with(node.status().withOsVersion(node.status().osVersion().withCurrent(Optional.of(currentVersion)))));
+    }
+
+    private void writeNode(List<Node> nodes, UnaryOperator<Node> updateFunc) {
         for (var node : nodes) {
             try (var lock = tester.nodeRepository().lock(node)) {
                 node = tester.nodeRepository().getNode(node.hostname()).get();
-                node = node.with(node.status().withOsVersion(node.status().osVersion().withCurrent(Optional.of(currentVersion))));
-                tester.nodeRepository().write(node, lock);
+                tester.nodeRepository().write(updateFunc.apply(node), lock);
             }
         }
     }
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/SerializationTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/SerializationTest.java
index 8e048001b98..58ebfa555d6 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/SerializationTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/SerializationTest.java
@@ -78,7 +78,7 @@ public class SerializationTest {
         node = node.allocate(ApplicationId.from(TenantName.from("myTenant"),
                                                 ApplicationName.from("myApplication"),
                                                 InstanceName.from("myInstance")),
-                             ClusterMembership.from("content/myId/0/0", Vtag.currentVersion),
+                             ClusterMembership.from("content/myId/0/0", Vtag.currentVersion, Optional.empty()),
                              requestedResources,
                              clock.instant());
         assertEquals(1, node.history().events().size());
@@ -166,7 +166,7 @@ public class SerializationTest {
         node = node.allocate(ApplicationId.from(TenantName.from("myTenant"),
                                                 ApplicationName.from("myApplication"),
                                                 InstanceName.from("myInstance")),
-                             ClusterMembership.from("content/myId/0/0", Vtag.currentVersion),
+                             ClusterMembership.from("content/myId/0/0", Vtag.currentVersion, Optional.empty()),
                              node.flavor().resources(),
                              clock.instant());
         assertEquals(1, node.history().events().size());
@@ -216,11 +216,11 @@ public class SerializationTest {
         node = node.allocate(ApplicationId.from(TenantName.from("myTenant"),
                              ApplicationName.from("myApplication"),
                              InstanceName.from("myInstance")),
-                             ClusterMembership.from("content/myId/0/0", Vtag.currentVersion),
+                             ClusterMembership.from("content/myId/0/0", Vtag.currentVersion, Optional.empty()),
                              node.flavor().resources(),
                              clock.instant());
 
-        node = node.with(node.status().setFailCount(0));
+        node = node.with(node.status().withFailCount(0));
         Node copy2 = nodeSerializer.fromJson(Node.State.provisioned, nodeSerializer.toJson(node));
 
         assertEquals(0, copy2.status().failCount());
@@ -401,7 +401,7 @@ public class SerializationTest {
         node = node.allocate(ApplicationId.from(TenantName.from("myTenant"),
                 ApplicationName.from("myApplication"),
                 InstanceName.from("myInstance")),
-                ClusterMembership.from("content/myId/0/0", Vtag.currentVersion),
+                ClusterMembership.from("content/myId/0/0", Vtag.currentVersion, Optional.empty()),
                 node.flavor().resources(),
                 clock.instant());
         assertTrue(node.allocation().isPresent());
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AclProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AclProvisioningTest.java
index 92d066e5f16..2a87f513cd9 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AclProvisioningTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AclProvisioningTest.java
@@ -4,8 +4,7 @@ package com.yahoo.vespa.hosted.provision.provisioning;
 import com.yahoo.component.Version;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.Capacity;
-import com.yahoo.config.provision.ClusterSpec;
-import com.yahoo.config.provision.HostSpec;
+import com.yahoo.config.provision.ClusterResources;
 import com.yahoo.config.provision.NodeResources;
 import com.yahoo.config.provision.NodeType;
 import com.yahoo.vespa.hosted.provision.Node;
@@ -40,14 +39,14 @@ public class AclProvisioningTest {
         tester.makeReadyNodes(10, new NodeResources(1, 4, 10, 1));
         List<Node> dockerHost = tester.makeReadyNodes(1, new NodeResources(1, 4, 10, 1), NodeType.host);
         ApplicationId zoneApplication = tester.makeApplicationId();
-        deploy(zoneApplication, Capacity.fromRequiredNodeType(NodeType.host));
+        tester.deploy(zoneApplication, Capacity.fromRequiredNodeType(NodeType.host));
         tester.makeReadyVirtualDockerNodes(1,new NodeResources(1, 4, 10, 1),
                                            dockerHost.get(0).hostname());
         List<Node> proxyNodes = tester.makeReadyNodes(3, new NodeResources(1, 4, 10, 1), NodeType.proxy);
 
         // Allocate 2 nodes
         ApplicationId application = tester.makeApplicationId();
-        List<Node> activeNodes = deploy(application, Capacity.fromCount(2, new NodeResources(1, 4, 10, 1), false, true));
+        List<Node> activeNodes = tester.deploy(application, Capacity.from(new ClusterResources(2, 1, new NodeResources(1, 4, 10, 1)), false, true));
         assertEquals(2, activeNodes.size());
 
         // Get trusted nodes for the first active node
@@ -112,7 +111,7 @@ public class AclProvisioningTest {
 
         // Deploy zone application
         ApplicationId zoneApplication = tester.makeApplicationId();
-        deploy(zoneApplication, Capacity.fromRequiredNodeType(NodeType.proxy));
+        tester.deploy(zoneApplication, Capacity.fromRequiredNodeType(NodeType.proxy));
 
         // Get trusted nodes for first proxy node
         List<Node> proxyNodes = tester.nodeRepository().getNodes(zoneApplication);
@@ -154,22 +153,43 @@ public class AclProvisioningTest {
 
         // Allocate
         ApplicationId controllerApplication = tester.makeApplicationId();
-        List<Node> controllers = deploy(controllerApplication, Capacity.fromRequiredNodeType(NodeType.controller));
+        List<Node> controllers = tester.deploy(controllerApplication, Capacity.fromRequiredNodeType(NodeType.controller));
 
         // Controllers and hosts all trust each other
         List<NodeAcl> controllerAcls = tester.nodeRepository().getNodeAcls(controllers.get(0), false);
         assertAcls(List.of(controllers), controllerAcls);
-        assertEquals(Set.of(22, 4443, 443), controllerAcls.get(0).trustedPorts());
+        assertEquals(Set.of(22, 80, 4443, 443), controllerAcls.get(0).trustedPorts());
     }
 
     @Test
     public void trusted_nodes_for_application_with_load_balancer() {
-        // Populate repo
-        tester.makeReadyNodes(10, nodeResources);
+        // Provision hosts and containers
+        var hosts = tester.makeReadyNodes(2, "default", NodeType.host);
+        tester.deployZoneApp();
+        for (var host : hosts) {
+            tester.makeReadyVirtualDockerNodes(2, new NodeResources(2, 8, 50, 1),
+                                               host.hostname());
+        }
 
-        // Allocate 2 nodes
-        List<Node> activeNodes = deploy(2);
+        // Deploy application
+        var application = tester.makeApplicationId();
+        List<Node> activeNodes = deploy(application, 2);
         assertEquals(2, activeNodes.size());
+
+        // Load balancer is allocated to application
+        var loadBalancers = tester.nodeRepository().loadBalancers(application);
+        assertEquals(1, loadBalancers.asList().size());
+        var lbNetworks = loadBalancers.asList().get(0).instance().networks();
+        assertEquals(2, lbNetworks.size());
+
+        // ACL for nodes with allocation trust their respective load balancer networks, if any
+        for (var host : hosts) {
+            var acls = tester.nodeRepository().getNodeAcls(host, true);
+            assertEquals(2, acls.size());
+            assertEquals(Set.of(), acls.get(0).trustedNetworks());
+            assertEquals(application, acls.get(1).node().allocation().get().owner());
+            assertEquals(lbNetworks, acls.get(1).trustedNetworks());
+        }
     }
 
     @Test
@@ -191,15 +211,7 @@ public class AclProvisioningTest {
     }
 
     private List<Node> deploy(ApplicationId application, int nodeCount) {
-        return deploy(application, Capacity.fromCount(nodeCount, nodeResources));
-    }
-
-    private List<Node> deploy(ApplicationId application, Capacity capacity) {
-        ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test"),
-                                                  Version.fromString("6.42"), false);
-        List<HostSpec> prepared = tester.prepare(application, cluster, capacity, 1);
-        tester.activate(application, Set.copyOf(prepared));
-        return tester.getNodes(application, Node.State.active).asList();
+        return tester.deploy(application, Capacity.from(new ClusterResources(nodeCount, 1, nodeResources)));
     }
 
     private static void assertAcls(List<List<Node>> expected, NodeAcl actual) {
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationSimulator.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationSimulator.java
index 2c01cdde932..b113c281289 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationSimulator.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AllocationSimulator.java
@@ -4,7 +4,6 @@ package com.yahoo.vespa.hosted.provision.provisioning;
 import com.yahoo.component.Version;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.ClusterMembership;
-import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.config.provision.Flavor;
 import com.yahoo.config.provision.NodeFlavors;
 import com.yahoo.config.provision.NodeType;
@@ -17,7 +16,7 @@ import com.yahoo.vespa.hosted.provision.node.IP;
 import com.yahoo.vespa.hosted.provision.node.Reports;
 import com.yahoo.vespa.hosted.provision.node.Status;
 
-import javax.swing.*;
+import javax.swing.JFrame;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashSet;
@@ -93,7 +92,7 @@ public class AllocationSimulator {
     private Optional<Allocation> allocation(Optional<String> tenant, Flavor flavor) {
         if (tenant.isPresent()) {
             Allocation allocation = new Allocation(app(tenant.get()),
-                                                   ClusterMembership.from("container/id1/3", new Version()),
+                                                   ClusterMembership.from("container/id1/3", new Version(), Optional.empty()),
                                                    flavor.resources(),
                                                    Generation.initial(),
                                                    false);
@@ -109,10 +108,6 @@ public class AllocationSimulator {
                 .instanceName("default").build();
     }
 
-    private ClusterSpec cluster() {
-        return ClusterSpec.from(ClusterSpec.Type.container, ClusterSpec.Id.from("test"), ClusterSpec.Group.from(1), Version.fromString("6.41"), false);
-    }
-
     /* ------------ Methods to add events to the system ----------------*/
 
     public void addCluster(String task, int count, Flavor flavor, String id) {
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacityTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacityTest.java
index 7d9ac230771..b3cda5bb3df 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacityTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacityTest.java
@@ -1,6 +1,7 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.provision.provisioning;
 
+import com.yahoo.config.provision.Flavor;
 import com.yahoo.config.provision.NodeFlavors;
 import com.yahoo.config.provision.NodeResources;
 import com.yahoo.config.provision.NodeType;
@@ -37,7 +38,7 @@ public class DockerHostCapacityTest {
 
     @Before
     public void setup() {
-        doAnswer(invocation -> invocation.getArguments()[0]).when(hostResourcesCalculator).availableCapacityOf(any());
+        doAnswer(invocation -> ((Flavor)invocation.getArguments()[0]).resources()).when(hostResourcesCalculator).advertisedResourcesOf(any());
 
         // Create flavors
         NodeFlavors nodeFlavors = FlavorConfigBuilder.createDummies("host", "docker", "docker2");
@@ -95,9 +96,9 @@ public class DockerHostCapacityTest {
                      capacity.freeCapacityOf(host3, false));
 
         doAnswer(invocation -> {
-            NodeResources totalHostResources = (NodeResources) invocation.getArguments()[0];
+            NodeResources totalHostResources = ((Flavor) invocation.getArguments()[0]).resources();
             return totalHostResources.subtract(new NodeResources(1, 2, 3, 0.5, NodeResources.DiskSpeed.any));
-        }).when(hostResourcesCalculator).availableCapacityOf(any());
+        }).when(hostResourcesCalculator).advertisedResourcesOf(any());
 
         assertEquals(new NodeResources(4, 2, 5, 1.5, NodeResources.DiskSpeed.fast, NodeResources.StorageType.remote),
                      capacity.freeCapacityOf(host1, false));
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerImagesTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerImagesTest.java
new file mode 100644
index 00000000000..cd6ae587b04
--- /dev/null
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerImagesTest.java
@@ -0,0 +1,51 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.provisioning;
+
+import com.yahoo.config.provision.DockerImage;
+import com.yahoo.config.provision.NodeResources;
+import com.yahoo.config.provision.NodeType;
+import com.yahoo.vespa.flags.InMemoryFlagSource;
+import org.junit.Test;
+
+import java.util.Optional;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author mpolden
+ */
+public class DockerImagesTest {
+
+    @Test
+    public void image_selection() {
+        var flagSource = new InMemoryFlagSource();
+        var tester = new ProvisioningTester.Builder().flagSource(flagSource).build();
+
+        var proxyImage = DockerImage.fromString("docker-registry.domain.tld:8080/dist/proxy");
+        tester.nodeRepository().dockerImages().setDockerImage(NodeType.proxy, Optional.of(proxyImage));
+
+        // Host uses tenant default image (for preload purposes)
+        var defaultImage = DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa");
+        var hosts = tester.makeReadyNodes(2, "default", NodeType.host);
+        tester.deployZoneApp();
+        for (var host : hosts) {
+            assertEquals(defaultImage, tester.nodeRepository().dockerImages().dockerImageFor(host.type()));
+        }
+
+        // Tenant node uses tenant default image
+        var resources = new NodeResources(2, 8, 50, 1);
+        for (var host : hosts) {
+            var nodes = tester.makeReadyVirtualDockerNodes(2, resources, host.hostname());
+            for (var node : nodes) {
+                assertEquals(defaultImage, tester.nodeRepository().dockerImages().dockerImageFor(node.type()));
+            }
+        }
+
+        // Proxy host uses image used by child nodes (proxy nodes), which is overridden in this case (for preload purposes)
+        var proxyHosts = tester.makeReadyNodes(2, "default", NodeType.proxyhost);
+        for (var host : proxyHosts) {
+            assertEquals(proxyImage, tester.nodeRepository().dockerImages().dockerImageFor(host.type()));
+        }
+    }
+
+}
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerProvisioningTest.java
index 1e59add8304..5079fce4418 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerProvisioningTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerProvisioningTest.java
@@ -5,6 +5,7 @@ import com.yahoo.component.Version;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.ApplicationName;
 import com.yahoo.config.provision.Capacity;
+import com.yahoo.config.provision.ClusterResources;
 import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.config.provision.Environment;
 import com.yahoo.config.provision.HostSpec;
@@ -52,7 +53,7 @@ public class DockerProvisioningTest {
         Version wantedVespaVersion = Version.fromString("6.39");
         int nodeCount = 7;
         List<HostSpec> hosts = tester.prepare(application1,
-                                              ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent"), wantedVespaVersion, false),
+                                              ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent")).vespaVersion(wantedVespaVersion).build(),
                                               nodeCount, 1, dockerFlavor);
         tester.activate(application1, new HashSet<>(hosts));
 
@@ -63,7 +64,7 @@ public class DockerProvisioningTest {
         // Upgrade Vespa version on nodes
         Version upgradedWantedVespaVersion = Version.fromString("6.40");
         List<HostSpec> upgradedHosts = tester.prepare(application1,
-                                                      ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent"), upgradedWantedVespaVersion, false),
+                                                      ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent")).vespaVersion(upgradedWantedVespaVersion).build(),
                                                       nodeCount, 1, dockerFlavor);
         tester.activate(application1, new HashSet<>(upgradedHosts));
         NodeList upgradedNodes = tester.getNodes(application1, Node.State.active);
@@ -85,7 +86,7 @@ public class DockerProvisioningTest {
         Version wantedVespaVersion = Version.fromString("6.39");
         int nodeCount = 7;
         List<HostSpec> nodes = tester.prepare(application1,
-                ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent"), wantedVespaVersion, false),
+                ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent")).vespaVersion(wantedVespaVersion).build(),
                 nodeCount, 1, dockerFlavor);
         try {
             tester.activate(application1, new HashSet<>(nodes));
@@ -94,13 +95,13 @@ public class DockerProvisioningTest {
 
         // Activate the zone-app, thereby allocating the parents
         List<HostSpec> hosts = tester.prepare(zoneApplication,
-                ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("zone-app"), wantedVespaVersion, false),
-                Capacity.fromRequiredNodeType(NodeType.host), 1);
+                ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("zone-app")).vespaVersion(wantedVespaVersion).build(),
+                Capacity.fromRequiredNodeType(NodeType.host));
         tester.activate(zoneApplication, hosts);
 
         // Try allocating tenants again
         nodes = tester.prepare(application1,
-                ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent"), wantedVespaVersion, false),
+                ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent")).vespaVersion(wantedVespaVersion).build(),
                 nodeCount, 1, dockerFlavor);
         tester.activate(application1, new HashSet<>(nodes));
 
@@ -124,14 +125,14 @@ public class DockerProvisioningTest {
 
         Version wantedVespaVersion = Version.fromString("6.39");
         List<HostSpec> nodes = tester.prepare(application2_1,
-                                              ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("myContent"), wantedVespaVersion, false),
+                                              ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("myContent")).vespaVersion(wantedVespaVersion).build(),
                                               6, 1, resources);
         assertHostSpecParentReservation(nodes, Optional.empty(), tester); // We do not get nodes on hosts reserved to tenant1
         tester.activate(application2_1, nodes);
 
         try {
             tester.prepare(application2_2,
-                           ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("myContent"), wantedVespaVersion, false),
+                           ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("myContent")).vespaVersion(wantedVespaVersion).build(),
                            5, 1, resources);
             fail("Expected exception");
         }
@@ -140,7 +141,7 @@ public class DockerProvisioningTest {
         }
 
         nodes = tester.prepare(application1_1,
-                               ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("myContent"), wantedVespaVersion, false),
+                               ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("myContent")).vespaVersion(wantedVespaVersion).build(),
                               10, 1, resources);
         assertHostSpecParentReservation(nodes, Optional.of(tenant1), tester);
         tester.activate(application1_1, nodes);
@@ -262,7 +263,9 @@ public class DockerProvisioningTest {
         ApplicationId application1 = tester.makeApplicationId();
         tester.makeReadyVirtualDockerNodes(1, dockerFlavor, "dockerHost");
 
-        List<HostSpec> hosts = tester.prepare(application1, ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent"), Version.fromString("6.42"), false), 1, 1, dockerFlavor);
+        List<HostSpec> hosts = tester.prepare(application1,
+                                              ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent")).vespaVersion("6.42").build(),
+                                              1, 1, dockerFlavor);
         tester.activate(application1, new HashSet<>(hosts));
 
         NodeList nodes = tester.getNodes(application1, Node.State.active);
@@ -279,8 +282,9 @@ public class DockerProvisioningTest {
             tester.makeReadyVirtualDockerNodes(1, dockerFlavor, "dockerHost1");
             tester.makeReadyVirtualDockerNodes(1, dockerFlavor, "dockerHost2");
 
-            List<HostSpec> hosts = tester.prepare(application1, ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent"),
-                                                                                    Version.fromString("6.42"), false), 2, 1,
+            List<HostSpec> hosts = tester.prepare(application1,
+                                                  ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent")).vespaVersion("6.42").build(),
+                                                  2, 1,
                                                   dockerFlavor.with(NodeResources.StorageType.remote));
         }
         catch (OutOfCapacityException e) {
@@ -294,9 +298,8 @@ public class DockerProvisioningTest {
 
     private void prepareAndActivate(ApplicationId application, int nodeCount, boolean exclusive, ProvisioningTester tester) {
         Set<HostSpec> hosts = new HashSet<>(tester.prepare(application,
-                                            ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("myContainer"), Version.fromString("6.39"), exclusive),
-                                            Capacity.fromCount(nodeCount, Optional.of(dockerFlavor), false, true),
-                                            1));
+                                            ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("myContainer")).vespaVersion("6.39").exclusive(exclusive).build(),
+                                            Capacity.from(new ClusterResources(nodeCount, 1, dockerFlavor), false, true)));
         tester.activate(application, hosts);
     }
 
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerAllocationTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerAllocationTest.java
index 53fecbf6095..4eca0542992 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerAllocationTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerAllocationTest.java
@@ -2,10 +2,10 @@
 package com.yahoo.vespa.hosted.provision.provisioning;
 
 import com.google.common.collect.ImmutableSet;
-import com.yahoo.component.Version;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.Capacity;
 import com.yahoo.config.provision.ClusterMembership;
+import com.yahoo.config.provision.ClusterResources;
 import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.config.provision.Environment;
 import com.yahoo.config.provision.Flavor;
@@ -343,7 +343,7 @@ public class DynamicDockerAllocationTest {
         tester.deployZoneApp();
 
         ApplicationId application = tester.makeApplicationId();
-        ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test"), Version.fromString("1"), false);
+        ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test")).vespaVersion("1").build();
         NodeResources resources = new NodeResources(1, 4, 10, 1, NodeResources.DiskSpeed.any);
 
         List<HostSpec> hosts = tester.prepare(application, cluster, 2, 1, resources);
@@ -360,7 +360,7 @@ public class DynamicDockerAllocationTest {
         tester.deployZoneApp();
 
         ApplicationId application = tester.makeApplicationId();
-        ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test"), Version.fromString("1"), false);
+        ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test")).vespaVersion("1").build();
         NodeResources resources = new NodeResources(1, 4, 10, 1, requestDiskSpeed);
 
         try {
@@ -382,7 +382,7 @@ public class DynamicDockerAllocationTest {
         tester.deployZoneApp();
 
         ApplicationId application = tester.makeApplicationId();
-        ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test"), Version.fromString("1"), false);
+        ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test")).vespaVersion("1").build();
         NodeResources resources = new NodeResources(1, 4, 10, 1, NodeResources.DiskSpeed.fast);
 
         List<HostSpec> hosts = tester.prepare(application, cluster, 4, 1, resources);
@@ -393,7 +393,6 @@ public class DynamicDockerAllocationTest {
                      NodeResources.DiskSpeed.slow, hosts.get(0).flavor().get().resources().diskSpeed());
     }
 
-    @SuppressWarnings("deprecation")
     @Test
     public void testSwitchingFromLegacyFlavorSyntaxToResourcesDoesNotCauseReallocation() {
         ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).flavorsConfig(flavorsConfig()).build();
@@ -401,13 +400,13 @@ public class DynamicDockerAllocationTest {
         tester.deployZoneApp();
 
         ApplicationId application = tester.makeApplicationId();
-        ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test"), Version.fromString("1"), false);
+        ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test")).vespaVersion("1").build();
 
-        List<HostSpec> hosts1 = tester.prepare(application, cluster, Capacity.fromCount(2, Optional.of(NodeResources.fromLegacyName("d-2-8-50")), false, true), 1);
+        List<HostSpec> hosts1 = tester.prepare(application, cluster, Capacity.from(new ClusterResources(2, 1, NodeResources.fromLegacyName("d-2-8-50")), false, true));
         tester.activate(application, hosts1);
 
         NodeResources resources = new NodeResources(1.5, 8, 50, 0.3);
-        List<HostSpec> hosts2 = tester.prepare(application, cluster, Capacity.fromCount(2, resources), 1);
+        List<HostSpec> hosts2 = tester.prepare(application, cluster, Capacity.from(new ClusterResources(2, 1, resources)));
         tester.activate(application, hosts2);
 
         assertEquals(hosts1, hosts2);
@@ -430,7 +429,7 @@ public class DynamicDockerAllocationTest {
                 clusterSpec.with(Optional.of(ClusterSpec.Group.from(0))), index); // Need to add group here so that group is serialized in node allocation
         Node node1aAllocation = node1a.allocate(id, clusterMembership1, node1a.flavor().resources(), Instant.now());
 
-        tester.nodeRepository().addNodes(Collections.singletonList(node1aAllocation));
+        tester.nodeRepository().addNodes(Collections.singletonList(node1aAllocation), Agent.system);
         NestedTransaction transaction = new NestedTransaction().add(new CuratorTransaction(tester.getCurator()));
         tester.nodeRepository().activate(Collections.singletonList(node1aAllocation), transaction);
         transaction.commit();
@@ -466,7 +465,7 @@ public class DynamicDockerAllocationTest {
     }
 
     private ClusterSpec clusterSpec(String clusterId) {
-        return ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from(clusterId), Version.fromString("6.42"), false);
+        return ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from(clusterId)).vespaVersion("6.42").build();
     }
 
 }
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisionTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisionTest.java
index 2b770625060..8f207ff9531 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisionTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicDockerProvisionTest.java
@@ -1,15 +1,15 @@
 // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.provision.provisioning;
 
-import com.google.common.collect.ImmutableSet;
-import com.yahoo.component.Version;
 import com.yahoo.config.provision.ApplicationId;
-import com.yahoo.config.provision.Capacity;
 import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.config.provision.Environment;
 import com.yahoo.config.provision.Flavor;
 import com.yahoo.config.provision.HostSpec;
 import com.yahoo.config.provision.NodeResources;
 import com.yahoo.config.provision.NodeType;
+import com.yahoo.config.provision.RegionName;
+import com.yahoo.config.provision.Zone;
 import com.yahoo.vespa.flags.Flags;
 import com.yahoo.vespa.flags.InMemoryFlagSource;
 import com.yahoo.vespa.hosted.provision.Node;
@@ -18,11 +18,15 @@ import com.yahoo.vespa.hosted.provision.node.IP;
 import com.yahoo.vespa.hosted.provision.testutils.MockNameResolver;
 import org.junit.Test;
 
+import java.time.Instant;
 import java.util.List;
 import java.util.Set;
+import java.util.function.Function;
 import java.util.stream.Collectors;
+import java.util.stream.IntStream;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.Mockito.doAnswer;
 import static org.mockito.Mockito.mock;
@@ -62,7 +66,7 @@ public class DynamicDockerProvisionTest {
     @Test
     public void does_not_allocate_to_available_empty_hosts() {
         tester.makeReadyNodes(3, "small", NodeType.host, 10);
-        deployZoneApp(tester);
+        tester.deployZoneApp();
 
         ApplicationId application = tester.makeApplicationId();
         NodeResources flavor = new NodeResources(1, 4, 10, 1);
@@ -82,7 +86,7 @@ public class DynamicDockerProvisionTest {
         tester.prepare(application, clusterSpec("myContent.t2.a2"), 2, 1, flavor);
         verify(hostProvisioner).provisionHosts(expectedProvisionIndexes, flavor, application);
 
-        // Ready the provisioned hosts, add an IP addreses to pool and activate them
+        // Ready the provisioned hosts, add an IP addresses to pool and activate them
         for (Integer i : expectedProvisionIndexes) {
             String hostname = "host-" + i;
             var ipConfig = new IP.Config(Set.of("::" + i + ":0"), Set.of("::" + i + ":2"));
@@ -90,7 +94,7 @@ public class DynamicDockerProvisionTest {
             tester.nodeRepository().setReady(List.of(host), Agent.system, getClass().getSimpleName());
             nameResolver.addRecord(hostname + "-2", "::" + i + ":2");
         }
-        deployZoneApp(tester);
+        tester.deployZoneApp();
 
         mockHostProvisioner(hostProvisioner, tester.nodeRepository().getAvailableFlavors().getFlavorOrThrow("small"));
         tester.prepare(application, clusterSpec("another-id"), 2, 1, flavor);
@@ -103,21 +107,43 @@ public class DynamicDockerProvisionTest {
         assertEquals(4, tester.nodeRepository().getNodes(NodeType.tenant, Node.State.reserved).size());
     }
 
-    private static void deployZoneApp(ProvisioningTester tester) {
-        ApplicationId applicationId = tester.makeApplicationId();
-        List<HostSpec> list = tester.prepare(applicationId,
-                ClusterSpec.request(ClusterSpec.Type.container,
-                        ClusterSpec.Id.from("node-admin"),
-                        Version.fromString("6.42"),
-                        false),
-                Capacity.fromRequiredNodeType(NodeType.host),
-                1);
-        tester.activate(applicationId, ImmutableSet.copyOf(list));
+    @Test
+    public void node_indices_are_unique_even_when_a_node_is_left_in_reserved_state() {
+        ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build();
+        NodeResources resources = new NodeResources(10, 10, 10, 10);
+        ApplicationId app = tester.makeApplicationId();
+
+        Function<Node, Node> retireNode = node ->
+                tester.nodeRepository().write(node.withWantToRetire(true, Agent.system, Instant.now()), () -> {});
+        Function<Integer, Node> getNodeInGroup = group -> tester.nodeRepository().getNodes(app).stream()
+                .filter(node -> node.allocation().get().membership().cluster().group().get().index() == group)
+                .findAny().orElseThrow();
+
+        // Allocate 10 hosts
+        tester.makeReadyNodes(10, resources, NodeType.host, 1);
+        tester.deployZoneApp();
+
+        // Prepare & activate an application with 8 nodes and 2 groups
+        tester.activate(app, tester.prepare(app, clusterSpec("content"), 8, 2, resources));
+
+        // Retire a node in group 1 and prepare the application
+        retireNode.apply(getNodeInGroup.apply(1));
+        tester.prepare(app, clusterSpec("content"), 8, 2, resources);
+        // App is not activated, to leave node '8' in reserved state
+
+        // Retire a node in group 0 and prepare the application
+        retireNode.apply(getNodeInGroup.apply(0));
+        tester.prepare(app, clusterSpec("content"), 8, 2, resources);
+
+        // Verify that nodes have unique indices from 0..9
+        var indices = tester.nodeRepository().getNodes(app).stream()
+                .map(node -> node.allocation().get().membership().index())
+                .collect(Collectors.toSet());
+        assertTrue(indices.containsAll(IntStream.range(0, 10).boxed().collect(Collectors.toList())));
     }
 
-
     private static ClusterSpec clusterSpec(String clusterId) {
-        return ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from(clusterId), Version.fromString("6.42"), false);
+        return ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from(clusterId)).vespaVersion("6.42").build();
     }
 
     @SuppressWarnings("unchecked")
@@ -130,4 +156,5 @@ public class DynamicDockerProvisionTest {
                     .collect(Collectors.toList());
         }).when(hostProvisioner).provisionHosts(any(), any(), any());
     }
+
 }
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InPlaceResizeProvisionTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InPlaceResizeProvisionTest.java
index 9145cb9d88f..7e9c3ef09dc 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InPlaceResizeProvisionTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InPlaceResizeProvisionTest.java
@@ -1,7 +1,6 @@
 // Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.provision.provisioning;
 
-import com.yahoo.component.Version;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.config.provision.Environment;
@@ -11,7 +10,6 @@ import com.yahoo.config.provision.NodeType;
 import com.yahoo.config.provision.OutOfCapacityException;
 import com.yahoo.config.provision.RegionName;
 import com.yahoo.config.provision.Zone;
-import com.yahoo.vespa.flags.Flags;
 import com.yahoo.vespa.flags.InMemoryFlagSource;
 import com.yahoo.vespa.hosted.provision.Node;
 import com.yahoo.vespa.hosted.provision.NodeList;
@@ -52,12 +50,11 @@ public class InPlaceResizeProvisionTest {
     private static final NodeResources mediumResources = new NodeResources(4, 8, 16, 1, NodeResources.DiskSpeed.any, NodeResources.StorageType.any);
     private static final NodeResources largeResources = new NodeResources(8, 16, 32, 1, NodeResources.DiskSpeed.any, NodeResources.StorageType.any);
 
-    private static final ClusterSpec container1 = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("container1"), Version.fromString("7.157.9"), false);
-    private static final ClusterSpec container2 = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("container2"), Version.fromString("7.157.9"), false);
-    private static final ClusterSpec content1 = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("content1"), Version.fromString("7.157.9"), false);
+    private static final ClusterSpec container1 = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("container1")).vespaVersion("7.157.9").build();
+    private static final ClusterSpec container2 = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("container2")).vespaVersion("7.157.9").build();
+    private static final ClusterSpec content1 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("content1")).vespaVersion("7.157.9").build();
 
-    private final InMemoryFlagSource flagSource = new InMemoryFlagSource()
-            .withBooleanFlag(Flags.ENABLE_IN_PLACE_RESIZE.id(), true);
+    private final InMemoryFlagSource flagSource = new InMemoryFlagSource();
     private final ProvisioningTester tester = new ProvisioningTester.Builder()
             .flagSource(flagSource)
             .zone(new Zone(Environment.prod, RegionName.from("us-east"))).build();
@@ -69,10 +66,10 @@ public class InPlaceResizeProvisionTest {
         addParentHosts(4, largeResources.with(fast).with(local));
 
         new PrepareHelper(tester, app).prepare(container1, 4, 1, mediumResources).activate();
-        assertClusterSizeAndResources(container1, 4, new NodeResources(4, 8, 16, 1, fast, local));
+        assertSizeAndResources(container1, 4, new NodeResources(4, 8, 16, 1, fast, local));
 
         new PrepareHelper(tester, app).prepare(container1, 4, 1, largeResources).activate();
-        assertClusterSizeAndResources(container1, 4, new NodeResources(8, 16, 32, 1, fast, local));
+        assertSizeAndResources(container1, 4, new NodeResources(8, 16, 32, 1, fast, local));
         assertEquals("No nodes are retired", 0, tester.getNodes(app, Node.State.active).retired().size());
     }
 
@@ -81,10 +78,10 @@ public class InPlaceResizeProvisionTest {
         addParentHosts(4, mediumResources.with(fast).with(local));
 
         new PrepareHelper(tester, app).prepare(container1, 4, 1, mediumResources).activate();
-        assertClusterSizeAndResources(container1, 4, new NodeResources(4, 8, 16, 1, fast, local));
+        assertSizeAndResources(container1, 4, new NodeResources(4, 8, 16, 1, fast, local));
 
         new PrepareHelper(tester, app).prepare(container1, 4, 1, smallResources).activate();
-        assertClusterSizeAndResources(container1, 4, new NodeResources(2, 4, 8, 1, fast, local));
+        assertSizeAndResources(container1, 4, new NodeResources(2, 4, 8, 1, fast, local));
         assertEquals("No nodes are retired", 0, tester.getNodes(app, Node.State.active).retired().size());
     }
 
@@ -97,16 +94,16 @@ public class InPlaceResizeProvisionTest {
                 .prepare(container2, 4, 1, mediumResources)
                 .activate();
         Set<String> container1Hostnames = listCluster(container1).stream().map(Node::hostname).collect(Collectors.toSet());
-        assertClusterSizeAndResources(container1, 4, new NodeResources(2, 4, 8, 1, fast, local));
-        assertClusterSizeAndResources(container2, 4, new NodeResources(4, 8, 16, 1, fast, local));
+        assertSizeAndResources(container1, 4, new NodeResources(2, 4, 8, 1, fast, local));
+        assertSizeAndResources(container2, 4, new NodeResources(4, 8, 16, 1, fast, local));
 
         new PrepareHelper(tester, app)
                 .prepare(container1, 4, 1, mediumResources)
                 .prepare(container2, 4, 1, smallResources)
                 .activate();
         assertEquals(container1Hostnames, listCluster(container1).stream().map(Node::hostname).collect(Collectors.toSet()));
-        assertClusterSizeAndResources(container1, 4, new NodeResources(4, 8, 16, 1, fast, local));
-        assertClusterSizeAndResources(container2, 4, new NodeResources(2, 4, 8, 1, fast, local));
+        assertSizeAndResources(container1, 4, new NodeResources(4, 8, 16, 1, fast, local));
+        assertSizeAndResources(container2, 4, new NodeResources(2, 4, 8, 1, fast, local));
         assertEquals("No nodes are retired", 0, tester.getNodes(app, Node.State.active).retired().size());
     }
 
@@ -120,7 +117,7 @@ public class InPlaceResizeProvisionTest {
         new PrepareHelper(tester, app)
                 .prepare(container1, 6, 1, largeResources).activate();
         assertTrue(listCluster(container1).stream().map(Node::hostname).collect(Collectors.toSet()).containsAll(initialHostnames));
-        assertClusterSizeAndResources(container1, 6, new NodeResources(8, 16, 32, 1, fast, local));
+        assertSizeAndResources(container1, 6, new NodeResources(8, 16, 32, 1, fast, local));
         assertEquals("No nodes are retired", 0, tester.getNodes(app, Node.State.active).retired().size());
     }
 
@@ -159,24 +156,34 @@ public class InPlaceResizeProvisionTest {
         assertTrue("All initial nodes should still be allocated to the application", initialHostnames.isEmpty());
     }
 
-    @Test(expected = OutOfCapacityException.class)
-    public void no_in_place_resize_if_flag_not_set() {
-        flagSource.withBooleanFlag(Flags.ENABLE_IN_PLACE_RESIZE.id(), false);
-        addParentHosts(4, mediumResources.with(fast).with(local));
+    /** In this scenario there should be no resizing */
+    @Test
+    public void increase_size_decrease_resources() {
+        addParentHosts(12, largeResources.with(fast));
 
-        new PrepareHelper(tester, app).prepare(container1, 4, 1, mediumResources).activate();
-        assertClusterSizeAndResources(container1, 4, new NodeResources(4, 8, 16, 1, fast, local));
+        NodeResources resources = new NodeResources(4, 8, 16, 1);
+        NodeResources halvedResources = new NodeResources(2, 4, 8, 1);
 
-        new PrepareHelper(tester, app).prepare(container1, 4, 1, smallResources);
-    }
+        new PrepareHelper(tester, app).prepare(container1, 4, 1, resources).activate();
+        assertSizeAndResources(container1, 4, resources);
 
+        // No resizing since it would initially (before redistribution) lead to too few resources:
+        new PrepareHelper(tester, app).prepare(container1, 8, 1, halvedResources).activate();
+        assertSizeAndResources(listCluster(container1).retired(), 4, resources);
+        assertSizeAndResources(listCluster(container1).not().retired(), 8, halvedResources);
+
+        // Redeploying the same capacity should also not lead to any resizing
+        new PrepareHelper(tester, app).prepare(container1, 8, 1, halvedResources).activate();
+        assertSizeAndResources(listCluster(container1).retired(), 4, resources);
+        assertSizeAndResources(listCluster(container1).not().retired(), 8, halvedResources);
+    }
 
     @Test(expected = OutOfCapacityException.class)
     public void cannot_inplace_decrease_resources_while_increasing_cluster_size() {
         addParentHosts(6, mediumResources.with(fast).with(local));
 
         new PrepareHelper(tester, app).prepare(container1, 4, 1, mediumResources).activate();
-        assertClusterSizeAndResources(container1, 4, new NodeResources(4, 8, 16, 1, fast, local));
+        assertSizeAndResources(container1, 4, new NodeResources(4, 8, 16, 1, fast, local));
 
         new PrepareHelper(tester, app).prepare(container1, 6, 1, smallResources);
     }
@@ -186,7 +193,7 @@ public class InPlaceResizeProvisionTest {
         addParentHosts(4, largeResources.with(fast).with(local));
 
         new PrepareHelper(tester, app).prepare(container1, 4, 1, mediumResources).activate();
-        assertClusterSizeAndResources(container1, 4, new NodeResources(4, 8, 16, 1, fast, local));
+        assertSizeAndResources(container1, 4, new NodeResources(4, 8, 16, 1, fast, local));
 
         new PrepareHelper(tester, app).prepare(container1, 2, 1, smallResources);
     }
@@ -196,7 +203,7 @@ public class InPlaceResizeProvisionTest {
         addParentHosts(4, largeResources.with(fast).with(local));
 
         new PrepareHelper(tester, app).prepare(container1, 4, 1, mediumResources).activate();
-        assertClusterSizeAndResources(container1, 4, new NodeResources(4, 8, 16, 1, fast, local));
+        assertSizeAndResources(container1, 4, new NodeResources(4, 8, 16, 1, fast, local));
 
         new PrepareHelper(tester, app).prepare(container1, 4, 2, smallResources);
     }
@@ -206,10 +213,13 @@ public class InPlaceResizeProvisionTest {
         tester.prepareAndActivateInfraApplication(infraApp, NodeType.host);
     }
 
-    private void assertClusterSizeAndResources(ClusterSpec cluster, int clusterSize, NodeResources resources) {
-        NodeList nodes = listCluster(cluster);
-        nodes.forEach(node -> assertEquals(node.toString(), node.flavor().resources(), resources));
-        assertEquals(clusterSize, nodes.size());
+    private void assertSizeAndResources(ClusterSpec cluster, int clusterSize, NodeResources resources) {
+        assertSizeAndResources(listCluster(cluster), clusterSize, resources);
+    }
+
+    private void assertSizeAndResources(NodeList nodes, int size, NodeResources resources) {
+        assertEquals(size, nodes.size());
+        nodes.forEach(n -> assertEquals(resources, n.flavor().resources()));
     }
 
     private NodeList listCluster(ClusterSpec cluster) {
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java
index a26e802dfe8..ad9d13355dc 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java
@@ -1,10 +1,10 @@
-// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.provision.provisioning;
 
 import com.google.common.collect.Iterators;
-import com.yahoo.component.Version;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.Capacity;
+import com.yahoo.config.provision.ClusterResources;
 import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.config.provision.HostName;
 import com.yahoo.config.provision.HostSpec;
@@ -24,6 +24,7 @@ import java.util.Collection;
 import java.util.Collections;
 import java.util.LinkedHashSet;
 import java.util.List;
+import java.util.Optional;
 import java.util.Set;
 import java.util.function.Supplier;
 import java.util.stream.Collectors;
@@ -40,15 +41,14 @@ public class LoadBalancerProvisionerTest {
 
     private final ApplicationId app1 = ApplicationId.from("tenant1", "application1", "default");
     private final ApplicationId app2 = ApplicationId.from("tenant2", "application2", "default");
-
     private final ApplicationId infraApp1 = ApplicationId.from("vespa", "tenant-host", "default");
 
-    private ProvisioningTester tester = new ProvisioningTester.Builder().build();
+    private final ProvisioningTester tester = new ProvisioningTester.Builder().build();
 
     @Test
     public void provision_load_balancer() {
-        Supplier<List<LoadBalancer>> lbApp1 = () -> tester.nodeRepository().loadBalancers().owner(app1).asList();
-        Supplier<List<LoadBalancer>> lbApp2 = () -> tester.nodeRepository().loadBalancers().owner(app2).asList();
+        Supplier<List<LoadBalancer>> lbApp1 = () -> tester.nodeRepository().loadBalancers(app1).asList();
+        Supplier<List<LoadBalancer>> lbApp2 = () -> tester.nodeRepository().loadBalancers(app2).asList();
         ClusterSpec.Id containerCluster1 = ClusterSpec.Id.from("qrs1");
         ClusterSpec.Id contentCluster = ClusterSpec.Id.from("content");
 
@@ -80,7 +80,7 @@ public class LoadBalancerProvisionerTest {
         tester.activate(app1, prepare(app1,
                                       clusterRequest(ClusterSpec.Type.container, containerCluster1),
                                       clusterRequest(ClusterSpec.Type.content, contentCluster)));
-        LoadBalancer loadBalancer = tester.nodeRepository().loadBalancers().owner(app1).asList().get(0);
+        LoadBalancer loadBalancer = tester.nodeRepository().loadBalancers(app1).asList().get(0);
         assertEquals(2, loadBalancer.instance().reals().size());
         assertTrue("Failed node is removed", loadBalancer.instance().reals().stream()
                                                          .map(Real::hostname)
@@ -155,10 +155,10 @@ public class LoadBalancerProvisionerTest {
 
     @Test
     public void provision_load_balancers_with_dynamic_node_provisioning() {
-        var nodes = prepare(app1, Capacity.fromCount(2, new NodeResources(1, 4, 10, 0.3), false, true),
+        var nodes = prepare(app1, Capacity.from(new ClusterResources(2, 1, new NodeResources(1, 4, 10, 0.3)), false, true),
                                            true,
                                            clusterRequest(ClusterSpec.Type.container, ClusterSpec.Id.from("qrs")));
-        Supplier<LoadBalancer> lb = () -> tester.nodeRepository().loadBalancers().owner(app1).asList().get(0);
+        Supplier<LoadBalancer> lb = () -> tester.nodeRepository().loadBalancers(app1).asList().get(0);
         assertTrue("Load balancer provisioned with empty reals", tester.loadBalancerService().instances().get(lb.get().id()).reals().isEmpty());
         assignIps(tester.nodeRepository().getNodes(app1));
         tester.activate(app1, nodes);
@@ -173,7 +173,7 @@ public class LoadBalancerProvisionerTest {
         assertSame("Load balancer is deactivated", LoadBalancer.State.inactive, lb.get().state());
 
         // Application is redeployed
-        nodes = prepare(app1, Capacity.fromCount(2, new NodeResources(1, 4, 10, 0.3), false, true),
+        nodes = prepare(app1, Capacity.from(new ClusterResources(2, 1, new NodeResources(1, 4, 10, 0.3)), false, true),
                         true,
                         clusterRequest(ClusterSpec.Type.container, ClusterSpec.Id.from("qrs")));
         assertTrue("Load balancer is reconfigured with empty reals", tester.loadBalancerService().instances().get(lb.get().id()).reals().isEmpty());
@@ -189,7 +189,7 @@ public class LoadBalancerProvisionerTest {
                                            clusterRequest(ClusterSpec.Type.container,
                                                           ClusterSpec.Id.from("tenant-host"))));
         assertTrue("No load balancer provisioned", tester.loadBalancerService().instances().isEmpty());
-        assertEquals(List.of(), tester.nodeRepository().loadBalancers().owner(infraApp1).asList());
+        assertEquals(List.of(), tester.nodeRepository().loadBalancers(infraApp1).asList());
     }
 
     @Test
@@ -197,12 +197,13 @@ public class LoadBalancerProvisionerTest {
         tester.activate(app1, prepare(app1, clusterRequest(ClusterSpec.Type.content,
                                                            ClusterSpec.Id.from("tenant-host"))));
         assertTrue("No load balancer provisioned", tester.loadBalancerService().instances().isEmpty());
-        assertEquals(List.of(), tester.nodeRepository().loadBalancers().owner(app1).asList());
+        assertEquals(List.of(), tester.nodeRepository().loadBalancers(app1).asList());
     }
 
+    // TODO(mpolden): Remove when ClusterSpec with combined type rejects empty combinedId
     @Test
-    public void provision_load_balancer_combined_cluster() {
-        Supplier<List<LoadBalancer>> lbs = () -> tester.nodeRepository().loadBalancers().owner(app1).asList();
+    public void provision_load_balancer_combined_cluster_without_id() {
+        Supplier<List<LoadBalancer>> lbs = () -> tester.nodeRepository().loadBalancers(app1).asList();
         ClusterSpec.Id cluster = ClusterSpec.Id.from("foo");
 
         var nodes = prepare(app1, clusterRequest(ClusterSpec.Type.combined, cluster));
@@ -212,12 +213,24 @@ public class LoadBalancerProvisionerTest {
         assertSame(LoadBalancer.State.active, lbs.get().get(0).state());
     }
 
+    @Test
+    public void provision_load_balancer_combined_cluster() {
+        Supplier<List<LoadBalancer>> lbs = () -> tester.nodeRepository().loadBalancers(app1).asList();
+        var combinedId = ClusterSpec.Id.from("container1");
+        var nodes = prepare(app1, clusterRequest(ClusterSpec.Type.combined, ClusterSpec.Id.from("content1"), Optional.of(combinedId)));
+        assertEquals(1, lbs.get().size());
+        assertEquals("Prepare provisions load balancer with reserved nodes", 2, lbs.get().get(0).instance().reals().size());
+        tester.activate(app1, nodes);
+        assertSame(LoadBalancer.State.active, lbs.get().get(0).state());
+        assertEquals(combinedId, lbs.get().get(0).id().cluster());
+    }
+
     private void dirtyNodesOf(ApplicationId application) {
         tester.nodeRepository().setDirty(tester.nodeRepository().getNodes(application), Agent.system, this.getClass().getSimpleName());
     }
 
     private Set<HostSpec> prepare(ApplicationId application, ClusterSpec... specs) {
-        return prepare(application, Capacity.fromCount(2, new NodeResources(1, 4, 10, 0.3), false, true), false, specs);
+        return prepare(application, Capacity.from(new ClusterResources(2, 1, new NodeResources(1, 4, 10, 0.3)), false, true), false, specs);
     }
 
     private Set<HostSpec> prepare(ApplicationId application, Capacity capacity, boolean dynamicDockerNodes, ClusterSpec... specs) {
@@ -228,7 +241,7 @@ public class LoadBalancerProvisionerTest {
         }
         Set<HostSpec> allNodes = new LinkedHashSet<>();
         for (ClusterSpec spec : specs) {
-            allNodes.addAll(tester.prepare(application, spec, capacity, 1, false));
+            allNodes.addAll(tester.prepare(application, spec, capacity, false));
         }
         return allNodes;
     }
@@ -255,7 +268,11 @@ public class LoadBalancerProvisionerTest {
     }
 
     private static ClusterSpec clusterRequest(ClusterSpec.Type type, ClusterSpec.Id id) {
-        return ClusterSpec.request(type, id, Version.fromString("6.42"), false);
+        return clusterRequest(type,  id, Optional.empty());
+    }
+
+    private static ClusterSpec clusterRequest(ClusterSpec.Type type, ClusterSpec.Id id, Optional<ClusterSpec.Id> combinedId) {
+        return ClusterSpec.request(type, id).vespaVersion("6.42").combinedId(combinedId).build();
     }
 
     private static <T> T get(Set<T> set, int position) {
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/MultigroupProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/MultigroupProvisioningTest.java
index 6ab027cd143..1265961e351 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/MultigroupProvisioningTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/MultigroupProvisioningTest.java
@@ -1,9 +1,9 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.provision.provisioning;
 
-import com.yahoo.component.Version;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.Capacity;
+import com.yahoo.config.provision.ClusterResources;
 import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.config.provision.Environment;
 import com.yahoo.config.provision.HostSpec;
@@ -12,6 +12,7 @@ import com.yahoo.config.provision.RegionName;
 import com.yahoo.config.provision.Zone;
 import com.yahoo.vespa.hosted.provision.Node;
 import com.yahoo.vespa.hosted.provision.maintenance.RetiredExpirer;
+import com.yahoo.vespa.hosted.provision.maintenance.TestMetric;
 import com.yahoo.vespa.hosted.provision.testutils.MockDeployer;
 import org.junit.Ignore;
 import org.junit.Test;
@@ -21,7 +22,6 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Map;
-import java.util.Optional;
 import java.util.Set;
 
 import static org.junit.Assert.assertEquals;
@@ -104,8 +104,8 @@ public class MultigroupProvisioningTest {
 
         tester.makeReadyNodes(10, small);
 
-        deploy(application1, Capacity.fromCount(1, Optional.of(small), true, true), 1, tester);
-        deploy(application1, Capacity.fromCount(2, Optional.of(small), true, true), 2, tester);
+        deploy(application1, Capacity.from(new ClusterResources(1, 1, small), true, true), tester);
+        deploy(application1, Capacity.from(new ClusterResources(2, 2, small), true, true), tester);
     }
 
     @Test
@@ -117,8 +117,8 @@ public class MultigroupProvisioningTest {
         tester.makeReadyNodes(10, small);
         tester.makeReadyNodes(10, large);
 
-        deploy(application1, Capacity.fromCount(1, Optional.of(small), true, true), 1, tester);
-        deploy(application1, Capacity.fromCount(2, Optional.of(large), true, true), 2, tester);
+        deploy(application1, Capacity.from(new ClusterResources(1, 1, small), true, true), tester);
+        deploy(application1, Capacity.from(new ClusterResources(2, 2, large), true, true), tester);
     }
 
     @Test
@@ -140,30 +140,35 @@ public class MultigroupProvisioningTest {
                              tester.clock(),
                              Collections.singletonMap(application1, 
                                                       new MockDeployer.ApplicationContext(application1, cluster(), 
-                                                                                          Capacity.fromCount(8, Optional.of(large), false, true), 1)));
-        new RetiredExpirer(tester.nodeRepository(), tester.orchestrator(), deployer, tester.clock(), Duration.ofDays(30),
-                Duration.ofHours(12)).run();
+                                                                                          Capacity.from(new ClusterResources(8, 1, large), false, true))));
+        new RetiredExpirer(tester.nodeRepository(),
+                           tester.orchestrator(),
+                           deployer,
+                           new TestMetric(),
+                           tester.clock(),
+                           Duration.ofDays(30),
+                           Duration.ofHours(12)).run();
 
         assertEquals(8, tester.getNodes(application1, Node.State.inactive).resources(small).size());
         deploy(application1, 8, 8, large, tester);
     }
 
     private void deploy(ApplicationId application, int nodeCount, int groupCount, NodeResources resources, ProvisioningTester tester) {
-        deploy(application, Capacity.fromCount(nodeCount, Optional.of(resources), false, true), groupCount, tester);
+        deploy(application, Capacity.from(new ClusterResources(nodeCount, groupCount, resources), false, true), tester);
     }
     private void deploy(ApplicationId application, int nodeCount, int groupCount, ProvisioningTester tester) {
-        deploy(application, Capacity.fromCount(nodeCount, Optional.of(large), false, true), groupCount, tester);
+        deploy(application, Capacity.from(new ClusterResources(nodeCount, groupCount, large), false, true), tester);
     }
 
-    private void deploy(ApplicationId application, Capacity capacity, int wantedGroups, ProvisioningTester tester) {
-        int nodeCount = capacity.nodeCount();
-        NodeResources nodeResources = capacity.nodeResources().get();
+    private void deploy(ApplicationId application, Capacity capacity, ProvisioningTester tester) {
+        int nodeCount = capacity.minResources().nodes();
+        NodeResources nodeResources = capacity.minResources().nodeResources();
 
         int previousActiveNodeCount = tester.getNodes(application, Node.State.active).resources(nodeResources).size();
 
-        tester.activate(application, prepare(application, capacity, wantedGroups, tester));
+        tester.activate(application, prepare(application, capacity, tester));
         assertEquals("Superfluous nodes are retired, but no others - went from " + previousActiveNodeCount + " to " + nodeCount + " nodes",
-                     Math.max(0, previousActiveNodeCount - capacity.nodeCount()),
+                     Math.max(0, previousActiveNodeCount - capacity.minResources().nodes()),
                      tester.getNodes(application, Node.State.active).retired().resources(nodeResources).size());
         assertEquals("Other flavors are retired",
                      0, tester.getNodes(application, Node.State.active).not().retired().not().resources(nodeResources).size());
@@ -188,26 +193,27 @@ public class MultigroupProvisioningTest {
             ClusterSpec.Group group = node.allocation().get().membership().cluster().group().get();
             nonretiredGroups.put(group, nonretiredGroups.getOrDefault(group, 0) + 1);
 
-            if (wantedGroups > 1)
-                assertTrue("Group indexes are always in [0, wantedGroups>", group.index() < wantedGroups);
+            if (capacity.minResources().groups() > 1)
+                assertTrue("Group indexes are always in [0, wantedGroups>",
+                           group.index() < capacity.minResources().groups());
         }
         assertEquals("Total nonretired nodes", nodeCount, indexes.size());
-        assertEquals("Total nonretired groups", wantedGroups, nonretiredGroups.size());
+        assertEquals("Total nonretired groups", capacity.minResources().groups(), nonretiredGroups.size());
         for (Integer groupSize : nonretiredGroups.values())
-            assertEquals("Group size", (long)nodeCount / wantedGroups, (long)groupSize);
+            assertEquals("Group size", (long)nodeCount / capacity.minResources().groups(), (long)groupSize);
 
         Map<ClusterSpec.Group, Integer> allGroups = new HashMap<>();
         for (Node node : tester.getNodes(application, Node.State.active).resources(nodeResources)) {
             ClusterSpec.Group group = node.allocation().get().membership().cluster().group().get();
             allGroups.put(group, nonretiredGroups.getOrDefault(group, 0) + 1);
         }
-        assertEquals("No additional groups are retained containing retired nodes", wantedGroups, allGroups.size());
+        assertEquals("No additional groups are retained containing retired nodes", capacity.minResources().groups(), allGroups.size());
     }
 
-    private ClusterSpec cluster() { return ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test"), Version.fromString("6.42"), false); }
+    private ClusterSpec cluster() { return ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("test")).vespaVersion("6.42").build(); }
 
-    private Set<HostSpec> prepare(ApplicationId application, Capacity capacity, int groupCount, ProvisioningTester tester) {
-        return new HashSet<>(tester.prepare(application, cluster(), capacity, groupCount));
+    private Set<HostSpec> prepare(ApplicationId application, Capacity capacity, ProvisioningTester tester) {
+        return new HashSet<>(tester.prepare(application, cluster(), capacity));
     }
 
 }
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodeTypeProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodeTypeProvisioningTest.java
index f0bd43b6bae..98133898cf6 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodeTypeProvisioningTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodeTypeProvisioningTest.java
@@ -1,7 +1,6 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.provision.provisioning;
 
-import com.yahoo.component.Version;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.Capacity;
 import com.yahoo.config.provision.ClusterSpec;
@@ -9,6 +8,7 @@ import com.yahoo.config.provision.HostSpec;
 import com.yahoo.config.provision.NodeType;
 import com.yahoo.vespa.hosted.provision.Node;
 import com.yahoo.vespa.hosted.provision.maintenance.RetiredExpirer;
+import com.yahoo.vespa.hosted.provision.maintenance.TestMetric;
 import com.yahoo.vespa.hosted.provision.node.Agent;
 import com.yahoo.vespa.hosted.provision.testutils.MockDeployer;
 import org.junit.Before;
@@ -35,10 +35,7 @@ public class NodeTypeProvisioningTest {
 
     private final ApplicationId application = tester.makeApplicationId(); // application using proxy nodes
     private final Capacity capacity = Capacity.fromRequiredNodeType(NodeType.proxy);
-    private final ClusterSpec clusterSpec = ClusterSpec.request(ClusterSpec.Type.container,
-                                                                ClusterSpec.Id.from("test"),
-                                                                Version.fromString("6.42"),
-                                                                false);
+    private final ClusterSpec clusterSpec = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test")).vespaVersion("6.42").build();
 
     @Before
     public void setup() {
@@ -97,9 +94,14 @@ public class NodeTypeProvisioningTest {
                 tester.provisioner(),
                 tester.clock(),
                 Collections.singletonMap(
-                        application, new MockDeployer.ApplicationContext(application, clusterSpec, capacity, 1)));
-        RetiredExpirer retiredExpirer =  new RetiredExpirer(tester.nodeRepository(), tester.orchestrator(), deployer,
-                tester.clock(), Duration.ofDays(30), Duration.ofMinutes(10));
+                        application, new MockDeployer.ApplicationContext(application, clusterSpec, capacity)));
+        RetiredExpirer retiredExpirer =  new RetiredExpirer(tester.nodeRepository(),
+                                                            tester.orchestrator(),
+                                                            deployer,
+                                                            new TestMetric(),
+                                                            tester.clock(),
+                                                            Duration.ofDays(30),
+                                                            Duration.ofMinutes(10));
 
         { // Deploy
             List<HostSpec> hosts = deployProxies(application, tester);
@@ -161,10 +163,11 @@ public class NodeTypeProvisioningTest {
         MockDeployer deployer = new MockDeployer(tester.provisioner(),
                                                  tester.clock(),
                                                  Collections.singletonMap(application,
-                                                                          new MockDeployer.ApplicationContext(application, clusterSpec, capacity, 1)));
+                                                                          new MockDeployer.ApplicationContext(application, clusterSpec, capacity)));
         RetiredExpirer retiredExpirer =  new RetiredExpirer(tester.nodeRepository(),
                                                             tester.orchestrator(),
                                                             deployer,
+                                                            new TestMetric(),
                                                             tester.clock(),
                                                             Duration.ofDays(30),
                                                             Duration.ofMinutes(10));
@@ -267,7 +270,7 @@ public class NodeTypeProvisioningTest {
     }
 
     private List<HostSpec> deployProxies(ApplicationId application, ProvisioningTester tester) {
-        return tester.prepare(application, clusterSpec, capacity, 1);
+        return tester.prepare(application, clusterSpec, capacity);
     }
 
 }
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java
index 166d5a1f654..8fc73d68785 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java
@@ -3,20 +3,19 @@ package com.yahoo.vespa.hosted.provision.provisioning;
 
 import com.yahoo.component.Version;
 import com.yahoo.config.provision.ApplicationId;
-import com.yahoo.config.provision.ApplicationName;
 import com.yahoo.config.provision.Capacity;
 import com.yahoo.config.provision.ClusterMembership;
+import com.yahoo.config.provision.ClusterResources;
 import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.config.provision.DockerImage;
 import com.yahoo.config.provision.Environment;
 import com.yahoo.config.provision.HostFilter;
 import com.yahoo.config.provision.HostSpec;
-import com.yahoo.config.provision.InstanceName;
 import com.yahoo.config.provision.NodeResources;
 import com.yahoo.config.provision.NodeType;
 import com.yahoo.config.provision.OutOfCapacityException;
 import com.yahoo.config.provision.RegionName;
 import com.yahoo.config.provision.SystemName;
-import com.yahoo.config.provision.TenantName;
 import com.yahoo.config.provision.Zone;
 import com.yahoo.transaction.NestedTransaction;
 import com.yahoo.vespa.hosted.provision.Node;
@@ -33,7 +32,6 @@ import java.util.Iterator;
 import java.util.List;
 import java.util.Optional;
 import java.util.Set;
-import java.util.UUID;
 import java.util.function.Function;
 import java.util.function.Predicate;
 import java.util.stream.Collectors;
@@ -161,6 +159,34 @@ public class ProvisioningTest {
     }
 
     @Test
+    public void dockerImageRepoIsReturnedIfSet() {
+        ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.dev, RegionName.from("us-east"))).build();
+
+        tester.makeReadyNodes(4, defaultResources, NodeType.host, 1);
+        tester.prepareAndActivateInfraApplication(tester.makeApplicationId(), NodeType.host);
+
+        // deploy
+        ApplicationId application1 = tester.makeApplicationId();
+        SystemState state1 = prepare(application1, tester, 1, 1, 1, 1, defaultResources, "1.2.3");
+        String dockerImageRepo = "docker.domain.tld/my/image";
+        prepare(application1, tester, 1, 1, 1 , 1 , false, defaultResources, "1.2.3", Optional.of(dockerImageRepo));
+        tester.activate(application1, state1.allHosts);
+
+        HostSpec host1 = state1.container0.iterator().next();
+        Node node1 = tester.nodeRepository().getNode(host1.hostname()).get();
+        DockerImage dockerImage = DockerImage.fromString(dockerImageRepo).withTag(Version.fromString("1.2.3"));
+        tester.nodeRepository().write(node1.with(node1.status().withDockerImage(dockerImage)), () -> {});
+
+        // redeploy
+        SystemState state2 = prepare(application1, tester, 1, 1, 1 ,1 , false, defaultResources, "1.2.3", Optional.of(dockerImageRepo));
+        tester.activate(application1, state2.allHosts);
+
+        host1 = state2.container0.iterator().next();
+        node1 = tester.nodeRepository().getNode(host1.hostname()).get();
+        assertEquals(dockerImage, node1.status().dockerImage().get());
+    }
+
+    @Test
     public void application_deployment_variable_application_size() {
         ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build();
 
@@ -328,6 +354,17 @@ public class ProvisioningTest {
             assertTrue(state.allHosts.stream().allMatch(host -> host.requestedResources().get().diskSpeed() == NodeResources.DiskSpeed.fast));
             assertTrue(tester.nodeRepository().getNodes(application).stream().allMatch(node -> node.allocation().get().requestedResources().diskSpeed() == NodeResources.DiskSpeed.fast));
         }
+
+        {
+            // Go back to any
+            SystemState state = prepare(application, 0, 0, 5, 3,
+                                        defaultResources.justNumbers(),
+                                        tester);
+            assertEquals(8, state.allHosts.size());
+            tester.activate(application, state.allHosts);
+            assertTrue(state.allHosts.stream().allMatch(host -> host.requestedResources().get().diskSpeed() == NodeResources.DiskSpeed.any));
+            assertTrue(tester.nodeRepository().getNodes(application).stream().allMatch(node -> node.allocation().get().requestedResources().diskSpeed() == NodeResources.DiskSpeed.any));
+        }
     }
 
     @Test
@@ -338,7 +375,21 @@ public class ProvisioningTest {
         tester.prepareAndActivateInfraApplication(tester.makeApplicationId(), NodeType.host);
 
         ApplicationId application = tester.makeApplicationId();
-        SystemState state = prepare(application, 2, 2, 3, 3, defaultResources, Version.fromString("6.91"), tester);
+        SystemState state = prepare(application, tester, 2, 2, 3, 3, defaultResources, "6.91");
+        assertEquals(4, state.allHosts.size());
+        tester.activate(application, state.allHosts);
+    }
+
+    @Test
+    public void deploy_specific_vespa_version_and_docker_image() {
+        ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.dev, RegionName.from("us-east"))).build();
+
+        tester.makeReadyNodes(4, defaultResources, NodeType.host, 1);
+        tester.prepareAndActivateInfraApplication(tester.makeApplicationId(), NodeType.host);
+
+        ApplicationId application = tester.makeApplicationId();
+        String dockerImageRepo = "docker.domain.tld/my/image";
+        SystemState state = prepare(application, tester, 2, 2, 3, 3, false, defaultResources, "6.91", Optional.of(dockerImageRepo));
         assertEquals(4, state.allHosts.size());
         tester.activate(application, state.allHosts);
     }
@@ -363,6 +414,21 @@ public class ProvisioningTest {
         prepare(application, 1, 2, 3, 3, defaultResources, tester);
     }
 
+    @Test
+    public void below_resource_limit() {
+        ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build();
+
+        ApplicationId application = tester.makeApplicationId();
+        tester.makeReadyNodes(10, defaultResources);
+        try {
+            prepare(application, 2, 2, 3, 3,
+                    new NodeResources(2, 2, 10, 2), tester);
+        }
+        catch (IllegalArgumentException e) {
+            assertEquals("Must specify at least 4.00 Gb of memory for container cluster 'container0', was: 2.00 Gb", e.getMessage());
+        }
+    }
+
     /** Dev always uses the zone default flavor */
     @Test
     public void dev_deployment_flavor() {
@@ -453,7 +519,7 @@ public class ProvisioningTest {
             fail("Expected exception");
         }
         catch (IllegalArgumentException e) {
-            assertEquals("6 nodes [vcpu: 1.0, memory: 4.0 Gb, disk 10.0 Gb, bandwidth: 4.0 Gbps] requested for content cluster 'content0' 6.42 exceeds your quota. Resolve this at https://cloud.vespa.ai/quota",
+            assertEquals("6 nodes with [vcpu: 1.0, memory: 4.0 Gb, disk 10.0 Gb, bandwidth: 4.0 Gbps] requested for content cluster 'content0' 6.42. Max value exceeds your quota. Resolve this at https://cloud.vespa.ai/quota",
                          e.getMessage());
         }
     }
@@ -474,11 +540,8 @@ public class ProvisioningTest {
         ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build();
         tester.makeReadyNodes(4, defaultResources);
         ApplicationId application = tester.makeApplicationId();
-        ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content,
-                                                  ClusterSpec.Id.from("music"),
-                                                  new com.yahoo.component.Version(4, 5, 6),
-                                                  false);
-        tester.prepare(application, cluster, Capacity.fromCount(5, Optional.empty(), false, false), 1);
+        ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("music")).vespaVersion("4.5.6").build();
+        tester.prepare(application, cluster, Capacity.from(new ClusterResources(5, 1, NodeResources.unspecified), false, false));
         // No exception; Success
     }
 
@@ -501,8 +564,8 @@ public class ProvisioningTest {
 
     @Test
     public void want_to_retire_but_cannot_fail() {
-        Capacity capacity =       Capacity.fromCount(5, Optional.of(defaultResources), false, true);
-        Capacity capacityFORCED = Capacity.fromCount(5, Optional.of(defaultResources), false, false);
+        Capacity capacity =       Capacity.from(new ClusterResources(5, 1, defaultResources), false, true);
+        Capacity capacityFORCED = Capacity.from(new ClusterResources(5, 1, defaultResources), false, false);
 
         ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build();
 
@@ -511,18 +574,15 @@ public class ProvisioningTest {
         // Create 10 nodes
         tester.makeReadyNodes(10, defaultResources);
         // Allocate 5 nodes
-        ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content,
-                                                  ClusterSpec.Id.from("music"),
-                                                  new com.yahoo.component.Version(4, 5, 6),
-                                                  false);
-        tester.activate(application, tester.prepare(application, cluster, capacity, 1));
+        ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("music")).vespaVersion("4.5.6").build();
+        tester.activate(application, tester.prepare(application, cluster, capacity));
         assertEquals(5, NodeList.copyOf(tester.nodeRepository().getNodes(application, Node.State.active)).not().retired().size());
         assertEquals(0, NodeList.copyOf(tester.nodeRepository().getNodes(application, Node.State.active)).retired().size());
 
         // Mark the nodes as want to retire
         tester.nodeRepository().getNodes(application, Node.State.active).forEach(node -> tester.patchNode(node.with(node.status().withWantToRetire(true))));
         // redeploy without allow failing
-        tester.activate(application, tester.prepare(application, cluster, capacityFORCED, 1));
+        tester.activate(application, tester.prepare(application, cluster, capacityFORCED));
 
         // Nodes are not retired since that is unsafe when we cannot fail
         assertEquals(5, NodeList.copyOf(tester.nodeRepository().getNodes(application, Node.State.active)).not().retired().size());
@@ -531,7 +591,7 @@ public class ProvisioningTest {
         tester.nodeRepository().getNodes(application, Node.State.active).forEach(node -> assertTrue(node.status().wantToRetire()));
 
         // redeploy with allowing failing
-        tester.activate(application, tester.prepare(application, cluster, capacity, 1));
+        tester.activate(application, tester.prepare(application, cluster, capacity));
         // ... old nodes are now retired
         assertEquals(5, NodeList.copyOf(tester.nodeRepository().getNodes(application, Node.State.active)).not().retired().size());
         assertEquals(5, NodeList.copyOf(tester.nodeRepository().getNodes(application, Node.State.active)).retired().size());
@@ -640,7 +700,7 @@ public class ProvisioningTest {
         ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build();
         ApplicationId application = tester.makeApplicationId();
         try {
-            prepare(application, 1, 0, 1, 0, true, defaultResources, Version.fromString("6.42"), tester);
+            prepare(application, tester, 1, 0, 1, 0, true, defaultResources, "6.42", Optional.empty());
             fail("Expected exception");
         } catch (IllegalArgumentException ignored) {}
     }
@@ -662,44 +722,74 @@ public class ProvisioningTest {
     public void cluster_spec_update_for_already_reserved_nodes() {
         ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.dev, RegionName.from("us-east"))).build();
         ApplicationId application = tester.makeApplicationId();
-        Version version1 = Version.fromString("6.42");
-        Version version2 = Version.fromString("6.43");
+        String version1 = "6.42";
+        String version2 = "6.43";
         tester.makeReadyNodes(2, defaultResources);
 
-        prepare(application, 1, 0, 1, 0, true, defaultResources, version1, tester);
+        prepare(application, tester, 1, 0, 1, 0, true, defaultResources, version1, Optional.empty());
         tester.getNodes(application, Node.State.reserved).forEach(node ->
-                assertEquals(version1, node.allocation().get().membership().cluster().vespaVersion()));
+                assertEquals(Version.fromString(version1), node.allocation().get().membership().cluster().vespaVersion()));
 
-        prepare(application, 1, 0, 1, 0, true, defaultResources, version2, tester);
+        prepare(application, tester, 1, 0, 1, 0, true, defaultResources, version2, Optional.empty());
         tester.getNodes(application, Node.State.reserved).forEach(node ->
-                assertEquals(version2, node.allocation().get().membership().cluster().vespaVersion()));
+                assertEquals(Version.fromString(version2), node.allocation().get().membership().cluster().vespaVersion()));
+    }
+
+    @Test
+    public void change_to_and_from_combined_cluster_does_not_change_node_allocation() {
+        var tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build();
+        var application = tester.makeApplicationId();
+
+        tester.makeReadyNodes(4, defaultResources);
+
+        // Application allocates two content nodes initially, with cluster type content
+        ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("music")).vespaVersion("1.2.3").build();
+        var initialNodes = tester.activate(application, tester.prepare(application, cluster,
+                                                                       Capacity.from(new ClusterResources(2, 1, defaultResources), false, false)));
+
+        // Application is redeployed with cluster type combined
+        cluster = ClusterSpec.request(ClusterSpec.Type.combined, ClusterSpec.Id.from("music")).vespaVersion("1.2.3").build();
+        var newNodes = tester.activate(application, tester.prepare(application, cluster,
+                                                                   Capacity.from(new ClusterResources(2, 1, defaultResources), false, false)));
+
+        assertEquals("Node allocation remains the same", initialNodes, newNodes);
+        assertEquals("Cluster type is updated",
+                     Set.of(ClusterSpec.Type.combined),
+                     newNodes.stream().map(n -> n.membership().get().cluster().type()).collect(Collectors.toSet()));
+
+        // Application is redeployed with cluster type content again
+        cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("music")).vespaVersion("1.2.3").build();
+        newNodes = tester.activate(application, tester.prepare(application, cluster,
+                                                               Capacity.from(new ClusterResources(2, 1, defaultResources), false, false)));
+        assertEquals("Node allocation remains the same", initialNodes, newNodes);
+        assertEquals("Cluster type is updated",
+                     Set.of(ClusterSpec.Type.content),
+                     newNodes.stream().map(n -> n.membership().get().cluster().type()).collect(Collectors.toSet()));
     }
 
     private SystemState prepare(ApplicationId application, int container0Size, int container1Size, int content0Size,
                                 int content1Size, NodeResources flavor, ProvisioningTester tester) {
-        return prepare(application, container0Size, container1Size, content0Size, content1Size, flavor,
-                       Version.fromString("6.42"), tester);
+        return prepare(application, tester, container0Size, container1Size, content0Size, content1Size, flavor, "6.42");
     }
 
-    private SystemState prepare(ApplicationId application, int container0Size, int container1Size, int content0Size,
-                                int content1Size, NodeResources nodeResources, Version wantedVersion, ProvisioningTester tester) {
-        return prepare(application, container0Size, container1Size, content0Size, content1Size, false, nodeResources,
-                       wantedVersion, tester);
+    private SystemState prepare(ApplicationId application, ProvisioningTester tester, int container0Size, int container1Size, int content0Size,
+                                int content1Size, NodeResources nodeResources, String wantedVersion) {
+        return prepare(application, tester, container0Size, container1Size, content0Size, content1Size, false, nodeResources,
+                       wantedVersion, Optional.empty());
     }
 
-    private SystemState prepare(ApplicationId application, int container0Size, int container1Size, int content0Size,
-                                int content1Size, boolean required, NodeResources nodeResources, Version wantedVersion,
-                                ProvisioningTester tester) {
+    private SystemState prepare(ApplicationId application, ProvisioningTester tester, int container0Size, int container1Size, int content0Size,
+                                int content1Size, boolean required, NodeResources nodeResources, String wantedVersion, Optional<String> dockerImageRepo) {
         // "deploy prepare" with a two container clusters and a storage cluster having of two groups
-        ClusterSpec containerCluster0 = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("container0"), wantedVersion, false);
-        ClusterSpec containerCluster1 = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("container1"), wantedVersion, false);
-        ClusterSpec contentCluster0 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("content0"), wantedVersion, false);
-        ClusterSpec contentCluster1 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("content1"), wantedVersion, false);
+        ClusterSpec containerCluster0 = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("container0")).vespaVersion(wantedVersion).build();
+        ClusterSpec containerCluster1 = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("container1")).vespaVersion(wantedVersion).build();
+        ClusterSpec contentCluster0 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("content0")).vespaVersion(wantedVersion).build();
+        ClusterSpec contentCluster1 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("content1")).vespaVersion(wantedVersion).build();
 
-        Set<HostSpec> container0 = prepare(application, containerCluster0, container0Size, 1, required, nodeResources, tester);
-        Set<HostSpec> container1 = prepare(application, containerCluster1, container1Size, 1, required, nodeResources, tester);
-        Set<HostSpec> content0 = prepare(application, contentCluster0, content0Size, 1, required, nodeResources, tester);
-        Set<HostSpec> content1 = prepare(application, contentCluster1, content1Size, 1, required, nodeResources, tester);
+        Set<HostSpec> container0 = prepare(application, tester, containerCluster0, container0Size, 1, required, nodeResources);
+        Set<HostSpec> container1 = prepare(application, tester, containerCluster1, container1Size, 1, required, nodeResources);
+        Set<HostSpec> content0 = prepare(application, tester, contentCluster0, content0Size, 1, required, nodeResources);
+        Set<HostSpec> content1 = prepare(application, tester, contentCluster1, content1Size, 1, required, nodeResources);
 
         Set<HostSpec> allHosts = new HashSet<>();
         allHosts.addAll(container0);
@@ -707,11 +797,11 @@ public class ProvisioningTest {
         allHosts.addAll(content0);
         allHosts.addAll(content1);
 
-        Function<Integer, Capacity> capacity = count -> Capacity.fromCount(count, Optional.empty(), required, true);
-        int expectedContainer0Size = tester.capacityPolicies().decideSize(capacity.apply(container0Size), containerCluster0.type(), application);
-        int expectedContainer1Size = tester.capacityPolicies().decideSize(capacity.apply(container1Size), containerCluster1.type(), application);
-        int expectedContent0Size = tester.capacityPolicies().decideSize(capacity.apply(content0Size), contentCluster0.type(), application);
-        int expectedContent1Size = tester.capacityPolicies().decideSize(capacity.apply(content1Size), contentCluster1.type(), application);
+        Function<Integer, Capacity> capacity = count -> Capacity.from(new ClusterResources(count, 1, NodeResources.unspecified), required, true);
+        int expectedContainer0Size = tester.capacityPolicies().decideSize(container0Size, capacity.apply(container0Size), containerCluster0, application);
+        int expectedContainer1Size = tester.capacityPolicies().decideSize(container1Size, capacity.apply(container1Size), containerCluster1, application);
+        int expectedContent0Size = tester.capacityPolicies().decideSize(content0Size, capacity.apply(content0Size), contentCluster0, application);
+        int expectedContent1Size = tester.capacityPolicies().decideSize(content1Size, capacity.apply(content1Size), contentCluster1, application);
 
         assertEquals("Hosts in each group cluster is disjunct and the total number of unretired nodes is correct",
                      expectedContainer0Size + expectedContainer1Size + expectedContent0Size + expectedContent1Size,
@@ -730,8 +820,8 @@ public class ProvisioningTest {
         return new SystemState(allHosts, container0, container1, content0, content1);
     }
 
-    private Set<HostSpec> prepare(ApplicationId application, ClusterSpec cluster, int nodeCount, int groups,
-                                  boolean required, NodeResources nodeResources, ProvisioningTester tester) {
+    private Set<HostSpec> prepare(ApplicationId application, ProvisioningTester tester, ClusterSpec cluster, int nodeCount, int groups,
+                                  boolean required, NodeResources nodeResources) {
         if (nodeCount == 0) return Collections.emptySet(); // this is a shady practice
         return new HashSet<>(tester.prepare(application, cluster, nodeCount, groups, required, nodeResources));
     }
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java
index 4e63d3cc79f..a8df47aab1a 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java
@@ -5,6 +5,7 @@ import com.yahoo.component.Version;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.ApplicationName;
 import com.yahoo.config.provision.Capacity;
+import com.yahoo.config.provision.ClusterResources;
 import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.config.provision.DockerImage;
 import com.yahoo.config.provision.Flavor;
@@ -87,7 +88,7 @@ public class ProvisioningTester {
         this.nodeFlavors = nodeFlavors;
         this.clock = new ManualClock();
         this.nodeRepository = new NodeRepository(nodeFlavors, curator, clock, zone, nameResolver,
-                DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), true);
+                                                 DockerImage.fromString("docker-registry.domain.tld:8080/dist/vespa"), true);
         this.orchestrator = orchestrator;
         ProvisionServiceProvider provisionServiceProvider = new MockProvisionServiceProvider(loadBalancerService, hostProvisioner);
         this.provisioner = new NodeRepositoryProvisioner(nodeRepository, zone, provisionServiceProvider, flagSource);
@@ -127,19 +128,19 @@ public class ProvisioningTester {
     }
 
     public List<HostSpec> prepare(ApplicationId application, ClusterSpec cluster, int nodeCount, int groups, boolean required, NodeResources resources) {
-        return prepare(application, cluster, Capacity.fromCount(nodeCount, Optional.ofNullable(resources), required, true), groups);
+        return prepare(application, cluster, Capacity.from(new ClusterResources(nodeCount, groups, resources), required, true));
     }
 
-    public List<HostSpec> prepare(ApplicationId application, ClusterSpec cluster, Capacity capacity, int groups) {
-        return prepare(application, cluster, capacity, groups, true);
+    public List<HostSpec> prepare(ApplicationId application, ClusterSpec cluster, Capacity capacity) {
+        return prepare(application, cluster, capacity, true);
     }
 
-    public List<HostSpec> prepare(ApplicationId application, ClusterSpec cluster, Capacity capacity, int groups, boolean idempotentPrepare) {
+    public List<HostSpec> prepare(ApplicationId application, ClusterSpec cluster, Capacity capacity, boolean idempotentPrepare) {
         Set<String> reservedBefore = toHostNames(nodeRepository.getNodes(application, Node.State.reserved));
         Set<String> inactiveBefore = toHostNames(nodeRepository.getNodes(application, Node.State.inactive));
-        List<HostSpec> hosts1 = provisioner.prepare(application, cluster, capacity, groups, provisionLogger);
+        List<HostSpec> hosts1 = provisioner.prepare(application, cluster, capacity, provisionLogger);
         if (idempotentPrepare) { // prepare twice to ensure idempotence
-            List<HostSpec> hosts2 = provisioner.prepare(application, cluster, capacity, groups, provisionLogger);
+            List<HostSpec> hosts2 = provisioner.prepare(application, cluster, capacity, provisionLogger);
             assertEquals(hosts1, hosts2);
         }
         Set<String> newlyActivated = toHostNames(nodeRepository.getNodes(application, Node.State.reserved));
@@ -158,9 +159,9 @@ public class ProvisioningTester {
     }
 
     public void prepareAndActivateInfraApplication(ApplicationId application, NodeType nodeType, Version version) {
-        ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from(nodeType.toString()), version, false);
+        ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from(nodeType.toString())).vespaVersion(version).build();
         Capacity capacity = Capacity.fromRequiredNodeType(nodeType);
-        List<HostSpec> hostSpecs = prepare(application, cluster, capacity, 1, true);
+        List<HostSpec> hostSpecs = prepare(application, cluster, capacity, true);
         activate(application, hostSpecs);
     }
 
@@ -232,6 +233,10 @@ public class ProvisioningTester {
                 InstanceName.from(UUID.randomUUID().toString()));
     }
 
+    public ApplicationId makeApplicationId(String applicationName) {
+        return ApplicationId.from("tenant", applicationName, "default");
+    }
+
     public List<Node> makeReadyNodes(int n, String flavor) {
         return makeReadyNodes(n, flavor, NodeType.tenant);
     }
@@ -305,7 +310,7 @@ public class ProvisioningTester {
                                                 reservedTo,
                                                 type));
         }
-        nodes = nodeRepository.addNodes(nodes);
+        nodes = nodeRepository.addNodes(nodes, Agent.system);
         return nodes;
     }
 
@@ -328,16 +333,14 @@ public class ProvisioningTester {
             nodes.add(node);
         }
 
-        nodes = nodeRepository.addNodes(nodes);
+        nodes = nodeRepository.addNodes(nodes, Agent.system);
         nodes = nodeRepository.setDirty(nodes, Agent.system, getClass().getSimpleName());
         nodeRepository.setReady(nodes, Agent.system, getClass().getSimpleName());
 
         ConfigServerApplication application = new ConfigServerApplication();
-        List<HostSpec> hosts = prepare(
-                application.getApplicationId(),
-                application.getClusterSpecWithVersion(configServersVersion),
-                application.getCapacity(),
-                1);
+        List<HostSpec> hosts = prepare(application.getApplicationId(),
+                                       application.getClusterSpecWithVersion(configServersVersion),
+                                       application.getCapacity());
         activate(application.getApplicationId(), new HashSet<>(hosts));
         return nodeRepository.getNodes(application.getApplicationId(), Node.State.active);
     }
@@ -400,7 +403,7 @@ public class ProvisioningTester {
             nodes.add(nodeRepository.createNode("openstack-id", hostname, parentHostId,
                                                 new Flavor(flavor), NodeType.tenant));
         }
-        nodes = nodeRepository.addNodes(nodes);
+        nodes = nodeRepository.addNodes(nodes, Agent.system);
         nodes = nodeRepository.setDirty(nodes, Agent.system, getClass().getSimpleName());
         nodeRepository.setReady(nodes, Agent.system, getClass().getSimpleName());
         return nodes;
@@ -409,30 +412,30 @@ public class ProvisioningTester {
     public void deployZoneApp() {
         ApplicationId applicationId = makeApplicationId();
         List<HostSpec> list = prepare(applicationId,
-                                             ClusterSpec.request(ClusterSpec.Type.container,
-                                                                 ClusterSpec.Id.from("node-admin"),
-                                                                 Version.fromString("6.42"),
-                                                                 false),
-                                             Capacity.fromRequiredNodeType(NodeType.host),
-                                             1);
+                                      ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("node-admin")).vespaVersion("6.42").build(),
+                                      Capacity.fromRequiredNodeType(NodeType.host));
         activate(applicationId, Set.copyOf(list));
     }
 
+    public ClusterSpec clusterSpec() {
+        return ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("test")).vespaVersion("6.42").build();
+    }
+
+    public List<Node> deploy(ApplicationId application, Capacity capacity) {
+        return deploy(application, clusterSpec(), capacity);
+    }
+
+    public List<Node> deploy(ApplicationId application, ClusterSpec cluster, Capacity capacity) {
+        List<HostSpec> prepared = prepare(application, cluster, capacity);
+        activate(application, Set.copyOf(prepared));
+        return getNodes(application, Node.State.active).asList();
+    }
+
     /** Returns the hosts from the input list which are not retired */
     public List<HostSpec> nonRetired(Collection<HostSpec> hosts) {
         return hosts.stream().filter(host -> ! host.membership().get().retired()).collect(Collectors.toList());
     }
 
-    public void assertNumberOfNodesWithFlavor(List<HostSpec> hostSpecs, String flavor, int expectedCount) {
-        long actualNodesWithFlavor = hostSpecs.stream()
-                .map(HostSpec::hostname)
-                .map(this::getNodeFlavor)
-                .map(Flavor::name)
-                .filter(name -> name.equals(flavor))
-                .count();
-        assertEquals(expectedCount, actualNodesWithFlavor);
-    }
-
     public void assertAllocatedOn(String explanation, String hostFlavor, ApplicationId app) {
         for (Node node : nodeRepository.getNodes(app)) {
             Node parent = nodeRepository.getNode(node.parentHostname().get()).get();
@@ -440,17 +443,6 @@ public class ProvisioningTester {
         }
     }
 
-    public void printFreeResources() {
-        for (Node host : nodeRepository().getNodes(NodeType.host)) {
-            NodeResources free = host.flavor().resources();
-            for (Node child : nodeRepository().getNodes(NodeType.tenant)) {
-                if (child.parentHostname().get().equals(host.hostname()))
-                    free = free.subtract(child.flavor().resources());
-            }
-            System.out.println(host.flavor().name() + " node. Free resources: " + free);
-        }
-    }
-
     public int hostFlavorCount(String hostFlavor, ApplicationId app) {
         return (int)nodeRepository().getNodes(app).stream()
                                                   .map(n -> nodeRepository().getNode(n.parentHostname().get()).get())
@@ -458,11 +450,8 @@ public class ProvisioningTester {
                                                   .count();
     }
 
-    private Flavor getNodeFlavor(String hostname) {
-        return nodeRepository.getNode(hostname).map(Node::flavor).orElseThrow(() -> new RuntimeException("No flavor for host " + hostname));
-    }
-
     public static final class Builder {
+
         private Curator curator;
         private FlavorsConfig flavorsConfig;
         private Zone zone;
@@ -540,4 +529,22 @@ public class ProvisioningTester {
         @Override public void log(Level level, String message) { }
     }
 
+    public IdentityHostResourcesCalculator identityHostResourcesCalculator() {
+        return new IdentityHostResourcesCalculator();
+    }
+
+    private static class IdentityHostResourcesCalculator implements HostResourcesCalculator {
+
+        @Override
+        public NodeResources realResourcesOf(Node node) {
+            return node.flavor().resources();
+        }
+
+        @Override
+        public NodeResources advertisedResourcesOf(Flavor flavor) {
+            return flavor.resources();
+        }
+
+    }
+
 }
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/VirtualNodeProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/VirtualNodeProvisioningTest.java
index f8c6e31cfd7..51261c29a71 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/VirtualNodeProvisioningTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/VirtualNodeProvisioningTest.java
@@ -1,7 +1,6 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.provision.provisioning;
 
-import com.yahoo.component.Version;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.config.provision.Environment;
@@ -36,8 +35,8 @@ import static org.junit.Assert.assertNotNull;
 public class VirtualNodeProvisioningTest {
 
     private static final NodeResources flavor = new NodeResources(4, 8, 100, 1);
-    private static final ClusterSpec contentClusterSpec = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent"), Version.fromString("6.42"), false);
-    private static final ClusterSpec containerClusterSpec = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("myContainer"), Version.fromString("6.42"), false);
+    private static final ClusterSpec contentClusterSpec = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent")).vespaVersion("6.42").build();
+    private static final ClusterSpec containerClusterSpec = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("myContainer")).vespaVersion("6.42").build();
 
     private ProvisioningTester tester = new ProvisioningTester.Builder().build();
     private ApplicationId applicationId = tester.makeApplicationId();
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java
index 1a1cbdea6b8..0001c344dd3 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java
@@ -9,10 +9,12 @@ import com.yahoo.config.provision.NodeType;
 import com.yahoo.config.provision.TenantName;
 import com.yahoo.io.IOUtils;
 import com.yahoo.text.Utf8;
+import com.yahoo.vespa.applicationmodel.HostName;
 import com.yahoo.vespa.hosted.provision.NodeRepository;
 import com.yahoo.vespa.hosted.provision.maintenance.OsUpgradeActivator;
 import com.yahoo.vespa.hosted.provision.testutils.ContainerConfig;
 import com.yahoo.vespa.hosted.provision.testutils.MockNodeRepository;
+import com.yahoo.vespa.hosted.provision.testutils.OrchestratorMock;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.ComparisonFailure;
@@ -93,17 +95,6 @@ public class RestApiTest {
         assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/host2.yahoo.com"),
                                "\"rebootGeneration\":4");
 
-        // POST deactivation of a maintenance job
-        assertResponse(new Request("http://localhost:8080/nodes/v2/maintenance/inactive/NodeFailer",
-                                   new byte[0], Request.Method.POST),
-                       "{\"message\":\"Deactivated job 'NodeFailer'\"}");
-        // GET a list of all maintenance jobs
-        assertFile(new Request("http://localhost:8080/nodes/v2/maintenance/"), "maintenance.json");
-        // DELETE deactivation of a maintenance job
-        assertResponse(new Request("http://localhost:8080/nodes/v2/maintenance/inactive/NodeFailer",
-                                   new byte[0], Request.Method.DELETE),
-                       "{\"message\":\"Re-activated job 'NodeFailer'\"}");
-
         // POST new nodes
         assertResponse(new Request("http://localhost:8080/nodes/v2/node",
                                    ("[" + asNodeJson("host8.yahoo.com", "default", "127.0.8.1") + "," + // test with only 1 ip address
@@ -122,7 +113,7 @@ public class RestApiTest {
         assertResponse(new Request("http://localhost:8080/nodes/v2/node",
                                    ("[" + asNodeJson("host8.yahoo.com", "default", "127.0.254.8") + "]").getBytes(StandardCharsets.UTF_8),
                                    Request.Method.POST), 400,
-                       "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Cannot add host8.yahoo.com: A node with this name already exists\"}");
+                       "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Cannot add provisioned host host8.yahoo.com: A node with this name already exists\"}");
 
         // DELETE a provisioned node
         assertResponse(new Request("http://localhost:8080/nodes/v2/node/host9.yahoo.com",
@@ -216,9 +207,6 @@ public class RestApiTest {
                                    Utf8.toBytes("{\"wantToRetire\": true}"), Request.Method.PATCH),
                        "{\"message\":\"Updated host4.yahoo.com\"}");
         assertResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com",
-                        Utf8.toBytes("{\"wantToDeprovision\": true}"), Request.Method.PATCH),
-                "{\"message\":\"Updated host4.yahoo.com\"}");
-        assertResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com",
                         Utf8.toBytes("{\"currentVespaVersion\": \"6.43.0\",\"currentDockerImage\": \"docker-registry.domain.tld:8080/dist/vespa:6.45.0\"}"), Request.Method.PATCH),
                         "{\"message\":\"Updated host4.yahoo.com\"}");
         assertResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com",
@@ -227,6 +215,9 @@ public class RestApiTest {
         assertResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com",
                                    Utf8.toBytes("{\"modelName\": \"foo\"}"), Request.Method.PATCH),
                        "{\"message\":\"Updated dockerhost1.yahoo.com\"}");
+        assertResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com",
+                        Utf8.toBytes("{\"wantToDeprovision\": true}"), Request.Method.PATCH),
+                "{\"message\":\"Updated dockerhost1.yahoo.com\"}");
         assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com"), "\"modelName\":\"foo\"");
         assertResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com",
                                    Utf8.toBytes("{\"modelName\": null}"), Request.Method.PATCH),
@@ -234,10 +225,39 @@ public class RestApiTest {
         assertPartialResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com"), "modelName", false);
         container.handleRequest((new Request("http://localhost:8080/nodes/v2/upgrade/tenant", Utf8.toBytes("{\"dockerImage\": \"docker.domain.tld/my/image\"}"), Request.Method.PATCH)));
 
+        ((OrchestratorMock) container.components().getComponent(OrchestratorMock.class.getName()))
+                .suspend(new HostName("host4.yahoo.com"));
+
         assertFile(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com"), "node4-after-changes.json");
     }
 
     @Test
+    public void maintenance_requests() throws Exception {
+        // POST deactivation of a maintenance job
+        assertResponse(new Request("http://localhost:8080/nodes/v2/maintenance/inactive/NodeFailer",
+                                   new byte[0], Request.Method.POST),
+                       "{\"message\":\"Deactivated job 'NodeFailer'\"}");
+        // GET a list of all maintenance jobs
+        assertFile(new Request("http://localhost:8080/nodes/v2/maintenance/"), "maintenance.json");
+
+        // DELETE deactivation of a maintenance job
+        assertResponse(new Request("http://localhost:8080/nodes/v2/maintenance/inactive/NodeFailer",
+                                   new byte[0], Request.Method.DELETE),
+                       "{\"message\":\"Re-activated job 'NodeFailer'\"}");
+
+        // POST run of a maintenance job
+        assertResponse(new Request("http://localhost:8080/nodes/v2/maintenance/run/PeriodicApplicationMaintainer",
+                                   new byte[0], Request.Method.POST),
+                       "{\"message\":\"Executed job 'PeriodicApplicationMaintainer'\"}");
+
+        // POST run of unknown maintenance job
+        assertResponse(new Request("http://localhost:8080/nodes/v2/maintenance/run/foo",
+                                   new byte[0], Request.Method.POST),
+                       400,
+                       "{\"error-code\":\"BAD_REQUEST\",\"message\":\"No such job 'foo'\"}");
+    }
+
+    @Test
     public void post_with_patch_method_override_in_header_is_handled_as_patch() throws Exception  {
         Request req = new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com",
                 Utf8.toBytes("{\"currentRestartGeneration\": 1}"), Request.Method.POST);
@@ -358,7 +378,7 @@ public class RestApiTest {
                 "{\"message\":\"Updated host12.yahoo.com\"}");
         assertResponse(new Request("http://localhost:8080/nodes/v2/state/ready/host12.yahoo.com", new byte[0], Request.Method.PUT),
                 400,
-                "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Node host12.yahoo.com cannot be readied because it has " +
+                "{\"error-code\":\"BAD_REQUEST\",\"message\":\"provisioned host host12.yahoo.com cannot be readied because it has " +
                         "hard failures: [diskSpace reported 1970-01-01T00:00:00.002Z: " + msg + "]\"}");
     }
 
@@ -423,7 +443,7 @@ public class RestApiTest {
         assertResponse(new Request("http://localhost:8080/nodes/v2/state/ready/host1.yahoo.com",
                                    new byte[0], Request.Method.PUT),
                        400,
-                        "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Cannot make host1.yahoo.com available for new allocation, must be in state dirty, but was in failed\"}");
+                        "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Cannot make failed host host1.yahoo.com allocated to tenant1.application1.instance1 as 'container/id1/0/0' available for new allocation as it is not in state [dirty]\"}");
 
         // (... while dirty then ready works (the ready move will be initiated by node maintenance))
         assertResponse(new Request("http://localhost:8080/nodes/v2/state/dirty/host1.yahoo.com",
@@ -439,7 +459,7 @@ public class RestApiTest {
                        "{\"message\":\"Moved host2.yahoo.com to parked\"}");
         assertResponse(new Request("http://localhost:8080/nodes/v2/state/ready/host2.yahoo.com",
                                    new byte[0], Request.Method.PUT),
-                       400, "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Cannot make host2.yahoo.com available for new allocation, must be in state dirty, but was in parked\"}");
+                       400, "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Cannot make parked host host2.yahoo.com allocated to tenant2.application2.instance2 as 'content/id2/0/0' available for new allocation as it is not in state [dirty]\"}");
         // (... while dirty then ready works (the ready move will be initiated by node maintenance))
         assertResponse(new Request("http://localhost:8080/nodes/v2/state/dirty/host2.yahoo.com",
                                    new byte[0], Request.Method.PUT),
@@ -456,7 +476,7 @@ public class RestApiTest {
         // Attempt to DELETE allocated node
         assertResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com",
                                    new byte[0], Request.Method.DELETE),
-                       400, "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Failed to delete host4.yahoo.com: Node is currently allocated and cannot be removed: allocated to tenant3.application3.instance3 as 'content/id3/0/0'\"}");
+                       400, "{\"error-code\":\"BAD_REQUEST\",\"message\":\"active child node host4.yahoo.com allocated to tenant3.application3.instance3 as 'content/id3/0/0' is currently allocated and cannot be removed\"}");
 
         // PUT current restart generation with string instead of long
         assertResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com",
@@ -478,7 +498,7 @@ public class RestApiTest {
                                    ("[" + asNodeJson("host8.yahoo.com", "default", "127.0.254.1", "::254:1") + "," +
                                     asNodeJson("host8.yahoo.com", "large-variant", "127.0.253.1", "::253:1") + "]").getBytes(StandardCharsets.UTF_8),
                                    Request.Method.POST), 400,
-                       "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Cannot add host8.yahoo.com: A node with this name already exists\"}");
+                       "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Cannot add nodes: provisioned host host8.yahoo.com is duplicated in the argument list\"}");
 
         // Attempt to PATCH field not relevant for child node
         assertResponse(new Request("http://localhost:8080/nodes/v2/node/test-node-pool-102-2",
@@ -536,11 +556,13 @@ public class RestApiTest {
                                 "    \"actualCpuCores\": {" +
                                 "      \"createdMillis\": 1, " +
                                 "      \"description\": \"Actual number of CPU cores (2) differs from spec (4)\"," +
+                                "      \"type\": \"HARD_FAIL\"," +
                                 "      \"value\":2" +
                                 "    }," +
                                 "    \"diskSpace\": {" +
                                 "      \"createdMillis\": 2, " +
                                 "      \"description\": \"Actual disk space (2TB) differs from spec (3TB)\"," +
+                                "      \"type\": \"HARD_FAIL\"," +
                                 "      \"details\": {" +
                                 "        \"inGib\": 3," +
                                 "        \"disks\": [\"/dev/sda1\", \"/dev/sdb3\"]" +
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/maintenance.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/maintenance.json
index 02746f1c79a..ab608bac2b4 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/maintenance.json
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/maintenance.json
@@ -1,6 +1,9 @@
 {
   "jobs": [
     {
+      "name": "AutoscalingMaintainer"
+    },
+    {
       "name": "CapacityReportMaintainer"
     },
     {
@@ -25,6 +28,9 @@
       "name": "NodeFailer"
     },
     {
+      "name": "NodeMetricsDbMaintainer"
+    },
+    {
       "name": "NodeRebooter"
     },
     {
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4-after-changes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4-after-changes.json
index f663f4adb8d..af3552945d9 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4-after-changes.json
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4-after-changes.json
@@ -27,14 +27,15 @@
   "wantedDockerImage": "docker.domain.tld/my/image:6.42.0",
   "wantedVespaVersion": "6.42.0",
   "requestedResources": { "vcpu":1.0, "memoryGb":4.0, "diskGb":100.0, "bandwidthGbps":1.0, "diskSpeed":"fast", "storageType":"any" },
-  "allowedToBeDown": false,
+  "allowedToBeDown": true,
+  "suspendedSinceMillis": 0,
   "rebootGeneration": 3,
   "currentRebootGeneration": 1,
   "vespaVersion": "6.43.0",
   "currentDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.45.0",
   "failCount": 1,
   "wantToRetire": true,
-  "wantToDeprovision": true,
+  "wantToDeprovision": false,
   "history": [
     {
       "event": "provisioned",
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-reports-2.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-reports-2.json
index d33c1c9e743..a3d53798d7c 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-reports-2.json
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-reports-2.json
@@ -30,7 +30,7 @@
   "currentRebootGeneration": 0,
   "failCount": 0,
   "wantToRetire": false,
-  "wantToDeprovision": false,
+  "wantToDeprovision": true,
   "history": [
     {
       "event": "provisioned",
@@ -65,6 +65,7 @@
     "diskSpace": {
       "createdMillis": 2,
       "description": "Actual disk space (2TB) differs from spec (3TB)",
+      "type":"HARD_FAIL",
       "details": {
         "inGib": 3,
         "disks": [
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-reports.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-reports.json
index 4119e46e225..67b8d67c7f1 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-reports.json
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-reports.json
@@ -30,7 +30,7 @@
   "currentRebootGeneration": 0,
   "failCount": 0,
   "wantToRetire": false,
-  "wantToDeprovision": false,
+  "wantToDeprovision": true,
   "history": [
     {
       "event": "provisioned",
@@ -62,11 +62,13 @@
     "actualCpuCores": {
       "createdMillis": 1,
       "description": "Actual number of CPU cores (2) differs from spec (4)",
+      "type":"HARD_FAIL",
       "value": 2
     },
     "diskSpace": {
       "createdMillis": 2,
       "description": "Actual disk space (2TB) differs from spec (3TB)",
+      "type":"HARD_FAIL",
       "details": {
         "inGib": 3,
         "disks": [
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/states-recursive.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/states-recursive.json
index c13dca5439a..f8343756559 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/states-recursive.json
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/states-recursive.json
@@ -58,6 +58,11 @@
       "url": "http://localhost:8080/nodes/v2/state/parked",
       "nodes": [
       ]
+    },
+    "deprovisioned": {
+      "url": "http://localhost:8080/nodes/v2/state/deprovisioned",
+      "nodes": [
+      ]
     }
   }
 }
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/states.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/states.json
index 4b2de7532dd..69579148df3 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/states.json
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/states.json
@@ -23,6 +23,9 @@
     },
     "parked": {
       "url": "http://localhost:8080/nodes/v2/state/parked"
+    },
+    "deprovisioned": {
+      "url": "http://localhost:8080/nodes/v2/state/deprovisioned"
     }
   }
 }
\ No newline at end of file
diff --git a/orchestrator-restapi/src/main/java/com/yahoo/vespa/orchestrator/restapi/wire/GetHostResponse.java b/orchestrator-restapi/src/main/java/com/yahoo/vespa/orchestrator/restapi/wire/GetHostResponse.java
index 3f14579dfba..2a582020bb3 100644
--- a/orchestrator-restapi/src/main/java/com/yahoo/vespa/orchestrator/restapi/wire/GetHostResponse.java
+++ b/orchestrator-restapi/src/main/java/com/yahoo/vespa/orchestrator/restapi/wire/GetHostResponse.java
@@ -18,6 +18,7 @@ public class GetHostResponse {
 
     public static final String FIELD_NAME_HOSTNAME = "hostname";
     public static final String FIELD_NAME_STATE = "state";
+    public static final String FIELD_NAME_SUSPENDED_SINCE = "suspendedSince";
     public static final String FIELD_NAME_APPLICATION_URL = "applicationUrl";
     public static final String FIELD_NAME_SERVICES = "services";
 
@@ -25,15 +26,18 @@ public class GetHostResponse {
     private final String state;
     private final String applicationUrl;
     private final List<HostService> services;
+    private final String suspendedSince;
 
     @JsonCreator
     public GetHostResponse(
             @JsonProperty(FIELD_NAME_HOSTNAME) String hostname,
             @JsonProperty(FIELD_NAME_STATE) String state,
+            @JsonProperty(FIELD_NAME_SUSPENDED_SINCE) String suspendedSince,
             @JsonProperty(FIELD_NAME_APPLICATION_URL) String applicationUrl,
             @JsonProperty(FIELD_NAME_SERVICES) List<HostService> services) {
         this.hostname = hostname;
         this.state = state;
+        this.suspendedSince = suspendedSince;
         this.applicationUrl = applicationUrl;
         this.services = services;
     }
@@ -48,6 +52,11 @@ public class GetHostResponse {
         return state;
     }
 
+    @JsonProperty(FIELD_NAME_SUSPENDED_SINCE)
+    public String suspendedSince() {
+        return suspendedSince;
+    }
+
     @JsonProperty(FIELD_NAME_APPLICATION_URL)
     public String applicationUrl() {
         return applicationUrl;
@@ -65,12 +74,13 @@ public class GetHostResponse {
         GetHostResponse that = (GetHostResponse) o;
         return Objects.equals(hostname, that.hostname) &&
                 Objects.equals(state, that.state) &&
+                Objects.equals(suspendedSince, that.suspendedSince) &&
                 Objects.equals(applicationUrl, that.applicationUrl) &&
                 Objects.equals(services, that.services);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(hostname, state, applicationUrl, services);
+        return Objects.hash(hostname, state, suspendedSince, applicationUrl, services);
     }
 }
diff --git a/orchestrator-restapi/src/main/java/com/yahoo/vespa/orchestrator/restapi/wire/WireHostInfo.java b/orchestrator-restapi/src/main/java/com/yahoo/vespa/orchestrator/restapi/wire/WireHostInfo.java
new file mode 100644
index 00000000000..39c93291bad
--- /dev/null
+++ b/orchestrator-restapi/src/main/java/com/yahoo/vespa/orchestrator/restapi/wire/WireHostInfo.java
@@ -0,0 +1,38 @@
+package com.yahoo.vespa.orchestrator.restapi.wire;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import java.time.Instant;
+import java.util.Objects;
+
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class WireHostInfo {
+    private static final String HOST_STATUS_FIELD = "hostStatus";
+    private static final String SUSPENDED_SINCE_FIELD = "suspendedSince";
+
+    private final String hostStatus;
+    private final String suspendedSinceUtcOrNull;
+
+    /**
+     * @param hostStatus              The host status, e.g. NO_REMARKS.
+     * @param suspendedSinceUtcOrNull The time the host was suspended in the format
+     *                                {@link Instant#toString()}, or null if not suspended
+     *                                (NO_REMARKS).
+     */
+    @JsonCreator
+    public WireHostInfo(@JsonProperty(HOST_STATUS_FIELD) String hostStatus,
+                        @JsonProperty(SUSPENDED_SINCE_FIELD) String suspendedSinceUtcOrNull) {
+        this.hostStatus = Objects.requireNonNull(hostStatus);
+        this.suspendedSinceUtcOrNull = suspendedSinceUtcOrNull;
+    }
+
+    @JsonProperty(HOST_STATUS_FIELD)
+    public String hostStatus() { return hostStatus; }
+
+    @JsonProperty(SUSPENDED_SINCE_FIELD)
+    public String getSuspendedSinceUtcOrNull() { return suspendedSinceUtcOrNull; }
+}
diff --git a/orchestrator/pom.xml b/orchestrator/pom.xml
index 3ebc87010fb..fed0e617c79 100644
--- a/orchestrator/pom.xml
+++ b/orchestrator/pom.xml
@@ -42,6 +42,12 @@
     </dependency>
     <dependency>
       <groupId>com.yahoo.vespa</groupId>
+      <artifactId>flags</artifactId>
+      <version>${project.version}</version>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>com.yahoo.vespa</groupId>
       <artifactId>vespajlib</artifactId>
       <version>${project.version}</version>
       <scope>provided</scope>
@@ -59,6 +65,12 @@
       <scope>test</scope>
     </dependency>
     <dependency>
+      <groupId>com.yahoo.vespa</groupId>
+      <artifactId>zookeeper-server-common</artifactId>
+      <version>${project.version}</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
       <groupId>junit</groupId>
       <artifactId>junit</artifactId>
       <scope>test</scope>
@@ -69,12 +81,6 @@
       <scope>test</scope>
     </dependency>
     <dependency>
-      <groupId>org.easytesting</groupId>
-      <artifactId>fest-assert</artifactId>
-      <version>1.4</version>
-      <scope>test</scope>
-    </dependency>
-    <dependency>
       <groupId>com.yahoo.vespa</groupId>
       <artifactId>config-provisioning</artifactId>
       <version>${project.version}</version>
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/Host.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/Host.java
index bda9505d72b..8f5f00af7a0 100644
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/Host.java
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/Host.java
@@ -4,23 +4,23 @@ package com.yahoo.vespa.orchestrator;
 import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference;
 import com.yahoo.vespa.applicationmodel.HostName;
 import com.yahoo.vespa.applicationmodel.ServiceInstance;
-import com.yahoo.vespa.orchestrator.status.HostStatus;
+import com.yahoo.vespa.orchestrator.status.HostInfo;
 
 import java.util.List;
 
 public class Host {
 
     private final HostName hostName;
-    private final HostStatus hostStatus;
+    private final HostInfo hostInfo;
     private final ApplicationInstanceReference applicationInstanceReference;
     private final List<ServiceInstance> serviceInstances;
 
     public Host(HostName hostName,
-                HostStatus hostStatus,
+                HostInfo hostInfo,
                 ApplicationInstanceReference applicationInstanceReference,
                 List<ServiceInstance> serviceInstances) {
         this.hostName = hostName;
-        this.hostStatus = hostStatus;
+        this.hostInfo = hostInfo;
         this.applicationInstanceReference = applicationInstanceReference;
         this.serviceInstances = serviceInstances;
     }
@@ -29,8 +29,8 @@ public class Host {
         return hostName;
     }
 
-    public HostStatus getHostStatus() {
-        return hostStatus;
+    public HostInfo getHostInfo() {
+        return hostInfo;
     }
 
     public ApplicationInstanceReference getApplicationInstanceReference() {
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/InstanceLookupService.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/InstanceLookupService.java
deleted file mode 100644
index 554b6e11501..00000000000
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/InstanceLookupService.java
+++ /dev/null
@@ -1,20 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.orchestrator;
-
-import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference;
-import com.yahoo.vespa.applicationmodel.ApplicationInstance;
-import com.yahoo.vespa.applicationmodel.HostName;
-
-import java.util.Optional;
-import java.util.Set;
-
-/**
- * @author oyving
- */
-public interface InstanceLookupService {
-
-    Optional<ApplicationInstance> findInstanceById(ApplicationInstanceReference applicationInstanceReference);
-    Optional<ApplicationInstance> findInstanceByHost(HostName hostName);
-    Set<ApplicationInstanceReference> knownInstances();
-
-}
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/Orchestrator.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/Orchestrator.java
index 59b320cf501..9d2d72277e5 100644
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/Orchestrator.java
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/Orchestrator.java
@@ -2,11 +2,12 @@
 package com.yahoo.vespa.orchestrator;
 
 import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference;
 import com.yahoo.vespa.applicationmodel.HostName;
-import com.yahoo.vespa.orchestrator.model.NodeGroup;
 import com.yahoo.vespa.orchestrator.policy.BatchHostStateChangeDeniedException;
 import com.yahoo.vespa.orchestrator.policy.HostStateChangeDeniedException;
 import com.yahoo.vespa.orchestrator.status.ApplicationInstanceStatus;
+import com.yahoo.vespa.orchestrator.status.HostInfo;
 import com.yahoo.vespa.orchestrator.status.HostStatus;
 
 import java.util.List;
@@ -44,17 +45,23 @@ public interface Orchestrator {
      *
      * @param hostName The FQDN which are used in the noderepo.
      * @return The enum describing the current state.
-     * @throws HostNameNotFoundException if hostName is unrecognized (in node repo)
+     * @throws HostNameNotFoundException if hostName is not associated with any application
      */
     HostStatus getNodeStatus(HostName hostName) throws HostNameNotFoundException;
 
+    /** Get host info for hostname in application, returning no-remarks if not in application. */
+    HostInfo getHostInfo(ApplicationInstanceReference reference, HostName hostname);
+
     /**
-     * Returns a not necessarily consistent mapping from host names to their statuses, for hosts known by this.
+     * Returns a lambda, which when invoked with a hostname, returns its current host info.
+     *
+     * <p>When invoked multiple times, the hostname/host info mapping is not necessarily consistent.
+     * Prefer this to {@link #getNodeStatus(HostName)} when consistency is not required,
+     * and when doing bulk reads.</p>
      *
-     * Prefer this to {@link #getNodeStatus(HostName)} when consistency is not required, and when doing bulk reads.
      * @return a mapping from host names to their statuses. Unknown hosts map to {@code Optional.empty()}.
      */
-    Function<HostName, Optional<HostStatus>> getNodeStatuses();
+    Function<HostName, Optional<HostInfo>> getHostResolver();
 
     void setNodeStatus(HostName hostName, HostStatus state) throws OrchestrationException;
 
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorContext.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorContext.java
index d3bdaa6dc64..b1f205f69d8 100644
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorContext.java
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorContext.java
@@ -1,13 +1,17 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.orchestrator;
 
+import com.yahoo.log.LogLevel;
 import com.yahoo.time.TimeBudget;
+import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference;
 import com.yahoo.vespa.orchestrator.controller.ClusterControllerClientTimeouts;
 
 import java.time.Clock;
 import java.time.Duration;
 import java.time.Instant;
+import java.util.HashMap;
 import java.util.Optional;
+import java.util.logging.Logger;
 
 /**
  * Context for an operation (or suboperation) of the Orchestrator that needs to pass through to the backend,
@@ -15,29 +19,60 @@ import java.util.Optional;
  *
  * @author hakonhall
  */
-public class OrchestratorContext {
+public class OrchestratorContext implements AutoCloseable {
+    private static final Logger logger = Logger.getLogger(OrchestratorContext.class.getName());
     private static final Duration DEFAULT_TIMEOUT_FOR_SINGLE_OP = Duration.ofSeconds(10);
     private static final Duration DEFAULT_TIMEOUT_FOR_BATCH_OP = Duration.ofSeconds(60);
+    private static final Duration DEFAULT_TIMEOUT_FOR_ADMIN_OP = Duration.ofMinutes(5);
     private static final Duration TIMEOUT_OVERHEAD = Duration.ofMillis(500);
 
+    private final Optional<OrchestratorContext> parent;
     private final Clock clock;
     private final TimeBudget timeBudget;
     private final boolean probe;
+    private final boolean largeLocks;
+    private final boolean usePermanentlyDownStatus;
+
+    // The key set is the set of applications locked by this context tree: Only the
+    // root context has a non-empty set. The value is an unlock callback to be called
+    // when the root context is closed.
+    private final HashMap<ApplicationInstanceReference, Runnable> locks = new HashMap<>();
 
     /** Create an OrchestratorContext for operations on multiple applications. */
     public static OrchestratorContext createContextForMultiAppOp(Clock clock) {
-        return new OrchestratorContext(clock, TimeBudget.fromNow(clock, DEFAULT_TIMEOUT_FOR_BATCH_OP), false);
+        return new OrchestratorContext(null, clock, TimeBudget.fromNow(clock, DEFAULT_TIMEOUT_FOR_BATCH_OP),
+                false, // probe
+                true, // large locks
+                false); // use permanently down status
     }
 
     /** Create an OrchestratorContext for an operation on a single application. */
     public static OrchestratorContext createContextForSingleAppOp(Clock clock) {
-        return new OrchestratorContext(clock, TimeBudget.fromNow(clock, DEFAULT_TIMEOUT_FOR_SINGLE_OP), false);
+        return createContextForSingleAppOp(clock, false);
+    }
+
+    public static OrchestratorContext createContextForSingleAppOp(Clock clock, boolean usePermanentlyDownStatus) {
+        return new OrchestratorContext(null, clock, TimeBudget.fromNow(clock, DEFAULT_TIMEOUT_FOR_SINGLE_OP),
+                false, false, usePermanentlyDownStatus);
+    }
+
+    public static OrchestratorContext createContextForAdminOp(Clock clock) {
+        return new OrchestratorContext(null, clock, TimeBudget.fromNow(clock, DEFAULT_TIMEOUT_FOR_ADMIN_OP),
+                false, false, false);
     }
 
-    private OrchestratorContext(Clock clock, TimeBudget timeBudget, boolean probe) {
+    private OrchestratorContext(OrchestratorContext parentOrNull,
+                                Clock clock,
+                                TimeBudget timeBudget,
+                                boolean probe,
+                                boolean largeLocks,
+                                boolean usePermanentlyDownStatus) {
+        this.parent = Optional.ofNullable(parentOrNull);
         this.clock = clock;
         this.timeBudget = timeBudget;
         this.probe = probe;
+        this.largeLocks = largeLocks;
+        this.usePermanentlyDownStatus = usePermanentlyDownStatus;
     }
 
     public Duration getTimeLeft() {
@@ -53,12 +88,50 @@ public class OrchestratorContext {
         return probe;
     }
 
+    /** Whether application locks acquired during probing of a batch suspend should be closed after the non-probe is done. */
+    public boolean largeLocks() { return largeLocks; }
+
+    /** Whether the PERMANENTLY_DOWN host status should be used (where appropriate). */
+    public boolean usePermanentlyDownStatus() { return usePermanentlyDownStatus; }
+
+    /**
+     * Returns true if 1. large locks is enabled, and 2.
+     * {@link #registerLockAcquisition(ApplicationInstanceReference, Runnable) registerLockAcquisition}
+     * has been invoked on any context below the root context that returned true.
+     */
+    public boolean hasLock(ApplicationInstanceReference application) {
+        return parent.map(p -> p.hasLock(application)).orElseGet(() -> locks.containsKey(application));
+    }
+
+    /**
+     * Returns true if large locks is enabled in the root context, and in case the unlock callback
+     * will be invoked when the root context is closed.
+     */
+    public boolean registerLockAcquisition(ApplicationInstanceReference application, Runnable unlock) {
+        if (parent.isPresent()) {
+            return parent.get().registerLockAcquisition(application, unlock);
+        }
+
+        if (!largeLocks) {
+            return false;
+        }
+
+        if (locks.containsKey(application)) {
+            unlock.run();
+            throw new IllegalStateException("Application " + application + " was already associated with a lock");
+        }
+
+        locks.put(application, unlock);
+
+        return true;
+    }
+
     /** Create OrchestratorContext to use within an application lock. */
     public OrchestratorContext createSubcontextWithinLock() {
         // Move deadline towards past by a fixed amount to ensure there's time to process exceptions and
         // access ZooKeeper before the lock times out.
         TimeBudget subTimeBudget = timeBudget.withDeadline(timeBudget.deadline().get().minus(TIMEOUT_OVERHEAD));
-        return new OrchestratorContext(clock, subTimeBudget, probe);
+        return new OrchestratorContext(this, clock, subTimeBudget, probe, largeLocks, usePermanentlyDownStatus);
     }
 
     /** Create an OrchestratorContext for an operation on a single application, but limited to current timeout. */
@@ -70,9 +143,18 @@ public class OrchestratorContext {
             deadline = maxDeadline;
         }
 
-        return new OrchestratorContext(
-                clock,
-                TimeBudget.from(clock, now, Optional.of(Duration.between(now, deadline))),
-                probe);
+        TimeBudget timeBudget = TimeBudget.from(clock, now, Optional.of(Duration.between(now, deadline)));
+        return new OrchestratorContext(this, clock, timeBudget, probe, largeLocks, usePermanentlyDownStatus);
+    }
+
+    @Override
+    public void close() {
+        locks.forEach((application, unlock) -> {
+            try {
+                unlock.run();
+            } catch (RuntimeException e) {
+                logger.log(LogLevel.ERROR, "Failed run on close : " + e.getMessage());
+            }
+        });
     }
 }
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java
index c9d234893cb..baa98790d28 100644
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java
@@ -5,13 +5,16 @@ import com.google.common.util.concurrent.UncheckedTimeoutException;
 import com.google.inject.Inject;
 import com.yahoo.cloud.config.ConfigserverConfig;
 import com.yahoo.config.provision.ApplicationId;
-import com.yahoo.log.LogLevel;
 import com.yahoo.vespa.applicationmodel.ApplicationInstance;
 import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference;
 import com.yahoo.vespa.applicationmodel.ClusterId;
 import com.yahoo.vespa.applicationmodel.HostName;
 import com.yahoo.vespa.applicationmodel.ServiceCluster;
 import com.yahoo.vespa.applicationmodel.ServiceInstance;
+import com.yahoo.vespa.flags.BooleanFlag;
+import com.yahoo.vespa.flags.FetchVector;
+import com.yahoo.vespa.flags.FlagSource;
+import com.yahoo.vespa.flags.Flags;
 import com.yahoo.vespa.orchestrator.config.OrchestratorConfig;
 import com.yahoo.vespa.orchestrator.controller.ClusterControllerClient;
 import com.yahoo.vespa.orchestrator.controller.ClusterControllerClientFactory;
@@ -27,9 +30,12 @@ import com.yahoo.vespa.orchestrator.policy.HostedVespaClusterPolicy;
 import com.yahoo.vespa.orchestrator.policy.HostedVespaPolicy;
 import com.yahoo.vespa.orchestrator.policy.Policy;
 import com.yahoo.vespa.orchestrator.status.ApplicationInstanceStatus;
+import com.yahoo.vespa.orchestrator.status.ApplicationLock;
+import com.yahoo.vespa.orchestrator.status.HostInfo;
+import com.yahoo.vespa.orchestrator.status.HostInfos;
 import com.yahoo.vespa.orchestrator.status.HostStatus;
-import com.yahoo.vespa.orchestrator.status.MutableStatusRegistry;
 import com.yahoo.vespa.orchestrator.status.StatusService;
+import com.yahoo.vespa.service.monitor.ServiceMonitor;
 
 import java.io.IOException;
 import java.time.Clock;
@@ -53,79 +59,95 @@ public class OrchestratorImpl implements Orchestrator {
 
     private final Policy policy;
     private final StatusService statusService;
-    private final InstanceLookupService instanceLookupService;
+    private final ServiceMonitor serviceMonitor;
     private final int serviceMonitorConvergenceLatencySeconds;
     private final ClusterControllerClientFactory clusterControllerClientFactory;
     private final Clock clock;
     private final ApplicationApiFactory applicationApiFactory;
+    private final BooleanFlag retireWithPermanentlyDownFlag;
 
     @Inject
     public OrchestratorImpl(ClusterControllerClientFactory clusterControllerClientFactory,
                             StatusService statusService,
                             OrchestratorConfig orchestratorConfig,
-                            InstanceLookupService instanceLookupService,
-                            ConfigserverConfig configServerConfig)
+                            ServiceMonitor serviceMonitor,
+                            ConfigserverConfig configServerConfig,
+                            FlagSource flagSource)
     {
-        this(new HostedVespaPolicy(new HostedVespaClusterPolicy(), clusterControllerClientFactory, new ApplicationApiFactory(configServerConfig.zookeeperserver().size())),
-             clusterControllerClientFactory,
-             statusService,
-             instanceLookupService,
-             orchestratorConfig.serviceMonitorConvergenceLatencySeconds(),
-             Clock.systemUTC(),
-             new ApplicationApiFactory(configServerConfig.zookeeperserver().size()));
+        this(new HostedVespaPolicy(new HostedVespaClusterPolicy(),
+                                   clusterControllerClientFactory,
+                                   new ApplicationApiFactory(configServerConfig.zookeeperserver().size())),
+                clusterControllerClientFactory,
+                statusService,
+                serviceMonitor,
+                orchestratorConfig.serviceMonitorConvergenceLatencySeconds(),
+                Clock.systemUTC(),
+                new ApplicationApiFactory(configServerConfig.zookeeperserver().size()),
+                flagSource);
     }
 
     public OrchestratorImpl(Policy policy,
                             ClusterControllerClientFactory clusterControllerClientFactory,
                             StatusService statusService,
-                            InstanceLookupService instanceLookupService,
+                            ServiceMonitor serviceMonitor,
                             int serviceMonitorConvergenceLatencySeconds,
                             Clock clock,
-                            ApplicationApiFactory applicationApiFactory)
+                            ApplicationApiFactory applicationApiFactory,
+                            FlagSource flagSource)
     {
         this.policy = policy;
         this.clusterControllerClientFactory = clusterControllerClientFactory;
         this.statusService = statusService;
         this.serviceMonitorConvergenceLatencySeconds = serviceMonitorConvergenceLatencySeconds;
-        this.instanceLookupService = instanceLookupService;
+        this.serviceMonitor = serviceMonitor;
         this.clock = clock;
         this.applicationApiFactory = applicationApiFactory;
+        this.retireWithPermanentlyDownFlag = Flags.RETIRE_WITH_PERMANENTLY_DOWN.bindTo(flagSource);
+
+        serviceMonitor.registerListener(statusService);
     }
 
     @Override
     public Host getHost(HostName hostName) throws HostNameNotFoundException {
-        ApplicationInstance applicationInstance = getApplicationInstance(hostName);
+        ApplicationInstance applicationInstance = serviceMonitor
+                .getApplicationNarrowedTo(hostName)
+                .orElseThrow(() -> new HostNameNotFoundException(hostName));
+
         List<ServiceInstance> serviceInstances = applicationInstance
                 .serviceClusters().stream()
                 .flatMap(cluster -> cluster.serviceInstances().stream())
                 .filter(serviceInstance -> hostName.equals(serviceInstance.hostName()))
                 .collect(Collectors.toList());
 
-        HostStatus hostStatus = getNodeStatus(applicationInstance.reference(), hostName);
+        HostInfo hostInfo = statusService.getHostInfo(applicationInstance.reference(), hostName);
 
-        return new Host(hostName, hostStatus, applicationInstance.reference(), serviceInstances);
+        return new Host(hostName, hostInfo, applicationInstance.reference(), serviceInstances);
     }
 
     @Override
     public HostStatus getNodeStatus(HostName hostName) throws HostNameNotFoundException {
-        return getNodeStatus(getApplicationInstance(hostName).reference(), hostName);
+        ApplicationInstanceReference reference = getApplicationInstanceReference(hostName);
+        return statusService.getHostInfo(reference, hostName).status();
+    }
+
+    @Override
+    public HostInfo getHostInfo(ApplicationInstanceReference reference, HostName hostname) {
+        return statusService.getHostInfo(reference, hostname);
     }
 
     @Override
-    public Function<HostName, Optional<HostStatus>> getNodeStatuses() {
-        Function<ApplicationInstanceReference, Set<HostName>> suspendedHosts = statusService.getSuspendedHostsByApplication();
-        return hostName -> instanceLookupService.findInstanceByHost(hostName)
-                                                .map(application -> suspendedHosts.apply(application.reference()).contains(hostName)
-                                                                    ? HostStatus.ALLOWED_TO_BE_DOWN : HostStatus.NO_REMARKS);
+    public Function<HostName, Optional<HostInfo>> getHostResolver() {
+        return hostName -> serviceMonitor
+                .getApplicationInstanceReference(hostName)
+                .map(reference -> statusService.getHostInfo(reference, hostName));
     }
 
     @Override
     public void setNodeStatus(HostName hostName, HostStatus status) throws OrchestrationException {
-        ApplicationInstanceReference reference = getApplicationInstance(hostName).reference();
+        ApplicationInstanceReference reference = getApplicationInstanceReference(hostName);
         OrchestratorContext context = OrchestratorContext.createContextForSingleAppOp(clock);
-        try (MutableStatusRegistry statusRegistry = statusService
-                .lockApplicationInstance_forCurrentThreadOnly(context, reference)) {
-            statusRegistry.setHostState(hostName, status);
+        try (ApplicationLock lock = statusService.lockApplication(context, reference)) {
+            lock.setHostState(hostName, status);
         }
     }
 
@@ -141,24 +163,35 @@ public class OrchestratorImpl implements Orchestrator {
         * monitoring will have had time to catch up. Since we don't want do the delay with the lock held,
         * and the host status service's locking functionality does not support something like condition
         * variables or Object.wait(), we break out here, releasing the lock before delaying.
+        *
+        * 2020-02-07: We should utilize suspendedSince timestamp on the HostInfo: The above
+        * is equivalent to guaranteeing a minimum time after suspendedSince, before checking
+        * the health with service monitor. This should for all practical purposes remove
+        * the amount of time in this sleep.
+        * Caveat: Cannot be implemented before lingering HostInfo has been fixed (VESPA-17546).
         */
         sleep(serviceMonitorConvergenceLatencySeconds, TimeUnit.SECONDS);
 
         ApplicationInstance appInstance = getApplicationInstance(hostName);
 
         OrchestratorContext context = OrchestratorContext.createContextForSingleAppOp(clock);
-        try (MutableStatusRegistry statusRegistry = statusService
-                .lockApplicationInstance_forCurrentThreadOnly(context, appInstance.reference())) {
-            HostStatus currentHostState = statusRegistry.getHostInfo(hostName).status();
-
-            if (HostStatus.NO_REMARKS == currentHostState) {
+        try (ApplicationLock lock = statusService.lockApplication(context, appInstance.reference())) {
+            HostStatus currentHostState = lock.getHostInfos().getOrNoRemarks(hostName).status();
+            if (currentHostState == HostStatus.NO_REMARKS) {
                 return;
             }
 
-            ApplicationInstanceStatus appStatus = statusRegistry.getStatus();
-            if (appStatus == ApplicationInstanceStatus.NO_REMARKS) {
-                policy.releaseSuspensionGrant(context.createSubcontextWithinLock(), appInstance, hostName, statusRegistry);
+            // In 2 cases the resume will appear to succeed (no exception thrown),
+            // but the host status and content cluster states will not be changed accordingly:
+            //  1. When host is permanently down: the host will be removed from the application asap.
+            //  2. The whole application is down: the content cluster states are set to maintenance,
+            //     and the host may be taken down manually at any moment.
+            if (currentHostState == HostStatus.PERMANENTLY_DOWN ||
+                    lock.getApplicationInstanceStatus() == ApplicationInstanceStatus.ALLOWED_TO_BE_DOWN) {
+                return;
             }
+
+            policy.releaseSuspensionGrant(context.createSubcontextWithinLock(), appInstance, hostName, lock);
         }
     }
 
@@ -174,11 +207,12 @@ public class OrchestratorImpl implements Orchestrator {
         ApplicationInstance appInstance = getApplicationInstance(hostName);
         NodeGroup nodeGroup = new NodeGroup(appInstance, hostName);
 
-        OrchestratorContext context = OrchestratorContext.createContextForSingleAppOp(clock);
-        try (MutableStatusRegistry statusRegistry = statusService
-                .lockApplicationInstance_forCurrentThreadOnly(context, appInstance.reference())) {
-            ApplicationApi applicationApi = applicationApiFactory.create(nodeGroup, statusRegistry,
-                                                                         clusterControllerClientFactory);
+        boolean usePermanentlyDownStatus = retireWithPermanentlyDownFlag
+                .with(FetchVector.Dimension.HOSTNAME, hostName.s())
+                .value();
+        OrchestratorContext context = OrchestratorContext.createContextForSingleAppOp(clock, usePermanentlyDownStatus);
+        try (ApplicationLock lock = statusService.lockApplication(context, appInstance.reference())) {
+            ApplicationApi applicationApi = applicationApiFactory.create(nodeGroup, lock, clusterControllerClientFactory);
 
             policy.acquirePermissionToRemove(context.createSubcontextWithinLock(), applicationApi);
         }
@@ -193,23 +227,22 @@ public class OrchestratorImpl implements Orchestrator {
     void suspendGroup(OrchestratorContext context, NodeGroup nodeGroup) throws HostStateChangeDeniedException {
         ApplicationInstanceReference applicationReference = nodeGroup.getApplicationReference();
 
-        try (MutableStatusRegistry hostStatusRegistry =
-                     statusService.lockApplicationInstance_forCurrentThreadOnly(context, applicationReference)) {
-            ApplicationInstanceStatus appStatus = hostStatusRegistry.getStatus();
+        try (ApplicationLock lock = statusService.lockApplication(context, applicationReference)) {
+            ApplicationInstanceStatus appStatus = lock.getApplicationInstanceStatus();
             if (appStatus == ApplicationInstanceStatus.ALLOWED_TO_BE_DOWN) {
                 return;
             }
 
-            ApplicationApi applicationApi = applicationApiFactory.create(nodeGroup, hostStatusRegistry,
-                                                                         clusterControllerClientFactory);
+            ApplicationApi applicationApi = applicationApiFactory.create(
+                    nodeGroup, lock, clusterControllerClientFactory);
             policy.grantSuspensionRequest(context.createSubcontextWithinLock(), applicationApi);
         }
     }
 
     @Override
     public ApplicationInstanceStatus getApplicationInstanceStatus(ApplicationId appId) throws ApplicationIdNotFoundException {
-        ApplicationInstanceReference appRef = OrchestratorUtil.toApplicationInstanceReference(appId, instanceLookupService);
-        return statusService.getApplicationInstanceStatus(appRef);
+        ApplicationInstanceReference reference = OrchestratorUtil.toApplicationInstanceReference(appId, serviceMonitor);
+        return statusService.getApplicationInstanceStatus(reference);
     }
 
     @Override
@@ -231,17 +264,17 @@ public class OrchestratorImpl implements Orchestrator {
     @Override
     public void suspendAll(HostName parentHostname, List<HostName> hostNames)
             throws BatchHostStateChangeDeniedException, BatchHostNameNotFoundException, BatchInternalErrorException {
-        OrchestratorContext context = OrchestratorContext.createContextForMultiAppOp(clock);
+        try (OrchestratorContext context = OrchestratorContext.createContextForMultiAppOp(clock)) {
+            List<NodeGroup> nodeGroupsOrderedByApplication;
+            try {
+                nodeGroupsOrderedByApplication = nodeGroupsOrderedForSuspend(hostNames);
+            } catch (HostNameNotFoundException e) {
+                throw new BatchHostNameNotFoundException(parentHostname, hostNames, e);
+            }
 
-        List<NodeGroup> nodeGroupsOrderedByApplication;
-        try {
-            nodeGroupsOrderedByApplication = nodeGroupsOrderedForSuspend(hostNames);
-        } catch (HostNameNotFoundException e) {
-            throw new BatchHostNameNotFoundException(parentHostname, hostNames, e);
+            suspendAllNodeGroups(context, parentHostname, nodeGroupsOrderedByApplication, true);
+            suspendAllNodeGroups(context, parentHostname, nodeGroupsOrderedByApplication, false);
         }
-
-        suspendAllNodeGroups(context, parentHostname, nodeGroupsOrderedByApplication, true);
-        suspendAllNodeGroups(context, parentHostname, nodeGroupsOrderedByApplication, false);
     }
 
     private void suspendAllNodeGroups(OrchestratorContext context,
@@ -254,6 +287,8 @@ public class OrchestratorImpl implements Orchestrator {
                 suspendGroup(context.createSubcontextForSingleAppOp(probe), nodeGroup);
             } catch (HostStateChangeDeniedException e) {
                 throw new BatchHostStateChangeDeniedException(parentHostname, nodeGroup, e);
+            } catch (UncheckedTimeoutException e) {
+                throw e;
             } catch (RuntimeException e) {
                 throw new BatchInternalErrorException(parentHostname, nodeGroup, e);
             }
@@ -317,34 +352,37 @@ public class OrchestratorImpl implements Orchestrator {
         return leftApplicationReference.asString().compareTo(rightApplicationReference.asString());
     }
 
-    private HostStatus getNodeStatus(ApplicationInstanceReference applicationRef, HostName hostName) {
-        return statusService.getHostInfo(applicationRef, hostName).status();
-    }
-
-    private void setApplicationStatus(ApplicationId appId, ApplicationInstanceStatus status) 
+    private void setApplicationStatus(ApplicationId appId, ApplicationInstanceStatus status)
             throws ApplicationStateChangeDeniedException, ApplicationIdNotFoundException{
         OrchestratorContext context = OrchestratorContext.createContextForSingleAppOp(clock);
-        ApplicationInstanceReference appRef = OrchestratorUtil.toApplicationInstanceReference(appId, instanceLookupService);
-        try (MutableStatusRegistry statusRegistry =
-                     statusService.lockApplicationInstance_forCurrentThreadOnly(context, appRef)) {
+        ApplicationInstanceReference reference = OrchestratorUtil.toApplicationInstanceReference(appId, serviceMonitor);
+
+        ApplicationInstance application = serviceMonitor.getApplication(reference)
+                .orElseThrow(ApplicationIdNotFoundException::new);
+
+        try (ApplicationLock lock = statusService.lockApplication(context, reference)) {
 
             // Short-circuit if already in wanted state
-            if (status == statusRegistry.getStatus()) return;
+            if (status == lock.getApplicationInstanceStatus()) return;
 
             // Set content clusters for this application in maintenance on suspend
             if (status == ApplicationInstanceStatus.ALLOWED_TO_BE_DOWN) {
-                ApplicationInstance application = getApplicationInstance(appRef);
+                HostInfos hostInfosSnapshot = lock.getHostInfos();
 
                 // Mark it allowed to be down before we manipulate the clustercontroller
                 OrchestratorUtil.getHostsUsedByApplicationInstance(application)
-                        .forEach(h -> statusRegistry.setHostState(h, HostStatus.ALLOWED_TO_BE_DOWN));
+                        .stream()
+                        // This filter also ensures host status is not modified if a suspended host
+                        // has status != ALLOWED_TO_BE_DOWN.
+                        .filter(hostname -> !hostInfosSnapshot.getOrNoRemarks(hostname).status().isSuspended())
+                        .forEach(hostname -> lock.setHostState(hostname, HostStatus.ALLOWED_TO_BE_DOWN));
 
                 // If the clustercontroller throws an error the nodes will be marked as allowed to be down
                 // and be set back up on next resume invocation.
                 setClusterStateInController(context.createSubcontextWithinLock(), application, ClusterControllerNodeState.MAINTENANCE);
             }
 
-            statusRegistry.setApplicationInstanceStatus(status);
+            lock.setApplicationInstanceStatus(status);
         }
     }
 
@@ -359,8 +397,6 @@ public class OrchestratorImpl implements Orchestrator {
                 .collect(Collectors.toSet());
 
         // For all content clusters set in maintenance
-        log.log(LogLevel.INFO, String.format("Setting content clusters %s for application %s to %s",
-                contentClusterIds,application.applicationInstanceId(),state));
         for (ClusterId clusterId : contentClusterIds) {
             List<HostName> clusterControllers = VespaModelUtil.getClusterControllerInstancesInOrder(application, clusterId);
             ClusterControllerClient client = clusterControllerClientFactory.createClient(
@@ -383,13 +419,14 @@ public class OrchestratorImpl implements Orchestrator {
         }
     }
 
-    private ApplicationInstance getApplicationInstance(HostName hostName) throws HostNameNotFoundException{
-        return instanceLookupService.findInstanceByHost(hostName).orElseThrow(
-                () -> new HostNameNotFoundException(hostName));
+    private ApplicationInstanceReference getApplicationInstanceReference(HostName hostname) throws HostNameNotFoundException {
+        return serviceMonitor.getApplicationInstanceReference(hostname)
+                .orElseThrow(() -> new HostNameNotFoundException(hostname));
     }
 
-    private ApplicationInstance getApplicationInstance(ApplicationInstanceReference appRef) throws ApplicationIdNotFoundException {
-        return instanceLookupService.findInstanceById(appRef).orElseThrow(ApplicationIdNotFoundException::new);
+    private ApplicationInstance getApplicationInstance(HostName hostName) throws HostNameNotFoundException{
+        return serviceMonitor.getApplication(hostName)
+                .orElseThrow(() -> new HostNameNotFoundException(hostName));
     }
 
     private static void sleep(long time, TimeUnit timeUnit) {
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorUtil.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorUtil.java
index 91da046840d..40a45627f4d 100644
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorUtil.java
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorUtil.java
@@ -12,6 +12,7 @@ import com.yahoo.vespa.applicationmodel.HostName;
 import com.yahoo.vespa.applicationmodel.ServiceCluster;
 import com.yahoo.vespa.applicationmodel.ServiceInstance;
 import com.yahoo.vespa.applicationmodel.TenantId;
+import com.yahoo.vespa.service.monitor.ServiceMonitor;
 
 import java.util.Collection;
 import java.util.List;
@@ -65,7 +66,7 @@ public class OrchestratorUtil {
     private static final Pattern APPLICATION_INSTANCE_REFERENCE_REST_FORMAT_PATTERN = Pattern.compile("^([^:]+):(.+)$");
 
     /** Returns an ApplicationInstanceReference constructed from the serialized format used in the REST API. */
-    public static ApplicationInstanceReference parseAppInstanceReference(String restFormat) {
+    public static ApplicationInstanceReference parseApplicationInstanceReference(String restFormat) {
         if (restFormat == null) {
             throw new IllegalArgumentException("Could not construct instance id from null string");
         }
@@ -84,26 +85,26 @@ public class OrchestratorUtil {
         return applicationInstanceReference.tenantId() + ":" + applicationInstanceReference.applicationInstanceId();
     }
 
-
-    public static ApplicationInstanceReference toApplicationInstanceReference(ApplicationId appId,
-                                                                              InstanceLookupService instanceLookupService)
+    public static ApplicationInstanceReference toApplicationInstanceReference(
+            ApplicationId applicationid,
+            ServiceMonitor serviceMonitor)
             throws ApplicationIdNotFoundException {
 
-        Set<ApplicationInstanceReference> appRefs = instanceLookupService.knownInstances();
-        List<ApplicationInstanceReference> appRefList = appRefs.stream()
-                .filter(a -> OrchestratorUtil.toApplicationId(a).equals(appId))
+        Set<ApplicationInstanceReference> references = serviceMonitor.getAllApplicationInstanceReferences();
+        List<ApplicationInstanceReference> referencesWithId = references.stream()
+                .filter(a -> OrchestratorUtil.toApplicationId(a).equals(applicationid))
                 .collect(Collectors.toList());
 
-        if (appRefList.size() > 1) {
-            String msg = String.format("ApplicationId '%s' was not unique but mapped to '%s'", appId, appRefList);
+        if (referencesWithId.size() > 1) {
+            String msg = String.format("ApplicationId '%s' was not unique but mapped to '%s'", applicationid, referencesWithId);
             throw new ApplicationIdNotFoundException(msg);
         }
 
-        if (appRefList.size() == 0) {
+        if (referencesWithId.size() == 0) {
             throw new ApplicationIdNotFoundException();
         }
 
-        return appRefList.get(0);
+        return referencesWithId.get(0);
     }
 
     public static ApplicationId toApplicationId(ApplicationInstanceReference appRef) {
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/ServiceMonitorInstanceLookupService.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/ServiceMonitorInstanceLookupService.java
deleted file mode 100644
index 1a859cfacc5..00000000000
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/ServiceMonitorInstanceLookupService.java
+++ /dev/null
@@ -1,45 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.orchestrator;
-
-import com.google.inject.Inject;
-import com.yahoo.vespa.applicationmodel.ApplicationInstance;
-import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference;
-import com.yahoo.vespa.applicationmodel.HostName;
-import com.yahoo.vespa.service.monitor.ServiceMonitor;
-
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-/**
- * Uses slobrok data (a.k.a. heartbeat) to implement {@link InstanceLookupService}.
- *
- * @author bakksjo
- */
-public class ServiceMonitorInstanceLookupService implements InstanceLookupService {
-
-    private final ServiceMonitor serviceMonitor;
-
-    @Inject
-    public ServiceMonitorInstanceLookupService(ServiceMonitor serviceMonitor) {
-        this.serviceMonitor = serviceMonitor;
-    }
-
-    @Override
-    public Optional<ApplicationInstance> findInstanceById(ApplicationInstanceReference applicationInstanceReference) {
-        return serviceMonitor.getServiceModelSnapshot().getApplicationInstance(applicationInstanceReference);
-    }
-
-    @Override
-    public Optional<ApplicationInstance> findInstanceByHost(HostName hostName) {
-        return Optional.ofNullable(serviceMonitor.getServiceModelSnapshot().getApplicationsByHostName().get(hostName));
-    }
-
-    @Override
-    public Set<ApplicationInstanceReference> knownInstances() {
-        return serviceMonitor.getServiceModelSnapshot().getAllApplicationInstances().keySet();
-    }
-
-}
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApi.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApi.java
index 2e85713d323..a6353e39610 100644
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApi.java
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApi.java
@@ -8,6 +8,7 @@ import com.yahoo.vespa.orchestrator.status.ApplicationInstanceStatus;
 import com.yahoo.vespa.orchestrator.status.HostStatus;
 
 import java.util.List;
+import java.util.function.Predicate;
 
 /**
  * The API a Policy has access to
@@ -29,10 +30,12 @@ public interface ApplicationApi {
     ApplicationInstanceStatus getApplicationStatus();
 
     void setHostState(OrchestratorContext context, HostName hostName, HostStatus status);
-    List<HostName> getNodesInGroupWithStatus(HostStatus status);
+    List<HostName> getNodesInGroupWith(Predicate<HostStatus> statusPredicate);
+    default List<HostName> getNodesInGroupWithStatus(HostStatus requiredStatus) {
+        return getNodesInGroupWith(status -> status == requiredStatus);
+    }
 
     List<StorageNode> getStorageNodesInGroupInClusterOrder();
     List<StorageNode> getUpStorageNodesInGroupInClusterOrder();
-    List<StorageNode> getStorageNodesAllowedToBeDownInGroupInReverseClusterOrder();
-
+    List<StorageNode> getSuspendedStorageNodesInGroupInReverseClusterOrder();
 }
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApiFactory.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApiFactory.java
index 5e6ec59c28d..fb52eed2048 100644
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApiFactory.java
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApiFactory.java
@@ -2,7 +2,7 @@
 package com.yahoo.vespa.orchestrator.model;
 
 import com.yahoo.vespa.orchestrator.controller.ClusterControllerClientFactory;
-import com.yahoo.vespa.orchestrator.status.MutableStatusRegistry;
+import com.yahoo.vespa.orchestrator.status.ApplicationLock;
 
 /**
  * @author mpolden
@@ -16,9 +16,9 @@ public class ApplicationApiFactory {
     }
 
     public ApplicationApi create(NodeGroup nodeGroup,
-                                 MutableStatusRegistry hostStatusService,
+                                 ApplicationLock lock,
                                  ClusterControllerClientFactory clusterControllerClientFactory) {
-        return new ApplicationApiImpl(nodeGroup, hostStatusService, clusterControllerClientFactory, numberOfConfigServers);
+        return new ApplicationApiImpl(nodeGroup, lock, clusterControllerClientFactory, numberOfConfigServers);
     }
 
 }
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApiImpl.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApiImpl.java
index eb5dcf790ba..ccc89fa9191 100644
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApiImpl.java
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApiImpl.java
@@ -10,17 +10,17 @@ import com.yahoo.vespa.orchestrator.OrchestratorContext;
 import com.yahoo.vespa.orchestrator.OrchestratorUtil;
 import com.yahoo.vespa.orchestrator.controller.ClusterControllerClientFactory;
 import com.yahoo.vespa.orchestrator.status.ApplicationInstanceStatus;
+import com.yahoo.vespa.orchestrator.status.HostInfos;
 import com.yahoo.vespa.orchestrator.status.HostStatus;
-import com.yahoo.vespa.orchestrator.status.MutableStatusRegistry;
+import com.yahoo.vespa.orchestrator.status.ApplicationLock;
 
 import java.util.Collection;
 import java.util.Comparator;
 import java.util.HashSet;
 import java.util.List;
-import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
-import java.util.function.Function;
+import java.util.function.Predicate;
 import java.util.stream.Collectors;
 
 import static com.yahoo.vespa.orchestrator.OrchestratorUtil.getHostsUsedByApplicationInstance;
@@ -32,22 +32,20 @@ public class ApplicationApiImpl implements ApplicationApi {
 
     private final ApplicationInstance applicationInstance;
     private final NodeGroup nodeGroup;
-    private final MutableStatusRegistry hostStatusService;
+    private final ApplicationLock lock;
     private final List<ClusterApi> clusterInOrder;
-    private final Map<HostName, HostStatus> hostStatusMap;
+    private final HostInfos hostInfos;
 
     public ApplicationApiImpl(NodeGroup nodeGroup,
-                              MutableStatusRegistry hostStatusService,
+                              ApplicationLock lock,
                               ClusterControllerClientFactory clusterControllerClientFactory,
                               int numberOfConfigServers) {
         this.applicationInstance = nodeGroup.getApplication();
         this.nodeGroup = nodeGroup;
-        this.hostStatusService = hostStatusService;
+        this.lock = lock;
         Collection<HostName> hosts = getHostsUsedByApplicationInstance(applicationInstance);
-        this.hostStatusMap = hosts.stream().collect(Collectors.toMap(Function.identity(),
-                                                                     hostName -> hostStatusService.getSuspendedHosts().contains(hostName)
-                                                                                 ? HostStatus.ALLOWED_TO_BE_DOWN : HostStatus.NO_REMARKS));
-        this.clusterInOrder = makeClustersInOrder(nodeGroup, hostStatusMap, clusterControllerClientFactory, numberOfConfigServers);
+        this.hostInfos = lock.getHostInfos();
+        this.clusterInOrder = makeClustersInOrder(nodeGroup, hostInfos, clusterControllerClientFactory, numberOfConfigServers);
     }
 
     @Override
@@ -56,7 +54,7 @@ public class ApplicationApiImpl implements ApplicationApi {
     }
 
     private HostStatus getHostStatus(HostName hostName) {
-        return hostStatusMap.getOrDefault(hostName, HostStatus.NO_REMARKS);
+        return hostInfos.getOrNoRemarks(hostName).status();
     }
 
     @Override
@@ -65,9 +63,9 @@ public class ApplicationApiImpl implements ApplicationApi {
     }
 
     @Override
-    public List<StorageNode> getStorageNodesAllowedToBeDownInGroupInReverseClusterOrder() {
+    public List<StorageNode> getSuspendedStorageNodesInGroupInReverseClusterOrder() {
         return getStorageNodesInGroupInClusterOrder().stream()
-                .filter(storageNode -> getHostStatus(storageNode.hostName()) == HostStatus.ALLOWED_TO_BE_DOWN)
+                .filter(storageNode -> getHostStatus(storageNode.hostName()).isSuspended())
                 .sorted(Comparator.reverseOrder())
                 .collect(Collectors.toList());
     }
@@ -97,22 +95,23 @@ public class ApplicationApiImpl implements ApplicationApi {
 
     @Override
     public ApplicationInstanceStatus getApplicationStatus() {
-        return hostStatusService.getStatus();
+        return lock.getApplicationInstanceStatus();
     }
 
     @Override
     public void setHostState(OrchestratorContext context, HostName hostName, HostStatus status) {
-        hostStatusService.setHostState(hostName, status);
+        lock.setHostState(hostName, status);
     }
 
     @Override
-    public List<HostName> getNodesInGroupWithStatus(HostStatus status) {
+    public List<HostName> getNodesInGroupWith(Predicate<HostStatus> statusPredicate) {
         return nodeGroup.getHostNames().stream()
-                .filter(hostName -> getHostStatus(hostName) == status)
+                .filter(hostName -> statusPredicate.test(getHostStatus(hostName)))
                 .collect(Collectors.toList());
     }
 
-    private List<ClusterApi> makeClustersInOrder(NodeGroup nodeGroup, Map<HostName, HostStatus> hostStatusMap,
+    private List<ClusterApi> makeClustersInOrder(NodeGroup nodeGroup,
+                                                 HostInfos hostInfos,
                                                  ClusterControllerClientFactory clusterControllerClientFactory,
                                                  int numberOfConfigServers) {
         Set<ServiceCluster> clustersInGroup = getServiceClustersInGroup(nodeGroup);
@@ -121,7 +120,7 @@ public class ApplicationApiImpl implements ApplicationApi {
                         this,
                         serviceCluster,
                         nodeGroup,
-                        hostStatusMap,
+                        hostInfos,
                         clusterControllerClientFactory,
                         numberOfConfigServers))
                 .sorted(ApplicationApiImpl::compareClusters)
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApi.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApi.java
index e9bb4984c2e..65c45c8df76 100644
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApi.java
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApi.java
@@ -26,7 +26,6 @@ public interface ClusterApi {
     Optional<StorageNode> storageNodeInGroup();
     Optional<StorageNode> upStorageNodeInGroup();
 
-    String servicesDownAndNotInGroupDescription();
-    String nodesAllowedToBeDownNotInGroupDescription();
+    String downDescription();
 
 }
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApiImpl.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApiImpl.java
index 2aaebb18aa6..24f56eac85d 100644
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApiImpl.java
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ClusterApiImpl.java
@@ -8,6 +8,7 @@ import com.yahoo.vespa.applicationmodel.ServiceInstance;
 import com.yahoo.vespa.applicationmodel.ServiceStatus;
 import com.yahoo.vespa.applicationmodel.ServiceType;
 import com.yahoo.vespa.orchestrator.controller.ClusterControllerClientFactory;
+import com.yahoo.vespa.orchestrator.status.HostInfos;
 import com.yahoo.vespa.orchestrator.status.HostStatus;
 
 import java.util.Collections;
@@ -27,7 +28,7 @@ class ClusterApiImpl implements ClusterApi {
     private final ApplicationApi applicationApi;
     private final ServiceCluster serviceCluster;
     private final NodeGroup nodeGroup;
-    private final Map<HostName, HostStatus> hostStatusMap;
+    private final HostInfos hostInfos;
     private final ClusterControllerClientFactory clusterControllerClientFactory;
     private final Set<ServiceInstance> servicesInGroup;
     private final Set<ServiceInstance> servicesDownInGroup;
@@ -50,13 +51,13 @@ class ClusterApiImpl implements ClusterApi {
     public ClusterApiImpl(ApplicationApi applicationApi,
                           ServiceCluster serviceCluster,
                           NodeGroup nodeGroup,
-                          Map<HostName, HostStatus> hostStatusMap,
+                          HostInfos hostInfos,
                           ClusterControllerClientFactory clusterControllerClientFactory,
                           int numberOfConfigServers) {
         this.applicationApi = applicationApi;
         this.serviceCluster = serviceCluster;
         this.nodeGroup = nodeGroup;
-        this.hostStatusMap = hostStatusMap;
+        this.hostInfos = hostInfos;
         this.clusterControllerClientFactory = clusterControllerClientFactory;
 
         Map<Boolean, Set<ServiceInstance>> serviceInstancesByLocality =
@@ -130,25 +131,56 @@ class ClusterApiImpl implements ClusterApi {
         return numberOfServicesDown * 100 / (serviceCluster.serviceInstances().size() + missingServices);
     }
 
+    /**
+     * A description of the hosts outside the group that are allowed to be down,
+     * and a description of the services outside the group and outside of the allowed services
+     * that are down.
+     */
     @Override
-    public String servicesDownAndNotInGroupDescription() {
-        // Sort these for readability and testing stability
-        return Stream
-                .concat(servicesDownAndNotInGroup.stream().map(ServiceInstance::toString).sorted(),
-                        missingServices > 0 ? Stream.of(descriptionOfMissingServices) : Stream.of())
-                .collect(Collectors.toList())
-                .toString();
-    }
+    public String downDescription() {
+        StringBuilder description = new StringBuilder();
 
-    @Override
-    public String nodesAllowedToBeDownNotInGroupDescription() {
-        return servicesNotInGroup.stream()
+        Set<HostName> suspended = servicesNotInGroup.stream()
                 .map(ServiceInstance::hostName)
-                .filter(hostName -> hostStatus(hostName) == HostStatus.ALLOWED_TO_BE_DOWN)
-                .sorted()
-                .distinct()
-                .collect(Collectors.toList())
-                .toString();
+                .filter(hostName -> hostStatus(hostName).isSuspended())
+                .collect(Collectors.toSet());
+
+        if (suspended.size() > 0) {
+            description.append(" ");
+
+            final int nodeLimit = 3;
+            description.append("Suspended hosts: ");
+            description.append(suspended.stream().sorted().distinct().limit(nodeLimit).collect(Collectors.toList()).toString());
+            if (suspended.size() > nodeLimit) {
+                description.append(", and " + (suspended.size() - nodeLimit) + " more");
+            }
+            description.append(".");
+        }
+
+        Set<ServiceInstance> downElsewhere = servicesDownAndNotInGroup.stream()
+                .filter(serviceInstance -> !suspended.contains(serviceInstance.hostName()))
+                .collect(Collectors.toSet());
+
+        final int downElsewhereTotal = downElsewhere.size() + missingServices;
+        if (downElsewhereTotal > 0) {
+            description.append(" ");
+
+            final int serviceLimit = 2; // services info is verbose
+            description.append("Services down on resumed hosts: ");
+            description.append(Stream.concat(
+                    downElsewhere.stream().map(ServiceInstance::toString).sorted(),
+                    missingServices > 0 ? Stream.of(descriptionOfMissingServices) : Stream.of())
+                    .limit(serviceLimit)
+                    .collect(Collectors.toList())
+                    .toString());
+
+            if (downElsewhereTotal > serviceLimit) {
+                description.append(", and " + (downElsewhereTotal - serviceLimit) + " more");
+            }
+            description.append(".");
+        }
+
+        return description.toString();
     }
 
     private Optional<StorageNode> storageNodeInGroup(Predicate<ServiceInstance> storageServicePredicate) {
@@ -206,11 +238,11 @@ class ClusterApiImpl implements ClusterApi {
     }
 
     private HostStatus hostStatus(HostName hostName) {
-        return hostStatusMap.getOrDefault(hostName, HostStatus.NO_REMARKS);
+        return hostInfos.getOrNoRemarks(hostName).status();
     }
 
     private boolean serviceEffectivelyDown(ServiceInstance service) {
-        if (hostStatus(service.hostName()) == HostStatus.ALLOWED_TO_BE_DOWN) {
+        if (hostStatus(service.hostName()).isSuspended()) {
             return true;
         }
 
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/StorageNodeImpl.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/StorageNodeImpl.java
index 0e4fe672725..9900c8de752 100644
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/StorageNodeImpl.java
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/StorageNodeImpl.java
@@ -104,12 +104,6 @@ public class StorageNodeImpl implements StorageNode {
                     HostedVespaPolicy.SET_NODE_STATE_CONSTRAINT,
                     "Failed to set state to " + wantedNodeState + " in cluster controller: " + response.reason);
         }
-
-        String logSuffix = context.isProbe() ?
-                " would have been set to " + wantedNodeState + " (this is a probe)" :
-                " has been set to " + wantedNodeState;
-        logger.log(LogLevel.INFO, "Storage node " + nodeIndex + " in cluster " + clusterId +
-                " application " + applicationInstance.reference().asString() + " on host " + hostName() + logSuffix);
     }
 
     @Override
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostStateChangeDeniedException.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostStateChangeDeniedException.java
index e13cf17d420..a6b3cbc87dc 100644
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostStateChangeDeniedException.java
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostStateChangeDeniedException.java
@@ -2,55 +2,43 @@
 package com.yahoo.vespa.orchestrator.policy;
 
 import com.yahoo.vespa.applicationmodel.HostName;
-import com.yahoo.vespa.applicationmodel.ServiceType;
 import com.yahoo.vespa.orchestrator.OrchestrationException;
 import com.yahoo.vespa.orchestrator.model.NodeGroup;
 
-import java.util.Optional;
-
 /**
  * @author bakksjo
  */
 public class HostStateChangeDeniedException extends OrchestrationException {
     private final String constraintName;
-    private final Optional<ServiceType> serviceType;
 
     public HostStateChangeDeniedException(HostName hostName, String constraintName, String message) {
         this(hostName, constraintName, message, null);
     }
 
     public HostStateChangeDeniedException(HostName hostName, String constraintName, String message, Exception e) {
-        this(hostName.s(), constraintName, Optional.empty(), message, e);
+        this(hostName.s(), constraintName, message, e);
     }
 
     public HostStateChangeDeniedException(NodeGroup nodeGroup, String constraintName, String message) {
-        this(nodeGroup.toCommaSeparatedString(), constraintName, Optional.empty(), message, null);
+        this(nodeGroup.toCommaSeparatedString(), constraintName, message, null);
     }
 
     private HostStateChangeDeniedException(String nodes,
                                            String constraintName,
-                                           Optional<ServiceType> serviceType,
                                            String message,
                                            Throwable cause) {
-        super(createMessage(nodes, constraintName, serviceType, message), cause);
+        super(createMessage(nodes, constraintName, message), cause);
         this.constraintName = constraintName;
-        this.serviceType = serviceType;
     }
 
     private static String createMessage(String nodes,
                                         String constraintName,
-                                        Optional<ServiceType> serviceType,
                                         String message) {
         return "Changing the state of " + nodes + " would violate " + constraintName
-                + (serviceType.isPresent() ? " for service type " + serviceType.get() : "")
                 + ": " + message;
     }
 
     public String getConstraintName() {
         return constraintName;
     }
-
-    public Optional<ServiceType> getServiceType() {
-        return serviceType;
-    }
 }
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicy.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicy.java
index 1e895d0e757..ccb0bb57186 100644
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicy.java
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicy.java
@@ -26,13 +26,11 @@ public class HostedVespaClusterPolicy implements ClusterPolicy {
         throw new HostStateChangeDeniedException(
                 clusterApi.getNodeGroup(),
                 ENOUGH_SERVICES_UP_CONSTRAINT,
-                "Suspension percentage for service type " + clusterApi.serviceType()
+                "Suspension for service type " + clusterApi.serviceType()
                         + " would increase from " + clusterApi.percentageOfServicesDown()
                         + "% to " + clusterApi.percentageOfServicesDownIfGroupIsAllowedToBeDown()
                         + "%, over the limit of " + percentageOfServicesAllowedToBeDown + "%."
-                        + " These instances may be down: " + clusterApi.servicesDownAndNotInGroupDescription()
-                        + " and these hosts are allowed to be down: "
-                        + clusterApi.nodesAllowedToBeDownNotInGroupDescription());
+                        + clusterApi.downDescription());
     }
 
     @Override
@@ -56,9 +54,7 @@ public class HostedVespaClusterPolicy implements ClusterPolicy {
                 "Down percentage for service type " + clusterApi.serviceType()
                         + " would increase to " + clusterApi.percentageOfServicesDownIfGroupIsAllowedToBeDown()
                         + "%, over the limit of " + percentageOfServicesAllowedToBeDown + "%."
-                        + " These instances may be down: " + clusterApi.servicesDownAndNotInGroupDescription()
-                        + " and these hosts are allowed to be down: "
-                        + clusterApi.nodesAllowedToBeDownNotInGroupDescription());
+                        + clusterApi.downDescription());
     }
 
     // Non-private for testing purposes
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicy.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicy.java
index a9b9736ebfb..8b74d8a40ef 100644
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicy.java
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicy.java
@@ -13,7 +13,7 @@ import com.yahoo.vespa.orchestrator.model.NodeGroup;
 import com.yahoo.vespa.orchestrator.model.StorageNode;
 import com.yahoo.vespa.orchestrator.status.ApplicationInstanceStatus;
 import com.yahoo.vespa.orchestrator.status.HostStatus;
-import com.yahoo.vespa.orchestrator.status.MutableStatusRegistry;
+import com.yahoo.vespa.orchestrator.status.ApplicationLock;
 
 /**
  * @author oyving
@@ -30,7 +30,9 @@ public class HostedVespaPolicy implements Policy {
     private final ClusterControllerClientFactory clusterControllerClientFactory;
     private final ApplicationApiFactory applicationApiFactory;
 
-    public HostedVespaPolicy(HostedVespaClusterPolicy clusterPolicy, ClusterControllerClientFactory clusterControllerClientFactory, ApplicationApiFactory applicationApiFactory) {
+    public HostedVespaPolicy(HostedVespaClusterPolicy clusterPolicy,
+                             ClusterControllerClientFactory clusterControllerClientFactory,
+                             ApplicationApiFactory applicationApiFactory) {
         this.clusterPolicy = clusterPolicy;
         this.clusterControllerClientFactory = clusterControllerClientFactory;
         this.applicationApiFactory = applicationApiFactory;
@@ -60,10 +62,11 @@ public class HostedVespaPolicy implements Policy {
     public void releaseSuspensionGrant(OrchestratorContext context, ApplicationApi application)
             throws HostStateChangeDeniedException {
         // Always defer to Cluster Controller whether it's OK to resume storage node
-        for (StorageNode storageNode : application.getStorageNodesAllowedToBeDownInGroupInReverseClusterOrder()) {
+        for (StorageNode storageNode : application.getSuspendedStorageNodesInGroupInReverseClusterOrder()) {
             storageNode.setNodeState(context, ClusterControllerNodeState.UP);
         }
 
+        // In particular, we're not modifying the state of PERMANENTLY_DOWN nodes.
         for (HostName hostName : application.getNodesInGroupWithStatus(HostStatus.ALLOWED_TO_BE_DOWN)) {
             application.setHostState(context, hostName, HostStatus.NO_REMARKS);
         }
@@ -92,9 +95,15 @@ public class HostedVespaPolicy implements Policy {
             storageNode.setNodeState(context, ClusterControllerNodeState.DOWN);
         }
 
-        // Ensure all nodes in the group are marked as allowed to be down
-        for (HostName hostName : applicationApi.getNodesInGroupWithStatus(HostStatus.NO_REMARKS)) {
-            applicationApi.setHostState(context, hostName, HostStatus.ALLOWED_TO_BE_DOWN);
+        if (context.usePermanentlyDownStatus()) {
+            // Ensure all nodes in the group are marked as permanently down
+            for (HostName hostName : applicationApi.getNodesInGroupWith(status -> status != HostStatus.PERMANENTLY_DOWN)) {
+                applicationApi.setHostState(context, hostName, HostStatus.PERMANENTLY_DOWN);
+            }
+        } else {
+            for (HostName hostName : applicationApi.getNodesInGroupWith(status -> status != HostStatus.ALLOWED_TO_BE_DOWN)) {
+                applicationApi.setHostState(context, hostName, HostStatus.ALLOWED_TO_BE_DOWN);
+            }
         }
     }
 
@@ -104,9 +113,9 @@ public class HostedVespaPolicy implements Policy {
             OrchestratorContext context,
             ApplicationInstance applicationInstance,
             HostName hostName,
-            MutableStatusRegistry hostStatusService) throws HostStateChangeDeniedException {
+            ApplicationLock lock) throws HostStateChangeDeniedException {
         NodeGroup nodeGroup = new NodeGroup(applicationInstance, hostName);
-        ApplicationApi applicationApi = applicationApiFactory.create(nodeGroup, hostStatusService, clusterControllerClientFactory);
+        ApplicationApi applicationApi = applicationApiFactory.create(nodeGroup, lock, clusterControllerClientFactory);
         releaseSuspensionGrant(context, applicationApi);
     }
 
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/Policy.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/Policy.java
index aa7636227c8..c410cda23a8 100644
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/Policy.java
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/Policy.java
@@ -5,8 +5,7 @@ import com.yahoo.vespa.applicationmodel.ApplicationInstance;
 import com.yahoo.vespa.applicationmodel.HostName;
 import com.yahoo.vespa.orchestrator.OrchestratorContext;
 import com.yahoo.vespa.orchestrator.model.ApplicationApi;
-import com.yahoo.vespa.orchestrator.status.MutableStatusRegistry;
-import com.yahoo.vespa.orchestrator.status.StatusService;
+import com.yahoo.vespa.orchestrator.status.ApplicationLock;
 
 /**
  * @author oyving
@@ -33,6 +32,6 @@ public interface Policy {
     void releaseSuspensionGrant(
             OrchestratorContext context, ApplicationInstance applicationInstance,
             HostName hostName,
-            MutableStatusRegistry hostStatusService) throws HostStateChangeDeniedException;
+            ApplicationLock hostStatusService) throws HostStateChangeDeniedException;
 
 }
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostResource.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostResource.java
index 4bb93ffa3cb..054d3320215 100644
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostResource.java
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostResource.java
@@ -31,6 +31,7 @@ import javax.ws.rs.core.MediaType;
 import javax.ws.rs.core.Response;
 import javax.ws.rs.core.UriInfo;
 import java.net.URI;
+import java.time.Instant;
 import java.util.List;
 import java.util.logging.Logger;
 import java.util.stream.Collectors;
@@ -72,14 +73,15 @@ public class HostResource implements HostApi {
 
             return new GetHostResponse(
                     host.getHostName().s(),
-                    host.getHostStatus().name(),
+                    host.getHostInfo().status().name(),
+                    host.getHostInfo().suspendedSince().map(Instant::toString).orElse(null),
                     applicationUri.toString(),
                     hostServices);
         } catch (UncheckedTimeoutException e) {
-            log.log(LogLevel.INFO, "Failed to get host " + hostName + ": " + e.getMessage());
+            log.log(LogLevel.DEBUG, "Failed to get host " + hostName + ": " + e.getMessage());
             throw webExceptionFromTimeout("getHost", hostName, e);
         } catch (HostNameNotFoundException e) {
-            log.log(LogLevel.INFO, "Host not found: " + hostName);
+            log.log(LogLevel.DEBUG, "Host not found: " + hostName);
             throw new NotFoundException(e);
         }
     }
@@ -99,14 +101,14 @@ public class HostResource implements HostApi {
             try {
                 orchestrator.setNodeStatus(hostName, state);
             } catch (HostNameNotFoundException e) {
-                log.log(LogLevel.INFO, "Host not found: " + hostName);
+                log.log(LogLevel.DEBUG, "Host not found: " + hostName);
                 throw new NotFoundException(e);
             } catch (UncheckedTimeoutException e) {
-                log.log(LogLevel.INFO, "Failed to patch " + hostName + ": " + e.getMessage());
+                log.log(LogLevel.DEBUG, "Failed to patch " + hostName + ": " + e.getMessage());
                 throw webExceptionFromTimeout("patch", hostName, e);
             } catch (OrchestrationException e) {
                 String message = "Failed to set " + hostName + " to " + state + ": " + e.getMessage();
-                log.log(LogLevel.INFO, message, e);
+                log.log(LogLevel.DEBUG, message, e);
                 throw new InternalServerErrorException(message);
             }
         }
@@ -122,13 +124,13 @@ public class HostResource implements HostApi {
         try {
             orchestrator.suspend(hostName);
         } catch (HostNameNotFoundException e) {
-            log.log(LogLevel.INFO, "Host not found: " + hostName);
+            log.log(LogLevel.DEBUG, "Host not found: " + hostName);
             throw new NotFoundException(e);
         } catch (UncheckedTimeoutException e) {
-            log.log(LogLevel.INFO, "Failed to suspend " + hostName + ": " + e.getMessage());
+            log.log(LogLevel.DEBUG, "Failed to suspend " + hostName + ": " + e.getMessage());
             throw webExceptionFromTimeout("suspend", hostName, e);
         } catch (HostStateChangeDeniedException e) {
-            log.log(LogLevel.INFO, "Failed to suspend " + hostName + ": " + e.getMessage());
+            log.log(LogLevel.DEBUG, "Failed to suspend " + hostName + ": " + e.getMessage());
             throw webExceptionWithDenialReason("suspend", hostName, e);
         }
         return new UpdateHostResponse(hostName.s(), null);
@@ -140,13 +142,13 @@ public class HostResource implements HostApi {
         try {
             orchestrator.resume(hostName);
         } catch (HostNameNotFoundException e) {
-            log.log(LogLevel.INFO, "Host not found: " + hostName);
+            log.log(LogLevel.DEBUG, "Host not found: " + hostName);
             throw new NotFoundException(e);
         } catch (UncheckedTimeoutException e) {
-            log.log(LogLevel.INFO, "Failed to resume " + hostName + ": " + e.getMessage());
+            log.log(LogLevel.DEBUG, "Failed to resume " + hostName + ": " + e.getMessage());
             throw webExceptionFromTimeout("resume", hostName, e);
         } catch (HostStateChangeDeniedException e) {
-            log.log(LogLevel.INFO, "Failed to resume " + hostName + ": " + e.getMessage());
+            log.log(LogLevel.DEBUG, "Failed to resume " + hostName + ": " + e.getMessage());
             throw webExceptionWithDenialReason("resume", hostName, e);
         }
         return new UpdateHostResponse(hostName.s(), null);
@@ -155,8 +157,9 @@ public class HostResource implements HostApi {
     private static WebApplicationException webExceptionFromTimeout(String operationDescription,
                                                                    HostName hostName,
                                                                    UncheckedTimeoutException e) {
-        return createWebException(operationDescription, hostName, e, HostedVespaPolicy.DEADLINE_CONSTRAINT, e.getMessage(),
-                Response.Status.GATEWAY_TIMEOUT);
+        // Return timeouts as 409 Conflict instead of 504 Gateway Timeout to reduce noise in 5xx graphs.
+        return createWebException(operationDescription, hostName, e,
+                HostedVespaPolicy.DEADLINE_CONSTRAINT, e.getMessage(), Response.Status.CONFLICT);
     }
 
     private static WebApplicationException webExceptionWithDenialReason(
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostSuspensionResource.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostSuspensionResource.java
index 79e0fc0f3e9..6e857563f9b 100644
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostSuspensionResource.java
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostSuspensionResource.java
@@ -18,7 +18,6 @@ import javax.ws.rs.WebApplicationException;
 import javax.ws.rs.core.MediaType;
 import javax.ws.rs.core.Response;
 import java.util.List;
-import java.util.concurrent.TimeoutException;
 import java.util.logging.Logger;
 import java.util.stream.Collectors;
 
@@ -48,7 +47,7 @@ public class HostSuspensionResource implements HostSuspensionApi {
             throw createWebApplicationException(e.getMessage(), Response.Status.CONFLICT);
         } catch (UncheckedTimeoutException e) {
             log.log(LogLevel.DEBUG, "Failed to suspend nodes " + hostnames + " with parent host " + parentHostname, e);
-            throw createWebApplicationException(e.getMessage(), Response.Status.GATEWAY_TIMEOUT);
+            throw createWebApplicationException(e.getMessage(), Response.Status.CONFLICT);
         } catch (BatchHostNameNotFoundException e) {
             log.log(LogLevel.DEBUG, "Failed to suspend nodes " + hostnames + " with parent host " + parentHostname, e);
             // Note that we're returning BAD_REQUEST instead of NOT_FOUND because the resource identified
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/InstanceResource.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/InstanceResource.java
index bd265ed39e4..1cf2a2a4965 100644
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/InstanceResource.java
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/InstanceResource.java
@@ -11,13 +11,15 @@ import com.yahoo.vespa.applicationmodel.ConfigId;
 import com.yahoo.vespa.applicationmodel.HostName;
 import com.yahoo.vespa.applicationmodel.ServiceStatusInfo;
 import com.yahoo.vespa.applicationmodel.ServiceType;
-import com.yahoo.vespa.orchestrator.InstanceLookupService;
 import com.yahoo.vespa.orchestrator.OrchestratorUtil;
 import com.yahoo.vespa.orchestrator.restapi.wire.SlobrokEntryResponse;
-import com.yahoo.vespa.orchestrator.status.HostStatus;
+import com.yahoo.vespa.orchestrator.restapi.wire.WireHostInfo;
+import com.yahoo.vespa.orchestrator.status.HostInfo;
+import com.yahoo.vespa.orchestrator.status.HostInfos;
 import com.yahoo.vespa.orchestrator.status.StatusService;
 import com.yahoo.vespa.service.manager.MonitorManager;
 import com.yahoo.vespa.service.manager.UnionMonitorManager;
+import com.yahoo.vespa.service.monitor.ServiceMonitor;
 import com.yahoo.vespa.service.monitor.SlobrokApi;
 
 import javax.inject.Inject;
@@ -29,13 +31,13 @@ import javax.ws.rs.QueryParam;
 import javax.ws.rs.WebApplicationException;
 import javax.ws.rs.core.MediaType;
 import javax.ws.rs.core.Response;
+import java.time.Instant;
 import java.util.List;
-import java.util.Map;
-import java.util.Set;
+import java.util.TreeMap;
 import java.util.stream.Collectors;
 
 import static com.yahoo.vespa.orchestrator.OrchestratorUtil.getHostsUsedByApplicationInstance;
-import static com.yahoo.vespa.orchestrator.OrchestratorUtil.parseAppInstanceReference;
+import static com.yahoo.vespa.orchestrator.OrchestratorUtil.parseApplicationInstanceReference;
 
 /**
  * Provides a read-only API for looking into the current state as seen by the Orchestrator.
@@ -52,14 +54,14 @@ public class InstanceResource {
     private final StatusService statusService;
     private final SlobrokApi slobrokApi;
     private final MonitorManager rootManager;
-    private final InstanceLookupService instanceLookupService;
+    private final ServiceMonitor serviceMonitor;
 
     @Inject
-    public InstanceResource(@Component InstanceLookupService instanceLookupService,
+    public InstanceResource(@Component ServiceMonitor serviceMonitor,
                             @Component StatusService statusService,
                             @Component SlobrokApi slobrokApi,
                             @Component UnionMonitorManager rootManager) {
-        this.instanceLookupService = instanceLookupService;
+        this.serviceMonitor = serviceMonitor;
         this.statusService = statusService;
         this.slobrokApi = slobrokApi;
         this.rootManager = rootManager;
@@ -67,8 +69,8 @@ public class InstanceResource {
 
     @GET
     @Produces(MediaType.APPLICATION_JSON)
-    public Set<ApplicationInstanceReference> getAllInstances() {
-        return instanceLookupService.knownInstances();
+    public List<ApplicationInstanceReference> getAllInstances() {
+        return serviceMonitor.getAllApplicationInstanceReferences().stream().sorted().collect(Collectors.toList());
     }
 
     @GET
@@ -78,18 +80,27 @@ public class InstanceResource {
         ApplicationInstanceReference instanceId = parseInstanceId(instanceIdString);
 
         ApplicationInstance applicationInstance
-                = instanceLookupService.findInstanceById(instanceId)
+                = serviceMonitor.getApplication(instanceId)
                 .orElseThrow(() -> new WebApplicationException(Response.status(Response.Status.NOT_FOUND).build()));
 
-        Set<HostName> suspendedHosts = statusService.getSuspendedHostsByApplication().apply(applicationInstance.reference());
-        Map<HostName, String> hostStatusMap = getHostsUsedByApplicationInstance(applicationInstance)
-                .stream()
-                .collect(Collectors.toMap(hostName -> hostName,
-                                          hostName -> suspendedHosts.contains(hostName) ? HostStatus.ALLOWED_TO_BE_DOWN.name()
-                                                                                        : HostStatus.NO_REMARKS.name()));
+        HostInfos hostInfos = statusService.getHostInfosByApplicationResolver().apply(applicationInstance.reference());
+        TreeMap<HostName, WireHostInfo> hostStatusMap =
+                getHostsUsedByApplicationInstance(applicationInstance)
+                        .stream()
+                        .collect(Collectors.toMap(
+                                hostName -> hostName,
+                                hostName -> hostInfoToWire(hostInfos.getOrNoRemarks(hostName)),
+                                (u, v) -> { throw new IllegalStateException(); },
+                                TreeMap::new));
         return InstanceStatusResponse.create(applicationInstance, hostStatusMap);
     }
 
+    private WireHostInfo hostInfoToWire(HostInfo hostInfo) {
+        String hostStatusString = hostInfo.status().asString();
+        String suspendedSinceUtcOrNull = hostInfo.suspendedSince().map(Instant::toString).orElse(null);
+        return new WireHostInfo(hostStatusString, suspendedSinceUtcOrNull);
+    }
+
     @GET
     @Path("/{instanceId}/slobrok")
     @Produces(MediaType.APPLICATION_JSON)
@@ -141,7 +152,7 @@ public class InstanceResource {
 
     static ApplicationInstanceReference parseInstanceId(String instanceIdString) {
         try {
-            return parseAppInstanceReference(instanceIdString);
+            return parseApplicationInstanceReference(instanceIdString);
         } catch (IllegalArgumentException e) {
             throwBadRequest(e.getMessage());
             return null;  // Necessary for compiler
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/InstanceStatusResponse.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/InstanceStatusResponse.java
index c2ea0c9eddf..313c73e5c68 100644
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/InstanceStatusResponse.java
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/InstanceStatusResponse.java
@@ -4,9 +4,12 @@ package com.yahoo.vespa.orchestrator.resources;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.yahoo.vespa.applicationmodel.ApplicationInstance;
 import com.yahoo.vespa.applicationmodel.HostName;
+import com.yahoo.vespa.orchestrator.restapi.wire.WireHostInfo;
 
 import java.util.Map;
 import java.util.Objects;
+import java.util.TreeMap;
+import java.util.stream.Collectors;
 
 /*
  * @author andreer
@@ -14,16 +17,16 @@ import java.util.Objects;
 public class InstanceStatusResponse {
 
     private final ApplicationInstance applicationInstance;
-    private final Map<HostName, String> hostStates;
+    private final TreeMap<HostName, WireHostInfo> hostInfos;
 
-    private InstanceStatusResponse(ApplicationInstance applicationInstance, Map<HostName, String> hostStates) {
+    private InstanceStatusResponse(ApplicationInstance applicationInstance, TreeMap<HostName, WireHostInfo> hostInfos) {
         this.applicationInstance = applicationInstance;
-        this.hostStates = hostStates;
+        this.hostInfos = hostInfos;
     }
 
     public static InstanceStatusResponse create(
             ApplicationInstance applicationInstance,
-            Map<HostName, String> hostStates) {
+            TreeMap<HostName, WireHostInfo> hostStates) {
         return new InstanceStatusResponse(applicationInstance, hostStates);
     }
 
@@ -34,14 +37,24 @@ public class InstanceStatusResponse {
 
     @JsonProperty("hostStates")
     public Map<HostName, String> hostStates() {
-        return hostStates;
+        // TODO: Remove this once all clients have been moved to hostStatus.
+        return hostInfos.entrySet().stream()
+                .collect(Collectors.toMap(
+                        entry -> entry.getKey(),
+                        entry -> entry.getValue().hostStatus()
+                ));
+    }
+
+    @JsonProperty("hostInfos")
+    public TreeMap<HostName, WireHostInfo> hostInfos() {
+        return hostInfos;
     }
 
     @Override
     public String toString() {
         return "InstanceStatusResponse{" +
                 "applicationInstance=" + applicationInstance +
-                ", hostStates=" + hostStates +
+                ", hostInfos=" + hostInfos +
                 '}';
     }
 
@@ -51,11 +64,11 @@ public class InstanceStatusResponse {
         if (o == null || getClass() != o.getClass()) return false;
         InstanceStatusResponse that = (InstanceStatusResponse) o;
         return Objects.equals(applicationInstance, that.applicationInstance) &&
-                Objects.equals(hostStates, that.hostStates);
+                Objects.equals(hostInfos, that.hostInfos);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(applicationInstance, hostStates);
+        return Objects.hash(applicationInstance, hostInfos);
     }
 }
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ApplicationLock.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ApplicationLock.java
new file mode 100644
index 00000000000..8883f78b693
--- /dev/null
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ApplicationLock.java
@@ -0,0 +1,36 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.orchestrator.status;
+
+import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference;
+import com.yahoo.vespa.applicationmodel.HostName;
+
+/**
+ * The exclusive lock of an application in the status service, and methods to mutate host status
+ * and data structures guarded by such a lock.
+ *
+ * @author oyving
+ * @author Tony Vaagenes
+ * @author bakksjo
+ */
+public interface ApplicationLock extends AutoCloseable {
+
+    /** The reference of the locked application. */
+    ApplicationInstanceReference getApplicationInstanceReference();
+
+    /** Returns all host infos for this application. */
+    HostInfos getHostInfos();
+
+    /** Sets the state for the given host. */
+    void setHostState(HostName hostName, HostStatus status);
+
+    /** Returns the application status. */
+    ApplicationInstanceStatus getApplicationInstanceStatus();
+
+    /** Sets the orchestration status for the application instance. */
+    void setApplicationInstanceStatus(ApplicationInstanceStatus applicationInstanceStatus);
+
+    /** WARNING: Must not throw an exception. */
+    @Override
+    void close();
+
+}
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/HostInfos.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/HostInfos.java
index cb3f9cb8afe..22c2bcf1a79 100644
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/HostInfos.java
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/HostInfos.java
@@ -5,13 +5,13 @@ import com.yahoo.vespa.applicationmodel.HostName;
 
 import java.util.Map;
 import java.util.Set;
-import java.util.stream.Collectors;
 
 /**
- * Collection of suspended hosts.
+ * Collection of the suspended hosts of an application.
  *
  * @author hakonhall
  */
+// @Immutable
 public class HostInfos {
     private final Map<HostName, HostInfo> hostInfos;
 
@@ -19,16 +19,17 @@ public class HostInfos {
         this.hostInfos = Map.copyOf(hostInfos);
     }
 
-    /** Get all suspended hostnames. */
-    public Set<HostName> suspendedHostsnames() {
-        return hostInfos.entrySet().stream()
-                .filter(entry -> entry.getValue().status().isSuspended())
-                .map(entry -> entry.getKey())
-                .collect(Collectors.toSet());
+    public HostInfos() {
+        this.hostInfos = Map.of();
     }
 
     /** Get host info for hostname, returning a NO_REMARKS HostInfo if unknown. */
-    public HostInfo get(HostName hostname) {
+    public HostInfo getOrNoRemarks(HostName hostname) {
         return hostInfos.getOrDefault(hostname, HostInfo.createNoRemarks());
     }
+
+    /** The set of hostnames that were set in ZooKeeper - used for removing orphaned hostnames. */
+    public Set<HostName> getZkHostnames() {
+        return Set.copyOf(hostInfos.keySet());
+    }
 }
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/HostInfosCache.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/HostInfosCache.java
index 9998e7de918..7ee65ebcd0b 100644
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/HostInfosCache.java
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/HostInfosCache.java
@@ -7,6 +7,7 @@ import com.yahoo.vespa.curator.Curator;
 import com.yahoo.vespa.curator.recipes.CuratorCounter;
 
 import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.atomic.AtomicLong;
 
@@ -28,22 +29,29 @@ public class HostInfosCache implements HostInfosService {
         this.cacheGeneration = new AtomicLong(counter.get());
     }
 
-    @Override
-    public HostInfos getHostInfos(ApplicationInstanceReference application) {
+    public void refreshCache() {
         long newCacheGeneration = counter.get();
         if (cacheGeneration.getAndSet(newCacheGeneration) != newCacheGeneration) {
             suspendedHosts.clear();
         }
+    }
+
+    public HostInfos getCachedHostInfos(ApplicationInstanceReference reference) {
+        return suspendedHosts.computeIfAbsent(reference, wrappedService::getHostInfos);
+    }
 
-        return suspendedHosts.computeIfAbsent(application, wrappedService::getHostInfos);
+    @Override
+    public HostInfos getHostInfos(ApplicationInstanceReference reference) {
+        refreshCache();
+        return getCachedHostInfos(reference);
     }
 
     @Override
-    public boolean setHostStatus(ApplicationInstanceReference application, HostName hostName, HostStatus hostStatus) {
+    public boolean setHostStatus(ApplicationInstanceReference reference, HostName hostName, HostStatus hostStatus) {
         boolean isException = true;
         boolean modified = false;
         try {
-            modified = wrappedService.setHostStatus(application, hostName, hostStatus);
+            modified = wrappedService.setHostStatus(reference, hostName, hostStatus);
             isException = false;
         } finally {
             if (modified || isException) {
@@ -54,4 +62,18 @@ public class HostInfosCache implements HostInfosService {
 
         return modified;
     }
+
+    @Override
+    public void removeApplication(ApplicationInstanceReference reference) {
+        wrappedService.removeApplication(reference);
+        suspendedHosts.remove(reference);
+    }
+
+    @Override
+    public void removeHosts(ApplicationInstanceReference reference, Set<HostName> hostnames) {
+        if (hostnames.size() > 0) {
+            wrappedService.removeHosts(reference, hostnames);
+            counter.next();
+        }
+    }
 }
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/HostInfosService.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/HostInfosService.java
index f5c079f9ba3..a796a55236c 100644
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/HostInfosService.java
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/HostInfosService.java
@@ -4,12 +4,20 @@ package com.yahoo.vespa.orchestrator.status;
 import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference;
 import com.yahoo.vespa.applicationmodel.HostName;
 
+import java.util.Set;
+
 /**
  * @author hakonhall
  */
 interface HostInfosService {
-    HostInfos getHostInfos(ApplicationInstanceReference application);
+    HostInfos getHostInfos(ApplicationInstanceReference reference);
 
     /** Returns false if it is known that the operation was a no-op. */
-    boolean setHostStatus(ApplicationInstanceReference application, HostName hostName, HostStatus hostStatus);
+    boolean setHostStatus(ApplicationInstanceReference reference, HostName hostName, HostStatus hostStatus);
+
+    /** Remove application. */
+    void removeApplication(ApplicationInstanceReference reference);
+
+    /** Remove hosts for application. */
+    void removeHosts(ApplicationInstanceReference reference, Set<HostName> hostnames);
 }
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/HostInfosServiceImpl.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/HostInfosServiceImpl.java
new file mode 100644
index 00000000000..99f1f00e7dc
--- /dev/null
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/HostInfosServiceImpl.java
@@ -0,0 +1,143 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+package com.yahoo.vespa.orchestrator.status;
+
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.jdisc.Timer;
+import com.yahoo.log.LogLevel;
+import com.yahoo.path.Path;
+import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference;
+import com.yahoo.vespa.applicationmodel.HostName;
+import com.yahoo.vespa.curator.Curator;
+import com.yahoo.vespa.orchestrator.OrchestratorUtil;
+import com.yahoo.vespa.orchestrator.status.json.WireHostInfo;
+import org.apache.zookeeper.KeeperException.NoNodeException;
+
+import java.time.Instant;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.logging.Logger;
+import java.util.stream.Collectors;
+
+/**
+ * Handles all ZooKeeper data structures related to each active application, including HostInfo.
+ * Cache (if any) is above and not visible here.
+ *
+ * @author hakonhall
+ */
+public class HostInfosServiceImpl implements HostInfosService {
+    private static final Logger log = Logger.getLogger(HostInfosServiceImpl.class.getName());
+
+    private final Curator curator;
+    private final Timer timer;
+
+    HostInfosServiceImpl(Curator curator, Timer timer) {
+        this.curator = curator;
+        this.timer = timer;
+    }
+
+    @Override
+    public HostInfos getHostInfos(ApplicationInstanceReference reference) {
+        ApplicationId application = OrchestratorUtil.toApplicationId(reference);
+        String hostsRootPath = hostsPath(application);
+        if (uncheck(() -> curator.framework().checkExists().forPath(hostsRootPath)) == null) {
+            return new HostInfos();
+        } else {
+            List<String> hostnames = uncheck(() -> curator.framework().getChildren().forPath(hostsRootPath));
+            Map<HostName, HostInfo> hostInfos = hostnames.stream().collect(Collectors.toMap(
+                    hostname -> new HostName(hostname),
+                    hostname -> {
+                        byte[] bytes = uncheck(() -> curator.framework().getData().forPath(hostsRootPath + "/" + hostname));
+                        return WireHostInfo.deserialize(bytes);
+                    }));
+            return new HostInfos(hostInfos);
+        }
+    }
+
+    @Override
+    public boolean setHostStatus(ApplicationInstanceReference reference, HostName hostname, HostStatus status) {
+        ApplicationId application = OrchestratorUtil.toApplicationId(reference);
+        String path = hostPath(application, hostname);
+
+        if (status == HostStatus.NO_REMARKS) {
+            return deleteNode_ignoreNoNodeException(path, "Host already has state NO_REMARKS, path = " + path);
+        }
+
+        Optional<HostInfo> currentHostInfo = uncheck(() -> readBytesFromZk(path)).map(WireHostInfo::deserialize);
+        if (currentHostInfo.isEmpty()) {
+            Instant suspendedSince = timer.currentTime();
+            HostInfo hostInfo = HostInfo.createSuspended(status, suspendedSince);
+            byte[] hostInfoBytes = WireHostInfo.serialize(hostInfo);
+            uncheck(() -> curator.framework().create().creatingParentsIfNeeded().forPath(path, hostInfoBytes));
+        } else if (currentHostInfo.get().status() == status) {
+            return false;
+        } else {
+            Instant suspendedSince = currentHostInfo.get().suspendedSince().orElseGet(timer::currentTime);
+            HostInfo hostInfo = HostInfo.createSuspended(status, suspendedSince);
+            byte[] hostInfoBytes = WireHostInfo.serialize(hostInfo);
+            uncheck(() -> curator.framework().setData().forPath(path, hostInfoBytes));
+        }
+
+        return true;
+    }
+
+    @Override
+    public void removeApplication(ApplicationInstanceReference reference) {
+        ApplicationId application = OrchestratorUtil.toApplicationId(reference);
+        curator.delete(Path.fromString(applicationPath(application)));
+    }
+
+    @Override
+    public void removeHosts(ApplicationInstanceReference reference, Set<HostName> hostnames) {
+        ApplicationId application = OrchestratorUtil.toApplicationId(reference);
+        // Remove /vespa/host-status/APPLICATION_ID/hosts/HOSTNAME
+        hostnames.forEach(hostname -> curator.delete(Path.fromString(hostPath(application, hostname))));
+    }
+
+    private Optional<byte[]> readBytesFromZk(String path) throws Exception {
+        try {
+            return Optional.of(curator.framework().getData().forPath(path));
+        } catch (NoNodeException e) {
+            return Optional.empty();
+        }
+    }
+
+    private boolean deleteNode_ignoreNoNodeException(String path, String debugLogMessageIfNotExists) {
+        try {
+            curator.framework().delete().forPath(path);
+            return true;
+        } catch (NoNodeException e) {
+            log.log(LogLevel.DEBUG, debugLogMessageIfNotExists, e);
+            return false;
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    static String applicationPath(ApplicationId application) {
+        return "/vespa/host-status/" + application.serializedForm();
+    }
+
+    private static String hostsPath(ApplicationId application) {
+        return applicationPath(application) + "/hosts";
+    }
+
+    private static String hostPath(ApplicationId application, HostName hostname) {
+        return hostsPath(application) + "/" + hostname.s();
+    }
+
+    private <T> T uncheck(SupplierThrowingException<T> supplier) {
+        try {
+            return supplier.get();
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    @FunctionalInterface
+    private interface SupplierThrowingException<T> {
+        T get() throws Exception;
+    }
+}
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/HostStatus.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/HostStatus.java
index 6764ffb48ea..330e53f9586 100644
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/HostStatus.java
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/HostStatus.java
@@ -23,4 +23,5 @@ public enum HostStatus {
 
     HostStatus(boolean suspended) { this.suspended = suspended; }
     public boolean isSuspended() { return suspended; }
+    public String asString() { return name(); }
 }
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/MutableStatusRegistry.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/MutableStatusRegistry.java
deleted file mode 100644
index 92b0ec50011..00000000000
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/MutableStatusRegistry.java
+++ /dev/null
@@ -1,47 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.orchestrator.status;
-
-import com.yahoo.vespa.applicationmodel.HostName;
-
-import java.util.Set;
-
-/**
- * Registry of the suspension and host statuses for an application instance.
- *
- * @author oyving
- * @author Tony Vaagenes
- * @author bakksjo
- */
-public interface MutableStatusRegistry extends AutoCloseable {
-
-    /**
-     * Returns the status of this application.
-     */
-    ApplicationInstanceStatus getStatus();
-
-    /** Returns the host info of the given host. */
-    HostInfo getHostInfo(HostName hostName);
-
-    /**
-     * Returns the set of all suspended hosts for this application.
-     */
-    Set<HostName> getSuspendedHosts();
-
-    /**
-     * Sets the state for the given host.
-     */
-    void setHostState(HostName hostName, HostStatus status);
-
-    /**
-     * Sets the orchestration status for the application instance.
-     */
-    void setApplicationInstanceStatus(ApplicationInstanceStatus applicationInstanceStatus);
-
-    /**
-     * We don't want {@link AutoCloseable#close()} to throw an exception (what to do about it anyway?),
-     * so we override it here to strip the exception from the signature.
-     */
-    @Override
-    void close();
-
-}
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/StatusService.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/StatusService.java
index 9e3ee84e1d9..f6d14f609ee 100644
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/StatusService.java
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/StatusService.java
@@ -5,6 +5,7 @@ import com.google.common.util.concurrent.UncheckedTimeoutException;
 import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference;
 import com.yahoo.vespa.applicationmodel.HostName;
 import com.yahoo.vespa.orchestrator.OrchestratorContext;
+import com.yahoo.vespa.service.monitor.ServiceHostListener;
 
 import java.util.Set;
 import java.util.function.Function;
@@ -18,35 +19,15 @@ import java.util.function.Function;
  * @author Tony Vaagenes
  * @author smorgrav
  */
-public interface StatusService {
+public interface StatusService extends ServiceHostListener {
 
     /**
      * Returns a mutable host status registry for a locked application instance. All operations performed on
      * the returned registry are executed in the context of a lock, including read operations. Hence, multi-step
      * operations (e.g. read-then-write) are guaranteed to be consistent.
-     *
-     * Some limitations/caveats apply for certain implementations, and since clients of this API must be aware of
-     * these limitations/caveats when using those implementations, they are expressed here, at interface level
-     * rather than at implementation level, because the interface represents the lowest common denominator
-     * of guarantees offered by implementations. Specifically, it is the zookeeper-based implementation's semantics
-     * that "leak through" in this spec. Now, to the specific caveats:
-     *
-     * Locking this application instance only guarantees that the holder is the only one that can mutate host statuses
-     * for the application instance.
-     * It is _not_ safe to assume that there is only one entity holding the lock for a given application instance
-     * reference at any given time.
-     *
-     * You cannot have multiple locks in a single thread, even if they are for different application instances,
-     * (i.e. different HostStatusRegistry instances). (This is due to a limitation in SessionFailRetryLoop.)
-     *
-     * While read-then-write-operations are consistent (i.e. the current value doesn't change between the read
-     * and the write), it is possible that the lock is lost before it is explicitly released by the code. In
-     * this case, subsequent mutating operations will fail, but previous mutating operations are NOT rolled back.
-     * This may leave the registry in an inconsistent state (as judged by the client code).
      */
-    MutableStatusRegistry lockApplicationInstance_forCurrentThreadOnly(
-            OrchestratorContext context,
-            ApplicationInstanceReference applicationInstanceReference) throws UncheckedTimeoutException;
+    ApplicationLock lockApplication(OrchestratorContext context, ApplicationInstanceReference reference)
+            throws UncheckedTimeoutException;
 
     /**
      * Returns all application instances that are allowed to be down. The intention is to use this
@@ -57,16 +38,16 @@ public interface StatusService {
     Set<ApplicationInstanceReference> getAllSuspendedApplications();
 
     /**
-     * Returns a fresh, but not necessarily consistent mapping from applications to their set of suspended hosts.
+     * Returns a lambda, which when invoked for an application, returns an up-to-date snapshot of {@link HostInfos host infos}.
      *
-     * If the lock for an application is held when this mapping is acquired, new sets returned for that application
-     * are consistent and up to date for as long as the lock is held. (The sets themselves don't reflect changes.)
+     * <p>Unless the lock for the application is held, the returned snapshot may already be out of date.
+     * (The snapshot itself is immutable.)</p>
      */
-    Function<ApplicationInstanceReference, Set<HostName>> getSuspendedHostsByApplication();
+    Function<ApplicationInstanceReference, HostInfos> getHostInfosByApplicationResolver();
 
     /** Returns the status of the given application. This is consistent if its lock is held.*/
     ApplicationInstanceStatus getApplicationInstanceStatus(ApplicationInstanceReference application);
 
     /** Get host info for hostname in application. This is consistent if its lock is held. */
-    HostInfo getHostInfo(ApplicationInstanceReference applicationInstanceReference, HostName hostName);
+    HostInfo getHostInfo(ApplicationInstanceReference reference, HostName hostName);
 }
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZkApplicationLock.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZkApplicationLock.java
new file mode 100644
index 00000000000..dfaaecbcf82
--- /dev/null
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZkApplicationLock.java
@@ -0,0 +1,111 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+package com.yahoo.vespa.orchestrator.status;
+
+import com.yahoo.log.LogLevel;
+import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference;
+import com.yahoo.vespa.applicationmodel.HostName;
+import com.yahoo.vespa.curator.Curator;
+import org.apache.zookeeper.KeeperException;
+
+import java.util.logging.Logger;
+
+/**
+ * ZooKeeper implementation of {@link ApplicationLock}.
+ *
+ * @author hakonhall
+ */
+class ZkApplicationLock implements ApplicationLock {
+
+    private static final Logger log = Logger.getLogger(ZkApplicationLock.class.getName());
+
+    private final ZkStatusService statusService;
+    private final Curator curator;
+    private final Runnable onClose;
+    private final ApplicationInstanceReference reference;
+    private final boolean probe;
+    private final HostInfosCache hostInfosCache;
+
+    ZkApplicationLock(ZkStatusService statusService,
+                      Curator curator,
+                      Runnable onClose,
+                      ApplicationInstanceReference reference,
+                      boolean probe,
+                      HostInfosCache hostInfosCache) {
+        this.statusService = statusService;
+        this.curator = curator;
+        this.onClose = onClose;
+        this.reference = reference;
+        this.probe = probe;
+        this.hostInfosCache = hostInfosCache;
+    }
+
+    @Override
+    public ApplicationInstanceReference getApplicationInstanceReference() {
+        return reference;
+    }
+
+    @Override
+    public ApplicationInstanceStatus getApplicationInstanceStatus() {
+        return statusService.getApplicationInstanceStatus(reference);
+    }
+
+    @Override
+    public HostInfos getHostInfos() {
+        return hostInfosCache.getHostInfos(reference);
+    }
+
+    @Override
+    public void setHostState(final HostName hostName, final HostStatus status) {
+        if (probe) return;
+        hostInfosCache.setHostStatus(reference, hostName, status);
+    }
+
+    @Override
+    public void setApplicationInstanceStatus(ApplicationInstanceStatus applicationInstanceStatus) {
+        if (probe) return;
+
+        String path = statusService.applicationInstanceSuspendedPath(reference);
+        switch (applicationInstanceStatus) {
+            case NO_REMARKS:
+                deleteNode_ignoreNoNodeException(path);
+                break;
+            case ALLOWED_TO_BE_DOWN:
+                createNode_ignoreNodeExistsException(path);
+                break;
+        }
+    }
+
+    @Override
+    public void close()  {
+        try {
+            onClose.run();
+        } catch (RuntimeException e) {
+            // We may want to avoid logging some exceptions that may be expected, like when session expires.
+            log.log(LogLevel.WARNING,
+                    "Failed close application lock in " +
+                    ZkApplicationLock.class.getSimpleName() + ", will ignore and continue",
+                    e);
+        }
+    }
+
+    void deleteNode_ignoreNoNodeException(String path) {
+        try {
+            curator.framework().delete().forPath(path);
+        } catch (KeeperException.NoNodeException e) {
+            // ok
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    void createNode_ignoreNodeExistsException(String path) {
+        try {
+            curator.framework().create().creatingParentsIfNeeded().forPath(path);
+        } catch (KeeperException.NodeExistsException e) {
+            // ok
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+}
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZkStatusService.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZkStatusService.java
new file mode 100644
index 00000000000..246a70992c7
--- /dev/null
+++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZkStatusService.java
@@ -0,0 +1,313 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.orchestrator.status;
+
+import com.google.common.util.concurrent.UncheckedTimeoutException;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.container.jaxrs.annotation.Component;
+import com.yahoo.jdisc.Metric;
+import com.yahoo.jdisc.Timer;
+import com.yahoo.log.LogLevel;
+import com.yahoo.path.Path;
+import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference;
+import com.yahoo.vespa.applicationmodel.HostName;
+import com.yahoo.vespa.curator.Curator;
+import com.yahoo.vespa.curator.Lock;
+import com.yahoo.vespa.orchestrator.OrchestratorContext;
+import com.yahoo.vespa.orchestrator.OrchestratorUtil;
+import com.yahoo.vespa.service.monitor.AntiServiceMonitor;
+import com.yahoo.vespa.service.monitor.CriticalRegion;
+import org.apache.zookeeper.data.Stat;
+
+import javax.inject.Inject;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.function.Function;
+import java.util.logging.Logger;
+
+/**
+ * Stores instance suspension status and which hosts are allowed to go down in zookeeper.
+ *
+ * TODO: expiry of old application instances
+ * @author Tony Vaagenes
+ */
+public class ZkStatusService implements StatusService {
+
+    private static final Logger log = Logger.getLogger(ZkStatusService.class.getName());
+
+    final static String HOST_STATUS_BASE_PATH = "/vespa/host-status-service";
+    final static String APPLICATION_STATUS_BASE_PATH = "/vespa/application-status-service";
+
+    private final Curator curator;
+    private final HostInfosCache hostInfosCache;
+    private final Metric metric;
+    private final Timer timer;
+    private final AntiServiceMonitor antiServiceMonitor;
+
+    /**
+     * A cache of metric contexts for each possible dimension map. In practice, there is one dimension map
+     * for each application, so up to hundreds of elements.
+     */
+    private final ConcurrentHashMap<Map<String, String>, Metric.Context> cachedContexts = new ConcurrentHashMap<>();
+
+    @Inject
+    public ZkStatusService(
+            @Component Curator curator,
+            @Component Metric metric,
+            @Component Timer timer,
+            @Component AntiServiceMonitor antiServiceMonitor) {
+        this(curator,
+                metric,
+                timer,
+                new HostInfosCache(curator, new HostInfosServiceImpl(curator, timer)),
+                antiServiceMonitor);
+    }
+
+    /** Non-private for testing only. */
+    ZkStatusService(Curator curator, Metric metric, Timer timer, HostInfosCache hostInfosCache,
+                    AntiServiceMonitor antiServiceMonitor) {
+        this.curator = curator;
+        this.metric = metric;
+        this.timer = timer;
+        this.hostInfosCache = hostInfosCache;
+        this.antiServiceMonitor = antiServiceMonitor;
+    }
+
+    @Override
+    public Set<ApplicationInstanceReference> getAllSuspendedApplications() {
+        try {
+            Set<ApplicationInstanceReference> resultSet = new HashSet<>();
+
+            // Return empty set if the base path does not exist
+            Stat stat = curator.framework().checkExists().forPath(APPLICATION_STATUS_BASE_PATH);
+            if (stat == null) return resultSet;
+
+            // The path exist and we may have children
+            for (String referenceString : curator.framework().getChildren().forPath(APPLICATION_STATUS_BASE_PATH)) {
+                ApplicationInstanceReference reference = OrchestratorUtil.parseApplicationInstanceReference(referenceString);
+                resultSet.add(reference);
+            }
+
+            return resultSet;
+        } catch (Exception e) {
+            log.log(LogLevel.DEBUG, "Something went wrong while listing out applications in suspend.", e);
+            throw new RuntimeException(e);
+        }
+    }
+
+    /**
+     * Cache is checked for freshness when this mapping is created, and may be invalidated again later
+     * by other users of the cache. Since this function is backed by the cache, any such invalidation
+     * will be reflected in the returned mapping; all users of the cache collaborate in repopulating it.
+     */
+    @Override
+    public Function<ApplicationInstanceReference, HostInfos> getHostInfosByApplicationResolver() {
+        hostInfosCache.refreshCache();
+        return hostInfosCache::getCachedHostInfos;
+    }
+
+
+    /**
+     *  1) locks the status service for an application instance.
+     *  2) fails all operations in this thread when the session is lost,
+     *     since session loss might cause the lock to be lost.
+     *     Since it only fails operations in this thread,
+     *     all operations depending on a lock, including the locking itself, must be done in this thread.
+     *     Note that since it is the thread that fails, all status operations in this thread will fail
+     *     even if they're not supposed to be guarded by this lock
+     *     (i.e. the request is for another applicationInstanceReference)
+     */
+    @Override
+    public ApplicationLock lockApplication(OrchestratorContext context, ApplicationInstanceReference reference)
+            throws UncheckedTimeoutException {
+
+        Runnable onRegistryClose;
+
+        // A multi-application operation, aka batch suspension, will first issue a probe
+        // then a non-probe. With "large locks", the lock is not release in between -
+        // no lock is taken on the non-probe. Instead, the release is done on the multi-application
+        // context close.
+        if (context.hasLock(reference)) {
+            onRegistryClose = () -> {};
+        } else {
+            Runnable unlock = acquireLock(context, reference);
+            if (context.registerLockAcquisition(reference, unlock)) {
+                onRegistryClose = () -> {};
+            } else {
+                onRegistryClose = unlock;
+            }
+        }
+
+        try {
+            return new ZkApplicationLock(
+                    this,
+                    curator,
+                    onRegistryClose,
+                    reference,
+                    context.isProbe(),
+                    hostInfosCache);
+        } catch (Throwable t) {
+            // In case the constructor throws an exception.
+            onRegistryClose.run();
+            throw t;
+        }
+    }
+
+    private Runnable acquireLock(OrchestratorContext context, ApplicationInstanceReference reference)
+            throws UncheckedTimeoutException {
+        ApplicationId applicationId = OrchestratorUtil.toApplicationId(reference);
+        String app = applicationId.application().value() + "." + applicationId.instance().value();
+        Map<String, String> dimensions = Map.of(
+                "tenantName", applicationId.tenant().value(),
+                "applicationId", applicationId.toFullString(),
+                "app", app);
+        Metric.Context metricContext = cachedContexts.computeIfAbsent(dimensions, metric::createContext);
+
+        Duration duration = context.getTimeLeft();
+        String lockPath = applicationInstanceLock2Path(reference);
+        Lock lock = new Lock(lockPath, curator);
+
+        Instant startTime = timer.currentTime();
+        Instant acquireEndTime;
+        boolean lockAcquired = false;
+        try {
+            lock.acquire(duration);
+            lockAcquired = true;
+        } finally {
+            acquireEndTime = timer.currentTime();
+            double seconds = durationInSeconds(startTime, acquireEndTime);
+            metric.set("orchestrator.lock.acquire-latency", seconds, metricContext);
+            metric.set("orchestrator.lock.acquired", lockAcquired ? 1 : 0, metricContext);
+
+            metric.add("orchestrator.lock.acquire", 1, metricContext);
+            String acquireResultMetricName = lockAcquired ? "orchestrator.lock.acquire-success" : "orchestrator.lock.acquire-timedout";
+            metric.add(acquireResultMetricName, 1, metricContext);
+        }
+
+        CriticalRegion inaccessibleDuperModelRegion = antiServiceMonitor
+                .disallowDuperModelLockAcquisition(ZkStatusService.class.getSimpleName() + " application lock");
+
+        return () -> {
+            try {
+                lock.close();
+            } catch (RuntimeException e) {
+                // We may want to avoid logging some exceptions that may be expected, like when session expires.
+                log.log(LogLevel.WARNING,
+                        "Failed to close application lock for " +
+                                ZkStatusService.class.getSimpleName() + ", will ignore and continue",
+                        e);
+            }
+
+            inaccessibleDuperModelRegion.close();
+
+            Instant lockReleasedTime = timer.currentTime();
+            double seconds = durationInSeconds(acquireEndTime, lockReleasedTime);
+            metric.set("orchestrator.lock.hold-latency", seconds, metricContext);
+        };
+    }
+
+    private double durationInSeconds(Instant startInstant, Instant endInstant) {
+        return Duration.between(startInstant, endInstant).toMillis() / 1000.0;
+    }
+
+    @Override
+    public HostInfo getHostInfo(ApplicationInstanceReference reference, HostName hostName) {
+        return hostInfosCache.getHostInfos(reference).getOrNoRemarks(hostName);
+    }
+
+    @Override
+    public ApplicationInstanceStatus getApplicationInstanceStatus(ApplicationInstanceReference reference) {
+        try {
+            Stat statOrNull = curator.framework().checkExists().forPath(
+                    applicationInstanceSuspendedPath(reference));
+
+            return (statOrNull == null) ? ApplicationInstanceStatus.NO_REMARKS : ApplicationInstanceStatus.ALLOWED_TO_BE_DOWN;
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    /**
+     * Remove all host-related data in ZooKeeper for all hostnames outside the given set.
+     */
+    @Override
+    public void onApplicationActivate(ApplicationInstanceReference reference, Set<HostName> hostnames) {
+        withLockForAdminOp(reference, " was activated", () -> {
+            HostInfos hostInfos = hostInfosCache.getCachedHostInfos(reference);
+            Set<HostName> toRemove = new HashSet<>(hostInfos.getZkHostnames());
+            toRemove.removeAll(hostnames);
+            if (toRemove.size() > 0) {
+                hostInfosCache.removeHosts(reference, toRemove);
+            }
+        });
+    }
+
+    /**
+     * Remove the application from ZooKeeper.
+     *
+     * <ol>
+     *     <li>/vespa/host-status/APPLICATION_ID (should just be ./hosts/*)</li>
+     *     <li>/vespa/host-status-service/REFERENCE/hosts-allowed-down  (should just be ./*)</li>
+     *     <li>/vespa/application-status-service/REFERENCE  (should just be .)</li>
+     * </ol>
+     */
+    @Override
+    public void onApplicationRemove(ApplicationInstanceReference reference) {
+        withLockForAdminOp(reference, " was removed", () -> {
+            // /vespa/application-status-service/REFERENCE
+            curator.delete(Path.fromString(applicationInstanceSuspendedPath(reference)));
+
+            // /vespa/host-status-service/REFERENCE/hosts-allowed-down
+            curator.delete(Path.fromString(hostsAllowedDownPath(reference)));
+
+            // /vespa/host-status/APPLICATION_ID
+            hostInfosCache.removeApplication(reference);
+        });
+    }
+
+    private void withLockForAdminOp(ApplicationInstanceReference reference,
+                                    String eventDescription,
+                                    Runnable runnable) {
+        OrchestratorContext context = OrchestratorContext.createContextForAdminOp(timer.toUtcClock());
+
+        final ApplicationLock lock;
+        try {
+            lock = lockApplication(context, reference);
+        } catch (RuntimeException e) {
+            log.log(LogLevel.ERROR, "Failed to get Orchestrator lock on when " + reference +
+                    eventDescription + ": " + e.getMessage());
+            return;
+        }
+
+        try (lock) {
+            runnable.run();
+        } catch (RuntimeException e) {
+            log.log(LogLevel.ERROR, "Failed to clean up after " + reference + eventDescription +
+                    ": " + e.getMessage());
+        }
+    }
+
+    static String applicationInstanceReferencePath(ApplicationInstanceReference reference) {
+        return HOST_STATUS_BASE_PATH + '/' + reference.asString();
+    }
+
+    private static String hostsAllowedDownPath(ApplicationInstanceReference reference) {
+        return applicationInstanceReferencePath(reference) + "/hosts-allowed-down";
+    }
+
+    private static String applicationInstanceLock2Path(ApplicationInstanceReference reference) {
+        return applicationInstanceReferencePath(reference) + "/lock2";
+    }
+
+    String applicationInstanceSuspendedPath(ApplicationInstanceReference reference) {
+        return APPLICATION_STATUS_BASE_PATH + "/" + OrchestratorUtil.toRestApiFormat(reference);
+    }
+
+    private static String hostAllowedDownPath(ApplicationInstanceReference reference, HostName hostname) {
+        return hostsAllowedDownPath(reference) + '/' + hostname.s();
+    }
+
+}
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZookeeperStatusService.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZookeeperStatusService.java
deleted file mode 100644
index ecf3a161c83..00000000000
--- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZookeeperStatusService.java
+++ /dev/null
@@ -1,461 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.orchestrator.status;
-
-import com.google.common.util.concurrent.UncheckedTimeoutException;
-import com.yahoo.config.provision.ApplicationId;
-import com.yahoo.container.jaxrs.annotation.Component;
-import com.yahoo.jdisc.Metric;
-import com.yahoo.jdisc.Timer;
-import com.yahoo.log.LogLevel;
-import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference;
-import com.yahoo.vespa.applicationmodel.HostName;
-import com.yahoo.vespa.curator.Curator;
-import com.yahoo.vespa.curator.Lock;
-import com.yahoo.vespa.orchestrator.OrchestratorContext;
-import com.yahoo.vespa.orchestrator.OrchestratorUtil;
-import com.yahoo.vespa.orchestrator.status.json.WireHostInfo;
-import org.apache.zookeeper.KeeperException.NoNodeException;
-import org.apache.zookeeper.KeeperException.NodeExistsException;
-import org.apache.zookeeper.data.Stat;
-
-import javax.inject.Inject;
-import java.time.Duration;
-import java.time.Instant;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.function.Function;
-import java.util.logging.Logger;
-import java.util.stream.Collectors;
-
-/**
- * Stores instance suspension status and which hosts are allowed to go down in zookeeper.
- *
- * TODO: expiry of old application instances
- * @author Tony Vaagenes
- */
-public class ZookeeperStatusService implements StatusService {
-
-    private static final Logger log = Logger.getLogger(ZookeeperStatusService.class.getName());
-
-    final static String HOST_STATUS_BASE_PATH = "/vespa/host-status-service";
-    final static String APPLICATION_STATUS_BASE_PATH = "/vespa/application-status-service";
-
-    private final Curator curator;
-    private final HostInfosCache hostInfosCache;
-    private final Metric metric;
-    private final Timer timer;
-
-    /**
-     * A cache of metric contexts for each possible dimension map. In practice, there is one dimension map
-     * for each application, so up to hundreds of elements.
-     */
-    private final ConcurrentHashMap<Map<String, String>, Metric.Context> cachedContexts = new ConcurrentHashMap<>();
-
-    @Inject
-    public ZookeeperStatusService(@Component Curator curator, @Component Metric metric, @Component Timer timer) {
-        this.curator = curator;
-        this.metric = metric;
-        this.timer = timer;
-
-        // Insert a cache above some ZooKeeper accesses
-        this.hostInfosCache = new HostInfosCache(curator, new HostInfosService() {
-            @Override
-            public HostInfos getHostInfos(ApplicationInstanceReference application) {
-                return ZookeeperStatusService.this.getHostInfosFromZk(application);
-            }
-
-            @Override
-            public boolean setHostStatus(ApplicationInstanceReference application, HostName hostName, HostStatus hostStatus) {
-                return ZookeeperStatusService.this.setHostStatusInZk(application, hostName, hostStatus);
-            }
-        });
-    }
-
-    @Override
-    public Set<ApplicationInstanceReference> getAllSuspendedApplications() {
-        try {
-            Set<ApplicationInstanceReference> resultSet = new HashSet<>();
-
-            // Return empty set if the base path does not exist
-            Stat stat = curator.framework().checkExists().forPath(APPLICATION_STATUS_BASE_PATH);
-            if (stat == null) return resultSet;
-
-            // The path exist and we may have children
-            for (String appRefStr : curator.framework().getChildren().forPath(APPLICATION_STATUS_BASE_PATH)) {
-                ApplicationInstanceReference appRef = OrchestratorUtil.parseAppInstanceReference(appRefStr);
-                resultSet.add(appRef);
-            }
-
-            return resultSet;
-        } catch (Exception e) {
-            log.log(LogLevel.DEBUG, "Something went wrong while listing out applications in suspend.", e);
-            throw new RuntimeException(e);
-        }
-    }
-
-    /**
-     * Cache is checked for freshness when this mapping is created, and may be invalidated again later
-     * by other users of the cache. Since this function is backed by the cache, any such invalidations
-     * will be reflected in the returned mapping; all users of the cache collaborate in repopulating it.
-     */
-    @Override
-    public Function<ApplicationInstanceReference, Set<HostName>> getSuspendedHostsByApplication() {
-        return application -> hostInfosCache.getHostInfos(application).suspendedHostsnames();
-    }
-
-
-    /**
-     *  1) locks the status service for an application instance.
-     *  2) fails all operations in this thread when the session is lost,
-     *     since session loss might cause the lock to be lost.
-     *     Since it only fails operations in this thread,
-     *     all operations depending on a lock, including the locking itself, must be done in this thread.
-     *     Note that since it is the thread that fails, all status operations in this thread will fail
-     *     even if they're not supposed to be guarded by this lock
-     *     (i.e. the request is for another applicationInstanceReference)
-     */
-    @Override
-    public MutableStatusRegistry lockApplicationInstance_forCurrentThreadOnly(
-            OrchestratorContext context,
-            ApplicationInstanceReference applicationInstanceReference) throws UncheckedTimeoutException {
-        ApplicationId applicationId = OrchestratorUtil.toApplicationId(applicationInstanceReference);
-        String app = applicationId.application().value() + "." + applicationId.instance().value();
-        Map<String, String> dimensions = Map.of(
-                "tenantName", applicationId.tenant().value(),
-                "applicationId", applicationId.toFullString(),
-                "app", app);
-        Metric.Context metricContext = cachedContexts.computeIfAbsent(dimensions, metric::createContext);
-
-        Duration duration = context.getTimeLeft();
-        String lockPath = applicationInstanceLock2Path(applicationInstanceReference);
-        Lock lock = new Lock(lockPath, curator);
-
-        Instant startTime = timer.currentTime();
-        Instant acquireEndTime;
-        boolean lockAcquired = false;
-        try {
-            lock.acquire(duration);
-            lockAcquired = true;
-        } finally {
-            acquireEndTime = timer.currentTime();
-            double seconds = durationInSeconds(startTime, acquireEndTime);
-            metric.set("orchestrator.lock.acquire-latency", seconds, metricContext);
-            metric.set("orchestrator.lock.acquired", lockAcquired ? 1 : 0, metricContext);
-
-            metric.add("orchestrator.lock.acquire", 1, metricContext);
-            String acquireResultMetricName = lockAcquired ? "orchestrator.lock.acquire-success" : "orchestrator.lock.acquire-timedout";
-            metric.add(acquireResultMetricName, 1, metricContext);
-        }
-
-        Runnable updateLockHoldMetric = () -> {
-            Instant lockReleasedTime = timer.currentTime();
-            double seconds = durationInSeconds(acquireEndTime, lockReleasedTime);
-            metric.set("orchestrator.lock.hold-latency", seconds, metricContext);
-        };
-
-        try {
-            return new ZkMutableStatusRegistry(lock, applicationInstanceReference, context.isProbe(), updateLockHoldMetric);
-        } catch (Throwable t) {
-            // In case the constructor throws an exception.
-            updateLockHoldMetric.run();
-            lock.close();
-            throw t;
-        }
-    }
-
-    private double durationInSeconds(Instant startInstant, Instant endInstant) {
-        return Duration.between(startInstant, endInstant).toMillis() / 1000.0;
-    }
-
-    /** Do not call this directly: should be called behind a cache. */
-    private boolean setHostStatusInZk(ApplicationInstanceReference applicationInstanceReference,
-                                      HostName hostName,
-                                      HostStatus status) {
-        String hostAllowedDownPath = hostAllowedDownPath(applicationInstanceReference, hostName);
-
-        boolean modified;
-        try {
-            switch (status) {
-                case NO_REMARKS:
-                    modified = deleteNode_ignoreNoNodeException(hostAllowedDownPath, "Host already has state NO_REMARKS, path = " + hostAllowedDownPath);
-                    break;
-                case ALLOWED_TO_BE_DOWN:
-                    modified = createNode_ignoreNodeExistsException(hostAllowedDownPath, "Host already has state ALLOWED_TO_BE_DOWN, path = " + hostAllowedDownPath);
-                    break;
-                default:
-                    throw new IllegalArgumentException("Unexpected status '" + status + "'.");
-            }
-
-            modified |= setHostInfoInZk(applicationInstanceReference, hostName, status);
-        } catch (Exception e) {
-            throw new RuntimeException(e);
-        }
-
-        return modified;
-    }
-
-    /** Returns false if no changes were made. */
-    private boolean setHostInfoInZk(ApplicationInstanceReference application, HostName hostname, HostStatus status)
-            throws Exception {
-        String path = hostPath(application, hostname);
-
-        if (status == HostStatus.NO_REMARKS) {
-            return deleteNode_ignoreNoNodeException(path, "Host already has state NO_REMARKS, path = " + path);
-        }
-
-        Optional<HostInfo> currentHostInfo = readBytesFromZk(path).map(WireHostInfo::deserialize);
-        if (currentHostInfo.isEmpty()) {
-            Instant suspendedSince = timer.currentTime();
-            HostInfo hostInfo = HostInfo.createSuspended(status, suspendedSince);
-            byte[] hostInfoBytes = WireHostInfo.serialize(hostInfo);
-            curator.framework().create().creatingParentsIfNeeded().forPath(path, hostInfoBytes);
-        } else if (currentHostInfo.get().status() == status) {
-            return false;
-        } else {
-            Instant suspendedSince = currentHostInfo.get().suspendedSince().orElseGet(timer::currentTime);
-            HostInfo hostInfo = HostInfo.createSuspended(status, suspendedSince);
-            byte[] hostInfoBytes = WireHostInfo.serialize(hostInfo);
-            curator.framework().setData().forPath(path, hostInfoBytes);
-        }
-
-        return true;
-    }
-
-    private boolean deleteNode_ignoreNoNodeException(String path, String debugLogMessageIfNotExists) throws Exception {
-        try {
-            curator.framework().delete().forPath(path);
-            return true;
-        } catch (NoNodeException e) {
-            log.log(LogLevel.DEBUG, debugLogMessageIfNotExists, e);
-            return false;
-        }
-    }
-
-    private boolean createNode_ignoreNodeExistsException(String path, String debugLogMessageIfExists) throws Exception {
-        try {
-            curator.framework().create()
-                    .creatingParentsIfNeeded()
-                    .forPath(path);
-            return true;
-        } catch (NodeExistsException e) {
-            log.log(LogLevel.DEBUG, debugLogMessageIfExists, e);
-            return false;
-        }
-    }
-
-    private Optional<byte[]> readBytesFromZk(String path) throws Exception {
-        try {
-            return Optional.of(curator.framework().getData().forPath(path));
-        } catch (NoNodeException e) {
-            return Optional.empty();
-        }
-    }
-
-    private void updateNodeInZk(String path, byte[] bytes) throws Exception {
-        curator.framework().setData().forPath(path, bytes);
-    }
-
-    private void createNodeInZk(String path, byte[] bytes) throws Exception {
-        curator.framework().create().creatingParentsIfNeeded().forPath(path, bytes);
-    }
-
-    @Override
-    public HostInfo getHostInfo(ApplicationInstanceReference applicationInstanceReference, HostName hostName) {
-        return hostInfosCache.getHostInfos(applicationInstanceReference).get(hostName);
-    }
-
-    private Set<HostName> hostsDownFor(ApplicationInstanceReference application) {
-        try {
-            if (curator.framework().checkExists().forPath(hostsAllowedDownPath(application)) == null)
-                return Collections.emptySet();
-
-            return curator.framework().getChildren().forPath(hostsAllowedDownPath(application))
-                          .stream().map(HostName::new)
-                          .collect(Collectors.toUnmodifiableSet());
-        }
-        catch (Exception e) {
-            throw new RuntimeException(e);
-        }
-    }
-
-    /** Do not call this directly: should be called behind a cache. */
-    private HostInfos getHostInfosFromZk(ApplicationInstanceReference application) {
-        Map<HostName, HostInfo> hostInfos;
-        String hostsRootPath = hostsPath(application);
-        if (uncheck(() -> curator.framework().checkExists().forPath(hostsRootPath)) == null) {
-            hostInfos = new HashMap<>();
-        } else {
-            List<String> hostnames = uncheck(() -> curator.framework().getChildren().forPath(hostsRootPath));
-            hostInfos = new HashMap<>(hostnames.stream().collect(Collectors.toMap(
-                    hostname -> new HostName(hostname),
-                    hostname -> {
-                        byte[] bytes = uncheck(() -> curator.framework().getData().forPath(hostsRootPath + "/" + hostname));
-                        return WireHostInfo.deserialize(bytes);
-                    })));
-        }
-
-        // For backwards compatibility we'll add HostInfos from the old hosts-allowed-down ZK path,
-        // using the creation time as the since path. The new hosts ZK path should contain a subset of
-        // the hostnames under hosts-allowed-down ZK path. Eventually these sets should be identical.
-        // Once that's true we can stop writing to hosts-allowed-down, remove this code, and all
-        // data in hosts-allowed-down can be removed.
-        Set<HostName> legacyHostsDown = hostsDownFor(application);
-        Map<HostName, HostInfo> legacyHostInfos = legacyHostsDown.stream()
-                .filter(hostname -> !hostInfos.containsKey(hostname))
-                .collect(Collectors.toMap(
-                hostname -> hostname,
-                hostname -> {
-                    Stat stat = uncheck(() -> curator.framework()
-                            .checkExists()
-                            .forPath(hostsAllowedDownPath(application) + "/" + hostname.s()));
-                    return HostInfo.createSuspended(HostStatus.ALLOWED_TO_BE_DOWN, Instant.ofEpochMilli(stat.getCtime()));
-                }
-        ));
-
-        hostInfos.putAll(legacyHostInfos);
-        return new HostInfos(hostInfos);
-    }
-
-    private <T> T uncheck(SupplierThrowingException<T> supplier) {
-        try {
-            return supplier.get();
-        } catch (Exception e) {
-            throw new RuntimeException(e);
-        }
-    }
-
-    @FunctionalInterface
-    interface SupplierThrowingException<T> {
-        T get() throws Exception;
-    }
-
-    @Override
-    public ApplicationInstanceStatus getApplicationInstanceStatus(ApplicationInstanceReference applicationInstanceReference) {
-        try {
-            Stat statOrNull = curator.framework().checkExists().forPath(
-                    applicationInstanceSuspendedPath(applicationInstanceReference));
-
-            return (statOrNull == null) ? ApplicationInstanceStatus.NO_REMARKS : ApplicationInstanceStatus.ALLOWED_TO_BE_DOWN;
-        } catch (Exception e) {
-            throw new RuntimeException(e);
-        }
-    }
-
-    static String applicationInstanceReferencePath(ApplicationInstanceReference applicationInstanceReference) {
-        return HOST_STATUS_BASE_PATH + '/' +
-                applicationInstanceReference.tenantId() + ":" + applicationInstanceReference.applicationInstanceId();
-    }
-
-    private static String applicationPath(ApplicationInstanceReference applicationInstanceReference) {
-        ApplicationId applicationId = OrchestratorUtil.toApplicationId(applicationInstanceReference);
-        return "/vespa/host-status/" + applicationId.serializedForm();
-    }
-
-    private static String hostsPath(ApplicationInstanceReference applicationInstanceReference) {
-        return applicationPath(applicationInstanceReference) + "/hosts";
-    }
-
-    private static String hostsAllowedDownPath(ApplicationInstanceReference applicationInstanceReference) {
-        return applicationInstanceReferencePath(applicationInstanceReference) + "/hosts-allowed-down";
-    }
-
-    private static String applicationInstanceLock2Path(ApplicationInstanceReference applicationInstanceReference) {
-        return applicationInstanceReferencePath(applicationInstanceReference) + "/lock2";
-    }
-
-    private String applicationInstanceSuspendedPath(ApplicationInstanceReference applicationInstanceReference) {
-        return APPLICATION_STATUS_BASE_PATH + "/" + OrchestratorUtil.toRestApiFormat(applicationInstanceReference);
-    }
-
-    private static String hostAllowedDownPath(ApplicationInstanceReference applicationInstanceReference, HostName hostname) {
-        return hostsAllowedDownPath(applicationInstanceReference) + '/' + hostname.s();
-    }
-
-    private static String hostPath(ApplicationInstanceReference application, HostName hostname) {
-        return hostsPath(application) + "/" + hostname.s();
-    }
-
-    private class ZkMutableStatusRegistry implements MutableStatusRegistry {
-
-        private final Lock lock;
-        private final ApplicationInstanceReference applicationInstanceReference;
-        private final boolean probe;
-        private final Runnable onLockRelease;
-
-        public ZkMutableStatusRegistry(Lock lock,
-                                       ApplicationInstanceReference applicationInstanceReference,
-                                       boolean probe,
-                                       Runnable onLockRelease) {
-            this.lock = lock;
-            this.applicationInstanceReference = applicationInstanceReference;
-            this.probe = probe;
-            this.onLockRelease = onLockRelease;
-        }
-
-        @Override
-        public ApplicationInstanceStatus getStatus() {
-            return getApplicationInstanceStatus(applicationInstanceReference);
-        }
-
-        @Override
-        public HostInfo getHostInfo(HostName hostName) {
-            return ZookeeperStatusService.this.getHostInfo(applicationInstanceReference, hostName);
-        }
-
-        @Override
-        public Set<HostName> getSuspendedHosts() {
-            return hostInfosCache.getHostInfos(applicationInstanceReference).suspendedHostsnames();
-        }
-
-        @Override
-        public void setHostState(final HostName hostName, final HostStatus status) {
-            if (probe) return;
-            log.log(LogLevel.INFO, "Setting host " + hostName + " to status " + status);
-            hostInfosCache.setHostStatus(applicationInstanceReference, hostName, status);
-        }
-
-        @Override
-        public void setApplicationInstanceStatus(ApplicationInstanceStatus applicationInstanceStatus) {
-            if (probe) return;
-
-            log.log(LogLevel.INFO, "Setting app " + applicationInstanceReference.asString() + " to status " + applicationInstanceStatus);
-
-            String path = applicationInstanceSuspendedPath(applicationInstanceReference);
-            try {
-                switch (applicationInstanceStatus) {
-                    case NO_REMARKS:
-                        deleteNode_ignoreNoNodeException(path,
-                                "Instance is already in state NO_REMARKS, path = " + path);
-                        break;
-                    case ALLOWED_TO_BE_DOWN:
-                        createNode_ignoreNodeExistsException(path,
-                                "Instance is already in state ALLOWED_TO_BE_DOWN, path = " + path);
-                        break;
-                }
-            } catch (Exception e) {
-                throw new RuntimeException(e);
-            }
-        }
-
-        @Override
-        public void close()  {
-            onLockRelease.run();
-            try {
-                lock.close();
-            } catch (RuntimeException e) {
-                // We may want to avoid logging some exceptions that may be expected, like when session expires.
-                log.log(LogLevel.WARNING,
-                        "Failed to close application lock for " +
-                        ZookeeperStatusService.class.getSimpleName() + ", will ignore and continue",
-                        e);
-            }
-        }
-    }
-
-}
diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/DummyAntiServiceMonitor.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/DummyAntiServiceMonitor.java
new file mode 100644
index 00000000000..d22d99b5c9c
--- /dev/null
+++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/DummyAntiServiceMonitor.java
@@ -0,0 +1,15 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.orchestrator;
+
+import com.yahoo.vespa.service.monitor.AntiServiceMonitor;
+import com.yahoo.vespa.service.monitor.CriticalRegion;
+
+/**
+ * @author hakonhall
+ */
+public class DummyAntiServiceMonitor implements AntiServiceMonitor {
+    @Override
+    public CriticalRegion disallowDuperModelLockAcquisition(String regionDescription) {
+        return () -> {};
+    }
+}
diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/DummyInstanceLookupService.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/DummyServiceMonitor.java
index a54f5284ee0..501a09f78ff 100644
--- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/DummyInstanceLookupService.java
+++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/DummyServiceMonitor.java
@@ -15,8 +15,14 @@ import com.yahoo.vespa.applicationmodel.ServiceType;
 import com.yahoo.vespa.applicationmodel.TenantId;
 import com.yahoo.vespa.orchestrator.model.NodeGroup;
 import com.yahoo.vespa.orchestrator.model.VespaModelUtil;
-
-import java.util.HashSet;
+import com.yahoo.vespa.service.monitor.AntiServiceMonitor;
+import com.yahoo.vespa.service.monitor.CriticalRegion;
+import com.yahoo.vespa.service.monitor.ServiceModel;
+import com.yahoo.vespa.service.monitor.ServiceMonitor;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
@@ -27,13 +33,13 @@ import java.util.stream.Collectors;
  * @author oyving
  * @author smorgrav
  */
-public class DummyInstanceLookupService implements InstanceLookupService {
+public class DummyServiceMonitor implements ServiceMonitor, AntiServiceMonitor {
 
     public static final HostName TEST1_HOST_NAME = new HostName("test1.hostname.tld");
     public static final HostName TEST3_HOST_NAME = new HostName("test3.hostname.tld");
     public static final HostName TEST6_HOST_NAME = new HostName("test6.hostname.tld");
 
-    private static final Set<ApplicationInstance> apps = new HashSet<>();
+    private static final List<ApplicationInstance> apps = new ArrayList<>();
 
     static {
         apps.add(new ApplicationInstance(
@@ -121,26 +127,27 @@ public class DummyInstanceLookupService implements InstanceLookupService {
     }
 
     // A node group is tied to an application, so we need to define them after we have populated the above applications.
-    public final static NodeGroup TEST1_NODE_GROUP = new NodeGroup(new DummyInstanceLookupService().findInstanceByHost(TEST1_HOST_NAME).get(), TEST1_HOST_NAME);
-    public final static NodeGroup TEST3_NODE_GROUP = new NodeGroup(new DummyInstanceLookupService().findInstanceByHost(TEST3_HOST_NAME).get(), TEST3_HOST_NAME);
-    public final static NodeGroup TEST6_NODE_GROUP = new NodeGroup(new DummyInstanceLookupService().findInstanceByHost(TEST6_HOST_NAME).get(), TEST6_HOST_NAME);
+    public final static NodeGroup TEST1_NODE_GROUP = new NodeGroup(new DummyServiceMonitor().getApplication(TEST1_HOST_NAME).get(), TEST1_HOST_NAME);
+    public final static NodeGroup TEST3_NODE_GROUP = new NodeGroup(new DummyServiceMonitor().getApplication(TEST3_HOST_NAME).get(), TEST3_HOST_NAME);
+    public final static NodeGroup TEST6_NODE_GROUP = new NodeGroup(new DummyServiceMonitor().getApplication(TEST6_HOST_NAME).get(), TEST6_HOST_NAME);
 
+    @Override
+    public ServiceModel getServiceModelSnapshot() {
+        throw new UnsupportedOperationException();
+    }
 
     @Override
-    public Optional<ApplicationInstance> findInstanceById(
-            final ApplicationInstanceReference applicationInstanceReference) {
-        for (ApplicationInstance app : apps) {
-            if (app.reference().equals(applicationInstanceReference)) return Optional.of(app);
-        }
-        return Optional.empty();
+    public Set<ApplicationInstanceReference> getAllApplicationInstanceReferences() {
+        return apps.stream().map(a ->
+                new ApplicationInstanceReference(a.tenantId(),a.applicationInstanceId())).collect(Collectors.toSet());
     }
 
     @Override
-    public Optional<ApplicationInstance> findInstanceByHost(HostName hostName) {
+    public Optional<ApplicationInstance> getApplication(HostName hostname) {
         for (ApplicationInstance app : apps) {
             for (ServiceCluster cluster : app.serviceClusters()) {
                 for (ServiceInstance service : cluster.serviceInstances()) {
-                    if (hostName.equals(service.hostName())) return Optional.of(app);
+                    if (hostname.equals(service.hostName())) return Optional.of(app);
                 }
             }
         }
@@ -148,10 +155,16 @@ public class DummyInstanceLookupService implements InstanceLookupService {
     }
 
     @Override
-    public Set<ApplicationInstanceReference> knownInstances() {
-        return apps.stream().map(a ->
-                new ApplicationInstanceReference(a.tenantId(),a.applicationInstanceId())).collect(Collectors.toSet());
+    public Optional<ApplicationInstance> getApplication(ApplicationInstanceReference reference) {
+        for (ApplicationInstance app : apps) {
+            if (app.reference().equals(reference)) return Optional.of(app);
+        }
+        return Optional.empty();
+    }
 
+    @Override
+    public CriticalRegion disallowDuperModelLockAcquisition(String regionDescription) {
+        return () -> {};
     }
 
     public static Set<HostName> getContentHosts(ApplicationInstanceReference appRef) {
@@ -166,7 +179,7 @@ public class DummyInstanceLookupService implements InstanceLookupService {
         return hosts;
     }
 
-    public static  Set<ApplicationInstance> getApplications() {
+    public static  List<ApplicationInstance> getApplications() {
        return apps;
     }
 }
diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorContextTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorContextTest.java
new file mode 100644
index 00000000000..16b66b2804e
--- /dev/null
+++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorContextTest.java
@@ -0,0 +1,44 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.orchestrator;
+
+import com.yahoo.test.ManualClock;
+import com.yahoo.vespa.applicationmodel.ApplicationInstanceId;
+import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference;
+import com.yahoo.vespa.applicationmodel.TenantId;
+import org.junit.Test;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * @author hakonhall
+ */
+public class OrchestratorContextTest {
+    private final ApplicationInstanceReference application = new ApplicationInstanceReference(
+            new TenantId("tenant"),
+            new ApplicationInstanceId("app:dev:us-east-1:default"));
+
+    @Test
+    public void testLargeLocks() {
+        var mutable = new Object() { boolean locked = true; };
+        Runnable unlock = () -> mutable.locked = false;
+
+        try (OrchestratorContext rootContext = OrchestratorContext.createContextForMultiAppOp(new ManualClock())) {
+            try (OrchestratorContext probeContext = rootContext.createSubcontextForSingleAppOp(true)) {
+                assertFalse(probeContext.hasLock(application));
+                assertTrue(probeContext.registerLockAcquisition(application, unlock));
+
+                assertTrue(probeContext.hasLock(application));
+                assertTrue(mutable.locked);
+            }
+
+            try (OrchestratorContext nonProbeContext = rootContext.createSubcontextForSingleAppOp(false)) {
+                assertTrue(nonProbeContext.hasLock(application));
+                assertTrue(mutable.locked);
+            }
+
+            assertTrue(mutable.locked);
+        }
+        assertFalse(mutable.locked);
+    }
+}
\ No newline at end of file
diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java
index 45d4c531898..b32a6aafa56 100644
--- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java
+++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorImplTest.java
@@ -17,6 +17,7 @@ import com.yahoo.vespa.applicationmodel.ServiceStatus;
 import com.yahoo.vespa.applicationmodel.ServiceType;
 import com.yahoo.vespa.applicationmodel.TenantId;
 import com.yahoo.vespa.curator.mock.MockCurator;
+import com.yahoo.vespa.flags.InMemoryFlagSource;
 import com.yahoo.vespa.orchestrator.controller.ClusterControllerClientFactory;
 import com.yahoo.vespa.orchestrator.controller.ClusterControllerClientFactoryMock;
 import com.yahoo.vespa.orchestrator.model.ApplicationApiFactory;
@@ -25,10 +26,12 @@ import com.yahoo.vespa.orchestrator.policy.BatchHostStateChangeDeniedException;
 import com.yahoo.vespa.orchestrator.policy.HostStateChangeDeniedException;
 import com.yahoo.vespa.orchestrator.policy.HostedVespaClusterPolicy;
 import com.yahoo.vespa.orchestrator.policy.HostedVespaPolicy;
+import com.yahoo.vespa.orchestrator.status.ApplicationLock;
 import com.yahoo.vespa.orchestrator.status.HostStatus;
 import com.yahoo.vespa.orchestrator.status.StatusService;
-import com.yahoo.vespa.orchestrator.status.ZookeeperStatusService;
+import com.yahoo.vespa.orchestrator.status.ZkStatusService;
 import com.yahoo.vespa.service.monitor.ServiceModel;
+import com.yahoo.vespa.service.monitor.ServiceMonitor;
 import org.junit.Before;
 import org.junit.Test;
 import org.mockito.ArgumentCaptor;
@@ -36,7 +39,9 @@ import org.mockito.InOrder;
 
 import java.util.Arrays;
 import java.util.Iterator;
+import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.Set;
 
 import static com.yahoo.vespa.orchestrator.status.ApplicationInstanceStatus.ALLOWED_TO_BE_DOWN;
@@ -55,6 +60,11 @@ import static org.mockito.Mockito.doThrow;
 import static org.mockito.Mockito.inOrder;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
+import static org.mockito.Mockito.when;
+import static org.mockito.internal.verification.VerificationModeFactory.atLeastOnce;
 
 /**
  * Test Orchestrator with a mock backend (the MockCurator)
@@ -64,6 +74,13 @@ import static org.mockito.Mockito.spy;
 public class OrchestratorImplTest {
 
     private final ApplicationApiFactory applicationApiFactory = new ApplicationApiFactory(3);
+    private final InMemoryFlagSource flagSource = new InMemoryFlagSource();
+    private final MockCurator curator = new MockCurator();
+    private ZkStatusService statusService = new ZkStatusService(
+            curator,
+            mock(Metric.class),
+            new TestTimer(),
+            new DummyAntiServiceMonitor());
 
     private ApplicationId app1;
     private ApplicationId app2;
@@ -75,20 +92,21 @@ public class OrchestratorImplTest {
     @Before
     public void setUp() {
         // Extract applications and hosts from dummy instance lookup service
-        Iterator<ApplicationInstance> iterator = DummyInstanceLookupService.getApplications().iterator();
+        Iterator<ApplicationInstance> iterator = DummyServiceMonitor.getApplications().iterator();
         ApplicationInstanceReference app1_ref = iterator.next().reference();
         app1 = OrchestratorUtil.toApplicationId(app1_ref);
-        app1_host1 = DummyInstanceLookupService.getContentHosts(app1_ref).iterator().next();
+        app1_host1 = DummyServiceMonitor.getContentHosts(app1_ref).iterator().next();
         app2 = OrchestratorUtil.toApplicationId(iterator.next().reference());
 
         clustercontroller = new ClusterControllerClientFactoryMock();
         orchestrator = new OrchestratorImpl(new HostedVespaPolicy(new HostedVespaClusterPolicy(), clustercontroller, applicationApiFactory),
                                             clustercontroller,
-                                            new ZookeeperStatusService(new MockCurator(), mock(Metric.class), new TestTimer()),
-                                            new DummyInstanceLookupService(),
+                                            statusService,
+                                            new DummyServiceMonitor(),
                                             0,
                                             new ManualClock(),
-                                            applicationApiFactory);
+                                            applicationApiFactory,
+                                            flagSource);
 
         clustercontroller.setAllDummyNodesAsUp();
     }
@@ -230,8 +248,8 @@ public class OrchestratorImplTest {
 
     @Test
     public void applicationReferenceHasTenantAndAppInstance() {
-        InstanceLookupService service = new DummyInstanceLookupService();
-        String applicationInstanceId = service.findInstanceByHost(DummyInstanceLookupService.TEST1_HOST_NAME).get()
+        ServiceMonitor service = new DummyServiceMonitor();
+        String applicationInstanceId = service.getApplication(DummyServiceMonitor.TEST1_HOST_NAME).get()
                 .reference().toString();
         assertEquals("test-tenant-id:application:prod:utopia-1:instance", applicationInstanceId);
     }
@@ -253,21 +271,21 @@ public class OrchestratorImplTest {
         orchestrator.suspendAll(
                 new HostName("parentHostname"),
                 Arrays.asList(
-                        DummyInstanceLookupService.TEST1_HOST_NAME,
-                        DummyInstanceLookupService.TEST3_HOST_NAME,
-                        DummyInstanceLookupService.TEST6_HOST_NAME));
+                        DummyServiceMonitor.TEST1_HOST_NAME,
+                        DummyServiceMonitor.TEST3_HOST_NAME,
+                        DummyServiceMonitor.TEST6_HOST_NAME));
 
         // As of 2016-06-07 the order of the node groups are as follows:
         //   TEST3: mediasearch:imagesearch:default
         //   TEST6: tenant-id-3:application-instance-3:default
         //   TEST1: test-tenant-id:application:instance
         InOrder order = inOrder(orchestrator);
-        verifySuspendGroup(order, orchestrator, DummyInstanceLookupService.TEST3_NODE_GROUP, true);
-        verifySuspendGroup(order, orchestrator, DummyInstanceLookupService.TEST6_NODE_GROUP, true);
-        verifySuspendGroup(order, orchestrator, DummyInstanceLookupService.TEST1_NODE_GROUP, true);
-        verifySuspendGroup(order, orchestrator, DummyInstanceLookupService.TEST3_NODE_GROUP, false);
-        verifySuspendGroup(order, orchestrator, DummyInstanceLookupService.TEST6_NODE_GROUP, false);
-        verifySuspendGroup(order, orchestrator, DummyInstanceLookupService.TEST1_NODE_GROUP, false);
+        verifySuspendGroup(order, orchestrator, DummyServiceMonitor.TEST3_NODE_GROUP, true);
+        verifySuspendGroup(order, orchestrator, DummyServiceMonitor.TEST6_NODE_GROUP, true);
+        verifySuspendGroup(order, orchestrator, DummyServiceMonitor.TEST1_NODE_GROUP, true);
+        verifySuspendGroup(order, orchestrator, DummyServiceMonitor.TEST3_NODE_GROUP, false);
+        verifySuspendGroup(order, orchestrator, DummyServiceMonitor.TEST6_NODE_GROUP, false);
+        verifySuspendGroup(order, orchestrator, DummyServiceMonitor.TEST1_NODE_GROUP, false);
         order.verifyNoMoreInteractions();
     }
 
@@ -284,18 +302,18 @@ public class OrchestratorImplTest {
         OrchestratorImpl orchestrator = spy(this.orchestrator);
 
         Throwable supensionFailure = new HostStateChangeDeniedException(
-                DummyInstanceLookupService.TEST6_HOST_NAME,
+                DummyServiceMonitor.TEST6_HOST_NAME,
                 "some-constraint",
                 "error message");
-        doThrow(supensionFailure).when(orchestrator).suspendGroup(any(), eq(DummyInstanceLookupService.TEST6_NODE_GROUP));
+        doThrow(supensionFailure).when(orchestrator).suspendGroup(any(), eq(DummyServiceMonitor.TEST6_NODE_GROUP));
 
         try {
             orchestrator.suspendAll(
                     new HostName("parentHostname"),
                     Arrays.asList(
-                            DummyInstanceLookupService.TEST1_HOST_NAME,
-                            DummyInstanceLookupService.TEST3_HOST_NAME,
-                            DummyInstanceLookupService.TEST6_HOST_NAME));
+                            DummyServiceMonitor.TEST1_HOST_NAME,
+                            DummyServiceMonitor.TEST3_HOST_NAME,
+                            DummyServiceMonitor.TEST6_HOST_NAME));
             fail();
         } catch (BatchHostStateChangeDeniedException e) {
             assertEquals("Failed to suspend NodeGroup{application=tenant-id-3:application-instance-3:prod:utopia-1:default, " +
@@ -306,15 +324,85 @@ public class OrchestratorImplTest {
         }
 
         InOrder order = inOrder(orchestrator);
-        order.verify(orchestrator).suspendGroup(any(), eq(DummyInstanceLookupService.TEST3_NODE_GROUP));
-        order.verify(orchestrator).suspendGroup(any(), eq(DummyInstanceLookupService.TEST6_NODE_GROUP));
+        order.verify(orchestrator).suspendGroup(any(), eq(DummyServiceMonitor.TEST3_NODE_GROUP));
+        order.verify(orchestrator).suspendGroup(any(), eq(DummyServiceMonitor.TEST6_NODE_GROUP));
         order.verifyNoMoreInteractions();
     }
 
     @Test
+    public void testLargeLocks() throws Exception {
+        var tenantId = new TenantId("tenant");
+        var applicationInstanceId = new ApplicationInstanceId("app:dev:us-east-1:default");
+        var applicationInstanceReference = new ApplicationInstanceReference(tenantId, applicationInstanceId);
+
+        var policy = mock(HostedVespaPolicy.class);
+        var zookeeperStatusService = mock(ZkStatusService.class);
+        var serviceMonitor = mock(ServiceMonitor.class);
+        var applicationInstance = mock(ApplicationInstance.class);
+        var clusterControllerClientFactory = mock(ClusterControllerClientFactory.class);
+        var clock = new ManualClock();
+        var applicationApiFactory = mock(ApplicationApiFactory.class);
+        var lock = mock(ApplicationLock.class);
+
+        when(serviceMonitor.getApplication(any(HostName.class))).thenReturn(Optional.of(applicationInstance));
+        when(applicationInstance.reference()).thenReturn(applicationInstanceReference);
+        when(zookeeperStatusService.lockApplication(any(), any())).thenReturn(lock);
+        when(lock.getApplicationInstanceStatus()).thenReturn(NO_REMARKS);
+
+        var orchestrator = new OrchestratorImpl(
+                policy,
+                clusterControllerClientFactory,
+                zookeeperStatusService,
+                serviceMonitor,
+                20,
+                clock,
+                applicationApiFactory,
+                flagSource);
+
+        HostName parentHostname = new HostName("parent.vespa.ai");
+
+        verify(serviceMonitor, atLeastOnce()).registerListener(zookeeperStatusService);
+        verifyNoMoreInteractions(serviceMonitor);
+
+        orchestrator.suspendAll(parentHostname, List.of(parentHostname));
+
+        ArgumentCaptor<OrchestratorContext> contextCaptor = ArgumentCaptor.forClass(OrchestratorContext.class);
+        verify(zookeeperStatusService, times(2)).lockApplication(contextCaptor.capture(), any());
+        List<OrchestratorContext> contexts = contextCaptor.getAllValues();
+
+        // First invocation is probe, second is not.
+        assertEquals(2, contexts.size());
+        assertTrue(contexts.get(0).isProbe());
+        assertTrue(contexts.get(0).largeLocks());
+        assertFalse(contexts.get(1).isProbe());
+        assertTrue(contexts.get(1).largeLocks());
+
+        verify(applicationApiFactory, times(2)).create(any(), any(), any());
+        verify(policy, times(2)).grantSuspensionRequest(any(), any());
+        verify(serviceMonitor, atLeastOnce()).getApplication(any(HostName.class));
+        verify(lock, times(2)).getApplicationInstanceStatus();
+
+        // Each zookeeperStatusService that is created, is closed.
+        verify(zookeeperStatusService, times(2)).lockApplication(any(), any());
+        verify(lock, times(2)).close();
+
+        verifyNoMoreInteractions(
+                policy,
+                clusterControllerClientFactory,
+                zookeeperStatusService,
+                lock,
+                serviceMonitor,
+                applicationApiFactory);
+    }
+
+    @Test
     public void testGetHost() throws Exception {
         ClusterControllerClientFactory clusterControllerClientFactory = new ClusterControllerClientFactoryMock();
-        StatusService statusService = new ZookeeperStatusService(new MockCurator(), mock(Metric.class), new TestTimer());
+        StatusService statusService = new ZkStatusService(
+                new MockCurator(),
+                mock(Metric.class),
+                new TestTimer(),
+                new DummyAntiServiceMonitor());
 
         HostName hostName = new HostName("host.yahoo.com");
         TenantId tenantId = new TenantId("tenant");
@@ -340,29 +428,30 @@ public class OrchestratorImplTest {
                                                hostName,
                                                ServiceStatus.NOT_CHECKED)))));
 
-        InstanceLookupService lookupService = new ServiceMonitorInstanceLookupService(
-                () -> new ServiceModel(Map.of(reference, applicationInstance)));
+        ServiceMonitor serviceMonitor = () -> new ServiceModel(Map.of(reference, applicationInstance));
 
         orchestrator = new OrchestratorImpl(new HostedVespaPolicy(new HostedVespaClusterPolicy(), clusterControllerClientFactory, applicationApiFactory),
                                             clusterControllerClientFactory,
                                             statusService,
-                                            lookupService,
+                                            serviceMonitor,
                                             0,
                                             new ManualClock(),
-                                            applicationApiFactory);
+                                            applicationApiFactory,
+                                            flagSource);
 
         orchestrator.setNodeStatus(hostName, HostStatus.ALLOWED_TO_BE_DOWN);
 
         Host host = orchestrator.getHost(hostName);
         assertEquals(reference, host.getApplicationInstanceReference());
         assertEquals(hostName, host.getHostName());
-        assertEquals(HostStatus.ALLOWED_TO_BE_DOWN, host.getHostStatus());
+        assertEquals(HostStatus.ALLOWED_TO_BE_DOWN, host.getHostInfo().status());
+        assertTrue(host.getHostInfo().suspendedSince().isPresent());
         assertEquals(2, host.getServiceInstances().size());
     }
 
     private boolean isInMaintenance(ApplicationId appId, HostName hostName) throws ApplicationIdNotFoundException {
-        for (ApplicationInstance app : DummyInstanceLookupService.getApplications()) {
-            if (app.reference().equals(OrchestratorUtil.toApplicationInstanceReference(appId, new DummyInstanceLookupService()))) {
+        for (ApplicationInstance app : DummyServiceMonitor.getApplications()) {
+            if (app.reference().equals(OrchestratorUtil.toApplicationInstanceReference(appId, new DummyServiceMonitor()))) {
                 return clustercontroller.isInMaintenance(app, hostName);
             }
         }
diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorUtilTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorUtilTest.java
index 230048f505d..76fbb5c9fe2 100644
--- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorUtilTest.java
+++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorUtilTest.java
@@ -39,14 +39,14 @@ public class OrchestratorUtilTest {
     public void applicationid_conversion_are_symmetric() throws Exception {
 
         // From appId to appRef and back
-        ApplicationInstanceReference appRef = OrchestratorUtil.toApplicationInstanceReference(APPID_1, new DummyInstanceLookupService());
+        ApplicationInstanceReference appRef = OrchestratorUtil.toApplicationInstanceReference(APPID_1, new DummyServiceMonitor());
         ApplicationId appIdRoundTrip = OrchestratorUtil.toApplicationId(appRef);
 
         Assert.assertEquals(APPID_1, appIdRoundTrip);
 
         // From appRef to appId and back
         ApplicationId appId = OrchestratorUtil.toApplicationId(APPREF_1);
-        ApplicationInstanceReference appRefRoundTrip = OrchestratorUtil.toApplicationInstanceReference(appId, new DummyInstanceLookupService());
+        ApplicationInstanceReference appRefRoundTrip = OrchestratorUtil.toApplicationInstanceReference(appId, new DummyServiceMonitor());
 
         Assert.assertEquals(APPREF_1, appRefRoundTrip);
     }
diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientFactoryMock.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientFactoryMock.java
index 5c5ee7d2260..502e42481aa 100644
--- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientFactoryMock.java
+++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientFactoryMock.java
@@ -4,7 +4,7 @@ package com.yahoo.vespa.orchestrator.controller;
 import com.yahoo.vespa.applicationmodel.ApplicationInstance;
 import com.yahoo.vespa.applicationmodel.ClusterId;
 import com.yahoo.vespa.applicationmodel.HostName;
-import com.yahoo.vespa.orchestrator.DummyInstanceLookupService;
+import com.yahoo.vespa.orchestrator.DummyServiceMonitor;
 import com.yahoo.vespa.orchestrator.OrchestratorContext;
 import com.yahoo.vespa.orchestrator.model.VespaModelUtil;
 
@@ -37,8 +37,8 @@ public class ClusterControllerClientFactoryMock implements ClusterControllerClie
     }
 
     public void setAllDummyNodesAsUp() {
-        for (ApplicationInstance app : DummyInstanceLookupService.getApplications()) {
-            Set<HostName> hosts = DummyInstanceLookupService.getContentHosts(app.reference());
+        for (ApplicationInstance app : DummyServiceMonitor.getApplications()) {
+            Set<HostName> hosts = DummyServiceMonitor.getContentHosts(app.reference());
             for (HostName host : hosts) {
                 ClusterId clusterName = VespaModelUtil.getContentClusterName(app, host);
                 int storageNodeIndex = VespaModelUtil.getStorageNodeIndex(app, host);
diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ApplicationApiImplTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ApplicationApiImplTest.java
index d5734a73de0..ee33a92367b 100644
--- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ApplicationApiImplTest.java
+++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ApplicationApiImplTest.java
@@ -327,7 +327,7 @@ public class ApplicationApiImplTest {
     private void verifyStorageNodesAllowedToBeDown(
             ApplicationApi applicationApi, HostName... hostNames) {
         List<HostName> actualStorageNodes =
-                applicationApi.getStorageNodesAllowedToBeDownInGroupInReverseClusterOrder().stream()
+                applicationApi.getSuspendedStorageNodesInGroupInReverseClusterOrder().stream()
                 .map(storageNode -> storageNode.hostName())
                 .collect(Collectors.toList());
         assertEquals(Arrays.asList(hostNames), actualStorageNodes);
diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ClusterApiImplTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ClusterApiImplTest.java
index 8e42ccdc79d..a5cb5cfa630 100644
--- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ClusterApiImplTest.java
+++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ClusterApiImplTest.java
@@ -13,6 +13,7 @@ import com.yahoo.vespa.applicationmodel.TenantId;
 import com.yahoo.vespa.orchestrator.OrchestratorUtil;
 import com.yahoo.vespa.orchestrator.policy.HostStateChangeDeniedException;
 import com.yahoo.vespa.orchestrator.policy.HostedVespaClusterPolicy;
+import com.yahoo.vespa.orchestrator.status.HostInfos;
 import com.yahoo.vespa.orchestrator.status.HostStatus;
 import org.junit.Test;
 
@@ -20,7 +21,6 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.EnumSet;
-import java.util.HashMap;
 import java.util.List;
 import java.util.Optional;
 import java.util.Set;
@@ -75,20 +75,15 @@ public class ClusterApiImplTest {
                 applicationApi,
                 serviceCluster,
                 new NodeGroup(modelUtils.createApplicationInstance(new ArrayList<>()), hostName5),
-                modelUtils.getHostStatusMap(),
+                modelUtils.getHostInfos(),
                 modelUtils.getClusterControllerClientFactory(), ModelTestUtils.NUMBER_OF_CONFIG_SERVERS);
 
         assertEquals("{ clusterId=cluster, serviceType=service-type }", clusterApi.clusterInfo());
         assertFalse(clusterApi.isStorageCluster());
-        assertEquals("[ServiceInstance{configId=service-2, hostName=host2, serviceStatus=" +
-                        "ServiceStatusInfo{status=DOWN, since=Optional.empty, lastChecked=Optional.empty}}, "
-                        + "ServiceInstance{configId=service-3, hostName=host3, serviceStatus=" +
-                        "ServiceStatusInfo{status=UP, since=Optional.empty, lastChecked=Optional.empty}}, "
-                        + "ServiceInstance{configId=service-4, hostName=host4, serviceStatus=" +
-                        "ServiceStatusInfo{status=DOWN, since=Optional.empty, lastChecked=Optional.empty}}]",
-                clusterApi.servicesDownAndNotInGroupDescription());
-        assertEquals("[host3, host4]",
-                clusterApi.nodesAllowedToBeDownNotInGroupDescription());
+        assertEquals(" Suspended hosts: [host3, host4]. Services down on resumed hosts: [" +
+                        "ServiceInstance{configId=service-2, hostName=host2, serviceStatus=" +
+                        "ServiceStatusInfo{status=DOWN, since=Optional.empty, lastChecked=Optional.empty}}].",
+                clusterApi.downDescription());
         assertEquals(60, clusterApi.percentageOfServicesDown());
         assertEquals(80, clusterApi.percentageOfServicesDownIfGroupIsAllowedToBeDown());
     }
@@ -109,9 +104,9 @@ public class ClusterApiImplTest {
             fail();
         } catch (HostStateChangeDeniedException e) {
             assertThat(e.getMessage(),
-                    containsString("Changing the state of cfg1 would violate enough-services-up: Suspension percentage " +
-                            "for service type configserver would increase from 33% to 66%, over the limit of 10%. " +
-                            "These instances may be down: [1 missing config server] and these hosts are allowed to be down: []"));
+                    containsString("Changing the state of cfg1 would violate enough-services-up: " +
+                            "Suspension for service type configserver would increase from 33% to 66%, " +
+                            "over the limit of 10%. Services down on resumed hosts: [1 missing config server]."));
         }
     }
 
@@ -184,7 +179,7 @@ public class ClusterApiImplTest {
                 applicationApi,
                 serviceCluster,
                 new NodeGroup(modelUtils.createApplicationInstance(new ArrayList<>()), groupNodes),
-                modelUtils.getHostStatusMap(),
+                modelUtils.getHostInfos(),
                 modelUtils.getClusterControllerClientFactory(), ModelTestUtils.NUMBER_OF_CONFIG_SERVERS);
 
         assertEquals(expectedNoServicesInGroupIsUp, clusterApi.noServicesInGroupIsUp());
@@ -214,7 +209,7 @@ public class ClusterApiImplTest {
                 applicationApi,
                 serviceCluster,
                 new NodeGroup(applicationInstance, hostName1, hostName3),
-                new HashMap<>(),
+                new HostInfos(),
                 modelUtils.getClusterControllerClientFactory(), ModelTestUtils.NUMBER_OF_CONFIG_SERVERS);
 
         assertTrue(clusterApi.isStorageCluster());
@@ -254,7 +249,7 @@ public class ClusterApiImplTest {
                 applicationApi,
                 serviceCluster,
                 new NodeGroup(application, hostnames.get(0)),
-                modelUtils.getHostStatusMap(),
+                modelUtils.getHostInfos(),
                 modelUtils.getClusterControllerClientFactory(), clusterSize);
 
         assertEquals(clusterSize - serviceStatusList.size(), clusterApi.missingServices());
diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ModelTestUtils.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ModelTestUtils.java
index 729b3ae79ff..8162542e540 100644
--- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ModelTestUtils.java
+++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/ModelTestUtils.java
@@ -16,28 +16,34 @@ import com.yahoo.vespa.applicationmodel.ServiceStatus;
 import com.yahoo.vespa.applicationmodel.ServiceType;
 import com.yahoo.vespa.applicationmodel.TenantId;
 import com.yahoo.vespa.curator.mock.MockCurator;
+import com.yahoo.vespa.flags.InMemoryFlagSource;
+import com.yahoo.vespa.orchestrator.DummyAntiServiceMonitor;
 import com.yahoo.vespa.orchestrator.OrchestrationException;
 import com.yahoo.vespa.orchestrator.Orchestrator;
 import com.yahoo.vespa.orchestrator.OrchestratorContext;
 import com.yahoo.vespa.orchestrator.OrchestratorImpl;
-import com.yahoo.vespa.orchestrator.ServiceMonitorInstanceLookupService;
 import com.yahoo.vespa.orchestrator.controller.ClusterControllerClientFactory;
 import com.yahoo.vespa.orchestrator.controller.ClusterControllerClientFactoryMock;
 import com.yahoo.vespa.orchestrator.policy.HostedVespaClusterPolicy;
 import com.yahoo.vespa.orchestrator.policy.HostedVespaPolicy;
+import com.yahoo.vespa.orchestrator.status.ApplicationLock;
+import com.yahoo.vespa.orchestrator.status.HostInfo;
+import com.yahoo.vespa.orchestrator.status.HostInfos;
 import com.yahoo.vespa.orchestrator.status.HostStatus;
-import com.yahoo.vespa.orchestrator.status.MutableStatusRegistry;
 import com.yahoo.vespa.orchestrator.status.StatusService;
-import com.yahoo.vespa.orchestrator.status.ZookeeperStatusService;
+import com.yahoo.vespa.orchestrator.status.ZkStatusService;
 import com.yahoo.vespa.service.monitor.ServiceModel;
+import com.yahoo.vespa.service.monitor.ServiceMonitor;
 import com.yahoo.yolean.Exceptions;
 
 import java.time.Clock;
+import java.time.Instant;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.stream.Collectors;
 
 import static org.mockito.Mockito.mock;
 
@@ -48,24 +54,46 @@ class ModelTestUtils {
 
     public static final int NUMBER_OF_CONFIG_SERVERS = 3;
 
+    private final InMemoryFlagSource flagSource = new InMemoryFlagSource();
     private final Map<ApplicationInstanceReference, ApplicationInstance> applications = new HashMap<>();
     private final ClusterControllerClientFactory clusterControllerClientFactory = new ClusterControllerClientFactoryMock();
     private final Map<HostName, HostStatus> hostStatusMap = new HashMap<>();
-    private final StatusService statusService = new ZookeeperStatusService(new MockCurator(), mock(Metric.class), new TestTimer());
+    private final ServiceMonitor serviceMonitor = () -> new ServiceModel(applications);
+    private final StatusService statusService = new ZkStatusService(
+            new MockCurator(),
+            mock(Metric.class),
+            new TestTimer(),
+            new DummyAntiServiceMonitor());
     private final Orchestrator orchestrator = new OrchestratorImpl(new HostedVespaPolicy(new HostedVespaClusterPolicy(), clusterControllerClientFactory, applicationApiFactory()),
                                                                    clusterControllerClientFactory,
                                                                    statusService,
-                                                                   new ServiceMonitorInstanceLookupService(() -> new ServiceModel(applications)),
+                                                                   serviceMonitor,
                                                                    0,
                                                                    new ManualClock(),
-                                                                   applicationApiFactory());
+                                                                   applicationApiFactory(),
+                                                                   flagSource);
 
     ApplicationApiFactory applicationApiFactory() {
         return new ApplicationApiFactory(NUMBER_OF_CONFIG_SERVERS);
     }
 
-    Map<HostName, HostStatus> getHostStatusMap() {
-        return hostStatusMap;
+    HostInfos getHostInfos() {
+        Instant now = Instant.now();
+
+        Map<HostName, HostInfo> hostInfosMap = hostStatusMap.entrySet().stream()
+                .collect(Collectors.toMap(
+                        entry -> entry.getKey(),
+                        entry -> {
+                            HostStatus status = entry.getValue();
+                            if (status == HostStatus.NO_REMARKS) {
+                                return HostInfo.createNoRemarks();
+                            } else {
+                                return HostInfo.createSuspended(status, now);
+                            }
+                        }
+                ));
+
+        return new HostInfos(hostInfosMap);
     }
 
     HostName createNode(String name, HostStatus hostStatus) {
@@ -88,10 +116,10 @@ class ModelTestUtils {
             ApplicationInstance applicationInstance,
             HostName... hostnames) {
         NodeGroup nodeGroup = new NodeGroup(applicationInstance, hostnames);
-        MutableStatusRegistry registry = statusService.lockApplicationInstance_forCurrentThreadOnly(
+        ApplicationLock lock = statusService.lockApplication(
                 OrchestratorContext.createContextForSingleAppOp(Clock.systemUTC()),
                 applicationInstance.reference());
-        return applicationApiFactory().create(nodeGroup, registry, clusterControllerClientFactory);
+        return applicationApiFactory().create(nodeGroup, lock, clusterControllerClientFactory);
     }
 
     ApplicationInstance createApplicationInstance(
diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicyTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicyTest.java
index d834034c9a8..4462e886d1b 100644
--- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicyTest.java
+++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaClusterPolicyTest.java
@@ -101,8 +101,7 @@ public class HostedVespaClusterPolicyTest {
         when(clusterApi.serviceType()).thenReturn(new ServiceType("service-type"));
         when(clusterApi.percentageOfServicesDown()).thenReturn(5);
         when(clusterApi.percentageOfServicesDownIfGroupIsAllowedToBeDown()).thenReturn(percentageOfServicesDownIfGroupIsAllowedToBeDown);
-        when(clusterApi.servicesDownAndNotInGroupDescription()).thenReturn("services-down-and-not-in-group");
-        when(clusterApi.nodesAllowedToBeDownNotInGroupDescription()).thenReturn("allowed-to-be-down");
+        when(clusterApi.downDescription()).thenReturn(" Down description");
 
         NodeGroup nodeGroup = mock(NodeGroup.class);
         when(clusterApi.getNodeGroup()).thenReturn(nodeGroup);
@@ -116,11 +115,9 @@ public class HostedVespaClusterPolicyTest {
             }
         } catch (HostStateChangeDeniedException e) {
             if (!expectSuccess) {
-                assertEquals("Changing the state of node-group would violate enough-services-up: "
-                        + "Suspension percentage for service type service-type would increase from "
-                        + "5% to 13%, over the limit of 10%. These instances may be down: "
-                        + "services-down-and-not-in-group and these hosts are allowed to be down: "
-                        + "allowed-to-be-down", e.getMessage());
+                assertEquals("Changing the state of node-group would violate enough-services-up: " +
+                        "Suspension for service type service-type would increase from 5% to 13%, " +
+                        "over the limit of 10%. Down description", e.getMessage());
                 assertEquals("enough-services-up", e.getConstraintName());
             }
         }
diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicyTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicyTest.java
index b27e37ac034..ed6917a3a4e 100644
--- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicyTest.java
+++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicyTest.java
@@ -120,7 +120,7 @@ public class HostedVespaPolicyTest {
         when(applicationApi.getStorageNodesInGroupInClusterOrder()).thenReturn(upStorageNodes);
 
         List<HostName> noRemarksHostNames = Arrays.asList(hostName1, hostName2, hostName3);
-        when(applicationApi.getNodesInGroupWithStatus(HostStatus.NO_REMARKS)).thenReturn(noRemarksHostNames);
+        when(applicationApi.getNodesInGroupWith(any())).thenReturn(noRemarksHostNames);
 
         InOrder order = inOrder(applicationApi, clusterPolicy, storageNode1, storageNode3);
 
@@ -136,7 +136,7 @@ public class HostedVespaPolicyTest {
         order.verify(storageNode1).setNodeState(context, ClusterControllerNodeState.DOWN);
         order.verify(storageNode3).setNodeState(context, ClusterControllerNodeState.DOWN);
 
-        order.verify(applicationApi).getNodesInGroupWithStatus(HostStatus.NO_REMARKS);
+        order.verify(applicationApi).getNodesInGroupWith(any());
         order.verify(applicationApi).setHostState(context, hostName1, HostStatus.ALLOWED_TO_BE_DOWN);
         order.verify(applicationApi).setHostState(context, hostName2, HostStatus.ALLOWED_TO_BE_DOWN);
         order.verify(applicationApi).setHostState(context, hostName3, HostStatus.ALLOWED_TO_BE_DOWN);
diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/ApplicationSuspensionResourceTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/ApplicationSuspensionResourceTest.java
index 89f421e9125..1fff8f976bb 100644
--- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/ApplicationSuspensionResourceTest.java
+++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/ApplicationSuspensionResourceTest.java
@@ -155,12 +155,14 @@ public class ApplicationSuspensionResourceTest {
 
         return "<services>\n" +
                 "    <container version=\"1.0\" jetty=\"true\">\n" +
+                "        <accesslog type=\"disabled\"/>\n" +
                 "        <config name=\"container.handler.threadpool\">\n" +
                 "            <maxthreads>10</maxthreads>\n" +
                 "        </config>\n" +
+                "        <component id=\"com.yahoo.vespa.flags.InMemoryFlagSource\" bundle=\"flags\" />\n" +
                 "        <component id=\"com.yahoo.vespa.curator.mock.MockCurator\" bundle=\"zkfacade\" />\n" +
-                "        <component id=\"com.yahoo.vespa.orchestrator.status.ZookeeperStatusService\" bundle=\"orchestrator\" />\n" +
-                "        <component id=\"com.yahoo.vespa.orchestrator.DummyInstanceLookupService\" bundle=\"orchestrator\" />\n" +
+                "        <component id=\"com.yahoo.vespa.orchestrator.status.ZkStatusService\" bundle=\"orchestrator\" />\n" +
+                "        <component id=\"com.yahoo.vespa.orchestrator.DummyServiceMonitor\" bundle=\"orchestrator\" />\n" +
                 "        <component id=\"com.yahoo.vespa.orchestrator.OrchestratorImpl\" bundle=\"orchestrator\" />\n" +
                 "        <component id=\"com.yahoo.vespa.orchestrator.controller.ClusterControllerClientFactoryMock\" bundle=\"orchestrator\" />\n" +
                 "\n" +
diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java
index fec1554396d..0605fcb8a0a 100644
--- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java
+++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/resources/HostResourceTest.java
@@ -16,11 +16,12 @@ import com.yahoo.vespa.applicationmodel.ServiceStatus;
 import com.yahoo.vespa.applicationmodel.ServiceType;
 import com.yahoo.vespa.applicationmodel.TenantId;
 import com.yahoo.vespa.curator.mock.MockCurator;
+import com.yahoo.vespa.flags.InMemoryFlagSource;
 import com.yahoo.vespa.orchestrator.BatchHostNameNotFoundException;
 import com.yahoo.vespa.orchestrator.BatchInternalErrorException;
+import com.yahoo.vespa.orchestrator.DummyAntiServiceMonitor;
 import com.yahoo.vespa.orchestrator.Host;
 import com.yahoo.vespa.orchestrator.HostNameNotFoundException;
-import com.yahoo.vespa.orchestrator.InstanceLookupService;
 import com.yahoo.vespa.orchestrator.OrchestrationException;
 import com.yahoo.vespa.orchestrator.Orchestrator;
 import com.yahoo.vespa.orchestrator.OrchestratorContext;
@@ -36,10 +37,13 @@ import com.yahoo.vespa.orchestrator.restapi.wire.GetHostResponse;
 import com.yahoo.vespa.orchestrator.restapi.wire.PatchHostRequest;
 import com.yahoo.vespa.orchestrator.restapi.wire.PatchHostResponse;
 import com.yahoo.vespa.orchestrator.restapi.wire.UpdateHostResponse;
+import com.yahoo.vespa.orchestrator.status.ApplicationLock;
+import com.yahoo.vespa.orchestrator.status.HostInfo;
 import com.yahoo.vespa.orchestrator.status.HostStatus;
-import com.yahoo.vespa.orchestrator.status.MutableStatusRegistry;
 import com.yahoo.vespa.orchestrator.status.StatusService;
-import com.yahoo.vespa.orchestrator.status.ZookeeperStatusService;
+import com.yahoo.vespa.orchestrator.status.ZkStatusService;
+import com.yahoo.vespa.service.monitor.ServiceModel;
+import com.yahoo.vespa.service.monitor.ServiceMonitor;
 import org.junit.Before;
 import org.junit.Test;
 
@@ -54,13 +58,13 @@ import java.time.Instant;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
+import java.util.Map;
 import java.util.Optional;
-import java.util.Set;
 
 import static com.yahoo.vespa.orchestrator.TestUtil.makeServiceClusterSet;
-import static org.fest.assertions.Assertions.assertThat;
-import static org.fest.assertions.Fail.fail;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.Mockito.doThrow;
 import static org.mockito.Mockito.mock;
@@ -76,13 +80,13 @@ public class HostResourceTest {
     private static final int SERVICE_MONITOR_CONVERGENCE_LATENCY_SECONDS = 0;
     private static final TenantId TENANT_ID = new TenantId("tenantId");
     private static final ApplicationInstanceId APPLICATION_INSTANCE_ID = new ApplicationInstanceId("applicationId");
-    private static final StatusService EVERY_HOST_IS_UP_HOST_STATUS_SERVICE = new ZookeeperStatusService(
-            new MockCurator(), mock(Metric.class), new TestTimer());
+    private static final ServiceMonitor serviceMonitor = mock(ServiceMonitor.class);
+    private static final StatusService EVERY_HOST_IS_UP_HOST_STATUS_SERVICE = new ZkStatusService(
+            new MockCurator(), mock(Metric.class), new TestTimer(), new DummyAntiServiceMonitor());
     private static final ApplicationApiFactory applicationApiFactory = new ApplicationApiFactory(3);
 
-    private static final InstanceLookupService mockInstanceLookupService = mock(InstanceLookupService.class);
     static {
-        when(mockInstanceLookupService.findInstanceByHost(any()))
+        when(serviceMonitor.getApplication(any(HostName.class)))
                 .thenReturn(Optional.of(
                         new ApplicationInstance(
                                 TENANT_ID,
@@ -90,21 +94,14 @@ public class HostResourceTest {
                                 makeServiceClusterSet())));
     }
 
-    private static final InstanceLookupService alwaysEmptyInstanceLookUpService = new InstanceLookupService() {
-        @Override
-        public Optional<ApplicationInstance> findInstanceById(
-                final ApplicationInstanceReference applicationInstanceReference) {
-            return Optional.empty();
-        }
+    private final InMemoryFlagSource flagSource = new InMemoryFlagSource();
 
-        @Override
-        public Optional<ApplicationInstance> findInstanceByHost(final HostName hostName) {
-            return Optional.empty();
-        }
+    private static final ServiceMonitor alwaysEmptyServiceMonitor = new ServiceMonitor() {
+        private final ServiceModel emptyServiceModel = new ServiceModel(Map.of());
 
         @Override
-        public Set<ApplicationInstanceReference> knownInstances() {
-            return Collections.emptySet();
+        public ServiceModel getServiceModelSnapshot() {
+            return emptyServiceModel;
         }
     };
 
@@ -125,27 +122,29 @@ public class HostResourceTest {
         public void releaseSuspensionGrant(
                 OrchestratorContext context, ApplicationInstance applicationInstance,
                 HostName hostName,
-                MutableStatusRegistry hostStatusRegistry) {
+                ApplicationLock hostStatusRegistry) {
         }
     }
 
-    private static final OrchestratorImpl alwaysAllowOrchestrator = new OrchestratorImpl(
+    private final OrchestratorImpl alwaysAllowOrchestrator = new OrchestratorImpl(
             new AlwaysAllowPolicy(),
             new ClusterControllerClientFactoryMock(),
-            EVERY_HOST_IS_UP_HOST_STATUS_SERVICE, mockInstanceLookupService,
+            EVERY_HOST_IS_UP_HOST_STATUS_SERVICE,
+            serviceMonitor,
             SERVICE_MONITOR_CONVERGENCE_LATENCY_SECONDS,
             clock,
-            applicationApiFactory
-    );
+            applicationApiFactory,
+            flagSource);
 
-    private static final OrchestratorImpl hostNotFoundOrchestrator = new OrchestratorImpl(
+    private final OrchestratorImpl hostNotFoundOrchestrator = new OrchestratorImpl(
             new AlwaysAllowPolicy(),
             new ClusterControllerClientFactoryMock(),
-            EVERY_HOST_IS_UP_HOST_STATUS_SERVICE, alwaysEmptyInstanceLookUpService,
+            EVERY_HOST_IS_UP_HOST_STATUS_SERVICE,
+            alwaysEmptyServiceMonitor,
             SERVICE_MONITOR_CONVERGENCE_LATENCY_SECONDS,
             clock,
-            applicationApiFactory
-    );
+            applicationApiFactory,
+            flagSource);
 
     private final UriInfo uriInfo = mock(UriInfo.class);
 
@@ -163,7 +162,7 @@ public class HostResourceTest {
 
         UpdateHostResponse response = hostResource.suspend(hostName);
 
-        assertThat(response.hostname()).isEqualTo(hostName);
+        assertEquals(hostName, response.hostname());
     }
 
     @Test
@@ -171,14 +170,14 @@ public class HostResourceTest {
         HostSuspensionResource hostSuspensionResource = new HostSuspensionResource(alwaysAllowOrchestrator);
         BatchOperationResult response = hostSuspensionResource.suspendAll("parentHostname",
                                                                           Arrays.asList("hostname1", "hostname2"));
-        assertThat(response.success());
+        assertTrue(response.success());
     }
 
     @Test
     public void returns_200_empty_batch() {
         HostSuspensionResource hostSuspensionResource = new HostSuspensionResource(alwaysAllowOrchestrator);
         BatchOperationResult response = hostSuspensionResource.suspendAll("parentHostname", List.of());
-        assertThat(response.success());
+        assertTrue(response.success());
     }
 
     @Test
@@ -189,7 +188,7 @@ public class HostResourceTest {
             hostResource.suspend("hostname");
             fail();
         } catch (WebApplicationException w) {
-            assertThat(w.getResponse().getStatus()).isEqualTo(404);
+            assertEquals(404, w.getResponse().getStatus());
         }
     }
 
@@ -203,7 +202,7 @@ public class HostResourceTest {
             hostSuspensionResource.suspendAll("parentHostname", Arrays.asList("hostname1", "hostname2"));
             fail();
         } catch (WebApplicationException w) {
-            assertThat(w.getResponse().getStatus()).isEqualTo(400);
+            assertEquals(400, w.getResponse().getStatus());
         }
     }
 
@@ -227,7 +226,7 @@ public class HostResourceTest {
         public void releaseSuspensionGrant(
                 OrchestratorContext context, ApplicationInstance applicationInstance,
                 HostName hostName,
-                MutableStatusRegistry hostStatusRegistry) throws HostStateChangeDeniedException {
+                ApplicationLock hostStatusRegistry) throws HostStateChangeDeniedException {
             doThrow();
         }
 
@@ -244,17 +243,19 @@ public class HostResourceTest {
         final OrchestratorImpl alwaysRejectResolver = new OrchestratorImpl(
                 new AlwaysFailPolicy(),
                 new ClusterControllerClientFactoryMock(),
-                EVERY_HOST_IS_UP_HOST_STATUS_SERVICE,mockInstanceLookupService,
+                EVERY_HOST_IS_UP_HOST_STATUS_SERVICE,
+                serviceMonitor,
                 SERVICE_MONITOR_CONVERGENCE_LATENCY_SECONDS,
                 clock,
-                applicationApiFactory);
+                applicationApiFactory,
+                flagSource);
 
         try {
             HostResource hostResource = new HostResource(alwaysRejectResolver, uriInfo);
             hostResource.suspend("hostname");
             fail();
         } catch (WebApplicationException w) {
-            assertThat(w.getResponse().getStatus()).isEqualTo(409);
+            assertEquals(409, w.getResponse().getStatus());
         }
     }
 
@@ -264,17 +265,18 @@ public class HostResourceTest {
                 new AlwaysFailPolicy(),
                 new ClusterControllerClientFactoryMock(),
                 EVERY_HOST_IS_UP_HOST_STATUS_SERVICE,
-                mockInstanceLookupService,
+                serviceMonitor,
                 SERVICE_MONITOR_CONVERGENCE_LATENCY_SECONDS,
                 clock,
-                applicationApiFactory);
+                applicationApiFactory,
+                flagSource);
 
         try {
             HostSuspensionResource hostSuspensionResource = new HostSuspensionResource(alwaysRejectResolver);
             hostSuspensionResource.suspendAll("parentHostname", Arrays.asList("hostname1", "hostname2"));
             fail();
         } catch (WebApplicationException w) {
-            assertThat(w.getResponse().getStatus()).isEqualTo(409);
+            assertEquals(409, w.getResponse().getStatus());
         }
     }
 
@@ -343,7 +345,7 @@ public class HostResourceTest {
 
         Host host = new Host(
                 hostName,
-                HostStatus.ALLOWED_TO_BE_DOWN,
+                HostInfo.createSuspended(HostStatus.ALLOWED_TO_BE_DOWN, Instant.EPOCH),
                 new ApplicationInstanceReference(
                         new TenantId("tenantId"),
                         new ApplicationInstanceId("applicationId")),
@@ -353,6 +355,7 @@ public class HostResourceTest {
         assertEquals("https://foo.com/bar", response.applicationUrl());
         assertEquals("hostname", response.hostname());
         assertEquals("ALLOWED_TO_BE_DOWN", response.state());
+        assertEquals("1970-01-01T00:00:00Z", response.suspendedSince());
         assertEquals(1, response.services().size());
         assertEquals("clusterId", response.services().get(0).clusterId);
         assertEquals("configId", response.services().get(0).configId);
@@ -361,7 +364,7 @@ public class HostResourceTest {
     }
 
     @Test
-    public void throws_504_on_timeout() throws HostNameNotFoundException, HostStateChangeDeniedException {
+    public void throws_409_on_timeout() throws HostNameNotFoundException, HostStateChangeDeniedException {
         Orchestrator orchestrator = mock(Orchestrator.class);
         doThrow(new UncheckedTimeoutException("Timeout Message")).when(orchestrator).resume(any(HostName.class));
 
@@ -370,13 +373,13 @@ public class HostResourceTest {
             hostResource.resume("hostname");
             fail();
         } catch (WebApplicationException w) {
-            assertThat(w.getResponse().getStatus()).isEqualTo(504);
+            assertEquals(409, w.getResponse().getStatus());
             assertEquals("resume failed: Timeout Message [deadline]", w.getMessage());
         }
     }
 
     @Test
-    public void throws_504_on_suspendAll_timeout() throws BatchHostStateChangeDeniedException, BatchHostNameNotFoundException, BatchInternalErrorException {
+    public void throws_409_on_suspendAll_timeout() throws BatchHostStateChangeDeniedException, BatchHostNameNotFoundException, BatchInternalErrorException {
         Orchestrator orchestrator = mock(Orchestrator.class);
         doThrow(new UncheckedTimeoutException("Timeout Message")).when(orchestrator).suspendAll(any(), any());
 
@@ -385,7 +388,7 @@ public class HostResourceTest {
             resource.suspendAll("parenthost", Arrays.asList("h1", "h2", "h3"));
             fail();
         } catch (WebApplicationException w) {
-            assertThat(w.getResponse().getStatus()).isEqualTo(504);
+            assertEquals(409, w.getResponse().getStatus());
         }
     }
 }
diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/status/ZkStatusService2Test.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/status/ZkStatusService2Test.java
new file mode 100644
index 00000000000..d3c366f5174
--- /dev/null
+++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/status/ZkStatusService2Test.java
@@ -0,0 +1,148 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.orchestrator.status;
+
+import com.yahoo.jdisc.Metric;
+import com.yahoo.jdisc.Timer;
+import com.yahoo.jdisc.test.TestTimer;
+import com.yahoo.vespa.applicationmodel.ApplicationInstanceId;
+import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference;
+import com.yahoo.vespa.applicationmodel.TenantId;
+import com.yahoo.vespa.curator.Curator;
+import com.yahoo.vespa.orchestrator.OrchestratorContext;
+import com.yahoo.vespa.service.monitor.AntiServiceMonitor;
+import com.yahoo.vespa.service.monitor.CriticalRegion;
+import org.apache.curator.framework.recipes.locks.InterProcessMutex;
+import org.junit.Test;
+import org.mockito.ArgumentCaptor;
+
+import java.time.Duration;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
+import static org.mockito.Mockito.when;
+
+/**
+ * @author hakonhall
+ */
+public class ZkStatusService2Test {
+    private final Curator curator = mock(Curator.class);
+    private final Timer timer = new TestTimer();
+    private final Metric metric = mock(Metric.class);
+    private final HostInfosCache cache = mock(HostInfosCache.class);
+    private final CriticalRegion criticalRegion = mock(CriticalRegion.class);
+    private final AntiServiceMonitor antiServiceMonitor = mock(AntiServiceMonitor.class);
+    private final ZkStatusService zkStatusService =
+            new ZkStatusService(curator, metric, timer, cache, antiServiceMonitor);
+
+    private final OrchestratorContext context = mock(OrchestratorContext.class);
+    private final InterProcessMutex mutex = mock(InterProcessMutex.class);
+    private final ApplicationInstanceReference reference = new ApplicationInstanceReference(
+            new TenantId("tenant"), new ApplicationInstanceId("app:dev:us-east-1:default"));
+
+    @Test
+    public void verifyLocks() throws Exception {
+        when(context.isProbe()).thenReturn(true);
+        when(context.hasLock(any())).thenReturn(false);
+        when(context.registerLockAcquisition(any(), any())).thenReturn(false);
+
+        when(curator.createMutex(any())).thenReturn(mutex);
+        when(mutex.acquire(anyLong(), any())).thenReturn(true);
+        when(antiServiceMonitor.disallowDuperModelLockAcquisition(any())).thenReturn(criticalRegion);
+
+        when(context.getTimeLeft()).thenReturn(Duration.ofSeconds(12));
+
+        verify(antiServiceMonitor, times(0)).disallowDuperModelLockAcquisition(any());
+        try (ApplicationLock lock = zkStatusService.lockApplication(context, reference)) {
+            verify(antiServiceMonitor, times(1)).disallowDuperModelLockAcquisition(any());
+
+            verify(criticalRegion, times(0)).close();
+        }
+        verify(criticalRegion, times(1)).close();
+
+        verify(curator, times(1)).createMutex(any());
+        verify(mutex, times(1)).acquire(anyLong(), any());
+        verify(mutex, times(1)).release();
+        verify(context, times(1)).hasLock(any());
+        verify(context, times(1)).registerLockAcquisition(any(), any());
+        verifyNoMoreInteractions(mutex, antiServiceMonitor, criticalRegion);
+
+        // Now the non-probe suspension
+
+        when(context.isProbe()).thenReturn(false);
+
+        verify(antiServiceMonitor, times(1)).disallowDuperModelLockAcquisition(any());
+        try (ApplicationLock lock = zkStatusService.lockApplication(context, reference)) {
+            verify(antiServiceMonitor, times(2)).disallowDuperModelLockAcquisition(any());
+
+            verify(criticalRegion, times(1)).close();
+        }
+        verify(criticalRegion, times(2)).close();
+
+        verify(mutex, times(2)).acquire(anyLong(), any());
+        verify(mutex, times(2)).release();
+        ArgumentCaptor<Runnable> runnableCaptor = ArgumentCaptor.forClass(Runnable.class);
+        verify(context, times(2)).hasLock(any());
+        verify(context, times(2)).registerLockAcquisition(any(), any());
+        verifyNoMoreInteractions(mutex, antiServiceMonitor, criticalRegion);
+    }
+
+    @Test
+    public void verifyLargeLocks() throws Exception {
+        when(context.isProbe()).thenReturn(true);
+        when(context.hasLock(any())).thenReturn(false);
+        when(context.registerLockAcquisition(any(), any())).thenReturn(true);
+
+        when(curator.createMutex(any())).thenReturn(mutex);
+        when(mutex.acquire(anyLong(), any())).thenReturn(true);
+        when(antiServiceMonitor.disallowDuperModelLockAcquisition(any())).thenReturn(criticalRegion);
+
+        when(context.getTimeLeft()).thenReturn(Duration.ofSeconds(12));
+
+        try (ApplicationLock lock = zkStatusService.lockApplication(context, reference)) {
+            // nothing
+        }
+
+        verify(curator, times(1)).createMutex(any());
+        verify(mutex, times(1)).acquire(anyLong(), any());
+        verify(mutex, times(0)).release();
+        verify(context, times(1)).hasLock(any());
+        verify(context, times(1)).registerLockAcquisition(any(), any());
+        verify(antiServiceMonitor, times(1)).disallowDuperModelLockAcquisition(any());
+        verify(criticalRegion, times(0)).close();
+        verifyNoMoreInteractions(mutex, antiServiceMonitor, criticalRegion);
+
+        // Now the non-probe suspension
+
+        when(context.isProbe()).thenReturn(false);
+        when(context.hasLock(any())).thenReturn(true);
+        when(context.registerLockAcquisition(any(), any())).thenReturn(false);
+
+        try (ApplicationLock lock = zkStatusService.lockApplication(context, reference)) {
+            // nothing
+        }
+
+        // No (additional) acquire, and no releases.
+        verify(mutex, times(1)).acquire(anyLong(), any());
+        verify(mutex, times(0)).release();
+        verify(context, times(2)).hasLock(any());
+        verify(context, times(1)).registerLockAcquisition(any(), any());
+        verify(antiServiceMonitor, times(1)).disallowDuperModelLockAcquisition(any());
+        verify(criticalRegion, times(0)).close();
+        verifyNoMoreInteractions(mutex, antiServiceMonitor, criticalRegion);
+
+        // Verify the context runnable releases the mutex
+        ArgumentCaptor<Runnable> runnableCaptor = ArgumentCaptor.forClass(Runnable.class);
+        verify(context, times(1)).registerLockAcquisition(any(), runnableCaptor.capture());
+        assertEquals(1, runnableCaptor.getAllValues().size());
+        runnableCaptor.getAllValues().forEach(Runnable::run);
+        verify(mutex, times(1)).acquire(anyLong(), any());
+        verify(mutex, times(1)).release();
+        verify(criticalRegion, times(1)).close();
+        verifyNoMoreInteractions(mutex, antiServiceMonitor, criticalRegion);
+    }
+}
\ No newline at end of file
diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/status/ZookeeperStatusServiceTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/status/ZkStatusServiceTest.java
index 12622f22837..ddff782cdb1 100644
--- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/status/ZookeeperStatusServiceTest.java
+++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/status/ZkStatusServiceTest.java
@@ -1,19 +1,29 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.orchestrator.status;
 
+import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.exception.ExceptionUtils;
 import com.yahoo.jdisc.Metric;
 import com.yahoo.jdisc.Timer;
 import com.yahoo.jdisc.test.TestTimer;
 import com.yahoo.log.LogLevel;
+import com.yahoo.test.ManualClock;
 import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference;
+import com.yahoo.vespa.applicationmodel.HostName;
 import com.yahoo.vespa.curator.Curator;
+import com.yahoo.vespa.flags.Flags;
+import com.yahoo.vespa.flags.InMemoryFlagSource;
 import com.yahoo.vespa.orchestrator.OrchestratorContext;
+import com.yahoo.vespa.orchestrator.OrchestratorUtil;
 import com.yahoo.vespa.orchestrator.TestIds;
+import com.yahoo.vespa.service.monitor.AntiServiceMonitor;
+import com.yahoo.vespa.service.monitor.CriticalRegion;
+import com.yahoo.vespa.service.monitor.ServiceMonitor;
 import com.yahoo.yolean.Exceptions;
 import org.apache.curator.framework.CuratorFramework;
 import org.apache.curator.test.KillSession;
 import org.apache.curator.test.TestingServer;
+import org.apache.zookeeper.data.Stat;
 import org.hamcrest.Description;
 import org.hamcrest.Matcher;
 import org.hamcrest.TypeSafeMatcher;
@@ -40,6 +50,8 @@ import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 import java.util.logging.Logger;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 import static org.hamcrest.core.Is.is;
 import static org.hamcrest.core.IsCollectionContaining.hasItem;
@@ -49,17 +61,21 @@ import static org.junit.Assert.fail;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.eq;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
 @RunWith(MockitoJUnitRunner.class)
-public class ZookeeperStatusServiceTest {
+public class ZkStatusServiceTest {
     private TestingServer testingServer;
-    private ZookeeperStatusService zookeeperStatusService;
+    private ZkStatusService statusService;
     private Curator curator;
     private final Timer timer = mock(Timer.class);
     private final Metric metric = mock(Metric.class);
     private final OrchestratorContext context = mock(OrchestratorContext.class);
+    private final InMemoryFlagSource flagSource = new InMemoryFlagSource();
+    private final CriticalRegion criticalRegion = mock(CriticalRegion.class);
+    private final AntiServiceMonitor antiServiceMonitor = mock(AntiServiceMonitor.class);
 
     @Captor
     private ArgumentCaptor<Map<String, String>> captor;
@@ -70,10 +86,12 @@ public class ZookeeperStatusServiceTest {
 
         testingServer = new TestingServer();
         curator = createConnectedCurator(testingServer);
-        zookeeperStatusService = new ZookeeperStatusService(curator, metric, timer);
+        statusService = new ZkStatusService(curator, metric, timer, antiServiceMonitor);
         when(context.getTimeLeft()).thenReturn(Duration.ofSeconds(10));
         when(context.isProbe()).thenReturn(false);
         when(timer.currentTime()).thenReturn(Instant.ofEpochMilli(1));
+        when(timer.toUtcClock()).thenReturn(new ManualClock(Instant.ofEpochMilli(1)));
+        when(antiServiceMonitor.disallowDuperModelLockAcquisition(any())).thenReturn(criticalRegion);
     }
 
     private static Curator createConnectedCurator(TestingServer server) throws InterruptedException {
@@ -94,30 +112,27 @@ public class ZookeeperStatusServiceTest {
     }
 
     @Test
-    public void host_state_for_unknown_hosts_is_no_remarks() {
+    public void host_state_for_unknown_hosts_is_no_remarks() throws Exception {
         assertThat(
-                zookeeperStatusService.getHostInfo(TestIds.APPLICATION_INSTANCE_REFERENCE, TestIds.HOST_NAME1).status(),
+                statusService.getHostInfo(TestIds.APPLICATION_INSTANCE_REFERENCE, TestIds.HOST_NAME1).status(),
                 is(HostStatus.NO_REMARKS));
     }
 
     @Test
-    public void setting_host_state_is_idempotent() {
+    public void setting_host_state_is_idempotent() throws Exception {
         when(timer.currentTime()).thenReturn(
                 Instant.ofEpochMilli((1)),
                 Instant.ofEpochMilli((3)),
                 Instant.ofEpochMilli(6));
 
-        try (MutableStatusRegistry statusRegistry = zookeeperStatusService
-                .lockApplicationInstance_forCurrentThreadOnly(context, TestIds.APPLICATION_INSTANCE_REFERENCE)) {
+        try (ApplicationLock lock = statusService.lockApplication(context, TestIds.APPLICATION_INSTANCE_REFERENCE)) {
 
             //shuffling to catch "clean database" failures for all cases.
             for (HostStatus hostStatus: shuffledList(HostStatus.NO_REMARKS, HostStatus.ALLOWED_TO_BE_DOWN)) {
                 for (int i = 0; i < 2; i++) {
-                    statusRegistry.setHostState(
-                            TestIds.HOST_NAME1,
-                            hostStatus);
+                    lock.setHostState(TestIds.HOST_NAME1, hostStatus);
 
-                    assertThat(statusRegistry.getHostInfo(TestIds.HOST_NAME1).status(),
+                    assertThat(lock.getHostInfos().getOrNoRemarks(TestIds.HOST_NAME1).status(),
                                is(hostStatus));
                 }
             }
@@ -141,15 +156,16 @@ public class ZookeeperStatusServiceTest {
 
     @Test
     public void locks_are_exclusive() throws Exception {
-        ZookeeperStatusService zookeeperStatusService2 = new ZookeeperStatusService(curator, mock(Metric.class), new TestTimer());
+        ZkStatusService zkStatusService2 =
+                new ZkStatusService(curator, mock(Metric.class), new TestTimer(), antiServiceMonitor);
 
         final CompletableFuture<Void> lockedSuccessfullyFuture;
-        try (MutableStatusRegistry statusRegistry = zookeeperStatusService
-                .lockApplicationInstance_forCurrentThreadOnly(context, TestIds.APPLICATION_INSTANCE_REFERENCE)) {
+        try (ApplicationLock lock = statusService
+                .lockApplication(context, TestIds.APPLICATION_INSTANCE_REFERENCE)) {
 
             lockedSuccessfullyFuture = CompletableFuture.runAsync(() -> {
-                try (MutableStatusRegistry statusRegistry2 = zookeeperStatusService2
-                        .lockApplicationInstance_forCurrentThreadOnly(context, TestIds.APPLICATION_INSTANCE_REFERENCE))
+                try (ApplicationLock lock2 = zkStatusService2
+                        .lockApplication(context, TestIds.APPLICATION_INSTANCE_REFERENCE))
                 {
                 }
             });
@@ -166,15 +182,16 @@ public class ZookeeperStatusServiceTest {
 
     @Test
     public void failing_to_get_lock_closes_SessionFailRetryLoop() throws Exception {
-        ZookeeperStatusService zookeeperStatusService2 = new ZookeeperStatusService(curator, mock(Metric.class), new TestTimer());
+        ZkStatusService zkStatusService2 =
+                new ZkStatusService(curator, mock(Metric.class), new TestTimer(), antiServiceMonitor);
 
-        try (MutableStatusRegistry statusRegistry = zookeeperStatusService
-                .lockApplicationInstance_forCurrentThreadOnly(context, TestIds.APPLICATION_INSTANCE_REFERENCE)) {
+        try (ApplicationLock lock = statusService
+                .lockApplication(context, TestIds.APPLICATION_INSTANCE_REFERENCE)) {
 
             //must run in separate thread, since having 2 locks in the same thread fails
             CompletableFuture<Void> resultOfZkOperationAfterLockFailure = CompletableFuture.runAsync(() -> {
                 try {
-                    zookeeperStatusService2.lockApplicationInstance_forCurrentThreadOnly(context, TestIds.APPLICATION_INSTANCE_REFERENCE);
+                    zkStatusService2.lockApplication(context, TestIds.APPLICATION_INSTANCE_REFERENCE);
                     fail("Both zookeeper host status services locked simultaneously for the same application instance");
                 } catch (RuntimeException e) {
                 }
@@ -182,7 +199,7 @@ public class ZookeeperStatusServiceTest {
                 killSession(curator.framework(), testingServer);
 
                 //Throws SessionFailedException if the SessionFailRetryLoop has not been closed.
-                statusRegistry.getHostInfo(TestIds.HOST_NAME1);
+                lock.getHostInfos().getOrNoRemarks(TestIds.HOST_NAME1);
             });
 
             assertThat(resultOfZkOperationAfterLockFailure, notHoldsException());
@@ -235,59 +252,166 @@ public class ZookeeperStatusServiceTest {
     }
 
     @Test
-    public void suspend_and_resume_application_works_and_is_symmetric() {
+    public void suspend_and_resume_application_works_and_is_symmetric() throws Exception {
 
         // Initial state is NO_REMARK
         assertThat(
-                zookeeperStatusService
+                statusService
                         .getApplicationInstanceStatus(TestIds.APPLICATION_INSTANCE_REFERENCE),
                 is(ApplicationInstanceStatus.NO_REMARKS));
 
         // Suspend
-        try (MutableStatusRegistry statusRegistry = zookeeperStatusService
-                .lockApplicationInstance_forCurrentThreadOnly(context, TestIds.APPLICATION_INSTANCE_REFERENCE)) {
-            statusRegistry.setApplicationInstanceStatus(ApplicationInstanceStatus.ALLOWED_TO_BE_DOWN);
+        try (ApplicationLock lock = statusService
+                .lockApplication(context, TestIds.APPLICATION_INSTANCE_REFERENCE)) {
+            lock.setApplicationInstanceStatus(ApplicationInstanceStatus.ALLOWED_TO_BE_DOWN);
         }
 
         assertThat(
-                zookeeperStatusService
+                statusService
                         .getApplicationInstanceStatus(TestIds.APPLICATION_INSTANCE_REFERENCE),
                 is(ApplicationInstanceStatus.ALLOWED_TO_BE_DOWN));
 
         // Resume
-        try (MutableStatusRegistry statusRegistry = zookeeperStatusService
-                .lockApplicationInstance_forCurrentThreadOnly(context, TestIds.APPLICATION_INSTANCE_REFERENCE)) {
-            statusRegistry.setApplicationInstanceStatus(ApplicationInstanceStatus.NO_REMARKS);
+        try (ApplicationLock lock = statusService
+                .lockApplication(context, TestIds.APPLICATION_INSTANCE_REFERENCE)) {
+            lock.setApplicationInstanceStatus(ApplicationInstanceStatus.NO_REMARKS);
         }
 
         assertThat(
-                zookeeperStatusService
+                statusService
                         .getApplicationInstanceStatus(TestIds.APPLICATION_INSTANCE_REFERENCE),
                 is(ApplicationInstanceStatus.NO_REMARKS));
     }
 
     @Test
-    public void suspending_two_applications_returns_two_applications() {
-        Set<ApplicationInstanceReference> suspendedApps
-                = zookeeperStatusService.getAllSuspendedApplications();
+    public void suspending_two_applications_returns_two_applications() throws Exception {
+        Set<ApplicationInstanceReference> suspendedApps = statusService.getAllSuspendedApplications();
         assertThat(suspendedApps.size(), is(0));
 
-        try (MutableStatusRegistry statusRegistry = zookeeperStatusService
-                .lockApplicationInstance_forCurrentThreadOnly(context, TestIds.APPLICATION_INSTANCE_REFERENCE)) {
+        try (ApplicationLock statusRegistry = statusService
+                .lockApplication(context, TestIds.APPLICATION_INSTANCE_REFERENCE)) {
             statusRegistry.setApplicationInstanceStatus(ApplicationInstanceStatus.ALLOWED_TO_BE_DOWN);
         }
 
-        try (MutableStatusRegistry statusRegistry = zookeeperStatusService
-                .lockApplicationInstance_forCurrentThreadOnly(context, TestIds.APPLICATION_INSTANCE_REFERENCE2)) {
-            statusRegistry.setApplicationInstanceStatus(ApplicationInstanceStatus.ALLOWED_TO_BE_DOWN);
+        try (ApplicationLock lock = statusService
+                .lockApplication(context, TestIds.APPLICATION_INSTANCE_REFERENCE2)) {
+            lock.setApplicationInstanceStatus(ApplicationInstanceStatus.ALLOWED_TO_BE_DOWN);
         }
 
-        suspendedApps = zookeeperStatusService.getAllSuspendedApplications();
+        suspendedApps = statusService.getAllSuspendedApplications();
         assertThat(suspendedApps.size(), is(2));
         assertThat(suspendedApps, hasItem(TestIds.APPLICATION_INSTANCE_REFERENCE));
         assertThat(suspendedApps, hasItem(TestIds.APPLICATION_INSTANCE_REFERENCE2));
     }
 
+    @Test
+    public void zookeeper_cleanup() throws Exception {
+        HostName strayHostname = new HostName("stray1.com");
+
+        verify(antiServiceMonitor, times(0)).disallowDuperModelLockAcquisition(any());
+        try (ApplicationLock lock = statusService.lockApplication(context, TestIds.APPLICATION_INSTANCE_REFERENCE)) {
+            verify(antiServiceMonitor, times(1)).disallowDuperModelLockAcquisition(any());
+
+            lock.setApplicationInstanceStatus(ApplicationInstanceStatus.ALLOWED_TO_BE_DOWN);
+            lock.setHostState(TestIds.HOST_NAME1, HostStatus.ALLOWED_TO_BE_DOWN);
+
+            lock.setHostState(strayHostname, HostStatus.PERMANENTLY_DOWN);
+
+            verify(criticalRegion, times(0)).close();
+        }
+        verify(criticalRegion, times(1)).close();
+
+        verify(antiServiceMonitor, times(1)).disallowDuperModelLockAcquisition(any());
+        try (ApplicationLock lock = statusService.lockApplication(context, TestIds.APPLICATION_INSTANCE_REFERENCE2)) {
+            verify(antiServiceMonitor, times(2)).disallowDuperModelLockAcquisition(any());
+
+            lock.setApplicationInstanceStatus(ApplicationInstanceStatus.ALLOWED_TO_BE_DOWN);
+
+            verify(criticalRegion, times(1)).close();
+        }
+        verify(criticalRegion, times(2)).close();
+
+        ApplicationId applicationId = OrchestratorUtil.toApplicationId(TestIds.APPLICATION_INSTANCE_REFERENCE);
+        assertEquals("test-tenant:test-application:test-environment:test-region:test-instance-key",
+                TestIds.APPLICATION_INSTANCE_REFERENCE.asString());
+        assertEquals("test-tenant:test-application:test-instance-key", applicationId.serializedForm());
+        assertEquals("host1", TestIds.HOST_NAME1.s());
+
+        String hostStatusPath = "/vespa/host-status/" + applicationId.serializedForm() + "/hosts/" + TestIds.HOST_NAME1.s();
+        String lock2Path = "/vespa/host-status-service/" + TestIds.APPLICATION_INSTANCE_REFERENCE.asString() + "/lock2";
+        String applicationStatusPath = "/vespa/application-status-service/" + TestIds.APPLICATION_INSTANCE_REFERENCE.asString();
+        assertZkPathExists(true, hostStatusPath);
+        assertZkPathExists(true, lock2Path);
+        assertZkPathExists(true, applicationStatusPath);
+
+        String strayHostStatusPath = "/vespa/host-status/" + applicationId.serializedForm() + "/hosts/" + strayHostname.s();
+        String strayHostStatusServicePath = "/vespa/host-status-service/" + TestIds.APPLICATION_INSTANCE_REFERENCE.asString() +
+                "/hosts-allowed-down/stray2.com";
+        String strayApplicationStatusPath = "/vespa/application-status-service/" + TestIds.APPLICATION_INSTANCE_REFERENCE2.asString();
+
+        createZkNodes(strayHostStatusServicePath);
+        assertZkPathExists(true, strayHostStatusPath);
+        assertZkPathExists(true, strayHostStatusServicePath);
+        assertZkPathExists(true, strayApplicationStatusPath);
+
+        statusService.onApplicationActivate(TestIds.APPLICATION_INSTANCE_REFERENCE2, makeHostnameSet("host1", "host2"));
+
+        // Nothing has been deleted
+        assertZkPathExists(true, hostStatusPath);
+        assertZkPathExists(true, lock2Path);
+        assertZkPathExists(true, applicationStatusPath);
+        assertZkPathExists(true, strayHostStatusPath);
+        assertZkPathExists(true, strayHostStatusServicePath);
+        assertZkPathExists(true, strayApplicationStatusPath);
+
+        statusService.onApplicationActivate(TestIds.APPLICATION_INSTANCE_REFERENCE,
+                makeHostnameSet(TestIds.HOST_NAME1.s(), "host3"));
+
+        // Stray hosts for app1 has been deleted
+        assertZkPathExists(true, hostStatusPath);
+        assertZkPathExists(true, lock2Path);
+        assertZkPathExists(true, applicationStatusPath);
+        assertZkPathExists(false, strayHostStatusPath);
+        assertZkPathExists(true, strayHostStatusServicePath);
+        assertZkPathExists(true, strayApplicationStatusPath);
+
+        statusService.onApplicationRemove(TestIds.APPLICATION_INSTANCE_REFERENCE);
+
+        // Application removed => only lock2 path and other apps are left
+        assertZkPathExists(false, hostStatusPath);
+        assertZkPathExists(true, lock2Path);
+        assertZkPathExists(false, applicationStatusPath);
+        assertZkPathExists(false, strayHostStatusPath);
+        assertZkPathExists(false, strayHostStatusServicePath);
+        assertZkPathExists(true, strayApplicationStatusPath);
+
+    }
+
+    private Set<HostName> makeHostnameSet(String... hostnames) {
+        return Stream.of(hostnames).map(HostName::new).collect(Collectors.toSet());
+    }
+
+    private void assertZkPathExists(boolean exists, String path) {
+        final Stat stat;
+        try {
+            stat = curator.framework().checkExists().forPath(path);
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+
+        assertEquals(exists, stat != null);
+    }
+
+    private void createZkNodes(String... paths) {
+        Stream.of(paths).forEach(path -> {
+            try {
+                curator.framework().create().creatingParentsIfNeeded().forPath(path);
+            } catch (Exception e) {
+                throw new RuntimeException(e);
+            }
+        });
+    }
+
     //TODO: move to vespajlib
     @SafeVarargs
     private static <T> List<T> shuffledList(T... values) {
diff --git a/parent/pom.xml b/parent/pom.xml
index b1ca2539ef5..454b4677cfa 100644
--- a/parent/pom.xml
+++ b/parent/pom.xml
@@ -222,6 +222,7 @@
                         <systemPropertyVariables>
                             <java.io.tmpdir>${project.build.directory}</java.io.tmpdir>
                         </systemPropertyVariables>
+                        <trimStackTrace>false</trimStackTrace>
                     </configuration>
                 </plugin>
                 <plugin>
@@ -414,6 +415,11 @@
                 <version>${aws.sdk.version}</version>
             </dependency>
             <dependency>
+                <groupId>com.auth0</groupId>
+                <artifactId>java-jwt</artifactId>
+                <version>3.10.0</version>
+            </dependency>
+            <dependency>
                 <groupId>com.github.cverges.expect4j</groupId>
                 <artifactId>expect4j</artifactId>
                 <version>1.6</version>
@@ -473,6 +479,11 @@
                 <version>${athenz.version}</version>
             </dependency>
             <dependency>
+                <groupId>org.apache.commons</groupId>
+                <artifactId>commons-math3</artifactId>
+                <version>${commons.math3.version}</version>
+            </dependency>
+            <dependency>
                 <groupId>commons-cli</groupId>
                 <artifactId>commons-cli</artifactId>
                 <version>1.4</version>
@@ -735,11 +746,11 @@
     <properties>
         <antlr.version>3.5.2</antlr.version>
         <antlr4.version>4.5</antlr4.version>
-        <apache.httpclient.version>4.5.10</apache.httpclient.version>
-        <apache.httpcore.version>4.4.12</apache.httpcore.version>
+        <apache.httpclient.version>4.5.11</apache.httpclient.version>
+        <apache.httpcore.version>4.4.13</apache.httpcore.version>
         <asm.version>7.0</asm.version>
         <!-- Athenz dependencies. Make sure these dependencies match those in Vespa's internal repositories -->
-        <athenz.version>1.8.44</athenz.version>
+        <athenz.version>1.8.49</athenz.version>
         <aws.sdk.version>1.11.542</aws.sdk.version>
         <!-- WARNING: If you change curator version, you also need to update
                  zkfacade/src/main/java/org/apache/curator/**/package-info.java
@@ -749,6 +760,7 @@
         -->
         <curator.version>2.9.1</curator.version>
         <jna.version>4.5.2</jna.version>
+        <commons.math3.version>3.6.1</commons.math3.version>
         <junit.version>5.4.2</junit.version>
         <maven-assembly-plugin.version>3.1.1</maven-assembly-plugin.version>
         <maven-bundle-plugin.version>3.5.0</maven-bundle-plugin.version>
diff --git a/persistence/src/vespa/persistence/CMakeLists.txt b/persistence/src/vespa/persistence/CMakeLists.txt
index ec569a23029..77f3f865b6f 100644
--- a/persistence/src/vespa/persistence/CMakeLists.txt
+++ b/persistence/src/vespa/persistence/CMakeLists.txt
@@ -9,7 +9,6 @@ vespa_add_library(persistence
 vespa_add_library(persistence_persistence_conformancetest
     SOURCES
     $<TARGET_OBJECTS:persistence_conformancetest_lib>
-    INSTALL lib64
     DEPENDS
     persistence
     gtest
diff --git a/persistence/src/vespa/persistence/spi/abstractpersistenceprovider.cpp b/persistence/src/vespa/persistence/spi/abstractpersistenceprovider.cpp
index 7f6bd835cd5..e35a6a74bde 100644
--- a/persistence/src/vespa/persistence/spi/abstractpersistenceprovider.cpp
+++ b/persistence/src/vespa/persistence/spi/abstractpersistenceprovider.cpp
@@ -6,9 +6,7 @@
 #include <vespa/document/fieldset/fieldsets.h>
 #include <vespa/document/fieldvalue/document.h>
 
-namespace storage {
-
-namespace spi {
+namespace storage::spi {
 
 UpdateResult
 AbstractPersistenceProvider::update(const Bucket& bucket, Timestamp ts,
@@ -66,5 +64,3 @@ AbstractPersistenceProvider::move(const Bucket& source, PartitionId target, Cont
 }
 
 }
-
-}
diff --git a/persistence/src/vespa/persistence/spi/abstractpersistenceprovider.h b/persistence/src/vespa/persistence/spi/abstractpersistenceprovider.h
index 27197c2adb8..557a9ec2edd 100644
--- a/persistence/src/vespa/persistence/spi/abstractpersistenceprovider.h
+++ b/persistence/src/vespa/persistence/spi/abstractpersistenceprovider.h
@@ -1,7 +1,7 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 #pragma once
 
-#include <vespa/persistence/spi/persistenceprovider.h>
+#include "persistenceprovider.h"
 
 namespace storage::spi {
 
diff --git a/persistence/src/vespa/persistence/spi/bucket.cpp b/persistence/src/vespa/persistence/spi/bucket.cpp
index 9265f995a45..ef94519cdb0 100644
--- a/persistence/src/vespa/persistence/spi/bucket.cpp
+++ b/persistence/src/vespa/persistence/spi/bucket.cpp
@@ -4,8 +4,7 @@
 #include <ostream>
 #include <vespa/vespalib/stllike/asciistream.h>
 
-namespace storage {
-namespace spi {
+namespace storage::spi {
 
 vespalib::string
 Bucket::toString() const {
@@ -30,5 +29,4 @@ operator<<(std::ostream& os, const Bucket& bucket) {
     return os << bucket.toString();
 }
 
-} // spi
-} // storage
+}
diff --git a/persistence/src/vespa/persistence/spi/bucket.h b/persistence/src/vespa/persistence/spi/bucket.h
index 54760304694..874074d7e24 100644
--- a/persistence/src/vespa/persistence/spi/bucket.h
+++ b/persistence/src/vespa/persistence/spi/bucket.h
@@ -17,8 +17,7 @@
 #include <persistence/spi/types.h>
 #include <vespa/document/bucket/bucket.h>
 
-namespace storage {
-namespace spi {
+namespace storage::spi {
 
 class Bucket {
     document::Bucket _bucket;
@@ -47,6 +46,4 @@ public:
 vespalib::asciistream& operator<<(vespalib::asciistream& out, const Bucket& bucket);
 std::ostream& operator<<(std::ostream& out, const Bucket& bucket);
 
-} // spi
-} // storage
-
+}
diff --git a/persistence/src/vespa/persistence/spi/bucketinfo.cpp b/persistence/src/vespa/persistence/spi/bucketinfo.cpp
index e60d6152058..5bef59a65f1 100644
--- a/persistence/src/vespa/persistence/spi/bucketinfo.cpp
+++ b/persistence/src/vespa/persistence/spi/bucketinfo.cpp
@@ -3,8 +3,7 @@
 #include "bucketinfo.h"
 #include <vespa/vespalib/stllike/asciistream.h>
 
-namespace storage {
-namespace spi {
+namespace storage::spi {
 
 BucketInfo::BucketInfo()
     : _checksum(0),
@@ -73,5 +72,4 @@ std::ostream& operator<<(std::ostream& out, const BucketInfo& info) {
     return out << info.toString();
 }
 
-} // spi
-} // storage
+}
diff --git a/persistence/src/vespa/persistence/spi/bucketinfo.h b/persistence/src/vespa/persistence/spi/bucketinfo.h
index fd4605229f8..827aad48d7f 100644
--- a/persistence/src/vespa/persistence/spi/bucketinfo.h
+++ b/persistence/src/vespa/persistence/spi/bucketinfo.h
@@ -8,9 +8,7 @@
 
 #include <persistence/spi/types.h>
 
-namespace vespalib {
-    class asciistream;
-}
+namespace vespalib { class asciistream; }
 
 namespace storage::spi {
 
diff --git a/persistence/src/vespa/persistence/spi/clusterstate.cpp b/persistence/src/vespa/persistence/spi/clusterstate.cpp
index bc30197978f..567d7ebc1ce 100644
--- a/persistence/src/vespa/persistence/spi/clusterstate.cpp
+++ b/persistence/src/vespa/persistence/spi/clusterstate.cpp
@@ -12,9 +12,9 @@ namespace storage::spi {
 ClusterState::ClusterState(const lib::ClusterState& state,
                            uint16_t nodeIndex,
                            const lib::Distribution& distribution)
-    : _state(new lib::ClusterState(state)),
+    : _state(std::make_unique<lib::ClusterState>(state)),
       _nodeIndex(nodeIndex),
-      _distribution(new lib::Distribution(distribution.serialize()))
+      _distribution(std::make_unique<lib::Distribution>(distribution.serialize()))
 {
 }
 
@@ -26,8 +26,8 @@ void ClusterState::deserialize(vespalib::nbostream& i) {
     i >> _nodeIndex;
     i >> distribution;
 
-    _state.reset(new lib::ClusterState(clusterState));
-    _distribution.reset(new lib::Distribution(distribution));
+    _state = std::make_unique<lib::ClusterState>(clusterState);
+    _distribution = std::make_unique<lib::Distribution>(distribution);
 }
 
 ClusterState::ClusterState(vespalib::nbostream& i) {
diff --git a/persistence/src/vespa/persistence/spi/clusterstateimpl.h b/persistence/src/vespa/persistence/spi/clusterstateimpl.h
deleted file mode 100644
index 281c37eb9d5..00000000000
--- a/persistence/src/vespa/persistence/spi/clusterstateimpl.h
+++ /dev/null
@@ -1,66 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-#pragma once
-
-#include <vespa/persistence/spi/bucket.h>
-#include <vespa/persistence/spi/clusterstate.h>
-
-namespace storage {
-
-namespace spi {
-
-/**
- * Used to determine the state of the current node and its buckets.
- */
-class ClusterStateImpl : public ClusterState{
-public:
-    ClusterStateImpl();
-
-    ClusterStateImpl(const lib::ClusterState& state,
-                     uint16_t nodeIndex,
-                     const lib::Distribution& distribution);
-
-    ClusterStateImpl(vespalib::nbostream& i);
-
-    ClusterStateImpl(const ClusterStateImpl& other);
-
-    ClusterStateImpl& operator=(const ClusterStateImpl& other);
-
-    /**
-     * Returns true if the given bucket is in the ideal state
-     * for readiness.
-     *
-     * @param b The bucket to check.
-     */
-    bool shouldBeReady(const Bucket& b) const;
-
-    /**
-     * Returns false if the cluster has been deemed down. This can happen
-     * if the fleet controller has detected that too many nodes are down
-     * compared to the complete list of nodes, and deigns the system to be
-     * unusable.
-     */
-    bool clusterUp() const;
-
-    /**
-     * Returns false if this node has been set in a state where it should not
-     * receive external load.
-     */
-    bool nodeUp() const;
-
-    /**
-     * Returns a serialized form of this object.
-     */
-    void serialize(vespalib::nbostream& o) const;
-
-private:
-    std::unique_ptr<lib::ClusterState> _state;
-    uint16_t _nodeIndex;
-    std::unique_ptr<lib::Distribution> _distribution;
-
-    void deserialize(vespalib::nbostream&);
-};
-
-}
-
-}
-
diff --git a/persistence/src/vespa/persistence/spi/context.cpp b/persistence/src/vespa/persistence/spi/context.cpp
index 5ce34d5d139..429e2fb9d4e 100644
--- a/persistence/src/vespa/persistence/spi/context.cpp
+++ b/persistence/src/vespa/persistence/spi/context.cpp
@@ -2,8 +2,7 @@
 
 #include "context.h"
 
-namespace storage {
-namespace spi {
+namespace storage::spi {
 
 Context::Context(const LoadType& loadType, Priority pri, int maxTraceLevel)
     : _loadType(&loadType),
@@ -12,7 +11,6 @@ Context::Context(const LoadType& loadType, Priority pri, int maxTraceLevel)
       _readConsistency(ReadConsistency::STRONG)
 { }
 
-Context::~Context() { }
+Context::~Context() = default;
 
-} // spi
-} // storage
+}
diff --git a/persistence/src/vespa/persistence/spi/context.h b/persistence/src/vespa/persistence/spi/context.h
index ca4c79e3005..8c31439ee75 100644
--- a/persistence/src/vespa/persistence/spi/context.h
+++ b/persistence/src/vespa/persistence/spi/context.h
@@ -29,13 +29,10 @@
 
 #pragma once
 
-#include <persistence/spi/types.h>
-#include <vespa/persistence/spi/read_consistency.h>
+#include "read_consistency.h"
 #include <vespa/vespalib/trace/trace.h>
 
-namespace metrics {
-    class LoadType;
-}
+namespace metrics { class LoadType; }
 
 namespace storage::spi {
 
@@ -62,10 +59,6 @@ public:
 
     const LoadType& getLoadType() const { return *_loadType; }
     Priority getPriority() const { return _priority; }
-    int getMaxTraceLevel() const { return _trace.getLevel(); }
-    void addTrace(const vespalib::TraceNode& traceNode) {
-        _trace.getRoot().addChild(traceNode);
-    }
 
     /**
      * A read operation might choose to relax its consistency requirements,
diff --git a/persistence/src/vespa/persistence/spi/exceptions.cpp b/persistence/src/vespa/persistence/spi/exceptions.cpp
index a1b9d57270c..d17c0f90ca0 100644
--- a/persistence/src/vespa/persistence/spi/exceptions.cpp
+++ b/persistence/src/vespa/persistence/spi/exceptions.cpp
@@ -2,11 +2,8 @@
 
 #include "exceptions.h"
 
-namespace storage {
-namespace spi {
+namespace storage::spi {
 
 VESPA_IMPLEMENT_EXCEPTION(HandledException, vespalib::Exception);
 
-} // spi
-} // storage
-
+}
diff --git a/persistence/src/vespa/persistence/spi/exceptions.h b/persistence/src/vespa/persistence/spi/exceptions.h
index 1c434fd8f00..e972e304567 100644
--- a/persistence/src/vespa/persistence/spi/exceptions.h
+++ b/persistence/src/vespa/persistence/spi/exceptions.h
@@ -3,8 +3,7 @@
 
 #include <vespa/vespalib/util/exceptions.h>
 
-namespace storage {
-namespace spi {
+namespace storage::spi {
 
 /**
  * Exception used where the cause has already been reported to the user, so
@@ -16,6 +15,4 @@ namespace spi {
  */
 VESPA_DEFINE_EXCEPTION(HandledException, vespalib::Exception);
 
-} // spi
-} // storage
-
+}
diff --git a/persistence/src/vespa/persistence/spi/matcher.h b/persistence/src/vespa/persistence/spi/matcher.h
index 02dc6db2261..bf989530f35 100644
--- a/persistence/src/vespa/persistence/spi/matcher.h
+++ b/persistence/src/vespa/persistence/spi/matcher.h
@@ -8,12 +8,11 @@
 
 #pragma once
 
-#include <vespa/persistence/spi/docentry.h>
+#include "docentry.h"
 #include <persistence/spi/documentsubset.h>
 #include <persistence/spi/types.h>
 
-namespace storage {
-namespace spi {
+namespace storage::spi {
 
 class Matcher {
     DocumentSubset _subset;
@@ -37,6 +36,4 @@ struct AllMatcher : public Matcher {
     bool match(const DocEntry&) const { return true; }
 };
 
-} // spi
-} // storage
-
+}
diff --git a/persistence/src/vespa/persistence/spi/partitionstate.cpp b/persistence/src/vespa/persistence/spi/partitionstate.cpp
index 123a82829ef..7cc42742019 100644
--- a/persistence/src/vespa/persistence/spi/partitionstate.cpp
+++ b/persistence/src/vespa/persistence/spi/partitionstate.cpp
@@ -4,8 +4,7 @@
 #include <vespa/vespalib/util/exceptions.h>
 #include <vespa/vespalib/stllike/asciistream.h>
 
-namespace storage {
-namespace spi {
+namespace storage::spi {
 
 PartitionState::PartitionState()
     : _state(UP),
@@ -21,7 +20,7 @@ PartitionStateList::PartitionStateList(PartitionId::Type partitionCount)
     : _states(partitionCount)
 { }
 
-PartitionStateList::~PartitionStateList() { }
+PartitionStateList::~PartitionStateList() = default;
 
 PartitionState&
 PartitionStateList::operator[](PartitionId::Type index)
@@ -34,5 +33,4 @@ PartitionStateList::operator[](PartitionId::Type index)
     return _states[index];
 }
 
-} // spi
-} // storage
+}
diff --git a/persistence/src/vespa/persistence/spi/partitionstate.h b/persistence/src/vespa/persistence/spi/partitionstate.h
index 6296a70b2c1..e5ce24abbe6 100644
--- a/persistence/src/vespa/persistence/spi/partitionstate.h
+++ b/persistence/src/vespa/persistence/spi/partitionstate.h
@@ -16,8 +16,7 @@
 
 #include <persistence/spi/types.h>
 
-namespace storage {
-namespace spi {
+namespace storage::spi {
 
 struct PartitionState {
     enum State { UP, DOWN };
@@ -49,6 +48,4 @@ public:
     PartitionId size() const { return PartitionId(_states.size()); }
 };
 
-} // spi
-} // storage
-
+}
diff --git a/persistence/src/vespa/persistence/spi/persistenceprovider.cpp b/persistence/src/vespa/persistence/spi/persistenceprovider.cpp
index ec71928baba..61d141c0229 100644
--- a/persistence/src/vespa/persistence/spi/persistenceprovider.cpp
+++ b/persistence/src/vespa/persistence/spi/persistenceprovider.cpp
@@ -2,11 +2,8 @@
 
 #include "persistenceprovider.h"
 
-namespace storage {
-namespace spi {
+namespace storage::spi {
 
-PersistenceProvider::~PersistenceProvider() { }
-
-} // spi
-} // storage
+PersistenceProvider::~PersistenceProvider() = default;
 
+}
diff --git a/persistence/src/vespa/persistence/spi/persistenceprovider.h b/persistence/src/vespa/persistence/spi/persistenceprovider.h
index 96b3d385b87..857630e2606 100644
--- a/persistence/src/vespa/persistence/spi/persistenceprovider.h
+++ b/persistence/src/vespa/persistence/spi/persistenceprovider.h
@@ -11,9 +11,7 @@
 #include "selection.h"
 #include "clusterstate.h"
 
-namespace document {
-    class FieldSet;
-}
+namespace document { class FieldSet; }
 
 namespace storage::spi {
 
diff --git a/persistence/src/vespa/persistence/spi/providerfactory.h b/persistence/src/vespa/persistence/spi/providerfactory.h
index 6b4f0b20ae9..8be143851dd 100644
--- a/persistence/src/vespa/persistence/spi/providerfactory.h
+++ b/persistence/src/vespa/persistence/spi/providerfactory.h
@@ -8,14 +8,11 @@
 
 #pragma once
 
-#include <vespa/persistence/spi/persistenceprovider.h>
+#include "persistenceprovider.h"
 
-namespace document {
-    class DocumentTypeRepo;
-}
+namespace document { class DocumentTypeRepo; }
 
-namespace storage {
-namespace spi {
+namespace storage::spi {
 
 struct ProviderFactory {
     virtual ~ProviderFactory() {}
diff --git a/persistence/src/vespa/persistence/spi/read_consistency.cpp b/persistence/src/vespa/persistence/spi/read_consistency.cpp
index cad15a5263d..c3b55e5a261 100644
--- a/persistence/src/vespa/persistence/spi/read_consistency.cpp
+++ b/persistence/src/vespa/persistence/spi/read_consistency.cpp
@@ -1,12 +1,11 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 #include "read_consistency.h"
-#include <iostream>
+#include <ostream>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".persistence.spi.read_consistency");
 
-namespace storage {
-namespace spi {
+namespace storage::spi {
 
 std::ostream&
 operator<<(std::ostream& os, ReadConsistency consistency)
@@ -24,6 +23,4 @@ operator<<(std::ostream& os, ReadConsistency consistency)
     return os;
 }
 
-} // spi
-} // storage
-
+}
diff --git a/persistence/src/vespa/persistence/spi/read_consistency.h b/persistence/src/vespa/persistence/spi/read_consistency.h
index 507e05027cc..d90c43af407 100644
--- a/persistence/src/vespa/persistence/spi/read_consistency.h
+++ b/persistence/src/vespa/persistence/spi/read_consistency.h
@@ -2,10 +2,9 @@
 #pragma once
 
 #include <iosfwd>
-#include <stdint.h>
+#include <cstdint>
 
-namespace storage {
-namespace spi {
+namespace storage::spi {
 
 enum class ReadConsistency : uint8_t {
     /**
@@ -28,9 +27,7 @@ enum class ReadConsistency : uint8_t {
     WEAK
 };
 
-std::ostream&
-operator<<(std::ostream&, ReadConsistency);
+std::ostream& operator<<(std::ostream&, ReadConsistency);
 
-} // spi
-} // storage
+}
 
diff --git a/persistence/src/vespa/persistence/spi/result.cpp b/persistence/src/vespa/persistence/spi/result.cpp
index 024f5595102..f5131f08b1f 100644
--- a/persistence/src/vespa/persistence/spi/result.cpp
+++ b/persistence/src/vespa/persistence/spi/result.cpp
@@ -9,7 +9,7 @@ namespace storage::spi {
 
 Result::Result(const Result &) = default;
 Result & Result::operator = (const Result &) = default;
-Result::~Result() { }
+Result::~Result() = default;
 
 vespalib::string
 Result::toString() const {
@@ -33,10 +33,10 @@ GetResult::GetResult(Document::UP doc, Timestamp timestamp)
       _doc(std::move(doc))
 { }
 
-GetResult::~GetResult() { }
-BucketIdListResult::~BucketIdListResult() { }
+GetResult::~GetResult() = default;
+BucketIdListResult::~BucketIdListResult() = default;
 
-IterateResult::~IterateResult() { }
+IterateResult::~IterateResult() = default;
 
 }
 
diff --git a/persistence/src/vespa/persistence/spi/selection.cpp b/persistence/src/vespa/persistence/spi/selection.cpp
index 0cd50446488..c7ed98b9d92 100644
--- a/persistence/src/vespa/persistence/spi/selection.cpp
+++ b/persistence/src/vespa/persistence/spi/selection.cpp
@@ -2,8 +2,7 @@
 
 #include "selection.h"
 
-namespace storage {
-namespace spi {
+namespace storage::spi {
 
 Selection::Selection(const DocumentSelection& docSel)
     : _documentSelection(docSel),
@@ -12,8 +11,7 @@ Selection::Selection(const DocumentSelection& docSel)
       _timestampSubset()
 { }
 
-Selection::~Selection() { }
+Selection::~Selection() = default;
 
-} // spi
-} // storage
+}
 
diff --git a/persistence/src/vespa/persistence/spi/selection.h b/persistence/src/vespa/persistence/spi/selection.h
index 5c562cabd34..0f809cf1641 100644
--- a/persistence/src/vespa/persistence/spi/selection.h
+++ b/persistence/src/vespa/persistence/spi/selection.h
@@ -10,12 +10,7 @@
 
 #include "documentselection.h"
 
-namespace storage {
-namespace spi {
-
-class MetaData {
-    Timestamp timestamp;
-};
+namespace storage::spi {
 
 class Selection {
 public:
@@ -67,13 +62,6 @@ public:
 
     Timestamp getFromTimestamp() const { return _fromTimestamp; }
     Timestamp getToTimestamp() const { return _toTimestamp; }
-
-    /**
-     * Regular usage.
-     */
-    bool match(const Document& doc, const MetaData& metaData) const;
 };
 
-} // spi
-} // storage
-
+}
diff --git a/persistencetypes/src/persistence/spi/types.cpp b/persistencetypes/src/persistence/spi/types.cpp
index 00aa95b9707..e746f02e869 100644
--- a/persistencetypes/src/persistence/spi/types.cpp
+++ b/persistencetypes/src/persistence/spi/types.cpp
@@ -1,11 +1,8 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 #include "types.h"
-
 #include <vespa/vespalib/objects/nbostream.h>
 
-namespace storage {
-
-namespace spi {
+namespace storage::spi {
 
 DEFINE_PRIMITIVE_WRAPPER_NBOSTREAM(NodeIndex);
 DEFINE_PRIMITIVE_WRAPPER_NBOSTREAM(PartitionId);
@@ -14,5 +11,3 @@ DEFINE_PRIMITIVE_WRAPPER_NBOSTREAM(Timestamp);
 DEFINE_PRIMITIVE_WRAPPER_NBOSTREAM(BucketChecksum);
 
 }
-
-}
diff --git a/processing/abi-spec.json b/processing/abi-spec.json
index 78058f1a8b7..8f77672faec 100644
--- a/processing/abi-spec.json
+++ b/processing/abi-spec.json
@@ -329,6 +329,10 @@
       "public final void set(java.lang.String, java.lang.Object, java.util.Map)",
       "public final void set(com.yahoo.processing.request.CompoundName, java.lang.Object)",
       "public final void set(java.lang.String, java.lang.Object)",
+      "public void clearAll(com.yahoo.processing.request.CompoundName, java.util.Map)",
+      "public final void clearAll(java.lang.String, java.lang.Object, java.util.Map)",
+      "public final void clearAll(com.yahoo.processing.request.CompoundName)",
+      "public final void clearAll(java.lang.String)",
       "public final boolean getBoolean(com.yahoo.processing.request.CompoundName)",
       "public final boolean getBoolean(java.lang.String)",
       "public final boolean getBoolean(com.yahoo.processing.request.CompoundName, boolean)",
diff --git a/processing/src/main/java/com/yahoo/processing/execution/Execution.java b/processing/src/main/java/com/yahoo/processing/execution/Execution.java
index ee1e9eb39e4..98bc3485084 100644
--- a/processing/src/main/java/com/yahoo/processing/execution/Execution.java
+++ b/processing/src/main/java/com/yahoo/processing/execution/Execution.java
@@ -206,7 +206,7 @@ public class Execution {
          * Creates an empty environment. Only useful for some limited testing
          */
         public static <C extends Processor> Environment<C> createEmpty() {
-            return new Environment<C>(new ChainRegistry<C>());
+            return new Environment<>(new ChainRegistry<>());
         }
 
         /**
@@ -244,7 +244,7 @@ public class Execution {
         /**
          * The node in the trace tree capturing this execution
          */
-        private TraceNode traceNode;
+        private final TraceNode traceNode;
 
         /**
          * The highest level of tracing this should record
@@ -254,7 +254,7 @@ public class Execution {
         /**
          * If true, do timing logic, even though trace level is low.
          */
-        private boolean forceTimestamps = false;
+        private boolean forceTimestamps;
 
         /**
          * Creates an empty root trace with a given level of tracing
diff --git a/processing/src/main/java/com/yahoo/processing/request/CompoundName.java b/processing/src/main/java/com/yahoo/processing/request/CompoundName.java
index 976fb3e2796..2185aac7adc 100644
--- a/processing/src/main/java/com/yahoo/processing/request/CompoundName.java
+++ b/processing/src/main/java/com/yahoo/processing/request/CompoundName.java
@@ -20,12 +20,6 @@ import static com.yahoo.text.Lowercase.toLowerCase;
  */
 public final class CompoundName {
 
-    /**
-     * The string name of this compound.
-     */
-    private final String name;
-    private final String lowerCasedName;
-
     private final ImmutableList<String> compounds;
 
     /** A hashcode which is always derived from the compounds (NEVER the string) */
@@ -43,7 +37,7 @@ public final class CompoundName {
      * @throws NullPointerException if name is null
      */
     public CompoundName(String name) {
-        this(name, parse(name));
+        this(parse(name));
     }
 
     /** Constructs this from an array of name components which are assumed not to contain dots */
@@ -53,21 +47,6 @@ public final class CompoundName {
 
     /** Constructs this from a list of compounds. */
     public CompoundName(List<String> compounds) {
-        this(toCompoundString(compounds), compounds);
-    }
-
-    /**
-     * Constructs this from a name with already parsed compounds.
-     * Private to avoid creating names with inconsistencies.
-     *
-     * @param name the string representation of the compounds
-     * @param compounds the compounds of this name
-     */
-    private CompoundName(String name, List<String> compounds) {
-        if (name == null) throw new NullPointerException("Name can not be null");
-
-        this.name = name;
-        this.lowerCasedName = toLowerCase(name);
         if (compounds.size()==1 && compounds.get(0).isEmpty())
             this.compounds = ImmutableList.of();
         else
@@ -111,7 +90,7 @@ public final class CompoundName {
         if (isEmpty()) return new CompoundName(name);
         List<String> newCompounds = new ArrayList<>(compounds);
         newCompounds.addAll(parse(name));
-        return new CompoundName(concat(this.name, name), newCompounds);
+        return new CompoundName(newCompounds);
     }
 
     /**
@@ -124,11 +103,7 @@ public final class CompoundName {
         if (isEmpty()) return name;
         List<String> newCompounds = new ArrayList<>(compounds);
         newCompounds.addAll(name.compounds);
-        return new CompoundName(concat(this.name, name.name), newCompounds);
-    }
-
-    private String concat(String name1, String name2) {
-        return name1 + "." + name2;
+        return new CompoundName(newCompounds);
     }
 
     /**
@@ -242,14 +217,11 @@ public final class CompoundName {
     public boolean hasPrefix(CompoundName prefix) {
         if (prefix.size() > this.size()) return false;
 
-        int prefixLength = prefix.name.length();
-        if (prefixLength == 0)
-            return true;
-
-        if (name.length() > prefixLength && name.charAt(prefixLength) != '.')
-            return false;
-
-        return name.startsWith(prefix.name);
+        for (int i = 0; i < prefix.size(); i++) {
+            if ( ! this.compounds.get(i).equals(prefix.get(i)))
+                return false;
+        }
+        return true;
     }
 
     /**
@@ -267,24 +239,22 @@ public final class CompoundName {
         if (o == this) return true;
         if ( ! (o instanceof CompoundName)) return false;
         CompoundName other = (CompoundName)o;
-        return this.name.equals(other.name);
+        return this.compounds.equals(other.compounds);
     }
 
     /**
      * Returns the string representation of this - all the name components in order separated by dots.
      */
     @Override
-    public String toString() { return name; }
-
-    public String getLowerCasedName() {
-        return lowerCasedName;
-    }
-
-    private static String toCompoundString(List<String> compounds) {
+    public String toString() {
         StringBuilder b = new StringBuilder();
         for (String compound : compounds)
             b.append(compound).append(".");
         return b.length()==0 ? "" : b.substring(0, b.length()-1);
     }
 
+    public String getLowerCasedName() {
+        return toString().toLowerCase();
+    }
+
 }
diff --git a/processing/src/main/java/com/yahoo/processing/request/Properties.java b/processing/src/main/java/com/yahoo/processing/request/Properties.java
index 095a0e51ce0..9362de59203 100644
--- a/processing/src/main/java/com/yahoo/processing/request/Properties.java
+++ b/processing/src/main/java/com/yahoo/processing/request/Properties.java
@@ -206,8 +206,8 @@ public class Properties implements Cloneable {
      * This default implementation forwards to the chained instance or throws
      * a RuntimeException if there is not chained instance.
      *
-     * @param name    the name of the value
-     * @param value   the value to set. Setting a name to null explicitly is legal.
+     * @param name    the name of the property
+     * @param value   the value to set. Setting a property to null clears it.
      * @param context the context used to resolve where the values should be set, or null if none
      * @throws RuntimeException if no instance in the chain accepted this name-value pair
      */
@@ -223,8 +223,8 @@ public class Properties implements Cloneable {
      * This default implementation forwards to the chained instance or throws
      * a RuntimeException if there is not chained instance.
      *
-     * @param name    the name of the value
-     * @param value   the value to set. Setting a name to null explicitly is legal.
+     * @param name    the name of the property
+     * @param value   the value to set. Setting a property to null clears it.
      * @param context the context used to resolve where the values should be set, or null if none
      * @throws RuntimeException if no instance in the chain accepted this name-value pair
      */
@@ -235,8 +235,8 @@ public class Properties implements Cloneable {
     /**
      * Sets a value to the first chained instance which accepts it by calling set(name,value,null).
      *
-     * @param name  the name of the value
-     * @param value the value to set. Setting a name to null explicitly is legal.
+     * @param name    the name of the property
+     * @param value   the value to set. Setting a property to null clears it.
      * @throws RuntimeException if no instance in the chain accepted this name-value pair
      */
     public final void set(CompoundName name, Object value) {
@@ -246,8 +246,8 @@ public class Properties implements Cloneable {
     /**
      * Sets a value to the first chained instance which accepts it by calling set(name,value,null).
      *
-     * @param name  the name of the value
-     * @param value the value to set. Setting a name to null explicitly is legal.
+     * @param name    the name of the property
+     * @param value   the value to set. Setting a property to null clears it.
      * @throws RuntimeException if no instance in the chain accepted this name-value pair
      */
     public final void set(String name, Object value) {
@@ -255,6 +255,56 @@ public class Properties implements Cloneable {
     }
 
     /**
+     * Sets all properties having this name as a compound prefix to null.
+     * I.e clearAll("a") will clear the value of "a" and "a.b" but not "ab".
+     * This default implementation forwards to the chained instance or throws
+     * a RuntimeException if there is not chained instance.
+     *
+     * @param name    the compound prefix of the properties to clear
+     * @param context the context used to resolve where the values should be cleared, or null if none
+     * @throws RuntimeException if no instance in the chain accepted this name-value pair
+     */
+    public void clearAll(CompoundName name, Map<String, String> context) {
+        if (chained == null) throw new RuntimeException("Property '" + name +
+                                                        "' was not accepted in this property chain");
+        chained.clearAll(name, context);
+    }
+
+    /**
+     * Sets all properties having this name as a compound prefix to null.
+     * I.e clearAll("a") will clear the value of "a" and "a.b" but not "ab".
+     *
+     * @param name    the compound prefix of the properties to clear
+     * @param context the context used to resolve where the values should be cleared, or null if none
+     * @throws RuntimeException if no instance in the chain accepted this name-value pair
+     */
+    public final void clearAll(String name, Object value, Map<String, String> context) {
+        set(new CompoundName(name), value, context);
+    }
+
+    /**
+     * Sets all properties having this name as a compound prefix to null.
+     * I.e clearAll("a") will clear the value of "a" and "a.b" but not "ab".
+     *
+     * @param name    the compound prefix of the properties to clear
+     * @throws RuntimeException if no instance in the chain accepted this name-value pair
+     */
+    public final void clearAll(CompoundName name) {
+        clearAll(name, null);
+    }
+
+    /**
+     * Sets all properties having this name as a compound prefix to null.
+     * I.e clearAll("a") will clear the value of "a" and "a.b" but not "ab".
+     *
+     * @param name    the compound prefix of the properties to clear
+     * @throws RuntimeException if no instance in the chain accepted this name-value pair
+     */
+    public final void clearAll(String name) {
+        clearAll(new CompoundName(name), Collections.<String,String>emptyMap());
+    }
+
+    /**
      * Gets a property as a boolean - if this value can reasonably be interpreted as a boolean, this will return
      * the value. Returns false if this property is null.
      */
@@ -493,7 +543,7 @@ public class Properties implements Cloneable {
     /**
      * Returns a property as a Double
      *
-     * @return the integer value of the name, or null if the property is null
+     * @return the double value of the name, or null if the property is null
      * @throws NumberFormatException if the given parameter exists but have a value which
      *                               is not parseable as a number
      */
@@ -504,7 +554,7 @@ public class Properties implements Cloneable {
     /**
      * Returns a property as a Double
      *
-     * @return the integer value of the name, or null if the property is null
+     * @return the double value of the name, or null if the property is null
      * @throws NumberFormatException if the given parameter exists but have a value which
      *                               is not parseable as a number
      */
@@ -571,14 +621,14 @@ public class Properties implements Cloneable {
         }
     }
 
-    /**
-     * Clones a map by deep cloning each value which is cloneable and shallow copying all other values.
-     */
+    /** Clones a map by deep cloning each value which is cloneable and shallow copying all other values. */
     public static Map<CompoundName, Object> cloneMap(Map<CompoundName, Object> map) {
         return cloneHelper.cloneMap(map);
     }
+
     /** Clones this object if it is clonable, and the clone is public. Returns null if not */
     public static Object clone(Object object) {
         return cloneHelper.clone(object);
     }
+
 }
diff --git a/searchcommon/src/vespa/searchcommon/attribute/config.cpp b/searchcommon/src/vespa/searchcommon/attribute/config.cpp
index 53e57fd9c66..b4e05875820 100644
--- a/searchcommon/src/vespa/searchcommon/attribute/config.cpp
+++ b/searchcommon/src/vespa/searchcommon/attribute/config.cpp
@@ -17,7 +17,8 @@ Config::Config() :
     _growStrategy(),
     _compactionStrategy(),
     _predicateParams(),
-    _tensorType(vespalib::eval::ValueType::error_type())
+    _tensorType(vespalib::eval::ValueType::error_type()),
+    _hnsw_index_params()
 {
 }
 
@@ -34,7 +35,8 @@ Config::Config(BasicType bt, CollectionType ct, bool fastSearch_, bool huge_)
       _growStrategy(),
       _compactionStrategy(),
       _predicateParams(),
-      _tensorType(vespalib::eval::ValueType::error_type())
+      _tensorType(vespalib::eval::ValueType::error_type()),
+      _hnsw_index_params()
 {
 }
 
@@ -60,7 +62,8 @@ Config::operator==(const Config &b) const
            _compactionStrategy == b._compactionStrategy &&
            _predicateParams == b._predicateParams &&
            (_basicType.type() != BasicType::Type::TENSOR ||
-            _tensorType == b._tensorType);
+            _tensorType == b._tensorType) &&
+            _hnsw_index_params == b._hnsw_index_params;
 }
 
 }
diff --git a/searchcommon/src/vespa/searchcommon/attribute/config.h b/searchcommon/src/vespa/searchcommon/attribute/config.h
index 2f767061f7a..836fcfed84a 100644
--- a/searchcommon/src/vespa/searchcommon/attribute/config.h
+++ b/searchcommon/src/vespa/searchcommon/attribute/config.h
@@ -4,15 +4,21 @@
 
 #include "basictype.h"
 #include "collectiontype.h"
+#include "hnsw_index_params.h"
 #include "predicate_params.h"
-#include <vespa/searchcommon/common/growstrategy.h>
 #include <vespa/searchcommon/common/compaction_strategy.h>
+#include <vespa/searchcommon/common/growstrategy.h>
 #include <vespa/eval/eval/value_type.h>
+#include <optional>
 
 namespace search::attribute {
 
-class Config
-{
+/**
+ * Configuration for an attribute vector.
+ *
+ * Used to determine which implementation to instantiate.
+ */
+class Config {
 public:
     Config();
     Config(BasicType bt, CollectionType ct = CollectionType::SINGLE,
@@ -29,6 +35,7 @@ public:
     bool huge()                           const { return _huge; }
     const PredicateParams &predicateParams() const { return _predicateParams; }
     vespalib::eval::ValueType tensorType() const { return _tensorType; }
+    const std::optional<HnswIndexParams>& hnsw_index_params() const { return _hnsw_index_params; }
 
     /**
      * Check if attribute posting list can consist of a bitvector in
@@ -60,6 +67,10 @@ public:
         _tensorType = tensorType_in;
         return *this;
     }
+    Config& set_hnsw_index_params(const HnswIndexParams& params) {
+        _hnsw_index_params = params;
+        return *this;
+    }
 
     /**
      * Enable attribute posting list to consist of a bitvector in
@@ -107,6 +118,7 @@ private:
     CompactionStrategy _compactionStrategy;
     PredicateParams    _predicateParams;
     vespalib::eval::ValueType _tensorType;
+    std::optional<HnswIndexParams> _hnsw_index_params;
 };
 
 }
diff --git a/searchcommon/src/vespa/searchcommon/attribute/distance_metric.h b/searchcommon/src/vespa/searchcommon/attribute/distance_metric.h
new file mode 100644
index 00000000000..9309a0a86dc
--- /dev/null
+++ b/searchcommon/src/vespa/searchcommon/attribute/distance_metric.h
@@ -0,0 +1,9 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+namespace search::attribute {
+
+enum class DistanceMetric { Euclidean, Angular, GeoDegrees };
+
+}
diff --git a/searchcommon/src/vespa/searchcommon/attribute/hnsw_index_params.h b/searchcommon/src/vespa/searchcommon/attribute/hnsw_index_params.h
new file mode 100644
index 00000000000..94d3fda49f3
--- /dev/null
+++ b/searchcommon/src/vespa/searchcommon/attribute/hnsw_index_params.h
@@ -0,0 +1,39 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "distance_metric.h"
+
+namespace search::attribute {
+
+/**
+ * Configuration parameters for a hnsw index used together with a 1-dimensional indexed tensor
+ * for approximate nearest neighbor search.
+ */
+class HnswIndexParams {
+private:
+    uint32_t _max_links_per_node;
+    uint32_t _neighbors_to_explore_at_insert;
+    DistanceMetric _distance_metric;
+
+public:
+    HnswIndexParams(uint32_t max_links_per_node_in,
+                    uint32_t neighbors_to_explore_at_insert_in,
+                    DistanceMetric distance_metric_in)
+            : _max_links_per_node(max_links_per_node_in),
+              _neighbors_to_explore_at_insert(neighbors_to_explore_at_insert_in),
+              _distance_metric(distance_metric_in)
+    {}
+
+    uint32_t max_links_per_node() const { return _max_links_per_node; }
+    uint32_t neighbors_to_explore_at_insert() const { return _neighbors_to_explore_at_insert; }
+    DistanceMetric distance_metric() const { return _distance_metric; }
+
+    bool operator==(const HnswIndexParams& rhs) const {
+        return (_max_links_per_node == rhs._max_links_per_node &&
+                _neighbors_to_explore_at_insert == rhs._neighbors_to_explore_at_insert &&
+                _distance_metric == rhs._distance_metric);
+    }
+};
+
+}
diff --git a/searchcommon/src/vespa/searchcommon/attribute/i_attribute_functor.h b/searchcommon/src/vespa/searchcommon/attribute/i_attribute_functor.h
index 90484f8cc3a..c802730e6fe 100644
--- a/searchcommon/src/vespa/searchcommon/attribute/i_attribute_functor.h
+++ b/searchcommon/src/vespa/searchcommon/attribute/i_attribute_functor.h
@@ -17,19 +17,19 @@ class IConstAttributeFunctor
 {
 public:
     virtual void operator()(const IAttributeVector &attributeVector) = 0;
-    virtual ~IConstAttributeFunctor() { }
+    virtual ~IConstAttributeFunctor() = default;
 };
 
 class IAttributeFunctor
 {
 public:
     virtual void operator()(IAttributeVector &attributeVector) = 0;
-    virtual ~IAttributeFunctor() { }
+    virtual ~IAttributeFunctor() = default;
 };
 
 class IAttributeExecutor {
 public:
-    virtual ~IAttributeExecutor() { }
+    virtual ~IAttributeExecutor() = default;
     virtual void asyncForAttribute(const vespalib::string &name, std::unique_ptr<IAttributeFunctor> func) const = 0;
 };
 
diff --git a/searchcore/src/apps/proton/proton.cpp b/searchcore/src/apps/proton/proton.cpp
index f80558a1bc6..8f2be3257fc 100644
--- a/searchcore/src/apps/proton/proton.cpp
+++ b/searchcore/src/apps/proton/proton.cpp
@@ -8,7 +8,6 @@
 #include <vespa/metrics/metricmanager.h>
 #include <vespa/vespalib/util/signalhandler.h>
 #include <vespa/vespalib/util/programoptions.h>
-#include <vespa/vespalib/util/time.h>
 #include <vespa/vespalib/io/fileutil.h>
 #include <vespa/config/common/exceptions.h>
 #include <vespa/fastos/app.h>
@@ -25,15 +24,21 @@ struct Params
     std::string identity;
     std::string serviceidentity;
     uint64_t subscribeTimeout;
+    Params();
     ~Params();
 };
 
+Params::Params()
+    : identity(),
+      serviceidentity(),
+      subscribeTimeout(60)
+{}
 Params::~Params() = default;
 
 class App : public FastOS_Application
 {
 private:
-    void setupSignals();
+    static void setupSignals();
     Params parseParams();
 public:
     int Main() override;
@@ -73,7 +78,7 @@ App::parseParams()
     vespalib::ProgramOptions parser(_argc, _argv);
     parser.setSyntaxMessage("proton -- the nextgen search core");
     parser.addOption("identity", params.identity, "Node identity and config id");
-    std::string empty("");
+    std::string empty;
     parser.addOption("serviceidentity", params.serviceidentity, empty, "Service node identity and config id");
     parser.addOption("subscribeTimeout", params.subscribeTimeout, UINT64_C(600000), "Initial config subscribe timeout");
     try {
@@ -102,7 +107,7 @@ public:
     ProtonServiceLayerProcess(const config::ConfigUri & configUri,
                               proton::Proton & proton,
                               PersistenceProvider *downPersistence);
-    ~ProtonServiceLayerProcess() { shutdown(); }
+    ~ProtonServiceLayerProcess() override { shutdown(); }
 
     void shutdown() override;
     void setupProvider() override;
@@ -126,7 +131,7 @@ ProtonServiceLayerProcess::ProtonServiceLayerProcess(const config::ConfigUri &
                                                      downPersistence)
     : ServiceLayerProcess(configUri),
       _proton(proton),
-      _metricManager(0),
+      _metricManager(nullptr),
       _downPersistence(downPersistence)
 {
     if (!downPersistence) {
@@ -143,7 +148,7 @@ ProtonServiceLayerProcess::shutdown()
 void
 ProtonServiceLayerProcess::setupProvider()
 {
-    if (_metricManager != 0) {
+    if (_metricManager != nullptr) {
         _context.getComponentRegister().setMetricManager(*_metricManager);
     }
 }
@@ -270,6 +275,9 @@ App::Main()
     } catch (const vespalib::NetworkSetupFailureException & e) {
         LOG(warning, "Network failure: '%s'", e.what());
         return 1;
+    } catch (const config::InvalidConfigException & e) {
+        LOG(warning, "Invalid config failure: '%s'", e.what());
+        return 1;
     } catch (const vespalib::IllegalStateException & e) {
         LOG(error, "Unknown IllegalStateException: '%s'", e.what());
         throw;
diff --git a/searchcore/src/apps/tests/persistenceconformance_test.cpp b/searchcore/src/apps/tests/persistenceconformance_test.cpp
index ba0d2047b88..4056e49e37c 100644
--- a/searchcore/src/apps/tests/persistenceconformance_test.cpp
+++ b/searchcore/src/apps/tests/persistenceconformance_test.cpp
@@ -331,7 +331,7 @@ public:
             LOG(info, "putHandler(%s)", itr->first.toString().c_str());
             IPersistenceHandler::SP proxy(
                     new PersistenceHandlerProxy(itr->second));
-            putHandler(itr->second->getBucketSpace(), itr->first, proxy);
+            putHandler(getWLock(), itr->second->getBucketSpace(), itr->first, proxy);
         }
     }
 
@@ -343,7 +343,7 @@ public:
         const DocumentDBMap &docDbs = _docDbRepo->getDocDbs();
         for (DocumentDBMap::const_iterator itr = docDbs.begin();
              itr != docDbs.end(); ++itr) {
-            IPersistenceHandler::SP proxy(removeHandler(itr->second->getBucketSpace(), itr->first));
+            IPersistenceHandler::SP proxy(removeHandler(getWLock(), itr->second->getBucketSpace(), itr->first));
         }
     }
 
diff --git a/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp b/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp
index c6d49c479c4..407b14c5f7d 100644
--- a/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp
+++ b/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp
@@ -1,7 +1,6 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 #include <vespa/config-attributes.h>
 #include <vespa/fastos/file.h>
-#include <vespa/searchcommon/attribute/attributecontent.h>
 #include <vespa/searchcommon/attribute/iattributevector.h>
 #include <vespa/searchcore/proton/attribute/attribute_collection_spec_factory.h>
 #include <vespa/searchcore/proton/attribute/attribute_manager_initializer.h>
@@ -24,18 +23,16 @@
 #include <vespa/searchlib/attribute/attribute_read_guard.h>
 #include <vespa/searchlib/attribute/imported_attribute_vector.h>
 #include <vespa/searchlib/attribute/imported_attribute_vector_factory.h>
-#include <vespa/searchlib/attribute/integerbase.h>
 #include <vespa/searchlib/attribute/predicate_attribute.h>
 #include <vespa/searchlib/attribute/reference_attribute.h>
 #include <vespa/searchlib/attribute/singlenumericattribute.hpp>
-#include <vespa/searchlib/common/foregroundtaskexecutor.h>
+#include <vespa/vespalib/util/foregroundtaskexecutor.h>
 #include <vespa/searchlib/common/indexmetainfo.h>
 #include <vespa/searchlib/index/dummyfileheadercontext.h>
 #include <vespa/searchlib/predicate/predicate_index.h>
 #include <vespa/searchlib/predicate/predicate_tree_annotator.h>
 #include <vespa/searchlib/test/directory_handler.h>
 #include <vespa/searchlib/test/mock_gid_to_lid_mapping.h>
-#include <vespa/searchlib/util/filekit.h>
 #include <vespa/vespalib/testkit/testapp.h>
 #include <vespa/vespalib/util/threadstackexecutor.h>
 
@@ -55,7 +52,7 @@ using proton::initializer::InitializerTask;
 using proton::test::AttributeUtils;
 using proton::test::createInt32Attribute;
 using proton::test::Int32Attribute;
-using search::ForegroundTaskExecutor;
+using vespalib::ForegroundTaskExecutor;
 using search::TuneFileAttributes;
 using search::attribute::BasicType;
 using search::attribute::IAttributeContext;
@@ -215,7 +212,7 @@ struct SequentialAttributeManager
     {
         mgr.addInitializedAttributes(initializer.getInitializedAttributes());
     }
-    ~SequentialAttributeManager() {}
+    ~SequentialAttributeManager() = default;
 };
 
 struct DummyInitializerTask : public InitializerTask
@@ -262,23 +259,33 @@ ParallelAttributeManager::ParallelAttributeManager(search::SerialNum configSeria
     initializer::TaskRunner taskRunner(executor);
     taskRunner.runTask(initializer);
 }
-ParallelAttributeManager::~ParallelAttributeManager() {}
+ParallelAttributeManager::~ParallelAttributeManager() = default;
 
 TEST_F("require that attributes are added", Fixture)
 {
-    EXPECT_TRUE(f.addAttribute("a1").get() != NULL);
-    EXPECT_TRUE(f.addAttribute("a2").get() != NULL);
+    EXPECT_TRUE(f.addAttribute("a1").get() != nullptr);
+    EXPECT_TRUE(f.addAttribute("a2").get() != nullptr);
     EXPECT_EQUAL("a1", (*f._m.getAttribute("a1"))->getName());
     EXPECT_EQUAL("a1", (*f._m.getAttributeReadGuard("a1", true))->getName());
     EXPECT_EQUAL("a2", (*f._m.getAttribute("a2"))->getName());
     EXPECT_EQUAL("a2", (*f._m.getAttributeReadGuard("a2", true))->getName());
     EXPECT_TRUE(!f._m.getAttribute("not")->valid());
+
+    auto rv = f._m.readable_attribute_vector("a1");
+    ASSERT_TRUE(rv.get() != nullptr);
+    EXPECT_EQUAL("a1", rv->makeReadGuard(true)->attribute()->getName());
+
+    rv = f._m.readable_attribute_vector("a2");
+    ASSERT_TRUE(rv.get() != nullptr);
+    EXPECT_EQUAL("a2", rv->makeReadGuard(true)->attribute()->getName());
+
+    EXPECT_TRUE(f._m.readable_attribute_vector("not_valid").get() == nullptr);
 }
 
 TEST_F("require that predicate attributes are added", Fixture)
 {
     EXPECT_TRUE(f._m.addAttribute({"p1", AttributeUtils::getPredicateConfig()},
-                                  createSerialNum).get() != NULL);
+                                  createSerialNum).get() != nullptr);
     EXPECT_EQUAL("p1", (*f._m.getAttribute("p1"))->getName());
     EXPECT_EQUAL("p1", (*f._m.getAttributeReadGuard("p1", true))->getName());
 }
@@ -376,7 +383,7 @@ TEST_F("require that predicate attributes are flushed and loaded", BaseFixture)
         AttributeVector::SP a1 = am.addAttribute({"a1", AttributeUtils::getPredicateConfig()}, createSerialNum);
         EXPECT_EQUAL(1u, a1->getNumDocs());
 
-        PredicateAttribute &pa = static_cast<PredicateAttribute &>(*a1);
+        auto &pa = static_cast<PredicateAttribute &>(*a1);
         PredicateIndex &index = pa.getIndex();
         uint32_t doc_id;
         a1->addDoc(doc_id);
@@ -396,7 +403,7 @@ TEST_F("require that predicate attributes are flushed and loaded", BaseFixture)
         AttributeVector::SP a1 = am.addAttribute({"a1", AttributeUtils::getPredicateConfig()}, createSerialNum); // loaded
         EXPECT_EQUAL(2u, a1->getNumDocs());
 
-        PredicateAttribute &pa = static_cast<PredicateAttribute &>(*a1);
+        auto &pa = static_cast<PredicateAttribute &>(*a1);
         PredicateIndex &index = pa.getIndex();
         uint32_t doc_id;
         a1->addDoc(doc_id);
@@ -746,7 +753,7 @@ TEST_F("require that we can acquire exclusive read access to attribute", Fixture
     EXPECT_TRUE(noneAccessor.get() == nullptr);
 }
 
-TEST_F("require that imported attributes are exposed via attribute context together vi regular attributes", Fixture)
+TEST_F("require that imported attributes are exposed via attribute context together with regular attributes", Fixture)
 {
     f.addAttribute("attr");
     f.addImportedAttribute("imported");
@@ -767,6 +774,17 @@ TEST_F("require that imported attributes are exposed via attribute context toget
     EXPECT_EQUAL("imported", all[1]->getName());
 }
 
+TEST_F("imported attributes are transparently returned from readable_attribute_vector", Fixture)
+{
+    f.addAttribute("attr");
+    f.addImportedAttribute("imported");
+    f.setImportedAttributes();
+    auto av = f._m.readable_attribute_vector("imported");
+    ASSERT_TRUE(av);
+    auto g = av->makeReadGuard(false);
+    EXPECT_EQUAL("imported", g->attribute()->getName());
+}
+
 TEST_F("require that attribute vector of wrong type is dropped", BaseFixture)
 {
     AVConfig generic_tensor(BasicType::TENSOR);
diff --git a/searchcore/src/tests/proton/attribute/attribute_populator/attribute_populator_test.cpp b/searchcore/src/tests/proton/attribute/attribute_populator/attribute_populator_test.cpp
index 01dc2306ef6..788aa4fb1ee 100644
--- a/searchcore/src/tests/proton/attribute/attribute_populator/attribute_populator_test.cpp
+++ b/searchcore/src/tests/proton/attribute/attribute_populator/attribute_populator_test.cpp
@@ -9,13 +9,14 @@ LOG_SETUP("attribute_populator_test");
 #include <vespa/searchcore/proton/attribute/attributemanager.h>
 #include <vespa/searchcore/proton/common/hw_info.h>
 #include <vespa/searchcore/proton/test/test.h>
-#include <vespa/searchlib/common/foregroundtaskexecutor.h>
+#include <vespa/vespalib/util/foregroundtaskexecutor.h>
 #include <vespa/searchlib/index/dummyfileheadercontext.h>
 #include <vespa/searchlib/test/directory_handler.h>
 #include <vespa/vespalib/util/stringfmt.h>
 
 using document::config_builder::DocumenttypesConfigBuilderHelper;
 using document::config_builder::Struct;
+using vespalib::ForegroundTaskExecutor;
 using namespace document;
 using namespace proton;
 using namespace search;
diff --git a/searchcore/src/tests/proton/attribute/attribute_test.cpp b/searchcore/src/tests/proton/attribute/attribute_test.cpp
index 14b72c9d8f8..839ef14fcb0 100644
--- a/searchcore/src/tests/proton/attribute/attribute_test.cpp
+++ b/searchcore/src/tests/proton/attribute/attribute_test.cpp
@@ -20,14 +20,15 @@
 #include <vespa/searchcore/proton/test/attribute_utils.h>
 #include <vespa/searchcorespi/flush/iflushtarget.h>
 #include <vespa/searchlib/attribute/attributefactory.h>
+#include <vespa/searchlib/attribute/attribute_read_guard.h>
 #include <vespa/searchlib/attribute/bitvector_search_cache.h>
 #include <vespa/searchlib/attribute/imported_attribute_vector.h>
 #include <vespa/searchlib/attribute/imported_attribute_vector_factory.h>
 #include <vespa/searchlib/attribute/integerbase.h>
 #include <vespa/searchlib/attribute/predicate_attribute.h>
-#include <vespa/searchlib/common/foregroundtaskexecutor.h>
+#include <vespa/vespalib/util/foregroundtaskexecutor.h>
 #include <vespa/searchlib/common/idestructorcallback.h>
-#include <vespa/searchlib/common/sequencedtaskexecutorobserver.h>
+#include <vespa/vespalib/util/sequencedtaskexecutorobserver.h>
 #include <vespa/searchlib/index/docbuilder.h>
 #include <vespa/searchlib/index/dummyfileheadercontext.h>
 #include <vespa/searchlib/predicate/predicate_hash.h>
@@ -42,8 +43,6 @@
 #include <vespa/vespalib/btree/btreeroot.hpp>
 #include <vespa/searchlib/attribute/singlenumericattribute.hpp>
 
-
-
 #include <vespa/log/log.h>
 LOG_SETUP("attribute_test");
 
@@ -75,6 +74,8 @@ using vespalib::eval::ValueType;
 using vespalib::eval::TensorSpec;
 using vespalib::tensor::Tensor;
 using vespalib::tensor::DefaultTensorEngine;
+using vespalib::ForegroundTaskExecutor;
+using vespalib::SequencedTaskExecutorObserver;
 
 using AVConfig = search::attribute::Config;
 using AVBasicType = search::attribute::BasicType;
@@ -588,8 +589,8 @@ struct FilterFixture
 
 TEST_F("require that filter attribute manager can filter attributes", FilterFixture)
 {
-    EXPECT_TRUE(f._filterMgr.getAttribute("a1").get() == NULL);
-    EXPECT_TRUE(f._filterMgr.getAttribute("a2").get() != NULL);
+    EXPECT_TRUE(f._filterMgr.getAttribute("a1").get() == nullptr);
+    EXPECT_TRUE(f._filterMgr.getAttribute("a2").get() != nullptr);
     std::vector<AttributeGuard> attrs;
     f._filterMgr.getAttributeList(attrs);
     EXPECT_EQUAL(1u, attrs.size());
@@ -607,6 +608,16 @@ TEST_F("require that filter attribute manager can return flushed serial number",
     EXPECT_EQUAL(100u, f._filterMgr.getFlushedSerialNum("a2"));
 }
 
+TEST_F("readable_attribute_vector filters attributes", FilterFixture)
+{
+    auto av = f._filterMgr.readable_attribute_vector("a2");
+    ASSERT_TRUE(av);
+    EXPECT_EQUAL("a2", av->makeReadGuard(false)->attribute()->getName());
+
+    av = f._filterMgr.readable_attribute_vector("a1");
+    EXPECT_FALSE(av);
+}
+
 namespace {
 
 Tensor::UP make_tensor(const TensorSpec &spec) {
diff --git a/searchcore/src/tests/proton/attribute/attributeflush_test.cpp b/searchcore/src/tests/proton/attribute/attributeflush_test.cpp
index 99d4651d13f..4604129248b 100644
--- a/searchcore/src/tests/proton/attribute/attributeflush_test.cpp
+++ b/searchcore/src/tests/proton/attribute/attributeflush_test.cpp
@@ -8,7 +8,7 @@
 #include <vespa/searchcore/proton/flushengine/shrink_lid_space_flush_target.h>
 #include <vespa/searchlib/attribute/attributefactory.h>
 #include <vespa/searchlib/attribute/integerbase.h>
-#include <vespa/searchlib/common/foregroundtaskexecutor.h>
+#include <vespa/vespalib/util/foregroundtaskexecutor.h>
 #include <vespa/searchlib/common/indexmetainfo.h>
 #include <vespa/searchlib/index/dummyfileheadercontext.h>
 #include <vespa/searchlib/test/directory_handler.h>
diff --git a/searchcore/src/tests/proton/attribute/attributes_state_explorer/attributes_state_explorer_test.cpp b/searchcore/src/tests/proton/attribute/attributes_state_explorer/attributes_state_explorer_test.cpp
index da5a8b31671..01452f1361e 100644
--- a/searchcore/src/tests/proton/attribute/attributes_state_explorer/attributes_state_explorer_test.cpp
+++ b/searchcore/src/tests/proton/attribute/attributes_state_explorer/attributes_state_explorer_test.cpp
@@ -6,10 +6,11 @@
 #include <vespa/searchcore/proton/common/hw_info.h>
 #include <vespa/searchcore/proton/test/attribute_vectors.h>
 #include <vespa/searchcore/proton/test/attribute_utils.h>
-#include <vespa/searchlib/common/foregroundtaskexecutor.h>
+#include <vespa/vespalib/util/foregroundtaskexecutor.h>
 #include <vespa/searchlib/index/dummyfileheadercontext.h>
 #include <vespa/searchlib/test/directory_handler.h>
 #include <vespa/vespalib/test/insertion_operators.h>
+#include <vespa/searchlib/attribute/singlenumericattribute.hpp>
 
 #include <vespa/log/log.h>
 LOG_SETUP("attributes_state_explorer_test");
@@ -17,7 +18,7 @@ LOG_SETUP("attributes_state_explorer_test");
 using namespace proton;
 using namespace proton::test;
 using search::AttributeVector;
-using search::ForegroundTaskExecutor;
+using vespalib::ForegroundTaskExecutor;
 using search::TuneFileAttributes;
 using search::index::DummyFileHeaderContext;
 using search::test::DirectoryHandler;
diff --git a/searchcore/src/tests/proton/attribute/exclusive_attribute_read_accessor/exclusive_attribute_read_accessor_test.cpp b/searchcore/src/tests/proton/attribute/exclusive_attribute_read_accessor/exclusive_attribute_read_accessor_test.cpp
index 420e18db5af..6f91b1a6f6f 100644
--- a/searchcore/src/tests/proton/attribute/exclusive_attribute_read_accessor/exclusive_attribute_read_accessor_test.cpp
+++ b/searchcore/src/tests/proton/attribute/exclusive_attribute_read_accessor/exclusive_attribute_read_accessor_test.cpp
@@ -5,7 +5,7 @@
 #include <vespa/searchcommon/attribute/config.h>
 #include <vespa/searchlib/attribute/attributefactory.h>
 #include <vespa/searchlib/attribute/attributevector.h>
-#include <vespa/searchlib/common/sequencedtaskexecutor.h>
+#include <vespa/vespalib/util/sequencedtaskexecutor.h>
 #include <vespa/vespalib/util/gate.h>
 
 using namespace proton;
@@ -24,13 +24,13 @@ createAttribute()
 struct Fixture
 {
     AttributeVector::SP attribute;
-    SequencedTaskExecutor writer;
+    std::unique_ptr<ISequencedTaskExecutor> writer;
     ExclusiveAttributeReadAccessor accessor;
 
     Fixture()
         : attribute(createAttribute()),
-          writer(1),
-          accessor(attribute, writer)
+          writer(SequencedTaskExecutor::create(1)),
+          accessor(attribute, *writer)
     {}
 };
 
@@ -38,7 +38,7 @@ TEST_F("require that attribute write thread is blocked while guard is held", Fix
 {
     ReadGuard::UP guard = f.accessor.takeGuard();
     Gate gate;
-    f.writer.execute(f.writer.getExecutorId(f.attribute->getNamePrefix()), [&gate]() { gate.countDown(); });
+    f.writer->execute(f.writer->getExecutorId(f.attribute->getNamePrefix()), [&gate]() { gate.countDown(); });
     bool reachedZero = gate.await(100);
     EXPECT_FALSE(reachedZero);
     EXPECT_EQUAL(1u, gate.getCount());
diff --git a/searchcore/src/tests/proton/common/attribute_updater/attribute_updater_test.cpp b/searchcore/src/tests/proton/common/attribute_updater/attribute_updater_test.cpp
index 9b930e271bd..a1dc619b3e6 100644
--- a/searchcore/src/tests/proton/common/attribute_updater/attribute_updater_test.cpp
+++ b/searchcore/src/tests/proton/common/attribute_updater/attribute_updater_test.cpp
@@ -85,13 +85,13 @@ makeDocumentTypeRepo()
     return std::make_unique<DocumentTypeRepo>(builder.config());
 }
 
+std::unique_ptr<DocumentTypeRepo> repo = makeDocumentTypeRepo();
+
 struct Fixture {
-    std::unique_ptr<DocumentTypeRepo> repo;
     const DocumentType *docType;
 
     Fixture()
-        : repo(makeDocumentTypeRepo()),
-          docType(repo->getDocumentType("testdoc"))
+        : docType(repo->getDocumentType("testdoc"))
     {
     }
 
@@ -465,6 +465,14 @@ TEST_F("require that tensor add update is applied",
     f.assertTensor(TensorSpec(f.type).add({{"x", "a"}}, 3));
 }
 
+TEST_F("require that tensor add update to non-existing tensor creates empty tensor first",
+       TensorFixture<GenericTensorAttribute>("tensor(x{})", "sparse_tensor"))
+{
+    f.applyValueUpdate(*f.attribute, 1,
+                       TensorAddUpdate(makeTensorFieldValue(TensorSpec(f.type).add({{"x", "a"}}, 3))));
+    f.assertTensor(TensorSpec(f.type).add({{"x", "a"}}, 3));
+}
+
 TEST_F("require that tensor remove update is applied",
         TensorFixture<GenericTensorAttribute>("tensor(x{})", "sparse_tensor"))
 {
diff --git a/searchcore/src/tests/proton/common/selectpruner_test.cpp b/searchcore/src/tests/proton/common/selectpruner_test.cpp
index 5b1fa3ed4bf..744159962ad 100644
--- a/searchcore/src/tests/proton/common/selectpruner_test.cpp
+++ b/searchcore/src/tests/proton/common/selectpruner_test.cpp
@@ -75,7 +75,9 @@ makeDocTypeRepo()
                      addField("aaa", Array(DataType::T_INT)).
                      addField("aaw", Wset(DataType::T_INT)).
                      addField("ab", DataType::T_INT).
-                     addField("ae", DataType::T_INT));
+                     addField("ae", DataType::T_INT)).
+                     imported_field("my_imported_field").
+                     imported_field("my_missing_imported_field");
     builder.document(doc_type_id + 1, type_name_2,
                      Struct(header_name_2), Struct(body_name_2).
                      addField("ic", DataType::T_STRING).
@@ -150,6 +152,9 @@ TestFixture::TestFixture()
     _amgr.addAttribute("aaa", AttributeFactory::createAttribute("aaa", { BasicType::INT32 , CollectionType::ARRAY}));
     _amgr.addAttribute("aaw", AttributeFactory::createAttribute("aaw", { BasicType::INT32 , CollectionType::WSET}));
     _amgr.addAttribute("ae", AttributeFactory::createAttribute("ae", { BasicType::INT32 }));
+    // We "fake" having an imported attribute to avoid having to set up reference attributes, mappings etc.
+    // This is fine since the attribute manager already abstracts away if an attribute is imported or not.
+    _amgr.addAttribute("my_imported_field", AttributeFactory::createAttribute("my_imported_field", { BasicType::INT32 }));
     _repoUP = makeDocTypeRepo();
 }
 
@@ -170,9 +175,9 @@ TestFixture::testParse(const string &selection)
         select = parser.parse(selection);
     } catch (document::select::ParsingFailedException &e) {
         LOG(info, "Parse failed: %s", e.what());
-        select.reset(0);
+        select.reset();
     }
-    ASSERT_TRUE(select.get() != NULL);
+    ASSERT_TRUE(select.get() != nullptr);
 }
 
 
@@ -189,9 +194,9 @@ TestFixture::testParseFail(const string &selection)
         select = parser.parse(selection);
     } catch (document::select::ParsingFailedException &e) {
         LOG(info, "Parse failed: %s", e.getMessage().c_str());
-        select.reset(0);
+        select.reset();
     }
-    ASSERT_TRUE(select.get() == NULL);
+    ASSERT_TRUE(select.get() == nullptr);
 }
 
 
@@ -208,15 +213,15 @@ TestFixture::testPrune(const string &selection, const string &exp, const string
         select = parser.parse(selection);
     } catch (document::select::ParsingFailedException &e) {
         LOG(info, "Parse failed: %s", e.what());
-        select.reset(0);
+        select.reset();
     }
-    ASSERT_TRUE(select.get() != NULL);
+    ASSERT_TRUE(select.get() != nullptr);
     std::ostringstream os;
     select->print(os, true, "");
     LOG(info, "ParseTree: '%s'", os.str().c_str());
     const DocumentType *docType = repo.getDocumentType(docTypeName);
-    ASSERT_TRUE(docType != NULL);
-    Document::UP emptyDoc(new Document(*docType, document::DocumentId("id:ns:" + docTypeName + "::1")));
+    ASSERT_TRUE(docType != nullptr);
+    auto emptyDoc = std::make_unique<Document>(*docType, document::DocumentId("id:ns:" + docTypeName + "::1"));
     emptyDoc->setRepo(repo);
     SelectPruner pruner(docTypeName, &_amgr, *emptyDoc, repo, _hasFields, _hasDocuments);
     pruner.process(*select);
@@ -788,6 +793,29 @@ TEST_F("Test that field values are invalid when disabling document access", Test
                 "test.aa == 4 and test.ae == 5 and invalid");
 }
 
+TEST_F("Imported fields with matching attribute names are supported", TestFixture)
+{
+    f.testPrune("test.my_imported_field > 0",
+                "test.my_imported_field > 0");
+}
+
+// Edge case: document type reconfigured but attribute not yet visible in Proton
+TEST_F("Imported fields without matching attribute are mapped to constant NullValue", TestFixture)
+{
+    f.testPrune("test.my_missing_imported_field != test.aa", "null != test.aa");
+    // Simplified to -> "null != null" -> "false"
+    f.testPrune("test.my_missing_imported_field != null", "false");
+    // Simplified to -> "null > 0" -> "invalid", as null is not well-defined
+    // for operators other than (in-)equality.
+    f.testPrune("test.my_missing_imported_field > 0", "invalid");
+}
+
+TEST_F("Complex imported field references return Invalid", TestFixture)
+{
+    f.testPrune("test.my_imported_field.foo", "invalid");
+    f.testPrune("test.my_imported_field[123]", "invalid");
+    f.testPrune("test.my_imported_field{foo}", "invalid");
+}
 
 }  // namespace
 
diff --git a/searchcore/src/tests/proton/docsummary/docsummary.cpp b/searchcore/src/tests/proton/docsummary/docsummary.cpp
index 3e86e703e1e..55d524519b8 100644
--- a/searchcore/src/tests/proton/docsummary/docsummary.cpp
+++ b/searchcore/src/tests/proton/docsummary/docsummary.cpp
@@ -197,9 +197,9 @@ public:
           _dummy(),
           _spec(TEST_PATH("")),
           _configMgr(_spec, getDocTypeName()),
-          _documenttypesConfig(new DocumenttypesConfig()),
+          _documenttypesConfig(std::make_shared<DocumenttypesConfig>()),
           _repo(repo),
-          _tuneFileDocumentDB(new TuneFileDocumentDB()),
+          _tuneFileDocumentDB(std::make_shared<TuneFileDocumentDB>()),
           _hwInfo(),
           _ddb(),
           _aw(),
@@ -774,8 +774,8 @@ Test::requireThatAttributesAreUsed()
                             "bd:[20,30],"
                             "be:[20.2,30.3],"
                             "bf:['bar','baz'],"
-                            "bg:[{item:50,weight:3},{item:40,weight:2}],"
-                            "bh:[{item:40.4,weight:4},{item:50.5,weight:5}],"
+                            "bg:[{item:40,weight:2},{item:50,weight:3}],"
+                            "bh:[{item:50.5,weight:5},{item:40.4,weight:4}],"
                             "bi:[{item:'quux',weight:7},{item:'qux',weight:6}],"
                             "bj:'0x01020178017901016601674008000000000000'}", *rep, 0, true));
     TEST_DO(assertTensor(make_tensor(TensorSpec("tensor(x{},y{})")
@@ -787,7 +787,7 @@ Test::requireThatAttributesAreUsed()
     TEST_DO(assertTensor(Tensor::UP(), "bj", *rep, 1, rclass));
 
     proton::IAttributeManager::SP attributeManager = dc._ddb->getReadySubDB()->getAttributeManager();
-    search::ISequencedTaskExecutor &attributeFieldWriter = attributeManager->getAttributeFieldWriter();
+    vespalib::ISequencedTaskExecutor &attributeFieldWriter = attributeManager->getAttributeFieldWriter();
     search::AttributeVector *bjAttr = attributeManager->getWritableAttribute("bj");
     auto bjTensorAttr = dynamic_cast<search::tensor::TensorAttribute *>(bjAttr);
 
diff --git a/searchcore/src/tests/proton/documentdb/combiningfeedview/combiningfeedview_test.cpp b/searchcore/src/tests/proton/documentdb/combiningfeedview/combiningfeedview_test.cpp
index 958be6a4686..fd8ce978d43 100644
--- a/searchcore/src/tests/proton/documentdb/combiningfeedview/combiningfeedview_test.cpp
+++ b/searchcore/src/tests/proton/documentdb/combiningfeedview/combiningfeedview_test.cpp
@@ -155,9 +155,9 @@ struct Fixture
         const test::Document &doc = userDocs().getDocs(userId)[0];
         return PutOperation(doc.getBucket(), doc.getTimestamp(), doc.getDoc());
     }
-    RemoveOperation remove(uint32_t userId) {
+    RemoveOperationWithDocId remove(uint32_t userId) {
         const test::Document &doc = userDocs().getDocs(userId)[0];
-        return RemoveOperation(doc.getBucket(), doc.getTimestamp(), doc.getDoc()->getId());
+        return RemoveOperationWithDocId(doc.getBucket(), doc.getTimestamp(), doc.getDoc()->getId());
     }
     UpdateOperation update(uint32_t userId) {
         const test::Document &doc = userDocs().getDocs(userId)[0];
@@ -234,7 +234,7 @@ TEST_F("require that handlePut() sends to 2 feed views", Fixture)
 
 TEST_F("require that prepareRemove() sends to removed view", Fixture)
 {
-    RemoveOperation op = f.remove(1);
+    RemoveOperationWithDocId op = f.remove(1);
     f._view.prepareRemove(op);
     EXPECT_EQUAL(0u, f._ready._view->_prepareRemove);
     EXPECT_EQUAL(1u, f._removed._view->_prepareRemove);
@@ -246,7 +246,7 @@ TEST_F("require that prepareRemove() sends to removed view", Fixture)
 TEST_F("require that prepareRemove() can fill previous dbdId", Fixture)
 {
     f._ready.insertDocs(f.userDocs(1));
-    RemoveOperation op = f.remove(1);
+    RemoveOperationWithDocId op = f.remove(1);
     f._view.prepareRemove(op);
     EXPECT_EQUAL(1u, op.getPrevLid());
     EXPECT_EQUAL(READY, op.getPrevSubDbId());
@@ -257,7 +257,7 @@ TEST_F("require that prepareRemove() can fill previous dbdId", Fixture)
 
 TEST_F("require that handleRemove() sends op with valid dbdId to 1 feed view", Fixture)
 {
-    RemoveOperation op = f.remove(1);
+    RemoveOperationWithDocId op = f.remove(1);
     op.setDbDocumentId(DbDocumentId(REMOVED, 1));
     f._view.handleRemove(FeedToken(), op);
     EXPECT_EQUAL(0u, f._ready._view->_handleRemove);
@@ -268,7 +268,7 @@ TEST_F("require that handleRemove() sends op with valid dbdId to 1 feed view", F
 
 TEST_F("require that handleRemove() sends op with valid dbdId to 2 feed views", Fixture)
 {
-    RemoveOperation op = f.remove(1);
+    RemoveOperationWithDocId op = f.remove(1);
     op.setDbDocumentId(DbDocumentId(REMOVED, 1));
     op.setPrevDbDocumentId(DbDocumentId(READY, 1));
     f._view.handleRemove(FeedToken(), op);
@@ -280,7 +280,7 @@ TEST_F("require that handleRemove() sends op with valid dbdId to 2 feed views",
 
 TEST_F("require that handleRemove() sends op with invalid dbdId to prev view", Fixture)
 {
-    RemoveOperation op = f.remove(1);
+    RemoveOperationWithDocId op = f.remove(1);
     // can be used in the case where removed feed view does not remember removes.
     op.setPrevDbDocumentId(DbDocumentId(READY, 1));
     f._view.handleRemove(FeedToken(), op);
diff --git a/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp b/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp
index 3c3ed33b8ea..89930a6b1a3 100644
--- a/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp
+++ b/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp
@@ -79,7 +79,7 @@ struct ViewPtrs
     ~ViewPtrs();
 };
 
-ViewPtrs::~ViewPtrs() {}
+ViewPtrs::~ViewPtrs() = default;
 
 struct ViewSet
 {
@@ -208,10 +208,10 @@ Fixture::initViewSet(ViewSet &views)
     views._lidReuseDelayer = std::make_unique<documentmetastore::LidReuseDelayer>(views._writeService, metaStore->get());
     IndexSearchable::SP indexSearchable;
     MatchView::SP matchView(new MatchView(matchers, indexSearchable, attrMgr, sesMgr, metaStore, views._docIdLimit));
-    views.searchView.set(make_shared<SearchView>
+    views.searchView.set(SearchView::create
                                  (summaryMgr->createSummarySetup(SummaryConfig(), SummarymapConfig(),
                                                                  JuniperrcConfig(), views.repo, attrMgr),
-                                  matchView));
+                                  std::move(matchView)));
     views.feedView.set(
             make_shared<SearchableFeedView>(StoreOnlyFeedView::Context(summaryAdapter,
                             schema,
diff --git a/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp b/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp
index 7139eb0d82d..2785b744265 100644
--- a/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp
+++ b/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp
@@ -544,7 +544,7 @@ requireThatAttributeManagerCanBeReconfigured(Fixture &f)
     f.basicReconfig(10);
     std::vector<AttributeGuard> attributes;
     f.getAttributeManager()->getAttributeList(attributes);
-    assertAttributes2(attributes);
+    TEST_DO(assertAttributes2(attributes));
 }
 
 TEST_F("require that attribute manager can be reconfigured", FastAccessFixture)
@@ -743,13 +743,13 @@ struct DocumentHandler
         op.setSerialNum(serialNum);
         return op;
     }
-    RemoveOperation createRemove(const DocumentId &docId, Timestamp timestamp, SerialNum serialNum)
+    RemoveOperationWithDocId createRemove(const DocumentId &docId, Timestamp timestamp, SerialNum serialNum)
     {
         const document::GlobalId &gid = docId.getGlobalId();
         BucketId bucket = gid.convertToBucketId();
         bucket.setUsedBits(BUCKET_USED_BITS);
         bucket = bucket.stripUnused();
-        RemoveOperation op(bucket, timestamp, docId);
+        RemoveOperationWithDocId op(bucket, timestamp, docId);
         op.setSerialNum(serialNum);
         return op;
     }
@@ -791,13 +791,13 @@ assertAttribute(const AttributeGuard &attr, const vespalib::string &name, uint32
 void
 assertAttribute1(const AttributeGuard &attr, SerialNum createSerialNum, SerialNum lastSerialNum)
 {
-    assertAttribute(attr, "attr1", 3, 22, 44, createSerialNum, lastSerialNum);
+    TEST_DO(assertAttribute(attr, "attr1", 3, 22, 44, createSerialNum, lastSerialNum));
 }
 
 void
 assertAttribute2(const AttributeGuard &attr, SerialNum createSerialNum, SerialNum lastSerialNum)
 {
-    assertAttribute(attr, "attr2", 3, 33, 55, createSerialNum, lastSerialNum);
+    TEST_DO(assertAttribute(attr, "attr2", 3, 33, 55, createSerialNum, lastSerialNum));
 }
 
 TEST_F("require that fast-access attributes are populated during feed", FastAccessOnlyFixture)
@@ -833,8 +833,8 @@ requireThatAttributesArePopulatedDuringReprocessing(FixtureType &f)
         std::vector<AttributeGuard> attrs;
         f.getAttributeManager()->getAttributeList(attrs);
         EXPECT_EQUAL(2u, attrs.size());
-        assertAttribute1(attrs[0], CFG_SERIAL, 40);
-        assertAttribute2(attrs[1], 40, 40);
+        TEST_DO(assertAttribute1(attrs[0], CFG_SERIAL, 40));
+        TEST_DO(assertAttribute2(attrs[1], 40, 40));
     }
 }
 
@@ -883,7 +883,7 @@ TEST_F("require that lid allocation uses lowest free lid", StoreOnlyFixture)
     DocumentHandler<StoreOnlyFixture> handler(f);
     Document::UP doc;
     PutOperation putOp;
-    RemoveOperation rmOp;
+    RemoveOperationWithDocId rmOp;
     MoveOperation moveOp;
 
     doc = handler.createEmptyDoc(1);
diff --git a/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp b/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp
index 30fed6fa49e..796ec4436fe 100644
--- a/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp
+++ b/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp
@@ -55,7 +55,6 @@ using storage::spi::RemoveResult;
 using storage::spi::Result;
 using storage::spi::Timestamp;
 using storage::spi::UpdateResult;
-using vespalib::BlockingThreadStackExecutor;
 using vespalib::ThreadStackExecutor;
 using vespalib::ThreadStackExecutorBase;
 using vespalib::makeClosure;
@@ -194,7 +193,7 @@ struct MyFeedView : public test::DummyFeedView {
     MyFeedView(const std::shared_ptr<const DocumentTypeRepo> &dtr,
                const DocTypeName &docTypeName);
     ~MyFeedView() override;
-    void resetPutLatch(uint32_t count) { putLatch.reset(new vespalib::CountDownLatch(count)); }
+    void resetPutLatch(uint32_t count) { putLatch = std::make_unique<vespalib::CountDownLatch>(count); }
     void preparePut(PutOperation &op) override {
         prepareDocumentOperation(op, op.getDocument()->getId().getGlobalId());
     }
@@ -533,7 +532,7 @@ TEST_F("require that heartBeat calls FeedView's heartBeat",
 TEST_F("require that outdated remove is ignored", FeedHandlerFixture)
 {
     DocumentContext doc_context("id:ns:searchdocument::foo", *f.schema.builder);
-    FeedOperation::UP op(new RemoveOperation(doc_context.bucketId, Timestamp(10), doc_context.doc->getId()));
+    auto op = std::make_unique<RemoveOperationWithDocId>(doc_context.bucketId, Timestamp(10), doc_context.doc->getId());
     static_cast<DocumentOperation &>(*op).setPrevDbDocumentId(DbDocumentId(4));
     static_cast<DocumentOperation &>(*op).setPrevTimestamp(Timestamp(10000));
     FeedTokenContext token_context;
@@ -545,8 +544,7 @@ TEST_F("require that outdated remove is ignored", FeedHandlerFixture)
 TEST_F("require that outdated put is ignored", FeedHandlerFixture)
 {
     DocumentContext doc_context("id:ns:searchdocument::foo", *f.schema.builder);
-    FeedOperation::UP op(new PutOperation(doc_context.bucketId,
-                                          Timestamp(10), doc_context.doc));
+    auto op =std::make_unique<PutOperation>(doc_context.bucketId, Timestamp(10), doc_context.doc);
     static_cast<DocumentOperation &>(*op).setPrevTimestamp(Timestamp(10000));
     FeedTokenContext token_context;
     f.handler.performOperation(std::move(token_context.token), std::move(op));
@@ -557,7 +555,7 @@ TEST_F("require that outdated put is ignored", FeedHandlerFixture)
 void
 addLidToRemove(RemoveDocumentsOperation &op)
 {
-    LidVectorContext::SP lids(new LidVectorContext(42));
+    auto lids = std::make_shared<LidVectorContext>(42);
     lids->addLid(4);
     op.setLidsToRemove(0, lids);
 }
@@ -626,7 +624,7 @@ TEST_F("require that flush cannot unprune", FeedHandlerFixture)
 TEST_F("require that remove of unknown document with known data type stores remove", FeedHandlerFixture)
 {
     DocumentContext doc_context("id:test:searchdocument::foo", *f.schema.builder);
-    FeedOperation::UP op(new RemoveOperation(doc_context.bucketId, Timestamp(10), doc_context.doc->getId()));
+    auto op = std::make_unique<RemoveOperationWithDocId>(doc_context.bucketId, Timestamp(10), doc_context.doc->getId());
     FeedTokenContext token_context;
     f.handler.performOperation(std::move(token_context.token), std::move(op));
     EXPECT_EQUAL(1, f.feedView.remove_count);
@@ -636,7 +634,7 @@ TEST_F("require that remove of unknown document with known data type stores remo
 TEST_F("require that partial update for non-existing document is tagged as such", FeedHandlerFixture)
 {
     UpdateContext upCtx("id:test:searchdocument::foo", *f.schema.builder);
-    FeedOperation::UP op(new UpdateOperation(upCtx.bucketId, Timestamp(10), upCtx.update));
+    auto  op = std::make_unique<UpdateOperation>(upCtx.bucketId, Timestamp(10), upCtx.update);
     FeedTokenContext token_context;
     f.handler.performOperation(std::move(token_context.token), std::move(op));
     const UpdateResult *result = static_cast<const UpdateResult *>(token_context.getResult());
@@ -654,7 +652,7 @@ TEST_F("require that partial update for non-existing document is created if spec
     UpdateContext upCtx("id:test:searchdocument::foo", *f.schema.builder);
     upCtx.update->setCreateIfNonExistent(true);
     f.feedView.metaStore.insert(upCtx.update->getId().getGlobalId(), MyDocumentMetaStore::Entry(5, 5, Timestamp(10)));
-    FeedOperation::UP op(new UpdateOperation(upCtx.bucketId, Timestamp(10), upCtx.update));
+    auto op = std::make_unique<UpdateOperation>(upCtx.bucketId, Timestamp(10), upCtx.update);
     FeedTokenContext token_context;
     f.handler.performOperation(std::move(token_context.token), std::move(op));
     const UpdateResult *result = static_cast<const UpdateResult *>(token_context.getResult());
@@ -675,7 +673,7 @@ TEST_F("require that put is rejected if resource limit is reached", FeedHandlerF
     f.writeFilter._message = "Attribute resource limit reached";
 
     DocumentContext docCtx("id:test:searchdocument::foo", *f.schema.builder);
-    FeedOperation::UP op = std::make_unique<PutOperation>(docCtx.bucketId, Timestamp(10), docCtx.doc);
+    auto op = std::make_unique<PutOperation>(docCtx.bucketId, Timestamp(10), docCtx.doc);
     FeedTokenContext token;
     f.handler.performOperation(std::move(token.token), std::move(op));
     EXPECT_EQUAL(0, f.feedView.put_count);
@@ -690,7 +688,7 @@ TEST_F("require that update is rejected if resource limit is reached", FeedHandl
     f.writeFilter._message = "Attribute resource limit reached";
 
     UpdateContext updCtx("id:test:searchdocument::foo", *f.schema.builder);
-    FeedOperation::UP op = std::make_unique<UpdateOperation>(updCtx.bucketId, Timestamp(10), updCtx.update);
+    auto op = std::make_unique<UpdateOperation>(updCtx.bucketId, Timestamp(10), updCtx.update);
     FeedTokenContext token;
     f.handler.performOperation(std::move(token.token), std::move(op));
     EXPECT_EQUAL(0, f.feedView.update_count);
@@ -706,7 +704,7 @@ TEST_F("require that remove is NOT rejected if resource limit is reached", FeedH
     f.writeFilter._message = "Attribute resource limit reached";
 
     DocumentContext docCtx("id:test:searchdocument::foo", *f.schema.builder);
-    FeedOperation::UP op = std::make_unique<RemoveOperation>(docCtx.bucketId, Timestamp(10), docCtx.doc->getId());
+    auto op = std::make_unique<RemoveOperationWithDocId>(docCtx.bucketId, Timestamp(10), docCtx.doc->getId());
     FeedTokenContext token;
     f.handler.performOperation(std::move(token.token), std::move(op));
     EXPECT_EQUAL(1, f.feedView.remove_count);
@@ -727,7 +725,7 @@ checkUpdate(FeedHandlerFixture &f, SchemaContext &schemaContext,
     } else {
         updCtx.update->setCreateIfNonExistent(true);
     }
-    FeedOperation::UP op = std::make_unique<UpdateOperation>(updCtx.bucketId, Timestamp(10), updCtx.update);
+    auto op = std::make_unique<UpdateOperation>(updCtx.bucketId, Timestamp(10), updCtx.update);
     FeedTokenContext token;
     f.handler.performOperation(std::move(token.token), std::move(op));
     EXPECT_TRUE(dynamic_cast<const UpdateResult *>(token.getResult()));
diff --git a/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp b/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp
index f380ce03152..ddf45d6a509 100644
--- a/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp
+++ b/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp
@@ -620,7 +620,7 @@ struct FixtureBase
 
     void removeAndWait(const DocumentContext &docCtx) {
         FeedTokenContext token(_tracer);
-        RemoveOperation op(docCtx.bid, docCtx.ts, docCtx.doc->getId());
+        RemoveOperationWithDocId op(docCtx.bid, docCtx.ts, docCtx.doc->getId());
         runInMaster([&] () { performRemove(token.ft, op); });
     }
 
@@ -692,15 +692,15 @@ struct FixtureBase
 FixtureBase::FixtureBase(vespalib::duration visibilityDelay)
     : _tracer(),
       sc(),
-      iw(new MyIndexWriter(_tracer)),
-      sa(new MySummaryAdapter(*sc._builder->getDocumentTypeRepo())),
-      aw(new MyAttributeWriter(_tracer)),
+      iw(std::make_shared<MyIndexWriter>(_tracer)),
+      sa(std::make_shared<MySummaryAdapter>(*sc._builder->getDocumentTypeRepo())),
+      aw(std::make_shared<MyAttributeWriter>(_tracer)),
       miw(static_cast<MyIndexWriter&>(*iw)),
       msa(static_cast<MySummaryAdapter&>(*sa)),
       maw(static_cast<MyAttributeWriter&>(*aw)),
       _docIdLimit(0u),
-      _dmscReal(new DocumentMetaStoreContext(std::make_shared<BucketDBOwner>())),
-      _dmsc(new test::DocumentMetaStoreContextObserver(*_dmscReal)),
+      _dmscReal(std::make_shared<DocumentMetaStoreContext>(std::make_shared<BucketDBOwner>())),
+      _dmsc(std::make_shared<test::DocumentMetaStoreContextObserver>(*_dmscReal)),
       pc(sc._builder->getDocumentType().getName(), "fileconfig_test"),
       _sharedExecutor(1, 0x10000),
       _writeServiceReal(_sharedExecutor),
diff --git a/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_compaction_test.cpp b/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_compaction_test.cpp
index 50d4106282c..64299c70588 100644
--- a/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_compaction_test.cpp
+++ b/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_compaction_test.cpp
@@ -1,6 +1,4 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-#include <vespa/log/log.h>
-LOG_SETUP("lid_space_compaction_test");
 
 #include <vespa/searchcore/proton/server/i_disk_mem_usage_notifier.h>
 #include <vespa/searchcore/proton/server/i_lid_space_compaction_handler.h>
@@ -15,6 +13,9 @@ LOG_SETUP("lid_space_compaction_test");
 #include <vespa/searchlib/index/docbuilder.h>
 #include <vespa/vespalib/gtest/gtest.h>
 
+#include <vespa/log/log.h>
+LOG_SETUP("lid_space_compaction_test");
+
 using namespace document;
 using namespace proton;
 using namespace search::index;
diff --git a/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp b/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp
index d4aebd0b8a7..7e2e258476f 100644
--- a/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp
+++ b/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp
@@ -110,7 +110,7 @@ public:
     MaintenanceDocumentSubDB getSubDB();
     void handlePruneRemovedDocuments(const PruneRemovedDocumentsOperation &op);
     void handlePut(PutOperation &op);
-    void handleRemove(RemoveOperation &op);
+    void handleRemove(RemoveOperationWithDocId &op);
     void prepareMove(MoveOperation &op);
     void handleMove(const MoveOperation &op);
     uint32_t getNumUsedLids() const;
@@ -565,11 +565,11 @@ MyDocumentSubDB::handlePut(PutOperation &op)
 
 
 void
-MyDocumentSubDB::handleRemove(RemoveOperation &op)
+MyDocumentSubDB::handleRemove(RemoveOperationWithDocId &op)
 {
     const SerialNum serialNum = op.getSerialNum();
     const DocumentId &docId = op.getDocumentId();
-    const document::GlobalId &gid = docId.getGlobalId();
+    const document::GlobalId &gid = op.getGlobalId();
     bool needCommit = false;
 
     if (op.getValidDbdId(_subDBId)) {
@@ -584,7 +584,7 @@ MyDocumentSubDB::handleRemove(RemoveOperation &op)
         assert(op.getLid() == putRes._lid);
         const document::DocumentType *docType =
             _repo->getDocumentType(_docTypeName.getName());
-        Document::UP doc(new Document(*docType, docId));
+        auto doc = std::make_unique<Document>(*docType, docId);
         doc->setRepo(*_repo);
         _docs[op.getLid()] = std::move(doc);
         needCommit = true;
@@ -948,7 +948,7 @@ MaintenanceControllerFixture::removeDocs(const test::UserDocuments &docs,
         const test::BucketDocuments &bucketDocs = itr->second;
         for (size_t i = 0; i < bucketDocs.getDocs().size(); ++i) {
             const test::Document &testDoc = bucketDocs.getDocs()[i];
-            RemoveOperation op(testDoc.getBucket(), timestamp, testDoc.getDoc()->getId());
+            RemoveOperationWithDocId op(testDoc.getBucket(), timestamp, testDoc.getDoc()->getId());
             op.setDbDocumentId(DbDocumentId(_removed.getSubDBId(), testDoc.getLid()));
             _fh.storeOperation(op, std::make_shared<search::IgnoreCallback>());
             _removed.handleRemove(op);
diff --git a/searchcore/src/tests/proton/documentdb/threading_service_config/threading_service_config_test.cpp b/searchcore/src/tests/proton/documentdb/threading_service_config/threading_service_config_test.cpp
index 340619f09bd..37ac5f0d65c 100644
--- a/searchcore/src/tests/proton/documentdb/threading_service_config/threading_service_config_test.cpp
+++ b/searchcore/src/tests/proton/documentdb/threading_service_config/threading_service_config_test.cpp
@@ -1,12 +1,13 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-#include <vespa/log/log.h>
-LOG_SETUP("threading_service_config_test");
 
 #include <vespa/searchcore/config/config-proton.h>
 #include <vespa/searchcore/proton/common/hw_info.h>
 #include <vespa/searchcore/proton/server/threading_service_config.h>
 #include <vespa/vespalib/testkit/testapp.h>
 
+#include <vespa/log/log.h>
+LOG_SETUP("threading_service_config_test");
+
 using namespace proton;
 using ProtonConfig = vespa::config::search::core::ProtonConfig;
 using ProtonConfigBuilder = vespa::config::search::core::ProtonConfigBuilder;
@@ -42,6 +43,7 @@ TEST_F("require that indexing threads are set based on cpu cores and feeding con
     TEST_DO(f.assertIndexingThreads(3, 18));
     TEST_DO(f.assertIndexingThreads(4, 19));
     TEST_DO(f.assertIndexingThreads(4, 24));
+    TEST_DO(f.assertIndexingThreads(4, 64)); // Ensure it is capped at 4
 }
 
 TEST_F("require that indexing threads is always >= 1", Fixture(0))
diff --git a/searchcore/src/tests/proton/documentmetastore/lidreusedelayer/CMakeLists.txt b/searchcore/src/tests/proton/documentmetastore/lidreusedelayer/CMakeLists.txt
index 0d226268d1a..49f929e0127 100644
--- a/searchcore/src/tests/proton/documentmetastore/lidreusedelayer/CMakeLists.txt
+++ b/searchcore/src/tests/proton/documentmetastore/lidreusedelayer/CMakeLists.txt
@@ -5,5 +5,6 @@ vespa_add_executable(searchcore_lidreusedelayer_test_app TEST
     DEPENDS
     searchcore_server
     searchcore_documentmetastore
+    searchcore_test
 )
 vespa_add_test(NAME searchcore_lidreusedelayer_test_app COMMAND searchcore_lidreusedelayer_test_app)
diff --git a/searchcore/src/tests/proton/documentmetastore/lidreusedelayer/lidreusedelayer_test.cpp b/searchcore/src/tests/proton/documentmetastore/lidreusedelayer/lidreusedelayer_test.cpp
index d305751f7c2..25668f56753 100644
--- a/searchcore/src/tests/proton/documentmetastore/lidreusedelayer/lidreusedelayer_test.cpp
+++ b/searchcore/src/tests/proton/documentmetastore/lidreusedelayer/lidreusedelayer_test.cpp
@@ -1,6 +1,5 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-#include <vespa/log/log.h>
-LOG_SETUP("lidreusedelayer_test");
+
 #include <vespa/vespalib/testkit/testapp.h>
 #include <vespa/searchcore/proton/documentmetastore/i_store.h>
 #include <vespa/searchcore/proton/documentmetastore/lidreusedelayer.h>
@@ -9,12 +8,14 @@ LOG_SETUP("lidreusedelayer_test");
 #include <vespa/searchcore/proton/test/threading_service_observer.h>
 #include <vespa/vespalib/util/lambdatask.h>
 
+#include <vespa/log/log.h>
+LOG_SETUP("lidreusedelayer_test");
+
 using vespalib::makeLambdaTask;
 
 namespace proton {
 
-namespace
-{
+namespace {
 
 bool
 assertThreadObserver(uint32_t masterExecuteCnt,
@@ -55,69 +56,52 @@ public:
     {
     }
 
-    virtual ~MyMetaStore() { }
+    ~MyMetaStore() override = default;
 
-    virtual Result inspectExisting(const GlobalId &) const override
-    {
+    Result inspectExisting(const GlobalId &) const override {
         return Result();
     }
 
-    virtual Result inspect(const GlobalId &) override
-    {
+    Result inspect(const GlobalId &) override {
         return Result();
     }
 
-    virtual Result put(const GlobalId &, const BucketId &, const Timestamp &,
-                       uint32_t, DocId) override
-    {
+    Result put(const GlobalId &, const BucketId &, const Timestamp &, uint32_t, DocId) override {
         return Result();
     }
 
-    virtual bool updateMetaData(DocId, const BucketId &,
-                                const Timestamp &) override
-    {
+    bool updateMetaData(DocId, const BucketId &, const Timestamp &) override {
         return true;
     }
 
-    virtual bool remove(DocId) override
-    {
+    bool remove(DocId) override {
         return true;
     }
 
-    virtual void removeComplete(DocId) override
-    {
+    void removeComplete(DocId) override {
         ++_removeCompleteCount;
         ++_removeCompleteLids;
     }
 
-    virtual void move(DocId, DocId) override
-    {
+    void move(DocId, DocId) override {
     }
 
-    virtual bool validLid(DocId) const override
-    {
+    bool validLid(DocId) const override {
         return true;
     }
 
-    virtual void removeBatch(const std::vector<DocId> &,
-                             const DocId) override
-    {
-    }
+    void removeBatch(const std::vector<DocId> &, const DocId) override {}
 
-    virtual void
-    removeBatchComplete(const std::vector<DocId> &lidsToRemove) override
-    {
+    void removeBatchComplete(const std::vector<DocId> &lidsToRemove) override{
         ++_removeBatchCompleteCount;
         _removeCompleteLids += lidsToRemove.size();
     }
 
-    virtual const RawDocumentMetaData &getRawMetaData(DocId) const override
-    {
+    const RawDocumentMetaData &getRawMetaData(DocId) const override {
         LOG_ABORT("should not be reached");
     }
 
-    virtual bool getFreeListActive() const override
-    {
+    bool getFreeListActive() const override {
         return _freeListActive;
     }
 
diff --git a/searchcore/src/tests/proton/feed_and_search/feed_and_search.cpp b/searchcore/src/tests/proton/feed_and_search/feed_and_search.cpp
index ca45108b698..7af9d8d816c 100644
--- a/searchcore/src/tests/proton/feed_and_search/feed_and_search.cpp
+++ b/searchcore/src/tests/proton/feed_and_search/feed_and_search.cpp
@@ -4,7 +4,7 @@
 #include <vespa/document/fieldvalue/document.h>
 #include <vespa/document/fieldvalue/fieldvalue.h>
 #include <vespa/searchlib/common/documentsummary.h>
-#include <vespa/searchlib/common/sequencedtaskexecutor.h>
+#include <vespa/vespalib/util/sequencedtaskexecutor.h>
 #include <vespa/searchlib/diskindex/diskindex.h>
 #include <vespa/searchlib/diskindex/fusion.h>
 #include <vespa/searchlib/diskindex/indexbuilder.h>
@@ -149,9 +149,9 @@ void Test::testSearch(Searchable &source,
 void Test::requireThatMemoryIndexCanBeDumpedAndSearched() {
     Schema schema = getSchema();
     vespalib::ThreadStackExecutor sharedExecutor(2, 0x10000);
-    search::SequencedTaskExecutor indexFieldInverter(2);
-    search::SequencedTaskExecutor indexFieldWriter(2);
-    MemoryIndex memory_index(schema, MockFieldLengthInspector(), indexFieldInverter, indexFieldWriter);
+    auto indexFieldInverter = vespalib::SequencedTaskExecutor::create(2);
+    auto indexFieldWriter = vespalib::SequencedTaskExecutor::create(2);
+    MemoryIndex memory_index(schema, MockFieldLengthInspector(), *indexFieldInverter, *indexFieldWriter);
     DocBuilder doc_builder(schema);
 
     Document::UP doc = buildDocument(doc_builder, doc_id1, word1);
@@ -160,7 +160,7 @@ void Test::requireThatMemoryIndexCanBeDumpedAndSearched() {
     doc = buildDocument(doc_builder, doc_id2, word2);
     memory_index.insertDocument(doc_id2, *doc.get());
     memory_index.commit(std::shared_ptr<search::IDestructorCallback>());
-    indexFieldWriter.sync();
+    indexFieldWriter->sync();
 
     testSearch(memory_index, word1, doc_id1);
     testSearch(memory_index, word2, doc_id2);
diff --git a/searchcore/src/tests/proton/feedoperation/feedoperation_test.cpp b/searchcore/src/tests/proton/feedoperation/feedoperation_test.cpp
index de2cd3c624f..5fffd70f11d 100644
--- a/searchcore/src/tests/proton/feedoperation/feedoperation_test.cpp
+++ b/searchcore/src/tests/proton/feedoperation/feedoperation_test.cpp
@@ -194,10 +194,10 @@ TEST("require that toString() on derived classes are meaningful")
 
     EXPECT_EQUAL("Remove(id::::, BucketId(0x0000000000000000), timestamp=0, dbdId=(subDbId=0, lid=0), "
                  "prevDbdId=(subDbId=0, lid=0), prevMarkedAsRemoved=false, prevTimestamp=0, serialNum=0)",
-                 RemoveOperation().toString());
+                 RemoveOperationWithDocId().toString());
     EXPECT_EQUAL("Remove(id:ns:foo:::bar, BucketId(0x000000000000002a), timestamp=10, dbdId=(subDbId=0, lid=0), "
                  "prevDbdId=(subDbId=0, lid=0), prevMarkedAsRemoved=false, prevTimestamp=0, serialNum=0)",
-                 RemoveOperation(bucket_id1, timestamp, doc_id).toString());
+                 RemoveOperationWithDocId(bucket_id1, timestamp, doc_id).toString());
 
     EXPECT_EQUAL("SplitBucket("
                  "source=BucketId(0x0000000000000000), "
@@ -311,7 +311,7 @@ TEST_F("require that we can serialize and deserialize remove operations", Fixtur
     uint32_t expSerializedDocSize = getDocIdSize(docId);
     EXPECT_NOT_EQUAL(0u, expSerializedDocSize);
     {
-        RemoveOperation op(bucket, Timestamp(10), docId);
+        RemoveOperationWithDocId op(bucket, Timestamp(10), docId);
         op.setDbDocumentId({1, 2});
         op.setPrevDbDocumentId({3, 4});
         EXPECT_EQUAL(0u, op.getSerializedDocSize());
@@ -319,13 +319,43 @@ TEST_F("require that we can serialize and deserialize remove operations", Fixtur
         EXPECT_EQUAL(expSerializedDocSize, op.getSerializedDocSize());
     }
     {
-        RemoveOperation op;
+        RemoveOperationWithDocId op;
         op.deserialize(stream, *f._repo);
         EXPECT_EQUAL(docId, op.getDocumentId());
         TEST_DO(assertDocumentOperation(op, bucket, expSerializedDocSize));
     }
 }
 
+TEST_F("require that we can serialize and deserialize remove by gid operations", Fixture)
+{
+    vespalib::nbostream stream;
+    GlobalId gid = docId.getGlobalId();
+    BucketId bucket(toBucket(gid));
+    uint32_t expSerializedDocSize = 25;
+    vespalib::string expDocType = "testdoc_type";
+    EXPECT_NOT_EQUAL(0u, expSerializedDocSize);
+    {
+        RemoveOperationWithGid op(bucket, Timestamp(10), gid, expDocType);
+        op.setPrevDbDocumentId({3, 4});
+        EXPECT_EQUAL(0u, op.getSerializedDocSize());
+        op.serialize(stream);
+        EXPECT_EQUAL(expSerializedDocSize, op.getSerializedDocSize());
+    }
+    {
+        RemoveOperationWithGid op;
+        op.deserialize(stream, *f._repo);
+        EXPECT_EQUAL(gid, op.getGlobalId());
+        EXPECT_EQUAL(expDocType, op.getDocType());
+        EXPECT_EQUAL(bucket, op.getBucketId());
+        EXPECT_EQUAL(10u, op.getTimestamp().getValue());
+        EXPECT_EQUAL(expSerializedDocSize, op.getSerializedDocSize());
+        EXPECT_FALSE( op.getValidDbdId());
+        EXPECT_EQUAL(3u, op.getPrevSubDbId());
+        EXPECT_EQUAL(4u, op.getPrevLid());
+        EXPECT_TRUE(stream.empty());
+    }
+}
+
 }  // namespace
 
 TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/searchcore/src/tests/proton/index/fusionrunner_test.cpp b/searchcore/src/tests/proton/index/fusionrunner_test.cpp
index 975061f9d88..6831dd60bd5 100644
--- a/searchcore/src/tests/proton/index/fusionrunner_test.cpp
+++ b/searchcore/src/tests/proton/index/fusionrunner_test.cpp
@@ -4,7 +4,7 @@
 #include <vespa/searchcore/proton/index/indexmanager.h>
 #include <vespa/searchcore/proton/server/executorthreadingservice.h>
 #include <vespa/searchcorespi/index/fusionrunner.h>
-#include <vespa/searchlib/common/isequencedtaskexecutor.h>
+#include <vespa/vespalib/util/isequencedtaskexecutor.h>
 #include <vespa/searchlib/diskindex/diskindex.h>
 #include <vespa/searchlib/diskindex/indexbuilder.h>
 #include <vespa/searchlib/fef/matchdatalayout.h>
diff --git a/searchcore/src/tests/proton/index/indexmanager_test.cpp b/searchcore/src/tests/proton/index/indexmanager_test.cpp
index 7542ef24c52..5b6f7ca9a30 100644
--- a/searchcore/src/tests/proton/index/indexmanager_test.cpp
+++ b/searchcore/src/tests/proton/index/indexmanager_test.cpp
@@ -7,7 +7,7 @@
 #include <vespa/searchcorespi/index/indexcollection.h>
 #include <vespa/searchcorespi/index/indexflushtarget.h>
 #include <vespa/searchcorespi/index/indexfusiontarget.h>
-#include <vespa/searchlib/common/sequencedtaskexecutor.h>
+#include <vespa/vespalib/util/sequencedtaskexecutor.h>
 #include <vespa/searchlib/common/serialnum.h>
 #include <vespa/searchlib/index/docbuilder.h>
 #include <vespa/searchlib/index/dummyfileheadercontext.h>
@@ -31,7 +31,7 @@ using document::Document;
 using document::FieldValue;
 using proton::index::IndexConfig;
 using proton::index::IndexManager;
-using search::SequencedTaskExecutor;
+using vespalib::SequencedTaskExecutor;
 using search::SerialNum;
 using search::TuneFileAttributes;
 using search::TuneFileIndexManager;
@@ -48,7 +48,6 @@ using search::memoryindex::FieldIndexCollection;
 using search::queryeval::Source;
 using std::set;
 using std::string;
-using vespalib::BlockingThreadStackExecutor;
 using vespalib::ThreadStackExecutor;
 using vespalib::makeLambdaTask;
 using std::chrono::duration_cast;
@@ -379,10 +378,9 @@ TEST_F(IndexManagerTest, require_that_flush_stats_are_calculated)
 {
     Schema schema(getSchema());
     FieldIndexCollection fic(schema, MockFieldLengthInspector());
-    SequencedTaskExecutor invertThreads(2);
-    SequencedTaskExecutor pushThreads(2);
-    search::memoryindex::DocumentInverter inverter(schema, invertThreads,
-                                                   pushThreads, fic);
+    auto invertThreads = SequencedTaskExecutor::create(2);
+    auto pushThreads = SequencedTaskExecutor::create(2);
+    search::memoryindex::DocumentInverter inverter(schema, *invertThreads, *pushThreads, fic);
 
     uint64_t fixed_index_size = fic.getMemoryUsage().allocatedBytes();
     uint64_t index_size = fic.getMemoryUsage().allocatedBytes() - fixed_index_size;
@@ -395,9 +393,9 @@ TEST_F(IndexManagerTest, require_that_flush_stats_are_calculated)
 
     Document::UP doc = addDocument(docid);
     inverter.invertDocument(docid, *doc);
-    invertThreads.sync();
+    invertThreads->sync();
     inverter.pushDocuments(std::shared_ptr<search::IDestructorCallback>());
-    pushThreads.sync();
+    pushThreads->sync();
     index_size = fic.getMemoryUsage().allocatedBytes() - fixed_index_size;
 
     /// Must account for both docid 0 being reserved and the extra after.
@@ -414,9 +412,9 @@ TEST_F(IndexManagerTest, require_that_flush_stats_are_calculated)
     inverter.invertDocument(docid + 10, *doc);
     doc = addDocument(docid + 100);
     inverter.invertDocument(docid + 100, *doc);
-    invertThreads.sync();
+    invertThreads->sync();
     inverter.pushDocuments(std::shared_ptr<search::IDestructorCallback>());
-    pushThreads.sync();
+    pushThreads->sync();
     index_size = fic.getMemoryUsage().allocatedBytes() - fixed_index_size;
     /// Must account for both docid 0 being reserved and the extra after.
     selector_size = (docid + 100 + 1) * sizeof(Source);
diff --git a/searchcore/src/tests/proton/matchengine/matchengine.cpp b/searchcore/src/tests/proton/matchengine/matchengine.cpp
index 9ae938981a4..fd48fffcb5c 100644
--- a/searchcore/src/tests/proton/matchengine/matchengine.cpp
+++ b/searchcore/src/tests/proton/matchengine/matchengine.cpp
@@ -18,18 +18,17 @@ class MySearchHandler : public ISearchHandler {
     std::string _reply;
 public:
     MySearchHandler(size_t numHits = 0) :
-        _numHits(numHits), _name("my"), _reply("myreply") {}
-    virtual DocsumReply::UP getDocsums(const DocsumRequest &) override {
-        return DocsumReply::UP(new DocsumReply);
+        _numHits(numHits), _name("my"), _reply("myreply")
+    {}
+    DocsumReply::UP getDocsums(const DocsumRequest &) override {
+        return std::make_unique<DocsumReply>();
     }
 
-    virtual search::engine::SearchReply::UP match(
-            const ISearchHandler::SP &,
-            const search::engine::SearchRequest &,
-            vespalib::ThreadBundle &) const override {
-        SearchReply::UP retval(new SearchReply);
+    SearchReply::UP match(const SearchRequest &, vespalib::ThreadBundle &) const override
+    {
+        auto retval = std::make_unique<SearchReply>();
         for (size_t i = 0; i < _numHits; ++i) {
-            retval->hits.push_back(SearchReply::Hit());
+            retval->hits.emplace_back();
         }
         return retval;
     }
@@ -43,7 +42,7 @@ private:
 
 public:
     LocalSearchClient();
-    ~LocalSearchClient();
+    ~LocalSearchClient() override;
     void searchDone(SearchReply::UP reply) override {
         std::lock_guard<std::mutex> guard(_lock);
         _reply = std::move(reply);
@@ -62,8 +61,8 @@ public:
     }
 };
 
-LocalSearchClient::LocalSearchClient() {}
-LocalSearchClient::~LocalSearchClient() {}
+LocalSearchClient::LocalSearchClient() = default;
+LocalSearchClient::~LocalSearchClient() = default;
 
 TEST("requireThatSearchesExecute")
 {
@@ -71,23 +70,23 @@ TEST("requireThatSearchesExecute")
     MatchEngine engine(numMatcherThreads, 1, 7);
     engine.setNodeUp(true);
 
-    MySearchHandler::SP handler(new MySearchHandler);
+    auto handler = std::make_shared<MySearchHandler>();
     DocTypeName dtnvfoo("foo");
     engine.putSearchHandler(dtnvfoo, handler);
 
     LocalSearchClient client;
     SearchRequest::Source request(new SearchRequest());
     SearchReply::UP reply = engine.search(std::move(request), client);
-    EXPECT_TRUE(reply.get() == NULL);
+    EXPECT_FALSE(reply);
 
     reply = client.getReply(10000);
-    EXPECT_TRUE(reply.get() != NULL);
+    EXPECT_TRUE(reply);
 }
 
 bool
 assertSearchReply(MatchEngine & engine, const std::string & searchDocType, size_t expHits)
 {
-    SearchRequest *request = new SearchRequest();
+    auto *request = new SearchRequest();
     request->propertiesMap.lookupCreate(search::MapNames::MATCH).add("documentdb.searchdoctype", searchDocType);
     LocalSearchClient client;
     engine.search(SearchRequest::Source(request), client);
@@ -99,9 +98,9 @@ TEST("requireThatCorrectHandlerIsUsed")
 {
     MatchEngine engine(1, 1, 7);
     engine.setNodeUp(true);
-    ISearchHandler::SP h1(new MySearchHandler(2));
-    ISearchHandler::SP h2(new MySearchHandler(4));
-    ISearchHandler::SP h3(new MySearchHandler(6));
+    auto h1 = std::make_shared<MySearchHandler>(2);
+    auto h2 = std::make_shared<MySearchHandler>(4);
+    auto h3 = std::make_shared<MySearchHandler>(6);
     DocTypeName dtnvfoo("foo");
     DocTypeName dtnvbar("bar");
     DocTypeName dtnvbaz("baz");
@@ -120,13 +119,12 @@ struct ObserveBundleMatchHandler : MySearchHandler {
     mutable size_t bundleSize;
     ObserveBundleMatchHandler() : bundleSize(0) {}
 
-    virtual search::engine::SearchReply::UP match(
-            const ISearchHandler::SP &,
+    search::engine::SearchReply::UP match(
             const search::engine::SearchRequest &,
             vespalib::ThreadBundle &threadBundle) const override
     {
         bundleSize = threadBundle.size();
-        return SearchReply::UP(new SearchReply);
+        return std::make_unique<SearchReply>();
     }
 };
 
@@ -135,7 +133,7 @@ TEST("requireThatBundlesAreUsed")
     MatchEngine engine(15, 5, 7);
     engine.setNodeUp(true);
 
-    ObserveBundleMatchHandler::SP handler(new ObserveBundleMatchHandler());
+    auto handler = std::make_shared<ObserveBundleMatchHandler>();
     DocTypeName dtnvfoo("foo");
     engine.putSearchHandler(dtnvfoo, handler);
 
@@ -151,20 +149,20 @@ TEST("requireThatHandlersCanBeRemoved")
 {
     MatchEngine engine(1, 1, 7);
     engine.setNodeUp(true);
-    ISearchHandler::SP h(new MySearchHandler(1));
+    auto h = std::make_shared<MySearchHandler>(1);
     DocTypeName docType("foo");
     engine.putSearchHandler(docType, h);
 
     ISearchHandler::SP r = engine.getSearchHandler(docType);
-    EXPECT_TRUE(r.get() != NULL);
+    EXPECT_TRUE(r);
     EXPECT_TRUE(h.get() == r.get());
 
     r = engine.removeSearchHandler(docType);
-    EXPECT_TRUE(r.get() != NULL);
+    EXPECT_TRUE(r);
     EXPECT_TRUE(h.get() == r.get());
 
     r = engine.getSearchHandler(docType);
-    EXPECT_TRUE(r.get() == NULL);
+    EXPECT_FALSE(r);
 }
 
 TEST("requireThatEmptySearchReplyIsReturnedWhenEngineIsClosed")
@@ -175,7 +173,7 @@ TEST("requireThatEmptySearchReplyIsReturnedWhenEngineIsClosed")
     LocalSearchClient client;
     SearchRequest::Source request(new SearchRequest());
     SearchReply::UP reply = engine.search(std::move(request), client);
-    EXPECT_TRUE(reply.get() != NULL);
+    EXPECT_TRUE(reply );
     EXPECT_EQUAL(0u, reply->hits.size());
     EXPECT_EQUAL(7u, reply->getDistributionKey());
 }
diff --git a/searchcore/src/tests/proton/matching/matching_stats_test.cpp b/searchcore/src/tests/proton/matching/matching_stats_test.cpp
index 48ab09ffcb2..607a644a302 100644
--- a/searchcore/src/tests/proton/matching/matching_stats_test.cpp
+++ b/searchcore/src/tests/proton/matching/matching_stats_test.cpp
@@ -48,43 +48,51 @@ TEST("requireThatAverageTimesAreRecorded") {
     EXPECT_APPROX(0.0, stats.groupingTimeAvg(), 0.00001);
     EXPECT_APPROX(0.0, stats.rerankTimeAvg(), 0.00001);
     EXPECT_APPROX(0.0, stats.queryCollateralTimeAvg(), 0.00001);
+    EXPECT_APPROX(0.0, stats.querySetupTimeAvg(), 0.00001);
     EXPECT_APPROX(0.0, stats.queryLatencyAvg(), 0.00001);
     EXPECT_EQUAL(0u, stats.matchTimeCount());
     EXPECT_EQUAL(0u, stats.groupingTimeCount());
     EXPECT_EQUAL(0u, stats.rerankTimeCount());
     EXPECT_EQUAL(0u, stats.queryCollateralTimeCount());
+    EXPECT_EQUAL(0u, stats.querySetupTimeCount());
     EXPECT_EQUAL(0u, stats.queryLatencyCount());
-    stats.matchTime(0.01).groupingTime(0.1).rerankTime(0.5).queryCollateralTime(2.0).queryLatency(1.0);
+    stats.matchTime(0.01).groupingTime(0.1).rerankTime(0.5).queryCollateralTime(2.0).querySetupTime(2.0).queryLatency(1.0);
     EXPECT_APPROX(0.01, stats.matchTimeAvg(), 0.00001);
     EXPECT_APPROX(0.1, stats.groupingTimeAvg(), 0.00001);
     EXPECT_APPROX(0.5, stats.rerankTimeAvg(), 0.00001);
     EXPECT_APPROX(2.0, stats.queryCollateralTimeAvg(), 0.00001);
+    EXPECT_APPROX(2.0, stats.querySetupTimeAvg(), 0.00001);
     EXPECT_APPROX(1.0, stats.queryLatencyAvg(), 0.00001);
-    stats.add(MatchingStats().matchTime(0.03).groupingTime(0.3).rerankTime(1.5).queryCollateralTime(6.0).queryLatency(3.0));
+    stats.add(MatchingStats().matchTime(0.03).groupingTime(0.3).rerankTime(1.5).queryCollateralTime(6.0).querySetupTime(6.0).queryLatency(3.0));
     EXPECT_APPROX(0.02, stats.matchTimeAvg(), 0.00001);
     EXPECT_APPROX(0.2, stats.groupingTimeAvg(), 0.00001);
     EXPECT_APPROX(1.0, stats.rerankTimeAvg(), 0.00001);
     EXPECT_APPROX(4.0, stats.queryCollateralTimeAvg(), 0.00001);
+    EXPECT_APPROX(4.0, stats.querySetupTimeAvg(), 0.00001);
     EXPECT_APPROX(2.0, stats.queryLatencyAvg(), 0.00001);
     stats.add(MatchingStats().matchTime(0.05)
               .groupingTime(0.5)
               .rerankTime(2.5)
               .queryCollateralTime(10.0)
+              .querySetupTime(10.0)
               .queryLatency(5.0));
     stats.add(MatchingStats().matchTime(0.05).matchTime(0.03)
               .groupingTime(0.5).groupingTime(0.3)
               .rerankTime(2.5).rerankTime(1.5)
               .queryCollateralTime(10.0).queryCollateralTime(6.0)
+              .querySetupTime(10.0).querySetupTime(6.0)
               .queryLatency(5.0).queryLatency(3.0));
     EXPECT_APPROX(0.03, stats.matchTimeAvg(), 0.00001);
     EXPECT_APPROX(0.3, stats.groupingTimeAvg(), 0.00001);
     EXPECT_APPROX(1.5, stats.rerankTimeAvg(), 0.00001);
     EXPECT_APPROX(6.0, stats.queryCollateralTimeAvg(), 0.00001);
+    EXPECT_APPROX(6.0, stats.querySetupTimeAvg(), 0.00001);
     EXPECT_APPROX(3.0, stats.queryLatencyAvg(), 0.00001);
     EXPECT_EQUAL(4u, stats.matchTimeCount());
     EXPECT_EQUAL(4u, stats.groupingTimeCount());
     EXPECT_EQUAL(4u, stats.rerankTimeCount());
     EXPECT_EQUAL(4u, stats.queryCollateralTimeCount());
+    EXPECT_EQUAL(4u, stats.querySetupTimeCount());
     EXPECT_EQUAL(4u, stats.queryLatencyCount());
 }
 
@@ -94,53 +102,63 @@ TEST("requireThatMinMaxTimesAreRecorded") {
     EXPECT_APPROX(0.0, stats.groupingTimeMin(), 0.00001);
     EXPECT_APPROX(0.0, stats.rerankTimeMin(), 0.00001);
     EXPECT_APPROX(0.0, stats.queryCollateralTimeMin(), 0.00001);
+    EXPECT_APPROX(0.0, stats.querySetupTimeMin(), 0.00001);
     EXPECT_APPROX(0.0, stats.queryLatencyMin(), 0.00001);
     EXPECT_APPROX(0.0, stats.matchTimeMax(), 0.00001);
     EXPECT_APPROX(0.0, stats.groupingTimeMax(), 0.00001);
     EXPECT_APPROX(0.0, stats.rerankTimeMax(), 0.00001);
     EXPECT_APPROX(0.0, stats.queryCollateralTimeMax(), 0.00001);
+    EXPECT_APPROX(0.0, stats.querySetupTimeMax(), 0.00001);
     EXPECT_APPROX(0.0, stats.queryLatencyMax(), 0.00001);
-    stats.matchTime(0.01).groupingTime(0.1).rerankTime(0.5).queryCollateralTime(2.0).queryLatency(1.0);
+    stats.matchTime(0.01).groupingTime(0.1).rerankTime(0.5).queryCollateralTime(2.0).querySetupTime(2.0).queryLatency(1.0);
     EXPECT_APPROX(0.01, stats.matchTimeMin(), 0.00001);
     EXPECT_APPROX(0.1, stats.groupingTimeMin(), 0.00001);
     EXPECT_APPROX(0.5, stats.rerankTimeMin(), 0.00001);
     EXPECT_APPROX(2.0, stats.queryCollateralTimeMin(), 0.00001);
+    EXPECT_APPROX(2.0, stats.querySetupTimeMin(), 0.00001);
     EXPECT_APPROX(1.0, stats.queryLatencyMin(), 0.00001);
     EXPECT_APPROX(0.01, stats.matchTimeMax(), 0.00001);
     EXPECT_APPROX(0.1, stats.groupingTimeMax(), 0.00001);
     EXPECT_APPROX(0.5, stats.rerankTimeMax(), 0.00001);
     EXPECT_APPROX(2.0, stats.queryCollateralTimeMax(), 0.00001);
+    EXPECT_APPROX(2.0, stats.querySetupTimeMax(), 0.00001);
     EXPECT_APPROX(1.0, stats.queryLatencyMax(), 0.00001);
-    stats.add(MatchingStats().matchTime(0.03).groupingTime(0.3).rerankTime(1.5).queryCollateralTime(6.0).queryLatency(3.0));
+    stats.add(MatchingStats().matchTime(0.03).groupingTime(0.3).rerankTime(1.5).queryCollateralTime(6.0).querySetupTime(6.0).queryLatency(3.0));
     EXPECT_APPROX(0.01, stats.matchTimeMin(), 0.00001);
     EXPECT_APPROX(0.1, stats.groupingTimeMin(), 0.00001);
     EXPECT_APPROX(0.5, stats.rerankTimeMin(), 0.00001);
     EXPECT_APPROX(2.0, stats.queryCollateralTimeMin(), 0.00001);
+    EXPECT_APPROX(2.0, stats.querySetupTimeMin(), 0.00001);
     EXPECT_APPROX(1.0, stats.queryLatencyMin(), 0.00001);
     EXPECT_APPROX(0.03, stats.matchTimeMax(), 0.00001);
     EXPECT_APPROX(0.3, stats.groupingTimeMax(), 0.00001);
     EXPECT_APPROX(1.5, stats.rerankTimeMax(), 0.00001);
     EXPECT_APPROX(6.0, stats.queryCollateralTimeMax(), 0.00001);
+    EXPECT_APPROX(6.0, stats.querySetupTimeMax(), 0.00001);
     EXPECT_APPROX(3.0, stats.queryLatencyMax(), 0.00001);
     stats.add(MatchingStats().matchTime(0.05)
               .groupingTime(0.5)
               .rerankTime(2.5)
               .queryCollateralTime(10.0)
+              .querySetupTime(10.0)
               .queryLatency(5.0));
     stats.add(MatchingStats().matchTime(0.05).matchTime(0.03)
               .groupingTime(0.5).groupingTime(0.3)
               .rerankTime(2.5).rerankTime(1.5)
               .queryCollateralTime(10.0).queryCollateralTime(6.0)
+              .querySetupTime(10.0).querySetupTime(6.0)
               .queryLatency(5.0).queryLatency(3.0));
     EXPECT_APPROX(0.01, stats.matchTimeMin(), 0.00001);
     EXPECT_APPROX(0.1, stats.groupingTimeMin(), 0.00001);
     EXPECT_APPROX(0.5, stats.rerankTimeMin(), 0.00001);
     EXPECT_APPROX(2.0, stats.queryCollateralTimeMin(), 0.00001);
+    EXPECT_APPROX(2.0, stats.querySetupTimeMin(), 0.00001);
     EXPECT_APPROX(1.0, stats.queryLatencyMin(), 0.00001);
     EXPECT_APPROX(0.05, stats.matchTimeMax(), 0.00001);
     EXPECT_APPROX(0.5, stats.groupingTimeMax(), 0.00001);
     EXPECT_APPROX(2.5, stats.rerankTimeMax(), 0.00001);
     EXPECT_APPROX(10.0, stats.queryCollateralTimeMax(), 0.00001);
+    EXPECT_APPROX(10.0, stats.querySetupTimeMax(), 0.00001);
     EXPECT_APPROX(5.0, stats.queryLatencyMax(), 0.00001);
 }
 
diff --git a/searchcore/src/tests/proton/matching/matching_test.cpp b/searchcore/src/tests/proton/matching/matching_test.cpp
index 95ab43dbcba..00a319db394 100644
--- a/searchcore/src/tests/proton/matching/matching_test.cpp
+++ b/searchcore/src/tests/proton/matching/matching_test.cpp
@@ -289,7 +289,7 @@ struct MyWorld {
         DocsumReply::UP getDocsums(const DocsumRequest &) override {
             return DocsumReply::UP();
         }
-        SearchReply::UP match(const ISearchHandler::SP &, const SearchRequest &, vespalib::ThreadBundle &) const override {
+        SearchReply::UP match(const SearchRequest &, vespalib::ThreadBundle &) const override {
             return SearchReply::UP();
         }
     };
diff --git a/searchcore/src/tests/proton/persistenceengine/persistence_handler_map/persistence_handler_map_test.cpp b/searchcore/src/tests/proton/persistenceengine/persistence_handler_map/persistence_handler_map_test.cpp
index ef12b694187..35590cc68f6 100644
--- a/searchcore/src/tests/proton/persistenceengine/persistence_handler_map/persistence_handler_map_test.cpp
+++ b/searchcore/src/tests/proton/persistenceengine/persistence_handler_map/persistence_handler_map_test.cpp
@@ -46,23 +46,31 @@ DummyPersistenceHandler::SP handler_c(std::make_shared<DummyPersistenceHandler>(
 DummyPersistenceHandler::SP handler_a_new(std::make_shared<DummyPersistenceHandler>());
 
 
+
+void
+assertHandler(const IPersistenceHandler::SP & lhs, const IPersistenceHandler * rhs)
+{
+    EXPECT_EQUAL(lhs.get(), rhs);
+}
+
 void
 assertHandler(const IPersistenceHandler::SP &lhs, const IPersistenceHandler::SP &rhs)
 {
     EXPECT_EQUAL(lhs.get(), rhs.get());
 }
 
+template <typename T>
 void
-assertNullHandler(const IPersistenceHandler::SP &handler)
+assertNullHandler(const T & handler)
 {
-    EXPECT_TRUE(handler.get() == nullptr);
+    EXPECT_TRUE(! handler);
 }
 
 void
-assertSnapshot(const std::vector<IPersistenceHandler::SP> &exp, const HandlerSnapshot::UP &snapshot)
+assertSnapshot(const std::vector<IPersistenceHandler::SP> &exp, HandlerSnapshot snapshot)
 {
-    EXPECT_EQUAL(exp.size(), snapshot->size());
-    auto &sequence = snapshot->handlers();
+    EXPECT_EQUAL(exp.size(), snapshot.size());
+    auto &sequence = snapshot.handlers();
     for (size_t i = 0; i < exp.size() && sequence.valid(); ++i, sequence.next()) {
         EXPECT_EQUAL(exp[i].get(), sequence.get());
     }
diff --git a/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp b/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp
index 569b36a425d..19d9d41c3e4 100644
--- a/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp
+++ b/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp
@@ -404,8 +404,8 @@ struct SimpleFixture {
           engine(_owner, _writeFilter, -1, false),
           hset()
     {
-        engine.putHandler(makeBucketSpace(), DocTypeName(doc1->getType()), hset.phandler1);
-        engine.putHandler(bucketSpace2, DocTypeName(doc2->getType()), hset.phandler2);
+        engine.putHandler(engine.getWLock(), makeBucketSpace(), DocTypeName(doc1->getType()), hset.phandler1);
+        engine.putHandler(engine.getWLock(), bucketSpace2, DocTypeName(doc2->getType()), hset.phandler2);
     }
     SimpleFixture()
         : SimpleFixture(makeBucketSpace())
diff --git a/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp b/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp
index dfb1268aaa6..c26b008f769 100644
--- a/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp
+++ b/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp
@@ -243,7 +243,6 @@ struct MyLog
 struct MyProtonConfigurerOwner : public IProtonConfigurerOwner,
                                  public MyLog
 {
-    using InitializeThreads = std::shared_ptr<vespalib::ThreadStackExecutorBase>;
     vespalib::ThreadStackExecutor _executor;
     std::map<DocTypeName, std::shared_ptr<MyDocumentDBConfigOwner>> _dbs;
 
@@ -254,9 +253,9 @@ struct MyProtonConfigurerOwner : public IProtonConfigurerOwner,
           _dbs()
     {
     }
-    virtual ~MyProtonConfigurerOwner() { }
+    ~MyProtonConfigurerOwner() { }
 
-    virtual std::shared_ptr<DocumentDBConfigOwner> addDocumentDB(const DocTypeName &docTypeName,
+    std::shared_ptr<DocumentDBConfigOwner> addDocumentDB(const DocTypeName &docTypeName,
                                                                  document::BucketSpace bucketSpace,
                                                                  const vespalib::string &configId,
                                                                  const std::shared_ptr<BootstrapConfig> &bootstrapConfig,
@@ -275,14 +274,14 @@ struct MyProtonConfigurerOwner : public IProtonConfigurerOwner,
         _log.push_back(os.str());
         return db;
     }
-    virtual void removeDocumentDB(const DocTypeName &docTypeName) override {
+    void removeDocumentDB(const DocTypeName &docTypeName) override {
         ASSERT_FALSE(_dbs.find(docTypeName) == _dbs.end());
         _dbs.erase(docTypeName);
         std::ostringstream os;
         os << "remove db " << docTypeName.getName();
         _log.push_back(os.str());
     }
-    virtual void applyConfig(const std::shared_ptr<BootstrapConfig> &bootstrapConfig) override {
+    void applyConfig(const std::shared_ptr<BootstrapConfig> &bootstrapConfig) override {
         std::ostringstream os;
         os << "apply config " << bootstrapConfig->getGeneration();
         _log.push_back(os.str());
diff --git a/searchcore/src/tests/proton/reference/document_db_reference_resolver/document_db_reference_resolver_test.cpp b/searchcore/src/tests/proton/reference/document_db_reference_resolver/document_db_reference_resolver_test.cpp
index b77d9b3f5ab..c0098c878b4 100644
--- a/searchcore/src/tests/proton/reference/document_db_reference_resolver/document_db_reference_resolver_test.cpp
+++ b/searchcore/src/tests/proton/reference/document_db_reference_resolver/document_db_reference_resolver_test.cpp
@@ -19,7 +19,7 @@
 #include <vespa/searchlib/attribute/reference_attribute.h>
 #include <vespa/searchlib/common/i_gid_to_lid_mapper.h>
 #include <vespa/searchlib/common/i_gid_to_lid_mapper_factory.h>
-#include <vespa/searchlib/common/sequencedtaskexecutor.h>
+#include <vespa/vespalib/util/sequencedtaskexecutor.h>
 #include <vespa/searchlib/test/mock_attribute_manager.h>
 #include <vespa/vespalib/test/insertion_operators.h>
 
@@ -33,6 +33,8 @@ using proton::test::MockDocumentDBReference;
 using search::attribute::test::MockAttributeManager;
 using vespa::config::search::ImportedFieldsConfig;
 using vespa::config::search::ImportedFieldsConfigBuilder;
+using vespalib::SequencedTaskExecutor;
+using vespalib::ISequencedTaskExecutor;
 
 const ReferenceAttribute *getReferenceAttribute(const IGidToLidChangeListener &listener) {
     auto mylistener = dynamic_cast<const GidToLidChangeListener *>(&listener);
@@ -42,7 +44,7 @@ const ReferenceAttribute *getReferenceAttribute(const IGidToLidChangeListener &l
 
 struct MyGidToLidMapperFactory : public IGidToLidMapperFactory {
     using SP = std::shared_ptr<MyGidToLidMapperFactory>;
-    virtual std::unique_ptr<IGidToLidMapper> getMapper() const override {
+    std::unique_ptr<IGidToLidMapper> getMapper() const override {
         return std::unique_ptr<IGidToLidMapper>();
     }
 };
@@ -60,14 +62,14 @@ struct MyDocumentDBReference : public MockDocumentDBReference {
 
     MyDocumentDBReference(MyGidToLidMapperFactory::SP factory_,
                           std::shared_ptr<MockGidToLidChangeHandler> gidToLidChangeHandler)
-        : factory(factory_),
+        : factory(std::move(factory_)),
           _gidToLidChangeHandler(std::move(gidToLidChangeHandler))
     {
     }
-    virtual IGidToLidMapperFactory::SP getGidToLidMapperFactory() override {
+    IGidToLidMapperFactory::SP getGidToLidMapperFactory() override {
         return factory;
     }
-    virtual std::shared_ptr<search::attribute::ReadableAttributeVector> getAttribute(vespalib::stringref name) override {
+    std::shared_ptr<search::attribute::ReadableAttributeVector> getAttribute(vespalib::stringref name) override {
         auto itr = attributes.find(name);
         if (itr != attributes.end()) {
             return itr->second;
@@ -78,7 +80,7 @@ struct MyDocumentDBReference : public MockDocumentDBReference {
     void addIntAttribute(vespalib::stringref name) {
         attributes[name] = AttributeFactory::createAttribute(name, Config(BasicType::INT32));
     }
-    virtual std::unique_ptr<GidToLidChangeRegistrator> makeGidToLidChangeRegistrator(const vespalib::string &docTypeName) override {
+    std::unique_ptr<GidToLidChangeRegistrator> makeGidToLidChangeRegistrator(const vespalib::string &docTypeName) override {
         return std::make_unique<GidToLidChangeRegistrator>(_gidToLidChangeHandler, docTypeName);
     }
 
@@ -93,12 +95,12 @@ struct MyDocumentDBReference : public MockDocumentDBReference {
 struct MyReferenceRegistry : public IDocumentDBReferenceRegistry {
     using ReferenceMap = std::map<vespalib::string, IDocumentDBReference::SP>;
     ReferenceMap map;
-    virtual IDocumentDBReference::SP get(vespalib::stringref name) const override {
+    IDocumentDBReference::SP get(vespalib::stringref name) const override {
         auto itr = map.find(name);
         ASSERT_TRUE(itr != map.end());
         return itr->second;
     }
-    virtual IDocumentDBReference::SP tryGet(vespalib::stringref name) const override {
+    IDocumentDBReference::SP tryGet(vespalib::stringref name) const override {
         auto itr = map.find(name);
         if (itr != map.end()) {
             return itr->second;
@@ -106,10 +108,10 @@ struct MyReferenceRegistry : public IDocumentDBReferenceRegistry {
             return IDocumentDBReference::SP();
         }
     }
-    virtual void add(vespalib::stringref name, IDocumentDBReference::SP reference) override {
+    void add(vespalib::stringref name, IDocumentDBReference::SP reference) override {
         map[name] = reference;
     }
-    virtual void remove(vespalib::stringref) override {}
+    void remove(vespalib::stringref) override {}
 };
 
 struct MyAttributeManager : public MockAttributeManager {
@@ -121,7 +123,7 @@ struct MyAttributeManager : public MockAttributeManager {
     }
     const ReferenceAttribute *getReferenceAttribute(const vespalib::string &name) const {
         AttributeGuard::UP guard = getAttribute(name);
-        const ReferenceAttribute *result = dynamic_cast<const ReferenceAttribute *>(guard->get());
+        auto *result = dynamic_cast<const ReferenceAttribute *>(guard->get());
         ASSERT_TRUE(result != nullptr);
         return result;
     }
@@ -155,8 +157,7 @@ struct DocumentModel {
     }
 };
 
-DocumentModel::~DocumentModel() {
-}
+DocumentModel::~DocumentModel() = default;
 
 void
 set(const vespalib::string &name,
@@ -182,7 +183,7 @@ createImportedFieldsConfig()
 const ImportedAttributeVector &
 asImportedAttribute(const IAttributeVector &attr)
 {
-    const ImportedAttributeVector *result = dynamic_cast<const ImportedAttributeVector *>(&attr);
+    auto *result = dynamic_cast<const ImportedAttributeVector *>(&attr);
     ASSERT_TRUE(result != nullptr);
     return *result;
 }
@@ -199,7 +200,7 @@ struct Fixture {
     MyAttributeManager oldAttrMgr;
     DocumentModel docModel;
     ImportedFieldsConfig importedFieldsCfg;
-    SequencedTaskExecutor _attributeFieldWriter;
+    std::unique_ptr<ISequencedTaskExecutor> _attributeFieldWriter;
     Fixture() :
         factory(std::make_shared<MyGidToLidMapperFactory>()),
         _gidToLidChangeListenerRefCount(),
@@ -211,7 +212,7 @@ struct Fixture {
         attrMgr(),
         docModel(),
         importedFieldsCfg(createImportedFieldsConfig()),
-        _attributeFieldWriter(1)
+        _attributeFieldWriter(SequencedTaskExecutor::create(1))
 
     {
         registry.add("parent", parentReference);
@@ -231,7 +232,7 @@ struct Fixture {
         oldAttrMgr.addReferenceAttribute("parent3_ref");
     }
     ImportedAttributesRepo::UP resolve(vespalib::duration visibilityDelay, bool useReferences) {
-        DocumentDBReferenceResolver resolver(registry, docModel.childDocType, importedFieldsCfg, docModel.childDocType, _gidToLidChangeListenerRefCount, _attributeFieldWriter, useReferences);
+        DocumentDBReferenceResolver resolver(registry, docModel.childDocType, importedFieldsCfg, docModel.childDocType, _gidToLidChangeListenerRefCount, *_attributeFieldWriter, useReferences);
         return resolver.resolve(attrMgr, oldAttrMgr, std::shared_ptr<search::IDocumentMetaStoreContext>(), visibilityDelay);
     }
     ImportedAttributesRepo::UP resolve(vespalib::duration visibilityDelay) {
@@ -244,7 +245,7 @@ struct Fixture {
         return resolve(vespalib::duration::zero());
     }
     void teardown() {
-        DocumentDBReferenceResolver resolver(registry, docModel.childDocType, importedFieldsCfg, docModel.childDocType, _gidToLidChangeListenerRefCount, _attributeFieldWriter, false);
+        DocumentDBReferenceResolver resolver(registry, docModel.childDocType, importedFieldsCfg, docModel.childDocType, _gidToLidChangeListenerRefCount, *_attributeFieldWriter, false);
         resolver.teardown(attrMgr);
     }
     const IGidToLidMapperFactory *getMapperFactoryPtr(const vespalib::string &attrName) {
@@ -254,7 +255,7 @@ struct Fixture {
                                  const vespalib::string &referenceField,
                                  const vespalib::string &targetField,
                                  bool useSearchCache,
-                                 ImportedAttributeVector::SP attr) {
+                                 const ImportedAttributeVector::SP & attr) {
         ASSERT_TRUE(attr.get());
         EXPECT_EQUAL(name, attr->getName());
         EXPECT_EQUAL(attrMgr.getReferenceAttribute(referenceField), attr->getReferenceAttribute().get());
diff --git a/searchcore/src/tests/proton/reference/gid_to_lid_change_listener/gid_to_lid_change_listener_test.cpp b/searchcore/src/tests/proton/reference/gid_to_lid_change_listener/gid_to_lid_change_listener_test.cpp
index 159ce17ef45..2fb439ea530 100644
--- a/searchcore/src/tests/proton/reference/gid_to_lid_change_listener/gid_to_lid_change_listener_test.cpp
+++ b/searchcore/src/tests/proton/reference/gid_to_lid_change_listener/gid_to_lid_change_listener_test.cpp
@@ -2,7 +2,7 @@
 #include <vespa/vespalib/testkit/testapp.h>
 #include <vespa/vespalib/stllike/string.h>
 #include <vespa/document/base/documentid.h>
-#include <vespa/searchlib/common/sequencedtaskexecutor.h>
+#include <vespa/vespalib/util/sequencedtaskexecutor.h>
 #include <vespa/searchcore/proton/common/monitored_refcount.h>
 #include <vespa/searchcore/proton/reference/gid_to_lid_change_listener.h>
 #include <vespa/searchlib/common/i_gid_to_lid_mapper_factory.h>
@@ -49,13 +49,13 @@ struct MyGidToLidMapperFactory : public MockGidToLidMapperFactory
 struct Fixture
 {
     std::shared_ptr<ReferenceAttribute> _attr;
-    search::SequencedTaskExecutor _writer;
+    std::unique_ptr<vespalib::ISequencedTaskExecutor> _writer;
     MonitoredRefCount _refCount;
     std::unique_ptr<GidToLidChangeListener>  _listener;
 
     Fixture()
         : _attr(std::make_shared<ReferenceAttribute>("test", Config(BasicType::REFERENCE))),
-          _writer(1),
+          _writer(vespalib::SequencedTaskExecutor::create(1)),
           _refCount(),
           _listener()
     {
@@ -91,7 +91,7 @@ struct Fixture
     }
 
     void allocListener() {
-        _listener = std::make_unique<GidToLidChangeListener>(_writer, _attr, _refCount, "test", "testdoc");
+        _listener = std::make_unique<GidToLidChangeListener>(*_writer, _attr, _refCount, "test", "testdoc");
     }
 
     void notifyPutDone(const GlobalId &gid, uint32_t referencedDoc) {
diff --git a/searchcore/src/tests/proton/reprocessing/attribute_reprocessing_initializer/attribute_reprocessing_initializer_test.cpp b/searchcore/src/tests/proton/reprocessing/attribute_reprocessing_initializer/attribute_reprocessing_initializer_test.cpp
index 95cbdafb8e4..898825016b3 100644
--- a/searchcore/src/tests/proton/reprocessing/attribute_reprocessing_initializer/attribute_reprocessing_initializer_test.cpp
+++ b/searchcore/src/tests/proton/reprocessing/attribute_reprocessing_initializer/attribute_reprocessing_initializer_test.cpp
@@ -11,7 +11,7 @@
 #include <vespa/searchcore/proton/reprocessing/i_reprocessing_handler.h>
 #include <vespa/searchcore/proton/test/attribute_utils.h>
 #include <vespa/searchlib/attribute/attributefactory.h>
-#include <vespa/searchlib/common/foregroundtaskexecutor.h>
+#include <vespa/vespalib/util/foregroundtaskexecutor.h>
 #include <vespa/searchlib/index/dummyfileheadercontext.h>
 #include <vespa/searchlib/test/directory_handler.h>
 #include <vespa/vespalib/gtest/gtest.h>
@@ -29,6 +29,7 @@ using search::attribute::BasicType;
 using search::attribute::Config;
 using search::index::schema::DataType;
 using search::test::DirectoryHandler;
+using vespalib::ForegroundTaskExecutor;
 
 const vespalib::string TEST_DIR = "test_output";
 const SerialNum INIT_SERIAL_NUM = 10;
@@ -41,10 +42,10 @@ struct MyReprocessingHandler : public IReprocessingHandler
     IReprocessingReader::SP _reader;
     std::vector<IReprocessingRewriter::SP> _rewriters;
     MyReprocessingHandler() : _reader(), _rewriters() {}
-    virtual void addReader(const IReprocessingReader::SP &reader) override {
+    void addReader(const IReprocessingReader::SP &reader) override {
         _reader = reader;
     }
-    virtual void addRewriter(const IReprocessingRewriter::SP &rewriter) override {
+    void addRewriter(const IReprocessingRewriter::SP &rewriter) override {
         _rewriters.push_back(rewriter);
     }
 };
diff --git a/searchcore/src/tests/proton/server/documentretriever_test.cpp b/searchcore/src/tests/proton/server/documentretriever_test.cpp
index fc12cde57af..30a87bd216e 100644
--- a/searchcore/src/tests/proton/server/documentretriever_test.cpp
+++ b/searchcore/src/tests/proton/server/documentretriever_test.cpp
@@ -36,6 +36,7 @@
 #include <vespa/searchlib/attribute/stringbase.h>
 #include <vespa/searchlib/predicate/predicate_index.h>
 #include <vespa/searchlib/tensor/tensor_attribute.h>
+#include <vespa/vespalib/geo/zcurve.h>
 #include <vespa/vespalib/testkit/testapp.h>
 #include <vespa/vespalib/util/stringfmt.h>
 
@@ -111,6 +112,8 @@ vespalib::string tensor_spec("tensor(x{})");
 std::unique_ptr<Tensor> static_tensor = makeTensor<Tensor>(TensorSpec(tensor_spec).add({{"x", "1"}}, 1.5));
 std::unique_ptr<Tensor> dynamic_tensor = makeTensor<Tensor>(TensorSpec(tensor_spec).add({{"x", "2"}}, 3.5));
 const char zcurve_field[] = "position_field_zcurve";
+const char position_array_field[] = "position_array";
+const char zcurve_array_field[] = "position_array_zcurve";
 const char dyn_field_p[] = "dynamic predicate field";
 const char dyn_arr_field_i[] = "dynamic int array field";
 const char dyn_arr_field_d[] = "dynamic double array field";
@@ -153,7 +156,7 @@ struct MyDocumentStore : proton::test::DummyDocumentStore {
             return std::move(_testDoc);
         }
         const DocumentType *doc_type = r.getDocumentType(doc_type_name);
-        Document::UP doc(new Document(*doc_type, doc_id));
+        auto doc = std::make_unique<Document>(*doc_type, doc_id);
         ASSERT_TRUE(doc.get());
         doc->set(static_field, static_value);
         doc->set(dyn_field_i, static_value);
@@ -208,10 +211,11 @@ document::DocumenttypesConfig getRepoConfig() {
                      .addField(dyn_wset_field_d, Wset(document::DataType::T_DOUBLE))
                      .addField(dyn_wset_field_s, Wset(document::DataType::T_STRING))
                      .addField(dyn_wset_field_n, Wset(document::DataType::T_FLOAT))
-                     .addField(position_field,
-                               PositionDataType::getInstance().getId())
+                     .addField(position_field, PositionDataType::getInstance().getId())
                      .addTensorField(dyn_field_tensor, tensor_spec)
-                     .addField(zcurve_field, document::DataType::T_LONG));
+                     .addField(zcurve_field, document::DataType::T_LONG)
+                     .addField(position_array_field, Array(PositionDataType::getInstance().getId()))
+                     .addField(zcurve_array_field, Array(document::DataType::T_LONG)));
      return builder.config();
 }
 
@@ -359,6 +363,8 @@ struct Fixture {
                 dyn_arr_field_s, dyn_value_s, DataType::STRING, ct);
         addAttribute<FloatingPointAttribute>(
                 dyn_arr_field_n, DataType::FLOAT, ct);
+        addAttribute<IntegerAttribute>(
+                zcurve_array_field, dynamic_zcurve_value, DataType::INT64, ct);
         ct = schema::CollectionType::WEIGHTEDSET;
         addAttribute<IntegerAttribute>(
                 dyn_wset_field_i, dyn_value_i, DataType::INT32, ct);
@@ -484,25 +490,51 @@ void verify_position_field_has_expected_values(Fixture& f) {
 
     FieldValue::UP value = doc->getValue(position_field);
     ASSERT_TRUE(value.get());
-    StructFieldValue *position = dynamic_cast<StructFieldValue *>(value.get());
+    const auto *position = dynamic_cast<StructFieldValue *>(value.get());
     ASSERT_TRUE(position);
     FieldValue::UP x = position->getValue(PositionDataType::FIELD_X);
     FieldValue::UP y = position->getValue(PositionDataType::FIELD_Y);
-    EXPECT_EQUAL(-123096000, static_cast<IntFieldValue&>(*x).getValue());
-    EXPECT_EQUAL(49401000, static_cast<IntFieldValue&>(*y).getValue());
+    EXPECT_EQUAL(-123096000, dynamic_cast<IntFieldValue&>(*x).getValue());
+    EXPECT_EQUAL(49401000, dynamic_cast<IntFieldValue&>(*y).getValue());
 
     checkFieldValue<LongFieldValue>(doc->getValue(zcurve_field), dynamic_zcurve_value);
 }
 
-TEST_F("require that position fields are regenerated from zcurves", Fixture) {
+TEST_F("require that single value position fields are regenerated from zcurves", Fixture) {
     verify_position_field_has_expected_values(f);
 }
 
-TEST_F("zcurve attribute is authoritative for position field existence", Fixture) {
+TEST_F("zcurve attribute is authoritative for single value position field existence", Fixture) {
     f.doc_store._set_position_struct_field = false;
     verify_position_field_has_expected_values(f);
 }
 
+TEST_F("require that array position field value is generated from zcurve array attribute", Fixture) {
+    DocumentMetaData meta_data = f._retriever->getDocumentMetaData(doc_id);
+    Document::UP doc = f._retriever->getDocument(meta_data.lid);
+    ASSERT_TRUE(doc.get());
+    FieldValue::UP value = doc->getValue(position_array_field);
+    ASSERT_TRUE(value.get());
+    const auto* array_value = dynamic_cast<const document::ArrayFieldValue*>(value.get());
+    ASSERT_TRUE(array_value != nullptr);
+    ASSERT_EQUAL(array_value->getNestedType(), document::PositionDataType::getInstance());
+    // Should have two elements prepopulated
+    ASSERT_EQUAL(2u, array_value->size());
+    for (uint32_t i = 0; i < array_value->size(); ++i) {
+        // Magic index-specific value set by collection fixture code.
+        int64_t zcurve_at_pos = ((i == 0) ? dynamic_zcurve_value + 1 : dynamic_zcurve_value);
+        int32_t zx, zy;
+        vespalib::geo::ZCurve::decode(zcurve_at_pos, &zx, &zy);
+
+        const auto *position = dynamic_cast<const StructFieldValue*>(&(*array_value)[i]);
+        ASSERT_TRUE(position != nullptr);
+        FieldValue::UP x = position->getValue(PositionDataType::FIELD_X);
+        FieldValue::UP y = position->getValue(PositionDataType::FIELD_Y);
+        EXPECT_EQUAL(zx, dynamic_cast<IntFieldValue&>(*x).getValue());
+        EXPECT_EQUAL(zy, dynamic_cast<IntFieldValue&>(*y).getValue());
+    }
+}
+
 TEST_F("require that non-existing lid returns null pointer", Fixture) {
     Document::UP doc = f._retriever->getDocument(0);
     ASSERT_FALSE(doc.get());
diff --git a/searchcore/src/tests/proton/server/feedstates_test.cpp b/searchcore/src/tests/proton/server/feedstates_test.cpp
index ca48fb773d8..6643eb925b9 100644
--- a/searchcore/src/tests/proton/server/feedstates_test.cpp
+++ b/searchcore/src/tests/proton/server/feedstates_test.cpp
@@ -91,7 +91,7 @@ Fixture::~Fixture() = default;
 struct RemoveOperationContext
 {
     DocumentId doc_id;
-    RemoveOperation op;
+    RemoveOperationWithDocId op;
     nbostream str;
     std::unique_ptr<Packet> packet;
 
diff --git a/searchcore/src/tests/proton/summaryengine/summaryengine.cpp b/searchcore/src/tests/proton/summaryengine/summaryengine.cpp
index a9cae7d8ab7..7cdd8d767c6 100644
--- a/searchcore/src/tests/proton/summaryengine/summaryengine.cpp
+++ b/searchcore/src/tests/proton/summaryengine/summaryengine.cpp
@@ -42,7 +42,7 @@ public:
         : _name(name), _reply(reply)
     {}
 
-    virtual DocsumReply::UP getDocsums(const DocsumRequest &request) override {
+    DocsumReply::UP getDocsums(const DocsumRequest &request) override {
         return (request.useRootSlime())
                ? std::make_unique<DocsumReply>(createSlimeReply(request.hits.size()))
                : createOldDocSum(request);
@@ -62,7 +62,7 @@ public:
     }
 
     DocsumReply::UP createOldDocSum(const DocsumRequest &request) {
-        DocsumReply::UP retval(new DocsumReply());
+        auto retval = std::make_unique<DocsumReply>();
         for (size_t i = 0; i < request.hits.size(); i++) {
             const DocsumRequest::Hit &h = request.hits[i];
             DocsumReply::Docsum docsum;
@@ -74,11 +74,8 @@ public:
         return retval;
     }
 
-    virtual search::engine::SearchReply::UP match(
-            const ISearchHandler::SP &,
-            const search::engine::SearchRequest &,
-            vespalib::ThreadBundle &) const override {
-        return SearchReply::UP(new SearchReply);
+    SearchReply::UP match(const SearchRequest &, vespalib::ThreadBundle &) const override {
+        return std::make_unique<SearchReply>();
     }
 };
 
@@ -90,8 +87,7 @@ private:
 
 public:
     MyDocsumClient();
-
-    ~MyDocsumClient();
+    ~MyDocsumClient() override;
 
     void getDocsumsDone(DocsumReply::UP reply) override {
         std::lock_guard<std::mutex> guard(_lock);
@@ -111,19 +107,19 @@ public:
     }
 };
 
-MyDocsumClient::MyDocsumClient() {}
+MyDocsumClient::MyDocsumClient() = default;
 
-MyDocsumClient::~MyDocsumClient() {}
+MyDocsumClient::~MyDocsumClient() = default;
 
 DocsumRequest::UP
 createRequest(size_t num = 1) {
-    DocsumRequest::UP r(new DocsumRequest());
+    auto r = std::make_unique<DocsumRequest>();
     if (num == 1) {
         r->hits.emplace_back(GlobalId("aaaaaaaaaaaa"));
     } else {
         for (size_t i = 0; i < num; i++) {
             vespalib::string s = vespalib::make_string("aaaaaaaaaaa%c", char('a' + i % 26));
-            r->hits.push_back(GlobalId(s.c_str()));
+            r->hits.emplace_back(GlobalId(s.c_str()));
         }
     }
     return r;
@@ -132,7 +128,7 @@ createRequest(size_t num = 1) {
 TEST("requireThatGetDocsumsExecute") {
     int numSummaryThreads = 2;
     SummaryEngine engine(numSummaryThreads);
-    ISearchHandler::SP handler(new MySearchHandler);
+    auto handler = std::make_shared<MySearchHandler>();
     DocTypeName dtnvfoo("foo");
     engine.putSearchHandler(dtnvfoo, handler);
 
@@ -140,9 +136,9 @@ TEST("requireThatGetDocsumsExecute") {
     { // async call when engine running
         DocsumRequest::Source request(createRequest());
         DocsumReply::UP reply = engine.getDocsums(std::move(request), client);
-        EXPECT_TRUE(reply.get() == NULL);
+        EXPECT_FALSE(reply);
         reply = client.getReply(10000);
-        EXPECT_TRUE(reply.get() != NULL);
+        EXPECT_TRUE(reply);
         EXPECT_EQUAL(1u, reply->docsums.size());
         EXPECT_EQUAL(10u, reply->docsums[0].docid);
         EXPECT_EQUAL(GlobalId("aaaaaaaaaaaa"), reply->docsums[0].gid);
@@ -152,7 +148,7 @@ TEST("requireThatGetDocsumsExecute") {
     { // sync call when engine closed
         DocsumRequest::Source request(createRequest());
         DocsumReply::UP reply = engine.getDocsums(std::move(request), client);
-        EXPECT_TRUE(reply.get() != NULL);
+        EXPECT_TRUE(reply);
     }
 }
 
@@ -161,23 +157,23 @@ TEST("requireThatHandlersAreStored") {
     DocTypeName dtnvbar("bar");
     int numSummaryThreads = 2;
     SummaryEngine engine(numSummaryThreads);
-    ISearchHandler::SP h1(new MySearchHandler("foo"));
-    ISearchHandler::SP h2(new MySearchHandler("bar"));
-    ISearchHandler::SP h3(new MySearchHandler("baz"));
+    auto h1 = std::make_shared<MySearchHandler>("foo");
+    auto h2 = std::make_shared<MySearchHandler>("bar");
+    auto h3 = std::make_shared<MySearchHandler>("baz");
     // not found
-    EXPECT_TRUE(engine.getSearchHandler(dtnvfoo).get() == NULL);
-    EXPECT_TRUE(engine.removeSearchHandler(dtnvfoo).get() == NULL);
+    EXPECT_FALSE(engine.getSearchHandler(dtnvfoo));
+    EXPECT_FALSE(engine.removeSearchHandler(dtnvfoo));
     // put & get
-    EXPECT_TRUE(engine.putSearchHandler(dtnvfoo, h1).get() == NULL);
+    EXPECT_FALSE(engine.putSearchHandler(dtnvfoo, h1));
     EXPECT_EQUAL(engine.getSearchHandler(dtnvfoo).get(), h1.get());
-    EXPECT_TRUE(engine.putSearchHandler(dtnvbar, h2).get() == NULL);
+    EXPECT_FALSE(engine.putSearchHandler(dtnvbar, h2));
     EXPECT_EQUAL(engine.getSearchHandler(dtnvbar).get(), h2.get());
     // replace
     EXPECT_TRUE(engine.putSearchHandler(dtnvfoo, h3).get() == h1.get());
     EXPECT_EQUAL(engine.getSearchHandler(dtnvfoo).get(), h3.get());
     // remove
     EXPECT_EQUAL(engine.removeSearchHandler(dtnvfoo).get(), h3.get());
-    EXPECT_TRUE(engine.getSearchHandler(dtnvfoo).get() == NULL);
+    EXPECT_FALSE(engine.getSearchHandler(dtnvfoo));
 }
 
 bool
@@ -196,9 +192,9 @@ TEST("requireThatCorrectHandlerIsUsed") {
     DocTypeName dtnvbar("bar");
     DocTypeName dtnvbaz("baz");
     SummaryEngine engine(1);
-    ISearchHandler::SP h1(new MySearchHandler("foo", "foo reply"));
-    ISearchHandler::SP h2(new MySearchHandler("bar", "bar reply"));
-    ISearchHandler::SP h3(new MySearchHandler("baz", "baz reply"));
+    auto h1 = std::make_shared<MySearchHandler>("foo", "foo reply");
+    auto h2 = std::make_shared<MySearchHandler>("bar", "bar reply");
+    auto h3 = std::make_shared<MySearchHandler>("baz", "baz reply");
     engine.putSearchHandler(dtnvfoo, h1);
     engine.putSearchHandler(dtnvbar, h2);
     engine.putSearchHandler(dtnvbaz, h3);
@@ -369,7 +365,7 @@ public:
 Server::Server()
     : BaseServer(),
       engine(2),
-      handler(new MySearchHandler("slime", stringref(buf.GetDrainPos(), buf.GetUsedLen()))),
+      handler(std::make_shared<MySearchHandler>("slime", stringref(buf.GetDrainPos(), buf.GetUsedLen()))),
       docsumBySlime(engine),
       docsumByRPC(docsumBySlime)
 {
@@ -377,7 +373,7 @@ Server::Server()
     engine.putSearchHandler(dtnvfoo, handler);
 }
 
-Server::~Server() {}
+Server::~Server() = default;
 
 vespalib::string
 getAnswer(size_t num) {
diff --git a/searchcore/src/vespa/searchcore/config/proton.def b/searchcore/src/vespa/searchcore/config/proton.def
index d123a5711ac..0501ab6ed7c 100644
--- a/searchcore/src/vespa/searchcore/config/proton.def
+++ b/searchcore/src/vespa/searchcore/config/proton.def
@@ -31,6 +31,11 @@ numsummarythreads int default=16 restart
 ## Stop on io errors ?
 stoponioerrors bool default=false restart
 
+## Perform extra validation of stored data on startup
+## It requires a restart to be turned, but no restart to turned off.
+## Hence it must always be followed by a manual restart.
+validate_and_sanitize_docstore enum {NO, YES} default = NO
+
 ## Maximum number of concurrent flushes outstanding.
 flush.maxconcurrent int default=2 restart
 
@@ -117,6 +122,10 @@ indexing.read.io enum {NORMAL, DIRECTIO} default=DIRECTIO restart
 ## Control number of threads used for indexing
 indexing.threads int default=1 restart
 
+## Option to specify what is most important during indexing.
+## This is experimental and will most likely be temporary.
+indexing.optimize enum {LATENCY, THROUGHPUT, ADAPTIVE} default=LATENCY restart
+
 ## Maximum number of pending operations for each of the internal
 ## indexing threads.  Only used when visibility delay is zero.
 indexing.tasklimit int default=1000 restart
@@ -128,6 +137,14 @@ indexing.tasklimit int default=1000 restart
 ## is 40000 then effective task limit is 10000.
 indexing.semiunboundtasklimit int default = 40000 restart
 
+## Kind of watermark for when to activate extra manpower
+## Utilized if optimize is set to either THROUGHPUT or ADAPTIVE
+indexing.kind_of_watermark int default = 0 restart
+
+## Controls minimum reaction time in seconds if using THROUGHPUT
+indexing.reactiontime double default = 0.005 restart
+
+
 ## How long a freshly loaded index shall be warmed up
 ## before being used for serving
 index.warmup.time double default=0.0 restart
@@ -397,8 +414,8 @@ writefilter.memorylimit double default = 0.8
 ## put and update portion of feed is blocked.
 writefilter.disklimit double default = 0.8
 
-## Interval between sampling of disk and memory usage. Default is 60 seconds.
-writefilter.sampleinterval double default = 60.0
+## Interval between sampling of disk and memory usage. Default is 20 seconds.
+writefilter.sampleinterval double default = 20.0
 
 ## The size of the disk partition (in bytes) on which proton basedir is located.
 ## If set to 0, the disk size is sampled by looking at the filesystem space info.
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_vector_explorer.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_vector_explorer.cpp
index ad49c12a556..495fc5d63b3 100644
--- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_vector_explorer.cpp
+++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_vector_explorer.cpp
@@ -5,6 +5,8 @@
 #include <vespa/searchlib/attribute/multi_value_mapping.h>
 #include <vespa/searchlib/attribute/attributevector.h>
 #include <vespa/searchlib/attribute/ipostinglistattributebase.h>
+#include <vespa/searchlib/util/state_explorer_utils.h>
+#include <vespa/searchlib/tensor/i_tensor_attribute.h>
 #include <vespa/vespalib/data/slime/cursor.h>
 
 using search::attribute::Status;
@@ -67,10 +69,7 @@ convertAddressSpaceUsageToSlime(const AddressSpaceUsage &usage, Cursor &object)
 void
 convertMemoryUsageToSlime(const MemoryUsage &usage, Cursor &object)
 {
-    object.setLong("allocated", usage.allocatedBytes());
-    object.setLong("used", usage.usedBytes());
-    object.setLong("dead", usage.deadBytes());
-    object.setLong("onHold", usage.allocatedBytesOnHold());
+    search::StateExplorerUtils::memory_usage_to_slime(usage, object);
 }
 
 void
@@ -119,6 +118,9 @@ AttributeVectorExplorer::get_state(const vespalib::slime::Inserter &inserter, bo
         convertStatusToSlime(status, object.setObject("status"));
         convertGenerationToSlime(attr, object.setObject("generation"));
         convertAddressSpaceUsageToSlime(attr.getAddressSpaceUsage(), object.setObject("addressSpaceUsage"));
+        // TODO: Consider making enum store, multivalue mapping, posting list attribute and tensor attribute
+        // explorable as children of this state explorer, and let them expose even more detailed information.
+        // In this case we must ensure that ExclusiveAttributeReadAccessor::Guard is held also when exploring children.
         const IEnumStore *enumStore = attr.getEnumStoreBase();
         if (enumStore) {
             convertEnumStoreToSlime(*enumStore, object.setObject("enumStore"));
@@ -131,6 +133,11 @@ AttributeVectorExplorer::get_state(const vespalib::slime::Inserter &inserter, bo
         if (postingBase) {
             convertPostingBaseToSlime(*postingBase, object.setObject("postingList"));
         }
+        const auto* tensor_attr = attr.asTensorAttribute();
+        if (tensor_attr) {
+            ObjectInserter tensor_inserter(object, "tensor");
+            tensor_attr->get_state(tensor_inserter);
+        }
         convertChangeVectorToSlime(attr, object.setObject("changeVector"));
         object.setLong("committedDocIdLimit", attr.getCommittedDocIdLimit());
         object.setLong("createSerialNum", attr.getCreateSerialNum());
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp
index 74e3e903540..33b9d162163 100644
--- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp
+++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.cpp
@@ -13,7 +13,6 @@
 #include <vespa/searchlib/attribute/attributevector.hpp>
 #include <vespa/searchlib/attribute/imported_attribute_vector.h>
 #include <vespa/searchlib/common/idestructorcallback.h>
-#include <vespa/searchlib/common/isequencedtaskexecutor.h>
 #include <vespa/vespalib/stllike/hash_map.hpp>
 
 #include <vespa/log/log.h>
@@ -22,7 +21,8 @@ LOG_SETUP(".proton.attribute.attribute_writer");
 using namespace document;
 using namespace search;
 using search::attribute::ImportedAttributeVector;
-using ExecutorId = search::ISequencedTaskExecutor::ExecutorId;
+using vespalib::ISequencedTaskExecutor;
+using ExecutorId = vespalib::ISequencedTaskExecutor::ExecutorId;
 
 namespace proton {
 
@@ -361,7 +361,7 @@ public:
           _immediateCommit(immediateCommit),
           _onWriteDone(onWriteDone)
     {}
-    ~BatchRemoveTask() override {}
+    ~BatchRemoveTask() override = default;
     void run() override {
         for (auto field : _writeCtx.getFields()) {
             auto &attr = field.getAttribute();
@@ -469,9 +469,9 @@ AttributeWriter::internalRemove(SerialNum serialNum, DocumentIdT lid, bool immed
     }
 }
 
-AttributeWriter::AttributeWriter(const proton::IAttributeManager::SP &mgr)
-    : _mgr(mgr),
-      _attributeFieldWriter(mgr->getAttributeFieldWriter()),
+AttributeWriter::AttributeWriter(proton::IAttributeManager::SP mgr)
+    : _mgr(std::move(mgr)),
+      _attributeFieldWriter(_mgr->getAttributeFieldWriter()),
       _writeContexts(),
       _dataType(nullptr),
       _hasStructFieldAttribute(false),
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.h b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.h
index 9e5d8f4ce5d..4a9726dd113 100644
--- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.h
+++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer.h
@@ -5,7 +5,7 @@
 #include "i_attribute_writer.h"
 #include <vespa/searchcore/proton/common/commit_time_tracker.h>
 #include <vespa/document/base/fieldpath.h>
-#include <vespa/searchlib/common/isequencedtaskexecutor.h>
+#include <vespa/vespalib/util/isequencedtaskexecutor.h>
 #include <vespa/vespalib/stllike/hash_map.h>
 
 namespace document { class DocumentType; }
@@ -25,8 +25,8 @@ private:
     typedef document::DocumentType DocumentType;
     typedef document::FieldValue FieldValue;
     const IAttributeManager::SP _mgr;
-    search::ISequencedTaskExecutor &_attributeFieldWriter;
-    using ExecutorId = search::ISequencedTaskExecutor::ExecutorId;
+    vespalib::ISequencedTaskExecutor &_attributeFieldWriter;
+    using ExecutorId = vespalib::ISequencedTaskExecutor::ExecutorId;
 public:
     class WriteField
     {
@@ -74,7 +74,7 @@ private:
                         bool immediateCommit, OnWriteDoneType onWriteDone);
 
 public:
-    AttributeWriter(const proton::IAttributeManager::SP &mgr);
+    AttributeWriter(proton::IAttributeManager::SP mgr);
     ~AttributeWriter();
 
     /* Only for in tests that add attributes after AttributeWriter construction. */
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp
index e39061a8389..5ff40fa5360 100644
--- a/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp
+++ b/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp
@@ -11,9 +11,10 @@
 #include <vespa/searchcore/proton/flushengine/shrink_lid_space_flush_target.h>
 #include <vespa/searchlib/attribute/attributecontext.h>
 #include <vespa/searchlib/attribute/attribute_read_guard.h>
+#include <vespa/searchlib/attribute/imported_attribute_vector.h>
 #include <vespa/searchcommon/attribute/i_attribute_functor.h>
 #include <vespa/searchlib/attribute/interlock.h>
-#include <vespa/searchlib/common/isequencedtaskexecutor.h>
+#include <vespa/vespalib/util/isequencedtaskexecutor.h>
 #include <vespa/searchlib/common/threaded_compactable_lid_space.h>
 #include <vespa/searchlib/attribute/attributevector.h>
 #include <vespa/vespalib/io/fileutil.h>
@@ -73,7 +74,7 @@ search::SerialNum estimateShrinkSerialNum(const AttributeVector &attr)
     return std::max(attr.getStatus().getLastSyncToken(), serialNum);
 }
 
-std::shared_ptr<ShrinkLidSpaceFlushTarget> allocShrinker(const AttributeVector::SP &attr, search::ISequencedTaskExecutor &attributeFieldWriter, AttributeDiskLayout &diskLayout)
+std::shared_ptr<ShrinkLidSpaceFlushTarget> allocShrinker(const AttributeVector::SP &attr, vespalib::ISequencedTaskExecutor &attributeFieldWriter, AttributeDiskLayout &diskLayout)
 {
     using Type = IFlushTarget::Type;
     using Component = IFlushTarget::Component;
@@ -237,8 +238,7 @@ AttributeManager::AttributeManager(const vespalib::string &baseDir,
                                    const vespalib::string &documentSubDbName,
                                    const TuneFileAttributes &tuneFileAttributes,
                                    const FileHeaderContext &fileHeaderContext,
-                                   search::ISequencedTaskExecutor &
-                                   attributeFieldWriter,
+                                   vespalib::ISequencedTaskExecutor &attributeFieldWriter,
                                    const HwInfo &hwInfo)
     : proton::IAttributeManager(),
       _attributes(),
@@ -261,8 +261,7 @@ AttributeManager::AttributeManager(const vespalib::string &baseDir,
                                    const vespalib::string &documentSubDbName,
                                    const search::TuneFileAttributes &tuneFileAttributes,
                                    const search::common::FileHeaderContext &fileHeaderContext,
-                                   search::ISequencedTaskExecutor &
-                                   attributeFieldWriter,
+                                   vespalib::ISequencedTaskExecutor &attributeFieldWriter,
                                    const IAttributeFactory::SP &factory,
                                    const HwInfo &hwInfo)
     : proton::IAttributeManager(),
@@ -547,7 +546,7 @@ AttributeManager::pruneRemovedFields(search::SerialNum serialNum)
     }
 }
 
-search::ISequencedTaskExecutor &
+vespalib::ISequencedTaskExecutor &
 AttributeManager::getAttributeFieldWriter() const
 {
     return _attributeFieldWriter;
@@ -613,4 +612,14 @@ AttributeManager::setImportedAttributes(std::unique_ptr<ImportedAttributesRepo>
     _importedAttributes = std::move(attributes);
 }
 
+std::shared_ptr<search::attribute::ReadableAttributeVector>
+AttributeManager::readable_attribute_vector(const string& name) const
+{
+    auto attribute = findAttribute(name);
+    if (attribute || !_importedAttributes) {
+        return attribute;
+    }
+    return _importedAttributes->get(name);
+}
+
 } // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.h b/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.h
index cf60a0a41e9..350b986add4 100644
--- a/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.h
+++ b/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.h
@@ -77,7 +77,7 @@ private:
     const search::common::FileHeaderContext &_fileHeaderContext;
     IAttributeFactory::SP _factory;
     std::shared_ptr<search::attribute::Interlock> _interlock;
-    search::ISequencedTaskExecutor &_attributeFieldWriter;
+    vespalib::ISequencedTaskExecutor &_attributeFieldWriter;
     HwInfo _hwInfo;
     std::unique_ptr<ImportedAttributesRepo> _importedAttributes;
 
@@ -103,14 +103,14 @@ public:
                      const vespalib::string &documentSubDbName,
                      const search::TuneFileAttributes &tuneFileAttributes,
                      const search::common::FileHeaderContext & fileHeaderContext,
-                     search::ISequencedTaskExecutor &attributeFieldWriter,
+                     vespalib::ISequencedTaskExecutor &attributeFieldWriter,
                      const HwInfo &hwInfo);
 
     AttributeManager(const vespalib::string &baseDir,
                      const vespalib::string &documentSubDbName,
                      const search::TuneFileAttributes &tuneFileAttributes,
                      const search::common::FileHeaderContext & fileHeaderContext,
-                     search::ISequencedTaskExecutor &attributeFieldWriter,
+                     vespalib::ISequencedTaskExecutor &attributeFieldWriter,
                      const IAttributeFactory::SP &factory,
                      const HwInfo &hwInfo);
 
@@ -164,7 +164,7 @@ public:
 
     const IAttributeFactory::SP &getFactory() const override { return _factory; }
 
-    search::ISequencedTaskExecutor &getAttributeFieldWriter() const override;
+    vespalib::ISequencedTaskExecutor &getAttributeFieldWriter() const override;
 
     search::AttributeVector *getWritableAttribute(const vespalib::string &name) const override;
 
@@ -178,6 +178,8 @@ public:
     void setImportedAttributes(std::unique_ptr<ImportedAttributesRepo> attributes) override;
 
     const ImportedAttributesRepo *getImportedAttributes() const override { return _importedAttributes.get(); }
+
+    std::shared_ptr<search::attribute::ReadableAttributeVector> readable_attribute_vector(const string& name) const override;
 };
 
 } // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/exclusive_attribute_read_accessor.cpp b/searchcore/src/vespa/searchcore/proton/attribute/exclusive_attribute_read_accessor.cpp
index 7a31580fb16..9543897407d 100644
--- a/searchcore/src/vespa/searchcore/proton/attribute/exclusive_attribute_read_accessor.cpp
+++ b/searchcore/src/vespa/searchcore/proton/attribute/exclusive_attribute_read_accessor.cpp
@@ -3,12 +3,12 @@
 #include "exclusive_attribute_read_accessor.h"
 #include <vespa/vespalib/util/gate.h>
 #include <vespa/searchlib/attribute/attributevector.h>
-#include <vespa/searchlib/common/isequencedtaskexecutor.h>
+#include <vespa/vespalib/util/isequencedtaskexecutor.h>
 
 namespace proton {
 
 using search::AttributeVector;
-using search::ISequencedTaskExecutor;
+using vespalib::ISequencedTaskExecutor;
 using vespalib::Executor;
 using vespalib::Gate;
 
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/exclusive_attribute_read_accessor.h b/searchcore/src/vespa/searchcore/proton/attribute/exclusive_attribute_read_accessor.h
index 00f3ad3e8ac..233d42b2194 100644
--- a/searchcore/src/vespa/searchcore/proton/attribute/exclusive_attribute_read_accessor.h
+++ b/searchcore/src/vespa/searchcore/proton/attribute/exclusive_attribute_read_accessor.h
@@ -6,9 +6,11 @@
 
 namespace search {
     class AttributeVector;
+}
+namespace vespalib {
+    class Gate;
     class ISequencedTaskExecutor;
 }
-namespace vespalib { class Gate; }
 
 namespace proton {
 
@@ -38,13 +40,13 @@ public:
 private:
     using AttributeVectorSP = std::shared_ptr<search::AttributeVector>;
     AttributeVectorSP _attribute;
-    search::ISequencedTaskExecutor &_attributeFieldWriter;
+    vespalib::ISequencedTaskExecutor &_attributeFieldWriter;
 
 public:
     using UP = std::unique_ptr<ExclusiveAttributeReadAccessor>;
 
     ExclusiveAttributeReadAccessor(const AttributeVectorSP &attribute,
-                                   search::ISequencedTaskExecutor &attributeFieldWriter);
+                                   vespalib::ISequencedTaskExecutor &attributeFieldWriter);
     Guard::UP takeGuard();
 };
 
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.cpp b/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.cpp
index fd1f95e13ba..0fd9849443a 100644
--- a/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.cpp
+++ b/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.cpp
@@ -1,7 +1,7 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include "filter_attribute_manager.h"
-#include <vespa/searchlib/common/isequencedtaskexecutor.h>
+#include <vespa/vespalib/util/isequencedtaskexecutor.h>
 #include <vespa/vespalib/util/exceptions.h>
 #include <vespa/searchlib/attribute/attributevector.h>
 #include <vespa/searchcommon/attribute/i_attribute_functor.h>
@@ -161,7 +161,7 @@ FilterAttributeManager::getFlushedSerialNum(const vespalib::string &name) const
 }
 
 
-search::ISequencedTaskExecutor &
+vespalib::ISequencedTaskExecutor &
 FilterAttributeManager::getAttributeFieldWriter() const
 {
     return _mgr->getAttributeFieldWriter();
@@ -190,7 +190,7 @@ FilterAttributeManager::asyncForEachAttribute(std::shared_ptr<IConstAttributeFun
     // Run by document db master thread
     std::vector<AttributeGuard> completeList;
     _mgr->getAttributeList(completeList);
-    search::ISequencedTaskExecutor &attributeFieldWriter = getAttributeFieldWriter();
+    vespalib::ISequencedTaskExecutor &attributeFieldWriter = getAttributeFieldWriter();
     for (auto &guard : completeList) {
         search::AttributeVector::SP attrsp = guard.getSP();
         // Name must be extracted in document db master thread or attribute
@@ -204,7 +204,7 @@ void
 FilterAttributeManager::asyncForAttribute(const vespalib::string &name, std::unique_ptr<IAttributeFunctor> func) const {
     AttributeGuard::UP attr = _mgr->getAttribute(name);
     if (!attr) { return; }
-    search::ISequencedTaskExecutor &attributeFieldWriter = getAttributeFieldWriter();
+    vespalib::ISequencedTaskExecutor &attributeFieldWriter = getAttributeFieldWriter();
     vespalib::string attrName = (*attr)->getNamePrefix();
     attributeFieldWriter.execute(attributeFieldWriter.getExecutorId(attrName),
                                   [attr=std::move(attr), func=std::move(func)]() mutable {
@@ -231,4 +231,13 @@ FilterAttributeManager::getImportedAttributes() const
     throw vespalib::IllegalArgumentException("Not implemented");
 }
 
+std::shared_ptr<search::attribute::ReadableAttributeVector>
+FilterAttributeManager::readable_attribute_vector(const string& name) const
+{
+    if (acceptAttribute(name)) {
+        return _mgr->readable_attribute_vector(name);
+    }
+    return {};
+}
+
 }
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.h b/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.h
index 099bb2e84ba..3755946037d 100644
--- a/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.h
+++ b/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.h
@@ -46,7 +46,7 @@ public:
     void getAttributeListAll(std::vector<search::AttributeGuard> &) const override;
     void pruneRemovedFields(search::SerialNum serialNum) override;
     const IAttributeFactory::SP &getFactory() const override;
-    search::ISequencedTaskExecutor & getAttributeFieldWriter() const override;
+    vespalib::ISequencedTaskExecutor & getAttributeFieldWriter() const override;
 
     search::AttributeVector * getWritableAttribute(const vespalib::string &name) const override;
     const std::vector<search::AttributeVector *> & getWritableAttributes() const override;
@@ -54,6 +54,7 @@ public:
     ExclusiveAttributeReadAccessor::UP getExclusiveReadAccessor(const vespalib::string &name) const override;
     void setImportedAttributes(std::unique_ptr<ImportedAttributesRepo> attributes) override;
     const ImportedAttributesRepo *getImportedAttributes() const override;
+    std::shared_ptr<search::attribute::ReadableAttributeVector> readable_attribute_vector(const string& name) const override;
 
     void asyncForAttribute(const vespalib::string &name, std::unique_ptr<IAttributeFunctor> func) const override;
 };
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/flushableattribute.cpp b/searchcore/src/vespa/searchcore/proton/attribute/flushableattribute.cpp
index ee1c06ef7c1..0a61ec8d882 100644
--- a/searchcore/src/vespa/searchcore/proton/attribute/flushableattribute.cpp
+++ b/searchcore/src/vespa/searchcore/proton/attribute/flushableattribute.cpp
@@ -10,9 +10,9 @@
 #include <vespa/vespalib/io/fileutil.h>
 #include <vespa/vespalib/util/closuretask.h>
 #include <vespa/searchlib/common/serialnumfileheadercontext.h>
-#include <vespa/searchlib/common/isequencedtaskexecutor.h>
 #include <vespa/searchlib/attribute/attributememorysavetarget.h>
 #include <vespa/searchlib/attribute/attributevector.h>
+#include <vespa/vespalib/util/isequencedtaskexecutor.h>
 #include <vespa/vespalib/util/stringfmt.h>
 #include <fstream>
 #include <future>
@@ -154,7 +154,7 @@ FlushableAttribute::FlushableAttribute(const AttributeVectorSP attr,
                                        tuneFileAttributes,
                                        const FileHeaderContext &
                                        fileHeaderContext,
-                                       search::ISequencedTaskExecutor &
+                                       vespalib::ISequencedTaskExecutor &
                                        attributeFieldWriter,
                                        const HwInfo &hwInfo)
     : IFlushTarget(make_string("attribute.flush.%s", attr->getName().c_str()), Type::SYNC, Component::ATTRIBUTE),
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/flushableattribute.h b/searchcore/src/vespa/searchcore/proton/attribute/flushableattribute.h
index 01178dbe86c..8d807c153c0 100644
--- a/searchcore/src/vespa/searchcore/proton/attribute/flushableattribute.h
+++ b/searchcore/src/vespa/searchcore/proton/attribute/flushableattribute.h
@@ -7,14 +7,10 @@
 #include <vespa/searchcore/proton/common/hw_info.h>
 
 
-namespace search {
+namespace search { class AttributeVector; }
 
-class ISequencedTaskExecutor;
-class AttributeVector;
-
-namespace common { class FileHeaderContext; }
-
-}
+namespace search::common { class FileHeaderContext; }
+namespace vespalib { class ISequencedTaskExecutor; }
 
 namespace proton {
 
@@ -39,7 +35,7 @@ private:
     FlushStats                  _lastStats;
     const search::TuneFileAttributes _tuneFileAttributes;
     const search::common::FileHeaderContext &_fileHeaderContext;
-    search::ISequencedTaskExecutor &_attributeFieldWriter;
+    vespalib::ISequencedTaskExecutor &_attributeFieldWriter;
     HwInfo                       _hwInfo;
     std::shared_ptr<AttributeDirectory> _attrDir;
 
@@ -59,7 +55,7 @@ public:
                        const search::TuneFileAttributes &tuneFileAttributes,
                        const search::common::FileHeaderContext &
                        fileHeaderContext,
-                       search::ISequencedTaskExecutor &attributeFieldWriter,
+                       vespalib::ISequencedTaskExecutor &attributeFieldWriter,
                        const HwInfo &hwInfo);
 
     virtual
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_manager.h b/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_manager.h
index 4e50c52c58b..25d0a508438 100644
--- a/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_manager.h
+++ b/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_manager.h
@@ -10,12 +10,12 @@
 #include <vespa/searchlib/attribute/iattributemanager.h>
 #include <vespa/searchlib/common/serialnum.h>
 
-namespace search {
-    class IDestructorCallback;
-    class ISequencedTaskExecutor;
-}
+namespace search { class IDestructorCallback;}
+
 namespace search::attribute { class IAttributeFunctor; }
 
+namespace vespalib { class ISequencedTaskExecutor; }
+
 namespace proton {
 
 class ImportedAttributesRepo;
@@ -72,7 +72,7 @@ struct IAttributeManager : public search::IAttributeManager
      */
     virtual const IAttributeFactory::SP &getFactory() const = 0;
 
-    virtual search::ISequencedTaskExecutor &getAttributeFieldWriter() const = 0;
+    virtual vespalib::ISequencedTaskExecutor &getAttributeFieldWriter() const = 0;
 
     /*
      * Get pointer to named writable attribute.  If attribute isn't
diff --git a/searchcore/src/vespa/searchcore/proton/bucketdb/bucketdb.cpp b/searchcore/src/vespa/searchcore/proton/bucketdb/bucketdb.cpp
index 3ced0f509b5..e80543368d4 100644
--- a/searchcore/src/vespa/searchcore/proton/bucketdb/bucketdb.cpp
+++ b/searchcore/src/vespa/searchcore/proton/bucketdb/bucketdb.cpp
@@ -119,6 +119,19 @@ BucketDB::cachedGet(const BucketId &bucketId) const
     return get(bucketId);
 }
 
+storage::spi::BucketInfo
+BucketDB::cachedGetBucketInfo(const BucketId &bucketId) const
+{
+    if (isCachedBucket(bucketId)) {
+        return _cachedBucketState;
+    }
+    auto itr = _map.find(bucketId);
+    if (itr != _map.end()) {
+        return itr->second;
+    }
+    return BucketState();
+}
+
 bool
 BucketDB::hasBucket(const BucketId &bucketId) const
 {
diff --git a/searchcore/src/vespa/searchcore/proton/bucketdb/bucketdb.h b/searchcore/src/vespa/searchcore/proton/bucketdb/bucketdb.h
index e64cc53d4a2..05388931e20 100644
--- a/searchcore/src/vespa/searchcore/proton/bucketdb/bucketdb.h
+++ b/searchcore/src/vespa/searchcore/proton/bucketdb/bucketdb.h
@@ -51,6 +51,7 @@ public:
     void cacheBucket(const BucketId &bucketId);
     void uncacheBucket();
     bool isCachedBucket(const BucketId &bucketId) const;
+    storage::spi::BucketInfo cachedGetBucketInfo(const BucketId &bucketId) const;
     BucketState cachedGet(const BucketId &bucketId) const;
     bool hasBucket(const BucketId &bucketId) const;
     void getBuckets(BucketId::List & buckets) const;
diff --git a/searchcore/src/vespa/searchcore/proton/common/attribute_updater.cpp b/searchcore/src/vespa/searchcore/proton/common/attribute_updater.cpp
index eaf76ebdc8f..8fd47c17acb 100644
--- a/searchcore/src/vespa/searchcore/proton/common/attribute_updater.cpp
+++ b/searchcore/src/vespa/searchcore/proton/common/attribute_updater.cpp
@@ -208,9 +208,13 @@ namespace {
 
 template <typename TensorUpdateType>
 void
-applyTensorUpdate(TensorAttribute &vec, uint32_t lid, const TensorUpdateType &update)
+applyTensorUpdate(TensorAttribute &vec, uint32_t lid, const TensorUpdateType &update,
+                  bool create_empty_if_non_existing)
 {
     auto oldTensor = vec.getTensor(lid);
+    if (!oldTensor && create_empty_if_non_existing) {
+        oldTensor = vec.getEmptyTensor();
+    }
     if (oldTensor) {
         auto newTensor = update.applyTo(*oldTensor);
         if (newTensor) {
@@ -235,11 +239,11 @@ AttributeUpdater::handleUpdate(TensorAttribute &vec, uint32_t lid, const ValueUp
             updateValue(vec, lid, assign.getValue());
         }
     } else if (op == ValueUpdate::TensorModifyUpdate) {
-        applyTensorUpdate(vec, lid, static_cast<const TensorModifyUpdate &>(upd));
+        applyTensorUpdate(vec, lid, static_cast<const TensorModifyUpdate &>(upd), false);
     } else if (op == ValueUpdate::TensorAddUpdate) {
-        applyTensorUpdate(vec, lid, static_cast<const TensorAddUpdate &>(upd));
+        applyTensorUpdate(vec, lid, static_cast<const TensorAddUpdate &>(upd), true);
     } else if (op == ValueUpdate::TensorRemoveUpdate) {
-        applyTensorUpdate(vec, lid, static_cast<const TensorRemoveUpdate &>(upd));
+        applyTensorUpdate(vec, lid, static_cast<const TensorRemoveUpdate &>(upd), false);
     } else if (op == ValueUpdate::Clear) {
         vec.clearDoc(lid);
     } else {
diff --git a/searchcore/src/vespa/searchcore/proton/common/attributefieldvaluenode.cpp b/searchcore/src/vespa/searchcore/proton/common/attributefieldvaluenode.cpp
index ab3b0588f3f..c516bbfc06c 100644
--- a/searchcore/src/vespa/searchcore/proton/common/attributefieldvaluenode.cpp
+++ b/searchcore/src/vespa/searchcore/proton/common/attributefieldvaluenode.cpp
@@ -4,6 +4,7 @@
 #include "selectcontext.h"
 #include <vespa/searchcommon/attribute/attributecontent.h>
 #include <vespa/searchlib/attribute/attributevector.h>
+#include <vespa/searchlib/attribute/attribute_read_guard.h>
 #include <vespa/vespalib/util/exceptions.h>
 
 #include <vespa/log/log.h>
@@ -31,14 +32,10 @@ using vespalib::make_string;
 AttributeFieldValueNode::
 AttributeFieldValueNode(const vespalib::string& doctype,
                         const vespalib::string& field,
-                        const std::shared_ptr<search::AttributeVector> &attribute)
+                        uint32_t attr_guard_index)
     : FieldValueNode(doctype, field),
-      _attribute(attribute)
+      _attr_guard_index(attr_guard_index)
 {
-    const AttributeVector &v(*_attribute);
-    // Only handle single value attribute vectors for now
-    assert(v.getCollectionType() == CollectionType::SINGLE);
-    (void) v;
 }
 
 
@@ -46,10 +43,10 @@ std::unique_ptr<document::select::Value>
 AttributeFieldValueNode::
 getValue(const Context &context) const
 {
-    const SelectContext &sc(static_cast<const SelectContext &>(context)); 
+    const auto &sc(static_cast<const SelectContext &>(context));
     uint32_t docId(sc._docId); 
     assert(docId != 0u);
-    const AttributeVector &v(*_attribute);
+    const auto& v = sc.guarded_attribute_at_index(_attr_guard_index);
     if (v.isUndefined(docId)) {
         return std::make_unique<NullValue>();
     }
@@ -86,10 +83,10 @@ getValue(const Context &context) const
         case BasicType::PREDICATE:
         case BasicType::TENSOR:
         case BasicType::REFERENCE:
-            throw new IllegalArgumentException(make_string("Attribute '%s' of type '%s' can not be used for selection",
-                                                           v.getName().c_str(), v.getInternalBasicType().asString()));
+            throw IllegalArgumentException(make_string("Attribute '%s' of type '%s' can not be used for selection",
+                                                       v.getName().c_str(), BasicType(v.getBasicType()).asString()));
         case BasicType::MAX_TYPE:
-            throw new IllegalStateException(make_string("Attribute '%s' has illegal type '%d'", v.getName().c_str(), v.getBasicType()));
+            throw IllegalStateException(make_string("Attribute '%s' has illegal type '%d'", v.getName().c_str(), v.getBasicType()));
     }
     return std::make_unique<NullValue>();;
 
@@ -106,7 +103,7 @@ AttributeFieldValueNode::traceValue(const Context &context, std::ostream& out) c
 document::select::ValueNode::UP
 AttributeFieldValueNode::clone() const
 {
-    return wrapParens(new AttributeFieldValueNode(getDocType(), getFieldName(), _attribute));
+    return wrapParens(new AttributeFieldValueNode(getDocType(), getFieldName(), _attr_guard_index));
 }
 
 } // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/common/attributefieldvaluenode.h b/searchcore/src/vespa/searchcore/proton/common/attributefieldvaluenode.h
index 9ac6ce0e32b..89d1dac321b 100644
--- a/searchcore/src/vespa/searchcore/proton/common/attributefieldvaluenode.h
+++ b/searchcore/src/vespa/searchcore/proton/common/attributefieldvaluenode.h
@@ -3,18 +3,19 @@
 
 #include <vespa/document/select/valuenodes.h>
 
-namespace search { class AttributeVector; }
+namespace search { class ReadableAttributeVector; }
 namespace proton {
 
 class AttributeFieldValueNode : public document::select::FieldValueNode
 {
     using Context = document::select::Context;
-    std::shared_ptr<search::AttributeVector> _attribute;
+    uint32_t _attr_guard_index;
 
 public:
+    // Precondition: attribute must be of a single-value type.
     AttributeFieldValueNode(const vespalib::string& doctype,
                             const vespalib::string& field,
-                            const std::shared_ptr<search::AttributeVector> &attribute);
+                            uint32_t attr_guard_index);
 
     std::unique_ptr<document::select::Value> getValue(const Context &context) const override;
     std::unique_ptr<document::select::Value> traceValue(const Context &context, std::ostream& out) const override;
diff --git a/searchcore/src/vespa/searchcore/proton/common/cachedselect.cpp b/searchcore/src/vespa/searchcore/proton/common/cachedselect.cpp
index d007b030e6b..21f757fba79 100644
--- a/searchcore/src/vespa/searchcore/proton/common/cachedselect.cpp
+++ b/searchcore/src/vespa/searchcore/proton/common/cachedselect.cpp
@@ -7,6 +7,7 @@
 #include "selectpruner.h"
 #include <vespa/document/select/parser.h>
 #include <vespa/searchlib/attribute/attributevector.h>
+#include <vespa/searchlib/attribute/attribute_read_guard.h>
 #include <vespa/searchlib/attribute/iattributemanager.h>
 
 namespace proton {
@@ -24,7 +25,7 @@ namespace {
 class AttrVisitor : public document::select::CloningVisitor
 {
 public:
-    typedef std::map<vespalib::string, uint32_t> AttrMap;
+    using AttrMap = std::map<vespalib::string, uint32_t>;
 
     AttrMap _amap;
     const search::IAttributeManager &_amgr;
@@ -62,7 +63,7 @@ AttrVisitor::AttrVisitor(const search::IAttributeManager &amgr, CachedSelect::At
 
 AttrVisitor::~AttrVisitor() = default;
 
-bool isSingleValueThatWEHandle(BasicType type) {
+bool isSingleValueThatWeHandle(BasicType type) {
     return (type != BasicType::PREDICATE) && (type != BasicType::TENSOR) && (type != BasicType::REFERENCE);
 }
 
@@ -75,17 +76,18 @@ AttrVisitor::visitFieldValueNode(const FieldValueNode &expr)
     bool complex = false;
     vespalib::string name = SelectUtils::extractFieldName(expr, complex);
 
-    AttributeGuard::UP ag(_amgr.getAttribute(name));
-    if (ag->valid()) {
+    auto av = _amgr.readable_attribute_vector(name);
+    if (av) {
         if (complex) {
             ++_complexAttrs;
             // Don't try to optimize complex attribute references yet.
             _valueNode = expr.clone();
             return;
         }
-        std::shared_ptr<search::AttributeVector> av(ag->getSP());
-        if (av->getCollectionType() == CollectionType::SINGLE) {
-            if (isSingleValueThatWEHandle(av->getBasicType())) {
+        auto guard = av->makeReadGuard(false);
+        const auto* attr = guard->attribute();
+        if (attr->getCollectionType() == CollectionType::SINGLE) {
+            if (isSingleValueThatWeHandle(attr->getBasicType())) {
                 ++_svAttrs;
                 auto it(_amap.find(name));
                 uint32_t idx(invalidIdx());
@@ -99,7 +101,7 @@ AttrVisitor::visitFieldValueNode(const FieldValueNode &expr)
                     idx = it->second;
                 }
                 assert(idx != invalidIdx());
-                _valueNode = std::make_unique<AttributeFieldValueNode>(expr.getDocType(), name, av);
+                _valueNode = std::make_unique<AttributeFieldValueNode>(expr.getDocType(), name, idx);
             } else {
                 ++_complexAttrs;
                 // Don't try to optimize predicate/tensor/reference attributes yet.
diff --git a/searchcore/src/vespa/searchcore/proton/common/cachedselect.h b/searchcore/src/vespa/searchcore/proton/common/cachedselect.h
index e9eb452889e..563eed75c32 100644
--- a/searchcore/src/vespa/searchcore/proton/common/cachedselect.h
+++ b/searchcore/src/vespa/searchcore/proton/common/cachedselect.h
@@ -15,6 +15,8 @@ namespace search {
     class IAttributeManager;
 }
 
+namespace search::attribute { class ReadableAttributeVector; }
+
 namespace proton {
 
 class SelectContext;
@@ -44,7 +46,7 @@ public:
         const document::select::Node &selectNode() const;
     };
 
-    using AttributeVectors = std::vector<std::shared_ptr<search::AttributeVector>>;
+    using AttributeVectors = std::vector<std::shared_ptr<search::attribute::ReadableAttributeVector>>;
 
 private:
     // Single value attributes referenced from selection expression
diff --git a/searchcore/src/vespa/searchcore/proton/common/doctypename.cpp b/searchcore/src/vespa/searchcore/proton/common/doctypename.cpp
index 70daeeebeca..21b27ef6ffd 100644
--- a/searchcore/src/vespa/searchcore/proton/common/doctypename.cpp
+++ b/searchcore/src/vespa/searchcore/proton/common/doctypename.cpp
@@ -4,20 +4,16 @@
 #include <vespa/searchlib/engine/request.h>
 #include <vespa/document/datatype/documenttype.h>
 
-namespace proton
-{
+namespace proton {
 
 
 DocTypeName::DocTypeName(const search::engine::Request &request)
     : _name(request.propertiesMap.matchProperties().lookup("documentdb", "searchdoctype").get(""))
-{
-}
+{}
 
 
 DocTypeName::DocTypeName(const document::DocumentType &docType)
     : _name(docType.getName())
-{
-}
+{}
 
-
-} // namespace proton
+}
diff --git a/searchcore/src/vespa/searchcore/proton/common/handlermap.hpp b/searchcore/src/vespa/searchcore/proton/common/handlermap.hpp
index bddbfed371f..e762ac527f0 100644
--- a/searchcore/src/vespa/searchcore/proton/common/handlermap.hpp
+++ b/searchcore/src/vespa/searchcore/proton/common/handlermap.hpp
@@ -5,7 +5,6 @@
 #include <vespa/vespalib/util/exceptions.h>
 #include <vespa/vespalib/util/sequence.h>
 #include <vespa/vespalib/stllike/string.h>
-#include <vespa/vespalib/stllike/hash_map.h>
 #include <map>
 #include <vector>
 
@@ -18,9 +17,8 @@ namespace proton {
 template <typename T>
 class HandlerMap {
 private:
-    typedef typename std::shared_ptr<T>    HandlerSP;
-    typedef std::map<DocTypeName, HandlerSP> StdMap;
-    typedef typename StdMap::iterator        MapIterator;
+    using HandlerSP = typename std::shared_ptr<T>;
+    using StdMap = std::map<DocTypeName, HandlerSP>;
 
     StdMap _handlers;
 
@@ -40,8 +38,7 @@ public:
         size_t                 _offset;
 
     public:
-        typedef std::unique_ptr<Snapshot> UP;
-
+        Snapshot() : _handlers(), _offset(0) { }
         Snapshot(const StdMap &map) : _handlers(), _offset(0) {
             _handlers.reserve(map.size());
             for (auto itr : map) {
@@ -49,9 +46,13 @@ public:
             }
         }
         Snapshot(std::vector<HandlerSP> &&handlers) : _handlers(std::move(handlers)), _offset(0) {}
+        Snapshot(Snapshot &&) noexcept = default;
+        Snapshot & operator = (Snapshot &&) noexcept = default;
+        Snapshot(const Snapshot &) = delete;
+        Snapshot & operator = (const Snapshot &) = delete;
+
         bool valid() const override { return (_offset < _handlers.size()); }
         T *get() const override { return _handlers[_offset].get(); }
-        HandlerSP getSP() const { return _handlers[_offset]; }
         void next() override { ++_offset; }
     };
 
@@ -64,11 +65,7 @@ public:
     /**
      * Constructs a new instance of this class.
      */
-    HandlerMap()
-        : _handlers()
-    {
-        // empty
-    }
+    HandlerMap() = default;
 
     /**
      * Registers a new handler for the given document type. If another handler
@@ -83,7 +80,7 @@ public:
     putHandler(const DocTypeName &docTypeNameVer,
                const HandlerSP &handler)
     {
-        if (handler.get() == NULL) {
+        if ( ! handler) {
             throw vespalib::IllegalArgumentException(vespalib::make_string(
                             "Handler is null for docType '%s'",
                             docTypeNameVer.toString().c_str()));
@@ -115,6 +112,20 @@ public:
         return HandlerSP();
     }
 
+    /**
+     * Returns the handler for the given document type. If no handler was
+     * registered, this method returns a null pointer.
+     *
+     * @param docType The document type whose handler to return.
+     * @return The registered handler, if any.
+     */
+    T *
+    getHandlerPtr(const DocTypeName &docTypeNameVer) const
+    {
+        const_iterator it = _handlers.find(docTypeNameVer);
+        return (it != _handlers.end()) ? it->second.get() : nullptr;
+    }
+
     bool hasHandler(const HandlerSP &handler) const {
         bool found = false;
         for (const auto &kv : _handlers) {
@@ -148,11 +159,7 @@ public:
     /**
      * Clear all handlers.
      */
-    void
-    clear()
-    {
-        _handlers.clear();
-    }
+    void clear() { _handlers.clear(); }
 
     /**
      * Create a snapshot of the handlers currently contained in this
@@ -161,68 +168,15 @@ public:
      *
      * @return handler sequence
      **/
-    std::unique_ptr<Snapshot>
-    snapshot() const
-    {
-        return std::unique_ptr<Snapshot>(new Snapshot(_handlers));
-    }
+    Snapshot snapshot() const { return Snapshot(_handlers); }
 
 // we want to use snapshots rather than direct iteration to reduce locking;
 // the below functions should be deprecated when possible.
 
-    /**
-     * Returns a bidirectional iterator that points at the first element of the
-     * sequence (or just beyond the end of an empty sequence).
-     *
-     * @return The beginning of this map.
-     */
-    iterator
-    begin()
-    {
-        return _handlers.begin();
-    }
-
-    /**
-     * Returns a const bidirectional iterator that points at the first element
-     * of the sequence (or just beyond the end of an empty sequence).
-     *
-     * @return The beginning of this map.
-     */
-    const_iterator
-    begin() const
-    {
-        return _handlers.begin();
-    }
-
-    /**
-     * Returns a bidirectional iterator that points just beyond the end of the
-     * sequence.
-     *
-     * @return The end of this map.
-     */
-    iterator
-    end()
-    {
-        return _handlers.end();
-    }
-
-    /**
-     * Returns a const bidirectional iterator that points just beyond the end of
-     * the sequence.
-     *
-     * @return The end of this map.
-     */
-    const_iterator
-    end() const
-    {
-        return _handlers.end();
-    }
-
-    /**
-     * Returns the number of handlers in this map.
-     *
-     * @return the number of handlers.
-     */
+    iterator begin() { return _handlers.begin(); }
+    const_iterator begin() const { return _handlers.begin(); }
+    iterator end() { return _handlers.end(); }
+    const_iterator end() const { return _handlers.end(); }
     size_t size() const { return _handlers.size(); }
 };
 
diff --git a/searchcore/src/vespa/searchcore/proton/common/selectcontext.cpp b/searchcore/src/vespa/searchcore/proton/common/selectcontext.cpp
index 3275fbe06d4..ee713e19385 100644
--- a/searchcore/src/vespa/searchcore/proton/common/selectcontext.cpp
+++ b/searchcore/src/vespa/searchcore/proton/common/selectcontext.cpp
@@ -3,20 +3,22 @@
 #include "selectcontext.h"
 #include "cachedselect.h"
 #include <vespa/document/select/value.h>
-#include <vespa/searchlib/attribute/attributeguard.h>
+#include <vespa/searchlib/attribute/attribute_read_guard.h>
+#include <vespa/searchlib/attribute/readable_attribute_vector.h>
+#include <cassert>
 
 namespace proton {
 
 using document::select::Value;
 using document::select::Context;
-using search::AttributeGuard;
 using search::AttributeVector;
+using search::attribute::AttributeReadGuard;
 
 namespace select {
-    struct Guards : public std::vector<AttributeGuard> {
-        using Parent = std::vector<AttributeGuard>;
-        using Parent::Parent;
-    };
+struct Guards : public std::vector<std::unique_ptr<AttributeReadGuard>> {
+    using Parent = std::vector<std::unique_ptr<AttributeReadGuard>>;
+    using Parent::Parent;
+};
 }
 
 SelectContext::SelectContext(const CachedSelect &cachedSelect)
@@ -26,23 +28,30 @@ SelectContext::SelectContext(const CachedSelect &cachedSelect)
       _cachedSelect(cachedSelect)
 { }
 
-SelectContext::~SelectContext() { }
+SelectContext::~SelectContext() = default;
 
 void
 SelectContext::getAttributeGuards()
 {
-    _guards->resize(_cachedSelect.attributes().size());
-    auto j(_cachedSelect.attributes().begin());
-    for (std::vector<AttributeGuard>::iterator i(_guards->begin()), ie(_guards->end()); i != ie; ++i, ++j) {
-        *i = AttributeGuard(*j);
+    _guards->clear();
+    _guards->reserve(_cachedSelect.attributes().size());
+    for (const auto& attr : _cachedSelect.attributes()) {
+        _guards->emplace_back(attr->makeReadGuard(false));
     }
 }
 
-
 void
 SelectContext::dropAttributeGuards()
 {
     _guards->clear();
 }
 
+const search::attribute::IAttributeVector&
+SelectContext::guarded_attribute_at_index(uint32_t index) const noexcept
+{
+    assert(index < _guards->size());
+    assert((*_guards)[index].get() != nullptr);
+    return *((*_guards)[index])->attribute();
+}
+
 }
diff --git a/searchcore/src/vespa/searchcore/proton/common/selectcontext.h b/searchcore/src/vespa/searchcore/proton/common/selectcontext.h
index d37c6eb6f14..4dd9f873088 100644
--- a/searchcore/src/vespa/searchcore/proton/common/selectcontext.h
+++ b/searchcore/src/vespa/searchcore/proton/common/selectcontext.h
@@ -3,6 +3,8 @@
 
 #include <vespa/document/select/context.h>
 
+namespace search::attribute { class IAttributeVector; }
+
 namespace proton {
 
 class CachedSelect;
@@ -19,6 +21,8 @@ public:
     void dropAttributeGuards();
 
     uint32_t _docId;
+
+    const search::attribute::IAttributeVector& guarded_attribute_at_index(uint32_t index) const noexcept;
 private:
     std::unique_ptr<select::Guards> _guards;
     const CachedSelect &_cachedSelect;
diff --git a/searchcore/src/vespa/searchcore/proton/common/selectpruner.cpp b/searchcore/src/vespa/searchcore/proton/common/selectpruner.cpp
index 6cad54f134b..2c237074907 100644
--- a/searchcore/src/vespa/searchcore/proton/common/selectpruner.cpp
+++ b/searchcore/src/vespa/searchcore/proton/common/selectpruner.cpp
@@ -12,6 +12,7 @@
 #include <vespa/document/select/invalidconstant.h>
 #include <vespa/document/select/valuenodes.h>
 #include <vespa/searchlib/attribute/attributevector.h>
+#include <vespa/searchlib/attribute/attribute_read_guard.h>
 #include <vespa/searchlib/attribute/iattributemanager.h>
 
 using document::select::And;
@@ -97,10 +98,7 @@ SelectPruner::SelectPruner(const SelectPruner *rhs)
 }
 
 
-SelectPruner::~SelectPruner()
-{
-}
-
+SelectPruner::~SelectPruner() = default;
 
 void
 SelectPruner::visitAndBranch(const And &expr)
@@ -395,7 +393,6 @@ SelectPruner::visitIdValueNode(const IdValueNode &expr)
     CloningVisitor::visitIdValueNode(expr);
 }
 
-
 void
 SelectPruner::visitFieldValueNode(const FieldValueNode &expr)
 {
@@ -406,32 +403,33 @@ SelectPruner::visitFieldValueNode(const FieldValueNode &expr)
     const document::DocumentType *docType = _repo.getDocumentType(_docType);
     bool complex = false; // Cannot handle attribute if complex expression
     vespalib::string name = SelectUtils::extractFieldName(expr, complex);
-    try {
-        std::unique_ptr<Field> fp(new Field(docType->getField(name)));
-        if (!fp) {
+    const bool is_imported = docType->has_imported_field_name(name);
+    if (complex || !is_imported) {
+        try {
+            std::unique_ptr<Field> fp(new Field(docType->getField(name)));
+            if (!fp) {
+                setInvalidVal();
+                return;
+            }
+        } catch (FieldNotFoundException &) {
+            setInvalidVal();
+            return;
+        }
+        try {
+            FieldPath path;
+            docType->buildFieldPath(path, expr.getFieldName());
+        } catch (vespalib::IllegalArgumentException &) {
+            setInvalidVal();
+            return;
+        } catch (FieldNotFoundException &) {
             setInvalidVal();
             return;
         }
-    } catch (FieldNotFoundException &) {
-        setInvalidVal();
-        return;
-    }
-    try {
-        FieldPath path;
-        docType->buildFieldPath(path, expr.getFieldName());
-    } catch (vespalib::IllegalArgumentException &) {
-        setInvalidVal();
-        return;
-    } catch (FieldNotFoundException &) {
-        setInvalidVal();
-        return;
     }
     _constVal = false;
     if (!_hasFields) {
         // If we're working on removed document sub db then we have no fields.
-        _constVal = true;
-        _valueNode.reset(new NullValueNode());
-        _priority = NullValPriority;
+        set_null_value_node();
         return;
     }
     
@@ -440,13 +438,18 @@ SelectPruner::visitFieldValueNode(const FieldValueNode &expr)
     bool svAttr = false;
     bool attrField = false;
     if (_amgr != nullptr) {
-        AttributeGuard::UP ag(_amgr->getAttribute(name));
-        if (ag->valid()) {
+        auto attr = _amgr->readable_attribute_vector(name);
+        if (attr) {
             attrField = true;
-            auto av(ag->getSP());
-            if (av->getCollectionType() == CollectionType::SINGLE && !complex) {
+            auto ag = attr->makeReadGuard(false);
+            if ((ag->attribute()->getCollectionType() == CollectionType::SINGLE) && !complex) {
                 svAttr = true;
             }
+        } else if (is_imported) {
+            // Imported field present in document config but not yet in attribute config.
+            // Treat as missing (null) in document, as this matches behavior elsewhere in the pipeline.
+            set_null_value_node();
+            return;
         }
     }
     if (!_hasDocuments && !svAttr) {
@@ -538,6 +541,13 @@ SelectPruner::setInvalidConst()
     _node.reset(new InvalidConstant("invalid"));
 }
 
+void
+SelectPruner::set_null_value_node()
+{
+    _constVal = true;
+    _valueNode = std::make_unique<NullValueNode>();
+    _priority = NullValPriority;
+}
 
 void
 SelectPruner::setTernaryConst(bool val)
diff --git a/searchcore/src/vespa/searchcore/proton/common/selectpruner.h b/searchcore/src/vespa/searchcore/proton/common/selectpruner.h
index 4350247d8b3..2c3729006c3 100644
--- a/searchcore/src/vespa/searchcore/proton/common/selectpruner.h
+++ b/searchcore/src/vespa/searchcore/proton/common/selectpruner.h
@@ -81,6 +81,7 @@ private:
     void setInvalidVal();
     void setInvalidConst();
     void setTernaryConst(bool val);
+    void set_null_value_node();
     void resolveTernaryConst(bool wantInverted);
     bool isInvalidVal() const;
     bool isNullVal() const;
diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp
index 93d8ef3fff4..3e9ef787f74 100644
--- a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp
+++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp
@@ -1,24 +1,25 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include "documentmetastore.h"
-#include "search_context.h"
 #include "documentmetastoresaver.h"
+#include "search_context.h"
+#include <vespa/fastos/file.h>
+#include <vespa/searchcore/proton/bucketdb/bucketsessionbase.h>
+#include <vespa/searchcore/proton/bucketdb/joinbucketssession.h>
+#include <vespa/searchcore/proton/bucketdb/splitbucketsession.h>
 #include <vespa/searchlib/attribute/attributevector.hpp>
+#include <vespa/searchlib/attribute/load_utils.h>
 #include <vespa/searchlib/attribute/readerbase.h>
+#include <vespa/searchlib/common/i_gid_to_lid_mapper.h>
+#include <vespa/searchlib/query/query_term_simple.h>
 #include <vespa/vespalib/btree/btree.hpp>
-#include <vespa/vespalib/btree/btreenodestore.hpp>
+#include <vespa/vespalib/btree/btreebuilder.hpp>
 #include <vespa/vespalib/btree/btreenodeallocator.hpp>
+#include <vespa/vespalib/btree/btreenodestore.hpp>
 #include <vespa/vespalib/btree/btreeroot.hpp>
-#include <vespa/vespalib/btree/btreebuilder.hpp>
-#include <vespa/searchlib/common/i_gid_to_lid_mapper.h>
-#include <vespa/searchcore/proton/bucketdb/bucketsessionbase.h>
-#include <vespa/searchcore/proton/bucketdb/joinbucketssession.h>
-#include <vespa/searchcore/proton/bucketdb/splitbucketsession.h>
-#include <vespa/searchlib/query/query_term_simple.h>
 #include <vespa/vespalib/util/bufferwriter.h>
 #include <vespa/vespalib/util/exceptions.h>
 #include <vespa/vespalib/util/rcuvector.hpp>
-#include <vespa/fastos/file.h>
 #include "document_meta_store_versions.h"
 
 #include <vespa/log/log.h>
@@ -32,7 +33,7 @@ using search::FileReader;
 using search::GrowStrategy;
 using search::IAttributeSaveTarget;
 using search::LidUsageStats;
-using vespalib::MemoryUsage;
+using search::attribute::LoadUtils;
 using search::attribute::SearchContextParams;
 using search::btree::BTreeNoLeafData;
 using search::fef::TermFieldMatchData;
@@ -42,6 +43,7 @@ using storage::spi::Timestamp;
 using vespalib::GenerationHandler;
 using vespalib::GenerationHeldBase;
 using vespalib::IllegalStateException;
+using vespalib::MemoryUsage;
 using vespalib::make_string;
 
 namespace proton {
@@ -260,7 +262,7 @@ DocumentMetaStore::readNextDoc(documentmetastore::Reader & reader, TreeType::Bui
 bool
 DocumentMetaStore::onLoad()
 {
-    documentmetastore::Reader reader(openDAT());
+    documentmetastore::Reader reader(LoadUtils::openDAT(*this));
     unload();
     size_t numElems = reader.getNumElems();
     size_t docIdLimit = reader.getDocIdLimit();
diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/documentoperation.cpp b/searchcore/src/vespa/searchcore/proton/feedoperation/documentoperation.cpp
index 37b23449315..f1161c8ebdd 100644
--- a/searchcore/src/vespa/searchcore/proton/feedoperation/documentoperation.cpp
+++ b/searchcore/src/vespa/searchcore/proton/feedoperation/documentoperation.cpp
@@ -41,9 +41,14 @@ DocumentOperation::DocumentOperation(Type type, const BucketId &bucketId, const
 void
 DocumentOperation::assertValidBucketId(const document::DocumentId &docId) const
 {
+   assertValidBucketId(docId.getGlobalId());
+}
+
+void
+DocumentOperation::assertValidBucketId(const document::GlobalId &gid) const
+{
     assert(_bucketId.valid());
     uint8_t bucketUsedBits = _bucketId.getUsedBits();
-    const GlobalId &gid = docId.getGlobalId();
     BucketId verId(gid.convertToBucketId());
     verId.setUsedBits(bucketUsedBits);
     assert(_bucketId.getRawId() == verId.getRawId() ||
diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/documentoperation.h b/searchcore/src/vespa/searchcore/proton/feedoperation/documentoperation.h
index 9a823c553bd..6847dbfd943 100644
--- a/searchcore/src/vespa/searchcore/proton/feedoperation/documentoperation.h
+++ b/searchcore/src/vespa/searchcore/proton/feedoperation/documentoperation.h
@@ -26,6 +26,7 @@ protected:
                       const storage::spi::Timestamp &timestamp);
 
     void assertValidBucketId(const document::DocumentId &docId) const;
+    void assertValidBucketId(const document::GlobalId &docId) const;
     vespalib::string docArgsToString() const;
 
 public:
diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/feedoperation.h b/searchcore/src/vespa/searchcore/proton/feedoperation/feedoperation.h
index 3509af0de5c..10518c74340 100644
--- a/searchcore/src/vespa/searchcore/proton/feedoperation/feedoperation.h
+++ b/searchcore/src/vespa/searchcore/proton/feedoperation/feedoperation.h
@@ -36,7 +36,8 @@ public:
         MOVE                    = 15,
         CREATE_BUCKET           = 16,
         COMPACT_LID_SPACE       = 17,
-        UPDATE                  = 18
+        UPDATE                  = 18,
+        REMOVE_GID              = 19
     };
 
 private:
diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/removedocumentsoperation.h b/searchcore/src/vespa/searchcore/proton/feedoperation/removedocumentsoperation.h
index df74de1bd16..ad058bb153d 100644
--- a/searchcore/src/vespa/searchcore/proton/feedoperation/removedocumentsoperation.h
+++ b/searchcore/src/vespa/searchcore/proton/feedoperation/removedocumentsoperation.h
@@ -18,22 +18,22 @@ protected:
     void serializeLidsToRemove(vespalib::nbostream &os) const;
     void deserializeLidsToRemove(vespalib::nbostream &is);
 public:
-    virtual ~RemoveDocumentsOperation() { }
+    ~RemoveDocumentsOperation() override { }
 
     void setLidsToRemove(uint32_t subDbId, const LidVectorContext::SP &lidsToRemove) {
         _lidsToRemoveMap[subDbId] = lidsToRemove;
     }
 
+    bool hasLidsToRemove() const {
+        return !_lidsToRemoveMap.empty();
+    }
+
     const LidVectorContext::SP
     getLidsToRemove(uint32_t subDbId) const {
         LidsToRemoveMap::const_iterator found(_lidsToRemoveMap.find(subDbId));
-        if (found != _lidsToRemoveMap.end())
-            return found->second;
-        else
-            return LidVectorContext::SP();
+        return (found != _lidsToRemoveMap.end()) ? found->second : LidVectorContext::SP();
     }
 
 };
 
 } // namespace proton
-
diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/removeoperation.cpp b/searchcore/src/vespa/searchcore/proton/feedoperation/removeoperation.cpp
index 006a7e2b035..c6bdeb08ad5 100644
--- a/searchcore/src/vespa/searchcore/proton/feedoperation/removeoperation.cpp
+++ b/searchcore/src/vespa/searchcore/proton/feedoperation/removeoperation.cpp
@@ -1,37 +1,37 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include "removeoperation.h"
+#include <cassert>
 
 using document::BucketId;
 using document::DocumentId;
+using document::GlobalId;
 using document::DocumentTypeRepo;
 using storage::spi::Timestamp;
 using vespalib::make_string;
 
 namespace proton {
 
-RemoveOperation::RemoveOperation()
-    : DocumentOperation(FeedOperation::REMOVE),
+RemoveOperationWithDocId::RemoveOperationWithDocId()
+    : RemoveOperation(FeedOperation::REMOVE),
       _docId()
 {
 }
 
 
-RemoveOperation::RemoveOperation(const BucketId &bucketId,
-                                 const Timestamp &timestamp,
-                                 const DocumentId &docId)
-    : DocumentOperation(FeedOperation::REMOVE,
-                        bucketId,
-                        timestamp),
+RemoveOperationWithDocId::RemoveOperationWithDocId(BucketId bucketId, Timestamp timestamp, const DocumentId &docId)
+    : RemoveOperation(FeedOperation::REMOVE, bucketId, timestamp),
       _docId(docId)
 {
 }
 
+RemoveOperationWithDocId::~RemoveOperationWithDocId() = default;
+
 void
-RemoveOperation::serialize(vespalib::nbostream &os) const
+RemoveOperationWithDocId::serialize(vespalib::nbostream &os) const
 {
     assertValidBucketId(_docId);
-    DocumentOperation::serialize(os);
+    RemoveOperation::serialize(os);
     size_t oldSize = os.size();
     vespalib::string rawId = _docId.toString();
     os.write(rawId.c_str(), rawId.size() + 1);
@@ -40,18 +40,64 @@ RemoveOperation::serialize(vespalib::nbostream &os) const
 
 
 void
-RemoveOperation::deserialize(vespalib::nbostream &is,
-                          const DocumentTypeRepo &repo)
+RemoveOperationWithDocId::deserialize(vespalib::nbostream &is, const DocumentTypeRepo &repo)
 {
-    DocumentOperation::deserialize(is, repo);
+    RemoveOperation::deserialize(is, repo);
     size_t oldSize = is.size();
     _docId = DocumentId(is);
     _serializedDocSize = oldSize - is.size();
 }
 
-vespalib::string RemoveOperation::toString() const {
+vespalib::string
+RemoveOperationWithDocId::toString() const {
     return make_string("Remove(%s, %s)",
-                       _docId.getScheme().toString().c_str(),
-                       docArgsToString().c_str());
+                       _docId.getScheme().toString().c_str(), docArgsToString().c_str());
+}
+
+RemoveOperationWithGid::RemoveOperationWithGid()
+    : RemoveOperation(FeedOperation::REMOVE_GID),
+      _gid(),
+      _docType()
+{}
+
+
+RemoveOperationWithGid::RemoveOperationWithGid(BucketId bucketId, Timestamp timestamp, const GlobalId &gid, vespalib::stringref docType)
+    : RemoveOperation(FeedOperation::REMOVE_GID, bucketId, timestamp),
+      _gid(gid),
+      _docType(docType)
+{}
+
+RemoveOperationWithGid::~RemoveOperationWithGid() = default;
+
+void
+RemoveOperationWithGid::serialize(vespalib::nbostream &os) const
+{
+    assertValidBucketId(_gid);
+    assert( ! getValidDbdId());
+    RemoveOperation::serialize(os);
+    size_t oldSize = os.size();
+    os.write(_gid.get(), GlobalId::LENGTH);
+    os.writeSmallString(_docType);
+    _serializedDocSize = os.size() - oldSize;
 }
+
+
+void
+RemoveOperationWithGid::deserialize(vespalib::nbostream &is, const DocumentTypeRepo &repo)
+{
+    RemoveOperation::deserialize(is, repo);
+    size_t oldSize = is.size();
+    char buf[GlobalId::LENGTH];
+    is.read(buf, sizeof(buf));
+    _gid.set(buf);
+    is.readSmallString(_docType);
+    _serializedDocSize = oldSize - is.size();
+}
+
+vespalib::string
+RemoveOperationWithGid::toString() const {
+    return make_string("RemoveGid(%s, %s, %s)",
+                       _gid.toString().c_str(), _docType.c_str(), docArgsToString().c_str());
+}
+
 } // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/removeoperation.h b/searchcore/src/vespa/searchcore/proton/feedoperation/removeoperation.h
index 8c10107d944..ea1e62b8651 100644
--- a/searchcore/src/vespa/searchcore/proton/feedoperation/removeoperation.h
+++ b/searchcore/src/vespa/searchcore/proton/feedoperation/removeoperation.h
@@ -7,22 +7,54 @@
 namespace proton {
 
 class RemoveOperation : public DocumentOperation {
+protected:
+    explicit RemoveOperation(Type type) : DocumentOperation(type) {}
+    RemoveOperation(Type type, document::BucketId bucketId, storage::spi::Timestamp timestamp)
+        : DocumentOperation(type, bucketId, timestamp)
+    {}
+public:
+    virtual bool hasDocType() const = 0;
+    virtual vespalib::stringref getDocType() const = 0;
+    virtual const document::GlobalId & getGlobalId() const = 0;
+};
+
+class RemoveOperationWithDocId : public RemoveOperation {
     document::DocumentId _docId;
 
 public:
-    RemoveOperation();
-    RemoveOperation(const document::BucketId &bucketId,
-                    const storage::spi::Timestamp &timestamp,
-                    const document::DocumentId &docId);
-    virtual ~RemoveOperation() {}
+    RemoveOperationWithDocId();
+    RemoveOperationWithDocId(document::BucketId bucketId,
+                             storage::spi::Timestamp timestamp,
+                             const document::DocumentId &docId);
+    ~RemoveOperationWithDocId() override;
     const document::DocumentId &getDocumentId() const { return _docId; }
-    virtual void serialize(vespalib::nbostream &os) const override;
-    virtual void deserialize(vespalib::nbostream &is,
-                             const document::DocumentTypeRepo &repo) override;
-    virtual vespalib::string toString() const override;
+    const document::GlobalId & getGlobalId() const override { return _docId.getGlobalId(); }
+    void serialize(vespalib::nbostream &os) const override;
+    void deserialize(vespalib::nbostream &is, const document::DocumentTypeRepo &repo) override;
+    vespalib::string toString() const override;
+
+    bool hasDocType() const override { return _docId.hasDocType(); }
+    vespalib::stringref getDocType() const override { return _docId.getDocType(); }
+};
+
+class RemoveOperationWithGid : public RemoveOperation {
+    document::GlobalId _gid;
+    vespalib::string   _docType;
+
+public:
+    RemoveOperationWithGid();
+    RemoveOperationWithGid(document::BucketId bucketId,
+                           storage::spi::Timestamp timestamp,
+                           const document::GlobalId & gid,
+                           vespalib::stringref docType);
+    ~RemoveOperationWithGid() override;
+    const document::GlobalId & getGlobalId() const override { return _gid; }
+    void serialize(vespalib::nbostream &os) const override;
+    void deserialize(vespalib::nbostream &is, const document::DocumentTypeRepo &repo) override;
+    vespalib::string toString() const override;
 
-    bool hasDocType() const { return _docId.hasDocType(); }
-    vespalib::string getDocType() const { return _docId.getDocType(); }
+    bool hasDocType() const override { return true; }
+    vespalib::stringref getDocType() const override { return _docType; }
 };
 
 } // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/matchengine/imatchhandler.h b/searchcore/src/vespa/searchcore/proton/matchengine/imatchhandler.h
deleted file mode 100644
index 63283de0a4a..00000000000
--- a/searchcore/src/vespa/searchcore/proton/matchengine/imatchhandler.h
+++ /dev/null
@@ -1,46 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-#pragma once
-
-#include <vespa/searchcore/proton/summaryengine/isearchhandler.h>
-#include <vespa/vespalib/util/thread_bundle.h>
-
-namespace searche {
-namespace engine {
-    class SearchRequest;
-    class SearchReply;
-}
-}
-
-namespace proton {
-
-/**
- * This interface describes a sync match operation handler. It is implemented by
- * the DocumentDB class, and used by the MatchEngine class to delegate
- * operations to the appropriate db.
- */
-class IMatchHandler {
-protected:
-    using SearchReply = search::engine::SearchReply;
-    using SearchRequest = search::engine::SearchRequest;
-    using ThreadBundle = vespalib::ThreadBundle;
-    IMatchHandler() = default;
-public:
-    IMatchHandler(const IMatchHandler &) = delete;
-    IMatchHandler & operator = (const IMatchHandler &) = delete;
-    /**
-     * Convenience typedefs.
-     */
-    typedef std::unique_ptr<IMatchHandler> UP;
-    typedef std::shared_ptr<IMatchHandler> SP;
-
-    virtual ~IMatchHandler() { }
-
-    /**
-     * @return Use the request and produce the matching result.
-     */
-    virtual std::unique_ptr<SearchReply>
-    match(const ISearchHandler::SP &searchHandler, const SearchRequest &req, ThreadBundle &threadBundle) const = 0;
-};
-
-} // namespace proton
-
diff --git a/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.cpp b/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.cpp
index 480359f8382..ad47fb98232 100644
--- a/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.cpp
+++ b/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.cpp
@@ -121,21 +121,20 @@ MatchEngine::performSearch(search::engine::SearchRequest::Source req,
         ISearchHandler::SP searchHandler;
         vespalib::SimpleThreadBundle::UP threadBundle = _threadBundlePool.obtain();
         { // try to find the match handler corresponding to the specified search doc type
-            std::lock_guard<std::mutex> guard(_lock);
             DocTypeName docTypeName(*searchRequest);
+            std::lock_guard<std::mutex> guard(_lock);
             searchHandler = _handlers.getHandler(docTypeName);
         }
         if (searchHandler) {
-            ret = searchHandler->match(searchHandler, *searchRequest, *threadBundle);
+            ret = searchHandler->match(*searchRequest, *threadBundle);
         } else {
-            HandlerMap<ISearchHandler>::Snapshot::UP snapshot;
+            HandlerMap<ISearchHandler>::Snapshot snapshot;
             {
                 std::lock_guard<std::mutex> guard(_lock);
                 snapshot = _handlers.snapshot();
             }
-            if (snapshot->valid()) {
-                ISearchHandler::SP handler = snapshot->getSP();
-                ret = handler->match(handler, *searchRequest, *threadBundle); // use the first handler
+            if (snapshot.valid()) {
+                ret = snapshot.get()->match(*searchRequest, *threadBundle); // use the first handler
             }
         }
         _threadBundlePool.release(std::move(threadBundle));
diff --git a/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.h b/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.h
index 896ac83403a..34aacdafcad 100644
--- a/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.h
+++ b/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.h
@@ -1,7 +1,7 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 #pragma once
 
-#include "imatchhandler.h"
+#include <vespa/searchcore/proton/summaryengine/isearchhandler.h>
 #include <vespa/searchcore/proton/common/doctypename.h>
 #include <vespa/searchcore/proton/common/handlermap.hpp>
 #include <vespa/searchcore/proton/common/statusreport.h>
@@ -50,7 +50,7 @@ public:
      * Frees any allocated resources. this will also stop all internal threads
      * and wait for them to finish. All pending search requests are deleted.
      */
-    ~MatchEngine();
+    ~MatchEngine() override;
 
     /**
      * Observe and reset internal executor stats
@@ -123,13 +123,11 @@ public:
 
     StatusReport::UP reportStatus() const;
 
-    // Implements SearchServer.
     search::engine::SearchReply::UP search(
             search::engine::SearchRequest::Source request,
             search::engine::SearchClient &client) override;
 
-    // Implements vespalib::StateExplorer
-    virtual void get_state(const vespalib::slime::Inserter &inserter, bool full) const override;
+    void get_state(const vespalib::slime::Inserter &inserter, bool full) const override;
 };
 
 } // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_thread.cpp b/searchcore/src/vespa/searchcore/proton/matching/match_thread.cpp
index 7c5e7584eed..0e80d31a063 100644
--- a/searchcore/src/vespa/searchcore/proton/matching/match_thread.cpp
+++ b/searchcore/src/vespa/searchcore/proton/matching/match_thread.cpp
@@ -35,7 +35,7 @@ namespace {
 struct WaitTimer {
     double &wait_time_s;
     vespalib::Timer wait_time;
-    WaitTimer(double &wait_time_s_in)
+    explicit WaitTimer(double &wait_time_s_in)
         : wait_time_s(wait_time_s_in), wait_time()
     { }
     void done() {
diff --git a/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp b/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp
index 6c97dc8c9ef..630ac66f4f1 100644
--- a/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp
+++ b/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp
@@ -287,7 +287,9 @@ Matcher::match(const SearchRequest &request, vespalib::ThreadBundle &threadBundl
             numThreadsPerSearch, _rankSetup->getNumThreadsPerSearch(), estHits, reply->totalHitCount,
             request.ranking.c_str());
     }
-    my_stats.queryCollateralTime(vespalib::to_s(total_matching_time.elapsed()) - my_stats.queryLatencyAvg());
+    double querySetupTime = vespalib::to_s(total_matching_time.elapsed()) - my_stats.queryLatencyAvg();
+    my_stats.queryCollateralTime(querySetupTime); // TODO: Remove in Vespa 8
+    my_stats.querySetupTime(querySetupTime);
     {
         vespalib::duration duration = request.getTimeUsed();
         std::lock_guard<std::mutex> guard(_statsLock);
diff --git a/searchcore/src/vespa/searchcore/proton/matching/matching_stats.cpp b/searchcore/src/vespa/searchcore/proton/matching/matching_stats.cpp
index fff8c94c8be..8cc98816241 100644
--- a/searchcore/src/vespa/searchcore/proton/matching/matching_stats.cpp
+++ b/searchcore/src/vespa/searchcore/proton/matching/matching_stats.cpp
@@ -29,7 +29,8 @@ MatchingStats::MatchingStats()
       _softDoomed(0),
       _doomOvertime(),
       _softDoomFactor(INITIAL_SOFT_DOOM_FACTOR),
-      _queryCollateralTime(),
+      _queryCollateralTime(), // TODO: Remove in Vespa 8
+      _querySetupTime(),
       _queryLatency(),
       _matchTime(),
       _groupingTime(),
@@ -69,7 +70,8 @@ MatchingStats::add(const MatchingStats &rhs)
     _softDoomed += rhs.softDoomed();
     _doomOvertime.add(rhs._doomOvertime);
 
-    _queryCollateralTime.add(rhs._queryCollateralTime);
+    _queryCollateralTime.add(rhs._queryCollateralTime); // TODO: Remove in Vespa 8
+    _querySetupTime.add(rhs._querySetupTime);
     _queryLatency.add(rhs._queryLatency);
     _matchTime.add(rhs._matchTime);
     _groupingTime.add(rhs._groupingTime);
diff --git a/searchcore/src/vespa/searchcore/proton/matching/matching_stats.h b/searchcore/src/vespa/searchcore/proton/matching/matching_stats.h
index f5eccdd1127..6b533aab7e5 100644
--- a/searchcore/src/vespa/searchcore/proton/matching/matching_stats.h
+++ b/searchcore/src/vespa/searchcore/proton/matching/matching_stats.h
@@ -125,7 +125,8 @@ private:
     size_t                 _softDoomed;
     Avg                    _doomOvertime;
     double                 _softDoomFactor;
-    Avg                    _queryCollateralTime;
+    Avg                    _queryCollateralTime; // TODO: Remove in Vespa 8
+    Avg                    _querySetupTime;
     Avg                    _queryLatency;
     Avg                    _matchTime;
     Avg                    _groupingTime;
@@ -168,12 +169,19 @@ public:
     double softDoomFactor() const { return _softDoomFactor; }
     MatchingStats &updatesoftDoomFactor(vespalib::duration hardLimit, vespalib::duration softLimit, vespalib::duration duration);
 
+    // TODO: Remove in Vespa 8
     MatchingStats &queryCollateralTime(double time_s) { _queryCollateralTime.set(time_s); return *this; }
     double queryCollateralTimeAvg() const { return _queryCollateralTime.avg(); }
     size_t queryCollateralTimeCount() const { return _queryCollateralTime.count(); }
     double queryCollateralTimeMin() const { return _queryCollateralTime.min(); }
     double queryCollateralTimeMax() const { return _queryCollateralTime.max(); }
 
+    MatchingStats &querySetupTime(double time_s) { _querySetupTime.set(time_s); return *this; }
+    double querySetupTimeAvg() const { return _querySetupTime.avg(); }
+    size_t querySetupTimeCount() const { return _querySetupTime.count(); }
+    double querySetupTimeMin() const { return _querySetupTime.min(); }
+    double querySetupTimeMax() const { return _querySetupTime.max(); }
+
     MatchingStats &queryLatency(double time_s) { _queryLatency.set(time_s); return *this; }
     double queryLatencyAvg() const { return _queryLatency.avg(); }
     size_t queryLatencyCount() const { return _queryLatency.count(); }
diff --git a/searchcore/src/vespa/searchcore/proton/matching/search_session.h b/searchcore/src/vespa/searchcore/proton/matching/search_session.h
index 0aec02e9d31..13d24624597 100644
--- a/searchcore/src/vespa/searchcore/proton/matching/search_session.h
+++ b/searchcore/src/vespa/searchcore/proton/matching/search_session.h
@@ -26,7 +26,7 @@ public:
         OwnershipBundle(OwnershipBundle &&) = default;
         OwnershipBundle & operator = (OwnershipBundle &&) = default;
         ~OwnershipBundle();
-        ISearchHandler::SP search_handler;
+        std::shared_ptr<const ISearchHandler> search_handler;
         std::unique_ptr<search::fef::Properties> feature_overrides;
         std::unique_ptr<MatchContext> context;
         IDocumentMetaStoreContext::IReadGuard::UP readGuard;
diff --git a/searchcore/src/vespa/searchcore/proton/metrics/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/metrics/CMakeLists.txt
index 6077a6ddd87..8a1dd7ea101 100644
--- a/searchcore/src/vespa/searchcore/proton/metrics/CMakeLists.txt
+++ b/searchcore/src/vespa/searchcore/proton/metrics/CMakeLists.txt
@@ -12,7 +12,6 @@ vespa_add_library(searchcore_proton_metrics STATIC
     job_tracker.cpp
     job_tracked_flush_target.cpp
     job_tracked_flush_task.cpp
-    memory_usage_metrics.cpp
     metrics_engine.cpp
     resource_usage_metrics.cpp
     sessionmanager_metrics.cpp
diff --git a/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.cpp b/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.cpp
index f4b8203f8e2..05370920354 100644
--- a/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.cpp
+++ b/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.cpp
@@ -119,6 +119,8 @@ DocumentDBTaggedMetrics::MatchingMetrics::update(const MatchingStats &stats)
     queries.inc(stats.queries());
     queryCollateralTime.addValueBatch(stats.queryCollateralTimeAvg(), stats.queryCollateralTimeCount(),
                                       stats.queryCollateralTimeMin(), stats.queryCollateralTimeMax());
+    querySetupTime.addValueBatch(stats.querySetupTimeAvg(), stats.querySetupTimeCount(),
+                                      stats.querySetupTimeMin(), stats.querySetupTimeMax());
     queryLatency.addValueBatch(stats.queryLatencyAvg(), stats.queryLatencyCount(),
                                stats.queryLatencyMin(), stats.queryLatencyMax());
 }
@@ -131,6 +133,7 @@ DocumentDBTaggedMetrics::MatchingMetrics::MatchingMetrics(MetricSet *parent)
       queries("queries", {}, "Number of queries executed", this),
       softDoomedQueries("soft_doomed_queries", {}, "Number of queries hitting the soft timeout", this),
       queryCollateralTime("query_collateral_time", {}, "Average time (sec) spent setting up and tearing down queries", this),
+      querySetupTime("query_setup_time", {}, "Average time (sec) spent setting up and tearing down queries", this),
       queryLatency("query_latency", {}, "Total average latency (sec) when matching and ranking a query", this)
 {
 }
@@ -152,6 +155,7 @@ DocumentDBTaggedMetrics::MatchingMetrics::RankProfileMetrics::RankProfileMetrics
       groupingTime("grouping_time", {}, "Average time (sec) spent on grouping", this),
       rerankTime("rerank_time", {}, "Average time (sec) spent on 2nd phase ranking", this),
       queryCollateralTime("query_collateral_time", {}, "Average time (sec) spent setting up and tearing down queries", this),
+      querySetupTime("query_setup_time", {}, "Average time (sec) spent setting up and tearing down queries", this),
       queryLatency("query_latency", {}, "Total average latency (sec) when matching and ranking a query", this)
 {
     softDoomFactor.set(MatchingStats::INITIAL_SOFT_DOOM_FACTOR);
@@ -204,6 +208,8 @@ DocumentDBTaggedMetrics::MatchingMetrics::RankProfileMetrics::update(const Match
                              stats.rerankTimeMin(), stats.rerankTimeMax());
     queryCollateralTime.addValueBatch(stats.queryCollateralTimeAvg(), stats.queryCollateralTimeCount(),
                                       stats.queryCollateralTimeMin(), stats.queryCollateralTimeMax());
+    querySetupTime.addValueBatch(stats.querySetupTimeAvg(), stats.querySetupTimeCount(),
+                                      stats.querySetupTimeMin(), stats.querySetupTimeMax());
     queryLatency.addValueBatch(stats.queryLatencyAvg(), stats.queryLatencyCount(),
                                stats.queryLatencyMin(), stats.queryLatencyMax());
     if (stats.getNumPartitions() > 0) {
diff --git a/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.h b/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.h
index 01ba271a08f..26dd52a8577 100644
--- a/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.h
+++ b/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.h
@@ -116,6 +116,7 @@ struct DocumentDBTaggedMetrics : metrics::MetricSet
         metrics::LongCountMetric queries;
         metrics::LongCountMetric softDoomedQueries;
         metrics::DoubleAverageMetric queryCollateralTime;
+        metrics::DoubleAverageMetric querySetupTime;
         metrics::DoubleAverageMetric queryLatency;
 
         struct RankProfileMetrics : metrics::MetricSet {
@@ -145,6 +146,7 @@ struct DocumentDBTaggedMetrics : metrics::MetricSet
             metrics::DoubleAverageMetric groupingTime;
             metrics::DoubleAverageMetric rerankTime;
             metrics::DoubleAverageMetric queryCollateralTime;
+            metrics::DoubleAverageMetric querySetupTime;
             metrics::DoubleAverageMetric queryLatency;
             DocIdPartitions              partitions;
 
diff --git a/searchcore/src/vespa/searchcore/proton/metrics/executor_metrics.cpp b/searchcore/src/vespa/searchcore/proton/metrics/executor_metrics.cpp
index 710c072aa53..d4204473578 100644
--- a/searchcore/src/vespa/searchcore/proton/metrics/executor_metrics.cpp
+++ b/searchcore/src/vespa/searchcore/proton/metrics/executor_metrics.cpp
@@ -7,16 +7,19 @@ namespace proton {
 void
 ExecutorMetrics::update(const vespalib::ThreadStackExecutorBase::Stats &stats)
 {
-    maxPending.set(stats.maxPendingTasks);
+    maxPending.set(stats.queueSize.max());
     accepted.inc(stats.acceptedTasks);
     rejected.inc(stats.rejectedTasks);
+    const auto & qSize = stats.queueSize;
+    queueSize.addValueBatch(qSize.average(), qSize.count(), qSize.min(), qSize.max());
 }
 
 ExecutorMetrics::ExecutorMetrics(const std::string &name, metrics::MetricSet *parent)
     : metrics::MetricSet(name, {}, "Instance specific thread executor metrics", parent),
       maxPending("maxpending", {}, "Maximum number of pending (active + queued) tasks", this),
       accepted("accepted", {}, "Number of accepted tasks", this),
-      rejected("rejected", {}, "Number of rejected tasks", this)
+      rejected("rejected", {}, "Number of rejected tasks", this),
+      queueSize("queuesize", {}, "Size of task queue", this)
 {
 }
 
diff --git a/searchcore/src/vespa/searchcore/proton/metrics/executor_metrics.h b/searchcore/src/vespa/searchcore/proton/metrics/executor_metrics.h
index a347edffd4b..6b638391d1e 100644
--- a/searchcore/src/vespa/searchcore/proton/metrics/executor_metrics.h
+++ b/searchcore/src/vespa/searchcore/proton/metrics/executor_metrics.h
@@ -11,9 +11,10 @@ namespace proton {
 
 struct ExecutorMetrics : metrics::MetricSet
 {
-    metrics::LongValueMetric maxPending;
+    metrics::LongValueMetric maxPending; // TODO Remove on Vespa 8 or sooner if possible.
     metrics::LongCountMetric accepted;
     metrics::LongCountMetric rejected;
+    metrics::LongAverageMetric queueSize;
 
     void update(const vespalib::ThreadStackExecutorBase::Stats &stats);
     ExecutorMetrics(const std::string &name, metrics::MetricSet *parent);
diff --git a/searchcore/src/vespa/searchcore/proton/metrics/memory_usage_metrics.cpp b/searchcore/src/vespa/searchcore/proton/metrics/memory_usage_metrics.cpp
deleted file mode 100644
index 1f687d14969..00000000000
--- a/searchcore/src/vespa/searchcore/proton/metrics/memory_usage_metrics.cpp
+++ /dev/null
@@ -1,28 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#include "memory_usage_metrics.h"
-#include <vespa/vespalib/util/memoryusage.h>
-
-namespace proton {
-
-MemoryUsageMetrics::MemoryUsageMetrics(metrics::MetricSet *parent)
-    : metrics::MetricSet("memory_usage", {}, "The memory usage for a given component", parent),
-      _allocatedBytes("allocated_bytes", {}, "The number of allocated bytes", this),
-      _usedBytes("used_bytes", {}, "The number of used bytes (<= allocatedbytes)", this),
-      _deadBytes("dead_bytes", {}, "The number of dead bytes (<= usedbytes)", this),
-      _onHoldBytes("onhold_bytes", {}, "The number of bytes on hold", this)
-{
-}
-
-MemoryUsageMetrics::~MemoryUsageMetrics() {}
-
-void
-MemoryUsageMetrics::update(const vespalib::MemoryUsage &usage)
-{
-    _allocatedBytes.set(usage.allocatedBytes());
-    _usedBytes.set(usage.usedBytes());
-    _deadBytes.set(usage.deadBytes());
-    _onHoldBytes.set(usage.allocatedBytesOnHold());
-}
-
-}
diff --git a/searchcore/src/vespa/searchcore/proton/metrics/memory_usage_metrics.h b/searchcore/src/vespa/searchcore/proton/metrics/memory_usage_metrics.h
index 89177c3a359..e82e55848a5 100644
--- a/searchcore/src/vespa/searchcore/proton/metrics/memory_usage_metrics.h
+++ b/searchcore/src/vespa/searchcore/proton/metrics/memory_usage_metrics.h
@@ -2,27 +2,12 @@
 
 #pragma once
 
-#include <vespa/metrics/metrics.h>
+#include <vespa/metrics/common/memory_usage_metrics.h>
 
 namespace vespalib { class MemoryUsage; }
 
 namespace proton {
 
-/**
- * Metric set for memory usage metrics.
- */
-class MemoryUsageMetrics : public metrics::MetricSet
-{
-private:
-    metrics::LongValueMetric _allocatedBytes;
-    metrics::LongValueMetric _usedBytes;
-    metrics::LongValueMetric _deadBytes;
-    metrics::LongValueMetric _onHoldBytes;
-
-public:
-    MemoryUsageMetrics(metrics::MetricSet *parent);
-    ~MemoryUsageMetrics();
-    void update(const vespalib::MemoryUsage &usage);
-};
+using MemoryUsageMetrics = metrics::MemoryUsageMetrics;
 
 } // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/metrics/metrics_engine.cpp b/searchcore/src/vespa/searchcore/proton/metrics/metrics_engine.cpp
index d34ec8d05a8..bf97bdda29a 100644
--- a/searchcore/src/vespa/searchcore/proton/metrics/metrics_engine.cpp
+++ b/searchcore/src/vespa/searchcore/proton/metrics/metrics_engine.cpp
@@ -106,7 +106,7 @@ void
 doCleanAttributes(AttributeMetrics &attributes)
 {
     auto entries = attributes.release();
-    for (const auto entry : entries) {
+    for (const auto &entry : entries) {
         attributes.parent()->unregisterMetric(*entry);
     }
 }
diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/document_iterator.cpp b/searchcore/src/vespa/searchcore/proton/persistenceengine/document_iterator.cpp
index 1bcbe4e9683..8175e612bd4 100644
--- a/searchcore/src/vespa/searchcore/proton/persistenceengine/document_iterator.cpp
+++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/document_iterator.cpp
@@ -1,6 +1,8 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include "document_iterator.h"
+#include <vespa/searchcore/proton/common/cachedselect.h>
+#include <vespa/searchcore/proton/common/selectcontext.h>
 #include <vespa/document/select/gid_filter.h>
 #include <vespa/document/select/node.h>
 #include <vespa/document/fieldvalue/document.h>
@@ -42,13 +44,6 @@ DocEntry *createDocEntry(Timestamp timestamp, bool removed, Document::UP doc, ss
 } // namespace proton::<unnamed>
 
 bool
-DocumentIterator::useDocumentSelection() const
-{
-    return (!_metaOnly &&
-            !_selection.getDocumentSelection().getDocumentSelection().empty());
-}
-
-bool
 DocumentIterator::checkMeta(const search::DocumentMetaData &meta) const
 {
     if (!meta.valid()) {
diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/document_iterator.h b/searchcore/src/vespa/searchcore/proton/persistenceengine/document_iterator.h
index 285b3cb2afa..67242e8220f 100644
--- a/searchcore/src/vespa/searchcore/proton/persistenceengine/document_iterator.h
+++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/document_iterator.h
@@ -3,8 +3,6 @@
 #pragma once
 
 #include "i_document_retriever.h"
-#include <vespa/searchcore/proton/common/cachedselect.h>
-#include <vespa/searchcore/proton/common/selectcontext.h>
 #include <vespa/searchlib/common/idocumentmetastore.h>
 #include <vespa/persistence/spi/bucket.h>
 #include <vespa/persistence/spi/selection.h>
@@ -32,10 +30,7 @@ private:
     storage::spi::IterateResult::List     _list;
 
 
-    bool useDocumentSelection() const;
     bool checkMeta(const search::DocumentMetaData &meta) const;
-    bool checkDoc(const document::Document &doc) const;
-    bool checkDoc(const SelectContext &sc) const;
     void fetchCompleteSource(const IDocumentRetriever & source, storage::spi::IterateResult::List & list);
     bool isWeakRead() const { return _readConsistency == ReadConsistency::WEAK; }
 
diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistence_handler_map.cpp b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistence_handler_map.cpp
index 06969167ab3..5b62a290353 100644
--- a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistence_handler_map.cpp
+++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistence_handler_map.cpp
@@ -20,15 +20,15 @@ PersistenceHandlerMap::putHandler(document::BucketSpace bucketSpace,
     return _map[bucketSpace].putHandler(docType, handler);
 }
 
-IPersistenceHandler::SP
+IPersistenceHandler *
 PersistenceHandlerMap::getHandler(document::BucketSpace bucketSpace,
                                   const DocTypeName &docType) const
 {
     auto itr = _map.find(bucketSpace);
     if (itr != _map.end()) {
-        return itr->second.getHandler(docType);
+        return itr->second.getHandlerPtr(docType);
     }
-    return IPersistenceHandler::SP();
+    return nullptr;
 }
 
 IPersistenceHandler::SP
@@ -42,7 +42,7 @@ PersistenceHandlerMap::removeHandler(document::BucketSpace bucketSpace,
     return IPersistenceHandler::SP();
 }
 
-HandlerSnapshot::UP
+HandlerSnapshot
 PersistenceHandlerMap::getHandlerSnapshot() const
 {
     std::vector<IPersistenceHandler::SP> handlers;
@@ -52,47 +52,17 @@ PersistenceHandlerMap::getHandlerSnapshot() const
         }
     }
     size_t handlersSize = handlers.size();
-    return std::make_unique<HandlerSnapshot>
-            (std::make_unique<DocTypeToHandlerMap::Snapshot>(std::move(handlers)),
-             handlersSize);
-}
-
-namespace {
-
-struct EmptySequence : public vespalib::Sequence<IPersistenceHandler *> {
-    virtual bool valid() const override { return false; }
-    virtual IPersistenceHandler *get() const override { return nullptr; }
-    virtual void next() override { }
-    static EmptySequence::UP make() { return std::make_unique<EmptySequence>(); }
-};
-
+    return HandlerSnapshot(DocTypeToHandlerMap::Snapshot(std::move(handlers)), handlersSize);
 }
 
-HandlerSnapshot::UP
+HandlerSnapshot
 PersistenceHandlerMap::getHandlerSnapshot(document::BucketSpace bucketSpace) const
 {
     auto itr = _map.find(bucketSpace);
     if (itr != _map.end()) {
-        return std::make_unique<HandlerSnapshot>(itr->second.snapshot(), itr->second.size());
+        return HandlerSnapshot(itr->second.snapshot(), itr->second.size());
     }
-    return std::make_unique<HandlerSnapshot>(EmptySequence::make(), 0);
-}
-
-namespace {
-
-class SequenceOfOne : public vespalib::Sequence<IPersistenceHandler *> {
-private:
-    bool _done;
-    IPersistenceHandler *_value;
-public:
-    SequenceOfOne(IPersistenceHandler *value) : _done(false), _value(value) {}
-
-    virtual bool valid() const override { return !_done; }
-    virtual IPersistenceHandler *get() const override { return _value; }
-    virtual void next() override { _done = true; }
-    static SequenceOfOne::UP make(IPersistenceHandler *value) { return std::make_unique<SequenceOfOne>(value); }
-};
-
+    return HandlerSnapshot();
 }
 
 }
diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistence_handler_map.h b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistence_handler_map.h
index 003c378d6a7..64241e1ad2b 100644
--- a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistence_handler_map.h
+++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistence_handler_map.h
@@ -20,16 +20,17 @@ class IPersistenceHandler;
  */
 class PersistenceHandlerMap {
 public:
-    using PersistenceHandlerSequence = vespalib::Sequence<IPersistenceHandler *>;
+    using DocTypeToHandlerMap = HandlerMap<IPersistenceHandler>;
+    using PersistenceHandlerSequence = DocTypeToHandlerMap::Snapshot;
     using PersistenceHandlerSP = std::shared_ptr<IPersistenceHandler>;
 
     class HandlerSnapshot {
     private:
-        PersistenceHandlerSequence::UP _handlers;
-        size_t                         _size;
+        PersistenceHandlerSequence  _handlers;
+        size_t                      _size;
     public:
-        using UP = std::unique_ptr<HandlerSnapshot>;
-        HandlerSnapshot(PersistenceHandlerSequence::UP handlers_, size_t size_)
+        HandlerSnapshot() : _handlers(), _size(0) {}
+        HandlerSnapshot(DocTypeToHandlerMap::Snapshot handlers_, size_t size_)
             : _handlers(std::move(handlers_)),
               _size(size_)
         {}
@@ -37,12 +38,12 @@ public:
         HandlerSnapshot & operator = (const HandlerSnapshot &) = delete;
 
         size_t size() const { return _size; }
-        PersistenceHandlerSequence &handlers() { return *_handlers; }
-        static PersistenceHandlerSequence::UP release(HandlerSnapshot &&rhs) { return std::move(rhs._handlers); }
+        PersistenceHandlerSequence &handlers() { return _handlers; }
+        static PersistenceHandlerSequence release(HandlerSnapshot &&rhs) { return std::move(rhs._handlers); }
     };
 
 private:
-    using DocTypeToHandlerMap = HandlerMap<IPersistenceHandler>;
+
 
     struct BucketSpaceHash {
         std::size_t operator() (const document::BucketSpace &bucketSpace) const { return bucketSpace.getId(); }
@@ -53,15 +54,11 @@ private:
 public:
     PersistenceHandlerMap();
 
-    PersistenceHandlerSP putHandler(document::BucketSpace bucketSpace,
-                                    const DocTypeName &docType,
-                                    const PersistenceHandlerSP &handler);
-    PersistenceHandlerSP removeHandler(document::BucketSpace bucketSpace,
-                                       const DocTypeName &docType);
-    PersistenceHandlerSP getHandler(document::BucketSpace bucketSpace,
-                                    const DocTypeName &docType) const;
-    HandlerSnapshot::UP getHandlerSnapshot() const;
-    HandlerSnapshot::UP getHandlerSnapshot(document::BucketSpace bucketSpace) const;
+    PersistenceHandlerSP putHandler(document::BucketSpace bucketSpace, const DocTypeName &docType, const PersistenceHandlerSP &handler);
+    PersistenceHandlerSP removeHandler(document::BucketSpace bucketSpace, const DocTypeName &docType);
+    IPersistenceHandler * getHandler(document::BucketSpace bucketSpace, const DocTypeName &docType) const;
+    HandlerSnapshot getHandlerSnapshot() const;
+    HandlerSnapshot getHandlerSnapshot(document::BucketSpace bucketSpace) const;
 };
 
 }
diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp
index 1f862b07048..2ede5d45f7e 100644
--- a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp
+++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp
@@ -40,7 +40,7 @@ protected:
     std::mutex               _lock;
     vespalib::CountDownLatch _latch;
 public:
-    ResultHandlerBase(uint32_t waitCnt);
+    explicit ResultHandlerBase(uint32_t waitCnt);
     ~ResultHandlerBase();
     void await() { _latch.await(); }
 };
@@ -55,11 +55,11 @@ class GenericResultHandler : public ResultHandlerBase, public IGenericResultHand
 private:
     Result _result;
 public:
-    GenericResultHandler(uint32_t waitCnt) :
+    explicit GenericResultHandler(uint32_t waitCnt) :
         ResultHandlerBase(waitCnt),
         _result()
     { }
-    ~GenericResultHandler();
+    ~GenericResultHandler() override;
     void handle(const Result &result) override {
         if (result.hasError()) {
             std::lock_guard<std::mutex> guard(_lock);
@@ -109,7 +109,7 @@ class SynchronizedBucketIdListResultHandler : public ResultHandlerBase,
                                               public BucketIdListResultHandler
 {
 public:
-    SynchronizedBucketIdListResultHandler(uint32_t waitCnt)
+    explicit SynchronizedBucketIdListResultHandler(uint32_t waitCnt)
         : ResultHandlerBase(waitCnt),
           BucketIdListResultHandler()
     { }
@@ -163,17 +163,21 @@ BucketInfoResultHandler::~BucketInfoResultHandler() = default;
 
 }
 
-PersistenceEngine::HandlerSnapshot::UP
-PersistenceEngine::getHandlerSnapshot() const
+PersistenceEngine::HandlerSnapshot
+PersistenceEngine::getHandlerSnapshot(const WriteGuard &) const
 {
-    std::lock_guard<std::mutex> guard(_lock);
     return _handlers.getHandlerSnapshot();
 }
 
-PersistenceEngine::HandlerSnapshot::UP
-PersistenceEngine::getHandlerSnapshot(document::BucketSpace bucketSpace) const
+PersistenceEngine::HandlerSnapshot
+PersistenceEngine::getHandlerSnapshot(const ReadGuard &, document::BucketSpace bucketSpace) const
+{
+    return _handlers.getHandlerSnapshot(bucketSpace);
+}
+
+PersistenceEngine::HandlerSnapshot
+PersistenceEngine::getHandlerSnapshot(const WriteGuard &, document::BucketSpace bucketSpace) const
 {
-    std::lock_guard<std::mutex> guard(_lock);
     return _handlers.getHandlerSnapshot(bucketSpace);
 }
 
@@ -202,27 +206,23 @@ PersistenceEngine::~PersistenceEngine()
 
 
 IPersistenceHandler::SP
-PersistenceEngine::putHandler(document::BucketSpace bucketSpace, const DocTypeName &docType,
-                              const IPersistenceHandler::SP &handler)
+PersistenceEngine::putHandler(const WriteGuard &, document::BucketSpace bucketSpace, const DocTypeName &docType,const IPersistenceHandler::SP &handler)
 {
-    std::lock_guard<std::mutex> guard(_lock);
     return _handlers.putHandler(bucketSpace, docType, handler);
 }
 
 
-IPersistenceHandler::SP
-PersistenceEngine::getHandler(document::BucketSpace bucketSpace, const DocTypeName &docType) const
+IPersistenceHandler *
+PersistenceEngine::getHandler(const ReadGuard &, document::BucketSpace bucketSpace, const DocTypeName &docType) const
 {
-    std::lock_guard<std::mutex> guard(_lock);
     return _handlers.getHandler(bucketSpace, docType);
 }
 
 
 IPersistenceHandler::SP
-PersistenceEngine::removeHandler(document::BucketSpace bucketSpace, const DocTypeName &docType)
+PersistenceEngine::removeHandler(const WriteGuard &, document::BucketSpace bucketSpace, const DocTypeName &docType)
 {
     // TODO: Grab bucket list and treat them as modified
-    std::lock_guard<std::mutex> guard(_lock);
     return _handlers.removeHandler(bucketSpace, docType);
 }
 
@@ -232,9 +232,9 @@ PersistenceEngine::initialize()
 {
     std::unique_lock<std::shared_timed_mutex> wguard(getWLock());
     LOG(debug, "Begin initializing persistence handlers");
-    HandlerSnapshot::UP snap = getHandlerSnapshot();
-    for (; snap->handlers().valid(); snap->handlers().next()) {
-        IPersistenceHandler *handler = snap->handlers().get();
+    HandlerSnapshot snap = getHandlerSnapshot(wguard);
+    for (; snap.handlers().valid(); snap.handlers().next()) {
+        IPersistenceHandler *handler = snap.handlers().get();
         handler->initialize();
     }
     LOG(debug, "Done initializing persistence handlers");
@@ -260,10 +260,10 @@ PersistenceEngine::listBuckets(BucketSpace bucketSpace, PartitionId id) const
         BucketIdListResult::List emptyList;
         return BucketIdListResult(emptyList);
     }
-    HandlerSnapshot::UP snap = getHandlerSnapshot(bucketSpace);
+    HandlerSnapshot snap = getHandlerSnapshot(rguard, bucketSpace);
     BucketIdListResultHandler resultHandler;
-    for (; snap->handlers().valid(); snap->handlers().next()) {
-        IPersistenceHandler *handler = snap->handlers().get();
+    for (; snap.handlers().valid(); snap.handlers().next()) {
+        IPersistenceHandler *handler = snap.handlers().get();
         handler->handleListBuckets(resultHandler);
     }
     return resultHandler.getResult();
@@ -275,10 +275,10 @@ PersistenceEngine::setClusterState(BucketSpace bucketSpace, const ClusterState &
 {
     std::shared_lock<std::shared_timed_mutex> rguard(_rwMutex);
     saveClusterState(bucketSpace, calc);
-    HandlerSnapshot::UP snap = getHandlerSnapshot(bucketSpace);
-    GenericResultHandler resultHandler(snap->size());
-    for (; snap->handlers().valid(); snap->handlers().next()) {
-        IPersistenceHandler *handler = snap->handlers().get();
+    HandlerSnapshot snap = getHandlerSnapshot(rguard, bucketSpace);
+    GenericResultHandler resultHandler(snap.size());
+    for (; snap.handlers().valid(); snap.handlers().next()) {
+        IPersistenceHandler *handler = snap.handlers().get();
         handler->handleSetClusterState(calc, resultHandler);
     }
     resultHandler.await();
@@ -292,10 +292,10 @@ PersistenceEngine::setActiveState(const Bucket& bucket,
                                   storage::spi::BucketInfo::ActiveState newState)
 {
     std::shared_lock<std::shared_timed_mutex> rguard(_rwMutex);
-    HandlerSnapshot::UP snap = getHandlerSnapshot(bucket.getBucketSpace());
-    GenericResultHandler resultHandler(snap->size());
-    for (; snap->handlers().valid(); snap->handlers().next()) {
-        IPersistenceHandler *handler = snap->handlers().get();
+    HandlerSnapshot snap = getHandlerSnapshot(rguard, bucket.getBucketSpace());
+    GenericResultHandler resultHandler(snap.size());
+    for (; snap.handlers().valid(); snap.handlers().next()) {
+        IPersistenceHandler *handler = snap.handlers().get();
         handler->handleSetActiveState(bucket, newState, resultHandler);
     }
     resultHandler.await();
@@ -309,10 +309,10 @@ PersistenceEngine::getBucketInfo(const Bucket& b) const
     // Runs in SPI thread.
     // No handover to write threads in persistence handlers.
     std::shared_lock<std::shared_timed_mutex> rguard(_rwMutex);
-    HandlerSnapshot::UP snap = getHandlerSnapshot(b.getBucketSpace());
+    HandlerSnapshot snap = getHandlerSnapshot(rguard, b.getBucketSpace());
     BucketInfoResultHandler resultHandler;
-    for (; snap->handlers().valid(); snap->handlers().next()) {
-        IPersistenceHandler *handler = snap->handlers().get();
+    for (; snap.handlers().valid(); snap.handlers().next()) {
+        IPersistenceHandler *handler = snap.handlers().get();
         handler->handleGetBucketInfo(b, resultHandler);
     }
     return resultHandler.getResult();
@@ -338,7 +338,7 @@ PersistenceEngine::put(const Bucket& b, Timestamp t, const document::Document::S
         return Result(Result::ErrorType::PERMANENT_ERROR,
                       make_string("Old id scheme not supported in elastic mode (%s)", doc->getId().toString().c_str()));
     }
-    IPersistenceHandler::SP handler = getHandler(b.getBucketSpace(), docType);
+    IPersistenceHandler * handler = getHandler(rguard, b.getBucketSpace(), docType);
     if (!handler) {
         return Result(Result::ErrorType::PERMANENT_ERROR,
                       make_string("No handler for document type '%s'", docType.toString().c_str()));
@@ -360,7 +360,7 @@ PersistenceEngine::remove(const Bucket& b, Timestamp t, const DocumentId& did, C
                             make_string("Old id scheme not supported in elastic mode (%s)", did.toString().c_str()));
     }
     DocTypeName docType(did.getDocType());
-    IPersistenceHandler::SP handler = getHandler(b.getBucketSpace(), docType);
+    IPersistenceHandler * handler = getHandler(rguard, b.getBucketSpace(), docType);
     if (!handler) {
         return RemoveResult(Result::ErrorType::PERMANENT_ERROR,
                             make_string("No handler for document type '%s'", docType.toString().c_str()));
@@ -414,7 +414,7 @@ PersistenceEngine::update(const Bucket& b, Timestamp t, const DocumentUpdate::SP
         return UpdateResult(Result::ErrorType::PERMANENT_ERROR,
                             make_string("Update operation rejected due to bad id (%s, %s)", upd->getId().toString().c_str(), docType.getName().c_str()));
     }
-    IPersistenceHandler::SP handler = getHandler(b.getBucketSpace(), docType);
+    IPersistenceHandler * handler = getHandler(rguard, b.getBucketSpace(), docType);
 
     if (handler) {
         TransportLatch latch(1);
@@ -432,9 +432,9 @@ PersistenceEngine::GetResult
 PersistenceEngine::get(const Bucket& b, const document::FieldSet& fields, const DocumentId& did, Context& context) const
 {
     std::shared_lock<std::shared_timed_mutex> rguard(_rwMutex);
-    HandlerSnapshot::UP snapshot = getHandlerSnapshot(b.getBucketSpace());
+    HandlerSnapshot snapshot = getHandlerSnapshot(rguard, b.getBucketSpace());
 
-    for (PersistenceHandlerSequence & handlers = snapshot->handlers(); handlers.valid(); handlers.next()) {
+    for (PersistenceHandlerSequence & handlers = snapshot.handlers(); handlers.valid(); handlers.next()) {
         BucketGuard::UP bucket_guard = handlers.get()->lockBucket(b);
         IPersistenceHandler::RetrieversSP retrievers = handlers.get()->getDocumentRetrievers(context.getReadConsistency());
         for (size_t i = 0; i < retrievers->size(); ++i) {
@@ -462,19 +462,19 @@ PersistenceEngine::createIterator(const Bucket &bucket, const document::FieldSet
                                   IncludedVersions versions, Context & context)
 {
     std::shared_lock<std::shared_timed_mutex> rguard(_rwMutex);
-    HandlerSnapshot::UP snapshot = getHandlerSnapshot(bucket.getBucketSpace());
+    HandlerSnapshot snapshot = getHandlerSnapshot(rguard, bucket.getBucketSpace());
 
     auto entry = std::make_unique<IteratorEntry>(context.getReadConsistency(), bucket, fields, selection,
                                                  versions, _defaultSerializedSize, _ignoreMaxBytes);
-    entry->bucket_guards.reserve(snapshot->size());
-    for (PersistenceHandlerSequence & handlers = snapshot->handlers(); handlers.valid(); handlers.next()) {
+    entry->bucket_guards.reserve(snapshot.size());
+    for (PersistenceHandlerSequence & handlers = snapshot.handlers(); handlers.valid(); handlers.next()) {
         entry->bucket_guards.push_back(handlers.get()->lockBucket(bucket));
         IPersistenceHandler::RetrieversSP retrievers = handlers.get()->getDocumentRetrievers(context.getReadConsistency());
         for (size_t i = 0; i < retrievers->size(); ++i) {
             entry->it.add((*retrievers)[i]);
         }
     }
-    entry->handler_sequence = HandlerSnapshot::release(std::move(*snapshot));
+    entry->handler_sequence = HandlerSnapshot::release(std::move(snapshot));
 
     std::lock_guard<std::mutex> guard(_iterators_lock);
     static IteratorId id_counter(0);
@@ -541,10 +541,10 @@ PersistenceEngine::createBucket(const Bucket &b, Context &)
 {
     std::shared_lock<std::shared_timed_mutex> rguard(_rwMutex);
     LOG(spam, "createBucket(%s)", b.toString().c_str());
-    HandlerSnapshot::UP snap = getHandlerSnapshot(b.getBucketSpace());
-    TransportLatch latch(snap->size());
-    for (; snap->handlers().valid(); snap->handlers().next()) {
-        IPersistenceHandler *handler = snap->handlers().get();
+    HandlerSnapshot snap = getHandlerSnapshot(rguard, b.getBucketSpace());
+    TransportLatch latch(snap.size());
+    for (; snap.handlers().valid(); snap.handlers().next()) {
+        IPersistenceHandler *handler = snap.handlers().get();
         handler->handleCreateBucket(feedtoken::make(latch), b);
     }
     latch.await();
@@ -557,10 +557,10 @@ PersistenceEngine::deleteBucket(const Bucket& b, Context&)
 {
     std::shared_lock<std::shared_timed_mutex> rguard(_rwMutex);
     LOG(spam, "deleteBucket(%s)", b.toString().c_str());
-    HandlerSnapshot::UP snap = getHandlerSnapshot(b.getBucketSpace());
-    TransportLatch latch(snap->size());
-    for (; snap->handlers().valid(); snap->handlers().next()) {
-        IPersistenceHandler *handler = snap->handlers().get();
+    HandlerSnapshot snap = getHandlerSnapshot(rguard, b.getBucketSpace());
+    TransportLatch latch(snap.size());
+    for (; snap.handlers().valid(); snap.handlers().next()) {
+        IPersistenceHandler *handler = snap.handlers().get();
         handler->handleDeleteBucket(feedtoken::make(latch), b);
     }
     latch.await();
@@ -578,10 +578,10 @@ PersistenceEngine::getModifiedBuckets(BucketSpace bucketSpace) const
         std::lock_guard<std::mutex> guard(_lock);
         extraModifiedBuckets.swap(_extraModifiedBuckets[bucketSpace]);
     }
-    HandlerSnapshot::UP snap = getHandlerSnapshot(bucketSpace);
-    SynchronizedBucketIdListResultHandler resultHandler(snap->size() + extraModifiedBuckets.size());
-    for (; snap->handlers().valid(); snap->handlers().next()) {
-        IPersistenceHandler *handler = snap->handlers().get();
+    HandlerSnapshot snap = getHandlerSnapshot(rguard, bucketSpace);
+    SynchronizedBucketIdListResultHandler resultHandler(snap.size() + extraModifiedBuckets.size());
+    for (; snap.handlers().valid(); snap.handlers().next()) {
+        IPersistenceHandler *handler = snap.handlers().get();
         handler->handleGetModifiedBuckets(resultHandler);
     }
     for (const auto & item : extraModifiedBuckets) {
@@ -599,10 +599,10 @@ PersistenceEngine::split(const Bucket& source, const Bucket& target1, const Buck
     LOG(spam, "split(%s, %s, %s)", source.toString().c_str(), target1.toString().c_str(), target2.toString().c_str());
     assert(source.getBucketSpace() == target1.getBucketSpace());
     assert(source.getBucketSpace() == target2.getBucketSpace());
-    HandlerSnapshot::UP snap = getHandlerSnapshot(source.getBucketSpace());
-    TransportLatch latch(snap->size());
-    for (; snap->handlers().valid(); snap->handlers().next()) {
-        IPersistenceHandler *handler = snap->handlers().get();
+    HandlerSnapshot snap = getHandlerSnapshot(rguard, source.getBucketSpace());
+    TransportLatch latch(snap.size());
+    for (; snap.handlers().valid(); snap.handlers().next()) {
+        IPersistenceHandler *handler = snap.handlers().get();
         handler->handleSplit(feedtoken::make(latch), source, target1, target2);
     }
     latch.await();
@@ -617,10 +617,10 @@ PersistenceEngine::join(const Bucket& source1, const Bucket& source2, const Buck
     LOG(spam, "join(%s, %s, %s)", source1.toString().c_str(), source2.toString().c_str(), target.toString().c_str());
     assert(source1.getBucketSpace() == target.getBucketSpace());
     assert(source2.getBucketSpace() == target.getBucketSpace());
-    HandlerSnapshot::UP snap = getHandlerSnapshot(target.getBucketSpace());
-    TransportLatch latch(snap->size());
-    for (; snap->handlers().valid(); snap->handlers().next()) {
-        IPersistenceHandler *handler = snap->handlers().get();
+    HandlerSnapshot snap = getHandlerSnapshot(rguard, target.getBucketSpace());
+    TransportLatch latch(snap.size());
+    for (; snap.handlers().valid(); snap.handlers().next()) {
+        IPersistenceHandler *handler = snap.handlers().get();
         handler->handleJoin(feedtoken::make(latch), source1, source2, target);
     }
     latch.await();
@@ -722,19 +722,18 @@ public:
 };
 
 void
-PersistenceEngine::populateInitialBucketDB(BucketSpace bucketSpace,
-                                           IPersistenceHandler &targetHandler)
+PersistenceEngine::populateInitialBucketDB(const WriteGuard & guard, BucketSpace bucketSpace, IPersistenceHandler &targetHandler)
 {
-    HandlerSnapshot::UP snap = getHandlerSnapshot(bucketSpace);
+    HandlerSnapshot snap = getHandlerSnapshot(guard, bucketSpace);
     
-    size_t snapSize(snap->size());
+    size_t snapSize(snap.size());
     size_t flawed = 0;
 
     // handleListActiveBuckets() runs in SPI thread.
     // No handover to write threads in persistence handlers.
     ActiveBucketIdListResultHandler resultHandler;
-    for (; snap->handlers().valid(); snap->handlers().next()) {
-        IPersistenceHandler *handler = snap->handlers().get();
+    for (; snap.handlers().valid(); snap.handlers().next()) {
+        IPersistenceHandler *handler = snap.handlers().get();
         handler->handleListActiveBuckets(resultHandler);
     }
     typedef std::map<document::BucketId, size_t> BucketIdMap;
diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.h b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.h
index a6c696d08fb..5d3be07c532 100644
--- a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.h
+++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.h
@@ -5,12 +5,9 @@
 #include "i_resource_write_filter.h"
 #include "persistence_handler_map.h"
 #include "ipersistencehandler.h"
-#include <vespa/document/bucket/bucketspace.h>
 #include <vespa/persistence/spi/abstractpersistenceprovider.h>
-#include <vespa/searchcore/proton/common/handlermap.hpp>
 #include <mutex>
 #include <shared_mutex>
-#include <unordered_map>
 
 namespace proton {
 
@@ -18,7 +15,7 @@ class IPersistenceEngineOwner;
 
 class PersistenceEngine : public storage::spi::AbstractPersistenceProvider {
 private:
-    using PersistenceHandlerSequence = vespalib::Sequence<IPersistenceHandler *>;
+    using PersistenceHandlerSequence = PersistenceHandlerMap::PersistenceHandlerSequence;
     using HandlerSnapshot = PersistenceHandlerMap::HandlerSnapshot;
     using DocumentUpdate = document::DocumentUpdate;
     using Bucket = storage::spi::Bucket;
@@ -43,7 +40,7 @@ private:
     using UpdateResult = storage::spi::UpdateResult;
 
     struct IteratorEntry {
-        PersistenceHandlerSequence::UP handler_sequence;
+        PersistenceHandlerSequence  handler_sequence;
         DocumentIterator it;
         bool in_use;
         std::vector<BucketGuard::UP> bucket_guards;
@@ -75,9 +72,13 @@ private:
     mutable ExtraModifiedBuckets            _extraModifiedBuckets;
     mutable std::shared_timed_mutex         _rwMutex;
 
-    IPersistenceHandler::SP getHandler(document::BucketSpace bucketSpace, const DocTypeName &docType) const;
-    HandlerSnapshot::UP getHandlerSnapshot() const;
-    HandlerSnapshot::UP getHandlerSnapshot(document::BucketSpace bucketSpace) const;
+    using ReadGuard = std::shared_lock<std::shared_timed_mutex>;
+    using WriteGuard = std::unique_lock<std::shared_timed_mutex>;
+
+    IPersistenceHandler * getHandler(const ReadGuard & guard, document::BucketSpace bucketSpace, const DocTypeName &docType) const;
+    HandlerSnapshot getHandlerSnapshot(const WriteGuard & guard) const;
+    HandlerSnapshot getHandlerSnapshot(const ReadGuard & guard, document::BucketSpace bucketSpace) const;
+    HandlerSnapshot getHandlerSnapshot(const WriteGuard & guard, document::BucketSpace bucketSpace) const;
 
     void saveClusterState(BucketSpace bucketSpace, const ClusterState &calc);
     ClusterState::SP savedClusterState(BucketSpace bucketSpace) const;
@@ -89,9 +90,8 @@ public:
                       ssize_t defaultSerializedSize, bool ignoreMaxBytes);
     ~PersistenceEngine() override;
 
-    IPersistenceHandler::SP putHandler(document::BucketSpace bucketSpace, const DocTypeName &docType,
-                                       const IPersistenceHandler::SP &handler);
-    IPersistenceHandler::SP removeHandler(document::BucketSpace bucketSpace, const DocTypeName &docType);
+    IPersistenceHandler::SP putHandler(const WriteGuard &, document::BucketSpace bucketSpace, const DocTypeName &docType, const IPersistenceHandler::SP &handler);
+    IPersistenceHandler::SP removeHandler(const WriteGuard &, document::BucketSpace bucketSpace, const DocTypeName &docType);
 
     // Implements PersistenceProvider
     Result initialize() override;
@@ -121,8 +121,8 @@ public:
     void destroyIterators();
     void propagateSavedClusterState(BucketSpace bucketSpace, IPersistenceHandler &handler);
     void grabExtraModifiedBuckets(BucketSpace bucketSpace, IPersistenceHandler &handler);
-    void populateInitialBucketDB(BucketSpace bucketSpace, IPersistenceHandler &targetHandler);
-    std::unique_lock<std::shared_timed_mutex> getWLock() const;
+    void populateInitialBucketDB(const WriteGuard & guard, BucketSpace bucketSpace, IPersistenceHandler &targetHandler);
+    WriteGuard getWLock() const;
 };
 
 }
diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/transport_latch.cpp b/searchcore/src/vespa/searchcore/proton/persistenceengine/transport_latch.cpp
index 96025f5eaad..4a5dfb0a2a5 100644
--- a/searchcore/src/vespa/searchcore/proton/persistenceengine/transport_latch.cpp
+++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/transport_latch.cpp
@@ -12,7 +12,11 @@ TransportLatch::TransportLatch(uint32_t cnt)
     : _latch(cnt),
       _lock(),
       _result()
-{}
+{
+    if (cnt == 0u) {
+        _result = std::make_unique<Result>();
+    }
+}
 
 TransportLatch::~TransportLatch() = default;
 
diff --git a/searchcore/src/vespa/searchcore/proton/reference/document_db_reference_resolver.cpp b/searchcore/src/vespa/searchcore/proton/reference/document_db_reference_resolver.cpp
index 7f2ac5db768..fa531385d52 100644
--- a/searchcore/src/vespa/searchcore/proton/reference/document_db_reference_resolver.cpp
+++ b/searchcore/src/vespa/searchcore/proton/reference/document_db_reference_resolver.cpp
@@ -28,7 +28,7 @@ using search::AttributeGuard;
 using search::AttributeVector;
 using search::IAttributeManager;
 using search::NotImplementedAttribute;
-using search::ISequencedTaskExecutor;
+using vespalib::ISequencedTaskExecutor;
 using vespa::config::search::ImportedFieldsConfig;
 
 namespace proton {
diff --git a/searchcore/src/vespa/searchcore/proton/reference/document_db_reference_resolver.h b/searchcore/src/vespa/searchcore/proton/reference/document_db_reference_resolver.h
index 6ac804f538a..03d6a64345c 100644
--- a/searchcore/src/vespa/searchcore/proton/reference/document_db_reference_resolver.h
+++ b/searchcore/src/vespa/searchcore/proton/reference/document_db_reference_resolver.h
@@ -6,11 +6,23 @@
 #include <vespa/searchlib/attribute/not_implemented_attribute.h>
 #include <map>
 
-namespace document { class DocumentType; class DocumentTypeRepo; }
-namespace search { class ISequencedTaskExecutor; class IAttributeManager; struct IDocumentMetaStoreContext;
-    namespace attribute { class IAttributeVector; class ReferenceAttribute; } }
-namespace vespa { namespace config { namespace search { namespace internal { class InternalImportedFieldsType; } } } }
+namespace document {
+    class DocumentType;
+    class DocumentTypeRepo;
+}
+namespace search {
+    class IAttributeManager;
+    struct IDocumentMetaStoreContext;
+}
+namespace search::attribute {
+    class IAttributeVector;
+    class ReferenceAttribute;
+}
+namespace vespa::config::search::internal { class InternalImportedFieldsType; }
 
+namespace vespalib {
+    class ISequencedTaskExecutor;
+}
 namespace proton {
 
 class IDocumentDBReference;
@@ -27,12 +39,12 @@ class DocumentDBReferenceResolver : public IDocumentDBReferenceResolver {
 private:
     using ImportedFieldsConfig = const vespa::config::search::internal::InternalImportedFieldsType;
     const IDocumentDBReferenceRegistry &_registry;
-    const document::DocumentType &_thisDocType;
-    const ImportedFieldsConfig &_importedFieldsCfg;
-    const document::DocumentType &_prevThisDocType;
-    MonitoredRefCount              &_refCount;
-    search::ISequencedTaskExecutor &_attributeFieldWriter;
-    bool                            _useReferences;
+    const document::DocumentType       &_thisDocType;
+    const ImportedFieldsConfig         &_importedFieldsCfg;
+    const document::DocumentType       &_prevThisDocType;
+    MonitoredRefCount                  &_refCount;
+    vespalib::ISequencedTaskExecutor     &_attributeFieldWriter;
+    bool                                _useReferences;
     std::map<vespalib::string, std::unique_ptr<GidToLidChangeRegistrator>> _registrators;
 
     GidToLidChangeRegistrator &getRegistrator(const vespalib::string &docTypeName);
@@ -50,7 +62,7 @@ public:
                                 const ImportedFieldsConfig &importedFieldsCfg,
                                 const document::DocumentType &prevThisDocType,
                                 MonitoredRefCount &refCount,
-                                search::ISequencedTaskExecutor &attributeFieldWriter,
+                                vespalib::ISequencedTaskExecutor &attributeFieldWriter,
                                 bool useReferences);
     ~DocumentDBReferenceResolver() override;
 
diff --git a/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_listener.cpp b/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_listener.cpp
index 46096fead05..7d5eabe7de8 100644
--- a/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_listener.cpp
+++ b/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_listener.cpp
@@ -3,10 +3,9 @@
 #include "gid_to_lid_change_listener.h"
 #include <future>
 
-
 namespace proton {
 
-GidToLidChangeListener::GidToLidChangeListener(search::ISequencedTaskExecutor &attributeFieldWriter,
+GidToLidChangeListener::GidToLidChangeListener(vespalib::ISequencedTaskExecutor &attributeFieldWriter,
                                                std::shared_ptr<search::attribute::ReferenceAttribute> attr,
                                                MonitoredRefCount &refCount,
                                                const vespalib::string &name,
diff --git a/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_listener.h b/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_listener.h
index d4c349bd1d7..7b033b9899f 100644
--- a/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_listener.h
+++ b/searchcore/src/vespa/searchcore/proton/reference/gid_to_lid_change_listener.h
@@ -4,7 +4,7 @@
 
 #include "i_gid_to_lid_change_listener.h"
 #include <vespa/searchlib/attribute/reference_attribute.h>
-#include <vespa/searchlib/common/sequencedtaskexecutor.h>
+#include <vespa/vespalib/util/sequencedtaskexecutor.h>
 #include <vespa/searchcore/proton/common/monitored_refcount.h>
 
 namespace proton {
@@ -15,25 +15,25 @@ namespace proton {
  */
 class GidToLidChangeListener : public IGidToLidChangeListener
 {
-    search::ISequencedTaskExecutor                        &_attributeFieldWriter;
-    search::ISequencedTaskExecutor::ExecutorId             _executorId;
+    vespalib::ISequencedTaskExecutor                      &_attributeFieldWriter;
+    vespalib::ISequencedTaskExecutor::ExecutorId           _executorId;
     std::shared_ptr<search::attribute::ReferenceAttribute> _attr;
     MonitoredRefCount                                     &_refCount;
     vespalib::string                                       _name;
     vespalib::string                                       _docTypeName;
 
 public:
-    GidToLidChangeListener(search::ISequencedTaskExecutor &attributeFieldWriter,
+    GidToLidChangeListener(vespalib::ISequencedTaskExecutor &attributeFieldWriter,
                            std::shared_ptr<search::attribute::ReferenceAttribute> attr,
                            MonitoredRefCount &refCount,
                            const vespalib::string &name,
                            const vespalib::string &docTypeName);
-    virtual ~GidToLidChangeListener();
-    virtual void notifyPutDone(document::GlobalId gid, uint32_t lid) override;
-    virtual void notifyRemove(document::GlobalId gid) override;
-    virtual void notifyRegistered() override;
-    virtual const vespalib::string &getName() const override;
-    virtual const vespalib::string &getDocTypeName() const override;
+    ~GidToLidChangeListener() override;
+    void notifyPutDone(document::GlobalId gid, uint32_t lid) override;
+    void notifyRemove(document::GlobalId gid) override;
+    void notifyRegistered() override;
+    const vespalib::string &getName() const override;
+    const vespalib::string &getDocTypeName() const override;
     const std::shared_ptr<search::attribute::ReferenceAttribute> &getReferenceAttribute() const { return _attr; }
 };
 
diff --git a/searchcore/src/vespa/searchcore/proton/server/buckethandler.cpp b/searchcore/src/vespa/searchcore/proton/server/buckethandler.cpp
index 1d3b2165c80..57f1ee74e60 100644
--- a/searchcore/src/vespa/searchcore/proton/server/buckethandler.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/buckethandler.cpp
@@ -116,7 +116,7 @@ BucketHandler::handleGetBucketInfo(const Bucket &bucket,
     // Called by SPI thread.
     // BucketDBOwner ensures synchronization between SPI thread and
     // master write thread in document database.
-    BucketInfo bucketInfo = _ready->getBucketDB().takeGuard()->cachedGet(bucket);
+    BucketInfo bucketInfo = _ready->getBucketDB().takeGuard()->cachedGetBucketInfo(bucket);
     LOG(spam, "handleGetBucketInfo(%s): %s",
         bucket.toString().c_str(), bucketInfo.toString().c_str());
     resultHandler.handle(BucketInfoResult(bucketInfo));
diff --git a/searchcore/src/vespa/searchcore/proton/server/combiningfeedview.cpp b/searchcore/src/vespa/searchcore/proton/server/combiningfeedview.cpp
index 20306e92ea8..d25570794fe 100644
--- a/searchcore/src/vespa/searchcore/proton/server/combiningfeedview.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/combiningfeedview.cpp
@@ -151,9 +151,7 @@ CombiningFeedView::prepareRemove(RemoveOperation &rmOp)
 {
     getRemFeedView()->prepareRemove(rmOp);
     if (!rmOp.getPrevDbDocumentId().valid()) {
-        const DocumentId &docId = rmOp.getDocumentId();
-        const document::GlobalId &gid = docId.getGlobalId();
-        findPrevDbdId(gid, rmOp);
+        findPrevDbdId(rmOp.getGlobalId(), rmOp);
     }
 }
 
diff --git a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.cpp b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.cpp
index 9445a0a5206..1a6fc6cdbfd 100644
--- a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.cpp
@@ -163,7 +163,7 @@ DiskMemUsageFilter::DiskMemUsageFilter(const HwInfo &hwInfo)
       _listeners()
 { }
 
-DiskMemUsageFilter::~DiskMemUsageFilter() { }
+DiskMemUsageFilter::~DiskMemUsageFilter() = default;
 
 void
 DiskMemUsageFilter::setMemoryStats(vespalib::ProcessMemoryStats memoryStats_in)
diff --git a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_state.h b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_state.h
index 40c74808e72..b05cc261728 100644
--- a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_state.h
+++ b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_state.h
@@ -15,11 +15,7 @@ class DiskMemUsageState
     ResourceUsageState _memoryState;
 
 public:
-    DiskMemUsageState()
-        : _diskState(),
-          _memoryState()
-    {
-    }
+    DiskMemUsageState() = default;
     DiskMemUsageState(const ResourceUsageState &diskState_,
                       const ResourceUsageState &memoryState_)
         : _diskState(diskState_),
diff --git a/searchcore/src/vespa/searchcore/proton/server/docstorevalidator.cpp b/searchcore/src/vespa/searchcore/proton/server/docstorevalidator.cpp
index a195471d493..cc2beed8de7 100644
--- a/searchcore/src/vespa/searchcore/proton/server/docstorevalidator.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/docstorevalidator.cpp
@@ -1,8 +1,16 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include "docstorevalidator.h"
+#include "feedhandler.h"
+#include <vespa/searchcore/proton/feedoperation/removeoperation.h>
 #include <vespa/searchlib/common/bitvector.h>
 #include <vespa/document/fieldvalue/document.h>
+#include <vespa/document/datatype/documenttype.h>
+#include <vespa/searchcore/proton/common/feedtoken.h>
+#include <vespa/searchcore/proton/feedoperation/lidvectorcontext.h>
+
+#include <vespa/log/log.h>
+LOG_SETUP(".server.docstorevalidator");
 
 namespace proton {
 
@@ -92,19 +100,39 @@ DocStoreValidator::killOrphans(search::IDocumentStore &store,
 }
 
 
-LidVectorContext::SP
+std::shared_ptr<LidVectorContext>
 DocStoreValidator::getInvalidLids() const
 {
-    LidVectorContext::SP res(new LidVectorContext(_docIdLimit));
+    auto res = std::make_unique<LidVectorContext>(_docIdLimit);
     assert(_invalid->size() == _docIdLimit);
     for (search::DocumentIdT lid(_invalid->getFirstTrueBit(1));
          lid < _docIdLimit;
-         lid = _invalid->getNextTrueBit(lid + 1)) {
-
+         lid = _invalid->getNextTrueBit(lid + 1))
+    {
         res->addLid(lid);
     }
     return res;
 }
 
+void
+DocStoreValidator::performRemoves(FeedHandler & feedHandler, const search::IDocumentStore &store, const document::DocumentTypeRepo & repo) const {
+    for (search::DocumentIdT lid(_invalid->getFirstTrueBit(1));
+         lid < _docIdLimit;
+         lid = _invalid->getNextTrueBit(lid + 1))
+    {
+        document::GlobalId gid;
+        bool found = _dms.getGid(lid, gid);
+        assert(found);
+        if (found) {
+            search::DocumentMetaData metaData = _dms.getMetaData(gid);
+            assert(metaData.valid());
+            document::Document::UP document = store.read(lid, repo);
+            assert(document);
+            LOG(info, "Removing document with id %s and lid %u with gid %s in bucket %s", document->getId().toString().c_str(), lid, metaData.gid.toString().c_str(), metaData.bucketId.toString().c_str());
+            auto remove = std::make_unique<RemoveOperationWithGid>(metaData.bucketId, metaData.timestamp, gid, document->getType().getName());
+            feedHandler.performOperation(FeedToken(), std::move(remove));
+        }
+    }
+}
 
 } // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/docstorevalidator.h b/searchcore/src/vespa/searchcore/proton/server/docstorevalidator.h
index fba57d4f718..d51d924655a 100644
--- a/searchcore/src/vespa/searchcore/proton/server/docstorevalidator.h
+++ b/searchcore/src/vespa/searchcore/proton/server/docstorevalidator.h
@@ -4,11 +4,13 @@
 #include <vespa/searchlib/common/serialnum.h>
 #include <vespa/searchlib/docstore/idocumentstore.h>
 #include <vespa/searchcore/proton/documentmetastore/i_document_meta_store.h>
-#include <vespa/searchcore/proton/feedoperation/lidvectorcontext.h>
 
 namespace search { class BitVector; }
 namespace proton {
 
+class FeedHandler;
+class LidVectorContext;
+
 class DocStoreValidator : public search::IDocumentStoreReadVisitor
 {
     IDocumentMetaStore                 &_dms;
@@ -17,7 +19,7 @@ class DocStoreValidator : public search::IDocumentStoreReadVisitor
     std::unique_ptr<search::BitVector>  _orphans;
     uint32_t                            _visitCount;
     uint32_t                            _visitEmptyCount;
-    
+
 public:
     DocStoreValidator(IDocumentMetaStore &dms);
 
@@ -30,9 +32,8 @@ public:
     uint32_t getOrphanCount() const;
     uint32_t getVisitCount() const { return _visitCount; }
     uint32_t getVisitEmptyCount() const { return _visitEmptyCount; }
-    LidVectorContext::SP getInvalidLids() const;
+    std::shared_ptr<LidVectorContext> getInvalidLids() const;
+    void performRemoves(FeedHandler & feedHandler, const search::IDocumentStore &store, const document::DocumentTypeRepo & repo) const;
 };
 
-
 } // namespace proton
-
diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp
index 1b182c3e618..55f95ce0518 100644
--- a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp
@@ -137,10 +137,7 @@ DocumentDB::DocumentDB(const vespalib::string &baseDir,
               ThreadingServiceConfig::make(protonCfg,
                       findDocumentDB(protonCfg.documentdb, docTypeName.getName())->feeding.concurrency,
                       hwInfo.cpu())),
-      _writeService(sharedExecutor,
-                    _writeServiceConfig.indexingThreads(),
-                    indexing_thread_stack_size,
-                    _writeServiceConfig.defaultTaskLimit()),
+      _writeService(sharedExecutor, _writeServiceConfig, indexing_thread_stack_size),
       _initializeThreads(std::move(initializeThreads)),
       _initConfigSnapshot(),
       _initConfigSerialNum(0u),
@@ -150,6 +147,7 @@ DocumentDB::DocumentDB(const vespalib::string &baseDir,
       _activeConfigSnapshot(),
       _activeConfigSnapshotGeneration(0),
       _activeConfigSnapshotSerialNum(0u),
+      _validateAndSanitizeDocStore(protonCfg.validateAndSanitizeDocstore == vespa::config::search::core::ProtonConfig::ValidateAndSanitizeDocstore::YES),
       _initGate(),
       _clusterStateHandler(_writeService.master()),
       _bucketHandler(_writeService.master()),
@@ -664,6 +662,12 @@ DocumentDB::onTransactionLogReplayDone()
         // must signal that all existing buckets must be checked.
         notifyAllBucketsChanged();
     }
+    if (_validateAndSanitizeDocStore) {
+        LOG(info, "Validating documentdb %s", getName().c_str());
+        SerialNum serialNum = _feedHandler.getSerialNum();
+        sync(serialNum);
+        _subDBs.validateDocStore(_feedHandler, serialNum);
+    }
 }
 
 
@@ -756,11 +760,11 @@ DocumentDB::getNewestFlushedSerial()
 }
 
 std::unique_ptr<SearchReply>
-DocumentDB::match(const ISearchHandler::SP &, const SearchRequest &req, vespalib::ThreadBundle &threadBundle) const
+DocumentDB::match(const SearchRequest &req, vespalib::ThreadBundle &threadBundle) const
 {
     // Ignore input searchhandler. Use readysubdb's searchhandler instead.
     ISearchHandler::SP view(_subDBs.getReadySubDB()->getSearchView());
-    return view->match(view, req, threadBundle);
+    return view->match(req, threadBundle);
 }
 
 std::unique_ptr<DocsumReply>
diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.h b/searchcore/src/vespa/searchcore/proton/server/documentdb.h
index 02ad144af68..917d753683a 100644
--- a/searchcore/src/vespa/searchcore/proton/server/documentdb.h
+++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.h
@@ -85,7 +85,7 @@ private:
         DocumentStoreCacheStats() : total(), readySubDb(), notReadySubDb(), removedSubDb() {}
     };
 
-    using InitializeThreads = std::shared_ptr<vespalib::ThreadStackExecutorBase>;
+    using InitializeThreads = std::shared_ptr<vespalib::SyncableThreadExecutor>;
     using IFlushTargetList = std::vector<std::shared_ptr<searchcorespi::IFlushTarget>>;
     using StatusReportUP = std::unique_ptr<StatusReport>;
     using ProtonConfig = const vespa::config::search::core::internal::InternalProtonType;
@@ -112,6 +112,7 @@ private:
     DocumentDBConfig::SP          _activeConfigSnapshot;
     int64_t                       _activeConfigSnapshotGeneration;
     SerialNum                     _activeConfigSnapshotSerialNum;
+    const bool                    _validateAndSanitizeDocStore;
 
     vespalib::Gate                _initGate;
 
@@ -368,8 +369,7 @@ public:
     virtual SerialNum getNewestFlushedSerial();
 
     std::unique_ptr<search::engine::SearchReply>
-    match(const ISearchHandler::SP &searchHandler,
-          const search::engine::SearchRequest &req,
+    match(const search::engine::SearchRequest &req,
           vespalib::ThreadBundle &threadBundle) const;
 
     std::unique_ptr<search::engine::DocsumReply>
diff --git a/searchcore/src/vespa/searchcore/proton/server/documentretriever.cpp b/searchcore/src/vespa/searchcore/proton/server/documentretriever.cpp
index 360a0b12111..abfc840fc7e 100644
--- a/searchcore/src/vespa/searchcore/proton/server/documentretriever.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/documentretriever.cpp
@@ -1,9 +1,12 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include "documentretriever.h"
+#include <vespa/document/datatype/arraydatatype.h>
 #include <vespa/document/datatype/positiondatatype.h>
 #include <vespa/document/datatype/documenttype.h>
+#include <vespa/document/fieldvalue/arrayfieldvalue.h>
 #include <vespa/document/repo/documenttyperepo.h>
+#include <vespa/searchcommon/attribute/attributecontent.h>
 #include <vespa/searchcore/proton/attribute/document_field_retriever.h>
 #include <vespa/vespalib/geo/zcurve.h>
 #include <vespa/searchlib/attribute/attributevector.h>
@@ -28,6 +31,18 @@ using vespalib::geo::ZCurve;
 
 namespace proton {
 
+namespace {
+
+bool is_array_of_position_type(const document::DataType& field_type) noexcept {
+    const auto* arr_type = dynamic_cast<const document::ArrayDataType*>(&field_type);
+    if (!arr_type) {
+        return false;
+    }
+    return (arr_type->getNestedType() == PositionDataType::getInstance());
+}
+
+}
+
 DocumentRetriever
 ::DocumentRetriever(const DocTypeName &docTypeName,
                     const DocumentTypeRepo &repo,
@@ -47,7 +62,7 @@ DocumentRetriever
     int32_t positionDataTypeId = PositionDataType::getInstance().getId();
     LOG(debug, "checking document type '%s' for position fields", docTypeName.getName().c_str());
     for (const document::Field * field : fields) {
-        if (field->getDataType().getId() == positionDataTypeId) {
+        if ((field->getDataType().getId() == positionDataTypeId) || is_array_of_position_type(field->getDataType())) {
             LOG(debug, "Field '%s' is a position field", field->getName().data());
             const vespalib::string & zcurve_name = PositionDataType::getZCurveFieldName(field->getName());
             AttributeGuard::UP attr = attr_manager.getAttribute(zcurve_name);
@@ -72,19 +87,40 @@ FieldValue::UP positionFromZcurve(int64_t zcurve) {
     ZCurve::decode(zcurve, &x, &y);
 
     FieldValue::UP value = PositionDataType::getInstance().createFieldValue();
-    StructFieldValue *position = static_cast<StructFieldValue *>(value.get());
+    auto *position = static_cast<StructFieldValue *>(value.get());
     position->set(PositionDataType::FIELD_X, x);
     position->set(PositionDataType::FIELD_Y, y);
     return value;
 }
 
+std::unique_ptr<document::FieldValue>
+zcurve_array_attribute_to_field_value(const document::Field& field,
+                                      const search::attribute::IAttributeVector& attr,
+                                      DocumentIdT lid)
+{
+    search::attribute::AttributeContent<int64_t> zc_elems;
+    zc_elems.fill(attr, lid);
+    auto new_fv = field.createValue();
+    auto& new_array_fv = dynamic_cast<document::ArrayFieldValue&>(*new_fv);
+    new_array_fv.reserve(zc_elems.size());
+    for (int64_t zc : zc_elems) {
+        new_array_fv.append(positionFromZcurve(zc));
+    }
+    return new_fv;
+}
+
 void fillInPositionFields(Document &doc, DocumentIdT lid, const DocumentRetriever::PositionFields & possiblePositionFields, const IAttributeManager & attr_manager)
 {
     for (const auto & it : possiblePositionFields) {
-        AttributeGuard::UP attr = attr_manager.getAttribute(it.second);
-        if (!(*attr)->isUndefined(lid)) {
-            int64_t zcurve = (*attr)->getInt(lid);
-            doc.setValue(*it.first, *positionFromZcurve(zcurve));
+        auto attr_guard = attr_manager.getAttribute(it.second);
+        auto& attr = *attr_guard;
+        if (!attr->isUndefined(lid)) {
+            if (attr->hasArrayType()) {
+                doc.setFieldValue(*it.first, zcurve_array_attribute_to_field_value(*it.first, *attr, lid));
+            } else {
+                int64_t zcurve = attr->getInt(lid);
+                doc.setValue(*it.first, *positionFromZcurve(zcurve));
+            }
         } else {
             doc.remove(*it.first); // Don't resurrect old values from the docstore.
         }
diff --git a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp
index 74010391fcb..1e01203b431 100644
--- a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp
@@ -315,4 +315,10 @@ DocumentSubDBCollection::tearDownReferences(IDocumentDBReferenceResolver &resolv
     }
 }
 
+void DocumentSubDBCollection::validateDocStore(FeedHandler & feedHandler, SerialNum serialNum) {
+    for (auto subDb : _subDBs) {
+        subDb->validateDocStore(feedHandler, serialNum);
+    }
+}
+
 } // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h
index d09afb92cc8..2936051538d 100644
--- a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h
+++ b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h
@@ -26,6 +26,7 @@ namespace searchcorespi {
 }
 
 namespace proton {
+
 class DocumentDBConfig;
 struct DocumentDBTaggedMetrics;
 class MaintenanceController;
@@ -41,6 +42,8 @@ class IDocumentSubDBOwner;
 class IDocumentSubDB;
 class IDocumentRetriever;
 class ReconfigParams;
+class RemoveDocumentsOperation;
+class FeedHandler;
 
 namespace matching {
     class QueryLimiter;
@@ -163,6 +166,7 @@ public:
     double getReprocessingProgress() const;
     void close();
     void tearDownReferences(IDocumentDBReferenceResolver &resolver);
+    void validateDocStore(FeedHandler & feedHandler, SerialNum serialNum);
 };
 
 } // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/emptysearchview.cpp b/searchcore/src/vespa/searchcore/proton/server/emptysearchview.cpp
index 90cf0011685..9f2ec26ad4b 100644
--- a/searchcore/src/vespa/searchcore/proton/server/emptysearchview.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/emptysearchview.cpp
@@ -14,10 +14,7 @@ using search::engine::SearchRequest;
 
 namespace proton {
 
-EmptySearchView::EmptySearchView()
-    : ISearchHandler()
-{
-}
+EmptySearchView::EmptySearchView() = default;
 
 
 DocsumReply::UP
@@ -25,20 +22,16 @@ EmptySearchView::getDocsums(const DocsumRequest &req)
 {
     LOG(debug, "getDocsums(): resultClass(%s), numHits(%zu)",
         req.resultClassName.c_str(), req.hits.size());
-    DocsumReply::UP reply(new DocsumReply());
-    for (size_t i = 0; i < req.hits.size(); ++i) {
-        reply->docsums.push_back(DocsumReply::Docsum());
-        reply->docsums.back().gid = req.hits[i].gid;
+    auto reply = std::make_unique<DocsumReply>();
+    for (const auto & hit : req.hits) {
+        reply->docsums.emplace_back(hit.gid);
     }
     return reply;
 }
 
 SearchReply::UP
-EmptySearchView::match(const ISearchHandler::SP &,
-                       const SearchRequest &,
-                       vespalib::ThreadBundle &) const {
-    SearchReply::UP reply(new SearchReply);
-    return reply;
+EmptySearchView::match(const SearchRequest &, vespalib::ThreadBundle &) const {
+    return std::make_unique<SearchReply>();
 }
 
 
diff --git a/searchcore/src/vespa/searchcore/proton/server/emptysearchview.h b/searchcore/src/vespa/searchcore/proton/server/emptysearchview.h
index a3e2a2b176c..92fb97f6177 100644
--- a/searchcore/src/vespa/searchcore/proton/server/emptysearchview.h
+++ b/searchcore/src/vespa/searchcore/proton/server/emptysearchview.h
@@ -13,15 +13,10 @@ public:
 
     EmptySearchView();
 
-    /**
-     * Implements ISearchHandler
-     */
-    virtual std::unique_ptr<DocsumReply> getDocsums(const DocsumRequest & req) override;
-
-    virtual std::unique_ptr<SearchReply>
-    match(const ISearchHandler::SP &searchHandler,
-          const SearchRequest &req,
-          vespalib::ThreadBundle &threadBundle) const override;
+    std::unique_ptr<DocsumReply> getDocsums(const DocsumRequest & req) override;
+
+    std::unique_ptr<SearchReply>
+    match(const SearchRequest &req, vespalib::ThreadBundle &threadBundle) const override;
 };
 
 } // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/executor_thread_service.cpp b/searchcore/src/vespa/searchcore/proton/server/executor_thread_service.cpp
index af8e16f657b..b13fa2baed3 100644
--- a/searchcore/src/vespa/searchcore/proton/server/executor_thread_service.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/executor_thread_service.cpp
@@ -1,15 +1,15 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include "executor_thread_service.h"
-#include <vespa/vespalib/util/closuretask.h>
+#include <vespa/vespalib/util/lambdatask.h>
+#include <vespa/vespalib/util/gate.h>
 #include <vespa/fastos/thread.h>
 
-using vespalib::makeClosure;
-using vespalib::makeTask;
+using vespalib::makeLambdaTask;
 using vespalib::Executor;
 using vespalib::Gate;
 using vespalib::Runnable;
-using vespalib::ThreadStackExecutorBase;
+using vespalib::SyncableThreadExecutor;
 
 namespace proton {
 
@@ -29,10 +29,10 @@ sampleThreadId(FastOS_ThreadId *threadId)
 }
 
 std::unique_ptr<internal::ThreadId>
-getThreadId(ThreadStackExecutorBase &executor)
+getThreadId(SyncableThreadExecutor &executor)
 {
     std::unique_ptr<internal::ThreadId> id = std::make_unique<internal::ThreadId>();
-    executor.execute(makeTask(makeClosure(&sampleThreadId, &id->_id)));
+    executor.execute(makeLambdaTask([threadId=&id->_id] { sampleThreadId(threadId);}));
     executor.sync();
     return id;
 }
@@ -46,13 +46,13 @@ runRunnable(Runnable *runnable, Gate *gate)
 
 } // namespace
 
-ExecutorThreadService::ExecutorThreadService(ThreadStackExecutorBase &executor)
+ExecutorThreadService::ExecutorThreadService(SyncableThreadExecutor &executor)
     : _executor(executor),
       _threadId(getThreadId(executor))
 {
 }
 
-ExecutorThreadService::~ExecutorThreadService() {}
+ExecutorThreadService::~ExecutorThreadService()  = default;
 
 void
 ExecutorThreadService::run(Runnable &runnable)
@@ -61,7 +61,7 @@ ExecutorThreadService::run(Runnable &runnable)
         runnable.run();
     } else {
         Gate gate;
-        _executor.execute(makeTask(makeClosure(&runRunnable, &runnable, &gate)));
+        _executor.execute(makeLambdaTask([runnablePtr=&runnable, gatePtr=&gate] { runRunnable(runnablePtr, gatePtr); }));
         gate.await();
     }
 }
@@ -73,4 +73,12 @@ ExecutorThreadService::isCurrentThread() const
     return FastOS_Thread::CompareThreadIds(_threadId->_id, currentThreadId);
 }
 
+vespalib::ThreadExecutor::Stats ExecutorThreadService::getStats() {
+    return _executor.getStats();
+}
+
+void ExecutorThreadService::setTaskLimit(uint32_t taskLimit) {
+    _executor.setTaskLimit(taskLimit);
+}
+
 } // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/executor_thread_service.h b/searchcore/src/vespa/searchcore/proton/server/executor_thread_service.h
index 521565358d9..26069b4b8dd 100644
--- a/searchcore/src/vespa/searchcore/proton/server/executor_thread_service.h
+++ b/searchcore/src/vespa/searchcore/proton/server/executor_thread_service.h
@@ -2,7 +2,7 @@
 #pragma once
 
 #include <vespa/searchcorespi/index/i_thread_service.h>
-#include <vespa/vespalib/util/threadstackexecutorbase.h>
+#include <vespa/vespalib/util/threadexecutor.h>
 
 namespace proton {
 
@@ -14,16 +14,15 @@ namespace internal { struct ThreadId; }
 class ExecutorThreadService : public searchcorespi::index::IThreadService
 {
 private:
-    vespalib::ThreadStackExecutorBase &_executor;
+    vespalib::SyncableThreadExecutor &_executor;
     std::unique_ptr<internal::ThreadId>   _threadId;
 
 public:
-    ExecutorThreadService(vespalib::ThreadStackExecutorBase &executor);
+    ExecutorThreadService(vespalib::SyncableThreadExecutor &executor);
     ~ExecutorThreadService();
 
-    /**
-     * Implements IThreadService
-     */
+    Stats getStats() override;
+
     vespalib::Executor::Task::UP execute(vespalib::Executor::Task::UP task) override {
         return _executor.execute(std::move(task));
     }
@@ -32,8 +31,14 @@ public:
         _executor.sync();
         return *this;
     }
+    ExecutorThreadService & shutdown() override {
+        _executor.shutdown();
+        return *this;
+    }
     bool isCurrentThread() const override;
     size_t getNumThreads() const override { return _executor.getNumThreads(); }
+
+    void setTaskLimit(uint32_t taskLimit) override;
 };
 
 } // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.cpp b/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.cpp
index 262363f4250..b29dd955ff3 100644
--- a/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.cpp
@@ -1,27 +1,52 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include "executorthreadingservice.h"
+#include "threading_service_config.h"
 #include <vespa/searchcore/proton/metrics/executor_threading_service_stats.h>
-#include <vespa/searchlib/common/sequencedtaskexecutor.h>
+#include <vespa/vespalib/util/sequencedtaskexecutor.h>
+#include <vespa/vespalib/util/singleexecutor.h>
+#include <vespa/vespalib/util/blockingthreadstackexecutor.h>
 
-using vespalib::ThreadStackExecutorBase;
-using search::SequencedTaskExecutor;
+
+using vespalib::SyncableThreadExecutor;
+using vespalib::BlockingThreadStackExecutor;
+using vespalib::SingleExecutor;
+using vespalib::SequencedTaskExecutor;
+using OptimizeFor = vespalib::Executor::OptimizeFor;
 
 namespace proton {
 
-ExecutorThreadingService::ExecutorThreadingService(vespalib::ThreadStackExecutorBase & sharedExecutor,
-                                                   uint32_t threads, uint32_t stackSize, uint32_t taskLimit)
+namespace {
+
+std::unique_ptr<SyncableThreadExecutor>
+createExecutorWithOneThread(uint32_t stackSize, uint32_t taskLimit, OptimizeFor optimize) {
+    if (optimize == OptimizeFor::THROUGHPUT) {
+        return std::make_unique<SingleExecutor>(taskLimit);
+    } else {
+        return std::make_unique<BlockingThreadStackExecutor>(1, stackSize, taskLimit);
+    }
+}
+
+}
+
+ExecutorThreadingService::ExecutorThreadingService(vespalib::SyncableThreadExecutor &sharedExecutor, uint32_t num_treads)
+    : ExecutorThreadingService(sharedExecutor, ThreadingServiceConfig::make(num_treads))
+{}
+
+ExecutorThreadingService::ExecutorThreadingService(vespalib::SyncableThreadExecutor & sharedExecutor,
+                                                   const ThreadingServiceConfig & cfg,  uint32_t stackSize)
 
     : _sharedExecutor(sharedExecutor),
       _masterExecutor(1, stackSize),
-      _indexExecutor(1, stackSize, taskLimit),
-      _summaryExecutor(1, stackSize, taskLimit),
+      _indexExecutor(createExecutorWithOneThread(stackSize, cfg.defaultTaskLimit(), cfg.optimize())),
+      _summaryExecutor(createExecutorWithOneThread(stackSize, cfg.defaultTaskLimit(), cfg.optimize())),
       _masterService(_masterExecutor),
-      _indexService(_indexExecutor),
-      _summaryService(_summaryExecutor),
-      _indexFieldInverter(std::make_unique<SequencedTaskExecutor>(threads, taskLimit)),
-      _indexFieldWriter(std::make_unique<SequencedTaskExecutor>(threads, taskLimit)),
-      _attributeFieldWriter(std::make_unique<SequencedTaskExecutor>(threads, taskLimit))
+      _indexService(*_indexExecutor),
+      _summaryService(*_summaryExecutor),
+      _indexFieldInverter(SequencedTaskExecutor::create(cfg.indexingThreads(), cfg.defaultTaskLimit())),
+      _indexFieldWriter(SequencedTaskExecutor::create(cfg.indexingThreads(), cfg.defaultTaskLimit())),
+      _attributeFieldWriter(SequencedTaskExecutor::create(cfg.indexingThreads(), cfg.defaultTaskLimit(), cfg.optimize(),
+                                                          cfg.kindOfwatermark(), cfg.reactionTime()))
 {
 }
 
@@ -35,8 +60,8 @@ ExecutorThreadingService::sync()
         _masterExecutor.sync();
     }
     _attributeFieldWriter->sync();
-    _indexExecutor.sync();
-    _summaryExecutor.sync();
+    _indexExecutor->sync();
+    _summaryExecutor->sync();
     _indexFieldInverter->sync();
     _indexFieldWriter->sync();
     if (!isMasterThread) {
@@ -51,10 +76,10 @@ ExecutorThreadingService::shutdown()
     _masterExecutor.shutdown();
     _masterExecutor.sync();
     _attributeFieldWriter->sync();
-    _summaryExecutor.shutdown();
-    _summaryExecutor.sync();
-    _indexExecutor.shutdown();
-    _indexExecutor.sync();
+    _summaryExecutor->shutdown();
+    _summaryExecutor->sync();
+    _indexExecutor->shutdown();
+    _indexExecutor->sync();
     _indexFieldInverter->sync();
     _indexFieldWriter->sync();
 }
@@ -62,8 +87,8 @@ ExecutorThreadingService::shutdown()
 void
 ExecutorThreadingService::setTaskLimit(uint32_t taskLimit, uint32_t summaryTaskLimit)
 {
-    _indexExecutor.setTaskLimit(taskLimit);
-    _summaryExecutor.setTaskLimit(summaryTaskLimit);
+    _indexExecutor->setTaskLimit(taskLimit);
+    _summaryExecutor->setTaskLimit(summaryTaskLimit);
     _indexFieldInverter->setTaskLimit(taskLimit);
     _indexFieldWriter->setTaskLimit(taskLimit);
     _attributeFieldWriter->setTaskLimit(taskLimit);
@@ -73,25 +98,25 @@ ExecutorThreadingServiceStats
 ExecutorThreadingService::getStats()
 {
     return ExecutorThreadingServiceStats(_masterExecutor.getStats(),
-                                         _indexExecutor.getStats(),
-                                         _summaryExecutor.getStats(),
+                                         _indexExecutor->getStats(),
+                                         _summaryExecutor->getStats(),
                                          _sharedExecutor.getStats(),
                                          _indexFieldInverter->getStats(),
                                          _indexFieldWriter->getStats(),
                                          _attributeFieldWriter->getStats());
 }
 
-search::ISequencedTaskExecutor &
+vespalib::ISequencedTaskExecutor &
 ExecutorThreadingService::indexFieldInverter() {
     return *_indexFieldInverter;
 }
 
-search::ISequencedTaskExecutor &
+vespalib::ISequencedTaskExecutor &
 ExecutorThreadingService::indexFieldWriter() {
     return *_indexFieldWriter;
 }
 
-search::ISequencedTaskExecutor &
+vespalib::ISequencedTaskExecutor &
 ExecutorThreadingService::attributeFieldWriter() {
     return *_attributeFieldWriter;
 }
diff --git a/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.h b/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.h
index 9ab6eba39e1..280e50aea56 100644
--- a/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.h
+++ b/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.h
@@ -3,13 +3,12 @@
 
 #include "executor_thread_service.h"
 #include <vespa/searchcorespi/index/ithreadingservice.h>
-#include <vespa/vespalib/util/blockingthreadstackexecutor.h>
 #include <vespa/vespalib/util/threadstackexecutor.h>
 
-namespace search { class SequencedTaskExecutor; }
 namespace proton {
 
 class ExecutorThreadingServiceStats;
+class ThreadingServiceConfig;
 
 /**
  * Implementation of IThreadingService using 2 underlying thread stack executors
@@ -18,28 +17,28 @@ class ExecutorThreadingServiceStats;
 class ExecutorThreadingService : public searchcorespi::index::IThreadingService
 {
 private:
-    vespalib::ThreadStackExecutorBase & _sharedExecutor;
-    vespalib::ThreadStackExecutor _masterExecutor;
-    vespalib::BlockingThreadStackExecutor _indexExecutor;
-    vespalib::BlockingThreadStackExecutor _summaryExecutor;
-    ExecutorThreadService _masterService;
-    ExecutorThreadService _indexService;
-    ExecutorThreadService _summaryService;
-    std::unique_ptr<search::SequencedTaskExecutor> _indexFieldInverter;
-    std::unique_ptr<search::SequencedTaskExecutor> _indexFieldWriter;
-    std::unique_ptr<search::SequencedTaskExecutor> _attributeFieldWriter;
+    vespalib::SyncableThreadExecutor                   & _sharedExecutor;
+    vespalib::ThreadStackExecutor                        _masterExecutor;
+    std::unique_ptr<vespalib::SyncableThreadExecutor>    _indexExecutor;
+    std::unique_ptr<vespalib::SyncableThreadExecutor>    _summaryExecutor;
+    ExecutorThreadService                                _masterService;
+    ExecutorThreadService                                _indexService;
+    ExecutorThreadService                                _summaryService;
+    std::unique_ptr<vespalib::ISequencedTaskExecutor>    _indexFieldInverter;
+    std::unique_ptr<vespalib::ISequencedTaskExecutor>    _indexFieldWriter;
+    std::unique_ptr<vespalib::ISequencedTaskExecutor>    _attributeFieldWriter;
 
 public:
+    using OptimizeFor = vespalib::Executor::OptimizeFor;
     /**
      * Constructor.
      *
      * @stackSize The size of the stack of the underlying executors.
-     * @taskLimit The task limit for the index executor.
+     * @cfg config used to set up all executors.
      */
-    ExecutorThreadingService(vespalib::ThreadStackExecutorBase &sharedExecutor,
-                             uint32_t threads = 1,
-                             uint32_t stackSize = 128 * 1024,
-                             uint32_t taskLimit = 1000);
+    ExecutorThreadingService(vespalib::SyncableThreadExecutor &sharedExecutor,
+                             const ThreadingServiceConfig & cfg, uint32_t stackSize = 128 * 1024);
+    ExecutorThreadingService(vespalib::SyncableThreadExecutor &sharedExecutor, uint32_t num_treads = 1);
     ~ExecutorThreadingService() override;
 
     /**
@@ -55,11 +54,11 @@ public:
     vespalib::ThreadStackExecutorBase &getMasterExecutor() {
         return _masterExecutor;
     }
-    vespalib::ThreadStackExecutorBase &getIndexExecutor() {
-        return _indexExecutor;
+    vespalib::SyncableThreadExecutor &getIndexExecutor() {
+        return *_indexExecutor;
     }
-    vespalib::ThreadStackExecutorBase &getSummaryExecutor() {
-        return _summaryExecutor;
+    vespalib::SyncableThreadExecutor &getSummaryExecutor() {
+        return *_summaryExecutor;
     }
 
     /**
@@ -75,13 +74,13 @@ public:
     searchcorespi::index::IThreadService &summary() override {
         return _summaryService;
     }
-    vespalib::ThreadExecutor &shared() override {
+    vespalib::SyncableThreadExecutor &shared() override {
         return _sharedExecutor;
     }
 
-    search::ISequencedTaskExecutor &indexFieldInverter() override;
-    search::ISequencedTaskExecutor &indexFieldWriter() override;
-    search::ISequencedTaskExecutor &attributeFieldWriter() override;
+    vespalib::ISequencedTaskExecutor &indexFieldInverter() override;
+    vespalib::ISequencedTaskExecutor &indexFieldWriter() override;
+    vespalib::ISequencedTaskExecutor &attributeFieldWriter() override;
     ExecutorThreadingServiceStats getStats();
 };
 
diff --git a/searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.cpp b/searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.cpp
index dfe98b44adc..c24790a61f4 100644
--- a/searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/fast_access_feed_view.cpp
@@ -6,7 +6,7 @@
 #include "removedonecontext.h"
 #include "putdonecontext.h"
 #include <vespa/searchcore/proton/feedoperation/operations.h>
-#include <vespa/searchlib/common/isequencedtaskexecutor.h>
+#include <vespa/vespalib/util/isequencedtaskexecutor.h>
 
 using document::Document;
 using document::DocumentUpdate;
diff --git a/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp b/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp
index 338fc738040..362de7ee780 100644
--- a/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp
@@ -169,8 +169,8 @@ FeedHandler::createNonExistingDocument(FeedToken token, const UpdateOperation &o
 void FeedHandler::performRemove(FeedToken token, RemoveOperation &op) {
     _activeFeedView->prepareRemove(op);
     if (ignoreOperation(op)) {
-        LOG(debug, "performRemove(): ignoreOperation: docId(%s), timestamp(%" PRIu64 "), prevTimestamp(%" PRIu64 ")",
-            op.getDocumentId().toString().c_str(), (uint64_t)op.getTimestamp(), (uint64_t)op.getPrevTimestamp());
+        LOG(debug, "performRemove(): ignoreOperation: remove(%s), timestamp(%" PRIu64 "), prevTimestamp(%" PRIu64 ")",
+            op.toString().c_str(), (uint64_t)op.getTimestamp(), (uint64_t)op.getPrevTimestamp());
         if (token) {
             token->setResult(make_unique<RemoveResult>(false), false);
         }
@@ -564,6 +564,7 @@ FeedHandler::performOperation(FeedToken token, FeedOperation::UP op)
         performPut(std::move(token), static_cast<PutOperation &>(*op));
         return;
     case FeedOperation::REMOVE:
+    case FeedOperation::REMOVE_GID:
         performRemove(std::move(token), static_cast<RemoveOperation &>(*op));
         return;
     case FeedOperation::UPDATE_42:
diff --git a/searchcore/src/vespa/searchcore/proton/server/i_blockable_maintenance_job.h b/searchcore/src/vespa/searchcore/proton/server/i_blockable_maintenance_job.h
index c70a6c502f1..63ff269580e 100644
--- a/searchcore/src/vespa/searchcore/proton/server/i_blockable_maintenance_job.h
+++ b/searchcore/src/vespa/searchcore/proton/server/i_blockable_maintenance_job.h
@@ -36,7 +36,7 @@ public:
      */
     virtual void unBlock(BlockedReason reason) = 0;
 
-    virtual IBlockableMaintenanceJob *asBlockable() override { return this; }
+    IBlockableMaintenanceJob *asBlockable() override { return this; }
 };
 
 }
diff --git a/searchcore/src/vespa/searchcore/proton/server/i_maintenance_job.h b/searchcore/src/vespa/searchcore/proton/server/i_maintenance_job.h
index 6d0739e1aed..df6889ecf04 100644
--- a/searchcore/src/vespa/searchcore/proton/server/i_maintenance_job.h
+++ b/searchcore/src/vespa/searchcore/proton/server/i_maintenance_job.h
@@ -31,7 +31,7 @@ public:
           _interval(interval)
     {}
 
-    virtual ~IMaintenanceJob() {}
+    virtual ~IMaintenanceJob() = default;
 
     virtual const vespalib::string &getName() const { return _name; }
     virtual vespalib::duration getDelay() const { return _delay; }
diff --git a/searchcore/src/vespa/searchcore/proton/server/i_proton_configurer_owner.h b/searchcore/src/vespa/searchcore/proton/server/i_proton_configurer_owner.h
index fec8430e41d..5a457b168ec 100644
--- a/searchcore/src/vespa/searchcore/proton/server/i_proton_configurer_owner.h
+++ b/searchcore/src/vespa/searchcore/proton/server/i_proton_configurer_owner.h
@@ -18,8 +18,8 @@ class DocumentDBConfigOwner;
  */
 class IProtonConfigurerOwner
 {
-    using InitializeThreads = std::shared_ptr<vespalib::ThreadStackExecutorBase>;
 public:
+    using InitializeThreads = std::shared_ptr<vespalib::SyncableThreadExecutor>;
     virtual ~IProtonConfigurerOwner() { }
     virtual std::shared_ptr<DocumentDBConfigOwner> addDocumentDB(const DocTypeName &docTypeName,
                                                                  document::BucketSpace bucketSpace,
diff --git a/searchcore/src/vespa/searchcore/proton/server/idocumentsubdb.h b/searchcore/src/vespa/searchcore/proton/server/idocumentsubdb.h
index eb297fc7987..65724e66913 100644
--- a/searchcore/src/vespa/searchcore/proton/server/idocumentsubdb.h
+++ b/searchcore/src/vespa/searchcore/proton/server/idocumentsubdb.h
@@ -40,6 +40,7 @@ class ISearchHandler;
 class ISummaryAdapter;
 class ISummaryManager;
 class ReconfigParams;
+class RemoveDocumentsOperation;
 
 /**
  * Interface for a document sub database that handles a subset of the documents that belong to a
@@ -117,6 +118,7 @@ public:
     virtual void close() = 0;
     virtual std::shared_ptr<IDocumentDBReference> getDocumentDBReference() = 0;
     virtual void tearDownReferences(IDocumentDBReferenceResolver &resolver) = 0;
+    virtual void validateDocStore(FeedHandler &op, SerialNum serialNum) const = 0;
 };
 
 } // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/matchview.cpp b/searchcore/src/vespa/searchcore/proton/server/matchview.cpp
index 6f32f886637..7ba9b971715 100644
--- a/searchcore/src/vespa/searchcore/proton/server/matchview.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/matchview.cpp
@@ -31,17 +31,17 @@ using matching::ISearchContext;
 using matching::Matcher;
 using matching::SessionManager;
 
-MatchView::MatchView(const Matchers::SP &matchers,
-                     const IndexSearchable::SP &indexSearchable,
-                     const IAttributeManager::SP &attrMgr,
-                     const SessionManagerSP &sessionMgr,
-                     const IDocumentMetaStoreContext::SP &metaStore,
+MatchView::MatchView(Matchers::SP matchers,
+                     IndexSearchable::SP indexSearchable,
+                     IAttributeManager::SP attrMgr,
+                     SessionManagerSP sessionMgr,
+                     IDocumentMetaStoreContext::SP metaStore,
                      DocIdLimit &docIdLimit)
-    : _matchers(matchers),
-      _indexSearchable(indexSearchable),
-      _attrMgr(attrMgr),
-      _sessionMgr(sessionMgr),
-      _metaStore(metaStore),
+    : _matchers(std::move(matchers)),
+      _indexSearchable(std::move(indexSearchable)),
+      _attrMgr(std::move(attrMgr)),
+      _sessionMgr(std::move(sessionMgr)),
+      _metaStore(std::move(metaStore)),
       _docIdLimit(docIdLimit)
 { }
 
@@ -68,12 +68,12 @@ MatchView::createContext() const {
 
 
 std::unique_ptr<SearchReply>
-MatchView::match(const ISearchHandler::SP &searchHandler, const SearchRequest &req,
+MatchView::match(std::shared_ptr<const ISearchHandler> searchHandler, const SearchRequest &req,
                  vespalib::ThreadBundle &threadBundle) const
 {
     Matcher::SP matcher = getMatcher(req.ranking);
     SearchSession::OwnershipBundle owned_objects;
-    owned_objects.search_handler = searchHandler;
+    owned_objects.search_handler = std::move(searchHandler);
     owned_objects.context = createContext();
     owned_objects.readGuard = _metaStore->getReadGuard();;
     MatchContext *ctx = owned_objects.context.get();
diff --git a/searchcore/src/vespa/searchcore/proton/server/matchview.h b/searchcore/src/vespa/searchcore/proton/server/matchview.h
index 983e06d3414..721cb439508 100644
--- a/searchcore/src/vespa/searchcore/proton/server/matchview.h
+++ b/searchcore/src/vespa/searchcore/proton/server/matchview.h
@@ -36,11 +36,11 @@ public:
     MatchView(const MatchView &) = delete;
     MatchView & operator = (const MatchView &) = delete;
 
-    MatchView(const Matchers::SP &matchers,
-              const searchcorespi::IndexSearchable::SP &indexSearchable,
-              const IAttributeManager::SP &attrMgr,
-              const SessionManagerSP &sessionMgr,
-              const IDocumentMetaStoreContext::SP &metaStore,
+    MatchView(Matchers::SP matchers,
+              searchcorespi::IndexSearchable::SP indexSearchable,
+              IAttributeManager::SP attrMgr,
+              SessionManagerSP sessionMgr,
+              IDocumentMetaStoreContext::SP metaStore,
               DocIdLimit &docIdLimit);
     ~MatchView();
 
@@ -62,7 +62,7 @@ public:
     matching::MatchContext::UP createContext() const;
 
     std::unique_ptr<search::engine::SearchReply>
-    match(const std::shared_ptr<ISearchHandler> &searchHandler,
+    match(std::shared_ptr<const ISearchHandler> searchHandler,
           const search::engine::SearchRequest &req,
           vespalib::ThreadBundle &threadBundle) const;
 };
diff --git a/searchcore/src/vespa/searchcore/proton/server/move_operation_limiter.cpp b/searchcore/src/vespa/searchcore/proton/server/move_operation_limiter.cpp
index 6c0c0863fe1..e535b05393c 100644
--- a/searchcore/src/vespa/searchcore/proton/server/move_operation_limiter.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/move_operation_limiter.cpp
@@ -41,9 +41,7 @@ MoveOperationLimiter::MoveOperationLimiter(IBlockableMaintenanceJob *job,
 {
 }
 
-MoveOperationLimiter::~MoveOperationLimiter()
-{
-}
+MoveOperationLimiter::~MoveOperationLimiter() = default;
 
 void
 MoveOperationLimiter::clearJob()
diff --git a/searchcore/src/vespa/searchcore/proton/server/persistencehandlerproxy.cpp b/searchcore/src/vespa/searchcore/proton/server/persistencehandlerproxy.cpp
index beee2716cc7..16af4e87795 100644
--- a/searchcore/src/vespa/searchcore/proton/server/persistencehandlerproxy.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/persistencehandlerproxy.cpp
@@ -41,21 +41,21 @@ PersistenceHandlerProxy::initialize()
 void
 PersistenceHandlerProxy::handlePut(FeedToken token, const Bucket &bucket, Timestamp timestamp, const DocumentSP &doc)
 {
-    FeedOperation::UP op(new PutOperation(bucket.getBucketId().stripUnused(), timestamp, doc));
+    auto op = std::make_unique<PutOperation>(bucket.getBucketId().stripUnused(), timestamp, doc);
     _feedHandler.handleOperation(token, std::move(op));
 }
 
 void
 PersistenceHandlerProxy::handleUpdate(FeedToken token, const Bucket &bucket, Timestamp timestamp, const DocumentUpdateSP &upd)
 {
-    FeedOperation::UP op(new UpdateOperation(bucket.getBucketId().stripUnused(), timestamp, upd));
+    auto op = std::make_unique<UpdateOperation>(bucket.getBucketId().stripUnused(), timestamp, upd);
     _feedHandler.handleOperation(token, std::move(op));
 }
 
 void
 PersistenceHandlerProxy::handleRemove(FeedToken token, const Bucket &bucket, Timestamp timestamp, const document::DocumentId &id)
 {
-    FeedOperation::UP op(new RemoveOperation(bucket.getBucketId().stripUnused(), timestamp, id));
+    auto op = std::make_unique<RemoveOperationWithDocId>(bucket.getBucketId().stripUnused(), timestamp, id);
     _feedHandler.handleOperation(token, std::move(op));
 }
 
@@ -88,14 +88,14 @@ PersistenceHandlerProxy::handleGetBucketInfo(const Bucket &bucket, IBucketInfoRe
 void
 PersistenceHandlerProxy::handleCreateBucket(FeedToken token, const Bucket &bucket)
 {
-    FeedOperation::UP op(new CreateBucketOperation(bucket.getBucketId().stripUnused()));
+    auto op = std::make_unique<CreateBucketOperation>(bucket.getBucketId().stripUnused());
     _feedHandler.handleOperation(token, std::move(op));
 }
 
 void
 PersistenceHandlerProxy::handleDeleteBucket(FeedToken token, const Bucket &bucket)
 {
-    FeedOperation::UP op(new DeleteBucketOperation(bucket.getBucketId().stripUnused()));
+    auto op = std::make_unique<DeleteBucketOperation>(bucket.getBucketId().stripUnused());
     _feedHandler.handleOperation(token, std::move(op));
 }
 
@@ -108,9 +108,9 @@ PersistenceHandlerProxy::handleGetModifiedBuckets(IBucketIdListResultHandler &re
 void
 PersistenceHandlerProxy::handleSplit(FeedToken token, const Bucket &source, const Bucket &target1, const Bucket &target2)
 {
-    FeedOperation::UP op(new SplitBucketOperation(source.getBucketId().stripUnused(),
-                                                  target1.getBucketId().stripUnused(),
-                                                  target2.getBucketId().stripUnused()));
+    auto op = std::make_unique<SplitBucketOperation>(source.getBucketId().stripUnused(),
+                                                     target1.getBucketId().stripUnused(),
+                                                     target2.getBucketId().stripUnused());
     _feedHandler.handleOperation(token, std::move(op));
 }
 
diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.cpp b/searchcore/src/vespa/searchcore/proton/server/proton.cpp
index 4daf3e895af..c886b371064 100644
--- a/searchcore/src/vespa/searchcore/proton/server/proton.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/proton.cpp
@@ -32,6 +32,7 @@
 #include <vespa/vespalib/util/host_name.h>
 #include <vespa/vespalib/util/random.h>
 #include <vespa/vespalib/net/state_server.h>
+#include <vespa/vespalib/util/blockingthreadstackexecutor.h>
 
 #include <vespa/searchlib/aggregation/forcelink.hpp>
 #include <vespa/searchlib/expression/forcelink.hpp>
@@ -594,10 +595,10 @@ Proton::addDocumentDB(const document::DocumentType &docType,
         auto persistenceHandler = std::make_shared<PersistenceHandlerProxy>(ret);
         if (!_isInitializing) {
             _persistenceEngine->propagateSavedClusterState(bucketSpace, *persistenceHandler);
-            _persistenceEngine->populateInitialBucketDB(bucketSpace, *persistenceHandler);
+            _persistenceEngine->populateInitialBucketDB(persistenceWGuard, bucketSpace, *persistenceHandler);
         }
         // TODO: Fix race with new cluster state setting.
-        _persistenceEngine->putHandler(bucketSpace, docTypeName, persistenceHandler);
+        _persistenceEngine->putHandler(persistenceWGuard, bucketSpace, docTypeName, persistenceHandler);
     }
     auto searchHandler = std::make_shared<SearchHandlerProxy>(ret);
     _summaryEngine->putSearchHandler(docTypeName, searchHandler);
@@ -628,8 +629,7 @@ Proton::removeDocumentDB(const DocTypeName &docTypeName)
         {
             // Not allowed to get to service layer to call pause().
             std::unique_lock<std::shared_timed_mutex> persistenceWguard(_persistenceEngine->getWLock());
-            IPersistenceHandler::SP oldHandler;
-            oldHandler = _persistenceEngine->removeHandler(old->getBucketSpace(), docTypeName);
+            IPersistenceHandler::SP  oldHandler = _persistenceEngine->removeHandler(persistenceWguard, old->getBucketSpace(), docTypeName);
             if (_initComplete && oldHandler) {
                 // TODO: Fix race with bucket db modifying ops.
                 _persistenceEngine->grabExtraModifiedBuckets(old->getBucketSpace(), *oldHandler);
@@ -662,6 +662,7 @@ Proton::ping(MonitorRequest::UP request, MonitorClient & client)
     ret.timestamp = (_matchEngine->isOnline()) ? 42 : 0;
     ret.activeDocs = (_matchEngine->isOnline()) ? getNumActiveDocs() : 0;
     ret.activeDocsRequested = request->reportActiveDocs;
+    ret.is_blocking_writes = !_diskMemUsageSampler->writeFilter().acceptWriteOperation();
     return reply;
 }
 
@@ -684,25 +685,6 @@ Proton::prepareRestart()
 
 namespace {
 
-int countOpenFiles()
-{
-    static const char * const fd_dir_name = "/proc/self/fd";
-    int count = 0;
-    DIR *dp = opendir(fd_dir_name);
-    if (dp != nullptr) {
-        struct dirent *ptr;
-        while ((ptr = readdir(dp)) != nullptr) {
-            if (strcmp(".", ptr->d_name) == 0) continue;
-            if (strcmp("..", ptr->d_name) == 0) continue;
-            ++count;
-        }
-        closedir(dp);
-    } else {
-        LOG(warning, "could not scan directory %s: %s", fd_dir_name, strerror(errno));
-    }
-    return count;
-}
-
 void
 updateExecutorMetrics(ExecutorMetrics &metrics,
                       const vespalib::ThreadStackExecutor::Stats &stats)
@@ -729,7 +711,7 @@ Proton::updateMetrics(const vespalib::MonitorGuard &)
         metrics.resourceUsage.memory.set(usageState.memoryState().usage());
         metrics.resourceUsage.memoryUtilization.set(usageState.memoryState().utilization());
         metrics.resourceUsage.memoryMappings.set(usageFilter.getMemoryStats().getMappingsCount());
-        metrics.resourceUsage.openFileDescriptors.set(countOpenFiles());
+        metrics.resourceUsage.openFileDescriptors.set(FastOS_File::count_open_files());
         metrics.resourceUsage.feedingBlocked.set((usageFilter.acceptWriteOperation() ? 0.0 : 1.0));
     }
     {
diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.h b/searchcore/src/vespa/searchcore/proton/server/proton.h
index 410f45162e4..4c9d4c77cc4 100644
--- a/searchcore/src/vespa/searchcore/proton/server/proton.h
+++ b/searchcore/src/vespa/searchcore/proton/server/proton.h
@@ -57,7 +57,7 @@ private:
     typedef search::engine::MonitorClient                 MonitorClient;
     typedef std::map<DocTypeName, DocumentDB::SP>         DocumentDBMap;
     typedef BootstrapConfig::ProtonConfigSP               ProtonConfigSP;
-    using InitializeThreads = std::shared_ptr<vespalib::ThreadStackExecutorBase>;
+    using InitializeThreads = std::shared_ptr<vespalib::SyncableThreadExecutor>;
     using BucketSpace = document::BucketSpace;
 
     struct MetricsUpdateHook : metrics::UpdateHook
diff --git a/searchcore/src/vespa/searchcore/proton/server/proton_configurer.cpp b/searchcore/src/vespa/searchcore/proton/server/proton_configurer.cpp
index 0b9293a4aab..45e3c978dd9 100644
--- a/searchcore/src/vespa/searchcore/proton/server/proton_configurer.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/proton_configurer.cpp
@@ -39,7 +39,7 @@ getBucketSpace(const BootstrapConfig &bootstrapConfig, const DocTypeName &name)
 }
 
 
-ProtonConfigurer::ProtonConfigurer(vespalib::ThreadStackExecutorBase &executor,
+ProtonConfigurer::ProtonConfigurer(vespalib::SyncableThreadExecutor &executor,
                                    IProtonConfigurerOwner &owner,
                                    const std::unique_ptr<IProtonDiskLayout> &diskLayout)
     : IProtonConfigurer(),
diff --git a/searchcore/src/vespa/searchcore/proton/server/proton_configurer.h b/searchcore/src/vespa/searchcore/proton/server/proton_configurer.h
index c896f12bd4f..54399a26365 100644
--- a/searchcore/src/vespa/searchcore/proton/server/proton_configurer.h
+++ b/searchcore/src/vespa/searchcore/proton/server/proton_configurer.h
@@ -25,7 +25,7 @@ class IProtonDiskLayout;
 class ProtonConfigurer : public IProtonConfigurer
 {
     using DocumentDBs = std::map<DocTypeName, std::pair<std::weak_ptr<IDocumentDBConfigOwner>, std::weak_ptr<DocumentDBDirectoryHolder>>>;
-    using InitializeThreads = std::shared_ptr<vespalib::ThreadStackExecutorBase>;
+    using InitializeThreads = std::shared_ptr<vespalib::SyncableThreadExecutor>;
 
     ExecutorThreadService _executor;
     IProtonConfigurerOwner &_owner;
@@ -48,11 +48,11 @@ class ProtonConfigurer : public IProtonConfigurer
     void pruneInitialDocumentDBDirs(const ProtonConfigSnapshot &configSnapshot);
 
 public:
-    ProtonConfigurer(vespalib::ThreadStackExecutorBase &executor,
+    ProtonConfigurer(vespalib::SyncableThreadExecutor &executor,
                      IProtonConfigurerOwner &owner,
                      const std::unique_ptr<IProtonDiskLayout> &diskLayout);
 
-    ~ProtonConfigurer();
+    ~ProtonConfigurer() override;
 
     void setAllowReconfig(bool allowReconfig);
 
diff --git a/searchcore/src/vespa/searchcore/proton/server/replaypacketdispatcher.cpp b/searchcore/src/vespa/searchcore/proton/server/replaypacketdispatcher.cpp
index fcf1cf2a58c..a1138503085 100644
--- a/searchcore/src/vespa/searchcore/proton/server/replaypacketdispatcher.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/replaypacketdispatcher.cpp
@@ -36,7 +36,11 @@ ReplayPacketDispatcher::replayEntry(const Packet::Entry &entry)
         replay(op, is, entry);
         break;
     } case FeedOperation::REMOVE: {
-        RemoveOperation op;
+        RemoveOperationWithDocId op;
+        replay(op, is, entry);
+        break;
+    } case FeedOperation::REMOVE_GID: {
+        RemoveOperationWithGid op;
         replay(op, is, entry);
         break;
     } case FeedOperation::UPDATE: {
@@ -84,7 +88,7 @@ ReplayPacketDispatcher::replayEntry(const Packet::Entry &entry)
         throw IllegalStateException
             (make_string("Got packet entry with unknown type id '%u' from TLS", entry.type()));
     }
-    if (is.size() > 0) {
+    if ( ! is.empty()) {
         throw document::DeserializeException
             (make_string("Too much data in packet entry (type id '%u', %ld bytes)",
                          entry.type(), is.size()));
diff --git a/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.cpp b/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.cpp
index 069d3727850..9ef038b7325 100644
--- a/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.cpp
@@ -84,17 +84,17 @@ reconfigureMatchView(const Matchers::SP &matchers,
 }
 
 void
-SearchableDocSubDBConfigurer::reconfigureSearchView(const MatchView::SP &matchView)
+SearchableDocSubDBConfigurer::reconfigureSearchView(MatchView::SP matchView)
 {
     SearchView::SP curr = _searchView.get();
-    _searchView.set(SearchView::SP(new SearchView(curr->getSummarySetup(), matchView)));
+    _searchView.set(SearchView::create(curr->getSummarySetup(), std::move(matchView)));
 }
 
 void
-SearchableDocSubDBConfigurer::reconfigureSearchView(const ISummaryManager::ISummarySetup::SP &summarySetup,
-                                                    const MatchView::SP &matchView)
+SearchableDocSubDBConfigurer::reconfigureSearchView(ISummaryManager::ISummarySetup::SP summarySetup,
+                                                    MatchView::SP matchView)
 {
-    _searchView.set(SearchView::SP(new SearchView(summarySetup, matchView)));
+    _searchView.set(SearchView::create(std::move(summarySetup), std::move(matchView)));
 }
 
 SearchableDocSubDBConfigurer::
diff --git a/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.h b/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.h
index 5aac069853b..459c4651e67 100644
--- a/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.h
+++ b/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.h
@@ -65,11 +65,10 @@ private:
                          const IAttributeManager::SP &attrMgr);
 
     void
-    reconfigureSearchView(const MatchView::SP &matchView);
+    reconfigureSearchView(MatchView::SP matchView);
 
     void
-    reconfigureSearchView(const ISummaryManager::ISummarySetup::SP &summarySetup,
-                           const MatchView::SP &matchView);
+    reconfigureSearchView(ISummaryManager::ISummarySetup::SP summarySetup, MatchView::SP matchView);
 
 public:
     SearchableDocSubDBConfigurer(const SearchableDocSubDBConfigurer &) = delete;
diff --git a/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.cpp b/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.cpp
index eab118c1188..e0d873152cd 100644
--- a/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.cpp
@@ -4,13 +4,9 @@
 #include "forcecommitcontext.h"
 #include "operationdonecontext.h"
 #include "removedonecontext.h"
-#include <vespa/searchcore/proton/common/feedtoken.h>
 #include <vespa/searchcore/proton/documentmetastore/ilidreusedelayer.h>
 #include <vespa/searchcore/proton/feedoperation/compact_lid_space_operation.h>
-#include <vespa/searchlib/common/isequencedtaskexecutor.h>
-#include <vespa/vespalib/text/stringtokenizer.h>
-#include <vespa/vespalib/util/closuretask.h>
-#include <vespa/vespalib/util/exceptions.h>
+#include <vespa/vespalib/util/isequencedtaskexecutor.h>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".proton.server.searchable_feed_view");
@@ -23,8 +19,6 @@ using document::DocumentUpdate;
 using search::index::Schema;
 using storage::spi::BucketInfoResult;
 using storage::spi::Timestamp;
-using vespalib::IllegalStateException;
-using vespalib::make_string;
 using vespalib::makeLambdaTask;
 
 namespace proton {
diff --git a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp
index 77852dcc918..3ca8a4cff49 100644
--- a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp
@@ -203,16 +203,16 @@ SearchableDocSubDB::initViews(const DocumentDBConfig &configSnapshot, const Sess
     const IIndexManager::SP &indexMgr = getIndexManager();
     _constantValueRepo.reconfigure(configSnapshot.getRankingConstants());
     Matchers::SP matchers(_configurer.createMatchers(schema, configSnapshot.getRankProfilesConfig()).release());
-    auto matchView = std::make_shared<MatchView>(matchers, indexMgr->getSearchable(), attrMgr,
+    auto matchView = std::make_shared<MatchView>(std::move(matchers), indexMgr->getSearchable(), attrMgr,
                                                  sessionManager, _metaStoreCtx, _docIdLimit);
-    _rSearchView.set(std::make_shared<SearchView>(
+    _rSearchView.set(SearchView::create(
                                       getSummaryManager()->createSummarySetup(
                                               configSnapshot.getSummaryConfig(),
                                               configSnapshot.getSummarymapConfig(),
                                               configSnapshot.getJuniperrcConfig(),
                                               configSnapshot.getDocumentTypeRepoSP(),
-                                              matchView->getAttributeManager()),
-                                      matchView));
+                                              attrMgr),
+                                      std::move(matchView)));
 
     auto attrWriter = std::make_shared<AttributeWriter>(attrMgr);
     {
diff --git a/searchcore/src/vespa/searchcore/proton/server/searchhandlerproxy.cpp b/searchcore/src/vespa/searchcore/proton/server/searchhandlerproxy.cpp
index f611b4e1f4c..6da6c09cdba 100644
--- a/searchcore/src/vespa/searchcore/proton/server/searchhandlerproxy.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/searchhandlerproxy.cpp
@@ -7,8 +7,8 @@
 
 namespace proton {
 
-SearchHandlerProxy::SearchHandlerProxy(const DocumentDB::SP &documentDB)
-    : _documentDB(documentDB)
+SearchHandlerProxy::SearchHandlerProxy(DocumentDB::SP documentDB)
+    : _documentDB(std::move(documentDB))
 {
     _documentDB->retain();
 }
@@ -25,11 +25,9 @@ SearchHandlerProxy::getDocsums(const DocsumRequest & request)
 }
 
 std::unique_ptr<search::engine::SearchReply>
-SearchHandlerProxy::match(const ISearchHandler::SP &searchHandler,
-                          const SearchRequest &req,
-                          vespalib::ThreadBundle &threadBundle) const
+SearchHandlerProxy::match(const SearchRequest &req, vespalib::ThreadBundle &threadBundle) const
 {
-    return _documentDB->match(searchHandler, req, threadBundle);
+    return _documentDB->match(req, threadBundle);
 }
 
 } // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/searchhandlerproxy.h b/searchcore/src/vespa/searchcore/proton/server/searchhandlerproxy.h
index 4e9f5bdab8a..fc4f517fb36 100644
--- a/searchcore/src/vespa/searchcore/proton/server/searchhandlerproxy.h
+++ b/searchcore/src/vespa/searchcore/proton/server/searchhandlerproxy.h
@@ -13,11 +13,11 @@ class SearchHandlerProxy : public ISearchHandler
 private:
     std::shared_ptr<DocumentDB> _documentDB;
 public:
-    SearchHandlerProxy(const std::shared_ptr<DocumentDB> &documentDB);
+    SearchHandlerProxy(std::shared_ptr<DocumentDB> documentDB);
 
-    virtual~SearchHandlerProxy();
+    ~SearchHandlerProxy() override;
     std::unique_ptr<DocsumReply> getDocsums(const DocsumRequest & request) override;
-    std::unique_ptr<SearchReply> match(const ISearchHandler::SP &searchHandler, const SearchRequest &req, ThreadBundle &threadBundle) const override;
+    std::unique_ptr<SearchReply> match(const SearchRequest &req, ThreadBundle &threadBundle) const override;
 };
 
 } // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/searchview.cpp b/searchcore/src/vespa/searchcore/proton/server/searchview.cpp
index 9830048cdd8..36d873d9948 100644
--- a/searchcore/src/vespa/searchcore/proton/server/searchview.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/searchview.cpp
@@ -105,11 +105,14 @@ createEmptyReply(const DocsumRequest & request)
 
 }
 
-SearchView::SearchView(const ISummaryManager::ISummarySetup::SP & summarySetup,
-                       const MatchView::SP & matchView)
+std::shared_ptr<SearchView>
+SearchView::create(ISummaryManager::ISummarySetup::SP summarySetup, MatchView::SP matchView) {
+    return std::shared_ptr<SearchView>( new SearchView(std::move(summarySetup), std::move(matchView)));
+}
+SearchView::SearchView(ISummaryManager::ISummarySetup::SP summarySetup, MatchView::SP matchView)
     : ISearchHandler(),
-      _summarySetup(summarySetup),
-      _matchView(matchView)
+      _summarySetup(std::move(summarySetup)),
+      _matchView(std::move(matchView))
 { }
 
 SearchView::~SearchView() = default;
@@ -161,8 +164,8 @@ SearchView::getDocsumsInternal(const DocsumRequest & req)
 }
 
 std::unique_ptr<SearchReply>
-SearchView::match(const ISearchHandler::SP &self, const SearchRequest &req, ThreadBundle &threadBundle) const {
-    return _matchView->match(self, req, threadBundle);
+SearchView::match(const SearchRequest &req, ThreadBundle &threadBundle) const {
+    return _matchView->match(shared_from_this(), req, threadBundle);
 }
 
 } // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/server/searchview.h b/searchcore/src/vespa/searchcore/proton/server/searchview.h
index 28c7ffd2b36..630d29c32fd 100644
--- a/searchcore/src/vespa/searchcore/proton/server/searchview.h
+++ b/searchcore/src/vespa/searchcore/proton/server/searchview.h
@@ -8,7 +8,7 @@
 
 namespace proton {
 
-class SearchView : public ISearchHandler
+class SearchView : public ISearchHandler, public std::enable_shared_from_this<SearchView>
 {
 public:
     using SessionManagerSP = std::shared_ptr<matching::SessionManager>;
@@ -16,12 +16,12 @@ public:
     using InternalDocsumReply = std::pair<std::unique_ptr<DocsumReply>, bool>;
     typedef std::shared_ptr<SearchView> SP;
 
-    SearchView(const ISummaryManager::ISummarySetup::SP &summarySetup, const MatchView::SP &matchView);
+    static std::shared_ptr<SearchView> create(ISummaryManager::ISummarySetup::SP summarySetup, MatchView::SP matchView);
     SearchView(const SearchView &) = delete;
     SearchView(SearchView &&) = delete;
     SearchView &operator=(const SearchView &) = delete;
     SearchView &operator=(SearchView &&) = delete;
-    ~SearchView();
+    ~SearchView() override;
 
     const ISummaryManager::ISummarySetup::SP & getSummarySetup() const { return _summarySetup; }
     const MatchView::SP & getMatchView() const { return _matchView; }
@@ -34,8 +34,9 @@ public:
     matching::MatchingStats getMatcherStats(const vespalib::string &rankProfile) const { return _matchView->getMatcherStats(rankProfile); }
 
     std::unique_ptr<DocsumReply> getDocsums(const DocsumRequest & req) override;
-    std::unique_ptr<SearchReply> match(const ISearchHandler::SP &self, const SearchRequest &req, vespalib::ThreadBundle &threadBundle) const override;
+    std::unique_ptr<SearchReply> match(const SearchRequest &req, vespalib::ThreadBundle &threadBundle) const override;
 private:
+    SearchView(ISummaryManager::ISummarySetup::SP summarySetup, MatchView::SP matchView);
     InternalDocsumReply getDocsumsInternal(const DocsumRequest & req);
     ISummaryManager::ISummarySetup::SP _summarySetup;
     MatchView::SP                      _matchView;
diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp
index 646346d53ad..59a115ce5d1 100644
--- a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp
@@ -367,6 +367,30 @@ StoreOnlyDocSubDB::initViews(const DocumentDBConfig &configSnapshot, const Sessi
 }
 
 void
+StoreOnlyDocSubDB::validateDocStore(FeedHandler & feedHandler, SerialNum serialNum) const
+{
+    LOG(info, "Validating document store for sub db %u doctype %s", _subDbId, _docTypeName.toString().c_str());
+
+    search::IDocumentStore &docStore = _iSummaryMgr->getBackingStore();
+    DocStoreValidator validator(_metaStoreCtx->get());
+    search::DocumentStoreVisitorProgress validatorProgress;
+
+    docStore.accept(validator, validatorProgress, *_iFeedView.get()->getDocumentTypeRepo());
+
+    validator.visitDone();
+
+    LOG(info, "Validated document store for sub db %u, doctype %s, %u orphans, %u invalid, %u visits, %u empty visits",
+        _subDbId, _docTypeName.toString().c_str(), validator.getOrphanCount(),
+        validator.getInvalidCount(), validator.getVisitCount(), validator.getVisitEmptyCount());
+
+    validator.killOrphans(docStore, serialNum);
+    if (validator.getInvalidCount() != 0u) {
+        validator.performRemoves(feedHandler, docStore, *_iFeedView.get()->getDocumentTypeRepo());
+    }
+}
+
+
+void
 StoreOnlyDocSubDB::initFeedView(const DocumentDBConfig &configSnapshot)
 {
     assert(_writeService.master().isCurrentThread());
diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h
index 9d5b8c18d01..700a6d29460 100644
--- a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h
+++ b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h
@@ -13,7 +13,6 @@
 #include <vespa/searchcore/proton/documentmetastore/documentmetastorecontext.h>
 #include <vespa/searchcore/proton/documentmetastore/documentmetastoreflushtarget.h>
 #include <vespa/searchcore/proton/documentmetastore/ilidreusedelayer.h>
-#include <vespa/searchcore/proton/matchengine/imatchhandler.h>
 #include <vespa/searchcore/proton/summaryengine/isearchhandler.h>
 #include <vespa/searchcore/proton/common/commit_time_tracker.h>
 #include <vespa/searchcore/proton/persistenceengine/i_document_retriever.h>
@@ -208,6 +207,8 @@ public:
     void setup(const DocumentSubDbInitializerResult &initResult) override;
     void initViews(const DocumentDBConfig &configSnapshot, const std::shared_ptr<matching::SessionManager> &sessionManager) override;
 
+    void validateDocStore(FeedHandler & feedHandler, SerialNum serialNum) const override;
+
     IReprocessingTask::List
     applyConfig(const DocumentDBConfig &newConfigSnapshot, const DocumentDBConfig &oldConfigSnapshot,
                 SerialNum serialNum, const ReconfigParams &params, IDocumentDBReferenceResolver &resolver) override;
diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp
index 6397195da93..bee8a0e0473 100644
--- a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp
@@ -18,7 +18,7 @@
 #include <vespa/searchcore/proton/documentmetastore/ilidreusedelayer.h>
 #include <vespa/searchcore/proton/feedoperation/operations.h>
 #include <vespa/searchcore/proton/reference/i_gid_to_lid_change_handler.h>
-#include <vespa/searchlib/common/isequencedtaskexecutor.h>
+#include <vespa/vespalib/util/isequencedtaskexecutor.h>
 #include <vespa/vespalib/util/exceptions.h>
 
 #include <vespa/log/log.h>
@@ -27,6 +27,7 @@ LOG_SETUP(".proton.server.storeonlyfeedview");
 using document::BucketId;
 using document::Document;
 using document::DocumentId;
+using document::GlobalId;
 using document::DocumentTypeRepo;
 using document::DocumentUpdate;
 using proton::attribute::isUpdateableInMemoryOnly;
@@ -55,7 +56,7 @@ public:
         : PutDoneContext(std::move(token), gidToLidChangeHandler, std::move(doc), gid, lid, serialNum, enableNotifyPut),
           _moveDoneCtx(std::move(moveDoneCtx))
     {}
-    ~PutDoneContextForMove() = default;
+    ~PutDoneContextForMove() override = default;
 };
 
 std::shared_ptr<PutDoneContext>
@@ -80,7 +81,7 @@ createPutDoneContext(FeedToken token, IGidToLidChangeHandler &gidToLidChangeHand
                      std::shared_ptr<const Document> doc,
                      const document::GlobalId &gid, uint32_t lid, SerialNum serialNum, bool enableNotifyPut)
 {
-    return createPutDoneContext(token, gidToLidChangeHandler, std::move(doc), gid, lid, serialNum, enableNotifyPut, IDestructorCallback::SP());
+    return createPutDoneContext(std::move(token), gidToLidChangeHandler, std::move(doc), gid, lid, serialNum, enableNotifyPut, IDestructorCallback::SP());
 }
 
 std::shared_ptr<UpdateDoneContext>
@@ -110,7 +111,7 @@ public:
         : RemoveDoneContext(std::move(token), executor, documentMetaStore, std::move(pendingNotifyRemoveDone) ,lid),
           _moveDoneCtx(std::move(moveDoneCtx))
     {}
-    ~RemoveDoneContextForMove() = default;
+    ~RemoveDoneContextForMove() override = default;
 };
 
 std::shared_ptr<RemoveDoneContext>
@@ -142,7 +143,7 @@ std::vector<document::GlobalId> getGidsToRemove(const IDocumentMetaStore &metaSt
     return gids;
 }
 
-void putMetaData(documentmetastore::IStore &meta_store, const DocumentId &doc_id,
+void putMetaData(documentmetastore::IStore &meta_store, const DocumentId & doc_id,
                  const DocumentOperation &op, bool is_removed_doc)
 {
     documentmetastore::IStore::Result putRes(
@@ -157,23 +158,23 @@ void putMetaData(documentmetastore::IStore &meta_store, const DocumentId &doc_id
     assert(op.getLid() == putRes._lid);
 }
 
-void removeMetaData(documentmetastore::IStore &meta_store, const DocumentId &doc_id,
+void removeMetaData(documentmetastore::IStore &meta_store, const GlobalId & gid, const DocumentId &doc_id,
                     const DocumentOperation &op, bool is_removed_doc) {
     assert(meta_store.validLid(op.getPrevLid()));
     assert(is_removed_doc == op.getPrevMarkedAsRemoved());
     const RawDocumentMetaData &meta(meta_store.getRawMetaData(op.getPrevLid()));
-    assert(meta.getGid() == doc_id.getGlobalId());
+    assert(meta.getGid() == gid);
     (void) meta;
     if (!meta_store.remove(op.getPrevLid())) {
         throw IllegalStateException(
                 make_string("Could not remove <lid, gid> pair for %sdocument with id '%s' and gid '%s'",
                             is_removed_doc ? "removed " : "", doc_id.toString().c_str(),
-                            doc_id.getGlobalId().toString().c_str()));
+                            gid.toString().c_str()));
     }
 }
 
 void
-moveMetaData(documentmetastore::IStore &meta_store, const DocumentId &doc_id, const DocumentOperation &op)
+moveMetaData(documentmetastore::IStore &meta_store, const DocumentId & doc_id, const DocumentOperation &op)
 {
     (void) doc_id;
     assert(op.getLid() != op.getPrevLid());
@@ -280,7 +281,7 @@ StoreOnlyFeedView::internalPut(FeedToken token, const PutOperation &putOp)
          putOp.getSubDbId(), putOp.getLid(), putOp.getPrevSubDbId(), putOp.getPrevLid(),
          _params._subDbId, doc->toString(true).size(), doc->toString(true).c_str());
 
-    PendingNotifyRemoveDone pendingNotifyRemoveDone = adjustMetaStore(putOp, docId);
+    PendingNotifyRemoveDone pendingNotifyRemoveDone = adjustMetaStore(putOp, docId.getGlobalId(), docId);
     considerEarlyAck(token);
 
     bool docAlreadyExists = putOp.getValidPrevDbdId(_params._subDbId);
@@ -530,10 +531,8 @@ StoreOnlyFeedView::removeIndexedFields(SerialNum, Lid, bool, OnRemoveDoneType) {
 void
 StoreOnlyFeedView::prepareRemove(RemoveOperation &rmOp)
 {
-    const DocumentId &id = rmOp.getDocumentId();
-    const document::GlobalId &gid = id.getGlobalId();
-    documentmetastore::IStore::Result inspectRes = _metaStore.inspect(gid);
-    if (_params._subDbType == SubDbType::REMOVED) {
+    documentmetastore::IStore::Result inspectRes = _metaStore.inspect(rmOp.getGlobalId());
+    if ((_params._subDbType == SubDbType::REMOVED) && (rmOp.getType() == FeedOperation::REMOVE)) {
         rmOp.setDbDocumentId(DbDocumentId(_params._subDbId, inspectRes._lid));
     }
     setPrev(rmOp, inspectRes, _params._subDbId, _params._subDbType == SubDbType::REMOVED);
@@ -541,11 +540,18 @@ StoreOnlyFeedView::prepareRemove(RemoveOperation &rmOp)
 
 void
 StoreOnlyFeedView::handleRemove(FeedToken token, const RemoveOperation &rmOp) {
-    internalRemove(std::move(token), rmOp);
+    if (rmOp.getType() == FeedOperation::REMOVE) {
+        internalRemove(std::move(token), dynamic_cast<const RemoveOperationWithDocId &>(rmOp));
+    } else if (rmOp.getType() == FeedOperation::REMOVE_GID) {
+        internalRemove(std::move(token), dynamic_cast<const RemoveOperationWithGid &>(rmOp));
+    } else {
+        assert(rmOp.getType() == FeedOperation::REMOVE);
+    }
+
 }
 
 void
-StoreOnlyFeedView::internalRemove(FeedToken token, const RemoveOperation &rmOp)
+StoreOnlyFeedView::internalRemove(FeedToken token, const RemoveOperationWithDocId &rmOp)
 {
     assert(rmOp.getValidNewOrPrevDbdId());
     assert(rmOp.notMovingLidInSameSubDb());
@@ -557,11 +563,11 @@ StoreOnlyFeedView::internalRemove(FeedToken token, const RemoveOperation &rmOp)
          _params._docTypeName.toString().c_str(), serialNum, docId.toString().c_str(),
          rmOp.getSubDbId(), rmOp.getLid(), rmOp.getPrevSubDbId(), rmOp.getPrevLid(), _params._subDbId);
 
-    PendingNotifyRemoveDone pendingNotifyRemoveDone = adjustMetaStore(rmOp, docId);
+    PendingNotifyRemoveDone pendingNotifyRemoveDone = adjustMetaStore(rmOp, docId.getGlobalId(), docId);
     considerEarlyAck(token);
 
     if (rmOp.getValidDbdId(_params._subDbId)) {
-        Document::UP clearDoc(new Document(*_docType, docId));
+        auto clearDoc = std::make_unique<Document>(*_docType, docId);
         clearDoc->setRepo(*_repo);
 
         putSummary(serialNum, rmOp.getLid(), std::move(clearDoc), std::shared_ptr<OperationDoneContext>());
@@ -576,6 +582,25 @@ StoreOnlyFeedView::internalRemove(FeedToken token, const RemoveOperation &rmOp)
 }
 
 void
+StoreOnlyFeedView::internalRemove(FeedToken token, const RemoveOperationWithGid &rmOp)
+{
+    assert(rmOp.getValidNewOrPrevDbdId());
+    assert(rmOp.notMovingLidInSameSubDb());
+    const SerialNum serialNum = rmOp.getSerialNum();
+    DocumentId dummy;
+    PendingNotifyRemoveDone pendingNotifyRemoveDone = adjustMetaStore(rmOp, rmOp.getGlobalId(), dummy);
+    considerEarlyAck(token);
+
+    if (rmOp.getValidPrevDbdId(_params._subDbId)) {
+        if (rmOp.changedDbdId()) {
+            assert(!rmOp.getValidDbdId(_params._subDbId));
+            internalRemove(std::move(token), serialNum, std::move(pendingNotifyRemoveDone),
+                           rmOp.getPrevLid(), IDestructorCallback::SP());
+        }
+    }
+}
+
+void
 StoreOnlyFeedView::internalRemove(FeedToken token, SerialNum serialNum,
                                   PendingNotifyRemoveDone &&pendingNotifyRemoveDone, Lid lid,
                                   IDestructorCallback::SP moveDoneCtx)
@@ -592,7 +617,7 @@ StoreOnlyFeedView::internalRemove(FeedToken token, SerialNum serialNum,
 }
 
 PendingNotifyRemoveDone
-StoreOnlyFeedView::adjustMetaStore(const DocumentOperation &op, const DocumentId &docId)
+StoreOnlyFeedView::adjustMetaStore(const DocumentOperation &op, const GlobalId & gid, const DocumentId &docId)
 {
     PendingNotifyRemoveDone pendingNotifyRemoveDone;
     const SerialNum serialNum = op.getSerialNum();
@@ -607,9 +632,9 @@ StoreOnlyFeedView::adjustMetaStore(const DocumentOperation &op, const DocumentId
                 putMetaData(_metaStore, docId, op, _params._subDbType == SubDbType::REMOVED);
             }
         } else if (op.getValidPrevDbdId(_params._subDbId)) {
-            _gidToLidChangeHandler.notifyRemove(docId.getGlobalId(), serialNum);
-            pendingNotifyRemoveDone.setup(_gidToLidChangeHandler, docId.getGlobalId(), serialNum);
-            removeMetaData(_metaStore, docId, op, _params._subDbType == SubDbType::REMOVED);
+            _gidToLidChangeHandler.notifyRemove(gid, serialNum);
+            pendingNotifyRemoveDone.setup(_gidToLidChangeHandler, gid, serialNum);
+            removeMetaData(_metaStore, gid, docId, op, _params._subDbType == SubDbType::REMOVED);
         }
         _metaStore.commit(serialNum, serialNum);
     }
@@ -728,7 +753,7 @@ StoreOnlyFeedView::handleMove(const MoveOperation &moveOp, IDestructorCallback::
          moveOp.getSubDbId(), moveOp.getLid(), moveOp.getPrevSubDbId(), moveOp.getPrevLid(),
          _params._subDbId, doc->toString(true).size(), doc->toString(true).c_str());
 
-    PendingNotifyRemoveDone pendingNotifyRemoveDone = adjustMetaStore(moveOp, docId);
+    PendingNotifyRemoveDone pendingNotifyRemoveDone = adjustMetaStore(moveOp, docId.getGlobalId(), docId);
     bool docAlreadyExists = moveOp.getValidPrevDbdId(_params._subDbId);
     if (moveOp.getValidDbdId(_params._subDbId)) {
         bool immediateCommit = _commitTimeTracker.needCommit();
diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h
index 4c1d50232dd..be2ed9af126 100644
--- a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h
+++ b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h
@@ -20,6 +20,7 @@
 #include <vespa/searchlib/query/base.h>
 #include <vespa/vespalib/util/threadstackexecutorbase.h>
 #include <future>
+#include <vespa/searchcore/proton/feedoperation/operations.h>
 
 namespace search { class IDestructorCallback; }
 
@@ -171,12 +172,13 @@ private:
         return replaySerialNum > _params._flushedDocumentMetaStoreSerialNum;
     }
 
-    PendingNotifyRemoveDone adjustMetaStore(const DocumentOperation &op, const document::DocumentId &docId);
+    PendingNotifyRemoveDone adjustMetaStore(const DocumentOperation &op, const document::GlobalId & gid, const document::DocumentId &docId);
     void internalPut(FeedToken token, const PutOperation &putOp);
     void internalUpdate(FeedToken token, const UpdateOperation &updOp);
 
     bool lookupDocId(const document::DocumentId &docId, Lid & lid) const;
-    void internalRemove(FeedToken token, const RemoveOperation &rmOp);
+    void internalRemove(FeedToken token, const RemoveOperationWithDocId &rmOp);
+    void internalRemove(FeedToken token, const RemoveOperationWithGid &rmOp);
 
     // Removes documents from meta store and document store.
     // returns the number of documents removed.
diff --git a/searchcore/src/vespa/searchcore/proton/server/threading_service_config.cpp b/searchcore/src/vespa/searchcore/proton/server/threading_service_config.cpp
index 8f1c3560e9b..22eb64acdee 100644
--- a/searchcore/src/vespa/searchcore/proton/server/threading_service_config.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/threading_service_config.cpp
@@ -7,13 +7,21 @@
 namespace proton {
 
 using ProtonConfig = ThreadingServiceConfig::ProtonConfig;
+using OptimizeFor = vespalib::Executor::OptimizeFor;
+
 
 ThreadingServiceConfig::ThreadingServiceConfig(uint32_t indexingThreads_,
                                                uint32_t defaultTaskLimit_,
-                                               uint32_t semiUnboundTaskLimit_)
+                                               uint32_t semiUnboundTaskLimit_,
+                                               OptimizeFor optimize_,
+                                               uint32_t kindOfWatermark_,
+                                               vespalib::duration reactionTime_)
     : _indexingThreads(indexingThreads_),
       _defaultTaskLimit(defaultTaskLimit_),
-      _semiUnboundTaskLimit(semiUnboundTaskLimit_)
+      _semiUnboundTaskLimit(semiUnboundTaskLimit_),
+      _optimize(optimize_),
+      _kindOfWatermark(kindOfWatermark_),
+      _reactionTime(reactionTime_)
 {
 }
 
@@ -22,11 +30,25 @@ namespace {
 uint32_t
 calculateIndexingThreads(uint32_t cfgIndexingThreads, double concurrency, const HwInfo::Cpu &cpuInfo)
 {
-    double scaledCores = cpuInfo.cores() * concurrency;
+    // We are capping at 12 threads to reduce cost of waking up threads
+    // to achieve a better throughput.
+    // TODO: Fix this in a simpler/better way.
+    double scaledCores = std::min(12.0, cpuInfo.cores() * concurrency);
     uint32_t indexingThreads = std::max((uint32_t)std::ceil(scaledCores / 3), cfgIndexingThreads);
     return std::max(indexingThreads, 1u);
 }
 
+OptimizeFor
+selectOptimization(ProtonConfig::Indexing::Optimize optimize) {
+    using CfgOptimize = ProtonConfig::Indexing::Optimize;
+    switch (optimize) {
+        case CfgOptimize::LATENCY: return OptimizeFor::LATENCY;
+        case CfgOptimize::THROUGHPUT: return OptimizeFor::THROUGHPUT;
+        case CfgOptimize::ADAPTIVE: return OptimizeFor::ADAPTIVE;
+    }
+    return OptimizeFor::LATENCY;
+}
+
 }
 
 ThreadingServiceConfig
@@ -34,7 +56,15 @@ ThreadingServiceConfig::make(const ProtonConfig &cfg, double concurrency, const
 {
     uint32_t indexingThreads = calculateIndexingThreads(cfg.indexing.threads, concurrency, cpuInfo);
     return ThreadingServiceConfig(indexingThreads, cfg.indexing.tasklimit,
-                                  (cfg.indexing.semiunboundtasklimit / indexingThreads));
+                                  (cfg.indexing.semiunboundtasklimit / indexingThreads),
+                                  selectOptimization(cfg.indexing.optimize),
+                                  cfg.indexing.kindOfWatermark,
+                                  vespalib::from_s(cfg.indexing.reactiontime));
+}
+
+ThreadingServiceConfig
+ThreadingServiceConfig::make(uint32_t indexingThreads) {
+    return ThreadingServiceConfig(indexingThreads, 100, 1000, OptimizeFor::LATENCY, 0, 10ms);
 }
 
 }
diff --git a/searchcore/src/vespa/searchcore/proton/server/threading_service_config.h b/searchcore/src/vespa/searchcore/proton/server/threading_service_config.h
index be39f516598..055c3b46549 100644
--- a/searchcore/src/vespa/searchcore/proton/server/threading_service_config.h
+++ b/searchcore/src/vespa/searchcore/proton/server/threading_service_config.h
@@ -2,6 +2,8 @@
 #pragma once
 
 #include <vespa/searchcore/proton/common/hw_info.h>
+#include <vespa/vespalib/util/executor.h>
+#include <vespa/vespalib/util/time.h>
 #include <cstdint>
 
 namespace vespa::config::search::core::internal { class InternalProtonType; }
@@ -13,21 +15,28 @@ namespace proton {
 class ThreadingServiceConfig {
 public:
     using ProtonConfig = const vespa::config::search::core::internal::InternalProtonType;
+    using OptimizeFor = vespalib::Executor::OptimizeFor;
 
 private:
-    uint32_t _indexingThreads;
-    uint32_t _defaultTaskLimit;
-    uint32_t _semiUnboundTaskLimit;
+    uint32_t           _indexingThreads;
+    uint32_t           _defaultTaskLimit;
+    uint32_t           _semiUnboundTaskLimit;
+    OptimizeFor        _optimize;
+    uint32_t           _kindOfWatermark;
+    vespalib::duration _reactionTime;         // Maximum reaction time to new tasks
 
 private:
-    ThreadingServiceConfig(uint32_t indexingThreads_, uint32_t defaultTaskLimit_, uint32_t semiUnboundTaskLimit_);
+    ThreadingServiceConfig(uint32_t indexingThreads_, uint32_t defaultTaskLimit_, uint32_t semiUnboundTaskLimit_, OptimizeFor optimize, uint32_t kindOfWatermark, vespalib::duration reactionTime);
 
 public:
     static ThreadingServiceConfig make(const ProtonConfig &cfg, double concurrency, const HwInfo::Cpu &cpuInfo);
-
+    static ThreadingServiceConfig make(uint32_t indexingThreads);
     uint32_t indexingThreads() const { return _indexingThreads; }
     uint32_t defaultTaskLimit() const { return _defaultTaskLimit; }
     uint32_t semiUnboundTaskLimit() const { return _semiUnboundTaskLimit; }
+    OptimizeFor optimize() const { return _optimize; }
+    uint32_t kindOfwatermark() const { return _kindOfWatermark; }
+    vespalib::duration reactionTime() const { return _reactionTime; }
 };
 
 }
diff --git a/searchcore/src/vespa/searchcore/proton/server/visibilityhandler.cpp b/searchcore/src/vespa/searchcore/proton/server/visibilityhandler.cpp
index 58000348ecf..1281785e898 100644
--- a/searchcore/src/vespa/searchcore/proton/server/visibilityhandler.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/visibilityhandler.cpp
@@ -1,7 +1,7 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include "visibilityhandler.h"
-#include <vespa/searchlib/common/isequencedtaskexecutor.h>
+#include <vespa/vespalib/util/isequencedtaskexecutor.h>
 #include <vespa/vespalib/util/closuretask.h>
 
 using vespalib::makeTask;
diff --git a/searchcore/src/vespa/searchcore/proton/summaryengine/isearchhandler.h b/searchcore/src/vespa/searchcore/proton/summaryengine/isearchhandler.h
index ffcdfbaf365..b7dd438ae29 100644
--- a/searchcore/src/vespa/searchcore/proton/summaryengine/isearchhandler.h
+++ b/searchcore/src/vespa/searchcore/proton/summaryengine/isearchhandler.h
@@ -26,12 +26,11 @@ protected:
     using DocsumRequest = search::engine::DocsumRequest;
     using ThreadBundle = vespalib::ThreadBundle;
 public:
-    typedef std::unique_ptr<ISearchHandler> UP;
     typedef std::shared_ptr<ISearchHandler> SP;
 
     ISearchHandler(const ISearchHandler &) = delete;
     ISearchHandler & operator = (const ISearchHandler &) = delete;
-    virtual ~ISearchHandler() { }
+    virtual ~ISearchHandler() = default;
 
     /**
      * @return Use the request and produce the document summary result.
@@ -39,7 +38,7 @@ public:
     virtual std::unique_ptr<DocsumReply> getDocsums(const DocsumRequest & request) = 0;
 
     virtual std::unique_ptr<SearchReply>
-    match(const ISearchHandler::SP &self, const SearchRequest &req, ThreadBundle &threadBundle) const = 0;
+    match(const SearchRequest &req, ThreadBundle &threadBundle) const = 0;
 };
 
 } // namespace proton
diff --git a/searchcore/src/vespa/searchcore/proton/summaryengine/summaryengine.cpp b/searchcore/src/vespa/searchcore/proton/summaryengine/summaryengine.cpp
index e154c6761e2..1189e8a550c 100644
--- a/searchcore/src/vespa/searchcore/proton/summaryengine/summaryengine.cpp
+++ b/searchcore/src/vespa/searchcore/proton/summaryengine/summaryengine.cpp
@@ -107,13 +107,13 @@ SummaryEngine::getDocsums(DocsumRequest::Source request, DocsumClient & client)
 {
     if (_closed) {
         LOG(warning, "Receiving docsumrequest after engine has been shutdown");
-        DocsumReply::UP ret(new DocsumReply());
+        auto ret = std::make_unique<DocsumReply>();
 
         // TODO: Notify closed.
 
         return ret;
     }
-    vespalib::Executor::Task::UP task(new DocsumTask(*this, std::move(request), client));
+    auto task =std::make_unique<DocsumTask>(*this, std::move(request), client);
     _executor.execute(std::move(task));
     return DocsumReply::UP();
 }
@@ -128,13 +128,13 @@ SummaryEngine::getDocsums(DocsumRequest::UP req)
         if (searchHandler) {
             reply = searchHandler->getDocsums(*req);
         } else {
-            vespalib::Sequence<ISearchHandler*>::UP snapshot;
+            HandlerMap<ISearchHandler>::Snapshot snapshot;
             {
                 std::lock_guard<std::mutex> guard(_lock);
                 snapshot = _handlers.snapshot();
             }
-            if (snapshot->valid()) {
-                reply = snapshot->get()->getDocsums(*req); // use the first handler
+            if (snapshot.valid()) {
+                reply = snapshot.get()->getDocsums(*req); // use the first handler
             }
         }
         updateDocsumMetrics(vespalib::to_s(req->getTimeUsed()), getNumDocs(*reply));
diff --git a/searchcore/src/vespa/searchcore/proton/test/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/test/CMakeLists.txt
index ad5ae54f6e1..488ac1041f1 100644
--- a/searchcore/src/vespa/searchcore/proton/test/CMakeLists.txt
+++ b/searchcore/src/vespa/searchcore/proton/test/CMakeLists.txt
@@ -8,6 +8,7 @@ vespa_add_library(searchcore_test STATIC
     documentdb_config_builder.cpp
     dummy_feed_view.cpp
     userdocumentsbuilder.cpp
+    threading_service_observer.cpp
     DEPENDS
     searchcore_fconfig
 )
diff --git a/searchcore/src/vespa/searchcore/proton/test/dummy_document_sub_db.h b/searchcore/src/vespa/searchcore/proton/test/dummy_document_sub_db.h
index 6c11ebbbf6b..7358a78de61 100644
--- a/searchcore/src/vespa/searchcore/proton/test/dummy_document_sub_db.h
+++ b/searchcore/src/vespa/searchcore/proton/test/dummy_document_sub_db.h
@@ -31,7 +31,7 @@ struct DummyDocumentSubDb : public IDocumentSubDB
 
     DummyDocumentSubDb(std::shared_ptr<BucketDBOwner> bucketDB, uint32_t subDbId)
         : _subDbId(subDbId),
-          _metaStoreCtx(bucketDB),
+          _metaStoreCtx(std::move(bucketDB)),
           _summaryManager(),
           _indexManager(),
           _summaryAdapter(),
@@ -40,7 +40,7 @@ struct DummyDocumentSubDb : public IDocumentSubDB
           _writeService(std::make_unique<ExecutorThreadingService>(_sharedExecutor, 1))
     {
     }
-    ~DummyDocumentSubDb() {}
+    ~DummyDocumentSubDb() override { }
     void close() override { }
     uint32_t getSubDbId() const override { return _subDbId; }
     vespalib::string getName() const override { return "dummysubdb"; }
@@ -64,6 +64,11 @@ struct DummyDocumentSubDb : public IDocumentSubDB
     proton::IAttributeManager::SP getAttributeManager() const override {
         return proton::IAttributeManager::SP();
     }
+
+    void validateDocStore(FeedHandler &, SerialNum ) const override {
+
+    }
+
     const IIndexManager::SP &getIndexManager() const override { return _indexManager; }
     const ISummaryAdapter::SP &getSummaryAdapter() const override { return _summaryAdapter; }
     const IIndexWriter::SP &getIndexWriter() const override { return _indexWriter; }
diff --git a/searchcore/src/vespa/searchcore/proton/test/mock_attribute_manager.h b/searchcore/src/vespa/searchcore/proton/test/mock_attribute_manager.h
index 87fa2053508..3e49bb449ff 100644
--- a/searchcore/src/vespa/searchcore/proton/test/mock_attribute_manager.h
+++ b/searchcore/src/vespa/searchcore/proton/test/mock_attribute_manager.h
@@ -55,7 +55,7 @@ public:
     const IAttributeFactory::SP &getFactory() const override {
         HDR_ABORT("should not be reached");
     }
-    search::ISequencedTaskExecutor &getAttributeFieldWriter() const override {
+    vespalib::ISequencedTaskExecutor &getAttributeFieldWriter() const override {
         HDR_ABORT("should not be reached");
     }
     search::AttributeVector *getWritableAttribute(const vespalib::string &) const override {
@@ -78,6 +78,9 @@ public:
     void asyncForAttribute(const vespalib::string & name, std::unique_ptr<IAttributeFunctor> func) const override {
         _mock.asyncForAttribute(name, std::move(func));
     }
+    std::shared_ptr<search::attribute::ReadableAttributeVector> readable_attribute_vector(const string& name) const override {
+        return _mock.readable_attribute_vector(name);
+    }
 };
 
 }
diff --git a/searchcore/src/vespa/searchcore/proton/test/thread_service_observer.h b/searchcore/src/vespa/searchcore/proton/test/thread_service_observer.h
index c8d0f8968c9..127b696c4ab 100644
--- a/searchcore/src/vespa/searchcore/proton/test/thread_service_observer.h
+++ b/searchcore/src/vespa/searchcore/proton/test/thread_service_observer.h
@@ -20,9 +20,6 @@ public:
 
     uint32_t getExecuteCnt() const { return _executeCnt; }
 
-    /**
-     * Implements IThreadService
-     */
     vespalib::Executor::Task::UP execute(vespalib::Executor::Task::UP task) override {
         ++_executeCnt;
         return _service.execute(std::move(task));
@@ -34,11 +31,23 @@ public:
         _service.sync();
         return *this;
     }
+    ThreadServiceObserver &shutdown() override {
+        _service.shutdown();
+        return *this;
+    }
     bool isCurrentThread() const override {
         return _service.isCurrentThread();
     }
     size_t getNumThreads() const override { return _service.getNumThreads(); }
 
+    Stats getStats() override {
+        return _service.getStats();
+    }
+
+    void setTaskLimit(uint32_t taskLimit) override {
+        _service.setTaskLimit(taskLimit);
+    }
+
 };
 
 }
diff --git a/searchcore/src/vespa/searchcore/proton/test/threading_service_observer.cpp b/searchcore/src/vespa/searchcore/proton/test/threading_service_observer.cpp
new file mode 100644
index 00000000000..f06a50cc61d
--- /dev/null
+++ b/searchcore/src/vespa/searchcore/proton/test/threading_service_observer.cpp
@@ -0,0 +1,21 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "threading_service_observer.h"
+
+namespace proton::test {
+
+ThreadingServiceObserver::ThreadingServiceObserver(searchcorespi::index::IThreadingService &service)
+    : _service(service),
+      _master(_service.master()),
+      _index(service.index()),
+      _summary(service.summary()),
+      _shared(service.shared()),
+      _indexFieldInverter(_service.indexFieldInverter()),
+      _indexFieldWriter(_service.indexFieldWriter()),
+      _attributeFieldWriter(_service.attributeFieldWriter())
+{
+}
+
+ThreadingServiceObserver::~ThreadingServiceObserver() = default;
+
+}
diff --git a/searchcore/src/vespa/searchcore/proton/test/threading_service_observer.h b/searchcore/src/vespa/searchcore/proton/test/threading_service_observer.h
index e1a4433ed4b..7f6e7b40635 100644
--- a/searchcore/src/vespa/searchcore/proton/test/threading_service_observer.h
+++ b/searchcore/src/vespa/searchcore/proton/test/threading_service_observer.h
@@ -4,7 +4,7 @@
 #include "executor_observer.h"
 #include "thread_service_observer.h"
 #include <vespa/searchcorespi/index/ithreadingservice.h>
-#include <vespa/searchlib/common/sequencedtaskexecutorobserver.h>
+#include <vespa/vespalib/util/sequencedtaskexecutorobserver.h>
 
 namespace proton:: test {
 
@@ -16,23 +16,13 @@ private:
     ThreadServiceObserver _index;
     ThreadServiceObserver _summary;
     vespalib::ThreadExecutor & _shared;
-    search::SequencedTaskExecutorObserver _indexFieldInverter;
-    search::SequencedTaskExecutorObserver _indexFieldWriter;
-    search::SequencedTaskExecutorObserver _attributeFieldWriter;
+    vespalib::SequencedTaskExecutorObserver _indexFieldInverter;
+    vespalib::SequencedTaskExecutorObserver _indexFieldWriter;
+    vespalib::SequencedTaskExecutorObserver _attributeFieldWriter;
 
 public:
-    ThreadingServiceObserver(searchcorespi::index::IThreadingService &service)
-        : _service(service),
-          _master(_service.master()),
-          _index(service.index()),
-          _summary(service.summary()),
-          _shared(service.shared()),
-          _indexFieldInverter(_service.indexFieldInverter()),
-          _indexFieldWriter(_service.indexFieldWriter()),
-          _attributeFieldWriter(_service.attributeFieldWriter())
-    {
-    }
-    ~ThreadingServiceObserver() override { }
+    ThreadingServiceObserver(searchcorespi::index::IThreadingService &service);
+    ~ThreadingServiceObserver() override;
     const ThreadServiceObserver &masterObserver() const {
         return _master;
     }
@@ -42,27 +32,21 @@ public:
     const ThreadServiceObserver &summaryObserver() const {
         return _summary;
     }
-    const search::SequencedTaskExecutorObserver &indexFieldInverterObserver() const {
+    const vespalib::SequencedTaskExecutorObserver &indexFieldInverterObserver() const {
         return _indexFieldInverter;
     }
-    const search::SequencedTaskExecutorObserver &indexFieldWriterObserver() const {
+    const vespalib::SequencedTaskExecutorObserver &indexFieldWriterObserver() const {
         return _indexFieldWriter;
     }
 
-    const search::SequencedTaskExecutorObserver &attributeFieldWriterObserver() const {
+    const vespalib::SequencedTaskExecutorObserver &attributeFieldWriterObserver() const {
         return _attributeFieldWriter;
     }
 
-    /**
-     * Implements vespalib::Syncable
-     */
     vespalib::Syncable &sync() override {
         return _service.sync();
     }
 
-    /**
-     * Implements IThreadingService
-     */
     searchcorespi::index::IThreadService &master() override {
         return _master;
     }
@@ -75,16 +59,17 @@ public:
     vespalib::ThreadExecutor &shared() override {
         return _shared;
     }
-    search::ISequencedTaskExecutor &indexFieldInverter() override {
+    vespalib::ISequencedTaskExecutor &indexFieldInverter() override {
         return _indexFieldInverter;
     }
-    search::ISequencedTaskExecutor &indexFieldWriter() override {
+    vespalib::ISequencedTaskExecutor &indexFieldWriter() override {
         return _indexFieldWriter;
     }
 
-    search::ISequencedTaskExecutor &attributeFieldWriter() override {
+    vespalib::ISequencedTaskExecutor &attributeFieldWriter() override {
         return _attributeFieldWriter;
     }
+
 };
 
 }
diff --git a/searchcorespi/src/vespa/searchcorespi/flush/iflushtarget.h b/searchcorespi/src/vespa/searchcorespi/flush/iflushtarget.h
index 31707643649..03d9ba8d55c 100644
--- a/searchcorespi/src/vespa/searchcorespi/flush/iflushtarget.h
+++ b/searchcorespi/src/vespa/searchcorespi/flush/iflushtarget.h
@@ -153,7 +153,7 @@ public:
     virtual Time getLastFlushTime() const = 0;
 
     /**
-     * Return if the traget itself is in bad need for a flush.
+     * Return if the target itself is in bad need for a flush.
      *
      * @return true if an urgent flush is needed
      */
diff --git a/searchcorespi/src/vespa/searchcorespi/index/ithreadingservice.h b/searchcorespi/src/vespa/searchcorespi/index/ithreadingservice.h
index eb80eff19ac..2ace9a8ac6b 100644
--- a/searchcorespi/src/vespa/searchcorespi/index/ithreadingservice.h
+++ b/searchcorespi/src/vespa/searchcorespi/index/ithreadingservice.h
@@ -4,7 +4,7 @@
 #include "i_thread_service.h"
 #include <vespa/vespalib/util/syncable.h>
 
-namespace search { class ISequencedTaskExecutor; }
+namespace vespalib { class ISequencedTaskExecutor; }
 namespace searchcorespi::index {
 
 /**
@@ -55,9 +55,9 @@ struct IThreadingService : public vespalib::Syncable
     virtual IThreadService &index() = 0;
     virtual IThreadService &summary() = 0;
     virtual vespalib::ThreadExecutor &shared() = 0;
-    virtual search::ISequencedTaskExecutor &indexFieldInverter() = 0;
-    virtual search::ISequencedTaskExecutor &indexFieldWriter() = 0;
-    virtual search::ISequencedTaskExecutor &attributeFieldWriter() = 0;
+    virtual vespalib::ISequencedTaskExecutor &indexFieldInverter() = 0;
+    virtual vespalib::ISequencedTaskExecutor &indexFieldWriter() = 0;
+    virtual vespalib::ISequencedTaskExecutor &attributeFieldWriter() = 0;
 };
 
 }
diff --git a/searchlib/CMakeLists.txt b/searchlib/CMakeLists.txt
index 2c8eff4f4ad..19cad2f3905 100644
--- a/searchlib/CMakeLists.txt
+++ b/searchlib/CMakeLists.txt
@@ -68,6 +68,7 @@ vespa_define_module(
     src/tests/aggregator
     src/tests/alignment
     src/tests/attribute
+    src/tests/attribute/attribute_header
     src/tests/attribute/attribute_operation
     src/tests/attribute/attributefilewriter
     src/tests/attribute/attributemanager
@@ -90,6 +91,7 @@ vespa_define_module(
     src/tests/attribute/postinglist
     src/tests/attribute/postinglistattribute
     src/tests/attribute/reference_attribute
+    src/tests/attribute/save_target
     src/tests/attribute/searchable
     src/tests/attribute/searchcontext
     src/tests/attribute/sourceselector
@@ -99,11 +101,9 @@ vespa_define_module(
     src/tests/bitvector
     src/tests/bytecomplens
     src/tests/common/bitvector
-    src/tests/common/foregroundtaskexecutor
     src/tests/common/location
     src/tests/common/matching_elements
     src/tests/common/resultset
-    src/tests/common/sequencedtaskexecutor
     src/tests/common/struct_field_mapper
     src/tests/common/summaryfeatures
     src/tests/diskindex/bitvector
@@ -134,6 +134,8 @@ vespa_define_module(
     src/tests/features/item_raw_score
     src/tests/features/max_reduce_prod_join_replacer
     src/tests/features/native_dot_product
+    src/tests/features/nns_closeness
+    src/tests/features/nns_distance
     src/tests/features/ranking_expression
     src/tests/features/raw_score
     src/tests/features/subqueries
@@ -210,6 +212,9 @@ vespa_define_module(
     src/tests/sortspec
     src/tests/stringenum
     src/tests/tensor/dense_tensor_store
+    src/tests/tensor/distance_functions
+    src/tests/tensor/hnsw_index
+    src/tests/tensor/hnsw_saver
     src/tests/transactionlog
     src/tests/transactionlogstress
     src/tests/true
diff --git a/searchlib/src/apps/docstore/benchmarkdatastore.cpp b/searchlib/src/apps/docstore/benchmarkdatastore.cpp
index 620a139d451..4f8a4ad7345 100644
--- a/searchlib/src/apps/docstore/benchmarkdatastore.cpp
+++ b/searchlib/src/apps/docstore/benchmarkdatastore.cpp
@@ -9,6 +9,7 @@
 #include <vespa/vespalib/data/databuffer.h>
 #include <vespa/fastos/app.h>
 #include <unistd.h>
+#include <random>
 
 #include <vespa/log/log.h>
 LOG_SETUP("documentstore.benchmark");
@@ -65,17 +66,13 @@ BenchmarkDataStoreApp::Main()
 void BenchmarkDataStoreApp::read(size_t numReads, size_t perChunk, const IDataStore * dataStore)
 {
     vespalib::DataBuffer buf;
-    struct random_data rstate;
-    char state[8];
-    memset(state, 0, sizeof(state));
-    memset(&rstate, 0, sizeof(rstate));
+    std::minstd_rand rng;
     const size_t docIdLimit(dataStore->getDocIdLimit());
     assert(docIdLimit > 0);
-    initstate_r(getpid(), state, sizeof(state), &rstate);
-    assert(srandom_r(getpid(), &rstate) == 0);
+    rng.seed(getpid());
     int32_t rnd(0);
     for ( size_t i(0); i < numReads; i++) {
-        random_r(&rstate, &rnd);
+        rnd = rng();
         uint32_t lid(rnd%docIdLimit);
         for (uint32_t j(lid); j < std::min(docIdLimit, lid+perChunk); j++) {
             dataStore->read(j, buf);
diff --git a/searchlib/src/apps/tests/memoryindexstress_test.cpp b/searchlib/src/apps/tests/memoryindexstress_test.cpp
index a7689cd6b9f..6bbd93bda84 100644
--- a/searchlib/src/apps/tests/memoryindexstress_test.cpp
+++ b/searchlib/src/apps/tests/memoryindexstress_test.cpp
@@ -9,14 +9,12 @@
 #include <vespa/document/repo/documenttyperepo.h>
 #include <vespa/document/repo/fixedtyperepo.h>
 #include <vespa/searchlib/common/scheduletaskcallback.h>
-#include <vespa/searchlib/common/sequencedtaskexecutor.h>
 #include <vespa/searchlib/fef/matchdata.h>
 #include <vespa/searchlib/fef/matchdatalayout.h>
 #include <vespa/searchlib/fef/termfieldmatchdata.h>
 #include <vespa/searchlib/index/i_field_length_inspector.h>
 #include <vespa/searchlib/memoryindex/memory_index.h>
 #include <vespa/searchlib/query/tree/simplequery.h>
-#include <vespa/searchlib/queryeval/booleanmatchiteratorwrapper.h>
 #include <vespa/searchlib/queryeval/fake_requestcontext.h>
 #include <vespa/searchlib/queryeval/fake_search.h>
 #include <vespa/searchlib/queryeval/fake_searchable.h>
@@ -24,8 +22,8 @@
 #include <vespa/searchlib/test/index/mock_field_length_inspector.h>
 #include <vespa/searchlib/util/rand48.h>
 #include <vespa/vespalib/testkit/testapp.h>
-#include <vespa/vespalib/util/stringfmt.h>
 #include <vespa/vespalib/util/threadstackexecutor.h>
+#include <vespa/vespalib/util/sequencedtaskexecutor.h>
 
 #include <vespa/log/log.h>
 LOG_SETUP("memoryindexstress_test");
@@ -197,8 +195,8 @@ struct Fixture {
     Schema       schema;
     DocumentTypeRepo  repo;
     vespalib::ThreadStackExecutor _executor;
-    search::SequencedTaskExecutor _invertThreads;
-    search::SequencedTaskExecutor _pushThreads;
+    std::unique_ptr<vespalib::ISequencedTaskExecutor> _invertThreads;
+    std::unique_ptr<vespalib::ISequencedTaskExecutor> _pushThreads;
     MemoryIndex  index;
     uint32_t _readThreads;
     vespalib::ThreadStackExecutor _writer; // 1 write thread
@@ -249,9 +247,9 @@ Fixture::Fixture(uint32_t readThreads)
     : schema(makeSchema()),
       repo(makeDocTypeRepoConfig()),
       _executor(1, 128 * 1024),
-      _invertThreads(2),
-      _pushThreads(2),
-      index(schema, MockFieldLengthInspector(), _invertThreads, _pushThreads),
+      _invertThreads(vespalib::SequencedTaskExecutor::create(2)),
+      _pushThreads(vespalib::SequencedTaskExecutor::create(2)),
+      index(schema, MockFieldLengthInspector(), *_invertThreads, *_pushThreads),
       _readThreads(readThreads),
       _writer(1, 128 * 1024),
       _readers(readThreads, 128 * 1024),
diff --git a/searchlib/src/apps/vespa-ranking-expression-analyzer/vespa-ranking-expression-analyzer.cpp b/searchlib/src/apps/vespa-ranking-expression-analyzer/vespa-ranking-expression-analyzer.cpp
index 30177dbe693..65b27dd411a 100644
--- a/searchlib/src/apps/vespa-ranking-expression-analyzer/vespa-ranking-expression-analyzer.cpp
+++ b/searchlib/src/apps/vespa-ranking-expression-analyzer/vespa-ranking-expression-analyzer.cpp
@@ -153,7 +153,7 @@ struct FunctionInfo {
     size_t get_path_len(const TreeList &trees) const {
         size_t path = 0;
         for (const Node *tree: trees) {
-            InterpretedFunction ifun(DefaultTensorEngine::ref(), *tree, params.size(), NodeTypes());
+            InterpretedFunction ifun(DefaultTensorEngine::ref(), *tree, NodeTypes());
             InterpretedFunction::Context ctx(ifun);
             SimpleParams fun_params(params);
             ifun.eval(ctx, fun_params);
diff --git a/searchlib/src/main/java/com/yahoo/searchlib/aggregation/hll/NormalSketch.java b/searchlib/src/main/java/com/yahoo/searchlib/aggregation/hll/NormalSketch.java
index 37d5dd345ad..d32349906ee 100644
--- a/searchlib/src/main/java/com/yahoo/searchlib/aggregation/hll/NormalSketch.java
+++ b/searchlib/src/main/java/com/yahoo/searchlib/aggregation/hll/NormalSketch.java
@@ -22,6 +22,7 @@ public class NormalSketch extends Sketch<NormalSketch>  {
 
     private final byte[] data;
     private final int bucketMask;
+    private static final LZ4Factory lz4Factory = LZ4Factory.safeInstance();
 
     /**
      * Create a sketch with the default precision given by {@link HyperLogLog#DEFAULT_PRECISION}.
@@ -100,7 +101,7 @@ public class NormalSketch extends Sketch<NormalSketch>  {
         super.onSerialize(buf);
         buf.putInt(null, data.length);
         try {
-            LZ4Compressor c = LZ4Factory.safeInstance().highCompressor();
+            LZ4Compressor c = lz4Factory.highCompressor();
             byte[] compressedData = new byte[data.length];
             int compressedSize = c.compress(data, compressedData);
             serializeDataArray(compressedData, compressedSize, buf);
@@ -129,7 +130,7 @@ public class NormalSketch extends Sketch<NormalSketch>  {
         if (length == compressedLength) {
             deserializeDataArray(data, length, buf);
         } else {
-            LZ4FastDecompressor c = LZ4Factory.safeInstance().fastDecompressor();
+            LZ4FastDecompressor c = lz4Factory.fastDecompressor();
             byte[] compressedData = buf.getBytes(null, compressedLength);
             c.decompress(compressedData, data);
         }
diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/RankingExpression.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/RankingExpression.java
index 18f6c6f2ca2..1f27bc8750e 100755
--- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/RankingExpression.java
+++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/RankingExpression.java
@@ -10,6 +10,7 @@ import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode;
 import com.yahoo.searchlib.rankingexpression.rule.SerializationContext;
 import com.yahoo.tensor.TensorType;
 import com.yahoo.tensor.evaluation.TypeContext;
+import com.yahoo.text.Text;
 
 import java.io.File;
 import java.io.FileNotFoundException;
@@ -115,7 +116,7 @@ public class RankingExpression implements Serializable {
             root = parse(new StringReader(expression));
         }
         catch (ParseException e) {
-            ParseException p = new ParseException("Could not parse '" + expression + "'");
+            ParseException p = new ParseException("Could not parse '" + Text.truncate(expression, 50) + "'");
             p.initCause(e);
             throw p;
         }
diff --git a/searchlib/src/main/javacc/RankingExpressionParser.jj b/searchlib/src/main/javacc/RankingExpressionParser.jj
index 6fa915a134d..8aa10bf7b34 100755
--- a/searchlib/src/main/javacc/RankingExpressionParser.jj
+++ b/searchlib/src/main/javacc/RankingExpressionParser.jj
@@ -598,21 +598,21 @@ TensorFunctionNode tensorXwPlusB() :
 TensorFunctionNode tensorArgmax() :
 {
     ExpressionNode tensor;
-    String dimension;
+    List<String> dimensions = null;
 }
 {
-    <ARGMAX> <LBRACE> tensor = expression() <COMMA> dimension = identifier() <RBRACE>
-    { return new TensorFunctionNode(new Argmax(TensorFunctionNode.wrap(tensor), dimension)); }
+    <ARGMAX> <LBRACE> tensor = expression() dimensions = tagCommaLeadingList() <RBRACE>
+    { return new TensorFunctionNode(new Argmax(TensorFunctionNode.wrap(tensor), dimensions)); }
 }
 
 TensorFunctionNode tensorArgmin() :
 {
     ExpressionNode tensor;
-    String dimension;
+    List<String> dimensions = null;
 }
 {
-    <ARGMIN> <LBRACE> tensor = expression() <COMMA> dimension = identifier() <RBRACE>
-    { return new TensorFunctionNode(new Argmin(TensorFunctionNode.wrap(tensor), dimension)); }
+    <ARGMIN> <LBRACE> tensor = expression() dimensions = tagCommaLeadingList() <RBRACE>
+    { return new TensorFunctionNode(new Argmin(TensorFunctionNode.wrap(tensor), dimensions)); }
 }
 
 LambdaFunctionNode lambdaFunction() :
diff --git a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/EvaluationTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/EvaluationTestCase.java
index 6a87e0c6d46..807eb3aa7ce 100644
--- a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/EvaluationTestCase.java
+++ b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/EvaluationTestCase.java
@@ -386,6 +386,56 @@ public class EvaluationTestCase {
         // tensor result dimensions are given from argument dimensions, not the resulting values
         tester.assertEvaluates("tensor(x{}):{}", "tensor0 * tensor1", "{ {x:0}:1 }", "tensor(x{}):{ {x:1}:1 }");
         tester.assertEvaluates("tensor(x{},y{}):{}", "tensor0 * tensor1", "{ {x:0}:1 }", "tensor(x{},y{}):{ {x:1,y:0}:1, {x:2,y:1}:1 }");
+
+    }
+
+    @Test
+    public void testTake() {
+        EvaluationTester tester = new EvaluationTester();
+
+        // numpy.take(a, indices, axis) with tensors.
+
+        // 1 dim input, 1 dim indices
+        tester.assertEvaluates("tensor(d0[3]):[1, 3, 5]",
+                "tensor(d0[3])(tensor0{a0:(tensor1{indices0:(d0)})})",
+                "tensor(a0[6]):[1, 2, 3, 4, 5, 6]",
+                "tensor(indices0[3]):[0, 2, 4]");
+
+        // 1 dim input, 1 dim indices - negative indices
+        tester.assertEvaluates("tensor(d0[3]):[1, 5, 3]",
+                "tensor(d0[3])(tensor0{a0:(fmod(6 + tensor1{indices0:(d0)}, 6) ) })",
+                "tensor(a0[6]):[1, 2, 3, 4, 5, 6]",
+                "tensor(indices0[3]):[0, -2, -4]");
+
+        // 2 dim input, 1 dim indices - axis 0
+        tester.assertEvaluates("tensor(d0[4],d1[2]):[5, 6, 3, 4, 1, 2, 5, 6]",
+                "tensor(d0[4],d1[2])(tensor0{a0:(tensor1{indices0:(d0)}),a1:(d1)})",
+                "tensor(a0[3],a1[2]):[1, 2, 3, 4, 5, 6]",
+                "tensor(indices0[4]):[2, 1, 0, 2]");
+
+        // 1 dim input, 2 dim indices - axis 0
+        tester.assertEvaluates("tensor(d0[2],d1[2]):[1, 2, 4, 6]",
+                "tensor(d0[2],d1[2])(tensor0{a0:(tensor1{indices0:(d0),indices1:(d1)}) })",
+                "tensor(a0[6]):[1, 2, 3, 4, 5, 6]",
+                "tensor(indices0[2],indices1[2]):[0, 1, 3, 5]");
+
+        // 2 dim input, 2 dim indices - axis 0
+        tester.assertEvaluates("tensor(d0[2],d1[2],d2[2]):[1,2,3,4,3,4,5,6]",
+                "tensor(d0[2],d1[2],d2[2])(tensor0{a0:(tensor1{indices0:(d0),indices1:(d1)}),a1:(d2)})",
+                "tensor(a0[3],a1[2]):[1, 2, 3, 4, 5, 6]",
+                "tensor(indices0[2],indices1[2]):[0, 1, 1, 2]");
+
+        // 2 dim input, 1 dim indices - axis 1
+        tester.assertEvaluates("tensor(d0[3],d1[4]):[1,2,1,2,3,4,3,4,5,6,5,6]",
+                "tensor(d0[3],d1[4])(tensor0{a0:(d0), a1:(tensor1{indices0:(d1)}) })",
+                "tensor(a0[3],a1[2]):[1, 2, 3, 4, 5, 6]",
+                "tensor(indices0[4]):[0, 1, 0, 1]");
+
+        // 2 dim input, 2 dim indices - axis 1
+        tester.assertEvaluates("tensor(d0[3],d1[1],d2[2]):[1,3,4,6,7,9]",
+                "tensor(d0[3],d1[1],d2[2])(tensor0{a0:(d0), a1:(tensor1{indices0:(d1),indices1:(d2)}) })",  // can add an if
+                "tensor(a0[3],a1[3]):[1, 2, 3, 4, 5, 6, 7, 8, 9]",
+                "tensor(indices0[1],indices1[2]):[0, 2]");
     }
 
     @Test
diff --git a/searchlib/src/tests/aggregator/perdocexpr.cpp b/searchlib/src/tests/aggregator/perdocexpr.cpp
index 610fc58e98f..41465f991e3 100644
--- a/searchlib/src/tests/aggregator/perdocexpr.cpp
+++ b/searchlib/src/tests/aggregator/perdocexpr.cpp
@@ -1209,7 +1209,7 @@ TEST("testArithmeticOperations") {
     testAdd(createScalarInt(I1), createScalarInt(I2), 3469774562ull, 3469774562ull);
     testAdd(createScalarInt(I1), createScalarFloat(F2), 1793253251ull, 1793253250.767681239);
     testAdd(createScalarFloat(F1), createScalarFloat(F2), 11, 10.878668839 );
-    testMultiply(createScalarInt(I1), createScalarInt(I2), 3006427292488851361ull, 3006427292488851361ull);
+    testMultiply(createScalarInt(I1), createScalarInt(I2), 3006427292488851361ull, static_cast<double>(3006427292488851361ull));
     testMultiply(createScalarInt(I1), createScalarFloat(F2), 17515926039ull, 1793253241.0*9.767681239);
     testMultiply(createScalarFloat(F1), createScalarFloat(F2), 11, 10.8517727372816364 );
 
diff --git a/searchlib/src/tests/attribute/attribute_header/CMakeLists.txt b/searchlib/src/tests/attribute/attribute_header/CMakeLists.txt
new file mode 100644
index 00000000000..e72c0c6a528
--- /dev/null
+++ b/searchlib/src/tests/attribute/attribute_header/CMakeLists.txt
@@ -0,0 +1,9 @@
+# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchlib_attribute_header_test_app TEST
+    SOURCES
+    attribute_header_test.cpp
+    DEPENDS
+    searchlib
+    gtest
+)
+vespa_add_test(NAME searchlib_attribute_header_test_app COMMAND searchlib_attribute_header_test_app)
diff --git a/searchlib/src/tests/attribute/attribute_header/attribute_header_test.cpp b/searchlib/src/tests/attribute/attribute_header/attribute_header_test.cpp
new file mode 100644
index 00000000000..0f542d016a9
--- /dev/null
+++ b/searchlib/src/tests/attribute/attribute_header/attribute_header_test.cpp
@@ -0,0 +1,77 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/eval/eval/value_type.h>
+#include <vespa/searchcommon/attribute/config.h>
+#include <vespa/searchlib/attribute/attribute_header.h>
+#include <vespa/vespalib/data/fileheader.h>
+#include <vespa/vespalib/gtest/gtest.h>
+
+#include <vespa/log/log.h>
+LOG_SETUP("attribute_header_test");
+
+using namespace search;
+using namespace search::attribute;
+
+using HnswIPO = std::optional<HnswIndexParams>;
+using vespalib::eval::ValueType;
+
+const Config tensor_cfg(BasicType::TENSOR, CollectionType::SINGLE);
+const vespalib::string file_name = "my_file_name";
+const ValueType tensor_type = ValueType::from_spec("tensor<float>(x[4])");
+constexpr uint32_t num_docs = 23;
+constexpr uint64_t unique_value_count = 11;
+constexpr uint64_t total_value_count = 13;
+constexpr uint64_t create_serial_num = 17;
+constexpr uint32_t version = 19;
+
+vespalib::GenericHeader
+populate_header(const HnswIPO& hnsw_params)
+{
+    AttributeHeader header(file_name,
+                           tensor_cfg.basicType(),
+                           tensor_cfg.collectionType(),
+                           tensor_type,
+                           false,
+                           PersistentPredicateParams(),
+                           hnsw_params,
+                           num_docs,
+                           unique_value_count,
+                           total_value_count,
+                           create_serial_num,
+                           version);
+
+    vespalib::GenericHeader result;
+    header.addTags(result);
+    return result;
+}
+
+void
+verify_roundtrip_serialization(const HnswIPO& hnsw_params_in)
+{
+    auto gen_header = populate_header(hnsw_params_in);
+    auto attr_header = AttributeHeader::extractTags(gen_header);
+
+    EXPECT_EQ(tensor_cfg.basicType(), attr_header.getBasicType());
+    EXPECT_EQ(tensor_cfg.collectionType(), attr_header.getCollectionType());
+    EXPECT_EQ(tensor_type, attr_header.getTensorType());
+    EXPECT_EQ(num_docs, attr_header.getNumDocs());
+    EXPECT_EQ(create_serial_num, attr_header.getCreateSerialNum());
+    EXPECT_EQ(version, attr_header.getVersion());
+    EXPECT_EQ(false, attr_header.getPredicateParamsSet());
+    const auto& hnsw_params_out = attr_header.get_hnsw_index_params();
+    EXPECT_EQ(hnsw_params_in.has_value(), hnsw_params_out.has_value());
+    if (hnsw_params_in.has_value()) {
+        EXPECT_EQ(hnsw_params_in.value(), hnsw_params_out.value());
+    }
+}
+
+TEST(AttributeHeaderTest, can_be_added_to_and_extracted_from_generic_header)
+{
+    verify_roundtrip_serialization(HnswIPO({16, 100, DistanceMetric::Euclidean}));
+    verify_roundtrip_serialization(HnswIPO({16, 100, DistanceMetric::Angular}));
+    verify_roundtrip_serialization(HnswIPO({16, 100, DistanceMetric::GeoDegrees}));
+    verify_roundtrip_serialization(HnswIPO());
+}
+
+GTEST_MAIN_RUN_ALL_TESTS()
+
diff --git a/searchlib/src/tests/attribute/attribute_test.cpp b/searchlib/src/tests/attribute/attribute_test.cpp
index f402f536ae2..4715292f86a 100644
--- a/searchlib/src/tests/attribute/attribute_test.cpp
+++ b/searchlib/src/tests/attribute/attribute_test.cpp
@@ -3,6 +3,7 @@
 #include <vespa/document/fieldvalue/intfieldvalue.h>
 #include <vespa/document/fieldvalue/stringfieldvalue.h>
 #include <vespa/document/update/arithmeticvalueupdate.h>
+#include <vespa/document/update/assignvalueupdate.h>
 #include <vespa/document/update/mapvalueupdate.h>
 #include <vespa/fastlib/io/bufferedfile.h>
 #include <vespa/searchlib/attribute/attribute.h>
@@ -1493,20 +1494,21 @@ AttributeTest::testMapValueUpdate(const AttributePtr & ptr, BufferType initValue
     typedef ArithmeticValueUpdate ArithVU;
     auto & vec = static_cast<VectorType &>(*ptr.get());
 
-    addDocs(ptr, 6);
-    for (uint32_t doc = 0; doc < 6; ++doc) {
+    addDocs(ptr, 7);
+    for (uint32_t doc = 0; doc < 7; ++doc) {
         ASSERT_TRUE(vec.append(doc, initValue.getValue(), 100));
     }
-    EXPECT_EQUAL(ptr->getStatus().getUpdateCount(), 6u);
+    EXPECT_EQUAL(ptr->getStatus().getUpdateCount(), 7u);
     EXPECT_EQUAL(ptr->getStatus().getNonIdempotentUpdateCount(), 0u);
 
     EXPECT_TRUE(ptr->apply(0, MapVU(initFieldValue, ArithVU(ArithVU::Add, 10))));
     EXPECT_TRUE(ptr->apply(1, MapVU(initFieldValue, ArithVU(ArithVU::Sub, 10))));
     EXPECT_TRUE(ptr->apply(2, MapVU(initFieldValue, ArithVU(ArithVU::Mul, 10))));
     EXPECT_TRUE(ptr->apply(3, MapVU(initFieldValue, ArithVU(ArithVU::Div, 10))));
+    EXPECT_TRUE(ptr->apply(6, MapVU(initFieldValue, AssignValueUpdate(IntFieldValue(70)))));
     ptr->commit();
-    EXPECT_EQUAL(ptr->getStatus().getUpdateCount(), 10u);
-    EXPECT_EQUAL(ptr->getStatus().getNonIdempotentUpdateCount(), 4u);
+    EXPECT_EQUAL(ptr->getStatus().getUpdateCount(), 12u);
+    EXPECT_EQUAL(ptr->getStatus().getNonIdempotentUpdateCount(), 5u);
 
     std::vector<BufferType> buf(2);
     ptr->get(0, &buf[0], 2);
@@ -1517,6 +1519,8 @@ AttributeTest::testMapValueUpdate(const AttributePtr & ptr, BufferType initValue
     EXPECT_EQUAL(buf[0].getWeight(), 1000);
     ptr->get(3, &buf[0], 2);
     EXPECT_EQUAL(buf[0].getWeight(), 10);
+    ptr->get(6, &buf[0], 2);
+    EXPECT_EQUAL(buf[0].getWeight(), 70);
 
     // removeifzero
     EXPECT_TRUE(ptr->apply(4, MapVU(initFieldValue, ArithVU(ArithVU::Sub, 100))));
@@ -1527,8 +1531,8 @@ AttributeTest::testMapValueUpdate(const AttributePtr & ptr, BufferType initValue
         EXPECT_EQUAL(ptr->get(4, &buf[0], 2), uint32_t(1));
         EXPECT_EQUAL(buf[0].getWeight(), 0);
     }
-    EXPECT_EQUAL(ptr->getStatus().getUpdateCount(), 11u);
-    EXPECT_EQUAL(ptr->getStatus().getNonIdempotentUpdateCount(), 5u);
+    EXPECT_EQUAL(ptr->getStatus().getUpdateCount(), 13u);
+    EXPECT_EQUAL(ptr->getStatus().getNonIdempotentUpdateCount(), 6u);
 
     // createifnonexistant
     EXPECT_TRUE(ptr->apply(5, MapVU(nonExistant, ArithVU(ArithVU::Add, 10))));
@@ -1542,18 +1546,18 @@ AttributeTest::testMapValueUpdate(const AttributePtr & ptr, BufferType initValue
         EXPECT_EQUAL(ptr->get(5, &buf[0], 2), uint32_t(1));
         EXPECT_EQUAL(buf[0].getWeight(), 100);
     }
-    EXPECT_EQUAL(ptr->getStatus().getUpdateCount(), 12u);
-    EXPECT_EQUAL(ptr->getStatus().getNonIdempotentUpdateCount(), 6u);
+    EXPECT_EQUAL(ptr->getStatus().getUpdateCount(), 14u);
+    EXPECT_EQUAL(ptr->getStatus().getNonIdempotentUpdateCount(), 7u);
 
 
     // try divide by zero (should be ignored)
     vec.clearDoc(0);
-    EXPECT_EQUAL(ptr->getStatus().getUpdateCount(), 13u);
+    EXPECT_EQUAL(ptr->getStatus().getUpdateCount(), 15u);
     ASSERT_TRUE(vec.append(0, initValue.getValue(), 12345));
-    EXPECT_EQUAL(ptr->getStatus().getUpdateCount(), 14u);
+    EXPECT_EQUAL(ptr->getStatus().getUpdateCount(), 16u);
     EXPECT_TRUE(ptr->apply(0, MapVU(initFieldValue, ArithVU(ArithVU::Div, 0))));
-    EXPECT_EQUAL(ptr->getStatus().getUpdateCount(), 14u);
-    EXPECT_EQUAL(ptr->getStatus().getNonIdempotentUpdateCount(), 6u);
+    EXPECT_EQUAL(ptr->getStatus().getUpdateCount(), 16u);
+    EXPECT_EQUAL(ptr->getStatus().getNonIdempotentUpdateCount(), 7u);
     ptr->commit();
     ptr->get(0, &buf[0], 1);
     EXPECT_EQUAL(buf[0].getWeight(), 12345);
diff --git a/searchlib/src/tests/attribute/attributemanager/attributemanager_test.cpp b/searchlib/src/tests/attribute/attributemanager/attributemanager_test.cpp
index 62cde4d2c9c..1cb314165cd 100644
--- a/searchlib/src/tests/attribute/attributemanager/attributemanager_test.cpp
+++ b/searchlib/src/tests/attribute/attributemanager/attributemanager_test.cpp
@@ -34,6 +34,7 @@ private:
     void testGuards();
     void testConfigConvert();
     void testContext();
+    void can_get_readable_attribute_vector_by_name();
 
     bool
     assertDataType(BT::Type exp,
@@ -277,6 +278,26 @@ AttributeManagerTest::testConfigConvert()
         AttributeVector::Config out = ConfigConverter::convert(a);
         EXPECT_EQUAL("tensor(x[5])", out.tensorType().to_spec());
     }
+    { // hnsw index params (enabled)
+        auto dm_in = AttributesConfig::Attribute::Index::Hnsw::Distancemetric::ANGULAR;
+        auto dm_out = search::attribute::DistanceMetric::Angular;
+        CACA a;
+        a.index.hnsw.enabled = true;
+        a.index.hnsw.maxlinkspernode = 32;
+        a.index.hnsw.neighborstoexploreatinsert = 300;
+        a.index.hnsw.distancemetric = dm_in;
+        auto out = ConfigConverter::convert(a);
+        EXPECT_TRUE(out.hnsw_index_params().has_value());
+        EXPECT_EQUAL(32u, out.hnsw_index_params().value().max_links_per_node());
+        EXPECT_EQUAL(300u, out.hnsw_index_params().value().neighbors_to_explore_at_insert());
+        EXPECT_TRUE(out.hnsw_index_params().value().distance_metric() == dm_out);
+    }
+    { // hnsw index params (disabled)
+        CACA a;
+        a.index.hnsw.enabled = false;
+        auto out = ConfigConverter::convert(a);
+        EXPECT_FALSE(out.hnsw_index_params().has_value());
+    }
 }
 
 bool gt_attribute(const attribute::IAttributeVector * a, const attribute::IAttributeVector * b) {
@@ -377,6 +398,21 @@ AttributeManagerTest::testContext()
     }
 }
 
+void
+AttributeManagerTest::can_get_readable_attribute_vector_by_name()
+{
+    auto attr = AttributeFactory::createAttribute("cool_attr", Config(BT::INT32, CT::SINGLE));
+    // Ensure there's something to actually load, or fetching the attribute will throw.
+    attr->addDocs(64);
+    attr->commit();
+    AttributeManager manager;
+    manager.add(attr);
+    auto av = manager.readable_attribute_vector("cool_attr");
+    EXPECT_EQUAL(av.get(), static_cast<ReadableAttributeVector*>(attr.get()));
+    av = manager.readable_attribute_vector("uncool_attr");
+    EXPECT_TRUE(av.get() == nullptr);
+}
+
 int AttributeManagerTest::Main()
 {
     TEST_INIT("attributemanager_test");
@@ -385,6 +421,7 @@ int AttributeManagerTest::Main()
     testGuards();
     testConfigConvert();
     testContext();
+    can_get_readable_attribute_vector_by_name();
 
     TEST_DONE();
 }
diff --git a/searchlib/src/tests/attribute/enum_attribute_compaction/enum_attribute_compaction_test.cpp b/searchlib/src/tests/attribute/enum_attribute_compaction/enum_attribute_compaction_test.cpp
index 31af5945337..45d432c29be 100644
--- a/searchlib/src/tests/attribute/enum_attribute_compaction/enum_attribute_compaction_test.cpp
+++ b/searchlib/src/tests/attribute/enum_attribute_compaction/enum_attribute_compaction_test.cpp
@@ -221,7 +221,7 @@ TEST_P(IntegerCompactionTest, compact)
     test_enum_store_compaction();
 }
 
-INSTANTIATE_TEST_CASE_P(IntegerCompactionTestSet, IntegerCompactionTest, ::testing::Values(CollectionType::SINGLE, CollectionType::ARRAY, CollectionType::WSET));
+VESPA_GTEST_INSTANTIATE_TEST_SUITE_P(IntegerCompactionTestSet, IntegerCompactionTest, ::testing::Values(CollectionType::SINGLE, CollectionType::ARRAY, CollectionType::WSET));
 
 using StringCompactionTest = CompactionTest<StringAttribute>;
 
@@ -230,6 +230,6 @@ TEST_P(StringCompactionTest, compact)
     test_enum_store_compaction();
 }
 
-INSTANTIATE_TEST_CASE_P(StringCompactionTestSet, StringCompactionTest, ::testing::Values(CollectionType::SINGLE, CollectionType::ARRAY, CollectionType::WSET));
+VESPA_GTEST_INSTANTIATE_TEST_SUITE_P(StringCompactionTestSet, StringCompactionTest, ::testing::Values(CollectionType::SINGLE, CollectionType::ARRAY, CollectionType::WSET));
 
 GTEST_MAIN_RUN_ALL_TESTS()
diff --git a/searchlib/src/tests/attribute/enumeratedsave/enumeratedsave_test.cpp b/searchlib/src/tests/attribute/enumeratedsave/enumeratedsave_test.cpp
index bf829f6607a..41313fc7c53 100644
--- a/searchlib/src/tests/attribute/enumeratedsave/enumeratedsave_test.cpp
+++ b/searchlib/src/tests/attribute/enumeratedsave/enumeratedsave_test.cpp
@@ -108,6 +108,17 @@ public:
     }
     IAttributeFileWriter &udatWriter() override { return _udatWriter; }
 
+    bool setup_writer(const vespalib::string& file_suffix,
+                      const vespalib::string& desc) override {
+        (void) file_suffix;
+        (void) desc;
+        abort();
+    }
+    IAttributeFileWriter& get_writer(const vespalib::string& file_suffix) override {
+        (void) file_suffix;
+        abort();
+    }
+
     bool bufEqual(const Buffer &lhs, const Buffer &rhs) const;
  
     bool operator==(const MemAttr &rhs) const;
diff --git a/searchlib/src/tests/attribute/enumstore/enumstore_test.cpp b/searchlib/src/tests/attribute/enumstore/enumstore_test.cpp
index 3a885dda233..43e694f0bcd 100644
--- a/searchlib/src/tests/attribute/enumstore/enumstore_test.cpp
+++ b/searchlib/src/tests/attribute/enumstore/enumstore_test.cpp
@@ -72,7 +72,7 @@ public:
 #endif
 
 using FloatEnumStoreTestTypes = ::testing::Types<FloatEnumStore, DoubleEnumStore>;
-TYPED_TEST_CASE(FloatEnumStoreTest, FloatEnumStoreTestTypes);
+VESPA_GTEST_TYPED_TEST_SUITE(FloatEnumStoreTest, FloatEnumStoreTestTypes);
 
 TYPED_TEST(FloatEnumStoreTest, numbers_can_be_inserted_and_retrieved)
 {
@@ -452,7 +452,7 @@ LoaderTest<StringEnumStore>::load_values(enumstore::EnumeratedLoaderBase& loader
 #endif
 
 using LoaderTestTypes = ::testing::Types<NumericEnumStore, FloatEnumStore, StringEnumStore>;
-TYPED_TEST_CASE(LoaderTest, LoaderTestTypes);
+VESPA_GTEST_TYPED_TEST_SUITE(LoaderTest, LoaderTestTypes);
 
 TYPED_TEST(LoaderTest, store_is_instantiated_with_enumerated_loader)
 {
diff --git a/searchlib/src/tests/attribute/imported_attribute_vector/imported_attribute_vector_test.cpp b/searchlib/src/tests/attribute/imported_attribute_vector/imported_attribute_vector_test.cpp
index cfa05b7a765..48a06bec9e8 100644
--- a/searchlib/src/tests/attribute/imported_attribute_vector/imported_attribute_vector_test.cpp
+++ b/searchlib/src/tests/attribute/imported_attribute_vector/imported_attribute_vector_test.cpp
@@ -219,6 +219,15 @@ TEST_F("Weighted floating point attribute values can be retrieved via reference"
     assert_multi_value_matches<WeightedFloat>(f, DocId(3), doc7_values);
 }
 
+TEST_F("isUndefined() works for primitive attribute type", Fixture) {
+    reset_with_single_value_reference_mappings<IntegerAttribute, int32_t>(
+            f, BasicType::INT32,
+            {{DocId(3), dummy_gid(7), DocId(7), 5678}});
+
+    EXPECT_FALSE(f.get_imported_attr()->isUndefined(DocId(3))); // Mapped
+    EXPECT_TRUE(f.get_imported_attr()->isUndefined(DocId(2))); // Not mapped
+}
+
 struct SingleStringAttrFixture : Fixture {
     SingleStringAttrFixture() : Fixture() {
         setup();
@@ -255,6 +264,11 @@ TEST_F("findEnum() returns target vector enum via reference", SingleStringAttrFi
     EXPECT_EQUAL(expected_handle, actual_handle);
 }
 
+TEST_F("isUndefined() works for enumerated attribute type", SingleStringAttrFixture) {
+    EXPECT_FALSE(f.get_imported_attr()->isUndefined(DocId(2))); // Mapped
+    EXPECT_TRUE(f.get_imported_attr()->isUndefined(DocId(3))); // Not mapped
+}
+
 // Note: assumes that fixture has set up a string enum of value "foo" in target attribute
 template <typename FixtureType>
 void verify_get_string_from_enum_is_mapped(FixtureType& f) {
diff --git a/searchlib/src/tests/attribute/save_target/CMakeLists.txt b/searchlib/src/tests/attribute/save_target/CMakeLists.txt
new file mode 100644
index 00000000000..e127f66579e
--- /dev/null
+++ b/searchlib/src/tests/attribute/save_target/CMakeLists.txt
@@ -0,0 +1,9 @@
+# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchlib_attribute_save_target_test_app TEST
+    SOURCES
+    attribute_save_target_test.cpp
+    DEPENDS
+    searchlib
+    gtest
+)
+vespa_add_test(NAME searchlib_attribute_save_target_test_app COMMAND searchlib_attribute_save_target_test_app)
diff --git a/searchlib/src/tests/attribute/save_target/attribute_save_target_test.cpp b/searchlib/src/tests/attribute/save_target/attribute_save_target_test.cpp
new file mode 100644
index 00000000000..c746a0aa120
--- /dev/null
+++ b/searchlib/src/tests/attribute/save_target/attribute_save_target_test.cpp
@@ -0,0 +1,148 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/searchlib/attribute/attributefilesavetarget.h>
+#include <vespa/searchlib/attribute/attributememorysavetarget.h>
+#include <vespa/searchlib/common/tunefileinfo.h>
+#include <vespa/searchlib/index/dummyfileheadercontext.h>
+#include <vespa/searchlib/test/directory_handler.h>
+#include <vespa/searchlib/util/fileutil.h>
+#include <vespa/vespalib/gtest/gtest.h>
+#include <vespa/vespalib/util/bufferwriter.h>
+#include <vespa/vespalib/util/exceptions.h>
+
+#include <vespa/log/log.h>
+LOG_SETUP("attribute_save_target_test");
+
+using namespace search;
+using namespace search::attribute;
+
+using search::index::DummyFileHeaderContext;
+using search::test::DirectoryHandler;
+
+const vespalib::string test_dir = "test_data/";
+
+class SaveTargetTest : public ::testing::Test {
+public:
+    DirectoryHandler dir_handler;
+    TuneFileAttributes tune_file;
+    DummyFileHeaderContext file_header_ctx;
+    IAttributeSaveTarget& target;
+    vespalib::string base_file_name;
+
+    SaveTargetTest(IAttributeSaveTarget& target_in)
+        : dir_handler(test_dir),
+          tune_file(),
+          file_header_ctx(),
+          target(target_in),
+          base_file_name(test_dir + "test_file")
+    {
+    }
+    ~SaveTargetTest() {}
+    void set_header(const vespalib::string& file_name) {
+        target.setHeader(AttributeHeader(file_name));
+    }
+    IAttributeFileWriter& setup_writer(const vespalib::string& file_suffix,
+                                       const vespalib::string& desc) {
+        bool res = target.setup_writer(file_suffix, desc);
+        assert(res);
+        return target.get_writer(file_suffix);
+    }
+    void setup_writer_and_fill(const vespalib::string& file_suffix,
+                               const vespalib::string& desc,
+                               int value) {
+        auto& writer = setup_writer(file_suffix, desc);
+        auto buf = writer.allocBufferWriter();
+        buf->write(&value, sizeof(int));
+        buf->flush();
+    }
+    void validate_loaded_file(const vespalib::string& file_suffix,
+                              const vespalib::string& exp_desc,
+                              int exp_value)
+    {
+        vespalib::string file_name = base_file_name + "." + file_suffix;
+        EXPECT_TRUE(vespalib::fileExists(file_name));
+        auto loaded = FileUtil::loadFile(file_name);
+        EXPECT_FALSE(loaded->empty());
+
+        const auto& header = loaded->getHeader();
+        EXPECT_EQ(file_name, header.getTag("fileName").asString());
+        EXPECT_EQ(exp_desc, header.getTag("desc").asString());
+
+        EXPECT_EQ(sizeof(int), loaded->size());
+        int act_value = (reinterpret_cast<const int*>(loaded->buffer()))[0];
+        EXPECT_EQ(exp_value, act_value);
+    }
+};
+
+class FileSaveTargetTest : public SaveTargetTest {
+public:
+    AttributeFileSaveTarget file_target;
+
+    FileSaveTargetTest()
+        : SaveTargetTest(file_target),
+          file_target(tune_file, file_header_ctx)
+    {
+        set_header(base_file_name);
+    }
+};
+
+TEST_F(FileSaveTargetTest, can_setup_and_return_writers)
+{
+    setup_writer_and_fill("my1", "desc 1", 123);
+    setup_writer_and_fill("my2", "desc 2", 456);
+    target.close();
+
+    validate_loaded_file("my1", "desc 1", 123);
+    validate_loaded_file("my2", "desc 2", 456);
+}
+
+TEST_F(FileSaveTargetTest, setup_fails_if_writer_already_exists)
+{
+    setup_writer("my", "my desc");
+    EXPECT_FALSE(target.setup_writer("my", "my desc"));
+}
+
+TEST_F(FileSaveTargetTest, get_throws_if_writer_does_not_exists)
+{
+    EXPECT_THROW(target.get_writer("na"), vespalib::IllegalArgumentException);
+}
+
+class MemorySaveTargetTest : public SaveTargetTest {
+public:
+    AttributeMemorySaveTarget memory_target;
+
+    MemorySaveTargetTest()
+            : SaveTargetTest(memory_target),
+              memory_target()
+    {
+        set_header(base_file_name);
+    }
+    void write_to_file() {
+        bool res = memory_target.writeToFile(tune_file, file_header_ctx);
+        ASSERT_TRUE(res);
+    }
+};
+
+TEST_F(MemorySaveTargetTest, can_setup_and_return_writers)
+{
+    setup_writer_and_fill("my1", "desc 1", 123);
+    setup_writer_and_fill("my2", "desc 2", 456);
+    write_to_file();
+
+    validate_loaded_file("my1", "desc 1", 123);
+    validate_loaded_file("my2", "desc 2", 456);
+}
+
+TEST_F(MemorySaveTargetTest, setup_fails_if_writer_already_exists)
+{
+    setup_writer("my", "my desc");
+    EXPECT_FALSE(target.setup_writer("my", "my desc"));
+}
+
+TEST_F(MemorySaveTargetTest, get_throws_if_writer_does_not_exists)
+{
+    EXPECT_THROW(target.get_writer("na"), vespalib::IllegalArgumentException);
+}
+
+GTEST_MAIN_RUN_ALL_TESTS()
+
diff --git a/searchlib/src/tests/attribute/searchable/attribute_searchable_adapter_test.cpp b/searchlib/src/tests/attribute/searchable/attribute_searchable_adapter_test.cpp
index 4818287b429..2eafeab20bd 100644
--- a/searchlib/src/tests/attribute/searchable/attribute_searchable_adapter_test.cpp
+++ b/searchlib/src/tests/attribute/searchable/attribute_searchable_adapter_test.cpp
@@ -123,6 +123,15 @@ public:
         return IAttributeContext::UP();
     }
 
+    std::shared_ptr<attribute::ReadableAttributeVector> readable_attribute_vector(const string& name) const override {
+        if (name == field) {
+            return _attribute_vector;
+        } else if (name == other) {
+            return _other;
+        }
+        return {};
+    }
+
     void asyncForAttribute(const vespalib::string &name, std::unique_ptr<IAttributeFunctor> func) const override;
 };
 
diff --git a/searchlib/src/tests/attribute/searchable/attributeblueprint_test.cpp b/searchlib/src/tests/attribute/searchable/attributeblueprint_test.cpp
index e9addae07b9..665803b3057 100644
--- a/searchlib/src/tests/attribute/searchable/attributeblueprint_test.cpp
+++ b/searchlib/src/tests/attribute/searchable/attributeblueprint_test.cpp
@@ -100,6 +100,10 @@ public:
     void asyncForAttribute(const vespalib::string &, std::unique_ptr<IAttributeFunctor>) const override {
         assert(!"Not implemented");
     }
+
+    std::shared_ptr<attribute::ReadableAttributeVector> readable_attribute_vector(const string&) const override {
+        return _attribute_vector;
+    }
 };
 
 constexpr uint32_t DOCID_LIMIT = 3;
@@ -291,13 +295,15 @@ public:
         request_ctx.set_query_tensor("query_tensor", tensor_spec);
     }
     Blueprint::UP create_blueprint() {
-        query::NearestNeighborTerm term("query_tensor", attr_name, 0, Weight(0), 7);
+        query::NearestNeighborTerm term("query_tensor", attr_name, 0, Weight(0), 7, true, 33);
         return source.createBlueprint(request_ctx, FieldSpec(attr_name, 0, 0), term);
     }
 };
 
 void
-expect_nearest_neighbor_blueprint(const vespalib::string& attribute_tensor_type_spec, const TensorSpec& query_tensor)
+expect_nearest_neighbor_blueprint(const vespalib::string& attribute_tensor_type_spec,
+                                  const TensorSpec& query_tensor,
+                                  const TensorSpec& converted_query_tensor)
 {
     NearestNeighborFixture f(make_tensor_attribute(field, attribute_tensor_type_spec));
     f.set_query_tensor(query_tensor);
@@ -305,7 +311,7 @@ expect_nearest_neighbor_blueprint(const vespalib::string& attribute_tensor_type_
     auto result = f.create_blueprint();
     const auto& nearest = as_type<NearestNeighborBlueprint>(*result);
     EXPECT_EQ(attribute_tensor_type_spec, nearest.get_attribute_tensor().getTensorType().to_spec());
-    EXPECT_EQ(query_tensor, DefaultTensorEngine::ref().to_spec(nearest.get_query_tensor()));
+    EXPECT_EQ(converted_query_tensor, DefaultTensorEngine::ref().to_spec(nearest.get_query_tensor()));
     EXPECT_EQ(7u, nearest.get_target_num_hits());
 }
 
@@ -314,10 +320,12 @@ TEST(AttributeBlueprintTest, nearest_neighbor_blueprint_is_created_by_attribute_
     TensorSpec x_2_double = TensorSpec("tensor(x[2])").add({{"x", 0}}, 3).add({{"x", 1}}, 5);
     TensorSpec x_2_float = TensorSpec("tensor<float>(x[2])").add({{"x", 0}}, 3).add({{"x", 1}}, 5);
 
-    expect_nearest_neighbor_blueprint("tensor(x[2])", x_2_double);
-    expect_nearest_neighbor_blueprint("tensor<float>(x[2])", x_2_float);
-    expect_nearest_neighbor_blueprint("tensor(x[2])", x_2_float);
-    expect_nearest_neighbor_blueprint("tensor<float>(x[2])", x_2_double);
+    // same cell type:
+    expect_nearest_neighbor_blueprint("tensor(x[2])", x_2_double, x_2_double);
+    expect_nearest_neighbor_blueprint("tensor<float>(x[2])", x_2_float, x_2_float);
+    // convert cell type:
+    expect_nearest_neighbor_blueprint("tensor(x[2])", x_2_float, x_2_double);
+    expect_nearest_neighbor_blueprint("tensor<float>(x[2])", x_2_double, x_2_float);
 }
 
 void
diff --git a/searchlib/src/tests/attribute/searchcontext/CMakeLists.txt b/searchlib/src/tests/attribute/searchcontext/CMakeLists.txt
index a705fd5ecb5..377d91bf634 100644
--- a/searchlib/src/tests/attribute/searchcontext/CMakeLists.txt
+++ b/searchlib/src/tests/attribute/searchcontext/CMakeLists.txt
@@ -1,7 +1,7 @@
 # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 vespa_add_executable(searchlib_searchcontext_test_app TEST
     SOURCES
-    searchcontext.cpp
+    searchcontext_test.cpp
     DEPENDS
     searchlib
     searchlib_test
diff --git a/searchlib/src/tests/attribute/searchcontext/searchcontext.cpp b/searchlib/src/tests/attribute/searchcontext/searchcontext_test.cpp
index 7d4a2d63355..416ddb5fbc0 100644
--- a/searchlib/src/tests/attribute/searchcontext/searchcontext.cpp
+++ b/searchlib/src/tests/attribute/searchcontext/searchcontext_test.cpp
@@ -1,24 +1,27 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
 #include <vespa/searchlib/attribute/attribute.h>
 #include <vespa/searchlib/attribute/attributefactory.h>
 #include <vespa/searchlib/attribute/attributeiterators.h>
+#include <vespa/searchlib/attribute/attributevector.hpp>
+#include <vespa/searchlib/attribute/elementiterator.h>
 #include <vespa/searchlib/attribute/flagattribute.h>
+#include <vespa/searchlib/attribute/multistringattribute.h>
+#include <vespa/searchlib/attribute/singleboolattribute.h>
 #include <vespa/searchlib/attribute/singlenumericattribute.h>
 #include <vespa/searchlib/attribute/singlestringattribute.h>
-#include <vespa/searchlib/attribute/multistringattribute.h>
-#include <vespa/searchlib/attribute/elementiterator.h>
 #include <vespa/searchlib/common/bitvectoriterator.h>
 #include <vespa/searchlib/fef/matchdata.h>
 #include <vespa/searchlib/fef/termfieldmatchdataarray.h>
-#include <vespa/searchlib/queryeval/hitcollector.h>
+#include <vespa/searchlib/parsequery/parse.h>
+#include <vespa/searchlib/query/query_term_simple.h>
 #include <vespa/searchlib/queryeval/emptysearch.h>
 #include <vespa/searchlib/queryeval/executeinfo.h>
+#include <vespa/searchlib/queryeval/hitcollector.h>
+#include <vespa/searchlib/queryeval/simpleresult.h>
+#include <vespa/searchlib/test/searchiteratorverifier.h>
 #include <vespa/vespalib/testkit/testapp.h>
 #include <vespa/vespalib/util/compress.h>
-#include <vespa/searchlib/test/searchiteratorverifier.h>
-#include <vespa/searchlib/query/query_term_simple.h>
-#include <vespa/searchlib/parsequery/parse.h>
-#include <vespa/searchlib/attribute/attributevector.hpp>
 
 #include <vespa/log/log.h>
 LOG_SETUP("searchcontext_test");
@@ -43,22 +46,24 @@ isUnsignedSmallIntAttribute(const AttributeVector &a)
 
 }
 
-typedef AttributeVector::SP AttributePtr;
-typedef std::unique_ptr<AttributeVector::SearchContext> SearchContextPtr;
-typedef AttributeVector::SearchContext SearchContext;
-using attribute::Config;
+using AttributePtr = AttributeVector::SP;
+using ResultSetPtr = std::unique_ptr<ResultSet>;
+using SearchBasePtr = queryeval::SearchIterator::UP;
+using SearchContext = AttributeVector::SearchContext;
+using SearchContextPtr = std::unique_ptr<AttributeVector::SearchContext>;
+using largeint_t = AttributeVector::largeint_t;
+
 using attribute::BasicType;
 using attribute::CollectionType;
-typedef AttributeVector::largeint_t largeint_t;
-typedef queryeval::SearchIterator::UP SearchBasePtr;
-typedef std::unique_ptr<ResultSet> ResultSetPtr;
-
-using queryeval::HitCollector;
-using queryeval::SearchIterator;
+using attribute::Config;
+using attribute::SearchContextParams;
 using fef::MatchData;
 using fef::TermFieldMatchData;
 using fef::TermFieldMatchDataArray;
 using fef::TermFieldMatchDataPosition;
+using queryeval::HitCollector;
+using queryeval::SearchIterator;
+using queryeval::SimpleResult;
 
 class DocSet : public std::set<uint32_t>
 {
@@ -267,6 +272,9 @@ private:
     void requireThatOutOfBoundsSearchTermGivesZeroHits(const vespalib::string &name, const Config &cfg, int64_t maxValue);
     void requireThatOutOfBoundsSearchTermGivesZeroHits();
 
+    void single_bool_attribute_search_context_handles_true_and_false_queries();
+    void single_bool_attribute_search_iterator_handles_true_and_false_queries();
+
     // init maps with config objects
     void initIntegerConfig();
     void initFloatConfig();
@@ -1825,6 +1833,87 @@ SearchContextTest::requireThatOutOfBoundsSearchTermGivesZeroHits()
     }
 }
 
+class BoolAttributeFixture {
+private:
+    search::SingleBoolAttribute _attr;
+
+public:
+    BoolAttributeFixture(const SimpleResult& true_docs, uint32_t num_docs)
+        : _attr("bool_attr", search::GrowStrategy())
+    {
+        _attr.addDocs(num_docs);
+        for (uint32_t i = 0; i < true_docs.getHitCount(); ++i) {
+            uint32_t docid = true_docs.getHit(i);
+            _attr.update(docid, 1);
+        }
+        _attr.commit();
+    }
+    search::AttributeVector::SearchContext::UP create_search_context(const std::string& term) const {
+        return _attr.getSearch(std::make_unique<search::QueryTermSimple>(term, search::QueryTermSimple::WORD),
+                               SearchContextParams().useBitVector(true));
+    }
+    SimpleResult search_context(const std::string& term) const {
+        auto search_ctx = create_search_context(term);
+        SimpleResult result;
+        int32_t weight = 10;
+        for (uint32_t docid = 1; docid < _attr.getNumDocs(); ++docid) {
+            bool match_1 = search_ctx->matches(docid);
+            bool match_2 = search_ctx->matches(docid, weight);
+            EXPECT_EQUAL(match_1, match_2);
+            EXPECT_EQUAL(match_2 ? 1 : 0, weight);
+            if (match_1) {
+                result.addHit(docid);
+            }
+            weight = 10;
+        }
+        return result;
+    }
+    SimpleResult search_iterator(const std::string& term, bool strict) const {
+        auto search_ctx = create_search_context(term);
+        TermFieldMatchData tfmd;
+        auto itr = search_ctx->createIterator(&tfmd, strict);
+        SimpleResult result;
+        if (strict) {
+            result.searchStrict(*itr, _attr.getNumDocs());
+        } else {
+            result.search(*itr, _attr.getNumDocs());
+        }
+        return result;
+    }
+};
+
+void
+SearchContextTest::single_bool_attribute_search_context_handles_true_and_false_queries()
+{
+    BoolAttributeFixture f(SimpleResult().addHit(3).addHit(5).addHit(7), 9);
+
+    auto true_exp = SimpleResult().addHit(3).addHit(5).addHit(7);
+    EXPECT_EQUAL(true_exp, f.search_context("true"));
+    EXPECT_EQUAL(true_exp, f.search_context("1"));
+
+    auto false_exp = SimpleResult().addHit(1).addHit(2).addHit(4).addHit(6).addHit(8);
+    EXPECT_EQUAL(false_exp, f.search_context("false"));
+    EXPECT_EQUAL(false_exp, f.search_context("0"));
+}
+
+void
+SearchContextTest::single_bool_attribute_search_iterator_handles_true_and_false_queries()
+{
+    BoolAttributeFixture f(SimpleResult().addHit(3).addHit(5).addHit(7), 9);
+
+    auto true_exp = SimpleResult().addHit(3).addHit(5).addHit(7);
+    EXPECT_EQUAL(true_exp, f.search_iterator("true", false));
+    EXPECT_EQUAL(true_exp, f.search_iterator("1", false));
+    EXPECT_EQUAL(true_exp, f.search_iterator("true", true));
+    EXPECT_EQUAL(true_exp, f.search_iterator("1", true));
+
+    auto false_exp = SimpleResult().addHit(1).addHit(2).addHit(4).addHit(6).addHit(8);
+    EXPECT_EQUAL(false_exp, f.search_iterator("false", false));
+    EXPECT_EQUAL(false_exp, f.search_iterator("0", false));
+    EXPECT_EQUAL(false_exp, f.search_iterator("false", true));
+    EXPECT_EQUAL(false_exp, f.search_iterator("0", true));
+}
+
 void
 SearchContextTest::initIntegerConfig()
 {
@@ -1955,6 +2044,8 @@ SearchContextTest::Main()
     TEST_DO(requireThatInvalidSearchTermGivesZeroHits());
     TEST_DO(requireThatFlagAttributeHandlesTheByteRange());
     TEST_DO(requireThatOutOfBoundsSearchTermGivesZeroHits());
+    TEST_DO(single_bool_attribute_search_context_handles_true_and_false_queries());
+    TEST_DO(single_bool_attribute_search_iterator_handles_true_and_false_queries());
 
     TEST_DONE();
 }
diff --git a/searchlib/src/tests/attribute/tensorattribute/CMakeLists.txt b/searchlib/src/tests/attribute/tensorattribute/CMakeLists.txt
index 3794fd88fc3..44ff45d02d3 100644
--- a/searchlib/src/tests/attribute/tensorattribute/CMakeLists.txt
+++ b/searchlib/src/tests/attribute/tensorattribute/CMakeLists.txt
@@ -5,5 +5,4 @@ vespa_add_executable(searchlib_tensorattribute_test_app TEST
     DEPENDS
     searchlib
 )
-vespa_add_test(NAME searchlib_tensorattribute_test_app COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/tensorattribute_test.sh
-               DEPENDS searchlib_tensorattribute_test_app)
+vespa_add_test(NAME searchlib_tensorattribute_test_app COMMAND searchlib_tensorattribute_test_app)
diff --git a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp
index 7e0fcdc0ccc..39a7e53ca8c 100644
--- a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp
+++ b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp
@@ -1,34 +1,57 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
-#include <vespa/vespalib/testkit/test_kit.h>
 #include <vespa/document/base/exceptions.h>
-#include <vespa/searchlib/tensor/tensor_attribute.h>
-#include <vespa/searchlib/tensor/generic_tensor_attribute.h>
-#include <vespa/searchlib/tensor/dense_tensor_attribute.h>
-#include <vespa/searchlib/attribute/attributeguard.h>
-#include <vespa/eval/tensor/tensor.h>
-#include <vespa/eval/tensor/dense/dense_tensor.h>
 #include <vespa/eval/tensor/default_tensor_engine.h>
-#include <vespa/vespalib/io/fileutil.h>
-#include <vespa/vespalib/data/fileheader.h>
+#include <vespa/eval/tensor/dense/dense_tensor.h>
+#include <vespa/eval/tensor/tensor.h>
 #include <vespa/fastos/file.h>
+#include <vespa/searchlib/attribute/attribute_read_guard.h>
+#include <vespa/searchlib/attribute/attributeguard.h>
+#include <vespa/searchlib/tensor/default_nearest_neighbor_index_factory.h>
+#include <vespa/searchlib/tensor/dense_tensor_attribute.h>
+#include <vespa/searchlib/tensor/doc_vector_access.h>
+#include <vespa/searchlib/tensor/generic_tensor_attribute.h>
+#include <vespa/searchlib/tensor/hnsw_index.h>
+#include <vespa/searchlib/tensor/nearest_neighbor_index.h>
+#include <vespa/searchlib/tensor/nearest_neighbor_index_factory.h>
+#include <vespa/searchlib/tensor/nearest_neighbor_index_saver.h>
+#include <vespa/searchlib/tensor/tensor_attribute.h>
+#include <vespa/searchlib/test/directory_handler.h>
+#include <vespa/searchlib/util/fileutil.h>
+#include <vespa/vespalib/data/fileheader.h>
+#include <vespa/vespalib/io/fileutil.h>
+#include <vespa/vespalib/test/insertion_operators.h>
+#include <vespa/vespalib/testkit/test_kit.h>
+#include <vespa/vespalib/util/bufferwriter.h>
+
 #include <vespa/log/log.h>
 LOG_SETUP("tensorattribute_test");
 
 using document::WrongTensorTypeException;
-using search::tensor::TensorAttribute;
-using search::tensor::DenseTensorAttribute;
-using search::tensor::GenericTensorAttribute;
 using search::AttributeGuard;
 using search::AttributeVector;
-using vespalib::eval::ValueType;
+using search::attribute::DistanceMetric;
+using search::attribute::HnswIndexParams;
+using search::tensor::DefaultNearestNeighborIndexFactory;
+using search::tensor::DenseTensorAttribute;
+using search::tensor::DocVectorAccess;
+using search::tensor::GenericTensorAttribute;
+using search::tensor::HnswIndex;
+using search::tensor::HnswNode;
+using search::tensor::NearestNeighborIndex;
+using search::tensor::NearestNeighborIndexFactory;
+using search::tensor::NearestNeighborIndexSaver;
+using search::tensor::TensorAttribute;
 using vespalib::eval::TensorSpec;
-using vespalib::tensor::Tensor;
-using vespalib::tensor::DenseTensor;
+using vespalib::eval::ValueType;
 using vespalib::tensor::DefaultTensorEngine;
+using vespalib::tensor::DenseTensor;
+using vespalib::tensor::Tensor;
 
-namespace vespalib {
-namespace tensor {
+using DoubleVector = std::vector<double>;
+using generation_t = vespalib::GenerationHandler::generation_t;
+
+namespace vespalib::tensor {
 
 static bool operator==(const Tensor &lhs, const Tensor &rhs)
 {
@@ -36,10 +59,10 @@ static bool operator==(const Tensor &lhs, const Tensor &rhs)
 }
 
 }
-}
 
 vespalib::string sparseSpec("tensor(x{},y{})");
 vespalib::string denseSpec("tensor(x[2],y[3])");
+vespalib::string vec_2d_spec("tensor(x[2])");
 
 Tensor::UP createTensor(const TensorSpec &spec) {
     auto value = DefaultTensorEngine::ref().from_spec(spec);
@@ -52,25 +75,161 @@ Tensor::UP createTensor(const TensorSpec &spec) {
     return Tensor::UP(tensor);
 }
 
-struct Fixture
+TensorSpec
+vec_2d(double x0, double x1)
 {
+    return TensorSpec(vec_2d_spec).add({{"x", 0}}, x0).add({{"x", 1}}, x1);
+}
+
+class MockIndexSaver : public NearestNeighborIndexSaver {
+private:
+    int _index_value;
+
+public:
+    MockIndexSaver(int index_value) : _index_value(index_value) {}
+    void save(search::BufferWriter& writer) const override {
+        writer.write(&_index_value, sizeof(int));
+        writer.flush();
+    }
+};
+
+class MockNearestNeighborIndex : public NearestNeighborIndex {
+private:
+    using Entry = std::pair<uint32_t, DoubleVector>;
+    using EntryVector = std::vector<Entry>;
+
+    const DocVectorAccess& _vectors;
+    EntryVector _adds;
+    EntryVector _removes;
+    generation_t _transfer_gen;
+    generation_t _trim_gen;
+    mutable size_t _memory_usage_cnt;
+    int _index_value;
+
+public:
+    MockNearestNeighborIndex(const DocVectorAccess& vectors)
+        : _vectors(vectors),
+          _adds(),
+          _removes(),
+          _transfer_gen(std::numeric_limits<generation_t>::max()),
+          _trim_gen(std::numeric_limits<generation_t>::max()),
+          _memory_usage_cnt(0),
+          _index_value(0)
+    {
+    }
+    void clear() {
+        _adds.clear();
+        _removes.clear();
+    }
+    int get_index_value() const {
+        return _index_value;
+    }
+    void save_index_with_value(int value) {
+        _index_value = value;
+    }
+    void expect_empty_add() const {
+        EXPECT_TRUE(_adds.empty());
+    }
+    void expect_add(uint32_t exp_docid, const DoubleVector& exp_vector) const {
+        EXPECT_EQUAL(1u, _adds.size());
+        EXPECT_EQUAL(exp_docid, _adds.back().first);
+        EXPECT_EQUAL(exp_vector, _adds.back().second);
+    }
+    void expect_adds(const EntryVector &exp_adds) const {
+        EXPECT_EQUAL(exp_adds, _adds);
+    }
+    void expect_empty_remove() const {
+        EXPECT_TRUE(_removes.empty());
+    }
+    void expect_remove(uint32_t exp_docid, const DoubleVector& exp_vector) const {
+        EXPECT_EQUAL(1u, _removes.size());
+        EXPECT_EQUAL(exp_docid, _removes.back().first);
+        EXPECT_EQUAL(exp_vector, _removes.back().second);
+    }
+    generation_t get_transfer_gen() const { return _transfer_gen; }
+    generation_t get_trim_gen() const { return _trim_gen; }
+    size_t memory_usage_cnt() const { return _memory_usage_cnt; }
+
+    void add_document(uint32_t docid) override {
+        auto vector = _vectors.get_vector(docid).typify<double>();
+        _adds.emplace_back(docid, DoubleVector(vector.begin(), vector.end()));
+    }
+    void remove_document(uint32_t docid) override {
+        auto vector = _vectors.get_vector(docid).typify<double>();
+        _removes.emplace_back(docid, DoubleVector(vector.begin(), vector.end()));
+    }
+    void transfer_hold_lists(generation_t current_gen) override {
+        _transfer_gen = current_gen;
+    }
+    void trim_hold_lists(generation_t first_used_gen) override {
+        _trim_gen = first_used_gen;
+    }
+    vespalib::MemoryUsage memory_usage() const override {
+        ++_memory_usage_cnt;
+        return vespalib::MemoryUsage();
+    }
+    void get_state(const vespalib::slime::Inserter&) const override {}
+    std::unique_ptr<NearestNeighborIndexSaver> make_saver() const override {
+        if (_index_value != 0) {
+            return std::make_unique<MockIndexSaver>(_index_value);
+        }
+        return std::unique_ptr<NearestNeighborIndexSaver>();
+    }
+    bool load(const search::fileutil::LoadedBuffer& buf) override {
+        ASSERT_EQUAL(sizeof(int), buf.size());
+        _index_value = (reinterpret_cast<const int*>(buf.buffer()))[0];
+        return true;
+    }
+    std::vector<Neighbor> find_top_k(uint32_t k, vespalib::tensor::TypedCells vector, uint32_t explore_k) const override {
+        (void) k;
+        (void) vector;
+        (void) explore_k;
+        return std::vector<Neighbor>();
+    }
+    
+    const search::tensor::DistanceFunction *distance_function() const override { return nullptr; }
+};
+
+class MockNearestNeighborIndexFactory : public NearestNeighborIndexFactory {
+
+    std::unique_ptr<NearestNeighborIndex> make(const DocVectorAccess& vectors,
+                                               size_t vector_size,
+                                               ValueType::CellType cell_type,
+                                               const search::attribute::HnswIndexParams& params) const override {
+        (void) vector_size;
+        (void) params;
+        assert(cell_type == ValueType::CellType::DOUBLE);
+        return std::make_unique<MockNearestNeighborIndex>(vectors);
+    }
+};
+
+const vespalib::string test_dir = "test_data/";
+const vespalib::string attr_name = test_dir + "my_attr";
+
+struct Fixture {
     using BasicType = search::attribute::BasicType;
     using CollectionType = search::attribute::CollectionType;
     using Config = search::attribute::Config;
 
+    search::test::DirectoryHandler _dir_handler;
     Config _cfg;
     vespalib::string _name;
     vespalib::string _typeSpec;
+    std::unique_ptr<NearestNeighborIndexFactory> _index_factory;
     std::shared_ptr<TensorAttribute> _tensorAttr;
     std::shared_ptr<AttributeVector> _attr;
     bool _denseTensors;
     bool _useDenseTensorAttribute;
 
     Fixture(const vespalib::string &typeSpec,
-            bool useDenseTensorAttribute = false)
-        : _cfg(BasicType::TENSOR, CollectionType::SINGLE),
-          _name("test"),
+            bool useDenseTensorAttribute = false,
+            bool enable_hnsw_index = false,
+            bool use_mock_index = false)
+        : _dir_handler(test_dir),
+          _cfg(BasicType::TENSOR, CollectionType::SINGLE),
+          _name(attr_name),
           _typeSpec(typeSpec),
+          _index_factory(std::make_unique<DefaultNearestNeighborIndexFactory>()),
           _tensorAttr(),
           _attr(),
           _denseTensors(false),
@@ -80,25 +239,53 @@ struct Fixture
         if (_cfg.tensorType().is_dense()) {
             _denseTensors = true;
         }
+        if (enable_hnsw_index) {
+            _cfg.set_hnsw_index_params(HnswIndexParams(4, 20, DistanceMetric::Euclidean));
+            if (use_mock_index) {
+                _index_factory = std::make_unique<MockNearestNeighborIndexFactory>();
+            }
+        }
         _tensorAttr = makeAttr();
         _attr = _tensorAttr;
         _attr->addReservedDoc();
     }
+    ~Fixture() {}
 
     std::shared_ptr<TensorAttribute> makeAttr() {
         if (_useDenseTensorAttribute) {
             assert(_denseTensors);
-            return std::make_shared<DenseTensorAttribute>(_name, _cfg);
+            return std::make_shared<DenseTensorAttribute>(_name, _cfg, *_index_factory);
         } else {
             return std::make_shared<GenericTensorAttribute>(_name, _cfg);
         }
     }
 
+    const DenseTensorAttribute& as_dense_tensor() const {
+        auto result = dynamic_cast<const DenseTensorAttribute*>(_tensorAttr.get());
+        assert(result != nullptr);
+        return *result;
+    }
+
+    template <typename IndexType>
+    IndexType& get_nearest_neighbor_index() {
+        assert(as_dense_tensor().nearest_neighbor_index() != nullptr);
+        auto index = dynamic_cast<const IndexType*>(as_dense_tensor().nearest_neighbor_index());
+        assert(index != nullptr);
+        return *const_cast<IndexType*>(index);
+    }
+
+    HnswIndex& hnsw_index() {
+        return get_nearest_neighbor_index<HnswIndex>();
+    }
+
+    MockNearestNeighborIndex& mock_index() {
+        return get_nearest_neighbor_index<MockNearestNeighborIndex>();
+    }
+
     void ensureSpace(uint32_t docId) {
         while (_attr->getNumDocs() <= docId) {
             uint32_t newDocId = 0u;
             _attr->addDoc(newDocId);
-            _attr->commit();
         }
     }
 
@@ -108,38 +295,41 @@ struct Fixture
         _attr->commit();
     }
 
-    void setTensor(uint32_t docId, const Tensor &tensor) {
+    void set_tensor(uint32_t docid, const TensorSpec &spec) {
+        set_tensor_internal(docid, *createTensor(spec));
+    }
+
+    void set_empty_tensor(uint32_t docid) {
+        set_tensor_internal(docid, *_tensorAttr->getEmptyTensor());
+    }
+
+    void set_tensor_internal(uint32_t docId, const Tensor &tensor) {
         ensureSpace(docId);
         _tensorAttr->setTensor(docId, tensor);
         _attr->commit();
     }
 
+    generation_t get_current_gen() const {
+        return _attr->getCurrentGeneration();
+    }
+
     search::attribute::Status getStatus() {
         _attr->commit(true);
         return _attr->getStatus();
     }
 
-    void
-    assertGetNoTensor(uint32_t docId) {
+    void assertGetNoTensor(uint32_t docId) {
         AttributeGuard guard(_attr);
         Tensor::UP actTensor = _tensorAttr->getTensor(docId);
         EXPECT_FALSE(actTensor);
     }
 
-    void
-    assertGetTensor(const Tensor &expTensor, uint32_t docId)
-    {
+    void assertGetTensor(const TensorSpec &expSpec, uint32_t docId) {
+        Tensor::UP expTensor = createTensor(expSpec);
         AttributeGuard guard(_attr);
         Tensor::UP actTensor = _tensorAttr->getTensor(docId);
         EXPECT_TRUE(static_cast<bool>(actTensor));
-        EXPECT_EQUAL(expTensor, *actTensor);
-    }
-
-    void
-    assertGetTensor(const TensorSpec &expSpec, uint32_t docId)
-    {
-        Tensor::UP expTensor = createTensor(expSpec);
-        assertGetTensor(*expTensor, docId);
+        EXPECT_EQUAL(*expTensor, *actTensor);
     }
 
     void save() {
@@ -154,23 +344,20 @@ struct Fixture
         EXPECT_TRUE(loadok);
     }
 
-    Tensor::UP expDenseTensor3() const
-    {
-        return createTensor(TensorSpec(denseSpec)
-                            .add({{"x", 0}, {"y", 1}}, 11)
-                            .add({{"x", 1}, {"y", 2}}, 0));
+    TensorSpec expDenseTensor3() const {
+        return TensorSpec(denseSpec)
+                .add({{"x", 0}, {"y", 1}}, 11)
+                .add({{"x", 1}, {"y", 2}}, 0);
     }
 
-    Tensor::UP expDenseFillTensor() const
-    {
-        return createTensor(TensorSpec(denseSpec)
-                            .add({{"x", 0}, {"y", 0}}, 5)
-                            .add({{"x", 1}, {"y", 2}}, 0));
+    TensorSpec expDenseFillTensor() const {
+        return TensorSpec(denseSpec)
+                .add({{"x", 0}, {"y", 0}}, 5)
+                .add({{"x", 1}, {"y", 2}}, 0);
     }
 
-    Tensor::UP expEmptyDenseTensor() const
-    {
-        return createTensor(TensorSpec(denseSpec));
+    TensorSpec expEmptyDenseTensor() const {
+        return TensorSpec(denseSpec);
     }
 
     vespalib::string expEmptyDenseTensorSpec() const {
@@ -185,7 +372,6 @@ struct Fixture
     void testEmptyTensor();
 };
 
-
 void
 Fixture::testEmptyAttribute()
 {
@@ -198,23 +384,22 @@ Fixture::testSetTensorValue()
 {
     ensureSpace(4);
     EXPECT_EQUAL(5u, _attr->getNumDocs());
-    EXPECT_EQUAL(5u, _attr->getCommittedDocIdLimit());
     TEST_DO(assertGetNoTensor(4));
-    EXPECT_EXCEPTION(setTensor(4, *createTensor(TensorSpec("double"))),
+    EXPECT_EXCEPTION(set_tensor(4, TensorSpec("double")),
                      WrongTensorTypeException,
                      "but other tensor type is 'double'");
     TEST_DO(assertGetNoTensor(4));
-    setTensor(4, *_tensorAttr->getEmptyTensor());
+    set_empty_tensor(4);
     if (_denseTensors) {
-        TEST_DO(assertGetTensor(*expEmptyDenseTensor(), 4));
-        setTensor(3, *expDenseTensor3());
-        TEST_DO(assertGetTensor(*expDenseTensor3(), 3));
+        TEST_DO(assertGetTensor(expEmptyDenseTensor(), 4));
+        set_tensor(3, expDenseTensor3());
+        TEST_DO(assertGetTensor(expDenseTensor3(), 3));
     } else {
         TEST_DO(assertGetTensor(TensorSpec(sparseSpec), 4));
-        setTensor(3, *createTensor(TensorSpec(sparseSpec)
-                                   .add({{"x", ""}, {"y", ""}}, 11)));
+        set_tensor(3, TensorSpec(sparseSpec)
+                .add({{"x", ""}, {"y", ""}}, 11));
         TEST_DO(assertGetTensor(TensorSpec(sparseSpec)
-                                .add({{"x", ""}, {"y", ""}}, 11), 3));
+                                        .add({{"x", ""}, {"y", ""}}, 11), 3));
     }
     TEST_DO(assertGetNoTensor(2));
     TEST_DO(clearTensor(3));
@@ -225,29 +410,28 @@ void
 Fixture::testSaveLoad()
 {
     ensureSpace(4);
-    setTensor(4, *_tensorAttr->getEmptyTensor());
+    set_empty_tensor(4);
     if (_denseTensors) {
-        setTensor(3, *expDenseTensor3());
+        set_tensor(3, expDenseTensor3());
     } else {
-        setTensor(3, *createTensor(TensorSpec(sparseSpec)
-                                   .add({{"x", ""}, {"y", "1"}}, 11)));
+        set_tensor(3, TensorSpec(sparseSpec)
+                .add({{"x", ""}, {"y", "1"}}, 11));
     }
     TEST_DO(save());
     TEST_DO(load());
     EXPECT_EQUAL(5u, _attr->getNumDocs());
     EXPECT_EQUAL(5u, _attr->getCommittedDocIdLimit());
     if (_denseTensors) {
-        TEST_DO(assertGetTensor(*expDenseTensor3(), 3));
-        TEST_DO(assertGetTensor(*expEmptyDenseTensor(), 4));
+        TEST_DO(assertGetTensor(expDenseTensor3(), 3));
+        TEST_DO(assertGetTensor(expEmptyDenseTensor(), 4));
     } else {
         TEST_DO(assertGetTensor(TensorSpec(sparseSpec)
-                                .add({{"x", ""}, {"y", "1"}}, 11), 3));
+                                        .add({{"x", ""}, {"y", "1"}}, 11), 3));
         TEST_DO(assertGetTensor(TensorSpec(sparseSpec), 4));
     }
     TEST_DO(assertGetNoTensor(2));
 }
 
-
 void
 Fixture::testCompaction()
 {
@@ -256,29 +440,28 @@ Fixture::testCompaction()
         return;
     }
     ensureSpace(4);
-    Tensor::UP emptytensor = _tensorAttr->getEmptyTensor();
-    Tensor::UP emptyxytensor = createTensor(TensorSpec(sparseSpec));
-    Tensor::UP simpletensor = createTensor(TensorSpec(sparseSpec)
-                                           .add({{"x", ""}, {"y", "1"}}, 11));
-    Tensor::UP filltensor = createTensor(TensorSpec(sparseSpec)
-                                         .add({{"x", ""}, {"y", ""}}, 5));
+    TensorSpec empty_xy_tensor(sparseSpec);
+    TensorSpec simple_tensor = TensorSpec(sparseSpec)
+            .add({{"x", ""}, {"y", "1"}}, 11);
+    TensorSpec fill_tensor = TensorSpec(sparseSpec)
+            .add({{"x", ""}, {"y", ""}}, 5);
     if (_denseTensors) {
-        emptyxytensor = expEmptyDenseTensor();
-        simpletensor = expDenseTensor3();
-        filltensor = expDenseFillTensor();
+        empty_xy_tensor = expEmptyDenseTensor();
+        simple_tensor = expDenseTensor3();
+        fill_tensor = expDenseFillTensor();
     }
-    setTensor(4, *emptytensor);
-    setTensor(3, *simpletensor);
-    setTensor(2, *filltensor);
+    set_empty_tensor(4);
+    set_tensor(3, simple_tensor);
+    set_tensor(2, fill_tensor);
     clearTensor(2);
-    setTensor(2, *filltensor);
+    set_tensor(2, fill_tensor);
     search::attribute::Status oldStatus = getStatus();
     search::attribute::Status newStatus = oldStatus;
     uint64_t iter = 0;
     uint64_t iterLimit = 100000;
     for (; iter < iterLimit; ++iter) {
         clearTensor(2);
-        setTensor(2, *filltensor);
+        set_tensor(2, fill_tensor);
         newStatus = getStatus();
         if (newStatus.getUsed() < oldStatus.getUsed()) {
             break;
@@ -290,9 +473,9 @@ Fixture::testCompaction()
         "iter = %" PRIu64 ", memory usage %" PRIu64 ", -> %" PRIu64,
         iter, oldStatus.getUsed(), newStatus.getUsed());
     TEST_DO(assertGetNoTensor(1));
-    TEST_DO(assertGetTensor(*filltensor, 2));
-    TEST_DO(assertGetTensor(*simpletensor, 3));
-    TEST_DO(assertGetTensor(*emptyxytensor, 4));
+    TEST_DO(assertGetTensor(fill_tensor, 2));
+    TEST_DO(assertGetTensor(simple_tensor, 3));
+    TEST_DO(assertGetTensor(empty_xy_tensor, 4));
 }
 
 void
@@ -303,7 +486,8 @@ Fixture::testTensorTypeFileHeaderTag()
 
     vespalib::FileHeader header;
     FastOS_File file;
-    EXPECT_TRUE(file.OpenReadOnly("test.dat"));
+    vespalib::string file_name = attr_name + ".dat";
+    EXPECT_TRUE(file.OpenReadOnly(file_name.c_str()));
     (void) header.readFile(file);
     file.Close();
     EXPECT_TRUE(header.hasTag("tensortype"));
@@ -315,7 +499,6 @@ Fixture::testTensorTypeFileHeaderTag()
     }
 }
 
-
 void
 Fixture::testEmptyTensor()
 {
@@ -330,7 +513,6 @@ Fixture::testEmptyTensor()
     }
 }
 
-
 template <class MakeFixture>
 void testAll(MakeFixture &&f)
 {
@@ -357,4 +539,155 @@ TEST("Test dense tensors with dense tensor attribute")
     testAll([]() { return std::make_shared<Fixture>(denseSpec, true); });
 }
 
-TEST_MAIN() { TEST_RUN_ALL(); vespalib::unlink("test.dat"); }
+TEST_F("Hnsw index is NOT instantiated in dense tensor attribute by default",
+       Fixture(vec_2d_spec, true, false))
+{
+    const auto& tensor = f.as_dense_tensor();
+    EXPECT_TRUE(tensor.nearest_neighbor_index() == nullptr);
+}
+
+class DenseTensorAttributeHnswIndex : public Fixture {
+public:
+    DenseTensorAttributeHnswIndex() : Fixture(vec_2d_spec, true, true, false) {}
+};
+
+TEST_F("Hnsw index is instantiated in dense tensor attribute when specified in config", DenseTensorAttributeHnswIndex)
+{
+    auto& index = f.hnsw_index();
+
+    const auto& cfg = index.config();
+    EXPECT_EQUAL(8u, cfg.max_links_at_level_0());
+    EXPECT_EQUAL(4u, cfg.max_links_on_inserts());
+    EXPECT_EQUAL(20u, cfg.neighbors_to_explore_at_construction());
+    EXPECT_TRUE(cfg.heuristic_select_neighbors());
+}
+
+void
+expect_level_0(uint32_t exp_docid, const HnswNode& node)
+{
+    ASSERT_GREATER_EQUAL(node.size(), 1u);
+    ASSERT_EQUAL(1u, node.level(0).size());
+    EXPECT_EQUAL(exp_docid, node.level(0)[0]);
+}
+
+TEST_F("Hnsw index is integrated in dense tensor attribute and can be saved and loaded", DenseTensorAttributeHnswIndex)
+{
+    // Set two points that will be linked together in level 0 of the hnsw graph.
+    f.set_tensor(1, vec_2d(3, 5));
+    f.set_tensor(2, vec_2d(7, 9));
+
+    auto &index_a = f.hnsw_index();
+    expect_level_0(2, index_a.get_node(1));
+    expect_level_0(1, index_a.get_node(2));
+    f.save();
+    EXPECT_TRUE(vespalib::fileExists(attr_name + ".nnidx"));
+
+    f.load();
+    auto &index_b = f.hnsw_index();
+    EXPECT_NOT_EQUAL(&index_a, &index_b);
+    expect_level_0(2, index_b.get_node(1));
+    expect_level_0(1, index_b.get_node(2));
+}
+
+class DenseTensorAttributeMockIndex : public Fixture {
+public:
+    DenseTensorAttributeMockIndex() : Fixture(vec_2d_spec, true, true, true) {}
+};
+
+TEST_F("setTensor() updates nearest neighbor index", DenseTensorAttributeMockIndex)
+{
+    auto& index = f.mock_index();
+
+    f.set_tensor(1, vec_2d(3, 5));
+    index.expect_add(1, {3, 5});
+    index.expect_empty_remove();
+    index.clear();
+
+    // Replaces previous value.
+    f.set_tensor(1, vec_2d(7, 9));
+    index.expect_remove(1, {3, 5});
+    index.expect_add(1, {7, 9});
+}
+
+TEST_F("clearDoc() updates nearest neighbor index", DenseTensorAttributeMockIndex)
+{
+    auto& index = f.mock_index();
+
+    // Nothing to clear.
+    f.clearTensor(1);
+    index.expect_empty_remove();
+    index.expect_empty_add();
+
+    // Clears previous value.
+    f.set_tensor(1, vec_2d(3, 5));
+    index.clear();
+    f.clearTensor(1);
+    index.expect_remove(1, {3, 5});
+    index.expect_empty_add();
+}
+
+TEST_F("commit() ensures transfer and trim hold lists on nearest neighbor index", DenseTensorAttributeMockIndex)
+{
+    auto& index = f.mock_index();
+    TensorSpec spec = vec_2d(3, 5);
+
+    f.set_tensor(1, spec);
+    generation_t gen_1 = f.get_current_gen();
+    EXPECT_EQUAL(gen_1 - 1, index.get_transfer_gen());
+    EXPECT_EQUAL(gen_1, index.get_trim_gen());
+
+    generation_t gen_2 = 0;
+    {
+        // Takes guard on gen_1
+        auto guard = f._attr->makeReadGuard(false);
+        f.set_tensor(2, spec);
+        gen_2 = f.get_current_gen();
+        EXPECT_GREATER(gen_2, gen_1);
+        EXPECT_EQUAL(gen_2 - 1, index.get_transfer_gen());
+        EXPECT_EQUAL(gen_1, index.get_trim_gen());
+    }
+
+    f.set_tensor(3, spec);
+    generation_t gen_3 = f.get_current_gen();
+    EXPECT_GREATER(gen_3, gen_2);
+    EXPECT_EQUAL(gen_3 - 1, index.get_transfer_gen());
+    EXPECT_EQUAL(gen_3, index.get_trim_gen());
+}
+
+TEST_F("Memory usage is extracted from index when updating stats on attribute", DenseTensorAttributeMockIndex)
+{
+    size_t before = f.mock_index().memory_usage_cnt();
+    f.getStatus();
+    size_t after = f.mock_index().memory_usage_cnt();
+    EXPECT_EQUAL(before + 1, after);
+}
+
+TEST_F("Nearest neighbor index can be saved to disk and then loaded from file", DenseTensorAttributeMockIndex)
+{
+    f.set_tensor(1, vec_2d(3, 5));
+    f.set_tensor(2, vec_2d(7, 9));
+    f.mock_index().save_index_with_value(123);
+    f.save();
+    EXPECT_TRUE(vespalib::fileExists(attr_name + ".nnidx"));
+
+    f.load(); // index is loaded from saved file
+    auto& index = f.mock_index();
+    EXPECT_EQUAL(123, index.get_index_value());
+    index.expect_adds({});
+}
+
+TEST_F("onLoad() reconstructs nearest neighbor index if save file does not exists", DenseTensorAttributeMockIndex)
+{
+    f.set_tensor(1, vec_2d(3, 5));
+    f.set_tensor(2, vec_2d(7, 9));
+    f.save();
+    EXPECT_FALSE(vespalib::fileExists(attr_name + ".nnidx"));
+
+    f.load(); // index is reconstructed by adding all loaded tensors
+    auto& index = f.mock_index();
+    EXPECT_EQUAL(0, index.get_index_value());
+    index.expect_adds({{1, {3, 5}}, {2, {7, 9}}});
+}
+
+TEST_MAIN() { TEST_RUN_ALL(); }
+
diff --git a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.sh b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.sh
deleted file mode 100755
index dd9399dea78..00000000000
--- a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.sh
+++ /dev/null
@@ -1,5 +0,0 @@
-#!/bin/bash
-# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-set -e
-$VALGRIND ./searchlib_tensorattribute_test_app
-rm -rf *.dat
diff --git a/searchlib/src/tests/bitvector/bitvectorbenchmark.cpp b/searchlib/src/tests/bitvector/bitvectorbenchmark.cpp
index a9df188d417..842c42d4189 100644
--- a/searchlib/src/tests/bitvector/bitvectorbenchmark.cpp
+++ b/searchlib/src/tests/bitvector/bitvectorbenchmark.cpp
@@ -73,7 +73,7 @@ void BitVectorBenchmark::testCountSpeed1()
 {
     _bv[0]->invalidateCachedCount();
     unsigned int cnt = _bv[0]->countTrueBits();
-    assert(cnt = _bvc[0]);
+    assert(cnt == _bvc[0]);
     (void) cnt;
 }
 
diff --git a/searchlib/src/tests/common/bitvector/bitvector_test.cpp b/searchlib/src/tests/common/bitvector/bitvector_test.cpp
index d720b105671..409cc9f2725 100644
--- a/searchlib/src/tests/common/bitvector/bitvector_test.cpp
+++ b/searchlib/src/tests/common/bitvector/bitvector_test.cpp
@@ -542,15 +542,16 @@ TEST("requireThatGrowWorks")
 {
     vespalib::GenerationHolder g;
     GrowableBitVector v(200, 200, g);
+    EXPECT_EQUAL(0u, v.countTrueBits());
     
-    v.setBit(7);
-    v.setBit(39);
-    v.setBit(71);
-    v.setBit(103);
-   
-    EXPECT_EQUAL(200u, v.size()); 
+    v.setBitAndMaintainCount(7);
+    v.setBitAndMaintainCount(39);
+    v.setBitAndMaintainCount(71);
+    v.setBitAndMaintainCount(103);
+    EXPECT_EQUAL(4u, v.countTrueBits());
+
+    EXPECT_EQUAL(200u, v.size());
     EXPECT_EQUAL(1023u, v.capacity()); 
-    v.invalidateCachedCount();
     EXPECT_TRUE(assertBV("[7,39,71,103]", v));
     EXPECT_EQUAL(4u, v.countTrueBits());
     EXPECT_TRUE(v.reserve(1024));
@@ -584,6 +585,13 @@ TEST("requireThatGrowWorks")
     EXPECT_EQUAL(2047u, v.capacity()); 
     EXPECT_TRUE(assertBV("[7,39,71]", v));
     EXPECT_EQUAL(3u, v.countTrueBits());
+
+    v.invalidateCachedCount();
+    EXPECT_TRUE(v.reserve(3100));
+    EXPECT_EQUAL(100u, v.size());
+    EXPECT_EQUAL(4095u, v.capacity());
+    EXPECT_EQUAL(3u, v.countTrueBits());
+
     g.transferHoldLists(1);
     g.trimHoldLists(2);
 }
diff --git a/searchlib/src/tests/common/foregroundtaskexecutor/.gitignore b/searchlib/src/tests/common/foregroundtaskexecutor/.gitignore
deleted file mode 100644
index 0bd7759156b..00000000000
--- a/searchlib/src/tests/common/foregroundtaskexecutor/.gitignore
+++ /dev/null
@@ -1 +0,0 @@
-searchlib_foregroundtaskexecutor_test_app
diff --git a/searchlib/src/tests/common/foregroundtaskexecutor/CMakeLists.txt b/searchlib/src/tests/common/foregroundtaskexecutor/CMakeLists.txt
deleted file mode 100644
index 64354d54396..00000000000
--- a/searchlib/src/tests/common/foregroundtaskexecutor/CMakeLists.txt
+++ /dev/null
@@ -1,8 +0,0 @@
-# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-vespa_add_executable(searchlib_foregroundtaskexecutor_test_app TEST
-    SOURCES
-    foregroundtaskexecutor_test.cpp
-    DEPENDS
-    searchlib
-)
-vespa_add_test(NAME searchlib_foregroundtaskexecutor_test_app COMMAND searchlib_foregroundtaskexecutor_test_app)
diff --git a/searchlib/src/tests/common/sequencedtaskexecutor/.gitignore b/searchlib/src/tests/common/sequencedtaskexecutor/.gitignore
deleted file mode 100644
index 35d038b0b7c..00000000000
--- a/searchlib/src/tests/common/sequencedtaskexecutor/.gitignore
+++ /dev/null
@@ -1 +0,0 @@
-searchlib_sequencedtaskexecutor_test_app
diff --git a/searchlib/src/tests/common/sequencedtaskexecutor/CMakeLists.txt b/searchlib/src/tests/common/sequencedtaskexecutor/CMakeLists.txt
deleted file mode 100644
index 6ba30a1647f..00000000000
--- a/searchlib/src/tests/common/sequencedtaskexecutor/CMakeLists.txt
+++ /dev/null
@@ -1,8 +0,0 @@
-# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-vespa_add_executable(searchlib_sequencedtaskexecutor_test_app TEST
-    SOURCES
-    sequencedtaskexecutor_test.cpp
-    DEPENDS
-    searchlib
-)
-vespa_add_test(NAME searchlib_sequencedtaskexecutor_test_app COMMAND searchlib_sequencedtaskexecutor_test_app)
diff --git a/searchlib/src/tests/diskindex/fusion/fusion_test.cpp b/searchlib/src/tests/diskindex/fusion/fusion_test.cpp
index 1825c00ceda..c2fd048ce3e 100644
--- a/searchlib/src/tests/diskindex/fusion/fusion_test.cpp
+++ b/searchlib/src/tests/diskindex/fusion/fusion_test.cpp
@@ -1,6 +1,5 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
-#include <vespa/searchlib/common/sequencedtaskexecutor.h>
 #include <vespa/searchlib/diskindex/diskindex.h>
 #include <vespa/searchlib/diskindex/fusion.h>
 #include <vespa/searchlib/diskindex/indexbuilder.h>
@@ -19,6 +18,7 @@
 #include <vespa/vespalib/btree/btreeroot.hpp>
 #include <vespa/vespalib/io/fileutil.h>
 #include <vespa/vespalib/util/threadstackexecutor.h>
+#include <vespa/vespalib/util/sequencedtaskexecutor.h>
 #include <gtest/gtest.h>
 
 #include <vespa/log/log.h>
@@ -37,6 +37,7 @@ using search::common::FileHeaderContext;
 using search::index::schema::CollectionType;
 using search::index::schema::DataType;
 using search::index::test::MockFieldLengthInspector;
+using vespalib::SequencedTaskExecutor;
 
 using namespace index;
 
@@ -314,16 +315,16 @@ FusionTest::requireThatFusionIsWorking(const vespalib::string &prefix, bool dire
                                addField("f4"));
     FieldIndexCollection fic(schema, MockFieldLengthInspector());
     DocBuilder b(schema);
-    SequencedTaskExecutor invertThreads(2);
-    SequencedTaskExecutor pushThreads(2);
-    DocumentInverter inv(schema, invertThreads, pushThreads, fic);
+    auto invertThreads = SequencedTaskExecutor::create(2);
+    auto pushThreads = SequencedTaskExecutor::create(2);
+    DocumentInverter inv(schema, *invertThreads, *pushThreads, fic);
     Document::UP doc;
 
     doc = make_doc10(b);
     inv.invertDocument(10, *doc);
-    invertThreads.sync();
+    invertThreads->sync();
     myPushDocument(inv);
-    pushThreads.sync();
+    pushThreads->sync();
 
     b.startDocument("id:ns:searchdocument::11").
         startIndexField("f3").
@@ -331,9 +332,9 @@ FusionTest::requireThatFusionIsWorking(const vespalib::string &prefix, bool dire
         endField();
     doc = b.endDocument();
     inv.invertDocument(11, *doc);
-    invertThreads.sync();
+    invertThreads->sync();
     myPushDocument(inv);
-    pushThreads.sync();
+    pushThreads->sync();
 
     b.startDocument("id:ns:searchdocument::12").
         startIndexField("f3").
@@ -341,9 +342,9 @@ FusionTest::requireThatFusionIsWorking(const vespalib::string &prefix, bool dire
         endField();
     doc = b.endDocument();
     inv.invertDocument(12, *doc);
-    invertThreads.sync();
+    invertThreads->sync();
     myPushDocument(inv);
-    pushThreads.sync();
+    pushThreads->sync();
 
     IndexBuilder ib(schema);
     vespalib::string dump2dir = prefix + "dump2";
@@ -455,14 +456,14 @@ FusionTest::make_simple_index(const vespalib::string &dump_dir, const IFieldLeng
     uint32_t numDocs = 20;
     uint32_t numWords = 1000;
     DocBuilder b(_schema);
-    SequencedTaskExecutor invertThreads(2);
-    SequencedTaskExecutor pushThreads(2);
-    DocumentInverter inv(_schema, invertThreads, pushThreads, fic);
+    auto invertThreads = SequencedTaskExecutor::create(2);
+    auto pushThreads = SequencedTaskExecutor::create(2);
+    DocumentInverter inv(_schema, *invertThreads, *pushThreads, fic);
 
     inv.invertDocument(10, *make_doc10(b));
-    invertThreads.sync();
+    invertThreads->sync();
     myPushDocument(inv);
-    pushThreads.sync();
+    pushThreads->sync();
 
     IndexBuilder ib(_schema);
     TuneFileIndexing tuneFileIndexing;
diff --git a/searchlib/src/tests/engine/proto_converter/proto_converter_test.cpp b/searchlib/src/tests/engine/proto_converter/proto_converter_test.cpp
index 3e129457d46..16e4f93b6fb 100644
--- a/searchlib/src/tests/engine/proto_converter/proto_converter_test.cpp
+++ b/searchlib/src/tests/engine/proto_converter/proto_converter_test.cpp
@@ -535,6 +535,16 @@ TEST_F(MonitorReplyTest, require_that_distribution_key_is_converted) {
     EXPECT_EQ(proto.distribution_key(), 7);
 }
 
+TEST_F(MonitorReplyTest, require_that_is_blocking_writes_is_converted) {
+    reply.is_blocking_writes = false;
+    convert();
+    EXPECT_FALSE(proto.is_blocking_writes());
+
+    reply.is_blocking_writes = true;
+    convert();
+    EXPECT_TRUE(proto.is_blocking_writes());
+}
+
 //-----------------------------------------------------------------------------
 
 GTEST_MAIN_RUN_ALL_TESTS()
diff --git a/searchlib/src/tests/expression/attributenode/attribute_node_test.cpp b/searchlib/src/tests/expression/attributenode/attribute_node_test.cpp
index 503922bdb13..02d459dd0d8 100644
--- a/searchlib/src/tests/expression/attributenode/attribute_node_test.cpp
+++ b/searchlib/src/tests/expression/attributenode/attribute_node_test.cpp
@@ -267,7 +267,7 @@ Fixture::assertBools(std::vector<bool> expVals, const vespalib::string &attribut
 {
     auto node = makeNode(attributeName, false, preserveAccurateTypes);
     uint32_t docId = 0;
-    for (const auto &expDocVal : expVals) {
+    for (const auto expDocVal : expVals) {
         ++docId;
         node->setDocId(docId);
         node->execute();
diff --git a/searchlib/src/tests/features/item_raw_score/item_raw_score_test.cpp b/searchlib/src/tests/features/item_raw_score/item_raw_score_test.cpp
index a954710f153..2a96862b3dc 100644
--- a/searchlib/src/tests/features/item_raw_score/item_raw_score_test.cpp
+++ b/searchlib/src/tests/features/item_raw_score/item_raw_score_test.cpp
@@ -5,6 +5,7 @@
 #include <vespa/searchlib/fef/test/indexenvironment.h>
 #include <vespa/searchlib/fef/test/indexenvironmentbuilder.h>
 #include <vespa/searchlib/fef/test/queryenvironment.h>
+#include <vespa/searchlib/fef/test/labels.h>
 #include <vespa/searchlib/features/item_raw_score_feature.h>
 #include <vespa/searchlib/fef/fef.h>
 #include <vespa/searchlib/fef/test/dummy_dependency_handler.h>
@@ -44,26 +45,6 @@ struct FeatureDumpFixture : public IDumpFeatureVisitor {
     FeatureDumpFixture() : IDumpFeatureVisitor() {}
 };
 
-struct Labels {
-    virtual void inject(Properties &p) const = 0;
-    virtual ~Labels() {}
-};
-struct NoLabel : public Labels {
-    virtual void inject(Properties &) const override {}    
-};
-struct SingleLabel : public Labels {
-    vespalib::string label;
-    uint32_t uid;
-    SingleLabel(const vespalib::string &l, uint32_t x) : label(l), uid(x) {}
-    virtual void inject(Properties &p) const override {
-        vespalib::asciistream key;
-        key << "vespa.label." << label << ".id";
-        vespalib::asciistream value;
-        value << uid;
-        p.add(key.str(), value.str());
-    }
-};
-
 struct RankFixture : BlueprintFactoryFixture, IndexFixture {
     QueryEnvironment         queryEnv;
     RankSetup                rankSetup;
diff --git a/searchlib/src/tests/features/nns_closeness/CMakeLists.txt b/searchlib/src/tests/features/nns_closeness/CMakeLists.txt
new file mode 100644
index 00000000000..368c75813d0
--- /dev/null
+++ b/searchlib/src/tests/features/nns_closeness/CMakeLists.txt
@@ -0,0 +1,9 @@
+# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+vespa_add_executable(searchlib_nns_closeness_test_app TEST
+    SOURCES
+    nns_closeness_test.cpp
+    DEPENDS
+    searchlib
+)
+vespa_add_test(NAME searchlib_nns_closeness_test_app COMMAND searchlib_nns_closeness_test_app)
diff --git a/searchlib/src/tests/features/nns_closeness/nns_closeness_test.cpp b/searchlib/src/tests/features/nns_closeness/nns_closeness_test.cpp
new file mode 100644
index 00000000000..d50b1cfc88f
--- /dev/null
+++ b/searchlib/src/tests/features/nns_closeness/nns_closeness_test.cpp
@@ -0,0 +1,158 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/vespalib/testkit/test_kit.h>
+#include <vespa/searchlib/features/setup.h>
+#include <vespa/searchlib/fef/test/indexenvironment.h>
+#include <vespa/searchlib/fef/test/indexenvironmentbuilder.h>
+#include <vespa/searchlib/fef/test/queryenvironment.h>
+#include <vespa/searchlib/fef/test/labels.h>
+#include <vespa/searchlib/features/closenessfeature.h>
+#include <vespa/searchlib/fef/fef.h>
+#include <vespa/searchlib/fef/test/dummy_dependency_handler.h>
+#include <vespa/vespalib/stllike/asciistream.h>
+#include <vespa/vespalib/util/stringfmt.h>
+
+using search::feature_t;
+using namespace search::fef;
+using namespace search::fef::test;
+using namespace search::features;
+using CollectionType = FieldInfo::CollectionType;
+using DataType = FieldInfo::DataType;
+
+const vespalib::string labelFeatureName("closeness(label,nns)");
+const vespalib::string fieldFeatureName("closeness(bar)");
+
+struct BlueprintFactoryFixture {
+    BlueprintFactory factory;
+    BlueprintFactoryFixture() : factory()
+    {
+        setup_search_features(factory);
+    }
+};
+
+struct IndexFixture {
+    IndexEnvironment indexEnv;
+    IndexFixture() : indexEnv()
+    {
+        IndexEnvironmentBuilder builder(indexEnv);
+        builder.addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, DataType::INT64, "foo");
+        builder.addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, DataType::TENSOR, "bar");
+    }
+};
+
+struct FeatureDumpFixture : public IDumpFeatureVisitor {
+    virtual void visitDumpFeature(const vespalib::string &) override {
+        TEST_ERROR("no features should be dumped");
+    }
+    FeatureDumpFixture() : IDumpFeatureVisitor() {}
+};
+
+struct RankFixture : BlueprintFactoryFixture, IndexFixture {
+    QueryEnvironment         queryEnv;
+    RankSetup                rankSetup;
+    MatchDataLayout          mdl;
+    MatchData::UP            match_data;
+    RankProgram::UP          rankProgram;
+    std::vector<TermFieldHandle> fooHandles;
+    std::vector<TermFieldHandle> barHandles;
+    RankFixture(size_t fooCnt, size_t barCnt, const Labels &labels, const vespalib::string &featureName)
+        : queryEnv(&indexEnv), rankSetup(factory, indexEnv),
+          mdl(), match_data(), rankProgram(), fooHandles(), barHandles()
+    {
+        for (size_t i = 0; i < fooCnt; ++i) {
+            uint32_t fieldId = indexEnv.getFieldByName("foo")->id();
+            fooHandles.push_back(mdl.allocTermField(fieldId));
+            SimpleTermData term;
+            term.setUniqueId(i + 1);
+            term.addField(fieldId).setHandle(fooHandles.back());
+            queryEnv.getTerms().push_back(term);
+        }
+        for (size_t i = 0; i < barCnt; ++i) { 
+            uint32_t fieldId = indexEnv.getFieldByName("bar")->id();
+            barHandles.push_back(mdl.allocTermField(fieldId));
+            SimpleTermData term;
+            term.setUniqueId(fooCnt + i + 1);
+            term.addField(fieldId).setHandle(barHandles.back());
+            queryEnv.getTerms().push_back(term);
+        }
+        labels.inject(queryEnv.getProperties());
+        rankSetup.setFirstPhaseRank(featureName);
+        rankSetup.setIgnoreDefaultRankFeatures(true);
+        ASSERT_TRUE(rankSetup.compile());
+        match_data = mdl.createMatchData();
+        rankProgram = rankSetup.create_first_phase_program();
+        rankProgram->setup(*match_data, queryEnv);
+    }
+    feature_t getScore(uint32_t docId) {
+        return Utils::getScoreFeature(*rankProgram, docId);
+    }
+    void setScore(TermFieldHandle handle, uint32_t docId, feature_t score) {
+        match_data->resolveTermField(handle)->setRawScore(docId, score);
+    }
+    void setFooScore(uint32_t i, uint32_t docId, feature_t distance) {
+        ASSERT_LESS(i, fooHandles.size());
+        setScore(fooHandles[i], docId, 1.0/(1.0+distance));
+    }
+    void setBarScore(uint32_t i, uint32_t docId, feature_t distance) {
+        ASSERT_LESS(i, barHandles.size());
+        setScore(barHandles[i], docId, 1.0/(1.0+distance));
+    }
+};
+
+TEST_F("require that blueprint can be created from factory", BlueprintFactoryFixture) {
+    Blueprint::SP bp = f.factory.createBlueprint("closeness");
+    EXPECT_TRUE(bp.get() != 0);
+    EXPECT_TRUE(dynamic_cast<ClosenessBlueprint*>(bp.get()) != 0);
+}
+
+TEST_FFF("require that no features are dumped", ClosenessBlueprint, IndexFixture, FeatureDumpFixture) {
+    f1.visitDumpFeatures(f2.indexEnv, f3);
+}
+
+TEST_FF("require that setup can be done on random label", ClosenessBlueprint, IndexFixture) {
+    DummyDependencyHandler deps(f1);
+    f1.setName(vespalib::make_string("%s(label,random_label)", f1.getBaseName().c_str()));
+    EXPECT_TRUE(static_cast<Blueprint&>(f1).setup(f2.indexEnv, std::vector<vespalib::string>{"label", "random_label"}));
+}
+
+TEST_FF("require that no label gives 0 closeness", NoLabel(), RankFixture(2, 2, f1, labelFeatureName)) {
+    EXPECT_EQUAL(0.0, f2.getScore(10));
+}
+
+TEST_FF("require that unrelated label gives 0 closeness", SingleLabel("unrelated", 1), RankFixture(2, 2, f1, labelFeatureName)) {
+    EXPECT_EQUAL(0.0, f2.getScore(10));
+}
+
+TEST_FF("require that labeled item raw score can be obtained", SingleLabel("nns", 1), RankFixture(2, 2, f1, labelFeatureName)) {
+    f2.setFooScore(0, 10, 5.0);
+    EXPECT_EQUAL(1/(1+5.0), f2.getScore(10));
+}
+
+TEST_FF("require that field raw score can be obtained", NoLabel(), RankFixture(2, 2, f1, fieldFeatureName)) {
+    f2.setBarScore(0, 10, 5.0);
+    EXPECT_EQUAL(1/(1+5.0), f2.getScore(10));
+}
+
+TEST_FF("require that other raw scores are ignored", SingleLabel("nns", 2), RankFixture(2, 2, f1, labelFeatureName)) {
+    f2.setFooScore(0, 10, 1.0);
+    f2.setFooScore(1, 10, 2.0);
+    f2.setBarScore(0, 10, 5.0);
+    f2.setBarScore(1, 10, 6.0);
+    EXPECT_EQUAL(1/(1+2.0), f2.getScore(10));
+}
+
+TEST_FF("require that the correct raw score is used", NoLabel(), RankFixture(2, 2, f1, fieldFeatureName)) {
+    f2.setFooScore(0, 10, 3.0);
+    f2.setFooScore(1, 10, 4.0);
+    f2.setBarScore(0, 10, 8.0);
+    f2.setBarScore(1, 10, 7.0);
+    EXPECT_EQUAL(1/(1+7.0), f2.getScore(10));
+}
+
+TEST_FF("require that stale data is ignored", SingleLabel("nns", 2), RankFixture(2, 2, f1, labelFeatureName)) {
+    f2.setFooScore(0, 10, 1.0);
+    f2.setFooScore(1, 5, 2.0);
+    EXPECT_EQUAL(0, f2.getScore(10));
+}
+
+TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/searchlib/src/tests/features/nns_distance/CMakeLists.txt b/searchlib/src/tests/features/nns_distance/CMakeLists.txt
new file mode 100644
index 00000000000..5b4c8f86b44
--- /dev/null
+++ b/searchlib/src/tests/features/nns_distance/CMakeLists.txt
@@ -0,0 +1,9 @@
+# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+vespa_add_executable(searchlib_nns_distance_test_app TEST
+    SOURCES
+    nns_distance_test.cpp
+    DEPENDS
+    searchlib
+)
+vespa_add_test(NAME searchlib_nns_distance_test_app COMMAND searchlib_nns_distance_test_app)
diff --git a/searchlib/src/tests/features/nns_distance/nns_distance_test.cpp b/searchlib/src/tests/features/nns_distance/nns_distance_test.cpp
new file mode 100644
index 00000000000..a9a3f4b2ddd
--- /dev/null
+++ b/searchlib/src/tests/features/nns_distance/nns_distance_test.cpp
@@ -0,0 +1,164 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/vespalib/testkit/test_kit.h>
+#include <vespa/searchlib/features/setup.h>
+#include <vespa/searchlib/fef/test/indexenvironment.h>
+#include <vespa/searchlib/fef/test/indexenvironmentbuilder.h>
+#include <vespa/searchlib/fef/test/queryenvironment.h>
+#include <vespa/searchlib/fef/test/labels.h>
+#include <vespa/searchlib/features/distancefeature.h>
+#include <vespa/searchlib/fef/fef.h>
+#include <vespa/searchlib/fef/test/dummy_dependency_handler.h>
+#include <vespa/vespalib/stllike/asciistream.h>
+#include <vespa/vespalib/util/stringfmt.h>
+
+using search::feature_t;
+using namespace search::fef;
+using namespace search::fef::test;
+using namespace search::features;
+using CollectionType = FieldInfo::CollectionType;
+using DataType = FieldInfo::DataType;
+
+const vespalib::string labelFeatureName("distance(label,label)");
+const vespalib::string fieldFeatureName("distance(bar)");
+
+struct BlueprintFactoryFixture {
+    BlueprintFactory factory;
+    BlueprintFactoryFixture() : factory()
+    {
+        setup_search_features(factory);
+    }
+};
+
+struct IndexFixture {
+    IndexEnvironment indexEnv;
+    IndexFixture() : indexEnv()
+    {
+        IndexEnvironmentBuilder builder(indexEnv);
+        builder.addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, DataType::INT64, "foo");
+        builder.addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, DataType::TENSOR, "bar");
+    }
+};
+
+struct FeatureDumpFixture : public IDumpFeatureVisitor {
+    virtual void visitDumpFeature(const vespalib::string &) override {
+        TEST_ERROR("no features should be dumped");
+    }
+    FeatureDumpFixture() : IDumpFeatureVisitor() {}
+};
+
+struct RankFixture : BlueprintFactoryFixture, IndexFixture {
+    QueryEnvironment         queryEnv;
+    RankSetup                rankSetup;
+    MatchDataLayout          mdl;
+    MatchData::UP            match_data;
+    RankProgram::UP          rankProgram;
+    std::vector<TermFieldHandle> fooHandles;
+    std::vector<TermFieldHandle> barHandles;
+    RankFixture(size_t fooCnt, size_t barCnt, const Labels &labels, const vespalib::string &featureName)
+        : queryEnv(&indexEnv), rankSetup(factory, indexEnv),
+          mdl(), match_data(), rankProgram(), fooHandles(), barHandles()
+    {
+        for (size_t i = 0; i < fooCnt; ++i) {
+            uint32_t fieldId = indexEnv.getFieldByName("foo")->id();
+            fooHandles.push_back(mdl.allocTermField(fieldId));
+            SimpleTermData term;
+            term.setUniqueId(i + 1);
+            term.addField(fieldId).setHandle(fooHandles.back());
+            queryEnv.getTerms().push_back(term);
+        }
+        for (size_t i = 0; i < barCnt; ++i) { 
+            uint32_t fieldId = indexEnv.getFieldByName("bar")->id();
+            barHandles.push_back(mdl.allocTermField(fieldId));
+            SimpleTermData term;
+            term.setUniqueId(fooCnt + i + 1);
+            term.addField(fieldId).setHandle(barHandles.back());
+            queryEnv.getTerms().push_back(term);
+        }
+        labels.inject(queryEnv.getProperties());
+        rankSetup.setFirstPhaseRank(featureName);
+        rankSetup.setIgnoreDefaultRankFeatures(true);
+        ASSERT_TRUE(rankSetup.compile());
+        match_data = mdl.createMatchData();
+        rankProgram = rankSetup.create_first_phase_program();
+        rankProgram->setup(*match_data, queryEnv);
+    }
+    feature_t getScore(uint32_t docId) {
+        return Utils::getScoreFeature(*rankProgram, docId);
+    }
+    void setScore(TermFieldHandle handle, uint32_t docId, feature_t score) {
+        match_data->resolveTermField(handle)->setRawScore(docId, score);
+    }
+    void setFooScore(uint32_t i, uint32_t docId, feature_t distance) {
+        ASSERT_LESS(i, fooHandles.size());
+        setScore(fooHandles[i], docId, 1.0/(1.0+distance));
+    }
+    void setBarScore(uint32_t i, uint32_t docId, feature_t distance) {
+        ASSERT_LESS(i, barHandles.size());
+        setScore(barHandles[i], docId, 1.0/(1.0+distance));
+    }
+};
+
+TEST_F("require that blueprint can be created from factory", BlueprintFactoryFixture) {
+    Blueprint::SP bp = f.factory.createBlueprint("distance");
+    EXPECT_TRUE(bp.get() != 0);
+    EXPECT_TRUE(dynamic_cast<DistanceBlueprint*>(bp.get()) != 0);
+}
+
+TEST_FFF("require that no features are dumped", DistanceBlueprint, IndexFixture, FeatureDumpFixture) {
+    f1.visitDumpFeatures(f2.indexEnv, f3);
+}
+
+TEST_FF("require that setup can be done on random label", DistanceBlueprint, IndexFixture) {
+    DummyDependencyHandler deps(f1);
+    f1.setName(vespalib::make_string("%s(label,random_label)", f1.getBaseName().c_str()));
+    EXPECT_TRUE(static_cast<Blueprint&>(f1).setup(f2.indexEnv, std::vector<vespalib::string>{"label", "random_label"}));
+}
+
+TEST_FF("require that setup with unknown field fails", DistanceBlueprint, IndexFixture) {
+    DummyDependencyHandler deps(f1);
+    f1.setName(vespalib::make_string("%s(field,random_fieldname)", f1.getBaseName().c_str()));
+    EXPECT_FALSE(static_cast<Blueprint&>(f1).setup(f2.indexEnv, std::vector<vespalib::string>{"field", "random_fieldname"}));
+}
+
+TEST_FF("require that no label gives max-double distance", NoLabel(), RankFixture(2, 2, f1, labelFeatureName)) {
+    EXPECT_EQUAL(std::numeric_limits<feature_t>::max(), f2.getScore(10));
+}
+
+TEST_FF("require that unrelated label gives max-double distance", SingleLabel("unrelated", 1), RankFixture(2, 2, f1, labelFeatureName)) {
+    EXPECT_EQUAL(std::numeric_limits<feature_t>::max(), f2.getScore(10));
+}
+
+TEST_FF("require that labeled item raw score can be obtained", SingleLabel("label", 1), RankFixture(2, 2, f1, labelFeatureName)) {
+    f2.setFooScore(0, 10, 5.0);
+    EXPECT_EQUAL(5.0, f2.getScore(10));
+}
+
+TEST_FF("require that field raw score can be obtained", NoLabel(), RankFixture(2, 2, f1, fieldFeatureName)) {
+    f2.setBarScore(0, 10, 5.0);
+    EXPECT_EQUAL(5.0, f2.getScore(10));
+}
+
+TEST_FF("require that other raw scores are ignored", SingleLabel("label", 2), RankFixture(2, 2, f1, labelFeatureName)) {
+    f2.setFooScore(0, 10, 1.0);
+    f2.setFooScore(1, 10, 2.0);
+    f2.setBarScore(0, 10, 5.0);
+    f2.setBarScore(1, 10, 6.0);
+    EXPECT_EQUAL(2.0, f2.getScore(10));
+}
+
+TEST_FF("require that the correct raw score is used", NoLabel(), RankFixture(2, 2, f1, fieldFeatureName)) {
+    f2.setFooScore(0, 10, 3.0);
+    f2.setFooScore(1, 10, 4.0);
+    f2.setBarScore(0, 10, 8.0);
+    f2.setBarScore(1, 10, 7.0);
+    EXPECT_EQUAL(7.0, f2.getScore(10));
+}
+
+TEST_FF("require that stale data is ignored", SingleLabel("label", 2), RankFixture(2, 2, f1, labelFeatureName)) {
+    f2.setFooScore(0, 10, 1.0);
+    f2.setFooScore(1, 5, 2.0);
+    EXPECT_EQUAL(std::numeric_limits<feature_t>::max(), f2.getScore(10));
+}
+
+TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/searchlib/src/tests/features/prod_features.cpp b/searchlib/src/tests/features/prod_features.cpp
index 56aaf2dcbc9..1c07c81bc2f 100644
--- a/searchlib/src/tests/features/prod_features.cpp
+++ b/searchlib/src/tests/features/prod_features.cpp
@@ -9,16 +9,15 @@
 #include <vespa/searchlib/attribute/floatbase.h>
 #include <vespa/searchlib/attribute/integerbase.h>
 #include <vespa/searchlib/attribute/stringbase.h>
+#include <vespa/searchlib/attribute/singleboolattribute.h>
 #include <vespa/searchlib/features/agefeature.h>
 #include <vespa/searchlib/features/array_parser.hpp>
 #include <vespa/searchlib/features/attributefeature.h>
-#include <vespa/searchlib/features/attributematchfeature.h>
 #include <vespa/searchlib/features/closenessfeature.h>
 #include <vespa/searchlib/features/distancefeature.h>
 #include <vespa/searchlib/features/dotproductfeature.h>
 #include <vespa/searchlib/features/fieldlengthfeature.h>
 #include <vespa/searchlib/features/fieldmatchfeature.h>
-#include <vespa/searchlib/features/fieldtermmatchfeature.h>
 #include <vespa/searchlib/features/firstphasefeature.h>
 #include <vespa/searchlib/features/foreachfeature.h>
 #include <vespa/searchlib/features/freshnessfeature.h>
@@ -35,7 +34,6 @@
 #include <vespa/searchlib/features/setup.h>
 #include <vespa/searchlib/features/termfeature.h>
 #include <vespa/searchlib/features/utils.h>
-#include <vespa/searchlib/features/valuefeature.h>
 #include <vespa/searchlib/features/weighted_set_parser.hpp>
 #include <vespa/searchlib/fef/featurenamebuilder.h>
 #include <vespa/searchlib/fef/indexproperties.h>
@@ -60,6 +58,7 @@ using search::AttributeFactory;
 using search::IntegerAttribute;
 using search::FloatingPointAttribute;
 using search::StringAttribute;
+using search::SingleBoolAttribute;
 using search::WeightedSetStringExtAttribute;
 using search::attribute::WeightedEnumContent;
 
@@ -68,6 +67,7 @@ using AVC = search::attribute::Config;
 using AVBT = search::attribute::BasicType;
 using AVCT = search::attribute::CollectionType;
 using CollectionType = FieldInfo::CollectionType;
+using DataType = FieldInfo::DataType;
 
 const double EPS = 10e-6;
 
@@ -212,7 +212,7 @@ Test::setupForAgeTest(FtFeatureTest & ft, uint64_t docTime)
     doctime->addReservedDoc();
     doctime->addDocs(1);
     ft.getIndexEnv().getAttributeMap().add(doctime);
-    (static_cast<IntegerAttribute *>(doctime.get()))->update(1, docTime);
+    (dynamic_cast<IntegerAttribute *>(doctime.get()))->update(1, docTime);
     doctime->commit();
 }
 
@@ -240,7 +240,12 @@ Test::testAttribute()
         RankResult exp;
         exp.addScore("attribute(sint)", 10).
             addScore("attribute(sint,0)", 10).
+            addScore("attribute(slong)", 20).
+            addScore("attribute(sbyte)", 37).
+            addScore("attribute(sbool)", 1).
+            addScore("attribute(sebool)", 0).
             addScore("attribute(sfloat)", 60.5f).
+            addScore("attribute(sdouble)", 67.5f).
             addScore("attribute(sstr)", (feature_t)vespalib::hash_code("foo")).
             addScore("attribute(sint).count", 1).
             addScore("attribute(sfloat).count", 1).
@@ -250,12 +255,18 @@ Test::testAttribute()
             addScore("attribute(udefstr)", (feature_t)vespalib::hash_code(""));
 
         FtFeatureTest ft(_factory, exp.getKeys());
-        ft.getIndexEnv().getBuilder().addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "sint").
-            addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "sfloat").
-            addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "sstr").
-            addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "udefint").
-            addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "udeffloat").
-            addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "udefstr");
+        ft.getIndexEnv().getBuilder()
+            .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "sint")
+            .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "slong")
+            .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "sbyte")
+            .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, DataType::BOOL, "sbool")
+            .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, DataType::BOOL, "sebool")
+            .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "sfloat")
+            .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "sdouble")
+            .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "sstr")
+            .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "udefint")
+            .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "udeffloat")
+            .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "udefstr");
         setupForAttributeTest(ft);
         ASSERT_TRUE(ft.setup());
         ASSERT_TRUE(ft.execute(exp));
@@ -370,6 +381,11 @@ Test::setupForAttributeTest(FtFeatureTest &ft, bool setup_env)
     avs.push_back(AttributeFactory::createAttribute("udefint", AVC(AVBT::INT32, AVCT::SINGLE)));    // 9
     avs.push_back(AttributeFactory::createAttribute("udeffloat", AVC(AVBT::FLOAT, AVCT::SINGLE)));    // 10
     avs.push_back(AttributeFactory::createAttribute("udefstr", AVC(AVBT::STRING, AVCT::SINGLE)));    // 11
+    avs.push_back(AttributeFactory::createAttribute("sbyte",   AVC(AVBT::INT64,  AVCT::SINGLE))); // 12
+    avs.push_back(AttributeFactory::createAttribute("slong",   AVC(AVBT::INT64,  AVCT::SINGLE))); // 13
+    avs.push_back(AttributeFactory::createAttribute("sbool",   AVC(AVBT::BOOL,  AVCT::SINGLE))); // 14
+    avs.push_back(AttributeFactory::createAttribute("sebool",   AVC(AVBT::BOOL,  AVCT::SINGLE))); // 15
+    avs.push_back(AttributeFactory::createAttribute("sdouble",   AVC(AVBT::DOUBLE,  AVCT::SINGLE))); // 16
 
     // simulate a unique only attribute as specified in sd
     AVC cfg(AVBT::INT32, AVCT::SINGLE);
@@ -391,36 +407,46 @@ Test::setupForAttributeTest(FtFeatureTest &ft, bool setup_env)
             .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "udefint")
             .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "udeffloat")
             .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "udefstr")
-            .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "unique");
+            .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "unique")
+            .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "slong")
+            .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "sdouble")
+            .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "sbyte")
+            .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, DataType::BOOL,"sbool")
+            .addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, DataType::BOOL,"sebool");
     }
 
-    for (uint32_t i = 0; i < avs.size(); ++i) {
-        avs[i]->addReservedDoc();
-        avs[i]->addDocs(1);
-        ft.getIndexEnv().getAttributeMap().add(avs[i]);
+    for (const auto & attr : avs) {
+        attr->addReservedDoc();
+        attr->addDocs(1);
+        ft.getIndexEnv().getAttributeMap().add(attr);
     }
 
     // integer attributes
-    (static_cast<IntegerAttribute *>(avs[0].get()))->update(1, 10);
-    (static_cast<IntegerAttribute *>(avs[1].get()))->append(1, 20, 0);
-    (static_cast<IntegerAttribute *>(avs[1].get()))->append(1, 30, 0);
-    (static_cast<IntegerAttribute *>(avs[2].get()))->append(1, 40, 10);
-    (static_cast<IntegerAttribute *>(avs[2].get()))->append(1, 50, 20);
-    (static_cast<IntegerAttribute *>(avs[9].get()))->update(1, search::attribute::getUndefined<int32_t>());
+    (dynamic_cast<IntegerAttribute *>(avs[0].get()))->update(1, 10);
+    (dynamic_cast<IntegerAttribute *>(avs[12].get()))->update(1, 37);
+    (dynamic_cast<IntegerAttribute *>(avs[13].get()))->update(1, 20);
+    (dynamic_cast<SingleBoolAttribute *>(avs[14].get()))->update(1, 1);
+    (dynamic_cast<SingleBoolAttribute *>(avs[15].get()))->update(1, 0);
+    (dynamic_cast<IntegerAttribute *>(avs[1].get()))->append(1, 20, 0);
+    (dynamic_cast<IntegerAttribute *>(avs[1].get()))->append(1, 30, 0);
+    (dynamic_cast<IntegerAttribute *>(avs[2].get()))->append(1, 40, 10);
+    (dynamic_cast<IntegerAttribute *>(avs[2].get()))->append(1, 50, 20);
+    (dynamic_cast<IntegerAttribute *>(avs[9].get()))->update(1, search::attribute::getUndefined<int32_t>());
     // feature_t attributes
-    (static_cast<FloatingPointAttribute *>(avs[3].get()))->update(1, 60.5f);
-    (static_cast<FloatingPointAttribute *>(avs[4].get()))->append(1, 70.5f, 0);
-    (static_cast<FloatingPointAttribute *>(avs[4].get()))->append(1, 80.5f, 0);
-    (static_cast<FloatingPointAttribute *>(avs[5].get()))->append(1, 90.5f, -30);
-    (static_cast<FloatingPointAttribute *>(avs[5].get()))->append(1, 100.5f, -40);
-    (static_cast<FloatingPointAttribute *>(avs[10].get()))->update(1, search::attribute::getUndefined<float>());
+    (dynamic_cast<FloatingPointAttribute *>(avs[3].get()))->update(1, 60.5f);
+    (dynamic_cast<FloatingPointAttribute *>(avs[4].get()))->append(1, 70.5f, 0);
+    (dynamic_cast<FloatingPointAttribute *>(avs[4].get()))->append(1, 80.5f, 0);
+    (dynamic_cast<FloatingPointAttribute *>(avs[5].get()))->append(1, 90.5f, -30);
+    (dynamic_cast<FloatingPointAttribute *>(avs[5].get()))->append(1, 100.5f, -40);
+    (dynamic_cast<FloatingPointAttribute *>(avs[10].get()))->update(1, search::attribute::getUndefined<float>());
+    (dynamic_cast<FloatingPointAttribute *>(avs[16].get()))->update(1, 67.5);
     // string attributes
-    (static_cast<StringAttribute *>(avs[6].get()))->update(1, "foo");
-    (static_cast<StringAttribute *>(avs[7].get()))->append(1, "bar", 0);
-    (static_cast<StringAttribute *>(avs[7].get()))->append(1, "baz", 0);
-    (static_cast<StringAttribute *>(avs[8].get()))->append(1, "qux", 11);
-    (static_cast<StringAttribute *>(avs[8].get()))->append(1, "quux", 12);
-    (static_cast<StringAttribute *>(avs[11].get()))->update(1, "");
+    (dynamic_cast<StringAttribute *>(avs[6].get()))->update(1, "foo");
+    (dynamic_cast<StringAttribute *>(avs[7].get()))->append(1, "bar", 0);
+    (dynamic_cast<StringAttribute *>(avs[7].get()))->append(1, "baz", 0);
+    (dynamic_cast<StringAttribute *>(avs[8].get()))->append(1, "qux", 11);
+    (dynamic_cast<StringAttribute *>(avs[8].get()))->append(1, "quux", 12);
+    (dynamic_cast<StringAttribute *>(avs[11].get()))->update(1, "");
 
     for (uint32_t i = 0; i < avs.size() - 1; ++i) { // do not commit the noupdate attribute
         avs[i]->commit();
@@ -448,11 +474,13 @@ Test::testCloseness()
     }
 
     { // Test executor.
-        assertCloseness(1,   "pos", 0);
+        TEST_DO(assertCloseness(1,   "pos", 0));
         assertCloseness(0.8, "pos", 1802661);
         assertCloseness(0,   "pos", 9013306);
         // use non-existing attribute -> default distance
-        assertCloseness(0, "no", 0);
+        TEST_DO(assertCloseness(0, "no", 0));
+        // two-argument version
+        TEST_DO(assertCloseness(0.8, "field,pos", 1802661));
 
         // use non-default maxDistance
         assertCloseness(1,   "pos", 0,   100);
@@ -475,7 +503,7 @@ Test::assertCloseness(feature_t exp, const vespalib::string & attr, double dista
     FtFeatureTest ft(_factory, feature);
     std::vector<std::pair<int32_t, int32_t> > positions;
     int32_t x = 0;
-    positions.push_back(std::make_pair(x, x));
+    positions.emplace_back(x, x);
     setupForDistanceTest(ft, "pos", positions, false);
     ft.getQueryEnv().getLocation().setXPosition((int)distance);
     ft.getQueryEnv().getLocation().setValid(true);
@@ -572,7 +600,7 @@ Test::assertFieldMatch(const vespalib::string & spec,
                        const vespalib::string & field,
                        uint32_t totalTermWeight)
 {
-    assertFieldMatch(spec, query, field, NULL, totalTermWeight);
+    assertFieldMatch(spec, query, field, nullptr, totalTermWeight);
 }
 
 void
@@ -581,7 +609,7 @@ Test::assertFieldMatchTS(const vespalib::string & spec,
                          const vespalib::string & field,
                          feature_t totalSignificance)
 {
-    assertFieldMatch(spec, query, field, NULL, 0, totalSignificance);
+    assertFieldMatch(spec, query, field, nullptr, 0, totalSignificance);
 }
 
 
@@ -826,15 +854,23 @@ Test::testDistance()
         { // test default distance
             { // non-existing attribute
                 FtFeatureTest ft(_factory, "distance(pos)");
+                ft.getIndexEnv().getBuilder().addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, DataType::INT64, "pos");
                 ft.getQueryEnv().getLocation().setValid(true);
                 ASSERT_TRUE(ft.setup());
                 ASSERT_TRUE(ft.execute(RankResult().addScore("distance(pos)", 6400000000.0)));
             }
+            { // label
+                FtFeatureTest ft(_factory, "distance(label,foo)");
+                ft.getQueryEnv().getLocation().setValid(true);
+                ASSERT_TRUE(ft.setup());
+                ASSERT_TRUE(ft.execute(RankResult().addScore("distance(label,foo)", std::numeric_limits<feature_t>::max())));
+            }
             { // wrong attribute type (float)
                 FtFeatureTest ft(_factory, "distance(pos)");
                 AttributePtr pos = AttributeFactory::createAttribute("pos", AVC(AVBT::FLOAT,  AVCT::SINGLE));
                 pos->commit();
                 ft.getIndexEnv().getAttributeMap().add(pos);
+                ft.getIndexEnv().getBuilder().addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, DataType::INT64, "pos");
                 ft.getQueryEnv().getLocation().setValid(true);
                 ASSERT_TRUE(ft.setup());
                 ASSERT_TRUE(ft.execute(RankResult().addScore("distance(pos)", 6400000000.0)));
@@ -844,6 +880,7 @@ Test::testDistance()
                 AttributePtr pos = AttributeFactory::createAttribute("pos", AVC(AVBT::STRING,  AVCT::SINGLE));
                 pos->commit();
                 ft.getIndexEnv().getAttributeMap().add(pos);
+                ft.getIndexEnv().getBuilder().addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, DataType::INT64, "pos");
                 ft.getQueryEnv().getLocation().setValid(true);
                 ASSERT_TRUE(ft.setup());
                 ASSERT_TRUE(ft.execute(RankResult().addScore("distance(pos)", 6400000000.0)));
@@ -853,6 +890,7 @@ Test::testDistance()
                 AttributePtr pos = AttributeFactory::createAttribute("pos", AVC(AVBT::INT64,  AVCT::WSET));
                 pos->commit();
                 ft.getIndexEnv().getAttributeMap().add(pos);
+                ft.getIndexEnv().getBuilder().addField(FieldType::ATTRIBUTE, CollectionType::WEIGHTEDSET, DataType::INT64, "pos");
                 ft.getQueryEnv().getLocation().setValid(true);
                 ASSERT_TRUE(ft.setup());
                 ASSERT_TRUE(ft.execute(RankResult().addScore("distance(pos)", 6400000000.0)));
@@ -870,13 +908,14 @@ Test::setupForDistanceTest(FtFeatureTest &ft, const vespalib::string & attrName,
     pos->addReservedDoc();
     pos->addDocs(1);
     ft.getIndexEnv().getAttributeMap().add(pos);
+    ft.getIndexEnv().getBuilder().addField(FieldType::ATTRIBUTE, CollectionType::ARRAY, DataType::INT64, attrName);
 
-    IntegerAttribute * ia = static_cast<IntegerAttribute *>(pos.get());
-    for (uint32_t i = 0; i < positions.size(); ++i) {
+    auto ia = dynamic_cast<IntegerAttribute *>(pos.get());
+    for (const auto & p : positions) {
         if (zcurve) {
-            ia->append(1, vespalib::geo::ZCurve::encode(positions[i].first, positions[i].second), 0);
+            ia->append(1, vespalib::geo::ZCurve::encode(p.first, p.second), 0);
         } else {
-            ia->append(1, positions[i].first, 0);
+            ia->append(1, p.first, 0);
         }
     }
 
@@ -891,11 +930,11 @@ Test::assert2DZDistance(feature_t exp, const vespalib::string & positions,
     FtFeatureTest ft(_factory, "distance(pos)");
     std::vector<vespalib::string> ta = FtUtil::tokenize(positions, ",");
     std::vector<std::pair<int32_t, int32_t> > pos;
-    for (uint32_t i = 0; i < ta.size(); ++i) {
-        std::vector<vespalib::string> tb = FtUtil::tokenize(ta[i], ":");
-        int32_t x = util::strToNum<int32_t>(tb[0]);
-        int32_t y = util::strToNum<int32_t>(tb[1]);
-        pos.push_back(std::make_pair(x, y));
+    for (const auto & s : ta) {
+        std::vector<vespalib::string> tb = FtUtil::tokenize(s, ":");
+        auto x = util::strToNum<int32_t>(tb[0]);
+        auto y = util::strToNum<int32_t>(tb[1]);
+        pos.emplace_back(x, y);
     }
     setupForDistanceTest(ft, "pos", pos, true);
     ft.getQueryEnv().getLocation().setXPosition(xquery);
@@ -927,7 +966,7 @@ Test::testDistanceToPath()
     {
         // Test executor.
         std::vector<std::pair<int32_t, int32_t> > pos;
-        pos.push_back(std::make_pair(0, 0));
+        pos.emplace_back(0, 0);
 
         // invalid path
         assertDistanceToPath(pos, "a");
@@ -965,7 +1004,7 @@ Test::testDistanceToPath()
         assertDistanceToPath(pos, "(-3,2,2,2,2,-1,0,-1)", 1, 1, 2);
 
         // multiple document locations
-        pos.push_back(std::make_pair(0, 1));
+        pos.emplace_back(0, 1);
         assertDistanceToPath(pos, "(-1,1,1,1)", 0, 0.5);
         assertDistanceToPath(pos, "(-2,-1,-1,1)", 1, 1, 2);
         assertDistanceToPath(pos, "(-1,0.25,1,0.25)", 0.25, 0.5, 0.5);
@@ -1017,7 +1056,7 @@ Test::testDistanceToPath()
 }
 
 void
-Test::assertDistanceToPath(const std::vector<std::pair<int32_t, int32_t> > pos,
+Test::assertDistanceToPath(const std::vector<std::pair<int32_t, int32_t> > & pos,
                            const vespalib::string &path, feature_t distance, feature_t traveled, feature_t product)
 {
     LOG(info, "Testing distance to path '%s' with %zd document locations.", path.c_str(), pos.size());
@@ -1033,20 +1072,6 @@ Test::assertDistanceToPath(const std::vector<std::pair<int32_t, int32_t> > pos,
                            .addScore("distanceToPath(pos).product", product)));
 }
 
-void
-Test::setupForDocumentTest(FtFeatureTest &ft, const vespalib::string & attrName, const vespalib::string & docType)
-{
-    AttributePtr type = AttributeFactory::createAttribute(attrName, AVC(AVBT::STRING, AVCT::SINGLE));
-
-    type->addReservedDoc();
-    type->addDocs(1);
-    ft.getIndexEnv().getAttributeMap().add(type);
-
-    (static_cast<StringAttribute *>(type.get()))->update(1, docType);
-    type->commit();
-}
-
-
 namespace {
 
 void
@@ -1213,10 +1238,11 @@ Test::testDotProduct()
             assertDotProduct(0,   "(f:5,g:5)",             1, "wsextstr");
             assertDotProduct(550, "(a:1,b:2,c:3,d:4,e:5)", 1, "wsextstr");
         }
-        for (const char * name : {"wsbyte", "wsint"}) {
-            assertDotProduct(0,  "()",                    1, name);
-            assertDotProduct(0,  "(6:5,7:5)",             1, name);
-            assertDotProduct(55, "(1:1,2:2,3:3,4:4,5:5)", 1, name);
+        for (const char * name : {"wsbyte", "wsint", "wsint_fast"}) {
+            TEST_DO(assertDotProduct(0,  "()",                    1, name));
+            TEST_DO(assertDotProduct(0,  "(6:5,7:5)",             1, name));
+            TEST_DO(assertDotProduct(18, "(4:4.5)", 1, name));
+            TEST_DO(assertDotProduct(57, "(1:1,2:2,3:3,4:4.5,5:5)", 1, name));
         }
         for (const char * name : {"arrbyte", "arrint", "arrfloat", "arrint_fast", "arrfloat_fast"}) {
             assertDotProduct(0,  "()",                    1, name);
@@ -1237,10 +1263,12 @@ Test::testDotProduct()
         assertDotProduct(17, "(0:1,3:4,50:97)", 1, "sint", "arrfloat"); // attribute override
         assertDotProduct(0, "(0:1,3:4,50:97)", 1, "sint", "arrfloat_non_existing"); // incorrect attribute override
     }
-    verifyCorrectDotProductExecutor(_factory, "wsstr", "{a:1}", "search::features::dotproduct::wset::(anonymous namespace)::DotProductExecutorByEnum");
-    verifyCorrectDotProductExecutor(_factory, "wsstr", "{unknown:1}", "search::features::SingleZeroValueExecutor");
-    verifyCorrectDotProductExecutor(_factory, "wsint", "{1:1}", "search::features::dotproduct::wset::DotProductExecutor<search::MultiValueNumericAttribute<search::IntegerAttributeTemplate<int>, search::multivalue::WeightedValue<int> > >");
-    verifyCorrectDotProductExecutor(_factory, "wsint", "{}", "search::features::SingleZeroValueExecutor");
+    TEST_DO(verifyCorrectDotProductExecutor(_factory, "wsstr", "{a:1,b:2}", "search::features::dotproduct::wset::(anonymous namespace)::DotProductExecutorByEnum"));
+    TEST_DO(verifyCorrectDotProductExecutor(_factory, "wsstr", "{a:1}", "search::features::dotproduct::wset::(anonymous namespace)::SingleDotProductExecutorByEnum"));
+    TEST_DO(verifyCorrectDotProductExecutor(_factory, "wsstr", "{unknown:1}", "search::features::SingleZeroValueExecutor"));
+    TEST_DO(verifyCorrectDotProductExecutor(_factory, "wsint", "{1:1, 2:3}", "search::features::dotproduct::wset::DotProductExecutor<search::MultiValueNumericAttribute<search::IntegerAttributeTemplate<int>, search::multivalue::WeightedValue<int> > >"));
+    TEST_DO(verifyCorrectDotProductExecutor(_factory, "wsint", "{1:1}", "search::features::dotproduct::wset::(anonymous namespace)::SingleDotProductExecutorByValue<search::MultiValueNumericAttribute<search::IntegerAttributeTemplate<int>, search::multivalue::WeightedValue<int> > >"));
+    TEST_DO(verifyCorrectDotProductExecutor(_factory, "wsint", "{}", "search::features::SingleZeroValueExecutor"));
 
 }
 
@@ -1264,6 +1292,8 @@ void
 Test::setupForDotProductTest(FtFeatureTest & ft)
 {
     struct Config {
+        Config() : name(nullptr), dataType(AVBT::BOOL), collectionType(AVCT::SINGLE), fastSearch(false) {}
+        Config(const char *n, AVBT dt, AVCT ct, bool fs) : name(n), dataType(dt), collectionType(ct), fastSearch(fs) {}
         const char * name;
         AVBT dataType;
         AVCT collectionType;
@@ -1271,6 +1301,7 @@ Test::setupForDotProductTest(FtFeatureTest & ft)
     };
     std::vector<Config> cfgList = { {"wsint", AVBT::INT32, AVCT::WSET, false},
                                     {"wsbyte", AVBT::INT8, AVCT::WSET, false},
+                                    {"wsint_fast", AVBT::INT8, AVCT::WSET, true},
                                     {"arrbyte", AVBT::INT8, AVCT::ARRAY, false},
                                     {"arrint", AVBT::INT32, AVCT::ARRAY, false},
                                     {"arrfloat", AVBT::FLOAT, AVCT::ARRAY, false},
@@ -1296,11 +1327,11 @@ Test::setupForDotProductTest(FtFeatureTest & ft)
         baf->addDocs(2);
         ft.getIndexEnv().getAttributeMap().add(baf);
         for (size_t i(1); i < 6; i++) {
-            IntegerAttribute * ia = dynamic_cast<IntegerAttribute *>(baf.get());
+            auto ia = dynamic_cast<IntegerAttribute *>(baf.get());
             if (ia) {
                 ia->append(1, i, i);
             } else {
-                FloatingPointAttribute * fa = dynamic_cast<FloatingPointAttribute *>(baf.get());
+                auto fa = dynamic_cast<FloatingPointAttribute *>(baf.get());
                 fa->append(1, i, i);
             }
         }
@@ -1315,14 +1346,14 @@ Test::setupForDotProductTest(FtFeatureTest & ft)
     ft.getIndexEnv().getAttributeMap().add(c);
     ft.getIndexEnv().getAttributeMap().add(d);
 
-    StringAttribute * sa = static_cast<StringAttribute *>(a.get());
+    auto sa = dynamic_cast<StringAttribute *>(a.get());
     sa->append(1, "a", 1);
     sa->append(1, "b", 2);
     sa->append(1, "c", 3);
     sa->append(1, "d", 4);
     sa->append(1, "e", 5);
 
-    WeightedSetStringExtAttribute * ea = static_cast<WeightedSetStringExtAttribute *>(d.get());
+    auto ea = dynamic_cast<WeightedSetStringExtAttribute *>(d.get());
     EXPECT_TRUE(!ea->hasEnum());
     uint32_t docId;
     ea->addDoc(docId);  // reserved doc
@@ -1517,9 +1548,9 @@ Test::testMatchCount()
         ft.getIndexEnv().getBuilder().addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "foo");
         ft.getIndexEnv().getBuilder().addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "bar");
         ft.getIndexEnv().getBuilder().addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "baz");
-        ASSERT_TRUE(ft.getQueryEnv().getBuilder().addAttributeNode("foo") != NULL);   // query term 0, hit in foo
-        ASSERT_TRUE(ft.getQueryEnv().getBuilder().addAttributeNode("bar") != NULL);   // query term 1, hit in bar
-        ASSERT_TRUE(ft.getQueryEnv().getBuilder().addAttributeNode("foo") != NULL);   // query term 2, hit in foo
+        ASSERT_TRUE(ft.getQueryEnv().getBuilder().addAttributeNode("foo") != nullptr);   // query term 0, hit in foo
+        ASSERT_TRUE(ft.getQueryEnv().getBuilder().addAttributeNode("bar") != nullptr);   // query term 1, hit in bar
+        ASSERT_TRUE(ft.getQueryEnv().getBuilder().addAttributeNode("foo") != nullptr);   // query term 2, hit in foo
         ASSERT_TRUE(ft.setup());
 
         MatchDataBuilder::UP mdb = ft.createMatchDataBuilder();
@@ -1577,9 +1608,9 @@ Test::testMatches()
         ft.getIndexEnv().getBuilder().addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "foo");
         ft.getIndexEnv().getBuilder().addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "bar");
         ft.getIndexEnv().getBuilder().addField(FieldType::ATTRIBUTE, CollectionType::SINGLE, "baz");
-        ASSERT_TRUE(ft.getQueryEnv().getBuilder().addAttributeNode("foo") != NULL);   // query term 0, hit in foo
-        ASSERT_TRUE(ft.getQueryEnv().getBuilder().addAttributeNode("bar") != NULL);   // query term 1, hit in bar
-        ASSERT_TRUE(ft.getQueryEnv().getBuilder().addAttributeNode("foo") != NULL);   // query term 2, hit in foo
+        ASSERT_TRUE(ft.getQueryEnv().getBuilder().addAttributeNode("foo") != nullptr);   // query term 0, hit in foo
+        ASSERT_TRUE(ft.getQueryEnv().getBuilder().addAttributeNode("bar") != nullptr);   // query term 1, hit in bar
+        ASSERT_TRUE(ft.getQueryEnv().getBuilder().addAttributeNode("foo") != nullptr);   // query term 2, hit in foo
         ASSERT_TRUE(ft.setup());
 
         MatchDataBuilder::UP mdb = ft.createMatchDataBuilder();
@@ -1613,11 +1644,9 @@ Test::assertMatches(uint32_t output,
 
     ASSERT_TRUE(ft.execute(output, EPS, docId));
     // Execute and compare results.
-    if (!EXPECT_TRUE(ft.execute(output, EPS, docId))) return false;
-    return true;
+    return EXPECT_TRUE(ft.execute(output, EPS, docId));
 }
 
-
 void
 Test::testQuery()
 {
@@ -1721,7 +1750,7 @@ Test::testRandom()
         search::Rand48 rnd;
         rnd.srand48(100);
         for (uint32_t i = 0; i < 5; ++i) {
-            feature_t exp = rnd.lrand48() / (feature_t)0x80000000u;
+            feature_t exp = static_cast<feature_t>(rnd.lrand48()) / static_cast<feature_t>(0x80000000u);
             ASSERT_TRUE(ft.execute(exp, EPS, i + 1));
         }
     }
@@ -1744,7 +1773,7 @@ Test::testRandom()
         search::Rand48 rnd;
         for (uint32_t i = 1; i <= 5; ++i) {
             rnd.srand48(100 + i); // seed + lid
-            feature_t exp = rnd.lrand48() / (feature_t)0x80000000u;
+            feature_t exp = static_cast<feature_t>(rnd.lrand48()) / static_cast<feature_t>(0x80000000u);
             ASSERT_TRUE(ft.execute(exp, EPS, i));
         }
     }
@@ -2067,10 +2096,7 @@ Test::assertTermDistance(const TermDistanceCalculator::Result & exp,
     rr.addScore(feature + ".forwardTermPosition", exp.forwardTermPos);
     rr.addScore(feature + ".reverse",             exp.reverseDist);
     rr.addScore(feature + ".reverseTermPosition", exp.reverseTermPos);
-    if (!EXPECT_TRUE(ft.execute(rr, docId))) {
-        return false;
-    }
-    return true;
+    return EXPECT_TRUE(ft.execute(rr, docId));
 }
 
 void
diff --git a/searchlib/src/tests/features/prod_features.h b/searchlib/src/tests/features/prod_features.h
index 8e6578f34bd..6d30dd3fcd8 100644
--- a/searchlib/src/tests/features/prod_features.h
+++ b/searchlib/src/tests/features/prod_features.h
@@ -10,10 +10,10 @@ class Test : public FtTestApp
 {
 public:
     Test();
-    ~Test();
+    ~Test() override;
     int Main() override;
     void testFramework();
-    void testFtLib();
+    static void testFtLib();
     void testAge();
     void testAttribute();
     void testAttributeMatch();
@@ -39,10 +39,9 @@ public:
     void testRankingExpression();
     void testTerm();
     void testTermDistance();
-    void testUtils();
+    static void testUtils();
 
     static void setupForDotProductTest(FtFeatureTest & ft);
-    static void setupForDocumentTest(FtFeatureTest &ft, const vespalib::string & attrName, const vespalib::string & docType);
 
 private:
     void testFieldMatchBluePrint();
@@ -81,21 +80,21 @@ private:
     void testFieldMatchExecutorRemaining();
 
     void assertAge(feature_t expAge, const vespalib::string & attr, uint64_t now, uint64_t docTime);
-    void setupForAgeTest(FtFeatureTest & ft, uint64_t docTime);
-    void setupForAttributeTest(FtFeatureTest &ft, bool setup_env = true);
+    static void setupForAgeTest(FtFeatureTest & ft, uint64_t docTime);
+    static void setupForAttributeTest(FtFeatureTest &ft, bool setup_env = true);
     void assertCloseness(feature_t exp, const vespalib::string & attr, double distance, double maxDistance = 0, double halfResponse = 0);
-    void setupForDistanceTest(FtFeatureTest & ft, const vespalib::string & attrName,
-                              const std::vector<std::pair<int32_t, int32_t> > & positions, bool zcurve);
+    static void setupForDistanceTest(FtFeatureTest & ft, const vespalib::string & attrName,
+                                     const std::vector<std::pair<int32_t, int32_t> > & positions, bool zcurve);
     void assert2DZDistance(feature_t exp, const vespalib::string & positions,
                            int32_t xquery, int32_t yquery, uint32_t xAspect = 0);
-    void assertDistanceToPath(const std::vector<std::pair<int32_t, int32_t> > pos, const vespalib::string &path,
+    void assertDistanceToPath(const std::vector<std::pair<int32_t, int32_t> > & pos, const vespalib::string &path,
                               feature_t distance = search::features::DistanceToPathExecutor::DEFAULT_DISTANCE,
                               feature_t traveled = 1, feature_t product = 0);
     void assertDotProduct(feature_t exp, const vespalib::string & vector, uint32_t docId = 1,
                           const vespalib::string & attribute = "wsstr", const vespalib::string & attributeOverride="");
 
     void assertFieldMatch(const vespalib::string & spec, const vespalib::string & query, const vespalib::string & field,
-                          const search::features::fieldmatch::Params * params = NULL, uint32_t totalTermWeight = 0, feature_t totalSignificance = 0.0f);
+                          const search::features::fieldmatch::Params * params = nullptr, uint32_t totalTermWeight = 0, feature_t totalSignificance = 0.0f);
     void assertFieldMatch(const vespalib::string & spec, const vespalib::string & query, const vespalib::string & field,
                           uint32_t totalTermWeight);
     void assertFieldMatchTS(const vespalib::string & spec, const vespalib::string & query, const vespalib::string & field,
diff --git a/searchlib/src/tests/fef/fef_test.cpp b/searchlib/src/tests/fef/fef_test.cpp
index 896a917d6e3..4d1163de6ee 100644
--- a/searchlib/src/tests/fef/fef_test.cpp
+++ b/searchlib/src/tests/fef/fef_test.cpp
@@ -94,6 +94,7 @@ TEST("verify size of essential fef classes") {
     EXPECT_EQUAL(24u,sizeof(TermFieldMatchDataPosition));
     EXPECT_EQUAL(24u,sizeof(TermFieldMatchData::Features));
     EXPECT_EQUAL(40u,sizeof(TermFieldMatchData));
+    EXPECT_EQUAL(48u, sizeof(search::fef::FeatureExecutor));
 }
 
 TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/searchlib/src/tests/fef/object_passing/object_passing_test.cpp b/searchlib/src/tests/fef/object_passing/object_passing_test.cpp
index ce9585f11cb..793b2d40ed3 100644
--- a/searchlib/src/tests/fef/object_passing/object_passing_test.cpp
+++ b/searchlib/src/tests/fef/object_passing/object_passing_test.cpp
@@ -59,10 +59,13 @@ struct ProxyBlueprint : Blueprint {
     }
     bool setup(const IIndexEnvironment &, const std::vector<vespalib::string> &params) override {
         ASSERT_EQUAL(1u, params.size());
-        object_input = defineInput(params[0], accept_input);
-        describeOutput("value", "the value", object_output ? FeatureType::object(ValueType::double_type()) : FeatureType::number());
-        describeOutput("was_object", "whether input was object", FeatureType::number());
-        return true;
+        if (auto input = defineInput(params[0], accept_input)) {
+            object_input = input.value().is_object();
+            describeOutput("value", "the value", object_output ? FeatureType::object(ValueType::double_type()) : FeatureType::number());
+            describeOutput("was_object", "whether input was object", FeatureType::number());
+            return true;
+        }
+        return false;
     }
     FeatureExecutor &createExecutor(const IQueryEnvironment &, vespalib::Stash &stash) const override {
         return stash.create<ProxyExecutor>(object_input, object_output);
diff --git a/searchlib/src/tests/fef/resolver/resolver_test.cpp b/searchlib/src/tests/fef/resolver/resolver_test.cpp
index 955c072810f..72076f3999e 100644
--- a/searchlib/src/tests/fef/resolver/resolver_test.cpp
+++ b/searchlib/src/tests/fef/resolver/resolver_test.cpp
@@ -1,15 +1,14 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
-#include <vespa/vespalib/testkit/testapp.h>
+#include <vespa/vespalib/testkit/test_kit.h>
 #include <vespa/searchlib/fef/fef.h>
 #include <vespa/searchlib/fef/test/indexenvironment.h>
 #include <vespa/searchlib/features/valuefeature.h>
+#include <vespa/searchlib/features/rankingexpressionfeature.h>
 
-#include <vespa/log/log.h>
-LOG_SETUP("resolver_test");
-
-namespace search {
-namespace fef {
+using namespace search;
+using namespace search::fef;
+using search::features::RankingExpressionBlueprint;
 
 class BaseBlueprint : public Blueprint {
 public:
@@ -43,9 +42,9 @@ public:
     virtual bool setup(const IIndexEnvironment & indexEnv,
                        const ParameterList & params) override {
         (void) indexEnv; (void) params;
-        defineInput("base.foo");
-        defineInput("base.bar");
-        defineInput("base.baz");
+        ASSERT_TRUE(defineInput("base.foo"));
+        ASSERT_TRUE(defineInput("base.bar"));
+        ASSERT_TRUE(defineInput("base.baz"));
         describeOutput("out", "out");
         return true;
     }
@@ -54,48 +53,37 @@ public:
     }
 };
 
-class Test : public vespalib::TestApp {
-private:
-    BlueprintFactory _factory;
-    void requireThatWeGetUniqueBlueprints();
-public:
-    Test();
-    ~Test();
-    int Main() override;
+struct Fixture {
+    BlueprintFactory factory;    
+    Fixture() {
+        factory.addPrototype(Blueprint::SP(new BaseBlueprint()));
+        factory.addPrototype(Blueprint::SP(new CombineBlueprint()));
+        factory.addPrototype(std::make_shared<RankingExpressionBlueprint>());
+    }
 };
 
-Test::Test() :
-    _factory()
-{
-    _factory.addPrototype(Blueprint::SP(new BaseBlueprint()));
-    _factory.addPrototype(Blueprint::SP(new CombineBlueprint()));
-}
-Test::~Test() {}
-
-void
-Test::requireThatWeGetUniqueBlueprints()
-{
+TEST_F("requireThatWeGetUniqueBlueprints", Fixture()) {
     test::IndexEnvironment ienv;
-    BlueprintResolver::SP res(new BlueprintResolver(_factory, ienv));
+    BlueprintResolver::SP res(new BlueprintResolver(f.factory, ienv));
     res->addSeed("combine");
     EXPECT_TRUE(res->compile());
     const BlueprintResolver::ExecutorSpecList & spec = res->getExecutorSpecs();
-    EXPECT_EQUAL(2u, spec.size());
+    ASSERT_EQUAL(2u, spec.size());
     EXPECT_TRUE(dynamic_cast<BaseBlueprint *>(spec[0].blueprint.get()) != NULL);
     EXPECT_TRUE(dynamic_cast<CombineBlueprint *>(spec[1].blueprint.get()) != NULL);
 }
 
-int
-Test::Main()
-{
-    TEST_INIT("resolver_test");
-
-    requireThatWeGetUniqueBlueprints();
-
-    TEST_DONE();
-}
-
-}
+TEST_F("require_that_bad_input_is_handled", Fixture) {
+    test::IndexEnvironment ienv;
+    ienv.getProperties().add(indexproperties::eval::LazyExpressions::NAME, "false");
+    ienv.getProperties().add("rankingExpression(badinput).rankingScript", "base.foobad + base.bar");
+    BlueprintResolver::SP res(new BlueprintResolver(f.factory, ienv));
+    res->addSeed("rankingExpression(badinput)");
+    EXPECT_FALSE(res->compile());
+    const BlueprintResolver::ExecutorSpecList & spec = res->getExecutorSpecs();
+    ASSERT_EQUAL(2u, spec.size());
+    EXPECT_TRUE(dynamic_cast<BaseBlueprint *>(spec[0].blueprint.get()) != nullptr);
+    EXPECT_TRUE(dynamic_cast<RankingExpressionBlueprint *>(spec[1].blueprint.get()) != nullptr);
 }
 
-TEST_APPHOOK(search::fef::Test);
+TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/searchlib/src/tests/memoryindex/document_inverter/document_inverter_test.cpp b/searchlib/src/tests/memoryindex/document_inverter/document_inverter_test.cpp
index 3f798df3c05..58f800dec23 100644
--- a/searchlib/src/tests/memoryindex/document_inverter/document_inverter_test.cpp
+++ b/searchlib/src/tests/memoryindex/document_inverter/document_inverter_test.cpp
@@ -1,6 +1,5 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
-#include <vespa/searchlib/common/sequencedtaskexecutor.h>
 #include <vespa/searchlib/index/docbuilder.h>
 #include <vespa/searchlib/index/field_length_calculator.h>
 #include <vespa/searchlib/memoryindex/document_inverter.h>
@@ -9,6 +8,8 @@
 #include <vespa/searchlib/memoryindex/i_field_index_collection.h>
 #include <vespa/searchlib/memoryindex/word_store.h>
 #include <vespa/searchlib/test/memoryindex/ordered_field_index_inserter.h>
+#include <vespa/vespalib/util/sequencedtaskexecutor.h>
+
 #include <vespa/vespalib/gtest/gtest.h>
 
 namespace search {
@@ -18,6 +19,8 @@ using index::DocBuilder;
 using index::Schema;
 using index::schema::CollectionType;
 using index::schema::DataType;
+using vespalib::SequencedTaskExecutor;
+using vespalib::ISequencedTaskExecutor;
 
 using namespace index;
 
@@ -117,8 +120,8 @@ public:
 struct DocumentInverterTest : public ::testing::Test {
     Schema _schema;
     DocBuilder _b;
-    SequencedTaskExecutor _invertThreads;
-    SequencedTaskExecutor _pushThreads;
+    std::unique_ptr<ISequencedTaskExecutor> _invertThreads;
+    std::unique_ptr<ISequencedTaskExecutor> _pushThreads;
     WordStore                       _word_store;
     FieldIndexRemover               _remover;
     test::OrderedFieldIndexInserter _inserter;
@@ -138,26 +141,26 @@ struct DocumentInverterTest : public ::testing::Test {
     DocumentInverterTest()
         : _schema(makeSchema()),
           _b(_schema),
-          _invertThreads(2),
-          _pushThreads(2),
+          _invertThreads(SequencedTaskExecutor::create(2)),
+          _pushThreads(SequencedTaskExecutor::create(2)),
           _word_store(),
           _remover(_word_store),
           _inserter(),
           _calculator(),
           _fic(_remover, _inserter, _calculator),
-          _inv(_schema, _invertThreads, _pushThreads, _fic)
+          _inv(_schema, *_invertThreads, *_pushThreads, _fic)
     {
     }
 
     void pushDocuments() {
-        _invertThreads.sync();
+        _invertThreads->sync();
         uint32_t fieldId = 0;
         for (auto &inverter : _inv.getInverters()) {
             _inserter.setFieldId(fieldId);
             inverter->pushDocuments();
             ++fieldId;
         }
-        _pushThreads.sync();
+        _pushThreads->sync();
     }
 };
 
diff --git a/searchlib/src/tests/memoryindex/field_index/field_index_test.cpp b/searchlib/src/tests/memoryindex/field_index/field_index_test.cpp
index 512e1bd2051..ed6bd7168f2 100644
--- a/searchlib/src/tests/memoryindex/field_index/field_index_test.cpp
+++ b/searchlib/src/tests/memoryindex/field_index/field_index_test.cpp
@@ -1,6 +1,5 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
-#include <vespa/searchlib/common/sequencedtaskexecutor.h>
 #include <vespa/searchlib/diskindex/fusion.h>
 #include <vespa/searchlib/diskindex/indexbuilder.h>
 #include <vespa/searchlib/diskindex/zcposoccrandread.h>
@@ -19,6 +18,8 @@
 #include <vespa/searchlib/test/memoryindex/wrap_inserter.h>
 #include <vespa/vespalib/btree/btreenodeallocator.hpp>
 #include <vespa/vespalib/btree/btreeroot.hpp>
+#include <vespa/vespalib/util/sequencedtaskexecutor.h>
+
 #include <vespa/vespalib/gtest/gtest.h>
 
 #include <vespa/log/log.h>
@@ -38,6 +39,9 @@ using search::index::schema::CollectionType;
 using search::index::schema::DataType;
 using search::index::test::MockFieldLengthInspector;
 using vespalib::GenerationHandler;
+using vespalib::ISequencedTaskExecutor;
+using vespalib::SequencedTaskExecutor;
+
 
 namespace memoryindex {
 
@@ -517,7 +521,7 @@ struct FieldIndexTest : public ::testing::Test {
 };
 
 using FieldIndexTestTypes = ::testing::Types<FieldIndex<false>, FieldIndex<true>>;
-TYPED_TEST_CASE(FieldIndexTest, FieldIndexTestTypes);
+VESPA_GTEST_TYPED_TEST_SUITE(FieldIndexTest, FieldIndexTestTypes);
 
 // Disable warnings emitted by gtest generated files when using typed tests
 #pragma GCC diagnostic push
@@ -872,7 +876,7 @@ struct FieldIndexCollectionTypeTest : public ::testing::Test {
           fic(schema, MockFieldLengthInspector())
     {
     }
-    Schema make_schema() {
+    static Schema make_schema() {
         Schema result;
         result.addIndexField(Schema::IndexField("normal", DataType::STRING));
         Schema::IndexField interleaved("interleaved", DataType::STRING);
@@ -902,17 +906,17 @@ public:
     Schema _schema;
     FieldIndexCollection _fic;
     DocBuilder _b;
-    SequencedTaskExecutor _invertThreads;
-    SequencedTaskExecutor _pushThreads;
+    std::unique_ptr<ISequencedTaskExecutor> _invertThreads;
+    std::unique_ptr<ISequencedTaskExecutor> _pushThreads;
     DocumentInverter _inv;
 
     InverterTest(const Schema& schema)
         : _schema(schema),
           _fic(_schema, MockFieldLengthInspector()),
           _b(_schema),
-          _invertThreads(2),
-          _pushThreads(2),
-          _inv(_schema, _invertThreads, _pushThreads, _fic)
+          _invertThreads(SequencedTaskExecutor::create(2)),
+          _pushThreads(SequencedTaskExecutor::create(2)),
+          _inv(_schema, *_invertThreads, *_pushThreads, _fic)
     {
     }
     NormalFieldIndex::PostingList::Iterator find(const vespalib::stringref word, uint32_t field_id) const {
@@ -943,9 +947,9 @@ TEST_F(BasicInverterTest, require_that_inversion_is_working)
         endField();
     doc = _b.endDocument();
     _inv.invertDocument(10, *doc);
-    _invertThreads.sync();
+    _invertThreads->sync();
     myPushDocument(_inv);
-    _pushThreads.sync();
+    _pushThreads->sync();
 
     _b.startDocument("id:ns:searchdocument::20");
     _b.startIndexField("f0").
@@ -953,9 +957,9 @@ TEST_F(BasicInverterTest, require_that_inversion_is_working)
         endField();
     doc = _b.endDocument();
     _inv.invertDocument(20, *doc);
-    _invertThreads.sync();
+    _invertThreads->sync();
     myPushDocument(_inv);
-    _pushThreads.sync();
+    _pushThreads->sync();
 
     _b.startDocument("id:ns:searchdocument::30");
     _b.startIndexField("f0").
@@ -984,9 +988,9 @@ TEST_F(BasicInverterTest, require_that_inversion_is_working)
         endField();
     doc = _b.endDocument();
     _inv.invertDocument(30, *doc);
-    _invertThreads.sync();
+    _invertThreads->sync();
     myPushDocument(_inv);
-    _pushThreads.sync();
+    _pushThreads->sync();
 
     _b.startDocument("id:ns:searchdocument::40");
     _b.startIndexField("f0").
@@ -995,9 +999,9 @@ TEST_F(BasicInverterTest, require_that_inversion_is_working)
         endField();
     doc = _b.endDocument();
     _inv.invertDocument(40, *doc);
-    _invertThreads.sync();
+    _invertThreads->sync();
     myPushDocument(_inv);
-    _pushThreads.sync();
+    _pushThreads->sync();
 
     _b.startDocument("id:ns:searchdocument::999");
     _b.startIndexField("f0").
@@ -1025,12 +1029,12 @@ TEST_F(BasicInverterTest, require_that_inversion_is_working)
     doc = _b.endDocument();
     for (uint32_t docId = 10000; docId < 20000; ++docId) {
         _inv.invertDocument(docId, *doc);
-        _invertThreads.sync();
+        _invertThreads->sync();
         myPushDocument(_inv);
-        _pushThreads.sync();
+        _pushThreads->sync();
     }
 
-    _pushThreads.sync();
+    _pushThreads->sync();
     DataStoreBase::MemStats beforeStats = getFeatureStoreMemStats(_fic);
     LOG(info,
         "Before feature compaction: allocElems=%zu, usedElems=%zu"
@@ -1044,13 +1048,13 @@ TEST_F(BasicInverterTest, require_that_inversion_is_working)
         beforeStats._freeBuffers,
         beforeStats._activeBuffers,
         beforeStats._holdBuffers);
-    myCompactFeatures(_fic, _pushThreads);
+    myCompactFeatures(_fic, *_pushThreads);
     std::vector<std::unique_ptr<GenerationHandler::Guard>> guards;
     for (auto &fieldIndex : _fic.getFieldIndexes()) {
         guards.push_back(std::make_unique<GenerationHandler::Guard>
                          (fieldIndex->takeGenerationGuard()));
     }
-    myCommit(_fic, _pushThreads);
+    myCommit(_fic, *_pushThreads);
     DataStoreBase::MemStats duringStats = getFeatureStoreMemStats(_fic);
     LOG(info,
         "During feature compaction: allocElems=%zu, usedElems=%zu"
@@ -1065,7 +1069,7 @@ TEST_F(BasicInverterTest, require_that_inversion_is_working)
         duringStats._activeBuffers,
         duringStats._holdBuffers);
     guards.clear();
-    myCommit(_fic, _pushThreads);
+    myCommit(_fic, *_pushThreads);
     DataStoreBase::MemStats afterStats = getFeatureStoreMemStats(_fic);
     LOG(info,
         "After feature compaction: allocElems=%zu, usedElems=%zu"
@@ -1142,17 +1146,17 @@ TEST_F(BasicInverterTest, require_that_inverter_handles_remove_via_document_remo
     _b.startIndexField("f1").addStr("a").addStr("c").endField();
     Document::UP doc1 = _b.endDocument();
     _inv.invertDocument(1, *doc1.get());
-    _invertThreads.sync();
+    _invertThreads->sync();
     myPushDocument(_inv);
-    _pushThreads.sync();
+    _pushThreads->sync();
 
     _b.startDocument("id:ns:searchdocument::2");
     _b.startIndexField("f0").addStr("b").addStr("c").endField();
     Document::UP doc2 = _b.endDocument();
     _inv.invertDocument(2, *doc2.get());
-    _invertThreads.sync();
+    _invertThreads->sync();
     myPushDocument(_inv);
-    _pushThreads.sync();
+    _pushThreads->sync();
 
     EXPECT_TRUE(assertPostingList("[1]", find("a", 0)));
     EXPECT_TRUE(assertPostingList("[1,2]", find("b", 0)));
@@ -1160,8 +1164,8 @@ TEST_F(BasicInverterTest, require_that_inverter_handles_remove_via_document_remo
     EXPECT_TRUE(assertPostingList("[1]", find("a", 1)));
     EXPECT_TRUE(assertPostingList("[1]", find("c", 1)));
 
-    myremove(1, _inv, _invertThreads);
-    _pushThreads.sync();
+    myremove(1, _inv, *_invertThreads);
+    _pushThreads->sync();
 
     EXPECT_TRUE(assertPostingList("[]", find("a", 0)));
     EXPECT_TRUE(assertPostingList("[2]", find("b", 0)));
@@ -1311,10 +1315,10 @@ TEST_F(UriInverterTest, require_that_uri_indexing_is_working)
         endField();
     doc = _b.endDocument();
     _inv.invertDocument(10, *doc);
-    _invertThreads.sync();
+    _invertThreads->sync();
     myPushDocument(_inv);
 
-    _pushThreads.sync();
+    _pushThreads->sync();
 
     SimpleMatchData match_data;
     {
@@ -1387,10 +1391,10 @@ TEST_F(CjkInverterTest, require_that_cjk_indexing_is_working)
         endField();
     doc = _b.endDocument();
     _inv.invertDocument(10, *doc);
-    _invertThreads.sync();
+    _invertThreads->sync();
     myPushDocument(_inv);
 
-    _pushThreads.sync();
+    _pushThreads->sync();
 
     SimpleMatchData match_data;
     uint32_t fieldId = _schema.getIndexFieldId("f0");
@@ -1445,13 +1449,13 @@ TEST_F(FieldIndexCollectionTest, require_that_insert_tells_which_word_ref_that_w
 }
 
 struct RemoverTest : public FieldIndexCollectionTest {
-    SequencedTaskExecutor _invertThreads;
-    SequencedTaskExecutor _pushThreads;
+    std::unique_ptr<ISequencedTaskExecutor> _invertThreads;
+    std::unique_ptr<ISequencedTaskExecutor> _pushThreads;
 
     RemoverTest()
         : FieldIndexCollectionTest(),
-          _invertThreads(2),
-          _pushThreads(2)
+          _invertThreads(SequencedTaskExecutor::create(2)),
+          _pushThreads(SequencedTaskExecutor::create(2))
     {
     }
     void assertPostingLists(const vespalib::string &e1,
@@ -1462,9 +1466,9 @@ struct RemoverTest : public FieldIndexCollectionTest {
         EXPECT_TRUE(assertPostingList(e3, find("b", 1)));
     }
     void remove(uint32_t docId) {
-        DocumentInverter inv(schema, _invertThreads, _pushThreads, fic);
-        myremove(docId, inv, _invertThreads);
-        _pushThreads.sync();
+        DocumentInverter inv(schema, *_invertThreads, *_pushThreads, fic);
+        myremove(docId, inv, *_invertThreads);
+        _pushThreads->sync();
         EXPECT_FALSE(fic.getFieldIndex(0u)->getDocumentRemover().
                      getStore().get(docId).valid());
     }
diff --git a/searchlib/src/tests/memoryindex/memory_index/memory_index_test.cpp b/searchlib/src/tests/memoryindex/memory_index/memory_index_test.cpp
index 4bb0f91659a..6476ac7cf8a 100644
--- a/searchlib/src/tests/memoryindex/memory_index/memory_index_test.cpp
+++ b/searchlib/src/tests/memoryindex/memory_index/memory_index_test.cpp
@@ -1,7 +1,6 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include <vespa/searchlib/common/scheduletaskcallback.h>
-#include <vespa/searchlib/common/sequencedtaskexecutor.h>
 #include <vespa/searchlib/fef/matchdata.h>
 #include <vespa/searchlib/fef/matchdatalayout.h>
 #include <vespa/searchlib/fef/termfieldmatchdata.h>
@@ -14,9 +13,10 @@
 #include <vespa/searchlib/queryeval/fake_search.h>
 #include <vespa/searchlib/queryeval/fake_searchable.h>
 #include <vespa/searchlib/queryeval/searchiterator.h>
-#include <vespa/vespalib/gtest/gtest.h>
 #include <vespa/vespalib/util/stringfmt.h>
 #include <vespa/vespalib/util/threadstackexecutor.h>
+#include <vespa/vespalib/util/sequencedtaskexecutor.h>
+#include <vespa/vespalib/gtest/gtest.h>
 
 #include <vespa/log/log.h>
 LOG_SETUP("memory_index_test");
@@ -31,6 +31,8 @@ using vespalib::makeLambdaTask;
 using search::query::Node;
 using search::query::SimplePhrase;
 using search::query::SimpleStringTerm;
+using vespalib::ISequencedTaskExecutor;
+using vespalib::SequencedTaskExecutor;
 using namespace search::fef;
 using namespace search::index;
 using namespace search::memoryindex;
@@ -64,8 +66,8 @@ struct MySetup : public IFieldLengthInspector {
 struct Index {
     Schema       schema;
     vespalib::ThreadStackExecutor _executor;
-    search::SequencedTaskExecutor _invertThreads;
-    search::SequencedTaskExecutor _pushThreads;
+    std::unique_ptr<ISequencedTaskExecutor> _invertThreads;
+    std::unique_ptr<ISequencedTaskExecutor> _pushThreads;
     MemoryIndex  index;
     DocBuilder builder;
     uint32_t     docid;
@@ -123,15 +125,15 @@ private:
 Index::Index(const MySetup &setup)
     : schema(setup.schema),
       _executor(1, 128 * 1024),
-      _invertThreads(2),
-      _pushThreads(2),
-      index(schema, setup, _invertThreads, _pushThreads),
+      _invertThreads(SequencedTaskExecutor::create(2)),
+      _pushThreads(SequencedTaskExecutor::create(2)),
+      index(schema, setup, *_invertThreads, *_pushThreads),
       builder(schema),
       docid(1),
       currentField()
 {
 }
-Index::~Index() {}
+Index::~Index() = default;
 //-----------------------------------------------------------------------------
 
 std::string toString(SearchIterator & search)
diff --git a/searchlib/src/tests/predicate/document_features_store_test.cpp b/searchlib/src/tests/predicate/document_features_store_test.cpp
index d817b1668ff..72f83e533c9 100644
--- a/searchlib/src/tests/predicate/document_features_store_test.cpp
+++ b/searchlib/src/tests/predicate/document_features_store_test.cpp
@@ -208,9 +208,9 @@ TEST("require that serialization cleans up wordstore") {
     EXPECT_EQUAL(460u, features_store.getMemoryUsage().usedBytes());
     annotations.range_features.push_back({"bar", 100, 199});
     features_store.insert(annotations, doc_id + 1);
-    EXPECT_EQUAL(800u, features_store.getMemoryUsage().usedBytes());
+    EXPECT_EQUAL(848u, features_store.getMemoryUsage().usedBytes());
     features_store.remove(doc_id + 1);
-    EXPECT_EQUAL(752u, features_store.getMemoryUsage().usedBytes());
+    EXPECT_EQUAL(800u, features_store.getMemoryUsage().usedBytes());
 
     vespalib::DataBuffer buffer;
     features_store.serialize(buffer);
diff --git a/searchlib/src/tests/predicate/predicate_interval_store_test.cpp b/searchlib/src/tests/predicate/predicate_interval_store_test.cpp
index f90ed81cd7a..461bfd9ef30 100644
--- a/searchlib/src/tests/predicate/predicate_interval_store_test.cpp
+++ b/searchlib/src/tests/predicate/predicate_interval_store_test.cpp
@@ -135,7 +135,7 @@ TEST("require that interval refs are reused for identical data.") {
     PredicateIntervalStore store;
     auto ref = store.insert<Interval>({{0x00010001}, {0x0002ffff}});
     ASSERT_TRUE(ref.valid());
-    ASSERT_EQUAL(0x02000040u, ref.ref());
+    ASSERT_EQUAL(0x02000001u, ref.ref());
 
     auto ref2 = store.insert<Interval>({{0x00010001}, {0x0002ffff}});
     EXPECT_EQUAL(ref.ref(), ref2.ref());
diff --git a/searchlib/src/tests/query/query_visitor_test.cpp b/searchlib/src/tests/query/query_visitor_test.cpp
index 39e381c0942..edbc29be784 100644
--- a/searchlib/src/tests/query/query_visitor_test.cpp
+++ b/searchlib/src/tests/query/query_visitor_test.cpp
@@ -99,7 +99,7 @@ void Test::requireThatAllNodesCanBeVisited() {
     checkVisit<SuffixTerm>(new SimpleSuffixTerm("t", "field", 0, Weight(0)));
     checkVisit<PredicateQuery>(new SimplePredicateQuery(PredicateQueryTerm::UP(), "field", 0, Weight(0)));
     checkVisit<RegExpTerm>(new SimpleRegExpTerm("t", "field", 0, Weight(0)));
-    checkVisit<NearestNeighborTerm>(new SimpleNearestNeighborTerm("query_tensor", "doc_tensor", 0, Weight(0), 123));
+    checkVisit<NearestNeighborTerm>(new SimpleNearestNeighborTerm("query_tensor", "doc_tensor", 0, Weight(0), 123, true, 321));
 }
 
 }  // namespace
diff --git a/searchlib/src/tests/query/querybuilder_test.cpp b/searchlib/src/tests/query/querybuilder_test.cpp
index 7f496b3493c..8560cb0e091 100644
--- a/searchlib/src/tests/query/querybuilder_test.cpp
+++ b/searchlib/src/tests/query/querybuilder_test.cpp
@@ -111,7 +111,7 @@ Node::UP createQueryTree() {
             builder.addStringTerm(str[5], view[5], id[5], weight[6]);
             builder.addStringTerm(str[6], view[6], id[6], weight[7]);
         }
-        builder.add_nearest_neighbor_term("query_tensor", "doc_tensor", id[3], weight[5], 7);
+        builder.add_nearest_neighbor_term("query_tensor", "doc_tensor", id[3], weight[5], 7, true, 33);
     }
     Node::UP node = builder.build();
     ASSERT_TRUE(node.get());
@@ -395,8 +395,9 @@ struct MyRegExpTerm : RegExpTerm {
 };
 struct MyNearestNeighborTerm : NearestNeighborTerm {
     MyNearestNeighborTerm(vespalib::stringref query_tensor_name, vespalib::stringref field_name,
-                          int32_t i, Weight w, uint32_t target_num_hits)
-        : NearestNeighborTerm(query_tensor_name, field_name, i, w, target_num_hits)
+                          int32_t i, Weight w, uint32_t target_num_hits,
+                          bool allow_approximate, uint32_t explore_additional_hits)
+        : NearestNeighborTerm(query_tensor_name, field_name, i, w, target_num_hits, allow_approximate, explore_additional_hits)
     {}
 };
 
diff --git a/searchlib/src/tests/queryeval/multibitvectoriterator/multibitvectoriterator_test.cpp b/searchlib/src/tests/queryeval/multibitvectoriterator/multibitvectoriterator_test.cpp
index 28ba13bb4fc..12c38d3fe02 100644
--- a/searchlib/src/tests/queryeval/multibitvectoriterator/multibitvectoriterator_test.cpp
+++ b/searchlib/src/tests/queryeval/multibitvectoriterator/multibitvectoriterator_test.cpp
@@ -10,6 +10,7 @@
 #include <vespa/searchlib/fef/termfieldmatchdata.h>
 #include <vespa/searchlib/fef/termfieldmatchdataarray.h>
 #include <vespa/searchlib/test/searchiteratorverifier.h>
+#include <random>
 
 #include <vespa/log/log.h>
 LOG_SETUP("multibitvectoriterator_test");
@@ -52,6 +53,14 @@ private:
     BitVector * getBV(size_t index, bool inverted) {
         return inverted ? _bvs_inverted[index].get() : _bvs[index].get();
     }
+    void fixup_bitvectors() {
+        // Restore from inverted bitvectors after tampering
+        for (int i = 0; i < 3; ++i) {
+            if (_bvs_inverted[i]->testBit(1)) {
+                _bvs[i]->clearBit(1);
+            }
+        }
+    }
     std::vector< BitVector::UP > _bvs;
     std::vector< BitVector::UP > _bvs_inverted;
 };
@@ -61,12 +70,12 @@ Test::~Test() = default;
 
 void Test::setup()
 {
-    srand(7);
+    std::minstd_rand rnd(341);
     for(size_t i(0); i < 3; i++) {
         _bvs.push_back(BitVector::create(10000));
         BitVector & bv(*_bvs.back());
         for (size_t j(0); j < bv.size(); j++) {
-            int r = rand();
+            int r = rnd();
             if (r & 0x1) {
                 bv.setBit(j);
             }
@@ -132,7 +141,7 @@ Test::testAndWith(bool invert)
         s->initFullRange();
         H firstHits3 = seekNoReset(*s, 1, 130);
         H lastHits3 = seekNoReset(*s, 130, _bvs[0]->size());
-        //These constants will change if srand(7) is changed.
+        //These constants will change if rnd(341) is changed.
         EXPECT_EQUAL(30u, firstHits2.size());
         EXPECT_EQUAL(19u, firstHits3.size());
         EXPECT_EQUAL(1234u, lastHits2F.size());
@@ -208,6 +217,7 @@ Test::testBug7163266()
         EXPECT_TRUE(ms->needUnpack(i));
     }
     EXPECT_TRUE(ms->needUnpack(28)); // NB: force unpack all
+    fixup_bitvectors();
 }
 
 template<typename T>
@@ -240,6 +250,7 @@ Test::testThatOptimizePreservesUnpack()
     EXPECT_TRUE(ms != nullptr);
     EXPECT_EQUAL(2u, ms->getChildren().size());
     verifySelectiveUnpack(*s, tfmd);
+    fixup_bitvectors();
 }
 
 void
diff --git a/searchlib/src/tests/queryeval/nearest_neighbor/nearest_neighbor_test.cpp b/searchlib/src/tests/queryeval/nearest_neighbor/nearest_neighbor_test.cpp
index 7bc582ab442..5d933cb1285 100644
--- a/searchlib/src/tests/queryeval/nearest_neighbor/nearest_neighbor_test.cpp
+++ b/searchlib/src/tests/queryeval/nearest_neighbor/nearest_neighbor_test.cpp
@@ -11,19 +11,26 @@
 #include <vespa/searchlib/queryeval/nearest_neighbor_iterator.h>
 #include <vespa/searchlib/queryeval/simpleresult.h>
 #include <vespa/searchlib/tensor/dense_tensor_attribute.h>
+#include <vespa/searchlib/tensor/distance_function_factory.h>
 #include <vespa/vespalib/test/insertion_operators.h>
+#include <vespa/searchlib/queryeval/nns_index_iterator.h>
 
 #include <vespa/log/log.h>
 LOG_SETUP("nearest_neighbor_test");
 
+#define EPS 1.0e-6
+
 using search::feature_t;
 using search::tensor::DenseTensorAttribute;
 using search::AttributeVector;
 using vespalib::eval::ValueType;
+using CellType = vespalib::eval::ValueType::CellType;
 using vespalib::eval::TensorSpec;
 using vespalib::tensor::Tensor;
 using vespalib::tensor::DenseTensorView;
 using vespalib::tensor::DefaultTensorEngine;
+using search::tensor::DistanceFunction;
+using search::attribute::DistanceMetric;
 
 using namespace search::fef;
 using namespace search::queryeval;
@@ -31,6 +38,9 @@ using namespace search::queryeval;
 vespalib::string denseSpecDouble("tensor(x[2])");
 vespalib::string denseSpecFloat("tensor<float>(x[2])");
 
+DistanceFunction::UP euclid_d = search::tensor::make_distance_function(DistanceMetric::Euclidean, CellType::DOUBLE);
+DistanceFunction::UP euclid_f = search::tensor::make_distance_function(DistanceMetric::Euclidean, CellType::FLOAT);
+
 std::unique_ptr<DenseTensorView> createTensor(const TensorSpec &spec) {
     auto value = DefaultTensorEngine::ref().from_spec(spec);
     DenseTensorView *tensor = dynamic_cast<DenseTensorView*>(value.get());
@@ -93,6 +103,14 @@ struct Fixture
         auto t = createTensor(_typeSpec, v1, v2);
         setTensor(docId, *t);
     }
+
+    const DistanceFunction *dist_fun() const {
+        if (_cfg.tensorType().cell_type() == CellType::FLOAT) {
+            return euclid_f.get();
+        } else {
+            return euclid_d.get();
+        }
+    }
 };
 
 template <bool strict>
@@ -101,7 +119,7 @@ SimpleResult find_matches(Fixture &env, const DenseTensorView &qtv) {
     auto &tfmd = *(md->resolveTermField(0));
     auto &attr = *(env._tensorAttr);
     NearestNeighborDistanceHeap dh(2);
-    auto search = NearestNeighborIterator::create(strict, tfmd, qtv, attr, dh);
+    auto search = NearestNeighborIterator::create(strict, tfmd, qtv, attr, dh, env.dist_fun());
     if (strict) {
         return SimpleResult().searchStrict(*search, attr.getNumDocs());
     } else {
@@ -138,8 +156,6 @@ verify_iterator_returns_expected_results(const vespalib::string& attribute_tenso
 TEST("require that NearestNeighborIterator returns expected results") {
     TEST_DO(verify_iterator_returns_expected_results(denseSpecDouble, denseSpecDouble));
     TEST_DO(verify_iterator_returns_expected_results(denseSpecFloat, denseSpecFloat));
-    TEST_DO(verify_iterator_returns_expected_results(denseSpecDouble, denseSpecFloat));
-    TEST_DO(verify_iterator_returns_expected_results(denseSpecFloat, denseSpecDouble));
 }
 
 template <bool strict>
@@ -148,7 +164,7 @@ std::vector<feature_t> get_rawscores(Fixture &env, const DenseTensorView &qtv) {
     auto &tfmd = *(md->resolveTermField(0));
     auto &attr = *(env._tensorAttr);
     NearestNeighborDistanceHeap dh(2);
-    auto search = NearestNeighborIterator::create(strict, tfmd, qtv, attr, dh);
+    auto search = NearestNeighborIterator::create(strict, tfmd, qtv, attr, dh, env.dist_fun());
     uint32_t limit = attr.getNumDocs();
     uint32_t docid = 1;
     search->initRange(docid, limit);
@@ -178,16 +194,86 @@ verify_iterator_sets_expected_rawscore(const vespalib::string& attribute_tensor_
     auto nullTensor = createTensor(query_tensor_type_spec, 0.0, 0.0);
     std::vector<feature_t> got = get_rawscores<true>(fixture, *nullTensor);
     std::vector<feature_t> expected{5.0, 13.0, 10.0, 10.0, 5.0};
-    EXPECT_EQUAL(got, expected);
+    EXPECT_EQUAL(got.size(), expected.size());
+    for (size_t i = 0; i < expected.size(); ++i) {
+        EXPECT_APPROX(1.0/(1.0+expected[i]), got[i], EPS);
+    }
     got = get_rawscores<false>(fixture, *nullTensor);
-    EXPECT_EQUAL(got, expected);
+    EXPECT_EQUAL(got.size(), expected.size());
+    for (size_t i = 0; i < expected.size(); ++i) {
+        EXPECT_APPROX(1.0/(1.0+expected[i]), got[i], EPS);
+    }
 }
 
 TEST("require that NearestNeighborIterator sets expected rawscore") {
     TEST_DO(verify_iterator_sets_expected_rawscore(denseSpecDouble, denseSpecDouble));
     TEST_DO(verify_iterator_sets_expected_rawscore(denseSpecFloat, denseSpecFloat));
-    TEST_DO(verify_iterator_sets_expected_rawscore(denseSpecDouble, denseSpecFloat));
-    TEST_DO(verify_iterator_sets_expected_rawscore(denseSpecFloat, denseSpecDouble));
+}
+
+TEST("require that NnsIndexIterator works as expected") {
+    std::vector<NnsIndexIterator::Hit> hits{{2,4.0}, {3,9.0}, {5,1.0}, {8,16.0}, {9,36.0}};
+    auto md = MatchData::makeTestInstance(2, 2);
+    auto &tfmd = *(md->resolveTermField(0));
+    auto search = NnsIndexIterator::create(tfmd, hits, euclid_d.get());
+    uint32_t docid = 1;
+    search->initFullRange();
+    bool match = search->seek(docid);
+    EXPECT_FALSE(match);
+    EXPECT_FALSE(search->isAtEnd());
+    EXPECT_EQUAL(2u, search->getDocId());
+    docid = 2;
+    match = search->seek(docid);
+    EXPECT_TRUE(match);
+    EXPECT_FALSE(search->isAtEnd());
+    EXPECT_EQUAL(docid, search->getDocId());
+    search->unpack(docid);
+    EXPECT_APPROX(1.0/(1.0+2.0), tfmd.getRawScore(), EPS);
+
+    docid = 3;
+    match = search->seek(docid);
+    EXPECT_TRUE(match);
+    EXPECT_FALSE(search->isAtEnd());
+    EXPECT_EQUAL(docid, search->getDocId());
+    search->unpack(docid);
+    EXPECT_APPROX(1.0/(1.0+3.0), tfmd.getRawScore(), EPS);
+
+    docid = 4;
+    match = search->seek(docid);
+    EXPECT_FALSE(match);
+    EXPECT_FALSE(search->isAtEnd());
+    EXPECT_EQUAL(5u, search->getDocId());
+
+    docid = 6;
+    match = search->seek(docid);
+    EXPECT_FALSE(match);
+    EXPECT_FALSE(search->isAtEnd());
+    EXPECT_EQUAL(8u, search->getDocId());
+    docid = 8;
+    search->unpack(docid);
+    EXPECT_APPROX(1.0/(1.0+4.0), tfmd.getRawScore(), EPS);
+    docid = 9;
+    match = search->seek(docid);
+    EXPECT_TRUE(match);
+    EXPECT_FALSE(search->isAtEnd());
+    docid = 10;
+    match = search->seek(docid);
+    EXPECT_FALSE(match);
+    EXPECT_TRUE(search->isAtEnd());
+
+    docid = 4;
+    search->initRange(docid, 7);
+    match = search->seek(docid);
+    EXPECT_FALSE(match);
+    EXPECT_FALSE(search->isAtEnd());
+    EXPECT_EQUAL(5u, search->getDocId());
+    docid = 5;
+    search->unpack(docid);
+    EXPECT_APPROX(1.0/(1.0+1.0), tfmd.getRawScore(), EPS);
+    EXPECT_FALSE(search->isAtEnd());
+    docid = 6;
+    match = search->seek(docid);
+    EXPECT_FALSE(match);
+    EXPECT_TRUE(search->isAtEnd());
 }
 
 TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/searchlib/src/tests/queryeval/queryeval.cpp b/searchlib/src/tests/queryeval/queryeval.cpp
index 32dc9b1fa7e..5601baa9113 100644
--- a/searchlib/src/tests/queryeval/queryeval.cpp
+++ b/searchlib/src/tests/queryeval/queryeval.cpp
@@ -346,29 +346,12 @@ public:
         (void) tfmda;
         return _sc->createIterator(&_tfmd, strict);
     }
-    const search::AttributeVector::SearchContext & getSearchContext() const { return *_sc; }
 private:
     search::SingleBoolAttribute     _a;
     search::AttributeVector::SearchContext::UP _sc;
     mutable TermFieldMatchData _tfmd;
 };
 
-TEST("test bool attribute searchcontext") {
-    SimpleResult a;
-    a.addHit(5).addHit(17).addHit(30);
-    auto bp = std::make_unique<DummySingleValueBitNumericAttributeBlueprint>(a);
-    const search::AttributeVector::SearchContext & sc = bp->getSearchContext();
-    EXPECT_FALSE(sc.matches(7));
-    EXPECT_TRUE(sc.matches(17));
-    int32_t weight(0);
-    EXPECT_FALSE(sc.matches(7, weight));
-    EXPECT_EQUAL(0, weight);
-    EXPECT_TRUE(sc.matches(17, weight));
-    EXPECT_EQUAL(1, weight);
-    EXPECT_FALSE(sc.matches(27, weight));
-    EXPECT_EQUAL(0, weight);
-}
-
 
 TEST("testAndNot") {
     {
diff --git a/searchlib/src/tests/rankingexpression/intrinsic_blueprint_adapter/intrinsic_blueprint_adapter_test.cpp b/searchlib/src/tests/rankingexpression/intrinsic_blueprint_adapter/intrinsic_blueprint_adapter_test.cpp
index b10da86dd8c..9394cf085fc 100644
--- a/searchlib/src/tests/rankingexpression/intrinsic_blueprint_adapter/intrinsic_blueprint_adapter_test.cpp
+++ b/searchlib/src/tests/rankingexpression/intrinsic_blueprint_adapter/intrinsic_blueprint_adapter_test.cpp
@@ -5,6 +5,7 @@
 #include <vespa/searchlib/features/rankingexpression/intrinsic_blueprint_adapter.h>
 #include <vespa/searchlib/fef/test/indexenvironment.h>
 #include <vespa/searchlib/fef/test/queryenvironment.h>
+#include <vespa/vespalib/util/stash.h>
 #include <set>
 
 using namespace search::features::rankingexpression;
@@ -38,7 +39,7 @@ struct MyBlueprint : Blueprint {
         EXPECT_EQUAL(params[0], "foo");
         EXPECT_EQUAL(params[1], "bar");
         if (is_set(extra_input)) {
-            defineInput("my_input", AcceptInput::ANY);
+            EXPECT_TRUE(!defineInput("my_input", AcceptInput::ANY).has_value());
         }
         if (!is_set(no_output)) {
             if (is_set(error_result)) {
diff --git a/searchlib/src/tests/ranksetup/ranksetup_test.cpp b/searchlib/src/tests/ranksetup/ranksetup_test.cpp
index 7a26180eed2..0a20ddb3739 100644
--- a/searchlib/src/tests/ranksetup/ranksetup_test.cpp
+++ b/searchlib/src/tests/ranksetup/ranksetup_test.cpp
@@ -1,6 +1,7 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include <vespa/vespalib/testkit/testapp.h>
+#include <vespa/vespalib/util/stringfmt.h>
 #include <vespa/searchlib/common/feature.h>
 
 #include <vespa/searchlib/attribute/attributeguard.h>
@@ -39,6 +40,7 @@ using namespace search::fef;
 using namespace search::features;
 using namespace search::fef::test;
 using search::feature_t;
+using vespalib::make_string_short::fmt;
 
 typedef FeatureNameBuilder FNB;
 
@@ -477,12 +479,22 @@ RankSetupTest::testCompilation()
         rs.setFirstPhaseRank(oss.str());
         EXPECT_TRUE(!rs.compile());
     }
-    { // cycle
+    { // short cycle
         RankSetup rs(_factory, _indexEnv);
         // c(c,4,2) -> c(c,3,2) -> c(c,2,2) -> c(c,1,2) -> c(c,2,2)
         rs.setFirstPhaseRank("chain(cycle,4,2)");
         EXPECT_TRUE(!rs.compile());
     }
+    { // cycle with max back-trace
+        RankSetup rs(_factory, _indexEnv);
+        rs.setFirstPhaseRank(fmt("chain(cycle,%d,2)", BlueprintResolver::MAX_TRACE_SIZE));
+        EXPECT_TRUE(!rs.compile());
+    }
+    { // cycle with max+1 back-trace (skip 2)
+        RankSetup rs(_factory, _indexEnv);
+        rs.setFirstPhaseRank(fmt("chain(cycle,%d,2)", BlueprintResolver::MAX_TRACE_SIZE + 1));
+        EXPECT_TRUE(!rs.compile());
+    }
 }
 
 void RankSetupTest::testRankSetup()
diff --git a/searchlib/src/tests/ranksetup/verify_feature/verify_feature_test.cpp b/searchlib/src/tests/ranksetup/verify_feature/verify_feature_test.cpp
index 91bc5256b10..d8a821f1b3a 100644
--- a/searchlib/src/tests/ranksetup/verify_feature/verify_feature_test.cpp
+++ b/searchlib/src/tests/ranksetup/verify_feature/verify_feature_test.cpp
@@ -46,8 +46,8 @@ TEST_F("verify illegal feature name", RankFixture) {
 }
 
 TEST_F("verify too deep dependency graph", RankFixture) {
-    EXPECT_TRUE(f1.verify("chain(basic, 63, 4)"));
-    EXPECT_FALSE(f1.verify("chain(basic, 64, 4)"));
+    EXPECT_TRUE(f1.verify("chain(basic, 255, 4)"));
+    EXPECT_FALSE(f1.verify("chain(basic, 256, 4)"));
 }
 
 TEST_F("verify dependency cycle", RankFixture) {
diff --git a/searchlib/src/tests/tensor/distance_functions/CMakeLists.txt b/searchlib/src/tests/tensor/distance_functions/CMakeLists.txt
new file mode 100644
index 00000000000..9356658f90e
--- /dev/null
+++ b/searchlib/src/tests/tensor/distance_functions/CMakeLists.txt
@@ -0,0 +1,9 @@
+# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchlib_distance_functions_test_app TEST
+    SOURCES
+    distance_functions_test.cpp
+    DEPENDS
+    searchlib
+    gtest
+)
+vespa_add_test(NAME searchlib_distance_functions_test_app COMMAND searchlib_distance_functions_test_app)
diff --git a/searchlib/src/tests/tensor/distance_functions/distance_functions_test.cpp b/searchlib/src/tests/tensor/distance_functions/distance_functions_test.cpp
new file mode 100644
index 00000000000..59532919347
--- /dev/null
+++ b/searchlib/src/tests/tensor/distance_functions/distance_functions_test.cpp
@@ -0,0 +1,194 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/eval/tensor/dense/typed_cells.h>
+#include <vespa/searchlib/tensor/distance_functions.h>
+#include <vespa/searchlib/tensor/distance_function_factory.h>
+#include <vespa/vespalib/gtest/gtest.h>
+#include <vector>
+
+#include <vespa/log/log.h>
+LOG_SETUP("distance_function_test");
+
+using namespace search::tensor;
+using vespalib::tensor::TypedCells;
+using search::attribute::DistanceMetric;
+
+TypedCells t(const std::vector<double> &v) { return TypedCells(v); }
+
+void verify_geo_miles(const DistanceFunction *dist_fun,
+                      const std::vector<double> &p1,
+                      const std::vector<double> &p2,
+                      double exp_miles)
+{
+    TypedCells t1(p1);
+    TypedCells t2(p2);
+    double abstract_distance = dist_fun->calc(t1, t2);
+    double raw_score = dist_fun->to_rawscore(abstract_distance);
+    double m = ((1.0/raw_score)-1.0);
+    double d_miles = m / 1609.344;
+    EXPECT_GE(d_miles, exp_miles*0.99);
+    EXPECT_LE(d_miles, exp_miles*1.01);
+}
+
+
+TEST(DistanceFunctionsTest, gives_expected_score)
+{
+    auto ct = vespalib::eval::ValueType::CellType::DOUBLE;
+
+    auto euclid = make_distance_function(DistanceMetric::Euclidean, ct);
+    auto angular = make_distance_function(DistanceMetric::Angular, ct);
+
+    std::vector<double> p0{0.0, 0.0, 0.0};
+    std::vector<double> p1{1.0, 0.0, 0.0};
+    std::vector<double> p2{0.0, 1.0, 0.0};
+    std::vector<double> p3{0.0, 0.0, 1.0};
+    std::vector<double> p4{0.5, 0.5, 0.707107};
+    std::vector<double> p5{0.0,-1.0, 0.0};
+
+    double n4 = euclid->calc(t(p0), t(p4));
+    EXPECT_GT(n4, 0.99999);
+    EXPECT_LT(n4, 1.00001);
+    double d12 = euclid->calc(t(p1), t(p2));
+    EXPECT_EQ(d12, 2.0);
+
+    double a12 = angular->calc(t(p1), t(p2));
+    double a13 = angular->calc(t(p1), t(p3));
+    double a23 = angular->calc(t(p2), t(p3));
+    EXPECT_EQ(a12, 1.0);
+    EXPECT_EQ(a13, 1.0);
+    EXPECT_EQ(a23, 1.0);
+    double a14 = angular->calc(t(p1), t(p4));
+    double a24 = angular->calc(t(p2), t(p4));
+    EXPECT_EQ(a14, 0.5);
+    EXPECT_EQ(a24, 0.5);
+    double a34 = angular->calc(t(p3), t(p4));
+    EXPECT_GT(a34, 0.999999 - 0.707107);
+    EXPECT_LT(a34, 1.000001 - 0.707107);
+
+    double a25 = angular->calc(t(p2), t(p5));
+    EXPECT_EQ(a25, 2.0);
+
+    double a44 = angular->calc(t(p4), t(p4));
+    EXPECT_GE(a44, 0.0);
+    EXPECT_LT(a44, 0.000001);
+}
+
+TEST(GeoDegreesTest, gives_expected_score)
+{
+    auto ct = vespalib::eval::ValueType::CellType::DOUBLE;
+    auto geodeg = make_distance_function(DistanceMetric::GeoDegrees, ct);
+
+    std::vector<double> g1_sfo{37.61, -122.38};
+    std::vector<double> g2_lhr{51.47, -0.46};
+    std::vector<double> g3_osl{60.20, 11.08};
+    std::vector<double> g4_gig{-22.8, -43.25};
+    std::vector<double> g5_hkg{22.31, 113.91};
+    std::vector<double> g6_trd{63.45, 10.92};
+    std::vector<double> g7_syd{-33.95, 151.17};
+    std::vector<double> g8_lax{33.94, -118.41};
+    std::vector<double> g9_jfk{40.64, -73.78};
+
+    double g63_a = geodeg->calc(t(g6_trd), t(g3_osl));
+    double g63_r = geodeg->to_rawscore(g63_a);
+    double g63_km = ((1.0/g63_r)-1.0) *.001;
+    EXPECT_GT(g63_km, 350);
+    EXPECT_LT(g63_km, 375);
+
+    // all distances from gcmap.com, the
+    // Great Circle Mapper for airports using
+    // a more accurate formula - we should agree
+    // with < 1.0% deviation
+    verify_geo_miles(geodeg.get(), g1_sfo, g1_sfo, 0);
+    verify_geo_miles(geodeg.get(), g1_sfo, g2_lhr, 5367);
+    verify_geo_miles(geodeg.get(), g1_sfo, g3_osl, 5196);
+    verify_geo_miles(geodeg.get(), g1_sfo, g4_gig, 6604);
+    verify_geo_miles(geodeg.get(), g1_sfo, g5_hkg, 6927);
+    verify_geo_miles(geodeg.get(), g1_sfo, g6_trd, 5012);
+    verify_geo_miles(geodeg.get(), g1_sfo, g7_syd, 7417);
+    verify_geo_miles(geodeg.get(), g1_sfo, g8_lax, 337);
+    verify_geo_miles(geodeg.get(), g1_sfo, g9_jfk, 2586);
+
+    verify_geo_miles(geodeg.get(), g2_lhr, g1_sfo, 5367);
+    verify_geo_miles(geodeg.get(), g2_lhr, g2_lhr, 0);
+    verify_geo_miles(geodeg.get(), g2_lhr, g3_osl, 750);
+    verify_geo_miles(geodeg.get(), g2_lhr, g4_gig, 5734);
+    verify_geo_miles(geodeg.get(), g2_lhr, g5_hkg, 5994);
+    verify_geo_miles(geodeg.get(), g2_lhr, g6_trd, 928);
+    verify_geo_miles(geodeg.get(), g2_lhr, g7_syd, 10573);
+    verify_geo_miles(geodeg.get(), g2_lhr, g8_lax, 5456);
+    verify_geo_miles(geodeg.get(), g2_lhr, g9_jfk, 3451);
+
+    verify_geo_miles(geodeg.get(), g3_osl, g1_sfo, 5196);
+    verify_geo_miles(geodeg.get(), g3_osl, g2_lhr, 750);
+    verify_geo_miles(geodeg.get(), g3_osl, g3_osl, 0);
+    verify_geo_miles(geodeg.get(), g3_osl, g4_gig, 6479);
+    verify_geo_miles(geodeg.get(), g3_osl, g5_hkg, 5319);
+    verify_geo_miles(geodeg.get(), g3_osl, g6_trd, 226);
+    verify_geo_miles(geodeg.get(), g3_osl, g7_syd, 9888);
+    verify_geo_miles(geodeg.get(), g3_osl, g8_lax, 5345);
+    verify_geo_miles(geodeg.get(), g3_osl, g9_jfk, 3687);
+
+    verify_geo_miles(geodeg.get(), g4_gig, g1_sfo, 6604);
+    verify_geo_miles(geodeg.get(), g4_gig, g2_lhr, 5734);
+    verify_geo_miles(geodeg.get(), g4_gig, g3_osl, 6479);
+    verify_geo_miles(geodeg.get(), g4_gig, g4_gig, 0);
+    verify_geo_miles(geodeg.get(), g4_gig, g5_hkg, 10989);
+    verify_geo_miles(geodeg.get(), g4_gig, g6_trd, 6623);
+    verify_geo_miles(geodeg.get(), g4_gig, g7_syd, 8414);
+    verify_geo_miles(geodeg.get(), g4_gig, g8_lax, 6294);
+    verify_geo_miles(geodeg.get(), g4_gig, g9_jfk, 4786);
+
+    verify_geo_miles(geodeg.get(), g5_hkg, g1_sfo, 6927);
+    verify_geo_miles(geodeg.get(), g5_hkg, g2_lhr, 5994);
+    verify_geo_miles(geodeg.get(), g5_hkg, g3_osl, 5319);
+    verify_geo_miles(geodeg.get(), g5_hkg, g4_gig, 10989);
+    verify_geo_miles(geodeg.get(), g5_hkg, g5_hkg, 0);
+    verify_geo_miles(geodeg.get(), g5_hkg, g6_trd, 5240);
+    verify_geo_miles(geodeg.get(), g5_hkg, g7_syd, 4581);
+    verify_geo_miles(geodeg.get(), g5_hkg, g8_lax, 7260);
+    verify_geo_miles(geodeg.get(), g5_hkg, g9_jfk, 8072);
+
+    verify_geo_miles(geodeg.get(), g6_trd, g1_sfo, 5012);
+    verify_geo_miles(geodeg.get(), g6_trd, g2_lhr, 928);
+    verify_geo_miles(geodeg.get(), g6_trd, g3_osl, 226);
+    verify_geo_miles(geodeg.get(), g6_trd, g4_gig, 6623);
+    verify_geo_miles(geodeg.get(), g6_trd, g5_hkg, 5240);
+    verify_geo_miles(geodeg.get(), g6_trd, g6_trd, 0);
+    verify_geo_miles(geodeg.get(), g6_trd, g7_syd, 9782);
+    verify_geo_miles(geodeg.get(), g6_trd, g8_lax, 5171);
+    verify_geo_miles(geodeg.get(), g6_trd, g9_jfk, 3611);
+
+    verify_geo_miles(geodeg.get(), g7_syd, g1_sfo, 7417);
+    verify_geo_miles(geodeg.get(), g7_syd, g2_lhr, 10573);
+    verify_geo_miles(geodeg.get(), g7_syd, g3_osl, 9888);
+    verify_geo_miles(geodeg.get(), g7_syd, g4_gig, 8414);
+    verify_geo_miles(geodeg.get(), g7_syd, g5_hkg, 4581);
+    verify_geo_miles(geodeg.get(), g7_syd, g6_trd, 9782);
+    verify_geo_miles(geodeg.get(), g7_syd, g7_syd, 0);
+    verify_geo_miles(geodeg.get(), g7_syd, g8_lax, 7488);
+    verify_geo_miles(geodeg.get(), g7_syd, g9_jfk, 9950);
+
+    verify_geo_miles(geodeg.get(), g8_lax, g1_sfo, 337);
+    verify_geo_miles(geodeg.get(), g8_lax, g2_lhr, 5456);
+    verify_geo_miles(geodeg.get(), g8_lax, g3_osl, 5345);
+    verify_geo_miles(geodeg.get(), g8_lax, g4_gig, 6294);
+    verify_geo_miles(geodeg.get(), g8_lax, g5_hkg, 7260);
+    verify_geo_miles(geodeg.get(), g8_lax, g6_trd, 5171);
+    verify_geo_miles(geodeg.get(), g8_lax, g7_syd, 7488);
+    verify_geo_miles(geodeg.get(), g8_lax, g8_lax, 0);
+    verify_geo_miles(geodeg.get(), g8_lax, g9_jfk, 2475);
+
+    verify_geo_miles(geodeg.get(), g9_jfk, g1_sfo, 2586);
+    verify_geo_miles(geodeg.get(), g9_jfk, g2_lhr, 3451);
+    verify_geo_miles(geodeg.get(), g9_jfk, g3_osl, 3687);
+    verify_geo_miles(geodeg.get(), g9_jfk, g4_gig, 4786);
+    verify_geo_miles(geodeg.get(), g9_jfk, g5_hkg, 8072);
+    verify_geo_miles(geodeg.get(), g9_jfk, g6_trd, 3611);
+    verify_geo_miles(geodeg.get(), g9_jfk, g7_syd, 9950);
+    verify_geo_miles(geodeg.get(), g9_jfk, g8_lax, 2475);
+    verify_geo_miles(geodeg.get(), g9_jfk, g9_jfk, 0);
+
+}
+
+GTEST_MAIN_RUN_ALL_TESTS()
+
diff --git a/searchlib/src/tests/tensor/hnsw_index/CMakeLists.txt b/searchlib/src/tests/tensor/hnsw_index/CMakeLists.txt
new file mode 100644
index 00000000000..04c7312a63f
--- /dev/null
+++ b/searchlib/src/tests/tensor/hnsw_index/CMakeLists.txt
@@ -0,0 +1,9 @@
+# Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchlib_hnsw_index_test_app TEST
+    SOURCES
+    hnsw_index_test.cpp
+    DEPENDS
+    searchlib
+    gtest
+)
+vespa_add_test(NAME searchlib_hnsw_index_test_app COMMAND searchlib_hnsw_index_test_app)
diff --git a/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp b/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp
new file mode 100644
index 00000000000..7d38be7db4a
--- /dev/null
+++ b/searchlib/src/tests/tensor/hnsw_index/hnsw_index_test.cpp
@@ -0,0 +1,531 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/eval/tensor/dense/typed_cells.h>
+#include <vespa/searchlib/tensor/distance_functions.h>
+#include <vespa/searchlib/tensor/doc_vector_access.h>
+#include <vespa/searchlib/tensor/hnsw_index.h>
+#include <vespa/searchlib/tensor/random_level_generator.h>
+#include <vespa/searchlib/tensor/inv_log_level_generator.h>
+#include <vespa/vespalib/gtest/gtest.h>
+#include <vespa/vespalib/util/generationhandler.h>
+#include <vespa/vespalib/data/slime/slime.h>
+#include <vector>
+
+#include <vespa/log/log.h>
+LOG_SETUP("hnsw_index_test");
+
+using vespalib::GenerationHandler;
+using vespalib::MemoryUsage;
+using namespace search::tensor;
+using namespace vespalib::slime;
+using vespalib::Slime;
+
+
+template <typename FloatType>
+class MyDocVectorAccess : public DocVectorAccess {
+private:
+    using Vector = std::vector<FloatType>;
+    using ArrayRef = vespalib::ConstArrayRef<FloatType>;
+    std::vector<Vector> _vectors;
+
+public:
+    MyDocVectorAccess() : _vectors() {}
+    MyDocVectorAccess& set(uint32_t docid, const Vector& vec) {
+        if (docid >= _vectors.size()) {
+            _vectors.resize(docid + 1);
+        }
+        _vectors[docid] = vec;
+        return *this;
+    }
+    vespalib::tensor::TypedCells get_vector(uint32_t docid) const override {
+        ArrayRef ref(_vectors[docid]);
+        return vespalib::tensor::TypedCells(ref);
+    }
+};
+
+struct LevelGenerator : public RandomLevelGenerator {
+    uint32_t level;
+    LevelGenerator() : level(0) {}
+    uint32_t max_level() override { return level; }
+};
+
+using FloatVectors = MyDocVectorAccess<float>;
+using FloatSqEuclideanDistance = SquaredEuclideanDistance<float>;
+using HnswIndexUP = std::unique_ptr<HnswIndex>;
+
+class HnswIndexTest : public ::testing::Test {
+public:
+    FloatVectors vectors;
+    LevelGenerator* level_generator;
+    GenerationHandler gen_handler;
+    HnswIndexUP index;
+
+    HnswIndexTest()
+        : vectors(),
+          level_generator(),
+          gen_handler(),
+          index()
+    {
+        vectors.set(1, {2, 2}).set(2, {3, 2}).set(3, {2, 3})
+               .set(4, {1, 2}).set(5, {8, 3}).set(6, {7, 2})
+               .set(7, {3, 5}).set(8, {0, 3}).set(9, {4, 5});
+    }
+
+    ~HnswIndexTest() {}
+
+    void init(bool heuristic_select_neighbors) {
+        auto generator = std::make_unique<LevelGenerator>();
+        level_generator = generator.get();
+        index = std::make_unique<HnswIndex>(vectors, std::make_unique<FloatSqEuclideanDistance>(),
+                                            std::move(generator),
+                                            HnswIndex::Config(5, 2, 10, heuristic_select_neighbors));
+    }
+    void add_document(uint32_t docid, uint32_t max_level = 0) {
+        level_generator->level = max_level;
+        index->add_document(docid);
+        commit();
+    }
+    void remove_document(uint32_t docid) {
+        index->remove_document(docid);
+        commit();
+    }
+    void commit() {
+        index->transfer_hold_lists(gen_handler.getCurrentGeneration());
+        gen_handler.incGeneration();
+        gen_handler.updateFirstUsedGeneration();
+        index->trim_hold_lists(gen_handler.getFirstUsedGeneration());
+    }
+    GenerationHandler::Guard take_read_guard() {
+        return gen_handler.takeGuard();
+    }
+    MemoryUsage memory_usage() const {
+        return index->memory_usage();
+    }
+    void expect_entry_point(uint32_t exp_docid, uint32_t exp_level) {
+        EXPECT_EQ(exp_docid, index->get_entry_docid());
+        EXPECT_EQ(exp_level, index->get_entry_level());
+    }
+    void expect_level_0(uint32_t docid, const HnswNode::LinkArray& exp_links) {
+        auto node = index->get_node(docid);
+        ASSERT_EQ(1, node.size());
+        EXPECT_EQ(exp_links, node.level(0));
+    }
+    void expect_empty_level_0(uint32_t docid) {
+        auto node = index->get_node(docid);
+        EXPECT_TRUE(node.empty());
+    }
+    void expect_levels(uint32_t docid, const HnswNode::LevelArray& exp_levels) {
+        auto act_node = index->get_node(docid);
+        ASSERT_EQ(exp_levels.size(), act_node.size());
+        EXPECT_EQ(exp_levels, act_node.levels());
+    }
+    void expect_top_3(uint32_t docid, std::vector<uint32_t> exp_hits) {
+        uint32_t k = 3;
+        auto qv = vectors.get_vector(docid);
+        auto rv = index->top_k_candidates(qv, k).peek();
+        std::sort(rv.begin(), rv.end(), LesserDistance());
+        size_t idx = 0;
+        for (const auto & hit : rv) {
+            if (idx < exp_hits.size()) {
+                EXPECT_EQ(hit.docid, exp_hits[idx++]);
+            }
+        }
+        if (exp_hits.size() == k) {
+            std::vector<uint32_t> expected_by_docid = exp_hits;
+            std::sort(expected_by_docid.begin(), expected_by_docid.end());
+            auto got_by_docid = index->find_top_k(k, qv, k);
+            for (idx = 0; idx < k; ++idx) {
+                EXPECT_EQ(expected_by_docid[idx], got_by_docid[idx].docid);
+            }
+        }
+    }
+};
+
+
+TEST_F(HnswIndexTest, 2d_vectors_inserted_in_level_0_graph_with_simple_select_neighbors)
+{
+    init(false);
+
+    add_document(1);
+    expect_level_0(1, {});
+
+    add_document(2);
+    expect_level_0(1, {2});
+    expect_level_0(2, {1});
+
+    add_document(3);
+    expect_level_0(1, {2, 3});
+    expect_level_0(2, {1, 3});
+    expect_level_0(3, {1, 2});
+
+    add_document(4);
+    expect_level_0(1, {2, 3, 4});
+    expect_level_0(2, {1, 3});
+    expect_level_0(3, {1, 2, 4});
+    expect_level_0(4, {1, 3});
+
+    add_document(5);
+    expect_level_0(1, {2, 3, 4});
+    expect_level_0(2, {1, 3, 5});
+    expect_level_0(3, {1, 2, 4, 5});
+    expect_level_0(4, {1, 3});
+    expect_level_0(5, {2, 3});
+
+    add_document(6);
+    expect_level_0(1, {2, 3, 4});
+    expect_level_0(2, {1, 3, 5, 6});
+    expect_level_0(3, {1, 2, 4, 5});
+    expect_level_0(4, {1, 3});
+    expect_level_0(5, {2, 3, 6});
+    expect_level_0(6, {2, 5});
+
+    add_document(7);
+    expect_level_0(1, {2, 3, 4});
+    expect_level_0(2, {1, 3, 5, 6, 7});
+    expect_level_0(3, {1, 2, 4, 5, 7});
+    expect_level_0(4, {1, 3});
+    expect_level_0(5, {2, 3, 6});
+    expect_level_0(6, {2, 5});
+    expect_level_0(7, {2, 3});
+
+    expect_top_3(1, {1});
+    expect_top_3(2, {2, 1, 3});
+    expect_top_3(3, {3});
+    expect_top_3(4, {4, 1, 3});
+    expect_top_3(5, {5, 6, 2});
+    expect_top_3(6, {6, 5, 2});
+    expect_top_3(7, {7, 3, 2});
+    expect_top_3(8, {4, 3, 1});
+    expect_top_3(9, {7, 3, 2});
+}
+
+TEST_F(HnswIndexTest, 2d_vectors_inserted_and_removed)
+{
+    init(false);
+
+    add_document(1);
+    expect_level_0(1, {});
+    expect_entry_point(1, 0);
+
+    add_document(2);
+    expect_level_0(1, {2});
+    expect_level_0(2, {1});
+    expect_entry_point(1, 0);
+
+    add_document(3);
+    expect_level_0(1, {2, 3});
+    expect_level_0(2, {1, 3});
+    expect_level_0(3, {1, 2});
+    expect_entry_point(1, 0);
+
+    remove_document(2);
+    expect_level_0(1, {3});
+    expect_level_0(3, {1});
+    expect_entry_point(1, 0);
+
+    remove_document(1);
+    expect_level_0(3, {});
+    expect_entry_point(3, 0);
+
+    remove_document(3);
+    expect_entry_point(0, -1);
+}
+
+TEST_F(HnswIndexTest, 2d_vectors_inserted_in_hierarchic_graph_with_heuristic_select_neighbors)
+{
+    init(true);
+
+    add_document(1);
+    expect_entry_point(1, 0);
+    expect_level_0(1, {});
+
+    add_document(2);
+    expect_entry_point(1, 0);
+    expect_level_0(1, {2});
+    expect_level_0(2, {1});
+
+    // Doc 3 is also added to level 1
+    add_document(3, 1);
+    expect_entry_point(3, 1);
+    // Doc 3 is closest to 1 and they are linked.
+    // Doc 3 is NOT linked to 2, since that is closer to 1 also.
+    expect_level_0(1, {2, 3});
+    expect_level_0(2, {1});
+    expect_levels(3, {{1}, {}});
+
+    // Doc 4 is closest to 1 and they are linked.
+    // Doc 4 is NOT linked to 3 as the distance between 4 and 3 is greater than the distance between 3 and 1.
+    // Doc 3 is therefore reachable via 1. Same argument for why doc 4 is not linked to 2.
+    add_document(4);
+    expect_entry_point(3, 1);
+    expect_level_0(1, {2, 3, 4});
+    expect_level_0(2, {1});
+    expect_levels(3, {{1}, {}});
+    expect_level_0(4, {1});
+
+    // Doc 5 is closest to 2 and they are linked.
+    // The other docs are reachable via 2, and no other links are created. Same argument as with doc 4 above.
+    add_document(5);
+    expect_entry_point(3, 1);
+    expect_level_0(1, {2, 3, 4});
+    expect_level_0(2, {1, 5});
+    expect_levels(3, {{1}, {}});
+    expect_level_0(4, {1});
+    expect_level_0(5, {2});
+
+    // Doc 6 is closest to 5 and they are linked.
+    // Doc 6 is also linked to 2 as the distance between 6 and 2 is less than the distance between 2 and 5.
+    // Doc 6 is also added to level 1 and 2, and linked to doc 3 in level 1.
+    add_document(6, 2);
+    expect_entry_point(6, 2);
+    expect_level_0(1, {2, 3, 4});
+    expect_level_0(2, {1, 5, 6});
+    expect_levels(3, {{1}, {6}});
+    expect_level_0(4, {1});
+    expect_level_0(5, {2, 6});
+    expect_levels(6, {{2, 5}, {3}, {}});
+
+    // Doc 7 is closest to 3 and they are linked.
+    // Doc 7 is also linked to 6 as the distance between 7 and 6 is less than the distance between 6 and 3.
+    // Docs 1, 2, 4 are reachable via 3.
+    add_document(7);
+    expect_entry_point(6, 2);
+    expect_level_0(1, {2, 3, 4});
+    expect_level_0(2, {1, 5, 6});
+    expect_levels(3, {{1, 7}, {6}});
+    expect_level_0(4, {1});
+    expect_level_0(5, {2, 6});
+    expect_levels(6, {{2, 5, 7}, {3}, {}});
+    expect_level_0(7, {3, 6});
+    {
+        Slime actualSlime;
+        SlimeInserter inserter(actualSlime);
+        index->get_state(inserter);
+        const auto &root = actualSlime.get();
+        EXPECT_EQ(0, root["memory_usage"]["onHold"].asLong());
+        EXPECT_EQ(8, root["nodes"].asLong());
+        EXPECT_EQ(7, root["valid_nodes"].asLong());
+        EXPECT_EQ(0, root["level_histogram"][0].asLong());
+        EXPECT_EQ(5, root["level_histogram"][1].asLong());
+        EXPECT_EQ(0, root["level_0_links_histogram"][0].asLong());
+        EXPECT_EQ(1, root["level_0_links_histogram"][1].asLong());
+        EXPECT_EQ(3, root["level_0_links_histogram"][2].asLong());
+        EXPECT_EQ(3, root["level_0_links_histogram"][3].asLong());
+        EXPECT_EQ(0, root["level_0_links_histogram"][4].asLong());
+        EXPECT_EQ(0, root["unreachable_nodes"].asLong());
+    }
+
+    // removing 1, its neighbors {2,3,4} will try to
+    // link together, but since 2 already has enough links
+    // only 3 and 4 will become neighbors:
+    remove_document(1);
+    expect_entry_point(6, 2);
+    expect_level_0(2, {5, 6});
+    expect_levels(3, {{4, 7}, {6}});
+    expect_level_0(4, {3});
+    expect_level_0(5, {2, 6});
+    expect_levels(6, {{2, 5, 7}, {3}, {}});
+    expect_level_0(7, {3, 6});
+    {
+        Slime actualSlime;
+        SlimeInserter inserter(actualSlime);
+        index->get_state(inserter);
+        const auto &root = actualSlime.get();
+        EXPECT_EQ(0, root["memory_usage"]["onHold"].asLong());
+        EXPECT_EQ(8, root["nodes"].asLong());
+        EXPECT_EQ(6, root["valid_nodes"].asLong());
+        EXPECT_EQ(0, root["level_histogram"][0].asLong());
+        EXPECT_EQ(4, root["level_histogram"][1].asLong());
+        EXPECT_EQ(0, root["level_0_links_histogram"][0].asLong());
+        EXPECT_EQ(1, root["level_0_links_histogram"][1].asLong());
+        EXPECT_EQ(4, root["level_0_links_histogram"][2].asLong());
+        EXPECT_EQ(1, root["level_0_links_histogram"][3].asLong());
+        EXPECT_EQ(0, root["level_0_links_histogram"][4].asLong());
+        EXPECT_EQ(0, root["unreachable_nodes"].asLong());
+    }
+}
+
+TEST_F(HnswIndexTest, manual_insert)
+{
+    init(false);
+
+    std::vector<uint32_t> nbl;
+    HnswNode empty{nbl};
+    index->set_node(1, empty);
+    index->set_node(2, empty);
+
+    HnswNode three{{1,2}};
+    index->set_node(3, three);
+    expect_level_0(1, {3});
+    expect_level_0(2, {3});
+    expect_level_0(3, {1,2});
+
+    expect_entry_point(1, 0);
+
+    HnswNode twolevels{{{1},nbl}};
+    index->set_node(4, twolevels);
+
+    expect_entry_point(4, 1);
+    expect_level_0(1, {3,4});
+
+    HnswNode five{{{1,2}, {4}}};
+    index->set_node(5, five);
+
+    expect_levels(1, {{3,4,5}});
+    expect_levels(2, {{3,5}});
+    expect_levels(3, {{1,2}});
+    expect_levels(4, {{1}, {5}});
+    expect_levels(5, {{1,2}, {4}});
+}
+
+TEST_F(HnswIndexTest, memory_is_reclaimed_when_doing_changes_to_graph)
+{
+    init(false);
+
+    add_document(1);
+    add_document(3);
+    auto mem_1 = memory_usage();
+
+    add_document(2);
+    expect_level_0(1, {2,3});
+    expect_level_0(2, {1,3});
+    expect_level_0(3, {1,2});
+    auto mem_2 = memory_usage();
+    // We should use more memory with larger link arrays and extra document.
+    EXPECT_GT((mem_2.usedBytes() - mem_2.deadBytes()), (mem_1.usedBytes() - mem_1.deadBytes()));
+    EXPECT_EQ(0, mem_2.allocatedBytesOnHold());
+
+    remove_document(2);
+    expect_level_0(1, {3});
+    expect_empty_level_0(2);
+    expect_level_0(3, {1});
+    auto mem_3 = memory_usage();
+    // We end up in the same state as before document 2 was added and effectively use the same amount of memory.
+    EXPECT_EQ((mem_1.usedBytes() - mem_1.deadBytes()), (mem_3.usedBytes() - mem_3.deadBytes()));
+    EXPECT_EQ(0, mem_3.allocatedBytesOnHold());
+}
+
+TEST_F(HnswIndexTest, memory_is_put_on_hold_while_read_guard_is_held)
+{
+    init(true);
+
+    add_document(1);
+    add_document(3);
+    {
+        auto guard = take_read_guard();
+        add_document(2);
+        auto mem = memory_usage();
+        // As read guard is held memory to reclaim is put on hold
+        EXPECT_GT(mem.allocatedBytesOnHold(), 0);
+    }
+    commit();
+    auto mem = memory_usage();
+    EXPECT_EQ(0, mem.allocatedBytesOnHold());
+}
+
+TEST_F(HnswIndexTest, shrink_called_simple)
+{
+    init(false);
+    std::vector<uint32_t> nbl;
+    HnswNode empty{nbl};
+    index->set_node(1, empty);
+    nbl.push_back(1);
+    HnswNode nb1{nbl};
+    index->set_node(2, nb1);
+    index->set_node(3, nb1);
+    index->set_node(4, nb1);
+    index->set_node(5, nb1);
+    expect_level_0(1, {2,3,4,5});
+    index->set_node(6, nb1);
+    expect_level_0(1, {2,3,4,5,6});
+    expect_level_0(2, {1});
+    expect_level_0(3, {1});
+    expect_level_0(4, {1});
+    expect_level_0(5, {1});
+    expect_level_0(6, {1});
+    index->set_node(7, nb1);
+    expect_level_0(1, {2,3,4,6,7});
+    expect_level_0(5, {});
+    expect_level_0(6, {1});
+    index->set_node(8, nb1);
+    expect_level_0(1, {2,3,4,7,8});
+    expect_level_0(6, {});
+    index->set_node(9, nb1);
+    expect_level_0(1, {2,3,4,7,8});
+    expect_level_0(2, {1});
+    expect_level_0(3, {1});
+    expect_level_0(4, {1});
+    expect_level_0(5, {});
+    expect_level_0(6, {});
+    expect_level_0(7, {1});
+    expect_level_0(8, {1});
+    expect_level_0(9, {});
+    EXPECT_TRUE(index->check_link_symmetry());
+}
+
+TEST_F(HnswIndexTest, shrink_called_heuristic)
+{
+    init(true);
+    std::vector<uint32_t> nbl;
+    HnswNode empty{nbl};
+    index->set_node(1, empty);
+    nbl.push_back(1);
+    HnswNode nb1{nbl};
+    index->set_node(2, nb1);
+    index->set_node(3, nb1);
+    index->set_node(4, nb1);
+    index->set_node(5, nb1);
+    expect_level_0(1, {2,3,4,5});
+    index->set_node(6, nb1);
+    expect_level_0(1, {2,3,4,5,6});
+    expect_level_0(2, {1});
+    expect_level_0(3, {1});
+    expect_level_0(4, {1});
+    expect_level_0(5, {1});
+    expect_level_0(6, {1});
+    index->set_node(7, nb1);
+    expect_level_0(1, {2,3,4});
+    expect_level_0(2, {1});
+    expect_level_0(3, {1});
+    expect_level_0(4, {1});
+    expect_level_0(5, {});
+    expect_level_0(6, {});
+    expect_level_0(7, {});
+    index->set_node(8, nb1);
+    index->set_node(9, nb1);
+    expect_level_0(1, {2,3,4,8,9});
+    EXPECT_TRUE(index->check_link_symmetry());
+}
+
+TEST(LevelGeneratorTest, gives_various_levels)
+{
+    InvLogLevelGenerator generator(4);
+    EXPECT_EQ(2u, generator.max_level());
+    EXPECT_EQ(1u, generator.max_level());
+    EXPECT_EQ(0u, generator.max_level());
+    EXPECT_EQ(1u, generator.max_level());
+    EXPECT_EQ(0u, generator.max_level());
+    EXPECT_EQ(1u, generator.max_level());
+    EXPECT_EQ(0u, generator.max_level());
+    EXPECT_EQ(0u, generator.max_level());
+    EXPECT_EQ(0u, generator.max_level());
+
+    uint32_t left = 1000000;
+    std::vector<uint32_t> hist;
+    for (uint32_t i = 0; i < left; ++i) {
+        uint32_t l = generator.max_level();
+        if (hist.size() <= l) {
+            hist.resize(l+1);
+        }
+        hist[l]++;
+    }
+    for (uint32_t l = 0; l < hist.size(); ++l) {
+        double expected = left * 0.75;
+        EXPECT_TRUE(hist[l] < expected*1.01 + 100);
+        EXPECT_TRUE(hist[l] > expected*0.99 - 100);
+        left *= 0.25;
+    }
+    EXPECT_TRUE(hist.size() < 14);
+}
+
+GTEST_MAIN_RUN_ALL_TESTS()
diff --git a/searchlib/src/tests/tensor/hnsw_saver/CMakeLists.txt b/searchlib/src/tests/tensor/hnsw_saver/CMakeLists.txt
new file mode 100644
index 00000000000..90202e222a7
--- /dev/null
+++ b/searchlib/src/tests/tensor/hnsw_saver/CMakeLists.txt
@@ -0,0 +1,9 @@
+# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(searchlib_hnsw_save_load_test_app TEST
+    SOURCES
+    hnsw_save_load_test.cpp
+    DEPENDS
+    searchlib
+    gtest
+)
+vespa_add_test(NAME searchlib_hnsw_save_load_test_app COMMAND searchlib_hnsw_save_load_test_app)
diff --git a/searchlib/src/tests/tensor/hnsw_saver/hnsw_save_load_test.cpp b/searchlib/src/tests/tensor/hnsw_saver/hnsw_save_load_test.cpp
new file mode 100644
index 00000000000..b9e27d413f3
--- /dev/null
+++ b/searchlib/src/tests/tensor/hnsw_saver/hnsw_save_load_test.cpp
@@ -0,0 +1,150 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/searchlib/tensor/hnsw_graph.h>
+#include <vespa/searchlib/tensor/hnsw_index_saver.h>
+#include <vespa/searchlib/tensor/hnsw_index_loader.h>
+#include <vespa/vespalib/util/bufferwriter.h>
+#include <vespa/searchlib/util/fileutil.h>
+#include <vespa/vespalib/gtest/gtest.h>
+#include <vector>
+
+#include <vespa/log/log.h>
+LOG_SETUP("hnsw_save_load_test");
+
+using namespace search::tensor;
+using search::BufferWriter;
+using search::fileutil::LoadedBuffer;
+
+class VectorBufferWriter : public BufferWriter {
+private:
+    char tmp[1024];
+public:
+    std::vector<char> output;
+    VectorBufferWriter() {
+        setup(tmp, 1024);
+    }
+    ~VectorBufferWriter() {}
+    void flush() override {
+        for (size_t i = 0; i < usedLen(); ++i) {
+            output.push_back(tmp[i]);
+        }
+        rewind();
+    }
+};
+
+using V = std::vector<uint32_t>;
+
+void populate(HnswGraph &graph) {
+    // no 0
+    graph.make_node_for_document(1, 1);
+    graph.make_node_for_document(2, 2);
+    // no 3
+    graph.make_node_for_document(4, 2);
+    graph.make_node_for_document(5, 0);
+    graph.make_node_for_document(6, 1);
+
+    graph.set_link_array(1, 0, V{2, 4, 6});
+    graph.set_link_array(2, 0, V{1, 4, 6});
+    graph.set_link_array(4, 0, V{1, 2, 6});
+    graph.set_link_array(6, 0, V{1, 2, 4});
+    graph.set_link_array(2, 1, V{4});
+    graph.set_link_array(4, 1, V{2});
+    graph.set_entry_node(2, 1);
+}
+
+void modify(HnswGraph &graph) {
+    graph.remove_node_for_document(2);
+    graph.remove_node_for_document(6);
+    graph.make_node_for_document(7, 2);
+
+    graph.set_link_array(1, 0, V{7, 4});
+    graph.set_link_array(4, 0, V{7, 2});
+    graph.set_link_array(7, 0, V{4, 2});
+    graph.set_link_array(4, 1, V{7});
+    graph.set_link_array(7, 1, V{4});
+
+    graph.set_entry_node(4, 1);
+}
+
+
+class CopyGraphTest : public ::testing::Test {
+public:
+    HnswGraph original;
+    HnswGraph copy;
+
+    void expect_empty_d(uint32_t docid) const {
+        EXPECT_FALSE(copy.node_refs[docid].load_acquire().valid());
+    }
+
+    void expect_level_0(uint32_t docid, const V& exp_links) const {
+        auto levels = copy.get_level_array(docid);
+        EXPECT_GE(levels.size(), 1);
+        auto links = copy.get_link_array(docid, 0);
+        EXPECT_EQ(exp_links.size(), links.size());
+        for (size_t i = 0; i < exp_links.size() && i < links.size(); ++i) {
+            EXPECT_EQ(exp_links[i], links[i]);
+        }
+    }
+
+    void expect_level_1(uint32_t docid, const V& exp_links) const {
+        auto levels = copy.get_level_array(docid);
+        EXPECT_EQ(2, levels.size());
+        auto links = copy.get_link_array(docid, 1);
+        EXPECT_EQ(exp_links.size(), links.size());
+        for (size_t i = 0; i < exp_links.size() && i < links.size(); ++i) {
+            EXPECT_EQ(exp_links[i], links[i]);
+        }
+    }
+
+    std::vector<char> save_original() const {
+        HnswIndexSaver saver(original);
+        VectorBufferWriter vector_writer;
+        saver.save(vector_writer);
+        return vector_writer.output;
+    }
+    void load_copy(std::vector<char> data) {
+        HnswIndexLoader loader(copy);
+        LoadedBuffer buffer(&data[0], data.size());
+        loader.load(buffer);
+    }
+
+    void expect_copy_as_populated() const {
+        EXPECT_EQ(copy.size(), 7);
+        EXPECT_EQ(copy.entry_docid, 2);
+        EXPECT_EQ(copy.entry_level, 1);
+
+        expect_empty_d(0);
+        expect_empty_d(3);
+        expect_empty_d(5);
+
+        expect_level_0(1, {2, 4, 6});
+        expect_level_0(2, {1, 4, 6});
+        expect_level_0(4, {1, 2, 6});
+        expect_level_0(6, {1, 2, 4});
+        
+        expect_level_1(2, {4});
+        expect_level_1(4, {2});
+    }
+};
+
+TEST_F(CopyGraphTest, reconstructs_graph)
+{
+    populate(original);
+    auto data = save_original();
+    load_copy(data);
+    expect_copy_as_populated();
+}
+
+TEST_F(CopyGraphTest, later_changes_ignored)
+{
+    populate(original);
+    HnswIndexSaver saver(original);
+    modify(original);
+    VectorBufferWriter vector_writer;
+    saver.save(vector_writer);
+    auto data = vector_writer.output;
+    load_copy(data);
+    expect_copy_as_populated();
+}
+
+GTEST_MAIN_RUN_ALL_TESTS()
diff --git a/searchlib/src/tests/util/ioerrorhandler/ioerrorhandler_test.cpp b/searchlib/src/tests/util/ioerrorhandler/ioerrorhandler_test.cpp
index fb1c1a356f6..ec2831360c5 100644
--- a/searchlib/src/tests/util/ioerrorhandler/ioerrorhandler_test.cpp
+++ b/searchlib/src/tests/util/ioerrorhandler/ioerrorhandler_test.cpp
@@ -11,95 +11,66 @@
 #include <iostream>
 #include <fstream>
 #include <string>
-#include <setjmp.h>
-#include <dlfcn.h>
 #include <unistd.h>
+#include <vespa/fastos/file_rw_ops.h>
 
 #include <vespa/log/log.h>
 LOG_SETUP("ioerrorhandler_test");
 
-extern "C" {
-
-ssize_t read(int fd, void *buf, size_t count);
-ssize_t write(int fd, const void *buf, size_t count);
-ssize_t pread(int fd, void *buf, size_t count, off_t offset);
-ssize_t pwrite(int fd, const void *buf, size_t count, off_t offset);
-
-}
-
-using ReadFunc = ssize_t (*)(int fd, void *buf, size_t count);
-using WriteFunc = ssize_t (*)(int fd, const void *buf, size_t count);
-using PreadFunc = ssize_t (*)(int fd, void *buf, size_t count, off_t offset);
-using PwriteFunc = ssize_t (*)(int fd, const void *buf, size_t count, off_t offset);
-
 using namespace search::test::statefile;
 using namespace search::test::statestring;
 
-namespace {
-
-ReadFunc libc_read;
-WriteFunc libc_write;
-PreadFunc libc_pread;
-PwriteFunc libc_pwrite;
-
-}
-
 int injectErrno;
 std::atomic<int> injectreadErrnoTrigger;
 std::atomic<int> injectpreadErrnoTrigger;
 std::atomic<int> injectwriteErrnoTrigger;
 std::atomic<int> injectpwriteErrnoTrigger;
 
-ssize_t read(int fd, void *buf, size_t count)
+ssize_t error_injecting_read(int fd, void* buf, size_t count)
 {
     if (--injectreadErrnoTrigger == 0) {
         errno = injectErrno;
         return -1;
     }
-    if (!libc_read) {
-        libc_read = reinterpret_cast<ReadFunc>(dlsym(RTLD_NEXT, "read"));
-    }
-    return libc_read(fd, buf, count);
+    return read(fd, buf, count);
 }
 
-ssize_t write(int fd, const void *buf, size_t count)
+ssize_t error_injecting_write(int fd, const void* buf, size_t count)
 {
     if (--injectwriteErrnoTrigger == 0) {
         errno = injectErrno;
         return -1;
     }
-    if (!libc_write) {
-        libc_write = reinterpret_cast<WriteFunc>(dlsym(RTLD_NEXT, "write"));
-    }
-    return libc_write(fd, buf, count);
+    return write(fd, buf, count);
 }
 
-ssize_t pread(int fd, void *buf, size_t count, off_t offset)
+ssize_t error_injecting_pread(int fd, void* buf, size_t count, off_t offset)
 {
     if (--injectpreadErrnoTrigger == 0) {
         errno = injectErrno;
         return -1;
     }
-    if (!libc_pread) {
-        libc_pread = reinterpret_cast<PreadFunc>(dlsym(RTLD_NEXT, "pread"));
-    }
-    return libc_pread(fd, buf, count, offset);
+    return pread(fd, buf, count, offset);
 }
 
 
-ssize_t pwrite(int fd, const void *buf, size_t count, off_t offset)
+ssize_t error_injecting_pwrite(int fd, const void* buf, size_t count, off_t offset)
 {
     if (--injectpwriteErrnoTrigger == 0) {
         errno = injectErrno;
         return -1;
     }
-    if (!libc_pwrite) {
-        libc_pwrite = reinterpret_cast<PwriteFunc>(dlsym(RTLD_NEXT, "pwrite"));
-    }
-    return libc_pwrite(fd, buf, count, offset);
+    return pwrite(fd, buf, count, offset);
 }
 
-
+void setup_error_injections()
+{
+    using Ops = fastos::File_RW_Ops;
+    Ops::_read = error_injecting_read;
+    Ops::_write = error_injecting_write;
+    Ops::_pread = error_injecting_pread;
+    Ops::_pwrite = error_injecting_pwrite;
+}
 
 namespace search
 {
@@ -171,14 +142,6 @@ Fixture::openFile()
 }
 
 void
-Fixture::openFileDIO()
-{
-    file.reset(new FastOS_File);
-    file->EnableDirectIO();
-    file->OpenReadWrite("testfile");
-}
-
-void
 Fixture::writeTestString()
 {
     file->WriteBuf(testString, strlen(testString));
@@ -310,6 +273,15 @@ TEST_F("Test that ioerror handler can process write error", Fixture)
 }
 
 
+#ifdef __linux__
+void
+Fixture::openFileDIO()
+{
+    file.reset(new FastOS_File);
+    file->EnableDirectIO();
+    file->OpenReadWrite("testfile");
+}
+
 TEST_F("Test that ioerror handler can process pwrite error", Fixture)
 {
     IOErrorHandler ioeh(f.sf.get());
@@ -342,11 +314,13 @@ TEST_F("Test that ioerror handler can process pwrite error", Fixture)
         TEST_DO(assertHistory(exp, act));
     }
 }
+#endif
 
 }
 
 TEST_MAIN()
 {
+    setup_error_injections();
     TEST_RUN_ALL();
     search::StateFile::erase("state");
     unlink("testfile");
diff --git a/searchlib/src/tests/util/sigbushandler/sigbushandler_test.cpp b/searchlib/src/tests/util/sigbushandler/sigbushandler_test.cpp
index 658ad17544a..e687eff3ca7 100644
--- a/searchlib/src/tests/util/sigbushandler/sigbushandler_test.cpp
+++ b/searchlib/src/tests/util/sigbushandler/sigbushandler_test.cpp
@@ -66,16 +66,22 @@ TEST("Test that sigbus handler can trap synthetic sigbus")
         LOG_ABORT("Should never get here");
     }
     EXPECT_TRUE(sbh.fired());
+#ifdef __APPLE__
+    vespalib::string exp_state = "state=down ts=0.0 operation=sigbus errno=0 code=2 addr=0x0000000000000000\n";
+#else
+    vespalib::string exp_state = "state=down ts=0.0 operation=sigbus errno=0 code=0\n";
+#endif
     {
         vespalib::string act = readState(sf);
         normalizeTimestamp(act);
-        EXPECT_EQUAL("state=down ts=0.0 operation=sigbus errno=0 code=0\n",
-                     act);
+        normalizeAddr(act, nullptr);
+        EXPECT_EQUAL(exp_state, act);
     }
     {
-        strvec exp({"state=down ts=0.0 operation=sigbus errno=0 code=0\n" });
+        strvec exp({exp_state});
         std::vector<vespalib::string> act(readHistory("state.history"));
         normalizeTimestamps(act);
+        normalizeAddrs(act, nullptr);
         TEST_DO(assertHistory(exp, act));
     }
 }
diff --git a/searchlib/src/vespa/searchlib/aggregation/group.cpp b/searchlib/src/vespa/searchlib/aggregation/group.cpp
index 1808af79b4d..d6d3a8f1deb 100644
--- a/searchlib/src/vespa/searchlib/aggregation/group.cpp
+++ b/searchlib/src/vespa/searchlib/aggregation/group.cpp
@@ -291,7 +291,6 @@ void
 Group::Value::addChild(Group * child)
 {
     const size_t sz(getChildrenSize());
-    assert(sz < 0xffffff);
     if (_children == nullptr) {
         _children = new ChildP[4];
     } else if ((sz >=4) && vespalib::Optimized::msbIdx(sz) == vespalib::Optimized::lsbIdx(sz)) {
@@ -631,11 +630,12 @@ Group::Value::visitMembers(vespalib::ObjectVisitor &visitor) const {
 }
 
 Group::Value::Value() :
-    _packedLength(0),
-    _tag(-1),
     _aggregationResults(nullptr),
     _children(nullptr),
     _childInfo(),
+    _childrenLength(0),
+    _tag(-1),
+    _packedLength(0),
     _orderBy()
 {
     memset(_orderBy, 0, sizeof(_orderBy));
@@ -643,11 +643,12 @@ Group::Value::Value() :
 }
 
 Group::Value::Value(const Value & rhs) :
-    _packedLength(rhs._packedLength),
-    _tag(rhs._tag),
     _aggregationResults(nullptr),
     _children(nullptr),
     _childInfo(),
+    _childrenLength(rhs._childrenLength),
+    _tag(rhs._tag),
+    _packedLength(rhs._packedLength),
     _orderBy()
 {
     _childInfo._childMap = nullptr;
@@ -671,11 +672,12 @@ Group::Value::Value(const Value & rhs) :
 }
 
 Group::Value::Value(Value && rhs) noexcept :
-    _packedLength(rhs._packedLength),
-    _tag(rhs._tag),
     _aggregationResults(rhs._aggregationResults),
     _children(rhs._children),
     _childInfo(rhs._childInfo),
+    _childrenLength(rhs._childrenLength),
+    _tag(rhs._tag),
+    _packedLength(rhs._packedLength),
     _orderBy()
 {
     memcpy(_orderBy, rhs._orderBy, sizeof(_orderBy));
@@ -688,8 +690,9 @@ Group::Value::Value(Value && rhs) noexcept :
 
 Group::Value &
 Group::Value::operator =(Value && rhs) noexcept {
-    _packedLength = rhs._packedLength;
+    _childrenLength = rhs._childrenLength;
     _tag = rhs._tag;
+    _packedLength = rhs._packedLength;
     _aggregationResults = rhs._aggregationResults;
     _children = rhs._children;
     _childInfo = rhs._childInfo;
@@ -729,6 +732,7 @@ Group::Value::swap(Value & rhs)
         memcpy(_orderBy, rhs._orderBy, sizeof(_orderBy));
         memcpy(rhs._orderBy, tmp, sizeof(_orderBy));
     }
+    std::swap(_childrenLength, rhs._childrenLength);
     std::swap(_tag, rhs._tag);
     std::swap(_packedLength, rhs._packedLength);
 }
diff --git a/searchlib/src/vespa/searchlib/aggregation/group.h b/searchlib/src/vespa/searchlib/aggregation/group.h
index f302346f211..f6b6bc732af 100644
--- a/searchlib/src/vespa/searchlib/aggregation/group.h
+++ b/searchlib/src/vespa/searchlib/aggregation/group.h
@@ -101,7 +101,7 @@ private:
         void addChild(Group * child);
         uint32_t getAggrSize()    const { return _packedLength & 0x0f; }
         uint32_t getOrderBySize() const { return (_packedLength >> 6) & 0x03; }
-        uint32_t getChildrenSize()   const { return (_packedLength >> 8); }
+        uint32_t getChildrenSize()   const { return _childrenLength; }
         uint32_t getExpr(uint32_t i) const { return getAggrSize() + i; }
         int32_t getOrderBy(uint32_t i) const {
             int32_t v((_orderBy[i/2] >> (4*(i%2))) & 0x0f);
@@ -122,7 +122,7 @@ private:
         void setAggrSize(uint32_t v)    { _packedLength = (_packedLength & ~0x0f) | v; }
         void setExprSize(uint32_t v)    { _packedLength = (_packedLength & ~0x30) | (v << 4); }
         void setOrderBySize(uint32_t v) { _packedLength = (_packedLength & ~0xc0) | (v << 6); }
-        void setChildrenSize(uint32_t v) { _packedLength = (_packedLength & ~0xffffff00) | (v << 8); }
+        void setChildrenSize(uint32_t v) { _childrenLength = v; }
         AggregationResult * getAggr(size_t i) { return static_cast<AggregationResult *>(_aggregationResults[i].get()); }
         const AggregationResult & getAggr(size_t i) const { return static_cast<const AggregationResult &>(*_aggregationResults[i]); }
         const ExpressionNode::CP & getAggrCP(size_t i) const { return _aggregationResults[i]; }
@@ -139,9 +139,6 @@ private:
         }
         bool needFullRank() const { return getOrderBySize() != 0; }
 
-        uint32_t         _packedLength;    // Length of the 3 vectors below
-        uint32_t         _tag;             // Opaque tag used to identify the group by the client.
-
         // The collectors and expressions stored by this group. Currently, both aggregation results and expressions used by orderby() are stored in this
         // array to save 8 bytes in the Group size. This makes it important to use the getAggr() and expr() methods for accessing elements,
         // as they will correctly offset the index to the correct place in the array.
@@ -152,6 +149,9 @@ private:
             GroupHash *_childMap;               // child map used during aggregation
             size_t     _allChildren;            // Keep real number of children.
         }                _childInfo;
+        uint32_t         _childrenLength;
+        uint32_t         _tag;             // Opaque tag used to identify the group by the client.
+        uint8_t          _packedLength;    // Length of aggr and expr vectors.
         uint8_t          _orderBy[2];           // How this group is ranked, negative means reverse rank.
     };
 
diff --git a/searchlib/src/vespa/searchlib/aggregation/hitlist.cpp b/searchlib/src/vespa/searchlib/aggregation/hitlist.cpp
index 0efe2adfaf9..2cc82d59fff 100644
--- a/searchlib/src/vespa/searchlib/aggregation/hitlist.cpp
+++ b/searchlib/src/vespa/searchlib/aggregation/hitlist.cpp
@@ -3,6 +3,7 @@
 #include "hitlist.h"
 #include <vespa/vespalib/objects/visit.hpp>
 #include <algorithm>
+#include <stdexcept>
 
 namespace search::aggregation {
 
diff --git a/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp b/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp
index 8595b0eff7f..14d33914d05 100644
--- a/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp
+++ b/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp
@@ -519,12 +519,12 @@ public:
 
     void visit(StringTerm & n) override { visitTerm(n, true); }
     void visit(SubstringTerm & n) override {
-        query::SimpleRegExpTerm re(vespalib::Regexp::make_from_substring(n.getTerm()),
+        query::SimpleRegExpTerm re(vespalib::RegexpUtil::make_from_substring(n.getTerm()),
                                    n.getView(), n.getId(), n.getWeight());
         visitTerm(re);
     }
     void visit(SuffixTerm & n) override {
-        query::SimpleRegExpTerm re(vespalib::Regexp::make_from_suffix(n.getTerm()),
+        query::SimpleRegExpTerm re(vespalib::RegexpUtil::make_from_suffix(n.getTerm()),
                                    n.getView(), n.getId(), n.getWeight());
         visitTerm(re);
     }
@@ -646,7 +646,9 @@ public:
         query_tensor.release();
         setResult(std::make_unique<queryeval::NearestNeighborBlueprint>(_field, *dense_attr_tensor,
                                                                         std::move(dense_query_tensor_up),
-                                                                        n.get_target_num_hits()));
+                                                                        n.get_target_num_hits(),
+                                                                        n.get_allow_approximate(),
+                                                                        n.get_explore_additional_hits()));
     }
 };
 
diff --git a/searchlib/src/vespa/searchlib/attribute/attribute_header.cpp b/searchlib/src/vespa/searchlib/attribute/attribute_header.cpp
index 224d5758028..b35a88fab77 100644
--- a/searchlib/src/vespa/searchlib/attribute/attribute_header.cpp
+++ b/searchlib/src/vespa/searchlib/attribute/attribute_header.cpp
@@ -3,6 +3,7 @@
 #include "attribute_header.h"
 #include <vespa/vespalib/data/fileheader.h>
 #include <vespa/vespalib/data/databuffer.h>
+#include <vespa/vespalib/util/exceptions.h>
 
 namespace search::attribute {
 
@@ -18,11 +19,23 @@ const vespalib::string tensorTypeTag = "tensortype";
 const vespalib::string predicateArityTag = "predicate.arity";
 const vespalib::string predicateLowerBoundTag = "predicate.lower_bound";
 const vespalib::string predicateUpperBoundTag = "predicate.upper_bound";
+const vespalib::string hnsw_max_links_tag = "hnsw.max_links_per_node";
+const vespalib::string hnsw_neighbors_to_explore_tag = "hnsw.neighbors_to_explore_at_insert";
+const vespalib::string hnsw_distance_metric = "hnsw.distance_metric";
+const vespalib::string euclidean = "euclidean";
+const vespalib::string angular = "angular";
+const vespalib::string geodegrees = "geodegrees";
+const vespalib::string doc_id_limit_tag = "docIdLimit";
 
 }
 
 AttributeHeader::AttributeHeader()
-    : _fileName(""),
+    : AttributeHeader("")
+{
+}
+
+AttributeHeader::AttributeHeader(const vespalib::string &fileName)
+    : _fileName(fileName),
       _basicType(attribute::BasicType::Type::NONE),
       _collectionType(attribute::CollectionType::Type::SINGLE),
       _tensorType(vespalib::eval::ValueType::error_type()),
@@ -30,6 +43,7 @@ AttributeHeader::AttributeHeader()
       _collectionTypeParamsSet(false),
       _predicateParamsSet(false),
       _predicateParams(),
+      _hnsw_index_params(),
       _numDocs(0),
       _uniqueValueCount(0),
       _totalValueCount(0),
@@ -38,11 +52,18 @@ AttributeHeader::AttributeHeader()
 {
 }
 
-AttributeHeader::AttributeHeader(const vespalib::string &fileName, attribute::BasicType basicType,
-                                 attribute::CollectionType collectionType, const vespalib::eval::ValueType &tensorType,
-                                 bool enumerated, const attribute::PersistentPredicateParams &predicateParams,
-                                 uint32_t numDocs, [[maybe_unused]] uint32_t fixedWidth, uint64_t uniqueValueCount,
-                                 uint64_t totalValueCount, uint64_t createSerialNum, uint32_t version)
+AttributeHeader::AttributeHeader(const vespalib::string &fileName,
+                                 attribute::BasicType basicType,
+                                 attribute::CollectionType collectionType,
+                                 const vespalib::eval::ValueType &tensorType,
+                                 bool enumerated,
+                                 const attribute::PersistentPredicateParams &predicateParams,
+                                 const std::optional<HnswIndexParams>& hnsw_index_params,
+                                 uint32_t numDocs,
+                                 uint64_t uniqueValueCount,
+                                 uint64_t totalValueCount,
+                                 uint64_t createSerialNum,
+                                 uint32_t version)
     : _fileName(fileName),
       _basicType(basicType),
       _collectionType(collectionType),
@@ -51,6 +72,7 @@ AttributeHeader::AttributeHeader(const vespalib::string &fileName, attribute::Ba
       _collectionTypeParamsSet(false),
       _predicateParamsSet(false),
       _predicateParams(predicateParams),
+      _hnsw_index_params(hnsw_index_params),
       _numDocs(numDocs),
       _uniqueValueCount(uniqueValueCount),
       _totalValueCount(totalValueCount),
@@ -61,6 +83,35 @@ AttributeHeader::AttributeHeader(const vespalib::string &fileName, attribute::Ba
 
 AttributeHeader::~AttributeHeader() = default;
 
+namespace {
+
+vespalib::string
+to_string(DistanceMetric metric)
+{
+    switch (metric) {
+        case DistanceMetric::Euclidean: return euclidean;
+        case DistanceMetric::Angular: return angular;
+        case DistanceMetric::GeoDegrees: return geodegrees;
+    }
+    throw vespalib::IllegalArgumentException("Unknown distance metric " + std::to_string(static_cast<int>(metric)));
+}
+
+DistanceMetric
+to_distance_metric(const vespalib::string& metric)
+{
+    if (metric == euclidean) {
+        return DistanceMetric::Euclidean;
+    } else if (metric == angular) {
+        return DistanceMetric::Angular;
+    } else if (metric == geodegrees) {
+        return DistanceMetric::GeoDegrees;
+    } else {
+        throw vespalib::IllegalStateException("Unknown distance metric '" + metric + "'");
+    }
+}
+
+}
+
 void
 AttributeHeader::internalExtractTags(const vespalib::GenericHeader &header)
 {
@@ -86,6 +137,15 @@ AttributeHeader::internalExtractTags(const vespalib::GenericHeader &header)
     if (_basicType.type() == BasicType::Type::TENSOR) {
         assert(header.hasTag(tensorTypeTag));
         _tensorType = vespalib::eval::ValueType::from_spec(header.getTag(tensorTypeTag).asString());
+        if (header.hasTag(hnsw_max_links_tag)) {
+            assert(header.hasTag(hnsw_neighbors_to_explore_tag));
+            assert(header.hasTag(hnsw_distance_metric));
+
+            uint32_t max_links = header.getTag(hnsw_max_links_tag).asInteger();
+            uint32_t neighbors_to_explore = header.getTag(hnsw_neighbors_to_explore_tag).asInteger();
+            DistanceMetric distance_metric = to_distance_metric(header.getTag(hnsw_distance_metric).asString());
+            _hnsw_index_params.emplace(max_links, neighbors_to_explore, distance_metric);
+        }
     }
     if (_basicType.type() == BasicType::Type::PREDICATE) {
         if (header.hasTag(predicateArityTag)) {
@@ -100,6 +160,9 @@ AttributeHeader::internalExtractTags(const vespalib::GenericHeader &header)
             assert(!header.hasTag(predicateUpperBoundTag));
         }
     }
+    if (header.hasTag(doc_id_limit_tag)) {
+        _numDocs = header.getTag(doc_id_limit_tag).asInteger();
+    }
     if (header.hasTag(versionTag)) {
         _version = header.getTag(versionTag).asInteger();
     }
@@ -125,7 +188,7 @@ AttributeHeader::addTags(vespalib::GenericHeader &header) const
     }
     header.putTag(Tag("uniqueValueCount", _uniqueValueCount));
     header.putTag(Tag("totalValueCount", _totalValueCount));
-    header.putTag(Tag("docIdLimit", _numDocs));
+    header.putTag(Tag(doc_id_limit_tag, _numDocs));
     header.putTag(Tag("frozen", 0));
     header.putTag(Tag("fileBitSize", 0));
     header.putTag(Tag(versionTag, _version));
@@ -137,6 +200,12 @@ AttributeHeader::addTags(vespalib::GenericHeader &header) const
     }
     if (_basicType.type() == attribute::BasicType::Type::TENSOR) {
         header.putTag(Tag(tensorTypeTag, _tensorType.to_spec()));;
+        if (_hnsw_index_params.has_value()) {
+            const auto& params = *_hnsw_index_params;
+            header.putTag(Tag(hnsw_max_links_tag, params.max_links_per_node()));
+            header.putTag(Tag(hnsw_neighbors_to_explore_tag, params.neighbors_to_explore_at_insert()));
+            header.putTag(Tag(hnsw_distance_metric, to_string(params.distance_metric())));
+        }
     }
     if (_basicType.type() == attribute::BasicType::Type::PREDICATE) {
         const auto & params = _predicateParams;
diff --git a/searchlib/src/vespa/searchlib/attribute/attribute_header.h b/searchlib/src/vespa/searchlib/attribute/attribute_header.h
index 303c469e755..583253eea0f 100644
--- a/searchlib/src/vespa/searchlib/attribute/attribute_header.h
+++ b/searchlib/src/vespa/searchlib/attribute/attribute_header.h
@@ -5,8 +5,10 @@
 #include <vespa/vespalib/stllike/string.h>
 #include <vespa/searchcommon/attribute/basictype.h>
 #include <vespa/searchcommon/attribute/collectiontype.h>
+#include <vespa/searchcommon/attribute/hnsw_index_params.h>
 #include <vespa/searchcommon/attribute/predicate_params.h>
 #include <vespa/eval/eval/value_type.h>
+#include <optional>
 
 namespace vespalib { class GenericHeader; }
 
@@ -26,6 +28,7 @@ private:
     bool        _collectionTypeParamsSet;
     bool        _predicateParamsSet;
     PersistentPredicateParams _predicateParams;
+    std::optional<HnswIndexParams> _hnsw_index_params;
     uint32_t    _numDocs;
     uint64_t    _uniqueValueCount;
     uint64_t    _totalValueCount;
@@ -35,14 +38,15 @@ private:
     void internalExtractTags(const vespalib::GenericHeader &header);
 public:
     AttributeHeader();
+    AttributeHeader(const vespalib::string &fileName);
     AttributeHeader(const vespalib::string &fileName,
                     BasicType basicType,
                     CollectionType collectionType,
                     const vespalib::eval::ValueType &tensorType,
                     bool enumerated,
                     const PersistentPredicateParams &predicateParams,
+                    const std::optional<HnswIndexParams>& hnsw_index_params,
                     uint32_t numDocs,
-                    uint32_t fixedWidth,
                     uint64_t uniqueValueCount,
                     uint64_t totalValueCount,
                     uint64_t createSerialNum,
@@ -62,6 +66,7 @@ public:
     const PersistentPredicateParams &getPredicateParams() const { return _predicateParams; }
     bool getPredicateParamsSet() const { return _predicateParamsSet; }
     bool getCollectionTypeParamsSet() const { return _collectionTypeParamsSet; }
+    const std::optional<HnswIndexParams>& get_hnsw_index_params() const { return _hnsw_index_params; }
     static AttributeHeader extractTags(const vespalib::GenericHeader &header);
     void addTags(vespalib::GenericHeader &header) const;
 };
diff --git a/searchlib/src/vespa/searchlib/attribute/attribute_weighted_set_blueprint.cpp b/searchlib/src/vespa/searchlib/attribute/attribute_weighted_set_blueprint.cpp
index 9f9d5535014..af3fbca6943 100644
--- a/searchlib/src/vespa/searchlib/attribute/attribute_weighted_set_blueprint.cpp
+++ b/searchlib/src/vespa/searchlib/attribute/attribute_weighted_set_blueprint.cpp
@@ -7,8 +7,9 @@
 #include <vespa/searchlib/query/query_term_ucs4.h>
 #include <vespa/searchlib/queryeval/weighted_set_term_search.h>
 #include <vespa/vespalib/objects/visit.h>
-#include <vespa/vespalib/stllike/hash_map.h>
 #include <vespa/vespalib/util/stringfmt.h>
+#include <vespa/vespalib/stllike/hash_map.hpp>
+
 
 namespace search {
 
@@ -36,12 +37,13 @@ public:
 class UseStringEnum : public UseAttr
 {
 public:
+    using TokenT = uint32_t;
     UseStringEnum(const IAttributeVector & attr)
         : UseAttr(attr) {}
     auto mapToken(const ISearchContext &context) const {
         return attribute().findFoldedEnums(context.queryTerm()->getTerm());
     }
-    int64_t getToken(uint32_t docId) const {
+    TokenT getToken(uint32_t docId) const {
         return attribute().getEnum(docId);
     }
 };
@@ -51,6 +53,7 @@ public:
 class UseInteger : public UseAttr
 {
 public:
+    using TokenT = uint64_t;
     UseInteger(const IAttributeVector & attr) : UseAttr(attr) {}
     std::vector<int64_t> mapToken(const ISearchContext &context) const {
         std::vector<int64_t> result;
@@ -60,7 +63,7 @@ public:
         }
         return result;
     }
-    int64_t getToken(uint32_t docId) const {
+    TokenT getToken(uint32_t docId) const {
         return attribute().getInt(docId);
     }
 };
@@ -71,8 +74,9 @@ template <typename T>
 class AttributeFilter final : public queryeval::SearchIterator
 {
 private:
-    typedef vespalib::hash_map<int64_t, int32_t> Map;
-    typedef fef::TermFieldMatchData TFMD;
+    using Key = typename T::TokenT;
+    using Map = vespalib::hash_map<Key, int32_t, vespalib::hash<Key>, std::equal_to<Key>, vespalib::hashtable_base::and_modulator>;
+    using TFMD = fef::TermFieldMatchData;
 
     TFMD    &_tfmd;
     T        _attr;
@@ -93,12 +97,12 @@ public:
         }
     }
     void and_hits_into(BitVector & result,uint32_t begin_id) override {
-        Map::iterator end = _map.end();
+        typename Map::iterator end = _map.end();
         result.foreach_truebit([&, end](uint32_t key) { if ( _map.find(_attr.getToken(key)) == end) { result.clearBit(key); }}, begin_id);
     }
 
     void doSeek(uint32_t docId) override {
-        Map::const_iterator pos = _map.find(_attr.getToken(docId));
+        typename Map::const_iterator pos = _map.find(_attr.getToken(docId));
         if (pos != _map.end()) {
             _weight = pos->second;
             setDocId(docId);
diff --git a/searchlib/src/vespa/searchlib/attribute/attributefilesavetarget.cpp b/searchlib/src/vespa/searchlib/attribute/attributefilesavetarget.cpp
index f57094ae592..f284fecbf98 100644
--- a/searchlib/src/vespa/searchlib/attribute/attributefilesavetarget.cpp
+++ b/searchlib/src/vespa/searchlib/attribute/attributefilesavetarget.cpp
@@ -3,14 +3,16 @@
 #include "attributefilesavetarget.h"
 #include "attributevector.h"
 #include <vespa/searchlib/common/fileheadercontext.h>
-#include <vespa/vespalib/data/fileheader.h>
 #include <vespa/vespalib/data/databuffer.h>
+#include <vespa/vespalib/data/fileheader.h>
 #include <vespa/vespalib/util/error.h>
+#include <vespa/vespalib/util/exceptions.h>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".searchlib.attribute.attributefilesavetarget");
 
 using vespalib::getLastErrorString;
+using vespalib::IllegalArgumentException;
 
 namespace search {
 
@@ -18,13 +20,16 @@ using common::FileHeaderContext;
 
 
 AttributeFileSaveTarget::
-AttributeFileSaveTarget(const TuneFileAttributes &tuneFileAttributes,
-                        const FileHeaderContext &fileHeaderContext)
+AttributeFileSaveTarget(const TuneFileAttributes& tune_file,
+                        const FileHeaderContext& file_header_ctx)
     : IAttributeSaveTarget(),
-      _datWriter(tuneFileAttributes, fileHeaderContext, _header, "Attribute vector data file"),
-      _idxWriter(tuneFileAttributes, fileHeaderContext, _header, "Attribute vector idx file"),
-      _weightWriter(tuneFileAttributes, fileHeaderContext, _header, "Attribute vector weight file"),
-      _udatWriter(tuneFileAttributes, fileHeaderContext, _header, "Attribute vector unique data file")
+      _tune_file(tune_file),
+      _file_header_ctx(file_header_ctx),
+      _datWriter(tune_file, file_header_ctx, _header, "Attribute vector data file"),
+      _idxWriter(tune_file, file_header_ctx, _header, "Attribute vector idx file"),
+      _weightWriter(tune_file, file_header_ctx, _header, "Attribute vector weight file"),
+      _udatWriter(tune_file, file_header_ctx, _header, "Attribute vector unique data file"),
+      _writers()
 {
 }
 
@@ -66,23 +71,23 @@ AttributeFileSaveTarget::close()
     _udatWriter.close();
     _idxWriter.close();
     _weightWriter.close();
+    for (auto& writer : _writers) {
+        writer.second->close();
+    }
 }
 
-
 IAttributeFileWriter &
 AttributeFileSaveTarget::datWriter()
 {
     return _datWriter;
 }
 
-
 IAttributeFileWriter &
 AttributeFileSaveTarget::idxWriter()
 {
     return _idxWriter;
 }
 
-
 IAttributeFileWriter &
 AttributeFileSaveTarget::weightWriter()
 {
@@ -95,6 +100,33 @@ AttributeFileSaveTarget::udatWriter()
     return _udatWriter;
 }
 
+bool
+AttributeFileSaveTarget::setup_writer(const vespalib::string& file_suffix,
+                                      const vespalib::string& desc)
+{
+    vespalib::string file_name(_header.getFileName() + "." + file_suffix);
+    auto writer = std::make_unique<AttributeFileWriter>(_tune_file, _file_header_ctx,
+                                                        _header, desc);
+    if (!writer->open(file_name)) {
+        return false;
+    }
+    auto itr = _writers.find(file_suffix);
+    if (itr != _writers.end()) {
+        return false;
+    }
+    _writers.insert(std::make_pair(file_suffix, std::move(writer)));
+    return true;
+}
+
+IAttributeFileWriter&
+AttributeFileSaveTarget::get_writer(const vespalib::string& file_suffix)
+{
+    auto itr = _writers.find(file_suffix);
+    if (itr == _writers.end()) {
+        throw IllegalArgumentException("File writer with suffix '" + file_suffix + "' does not exist");
+    }
+    return *itr->second;
+}
 
 } // namespace search
 
diff --git a/searchlib/src/vespa/searchlib/attribute/attributefilesavetarget.h b/searchlib/src/vespa/searchlib/attribute/attributefilesavetarget.h
index acb3daf82e0..9a9d38615ea 100644
--- a/searchlib/src/vespa/searchlib/attribute/attributefilesavetarget.h
+++ b/searchlib/src/vespa/searchlib/attribute/attributefilesavetarget.h
@@ -4,24 +4,30 @@
 
 #include "iattributesavetarget.h"
 #include "attributefilewriter.h"
+#include <vespa/vespalib/stllike/hash_fun.h>
+#include <unordered_map>
 
-namespace search
-{
+namespace search {
 
 /**
  * Class used to save an attribute vector to file(s).
  **/
-class AttributeFileSaveTarget : public IAttributeSaveTarget
-{
+class AttributeFileSaveTarget : public IAttributeSaveTarget {
 private:
+    using FileWriterUP = std::unique_ptr<AttributeFileWriter>;
+    using WriterMap = std::unordered_map<vespalib::string, FileWriterUP, vespalib::hash<vespalib::string>>;
+
+    const TuneFileAttributes& _tune_file;
+    const search::common::FileHeaderContext& _file_header_ctx;
     AttributeFileWriter _datWriter;
     AttributeFileWriter _idxWriter;
     AttributeFileWriter _weightWriter;
     AttributeFileWriter _udatWriter;
+    WriterMap _writers;
 
 public:
-    AttributeFileSaveTarget(const TuneFileAttributes &tuneFileAttributes,
-                            const search::common::FileHeaderContext &fileHeaderContext);
+    AttributeFileSaveTarget(const TuneFileAttributes& tune_file,
+                            const search::common::FileHeaderContext& file_header_ctx);
     ~AttributeFileSaveTarget() override;
 
     // Implements IAttributeSaveTarget
@@ -35,6 +41,11 @@ public:
     IAttributeFileWriter &idxWriter() override;
     IAttributeFileWriter &weightWriter() override;
     IAttributeFileWriter &udatWriter() override;
+
+    bool setup_writer(const vespalib::string& file_suffix,
+                      const vespalib::string& desc) override;
+    IAttributeFileWriter& get_writer(const vespalib::string& file_suffix) override;
+
 };
 
 } // namespace search
diff --git a/searchlib/src/vespa/searchlib/attribute/attributemanager.cpp b/searchlib/src/vespa/searchlib/attribute/attributemanager.cpp
index da280aa7bda..5fa32e8ad75 100644
--- a/searchlib/src/vespa/searchlib/attribute/attributemanager.cpp
+++ b/searchlib/src/vespa/searchlib/attribute/attributemanager.cpp
@@ -131,7 +131,7 @@ bool AttributeManager::hasReaders() const
 const AttributeManager::VectorHolder *
 AttributeManager::findAndLoadAttribute(const string & name) const
 {
-    const VectorHolder * loadedVector(NULL);
+    const VectorHolder * loadedVector(nullptr);
     AttributeMap::const_iterator found = _attributes.find(name);
     if (found != _attributes.end()) {
         AttributeVector & vec = *found->second;
@@ -263,5 +263,14 @@ AttributeManager::asyncForAttribute(const vespalib::string &, std::unique_ptr<at
     throw std::runtime_error("search::AttributeManager::asyncForAttribute should never be called.");
 }
 
+std::shared_ptr<attribute::ReadableAttributeVector>
+AttributeManager::readable_attribute_vector(const string& name) const
+{
+    const auto* attr = findAndLoadAttribute(name);
+    if (attr) {
+        return *attr;
+    }
+    return {};
+}
 
 }
diff --git a/searchlib/src/vespa/searchlib/attribute/attributemanager.h b/searchlib/src/vespa/searchlib/attribute/attributemanager.h
index 1c60ab00585..9f30ff87a70 100644
--- a/searchlib/src/vespa/searchlib/attribute/attributemanager.h
+++ b/searchlib/src/vespa/searchlib/attribute/attributemanager.h
@@ -49,6 +49,8 @@ public:
     void getAttributeList(AttributeList & list) const override;
     attribute::IAttributeContext::UP createContext() const override;
 
+    std::shared_ptr<attribute::ReadableAttributeVector> readable_attribute_vector(const string& name) const override;
+
     const Snapshot & getSnapshot()         const { return _snapShot; }
     const string & getBaseDir()       const { return _baseDir; }
     void setBaseDir(const string & base);
diff --git a/searchlib/src/vespa/searchlib/attribute/attributememorysavetarget.cpp b/searchlib/src/vespa/searchlib/attribute/attributememorysavetarget.cpp
index 372168143ab..b28887691e5 100644
--- a/searchlib/src/vespa/searchlib/attribute/attributememorysavetarget.cpp
+++ b/searchlib/src/vespa/searchlib/attribute/attributememorysavetarget.cpp
@@ -1,24 +1,25 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
-#include "attributememorysavetarget.h"
 #include "attributefilesavetarget.h"
+#include "attributememorysavetarget.h"
 #include "attributevector.h"
+#include <vespa/vespalib/util/exceptions.h>
 
 namespace search {
 
 using search::common::FileHeaderContext;
+using vespalib::IllegalArgumentException;
 
 AttributeMemorySaveTarget::AttributeMemorySaveTarget()
     : _datWriter(),
       _idxWriter(),
       _weightWriter(),
-      _udatWriter()
+      _udatWriter(),
+      _writers()
 {
 }
 
-AttributeMemorySaveTarget::~AttributeMemorySaveTarget() {
-}
-
+AttributeMemorySaveTarget::~AttributeMemorySaveTarget() = default;
 
 IAttributeFileWriter &
 AttributeMemorySaveTarget::datWriter()
@@ -26,28 +27,24 @@ AttributeMemorySaveTarget::datWriter()
     return _datWriter;
 }
 
-
 IAttributeFileWriter &
 AttributeMemorySaveTarget::idxWriter()
 {
     return _idxWriter;
 }
 
-
 IAttributeFileWriter &
 AttributeMemorySaveTarget::weightWriter()
 {
     return _weightWriter;
 }
 
-
 IAttributeFileWriter &
 AttributeMemorySaveTarget::udatWriter()
 {
     return _udatWriter;
 }
 
-
 bool
 AttributeMemorySaveTarget::
 writeToFile(const TuneFileAttributes &tuneFileAttributes,
@@ -68,9 +65,39 @@ writeToFile(const TuneFileAttributes &tuneFileAttributes,
             _weightWriter.writeTo(saveTarget.weightWriter());
         }
     }
+    for (const auto& entry : _writers) {
+        if (!saveTarget.setup_writer(entry.first, entry.second.desc)) {
+            return false;
+        }
+        auto& file_writer = saveTarget.get_writer(entry.first);
+        entry.second.writer->writeTo(file_writer);
+    }
     saveTarget.close();
     return true;
 }
 
+bool
+AttributeMemorySaveTarget::setup_writer(const vespalib::string& file_suffix,
+                                        const vespalib::string& desc)
+{
+    auto writer = std::make_unique<AttributeMemoryFileWriter>();
+    auto itr = _writers.find(file_suffix);
+    if (itr != _writers.end()) {
+        return false;
+    }
+    _writers.insert(std::make_pair(file_suffix, WriterEntry(std::move(writer), desc)));
+    return true;
+}
+
+IAttributeFileWriter&
+AttributeMemorySaveTarget::get_writer(const vespalib::string& file_suffix)
+{
+    auto itr = _writers.find(file_suffix);
+    if (itr == _writers.end()) {
+        throw IllegalArgumentException("File writer with suffix '" + file_suffix + "' does not exist");
+    }
+    return *itr->second.writer;
+}
+
 } // namespace search
 
diff --git a/searchlib/src/vespa/searchlib/attribute/attributememorysavetarget.h b/searchlib/src/vespa/searchlib/attribute/attributememorysavetarget.h
index f06764fa34b..9533b881099 100644
--- a/searchlib/src/vespa/searchlib/attribute/attributememorysavetarget.h
+++ b/searchlib/src/vespa/searchlib/attribute/attributememorysavetarget.h
@@ -2,11 +2,13 @@
 
 #pragma once
 
-#include "iattributesavetarget.h"
 #include "attributememoryfilewriter.h"
+#include "iattributesavetarget.h"
+#include <vespa/searchlib/common/tunefileinfo.h>
 #include <vespa/searchlib/util/rawbuf.h>
+#include <vespa/vespalib/stllike/hash_fun.h>
 #include <memory>
-#include <vespa/searchlib/common/tunefileinfo.h>
+#include <unordered_map>
 
 namespace search::common { class FileHeaderContext; }
 
@@ -16,13 +18,22 @@ class AttributeVector;
 /**
  * Class used to save an attribute vector to memory buffer(s).
  **/
-class AttributeMemorySaveTarget : public IAttributeSaveTarget
-{
+class AttributeMemorySaveTarget : public IAttributeSaveTarget {
 private:
+    using FileWriterUP = std::unique_ptr<AttributeMemoryFileWriter>;
+    struct WriterEntry {
+        FileWriterUP writer;
+        vespalib::string desc;
+        WriterEntry(FileWriterUP writer_in, const vespalib::string& desc_in)
+            : writer(std::move(writer_in)), desc(desc_in) {}
+    };
+    using WriterMap = std::unordered_map<vespalib::string, WriterEntry, vespalib::hash<vespalib::string>>;
+
     AttributeMemoryFileWriter _datWriter;
     AttributeMemoryFileWriter _idxWriter;
     AttributeMemoryFileWriter _weightWriter;
     AttributeMemoryFileWriter _udatWriter;
+    WriterMap _writers;
 
 public:
     AttributeMemorySaveTarget();
@@ -40,6 +51,11 @@ public:
     IAttributeFileWriter &idxWriter() override;
     IAttributeFileWriter &weightWriter() override;
     IAttributeFileWriter &udatWriter() override;
+
+    bool setup_writer(const vespalib::string& file_suffix,
+                      const vespalib::string& desc) override;
+    IAttributeFileWriter& get_writer(const vespalib::string& file_suffix) override;
+
 };
 
 } // namespace search
diff --git a/searchlib/src/vespa/searchlib/attribute/attributevector.cpp b/searchlib/src/vespa/searchlib/attribute/attributevector.cpp
index b043bb4aaf8..08c7186e8c7 100644
--- a/searchlib/src/vespa/searchlib/attribute/attributevector.cpp
+++ b/searchlib/src/vespa/searchlib/attribute/attributevector.cpp
@@ -11,6 +11,7 @@
 #include "ipostinglistattributebase.h"
 #include "ipostinglistsearchcontext.h"
 #include "stringbase.h"
+#include <vespa/document/update/assignvalueupdate.h>
 #include <vespa/document/update/mapvalueupdate.h>
 #include <vespa/fastlib/io/bufferedfile.h>
 #include <vespa/searchlib/common/tunefileinfo.h>
@@ -28,6 +29,7 @@ LOG_SETUP(".searchlib.attribute.attributevector");
 using vespalib::getLastErrorString;
 
 using document::ValueUpdate;
+using document::AssignValueUpdate;
 using vespalib::make_string;
 using vespalib::Array;
 using vespalib::IllegalStateException;
@@ -266,79 +268,6 @@ const IEnumStore* AttributeVector::getEnumStoreBase() const { return nullptr; }
 IEnumStore* AttributeVector::getEnumStoreBase() { return nullptr; }
 const attribute::MultiValueMappingBase * AttributeVector::getMultiValueBase() const { return nullptr; }
 
-std::unique_ptr<FastOS_FileInterface>
-AttributeVector::openFile(const char *suffix)
-{
-    BaseName::string fileName(getBaseFileName());
-    fileName += suffix;
-    return FileUtil::openFile(fileName);
-}
-
-
-std::unique_ptr<FastOS_FileInterface>
-AttributeVector::openDAT()
-{
-    return openFile(".dat");
-}
-
-
-std::unique_ptr<FastOS_FileInterface>
-AttributeVector::openIDX()
-{
-    return openFile(".idx");
-}
-
-
-std::unique_ptr<FastOS_FileInterface>
-AttributeVector::openWeight()
-{
-    return openFile(".weight");
-}
-
-
-std::unique_ptr<FastOS_FileInterface>
-AttributeVector::openUDAT()
-{
-    return openFile(".dat");
-}
-
-fileutil::LoadedBuffer::UP
-AttributeVector::loadDAT()
-{
-    return loadFile(".dat");
-}
-
-
-fileutil::LoadedBuffer::UP
-AttributeVector::loadIDX()
-{
-    return loadFile(".idx");
-}
-
-
-fileutil::LoadedBuffer::UP
-AttributeVector::loadWeight()
-{
-    return loadFile(".weight");
-}
-
-
-fileutil::LoadedBuffer::UP
-AttributeVector::loadUDAT()
-{
-    return loadFile(".udat");
-}
-
-
-fileutil::LoadedBuffer::UP
-AttributeVector::loadFile(const char *suffix)
-{
-    BaseName::string fileName(getBaseFileName());
-    fileName += suffix;
-    return FileUtil::loadFile(fileName);
-}
-
-
 bool
 AttributeVector::save(vespalib::stringref fileName)
 {
@@ -379,19 +308,19 @@ AttributeVector::save(IAttributeSaveTarget &saveTarget, vespalib::stringref file
 attribute::AttributeHeader
 AttributeVector::createAttributeHeader(vespalib::stringref fileName) const {
     return attribute::AttributeHeader(fileName,
-                                   getConfig().basicType(),
-                                   getConfig().collectionType(),
-                                   getConfig().basicType().type() == BasicType::Type::TENSOR
-                                      ? getConfig().tensorType()
-                                      : vespalib::eval::ValueType::error_type(),
-                                   getEnumeratedSave(),
-                                   getConfig().predicateParams(),
-                                   getCommittedDocIdLimit(),
-                                   getFixedWidth(),
-                                   getUniqueValueCount(),
-                                   getTotalValueCount(),
-                                   getCreateSerialNum(),
-                                   getVersion());
+                                      getConfig().basicType(),
+                                      getConfig().collectionType(),
+                                      (getConfig().basicType().type() == BasicType::Type::TENSOR
+                                       ? getConfig().tensorType()
+                                       : vespalib::eval::ValueType::error_type()),
+                                      getEnumeratedSave(),
+                                      getConfig().predicateParams(),
+                                      getConfig().hnsw_index_params(),
+                                      getCommittedDocIdLimit(),
+                                      getUniqueValueCount(),
+                                      getTotalValueCount(),
+                                      getCreateSerialNum(),
+                                      getVersion());
 }
 
 void AttributeVector::onSave(IAttributeSaveTarget &)
@@ -540,6 +469,9 @@ AttributeVector::apply(DocId doc, const MapValueUpdate &map) {
         if (vu.inherits(ArithmeticValueUpdate::classId)) {
             const ArithmeticValueUpdate &au(static_cast<const ArithmeticValueUpdate &>(vu));
             retval = applyWeight(doc, map.getKey(), au);
+        } else if (vu.inherits(AssignValueUpdate::classId)) {
+            const AssignValueUpdate &au(static_cast<const AssignValueUpdate &>(vu));
+            retval = applyWeight(doc, map.getKey(), au);
         } else {
             retval = false;
         }
@@ -550,6 +482,7 @@ AttributeVector::apply(DocId doc, const MapValueUpdate &map) {
 
 bool AttributeVector::applyWeight(DocId, const FieldValue &, const ArithmeticValueUpdate &) { return false; }
 
+bool AttributeVector::applyWeight(DocId, const FieldValue&, const AssignValueUpdate&) { return false; }
 
 void
 AttributeVector::removeAllOldGenerations() {
diff --git a/searchlib/src/vespa/searchlib/attribute/attributevector.h b/searchlib/src/vespa/searchlib/attribute/attributevector.h
index b5f6beaa718..75699868691 100644
--- a/searchlib/src/vespa/searchlib/attribute/attributevector.h
+++ b/searchlib/src/vespa/searchlib/attribute/attributevector.h
@@ -30,6 +30,7 @@ class FastOS_FileInterface;
 
 namespace document {
     class ArithmeticValueUpdate;
+    class AssignValueUpdate;
     class MapValueUpdate;
     class FieldValue;
 }
@@ -212,11 +213,6 @@ protected:
     void setNumDocs(uint32_t n)          { _status.setNumDocs(n); }
     void incNumDocs()                    { _status.incNumDocs(); }
 
-    LoadedBufferUP loadDAT();
-    LoadedBufferUP loadIDX();
-    LoadedBufferUP loadWeight();
-    LoadedBufferUP loadUDAT();
-
     class ValueModifier
     {
     public:
@@ -269,10 +265,6 @@ protected:
     }
     
 public:
-    std::unique_ptr<FastOS_FileInterface> openDAT();
-    std::unique_ptr<FastOS_FileInterface> openIDX();
-    std::unique_ptr<FastOS_FileInterface> openWeight();
-    std::unique_ptr<FastOS_FileInterface> openUDAT();
     void incGeneration();
     void removeAllOldGenerations();
 
@@ -320,6 +312,10 @@ protected:
         return _genHolder;
     }
 
+    const GenerationHolder& getGenerationHolder() const {
+        return _genHolder;
+    }
+
     template<typename T>
     bool clearDoc(ChangeVectorT< ChangeTemplate<T> > &changes, DocId doc);
 
@@ -337,6 +333,9 @@ protected:
     template<typename T>
     bool adjustWeight(ChangeVectorT< ChangeTemplate<T> > &changes, DocId doc, const T &v, const ArithmeticValueUpdate &wd);
 
+    template<typename T>
+    bool adjustWeight(ChangeVectorT< ChangeTemplate<T> > &changes, DocId doc, const T &v, const document::AssignValueUpdate &wu);
+
     template <typename T>
     static int32_t
     applyWeightChange(int32_t weight, const ChangeTemplate<T> &weightChange) {
@@ -346,6 +345,8 @@ protected:
             return weight * weightChange._weight;
         } else if (weightChange._type == ChangeBase::DIVWEIGHT) {
             return weight / weightChange._weight;
+        } else if (weightChange._type == ChangeBase::SETWEIGHT) {
+            return weightChange._weight;
         }
         return weight;
     }
@@ -566,10 +567,9 @@ private:
     virtual void onAddDocs(DocId docIdLimit) = 0;
     void divideByZeroWarning();
     virtual bool applyWeight(DocId doc, const FieldValue &fv, const ArithmeticValueUpdate &wAdjust);
+    virtual bool applyWeight(DocId doc, const FieldValue& fv, const document::AssignValueUpdate& wAdjust);
     virtual void onSave(IAttributeSaveTarget & saveTarget);
     virtual bool onLoad();
-    std::unique_ptr<FastOS_FileInterface> openFile(const char *suffix);
-    LoadedBufferUP loadFile(const char *suffix);
 
 
     BaseName                              _baseFileName;
diff --git a/searchlib/src/vespa/searchlib/attribute/attributevector.hpp b/searchlib/src/vespa/searchlib/attribute/attributevector.hpp
index 66c2678497a..efc96bc57c2 100644
--- a/searchlib/src/vespa/searchlib/attribute/attributevector.hpp
+++ b/searchlib/src/vespa/searchlib/attribute/attributevector.hpp
@@ -3,7 +3,9 @@
 
 #include "attributevector.h"
 #include "integerbase.h"
+#include <vespa/document/fieldvalue/intfieldvalue.h>
 #include <vespa/document/update/arithmeticvalueupdate.h>
+#include <vespa/document/update/assignvalueupdate.h>
 #include <cmath>
 
 namespace search {
@@ -61,6 +63,32 @@ AttributeVector::adjustWeight(ChangeVectorT< ChangeTemplate<T> > & changes, DocI
 
 template<typename T>
 bool
+AttributeVector::adjustWeight(ChangeVectorT< ChangeTemplate<T> >& changes, DocId doc, const T& v, const document::AssignValueUpdate& wu)
+{
+    bool retval(hasWeightedSetType() && (doc < getNumDocs()));
+    if (retval) {
+        size_t oldSz(changes.size());
+        if (wu.hasValue()) {
+            const FieldValue &wv = wu.getValue();
+            if (wv.inherits(document::IntFieldValue::classId)) {
+                changes.push_back(ChangeTemplate<T>(ChangeBase::SETWEIGHT, doc, v, wv.getAsInt()));
+            } else {
+                retval = false;
+            }
+        } else {
+            retval = false;
+        }
+        if (retval) {
+            const size_t diff = changes.size() - oldSz;
+            _status.incNonIdempotentUpdates(diff);
+            _status.incUpdates(diff);
+        }
+    }
+    return retval;
+}
+
+template<typename T>
+bool
 AttributeVector::applyArithmetic(ChangeVectorT< ChangeTemplate<T> > & changes, DocId doc, const T & v,
                                  const ArithmeticValueUpdate & arithm)
 {
diff --git a/searchlib/src/vespa/searchlib/attribute/attrvector.cpp b/searchlib/src/vespa/searchlib/attribute/attrvector.cpp
index 0304aa8f38e..59771d7ffae 100644
--- a/searchlib/src/vespa/searchlib/attribute/attrvector.cpp
+++ b/searchlib/src/vespa/searchlib/attribute/attrvector.cpp
@@ -3,6 +3,7 @@
 #include "attrvector.h"
 #include "attrvector.hpp"
 #include "iattributesavetarget.h"
+#include "load_utils.h"
 
 #include <vespa/log/log.h>
 LOG_SETUP(".searchlib.attribute.attr_vector");
@@ -123,7 +124,7 @@ bool StringDirectAttribute::onLoad()
         setCommittedDocIdLimit(0);
     }
 
-    fileutil::LoadedBuffer::UP tmpBuffer(loadDAT());
+    auto tmpBuffer = attribute::LoadUtils::loadDAT(*this);
     bool rc(tmpBuffer.get());
     if (rc) {
         if ( ! tmpBuffer->empty()) {
@@ -158,7 +159,7 @@ bool StringDirectAttribute::onLoad()
         }
 
         if (hasMultiValue()) {
-            fileutil::LoadedBuffer::UP tmpIdx(loadIDX());
+            auto tmpIdx = attribute::LoadUtils::loadIDX(*this);
             size_t tmpIdxLen(tmpIdx->size(sizeof(uint32_t)));
             _idx.clear();
             _idx.reserve(tmpIdxLen);
diff --git a/searchlib/src/vespa/searchlib/attribute/attrvector.hpp b/searchlib/src/vespa/searchlib/attribute/attrvector.hpp
index cdd34725e69..4ce7575b28d 100644
--- a/searchlib/src/vespa/searchlib/attribute/attrvector.hpp
+++ b/searchlib/src/vespa/searchlib/attribute/attrvector.hpp
@@ -2,6 +2,7 @@
 #pragma once
 
 #include "attrvector.h"
+#include "load_utils.h"
 #include <vespa/vespalib/util/hdr_abort.h>
 #include <vespa/fastlib/io/bufferedfile.h>
 #include <vespa/searchlib/util/filekit.h>
@@ -23,7 +24,7 @@ NumericDirectAttribute<B>::~NumericDirectAttribute() = default;
 template <typename B>
 bool NumericDirectAttribute<B>::onLoad()
 {
-    fileutil::LoadedBuffer::UP dataBuffer(B::loadDAT());
+    auto dataBuffer = attribute::LoadUtils::loadDAT(*this);
     bool rc(dataBuffer.get());
     if (rc) {
         const BaseType * tmpData(static_cast <const BaseType *>(dataBuffer->buffer()));
@@ -56,7 +57,7 @@ bool NumericDirectAttribute<B>::onLoad()
         }
         dataBuffer.reset();
         if (this->hasMultiValue()) {
-            fileutil::LoadedBuffer::UP idxBuffer(B::loadIDX());
+            auto idxBuffer = attribute::LoadUtils::loadIDX(*this);
             rc = idxBuffer.get();
             if (rc) {
                 const uint32_t * tmpIdx(static_cast<const uint32_t *>(idxBuffer->buffer()));
diff --git a/searchlib/src/vespa/searchlib/attribute/changevector.h b/searchlib/src/vespa/searchlib/attribute/changevector.h
index 75eac911f6c..af4a4fd6618 100644
--- a/searchlib/src/vespa/searchlib/attribute/changevector.h
+++ b/searchlib/src/vespa/searchlib/attribute/changevector.h
@@ -18,6 +18,7 @@ struct ChangeBase {
         INCREASEWEIGHT,
         MULWEIGHT,
         DIVWEIGHT,
+        SETWEIGHT,
         ADD,
         SUB,
         MUL,
diff --git a/searchlib/src/vespa/searchlib/attribute/configconverter.cpp b/searchlib/src/vespa/searchlib/attribute/configconverter.cpp
index 535e81fc032..c573f0b4210 100644
--- a/searchlib/src/vespa/searchlib/attribute/configconverter.cpp
+++ b/searchlib/src/vespa/searchlib/attribute/configconverter.cpp
@@ -73,6 +73,24 @@ ConfigConverter::convert(const AttributesConfig::Attribute & cfg)
     predicateParams.setBounds(cfg.lowerbound, cfg.upperbound);
     predicateParams.setDensePostingListThreshold(cfg.densepostinglistthreshold);
     retval.setPredicateParams(predicateParams);
+    if (cfg.index.hnsw.enabled) {
+        using CfgDm = AttributesConfig::Attribute::Index::Hnsw::Distancemetric;
+        DistanceMetric dm;
+        switch (cfg.index.hnsw.distancemetric) {
+        case CfgDm::EUCLIDEAN:
+            dm = DistanceMetric::Euclidean;
+            break;
+        case CfgDm::ANGULAR:
+            dm = DistanceMetric::Angular;
+            break;
+        case CfgDm::GEODEGREES:
+            dm = DistanceMetric::GeoDegrees;
+            break;
+        }
+        retval.set_hnsw_index_params(HnswIndexParams(cfg.index.hnsw.maxlinkspernode,
+                                                     cfg.index.hnsw.neighborstoexploreatinsert,
+                                                     dm));
+    }
     if (retval.basicType().type() == BasicType::Type::TENSOR) {
         if (!cfg.tensortype.empty()) {
             retval.setTensorType(ValueType::from_spec(cfg.tensortype));
diff --git a/searchlib/src/vespa/searchlib/attribute/createsinglefastsearch.cpp b/searchlib/src/vespa/searchlib/attribute/createsinglefastsearch.cpp
index cdb7e1807ba..0e1cfaa3c09 100644
--- a/searchlib/src/vespa/searchlib/attribute/createsinglefastsearch.cpp
+++ b/searchlib/src/vespa/searchlib/attribute/createsinglefastsearch.cpp
@@ -5,15 +5,9 @@
 #include "floatbase.h"
 #include "defines.h"
 #include "singlestringattribute.h"
+#include "singleboolattribute.h"
 #include "singlestringpostattribute.hpp"
-#include "singlenumericenumattribute.hpp"
 #include "singlenumericpostattribute.hpp"
-#include "enumstore.hpp"
-#include "enumattribute.hpp"
-#include "singleenumattribute.hpp"
-
-#include <vespa/log/log.h>
-LOG_SETUP(".searchlib.attribute.create_single_fast_search");
 
 #define INTPOSTING(T)   SingleValueNumericPostingAttribute< ENUM_ATTRIBUTE(IntegerAttributeTemplate<T>) >
 #define FLOATPOSTING(T) SingleValueNumericPostingAttribute< ENUM_ATTRIBUTE(FloatingPointAttributeTemplate<T>) >
@@ -27,37 +21,30 @@ AttributeFactory::createSingleFastSearch(stringref name, const Config & info)
 {
     assert(info.collectionType().type() == attribute::CollectionType::SINGLE);
     assert(info.fastSearch());
-    AttributeVector::SP ret;
     switch(info.basicType().type()) {
     case BasicType::BOOL:
+        return std::make_shared<SingleBoolAttribute>(name, info.getGrowStrategy());
     case BasicType::UINT2:
     case BasicType::UINT4:
         break;
     case BasicType::INT8:
-        ret.reset(new INTPOSTING(int8_t)(name, info));
-        break;
+        return std::make_shared<INTPOSTING(int8_t)>(name, info);
     case BasicType::INT16:
-        ret.reset(new INTPOSTING(int16_t)(name, info));
-        break;
+        return std::make_shared<INTPOSTING(int16_t)>(name, info);
     case BasicType::INT32:
-        ret.reset(new INTPOSTING(int32_t)(name, info));
-        break;
+        return std::make_shared<INTPOSTING(int32_t)>(name, info);
     case BasicType::INT64:
-        ret.reset(new INTPOSTING(int64_t)(name, info));
-        break;
+        return std::make_shared<INTPOSTING(int64_t)>(name, info);
     case BasicType::FLOAT:
-        ret.reset(new FLOATPOSTING(float)(name, info));
-        break;
+        return std::make_shared<FLOATPOSTING(float)>(name, info);
     case BasicType::DOUBLE:
-        ret.reset(new FLOATPOSTING(double)(name, info));
-        break;
+        return std::make_shared<FLOATPOSTING(double)>(name, info);
     case BasicType::STRING:
-        ret.reset(new SingleValueStringPostingAttribute(name, info));
-        break;
+        return std::make_shared<SingleValueStringPostingAttribute>(name, info);
     default:
         break;
     }
-    return ret;
+    return AttributeVector::SP();
 }
 
 }
diff --git a/searchlib/src/vespa/searchlib/attribute/createsinglestd.cpp b/searchlib/src/vespa/searchlib/attribute/createsinglestd.cpp
index 7bd96b5cbb2..a0cf47f64e0 100644
--- a/searchlib/src/vespa/searchlib/attribute/createsinglestd.cpp
+++ b/searchlib/src/vespa/searchlib/attribute/createsinglestd.cpp
@@ -4,16 +4,12 @@
 #include "predicate_attribute.h"
 #include "singlesmallnumericattribute.h"
 #include "reference_attribute.h"
-#include "attributevector.hpp"
 #include "singlenumericattribute.hpp"
 #include "singlestringattribute.h"
 #include "singleboolattribute.h"
 #include <vespa/searchlib/tensor/generic_tensor_attribute.h>
 #include <vespa/searchlib/tensor/dense_tensor_attribute.h>
 
-#include <vespa/log/log.h>
-LOG_SETUP(".searchlib.attribute.create_single_std");
-
 namespace search {
 
 using attribute::BasicType;
@@ -22,55 +18,42 @@ AttributeVector::SP
 AttributeFactory::createSingleStd(stringref name, const Config & info)
 {
     assert(info.collectionType().type() == attribute::CollectionType::SINGLE);
-    AttributeVector::SP ret;
     switch(info.basicType().type()) {
     case BasicType::BOOL:
-        ret.reset(new SingleBoolAttribute(name, info.getGrowStrategy()));
-        break;
+        return std::make_shared<SingleBoolAttribute>(name, info.getGrowStrategy());
     case BasicType::UINT2:
-        ret.reset(new SingleValueSemiNibbleNumericAttribute(name, info.getGrowStrategy()));
-        break;
+        return std::make_shared<SingleValueSemiNibbleNumericAttribute>(name, info.getGrowStrategy());
     case BasicType::UINT4:
-        ret.reset(new SingleValueNibbleNumericAttribute(name, info.getGrowStrategy()));
-        break;
+        return std::make_shared<SingleValueNibbleNumericAttribute>(name, info.getGrowStrategy());
     case BasicType::INT8:
-        ret.reset(new SingleValueNumericAttribute<IntegerAttributeTemplate<int8_t> >(name, info));
-        break;
+        return std::make_shared<SingleValueNumericAttribute<IntegerAttributeTemplate<int8_t>>>(name, info);
     case BasicType::INT16:
         // XXX: Unneeded since we don't have short document fields in java.
-        ret.reset(new SingleValueNumericAttribute<IntegerAttributeTemplate<int16_t> >(name, info));
-        break;
+        return std::make_shared<SingleValueNumericAttribute<IntegerAttributeTemplate<int16_t>>>(name, info);
     case BasicType::INT32:
-        ret.reset(new SingleValueNumericAttribute<IntegerAttributeTemplate<int32_t> >(name, info));
-        break;
+        return std::make_shared<SingleValueNumericAttribute<IntegerAttributeTemplate<int32_t>>>(name, info);
     case BasicType::INT64:
-        ret.reset(new SingleValueNumericAttribute<IntegerAttributeTemplate<int64_t> >(name, info));
-        break;
+        return std::make_shared<SingleValueNumericAttribute<IntegerAttributeTemplate<int64_t>>>(name, info);
     case BasicType::FLOAT:
-        ret.reset(new SingleValueNumericAttribute<FloatingPointAttributeTemplate<float> >(name, info));
-        break;
+        return std::make_shared<SingleValueNumericAttribute<FloatingPointAttributeTemplate<float>>>(name, info);
     case BasicType::DOUBLE:
-        ret.reset(new SingleValueNumericAttribute<FloatingPointAttributeTemplate<double> >(name, info));
-        break;
+        return std::make_shared<SingleValueNumericAttribute<FloatingPointAttributeTemplate<double>>>(name, info);
     case BasicType::STRING:
-        ret.reset(new SingleValueStringAttribute(name, info));
-        break;
+        return std::make_shared<SingleValueStringAttribute>(name, info);
     case BasicType::PREDICATE:
-        ret.reset(new PredicateAttribute(name, info));
-        break;
+        return std::make_shared<PredicateAttribute>(name, info);
     case BasicType::TENSOR:
         if (info.tensorType().is_dense()) {
-            ret.reset(new tensor::DenseTensorAttribute(name, info));
+            return std::make_shared<tensor::DenseTensorAttribute>(name, info);
         } else {
-            ret.reset(new tensor::GenericTensorAttribute(name, info));
+            return std::make_shared<tensor::GenericTensorAttribute>(name, info);
         }
-        break;
     case BasicType::REFERENCE:
-        ret = std::make_shared<attribute::ReferenceAttribute>(name, info);
-        break;
+        return std::make_shared<attribute::ReferenceAttribute>(name, info);
     default:
         break;
     }
-    return ret;
+    return AttributeVector::SP();
 }
+
 }  // namespace search
diff --git a/searchlib/src/vespa/searchlib/attribute/flagattribute.cpp b/searchlib/src/vespa/searchlib/attribute/flagattribute.cpp
index 1e4bba95b4b..895e6a6f4c0 100644
--- a/searchlib/src/vespa/searchlib/attribute/flagattribute.cpp
+++ b/searchlib/src/vespa/searchlib/attribute/flagattribute.cpp
@@ -89,7 +89,7 @@ FlagAttributeT<B>::onLoadEnumerated(ReaderBase &attrReader)
     if (numValues > 0)
         _bitVectorSize = numDocs;
 
-    fileutil::LoadedBuffer::UP udatBuffer(this->loadUDAT());
+    auto udatBuffer = attribute::LoadUtils::loadUDAT(*this);
     assert((udatBuffer->size() % sizeof(TT)) == 0);
     vespalib::ConstArrayRef<TT> map(reinterpret_cast<const TT *>(udatBuffer->buffer()),
                                     udatBuffer->size() / sizeof(TT));
diff --git a/searchlib/src/vespa/searchlib/attribute/floatbase.cpp b/searchlib/src/vespa/searchlib/attribute/floatbase.cpp
index 813180bc5d7..4ffc6e333b6 100644
--- a/searchlib/src/vespa/searchlib/attribute/floatbase.cpp
+++ b/searchlib/src/vespa/searchlib/attribute/floatbase.cpp
@@ -76,6 +76,12 @@ bool FloatingPointAttribute::applyWeight(DocId doc, const FieldValue & fv, const
     return AttributeVector::adjustWeight(_changes, doc, NumericChangeData<double>(v), wAdjust);
 }
 
+bool FloatingPointAttribute::applyWeight(DocId doc, const FieldValue& fv, const document::AssignValueUpdate& wAdjust)
+{
+    double v = fv.getAsDouble();
+    return AttributeVector::adjustWeight(_changes, doc, NumericChangeData<double>(v), wAdjust);
+}
+
 bool FloatingPointAttribute::apply(DocId doc, const ArithmeticValueUpdate & op)
 {
     bool retval(doc < getNumDocs());
diff --git a/searchlib/src/vespa/searchlib/attribute/floatbase.h b/searchlib/src/vespa/searchlib/attribute/floatbase.h
index ce8d22c2706..65a146d7ba7 100644
--- a/searchlib/src/vespa/searchlib/attribute/floatbase.h
+++ b/searchlib/src/vespa/searchlib/attribute/floatbase.h
@@ -28,6 +28,7 @@ public:
     }
     bool apply(DocId doc, const ArithmeticValueUpdate & op);
     bool applyWeight(DocId doc, const FieldValue & fv, const ArithmeticValueUpdate & wAdjust) override;
+    bool applyWeight(DocId doc, const FieldValue& fv, const document::AssignValueUpdate& wAdjust) override;
     uint32_t clearDoc(DocId doc) override;
 protected:
     const char * getString(DocId doc, char * s, size_t sz) const override;
diff --git a/searchlib/src/vespa/searchlib/attribute/iattributemanager.h b/searchlib/src/vespa/searchlib/attribute/iattributemanager.h
index ef9403e1c4b..51e432f2035 100644
--- a/searchlib/src/vespa/searchlib/attribute/iattributemanager.h
+++ b/searchlib/src/vespa/searchlib/attribute/iattributemanager.h
@@ -8,7 +8,10 @@
 
 namespace search {
 
-namespace attribute { class AttributeReadGuard; }
+namespace attribute {
+class AttributeReadGuard;
+class ReadableAttributeVector;
+}
 
 /**
  * This is an interface used to access all registered attribute vectors.
@@ -17,12 +20,17 @@ class IAttributeManager : public attribute::IAttributeExecutor {
 public:
     IAttributeManager(const IAttributeManager &) = delete;
     IAttributeManager & operator = (const IAttributeManager &) = delete;
-    typedef std::shared_ptr<IAttributeManager> SP;
-    typedef vespalib::string string;
+    using SP = std::shared_ptr<IAttributeManager>;
+    using string = vespalib::string;
 
     /**
      * Returns a view of the attribute vector with the given name.
      *
+     * NOTE: this method is deprecated! Prefer using readable_attribute_vector(name) instead,
+     * as that enforces appropriate guards to be taken before accessing the underlying vector.
+     *
+     * TODO remove this when all usages are gone.
+     *
      * @param name name of the attribute vector.
      * @return view of the attribute vector or empty view if the attribute vector does not exists.
      **/
@@ -52,9 +60,15 @@ public:
     virtual attribute::IAttributeContext::UP createContext() const = 0;
 
     /**
-     * Virtual destructor to allow safe subclassing.
-     **/
-    virtual ~IAttributeManager() {}
+     * Looks up and returns a readable attribute vector shared_ptr with the provided name.
+     * This transparently supports imported attribute vectors.
+     *
+     * @param name name of the attribute vector.
+     * @return The attribute vector, or an empty shared_ptr if no vector was found with the given name.
+     */
+    virtual std::shared_ptr<attribute::ReadableAttributeVector> readable_attribute_vector(const string& name) const = 0;
+
+    ~IAttributeManager() override = default;
 protected:
     IAttributeManager() = default;
 };
diff --git a/searchlib/src/vespa/searchlib/attribute/iattributesavetarget.h b/searchlib/src/vespa/searchlib/attribute/iattributesavetarget.h
index 9f90544bb83..8946fc2fcdb 100644
--- a/searchlib/src/vespa/searchlib/attribute/iattributesavetarget.h
+++ b/searchlib/src/vespa/searchlib/attribute/iattributesavetarget.h
@@ -37,6 +37,19 @@ public:
     virtual IAttributeFileWriter &weightWriter() = 0;
     virtual IAttributeFileWriter &udatWriter() = 0;
 
+    /**
+     * Setups a custom file writer with the given file suffix and description in the file header.
+     * Returns false if the file writer cannot be setup or if it already exists, true otherwise.
+     */
+    virtual bool setup_writer(const vespalib::string& file_suffix,
+                              const vespalib::string& desc) = 0;
+
+    /**
+     * Returns the file writer with the given file suffix.
+     * Throws vespalib::IllegalArgumentException if the file writer does not exists.
+     */
+    virtual IAttributeFileWriter& get_writer(const vespalib::string& file_suffix) = 0;
+
     virtual ~IAttributeSaveTarget();
 };
 
diff --git a/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.cpp b/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.cpp
index b097b27ea53..f2949119ae2 100644
--- a/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.cpp
+++ b/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.cpp
@@ -152,6 +152,10 @@ bool ImportedAttributeVectorReadGuard::isImported() const
     return true;
 }
 
+bool ImportedAttributeVectorReadGuard::isUndefined(DocId doc) const {
+    return _target_attribute.isUndefined(getTargetLid(doc));
+}
+
 long ImportedAttributeVectorReadGuard::onSerializeForAscendingSort(DocId doc,
                                                                    void *serTo,
                                                                    long available,
diff --git a/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.h b/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.h
index f130a095006..b31b11b8c74 100644
--- a/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.h
+++ b/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.h
@@ -81,6 +81,7 @@ public:
     bool getIsFastSearch() const override;
     uint32_t getCommittedDocIdLimit() const override;
     bool isImported() const override;
+    bool isUndefined(DocId doc) const override;
 
 protected:
     long onSerializeForAscendingSort(DocId doc, void * serTo, long available,
diff --git a/searchlib/src/vespa/searchlib/attribute/integerbase.cpp b/searchlib/src/vespa/searchlib/attribute/integerbase.cpp
index f065bc63648..a150b5e1699 100644
--- a/searchlib/src/vespa/searchlib/attribute/integerbase.cpp
+++ b/searchlib/src/vespa/searchlib/attribute/integerbase.cpp
@@ -80,6 +80,12 @@ bool IntegerAttribute::applyWeight(DocId doc, const FieldValue & fv, const Arith
     return AttributeVector::adjustWeight(_changes, doc, NumericChangeData<largeint_t>(v), wAdjust);
 }
 
+bool IntegerAttribute::applyWeight(DocId doc, const FieldValue& fv, const document::AssignValueUpdate& wAdjust)
+{
+    largeint_t v = fv.getAsLong();
+    return AttributeVector::adjustWeight(_changes, doc, NumericChangeData<largeint_t>(v), wAdjust);
+}
+
 bool IntegerAttribute::apply(DocId doc, const ArithmeticValueUpdate & op)
 {
     bool retval(doc < getNumDocs());
diff --git a/searchlib/src/vespa/searchlib/attribute/integerbase.h b/searchlib/src/vespa/searchlib/attribute/integerbase.h
index 46774f9cebf..4e5fe8fec19 100644
--- a/searchlib/src/vespa/searchlib/attribute/integerbase.h
+++ b/searchlib/src/vespa/searchlib/attribute/integerbase.h
@@ -28,6 +28,7 @@ public:
     }
     bool apply(DocId doc, const ArithmeticValueUpdate & op);
     bool applyWeight(DocId doc, const FieldValue & fv, const ArithmeticValueUpdate & wAdjust) override;
+    bool applyWeight(DocId doc, const FieldValue& fv, const document::AssignValueUpdate& wAdjust) override;
     uint32_t clearDoc(DocId doc) override;
 protected:
     IntegerAttribute(const vespalib::string & name, const Config & c);
diff --git a/searchlib/src/vespa/searchlib/attribute/load_utils.cpp b/searchlib/src/vespa/searchlib/attribute/load_utils.cpp
index 041daa08cd5..701c8eaf702 100644
--- a/searchlib/src/vespa/searchlib/attribute/load_utils.cpp
+++ b/searchlib/src/vespa/searchlib/attribute/load_utils.cpp
@@ -5,13 +5,81 @@
 #include "loadedenumvalue.h"
 #include "multi_value_mapping.h"
 #include "multivalue.h"
+#include <vespa/fastos/file.h>
+#include <vespa/searchlib/util/fileutil.h>
+#include <vespa/vespalib/io/fileutil.h>
 #include <vespa/vespalib/util/array.hpp>
 
 using search::multivalue::Value;
 using search::multivalue::WeightedValue;
 
-namespace search {
-namespace attribute {
+namespace search::attribute {
+
+using FileInterfaceUP = LoadUtils::FileInterfaceUP;
+using LoadedBufferUP = LoadUtils::LoadedBufferUP;
+
+FileInterfaceUP
+LoadUtils::openFile(const AttributeVector& attr, const vespalib::string& suffix)
+{
+    return FileUtil::openFile(attr.getBaseFileName() + "." + suffix);
+}
+
+
+
+FileInterfaceUP
+LoadUtils::openDAT(const AttributeVector& attr)
+{
+    return openFile(attr, "dat");
+}
+
+FileInterfaceUP
+LoadUtils::openIDX(const AttributeVector& attr)
+{
+    return openFile(attr, "idx");
+}
+
+FileInterfaceUP
+LoadUtils::openWeight(const AttributeVector& attr)
+{
+    return openFile(attr, "weight");
+}
+
+bool
+LoadUtils::file_exists(const AttributeVector& attr, const vespalib::string& suffix)
+{
+    return vespalib::fileExists(attr.getBaseFileName() + "." + suffix);
+}
+
+LoadedBufferUP
+LoadUtils::loadFile(const AttributeVector& attr, const vespalib::string& suffix)
+{
+    return FileUtil::loadFile(attr.getBaseFileName() + "." + suffix);
+}
+
+LoadedBufferUP
+LoadUtils::loadDAT(const AttributeVector& attr)
+{
+    return loadFile(attr, "dat");
+}
+
+LoadedBufferUP
+LoadUtils::loadIDX(const AttributeVector& attr)
+{
+    return loadFile(attr, "idx");
+}
+
+LoadedBufferUP
+LoadUtils::loadWeight(const AttributeVector& attr)
+{
+    return loadFile(attr, "weight");
+}
+
+LoadedBufferUP
+LoadUtils::loadUDAT(const AttributeVector& attr)
+{
+    return loadFile(attr, "udat");
+}
+
 
 #define INSTANTIATE_ARRAY(ValueType, Saver) \
 template uint32_t loadFromEnumeratedMultiValue(MultiValueMapping<Value<ValueType>> &, ReaderBase &, vespalib::ConstArrayRef<ValueType>, Saver)
@@ -40,5 +108,4 @@ INSTANTIATE_VALUE(int64_t);
 INSTANTIATE_VALUE(float);
 INSTANTIATE_VALUE(double);
 
-} // namespace search::attribute
-} // namespace search
+}
diff --git a/searchlib/src/vespa/searchlib/attribute/load_utils.h b/searchlib/src/vespa/searchlib/attribute/load_utils.h
index 050d7726ecd..cd9d98084d5 100644
--- a/searchlib/src/vespa/searchlib/attribute/load_utils.h
+++ b/searchlib/src/vespa/searchlib/attribute/load_utils.h
@@ -6,10 +6,34 @@
 #include "readerbase.h"
 #include <vespa/vespalib/util/arrayref.h>
 
-namespace search {
-namespace attribute {
+namespace search::attribute {
 
-/*
+/**
+ * Helper functions used to open / load attribute vector data files from disk.
+ */
+class LoadUtils {
+public:
+    using FileInterfaceUP = std::unique_ptr<FastOS_FileInterface>;
+    using LoadedBufferUP = std::unique_ptr<fileutil::LoadedBuffer>;
+
+private:
+    static FileInterfaceUP openFile(const AttributeVector& attr, const vespalib::string& suffix);
+
+public:
+    static FileInterfaceUP openDAT(const AttributeVector& attr);
+    static FileInterfaceUP openIDX(const AttributeVector& attr);
+    static FileInterfaceUP openWeight(const AttributeVector& attr);
+
+    static bool file_exists(const AttributeVector& attr, const vespalib::string& suffix);
+    static LoadedBufferUP loadFile(const AttributeVector& attr, const vespalib::string& suffix);
+
+    static LoadedBufferUP loadDAT(const AttributeVector& attr);
+    static LoadedBufferUP loadIDX(const AttributeVector& attr);
+    static LoadedBufferUP loadWeight(const AttributeVector& attr);
+    static LoadedBufferUP loadUDAT(const AttributeVector& attr);
+};
+
+/**
  * Function for loading mapping from document id to array of enum indexes
  * or values from enumerated attribute reader.
  */
@@ -20,7 +44,7 @@ loadFromEnumeratedMultiValue(MvMapping &mapping,
                              vespalib::ConstArrayRef<typename MvMapping::MultiValueType::ValueType> enumValueToValueMap,
                              Saver saver) __attribute((noinline));
 
-/*
+/**
  * Function for loading mapping from document id to enum index or
  * value from enumerated attribute reader.
  */
@@ -32,5 +56,4 @@ loadFromEnumeratedSingleValue(Vector &vector,
                               vespalib::ConstArrayRef<typename Vector::ValueType> enumValueToValueMap,
                               Saver saver) __attribute((noinline));
 
-} // namespace search::attribute
-} // namespace search
+}
diff --git a/searchlib/src/vespa/searchlib/attribute/loadedenumvalue.h b/searchlib/src/vespa/searchlib/attribute/loadedenumvalue.h
index 275fadd1e7f..c91fdd297f8 100644
--- a/searchlib/src/vespa/searchlib/attribute/loadedenumvalue.h
+++ b/searchlib/src/vespa/searchlib/attribute/loadedenumvalue.h
@@ -6,6 +6,7 @@
 #include <vespa/vespalib/util/array.h>
 #include <vespa/vespalib/util/arrayref.h>
 #include <cassert>
+#include <limits>
 
 namespace search
 {
diff --git a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.hpp b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.hpp
index 3dcc0df477d..07f8077ae97 100644
--- a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.hpp
+++ b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.hpp
@@ -42,9 +42,9 @@ template <typename EntryT, typename RefT>
 void
 MultiValueMapping<EntryT,RefT>::replace(uint32_t docId, ConstArrayRef values)
 {
-    ConstArrayRef oldValues = _store.get(_indices[docId]);
+    auto oldValues = _store.get_writable(_indices[docId]);
     assert(oldValues.size() == values.size());
-    EntryT *dst = const_cast<EntryT *>(&oldValues[0]);
+    EntryT *dst = &oldValues[0];
     for (auto &src : values) {
         *dst = src;
         ++dst;
diff --git a/searchlib/src/vespa/searchlib/attribute/multienumattribute.hpp b/searchlib/src/vespa/searchlib/attribute/multienumattribute.hpp
index 219b3dd2473..40f211c2621 100644
--- a/searchlib/src/vespa/searchlib/attribute/multienumattribute.hpp
+++ b/searchlib/src/vespa/searchlib/attribute/multienumattribute.hpp
@@ -37,7 +37,7 @@ MultiValueEnumAttribute<B, M>::considerAttributeChange(const Change & c, UniqueS
 {
     if (c._type == ChangeBase::APPEND ||
         (this->getInternalCollectionType().createIfNonExistant() &&
-         (c._type >= ChangeBase::INCREASEWEIGHT && c._type <= ChangeBase::DIVWEIGHT)))
+         (c._type >= ChangeBase::INCREASEWEIGHT && c._type <= ChangeBase::SETWEIGHT)))
     {
         EnumIndex idx;
         if (!this->_enumStore.find_index(c._data.raw(), idx)) {
diff --git a/searchlib/src/vespa/searchlib/attribute/multinumericattribute.hpp b/searchlib/src/vespa/searchlib/attribute/multinumericattribute.hpp
index 1efa2789fcb..3ca7423c38c 100644
--- a/searchlib/src/vespa/searchlib/attribute/multinumericattribute.hpp
+++ b/searchlib/src/vespa/searchlib/attribute/multinumericattribute.hpp
@@ -117,7 +117,7 @@ MultiValueNumericAttribute<B, M>::onLoadEnumerated(ReaderBase & attrReader)
     this->setCommittedDocIdLimit(numDocs);
     this->_mvMapping.reserve(numDocs+1);
 
-    LoadedBuffer::UP udatBuffer(this->loadUDAT());
+    auto udatBuffer = attribute::LoadUtils::loadUDAT(*this);
     assert((udatBuffer->size() % sizeof(T)) == 0);
     vespalib::ConstArrayRef<T> map(reinterpret_cast<const T *>(udatBuffer->buffer()), udatBuffer->size() / sizeof(T));
     uint32_t maxvc = attribute::loadFromEnumeratedMultiValue(this->_mvMapping, attrReader, map, attribute::NoSaveLoadedEnum());
diff --git a/searchlib/src/vespa/searchlib/attribute/multinumericenumattribute.hpp b/searchlib/src/vespa/searchlib/attribute/multinumericenumattribute.hpp
index 9ee365fc7cc..e17d41a5521 100644
--- a/searchlib/src/vespa/searchlib/attribute/multinumericenumattribute.hpp
+++ b/searchlib/src/vespa/searchlib/attribute/multinumericenumattribute.hpp
@@ -2,13 +2,14 @@
 
 #pragma once
 
-#include "multinumericenumattribute.h"
-#include "loadednumericvalue.h"
 #include "attributeiterators.hpp"
-#include <vespa/searchlib/util/fileutil.hpp>
+#include "load_utils.h"
+#include "loadednumericvalue.h"
+#include "multinumericenumattribute.h"
 #include <vespa/fastlib/io/bufferedfile.h>
 #include <vespa/searchlib/query/query_term_simple.h>
 #include <vespa/searchlib/queryeval/emptysearch.h>
+#include <vespa/searchlib/util/fileutil.hpp>
 
 namespace search {
 
@@ -52,7 +53,7 @@ template <typename B, typename M>
 bool
 MultiValueNumericEnumAttribute<B, M>::onLoadEnumerated(ReaderBase &attrReader)
 {
-    LoadedBuffer::UP udatBuffer(this->loadUDAT());
+    auto udatBuffer = attribute::LoadUtils::loadUDAT(*this);
 
     uint32_t numDocs = attrReader.getNumIdx() - 1;
     uint64_t numValues = attrReader.getNumValues();
diff --git a/searchlib/src/vespa/searchlib/attribute/multistringattribute.hpp b/searchlib/src/vespa/searchlib/attribute/multistringattribute.hpp
index 6cefc03dd70..eafa5bf0e1f 100644
--- a/searchlib/src/vespa/searchlib/attribute/multistringattribute.hpp
+++ b/searchlib/src/vespa/searchlib/attribute/multistringattribute.hpp
@@ -123,7 +123,7 @@ StringTemplSearchContext(QueryTermSimpleUP qTerm, const AttrType & toBeSearched)
             auto comp = enumStore.make_folded_comparator(queryTerm()->getTerm(), true);
             lookupRange(comp, comp);
         } else if (this->isRegex()) {
-            vespalib::string prefix(vespalib::Regexp::get_prefix(this->queryTerm()->getTerm()));
+            vespalib::string prefix(vespalib::RegexpUtil::get_prefix(this->queryTerm()->getTerm()));
             auto comp = enumStore.make_folded_comparator(prefix.c_str(), true);
             lookupRange(comp, comp);
         } else {
diff --git a/searchlib/src/vespa/searchlib/attribute/multivalueattribute.hpp b/searchlib/src/vespa/searchlib/attribute/multivalueattribute.hpp
index b23860ebd31..0ae536aee23 100644
--- a/searchlib/src/vespa/searchlib/attribute/multivalueattribute.hpp
+++ b/searchlib/src/vespa/searchlib/attribute/multivalueattribute.hpp
@@ -172,7 +172,7 @@ MultiValueAttribute<B, M>::apply_attribute_changes_to_wset(DocumentValues& docVa
                 wset_inserted[data] = current->_weight;
             } else if (current->_type == ChangeBase::REMOVE) {
                 wset_inserted.erase(data);
-            } else if ((current->_type >= ChangeBase::INCREASEWEIGHT) && (current->_type <= ChangeBase::DIVWEIGHT)) {
+            } else if ((current->_type >= ChangeBase::INCREASEWEIGHT) && (current->_type <= ChangeBase::SETWEIGHT)) {
                 auto existing = wset_inserted.find(data);
                 if (existing != wset_inserted.end()) {
                     existing->second = this->applyWeightChange(existing->second, *current);
diff --git a/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.h b/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.h
index 43d8b7ce9d2..e94de44e45b 100644
--- a/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.h
+++ b/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.h
@@ -183,14 +183,14 @@ private:
     using PostingList = typename AggregationTraits::PostingList;
     using Parent = PostingSearchContext<BaseSC, PostingListFoldedSearchContextT<DataT>, AttrT>;
     using FoldedComparatorType = typename Parent::EnumStore::FoldedComparatorType;
-    using Regexp = vespalib::Regexp;
+    using RegexpUtil = vespalib::RegexpUtil;
     using QueryTermSimpleUP = typename Parent::QueryTermSimpleUP;
     using Parent::_toBeSearched;
     using Parent::_enumStore;
     using Parent::isRegex;
     using Parent::getRegex;
     bool useThis(const PostingListSearchContext::DictionaryConstIterator & it) const override {
-        return isRegex() ? (getRegex() ? std::regex_search(_enumStore.get_value(it.getKey()), *getRegex()) : false ) : true;
+        return isRegex() ? (getRegex() ? getRegex()->partial_match(_enumStore.get_value(it.getKey())) : false ) : true;
     }
 public:
     StringPostingSearchContext(QueryTermSimpleUP qTerm, bool useBitVector, const AttrT &toBeSearched);
@@ -288,7 +288,7 @@ StringPostingSearchContext(QueryTermSimpleUP qTerm, bool useBitVector, const Att
             auto comp = _enumStore.make_folded_comparator(this->queryTerm()->getTerm(), true);
             this->lookupRange(comp, comp);
         } else if (this->isRegex()) {
-            vespalib::string prefix(Regexp::get_prefix(this->queryTerm()->getTerm()));
+            vespalib::string prefix(RegexpUtil::get_prefix(this->queryTerm()->getTerm()));
             auto comp = _enumStore.make_folded_comparator(prefix.c_str(), true);
             this->lookupRange(comp, comp);
         } else {
diff --git a/searchlib/src/vespa/searchlib/attribute/predicate_attribute.cpp b/searchlib/src/vespa/searchlib/attribute/predicate_attribute.cpp
index b0e4df65c2b..72cc6e38fac 100644
--- a/searchlib/src/vespa/searchlib/attribute/predicate_attribute.cpp
+++ b/searchlib/src/vespa/searchlib/attribute/predicate_attribute.cpp
@@ -1,12 +1,13 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
-#include "predicate_attribute.h"
-#include "iattributesavetarget.h"
 #include "attribute_header.h"
-#include <vespa/searchlib/predicate/predicate_index.h>
-#include <vespa/searchlib/util/fileutil.h>
+#include "iattributesavetarget.h"
+#include "load_utils.h"
+#include "predicate_attribute.h"
 #include <vespa/document/fieldvalue/predicatefieldvalue.h>
 #include <vespa/document/predicate/predicate.h>
+#include <vespa/searchlib/predicate/predicate_index.h>
+#include <vespa/searchlib/util/fileutil.h>
 #include <vespa/vespalib/data/slime/slime.h>
 
 #include <vespa/log/log.h>
@@ -183,7 +184,7 @@ struct DummyObserver : SimpleIndexDeserializeObserver<> {
 
 bool PredicateAttribute::onLoad()
 {
-    fileutil::LoadedBuffer::UP loaded_buffer = loadDAT();
+    auto loaded_buffer = attribute::LoadUtils::loadDAT(*this);
     char *rawBuffer = const_cast<char *>(static_cast<const char *>(loaded_buffer->buffer()));
     size_t size = loaded_buffer->size();
     DataBuffer buffer(rawBuffer, size);
diff --git a/searchlib/src/vespa/searchlib/attribute/readable_attribute_vector.h b/searchlib/src/vespa/searchlib/attribute/readable_attribute_vector.h
index 03058362cb1..0e3954bdcb5 100644
--- a/searchlib/src/vespa/searchlib/attribute/readable_attribute_vector.h
+++ b/searchlib/src/vespa/searchlib/attribute/readable_attribute_vector.h
@@ -13,7 +13,7 @@ class AttributeReadGuard;
  */
 class ReadableAttributeVector {
 public:
-    virtual ~ReadableAttributeVector() {}
+    virtual ~ReadableAttributeVector() = default;
     virtual std::unique_ptr<AttributeReadGuard> makeReadGuard(bool stableEnumGuard) const = 0;
 };
 
diff --git a/searchlib/src/vespa/searchlib/attribute/readerbase.cpp b/searchlib/src/vespa/searchlib/attribute/readerbase.cpp
index 62936ecaaf4..a396fe9efd8 100644
--- a/searchlib/src/vespa/searchlib/attribute/readerbase.cpp
+++ b/searchlib/src/vespa/searchlib/attribute/readerbase.cpp
@@ -1,10 +1,11 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
-#include "readerbase.h"
 #include "attributevector.h"
+#include "load_utils.h"
+#include "readerbase.h"
 #include <vespa/fastlib/io/bufferedfile.h>
-#include <vespa/vespalib/util/exceptions.h>
 #include <vespa/searchlib/util/filesizecalculator.h>
+#include <vespa/vespalib/util/exceptions.h>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".search.attribute.readerbase");
@@ -12,28 +13,27 @@ LOG_SETUP(".search.attribute.readerbase");
 namespace search {
 
 namespace {
-    const vespalib::string versionTag = "version";
-    const vespalib::string docIdLimitTag = "docIdLimit";
-    const vespalib::string createSerialNumTag = "createSerialNum";
 
-    constexpr size_t DIRECTIO_ALIGNMENT(4096);
+const vespalib::string versionTag = "version";
+const vespalib::string docIdLimitTag = "docIdLimit";
+const vespalib::string createSerialNumTag = "createSerialNum";
 
-    uint64_t
-    extractCreateSerialNum(const vespalib::GenericHeader &header)
-    {
-        return (header.hasTag(createSerialNumTag)) ? header.getTag(createSerialNumTag).asInteger() : 0u;
-    }
+constexpr size_t DIRECTIO_ALIGNMENT(4096);
 
+uint64_t
+extractCreateSerialNum(const vespalib::GenericHeader &header)
+{
+    return (header.hasTag(createSerialNumTag)) ? header.getTag(createSerialNumTag).asInteger() : 0u;
+}
 
 }
 
 ReaderBase::ReaderBase(AttributeVector &attr)
-    : _datFile(attr.openDAT()),
+    : _datFile(attribute::LoadUtils::openDAT(attr)),
       _weightFile(attr.hasWeightedSetType() ?
-                  attr.openWeight() : std::unique_ptr<Fast_BufferedFile>()),
+                  attribute::LoadUtils::openWeight(attr) : std::unique_ptr<Fast_BufferedFile>()),
       _idxFile(attr.hasMultiValue() ?
-               attr.openIDX() : std::unique_ptr<Fast_BufferedFile>()),
-      _udatFile(),
+               attribute::LoadUtils::openIDX(attr) : std::unique_ptr<Fast_BufferedFile>()),
       _weightReader(*_weightFile),
       _idxReader(*_idxFile),
       _enumReader(*_datFile),
@@ -41,7 +41,6 @@ ReaderBase::ReaderBase(AttributeVector &attr)
       _datHeaderLen(0u),
       _idxHeaderLen(0u),
       _weightHeaderLen(0u),
-      _udatHeaderLen(0u),
       _createSerialNum(0u),
       _fixedWidth(attr.getFixedWidth()),
       _enumerated(false),
@@ -83,20 +82,12 @@ ReaderBase::ReaderBase(AttributeVector &attr)
     }
     if (hasData() && AttributeVector::isEnumerated(_datHeader)) {
         _enumerated = true;
-        _udatFile = attr.openUDAT();
-        vespalib::FileHeader udatHeader(DIRECTIO_ALIGNMENT);
-        _udatHeaderLen = udatHeader.readFile(*_udatFile);
-        _udatFile->SetPosition(_udatHeaderLen);
-        if (!attr.headerTypeOK(udatHeader))
-            _udatFile->Close();
     }
     _hasLoadData = hasData() &&
                    (!attr.hasMultiValue() || hasIdx()) &&
-                   (!attr.hasWeightedSetType() || hasWeight()) &&
-                   (!getEnumerated() || hasUData());
+                   (!attr.hasWeightedSetType() || hasWeight());
 }
 
-
 ReaderBase::~ReaderBase() = default;
 
 bool
@@ -115,11 +106,6 @@ ReaderBase::hasData() const {
 }
 
 bool
-ReaderBase::hasUData() const {
-    return _udatFile.get() && _udatFile->IsOpened();
-}
-
-bool
 ReaderBase::
 extractFileSize(const vespalib::GenericHeader &header,
                 FastOS_FileInterface &file, uint64_t &fileSize)
@@ -129,7 +115,6 @@ extractFileSize(const vespalib::GenericHeader &header,
                                                file.GetFileName(), fileSize);
 }
 
-
 void
 ReaderBase::rewind()
 {
@@ -142,12 +127,8 @@ ReaderBase::rewind()
     if (hasWeight()) {
         _weightFile->SetPosition(_weightHeaderLen);
     }
-    if (getEnumerated()) {
-        _udatFile->SetPosition(_udatHeaderLen);
-    }
 }
 
-
 size_t
 ReaderBase::getNumValues()
 {
@@ -169,7 +150,6 @@ ReaderBase::getNumValues()
     }
 }
 
-
 uint32_t
 ReaderBase::getNextValueCount()
 {
diff --git a/searchlib/src/vespa/searchlib/attribute/readerbase.h b/searchlib/src/vespa/searchlib/attribute/readerbase.h
index 09db52f5e25..a7685e4532a 100644
--- a/searchlib/src/vespa/searchlib/attribute/readerbase.h
+++ b/searchlib/src/vespa/searchlib/attribute/readerbase.h
@@ -19,7 +19,6 @@ public:
     bool hasWeight() const;
     bool hasIdx() const;
     bool hasData() const;
-    bool hasUData() const;
 
     uint32_t getNumIdx() const {
         return (_idxFileSize - _idxHeaderLen) /sizeof(uint32_t);
@@ -51,7 +50,6 @@ protected:
 private:
     std::unique_ptr<FastOS_FileInterface>  _weightFile;
     std::unique_ptr<FastOS_FileInterface>  _idxFile;
-    std::unique_ptr<FastOS_FileInterface>  _udatFile;
     FileReader<int32_t>   _weightReader;
     FileReader<uint32_t>  _idxReader;
     FileReader<uint32_t>  _enumReader;
@@ -59,7 +57,6 @@ private:
     uint32_t              _datHeaderLen;
     uint32_t              _idxHeaderLen;
     uint32_t              _weightHeaderLen;
-    uint32_t              _udatHeaderLen;
     uint64_t              _createSerialNum;
     size_t                _fixedWidth;
     bool                  _enumerated;
diff --git a/searchlib/src/vespa/searchlib/attribute/reference_attribute.cpp b/searchlib/src/vespa/searchlib/attribute/reference_attribute.cpp
index b055af7c084..9421730f335 100644
--- a/searchlib/src/vespa/searchlib/attribute/reference_attribute.cpp
+++ b/searchlib/src/vespa/searchlib/attribute/reference_attribute.cpp
@@ -1,6 +1,7 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include "attributesaver.h"
+#include "load_utils.h"
 #include "readerbase.h"
 #include "reference_attribute.h"
 #include "reference_attribute_saver.h"
@@ -223,7 +224,7 @@ ReferenceAttribute::onLoad()
     uint64_t numValues(0);
     numValues = attrReader.getEnumCount();
     numDocs = numValues;
-    fileutil::LoadedBuffer::UP udatBuffer(loadUDAT());
+    auto udatBuffer = attribute::LoadUtils::loadUDAT(*this);
     const GenericHeader &header = udatBuffer->getHeader();
     uint32_t uniqueValueCount = extractUniqueValueCount(header);
     assert(uniqueValueCount * sizeof(GlobalId) == udatBuffer->size());
diff --git a/searchlib/src/vespa/searchlib/attribute/singleboolattribute.cpp b/searchlib/src/vespa/searchlib/attribute/singleboolattribute.cpp
index fd6e68f3473..e1ab47ed434 100644
--- a/searchlib/src/vespa/searchlib/attribute/singleboolattribute.cpp
+++ b/searchlib/src/vespa/searchlib/attribute/singleboolattribute.cpp
@@ -104,7 +104,7 @@ private:
     bool _valid;
     bool valid() const override { return _valid; }
     int32_t onFind(DocId docId, int32_t elemId, int32_t & weight) const override final {
-        if ((elemId == 0) && _bv.testBit(docId)) {
+        if ((elemId == 0) && (_invert != _bv.testBit(docId))) {
             weight = 1;
             return 0;
         }
@@ -113,7 +113,7 @@ private:
     }
 
     int32_t onFind(DocId docId, int32_t elemId) const override final {
-        return ((elemId == 0) && _bv.testBit(docId)) ? 0 : -1;
+        return ((elemId == 0) && (_invert != _bv.testBit(docId))) ? 0 : -1;
     }
 
 public:
diff --git a/searchlib/src/vespa/searchlib/attribute/singleboolattribute.h b/searchlib/src/vespa/searchlib/attribute/singleboolattribute.h
index 78c297d55c3..579c77a382d 100644
--- a/searchlib/src/vespa/searchlib/attribute/singleboolattribute.h
+++ b/searchlib/src/vespa/searchlib/attribute/singleboolattribute.h
@@ -12,7 +12,7 @@ namespace search {
  * Attributevector for boolean field values occupying a bit per document
  * and backed by a growable rcu bit vector.
  */
-class SingleBoolAttribute : public IntegerAttributeTemplate<int8_t>
+class SingleBoolAttribute final : public IntegerAttributeTemplate<int8_t>
 {
 public:
     SingleBoolAttribute(const vespalib::string & baseFileName, const search::GrowStrategy & grow);
diff --git a/searchlib/src/vespa/searchlib/attribute/singlenumericattribute.hpp b/searchlib/src/vespa/searchlib/attribute/singlenumericattribute.hpp
index 69d4e6a5ee9..681c2af1f07 100644
--- a/searchlib/src/vespa/searchlib/attribute/singlenumericattribute.hpp
+++ b/searchlib/src/vespa/searchlib/attribute/singlenumericattribute.hpp
@@ -1,12 +1,12 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 #pragma once
 
-#include "singlenumericattribute.h"
+#include "attributeiterators.hpp"
 #include "attributevector.hpp"
-#include "singlenumericattributesaver.h"
 #include "load_utils.h"
 #include "primitivereader.h"
-#include "attributeiterators.hpp"
+#include "singlenumericattribute.h"
+#include "singlenumericattributesaver.h"
 #include <vespa/searchlib/query/query_term_simple.h>
 #include <vespa/searchlib/queryeval/emptysearch.h>
 
@@ -114,7 +114,7 @@ SingleValueNumericAttribute<B>::onLoadEnumerated(ReaderBase &attrReader)
     this->setCommittedDocIdLimit(numDocs);
     _data.unsafe_reserve(numDocs);
 
-    fileutil::LoadedBuffer::UP udatBuffer(this->loadUDAT());
+    auto udatBuffer = attribute::LoadUtils::loadUDAT(*this);
     assert((udatBuffer->size() % sizeof(T)) == 0);
     vespalib::ConstArrayRef<T> map(reinterpret_cast<const T *>(udatBuffer->buffer()),
                                    udatBuffer->size() / sizeof(T));
diff --git a/searchlib/src/vespa/searchlib/attribute/singlenumericenumattribute.hpp b/searchlib/src/vespa/searchlib/attribute/singlenumericenumattribute.hpp
index 990388d2a12..5fb587c908e 100644
--- a/searchlib/src/vespa/searchlib/attribute/singlenumericenumattribute.hpp
+++ b/searchlib/src/vespa/searchlib/attribute/singlenumericenumattribute.hpp
@@ -2,14 +2,15 @@
 
 #pragma once
 
-#include "singlenumericenumattribute.h"
-#include <vespa/searchlib/common/sort.h>
-#include "singleenumattribute.hpp"
+#include "attributeiterators.hpp"
+#include "load_utils.h"
 #include "loadednumericvalue.h"
 #include "primitivereader.h"
-#include "attributeiterators.hpp"
-#include <vespa/searchlib/queryeval/emptysearch.h>
+#include "singleenumattribute.hpp"
+#include "singlenumericenumattribute.h"
+#include <vespa/searchlib/common/sort.h>
 #include <vespa/searchlib/query/query_term_simple.h>
+#include <vespa/searchlib/queryeval/emptysearch.h>
 #include <vespa/searchlib/util/fileutil.hpp>
 
 namespace search {
@@ -79,7 +80,7 @@ template <typename B>
 bool
 SingleValueNumericEnumAttribute<B>::onLoadEnumerated(ReaderBase &attrReader)
 {
-    fileutil::LoadedBuffer::UP udatBuffer(this->loadUDAT());
+    auto udatBuffer = attribute::LoadUtils::loadUDAT(*this);
 
     uint64_t numValues = attrReader.getEnumCount();
     uint32_t numDocs = numValues;
diff --git a/searchlib/src/vespa/searchlib/attribute/singlenumericpostattribute.hpp b/searchlib/src/vespa/searchlib/attribute/singlenumericpostattribute.hpp
index ca6b6caba32..b330be3a2bb 100644
--- a/searchlib/src/vespa/searchlib/attribute/singlenumericpostattribute.hpp
+++ b/searchlib/src/vespa/searchlib/attribute/singlenumericpostattribute.hpp
@@ -2,10 +2,10 @@
 
 #pragma once
 
-#include <vespa/searchlib/attribute/singlenumericpostattribute.h>
-#include <vespa/searchlib/attribute/enumstore.h>
-#include <vespa/searchlib/attribute/enumcomparator.h>
-#include <vespa/searchlib/attribute/singlenumericenumattribute.hpp>
+#include "singlenumericpostattribute.h"
+#include "enumstore.h"
+#include "enumcomparator.h"
+#include "singlenumericenumattribute.hpp"
 
 namespace search {
 
@@ -141,9 +141,7 @@ AttributeVector::SearchContext::UP
 SingleValueNumericPostingAttribute<B>::getSearch(QueryTermSimple::UP qTerm,
                                                  const attribute::SearchContextParams & params) const
 {
-    return std::make_unique<SinglePostingSearchContext>(std::move(qTerm),
-                                                        params,
-                                                        *this);
+    return std::make_unique<SinglePostingSearchContext>(std::move(qTerm), params, *this);
 }
 
 } // namespace search
diff --git a/searchlib/src/vespa/searchlib/attribute/singlestringattribute.hpp b/searchlib/src/vespa/searchlib/attribute/singlestringattribute.hpp
index 214da6bf230..406cbbbe447 100644
--- a/searchlib/src/vespa/searchlib/attribute/singlestringattribute.hpp
+++ b/searchlib/src/vespa/searchlib/attribute/singlestringattribute.hpp
@@ -59,7 +59,7 @@ SingleValueStringAttributeT<B>::StringTemplSearchContext::StringTemplSearchConte
             auto comp = enumStore.make_folded_comparator(queryTerm()->getTerm(), true);
             lookupRange(comp, comp);
         } else if (this->isRegex()) {
-            vespalib::string prefix(vespalib::Regexp::get_prefix(this->queryTerm()->getTerm()));
+            vespalib::string prefix(vespalib::RegexpUtil::get_prefix(this->queryTerm()->getTerm()));
             auto comp = enumStore.make_folded_comparator(prefix.c_str(), true);
             lookupRange(comp, comp);
         } else {
diff --git a/searchlib/src/vespa/searchlib/attribute/singlestringpostattribute.hpp b/searchlib/src/vespa/searchlib/attribute/singlestringpostattribute.hpp
index 59a5a9b8cac..cdf1acedb24 100644
--- a/searchlib/src/vespa/searchlib/attribute/singlestringpostattribute.hpp
+++ b/searchlib/src/vespa/searchlib/attribute/singlestringpostattribute.hpp
@@ -2,7 +2,7 @@
 
 #pragma once
 
-#include <vespa/searchlib/attribute/singlestringpostattribute.h>
+#include "singlestringpostattribute.h"
 #include <vespa/searchlib/query/query_term_ucs4.h>
 
 namespace search {
diff --git a/searchlib/src/vespa/searchlib/attribute/stringbase.cpp b/searchlib/src/vespa/searchlib/attribute/stringbase.cpp
index d7523c86e29..d888ead21bf 100644
--- a/searchlib/src/vespa/searchlib/attribute/stringbase.cpp
+++ b/searchlib/src/vespa/searchlib/attribute/stringbase.cpp
@@ -1,11 +1,12 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
-#include "stringbase.h"
 #include "attributevector.hpp"
+#include "load_utils.h"
 #include "readerbase.h"
+#include "stringbase.h"
 #include <vespa/document/fieldvalue/fieldvalue.h>
-#include <vespa/searchlib/util/fileutil.hpp>
 #include <vespa/searchlib/query/query_term_ucs4.h>
+#include <vespa/searchlib/util/fileutil.hpp>
 #include <vespa/vespalib/locale/c.h>
 #include <vespa/vespalib/util/array.hpp>
 
@@ -231,10 +232,7 @@ StringAttribute::StringSearchContext::StringSearchContext(QueryTermSimple::UP qT
     _regex()
 {
     if (isRegex()) {
-        try {
-            _regex = std::regex(_queryTerm->getTerm(), std::regex::icase);
-        } catch (std::regex_error &) {
-        }
+        _regex = vespalib::Regex::from_pattern(_queryTerm->getTerm(), vespalib::Regex::Options::IgnoreCase);
     }
 }
 
@@ -311,6 +309,12 @@ bool StringAttribute::applyWeight(DocId doc, const FieldValue & fv, const Arithm
     return AttributeVector::adjustWeight(_changes, doc, StringChangeData(v), wAdjust);
 }
 
+bool StringAttribute::applyWeight(DocId doc, const FieldValue& fv, const document::AssignValueUpdate& wAdjust)
+{
+    vespalib::string v = fv.getAsString();
+    return AttributeVector::adjustWeight(_changes, doc, StringChangeData(v), wAdjust);
+}
+
 bool StringAttribute::apply(DocId, const ArithmeticValueUpdate & )
 {
     return false;
@@ -319,7 +323,7 @@ bool StringAttribute::apply(DocId, const ArithmeticValueUpdate & )
 bool
 StringAttribute::onLoadEnumerated(ReaderBase &attrReader)
 {
-    fileutil::LoadedBuffer::UP udatBuffer(loadUDAT());
+    auto udatBuffer = attribute::LoadUtils::loadUDAT(*this);
 
     bool hasIdx(attrReader.hasIdx());
     size_t numDocs(0);
diff --git a/searchlib/src/vespa/searchlib/attribute/stringbase.h b/searchlib/src/vespa/searchlib/attribute/stringbase.h
index cf0a92253de..e6238cc0f94 100644
--- a/searchlib/src/vespa/searchlib/attribute/stringbase.h
+++ b/searchlib/src/vespa/searchlib/attribute/stringbase.h
@@ -9,10 +9,10 @@
 #include <vespa/searchlib/attribute/i_enum_store.h>
 #include <vespa/searchlib/attribute/loadedenumvalue.h>
 #include <vespa/searchlib/util/foldedstringcompare.h>
+#include <vespa/vespalib/regex/regex.h>
 #include <vespa/vespalib/text/lowercase.h>
 #include <vespa/vespalib/text/utf8.h>
 #include <optional>
-#include <regex>
 
 namespace search {
 
@@ -44,6 +44,7 @@ public:
     }
     bool apply(DocId doc, const ArithmeticValueUpdate & op);
     bool applyWeight(DocId doc, const FieldValue & fv, const ArithmeticValueUpdate & wAdjust) override;
+    bool applyWeight(DocId doc, const FieldValue& fv, const document::AssignValueUpdate& wAdjust) override;
     bool findEnum(const char * value, EnumHandle & e) const override = 0;
     std::vector<EnumHandle> findFoldedEnums(const char *value) const override = 0;
     uint32_t get(DocId doc, largeint_t * v, uint32_t sz) const override;
@@ -103,7 +104,7 @@ protected:
         const QueryTermUCS4 * queryTerm() const override;
         bool isMatch(const char *src) const {
             if (__builtin_expect(isRegex(), false)) {
-                return _regex ? std::regex_search(src, *_regex) : false;
+                return _regex ? _regex->partial_match(std::string_view(src)) : false;
             }
             vespalib::Utf8ReaderForZTS u8reader(src);
             uint32_t j = 0;
@@ -162,7 +163,7 @@ protected:
         bool  isRegex() const { return _isRegex; }
         QueryTermSimpleUP         _queryTerm;
         std::vector<ucs4_t>       _termUCS4;
-        const std::optional<std::regex>& getRegex() const { return _regex; }
+        const std::optional<vespalib::Regex>& getRegex() const { return _regex; }
     private:
         WeightedConstChar * getBuffer() const {
             if (_buffer == nullptr) {
@@ -170,9 +171,9 @@ protected:
             }
             return _buffer;
         }
-        unsigned                    _bufferLen;
-        mutable WeightedConstChar * _buffer;
-        std::optional<std::regex>   _regex;
+        unsigned                       _bufferLen;
+        mutable WeightedConstChar *    _buffer;
+        std::optional<vespalib::Regex> _regex;
     };
 private:
     SearchContext::UP getSearch(QueryTermSimpleUP term, const attribute::SearchContextParams & params) const override;
diff --git a/searchlib/src/vespa/searchlib/common/CMakeLists.txt b/searchlib/src/vespa/searchlib/common/CMakeLists.txt
index 2ee722902c8..36264a2035b 100644
--- a/searchlib/src/vespa/searchlib/common/CMakeLists.txt
+++ b/searchlib/src/vespa/searchlib/common/CMakeLists.txt
@@ -11,11 +11,9 @@ vespa_add_library(searchlib_common OBJECT
     documentsummary.cpp
     featureset.cpp
     fileheadercontext.cpp
-    foregroundtaskexecutor.cpp
     gatecallback.cpp
     growablebitvector.cpp
     indexmetainfo.cpp
-    isequencedtaskexecutor.cpp
     location.cpp
     locationiterators.cpp
     mapnames.cpp
@@ -23,8 +21,6 @@ vespa_add_library(searchlib_common OBJECT
     packets.cpp
     partialbitvector.cpp
     resultset.cpp
-    sequencedtaskexecutor.cpp
-    sequencedtaskexecutorobserver.cpp
     serialnumfileheadercontext.cpp
     sort.cpp
     sortdata.cpp
diff --git a/searchlib/src/vespa/searchlib/common/allocatedbitvector.cpp b/searchlib/src/vespa/searchlib/common/allocatedbitvector.cpp
index 3de0c1f1320..08a3847f00d 100644
--- a/searchlib/src/vespa/searchlib/common/allocatedbitvector.cpp
+++ b/searchlib/src/vespa/searchlib/common/allocatedbitvector.cpp
@@ -52,6 +52,7 @@ AllocatedBitVector::AllocatedBitVector(Index numberOfElements, Index capacityBit
         }
         setBit(size()); // Guard bit
     }
+    updateCount();
 }
 
 AllocatedBitVector::AllocatedBitVector(const AllocatedBitVector & rhs) :
@@ -70,6 +71,8 @@ AllocatedBitVector::AllocatedBitVector(const BitVector & rhs, Index capacity_) :
     _capacityBits = computeCapacity(_capacityBits, _alloc.size());
     memcpy(_alloc.get(),  rhs.getStart(), rhs.sizeBytes());
     init(_alloc.get(), 0, rhs.size());
+    setBit(size());
+    updateCount();
 }
 
 //////////////////////////////////////////////////////////////////////
@@ -121,12 +124,9 @@ AllocatedBitVector::grow(Index newSize, Index newCapacity)
     if (newCapacity != capacity()) {
         AllocatedBitVector tbv(newSize, newCapacity, _alloc.get(), size());
         if (newSize > size()) {
-            tbv.clearBit(size());  // Clear old guard bit.
-        }
-        ret.reset(new GenerationHeldAlloc<Alloc>(_alloc));
-        if (( newSize >= size()) && isValidCount()) {
-            tbv.setTrueBits(countTrueBits());
+            tbv.clearBitAndMaintainCount(size());  // Clear old guard bit.
         }
+        ret = std::make_unique<GenerationHeldAlloc<Alloc>>(_alloc);
         swap(tbv);
     } else {
         if (newSize > size()) {
diff --git a/searchlib/src/vespa/searchlib/common/bitvector.cpp b/searchlib/src/vespa/searchlib/common/bitvector.cpp
index e28ebe6682f..2e70e9f2603 100644
--- a/searchlib/src/vespa/searchlib/common/bitvector.cpp
+++ b/searchlib/src/vespa/searchlib/common/bitvector.cpp
@@ -165,7 +165,7 @@ BitVector::countInterval(Index start, Index end) const
         ++endw;
     }
     if (startw < endw) {
-        res += IAccelrated::getAccelrator()->populationCount(bitValues + startw, endw - startw);
+        res += IAccelrated::getAccelrator().populationCount(bitValues + startw, endw - startw);
     }
     if (partialEnd) {
         res += Optimized::popCount(bitValues[endw] & ~endBits(last));
@@ -178,7 +178,7 @@ void
 BitVector::orWith(const BitVector & right)
 {
     verifyContains(*this, right);
-    IAccelrated::getAccelrator()->orBit(getActiveStart(), right.getWordIndex(getStartIndex()), getActiveBytes());
+    IAccelrated::getAccelrator().orBit(getActiveStart(), right.getWordIndex(getStartIndex()), getActiveBytes());
 
     repairEnds();
     invalidateCachedCount();
@@ -201,7 +201,7 @@ BitVector::andWith(const BitVector & right)
 {
     verifyContains(*this, right);
 
-    IAccelrated::getAccelrator()->andBit(getActiveStart(), right.getWordIndex(getStartIndex()), getActiveBytes());
+    IAccelrated::getAccelrator().andBit(getActiveStart(), right.getWordIndex(getStartIndex()), getActiveBytes());
 
     setGuardBit();
     invalidateCachedCount();
@@ -213,7 +213,7 @@ BitVector::andNotWith(const BitVector& right)
 {
     verifyContains(*this, right);
 
-    IAccelrated::getAccelrator()->andNotBit(getActiveStart(), right.getWordIndex(getStartIndex()), getActiveBytes());
+    IAccelrated::getAccelrator().andNotBit(getActiveStart(), right.getWordIndex(getStartIndex()), getActiveBytes());
 
     setGuardBit();
     invalidateCachedCount();
@@ -221,7 +221,7 @@ BitVector::andNotWith(const BitVector& right)
 
 void
 BitVector::notSelf() {
-    IAccelrated::getAccelrator()->notBit(getActiveStart(), getActiveBytes());
+    IAccelrated::getAccelrator().notBit(getActiveStart(), getActiveBytes());
     setGuardBit();
     invalidateCachedCount();
 }
diff --git a/searchlib/src/vespa/searchlib/common/bitvector.h b/searchlib/src/vespa/searchlib/common/bitvector.h
index 98e8c9adad3..9671e41df24 100644
--- a/searchlib/src/vespa/searchlib/common/bitvector.h
+++ b/searchlib/src/vespa/searchlib/common/bitvector.h
@@ -45,7 +45,7 @@ public:
     }
     Index countTrueBits() const {
         if ( ! isValidCount()) {
-            _numTrueBits.store(count(), std::memory_order_relaxed);
+            updateCount();
         }
         return _numTrueBits.load(std::memory_order_relaxed);
     }
@@ -255,6 +255,7 @@ protected:
     BitVector(void * buf, Index sz) : BitVector(buf, 0, sz) { }
     BitVector() : BitVector(nullptr, 0) { }
     void init(void * buf,  Index start, Index end);
+    void updateCount() const { _numTrueBits.store(count(), std::memory_order_relaxed); }
     void setTrueBits(Index numTrueBits) { _numTrueBits.store(numTrueBits, std::memory_order_relaxed); }
     VESPA_DLL_LOCAL void clearIntervalNoInvalidation(Index start, Index end);
     bool isValidCount() const { return isValidCount(_numTrueBits.load(std::memory_order_relaxed)); }
@@ -340,7 +341,7 @@ private:
             func(start+pos);
             start += pos + 1;
             word >>= pos;
-            word >>= 1;
+            word >>= 1u;
         }
     }
 
@@ -357,8 +358,6 @@ protected:
     operator>>(vespalib::nbostream &in, BitVector &bv);
 };
 
-typedef BitVector ConstBitVectorReference;
-
 vespalib::nbostream &
 operator<<(vespalib::nbostream &out, const BitVector &bv);
 
diff --git a/searchlib/src/vespa/searchlib/common/bitvectorcache.cpp b/searchlib/src/vespa/searchlib/common/bitvectorcache.cpp
index e124cc05448..4b5fc8a06c9 100644
--- a/searchlib/src/vespa/searchlib/common/bitvectorcache.cpp
+++ b/searchlib/src/vespa/searchlib/common/bitvectorcache.cpp
@@ -18,9 +18,7 @@ BitVectorCache::BitVectorCache(GenerationHolder &genHolder) :
 {
 }
 
-BitVectorCache::~BitVectorCache()
-{
-}
+BitVectorCache::~BitVectorCache() = default;
 
 void
 BitVectorCache::computeCountVector(KeySet & keys, CountVector & v) const
@@ -148,14 +146,19 @@ BitVectorCache::populate(Key2Index & newKeys, CondensedBitVector & chunk, const
             m.chunkIndex(index);
             LOG(info, "Populating bitvector %2d with feature %" PRIu64 " and %ld bits set. Cost is %8f = %2.2f%%, accumulated cost is %2.2f%%",
                        index, e.first, m.bitCount(), m.cost(), percentage, accum);
-            index++;
             assert(m.isCached());
             assert(newKeys[e.first].isCached());
             assert(&m == &newKeys[e.first]);
             PopulateInterface::Iterator::UP iterator = lookup.lookup(e.first);
-            for (int32_t docId(iterator->getNext()); docId >= 0; docId = iterator->getNext()) {
-                chunk.set(m.chunkIndex(), docId, true);
+            if (iterator) {
+                for (int32_t docId(iterator->getNext()); docId >= 0; docId = iterator->getNext()) {
+                    chunk.set(m.chunkIndex(), docId, true);
+                }
+            } else {
+                LOG(error, "Unable to to find a valid iterator for feature %" PRIu64 " and %ld bits set at while populating bitvector %2d. This should in theory be impossible.",
+                           e.first, m.bitCount(), index);
             }
+            index++;
         }
     }
 }
diff --git a/searchlib/src/vespa/searchlib/common/location.h b/searchlib/src/vespa/searchlib/common/location.h
index 96821e204e2..a00bb83648a 100644
--- a/searchlib/src/vespa/searchlib/common/location.h
+++ b/searchlib/src/vespa/searchlib/common/location.h
@@ -2,12 +2,12 @@
 
 #pragma once
 
-#include <vespa/vespalib/geo/zcurve.h>
 #include "documentlocations.h"
+#include <vespa/vespalib/geo/zcurve.h>
+
 #include <vespa/vespalib/stllike/string.h>
 
-namespace search {
-namespace common {
+namespace search::common {
 
 class Location : public DocumentLocations
 {
@@ -39,7 +39,7 @@ private:
     int32_t  _x;        /* Query X position */
     int32_t  _y;        /* Query Y position */
     uint32_t _xAspect;      /* X distance multiplier fraction */
-    uint32_t _radius;       /* Radius for euclidian distance */
+    uint32_t _radius;       /* Radius for euclidean distance */
     int32_t  _minx;     /* Min X coordinate */
     int32_t  _maxx;     /* Max X coordinate */
     int32_t  _miny;     /* Min Y coordinate */
@@ -51,5 +51,3 @@ private:
 };
 
 }
-}
-
diff --git a/searchlib/src/vespa/searchlib/common/sequencedtaskexecutor.cpp b/searchlib/src/vespa/searchlib/common/sequencedtaskexecutor.cpp
deleted file mode 100644
index bb779b659ab..00000000000
--- a/searchlib/src/vespa/searchlib/common/sequencedtaskexecutor.cpp
+++ /dev/null
@@ -1,71 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#include "sequencedtaskexecutor.h"
-#include <vespa/vespalib/util/blockingthreadstackexecutor.h>
-#include <vespa/vespalib/stllike/hash_map.hpp>
-
-using vespalib::BlockingThreadStackExecutor;
-
-namespace search {
-
-namespace {
-
-constexpr uint32_t stackSize = 128 * 1024;
-
-}
-
-
-SequencedTaskExecutor::SequencedTaskExecutor(uint32_t threads, uint32_t taskLimit)
-    : ISequencedTaskExecutor(threads),
-      _executors()
-{
-    for (uint32_t id = 0; id < threads; ++id) {
-        auto executor = std::make_unique<BlockingThreadStackExecutor>(1, stackSize, taskLimit);
-        _executors.push_back(std::move(executor));
-    }
-}
-
-SequencedTaskExecutor::~SequencedTaskExecutor()
-{
-    sync();
-}
-
-void
-SequencedTaskExecutor::setTaskLimit(uint32_t taskLimit)
-{
-    for (const auto &executor : _executors) {
-        executor->setTaskLimit(taskLimit);
-    }
-}
-
-void
-SequencedTaskExecutor::executeTask(ExecutorId id, vespalib::Executor::Task::UP task)
-{
-    assert(id.getId() < _executors.size());
-    vespalib::ThreadStackExecutorBase &executor(*_executors[id.getId()]);
-    auto rejectedTask = executor.execute(std::move(task));
-    assert(!rejectedTask);
-}
-
-void
-SequencedTaskExecutor::sync()
-{
-    for (auto &executor : _executors) {
-        executor->sync();
-    }
-}
-
-SequencedTaskExecutor::Stats
-SequencedTaskExecutor::getStats()
-{
-    Stats accumulatedStats;
-    for (auto &executor : _executors) {
-        Stats stats = executor->getStats();
-        accumulatedStats.maxPendingTasks += stats.maxPendingTasks;
-        accumulatedStats.acceptedTasks += stats.acceptedTasks;
-        accumulatedStats.rejectedTasks += stats.rejectedTasks;
-    }
-    return accumulatedStats;
-}
-
-} // namespace search
diff --git a/searchlib/src/vespa/searchlib/common/sequencedtaskexecutor.h b/searchlib/src/vespa/searchlib/common/sequencedtaskexecutor.h
deleted file mode 100644
index 2b7e70d69c7..00000000000
--- a/searchlib/src/vespa/searchlib/common/sequencedtaskexecutor.h
+++ /dev/null
@@ -1,35 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-#pragma once
-
-#include "isequencedtaskexecutor.h"
-#include <vespa/vespalib/stllike/hash_map.h>
-#include <vector>
-
-namespace vespalib {
-    struct ExecutorStats;
-    class BlockingThreadStackExecutor;
-}
-
-namespace search {
-
-/**
- * Class to run multiple tasks in parallel, but tasks with same
- * id has to be run in sequence.
- */
-class SequencedTaskExecutor : public ISequencedTaskExecutor
-{
-    using Stats = vespalib::ExecutorStats;
-    std::vector<std::shared_ptr<vespalib::BlockingThreadStackExecutor>> _executors;
-public:
-    using ISequencedTaskExecutor::getExecutorId;
-
-    SequencedTaskExecutor(uint32_t threads, uint32_t taskLimit = 1000);
-    ~SequencedTaskExecutor();
-
-    void setTaskLimit(uint32_t taskLimit);
-    void executeTask(ExecutorId id, vespalib::Executor::Task::UP task) override;
-    void sync() override;
-    Stats getStats();
-};
-
-} // namespace search
diff --git a/searchlib/src/vespa/searchlib/common/sortspec.cpp b/searchlib/src/vespa/searchlib/common/sortspec.cpp
index 694443b00ba..f6ac468b485 100644
--- a/searchlib/src/vespa/searchlib/common/sortspec.cpp
+++ b/searchlib/src/vespa/searchlib/common/sortspec.cpp
@@ -3,6 +3,7 @@
 #include <vespa/vespalib/util/stringfmt.h>
 #include <vespa/fastlib/text/normwordfolder.h>
 #include <vespa/vespalib/text/utf8.h>
+#include <stdexcept>
 
 namespace search {
 namespace common {
diff --git a/searchlib/src/vespa/searchlib/common/threaded_compactable_lid_space.cpp b/searchlib/src/vespa/searchlib/common/threaded_compactable_lid_space.cpp
index defb537be0e..4b253387f15 100644
--- a/searchlib/src/vespa/searchlib/common/threaded_compactable_lid_space.cpp
+++ b/searchlib/src/vespa/searchlib/common/threaded_compactable_lid_space.cpp
@@ -1,9 +1,6 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
-#include <cstddef>
-#include <cstdint>
 #include "threaded_compactable_lid_space.h"
-#include "isequencedtaskexecutor.h"
 #include <future>
 
 namespace search::common {
diff --git a/searchlib/src/vespa/searchlib/common/threaded_compactable_lid_space.h b/searchlib/src/vespa/searchlib/common/threaded_compactable_lid_space.h
index 02d54acf666..f22d5946295 100644
--- a/searchlib/src/vespa/searchlib/common/threaded_compactable_lid_space.h
+++ b/searchlib/src/vespa/searchlib/common/threaded_compactable_lid_space.h
@@ -3,7 +3,7 @@
 #pragma once
 
 #include "i_compactable_lid_space.h"
-#include "isequencedtaskexecutor.h"
+#include <vespa/vespalib/util/isequencedtaskexecutor.h>
 #include <memory>
 
 namespace search::common {
@@ -15,6 +15,7 @@ namespace search::common {
  */
 class ThreadedCompactableLidSpace : public ICompactableLidSpace
 {
+    using ISequencedTaskExecutor = vespalib::ISequencedTaskExecutor;
     std::shared_ptr<ICompactableLidSpace> _target;
     ISequencedTaskExecutor               &_executor;
     ISequencedTaskExecutor::ExecutorId    _executorId;
diff --git a/searchlib/src/vespa/searchlib/diskindex/field_length_scanner.h b/searchlib/src/vespa/searchlib/diskindex/field_length_scanner.h
index e282a85b64f..3ef081a2820 100644
--- a/searchlib/src/vespa/searchlib/diskindex/field_length_scanner.h
+++ b/searchlib/src/vespa/searchlib/diskindex/field_length_scanner.h
@@ -5,6 +5,7 @@
 #include <vector>
 #include <unordered_map>
 #include <limits>
+#include <cstdint>
 
 namespace search::index { class DocIdAndFeatures; }
 
diff --git a/searchlib/src/vespa/searchlib/docstore/chunk.cpp b/searchlib/src/vespa/searchlib/docstore/chunk.cpp
index c7976fd15ae..847cd3623c3 100644
--- a/searchlib/src/vespa/searchlib/docstore/chunk.cpp
+++ b/searchlib/src/vespa/searchlib/docstore/chunk.cpp
@@ -79,7 +79,7 @@ Chunk::Chunk(uint32_t id, const void * buffer, size_t len, bool skipcrc) :
     os >> _lastSerial;
 }
 
-Chunk::~Chunk() { }
+Chunk::~Chunk() = default;
 
 vespalib::ConstBufferRef
 Chunk::getLid(uint32_t lid) const
diff --git a/searchlib/src/vespa/searchlib/docstore/documentstore.cpp b/searchlib/src/vespa/searchlib/docstore/documentstore.cpp
index 4c4405fabb0..6644533550f 100644
--- a/searchlib/src/vespa/searchlib/docstore/documentstore.cpp
+++ b/searchlib/src/vespa/searchlib/docstore/documentstore.cpp
@@ -42,12 +42,6 @@ DocumentVisitorAdapter::visit(uint32_t lid, vespalib::ConstBufferRef buf) {
     }
 }
 
-document::Document::UP
-deserializeDocument(const vespalib::DataBuffer & uncompressed, const DocumentTypeRepo &repo) {
-    vespalib::nbostream is(uncompressed.getData(), uncompressed.getDataLen());
-    return std::make_unique<document::Document>(repo, is);
-}
-
 }
 
 using vespalib::nbostream;
@@ -183,7 +177,7 @@ DocumentStore::read(DocumentIdT lid, const DocumentTypeRepo &repo) const
         }
         Value::Result result = value.decompressed();
         if ( result.second ) {
-            return deserializeDocument(result.first, repo);
+            return std::make_unique<document::Document>(repo, std::move(result.first));
         } else {
             LOG(warning, "Summary cache for lid %u is corrupt. Invalidating and reading directly from backing store", lid);
             _cache->invalidate(lid);
@@ -195,7 +189,7 @@ DocumentStore::read(DocumentIdT lid, const DocumentTypeRepo &repo) const
     if ( ! value.empty() ) {
         Value::Result result = value.decompressed();
         assert(result.second);
-        return deserializeDocument(result.first, repo);
+        return std::make_unique<document::Document>(repo, std::move(result.first));
     }
     return std::unique_ptr<document::Document>();
 }
@@ -309,7 +303,7 @@ public:
         _visitorProgress.updateProgress(progress);
     }
     
-    WrapVisitorProgress(IDocumentStoreVisitorProgress &visitProgress)
+    explicit WrapVisitorProgress(IDocumentStoreVisitorProgress &visitProgress)
         : _visitorProgress(visitProgress)
     {
     }
@@ -369,7 +363,7 @@ DocumentStore::WrapVisitor<Visitor>::visit(uint32_t lid, const void *buffer, siz
         value.set(std::move(buf), len);
     }
     if (! value.empty()) {
-        std::shared_ptr<document::Document> doc(deserializeDocument(value.decompressed().first, _repo));
+        auto doc = std::make_shared<document::Document>(_repo, value.decompressed().first);
         _visitor.visit(lid, doc);
         rewrite(lid, *doc);
     } else {
diff --git a/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.cpp b/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.cpp
index 70295d81a93..d12a3a54a93 100644
--- a/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.cpp
+++ b/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.cpp
@@ -203,6 +203,13 @@ seek_past(LidInfoWithLidV::const_iterator begin, LidInfoWithLidV::const_iterator
    return begin;
 }
 
+struct LidAndBuffer {
+    LidAndBuffer(uint32_t lid, uint32_t sz, vespalib::alloc::Alloc buf) : _lid(lid), _size(sz), _buf(std::move(buf)) {}
+    uint32_t _lid;
+    uint32_t _size;
+    vespalib::alloc::Alloc _buf;
+};
+
 }
 
 void
@@ -211,6 +218,7 @@ WriteableFileChunk::read(LidInfoWithLidV::const_iterator begin, size_t count, IB
     if (count == 0) { return; }
     if (!frozen()) {
         vespalib::hash_map<uint32_t, ChunkInfo> chunksOnFile;
+        std::vector<LidAndBuffer> buffers;
         {
             LockGuard guard(_lock);
             for (size_t i(0); i < count; i++) {
@@ -225,12 +233,18 @@ WriteableFileChunk::read(LidInfoWithLidV::const_iterator begin, size_t count, IB
                         assert(chunk == _active->getId());
                         buffer = _active->getLid(li.getLid());
                     }
-                    visitor.visit(li.getLid(), buffer);
+                    auto copy = vespalib::alloc::Alloc::alloc(buffer.size());
+                    memcpy(copy.get(), buffer.data(), buffer.size());
+                    buffers.emplace_back(li.getLid(), buffer.size(), std::move(copy));
                 } else {
                     chunksOnFile[chunk] = _chunkInfo[chunk];
                 }
             }
         }
+        for (auto & entry : buffers) {
+            visitor.visit(entry._lid, vespalib::ConstBufferRef(entry._buf.get(), entry._size));
+            entry._buf = vespalib::alloc::Alloc();
+        }
         for (auto & it : chunksOnFile) {
             auto first = find_first(begin, it.first);
             auto last = seek_past(first, begin + count, it.first);
diff --git a/searchlib/src/vespa/searchlib/engine/monitorreply.cpp b/searchlib/src/vespa/searchlib/engine/monitorreply.cpp
index c0f7fa0d42a..3414603a893 100644
--- a/searchlib/src/vespa/searchlib/engine/monitorreply.cpp
+++ b/searchlib/src/vespa/searchlib/engine/monitorreply.cpp
@@ -15,7 +15,8 @@ MonitorReply::MonitorReply()
       totalParts(),
       activeParts(),
       activeDocs(0),
-      flags()
+      flags(),
+      is_blocking_writes(false)
 { }
 
 }
diff --git a/searchlib/src/vespa/searchlib/engine/monitorreply.h b/searchlib/src/vespa/searchlib/engine/monitorreply.h
index 7d1f0ce1cef..f66a30fdd89 100644
--- a/searchlib/src/vespa/searchlib/engine/monitorreply.h
+++ b/searchlib/src/vespa/searchlib/engine/monitorreply.h
@@ -21,6 +21,7 @@ struct MonitorReply
     uint32_t                  activeParts; // mld
     uint64_t                  activeDocs;
     uint32_t                  flags;
+    bool                      is_blocking_writes;
 
     MonitorReply();
 };
diff --git a/searchlib/src/vespa/searchlib/engine/proto_converter.cpp b/searchlib/src/vespa/searchlib/engine/proto_converter.cpp
index e5846734a8d..2876fa99434 100644
--- a/searchlib/src/vespa/searchlib/engine/proto_converter.cpp
+++ b/searchlib/src/vespa/searchlib/engine/proto_converter.cpp
@@ -170,6 +170,7 @@ ProtoConverter::monitor_reply_to_proto(const MonitorReply &reply, ProtoMonitorRe
     proto.set_online(reply.timestamp != 0);
     proto.set_active_docs(reply.activeDocs);
     proto.set_distribution_key(reply.distribution_key);
+    proto.set_is_blocking_writes(reply.is_blocking_writes);
 }
 
 //-----------------------------------------------------------------------------
diff --git a/searchlib/src/vespa/searchlib/expression/aggregationrefnode.cpp b/searchlib/src/vespa/searchlib/expression/aggregationrefnode.cpp
index 4c36960c754..1962cad870a 100644
--- a/searchlib/src/vespa/searchlib/expression/aggregationrefnode.cpp
+++ b/searchlib/src/vespa/searchlib/expression/aggregationrefnode.cpp
@@ -1,6 +1,7 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 #include "aggregationrefnode.h"
 #include <vespa/vespalib/util/stringfmt.h>
+#include <stdexcept>
 
 namespace search {
 namespace expression {
diff --git a/searchlib/src/vespa/searchlib/expression/integerresultnode.h b/searchlib/src/vespa/searchlib/expression/integerresultnode.h
index e428c0448bb..7f2ec823432 100644
--- a/searchlib/src/vespa/searchlib/expression/integerresultnode.h
+++ b/searchlib/src/vespa/searchlib/expression/integerresultnode.h
@@ -4,6 +4,7 @@
 #include "numericresultnode.h"
 #include <vespa/vespalib/util/sort.h>
 #include <limits>
+#include <type_traits>
 
 namespace search::expression {
 
@@ -29,7 +30,13 @@ public:
     }
     void add(const ResultNode & b) override { _value += b.getInteger(); }
     void negate() override { _value = - _value; }
-    void multiply(const ResultNode & b) override { _value *= b.getInteger(); }
+    void multiply(const ResultNode & b) override {
+        if constexpr (std::is_same_v<T, bool>) {
+         _value = (_value && (b.getInteger() != 0));
+       } else {
+         _value *= b.getInteger();
+       }
+    }
     void divide(const ResultNode & b) override {
         int64_t val = b.getInteger();
         _value = (val == 0) ? 0 : (_value / val);
diff --git a/searchlib/src/vespa/searchlib/expression/numericfunctionnode.cpp b/searchlib/src/vespa/searchlib/expression/numericfunctionnode.cpp
index eea179f67ff..5b6989b5dc9 100644
--- a/searchlib/src/vespa/searchlib/expression/numericfunctionnode.cpp
+++ b/searchlib/src/vespa/searchlib/expression/numericfunctionnode.cpp
@@ -1,5 +1,6 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 #include "numericfunctionnode.h"
+#include <stdexcept>
 
 namespace search::expression {
 
diff --git a/searchlib/src/vespa/searchlib/expression/resultnode.cpp b/searchlib/src/vespa/searchlib/expression/resultnode.cpp
index 66ac81902d8..a99ac01ce2a 100644
--- a/searchlib/src/vespa/searchlib/expression/resultnode.cpp
+++ b/searchlib/src/vespa/searchlib/expression/resultnode.cpp
@@ -3,6 +3,7 @@
 #include "resultnode.h"
 #include <vespa/vespalib/util/stringfmt.h>
 #include <vespa/vespalib/util/exception.h>
+#include <stdexcept>
 
 namespace search {
 namespace expression {
diff --git a/searchlib/src/vespa/searchlib/expression/resultnodes.cpp b/searchlib/src/vespa/searchlib/expression/resultnodes.cpp
index fb4652a4d62..69cad65795f 100644
--- a/searchlib/src/vespa/searchlib/expression/resultnodes.cpp
+++ b/searchlib/src/vespa/searchlib/expression/resultnodes.cpp
@@ -11,6 +11,7 @@
 #include <vespa/vespalib/objects/visit.hpp>
 #include <vespa/vespalib/objects/serializer.hpp>
 #include <vespa/vespalib/objects/deserializer.hpp>
+#include <stdexcept>
 
 namespace search::expression {
 
diff --git a/searchlib/src/vespa/searchlib/features/agefeature.cpp b/searchlib/src/vespa/searchlib/features/agefeature.cpp
index 258648408f8..e93691e7241 100644
--- a/searchlib/src/vespa/searchlib/features/agefeature.cpp
+++ b/searchlib/src/vespa/searchlib/features/agefeature.cpp
@@ -4,6 +4,7 @@
 #include "valuefeature.h"
 #include <vespa/searchlib/fef/featurenamebuilder.h>
 #include <vespa/searchlib/fef/matchdata.h>
+#include <vespa/vespalib/util/stash.h>
 
 using search::attribute::IAttributeVector;
 
@@ -18,20 +19,18 @@ AgeExecutor::AgeExecutor(const IAttributeVector *attribute) :
     _attribute(attribute),
     _buf()
 {
-    if (_attribute != NULL) {
+    if (_attribute != nullptr) {
         _buf.allocate(attribute->getMaxValueCount());
     }
 }
 
-AgeBlueprint::~AgeBlueprint()
-{
-}
+AgeBlueprint::~AgeBlueprint() = default;
 
 void
 AgeExecutor::execute(uint32_t docId)
 {
     feature_t age = 10000000000.0;
-    if (_attribute != NULL) {
+    if (_attribute != nullptr) {
         _buf.fill(*_attribute, docId);
         int64_t docTime = _buf[0];
         feature_t currTime = inputs().get_number(0);
@@ -65,7 +64,7 @@ AgeBlueprint::setup(const search::fef::IIndexEnvironment &env,
 search::fef::Blueprint::UP
 AgeBlueprint::createInstance() const
 {
-    return search::fef::Blueprint::UP(new AgeBlueprint());
+    return std::make_unique<AgeBlueprint>();
 }
 
 search::fef::FeatureExecutor &
diff --git a/searchlib/src/vespa/searchlib/features/attributefeature.cpp b/searchlib/src/vespa/searchlib/features/attributefeature.cpp
index b1d3356dd62..78804513ab7 100644
--- a/searchlib/src/vespa/searchlib/features/attributefeature.cpp
+++ b/searchlib/src/vespa/searchlib/features/attributefeature.cpp
@@ -13,6 +13,7 @@
 #include <vespa/searchlib/fef/indexproperties.h>
 #include <vespa/searchlib/attribute/singlenumericattribute.h>
 #include <vespa/searchlib/attribute/multinumericattribute.h>
+#include <vespa/searchlib/attribute/singleboolattribute.h>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".features.attributefeature");
@@ -33,6 +34,7 @@ using search::fef::FeatureExecutor;
 using search::features::util::ConstCharPtr;
 using vespalib::eval::ValueType;
 using search::fef::FeatureType;
+using namespace search::index;
 
 using namespace search::fef::indexproperties;
 
@@ -98,7 +100,7 @@ considerUndefined<ConstCharPtr>(ConstCharPtr value, BasicType::Type )
  * Implements the executor for fetching values from a single or array attribute vector
  */
 template <typename T>
-class SingleAttributeExecutor : public fef::FeatureExecutor {
+class SingleAttributeExecutor final : public fef::FeatureExecutor {
 private:
     const T & _attribute;
 public:
@@ -108,50 +110,73 @@ public:
      * @param attribute The attribute vector to use.
      */
     SingleAttributeExecutor(const T & attribute) : _attribute(attribute) { }
+    void handle_bind_outputs(vespalib::ArrayRef<fef::NumberOrObject> outputs_in) override {
+        fef::FeatureExecutor::handle_bind_outputs(outputs_in);
+        auto o = outputs().get_bound();
+        o[1].as_number = 0;  // weight
+        o[2].as_number = 0;  // contains
+        o[3].as_number = 1;  // count
+    }
     void execute(uint32_t docId) override;
 };
 
+class BoolAttributeExecutor final : public fef::FeatureExecutor {
+private:
+    const SingleBoolAttribute & _attribute;
+public:
+    BoolAttributeExecutor(const SingleBoolAttribute & attribute)
+        : _attribute(attribute)
+    {}
+    void execute(uint32_t docId) override {
+        outputs().set_number(0, _attribute.getFloat(docId));
+    }
+};
+
 /**
  * Implements the executor for fetching values from a single or array attribute vector
  */
 template <typename T>
-class MultiAttributeExecutor : public fef::FeatureExecutor {
+class MultiAttributeExecutor final : public fef::FeatureExecutor {
 private:
     const T & _attribute;
     uint32_t  _idx;
 public:
-    /**
-     * Constructs an executor.
-     *
-     * @param attribute The attribute vector to use.
-     */
     MultiAttributeExecutor(const T & attribute, uint32_t idx) : _attribute(attribute), _idx(idx) { }
     void execute(uint32_t docId) override;
+    void handle_bind_outputs(vespalib::ArrayRef<fef::NumberOrObject> outputs_in) override {
+        fef::FeatureExecutor::handle_bind_outputs(outputs_in);
+        auto o = outputs().get_bound();
+        o[1].as_number = 0;  // weight
+        o[2].as_number = 0;  // contains
+        o[3].as_number = 0;  // count
+    }
 };
 
-class CountOnlyAttributeExecutor : public fef::FeatureExecutor {
+class CountOnlyAttributeExecutor final : public fef::FeatureExecutor {
 private:
     const attribute::IAttributeVector & _attribute;
 
 public:
-    /**
-     * Constructs an executor.
-     *
-     * @param attribute The attribute vector to use.
-     */
     CountOnlyAttributeExecutor(const attribute::IAttributeVector & attribute) : _attribute(attribute) { }
     void execute(uint32_t docId) override;
+    void handle_bind_outputs(vespalib::ArrayRef<fef::NumberOrObject> outputs_in) override {
+        fef::FeatureExecutor::handle_bind_outputs(outputs_in);
+        auto o = outputs().get_bound();
+        o[0].as_number = 0;  // value
+        o[1].as_number = 0;  // weight
+        o[2].as_number = 0;  // contains
+    }
 };
 /**
  * Implements the executor for fetching values from a single or array attribute vector
  */
 template <typename T>
-class AttributeExecutor : public fef::FeatureExecutor {
+class AttributeExecutor final : public fef::FeatureExecutor {
 private:
     const attribute::IAttributeVector * _attribute;
     attribute::BasicType::Type _attrType;
-    uint32_t _idx;
-    T _buffer; // used when fetching values from the attribute
+    uint32_t  _idx;
+    T         _buffer; // used when fetching values from the attribute
     feature_t _defaultCount;
 
 public:
@@ -163,6 +188,13 @@ public:
      */
     AttributeExecutor(const attribute::IAttributeVector * attribute, uint32_t idx);
     void execute(uint32_t docId) override;
+    void handle_bind_outputs(vespalib::ArrayRef<fef::NumberOrObject> outputs_in) override {
+        fef::FeatureExecutor::handle_bind_outputs(outputs_in);
+        auto o = outputs().get_bound();
+        o[1].as_number = 0;  // weight
+        o[2].as_number = 0;  // contains
+        o[3].as_number = _defaultCount; // count
+    }
 };
 
 
@@ -200,9 +232,6 @@ SingleAttributeExecutor<T>::execute(uint32_t docId)
     o[0].as_number = __builtin_expect(attribute::isUndefined(v), false)
                      ? attribute::getUndefined<feature_t>()
                      : util::getAsFeature(v);
-    o[1].as_number = 0;  // weight
-    o[2].as_number = 0;  // contains
-    o[3].as_number = 1;  // contains
 }
 
 template <typename T>
@@ -214,18 +243,12 @@ MultiAttributeExecutor<T>::execute(uint32_t docId)
 
     auto o = outputs().get_bound();
     o[0].as_number = __builtin_expect(_idx < numValues, true) ? values[_idx].value() : 0;
-    o[1].as_number = 0;  // weight
-    o[2].as_number = 0;  // contains
-    o[3].as_number = 0;  // count
 }
 
 void
 CountOnlyAttributeExecutor::execute(uint32_t docId)
 {
     auto o = outputs().get_bound();
-    o[0].as_number = 0;  // value
-    o[1].as_number = 0;  // weight
-    o[2].as_number = 0;  // contains
     o[3].as_number = _attribute.getValueCount(docId); // count
 }
 
@@ -252,9 +275,6 @@ AttributeExecutor<T>::execute(uint32_t docId)
     }
     auto o = outputs().get_bound();
     o[0].as_number = value;  // value
-    o[1].as_number = 0;  // weight
-    o[2].as_number = 0;  // contains
-    o[3].as_number = _defaultCount; // count
 }
 
 
@@ -331,15 +351,17 @@ private:
 };
 
 fef::FeatureExecutor &
-createAttributeExecutor(const IAttributeVector *attribute, const vespalib::string &attrName, const vespalib::string &extraParam, vespalib::Stash &stash)
+createAttributeExecutor(uint32_t numOutputs, const IAttributeVector *attribute, const vespalib::string &attrName, const vespalib::string &extraParam, vespalib::Stash &stash)
 {
     if (attribute == nullptr) {
         LOG(warning, "The attribute vector '%s' was not found in the attribute manager, returning default values.",
                 attrName.c_str());
-        std::vector<feature_t> values(4, 0.0f);
+        std::vector<feature_t> values(numOutputs, 0.0f);
         return stash.create<ValueExecutor>(values);
     }
-    if (attribute->getCollectionType() == CollectionType::WSET) {
+    CollectionType collectionType = attribute->getCollectionType();
+    if (collectionType == CollectionType::WSET) {
+        assert(numOutputs == 4);
         bool useKey = !extraParam.empty();
         if (useKey) {
             if (attribute->isStringType()) {
@@ -353,30 +375,63 @@ createAttributeExecutor(const IAttributeVector *attribute, const vespalib::strin
             return stash.create<CountOnlyAttributeExecutor>(*attribute);
         }
     } else { // SINGLE or ARRAY
-        if ((attribute->getCollectionType() == CollectionType::SINGLE) && (attribute->isIntegerType() || attribute->isFloatingPointType())) {
-            { SingleValueExecutorCreator<FloatingPointAttributeTemplate<double>> creator; if (creator.handle(attribute)) return creator.create(stash); }
-            { SingleValueExecutorCreator<FloatingPointAttributeTemplate<float>> creator;  if (creator.handle(attribute)) return creator.create(stash); }
-            { SingleValueExecutorCreator<IntegerAttributeTemplate<int32_t>> creator;      if (creator.handle(attribute)) return creator.create(stash); }
-            { SingleValueExecutorCreator<IntegerAttributeTemplate<int64_t>> creator;      if (creator.handle(attribute)) return creator.create(stash); }
+        BasicType basicType = attribute->getBasicType();
+        if (collectionType == CollectionType::SINGLE) {
+            if (attribute->isIntegerType()) {
+                if (basicType == BasicType::BOOL) {
+                    auto boolAttribute = dynamic_cast<const SingleBoolAttribute *>(attribute);
+                    assert (boolAttribute && (numOutputs == 1));
+                    return stash.create<BoolAttributeExecutor>(*boolAttribute);
+                } else {
+                    assert(numOutputs == 4);
+                    if (basicType == BasicType::INT8) {
+                        SingleValueExecutorCreator<IntegerAttributeTemplate<int8_t>> creator;
+                        if (creator.handle(attribute)) return creator.create(stash);
+                    } else if (basicType == BasicType::INT32) {
+                        SingleValueExecutorCreator<IntegerAttributeTemplate<int32_t>> creator;
+                        if (creator.handle(attribute)) return creator.create(stash);
+                    }
+                    SingleValueExecutorCreator<IntegerAttributeTemplate<int64_t>> creator;
+                    if (creator.handle(attribute)) return creator.create(stash);
+                }
+            } else if (attribute->isFloatingPointType()) {
+                assert(numOutputs == 4);
+                if (basicType == BasicType::DOUBLE) {
+                    SingleValueExecutorCreator<FloatingPointAttributeTemplate<double>> creator;
+                    if (creator.handle(attribute)) return creator.create(stash);
+                } else {
+                    SingleValueExecutorCreator<FloatingPointAttributeTemplate<float>> creator;
+                    if (creator.handle(attribute)) return creator.create(stash);
+                }
+            }
+        }
+        assert(numOutputs == 4);
+        uint32_t idx = 0;
+        if (!extraParam.empty()) {
+            idx = util::strToNum<uint32_t>(extraParam);
+        } else if (attribute->getCollectionType() == CollectionType::ARRAY) {
+            return stash.create<CountOnlyAttributeExecutor>(*attribute);
         }
-        {
-            uint32_t idx = 0;
-            if (!extraParam.empty()) {
-                idx = util::strToNum<uint32_t>(extraParam);
-            } else if (attribute->getCollectionType() == CollectionType::ARRAY) {
-                return stash.create<CountOnlyAttributeExecutor>(*attribute);
+        if (attribute->isStringType()) {
+            return stash.create<AttributeExecutor<ConstCharContent>>(attribute, idx);
+        } else if (attribute->isIntegerType()) {
+            if (basicType == BasicType::INT32) {
+                MultiValueExecutorCreator<IntegerAttributeTemplate<int32_t>> creator;
+                if (creator.handle(attribute)) return creator.create(stash, idx);
+            } else if (basicType == BasicType::INT64) {
+                MultiValueExecutorCreator<IntegerAttributeTemplate<int64_t>> creator;
+                if (creator.handle(attribute)) return creator.create(stash, idx);
             }
-            if (attribute->isStringType()) {
-                return stash.create<AttributeExecutor<ConstCharContent>>(attribute, idx);
-            } else if (attribute->isIntegerType()) {
-                { MultiValueExecutorCreator<IntegerAttributeTemplate<int32_t>> creator; if (creator.handle(attribute)) return creator.create(stash, idx); }
-                { MultiValueExecutorCreator<IntegerAttributeTemplate<int64_t>> creator; if (creator.handle(attribute)) return creator.create(stash, idx); }
-                return stash.create<AttributeExecutor<IntegerContent>>(attribute, idx);
-            } else { // FLOAT
-                { MultiValueExecutorCreator<FloatingPointAttributeTemplate<double>> creator; if (creator.handle(attribute)) return creator.create(stash, idx); }
-                { MultiValueExecutorCreator<FloatingPointAttributeTemplate<float>> creator; if (creator.handle(attribute)) return creator.create(stash, idx); }
-                return stash.create<AttributeExecutor<FloatContent>>(attribute, idx);
+            return stash.create<AttributeExecutor<IntegerContent>>(attribute, idx);
+        } else { // FLOAT
+            if (basicType == BasicType::DOUBLE) {
+                MultiValueExecutorCreator<FloatingPointAttributeTemplate<double>> creator;
+                if (creator.handle(attribute)) return creator.create(stash, idx);
+            } else {
+                MultiValueExecutorCreator<FloatingPointAttributeTemplate<float>> creator;
+                if (creator.handle(attribute)) return creator.create(stash, idx);
             }
+            return stash.create<AttributeExecutor<FloatContent>>(attribute, idx);
         }
     }
 }
@@ -417,6 +472,12 @@ createTensorAttributeExecutor(const IAttributeVector *attribute, const vespalib:
     return stash.create<TensorAttributeExecutor>(tensorAttribute);
 }
 
+bool
+isSingleValueBoolField(const fef::FieldInfo & fInfo) {
+    return (fInfo.collection() == schema::CollectionType::SINGLE)
+           && (fInfo.get_data_type() == schema::DataType::BOOL);
+}
+
 }
 
 AttributeBlueprint::AttributeBlueprint() :
@@ -424,7 +485,8 @@ AttributeBlueprint::AttributeBlueprint() :
     _attrName(),
     _attrKey(),
     _extra(),
-    _tensorType(ValueType::double_type())
+    _tensorType(ValueType::double_type()),
+    _numOutputs(0)
 {
 }
 
@@ -461,10 +523,14 @@ AttributeBlueprint::setup(const fef::IIndexEnvironment & env,
                    "the value at the given index of an array attribute, "
                    "the given key of a weighted set attribute, or"
                    "the tensor of a tensor attribute", output_type);
-    if (!_tensorType.is_tensor()) {
+    const fef::FieldInfo * fInfo = env.getFieldByName(_attrName);
+    if (_tensorType.is_tensor() || isSingleValueBoolField(*fInfo)) {
+        _numOutputs = 1;
+    } else {
         describeOutput("weight", "The weight associated with the given key in a weighted set attribute.");
         describeOutput("contains", "1 if the given key is present in a weighted set attribute, 0 otherwise.");
         describeOutput("count", "Returns the number of elements in this array or weighted set attribute.");
+        _numOutputs = 4;
     }
     env.hintAttributeAccess(_attrName);
     return !_tensorType.is_error();
@@ -489,7 +555,7 @@ AttributeBlueprint::createExecutor(const fef::IQueryEnvironment &env, vespalib::
     if (_tensorType.is_tensor()) {
         return createTensorAttributeExecutor(attribute, _attrName, _tensorType, stash);
     } else {
-        return createAttributeExecutor(attribute, _attrName, _extra, stash);
+        return createAttributeExecutor(_numOutputs, attribute, _attrName, _extra, stash);
     }
 }
 
diff --git a/searchlib/src/vespa/searchlib/features/attributefeature.h b/searchlib/src/vespa/searchlib/features/attributefeature.h
index e1e3ddf7300..04ad698a63c 100644
--- a/searchlib/src/vespa/searchlib/features/attributefeature.h
+++ b/searchlib/src/vespa/searchlib/features/attributefeature.h
@@ -15,10 +15,12 @@ namespace search::features {
  */
 class AttributeBlueprint : public fef::Blueprint {
 private:
-    vespalib::string _attrName; // the name of the attribute vector
-    vespalib::string _attrKey;  // Used for looking up the attribute in the ObjectStore.
-    vespalib::string _extra;    // the index or key
+    vespalib::string          _attrName; // the name of the attribute vector
+    vespalib::string          _attrKey;  // Used for looking up the attribute in the ObjectStore.
+    vespalib::string          _extra;    // the index or key
     vespalib::eval::ValueType _tensorType;
+    uint8_t                   _numOutputs;
+
 
 public:
     AttributeBlueprint();
diff --git a/searchlib/src/vespa/searchlib/features/attributematchfeature.cpp b/searchlib/src/vespa/searchlib/features/attributematchfeature.cpp
index 4776437a14b..c26d18eb11c 100644
--- a/searchlib/src/vespa/searchlib/features/attributematchfeature.cpp
+++ b/searchlib/src/vespa/searchlib/features/attributematchfeature.cpp
@@ -8,6 +8,7 @@
 #include <vespa/searchlib/fef/properties.h>
 #include <vespa/searchlib/fef/parameterdescriptions.h>
 #include <vespa/searchcommon/attribute/attributecontent.h>
+#include <vespa/vespalib/util/stash.h>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".features.attributematchfeature");
diff --git a/searchlib/src/vespa/searchlib/features/bm25_feature.cpp b/searchlib/src/vespa/searchlib/features/bm25_feature.cpp
index e16b4bba996..64e365b25ac 100644
--- a/searchlib/src/vespa/searchlib/features/bm25_feature.cpp
+++ b/searchlib/src/vespa/searchlib/features/bm25_feature.cpp
@@ -6,8 +6,8 @@
 #include <vespa/searchlib/fef/itermfielddata.h>
 #include <vespa/searchlib/fef/objectstore.h>
 #include <vespa/searchlib/fef/properties.h>
+#include <vespa/vespalib/util/stash.h>
 #include <cmath>
-#include <memory>
 #include <stdexcept>
 
 #include <vespa/log/log.h>
diff --git a/searchlib/src/vespa/searchlib/features/closenessfeature.cpp b/searchlib/src/vespa/searchlib/features/closenessfeature.cpp
index 2358e54e9f5..ca5b5a48ecb 100644
--- a/searchlib/src/vespa/searchlib/features/closenessfeature.cpp
+++ b/searchlib/src/vespa/searchlib/features/closenessfeature.cpp
@@ -2,15 +2,75 @@
 
 #include "closenessfeature.h"
 #include "utils.h"
+#include <vespa/searchcommon/common/schema.h>
 #include <vespa/searchlib/fef/properties.h>
+#include <vespa/vespalib/util/stash.h>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".features.closenessfeature");
 
 using namespace search::fef;
 
-namespace search {
-namespace features {
+namespace search::features {
+
+/** Implements the executor for converting NNS rawscore to a closeness feature. */
+class ConvertRawScoreToCloseness : public fef::FeatureExecutor {
+private:
+    std::vector<fef::TermFieldHandle> _handles;
+    const fef::MatchData             *_md;
+    void handle_bind_match_data(const fef::MatchData &md) override {
+        _md = &md;
+    }
+public:
+    ConvertRawScoreToCloseness(const fef::IQueryEnvironment &env, uint32_t fieldId);
+    ConvertRawScoreToCloseness(const fef::IQueryEnvironment &env, const vespalib::string &label);
+    void execute(uint32_t docId) override;
+};
+
+ConvertRawScoreToCloseness::ConvertRawScoreToCloseness(const fef::IQueryEnvironment &env, uint32_t fieldId)
+  : _handles(),
+    _md(nullptr)
+{
+    _handles.reserve(env.getNumTerms());
+    for (uint32_t i = 0; i < env.getNumTerms(); ++i) {
+        search::fef::TermFieldHandle handle = util::getTermFieldHandle(env, i, fieldId);
+        if (handle != search::fef::IllegalHandle) {
+            _handles.push_back(handle);
+        }
+    }
+}
+
+ConvertRawScoreToCloseness::ConvertRawScoreToCloseness(const fef::IQueryEnvironment &env, const vespalib::string &label)
+  : _handles(),
+    _md(nullptr)
+{
+    const ITermData *term = util::getTermByLabel(env, label);
+    if (term != nullptr) {
+        // expect numFields() == 1
+        for (uint32_t i = 0; i < term->numFields(); ++i) {
+            TermFieldHandle handle = term->field(i).getHandle();
+            if (handle != IllegalHandle) {
+                _handles.push_back(handle);
+            }
+        }
+    }
+}
+
+void
+ConvertRawScoreToCloseness::execute(uint32_t docId)
+{
+    feature_t max_closeness = 0.0;
+    assert(_md);
+    for (auto handle : _handles) {
+        const TermFieldMatchData *tfmd = _md->resolveTermField(handle);
+        if (tfmd->getDocId() == docId) {
+            feature_t converted = tfmd->getRawScore();
+            max_closeness = std::max(max_closeness, converted);
+        }
+    }
+    outputs().set_number(0, max_closeness);
+}
+
 
 ClosenessExecutor::ClosenessExecutor(feature_t maxDistance, feature_t scaleDistance) :
     FeatureExecutor(),
@@ -38,7 +98,12 @@ ClosenessBlueprint::ClosenessBlueprint() :
     Blueprint("closeness"),
     _maxDistance(9013305.0),     // default value (about 250 km)
     _scaleDistance(5.0*9013.305), // default value (about 5 km)
-    _halfResponse(1)
+    _halfResponse(1),
+    _arg_string(),
+    _attr_id(search::index::Schema::UNKNOWN_FIELD_ID),
+    _use_geo_pos(false),
+    _use_nns_tensor(false),
+    _use_item_label(false)
 {
 }
 
@@ -53,6 +118,38 @@ ClosenessBlueprint::setup(const IIndexEnvironment & env,
                           const search::fef::ParameterList & params)
 {
     // params[0] = attribute name
+    vespalib::string arg = params[0].getValue();
+    if (params.size() == 2) {
+        // params[0] = field / label
+        // params[0] = attribute name / label value
+        if (arg == "label") {
+            _arg_string = params[1].getValue();
+            _use_item_label = true;
+            describeOutput("out", "The closeness from the labeled query item.");
+            return true;
+        } else if (arg == "field") {
+            arg = params[1].getValue();
+            // sanity checking happens in distance feature
+        } else {
+            LOG(error, "first argument must be 'field' or 'label', but was '%s'",
+                arg.c_str());
+            return false;
+        }
+    }
+    const FieldInfo *fi = env.getFieldByName(arg);
+    if (fi != nullptr && fi->hasAttribute()) {
+        auto dt = fi->get_data_type();
+        auto ct = fi->collection();
+        if (dt == search::index::schema::DataType::TENSOR &&
+            ct == search::index::schema::CollectionType::SINGLE)
+        {
+            _arg_string = arg;
+            _use_nns_tensor = true;
+            _attr_id = fi->id();
+            describeOutput("out", "The closeness for the given tensor field.");
+            return true;
+        }
+    }
     Property p = env.getProperties().lookup(getName(), "maxDistance");
     if (p.found()) {
         _maxDistance = util::strToNum<feature_t>(p.get());
@@ -85,8 +182,12 @@ ClosenessBlueprint::setup(const IIndexEnvironment & env,
         _scaleDistance = LogarithmCalculator::getScale(_halfResponse, _maxDistance);
     }
 
-
-    defineInput("distance(" + params[0].getValue() + ")");
+    _use_geo_pos = true;
+    if (params.size() == 2) {
+        defineInput("distance(field," + arg + ")");
+    } else {
+        defineInput("distance(" + arg + ")");
+    }
     describeOutput("out", "The closeness of the document (linear)");
     describeOutput("logscale", "The closeness of the document (logarithmic shape)");
 
@@ -96,15 +197,20 @@ ClosenessBlueprint::setup(const IIndexEnvironment & env,
 Blueprint::UP
 ClosenessBlueprint::createInstance() const
 {
-    return Blueprint::UP(new ClosenessBlueprint());
+    return std::make_unique<ClosenessBlueprint>();
 }
 
 FeatureExecutor &
-ClosenessBlueprint::createExecutor(const IQueryEnvironment &, vespalib::Stash &stash) const
+ClosenessBlueprint::createExecutor(const IQueryEnvironment &env, vespalib::Stash &stash) const
 {
+    if (_use_item_label) {
+        return stash.create<ConvertRawScoreToCloseness>(env, _arg_string);
+    }
+    if (_use_nns_tensor) {
+        return stash.create<ConvertRawScoreToCloseness>(env, _attr_id);
+    }
+    assert(_use_geo_pos);
     return stash.create<ClosenessExecutor>(_maxDistance, _scaleDistance);
 }
 
-
-} // namespace features
-} // namespace search
+}
diff --git a/searchlib/src/vespa/searchlib/features/closenessfeature.h b/searchlib/src/vespa/searchlib/features/closenessfeature.h
index d5e3cb36505..8bf63a2c3b6 100644
--- a/searchlib/src/vespa/searchlib/features/closenessfeature.h
+++ b/searchlib/src/vespa/searchlib/features/closenessfeature.h
@@ -29,6 +29,11 @@ private:
     feature_t _maxDistance;
     feature_t _scaleDistance;
     feature_t _halfResponse;
+    vespalib::string _arg_string;
+    uint32_t _attr_id;
+    bool _use_geo_pos;
+    bool _use_nns_tensor;
+    bool _use_item_label;
 
 public:
 
@@ -36,7 +41,7 @@ public:
     void visitDumpFeatures(const fef::IIndexEnvironment & env, fef::IDumpFeatureVisitor & visitor) const override;
     fef::Blueprint::UP createInstance() const override;
     fef::ParameterDescriptions getDescriptions() const override {
-        return fef::ParameterDescriptions().desc().string();
+        return fef::ParameterDescriptions().desc().string().desc().string().string();
     }
     bool setup(const fef::IIndexEnvironment & env, const fef::ParameterList & params) override;
     fef::FeatureExecutor &createExecutor(const fef::IQueryEnvironment &env, vespalib::Stash &stash) const override;
diff --git a/searchlib/src/vespa/searchlib/features/constant_feature.cpp b/searchlib/src/vespa/searchlib/features/constant_feature.cpp
index ced9d95fb33..5eedb5834bf 100644
--- a/searchlib/src/vespa/searchlib/features/constant_feature.cpp
+++ b/searchlib/src/vespa/searchlib/features/constant_feature.cpp
@@ -4,14 +4,14 @@
 #include "valuefeature.h"
 #include <vespa/searchlib/fef/featureexecutor.h>
 #include <vespa/eval/eval/value_cache/constant_value.h>
+#include <vespa/vespalib/util/stash.h>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".features.constant_feature");
 
 using namespace search::fef;
 
-namespace search {
-namespace features {
+namespace search::features {
 
 /**
  * Feature executor that returns a constant value.
@@ -25,8 +25,8 @@ public:
     ConstantFeatureExecutor(const vespalib::eval::Value &value)
         : _value(value)
     {}
-    virtual bool isPure() override { return true; }
-    virtual void execute(uint32_t) override {
+    bool isPure() override { return true; }
+    void execute(uint32_t) override {
         outputs().set_object(0, _value);
     }
     static FeatureExecutor &create(const vespalib::eval::Value &value, vespalib::Stash &stash) {
@@ -41,9 +41,7 @@ ConstantBlueprint::ConstantBlueprint()
 {
 }
 
-ConstantBlueprint::~ConstantBlueprint()
-{
-}
+ConstantBlueprint::~ConstantBlueprint() = default;
 
 void
 ConstantBlueprint::visitDumpFeatures(const IIndexEnvironment &,
@@ -54,7 +52,7 @@ ConstantBlueprint::visitDumpFeatures(const IIndexEnvironment &,
 Blueprint::UP
 ConstantBlueprint::createInstance() const
 {
-    return Blueprint::UP(new ConstantBlueprint());
+    return std::make_unique<ConstantBlueprint>();
 }
 
 bool
@@ -64,9 +62,9 @@ ConstantBlueprint::setup(const IIndexEnvironment &env,
     _key = params[0].getValue();
     _value = env.getConstantValue(_key);
     if (!_value) {
-        LOG(error, "Constant '%s' not found", _key.c_str());
+        fail("Constant '%s' not found", _key.c_str());
     } else if (_value->type().is_error()) {
-        LOG(error, "Constant '%s' has invalid type", _key.c_str());
+        fail("Constant '%s' has invalid type", _key.c_str());
     }
     FeatureType output_type = _value ?
                               FeatureType::object(_value->type()) :
@@ -88,5 +86,4 @@ ConstantBlueprint::createExecutor(const IQueryEnvironment &env, vespalib::Stash
     }
 }
 
-} // namespace features
-} // namespace search
+}
diff --git a/searchlib/src/vespa/searchlib/features/constant_tensor_executor.h b/searchlib/src/vespa/searchlib/features/constant_tensor_executor.h
index 2ab7b98fabe..785eb357795 100644
--- a/searchlib/src/vespa/searchlib/features/constant_tensor_executor.h
+++ b/searchlib/src/vespa/searchlib/features/constant_tensor_executor.h
@@ -7,7 +7,7 @@
 #include <vespa/eval/eval/value.h>
 #include <vespa/eval/eval/value_type.h>
 #include <vespa/eval/tensor/default_tensor_engine.h>
-#include <memory>
+#include <vespa/vespalib/util/stash.h>
 
 namespace search::features {
 
diff --git a/searchlib/src/vespa/searchlib/features/debug_attribute_wait.cpp b/searchlib/src/vespa/searchlib/features/debug_attribute_wait.cpp
index 86a71184f43..8cc75a9a424 100644
--- a/searchlib/src/vespa/searchlib/features/debug_attribute_wait.cpp
+++ b/searchlib/src/vespa/searchlib/features/debug_attribute_wait.cpp
@@ -2,6 +2,8 @@
 
 #include "debug_attribute_wait.h"
 #include <vespa/vespalib/util/time.h>
+#include <vespa/vespalib/util/stash.h>
+
 
 using search::attribute::IAttributeVector;
 using namespace search::fef;
diff --git a/searchlib/src/vespa/searchlib/features/debug_wait.cpp b/searchlib/src/vespa/searchlib/features/debug_wait.cpp
index fb002564572..57d19618ba4 100644
--- a/searchlib/src/vespa/searchlib/features/debug_wait.cpp
+++ b/searchlib/src/vespa/searchlib/features/debug_wait.cpp
@@ -2,6 +2,7 @@
 
 #include "debug_wait.h"
 #include <vespa/vespalib/util/time.h>
+#include <vespa/vespalib/util/stash.h>
 
 using namespace search::fef;
 
diff --git a/searchlib/src/vespa/searchlib/features/distancefeature.cpp b/searchlib/src/vespa/searchlib/features/distancefeature.cpp
index 501bfd7cd14..7a624e64d67 100644
--- a/searchlib/src/vespa/searchlib/features/distancefeature.cpp
+++ b/searchlib/src/vespa/searchlib/features/distancefeature.cpp
@@ -1,25 +1,88 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include "distancefeature.h"
+#include <vespa/searchcommon/common/schema.h>
 #include <vespa/searchlib/fef/location.h>
 #include <vespa/searchlib/fef/matchdata.h>
 #include <vespa/document/datatype/positiondatatype.h>
 #include <vespa/vespalib/geo/zcurve.h>
+#include <vespa/vespalib/util/stash.h>
 #include <cmath>
 #include <limits>
+#include "utils.h"
 
 #include <vespa/log/log.h>
 LOG_SETUP(".features.distancefeature");
 
 using namespace search::fef;
+using namespace search::index::schema;
+
+namespace search::features {
+
+/** Implements the executor for converting NNS rawscore to a distance feature. */
+class ConvertRawscoreToDistance : public fef::FeatureExecutor {
+private:
+    std::vector<fef::TermFieldHandle> _handles;
+    const fef::MatchData             *_md;
+    void handle_bind_match_data(const fef::MatchData &md) override {
+        _md = &md;
+    }
+public:
+    ConvertRawscoreToDistance(const fef::IQueryEnvironment &env, uint32_t fieldId);
+    ConvertRawscoreToDistance(const fef::IQueryEnvironment &env, const vespalib::string &label);
+    void execute(uint32_t docId) override;
+};
+
+ConvertRawscoreToDistance::ConvertRawscoreToDistance(const fef::IQueryEnvironment &env, uint32_t fieldId)
+  : _handles(),
+    _md(nullptr)
+{
+    _handles.reserve(env.getNumTerms());
+    for (uint32_t i = 0; i < env.getNumTerms(); ++i) {
+        search::fef::TermFieldHandle handle = util::getTermFieldHandle(env, i, fieldId);
+        if (handle != search::fef::IllegalHandle) {
+            _handles.push_back(handle);
+        }
+    }
+}
+
+ConvertRawscoreToDistance::ConvertRawscoreToDistance(const fef::IQueryEnvironment &env, const vespalib::string &label)
+  : _handles(),
+    _md(nullptr)
+{
+    const ITermData *term = util::getTermByLabel(env, label);
+    if (term != nullptr) {
+        // expect numFields() == 1
+        for (uint32_t i = 0; i < term->numFields(); ++i) {
+            TermFieldHandle handle = term->field(i).getHandle();
+            if (handle != IllegalHandle) {
+                _handles.push_back(handle);
+            }
+        }
+    }
+}
+
+void
+ConvertRawscoreToDistance::execute(uint32_t docId)
+{
+    feature_t min_distance = std::numeric_limits<feature_t>::max();
+    assert(_md);
+    for (auto handle : _handles) {
+        const TermFieldMatchData *tfmd = _md->resolveTermField(handle);
+        if (tfmd->getDocId() == docId) {
+            feature_t invdist = tfmd->getRawScore();
+            feature_t converted = (1.0 / invdist) - 1.0;
+            min_distance = std::min(min_distance, converted);
+        }
+    }
+    outputs().set_number(0, min_distance);
+}
 
-namespace search {
-namespace features {
 
 feature_t
 DistanceExecutor::calculateDistance(uint32_t docId)
 {
-    if (_location.isValid() && _pos != NULL) {
+    if (_location.isValid() && _pos != nullptr) {
         return calculate2DZDistance(docId);
     }
     return DEFAULT_DISTANCE;
@@ -66,7 +129,7 @@ DistanceExecutor::DistanceExecutor(const Location & location,
     _pos(pos),
     _intBuf()
 {
-    if (_pos != NULL) {
+    if (_pos != nullptr) {
         _intBuf.allocate(_pos->getMaxValueCount());
     }
 }
@@ -82,13 +145,15 @@ const feature_t DistanceExecutor::DEFAULT_DISTANCE(6400000000.0);
 
 DistanceBlueprint::DistanceBlueprint() :
     Blueprint("distance"),
-    _posAttr()
+    _arg_string(),
+    _attr_id(search::index::Schema::UNKNOWN_FIELD_ID),
+    _use_geo_pos(false),
+    _use_nns_tensor(false),
+    _use_item_label(false)
 {
 }
 
-DistanceBlueprint::~DistanceBlueprint()
-{
-}
+DistanceBlueprint::~DistanceBlueprint() = default;
 
 void
 DistanceBlueprint::visitDumpFeatures(const IIndexEnvironment &,
@@ -99,53 +164,115 @@ DistanceBlueprint::visitDumpFeatures(const IIndexEnvironment &,
 Blueprint::UP
 DistanceBlueprint::createInstance() const
 {
-    return Blueprint::UP(new DistanceBlueprint());
+    return std::make_unique<DistanceBlueprint>();
+}
+
+bool
+DistanceBlueprint::setup_geopos(const IIndexEnvironment & env,
+                                const vespalib::string &attr)
+{
+    _arg_string = attr;
+    _use_geo_pos = true;
+    describeOutput("out", "The euclidean distance from the query position.");
+    env.hintAttributeAccess(_arg_string);
+    return true;
+}
+
+bool
+DistanceBlueprint::setup_nns(const IIndexEnvironment & env,
+                             const vespalib::string &attr)
+{
+    _arg_string = attr;
+    _use_nns_tensor = true;
+    describeOutput("out", "The euclidean distance from the query position.");
+    env.hintAttributeAccess(_arg_string);
+    return true;
 }
 
 bool
 DistanceBlueprint::setup(const IIndexEnvironment & env,
                          const ParameterList & params)
 {
-    _posAttr = params[0].getValue();
-    describeOutput("out", "The euclidian distance from the query position.");
-    env.hintAttributeAccess(_posAttr);
-    env.hintAttributeAccess(document::PositionDataType::getZCurveFieldName(_posAttr));
-    return true;
+    // params[0] = attribute name
+    vespalib::string arg = params[0].getValue();
+    bool allow_bad_field = true;
+    if (params.size() == 2) {
+        // params[0] = field / label
+        // params[0] = attribute name / label value
+        if (arg == "label") {
+            _arg_string = params[1].getValue();
+            _use_item_label = true;
+            describeOutput("out", "The euclidean distance from the labeled query item.");
+            return true;
+        } else if (arg == "field") {
+            arg = params[1].getValue();
+            allow_bad_field = false;
+        } else {
+            LOG(error, "first argument must be 'field' or 'label', but was '%s'",
+                arg.c_str());
+            return false;
+        }
+    }
+    const FieldInfo *fi = env.getFieldByName(arg);
+    if (fi != nullptr && fi->hasAttribute()) {
+        auto dt = fi->get_data_type();
+        auto ct = fi->collection();
+        if (dt == DataType::TENSOR && ct == CollectionType::SINGLE) {
+            _attr_id = fi->id();
+            return setup_nns(env, arg);
+        }
+        // could check if dt is DataType::INT64
+        // could check if ct is CollectionType::SINGLE or CollectionType::ARRAY)
+        return setup_geopos(env, arg);
+    }
+    vespalib::string z = document::PositionDataType::getZCurveFieldName(arg);
+    fi = env.getFieldByName(z);
+    if (fi != nullptr && fi->hasAttribute()) {
+        return setup_geopos(env, z);
+    }
+    if (allow_bad_field) {
+        // backwards compatibility fallback:
+        return setup_geopos(env, arg);
+    }
+    if (env.getFieldByName(arg) == nullptr && fi == nullptr) {
+        LOG(error, "unknown field '%s' for rank feature %s\n", arg.c_str(), getName().c_str());
+    } else {
+        LOG(error, "field '%s' must be an attribute for rank feature %s\n", arg.c_str(), getName().c_str());
+    }
+    return false;
 }
 
 FeatureExecutor &
 DistanceBlueprint::createExecutor(const IQueryEnvironment &env, vespalib::Stash &stash) const
 {
-    const search::attribute::IAttributeVector * pos = NULL;
+    if (_use_nns_tensor) {
+        return stash.create<ConvertRawscoreToDistance>(env, _attr_id);
+    }
+    if (_use_item_label) {
+        return stash.create<ConvertRawscoreToDistance>(env, _arg_string);
+    }
+    const search::attribute::IAttributeVector * pos = nullptr;
     const Location & location = env.getLocation();
-    LOG(debug, "DistanceBlueprint::createExecutor location.valid='%s', '%s', alternatively '%s'",
-               location.isValid() ? "true" : "false", _posAttr.c_str(), document::PositionDataType::getZCurveFieldName(_posAttr).c_str());
-    if (location.isValid()) {
-        pos = env.getAttributeContext().getAttribute(_posAttr);
-        if (pos == NULL) {
-            LOG(debug, "Failed to find attribute '%s', resorting too '%s'",
-                       _posAttr.c_str(), document::PositionDataType::getZCurveFieldName(_posAttr).c_str());
-            pos = env.getAttributeContext().getAttribute(document::PositionDataType::getZCurveFieldName(_posAttr));
-        }
-        if (pos != NULL) {
+    LOG(debug, "DistanceBlueprint::createExecutor location.valid='%s', attribute='%s'",
+        location.isValid() ? "true" : "false", _arg_string.c_str());
+    if (_use_geo_pos && location.isValid()) {
+        pos = env.getAttributeContext().getAttribute(_arg_string);
+        if (pos != nullptr) {
             if (!pos->isIntegerType()) {
                 LOG(warning, "The position attribute '%s' is not an integer attribute. Will use default distance.",
                     pos->getName().c_str());
-                pos = NULL;
+                pos = nullptr;
             } else if (pos->getCollectionType() == attribute::CollectionType::WSET) {
                 LOG(warning, "The position attribute '%s' is a weighted set attribute. Will use default distance.",
                     pos->getName().c_str());
-                pos = NULL;
+                pos = nullptr;
             }
         } else {
-            LOG(warning, "The position attribute '%s' was not found. Will use default distance.", _posAttr.c_str());
+            LOG(warning, "The position attribute '%s' was not found. Will use default distance.", _arg_string.c_str());
         }
     }
 
     return stash.create<DistanceExecutor>(location, pos);
 }
 
-
-
-} // namespace features
-} // namespace search
+}
diff --git a/searchlib/src/vespa/searchlib/features/distancefeature.h b/searchlib/src/vespa/searchlib/features/distancefeature.h
index 3c75d53ad77..3a8edd5ee94 100644
--- a/searchlib/src/vespa/searchlib/features/distancefeature.h
+++ b/searchlib/src/vespa/searchlib/features/distancefeature.h
@@ -12,7 +12,7 @@ namespace search::features {
  */
 class DistanceExecutor : public fef::FeatureExecutor {
 private:
-    const fef::Location         & _location;
+    const fef::Location               & _location;
     const attribute::IAttributeVector * _pos;
     attribute::IntegerContent           _intBuf;
 
@@ -37,7 +37,14 @@ public:
  */
 class DistanceBlueprint : public fef::Blueprint {
 private:
-    vespalib::string _posAttr;
+    vespalib::string _arg_string;
+    uint32_t _attr_id;
+    bool _use_geo_pos;
+    bool _use_nns_tensor;
+    bool _use_item_label;
+
+    bool setup_geopos(const fef::IIndexEnvironment & env, const vespalib::string &attr);
+    bool setup_nns(const fef::IIndexEnvironment & env, const vespalib::string &attr);
 
 public:
     DistanceBlueprint();
@@ -45,7 +52,7 @@ public:
     void visitDumpFeatures(const fef::IIndexEnvironment & env, fef::IDumpFeatureVisitor & visitor) const override;
     fef::Blueprint::UP createInstance() const override;
     fef::ParameterDescriptions getDescriptions() const override {
-        return fef::ParameterDescriptions().desc().string();
+        return fef::ParameterDescriptions().desc().string().desc().string().string();
     }
     bool setup(const fef::IIndexEnvironment & env, const fef::ParameterList & params) override;
     fef::FeatureExecutor &createExecutor(const fef::IQueryEnvironment &env, vespalib::Stash &stash) const override;
diff --git a/searchlib/src/vespa/searchlib/features/distancetopathfeature.cpp b/searchlib/src/vespa/searchlib/features/distancetopathfeature.cpp
index 834f5913af9..7112581517e 100644
--- a/searchlib/src/vespa/searchlib/features/distancetopathfeature.cpp
+++ b/searchlib/src/vespa/searchlib/features/distancetopathfeature.cpp
@@ -6,10 +6,10 @@
 #include <vespa/searchlib/fef/properties.h>
 #include <vespa/document/datatype/positiondatatype.h>
 #include <vespa/vespalib/geo/zcurve.h>
+#include <vespa/vespalib/util/stash.h>
 #include <boost/algorithm/string/split.hpp>
 #include <boost/algorithm/string/classification.hpp>
 #include <cmath>
-#include <sstream>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".features.distancetopathfeature");
@@ -25,7 +25,7 @@ DistanceToPathExecutor::DistanceToPathExecutor(std::vector<Vector2> &path,
     _path(),
     _pos(pos)
 {
-    if (_pos != NULL) {
+    if (_pos != nullptr) {
         _intBuf.allocate(_pos->getMaxValueCount());
     }
     _path.swap(path); // avoid copy
@@ -34,7 +34,7 @@ DistanceToPathExecutor::DistanceToPathExecutor(std::vector<Vector2> &path,
 void
 DistanceToPathExecutor::execute(uint32_t docId)
 {
-    if (_path.size() > 1 && _pos != NULL) {
+    if (_path.size() > 1 && _pos != nullptr) {
         double pos = -1, trip = 0, product = 0;
         double minSqDist = std::numeric_limits<double>::max();
         _intBuf.fill(*_pos, docId);
@@ -114,7 +114,7 @@ DistanceToPathBlueprint::setup(const search::fef::IIndexEnvironment & env,
                                const search::fef::ParameterList & params)
 {
     _posAttr = params[0].getValue();
-    describeOutput("distance", "The euclidian distance from the query path.");
+    describeOutput("distance", "The euclidean distance from the query path.");
     describeOutput("traveled", "The normalized distance traveled along the path before intersection.");
     describeOutput("product",  "The cross-product of the intersecting line segment and the intersection-to-document vector.");
     env.hintAttributeAccess(_posAttr);
@@ -145,21 +145,21 @@ DistanceToPathBlueprint::createExecutor(const search::fef::IQueryEnvironment &en
     }
 
     // Lookup the attribute vector that holds document positions.
-    const search::attribute::IAttributeVector *pos = NULL;
+    const search::attribute::IAttributeVector *pos = nullptr;
     if (path.size() > 1) {
         pos = env.getAttributeContext().getAttribute(_posAttr);
-        if (pos == NULL) {
+        if (pos == nullptr) {
             pos = env.getAttributeContext().getAttribute(document::PositionDataType::getZCurveFieldName(_posAttr));
         }
-        if (pos != NULL) {
+        if (pos != nullptr) {
             if (!pos->isIntegerType()) {
                 LOG(warning, "The position attribute '%s' is not an integer attribute. Will use default distance.",
                     pos->getName().c_str());
-                pos = NULL;
+                pos = nullptr;
             } else if (pos->getCollectionType() == attribute::CollectionType::WSET) {
                 LOG(warning, "The position attribute '%s' is a weighted set attribute. Will use default distance.",
                     pos->getName().c_str());
-                pos = NULL;
+                pos = nullptr;
             }
         } else {
             LOG(warning, "The position attribute '%s' was not found. Will use default distance.", _posAttr.c_str());
diff --git a/searchlib/src/vespa/searchlib/features/dotproductfeature.cpp b/searchlib/src/vespa/searchlib/features/dotproductfeature.cpp
index 8998f01b59e..a8737a19eec 100644
--- a/searchlib/src/vespa/searchlib/features/dotproductfeature.cpp
+++ b/searchlib/src/vespa/searchlib/features/dotproductfeature.cpp
@@ -10,12 +10,11 @@
 #include <vespa/searchlib/attribute/floatbase.h>
 #include <vespa/searchlib/attribute/multinumericattribute.h>
 #include <vespa/searchlib/attribute/multienumattribute.h>
-#include <type_traits>
-
-#include <vespa/log/log.h>
 #include <vespa/eval/tensor/serialization/typed_binary_format.h>
 #include <vespa/vespalib/objects/nbostream.h>
+#include <vespa/vespalib/util/stash.h>
 
+#include <vespa/log/log.h>
 LOG_SETUP(".features.dotproduct");
 
 using namespace search::attribute;
@@ -147,7 +146,7 @@ DotProductExecutor<A>::getAttributeValues(uint32_t docId, const AT * & values)
 
 namespace {
 
-class DotProductExecutorByEnum : public fef::FeatureExecutor {
+class DotProductExecutorByEnum final : public fef::FeatureExecutor {
 public:
     using V  = VectorBase<EnumHandle, EnumHandle, feature_t>;
 private:
@@ -183,7 +182,8 @@ DotProductExecutorByEnum::DotProductExecutorByEnum(const IWeightedIndexVector *
 
 DotProductExecutorByEnum::~DotProductExecutorByEnum() = default;
 
-void DotProductExecutorByEnum::execute(uint32_t docId) {
+void
+DotProductExecutorByEnum::execute(uint32_t docId) {
     feature_t val = 0;
     const IWeightedIndexVector::WeightedIndex *values(nullptr);
     uint32_t sz = _attribute->getEnumHandles(docId, values);
@@ -196,6 +196,57 @@ void DotProductExecutorByEnum::execute(uint32_t docId) {
     outputs().set_number(0, val);
 }
 
+class SingleDotProductExecutorByEnum final : public fef::FeatureExecutor {
+public:
+    SingleDotProductExecutorByEnum(const IWeightedIndexVector * attribute, EnumHandle key, feature_t value)
+        : _attribute(attribute),
+          _key(key),
+          _value(value)
+    {}
+
+    void execute(uint32_t docId) override {
+        const IWeightedIndexVector::WeightedIndex *values(nullptr);
+        uint32_t sz = _attribute->getEnumHandles(docId, values);
+        for (size_t i = 0; i < sz; ++i) {
+            if (values[i].value().ref() == _key) {
+                outputs().set_number(0, values[i].weight()*_value);
+                return;
+            }
+        }
+        outputs().set_number(0, 0);
+    }
+private:
+    const IWeightedIndexVector * _attribute;
+    EnumHandle                   _key;
+    feature_t                    _value;
+};
+
+template <typename A>
+class SingleDotProductExecutorByValue final : public fef::FeatureExecutor {
+public:
+    SingleDotProductExecutorByValue(const A * attribute, typename A::BaseType key, feature_t value)
+        : _attribute(attribute),
+          _key(key),
+          _value(value)
+    {}
+
+    void execute(uint32_t docId) override {
+        const multivalue::WeightedValue<typename A::BaseType> *values(nullptr);
+        uint32_t sz = _attribute->getRawValues(docId, values);
+        for (size_t i = 0; i < sz; ++i) {
+            if (values[i].value() == _key) {
+                outputs().set_number(0, values[i].weight() * _value);
+                return;
+            }
+        }
+        outputs().set_number(0, 0);
+    }
+private:
+    const A               * _attribute;
+    typename A::BaseType    _key;
+    feature_t               _value;
+};
+
 }
 
 }
@@ -219,7 +270,7 @@ void DotProductExecutorBase<BaseType>::execute(uint32_t docId) {
     size_t count = getAttributeValues(docId, values);
     size_t commonRange = std::min(count, _queryVector.size());
     static_assert(std::is_same<typename AT::ValueType, BaseType>::value);
-    outputs().set_number(0, _multiplier->dotProduct(
+    outputs().set_number(0, _multiplier.dotProduct(
             &_queryVector[0], reinterpret_cast<const typename AT::ValueType *>(values), commonRange));
 }
 
@@ -573,6 +624,27 @@ createForDirectArray(const IAttributeVector * attribute,
     return createForDirectArrayImpl<A>(attribute, arguments.values, arguments.indexes, stash);
 }
 
+template<typename T>
+size_t extractSize(const dotproduct::wset::IntegerVectorT<T> & v) {
+    return v.getVector().size();
+}
+
+template<typename T>
+std::pair<T, feature_t> extractElem(const dotproduct::wset::IntegerVectorT<T> & v, size_t idx) {
+    const auto & pair = v.getVector()[idx];
+    return std::pair<T, feature_t>(pair.first, pair.second);
+}
+
+template<typename T>
+size_t extractSize(const std::unique_ptr<dotproduct::wset::IntegerVectorT<T>> & v) {
+    return extractSize(*v);
+}
+
+template<typename T>
+std::pair<T, feature_t> extractElem(const std::unique_ptr<dotproduct::wset::IntegerVectorT<T>> & v, size_t idx) {
+    return extractElem(*v, idx);
+}
+
 template <typename A, typename V>
 FeatureExecutor &
 createForDirectWSetImpl(const IAttributeVector * attribute, V && vector, vespalib::Stash & stash)
@@ -585,6 +657,10 @@ createForDirectWSetImpl(const IAttributeVector * attribute, V && vector, vespali
     if (!attribute->isImported() && (iattr != nullptr) && supportsGetRawValues<A, VT>(*iattr)) {
         auto * exactA = dynamic_cast<const ExactA *>(iattr);
         if (exactA != nullptr) {
+            if (extractSize(vector) == 1) {
+                auto elem = extractElem(vector, 0ul);
+                return stash.create<SingleDotProductExecutorByValue<ExactA>>(exactA, elem.first, elem.second);
+            }
             return stash.create<DotProductExecutor<ExactA>>(exactA, std::forward<V>(vector));
         }
         return stash.create<DotProductExecutor<A>>(iattr, std::forward<V>(vector));
@@ -643,6 +719,10 @@ createFromObject(const IAttributeVector * attribute, const fef::Anything & objec
             }
             const auto * getEnumHandles = dynamic_cast<const IWeightedIndexVector *>(attribute);
             if (supportsGetEnumHandles(getEnumHandles)) {
+                if (vector.getVector().size() == 1) {
+                    const auto & elem = vector.getVector()[0];
+                    return stash.create<SingleDotProductExecutorByEnum>(getEnumHandles, elem.first, elem.second);
+                }
                 return stash.create<DotProductExecutorByEnum>(getEnumHandles, vector);
             }
             return stash.create<DotProductExecutorByCopy<EnumVector, WeightedEnumContent>>(attribute, vector);
@@ -733,6 +813,10 @@ createTypedWsetExecutor(const IAttributeVector * attribute, const Property & pro
         vector->syncMap();
         auto * getEnumHandles = dynamic_cast<const IWeightedIndexVector *>(attribute);
         if (supportsGetEnumHandles(getEnumHandles)) {
+            if (vector->getVector().size() == 1) {
+                const auto & elem = vector->getVector()[0];
+                return stash.create<SingleDotProductExecutorByEnum>(getEnumHandles, elem.first, elem.second);
+            }
             return stash.create<DotProductExecutorByEnum>(getEnumHandles, std::move(vector));
         }
         return stash.create<DotProductExecutorByCopy<EnumVector, WeightedEnumContent>>(attribute, std::move(vector));
diff --git a/searchlib/src/vespa/searchlib/features/dotproductfeature.h b/searchlib/src/vespa/searchlib/features/dotproductfeature.h
index 8d2b3d9de72..bca6983877c 100644
--- a/searchlib/src/vespa/searchlib/features/dotproductfeature.h
+++ b/searchlib/src/vespa/searchlib/features/dotproductfeature.h
@@ -49,7 +49,7 @@ class VectorBase : public fef::Anything {
 public:
     typedef std::pair<DimensionVType, ComponentType> Element; // <dimension, component>
     typedef std::vector<Element>                    Vector;
-    typedef vespalib::hash_map<DimensionHType, ComponentType, vespalib::hash<DimensionHType>, HashMapComparator> HashMap;
+    typedef vespalib::hash_map<DimensionHType, ComponentType, vespalib::hash<DimensionHType>, HashMapComparator, vespalib::hashtable_base::and_modulator> HashMap;
 protected:
     VectorBase();
     Vector _vector;
@@ -181,8 +181,8 @@ public:
     using AT = multivalue::Value<BaseType>;
     using V  = std::vector<BaseType>;
 private:
-    vespalib::hwaccelrated::IAccelrated::UP _multiplier;
-    V                                       _queryVector;
+    const vespalib::hwaccelrated::IAccelrated   & _multiplier;
+    V                                             _queryVector;
     virtual size_t getAttributeValues(uint32_t docid, const AT * & count) = 0;
 public:
     DotProductExecutorBase(const V & queryVector);
diff --git a/searchlib/src/vespa/searchlib/features/element_completeness_feature.cpp b/searchlib/src/vespa/searchlib/features/element_completeness_feature.cpp
index 1622a87e733..1014fe4679a 100644
--- a/searchlib/src/vespa/searchlib/features/element_completeness_feature.cpp
+++ b/searchlib/src/vespa/searchlib/features/element_completeness_feature.cpp
@@ -5,6 +5,7 @@
 #include <vespa/searchlib/fef/featurenamebuilder.h>
 #include <vespa/searchlib/fef/properties.h>
 #include <vespa/vespalib/locale/c.h>
+#include <vespa/vespalib/util/stash.h>
 #include <cassert>
 
 namespace search::features {
@@ -116,7 +117,7 @@ ElementCompletenessBlueprint::visitDumpFeatures(const fef::IIndexEnvironment &en
 fef::Blueprint::UP
 ElementCompletenessBlueprint::createInstance() const
 {
-    return Blueprint::UP(new ElementCompletenessBlueprint());
+    return std::make_unique<ElementCompletenessBlueprint>();
 }
 
 bool
diff --git a/searchlib/src/vespa/searchlib/features/element_similarity_feature.cpp b/searchlib/src/vespa/searchlib/features/element_similarity_feature.cpp
index 0676b0a46c4..45fcd013fbf 100644
--- a/searchlib/src/vespa/searchlib/features/element_similarity_feature.cpp
+++ b/searchlib/src/vespa/searchlib/features/element_similarity_feature.cpp
@@ -6,6 +6,7 @@
 #include <vespa/searchlib/fef/properties.h>
 #include <vespa/eval/eval/llvm/compiled_function.h>
 #include <vespa/eval/eval/llvm/compile_cache.h>
+#include <vespa/vespalib/util/stash.h>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".features.elementsimilarity");
diff --git a/searchlib/src/vespa/searchlib/features/euclidean_distance_feature.cpp b/searchlib/src/vespa/searchlib/features/euclidean_distance_feature.cpp
index cd007f1396f..9d71d358b46 100644
--- a/searchlib/src/vespa/searchlib/features/euclidean_distance_feature.cpp
+++ b/searchlib/src/vespa/searchlib/features/euclidean_distance_feature.cpp
@@ -4,9 +4,9 @@
 #include "euclidean_distance_feature.h"
 #include "array_parser.hpp"
 #include <vespa/searchlib/attribute/integerbase.h>
-#include <vespa/searchlib/attribute/floatbase.h>
 #include <vespa/searchlib/fef/properties.h>
 #include <vespa/searchcommon/attribute/attributecontent.h>
+#include <vespa/vespalib/util/stash.h>
 #include <cmath>
 
 #include <vespa/log/log.h>
@@ -15,8 +15,7 @@ LOG_SETUP(".features.euclidean_distance_feature");
 using namespace search::attribute;
 using namespace search::fef;
 
-namespace search {
-namespace features {
+namespace search::features {
 
 
 template <typename DataType>
@@ -57,7 +56,7 @@ EuclideanDistanceBlueprint::EuclideanDistanceBlueprint() :
 {
 }
 
-EuclideanDistanceBlueprint::~EuclideanDistanceBlueprint() {}
+EuclideanDistanceBlueprint::~EuclideanDistanceBlueprint() = default;
 
 void
 EuclideanDistanceBlueprint::visitDumpFeatures(const IIndexEnvironment &, IDumpFeatureVisitor &) const
@@ -78,7 +77,7 @@ EuclideanDistanceBlueprint::setup(const IIndexEnvironment &env, const ParameterL
 Blueprint::UP
 EuclideanDistanceBlueprint::createInstance() const
 {
-    return Blueprint::UP(new EuclideanDistanceBlueprint());
+    return std::make_unique<EuclideanDistanceBlueprint>();
 }
 
 namespace {
@@ -97,7 +96,7 @@ FeatureExecutor &
 EuclideanDistanceBlueprint::createExecutor(const IQueryEnvironment &env, vespalib::Stash &stash) const
 {
     const IAttributeVector * attribute = env.getAttributeContext().getAttribute(_attributeName);
-    if (attribute == NULL) {
+    if (attribute == nullptr) {
         LOG(warning, "The attribute vector '%s' was not found in the attribute manager, returning executor with default value.",
             _attributeName.c_str());
         return stash.create<SingleZeroValueExecutor>();
@@ -118,6 +117,5 @@ EuclideanDistanceBlueprint::createExecutor(const IQueryEnvironment &env, vespali
 
 }
 
-} // namespace features
-} // namespace search
+}
 
diff --git a/searchlib/src/vespa/searchlib/features/fieldinfofeature.cpp b/searchlib/src/vespa/searchlib/features/fieldinfofeature.cpp
index d7e17187ff4..18a15ccf541 100644
--- a/searchlib/src/vespa/searchlib/features/fieldinfofeature.cpp
+++ b/searchlib/src/vespa/searchlib/features/fieldinfofeature.cpp
@@ -7,12 +7,11 @@
 #include <vespa/searchlib/fef/fieldinfo.h>
 #include <vespa/searchlib/fef/fieldtype.h>
 #include <vespa/searchlib/fef/featurenamebuilder.h>
-#include <vespa/searchlib/fef/itermdata.h>
 #include <vespa/searchlib/fef/handle.h>
+#include <vespa/vespalib/util/stash.h>
 #include <sstream>
 
-namespace search {
-namespace features {
+namespace search::features {
 
 IndexFieldInfoExecutor::IndexFieldInfoExecutor(feature_t type, feature_t isFilter,
                                                [[maybe_unused]] uint32_t field, uint32_t fieldHandle)
@@ -238,5 +237,4 @@ FieldInfoBlueprint::createExecutor(const fef::IQueryEnvironment &queryEnv, vespa
     return stash.create<ValueExecutor>(values);
 }
 
-} // namespace features
-} // namespace search
+}
diff --git a/searchlib/src/vespa/searchlib/features/fieldlengthfeature.cpp b/searchlib/src/vespa/searchlib/features/fieldlengthfeature.cpp
index d0680e8fc19..0cbcf62de6b 100644
--- a/searchlib/src/vespa/searchlib/features/fieldlengthfeature.cpp
+++ b/searchlib/src/vespa/searchlib/features/fieldlengthfeature.cpp
@@ -6,13 +6,12 @@
 #include <vespa/searchlib/fef/itermdata.h>
 #include <vespa/searchlib/fef/featurenamebuilder.h>
 #include <vespa/searchlib/fef/fieldinfo.h>
-#include <vespa/searchlib/fef/fieldtype.h>
-#include <vespa/vespalib/util/stringfmt.h>
+#include <vespa/vespalib/util/stash.h>
+
 
 using namespace search::fef;
 
-namespace search {
-namespace features {
+namespace search::features {
 
 FieldLengthExecutor::
 FieldLengthExecutor(const IQueryEnvironment &env,
@@ -63,7 +62,7 @@ FieldLengthExecutor::handle_bind_match_data(const MatchData &md)
 
 FieldLengthBlueprint::FieldLengthBlueprint()
     : Blueprint("fieldLength"),
-      _field(NULL)
+      _field(nullptr)
 {
 }
 
@@ -86,18 +85,16 @@ FieldLengthBlueprint::setup(const IIndexEnvironment &env,
 Blueprint::UP
 FieldLengthBlueprint::createInstance() const
 {
-    return Blueprint::UP(new FieldLengthBlueprint());
+    return std::make_unique<FieldLengthBlueprint>();
 }
 
 FeatureExecutor &
 FieldLengthBlueprint::createExecutor(const IQueryEnvironment &env, vespalib::Stash &stash) const
 {
     if (_field == 0) {
-        std::vector<feature_t> values;
-        values.push_back(fef::FieldPositionsIterator::UNKNOWN_LENGTH);
-        return stash.create<ValueExecutor>(values);
+        return stash.create<SingleValueExecutor>(fef::FieldPositionsIterator::UNKNOWN_LENGTH);
     }
     return stash.create<FieldLengthExecutor>(env, _field->id());
 }
 
-}}
+}
diff --git a/searchlib/src/vespa/searchlib/features/fieldmatchfeature.cpp b/searchlib/src/vespa/searchlib/features/fieldmatchfeature.cpp
index 583afa6e698..7ad9eef0506 100644
--- a/searchlib/src/vespa/searchlib/features/fieldmatchfeature.cpp
+++ b/searchlib/src/vespa/searchlib/features/fieldmatchfeature.cpp
@@ -7,6 +7,7 @@
 #include <vespa/searchlib/fef/properties.h>
 #include <vespa/vespalib/util/stringfmt.h>
 #include <vespa/vespalib/locale/c.h>
+#include <vespa/vespalib/util/stash.h>
 
 using namespace search::fef;
 using CollectionType = FieldInfo::CollectionType;
@@ -92,14 +93,12 @@ FieldMatchExecutor::handle_bind_match_data(const fef::MatchData &md)
 
 FieldMatchBlueprint::FieldMatchBlueprint() :
     Blueprint("fieldMatch"),
-    _field(NULL),
+    _field(nullptr),
     _params()
 {
 }
 
-FieldMatchBlueprint::~FieldMatchBlueprint()
-{
-}
+FieldMatchBlueprint::~FieldMatchBlueprint() = default;
 
 void
 FieldMatchBlueprint::visitDumpFeatures(const IIndexEnvironment & env,
@@ -158,7 +157,7 @@ FieldMatchBlueprint::visitDumpFeatures(const IIndexEnvironment & env,
 Blueprint::UP
 FieldMatchBlueprint::createInstance() const
 {
-    return Blueprint::UP(new FieldMatchBlueprint());
+    return std::make_unique<FieldMatchBlueprint>();
 }
 
 bool
@@ -306,5 +305,4 @@ FieldMatchBlueprint::createExecutor(const IQueryEnvironment & env, vespalib::Sta
     return stash.create<FieldMatchExecutor>(env, *_field, _params);
 }
 
-
 }
diff --git a/searchlib/src/vespa/searchlib/features/fieldtermmatchfeature.cpp b/searchlib/src/vespa/searchlib/features/fieldtermmatchfeature.cpp
index a7a00bee956..2f065fee289 100644
--- a/searchlib/src/vespa/searchlib/features/fieldtermmatchfeature.cpp
+++ b/searchlib/src/vespa/searchlib/features/fieldtermmatchfeature.cpp
@@ -8,9 +8,10 @@
 #include <vespa/searchlib/fef/properties.h>
 #include <vespa/searchlib/fef/itermdata.h>
 #include <vespa/vespalib/util/stringfmt.h>
+#include <vespa/vespalib/util/stash.h>
 
-namespace search {
-namespace features {
+
+namespace search::features {
 
 FieldTermMatchExecutor::FieldTermMatchExecutor(const search::fef::IQueryEnvironment &env,
                                                uint32_t fieldId, uint32_t termId) :
@@ -122,7 +123,7 @@ FieldTermMatchBlueprint::setup(const search::fef::IIndexEnvironment &env,
 search::fef::Blueprint::UP
 FieldTermMatchBlueprint::createInstance() const
 {
-    return search::fef::Blueprint::UP(new FieldTermMatchBlueprint());
+    return std::make_unique<FieldTermMatchBlueprint>();
 }
 
 search::fef::FeatureExecutor &
@@ -131,4 +132,4 @@ FieldTermMatchBlueprint::createExecutor(const search::fef::IQueryEnvironment &en
     return stash.create<FieldTermMatchExecutor>(env, _fieldId, _termId);
 }
 
-}}
+}
diff --git a/searchlib/src/vespa/searchlib/features/firstphasefeature.cpp b/searchlib/src/vespa/searchlib/features/firstphasefeature.cpp
index 2e6bae14a44..8025b443206 100644
--- a/searchlib/src/vespa/searchlib/features/firstphasefeature.cpp
+++ b/searchlib/src/vespa/searchlib/features/firstphasefeature.cpp
@@ -4,11 +4,12 @@
 #include <vespa/searchlib/fef/featureexecutor.h>
 #include <vespa/searchlib/fef/indexproperties.h>
 #include <vespa/searchlib/fef/properties.h>
+#include <vespa/vespalib/util/stash.h>
+
 
 using namespace search::fef;
 
-namespace search {
-namespace features {
+namespace search::features {
 
 void
 FirstPhaseExecutor::execute(uint32_t)
@@ -34,17 +35,21 @@ FirstPhaseBlueprint::visitDumpFeatures(const IIndexEnvironment &,
 Blueprint::UP
 FirstPhaseBlueprint::createInstance() const
 {
-    return Blueprint::UP(new FirstPhaseBlueprint());
+    return std::make_unique<FirstPhaseBlueprint>();
 }
 
 bool
 FirstPhaseBlueprint::setup(const IIndexEnvironment & env,
                            const ParameterList &)
 {
-    describeOutput("score", "The ranking score for first phase.",
-                   defineInput(indexproperties::rank::FirstPhase::lookup(env.getProperties()),
-                               AcceptInput::ANY));
-    return true;
+    if (auto maybe_input = defineInput(indexproperties::rank::FirstPhase::lookup(env.getProperties()),
+                                       AcceptInput::ANY))
+    {
+        describeOutput("score", "The ranking score for first phase.", maybe_input.value());
+        return true;
+    } else {
+        return false;
+    }
 }
 
 FeatureExecutor &
@@ -54,5 +59,4 @@ FirstPhaseBlueprint::createExecutor(const IQueryEnvironment &, vespalib::Stash &
 }
 
 
-} // namespace features
-} // namespace search
+}
diff --git a/searchlib/src/vespa/searchlib/features/flow_completeness_feature.cpp b/searchlib/src/vespa/searchlib/features/flow_completeness_feature.cpp
index eda83b991bf..e43faaec4e1 100644
--- a/searchlib/src/vespa/searchlib/features/flow_completeness_feature.cpp
+++ b/searchlib/src/vespa/searchlib/features/flow_completeness_feature.cpp
@@ -7,6 +7,8 @@
 #include <vespa/searchlib/fef/indexproperties.h>
 #include <vespa/vespalib/stllike/hash_map.h>
 #include <vespa/vespalib/locale/c.h>
+#include <vespa/vespalib/util/stash.h>
+
 #include <cassert>
 
 #include <vespa/log/log.h>
@@ -285,7 +287,7 @@ FlowCompletenessBlueprint::visitDumpFeatures(const fef::IIndexEnvironment &env,
 fef::Blueprint::UP
 FlowCompletenessBlueprint::createInstance() const
 {
-    return Blueprint::UP(new FlowCompletenessBlueprint());
+    return std::make_unique<FlowCompletenessBlueprint>();
 }
 
 bool
@@ -318,6 +320,4 @@ FlowCompletenessBlueprint::createExecutor(const fef::IQueryEnvironment &env, ves
     return stash.create<FlowCompletenessExecutor>(env, _params);
 }
 
-//-----------------------------------------------------------------------------
-
 }
diff --git a/searchlib/src/vespa/searchlib/features/foreachfeature.cpp b/searchlib/src/vespa/searchlib/features/foreachfeature.cpp
index a67d8001a36..21167dd23d4 100644
--- a/searchlib/src/vespa/searchlib/features/foreachfeature.cpp
+++ b/searchlib/src/vespa/searchlib/features/foreachfeature.cpp
@@ -6,6 +6,7 @@
 
 #include <vespa/searchlib/fef/properties.h>
 #include <vespa/vespalib/util/stringfmt.h>
+#include <vespa/vespalib/util/stash.h>
 #include <boost/algorithm/string/replace.hpp>
 
 #include <vespa/log/log.h>
@@ -120,9 +121,7 @@ ForeachBlueprint::ForeachBlueprint() :
 {
 }
 
-ForeachBlueprint::~ForeachBlueprint()
-{
-}
+ForeachBlueprint::~ForeachBlueprint() = default;
 
 void
 ForeachBlueprint::visitDumpFeatures(const IIndexEnvironment &,
@@ -171,13 +170,13 @@ ForeachBlueprint::setup(const IIndexEnvironment & env,
 Blueprint::UP
 ForeachBlueprint::createInstance() const
 {
-    return Blueprint::UP(new ForeachBlueprint());
+    return std::make_unique<ForeachBlueprint>();
 }
 
 FeatureExecutor &
 ForeachBlueprint::createExecutor(const IQueryEnvironment &, vespalib::Stash &stash) const
 {
-    if (_executorCreator.get() != NULL) {
+    if (_executorCreator) {
         return _executorCreator->create(_num_inputs, stash);
     }
     return stash.create<SingleZeroValueExecutor>();
diff --git a/searchlib/src/vespa/searchlib/features/freshnessfeature.cpp b/searchlib/src/vespa/searchlib/features/freshnessfeature.cpp
index 11ae8305e16..6e621f61034 100644
--- a/searchlib/src/vespa/searchlib/features/freshnessfeature.cpp
+++ b/searchlib/src/vespa/searchlib/features/freshnessfeature.cpp
@@ -3,14 +3,14 @@
 #include "freshnessfeature.h"
 #include "utils.h"
 #include <vespa/searchlib/fef/properties.h>
+#include <vespa/vespalib/util/stash.h>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".features.freshnessfeature");
 
 using namespace search::fef;
 
-namespace search {
-namespace features {
+namespace search::features {
 
 FreshnessExecutor::FreshnessExecutor(feature_t maxAge, feature_t scaleAge) :
     FeatureExecutor(),
@@ -86,7 +86,7 @@ FreshnessBlueprint::setup(const IIndexEnvironment & env,
 Blueprint::UP
 FreshnessBlueprint::createInstance() const
 {
-    return Blueprint::UP(new FreshnessBlueprint());
+    return std::make_unique<FreshnessBlueprint>();
 }
 
 fef::ParameterDescriptions
@@ -101,7 +101,4 @@ FreshnessBlueprint::createExecutor(const IQueryEnvironment &, vespalib::Stash &s
     return stash.create<FreshnessExecutor>(_maxAge, _scaleAge);
 }
 
-
-} // namespace features
-} // namespace search
-
+}
diff --git a/searchlib/src/vespa/searchlib/features/internal_max_reduce_prod_join_feature.cpp b/searchlib/src/vespa/searchlib/features/internal_max_reduce_prod_join_feature.cpp
index fd1faeae5ea..e2e8a206099 100644
--- a/searchlib/src/vespa/searchlib/features/internal_max_reduce_prod_join_feature.cpp
+++ b/searchlib/src/vespa/searchlib/features/internal_max_reduce_prod_join_feature.cpp
@@ -5,12 +5,13 @@
 #include "weighted_set_parser.h"
 #include "dotproductfeature.h"
 
-#include <vespa/searchlib/attribute/attribute.h>
 #include <vespa/searchlib/attribute/imported_attribute_vector_read_guard.h>
 #include <vespa/searchlib/attribute/multinumericattribute.h>
 #include <vespa/searchlib/fef/properties.h>
 #include <vespa/searchlib/fef/featureexecutor.h>
 #include <vespa/searchcommon/common/datatype.h>
+#include <vespa/vespalib/util/stash.h>
+
 
 #include <vespa/log/log.h>
 LOG_SETUP(".features.internalmaxreduceprodjoin");
diff --git a/searchlib/src/vespa/searchlib/features/item_raw_score_feature.cpp b/searchlib/src/vespa/searchlib/features/item_raw_score_feature.cpp
index 45baf646656..89f7751a369 100644
--- a/searchlib/src/vespa/searchlib/features/item_raw_score_feature.cpp
+++ b/searchlib/src/vespa/searchlib/features/item_raw_score_feature.cpp
@@ -3,11 +3,11 @@
 #include "item_raw_score_feature.h"
 #include "valuefeature.h"
 #include "utils.h"
+#include <vespa/vespalib/util/stash.h>
 
 using namespace search::fef;
 
-namespace search {
-namespace features {
+namespace search::features {
 
 void
 ItemRawScoreExecutor::execute(uint32_t docId)
@@ -89,6 +89,4 @@ ItemRawScoreBlueprint::resolve(const IQueryEnvironment &env,
     return handles;
 }
 
-
-} // namespace features
-} // namespace search
+}
diff --git a/searchlib/src/vespa/searchlib/features/jarowinklerdistancefeature.cpp b/searchlib/src/vespa/searchlib/features/jarowinklerdistancefeature.cpp
index a5e3e2da5ba..dc689459ff3 100644
--- a/searchlib/src/vespa/searchlib/features/jarowinklerdistancefeature.cpp
+++ b/searchlib/src/vespa/searchlib/features/jarowinklerdistancefeature.cpp
@@ -6,6 +6,8 @@
 #include <vespa/searchlib/fef/properties.h>
 #include <vespa/vespalib/util/stringfmt.h>
 #include <vespa/vespalib/locale/c.h>
+#include <vespa/vespalib/util/stash.h>
+
 
 namespace search::features {
 
@@ -168,7 +170,7 @@ JaroWinklerDistanceBlueprint::setup(const search::fef::IIndexEnvironment &env,
 search::fef::Blueprint::UP
 JaroWinklerDistanceBlueprint::createInstance() const
 {
-    return search::fef::Blueprint::UP(new JaroWinklerDistanceBlueprint());
+    return std::make_unique<JaroWinklerDistanceBlueprint>();
 }
 
 search::fef::FeatureExecutor &
diff --git a/searchlib/src/vespa/searchlib/features/matchcountfeature.cpp b/searchlib/src/vespa/searchlib/features/matchcountfeature.cpp
index fd453e17eb1..f19e32793a1 100644
--- a/searchlib/src/vespa/searchlib/features/matchcountfeature.cpp
+++ b/searchlib/src/vespa/searchlib/features/matchcountfeature.cpp
@@ -3,11 +3,11 @@
 #include "matchcountfeature.h"
 #include "utils.h"
 #include "valuefeature.h"
+#include <vespa/vespalib/util/stash.h>
 
 using namespace search::fef;
 
-namespace search {
-namespace features {
+namespace search::features {
 
 MatchCountExecutor::MatchCountExecutor(uint32_t fieldId, const IQueryEnvironment &env)
     : FeatureExecutor(),
@@ -43,7 +43,7 @@ MatchCountExecutor::handle_bind_match_data(const MatchData &md)
 
 MatchCountBlueprint::MatchCountBlueprint() :
     Blueprint("matchCount"),
-    _field(NULL)
+    _field(nullptr)
 {
 }
 
@@ -63,17 +63,16 @@ MatchCountBlueprint::setup(const IIndexEnvironment &, const ParameterList & para
 Blueprint::UP
 MatchCountBlueprint::createInstance() const
 {
-    return Blueprint::UP(new MatchCountBlueprint());
+    return std::make_unique<MatchCountBlueprint>();
 }
 
 FeatureExecutor &
 MatchCountBlueprint::createExecutor(const IQueryEnvironment & queryEnv, vespalib::Stash &stash) const
 {
     if (_field == nullptr) {
-        return stash.create<ValueExecutor>(std::vector<feature_t>(1, 0.0));
+        return stash.create<SingleZeroValueExecutor>();
     }
     return stash.create<MatchCountExecutor>(_field->id(), queryEnv);
 }
 
-} // namespace features
-} // namespace search
+}
diff --git a/searchlib/src/vespa/searchlib/features/matchesfeature.cpp b/searchlib/src/vespa/searchlib/features/matchesfeature.cpp
index f4788ee74c8..8d72978adf7 100644
--- a/searchlib/src/vespa/searchlib/features/matchesfeature.cpp
+++ b/searchlib/src/vespa/searchlib/features/matchesfeature.cpp
@@ -4,11 +4,12 @@
 #include "utils.h"
 #include "valuefeature.h"
 #include <vespa/searchlib/fef/fieldinfo.h>
+#include <vespa/vespalib/util/stash.h>
+
 
 using namespace search::fef;
 
-namespace search {
-namespace features {
+namespace search::features {
 
 MatchesExecutor::MatchesExecutor(uint32_t fieldId,
                                  const search::fef::IQueryEnvironment &env,
@@ -47,7 +48,7 @@ MatchesExecutor::handle_bind_match_data(const MatchData &md)
 
 MatchesBlueprint::MatchesBlueprint() :
     Blueprint("matches"),
-    _field(NULL),
+    _field(nullptr),
     _termIdx(std::numeric_limits<uint32_t>::max())
 {
 }
@@ -73,14 +74,14 @@ MatchesBlueprint::setup(const IIndexEnvironment &,
 Blueprint::UP
 MatchesBlueprint::createInstance() const
 {
-    return Blueprint::UP(new MatchesBlueprint());
+    return std::make_unique<MatchesBlueprint>();
 }
 
 FeatureExecutor &
 MatchesBlueprint::createExecutor(const IQueryEnvironment & queryEnv, vespalib::Stash &stash) const
 {
     if (_field == 0) {
-        return stash.create<ValueExecutor>(std::vector<feature_t>(1, 0.0));
+        return stash.create<SingleZeroValueExecutor>();
     }
     if (_termIdx != std::numeric_limits<uint32_t>::max()) {
         return stash.create<MatchesExecutor>(_field->id(), queryEnv, _termIdx, _termIdx + 1);
@@ -89,5 +90,4 @@ MatchesBlueprint::createExecutor(const IQueryEnvironment & queryEnv, vespalib::S
     }
 }
 
-} // namespace features
-} // namespace search
+}
diff --git a/searchlib/src/vespa/searchlib/features/matchfeature.cpp b/searchlib/src/vespa/searchlib/features/matchfeature.cpp
index 7210b8b67e9..f6843df1a2f 100644
--- a/searchlib/src/vespa/searchlib/features/matchfeature.cpp
+++ b/searchlib/src/vespa/searchlib/features/matchfeature.cpp
@@ -2,17 +2,16 @@
 
 #include "matchfeature.h"
 #include "utils.h"
-#include <vespa/searchlib/fef/featurenamebuilder.h>
 #include <vespa/searchlib/fef/fieldinfo.h>
 #include <vespa/searchlib/fef/indexproperties.h>
 #include <vespa/searchlib/fef/properties.h>
-#include <vespa/vespalib/util/stringfmt.h>
+#include <vespa/vespalib/util/stash.h>
+
 
 using namespace search::fef;
 using CollectionType = FieldInfo::CollectionType;
 
-namespace search {
-namespace features {
+namespace search::features {
 
 MatchExecutor::MatchExecutor(const MatchParams & params) :
     FeatureExecutor(),
@@ -46,9 +45,7 @@ MatchBlueprint::MatchBlueprint() :
 {
 }
 
-MatchBlueprint::~MatchBlueprint()
-{
-}
+MatchBlueprint::~MatchBlueprint() = default;
 
 void
 MatchBlueprint::visitDumpFeatures(const IIndexEnvironment & env,
@@ -61,7 +58,7 @@ MatchBlueprint::visitDumpFeatures(const IIndexEnvironment & env,
 Blueprint::UP
 MatchBlueprint::createInstance() const
 {
-    return Blueprint::UP(new MatchBlueprint());
+    return std::make_unique<MatchBlueprint>();
 }
 
 bool
@@ -101,6 +98,4 @@ MatchBlueprint::createExecutor(const IQueryEnvironment &env, vespalib::Stash &st
     return stash.create<MatchExecutor>(_params);
 }
 
-
-} // namespace features
-} // namespace search
+}
diff --git a/searchlib/src/vespa/searchlib/features/native_dot_product_feature.cpp b/searchlib/src/vespa/searchlib/features/native_dot_product_feature.cpp
index 7865e32849f..5ede14130ec 100644
--- a/searchlib/src/vespa/searchlib/features/native_dot_product_feature.cpp
+++ b/searchlib/src/vespa/searchlib/features/native_dot_product_feature.cpp
@@ -2,11 +2,11 @@
 
 #include "native_dot_product_feature.h"
 #include "utils.h"
+#include <vespa/vespalib/util/stash.h>
 
 using namespace search::fef;
 
-namespace search {
-namespace features {
+namespace search::features {
 
 NativeDotProductExecutor::NativeDotProductExecutor(const search::fef::IQueryEnvironment &env)
     : FeatureExecutor(),
@@ -80,5 +80,4 @@ NativeDotProductBlueprint::createExecutor(const IQueryEnvironment &queryEnv, ves
     }
 }
 
-} // namespace features
-} // namespace search
+}
diff --git a/searchlib/src/vespa/searchlib/features/nativeattributematchfeature.cpp b/searchlib/src/vespa/searchlib/features/nativeattributematchfeature.cpp
index 5dda159f629..ab8544634b5 100644
--- a/searchlib/src/vespa/searchlib/features/nativeattributematchfeature.cpp
+++ b/searchlib/src/vespa/searchlib/features/nativeattributematchfeature.cpp
@@ -7,11 +7,11 @@
 #include <vespa/searchlib/fef/indexproperties.h>
 #include <vespa/searchlib/fef/itablemanager.h>
 #include <vespa/searchlib/fef/properties.h>
+#include <vespa/vespalib/util/stash.h>
 
 using namespace search::fef;
 
-namespace search {
-namespace features {
+namespace search::features {
 
 feature_t
 NativeAttributeMatchExecutor::calculateScore(const CachedTermData &td, const TermFieldMatchData &tfmd)
@@ -51,7 +51,7 @@ NativeAttributeMatchExecutor::createExecutor(const IQueryEnvironment & env,
 {
     Precomputed setup = preComputeSetup(env, params);
     if (setup.first.size() == 0) {
-        return stash.create<ValueExecutor>(std::vector<feature_t>(1, 0.0));
+        return stash.create<SingleZeroValueExecutor>();
     } else if (setup.first.size() == 1) {
         return stash.create<NativeAttributeMatchExecutorSingle>(setup);
     } else {
@@ -115,7 +115,7 @@ NativeAttributeMatchBlueprint::visitDumpFeatures(const IIndexEnvironment & env,
 Blueprint::UP
 NativeAttributeMatchBlueprint::createInstance() const
 {
-    return Blueprint::UP(new NativeAttributeMatchBlueprint());
+    return std::make_unique<NativeAttributeMatchBlueprint>();
 }
 
 fef::ParameterDescriptions
@@ -160,6 +160,4 @@ NativeAttributeMatchBlueprint::createExecutor(const IQueryEnvironment &env, vesp
     return NativeAttributeMatchExecutor::createExecutor(env, _params, stash);
 }
 
-
-} // namespace features
-} // namespace search
+}
diff --git a/searchlib/src/vespa/searchlib/features/nativefieldmatchfeature.cpp b/searchlib/src/vespa/searchlib/features/nativefieldmatchfeature.cpp
index 089a8102d6e..2cdda669dd7 100644
--- a/searchlib/src/vespa/searchlib/features/nativefieldmatchfeature.cpp
+++ b/searchlib/src/vespa/searchlib/features/nativefieldmatchfeature.cpp
@@ -7,11 +7,11 @@
 #include <vespa/searchlib/fef/indexproperties.h>
 #include <vespa/searchlib/fef/itablemanager.h>
 #include <vespa/searchlib/fef/properties.h>
+#include <vespa/vespalib/util/stash.h>
 
 using namespace search::fef;
 
-namespace search {
-namespace features {
+namespace search::features {
 
 const uint32_t NativeFieldMatchParam::NOT_DEF_FIELD_LENGTH(std::numeric_limits<uint32_t>::max());
 
@@ -95,9 +95,7 @@ NativeFieldMatchBlueprint::NativeFieldMatchBlueprint() :
 {
 }
 
-NativeFieldMatchBlueprint::~NativeFieldMatchBlueprint()
-{
-}
+NativeFieldMatchBlueprint::~NativeFieldMatchBlueprint() = default;
 
 void
 NativeFieldMatchBlueprint::visitDumpFeatures(const IIndexEnvironment & env,
@@ -110,7 +108,7 @@ NativeFieldMatchBlueprint::visitDumpFeatures(const IIndexEnvironment & env,
 Blueprint::UP
 NativeFieldMatchBlueprint::createInstance() const
 {
-    return Blueprint::UP(new NativeFieldMatchBlueprint());
+    return std::make_unique<NativeFieldMatchBlueprint>();
 }
 
 bool
@@ -175,11 +173,10 @@ NativeFieldMatchBlueprint::createExecutor(const IQueryEnvironment &env, vespalib
 {
     NativeFieldMatchExecutor &native = stash.create<NativeFieldMatchExecutor>(env, _params);
     if (native.empty()) {
-        return stash.create<ValueExecutor>(std::vector<feature_t>(1, 0.0));
+        return stash.create<SingleZeroValueExecutor>();
     } else {
         return native;
     }
 }
 
-} // namespace features
-} // namespace search
+}
diff --git a/searchlib/src/vespa/searchlib/features/nativeproximityfeature.cpp b/searchlib/src/vespa/searchlib/features/nativeproximityfeature.cpp
index a31d9207e05..4da819b4dd3 100644
--- a/searchlib/src/vespa/searchlib/features/nativeproximityfeature.cpp
+++ b/searchlib/src/vespa/searchlib/features/nativeproximityfeature.cpp
@@ -7,12 +7,12 @@
 #include <vespa/searchlib/fef/indexproperties.h>
 #include <vespa/searchlib/fef/itablemanager.h>
 #include <vespa/searchlib/fef/properties.h>
+#include <vespa/vespalib/util/stash.h>
 #include <map>
 
 using namespace search::fef;
 
-namespace search {
-namespace features {
+namespace search::features {
 
 feature_t
 NativeProximityExecutor::calculateScoreForField(const FieldSetup & fs, uint32_t docId)
@@ -136,9 +136,7 @@ NativeProximityBlueprint::NativeProximityBlueprint() :
 {
 }
 
-NativeProximityBlueprint::~NativeProximityBlueprint()
-{
-}
+NativeProximityBlueprint::~NativeProximityBlueprint() = default;
 
 void
 NativeProximityBlueprint::visitDumpFeatures(const IIndexEnvironment & env,
@@ -151,7 +149,7 @@ NativeProximityBlueprint::visitDumpFeatures(const IIndexEnvironment & env,
 Blueprint::UP
 NativeProximityBlueprint::createInstance() const
 {
-    return Blueprint::UP(new NativeProximityBlueprint());
+    return std::make_unique<NativeProximityBlueprint>();
 }
 
 bool
@@ -168,12 +166,12 @@ NativeProximityBlueprint::setup(const IIndexEnvironment & env,
         NativeProximityParam & param = _params.vector[fieldId];
         param.field = true;
         if ((param.proximityTable =
-             util::lookupTable(env, getBaseName(), "proximityTable", info->name(), _defaultProximityBoost)) == NULL)
+             util::lookupTable(env, getBaseName(), "proximityTable", info->name(), _defaultProximityBoost)) == nullptr)
         {
             return false;
         }
         if ((param.revProximityTable =
-             util::lookupTable(env, getBaseName(), "reverseProximityTable", info->name(), _defaultRevProximityBoost)) == NULL)
+             util::lookupTable(env, getBaseName(), "reverseProximityTable", info->name(), _defaultRevProximityBoost)) == nullptr)
         {
             return false;
         }
@@ -190,7 +188,7 @@ NativeProximityBlueprint::setup(const IIndexEnvironment & env,
         if (NativeRankBlueprint::useTableNormalization(env)) {
             const Table * fp = param.proximityTable;
             const Table * rp = param.revProximityTable;
-            if (fp != NULL && rp != NULL) {
+            if (fp != nullptr && rp != nullptr) {
                 double value = (fp->max() * param.proximityImportance) +
                     (rp->max() * (1 - param.proximityImportance));
                 _params.setMaxTableSums(fieldId, value);
@@ -210,13 +208,11 @@ NativeProximityBlueprint::createExecutor(const IQueryEnvironment &env, vespalib:
 {
     NativeProximityExecutor &native = stash.create<NativeProximityExecutor>(env, _params);
     if (native.empty()) {
-        return stash.create<ValueExecutor>(std::vector<feature_t>(1, 0.0));
+        return stash.create<SingleZeroValueExecutor>();
     } else {
         return native;
     }
 
 }
 
-
-} // namespace features
-} // namespace search
+}
diff --git a/searchlib/src/vespa/searchlib/features/nativerankfeature.cpp b/searchlib/src/vespa/searchlib/features/nativerankfeature.cpp
index b519c4f4b7f..07f1dfb9d15 100644
--- a/searchlib/src/vespa/searchlib/features/nativerankfeature.cpp
+++ b/searchlib/src/vespa/searchlib/features/nativerankfeature.cpp
@@ -4,6 +4,7 @@
 #include "valuefeature.h"
 #include "utils.h"
 #include <vespa/searchlib/fef/properties.h>
+#include <vespa/vespalib/util/stash.h>
 #include <sstream>
 
 #include <vespa/log/log.h>
@@ -30,8 +31,7 @@ buildFeatureName(const vespalib::string & baseName, const search::features::Fiel
 
 }
 
-namespace search {
-namespace features {
+namespace search::features {
 
 FieldWrapper::FieldWrapper(const IIndexEnvironment & env,
                            const ParameterList & fields,
@@ -93,7 +93,7 @@ NativeRankBlueprint::visitDumpFeatures(const IIndexEnvironment & env,
 Blueprint::UP
 NativeRankBlueprint::createInstance() const
 {
-    return Blueprint::UP(new NativeRankBlueprint());
+    return std::make_unique<NativeRankBlueprint>();
 }
 
 bool
@@ -157,7 +157,7 @@ NativeRankBlueprint::createExecutor(const IQueryEnvironment &, vespalib::Stash &
     if (_params.proximityWeight + _params.fieldMatchWeight + _params.attributeMatchWeight > 0) {
         return stash.create<NativeRankExecutor>(_params);
     } else {
-        return stash.create<ValueExecutor>(std::vector<feature_t>(1, 0.0));
+        return stash.create<SingleZeroValueExecutor>();
     }
 }
 
@@ -168,6 +168,4 @@ NativeRankBlueprint::useTableNormalization(const search::fef::IIndexEnvironment
     return (!(norm.found() && (norm.get() == vespalib::string("false"))));
 }
 
-
-} // namespace features
-} // namespace search
+}
diff --git a/searchlib/src/vespa/searchlib/features/nowfeature.cpp b/searchlib/src/vespa/searchlib/features/nowfeature.cpp
index 074acb2e890..d6059592cf3 100644
--- a/searchlib/src/vespa/searchlib/features/nowfeature.cpp
+++ b/searchlib/src/vespa/searchlib/features/nowfeature.cpp
@@ -3,6 +3,7 @@
 #include "nowfeature.h"
 #include <vespa/searchlib/fef/queryproperties.h>
 #include <vespa/searchlib/fef/properties.h>
+#include <vespa/vespalib/util/stash.h>
 #include <chrono>
 
 namespace search::features {
diff --git a/searchlib/src/vespa/searchlib/features/proximityfeature.cpp b/searchlib/src/vespa/searchlib/features/proximityfeature.cpp
index f625e30f378..daeb4af6569 100644
--- a/searchlib/src/vespa/searchlib/features/proximityfeature.cpp
+++ b/searchlib/src/vespa/searchlib/features/proximityfeature.cpp
@@ -2,13 +2,11 @@
 
 #include "proximityfeature.h"
 #include "utils.h"
-#include <vespa/searchlib/fef/featurenamebuilder.h>
 #include <vespa/searchlib/fef/fieldinfo.h>
 #include <vespa/searchlib/fef/itermdata.h>
-#include <vespa/vespalib/util/stringfmt.h>
+#include <vespa/vespalib/util/stash.h>
 
-namespace search {
-namespace features {
+namespace search::features {
 
 ProximityConfig::ProximityConfig() :
     fieldId(search::fef::IllegalHandle),
@@ -139,7 +137,7 @@ ProximityBlueprint::setup(const search::fef::IIndexEnvironment &env,
 search::fef::Blueprint::UP
 ProximityBlueprint::createInstance() const
 {
-    return search::fef::Blueprint::UP(new ProximityBlueprint());
+    return std::make_unique<ProximityBlueprint>();
 }
 
 search::fef::FeatureExecutor &
@@ -148,4 +146,4 @@ ProximityBlueprint::createExecutor(const search::fef::IQueryEnvironment &env, ve
     return stash.create<ProximityExecutor>(env, _config);
 }
 
-}}
+}
diff --git a/searchlib/src/vespa/searchlib/features/querycompletenessfeature.cpp b/searchlib/src/vespa/searchlib/features/querycompletenessfeature.cpp
index b4b6a1b0eb4..56e8810e14b 100644
--- a/searchlib/src/vespa/searchlib/features/querycompletenessfeature.cpp
+++ b/searchlib/src/vespa/searchlib/features/querycompletenessfeature.cpp
@@ -3,15 +3,14 @@
 #include "querycompletenessfeature.h"
 #include "utils.h"
 #include <vespa/searchlib/fef/featurenamebuilder.h>
-#include <vespa/searchlib/fef/fieldinfo.h>
 #include <vespa/searchlib/fef/itermdata.h>
+#include <vespa/vespalib/util/stash.h>
 #include <limits>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".features.querycompleteness");
 
-namespace search {
-namespace features {
+namespace search::features {
 
 QueryCompletenessConfig::QueryCompletenessConfig() :
     fieldId(search::fef::IllegalHandle),
@@ -106,7 +105,7 @@ QueryCompletenessBlueprint::setup(const search::fef::IIndexEnvironment &env,
 search::fef::Blueprint::UP
 QueryCompletenessBlueprint::createInstance() const
 {
-    return search::fef::Blueprint::UP(new QueryCompletenessBlueprint());
+    return std::make_unique<QueryCompletenessBlueprint>();
 }
 
 search::fef::FeatureExecutor &
@@ -115,4 +114,4 @@ QueryCompletenessBlueprint::createExecutor(const search::fef::IQueryEnvironment
     return stash.create<QueryCompletenessExecutor>(env, _config);
 }
 
-}}
+}
diff --git a/searchlib/src/vespa/searchlib/features/queryfeature.cpp b/searchlib/src/vespa/searchlib/features/queryfeature.cpp
index b927188c1aa..12cad613a80 100644
--- a/searchlib/src/vespa/searchlib/features/queryfeature.cpp
+++ b/searchlib/src/vespa/searchlib/features/queryfeature.cpp
@@ -146,11 +146,10 @@ QueryBlueprint::createExecutor(const IQueryEnvironment &env, vespalib::Stash &st
             p = env.getProperties().lookup(_key2);
         }
         if (p.found()) {
-            values.push_back(asFeature(p.get()));
+            return stash.create<SingleValueExecutor>(asFeature(p.get()));
         } else {
-            values.push_back(_defaultValue);
+            return stash.create<SingleValueExecutor>(_defaultValue);
         }
-        return stash.create<ValueExecutor>(values);
     }
 }
 
diff --git a/searchlib/src/vespa/searchlib/features/querytermcountfeature.cpp b/searchlib/src/vespa/searchlib/features/querytermcountfeature.cpp
index dfc4af059a1..741501d3b6d 100644
--- a/searchlib/src/vespa/searchlib/features/querytermcountfeature.cpp
+++ b/searchlib/src/vespa/searchlib/features/querytermcountfeature.cpp
@@ -4,15 +4,14 @@
 #include "valuefeature.h"
 #include <vespa/searchlib/fef/properties.h>
 #include <vespa/searchlib/fef/fieldinfo.h>
-#include <vespa/searchlib/fef/fieldtype.h>
 #include <vespa/searchlib/fef/featurenamebuilder.h>
 #include <vespa/searchlib/fef/itermdata.h>
-#include <vespa/searchlib/fef/handle.h>
+#include <vespa/vespalib/util/stash.h>
+
 
 using namespace search::fef;
 
-namespace search {
-namespace features {
+namespace search::features {
 
 QueryTermCountBlueprint::QueryTermCountBlueprint() :
     Blueprint("queryTermCount")
@@ -30,7 +29,7 @@ QueryTermCountBlueprint::visitDumpFeatures(const IIndexEnvironment & env,
 Blueprint::UP
 QueryTermCountBlueprint::createInstance() const
 {
-    return Blueprint::UP(new QueryTermCountBlueprint());
+    return std::make_unique<QueryTermCountBlueprint>();
 }
 
 bool
@@ -44,11 +43,7 @@ QueryTermCountBlueprint::setup(const IIndexEnvironment &,
 FeatureExecutor &
 QueryTermCountBlueprint::createExecutor(const IQueryEnvironment &env, vespalib::Stash &stash) const
 {
-    std::vector<feature_t> values;
-    values.push_back(static_cast<feature_t>(env.getNumTerms()));
-    return stash.create<ValueExecutor>(values);
+    return stash.create<SingleValueExecutor>(env.getNumTerms());
 }
 
-
-} // namespace features
-} // namespace search
+}
diff --git a/searchlib/src/vespa/searchlib/features/random_normal_feature.cpp b/searchlib/src/vespa/searchlib/features/random_normal_feature.cpp
index dd0c67df45c..8d4af8fd88d 100644
--- a/searchlib/src/vespa/searchlib/features/random_normal_feature.cpp
+++ b/searchlib/src/vespa/searchlib/features/random_normal_feature.cpp
@@ -3,6 +3,7 @@
 #include "random_normal_feature.h"
 #include "utils.h"
 #include <vespa/searchlib/fef/properties.h>
+#include <vespa/vespalib/util/stash.h>
 #include <chrono>
 
 #include <vespa/log/log.h>
diff --git a/searchlib/src/vespa/searchlib/features/random_normal_stable_feature.cpp b/searchlib/src/vespa/searchlib/features/random_normal_stable_feature.cpp
index bde0cedaed0..f3d33c7dc29 100644
--- a/searchlib/src/vespa/searchlib/features/random_normal_stable_feature.cpp
+++ b/searchlib/src/vespa/searchlib/features/random_normal_stable_feature.cpp
@@ -3,6 +3,7 @@
 #include "random_normal_stable_feature.h"
 #include "utils.h"
 #include <vespa/searchlib/fef/properties.h>
+#include <vespa/vespalib/util/stash.h>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".features.randomnormalstablefeature");
@@ -41,7 +42,7 @@ RandomNormalStableBlueprint::visitDumpFeatures(const search::fef::IIndexEnvironm
 search::fef::Blueprint::UP
 RandomNormalStableBlueprint::createInstance() const
 {
-    return search::fef::Blueprint::UP(new RandomNormalStableBlueprint());
+    return std::make_unique<RandomNormalStableBlueprint>();
 }
 
 bool
@@ -75,5 +76,4 @@ RandomNormalStableBlueprint::createExecutor(const search::fef::IQueryEnvironment
     return stash.create<RandomNormalStableExecutor>(seed, _mean, _stddev);
 }
 
-
 }
diff --git a/searchlib/src/vespa/searchlib/features/randomfeature.cpp b/searchlib/src/vespa/searchlib/features/randomfeature.cpp
index 18b0cf616d4..95daebd0452 100644
--- a/searchlib/src/vespa/searchlib/features/randomfeature.cpp
+++ b/searchlib/src/vespa/searchlib/features/randomfeature.cpp
@@ -3,7 +3,9 @@
 #include "randomfeature.h"
 #include "utils.h"
 #include <vespa/searchlib/fef/properties.h>
+#include <vespa/vespalib/util/stash.h>
 #include <chrono>
+
 #include <vespa/log/log.h>
 LOG_SETUP(".features.randomfeature");
 
@@ -75,5 +77,4 @@ RandomBlueprint::createExecutor(const fef::IQueryEnvironment &env, vespalib::Sta
     return stash.create<RandomExecutor>(seed, matchSeed);
 }
 
-
 }
diff --git a/searchlib/src/vespa/searchlib/features/rankingexpression/intrinsic_blueprint_adapter.cpp b/searchlib/src/vespa/searchlib/features/rankingexpression/intrinsic_blueprint_adapter.cpp
index 4ff9d2f4e30..4b777802985 100644
--- a/searchlib/src/vespa/searchlib/features/rankingexpression/intrinsic_blueprint_adapter.cpp
+++ b/searchlib/src/vespa/searchlib/features/rankingexpression/intrinsic_blueprint_adapter.cpp
@@ -12,14 +12,14 @@ namespace search::features::rankingexpression {
 
 namespace {
 
-bool is_valid(const FeatureType *type) {
-    if (type == nullptr) {
+bool is_valid(const std::optional<FeatureType> &type) {
+    if (!type.has_value()) {
         return false;
     }
-    if (!type->is_object()) {
+    if (!type.value().is_object()) {
         return true;
     }
-    return !type->type().is_error();
+    return !type.value().type().is_error();
 }
 
 struct IntrinsicBlueprint : IntrinsicExpression {
@@ -38,19 +38,21 @@ struct IntrinsicBlueprint : IntrinsicExpression {
 };
 
 struct ResultTypeExtractor : Blueprint::DependencyHandler {
-    std::unique_ptr<FeatureType> result_type;
+    std::optional<FeatureType> result_type;
     bool too_much;
-    ResultTypeExtractor() : result_type(), too_much(false) {}
-    const FeatureType &resolve_input(const vespalib::string &, Blueprint::AcceptInput) override {
+    bool failed;
+    ResultTypeExtractor() : result_type(), too_much(false), failed(false) {}
+    std::optional<FeatureType> resolve_input(const vespalib::string &, Blueprint::AcceptInput) override {
         too_much = true;
-        return FeatureType::number();
+        return std::nullopt;
     }
-    void define_output(const vespalib::string &, const FeatureType &type) override {
-        too_much = (too_much || bool(result_type));
-        result_type = std::make_unique<FeatureType>(type);
+    void define_output(const vespalib::string &, FeatureType type) override {
+        too_much = (too_much || result_type.has_value());
+        result_type.emplace(std::move(type));
     }
-    bool valid() const { return (is_valid(result_type.get()) && !too_much); }
-    const FeatureType &get() const { return *result_type; }
+    void fail(const vespalib::string &) override { failed = true; }
+    bool valid() const { return (is_valid(result_type) && !too_much && !failed); }
+    const FeatureType &get() const { return result_type.value(); }
 };
 
 } // namespace search::features::rankingexpression::<unnamed>
diff --git a/searchlib/src/vespa/searchlib/features/rankingexpressionfeature.cpp b/searchlib/src/vespa/searchlib/features/rankingexpressionfeature.cpp
index 62860e04c73..bf0a731eaef 100644
--- a/searchlib/src/vespa/searchlib/features/rankingexpressionfeature.cpp
+++ b/searchlib/src/vespa/searchlib/features/rankingexpressionfeature.cpp
@@ -253,13 +253,11 @@ RankingExpressionBlueprint::setup(const fef::IIndexEnvironment &env,
         script = params[0].getValue();
         //LOG(debug, "Script from param: '%s'\n", script.c_str());
     } else {
-        LOG(error, "No expression given.");
-        return false;
+        return fail("No expression given.");
     }
     auto rank_function = Function::parse(script, rankingexpression::FeatureNameExtractor());
     if (rank_function->has_error()) {
-        LOG(error, "Failed to parse expression '%s': %s", script.c_str(), rank_function->get_error().c_str());
-        return false;
+        return fail("Failed to parse expression '%s': %s", script.c_str(), rank_function->get_error().c_str());
     }
     _intrinsic_expression = _expression_replacer->maybe_replace(*rank_function, env);
     if (_intrinsic_expression) {
@@ -268,25 +266,36 @@ RankingExpressionBlueprint::setup(const fef::IIndexEnvironment &env,
         return true;
     }
     bool do_compile = true;
+    bool dependency_error = false;
     std::vector<ValueType> input_types;
     for (size_t i = 0; i < rank_function->num_params(); ++i) {
-        const FeatureType &input = defineInput(rank_function->param_name(i), AcceptInput::ANY);
-        _input_is_object.push_back(char(input.is_object()));
-        if (input.is_object()) {
-            do_compile = false;
-            input_types.push_back(input.type());
+        if (auto maybe_input = defineInput(rank_function->param_name(i), AcceptInput::ANY)) {
+            const FeatureType &input = maybe_input.value();
+            _input_is_object.push_back(char(input.is_object()));
+            if (input.is_object()) {
+                do_compile = false;
+                input_types.push_back(input.type());
+            } else {
+                input_types.push_back(ValueType::double_type());
+            }
         } else {
-            input_types.push_back(ValueType::double_type());
+            dependency_error = true;
+            input_types.push_back(ValueType::error_type());
         }
     }
+    if (dependency_error) {
+        return false;
+    }
     NodeTypes node_types(*rank_function, input_types);
     if (!node_types.all_types_are_double()) {
         do_compile = false;
     }
     ValueType root_type = node_types.get_type(rank_function->root());
     if (root_type.is_error()) {
-        LOG(error, "rank expression contains type errors: %s\n", script.c_str());
-        return false;
+        for (const auto &type_error: node_types.errors()) {
+            LOG(warning, "type error: %s", type_error.c_str());
+        }
+        return fail("rank expression contains type errors: %s", script.c_str());
     }
     auto compile_issues = CompiledFunction::detect_issues(*rank_function);
     auto interpret_issues = InterpretedFunction::detect_issues(*rank_function);
@@ -297,9 +306,8 @@ RankingExpressionBlueprint::setup(const fef::IIndexEnvironment &env,
     }
     const auto &issues = do_compile ? compile_issues : interpret_issues;
     if (issues) {
-        LOG(error, "rank expression cannot be evaluated: %s\n%s",
-            script.c_str(), list_issues(issues.list).c_str());
-        return false;
+        return fail("rank expression cannot be evaluated: %s\n%s",
+                    script.c_str(), list_issues(issues.list).c_str());
     }
     // avoid costly compilation when only verifying setup
     if (env.getFeatureMotivation() != env.FeatureMotivation::VERIFY_SETUP) {
diff --git a/searchlib/src/vespa/searchlib/features/raw_score_feature.cpp b/searchlib/src/vespa/searchlib/features/raw_score_feature.cpp
index 61355581214..ea696c75eff 100644
--- a/searchlib/src/vespa/searchlib/features/raw_score_feature.cpp
+++ b/searchlib/src/vespa/searchlib/features/raw_score_feature.cpp
@@ -2,6 +2,7 @@
 
 #include "raw_score_feature.h"
 #include "utils.h"
+#include <vespa/vespalib/util/stash.h>
 
 using namespace search::fef;
 
diff --git a/searchlib/src/vespa/searchlib/features/raw_score_feature.h b/searchlib/src/vespa/searchlib/features/raw_score_feature.h
index 2a4eb946a68..42813cf9d22 100644
--- a/searchlib/src/vespa/searchlib/features/raw_score_feature.h
+++ b/searchlib/src/vespa/searchlib/features/raw_score_feature.h
@@ -10,7 +10,7 @@ class RawScoreExecutor : public fef::FeatureExecutor
 {
 private:
     std::vector<fef::TermFieldHandle> _handles;
-    const fef::MatchData                     *_md;
+    const fef::MatchData             *_md;
 
     void handle_bind_match_data(const fef::MatchData &md) override;
 public:
diff --git a/searchlib/src/vespa/searchlib/features/reverseproximityfeature.cpp b/searchlib/src/vespa/searchlib/features/reverseproximityfeature.cpp
index c27936332d2..436e9bc0a0c 100644
--- a/searchlib/src/vespa/searchlib/features/reverseproximityfeature.cpp
+++ b/searchlib/src/vespa/searchlib/features/reverseproximityfeature.cpp
@@ -2,14 +2,11 @@
 
 #include "reverseproximityfeature.h"
 #include "utils.h"
-#include <vespa/searchlib/fef/featurenamebuilder.h>
 #include <vespa/searchlib/fef/fieldinfo.h>
-#include <vespa/searchlib/fef/fieldtype.h>
 #include <vespa/searchlib/fef/itermdata.h>
-#include <vespa/vespalib/util/stringfmt.h>
+#include <vespa/vespalib/util/stash.h>
 
-namespace search {
-namespace features {
+namespace search::features {
 
 ReverseProximityConfig::ReverseProximityConfig() :
     fieldId(search::fef::IllegalHandle),
@@ -126,7 +123,7 @@ ReverseProximityBlueprint::setup(const search::fef::IIndexEnvironment &env,
 search::fef::Blueprint::UP
 ReverseProximityBlueprint::createInstance() const
 {
-    return search::fef::Blueprint::UP(new ReverseProximityBlueprint());
+    return std::make_unique<ReverseProximityBlueprint>();
 }
 
 search::fef::FeatureExecutor &
@@ -135,4 +132,4 @@ ReverseProximityBlueprint::createExecutor(const search::fef::IQueryEnvironment &
     return stash.create<ReverseProximityExecutor>(env, _config);
 }
 
-}}
+}
diff --git a/searchlib/src/vespa/searchlib/features/setup.cpp b/searchlib/src/vespa/searchlib/features/setup.cpp
index e1e351d0726..bf8b59f100c 100644
--- a/searchlib/src/vespa/searchlib/features/setup.cpp
+++ b/searchlib/src/vespa/searchlib/features/setup.cpp
@@ -67,59 +67,59 @@ namespace search::features {
 void setup_search_features(fef::IBlueprintRegistry & registry)
 {
     // Prod features.
-    registry.addPrototype(Blueprint::SP(new AgeBlueprint()));
-    registry.addPrototype(Blueprint::SP(new AttributeBlueprint()));
-    registry.addPrototype(Blueprint::SP(new AttributeMatchBlueprint()));
-    registry.addPrototype(Blueprint::SP(new Bm25Blueprint()));
-    registry.addPrototype(Blueprint::SP(new ClosenessBlueprint()));
-    registry.addPrototype(Blueprint::SP(new DebugAttributeWaitBlueprint()));
-    registry.addPrototype(Blueprint::SP(new DebugWaitBlueprint()));
-    registry.addPrototype(Blueprint::SP(new DistanceBlueprint()));
-    registry.addPrototype(Blueprint::SP(new DistanceToPathBlueprint()));
-    registry.addPrototype(Blueprint::SP(new DotProductBlueprint()));
-    registry.addPrototype(Blueprint::SP(new ElementCompletenessBlueprint()));
-    registry.addPrototype(Blueprint::SP(new ElementSimilarityBlueprint()));
-    registry.addPrototype(Blueprint::SP(new EuclideanDistanceBlueprint()));
-    registry.addPrototype(Blueprint::SP(new FieldInfoBlueprint()));
-    registry.addPrototype(Blueprint::SP(new FieldLengthBlueprint()));
-    registry.addPrototype(Blueprint::SP(new FieldMatchBlueprint()));
-    registry.addPrototype(Blueprint::SP(new FieldTermMatchBlueprint()));
-    registry.addPrototype(Blueprint::SP(new FirstPhaseBlueprint()));
-    registry.addPrototype(Blueprint::SP(new FlowCompletenessBlueprint()));
-    registry.addPrototype(Blueprint::SP(new ForeachBlueprint()));
-    registry.addPrototype(Blueprint::SP(new FreshnessBlueprint()));
-    registry.addPrototype(Blueprint::SP(new ItemRawScoreBlueprint()));
-    registry.addPrototype(Blueprint::SP(new MatchBlueprint()));
-    registry.addPrototype(Blueprint::SP(new MatchCountBlueprint()));
-    registry.addPrototype(Blueprint::SP(new MatchesBlueprint()));
-    registry.addPrototype(Blueprint::SP(new NativeAttributeMatchBlueprint()));
-    registry.addPrototype(Blueprint::SP(new NativeDotProductBlueprint()));
-    registry.addPrototype(Blueprint::SP(new NativeFieldMatchBlueprint()));
-    registry.addPrototype(Blueprint::SP(new NativeProximityBlueprint()));
-    registry.addPrototype(Blueprint::SP(new NativeRankBlueprint()));
-    registry.addPrototype(Blueprint::SP(new NowBlueprint()));
-    registry.addPrototype(Blueprint::SP(new QueryBlueprint()));
-    registry.addPrototype(Blueprint::SP(new QueryTermCountBlueprint()));
-    registry.addPrototype(Blueprint::SP(new RandomBlueprint()));
-    registry.addPrototype(Blueprint::SP(new RandomNormalBlueprint()));
-    registry.addPrototype(Blueprint::SP(new RandomNormalStableBlueprint()));
-    registry.addPrototype(Blueprint::SP(new RawScoreBlueprint()));
-    registry.addPrototype(Blueprint::SP(new SubqueriesBlueprint));
-    registry.addPrototype(Blueprint::SP(new TensorFromLabelsBlueprint()));
-    registry.addPrototype(Blueprint::SP(new TensorFromWeightedSetBlueprint()));
-    registry.addPrototype(Blueprint::SP(new TermBlueprint()));
-    registry.addPrototype(Blueprint::SP(new TermDistanceBlueprint()));
-    registry.addPrototype(Blueprint::SP(new TermInfoBlueprint()));
-    registry.addPrototype(Blueprint::SP(new TextSimilarityBlueprint()));
-    registry.addPrototype(Blueprint::SP(new ValueBlueprint()));
+    registry.addPrototype(std::make_shared<AgeBlueprint>());
+    registry.addPrototype(std::make_shared<AttributeBlueprint>());
+    registry.addPrototype(std::make_shared<AttributeMatchBlueprint>());
+    registry.addPrototype(std::make_shared<Bm25Blueprint>());
+    registry.addPrototype(std::make_shared<ClosenessBlueprint>());
+    registry.addPrototype(std::make_shared<DebugAttributeWaitBlueprint>());
+    registry.addPrototype(std::make_shared<DebugWaitBlueprint>());
+    registry.addPrototype(std::make_shared<DistanceBlueprint>());
+    registry.addPrototype(std::make_shared<DistanceToPathBlueprint>());
+    registry.addPrototype(std::make_shared<DotProductBlueprint>());
+    registry.addPrototype(std::make_shared<ElementCompletenessBlueprint>());
+    registry.addPrototype(std::make_shared<ElementSimilarityBlueprint>());
+    registry.addPrototype(std::make_shared<EuclideanDistanceBlueprint>());
+    registry.addPrototype(std::make_shared<FieldInfoBlueprint>());
+    registry.addPrototype(std::make_shared<FieldLengthBlueprint>());
+    registry.addPrototype(std::make_shared<FieldMatchBlueprint>());
+    registry.addPrototype(std::make_shared<FieldTermMatchBlueprint>());
+    registry.addPrototype(std::make_shared<FirstPhaseBlueprint>());
+    registry.addPrototype(std::make_shared<FlowCompletenessBlueprint>());
+    registry.addPrototype(std::make_shared<ForeachBlueprint>());
+    registry.addPrototype(std::make_shared<FreshnessBlueprint>());
+    registry.addPrototype(std::make_shared<ItemRawScoreBlueprint>());
+    registry.addPrototype(std::make_shared<MatchBlueprint>());
+    registry.addPrototype(std::make_shared<MatchCountBlueprint>());
+    registry.addPrototype(std::make_shared<MatchesBlueprint>());
+    registry.addPrototype(std::make_shared<NativeAttributeMatchBlueprint>());
+    registry.addPrototype(std::make_shared<NativeDotProductBlueprint>());
+    registry.addPrototype(std::make_shared<NativeFieldMatchBlueprint>());
+    registry.addPrototype(std::make_shared<NativeProximityBlueprint>());
+    registry.addPrototype(std::make_shared<NativeRankBlueprint>());
+    registry.addPrototype(std::make_shared<NowBlueprint>());
+    registry.addPrototype(std::make_shared<QueryBlueprint>());
+    registry.addPrototype(std::make_shared<QueryTermCountBlueprint>());
+    registry.addPrototype(std::make_shared<RandomBlueprint>());
+    registry.addPrototype(std::make_shared<RandomNormalBlueprint>());
+    registry.addPrototype(std::make_shared<RandomNormalStableBlueprint>());
+    registry.addPrototype(std::make_shared<RawScoreBlueprint>());
+    registry.addPrototype(std::make_shared<SubqueriesBlueprint>());
+    registry.addPrototype(std::make_shared<TensorFromLabelsBlueprint>());
+    registry.addPrototype(std::make_shared<TensorFromWeightedSetBlueprint>());
+    registry.addPrototype(std::make_shared<TermBlueprint>());
+    registry.addPrototype(std::make_shared<TermDistanceBlueprint>());
+    registry.addPrototype(std::make_shared<TermInfoBlueprint>());
+    registry.addPrototype(std::make_shared<TextSimilarityBlueprint>());
+    registry.addPrototype(std::make_shared<ValueBlueprint>());
 
     // Beta features.
-    registry.addPrototype(Blueprint::SP(new JaroWinklerDistanceBlueprint()));
-    registry.addPrototype(Blueprint::SP(new ProximityBlueprint()));
-    registry.addPrototype(Blueprint::SP(new QueryCompletenessBlueprint()));
-    registry.addPrototype(Blueprint::SP(new ReverseProximityBlueprint()));
-    registry.addPrototype(Blueprint::SP(new TermEditDistanceBlueprint()));
-    registry.addPrototype(Blueprint::SP(new TermFieldMdBlueprint()));
+    registry.addPrototype(std::make_shared<JaroWinklerDistanceBlueprint>());
+    registry.addPrototype(std::make_shared<ProximityBlueprint>());
+    registry.addPrototype(std::make_shared<QueryCompletenessBlueprint>());
+    registry.addPrototype(std::make_shared<ReverseProximityBlueprint>());
+    registry.addPrototype(std::make_shared<TermEditDistanceBlueprint>());
+    registry.addPrototype(std::make_shared<TermFieldMdBlueprint>());
     registry.addPrototype(std::make_shared<ConstantBlueprint>());
 
     // Ranking Expression
diff --git a/searchlib/src/vespa/searchlib/features/subqueries_feature.cpp b/searchlib/src/vespa/searchlib/features/subqueries_feature.cpp
index 6c52b6edb76..aaff46af93e 100644
--- a/searchlib/src/vespa/searchlib/features/subqueries_feature.cpp
+++ b/searchlib/src/vespa/searchlib/features/subqueries_feature.cpp
@@ -2,11 +2,11 @@
 
 #include "subqueries_feature.h"
 #include "utils.h"
+#include <vespa/vespalib/util/stash.h>
 
 using namespace search::fef;
 
-namespace search {
-namespace features {
+namespace search::features {
 
 SubqueriesExecutor::SubqueriesExecutor(const IQueryEnvironment &env,
                                        uint32_t fieldId)
@@ -59,5 +59,4 @@ SubqueriesBlueprint::createExecutor(const IQueryEnvironment &queryEnv, vespalib:
     return stash.create<SubqueriesExecutor>(queryEnv, _field->id());
 }
 
-} // namespace features
-} // namespace search
+}
diff --git a/searchlib/src/vespa/searchlib/features/term_field_md_feature.cpp b/searchlib/src/vespa/searchlib/features/term_field_md_feature.cpp
index a4ca8524140..7fd487d8bdd 100644
--- a/searchlib/src/vespa/searchlib/features/term_field_md_feature.cpp
+++ b/searchlib/src/vespa/searchlib/features/term_field_md_feature.cpp
@@ -2,10 +2,10 @@
 
 #include "term_field_md_feature.h"
 #include "utils.h"
-#include <vespa/searchlib/fef/fieldinfo.h>
 #include <vespa/searchlib/fef/indexproperties.h>
 #include <vespa/searchlib/fef/itablemanager.h>
 #include <vespa/searchlib/fef/properties.h>
+#include <vespa/vespalib/util/stash.h>
 #include <cassert>
 
 using namespace search::fef;
@@ -83,7 +83,7 @@ TermFieldMdBlueprint::visitDumpFeatures(const IIndexEnvironment &,
 Blueprint::UP
 TermFieldMdBlueprint::createInstance() const
 {
-    return Blueprint::UP(new TermFieldMdBlueprint());
+    return std::make_unique<TermFieldMdBlueprint>();
 }
 
 bool
diff --git a/searchlib/src/vespa/searchlib/features/termdistancefeature.cpp b/searchlib/src/vespa/searchlib/features/termdistancefeature.cpp
index fb38a49d6eb..e9f48421fcf 100644
--- a/searchlib/src/vespa/searchlib/features/termdistancefeature.cpp
+++ b/searchlib/src/vespa/searchlib/features/termdistancefeature.cpp
@@ -5,11 +5,11 @@
 #include "utils.h"
 #include <vespa/searchlib/fef/fieldinfo.h>
 #include <vespa/searchlib/fef/properties.h>
+#include <vespa/vespalib/util/stash.h>
 
 using namespace search::fef;
 
-namespace search {
-namespace features {
+namespace search::features {
 
 
 TermDistanceExecutor::TermDistanceExecutor(const IQueryEnvironment & env,
@@ -61,7 +61,7 @@ TermDistanceBlueprint::visitDumpFeatures(const IIndexEnvironment &,
 Blueprint::UP
 TermDistanceBlueprint::createInstance() const
 {
-    return Blueprint::UP(new TermDistanceBlueprint());
+    return std::make_unique<TermDistanceBlueprint>();
 }
 
 bool
@@ -97,6 +97,4 @@ TermDistanceBlueprint::createExecutor(const IQueryEnvironment &env, vespalib::St
     }
 }
 
-
-} // namespace features
-} // namespace search
+}
diff --git a/searchlib/src/vespa/searchlib/features/termeditdistancefeature.cpp b/searchlib/src/vespa/searchlib/features/termeditdistancefeature.cpp
index 5990d62cb25..433bf6134b8 100644
--- a/searchlib/src/vespa/searchlib/features/termeditdistancefeature.cpp
+++ b/searchlib/src/vespa/searchlib/features/termeditdistancefeature.cpp
@@ -6,6 +6,8 @@
 #include <vespa/searchlib/fef/properties.h>
 #include <vespa/vespalib/util/stringfmt.h>
 #include <vespa/vespalib/locale/c.h>
+#include <vespa/vespalib/util/stash.h>
+
 
 #include <vespa/log/log.h>
 LOG_SETUP(".features.termeditdistance");
@@ -219,7 +221,7 @@ TermEditDistanceBlueprint::setup(const search::fef::IIndexEnvironment &env,
 search::fef::Blueprint::UP
 TermEditDistanceBlueprint::createInstance() const
 {
-    return search::fef::Blueprint::UP(new TermEditDistanceBlueprint());
+    return std::make_unique<TermEditDistanceBlueprint>();
 }
 
 search::fef::FeatureExecutor &
diff --git a/searchlib/src/vespa/searchlib/features/termfeature.cpp b/searchlib/src/vespa/searchlib/features/termfeature.cpp
index d6df25cc2b9..90540726227 100644
--- a/searchlib/src/vespa/searchlib/features/termfeature.cpp
+++ b/searchlib/src/vespa/searchlib/features/termfeature.cpp
@@ -4,15 +4,14 @@
 #include "utils.h"
 #include <vespa/searchlib/fef/featurenamebuilder.h>
 #include <vespa/searchlib/fef/fieldinfo.h>
-#include <vespa/searchlib/fef/fieldtype.h>
 #include <vespa/searchlib/fef/properties.h>
 #include <vespa/searchlib/fef/itermdata.h>
 #include <vespa/vespalib/util/stringfmt.h>
+#include <vespa/vespalib/util/stash.h>
 
 using namespace search::fef;
 
-namespace search {
-namespace features {
+namespace search::features {
 
 TermExecutor::TermExecutor(const search::fef::IQueryEnvironment &env,
                            uint32_t termId) :
@@ -21,7 +20,7 @@ TermExecutor::TermExecutor(const search::fef::IQueryEnvironment &env,
     _connectedness(util::lookupConnectedness(env, termId)),
     _significance(0)
 {
-    if (_termData != NULL) {
+    if (_termData != nullptr) {
         feature_t fallback = util::getSignificance(*_termData);
         _significance = util::lookupSignificance(env, termId, fallback);
     }
@@ -30,7 +29,7 @@ TermExecutor::TermExecutor(const search::fef::IQueryEnvironment &env,
 void
 TermExecutor::execute(uint32_t)
 {
-    if (_termData == NULL) { // this query term is not present in the query
+    if (_termData == nullptr) { // this query term is not present in the query
         outputs().set_number(0, 0.0f); // connectedness
         outputs().set_number(1, 0.0f); // significance (1 - frequency)
         outputs().set_number(2, 0.0f); // weight
@@ -76,7 +75,7 @@ TermBlueprint::setup(const search::fef::IIndexEnvironment &,
 search::fef::Blueprint::UP
 TermBlueprint::createInstance() const
 {
-    return search::fef::Blueprint::UP(new TermBlueprint());
+    return std::make_unique<TermBlueprint>();
 }
 
 search::fef::FeatureExecutor &
@@ -85,4 +84,4 @@ TermBlueprint::createExecutor(const search::fef::IQueryEnvironment &env, vespali
     return stash.create<TermExecutor>(env, _termId);
 }
 
-}}
+}
diff --git a/searchlib/src/vespa/searchlib/features/terminfofeature.cpp b/searchlib/src/vespa/searchlib/features/terminfofeature.cpp
index 4f32cda8c86..b61021c4aa2 100644
--- a/searchlib/src/vespa/searchlib/features/terminfofeature.cpp
+++ b/searchlib/src/vespa/searchlib/features/terminfofeature.cpp
@@ -3,30 +3,28 @@
 #include "terminfofeature.h"
 #include "valuefeature.h"
 #include <vespa/searchlib/fef/properties.h>
-#include <vespa/searchlib/fef/fieldinfo.h>
-#include <vespa/searchlib/fef/fieldtype.h>
 #include <vespa/searchlib/fef/featurenamebuilder.h>
 #include <vespa/searchlib/fef/itermdata.h>
-#include <vespa/searchlib/fef/handle.h>
+#include <vespa/vespalib/util/stash.h>
 
-namespace search {
-namespace features {
+using namespace search::fef;
+namespace search::features {
 
 TermInfoBlueprint::TermInfoBlueprint()
-    : search::fef::Blueprint("termInfo"),
+    : Blueprint("termInfo"),
       _termIdx(0)
 {
 }
 
 void
-TermInfoBlueprint::visitDumpFeatures(const search::fef::IIndexEnvironment &,
-                                     search::fef::IDumpFeatureVisitor &) const
+TermInfoBlueprint::visitDumpFeatures(const IIndexEnvironment &,
+                                     IDumpFeatureVisitor &) const
 {
 }
 
 bool
-TermInfoBlueprint::setup(const search::fef::IIndexEnvironment &,
-                         const search::fef::ParameterList & params)
+TermInfoBlueprint::setup(const IIndexEnvironment &,
+                         const ParameterList & params)
 {
     _termIdx = params[0].asInteger();
     describeOutput("queryidx", "The index of the first term with the given "
@@ -34,17 +32,14 @@ TermInfoBlueprint::setup(const search::fef::IIndexEnvironment &,
     return true;
 }
 
-search::fef::FeatureExecutor &
-TermInfoBlueprint::createExecutor(const search::fef::IQueryEnvironment &queryEnv, vespalib::Stash &stash) const
+FeatureExecutor &
+TermInfoBlueprint::createExecutor(const IQueryEnvironment &queryEnv, vespalib::Stash &stash) const
 {
     feature_t queryIdx = -1.0;
     if (queryEnv.getNumTerms() > _termIdx) {
         queryIdx = _termIdx;
     }
-    std::vector<feature_t> values;
-    values.push_back(queryIdx);
-    return stash.create<ValueExecutor>(values);
+    return stash.create<SingleValueExecutor>(queryIdx);
 }
 
-} // namespace features
-} // namespace search
+}
diff --git a/searchlib/src/vespa/searchlib/features/text_similarity_feature.cpp b/searchlib/src/vespa/searchlib/features/text_similarity_feature.cpp
index 3cdb20a16e5..a0e0a8759a0 100644
--- a/searchlib/src/vespa/searchlib/features/text_similarity_feature.cpp
+++ b/searchlib/src/vespa/searchlib/features/text_similarity_feature.cpp
@@ -3,6 +3,7 @@
 #include "text_similarity_feature.h"
 #include <vespa/searchlib/fef/itermdata.h>
 #include <vespa/searchlib/fef/featurenamebuilder.h>
+#include <vespa/vespalib/util/stash.h>
 
 namespace search::features {
 
@@ -194,7 +195,7 @@ TextSimilarityBlueprint::visitDumpFeatures(const fef::IIndexEnvironment &env,
 fef::Blueprint::UP
 TextSimilarityBlueprint::createInstance() const
 {
-    return Blueprint::UP(new TextSimilarityBlueprint());
+    return std::make_unique<TextSimilarityBlueprint>();
 }
 
 bool
@@ -218,6 +219,4 @@ TextSimilarityBlueprint::createExecutor(const fef::IQueryEnvironment &env, vespa
     return stash.create<TextSimilarityExecutor>(env, _field_id);
 }
 
-//-----------------------------------------------------------------------------
-
 }
diff --git a/searchlib/src/vespa/searchlib/features/valuefeature.cpp b/searchlib/src/vespa/searchlib/features/valuefeature.cpp
index 339cc5431f1..64f2ba862e2 100644
--- a/searchlib/src/vespa/searchlib/features/valuefeature.cpp
+++ b/searchlib/src/vespa/searchlib/features/valuefeature.cpp
@@ -2,12 +2,13 @@
 
 #include "valuefeature.h"
 #include <vespa/vespalib/stllike/asciistream.h>
+#include <vespa/vespalib/util/stash.h>
 
-namespace search {
-namespace features {
+using namespace search::fef;
+namespace search::features {
 
 ValueExecutor::ValueExecutor(const std::vector<feature_t> & values) :
-    search::fef::FeatureExecutor(),
+    FeatureExecutor(),
     _values(values)
 {
 }
@@ -21,28 +22,32 @@ ValueExecutor::execute(uint32_t)
 }
 
 void
+SingleValueExecutor::execute(uint32_t)
+{
+    outputs().set_number(0, _value);
+}
+
+void
 SingleZeroValueExecutor::execute(uint32_t)
 {
     outputs().set_number(0, 0.0);
 }
 
 ValueBlueprint::ValueBlueprint() :
-    search::fef::Blueprint("value"),
+    Blueprint("value"),
     _values()
 {
 }
 
-ValueBlueprint::~ValueBlueprint() {}
+ValueBlueprint::~ValueBlueprint() = default;
 
 void
-ValueBlueprint::visitDumpFeatures(const search::fef::IIndexEnvironment &,
-                                  search::fef::IDumpFeatureVisitor &) const
+ValueBlueprint::visitDumpFeatures(const IIndexEnvironment &, IDumpFeatureVisitor &) const
 {
 }
 
 bool
-ValueBlueprint::setup(const search::fef::IIndexEnvironment &,
-                      const search::fef::ParameterList & params)
+ValueBlueprint::setup(const IIndexEnvironment &, const ParameterList & params)
 {
     for (uint32_t i = 0; i < params.size(); ++i) {
         _values.push_back(params[i].asDouble());
@@ -56,13 +61,15 @@ ValueBlueprint::setup(const search::fef::IIndexEnvironment &,
     return true;
 }
 
-search::fef::FeatureExecutor &
-ValueBlueprint::createExecutor(const search::fef::IQueryEnvironment &queryEnv, vespalib::Stash &stash) const
+FeatureExecutor &
+ValueBlueprint::createExecutor(const IQueryEnvironment &, vespalib::Stash &stash) const
 {
-    (void) queryEnv;
-    return stash.create<ValueExecutor>(_values);
+    if (_values.size() == 1) {
+        return stash.create<SingleValueExecutor>(_values[0]);
+    } else {
+        return stash.create<ValueExecutor>(_values);
+    }
 }
 
 
-} // namespace features
-} // namespace search
+}
diff --git a/searchlib/src/vespa/searchlib/features/valuefeature.h b/searchlib/src/vespa/searchlib/features/valuefeature.h
index 1cc6bd55bc1..d8e2ab70d0b 100644
--- a/searchlib/src/vespa/searchlib/features/valuefeature.h
+++ b/searchlib/src/vespa/searchlib/features/valuefeature.h
@@ -7,7 +7,7 @@
 
 namespace search::features {
 
-class ValueExecutor : public fef::FeatureExecutor
+class ValueExecutor final : public fef::FeatureExecutor
 {
 private:
     std::vector<feature_t> _values;
@@ -19,7 +19,18 @@ public:
     const std::vector<feature_t> & getValues() const { return _values; }
 };
 
-class SingleZeroValueExecutor : public fef::FeatureExecutor
+class SingleValueExecutor final : public fef::FeatureExecutor
+{
+private:
+    feature_t _value;
+
+public:
+    SingleValueExecutor(feature_t value) : _value(value) { }
+    bool isPure() override { return true; }
+    void execute(uint32_t docId) override;
+};
+
+class SingleZeroValueExecutor final : public fef::FeatureExecutor
 {
 public:
     SingleZeroValueExecutor() : FeatureExecutor() {}
diff --git a/searchlib/src/vespa/searchlib/fef/blueprint.cpp b/searchlib/src/vespa/searchlib/fef/blueprint.cpp
index 7073d0c0ccd..fb758058fe0 100644
--- a/searchlib/src/vespa/searchlib/fef/blueprint.cpp
+++ b/searchlib/src/vespa/searchlib/fef/blueprint.cpp
@@ -3,13 +3,14 @@
 #include "blueprint.h"
 #include "parametervalidator.h"
 #include <cassert>
+#include <vespa/vespalib/util/stringfmt.h>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".fef.blueprint");
 
 namespace search::fef {
 
-const FeatureType &
+std::optional<FeatureType>
 Blueprint::defineInput(vespalib::stringref inName, AcceptInput accept)
 {
     assert(_dependency_handler != nullptr);
@@ -19,11 +20,23 @@ Blueprint::defineInput(vespalib::stringref inName, AcceptInput accept)
 void
 Blueprint::describeOutput(vespalib::stringref outName,
                           vespalib::stringref desc,
-                          const FeatureType &type)
+                          FeatureType type)
 {
     (void) desc;
     assert(_dependency_handler != nullptr);
-    _dependency_handler->define_output(outName, type);
+    _dependency_handler->define_output(outName, std::move(type));
+}
+
+bool
+Blueprint::fail(const char *format, ...)
+{
+    va_list ap;
+    va_start(ap, format);
+    vespalib::string msg = vespalib::make_string_va(format, ap);
+    va_end(ap);
+    assert(_dependency_handler != nullptr);
+    _dependency_handler->fail(msg);
+    return false;
 }
 
 Blueprint::Blueprint(vespalib::stringref baseName)
@@ -52,9 +65,8 @@ Blueprint::setup(const IIndexEnvironment &indexEnv,
     if (result.valid()) {
         return setup(indexEnv, result.getParameters());
     } else {
-        LOG(error, "The parameter list used for setting up rank feature %s is not valid: %s",
-            getBaseName().c_str(), result.getError().c_str());
-        return false;
+        return fail("The parameter list used for setting up rank feature %s is not valid: %s",
+                    getBaseName().c_str(), result.getError().c_str());
     }
 }
 
@@ -62,9 +74,8 @@ bool
 Blueprint::setup(const IIndexEnvironment &indexEnv, const ParameterList &params)
 {
     (void) indexEnv; (void) params;
-    LOG(error, "The setup function using a typed parameter list does not have a default implementation. "
-        "Make sure the setup function is implemented in the rank feature %s.", getBaseName().c_str());
-    return false;
+    return fail("The setup function using a typed parameter list does not have a default implementation. "
+                "Make sure the setup function is implemented in the rank feature %s.", getBaseName().c_str());
 }
 
 void
diff --git a/searchlib/src/vespa/searchlib/fef/blueprint.h b/searchlib/src/vespa/searchlib/fef/blueprint.h
index 5d7eb6eb2c0..81f37f7224d 100644
--- a/searchlib/src/vespa/searchlib/fef/blueprint.h
+++ b/searchlib/src/vespa/searchlib/fef/blueprint.h
@@ -9,6 +9,7 @@
 #include "parameter.h"
 #include "parameterdescriptions.h"
 #include "feature_type.h"
+#include <optional>
 
 namespace vespalib { class Stash; }
 
@@ -43,8 +44,9 @@ public:
      * executor setup.
      **/
     struct DependencyHandler {
-        virtual const FeatureType &resolve_input(const vespalib::string &feature_name, AcceptInput accept_type) = 0;
-        virtual void define_output(const vespalib::string &output_name, const FeatureType &type) = 0;
+        virtual std::optional<FeatureType> resolve_input(const vespalib::string &feature_name, AcceptInput accept_type) = 0;
+        virtual void define_output(const vespalib::string &output_name, FeatureType type) = 0;
+        virtual void fail(const vespalib::string &msg) = 0;
         virtual ~DependencyHandler() = default;
     };
 
@@ -87,8 +89,8 @@ protected:
      * @param inName feature name of input
      * @param type accepted input type
      **/
-    const FeatureType &defineInput(vespalib::stringref inName,
-                                   AcceptInput accept = AcceptInput::NUMBER);
+    std::optional<FeatureType> defineInput(vespalib::stringref inName,
+                                           AcceptInput accept = AcceptInput::NUMBER);
 
     /**
      * Describe an output for this blueprint. This method should be
@@ -104,7 +106,22 @@ protected:
      * @param desc output description
      **/
     void describeOutput(vespalib::stringref outName, vespalib::stringref desc,
-                        const FeatureType &type = FeatureType::number());
+                        FeatureType type = FeatureType::number());
+
+    /**
+     * Fail the setup of this blueprint with the given message. This
+     * function should be called by the @ref setup function when it
+     * fails. The failure is handled by the dependency handler to make
+     * sure we only report the first error for each feature.
+     *
+     * @return false
+     * @param format printf-style format string
+     **/
+    bool fail(const char *format, ...)
+#ifdef __GNUC__
+        __attribute__ ((format (printf,2,3)))
+#endif
+        ;
 
     /**
      * Used to store a reference to the attribute during prepareSharedState
diff --git a/searchlib/src/vespa/searchlib/fef/blueprintfactory.cpp b/searchlib/src/vespa/searchlib/fef/blueprintfactory.cpp
index 95813be1360..357aec3fc71 100644
--- a/searchlib/src/vespa/searchlib/fef/blueprintfactory.cpp
+++ b/searchlib/src/vespa/searchlib/fef/blueprintfactory.cpp
@@ -5,8 +5,7 @@
 #include <vespa/log/log.h>
 LOG_SETUP(".fef.blueprintfactory");
 
-namespace search {
-namespace fef {
+namespace search::fef {
 
 BlueprintFactory::BlueprintFactory()
     : _blueprintMap()
@@ -20,7 +19,7 @@ BlueprintFactory::addPrototype(Blueprint::SP proto)
     if (_blueprintMap.find(name) != _blueprintMap.end()) {
         LOG(warning, "Blueprint prototype overwritten: %s", name.c_str());
     }
-    _blueprintMap[name] = proto;
+    _blueprintMap[name] = std::move(proto);
 }
 
 void
@@ -41,9 +40,7 @@ BlueprintFactory::createBlueprint(const vespalib::string &name) const
     if (itr == _blueprintMap.end()) {
         return Blueprint::SP();
     }
-    Blueprint::UP bp = itr->second->createInstance();
-    return Blueprint::SP(bp.release());
+    return itr->second->createInstance();
 }
 
-} // namespace fef
-} // namespace search
+}
diff --git a/searchlib/src/vespa/searchlib/fef/blueprintfactory.h b/searchlib/src/vespa/searchlib/fef/blueprintfactory.h
index 221cca38f5f..346294ba5f5 100644
--- a/searchlib/src/vespa/searchlib/fef/blueprintfactory.h
+++ b/searchlib/src/vespa/searchlib/fef/blueprintfactory.h
@@ -6,8 +6,7 @@
 #include "iblueprintregistry.h"
 #include <map>
 
-namespace search {
-namespace fef {
+namespace search::fef {
 
 /**
  * This class implements the blueprint repository interface and acts
@@ -54,6 +53,4 @@ public:
     Blueprint::SP createBlueprint(const vespalib::string &name) const;
 };
 
-} // namespace fef
-} // namespace search
-
+}
diff --git a/searchlib/src/vespa/searchlib/fef/blueprintresolver.cpp b/searchlib/src/vespa/searchlib/fef/blueprintresolver.cpp
index a9e500fc802..d229a735f11 100644
--- a/searchlib/src/vespa/searchlib/fef/blueprintresolver.cpp
+++ b/searchlib/src/vespa/searchlib/fef/blueprintresolver.cpp
@@ -6,14 +6,21 @@
 #include <vespa/vespalib/util/stringfmt.h>
 #include <stack>
 #include <cassert>
+#include <set>
+#include <thread>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".fef.blueprintresolver");
 
+using vespalib::make_string_short::fmt;
+
 namespace search::fef {
 
 namespace {
 
+constexpr int MAX_TRACE_SIZE = BlueprintResolver::MAX_TRACE_SIZE;
+constexpr int TRACE_SKIP_POS = 10;
+
 using Accept = Blueprint::AcceptInput;
 
 bool is_compatible(bool is_object, Accept accept_type) {
@@ -44,22 +51,28 @@ struct Compiler : public Blueprint::DependencyHandler {
         ExecutorSpec spec;
         const FeatureNameParser &parser;
         Frame(Blueprint::SP blueprint, const FeatureNameParser &parser_in)
-            : spec(blueprint), parser(parser_in) {}
+            : spec(std::move(blueprint)), parser(parser_in) {}
     };
     using Stack = std::vector<Frame>;
 
     struct FrameGuard {
         Stack &stack;
-        FrameGuard(Stack &stack_in) : stack(stack_in) {}
-        ~FrameGuard() { stack.pop_back(); }
+        explicit FrameGuard(Stack &stack_in) : stack(stack_in) {}
+        ~FrameGuard() {
+            stack.back().spec.blueprint->detach_dependency_handler();
+            stack.pop_back();
+        }
     };
 
     const BlueprintFactory  &factory;
     const IIndexEnvironment &index_env;
-    bool                     compile_error;
     Stack                    resolve_stack;
     ExecutorSpecList        &spec_list;
     FeatureMap              &feature_map;
+    std::set<vespalib::string> setup_set;
+    std::set<vespalib::string> failed_set;
+    const char *min_stack;
+    const char *max_stack;
 
     Compiler(const BlueprintFactory &factory_in,
              const IIndexEnvironment &index_env_in,
@@ -67,94 +80,143 @@ struct Compiler : public Blueprint::DependencyHandler {
              FeatureMap &feature_map_out)
         : factory(factory_in),
           index_env(index_env_in),
-          compile_error(false),
           resolve_stack(),
           spec_list(spec_list_out),
-          feature_map(feature_map_out) {}
+          feature_map(feature_map_out),
+          setup_set(),
+          failed_set(),
+          min_stack(nullptr),
+          max_stack(nullptr) {}
+    ~Compiler();
+
+    void probe_stack() {
+        const char c = 'X';
+        min_stack = (min_stack == nullptr) ? &c : std::min(min_stack, &c);
+        max_stack = (max_stack == nullptr) ? &c : std::max(max_stack, &c);
+    }
+
+    int stack_usage() const {
+        return (max_stack - min_stack);
+    }
 
     Frame &self() { return resolve_stack.back(); }
+    bool failed() const { return !failed_set.empty(); }
 
-    FeatureRef failed(const vespalib::string &feature_name, const vespalib::string &reason) {
-        if (!compile_error) {
-            LOG(warning, "invalid rank feature: '%s' (%s)", feature_name.c_str(), reason.c_str());
-            for (size_t i = resolve_stack.size(); i > 0; --i) {
-                const auto &frame = resolve_stack[i - 1];
-                if (&frame != &self()) {
-                    LOG(warning, "  ... needed by rank feature '%s'", frame.parser.featureName().c_str());
-                }
+    vespalib::string make_trace(bool skip_self) {
+        vespalib::string trace;
+        auto pos = resolve_stack.rbegin();
+        auto end = resolve_stack.rend();
+        if ((pos != end) && skip_self) {
+            ++pos;
+        }
+        size_t i = 0;
+        size_t n = (end - pos);
+        for (; (pos != end); ++pos, ++i) {
+            failed_set.insert(pos->parser.featureName());
+            bool should_trace = (n <= MAX_TRACE_SIZE);
+            should_trace |= (i < TRACE_SKIP_POS);
+            should_trace |= ((end - pos) < (MAX_TRACE_SIZE - TRACE_SKIP_POS));
+            if (should_trace) {
+                trace += fmt("  ... needed by rank feature '%s'\n", pos->parser.featureName().c_str());
+            } else if (i == TRACE_SKIP_POS) {
+                trace += fmt("  (skipped %zu entries)\n", (n - MAX_TRACE_SIZE) + 1);
             }
-            compile_error = true;
         }
+        return trace;
+    }
+
+    FeatureRef fail(const vespalib::string &feature_name, const vespalib::string &reason, bool skip_self = false) {
+        if (failed_set.count(feature_name) == 0) {
+            failed_set.insert(feature_name);
+            auto trace = make_trace(skip_self);
+            if (trace.empty()) {
+                LOG(warning, "invalid rank feature '%s': %s", feature_name.c_str(), reason.c_str());
+            } else {
+                LOG(warning, "invalid rank feature '%s': %s\n%s", feature_name.c_str(), reason.c_str(), trace.c_str());
+            }
+        }
+        probe_stack();
         return FeatureRef();
     }
 
+    void fail_self(const vespalib::string &reason) {
+        fail(self().parser.featureName(), reason, true);
+    }
+
     FeatureRef verify_type(const FeatureNameParser &parser, FeatureRef ref, Accept accept_type) {
         const auto &spec = spec_list[ref.executor];
-        bool is_object = spec.output_types[ref.output];
+        bool is_object = spec.output_types[ref.output].is_object();
         if (!is_compatible(is_object, accept_type)) {
-            return failed(parser.featureName(),
-                          vespalib::make_string("output '%s' has wrong type: was %s, expected %s",
-                                  parser.output().c_str(), type_str(is_object), accept_type_str(accept_type)));
+            return fail(parser.featureName(),
+                        fmt("output '%s' has wrong type: was %s, expected %s",
+                            parser.output().c_str(), type_str(is_object), accept_type_str(accept_type)));
         }
+        probe_stack();
         return ref;
     }
 
-    FeatureRef setup_feature(const FeatureNameParser &parser, Accept accept_type) {
-        Blueprint::SP blueprint = factory.createBlueprint(parser.baseName());
-        if (blueprint.get() == nullptr) {
-            return failed(parser.featureName(),
-                          vespalib::make_string("unknown basename: '%s'", parser.baseName().c_str()));
-        }
-        resolve_stack.emplace_back(blueprint, parser);
-        FrameGuard frame_guard(resolve_stack);
-        self().spec.blueprint->setName(parser.executorName());
-        self().spec.blueprint->attach_dependency_handler(*this);
-        if (!self().spec.blueprint->setup(index_env, parser.parameters())) {
-            return failed(parser.featureName(), "invalid parameters");
-        }
-        if (parser.output().empty() && self().spec.output_types.empty()) {
-            return failed(parser.featureName(), "has no output value");
-        }
-        const auto &feature = feature_map.find(parser.featureName());
-        if (feature == feature_map.end()) {
-            return failed(parser.featureName(),
-                          vespalib::make_string("unknown output: '%s'", parser.output().c_str()));
+    void setup_executor(const FeatureNameParser &parser) {
+        if (setup_set.count(parser.executorName()) == 0) {
+            setup_set.insert(parser.executorName());
+            if (Blueprint::SP blueprint = factory.createBlueprint(parser.baseName())) {
+                resolve_stack.emplace_back(std::move(blueprint), parser);
+                FrameGuard frame_guard(resolve_stack);
+                self().spec.blueprint->setName(parser.executorName());
+                self().spec.blueprint->attach_dependency_handler(*this);
+                if (!self().spec.blueprint->setup(index_env, parser.parameters())) {
+                    fail_self("invalid parameters");
+                }
+                if (parser.output().empty() && self().spec.output_types.empty()) {
+                    fail_self("has no output value");
+                }
+                spec_list.push_back(self().spec); // keep all feature_map refs valid
+            } else {
+                fail(parser.featureName(), fmt("unknown basename: '%s'", parser.baseName().c_str()));
+            }
         }
-        spec_list.push_back(self().spec);
-        return verify_type(parser, feature->second, accept_type);
     }
 
     FeatureRef resolve_feature(const vespalib::string &feature_name, Accept accept_type) {
         FeatureNameParser parser(feature_name);
         if (!parser.valid()) {
-            return failed(feature_name, "malformed name");
+            return fail(feature_name, "malformed name");
         }
-        const auto &feature = feature_map.find(parser.featureName());
-        if (feature != feature_map.end()) {
-            return verify_type(parser, feature->second, accept_type);
+        if (failed_set.count(parser.featureName()) > 0) {
+            return fail(parser.featureName(), "already failed");
+        }
+        auto old_feature = feature_map.find(parser.featureName());
+        if (old_feature != feature_map.end()) {
+            return verify_type(parser, old_feature->second, accept_type);
         }
         if ((resolve_stack.size() + 1) > BlueprintResolver::MAX_DEP_DEPTH) {
-            return failed(parser.featureName(), "dependency graph too deep");
+            return fail(parser.featureName(), "dependency graph too deep");
         }
         for (const Frame &frame: resolve_stack) {
             if (frame.parser.executorName() == parser.executorName()) {
-                return failed(parser.featureName(), "dependency cycle detected");
+                return fail(parser.featureName(), "dependency cycle detected");
             }
         }
-        return setup_feature(parser, accept_type);
+        setup_executor(parser);
+        auto new_feature = feature_map.find(parser.featureName());
+        if (new_feature != feature_map.end()) {
+            return verify_type(parser, new_feature->second, accept_type);
+        }
+        return fail(parser.featureName(), fmt("unknown output: '%s'", parser.output().c_str()));
     }
 
-    const FeatureType &resolve_input(const vespalib::string &feature_name, Accept accept_type) override {
+    std::optional<FeatureType> resolve_input(const vespalib::string &feature_name, Accept accept_type) override {
         assert(self().spec.output_types.empty()); // require: 'resolve inputs' before 'define outputs'
         auto ref = resolve_feature(feature_name, accept_type);
         if (!ref.valid()) {
-            return FeatureType::number();
+            // fail silently here to avoid mutiple traces for the same root error
+            failed_set.insert(self().parser.featureName());
+            return std::nullopt;
         }
         self().spec.inputs.push_back(ref);
         return spec_list[ref.executor].output_types[ref.output];
     }
 
-    void define_output(const vespalib::string &output_name, const FeatureType &type) override {
+    void define_output(const vespalib::string &output_name, FeatureType type) override {
         vespalib::string feature_name = self().parser.executorName();
         if (!output_name.empty()) {
             feature_name.push_back('.');
@@ -165,20 +227,26 @@ struct Compiler : public Blueprint::DependencyHandler {
             feature_map.emplace(self().parser.executorName(), output_ref);
         }
         feature_map.emplace(feature_name, output_ref);
-        self().spec.output_types.push_back(type);
+        self().spec.output_types.push_back(std::move(type));
+    }
+
+    void fail(const vespalib::string &msg) override {
+        fail_self(msg);
     }
 };
 
+Compiler::~Compiler() = default;
+
 } // namespace search::fef::<unnamed>
 
 BlueprintResolver::ExecutorSpec::ExecutorSpec(Blueprint::SP blueprint_in)
-    : blueprint(blueprint_in),
+    : blueprint(std::move(blueprint_in)),
       inputs(),
       output_types()
 { }
 
-BlueprintResolver::ExecutorSpec::~ExecutorSpec() { }
-BlueprintResolver::~BlueprintResolver() { }
+BlueprintResolver::ExecutorSpec::~ExecutorSpec() = default;
+BlueprintResolver::~BlueprintResolver() = default;
 
 BlueprintResolver::BlueprintResolver(const BlueprintFactory &factory,
                                      const IIndexEnvironment &indexEnv)
@@ -194,7 +262,7 @@ BlueprintResolver::BlueprintResolver(const BlueprintFactory &factory,
 void
 BlueprintResolver::addSeed(vespalib::stringref feature)
 {
-    _seeds.push_back(feature);
+    _seeds.emplace_back(feature);
 }
 
 bool
@@ -202,14 +270,23 @@ BlueprintResolver::compile()
 {
     assert(_executorSpecs.empty()); // only one compilation allowed
     Compiler compiler(_factory, _indexEnv, _executorSpecs, _featureMap);
-    for (const auto &seed: _seeds) {
-        auto ref = compiler.resolve_feature(seed, Blueprint::AcceptInput::ANY);
-        if (compiler.compile_error) {
-            return false;
-        }
-        _seedMap.emplace(FeatureNameParser(seed).featureName(), ref);
+    std::thread compile_thread([&]()
+                               {
+                                   compiler.probe_stack();
+                                   for (const auto &seed: _seeds) {
+                                       auto ref = compiler.resolve_feature(seed, Blueprint::AcceptInput::ANY);
+                                       if (compiler.failed()) {
+                                           return;
+                                       }
+                                       _seedMap.emplace(FeatureNameParser(seed).featureName(), ref);
+                                   }
+                               });
+    compile_thread.join();
+    int stack_usage = compiler.stack_usage();
+    if (stack_usage > (128 * 1024)) {
+        LOG(warning, "high stack usage: %d bytes", stack_usage);
     }
-    return true;
+    return !compiler.failed();
 }
 
 const BlueprintResolver::ExecutorSpecList &
diff --git a/searchlib/src/vespa/searchlib/fef/blueprintresolver.h b/searchlib/src/vespa/searchlib/fef/blueprintresolver.h
index 317263260fe..60b1eacb803 100644
--- a/searchlib/src/vespa/searchlib/fef/blueprintresolver.h
+++ b/searchlib/src/vespa/searchlib/fef/blueprintresolver.h
@@ -8,8 +8,7 @@
 #include "blueprint.h"
 #include "feature_type.h"
 
-namespace search {
-namespace fef {
+namespace search::fef {
 
 class BlueprintFactory;
 class IIndexEnvironment;
@@ -69,7 +68,14 @@ public:
      * problems for 'sane' developers and low enough to avoid stack
      * overflow.
      **/
-    static const uint32_t MAX_DEP_DEPTH = 64;
+    static constexpr uint32_t MAX_DEP_DEPTH = 256;
+
+    /**
+     * The maximum size of back-traces. Longer back-traces will be
+     * logged with skipped entries somewhere in the middle. Exposed
+     * for testing purposes.
+     **/
+    static constexpr int MAX_TRACE_SIZE = 16;
 
 private:
     const BlueprintFactory       &_factory;
@@ -147,5 +153,4 @@ public:
     const FeatureMap &getSeedMap() const;
 };
 
-} // namespace fef
-} // namespace search
+}
diff --git a/searchlib/src/vespa/searchlib/fef/feature_type.h b/searchlib/src/vespa/searchlib/fef/feature_type.h
index cdcff93c821..5763df1fe81 100644
--- a/searchlib/src/vespa/searchlib/fef/feature_type.h
+++ b/searchlib/src/vespa/searchlib/fef/feature_type.h
@@ -13,9 +13,9 @@ namespace fef {
  * the low-level eval library. A feature can either be a simple number
  * represented by a double or a polymorph value represented with an
  * object. The ranking framework itself will mostly care about the
- * representation (number/object) and not the specific type, hence the
- * implicit cast to bool. The type function is used to extract the
- * underlying type and is only allowed for features that are objects.
+ * representation (number/object) and not the specific type. The type
+ * function is used to extract the underlying type and is only allowed
+ * for features that are objects.
  **/
 class FeatureType {
 private:
@@ -26,8 +26,8 @@ private:
     FeatureType(TYPE_UP type_in) : _type(std::move(type_in)) {}
 public:
     FeatureType(const FeatureType &rhs);
+    FeatureType(FeatureType &&rhs) = default;
     bool is_object() const { return (_type.get() != nullptr); }
-    operator bool() const { return is_object(); }
     const TYPE &type() const;
     static const FeatureType &number() { return _number; }
     static FeatureType object(const TYPE &type_in);
diff --git a/searchlib/src/vespa/searchlib/fef/iblueprintregistry.h b/searchlib/src/vespa/searchlib/fef/iblueprintregistry.h
index 3b7634d3e93..43ca5a44303 100644
--- a/searchlib/src/vespa/searchlib/fef/iblueprintregistry.h
+++ b/searchlib/src/vespa/searchlib/fef/iblueprintregistry.h
@@ -2,8 +2,7 @@
 
 #pragma once
 
-namespace search {
-namespace fef {
+namespace search::fef {
 
 /**
  * This is an interface used during plugin setup to register blueprint
@@ -23,6 +22,4 @@ public:
     virtual ~IBlueprintRegistry() {}
 };
 
-} // namespace fef
-} // namespace search
-
+}
diff --git a/searchlib/src/vespa/searchlib/fef/parameterdescriptions.h b/searchlib/src/vespa/searchlib/fef/parameterdescriptions.h
index c91d85c1a83..fb8f8e06d4d 100644
--- a/searchlib/src/vespa/searchlib/fef/parameterdescriptions.h
+++ b/searchlib/src/vespa/searchlib/fef/parameterdescriptions.h
@@ -6,8 +6,7 @@
 #include <vector>
 #include <cstddef>
 
-namespace search {
-namespace fef {
+namespace search::fef {
 
 /**
  * Represents the type of a parameter.
@@ -248,5 +247,4 @@ public:
     ParameterDescriptions & repeat(size_t n = 1);
 };
 
-} // namespace fef
-} // namespace search
+}
diff --git a/searchlib/src/vespa/searchlib/fef/parametervalidator.h b/searchlib/src/vespa/searchlib/fef/parametervalidator.h
index 6e0fd7199c8..12044905dd0 100644
--- a/searchlib/src/vespa/searchlib/fef/parametervalidator.h
+++ b/searchlib/src/vespa/searchlib/fef/parametervalidator.h
@@ -7,8 +7,7 @@
 #include "parameter.h"
 #include "parameterdescriptions.h"
 
-namespace search {
-namespace fef {
+namespace search::fef {
 
 /**
  * This class is a validator for a string parameter list given an index environment and a set of parameter descriptions.
@@ -83,6 +82,4 @@ public:
     Result validate();
 };
 
-} // namespace fef
-} // namespace search
-
+}
diff --git a/searchlib/src/vespa/searchlib/fef/properties.h b/searchlib/src/vespa/searchlib/fef/properties.h
index 1cbc0ba9064..377926509bd 100644
--- a/searchlib/src/vespa/searchlib/fef/properties.h
+++ b/searchlib/src/vespa/searchlib/fef/properties.h
@@ -129,9 +129,10 @@ public:
 class Properties
 {
 private:
-    typedef vespalib::string      Key;
-    typedef Property::Values      Value;
-    typedef vespalib::hash_map<Key, Value>  Map;
+    using Key = vespalib::string;
+    using Value =  Property::Values;
+    using Map = vespalib::hash_map<Key, Value, vespalib::hash<Key>,
+                                   std::equal_to<>, vespalib::hashtable_base::and_modulator>;
 
     uint32_t _numValues;
     Map      _data;
diff --git a/searchlib/src/vespa/searchlib/fef/rank_program.cpp b/searchlib/src/vespa/searchlib/fef/rank_program.cpp
index 80729d90bf5..0bc85a63ceb 100644
--- a/searchlib/src/vespa/searchlib/fef/rank_program.cpp
+++ b/searchlib/src/vespa/searchlib/fef/rank_program.cpp
@@ -3,15 +3,18 @@
 #include "rank_program.h"
 #include "featureoverrider.h"
 #include <vespa/vespalib/locale/c.h>
+#include <vespa/vespalib/stllike/hash_set.hpp>
 #include <algorithm>
 #include <cassert>
 
+#include <vespa/log/log.h>
+LOG_SETUP(".fef.rankprogram");
+
 using vespalib::Stash;
 
 namespace search::fef {
 
 using MappedValues = std::map<const NumberOrObject *, LazyValue>;
-using ValueSet = std::set<const NumberOrObject *>;
 
 namespace {
 
@@ -42,7 +45,7 @@ struct OverrideVisitor : public IPropertiesVisitor
     {
         auto pos = feature_map.find(key);
         if (pos != feature_map.end()) {
-            overrides.push_back(Override(pos->second, vespalib::locale::c::strtod(values.get().c_str(), nullptr)));
+            overrides.emplace_back(pos->second, vespalib::locale::c::strtod(values.get().c_str(), nullptr));
         }
     }
 };
@@ -136,7 +139,7 @@ RankProgram::resolve(const BlueprintResolver::FeatureMap &features, bool unbox_s
     for (const auto &entry: features) {
         const auto &name = entry.first;
         auto ref = entry.second;
-        bool is_object = specs[ref.executor].output_types[ref.output];
+        bool is_object = specs[ref.executor].output_types[ref.output].is_object();
         FeatureExecutor *executor = _executors[ref.executor];
         const NumberOrObject *raw_value = executor->outputs().get_raw(ref.output);
         LazyValue lazy_value = check_const(raw_value) ? LazyValue(raw_value) : LazyValue(raw_value, executor);
@@ -153,7 +156,7 @@ RankProgram::resolve(const BlueprintResolver::FeatureMap &features, bool unbox_s
 }
 
 RankProgram::RankProgram(BlueprintResolver::SP resolver)
-    : _resolver(resolver),
+    : _resolver(std::move(resolver)),
       _hot_stash(32768),
       _cold_stash(),
       _executors(),
@@ -162,7 +165,7 @@ RankProgram::RankProgram(BlueprintResolver::SP resolver)
 {
 }
 
-RankProgram::~RankProgram() {}
+RankProgram::~RankProgram() = default;
 
 void
 RankProgram::setup(const MatchData &md,
@@ -175,6 +178,8 @@ RankProgram::setup(const MatchData &md,
     auto override_end = overrides.end();
 
     const auto &specs = _resolver->getExecutorSpecs();
+    _executors.reserve(specs.size());
+    _is_const.resize(specs.size()*2); // Reserve space in hashmap for executors to be const
     for (uint32_t i = 0; i < specs.size(); ++i) {
         vespalib::ArrayRef<NumberOrObject> outputs = _hot_stash.create_array<NumberOrObject>(specs[i].output_types.size());
         StashSelector stash(_hot_stash, _cold_stash);
@@ -211,11 +216,27 @@ RankProgram::setup(const MatchData &md,
     }
     for (const auto &seed_entry: _resolver->getSeedMap()) {
         auto seed = seed_entry.second;
-        if (specs[seed.executor].output_types[seed.output]) {
+        if (specs[seed.executor].output_types[seed.output].is_object()) {
             unbox(seed, md);
         }
     }
     assert(_executors.size() == specs.size());
+    LOG(debug, "Num executors = %ld, hot stash = %ld, cold stash = %ld, match data fields = %d",
+               _executors.size(), _hot_stash.count_used(), _cold_stash.count_used(), md.getNumTermFields());
+    if (LOG_WOULD_LOG(debug)) {
+        vespalib::hash_map<vespalib::string, size_t> executorStats;
+        for (const FeatureExecutor * executor : _executors) {
+            vespalib::string name = executor->getClassName();
+            if (executorStats.find(name) == executorStats.end()) {
+                executorStats[name] = 1;
+            } else {
+                executorStats[name]++;
+            }
+        }
+        for (const auto & stat : executorStats) {
+            LOG(debug, "There are %ld executors of type %s", stat.second, stat.first.c_str());
+        }
+    }
 }
 
 FeatureResolver
diff --git a/searchlib/src/vespa/searchlib/fef/rank_program.h b/searchlib/src/vespa/searchlib/fef/rank_program.h
index 0e5c390162a..15b7912bfb1 100644
--- a/searchlib/src/vespa/searchlib/fef/rank_program.h
+++ b/searchlib/src/vespa/searchlib/fef/rank_program.h
@@ -9,8 +9,8 @@
 #include "feature_resolver.h"
 #include <vespa/vespalib/stllike/string.h>
 #include <vespa/vespalib/util/array.h>
-#include <set>
-#include <vector>
+#include <vespa/vespalib/util/stash.h>
+#include <vespa/vespalib/stllike/hash_set.h>
 
 namespace search::fef {
 
@@ -31,7 +31,8 @@ private:
     RankProgram &operator=(const RankProgram &) = delete;
 
     using MappedValues = std::map<const NumberOrObject *, LazyValue>;
-    using ValueSet = std::set<const NumberOrObject *>;
+    using ValueSet = vespalib::hash_set<const NumberOrObject *, vespalib::hash<const NumberOrObject *>,
+                                        std::equal_to<>, vespalib::hashtable_base::and_modulator>;
 
     BlueprintResolver::SP            _resolver;
     vespalib::Stash                  _hot_stash;
diff --git a/searchlib/src/vespa/searchlib/fef/ranksetup.cpp b/searchlib/src/vespa/searchlib/fef/ranksetup.cpp
index 440ea3e0ea1..88f4a07d95d 100644
--- a/searchlib/src/vespa/searchlib/fef/ranksetup.cpp
+++ b/searchlib/src/vespa/searchlib/fef/ranksetup.cpp
@@ -12,7 +12,7 @@ class VisitorAdapter : public search::fef::IDumpFeatureVisitor
 {
     search::fef::BlueprintResolver &_resolver;
 public:
-    VisitorAdapter(search::fef::BlueprintResolver &resolver)
+    explicit VisitorAdapter(search::fef::BlueprintResolver &resolver)
         : _resolver(resolver) {}
     void visitDumpFeature(const vespalib::string &name) override {
         _resolver.addSeed(name);
@@ -27,10 +27,10 @@ using namespace indexproperties;
 RankSetup::RankSetup(const BlueprintFactory &factory, const IIndexEnvironment &indexEnv)
     : _factory(factory),
       _indexEnv(indexEnv),
-      _first_phase_resolver(new BlueprintResolver(factory, indexEnv)),
-      _second_phase_resolver(new BlueprintResolver(factory, indexEnv)),
-      _summary_resolver(new BlueprintResolver(factory, indexEnv)),
-      _dumpResolver(new BlueprintResolver(factory, indexEnv)),
+      _first_phase_resolver(std::make_shared<BlueprintResolver>(factory, indexEnv)),
+      _second_phase_resolver(std::make_shared<BlueprintResolver>(factory, indexEnv)),
+      _summary_resolver(std::make_shared<BlueprintResolver>(factory, indexEnv)),
+      _dumpResolver(std::make_shared<BlueprintResolver>(factory, indexEnv)),
       _firstPhaseRankFeature(),
       _secondPhaseRankFeature(),
       _degradationAttribute(),
@@ -60,7 +60,8 @@ RankSetup::RankSetup(const BlueprintFactory &factory, const IIndexEnvironment &i
       _diversityCutoffFactor(10.0),
       _diversityCutoffStrategy("loose"),
       _softTimeoutEnabled(false),
-      _softTimeoutTailCost(0.1)
+      _softTimeoutTailCost(0.1),
+      _softTimeoutFactor(0.5)
 { }
 
 RankSetup::~RankSetup() = default;
@@ -71,13 +72,13 @@ RankSetup::configure()
     setFirstPhaseRank(rank::FirstPhase::lookup(_indexEnv.getProperties()));
     setSecondPhaseRank(rank::SecondPhase::lookup(_indexEnv.getProperties()));
     std::vector<vespalib::string> summaryFeatures = summary::Feature::lookup(_indexEnv.getProperties());
-    for (uint32_t i = 0; i < summaryFeatures.size(); ++i) {
-        addSummaryFeature(summaryFeatures[i]);
+    for (const auto & feature : summaryFeatures) {
+        addSummaryFeature(feature);
     }
     setIgnoreDefaultRankFeatures(dump::IgnoreDefaultFeatures::check(_indexEnv.getProperties()));
     std::vector<vespalib::string> dumpFeatures = dump::Feature::lookup(_indexEnv.getProperties());
-    for (uint32_t i = 0; i < dumpFeatures.size(); ++i) {
-        addDumpFeature(dumpFeatures[i]);
+    for (const auto & feature : dumpFeatures) {
+        addDumpFeature(feature);
     }
     split_unpacking_iterators(matching::SplitUnpackingIterators::check(_indexEnv.getProperties()));
     delay_unpacking_iterators(matching::DelayUnpackingIterators::check(_indexEnv.getProperties()));
@@ -159,15 +160,15 @@ RankSetup::compile()
             _compileError = true;
         }
     }
-    for (uint32_t i = 0; i < _summaryFeatures.size(); ++i) {
-        _summary_resolver->addSeed(_summaryFeatures[i]);
+    for (const auto & feature :_summaryFeatures) {
+        _summary_resolver->addSeed(feature);
     }
     if (!_ignoreDefaultRankFeatures) {
         VisitorAdapter adapter(*_dumpResolver);
         _factory.visitDumpFeatures(_indexEnv, adapter);
     }
-    for (uint32_t i = 0; i < _dumpFeatures.size(); ++i) {
-        _dumpResolver->addSeed(_dumpFeatures[i]);
+    for (const auto & feature : _dumpFeatures) {
+        _dumpResolver->addSeed(feature);
     }
     _indexEnv.hintFeatureMotivation(IIndexEnvironment::RANK);
     _compileError |= !_first_phase_resolver->compile();
diff --git a/searchlib/src/vespa/searchlib/fef/ranksetup.h b/searchlib/src/vespa/searchlib/fef/ranksetup.h
index d543794b347..e1cd78d41a9 100644
--- a/searchlib/src/vespa/searchlib/fef/ranksetup.h
+++ b/searchlib/src/vespa/searchlib/fef/ranksetup.h
@@ -397,10 +397,10 @@ public:
     // them to be ready to use. Also keep in mind that creating a rank
     // program is cheap while setting it up is more expensive.
 
-    RankProgram::UP create_first_phase_program() const { return RankProgram::UP(new RankProgram(_first_phase_resolver)); }
-    RankProgram::UP create_second_phase_program() const { return RankProgram::UP(new RankProgram(_second_phase_resolver)); }
-    RankProgram::UP create_summary_program() const { return RankProgram::UP(new RankProgram(_summary_resolver)); }
-    RankProgram::UP create_dump_program() const { return RankProgram::UP(new RankProgram(_dumpResolver)); }
+    RankProgram::UP create_first_phase_program() const { return std::make_unique<RankProgram>(_first_phase_resolver); }
+    RankProgram::UP create_second_phase_program() const { return std::make_unique<RankProgram>(_second_phase_resolver); }
+    RankProgram::UP create_summary_program() const { return std::make_unique<RankProgram>(_summary_resolver); }
+    RankProgram::UP create_dump_program() const { return std::make_unique<RankProgram>(_dumpResolver); }
 
     /**
      * Here you can do some preprocessing. State must be stored in the IObjectStore.
diff --git a/searchlib/src/vespa/searchlib/fef/termfieldmatchdata.h b/searchlib/src/vespa/searchlib/fef/termfieldmatchdata.h
index 45de0d654fb..efa2e3f5bfa 100644
--- a/searchlib/src/vespa/searchlib/fef/termfieldmatchdata.h
+++ b/searchlib/src/vespa/searchlib/fef/termfieldmatchdata.h
@@ -6,6 +6,7 @@
 #include "fieldinfo.h"
 #include <vespa/searchlib/common/feature.h>
 #include <cstring>
+#include <limits>
 
 class MatchDataHeapTest;
 
diff --git a/searchlib/src/vespa/searchlib/fef/test/CMakeLists.txt b/searchlib/src/vespa/searchlib/fef/test/CMakeLists.txt
index dd6fda7fdf8..17179186dfd 100644
--- a/searchlib/src/vespa/searchlib/fef/test/CMakeLists.txt
+++ b/searchlib/src/vespa/searchlib/fef/test/CMakeLists.txt
@@ -7,6 +7,7 @@ vespa_add_library(searchlib_fef_test OBJECT
     ftlib.cpp
     indexenvironment.cpp
     indexenvironmentbuilder.cpp
+    labels.cpp
     matchdatabuilder.cpp
     mock_attribute_context.cpp
     queryenvironment.cpp
diff --git a/searchlib/src/vespa/searchlib/fef/test/dummy_dependency_handler.cpp b/searchlib/src/vespa/searchlib/fef/test/dummy_dependency_handler.cpp
index 3a6e4c8db2d..4fd36a69c71 100644
--- a/searchlib/src/vespa/searchlib/fef/test/dummy_dependency_handler.cpp
+++ b/searchlib/src/vespa/searchlib/fef/test/dummy_dependency_handler.cpp
@@ -13,7 +13,8 @@ DummyDependencyHandler::DummyDependencyHandler(Blueprint &blueprint_in)
       input(),
       accept_input(),
       output(),
-      output_type()
+      output_type(),
+      fail_msg()
 {
     blueprint.attach_dependency_handler(*this);
 }
@@ -29,7 +30,7 @@ DummyDependencyHandler::define_object_input(const vespalib::string &name, const
     object_type_map.emplace(name, FeatureType::object(type));
 }
 
-const FeatureType &
+std::optional<FeatureType>
 DummyDependencyHandler::resolve_input(const vespalib::string &feature_name, Blueprint::AcceptInput accept_type)
 {
     input.push_back(feature_name);
@@ -38,19 +39,28 @@ DummyDependencyHandler::resolve_input(const vespalib::string &feature_name, Blue
     if (pos == object_type_map.end()) {
         if (accept_type == Blueprint::AcceptInput::OBJECT) {
             accept_type_mismatch = true;
+            return std::nullopt;
         }
         return FeatureType::number();
     }
     if (accept_type == Blueprint::AcceptInput::NUMBER) {
         accept_type_mismatch = true;
+        return std::nullopt;
     }
     return pos->second;
 }
 
-void DummyDependencyHandler::define_output(const vespalib::string &output_name, const FeatureType &type)
+void
+DummyDependencyHandler::define_output(const vespalib::string &output_name, FeatureType type)
 {
     output.push_back(output_name);
-    output_type.push_back(type);
+    output_type.push_back(std::move(type));
+}
+
+void
+DummyDependencyHandler::fail(const vespalib::string &msg)
+{
+    fail_msg = msg;
 }
 
 } // namespace search::fef::test
diff --git a/searchlib/src/vespa/searchlib/fef/test/dummy_dependency_handler.h b/searchlib/src/vespa/searchlib/fef/test/dummy_dependency_handler.h
index 1c24858b12b..ca70d0943cf 100644
--- a/searchlib/src/vespa/searchlib/fef/test/dummy_dependency_handler.h
+++ b/searchlib/src/vespa/searchlib/fef/test/dummy_dependency_handler.h
@@ -26,12 +26,14 @@ struct DummyDependencyHandler : public Blueprint::DependencyHandler
     std::vector<Blueprint::AcceptInput>    accept_input;
     std::vector<vespalib::string>          output;
     std::vector<FeatureType>               output_type;
+    vespalib::string                       fail_msg;
 
     explicit DummyDependencyHandler(Blueprint &blueprint_in);
     ~DummyDependencyHandler();
     void define_object_input(const vespalib::string &name, const vespalib::eval::ValueType &type);
-    const FeatureType &resolve_input(const vespalib::string &feature_name, Blueprint::AcceptInput accept_type) override;
-    void define_output(const vespalib::string &output_name, const FeatureType &type) override;
+    std::optional<FeatureType> resolve_input(const vespalib::string &feature_name, Blueprint::AcceptInput accept_type) override;
+    void define_output(const vespalib::string &output_name, FeatureType type) override;
+    void fail(const vespalib::string &msg) override;
 };
 
 } // namespace search::fef::test
diff --git a/searchlib/src/vespa/searchlib/fef/test/indexenvironmentbuilder.cpp b/searchlib/src/vespa/searchlib/fef/test/indexenvironmentbuilder.cpp
index be78e8c45f8..c0ca3b637b1 100644
--- a/searchlib/src/vespa/searchlib/fef/test/indexenvironmentbuilder.cpp
+++ b/searchlib/src/vespa/searchlib/fef/test/indexenvironmentbuilder.cpp
@@ -3,14 +3,11 @@
 #include <vespa/searchcommon/common/datatype.h>
 #include "indexenvironmentbuilder.h"
 
-namespace search {
-namespace fef {
-namespace test {
+namespace search::fef::test {
 
 IndexEnvironmentBuilder::IndexEnvironmentBuilder(IndexEnvironment &env) :
     _env(env)
 {
-    // empty
 }
 
 IndexEnvironmentBuilder &
@@ -34,6 +31,4 @@ IndexEnvironmentBuilder::addField(const FieldType &type,
     return *this;
 }
 
-} // namespace test
-} // namespace fef
-} // namespace search
+}
diff --git a/searchlib/src/vespa/searchlib/fef/test/indexenvironmentbuilder.h b/searchlib/src/vespa/searchlib/fef/test/indexenvironmentbuilder.h
index e6a81a46008..edfa78e20f0 100644
--- a/searchlib/src/vespa/searchlib/fef/test/indexenvironmentbuilder.h
+++ b/searchlib/src/vespa/searchlib/fef/test/indexenvironmentbuilder.h
@@ -3,9 +3,7 @@
 
 #include "indexenvironment.h"
 
-namespace search {
-namespace fef {
-namespace test {
+namespace search::fef::test {
 
 /**
  * This class is used to setup an IndexEnvironment for testing.
@@ -57,7 +55,4 @@ private:
     IndexEnvironment &_env;
 };
 
-} // namespace test
-} // namespace fef
-} // namespace search
-
+}
diff --git a/searchlib/src/vespa/searchlib/fef/test/labels.cpp b/searchlib/src/vespa/searchlib/fef/test/labels.cpp
new file mode 100644
index 00000000000..83b9cf4bc11
--- /dev/null
+++ b/searchlib/src/vespa/searchlib/fef/test/labels.cpp
@@ -0,0 +1,3 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "labels.h"
diff --git a/searchlib/src/vespa/searchlib/fef/test/labels.h b/searchlib/src/vespa/searchlib/fef/test/labels.h
new file mode 100644
index 00000000000..2ed8730a4a4
--- /dev/null
+++ b/searchlib/src/vespa/searchlib/fef/test/labels.h
@@ -0,0 +1,28 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/searchlib/fef/properties.h>
+#include <vespa/vespalib/stllike/asciistream.h>
+
+namespace search::fef::test {
+
+struct Labels {
+    virtual void inject(Properties &p) const = 0;
+    virtual ~Labels() {}
+};
+struct NoLabel : public Labels {
+    virtual void inject(Properties &) const override {}    
+};
+struct SingleLabel : public Labels {
+    vespalib::string label;
+    uint32_t uid;
+    SingleLabel(const vespalib::string &l, uint32_t x) : label(l), uid(x) {}
+    virtual void inject(Properties &p) const override {
+        vespalib::asciistream key;
+        key << "vespa.label." << label << ".id";
+        vespalib::asciistream value;
+        value << uid;
+        p.add(key.str(), value.str());
+    }
+};
+
+}
diff --git a/searchlib/src/vespa/searchlib/fef/test/plugin/cfgvalue.cpp b/searchlib/src/vespa/searchlib/fef/test/plugin/cfgvalue.cpp
index 31e99ef9953..8be0961f999 100644
--- a/searchlib/src/vespa/searchlib/fef/test/plugin/cfgvalue.cpp
+++ b/searchlib/src/vespa/searchlib/fef/test/plugin/cfgvalue.cpp
@@ -2,11 +2,10 @@
 
 #include "cfgvalue.h"
 #include <vespa/searchlib/fef/properties.h>
+#include <vespa/vespalib/util/stash.h>
 #include <sstream>
 
-namespace search {
-namespace fef {
-namespace test {
+namespace search::fef::test {
 
 CfgValueBlueprint::CfgValueBlueprint() :
     Blueprint("test_cfgvalue"),
@@ -14,9 +13,7 @@ CfgValueBlueprint::CfgValueBlueprint() :
 {
 }
 
-CfgValueBlueprint::~CfgValueBlueprint()
-{
-}
+CfgValueBlueprint::~CfgValueBlueprint() = default;
 
 void
 CfgValueBlueprint::visitDumpFeatures(const IIndexEnvironment &indexEnv, IDumpFeatureVisitor &visitor) const
@@ -59,6 +56,4 @@ CfgValueBlueprint::createExecutor(const IQueryEnvironment & queryEnv, vespalib::
     return stash.create<search::features::ValueExecutor>(_values);
 }
 
-} // namespace test
-} // namespace fef
-} // namespace search
+}
diff --git a/searchlib/src/vespa/searchlib/fef/test/plugin/chain.cpp b/searchlib/src/vespa/searchlib/fef/test/plugin/chain.cpp
index 86754c2c22d..017b916ad76 100644
--- a/searchlib/src/vespa/searchlib/fef/test/plugin/chain.cpp
+++ b/searchlib/src/vespa/searchlib/fef/test/plugin/chain.cpp
@@ -1,11 +1,10 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include "chain.h"
+#include <vespa/vespalib/util/stash.h>
 #include <sstream>
 
-namespace search {
-namespace fef {
-namespace test {
+namespace search::fef::test {
 
 ChainExecutor::ChainExecutor() :
     FeatureExecutor()
@@ -67,6 +66,4 @@ ChainBlueprint::createExecutor(const IQueryEnvironment &queryEnv, vespalib::Stas
     return stash.create<ChainExecutor>();
 }
 
-} // namespace test
-} // namespace fef
-} // namespace search
+}
diff --git a/searchlib/src/vespa/searchlib/fef/test/plugin/double.cpp b/searchlib/src/vespa/searchlib/fef/test/plugin/double.cpp
index d9ec8b13e57..6f8ebd57fb0 100644
--- a/searchlib/src/vespa/searchlib/fef/test/plugin/double.cpp
+++ b/searchlib/src/vespa/searchlib/fef/test/plugin/double.cpp
@@ -3,6 +3,7 @@
 #include "double.h"
 #include <vespa/searchlib/fef/featurenamebuilder.h>
 #include <vespa/vespalib/stllike/asciistream.h>
+#include <vespa/vespalib/util/stash.h>
 #include <cassert>
 
 namespace search::fef::test {
diff --git a/searchlib/src/vespa/searchlib/fef/test/plugin/query.cpp b/searchlib/src/vespa/searchlib/fef/test/plugin/query.cpp
index 4b4b10c4d25..1e57d6252ee 100644
--- a/searchlib/src/vespa/searchlib/fef/test/plugin/query.cpp
+++ b/searchlib/src/vespa/searchlib/fef/test/plugin/query.cpp
@@ -4,6 +4,7 @@
 #include <vespa/searchlib/features/valuefeature.h>
 #include <vespa/searchlib/fef/properties.h>
 #include <vespa/vespalib/locale/c.h>
+#include <vespa/vespalib/util/stash.h>
 #include <sstream>
 
 namespace search::fef::test {
@@ -32,7 +33,7 @@ QueryBlueprint::createExecutor(const IQueryEnvironment &queryEnv, vespalib::Stas
 {
     std::vector<feature_t> values;
     std::string val = queryEnv.getProperties().lookup(_key).get("0.0");
-    values.push_back(vespalib::locale::c::strtod(val.data(), NULL));
+    values.push_back(vespalib::locale::c::strtod(val.data(), nullptr));
     return stash.create<search::features::ValueExecutor>(values);
 }
 
diff --git a/searchlib/src/vespa/searchlib/fef/test/plugin/staticrank.cpp b/searchlib/src/vespa/searchlib/fef/test/plugin/staticrank.cpp
index c1b8b940245..c5871b23e77 100644
--- a/searchlib/src/vespa/searchlib/fef/test/plugin/staticrank.cpp
+++ b/searchlib/src/vespa/searchlib/fef/test/plugin/staticrank.cpp
@@ -2,10 +2,9 @@
 
 #include "staticrank.h"
 #include <vespa/searchcommon/attribute/attributecontent.h>
+#include <vespa/vespalib/util/stash.h>
 
-namespace search {
-namespace fef {
-namespace test {
+namespace search::fef::test {
 
 StaticRankExecutor::StaticRankExecutor(const search::attribute::IAttributeVector * attribute) :
     FeatureExecutor(),
@@ -17,7 +16,7 @@ void
 StaticRankExecutor::execute(uint32_t docId)
 {
     search::attribute::FloatContent staticRank;
-    if (_attribute != NULL) {
+    if (_attribute != nullptr) {
         staticRank.allocate(_attribute->getMaxValueCount());
         staticRank.fill(*_attribute, docId);
     }
@@ -31,9 +30,7 @@ StaticRankBlueprint::StaticRankBlueprint() :
 {
 }
 
-StaticRankBlueprint::~StaticRankBlueprint()
-{
-}
+StaticRankBlueprint::~StaticRankBlueprint() = default;
 
 bool
 StaticRankBlueprint::setup(const IIndexEnvironment & indexEnv, const StringVector & params)
@@ -54,6 +51,4 @@ StaticRankBlueprint::createExecutor(const IQueryEnvironment & queryEnv, vespalib
     return stash.create<StaticRankExecutor>(av);
 }
 
-} // namespace test
-} // namespace fef
-} // namespace search
+}
diff --git a/searchlib/src/vespa/searchlib/fef/test/plugin/sum.cpp b/searchlib/src/vespa/searchlib/fef/test/plugin/sum.cpp
index b5025d53cbd..4362a7f0860 100644
--- a/searchlib/src/vespa/searchlib/fef/test/plugin/sum.cpp
+++ b/searchlib/src/vespa/searchlib/fef/test/plugin/sum.cpp
@@ -2,10 +2,9 @@
 
 #include "sum.h"
 #include <vespa/searchlib/fef/featurenamebuilder.h>
+#include <vespa/vespalib/util/stash.h>
 
-namespace search {
-namespace fef {
-namespace test {
+namespace search::fef::test {
 
 void
 SumExecutor::execute(uint32_t)
@@ -73,6 +72,4 @@ SumBlueprint::createExecutor(const IQueryEnvironment &queryEnv, vespalib::Stash
     return stash.create<SumExecutor>();
 }
 
-} // namespace test
-} // namespace fef
-} // namespace search
+}
diff --git a/searchlib/src/vespa/searchlib/fef/test/plugin/unbox.cpp b/searchlib/src/vespa/searchlib/fef/test/plugin/unbox.cpp
index 7b3876fada0..e30b4893e15 100644
--- a/searchlib/src/vespa/searchlib/fef/test/plugin/unbox.cpp
+++ b/searchlib/src/vespa/searchlib/fef/test/plugin/unbox.cpp
@@ -1,6 +1,7 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include "unbox.h"
+#include <vespa/vespalib/util/stash.h>
 
 namespace search::fef::test {
 
diff --git a/searchlib/src/vespa/searchlib/fef/test/test_features.cpp b/searchlib/src/vespa/searchlib/fef/test/test_features.cpp
index f924acd65de..c32776dc88d 100644
--- a/searchlib/src/vespa/searchlib/fef/test/test_features.cpp
+++ b/searchlib/src/vespa/searchlib/fef/test/test_features.cpp
@@ -3,6 +3,8 @@
 #include <vespa/vespalib/testkit/test_kit.h>
 #include "test_features.h"
 #include <vespa/vespalib/locale/c.h>
+#include <vespa/vespalib/util/stash.h>
+
 
 using vespalib::eval::DoubleValue;
 using vespalib::eval::ValueType;
diff --git a/searchlib/src/vespa/searchlib/index/postinglisthandle.h b/searchlib/src/vespa/searchlib/index/postinglisthandle.h
index 8eb42dc91fe..a9176d7cf13 100644
--- a/searchlib/src/vespa/searchlib/index/postinglisthandle.h
+++ b/searchlib/src/vespa/searchlib/index/postinglisthandle.h
@@ -3,6 +3,7 @@
 
 #include <vespa/searchlib/index/postinglistcounts.h>
 #include <memory>
+#include <cstdlib>
 
 namespace search { class BitVector; }
 namespace search::queryeval { class SearchIterator; }
diff --git a/searchlib/src/vespa/searchlib/memoryindex/document_inverter.cpp b/searchlib/src/vespa/searchlib/memoryindex/document_inverter.cpp
index d032f06fc58..c8b23b2256f 100644
--- a/searchlib/src/vespa/searchlib/memoryindex/document_inverter.cpp
+++ b/searchlib/src/vespa/searchlib/memoryindex/document_inverter.cpp
@@ -8,7 +8,7 @@
 #include <vespa/document/annotation/alternatespanlist.h>
 #include <vespa/document/datatype/urldatatype.h>
 #include <vespa/document/repo/fixedtyperepo.h>
-#include <vespa/searchlib/common/isequencedtaskexecutor.h>
+#include <vespa/vespalib/util/isequencedtaskexecutor.h>
 #include <vespa/searchlib/common/sort.h>
 #include <vespa/searchlib/util/url.h>
 #include <vespa/vespalib/text/lowercase.h>
diff --git a/searchlib/src/vespa/searchlib/memoryindex/document_inverter.h b/searchlib/src/vespa/searchlib/memoryindex/document_inverter.h
index ffa9dd0fab8..57f51e7fdb0 100644
--- a/searchlib/src/vespa/searchlib/memoryindex/document_inverter.h
+++ b/searchlib/src/vespa/searchlib/memoryindex/document_inverter.h
@@ -6,18 +6,21 @@
 #include <vespa/searchlib/index/schema_index_fields.h>
 
 namespace document {
-class DataType;
-class Document;
-class DocumentType;
-class Field;
-class FieldValue;
+    class DataType;
+    class Document;
+    class DocumentType;
+    class Field;
+    class FieldValue;
 }
 
 namespace search {
-    class ISequencedTaskExecutor;
     class IDestructorCallback;
 }
 
+namespace vespalib {
+    class ISequencedTaskExecutor;
+}
+
 namespace search::memoryindex {
 
 class FieldInverter;
@@ -31,6 +34,7 @@ class IFieldIndexCollection;
  */
 class DocumentInverter {
 private:
+    using ISequencedTaskExecutor = vespalib::ISequencedTaskExecutor;
     DocumentInverter(const DocumentInverter &) = delete;
     DocumentInverter &operator=(const DocumentInverter &) = delete;
 
diff --git a/searchlib/src/vespa/searchlib/memoryindex/memory_index.cpp b/searchlib/src/vespa/searchlib/memoryindex/memory_index.cpp
index d8e48e84fb7..34c5d65ab23 100644
--- a/searchlib/src/vespa/searchlib/memoryindex/memory_index.cpp
+++ b/searchlib/src/vespa/searchlib/memoryindex/memory_index.cpp
@@ -5,7 +5,7 @@
 #include "memory_index.h"
 #include <vespa/document/fieldvalue/arrayfieldvalue.h>
 #include <vespa/document/fieldvalue/document.h>
-#include <vespa/searchlib/common/sequencedtaskexecutor.h>
+#include <vespa/vespalib/util/isequencedtaskexecutor.h>
 #include <vespa/searchlib/index/field_length_calculator.h>
 #include <vespa/searchlib/index/schemautil.h>
 #include <vespa/searchlib/queryeval/create_blueprint_visitor_helper.h>
@@ -44,6 +44,7 @@ using queryeval::EmptyBlueprint;
 using queryeval::FieldSpec;
 using queryeval::IRequestContext;
 using queryeval::Searchable;
+using vespalib::ISequencedTaskExecutor;
 
 }
 
diff --git a/searchlib/src/vespa/searchlib/memoryindex/memory_index.h b/searchlib/src/vespa/searchlib/memoryindex/memory_index.h
index 4e20bcf81e3..a9f153f7dd8 100644
--- a/searchlib/src/vespa/searchlib/memoryindex/memory_index.h
+++ b/searchlib/src/vespa/searchlib/memoryindex/memory_index.h
@@ -14,7 +14,7 @@ namespace search::index {
     class IndexBuilder;
 }
 
-namespace search { class ISequencedTaskExecutor; }
+namespace vespalib { class ISequencedTaskExecutor; }
 
 namespace document { class Document; }
 
@@ -40,6 +40,7 @@ class FieldIndexCollection;
  */
 class MemoryIndex : public queryeval::Searchable {
 private:
+    using ISequencedTaskExecutor = vespalib::ISequencedTaskExecutor;
     index::Schema     _schema;
     ISequencedTaskExecutor &_invertThreads;
     ISequencedTaskExecutor &_pushThreads;
diff --git a/searchlib/src/vespa/searchlib/parsequery/stackdumpiterator.cpp b/searchlib/src/vespa/searchlib/parsequery/stackdumpiterator.cpp
index 70a3097ae05..c42cf8fc370 100644
--- a/searchlib/src/vespa/searchlib/parsequery/stackdumpiterator.cpp
+++ b/searchlib/src/vespa/searchlib/parsequery/stackdumpiterator.cpp
@@ -21,9 +21,11 @@ SimpleQueryStackDumpIterator::SimpleQueryStackDumpIterator(vespalib::stringref b
     _currUniqueId(0),
     _currFlags(0),
     _currArity(0),
-    _currArg1(0),
-    _currArg2(0),
-    _currArg3(0),
+    _extraIntArg1(0),
+    _extraIntArg2(0),
+    _extraIntArg3(0),
+    _extraDoubleArg4(0),
+    _extraDoubleArg5(0),
     _predicate_query_term(),
     _curr_index_name(),
     _curr_term(),
@@ -138,7 +140,6 @@ SimpleQueryStackDumpIterator::next()
     case ParseItem::ITEM_ANY:
         try {
             _currArity = readCompressedPositiveInt(p);
-            _currArg1 = 0;
             _curr_index_name = vespalib::stringref();
             _curr_term = vespalib::stringref();
         } catch (...) {
@@ -150,7 +151,7 @@ SimpleQueryStackDumpIterator::next()
     case ParseItem::ITEM_ONEAR:
         try {
             _currArity = readCompressedPositiveInt(p);
-            _currArg1 = readCompressedPositiveInt(p);
+            _extraIntArg1 = readCompressedPositiveInt(p);
             _curr_index_name = vespalib::stringref();
             _curr_term = vespalib::stringref();
         } catch (...) {
@@ -161,7 +162,7 @@ SimpleQueryStackDumpIterator::next()
     case ParseItem::ITEM_WEAK_AND:
         try {
             _currArity = readCompressedPositiveInt(p);
-            _currArg1 = readCompressedPositiveInt(p);
+            _extraIntArg1 = readCompressedPositiveInt(p); // targetNumHits
             _curr_index_name = read_stringref(p);
             _curr_term = vespalib::stringref();
         } catch (...) {
@@ -171,7 +172,6 @@ SimpleQueryStackDumpIterator::next()
     case ParseItem::ITEM_SAME_ELEMENT:
         try {
             _currArity = readCompressedPositiveInt(p);
-            _currArg1 = 0;
             _curr_index_name = read_stringref(p);
             _curr_term = vespalib::stringref();
         } catch (...) {
@@ -182,7 +182,6 @@ SimpleQueryStackDumpIterator::next()
     case ParseItem::ITEM_PURE_WEIGHTED_STRING:
         try {
             _curr_term = read_stringref(p);
-            _currArg1 = 0;
             _currArity = 0;
         } catch (...) {
             return false;
@@ -196,7 +195,6 @@ SimpleQueryStackDumpIterator::next()
         p += sizeof(int64_t);
         if (p > _bufEnd) return false;
 
-        _currArg1 = 0;
         _currArity = 0;
         break;
     case ParseItem::ITEM_WORD_ALTERNATIVES:
@@ -218,7 +216,6 @@ SimpleQueryStackDumpIterator::next()
         try {
             _curr_index_name = read_stringref(p);
             _curr_term = read_stringref(p);
-            _currArg1 = 0;
             _currArity = 0;
         } catch (...) {
             return false;
@@ -258,11 +255,9 @@ SimpleQueryStackDumpIterator::next()
             _currArity = readCompressedPositiveInt(p);
             _curr_index_name = read_stringref(p);
             if (_currType == ParseItem::ITEM_WAND) {
-                _currArg1 = readCompressedPositiveInt(p); // targetNumHits
-                _currArg2 = read_double(p); // scoreThreshold
-                _currArg3 = read_double(p); // thresholdBoostFactor
-            } else {
-                _currArg1 = 0;
+                _extraIntArg1 = readCompressedPositiveInt(p); // targetNumHits
+                _extraDoubleArg4 = read_double(p); // scoreThreshold
+                _extraDoubleArg5 = read_double(p); // thresholdBoostFactor
             }
             _curr_term = vespalib::stringref();
         } catch (...) {
@@ -274,7 +269,9 @@ SimpleQueryStackDumpIterator::next()
         try {
             _curr_index_name = read_stringref(p);
             _curr_term = read_stringref(p); // query_tensor_name
-            _currArg1 = readCompressedPositiveInt(p); // target_num_hits;
+            _extraIntArg1 = readCompressedPositiveInt(p); // targetNumHits
+            _extraIntArg2 = readCompressedPositiveInt(p); // allow_approximate
+            _extraIntArg3 = readCompressedPositiveInt(p); // explore_additional_hits
             _currArity = 0;
         } catch (...) {
             return false;
diff --git a/searchlib/src/vespa/searchlib/parsequery/stackdumpiterator.h b/searchlib/src/vespa/searchlib/parsequery/stackdumpiterator.h
index 6eb3fb7777d..73c97bb5fb3 100644
--- a/searchlib/src/vespa/searchlib/parsequery/stackdumpiterator.h
+++ b/searchlib/src/vespa/searchlib/parsequery/stackdumpiterator.h
@@ -42,12 +42,13 @@ private:
 
     /** The arity of the current item */
     uint32_t _currArity;
-    /** The first argument of the current item (length of NEAR/ONEAR area for example) */
-    uint32_t _currArg1;
-    /** The second argument of the current item (score threshold of WAND for example) */
-    double _currArg2;
-    /** The third argument of the current item (threshold boost factor of WAND for example) */
-    double _currArg3;
+
+    /* extra arguments */
+    uint32_t _extraIntArg1;
+    uint32_t _extraIntArg2;
+    uint32_t _extraIntArg3;
+    double _extraDoubleArg4;
+    double _extraDoubleArg5;
     /** The predicate query specification */
     query::PredicateQueryTerm::UP _predicate_query_term;
     /** The index name (field name) in the current item */
@@ -118,11 +119,12 @@ public:
 
     uint32_t getArity() const { return _currArity; }
 
-    uint32_t getArg1() const { return _currArg1; }
-
-    double getArg2() const { return _currArg2; }
-
-    double getArg3() const { return _currArg3; }
+    uint32_t getNearDistance() const { return _extraIntArg1; }
+    uint32_t getTargetNumHits() const { return _extraIntArg1; }
+    double getScoreThreshold() const { return _extraDoubleArg4; }
+    double getThresholdBoostFactor() const { return _extraDoubleArg5; }
+    bool getAllowApproximate() const { return (_extraIntArg2 != 0); }
+    uint32_t getExploreAdditionalHits() const { return _extraIntArg3; }
 
     query::PredicateQueryTerm::UP getPredicateQueryTerm()
     { return std::move(_predicate_query_term); }
diff --git a/searchlib/src/vespa/searchlib/predicate/predicate_index.cpp b/searchlib/src/vespa/searchlib/predicate/predicate_index.cpp
index 99861db31c9..19935d2e339 100644
--- a/searchlib/src/vespa/searchlib/predicate/predicate_index.cpp
+++ b/searchlib/src/vespa/searchlib/predicate/predicate_index.cpp
@@ -194,10 +194,8 @@ void PredicateIndex::removeDocument(uint32_t doc_id) {
     auto features = _features_store.get(doc_id);
     if (!features.empty()) {
         for (auto feature : features) {
-            removeFromIndex(feature, doc_id, _interval_index,
-                            _interval_store);
-            removeFromIndex(feature, doc_id, _bounds_index,
-                            _interval_store);
+            removeFromIndex(feature, doc_id, _interval_index, _interval_store);
+            removeFromIndex(feature, doc_id, _bounds_index, _interval_store);
         }
         _cache.removeIndex(doc_id);
     }
@@ -242,7 +240,7 @@ PredicateIndex::lookup(uint64_t key) const
     if (dictIterator.valid()) {
         auto it = _interval_index.getBTreePostingList(dictIterator.getData());
         if (it.valid()) {
-            return PopulateInterface::Iterator::UP(new DocIdIterator(it));
+            return std::make_unique<DocIdIterator>(it);
         }
     }
     return PopulateInterface::Iterator::UP();
diff --git a/searchlib/src/vespa/searchlib/query/streaming/querynode.cpp b/searchlib/src/vespa/searchlib/query/streaming/querynode.cpp
index 24c458c7e32..3db6c8e68c8 100644
--- a/searchlib/src/vespa/searchlib/query/streaming/querynode.cpp
+++ b/searchlib/src/vespa/searchlib/query/streaming/querynode.cpp
@@ -41,7 +41,7 @@ QueryNode::Build(const QueryNode * parent, const QueryNodeResultFactory & factor
             QueryConnector * qc = dynamic_cast<QueryConnector *> (qn.get());
             NearQueryNode * nqn = dynamic_cast<NearQueryNode *> (qc);
             if (nqn) {
-                nqn->distance(queryRep.getArg1());
+                nqn->distance(queryRep.getNearDistance());
             }
             if ((type == ParseItem::ITEM_WEAK_AND) ||
                 (type == ParseItem::ITEM_WEIGHTED_SET) ||
diff --git a/searchlib/src/vespa/searchlib/query/tree/querybuilder.h b/searchlib/src/vespa/searchlib/query/tree/querybuilder.h
index 797defc39f5..8e6f2944ec9 100644
--- a/searchlib/src/vespa/searchlib/query/tree/querybuilder.h
+++ b/searchlib/src/vespa/searchlib/query/tree/querybuilder.h
@@ -205,8 +205,11 @@ createRegExpTerm(vespalib::stringref term, vespalib::stringref view, int32_t id,
 template <class NodeTypes>
 typename NodeTypes::NearestNeighborTerm *
 create_nearest_neighbor_term(vespalib::stringref query_tensor_name, vespalib::stringref field_name,
-                             int32_t id, Weight weight, uint32_t target_num_hits) {
-    return new typename NodeTypes::NearestNeighborTerm(query_tensor_name, field_name, id, weight, target_num_hits);
+                             int32_t id, Weight weight, uint32_t target_num_hits,
+                             bool allow_approximate, uint32_t explore_additional_hits)
+{
+    return new typename NodeTypes::NearestNeighborTerm(query_tensor_name, field_name, id, weight,
+                                                       target_num_hits, allow_approximate, explore_additional_hits);
 }
 
 template <class NodeTypes>
@@ -317,9 +320,10 @@ public:
         return addTerm(createRegExpTerm<NodeTypes>(term, view, id, weight));
     }
     typename NodeTypes::NearestNeighborTerm &add_nearest_neighbor_term(stringref query_tensor_name, stringref field_name,
-                                                                       int32_t id, Weight weight, uint32_t target_num_hits) {
+                                                                       int32_t id, Weight weight, uint32_t target_num_hits,
+                                                                       bool allow_approximate, uint32_t explore_additional_hits) {
         adjustWeight(weight);
-        return addTerm(create_nearest_neighbor_term<NodeTypes>(query_tensor_name, field_name, id, weight, target_num_hits));
+        return addTerm(create_nearest_neighbor_term<NodeTypes>(query_tensor_name, field_name, id, weight, target_num_hits, allow_approximate, explore_additional_hits));
     }
 };
 
diff --git a/searchlib/src/vespa/searchlib/query/tree/queryreplicator.h b/searchlib/src/vespa/searchlib/query/tree/queryreplicator.h
index 0bf923960b9..9289df7cbe9 100644
--- a/searchlib/src/vespa/searchlib/query/tree/queryreplicator.h
+++ b/searchlib/src/vespa/searchlib/query/tree/queryreplicator.h
@@ -165,7 +165,8 @@ private:
 
     void visit(NearestNeighborTerm &node) override {
         replicate(node, _builder.add_nearest_neighbor_term(node.get_query_tensor_name(), node.getView(),
-                                                           node.getId(), node.getWeight(), node.get_target_num_hits()));
+                                                           node.getId(), node.getWeight(), node.get_target_num_hits(),
+                                                           node.get_allow_approximate(), node.get_explore_additional_hits()));
     }
 };
 
diff --git a/searchlib/src/vespa/searchlib/query/tree/simplequery.h b/searchlib/src/vespa/searchlib/query/tree/simplequery.h
index 8663bede4d6..4953f1a5b7c 100644
--- a/searchlib/src/vespa/searchlib/query/tree/simplequery.h
+++ b/searchlib/src/vespa/searchlib/query/tree/simplequery.h
@@ -105,8 +105,10 @@ struct SimpleRegExpTerm : RegExpTerm {
 };
 struct SimpleNearestNeighborTerm : NearestNeighborTerm {
     SimpleNearestNeighborTerm(vespalib::stringref query_tensor_name, vespalib::stringref field_name,
-                              int32_t id, Weight weight, uint32_t target_num_hits)
-        : NearestNeighborTerm(query_tensor_name, field_name, id, weight, target_num_hits)
+                              int32_t id, Weight weight, uint32_t target_num_hits,
+                              bool allow_approximate, uint32_t explore_additional_hits)
+        : NearestNeighborTerm(query_tensor_name, field_name, id, weight,
+                              target_num_hits, allow_approximate, explore_additional_hits)
     {}
 };
 
diff --git a/searchlib/src/vespa/searchlib/query/tree/stackdumpcreator.cpp b/searchlib/src/vespa/searchlib/query/tree/stackdumpcreator.cpp
index 63acf532144..aafeaa46a22 100644
--- a/searchlib/src/vespa/searchlib/query/tree/stackdumpcreator.cpp
+++ b/searchlib/src/vespa/searchlib/query/tree/stackdumpcreator.cpp
@@ -263,6 +263,8 @@ class QueryNodeConverter : public QueryVisitor {
         createTermNode(node, ParseItem::ITEM_NEAREST_NEIGHBOR);
         appendString(node.get_query_tensor_name());
         appendCompressedPositiveNumber(node.get_target_num_hits());
+        appendCompressedPositiveNumber(node.get_allow_approximate() ? 1 : 0);
+        appendCompressedPositiveNumber(node.get_explore_additional_hits());
     }
 
 public:
diff --git a/searchlib/src/vespa/searchlib/query/tree/stackdumpquerycreator.h b/searchlib/src/vespa/searchlib/query/tree/stackdumpquerycreator.h
index 791da010720..65d6abeeaad 100644
--- a/searchlib/src/vespa/searchlib/query/tree/stackdumpquerycreator.h
+++ b/searchlib/src/vespa/searchlib/query/tree/stackdumpquerycreator.h
@@ -48,9 +48,6 @@ public:
 private: 
     static Term * createQueryTerm(search::SimpleQueryStackDumpIterator &queryStack, QueryBuilder<NodeTypes> & builder, vespalib::stringref & pureTermView) {
         uint32_t arity = queryStack.getArity();
-        uint32_t arg1 = queryStack.getArg1();
-        double arg2 = queryStack.getArg2();
-        double arg3 = queryStack.getArg3();
         ParseItem::ItemType type = queryStack.getType();
         Node::UP node;
         Term *t = 0;
@@ -68,16 +65,19 @@ private:
             pureTermView = view;
         } else if (type == ParseItem::ITEM_WEAK_AND) {
             vespalib::stringref view = queryStack.getIndexName();
-            builder.addWeakAnd(arity, arg1, view);
+            uint32_t targetNumHits = queryStack.getTargetNumHits();
+            builder.addWeakAnd(arity, targetNumHits, view);
             pureTermView = view;
         } else if (type == ParseItem::ITEM_EQUIV) {
             int32_t id = queryStack.getUniqueId();
             Weight weight = queryStack.GetWeight();
             builder.addEquiv(arity, id, weight);
         } else if (type == ParseItem::ITEM_NEAR) {
-            builder.addNear(arity, arg1);
+            uint32_t nearDistance = queryStack.getNearDistance();
+            builder.addNear(arity, nearDistance);
         } else if (type == ParseItem::ITEM_ONEAR) {
-            builder.addONear(arity, arg1);
+            uint32_t nearDistance = queryStack.getNearDistance();
+            builder.addONear(arity, nearDistance);
         } else if (type == ParseItem::ITEM_PHRASE) {
             vespalib::stringref view = queryStack.getIndexName();
             int32_t id = queryStack.getUniqueId();
@@ -104,17 +104,24 @@ private:
             vespalib::stringref view = queryStack.getIndexName();
             int32_t id = queryStack.getUniqueId();
             Weight weight = queryStack.GetWeight();
-            t = &builder.addWandTerm(arity, view, id, weight, arg1, arg2, arg3);
+            uint32_t targetNumHits = queryStack.getTargetNumHits();
+            double scoreThreshold = queryStack.getScoreThreshold();
+            double thresholdBoostFactor = queryStack.getThresholdBoostFactor();
+            t = &builder.addWandTerm(arity, view, id, weight,
+                                     targetNumHits, scoreThreshold, thresholdBoostFactor);
             pureTermView = vespalib::stringref();
         } else if (type == ParseItem::ITEM_NOT) {
             builder.addAndNot(arity);
         } else if (type == ParseItem::ITEM_NEAREST_NEIGHBOR) {
             vespalib::stringref query_tensor_name = queryStack.getTerm();
             vespalib::stringref field_name = queryStack.getIndexName();
-            uint32_t target_num_hits = queryStack.getArg1();
+            uint32_t target_num_hits = queryStack.getTargetNumHits();
             int32_t id = queryStack.getUniqueId();
             Weight weight = queryStack.GetWeight();
-            builder.add_nearest_neighbor_term(query_tensor_name, field_name, id, weight, target_num_hits);
+            bool allow_approximate = queryStack.getAllowApproximate();
+            uint32_t explore_additional_hits = queryStack.getExploreAdditionalHits();
+            builder.add_nearest_neighbor_term(query_tensor_name, field_name, id, weight,
+                                              target_num_hits, allow_approximate, explore_additional_hits);
         } else {
             vespalib::stringref term = queryStack.getTerm();
             vespalib::stringref view = queryStack.getIndexName();
diff --git a/searchlib/src/vespa/searchlib/query/tree/termnodes.h b/searchlib/src/vespa/searchlib/query/tree/termnodes.h
index a82b1e14d76..9af424716fb 100644
--- a/searchlib/src/vespa/searchlib/query/tree/termnodes.h
+++ b/searchlib/src/vespa/searchlib/query/tree/termnodes.h
@@ -128,17 +128,24 @@ class NearestNeighborTerm : public QueryNodeMixin<NearestNeighborTerm, TermNode>
 private:
     vespalib::string _query_tensor_name;
     uint32_t _target_num_hits;
+    bool _allow_approximate;
+    uint32_t _explore_additional_hits;
 
 public:
     NearestNeighborTerm(vespalib::stringref query_tensor_name, vespalib::stringref field_name,
-                        int32_t id, Weight weight, uint32_t target_num_hits)
+                        int32_t id, Weight weight, uint32_t target_num_hits,
+                        bool allow_approximate, uint32_t explore_additional_hits)
         : QueryNodeMixinType(field_name, id, weight),
           _query_tensor_name(query_tensor_name),
-          _target_num_hits(target_num_hits)
+          _target_num_hits(target_num_hits),
+          _allow_approximate(allow_approximate),
+          _explore_additional_hits(explore_additional_hits)
     {}
     virtual ~NearestNeighborTerm() {}
     const vespalib::string& get_query_tensor_name() const { return _query_tensor_name; }
     uint32_t get_target_num_hits() const { return _target_num_hits; }
+    bool get_allow_approximate() const { return _allow_approximate; }
+    uint32_t get_explore_additional_hits() const { return _explore_additional_hits; }
 };
 
 
diff --git a/searchlib/src/vespa/searchlib/queryeval/CMakeLists.txt b/searchlib/src/vespa/searchlib/queryeval/CMakeLists.txt
index de2919443ff..0dcb0393473 100644
--- a/searchlib/src/vespa/searchlib/queryeval/CMakeLists.txt
+++ b/searchlib/src/vespa/searchlib/queryeval/CMakeLists.txt
@@ -32,6 +32,7 @@ vespa_add_library(searchlib_queryeval OBJECT
     nearest_neighbor_blueprint.cpp
     nearest_neighbor_iterator.cpp
     nearsearch.cpp
+    nns_index_iterator.cpp
     orsearch.cpp
     predicate_blueprint.cpp
     predicate_search.cpp
diff --git a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.cpp b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.cpp
index 8be6263221a..4035da5f435 100644
--- a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.cpp
+++ b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.cpp
@@ -3,35 +3,117 @@
 #include "emptysearch.h"
 #include "nearest_neighbor_blueprint.h"
 #include "nearest_neighbor_iterator.h"
+#include "nns_index_iterator.h"
 #include <vespa/searchlib/fef/termfieldmatchdataarray.h>
 #include <vespa/eval/tensor/dense/dense_tensor_view.h>
+#include <vespa/eval/tensor/dense/dense_tensor.h>
 #include <vespa/searchlib/tensor/dense_tensor_attribute.h>
+#include <vespa/searchlib/tensor/distance_function_factory.h>
+
+using vespalib::tensor::DenseTensorView;
+using vespalib::tensor::DenseTensor;
 
 namespace search::queryeval {
 
+namespace {
+
+template<typename LCT, typename RCT>
+void
+convert_cells(std::unique_ptr<DenseTensorView> &original, vespalib::eval::ValueType want_type)
+{
+    auto old_cells = original->cellsRef().typify<LCT>();
+    std::vector<RCT> new_cells;
+    new_cells.reserve(old_cells.size());
+    for (LCT value : old_cells) {
+        RCT conv = value;
+        new_cells.push_back(conv);
+    }
+    original = std::make_unique<DenseTensor<RCT>>(want_type, std::move(new_cells));
+}
+
+template<>
+void
+convert_cells<float,float>(std::unique_ptr<DenseTensorView> &, vespalib::eval::ValueType) {}
+
+template<>
+void
+convert_cells<double,double>(std::unique_ptr<DenseTensorView> &, vespalib::eval::ValueType) {}
+
+struct ConvertCellsSelector
+{
+    template <typename LCT, typename RCT>
+    static auto get_fun() { return convert_cells<LCT, RCT>; }
+};
+
+} // namespace <unnamed>
+
 NearestNeighborBlueprint::NearestNeighborBlueprint(const queryeval::FieldSpec& field,
                                                    const tensor::DenseTensorAttribute& attr_tensor,
                                                    std::unique_ptr<vespalib::tensor::DenseTensorView> query_tensor,
-                                                   uint32_t target_num_hits)
+                                                   uint32_t target_num_hits, bool approximate, uint32_t explore_additional_hits)
     : ComplexLeafBlueprint(field),
       _attr_tensor(attr_tensor),
       _query_tensor(std::move(query_tensor)),
       _target_num_hits(target_num_hits),
-      _distance_heap(target_num_hits)
+      _approximate(approximate),
+      _explore_additional_hits(explore_additional_hits),
+      _fallback_dist_fun(),
+      _distance_heap(target_num_hits),
+      _found_hits()
 {
-    setEstimate(HitEstimate(_attr_tensor.getNumDocs(), false));
+    auto lct = _query_tensor->cellsRef().type;
+    auto rct = _attr_tensor.getTensorType().cell_type();
+    auto fixup_fun = vespalib::tensor::select_2<ConvertCellsSelector>(lct, rct);
+    fixup_fun(_query_tensor, _attr_tensor.getTensorType());
+    auto def_dm = search::attribute::DistanceMetric::Euclidean;
+    _fallback_dist_fun = search::tensor::make_distance_function(def_dm, rct);
+    _dist_fun = _fallback_dist_fun.get();
+    auto nns_index = _attr_tensor.nearest_neighbor_index();
+    if (nns_index) {
+        _dist_fun = nns_index->distance_function();
+    }
+    uint32_t est_hits = _attr_tensor.getNumDocs();
+    if (_approximate && nns_index) {
+        est_hits = std::min(target_num_hits, est_hits);
+    }
+    setEstimate(HitEstimate(est_hits, false));
 }
 
 NearestNeighborBlueprint::~NearestNeighborBlueprint() = default;
 
+void
+NearestNeighborBlueprint::perform_top_k()
+{
+    auto nns_index = _attr_tensor.nearest_neighbor_index();
+    if (_approximate && nns_index) {
+        auto lhs_type = _query_tensor->fast_type();
+        auto rhs_type = _attr_tensor.getTensorType();
+        // XXX deal with different cell types later
+        if (lhs_type == rhs_type) {
+            auto lhs = _query_tensor->cellsRef();
+            uint32_t k = _target_num_hits;
+            _found_hits = nns_index->find_top_k(k, lhs, k + _explore_additional_hits);
+        }
+    }
+}
+
+void
+NearestNeighborBlueprint::fetchPostings(const ExecuteInfo &execInfo) {
+    if (execInfo.isStrict()) {
+        perform_top_k();
+    }
+}
+
 std::unique_ptr<SearchIterator>
 NearestNeighborBlueprint::createLeafSearch(const search::fef::TermFieldMatchDataArray& tfmda, bool strict) const
 {
     assert(tfmda.size() == 1);
     fef::TermFieldMatchData &tfmd = *tfmda[0]; // always search in only one field
+    if (strict && ! _found_hits.empty()) {
+        return NnsIndexIterator::create(tfmd, _found_hits, _dist_fun);
+    }
     const vespalib::tensor::DenseTensorView &qT = *_query_tensor;
-
-    return NearestNeighborIterator::create(strict, tfmd, qT, _attr_tensor, _distance_heap);
+    return NearestNeighborIterator::create(strict, tfmd, qT, _attr_tensor, _distance_heap, _dist_fun);
 }
 
 void
@@ -41,6 +123,8 @@ NearestNeighborBlueprint::visitMembers(vespalib::ObjectVisitor& visitor) const
     visitor.visitString("attribute_tensor", _attr_tensor.getTensorType().to_spec());
     visitor.visitString("query_tensor", _query_tensor->type().to_spec());
     visitor.visitInt("target_num_hits", _target_num_hits);
+    visitor.visitBool("approximate", _approximate);
+    visitor.visitInt("explore_additional_hits", _explore_additional_hits);
 }
 
 bool
diff --git a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.h b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.h
index 019f8e31842..7f8d4f7f020 100644
--- a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.h
+++ b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.h
@@ -3,6 +3,8 @@
 
 #include "blueprint.h"
 #include "nearest_neighbor_distance_heap.h"
+#include <vespa/searchlib/tensor/distance_function.h>
+#include <vespa/searchlib/tensor/nearest_neighbor_index.h>
 
 namespace vespalib::tensor { class DenseTensorView; }
 namespace search::tensor { class DenseTensorAttribute; }
@@ -20,13 +22,19 @@ private:
     const tensor::DenseTensorAttribute& _attr_tensor;
     std::unique_ptr<vespalib::tensor::DenseTensorView> _query_tensor;
     uint32_t _target_num_hits;
+    bool _approximate;
+    uint32_t _explore_additional_hits;
+    search::tensor::DistanceFunction::UP _fallback_dist_fun;
+    const search::tensor::DistanceFunction *_dist_fun;
     mutable NearestNeighborDistanceHeap _distance_heap;
+    std::vector<search::tensor::NearestNeighborIndex::Neighbor> _found_hits;
 
+    void perform_top_k();
 public:
     NearestNeighborBlueprint(const queryeval::FieldSpec& field,
                              const tensor::DenseTensorAttribute& attr_tensor,
                              std::unique_ptr<vespalib::tensor::DenseTensorView> query_tensor,
-                             uint32_t target_num_hits);
+                             uint32_t target_num_hits, bool approximate, uint32_t explore_additional_hits);
     NearestNeighborBlueprint(const NearestNeighborBlueprint&) = delete;
     NearestNeighborBlueprint& operator=(const NearestNeighborBlueprint&) = delete;
     ~NearestNeighborBlueprint();
@@ -38,6 +46,7 @@ public:
                                                      bool strict) const override;
     void visitMembers(vespalib::ObjectVisitor& visitor) const override;
     bool always_needs_unpack() const override;
+    void fetchPostings(const ExecuteInfo &execInfo) override;
 };
 
 }
diff --git a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_iterator.cpp b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_iterator.cpp
index 2b78fce3625..68c6a1603d0 100644
--- a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_iterator.cpp
+++ b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_iterator.cpp
@@ -18,7 +18,7 @@ bool
 is_compatible(const vespalib::eval::ValueType& lhs,
               const vespalib::eval::ValueType& rhs)
 {
-    return (lhs.dimensions() == rhs.dimensions());
+    return (lhs == rhs);
 }
 
 }
@@ -29,14 +29,14 @@ is_compatible(const vespalib::eval::ValueType& lhs,
  * Keeps a heap of the K best hit distances.
  * Currently always does brute-force scanning, which is very expensive.
  **/
-template <bool strict, typename LCT, typename RCT>
+template <bool strict>
 class NearestNeighborImpl : public NearestNeighborIterator
 {
 public:
 
     NearestNeighborImpl(Params params_in)
         : NearestNeighborIterator(params_in),
-          _lhs(params().queryTensor.cellsRef().template typify<LCT>()),
+          _lhs(params().queryTensor.cellsRef()),
           _fieldTensor(params().tensorAttribute.getTensorType()),
           _lastScore(0.0)
     {
@@ -64,70 +64,42 @@ public:
     }
 
     void doUnpack(uint32_t docId) override {
-        params().tfmd.setRawScore(docId, sqrt(_lastScore));
+        double score = params().distanceFunction->to_rawscore(_lastScore);
+        params().tfmd.setRawScore(docId, score);
         params().distanceHeap.used(_lastScore);
     }
 
     Trinary is_strict() const override { return strict ? Trinary::True : Trinary::False ; }
 
 private:
-    static double computeSum(ConstArrayRef<LCT> lhs, ConstArrayRef<RCT> rhs, double limit) {
-        double sum = 0.0;
-        size_t sz = lhs.size();
-        assert(sz == rhs.size());
-        for (size_t i = 0; i < sz && sum <= limit; ++i) {
-            double diff = lhs[i] - rhs[i];
-            sum += diff*diff;
-        }
-        return sum;
-    }
-
     double computeDistance(uint32_t docId, double limit) {
         params().tensorAttribute.getTensor(docId, _fieldTensor);
-        return computeSum(_lhs, _fieldTensor.cellsRef().template typify<RCT>(), limit);
+        auto rhs = _fieldTensor.cellsRef();
+        return params().distanceFunction->calc_with_limit(_lhs, rhs, limit);
     }
 
-    ConstArrayRef<LCT>     _lhs;
+    TypedCells             _lhs;
     MutableDenseTensorView _fieldTensor;
     double                 _lastScore;
 };
 
-template <bool strict, typename LCT, typename RCT>
-NearestNeighborImpl<strict, LCT, RCT>::~NearestNeighborImpl() = default;
+template <bool strict>
+NearestNeighborImpl<strict>::~NearestNeighborImpl() = default;
 
 namespace {
 
-template<bool strict, typename LCT, typename RCT>
-std::unique_ptr<NearestNeighborIterator>
-create_impl(const NearestNeighborIterator::Params &params)
-{
-    using NNI = NearestNeighborImpl<strict, LCT, RCT>;
-    return std::make_unique<NNI>(params);
-}
-
-using Creator = std::unique_ptr<NearestNeighborIterator>(*)(const NearestNeighborIterator::Params &params);
-
-template <bool strict>
-struct CellTypeResolver
-{
-    template <typename LCT, typename RCT>
-    static Creator
-    get_fun() { return create_impl<strict, LCT, RCT>; }
-};
-
 std::unique_ptr<NearestNeighborIterator>
 resolve_strict_LCT_RCT(bool strict, const NearestNeighborIterator::Params &params)
 {
     CellType lct = params.queryTensor.fast_type().cell_type();
     CellType rct = params.tensorAttribute.getTensorType().cell_type();
+    if (lct != rct) abort();
     if (strict) {
-        using Resolver = CellTypeResolver<true>;
-        auto fun = vespalib::tensor::select_2<Resolver>(lct, rct);
-        return fun(params);
+        using NNI = NearestNeighborImpl<true>;
+        return std::make_unique<NNI>(params);
     } else {
-        using Resolver = CellTypeResolver<false>;
-        auto fun = vespalib::tensor::select_2<Resolver>(lct, rct);
-        return fun(params);
+        using NNI = NearestNeighborImpl<false>;
+        return std::make_unique<NNI>(params);
     }
 }
 
@@ -139,9 +111,11 @@ NearestNeighborIterator::create(
         fef::TermFieldMatchData &tfmd,
         const vespalib::tensor::DenseTensorView &queryTensor,
         const search::tensor::DenseTensorAttribute &tensorAttribute,
-        NearestNeighborDistanceHeap &distanceHeap)
+        NearestNeighborDistanceHeap &distanceHeap,
+        const search::tensor::DistanceFunction *dist_fun)
+
 {
-    Params params(tfmd, queryTensor, tensorAttribute, distanceHeap);
+    Params params(tfmd, queryTensor, tensorAttribute, distanceHeap, dist_fun);
     return resolve_strict_LCT_RCT(strict, params);
 }
 
diff --git a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_iterator.h b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_iterator.h
index 34eb547fe39..2a800f96710 100644
--- a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_iterator.h
+++ b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_iterator.h
@@ -8,6 +8,7 @@
 #include <vespa/eval/tensor/dense/mutable_dense_tensor_view.h>
 #include <vespa/searchlib/fef/termfieldmatchdata.h>
 #include <vespa/searchlib/tensor/dense_tensor_attribute.h>
+#include <vespa/searchlib/tensor/distance_function.h>
 #include <vespa/vespalib/util/priority_queue.h>
 #include <cmath>
 
@@ -24,15 +25,18 @@ public:
         const DenseTensorView &queryTensor;
         const DenseTensorAttribute &tensorAttribute;
         NearestNeighborDistanceHeap &distanceHeap;
+        const search::tensor::DistanceFunction *distanceFunction;
         
         Params(fef::TermFieldMatchData &tfmd_in,
                const DenseTensorView &queryTensor_in,
                const DenseTensorAttribute &tensorAttribute_in,
-               NearestNeighborDistanceHeap &distanceHeap_in)
+               NearestNeighborDistanceHeap &distanceHeap_in,
+               const search::tensor::DistanceFunction *distanceFunction_in)
           : tfmd(tfmd_in),
             queryTensor(queryTensor_in),
             tensorAttribute(tensorAttribute_in),
-            distanceHeap(distanceHeap_in)
+            distanceHeap(distanceHeap_in),
+            distanceFunction(distanceFunction_in)
         {}
     };
 
@@ -45,7 +49,8 @@ public:
             fef::TermFieldMatchData &tfmd,
             const vespalib::tensor::DenseTensorView &queryTensor,
             const search::tensor::DenseTensorAttribute &tensorAttribute,
-            NearestNeighborDistanceHeap &distanceHeap);
+            NearestNeighborDistanceHeap &distanceHeap,
+            const search::tensor::DistanceFunction *dist_fun);
 
     const Params& params() const { return _params; }
 private:
diff --git a/searchlib/src/vespa/searchlib/queryeval/nns_index_iterator.cpp b/searchlib/src/vespa/searchlib/queryeval/nns_index_iterator.cpp
new file mode 100644
index 00000000000..50c562ebaad
--- /dev/null
+++ b/searchlib/src/vespa/searchlib/queryeval/nns_index_iterator.cpp
@@ -0,0 +1,73 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "nns_index_iterator.h"
+#include <vespa/searchlib/tensor/nearest_neighbor_index.h>
+#include <cmath>
+
+using Neighbor = search::tensor::NearestNeighborIndex::Neighbor;
+
+namespace search::queryeval {
+
+/**
+ * Search iterator for K nearest neighbor matching,
+ * where the actual search is done up front and this class
+ * just iterates over a vector held by the blueprint.
+ **/
+class NeighborVectorIterator : public NnsIndexIterator
+{
+private:
+    fef::TermFieldMatchData &_tfmd;
+    const std::vector<Neighbor> &_hits;
+    const search::tensor::DistanceFunction * const _dist_fun;
+    uint32_t _idx;
+    double _last_abstract_dist;
+public:
+    NeighborVectorIterator(fef::TermFieldMatchData &tfmd,
+                           const std::vector<Neighbor> &hits,
+                           const search::tensor::DistanceFunction *dist_fun)
+        : _tfmd(tfmd),
+          _hits(hits),
+          _dist_fun(dist_fun),
+          _idx(0),
+          _last_abstract_dist(0.0)
+    {}
+
+    void initRange(uint32_t begin_id, uint32_t end_id) override {
+        SearchIterator::initRange(begin_id, end_id);
+        _idx = 0;
+    }
+
+    void doSeek(uint32_t docId) override {
+        while (_idx < _hits.size()) {
+            uint32_t hit_id = _hits[_idx].docid;
+            if (hit_id < docId) {
+                ++_idx;
+            } else if (hit_id < getEndId()) {
+                setDocId(hit_id);
+                _last_abstract_dist = _hits[_idx].distance;
+                return;
+            } else {
+                _idx = _hits.size();
+            }
+        }
+        setAtEnd();
+    }
+
+    void doUnpack(uint32_t docId) override {
+        double score = _dist_fun->to_rawscore(_last_abstract_dist);
+        _tfmd.setRawScore(docId, score);
+    }
+
+    Trinary is_strict() const override { return Trinary::True; }
+};
+
+std::unique_ptr<NnsIndexIterator>
+NnsIndexIterator::create(
+        fef::TermFieldMatchData &tfmd,
+        const std::vector<Neighbor> &hits,
+        const search::tensor::DistanceFunction *dist_fun)
+{
+    return std::make_unique<NeighborVectorIterator>(tfmd, hits, dist_fun);
+}
+
+} // namespace
diff --git a/searchlib/src/vespa/searchlib/queryeval/nns_index_iterator.h b/searchlib/src/vespa/searchlib/queryeval/nns_index_iterator.h
new file mode 100644
index 00000000000..ce64d003b3a
--- /dev/null
+++ b/searchlib/src/vespa/searchlib/queryeval/nns_index_iterator.h
@@ -0,0 +1,22 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "searchiterator.h"
+#include <vespa/searchlib/fef/termfieldmatchdata.h>
+#include <vespa/searchlib/tensor/distance_function.h>
+#include <vespa/searchlib/tensor/nearest_neighbor_index.h>
+
+namespace search::queryeval {
+
+class NnsIndexIterator : public SearchIterator
+{
+public:
+    using Hit = search::tensor::NearestNeighborIndex::Neighbor;
+    static std::unique_ptr<NnsIndexIterator> create(
+            fef::TermFieldMatchData &tfmd,
+            const std::vector<Hit> &hits,
+            const search::tensor::DistanceFunction *dist_fun);
+};
+
+} // namespace
diff --git a/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt b/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt
index ea5d30d9a47..0f106f693f8 100644
--- a/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt
+++ b/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt
@@ -1,15 +1,24 @@
 # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 vespa_add_library(searchlib_tensor OBJECT
     SOURCES
+    default_nearest_neighbor_index_factory.cpp
     dense_tensor_attribute.cpp
     dense_tensor_attribute_saver.cpp
     dense_tensor_store.cpp
+    distance_function_factory.cpp
     generic_tensor_attribute.cpp
+    generic_tensor_attribute_saver.cpp
     generic_tensor_store.cpp
+    hnsw_graph.cpp
+    hnsw_index.cpp
+    hnsw_index_loader.cpp
+    hnsw_index_saver.cpp
     imported_tensor_attribute_vector.cpp
     imported_tensor_attribute_vector_read_guard.cpp
+    inv_log_level_generator.cpp
+    nearest_neighbor_index.cpp
+    nearest_neighbor_index_saver.cpp
     tensor_attribute.cpp
-    generic_tensor_attribute_saver.cpp
     tensor_store.cpp
     DEPENDS
 )
diff --git a/searchlib/src/vespa/searchlib/tensor/default_nearest_neighbor_index_factory.cpp b/searchlib/src/vespa/searchlib/tensor/default_nearest_neighbor_index_factory.cpp
new file mode 100644
index 00000000000..067280e9a23
--- /dev/null
+++ b/searchlib/src/vespa/searchlib/tensor/default_nearest_neighbor_index_factory.cpp
@@ -0,0 +1,47 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "default_nearest_neighbor_index_factory.h"
+#include "hnsw_index.h"
+#include "random_level_generator.h"
+#include "inv_log_level_generator.h"
+#include "distance_function_factory.h"
+#include <vespa/searchcommon/attribute/config.h>
+
+namespace search::tensor {
+
+using vespalib::eval::ValueType;
+
+namespace {
+
+class LevelZeroGenerator : public RandomLevelGenerator {
+    uint32_t max_level() override { return 0; }
+};
+
+RandomLevelGenerator::UP
+make_random_level_generator(uint32_t m)
+{
+    return std::make_unique<InvLogLevelGenerator>(m);
+}
+
+} // namespace <unnamed>
+
+std::unique_ptr<NearestNeighborIndex>
+DefaultNearestNeighborIndexFactory::make(const DocVectorAccess& vectors,
+                                         size_t vector_size,
+                                         vespalib::eval::ValueType::CellType cell_type,
+                                         const search::attribute::HnswIndexParams& params) const
+{
+    (void) vector_size;
+    uint32_t m = params.max_links_per_node();
+    HnswIndex::Config cfg(m * 2,
+                          m,
+                          params.neighbors_to_explore_at_insert(),
+                          true);
+    return std::make_unique<HnswIndex>(vectors,
+                                       make_distance_function(params.distance_metric(), cell_type),
+                                       make_random_level_generator(m),
+                                       cfg);
+}
+
+}
+
diff --git a/searchlib/src/vespa/searchlib/tensor/default_nearest_neighbor_index_factory.h b/searchlib/src/vespa/searchlib/tensor/default_nearest_neighbor_index_factory.h
new file mode 100644
index 00000000000..6a9ded92b60
--- /dev/null
+++ b/searchlib/src/vespa/searchlib/tensor/default_nearest_neighbor_index_factory.h
@@ -0,0 +1,20 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "nearest_neighbor_index_factory.h"
+
+namespace search::tensor {
+
+/**
+ * Factory that instantiates the production hnsw index.
+ */
+class DefaultNearestNeighborIndexFactory : public NearestNeighborIndexFactory {
+public:
+    std::unique_ptr<NearestNeighborIndex> make(const DocVectorAccess& vectors,
+                                               size_t vector_size,
+                                               vespalib::eval::ValueType::CellType cell_type,
+                                               const search::attribute::HnswIndexParams& params) const override;
+};
+
+}
diff --git a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp
index a2b9f136ed9..68ce0c1bb00 100644
--- a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp
+++ b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp
@@ -2,16 +2,22 @@
 
 #include "dense_tensor_attribute.h"
 #include "dense_tensor_attribute_saver.h"
+#include "nearest_neighbor_index.h"
+#include "nearest_neighbor_index_saver.h"
 #include "tensor_attribute.hpp"
-#include <vespa/eval/tensor/tensor.h>
 #include <vespa/eval/tensor/dense/mutable_dense_tensor_view.h>
+#include <vespa/eval/tensor/tensor.h>
 #include <vespa/fastlib/io/bufferedfile.h>
+#include <vespa/searchlib/attribute/load_utils.h>
 #include <vespa/searchlib/attribute/readerbase.h>
+#include <vespa/vespalib/data/slime/inserter.h>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".searchlib.tensor.dense_tensor_attribute");
 
+using search::attribute::LoadUtils;
 using vespalib::eval::ValueType;
+using vespalib::slime::ObjectInserter;
 using vespalib::tensor::MutableDenseTensorView;
 using vespalib::tensor::Tensor;
 
@@ -55,11 +61,37 @@ TensorReader::is_present() {
 
 }
 
-DenseTensorAttribute::DenseTensorAttribute(vespalib::stringref baseFileName,
-                                 const Config &cfg)
+void
+DenseTensorAttribute::consider_remove_from_index(DocId docid)
+{
+    if (_index && _refVector[docid].valid()) {
+        _index->remove_document(docid);
+    }
+}
+
+vespalib::MemoryUsage
+DenseTensorAttribute::memory_usage() const
+{
+    vespalib::MemoryUsage result = TensorAttribute::memory_usage();
+    if (_index) {
+        result.merge(_index->memory_usage());
+    }
+    return result;
+}
+
+DenseTensorAttribute::DenseTensorAttribute(vespalib::stringref baseFileName, const Config& cfg,
+                                           const NearestNeighborIndexFactory& index_factory)
     : TensorAttribute(baseFileName, cfg, _denseTensorStore),
-      _denseTensorStore(cfg.tensorType())
+      _denseTensorStore(cfg.tensorType()),
+      _index()
 {
+    if (cfg.hnsw_index_params().has_value()) {
+        auto tensor_type = cfg.tensorType();
+        assert(tensor_type.dimensions().size() == 1);
+        assert(tensor_type.is_dense());
+        size_t vector_size = tensor_type.dimensions()[0].size;
+        _index = index_factory.make(*this, vector_size, tensor_type.cell_type(), cfg.hnsw_index_params().value());
+    }
 }
 
 
@@ -69,12 +101,23 @@ DenseTensorAttribute::~DenseTensorAttribute()
     _tensorStore.clearHoldLists();
 }
 
+uint32_t
+DenseTensorAttribute::clearDoc(DocId docId)
+{
+    consider_remove_from_index(docId);
+    return TensorAttribute::clearDoc(docId);
+}
+
 void
 DenseTensorAttribute::setTensor(DocId docId, const Tensor &tensor)
 {
     checkTensorType(tensor);
+    consider_remove_from_index(docId);
     EntryRef ref = _denseTensorStore.setTensor(tensor);
     setTensorRef(docId, ref);
+    if (_index) {
+        _index->add_document(docId);
+    }
 }
 
 
@@ -108,6 +151,8 @@ DenseTensorAttribute::onLoad()
     if (!tensorReader.hasData()) {
         return false;
     }
+    bool has_index_file = LoadUtils::file_exists(*this, DenseTensorAttributeSaver::index_file_suffix());
+
     setCreateSerialNum(tensorReader.getCreateSerialNum());
     assert(tensorReader.getVersion() == DENSE_TENSOR_ATTRIBUTE_VERSION);
     assert(getConfig().tensorType().to_spec() ==
@@ -120,12 +165,23 @@ DenseTensorAttribute::onLoad()
             auto raw = _denseTensorStore.allocRawBuffer();
             tensorReader.readTensor(raw.data, _denseTensorStore.getBufSize());
             _refVector.push_back(raw.ref);
+            if (_index && !has_index_file) {
+                // This ensures that get_vector() (via getTensor()) is able to find the newly added tensor.
+                setCommittedDocIdLimit(lid + 1);
+                _index->add_document(lid);
+            }
         } else {
             _refVector.push_back(EntryRef());
         }
     }
     setNumDocs(numDocs);
     setCommittedDocIdLimit(numDocs);
+    if (_index && has_index_file) {
+        auto buffer = LoadUtils::loadFile(*this, DenseTensorAttributeSaver::index_file_suffix());
+        if (!_index->load(*buffer)) {
+            return false;
+        }
+    }
     return true;
 }
 
@@ -135,11 +191,13 @@ DenseTensorAttribute::onInitSave(vespalib::stringref fileName)
 {
     vespalib::GenerationHandler::Guard guard(getGenerationHandler().
                                              takeGuard());
+    auto index_saver = (_index ? _index->make_saver() : std::unique_ptr<NearestNeighborIndexSaver>());
     return std::make_unique<DenseTensorAttributeSaver>
         (std::move(guard),
          this->createAttributeHeader(fileName),
          getRefCopy(),
-         _denseTensorStore);
+         _denseTensorStore,
+         std::move(index_saver));
 }
 
 void
@@ -154,4 +212,43 @@ DenseTensorAttribute::getVersion() const
     return DENSE_TENSOR_ATTRIBUTE_VERSION;
 }
 
+void
+DenseTensorAttribute::onGenerationChange(generation_t next_gen)
+{
+    // TODO: Change onGenerationChange() to send current generation instead of next generation.
+    //       This applies for entire attribute vector code.
+    TensorAttribute::onGenerationChange(next_gen);
+    if (_index) {
+        _index->transfer_hold_lists(next_gen - 1);
+    }
+}
+
+void
+DenseTensorAttribute::removeOldGenerations(generation_t first_used_gen)
+{
+    TensorAttribute::removeOldGenerations(first_used_gen);
+    if (_index) {
+        _index->trim_hold_lists(first_used_gen);
+    }
+}
+
+void
+DenseTensorAttribute::get_state(const vespalib::slime::Inserter& inserter) const
+{
+    auto& object = inserter.insertObject();
+    populate_state(object);
+    if (_index) {
+        ObjectInserter index_inserter(object, "nearest_neighbor_index");
+        _index->get_state(index_inserter);
+    }
+}
+
+vespalib::tensor::TypedCells
+DenseTensorAttribute::get_vector(uint32_t docid) const
+{
+    assert(docid < _refVector.size());
+    EntryRef ref = _refVector[docid];
+    return _denseTensorStore.get_typed_cells(ref);
+}
+
 }
diff --git a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.h b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.h
index 593741cef39..f0383627ea2 100644
--- a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.h
+++ b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.h
@@ -2,35 +2,51 @@
 
 #pragma once
 
-#include "tensor_attribute.h"
+#include "default_nearest_neighbor_index_factory.h"
 #include "dense_tensor_store.h"
+#include "doc_vector_access.h"
+#include "tensor_attribute.h"
+#include <memory>
 
-namespace vespalib { namespace tensor { class MutableDenseTensorView; }}
+namespace vespalib::tensor { class MutableDenseTensorView; }
 
-namespace search {
+namespace search::tensor {
 
-namespace tensor {
+class NearestNeighborIndex;
 
 /**
  * Attribute vector class used to store dense tensors for all
  * documents in memory.
  */
-class DenseTensorAttribute : public TensorAttribute
-{
+class DenseTensorAttribute : public TensorAttribute, public DocVectorAccess {
+private:
     DenseTensorStore _denseTensorStore;
+    std::unique_ptr<NearestNeighborIndex> _index;
+
+    void consider_remove_from_index(DocId docid);
+    vespalib::MemoryUsage memory_usage() const override;
+
 public:
-    DenseTensorAttribute(vespalib::stringref baseFileName, const Config &cfg);
+    DenseTensorAttribute(vespalib::stringref baseFileName, const Config& cfg,
+                         const NearestNeighborIndexFactory& index_factory = DefaultNearestNeighborIndexFactory());
     virtual ~DenseTensorAttribute();
-    virtual void setTensor(DocId docId, const Tensor &tensor) override;
-    virtual std::unique_ptr<Tensor> getTensor(DocId docId) const override;
-    virtual void getTensor(DocId docId, vespalib::tensor::MutableDenseTensorView &tensor) const override;
-    virtual bool onLoad() override;
-    virtual std::unique_ptr<AttributeSaver> onInitSave(vespalib::stringref fileName) override;
-    virtual void compactWorst() override;
-    virtual uint32_t getVersion() const override;
+    // Implements AttributeVector and ITensorAttribute
+    uint32_t clearDoc(DocId docId) override;
+    void setTensor(DocId docId, const Tensor &tensor) override;
+    std::unique_ptr<Tensor> getTensor(DocId docId) const override;
+    void getTensor(DocId docId, vespalib::tensor::MutableDenseTensorView &tensor) const override;
+    bool onLoad() override;
+    std::unique_ptr<AttributeSaver> onInitSave(vespalib::stringref fileName) override;
+    void compactWorst() override;
+    uint32_t getVersion() const override;
+    void onGenerationChange(generation_t next_gen) override;
+    void removeOldGenerations(generation_t first_used_gen) override;
+    void get_state(const vespalib::slime::Inserter& inserter) const override;
+
+    // Implements DocVectorAccess
+    vespalib::tensor::TypedCells get_vector(uint32_t docid) const override;
+
+    const NearestNeighborIndex* nearest_neighbor_index() const { return _index.get(); }
 };
 
-
-}  // namespace search::tensor
-
-}  // namespace search
+}
diff --git a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute_saver.cpp b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute_saver.cpp
index d78adab81b5..fd8d6162f01 100644
--- a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute_saver.cpp
+++ b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute_saver.cpp
@@ -1,20 +1,19 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include "dense_tensor_attribute_saver.h"
-#include <vespa/vespalib/util/bufferwriter.h>
 #include "dense_tensor_store.h"
+#include "nearest_neighbor_index_saver.h"
+#include <vespa/vespalib/util/bufferwriter.h>
 #include <vespa/searchlib/attribute/iattributesavetarget.h>
 
 using vespalib::GenerationHandler;
 
-namespace search {
-
-namespace tensor {
+namespace search::tensor {
 
 namespace {
 
-static const uint8_t tensorIsNotPresent = 0;
-static const uint8_t tensorIsPresent = 1;
+constexpr uint8_t tensorIsNotPresent = 0;
+constexpr uint8_t tensorIsPresent = 1;
 
 }
 
@@ -22,42 +21,60 @@ DenseTensorAttributeSaver::
 DenseTensorAttributeSaver(GenerationHandler::Guard &&guard,
                           const attribute::AttributeHeader &header,
                           RefCopyVector &&refs,
-                          const DenseTensorStore &tensorStore)
+                          const DenseTensorStore &tensorStore,
+                          IndexSaverUP index_saver)
     : AttributeSaver(std::move(guard), header),
       _refs(std::move(refs)),
-      _tensorStore(tensorStore)
+      _tensorStore(tensorStore),
+      _index_saver(std::move(index_saver))
 {
 }
 
+DenseTensorAttributeSaver::~DenseTensorAttributeSaver() = default;
 
-DenseTensorAttributeSaver::~DenseTensorAttributeSaver()
+vespalib::string
+DenseTensorAttributeSaver::index_file_suffix()
 {
+    return "nnidx";
 }
 
-
 bool
 DenseTensorAttributeSaver::onSave(IAttributeSaveTarget &saveTarget)
 {
-    std::unique_ptr<BufferWriter>
-        datWriter(saveTarget.datWriter().allocBufferWriter());
+    if (_index_saver) {
+        if (!saveTarget.setup_writer(index_file_suffix(), "Binary data file for nearest neighbor index")) {
+            return false;
+        }
+    }
+
+    auto dat_writer = saveTarget.datWriter().allocBufferWriter();
+    save_tensor_store(*dat_writer);
+
+    if (_index_saver) {
+        auto index_writer = saveTarget.get_writer(index_file_suffix()).allocBufferWriter();
+        // Note: Implementation of save() is responsible to call BufferWriter::flush().
+        _index_saver->save(*index_writer);
+    }
+    return true;
+}
+
+void
+DenseTensorAttributeSaver::save_tensor_store(BufferWriter& writer) const
+{
     const uint32_t docIdLimit(_refs.size());
     const uint32_t cellSize = _tensorStore.getCellSize();
     for (uint32_t lid = 0; lid < docIdLimit; ++lid) {
         if (_refs[lid].valid()) {
             auto raw = _tensorStore.getRawBuffer(_refs[lid]);
-            datWriter->write(&tensorIsPresent, sizeof(tensorIsPresent));
+            writer.write(&tensorIsPresent, sizeof(tensorIsPresent));
             size_t numCells = _tensorStore.getNumCells();
             size_t rawLen = numCells * cellSize;
-            datWriter->write(static_cast<const char *>(raw), rawLen);
+            writer.write(static_cast<const char *>(raw), rawLen);
         } else {
-            datWriter->write(&tensorIsNotPresent, sizeof(tensorIsNotPresent));
+            writer.write(&tensorIsNotPresent, sizeof(tensorIsNotPresent));
         }
     }
-    datWriter->flush();
-    return true;
+    writer.flush();
 }
 
-
-}  // namespace search::tensor
-
-}  // namespace search
+}
diff --git a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute_saver.h b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute_saver.h
index 1f6596e82f5..895e2951cea 100644
--- a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute_saver.h
+++ b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute_saver.h
@@ -5,28 +5,41 @@
 #include "tensor_attribute.h"
 #include <vespa/searchlib/attribute/attributesaver.h>
 
+namespace search { class BufferWriter; }
+
 namespace search::tensor {
 
 class DenseTensorStore;
+class NearestNeighborIndexSaver;
 
-/*
- * Class for saving a tensor attribute.
+/**
+ * Class for saving a dense tensor attribute.
+ * Will also save the nearest neighbor index if existing.
  */
-class DenseTensorAttributeSaver : public AttributeSaver
-{
+class DenseTensorAttributeSaver : public AttributeSaver {
 public:
     using RefCopyVector = TensorAttribute::RefCopyVector;
 private:
+    using GenerationHandler = vespalib::GenerationHandler;
+    using IndexSaverUP = std::unique_ptr<NearestNeighborIndexSaver>;
+
     RefCopyVector      _refs;
     const DenseTensorStore &_tensorStore;
-    using GenerationHandler = vespalib::GenerationHandler;
+    IndexSaverUP _index_saver;
 
     bool onSave(IAttributeSaveTarget &saveTarget) override;
+    void save_tensor_store(BufferWriter& writer) const;
+
 public:
-    DenseTensorAttributeSaver(GenerationHandler::Guard &&guard, const attribute::AttributeHeader &header,
-                              RefCopyVector &&refs, const DenseTensorStore &tensorStore);
+    DenseTensorAttributeSaver(GenerationHandler::Guard &&guard,
+                              const attribute::AttributeHeader &header,
+                              RefCopyVector &&refs,
+                              const DenseTensorStore &tensorStore,
+                              IndexSaverUP index_saver);
 
     ~DenseTensorAttributeSaver() override;
+
+    static vespalib::string index_file_suffix();
 };
 
 }
diff --git a/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.cpp b/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.cpp
index dc459d7d246..b026382994b 100644
--- a/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.cpp
+++ b/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.cpp
@@ -153,6 +153,15 @@ DenseTensorStore::getTensor(EntryRef ref, MutableDenseTensorView &tensor) const
     }
 }
 
+vespalib::tensor::TypedCells
+DenseTensorStore::get_typed_cells(EntryRef ref) const
+{
+    if (!ref.valid()) {
+        return vespalib::tensor::TypedCells(&_emptySpace[0], _type.cell_type(), getNumCells());
+    }
+    return vespalib::tensor::TypedCells(getRawBuffer(ref), _type.cell_type(), getNumCells());
+}
+
 template <class TensorType>
 TensorStore::EntryRef
 DenseTensorStore::setDenseTensor(const TensorType &tensor)
diff --git a/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.h b/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.h
index bd52709e423..d5e4152f6d7 100644
--- a/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.h
+++ b/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.h
@@ -4,6 +4,7 @@
 
 #include "tensor_store.h"
 #include <vespa/eval/eval/value_type.h>
+#include <vespa/eval/tensor/dense/typed_cells.h>
 
 namespace vespalib { namespace tensor { class MutableDenseTensorView; }}
 
@@ -65,6 +66,7 @@ public:
     EntryRef move(EntryRef ref) override;
     std::unique_ptr<Tensor> getTensor(EntryRef ref) const;
     void getTensor(EntryRef ref, vespalib::tensor::MutableDenseTensorView &tensor) const;
+    vespalib::tensor::TypedCells get_typed_cells(EntryRef ref) const;
     EntryRef setTensor(const Tensor &tensor);
     // The following method is meant to be used only for unit tests.
     uint32_t getArraySize() const { return _bufferType.getArraySize(); }
diff --git a/searchlib/src/vespa/searchlib/tensor/distance_function.h b/searchlib/src/vespa/searchlib/tensor/distance_function.h
new file mode 100644
index 00000000000..7665ca09f5a
--- /dev/null
+++ b/searchlib/src/vespa/searchlib/tensor/distance_function.h
@@ -0,0 +1,28 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <memory>
+
+namespace vespalib::tensor { struct TypedCells; }
+
+namespace search::tensor {
+
+/**
+ * Interface used to calculate the distance between two n-dimensional vectors.
+ *
+ * The vectors must be of same size and same type (float or double).
+ * The actual implementation must know which type the vectors are.
+ */
+class DistanceFunction {
+public:
+    using UP = std::unique_ptr<DistanceFunction>;
+    virtual ~DistanceFunction() {}
+    virtual double calc(const vespalib::tensor::TypedCells& lhs, const vespalib::tensor::TypedCells& rhs) const = 0;
+    virtual double to_rawscore(double distance) const = 0;
+    virtual double calc_with_limit(const vespalib::tensor::TypedCells& lhs,
+                                   const vespalib::tensor::TypedCells& rhs,
+                                   double limit) const = 0;
+};
+
+}
diff --git a/searchlib/src/vespa/searchlib/tensor/distance_function_factory.cpp b/searchlib/src/vespa/searchlib/tensor/distance_function_factory.cpp
new file mode 100644
index 00000000000..6b24a062727
--- /dev/null
+++ b/searchlib/src/vespa/searchlib/tensor/distance_function_factory.cpp
@@ -0,0 +1,41 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "distance_function_factory.h"
+#include "distance_functions.h"
+
+using search::attribute::DistanceMetric;
+using vespalib::eval::ValueType;
+
+namespace search::tensor {
+
+DistanceFunction::UP
+make_distance_function(DistanceMetric variant, ValueType::CellType cell_type)
+{
+    switch (variant) {
+        case DistanceMetric::Euclidean:
+            if (cell_type == ValueType::CellType::FLOAT) {
+                return std::make_unique<SquaredEuclideanDistance<float>>();
+            } else {
+                return std::make_unique<SquaredEuclideanDistance<double>>();
+            }
+            break;
+        case DistanceMetric::Angular:
+            if (cell_type == ValueType::CellType::FLOAT) {
+                return std::make_unique<AngularDistance<float>>();
+            } else {
+                return std::make_unique<AngularDistance<double>>();
+            }
+            break;
+        case DistanceMetric::GeoDegrees:
+            if (cell_type == ValueType::CellType::FLOAT) {
+                return std::make_unique<GeoDegreesDistance<float>>();
+            } else {
+                return std::make_unique<GeoDegreesDistance<double>>();
+            }
+            break;
+    }
+    // not reached:
+    return DistanceFunction::UP();
+}
+
+}
diff --git a/searchlib/src/vespa/searchlib/tensor/distance_function_factory.h b/searchlib/src/vespa/searchlib/tensor/distance_function_factory.h
new file mode 100644
index 00000000000..c86e40279bc
--- /dev/null
+++ b/searchlib/src/vespa/searchlib/tensor/distance_function_factory.h
@@ -0,0 +1,19 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "distance_function.h"
+#include <vespa/eval/eval/value_type.h>
+#include <vespa/searchcommon/attribute/distance_metric.h>
+
+namespace search::tensor {
+
+/**
+ * Create a distance function object customized for the given metric
+ * variant and cell type.
+ **/
+DistanceFunction::UP
+make_distance_function(search::attribute::DistanceMetric variant,
+                       vespalib::eval::ValueType::CellType cell_type);
+
+}
diff --git a/searchlib/src/vespa/searchlib/tensor/distance_functions.h b/searchlib/src/vespa/searchlib/tensor/distance_functions.h
new file mode 100644
index 00000000000..79f987c740c
--- /dev/null
+++ b/searchlib/src/vespa/searchlib/tensor/distance_functions.h
@@ -0,0 +1,145 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "distance_function.h"
+#include <vespa/eval/tensor/dense/typed_cells.h>
+#include <vespa/vespalib/hwaccelrated/iaccelrated.h>
+#include <cmath>
+
+namespace search::tensor {
+
+/**
+ * Calculates the square of the standard Euclidean distance.
+ * Will use instruction optimal for the cpu it is running on.
+ */
+template <typename FloatType>
+class SquaredEuclideanDistance : public DistanceFunction {
+public:
+    SquaredEuclideanDistance()
+        : _computer(vespalib::hwaccelrated::IAccelrated::getAccelrator())
+    {}
+    double calc(const vespalib::tensor::TypedCells& lhs, const vespalib::tensor::TypedCells& rhs) const override {
+        auto lhs_vector = lhs.typify<FloatType>();
+        auto rhs_vector = rhs.typify<FloatType>();
+        size_t sz = lhs_vector.size();
+        assert(sz == rhs_vector.size());
+        return _computer.squaredEuclideanDistance(&lhs_vector[0], &rhs_vector[0], sz);
+    }
+    double to_rawscore(double distance) const override {
+        double d = sqrt(distance);
+        double score = 1.0 / (1.0 + d);
+        return score;
+    }
+    double calc_with_limit(const vespalib::tensor::TypedCells& lhs,
+                           const vespalib::tensor::TypedCells& rhs,
+                           double limit) const override
+    {
+        auto lhs_vector = lhs.typify<FloatType>();
+        auto rhs_vector = rhs.typify<FloatType>();
+        double sum = 0.0;
+        size_t sz = lhs_vector.size();
+        assert(sz == rhs_vector.size());
+        for (size_t i = 0; i < sz && sum <= limit; ++i) {
+            double diff = lhs_vector[i] - rhs_vector[i];
+            sum += diff*diff;
+        }
+        return sum;
+    }
+
+    const vespalib::hwaccelrated::IAccelrated & _computer;
+};
+
+template class SquaredEuclideanDistance<float>;
+template class SquaredEuclideanDistance<double>;
+
+/**
+ * Calculates angular distance between vectors with assumed norm 1.
+ */
+template <typename FloatType>
+class AngularDistance : public DistanceFunction {
+public:
+    AngularDistance()
+        : _computer(vespalib::hwaccelrated::IAccelrated::getAccelrator())
+    {}
+    double calc(const vespalib::tensor::TypedCells& lhs, const vespalib::tensor::TypedCells& rhs) const override {
+        auto lhs_vector = lhs.typify<FloatType>();
+        auto rhs_vector = rhs.typify<FloatType>();
+        size_t sz = lhs_vector.size();
+        assert(sz == rhs_vector.size());
+        double score = 1.0 - _computer.dotProduct(&lhs_vector[0], &rhs_vector[0], sz);
+        return std::max(0.0, score);
+    }
+    double to_rawscore(double distance) const override {
+        double score = 1.0 / (1.0 + distance);
+        return score;
+    }
+    double calc_with_limit(const vespalib::tensor::TypedCells& lhs,
+                           const vespalib::tensor::TypedCells& rhs,
+                           double /*limit*/) const override
+    {
+        return calc(lhs, rhs);
+    }
+
+    const vespalib::hwaccelrated::IAccelrated & _computer;
+};
+
+template class AngularDistance<float>;
+template class AngularDistance<double>;
+
+/**
+ * Calculates great-circle distance between Latitude/Longitude pairs,
+ * measured in degrees.  Output distance is measured in meters.
+ * Uses the haversine formula directly from:
+ * https://en.wikipedia.org/wiki/Haversine_formula
+ **/
+template <typename FloatType>
+class GeoDegreesDistance : public DistanceFunction {
+public:
+    GeoDegreesDistance() {}
+    // haversine function:
+    static double hav(double angle) {
+        double s = sin(0.5*angle);
+        return s*s;
+    }
+    double calc(const vespalib::tensor::TypedCells& lhs, const vespalib::tensor::TypedCells& rhs) const override {
+        auto lhs_vector = lhs.typify<FloatType>();
+        auto rhs_vector = rhs.typify<FloatType>();
+        assert(2 == lhs_vector.size());
+        assert(2 == rhs_vector.size());
+        // convert to radians:
+        double lat_A = lhs_vector[0] * M_PI / 180.0;
+        double lat_B = rhs_vector[0] * M_PI / 180.0;
+        double lon_A = lhs_vector[1] * M_PI / 180.0;
+        double lon_B = rhs_vector[1] * M_PI / 180.0;
+
+        double lat_diff = lat_A - lat_B;
+        double lon_diff = lon_A - lon_B;
+
+        // haversines of differences:
+        double hav_lat = hav(lat_diff);
+        double hav_lon = hav(lon_diff);
+
+        // haversine of central angle between the two points:
+        double hav_central_angle = hav_lat + cos(lat_A)*cos(lat_B)*hav_lon;
+        return hav_central_angle;
+    }
+    double to_rawscore(double distance) const override {
+        double hav_diff = sqrt(distance);
+        // distance in meters:
+        double d = 2 * asin(hav_diff) * 6371008.8; // Earth mean radius
+        return 1.0 / (1.0 + d);
+    }
+    double calc_with_limit(const vespalib::tensor::TypedCells& lhs,
+                           const vespalib::tensor::TypedCells& rhs,
+                           double /*limit*/) const override
+    {
+        return calc(lhs, rhs);
+    }
+
+};
+
+template class GeoDegreesDistance<float>;
+template class GeoDegreesDistance<double>;
+
+}
diff --git a/searchlib/src/vespa/searchlib/tensor/doc_vector_access.h b/searchlib/src/vespa/searchlib/tensor/doc_vector_access.h
new file mode 100644
index 00000000000..e5dfa35a529
--- /dev/null
+++ b/searchlib/src/vespa/searchlib/tensor/doc_vector_access.h
@@ -0,0 +1,21 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/eval/tensor/dense/typed_cells.h>
+#include <cstdint>
+
+namespace search::tensor {
+
+/**
+ * Interface that provides access to the vector that is associated with the the given document id.
+ *
+ * All vectors should be the same size and either of type float or double.
+ */
+class DocVectorAccess {
+public:
+    virtual ~DocVectorAccess() {}
+    virtual vespalib::tensor::TypedCells get_vector(uint32_t docid) const = 0;
+};
+
+}
diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_graph.cpp b/searchlib/src/vespa/searchlib/tensor/hnsw_graph.cpp
new file mode 100644
index 00000000000..10113160263
--- /dev/null
+++ b/searchlib/src/vespa/searchlib/tensor/hnsw_graph.cpp
@@ -0,0 +1,89 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "hnsw_graph.h"
+#include "hnsw_index.h"
+#include <vespa/vespalib/datastore/array_store.hpp>
+#include <vespa/vespalib/util/rcuvector.hpp>
+
+namespace search::tensor {
+
+HnswGraph::HnswGraph()
+  : node_refs(),
+    nodes(HnswIndex::make_default_node_store_config()),
+    links(HnswIndex::make_default_link_store_config()),
+    entry_docid(0), // Note that docid 0 is reserved and never used
+    entry_level(-1)
+{}
+
+HnswGraph::~HnswGraph() {}
+
+void
+HnswGraph::make_node_for_document(uint32_t docid, uint32_t num_levels)
+{
+    node_refs.ensure_size(docid + 1, AtomicEntryRef());
+    // A document cannot be added twice.
+    assert(!node_refs[docid].load_acquire().valid());
+    // Note: The level array instance lives as long as the document is present in the index.
+    vespalib::Array<AtomicEntryRef> levels(num_levels, AtomicEntryRef());
+    auto node_ref = nodes.add(levels);
+    node_refs[docid].store_release(node_ref);
+}
+
+void
+HnswGraph::remove_node_for_document(uint32_t docid)
+{
+    auto node_ref = node_refs[docid].load_acquire();
+    nodes.remove(node_ref);
+    search::datastore::EntryRef invalid;
+    node_refs[docid].store_release(invalid);
+}
+
+void     
+HnswGraph::set_link_array(uint32_t docid, uint32_t level, const LinkArrayRef& new_links)
+{
+    auto new_links_ref = links.add(new_links);
+    auto node_ref = node_refs[docid].load_acquire();
+    assert(node_ref.valid());
+    auto levels = nodes.get_writable(node_ref);
+    auto old_links_ref = levels[level].load_acquire();
+    levels[level].store_release(new_links_ref);
+    links.remove(old_links_ref);
+}
+
+HnswGraph::Histograms
+HnswGraph::histograms() const
+{
+    Histograms result;
+    size_t num_nodes = node_refs.size();
+    for (size_t i = 0; i < num_nodes; ++i) {
+        auto node_ref = node_refs[i].load_acquire();
+        if (node_ref.valid()) {
+            uint32_t levels = 0;
+            uint32_t l0links = 0;
+            auto level_array = nodes.get(node_ref);
+            levels = level_array.size();
+            if (levels > 0) {
+                auto links_ref = level_array[0].load_acquire();
+                auto link_array = links.get(links_ref);
+                l0links = link_array.size();
+            }
+            while (result.level_histogram.size() <= levels) {
+                result.level_histogram.push_back(0);
+            }
+            ++result.level_histogram[levels];
+            while (result.links_histogram.size() <= l0links) {
+                result.links_histogram.push_back(0);
+            }
+            ++result.links_histogram[l0links];
+        }
+    }
+    return result;
+}
+
+} // namespace
+
+namespace vespalib {
+
+template class RcuVectorBase<search::datastore::AtomicEntryRef>;
+
+}
diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_graph.h b/searchlib/src/vespa/searchlib/tensor/hnsw_graph.h
new file mode 100644
index 00000000000..f38f5ca0baf
--- /dev/null
+++ b/searchlib/src/vespa/searchlib/tensor/hnsw_graph.h
@@ -0,0 +1,80 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/vespalib/datastore/array_store.h>
+#include <vespa/vespalib/datastore/atomic_entry_ref.h>
+#include <vespa/vespalib/datastore/entryref.h>
+#include <vespa/vespalib/util/rcuvector.h>
+
+namespace search::tensor {
+
+/**
+ * Stroage of a hierarchical navigable small world graph (HNSW)
+ * that is used for approximate K-nearest neighbor search.
+ */
+struct HnswGraph {
+    using AtomicEntryRef = search::datastore::AtomicEntryRef;
+
+    // This uses 10 bits for buffer id -> 1024 buffers.
+    // As we have very short arrays we get less fragmentation with fewer and larger buffers.
+    using EntryRefType = search::datastore::EntryRefT<22>;
+
+    // Provides mapping from document id -> node reference.
+    // The reference is used to lookup the node data in NodeStore.
+    using NodeRefVector = vespalib::RcuVector<AtomicEntryRef>;
+
+    // This stores the level arrays for all nodes.
+    // Each node consists of an array of levels (from level 0 to n) where each entry is a reference to the link array at that level.
+    using NodeStore = search::datastore::ArrayStore<AtomicEntryRef, EntryRefType>;
+    using StoreConfig = search::datastore::ArrayStoreConfig;
+    using LevelArrayRef = NodeStore::ConstArrayRef;
+
+    // This stores all link arrays.
+    // A link array consists of the document ids of the nodes a particular node is linked to.
+    using LinkStore = search::datastore::ArrayStore<uint32_t, EntryRefType>;
+    using LinkArrayRef = LinkStore::ConstArrayRef;
+
+    NodeRefVector node_refs;
+    NodeStore     nodes;
+    LinkStore     links;
+    uint32_t      entry_docid;
+    int32_t       entry_level;
+
+    HnswGraph();
+
+    ~HnswGraph();
+
+    void make_node_for_document(uint32_t docid, uint32_t num_levels);
+
+    void remove_node_for_document(uint32_t docid);
+
+    LevelArrayRef get_level_array(uint32_t docid) const {
+        auto node_ref = node_refs[docid].load_acquire();
+        assert(node_ref.valid());
+        return nodes.get(node_ref);
+    }
+
+    LinkArrayRef get_link_array(uint32_t docid, uint32_t level) const {
+        auto levels = get_level_array(docid);
+        assert(level < levels.size());
+        return links.get(levels[level].load_acquire());
+    }
+    
+    void set_link_array(uint32_t docid, uint32_t level, const LinkArrayRef& new_links);
+
+    void set_entry_node(uint32_t docid, int32_t level) {
+        entry_docid = docid;
+        entry_level = level;
+    }
+
+    size_t size() const { return node_refs.size(); }
+
+    struct Histograms {
+        std::vector<uint32_t> level_histogram;
+        std::vector<uint32_t> links_histogram;
+    };
+    Histograms histograms() const;
+};
+
+}
diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp b/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp
new file mode 100644
index 00000000000..c1f6589b025
--- /dev/null
+++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index.cpp
@@ -0,0 +1,551 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "distance_function.h"
+#include "hnsw_index.h"
+#include "hnsw_index_loader.h"
+#include "hnsw_index_saver.h"
+#include "random_level_generator.h"
+#include <vespa/searchlib/util/state_explorer_utils.h>
+#include <vespa/eval/tensor/dense/typed_cells.h>
+#include <vespa/vespalib/data/slime/cursor.h>
+#include <vespa/vespalib/data/slime/inserter.h>
+#include <vespa/vespalib/datastore/array_store.hpp>
+#include <vespa/vespalib/util/rcuvector.hpp>
+
+namespace search::tensor {
+
+using search::StateExplorerUtils;
+using search::datastore::EntryRef;
+
+namespace {
+
+// TODO: Move this to MemoryAllocator, with name PAGE_SIZE.
+constexpr size_t small_page_size = 4 * 1024;
+constexpr size_t min_num_arrays_for_new_buffer = 8 * 1024;
+constexpr float alloc_grow_factor = 0.2;
+// TODO: Adjust these numbers to what we accept as max in config.
+constexpr size_t max_level_array_size = 16;
+constexpr size_t max_link_array_size = 64;
+
+bool has_link_to(vespalib::ConstArrayRef<uint32_t> links, uint32_t id) {
+    for (uint32_t link : links) {
+        if (link == id) return true;
+    }
+    return false;
+}
+
+struct PairDist {
+    uint32_t id_first;
+    uint32_t id_second;
+    double distance;
+    PairDist(uint32_t i1, uint32_t i2, double d)
+      : id_first(i1), id_second(i2), distance(d)
+    {}
+};
+bool operator< (const PairDist &a, const PairDist &b) {
+    return (a.distance < b.distance);
+}
+
+}
+
+search::datastore::ArrayStoreConfig
+HnswIndex::make_default_node_store_config()
+{
+    return NodeStore::optimizedConfigForHugePage(max_level_array_size, vespalib::alloc::MemoryAllocator::HUGEPAGE_SIZE,
+                                                 small_page_size, min_num_arrays_for_new_buffer, alloc_grow_factor).enable_free_lists(true);
+}
+
+search::datastore::ArrayStoreConfig
+HnswIndex::make_default_link_store_config()
+{
+    return LinkStore::optimizedConfigForHugePage(max_link_array_size, vespalib::alloc::MemoryAllocator::HUGEPAGE_SIZE,
+                                                 small_page_size, min_num_arrays_for_new_buffer, alloc_grow_factor).enable_free_lists(true);
+}
+
+uint32_t
+HnswIndex::max_links_for_level(uint32_t level) const
+{
+    return (level == 0) ? _cfg.max_links_at_level_0() : _cfg.max_links_on_inserts();
+}
+
+bool
+HnswIndex::have_closer_distance(HnswCandidate candidate, const LinkArrayRef& result) const
+{
+    for (uint32_t result_docid : result) {
+        double dist = calc_distance(candidate.docid, result_docid);
+        if (dist < candidate.distance) {
+            return true;
+        }
+    }
+    return false;
+}
+
+HnswIndex::SelectResult
+HnswIndex::select_neighbors_simple(const HnswCandidateVector& neighbors, uint32_t max_links) const
+{
+    HnswCandidateVector sorted(neighbors);
+    std::sort(sorted.begin(), sorted.end(), LesserDistance());
+    SelectResult result;
+    for (const auto & candidate : sorted) {
+        if (result.used.size() < max_links) {
+            result.used.push_back(candidate.docid);
+        } else {
+            result.unused.push_back(candidate.docid);
+        }
+    }
+    return result;
+}
+
+HnswIndex::SelectResult
+HnswIndex::select_neighbors_heuristic(const HnswCandidateVector& neighbors, uint32_t max_links) const
+{
+    SelectResult result;
+    NearestPriQ nearest;
+    for (const auto& entry : neighbors) {
+        nearest.push(entry);
+    }
+    while (!nearest.empty()) {
+        auto candidate = nearest.top();
+        nearest.pop();
+        if (have_closer_distance(candidate, result.used)) {
+            result.unused.push_back(candidate.docid);
+            continue;
+        }
+        result.used.push_back(candidate.docid);
+        if (result.used.size() == max_links) {
+            while (!nearest.empty()) {
+                candidate = nearest.top();
+                nearest.pop();
+                result.unused.push_back(candidate.docid);
+            }
+        }
+    }
+    return result;
+}
+
+HnswIndex::SelectResult
+HnswIndex::select_neighbors(const HnswCandidateVector& neighbors, uint32_t max_links) const
+{
+    if (_cfg.heuristic_select_neighbors()) {
+        return select_neighbors_heuristic(neighbors, max_links);
+    } else {
+        return select_neighbors_simple(neighbors, max_links);
+    }
+}
+
+void
+HnswIndex::shrink_if_needed(uint32_t docid, uint32_t level)
+{
+    auto old_links = _graph.get_link_array(docid, level);
+    uint32_t max_links = max_links_for_level(level);
+    if (old_links.size() > max_links) {
+        HnswCandidateVector neighbors;
+        for (uint32_t neighbor_docid : old_links) {
+            double dist = calc_distance(docid, neighbor_docid);
+            neighbors.emplace_back(neighbor_docid, dist);
+        }
+        auto split = select_neighbors(neighbors, max_links);
+        _graph.set_link_array(docid, level, split.used);
+        for (uint32_t removed_docid : split.unused) {
+            remove_link_to(removed_docid, docid, level);
+        }
+    }
+}
+
+void
+HnswIndex::connect_new_node(uint32_t docid, const LinkArrayRef &neighbors, uint32_t level)
+{
+    _graph.set_link_array(docid, level, neighbors);
+    for (uint32_t neighbor_docid : neighbors) {
+        auto old_links = _graph.get_link_array(neighbor_docid, level);
+        add_link_to(neighbor_docid, level, old_links, docid);
+    }
+    for (uint32_t neighbor_docid : neighbors) {
+        shrink_if_needed(neighbor_docid, level);
+    }
+}
+
+void
+HnswIndex::remove_link_to(uint32_t remove_from, uint32_t remove_id, uint32_t level)
+{
+    LinkArray new_links;
+    auto old_links = _graph.get_link_array(remove_from, level);
+    for (uint32_t id : old_links) {
+        if (id != remove_id) new_links.push_back(id);
+    }
+    _graph.set_link_array(remove_from, level, new_links);
+}
+
+
+double
+HnswIndex::calc_distance(uint32_t lhs_docid, uint32_t rhs_docid) const
+{
+    auto lhs = get_vector(lhs_docid);
+    return calc_distance(lhs, rhs_docid);
+}
+
+double
+HnswIndex::calc_distance(const TypedCells& lhs, uint32_t rhs_docid) const
+{
+    auto rhs = get_vector(rhs_docid);
+    return _distance_func->calc(lhs, rhs);
+}
+
+HnswCandidate
+HnswIndex::find_nearest_in_layer(const TypedCells& input, const HnswCandidate& entry_point, uint32_t level) const
+{
+    HnswCandidate nearest = entry_point;
+    bool keep_searching = true;
+    while (keep_searching) {
+        keep_searching = false;
+        for (uint32_t neighbor_docid : _graph.get_link_array(nearest.docid, level)) {
+            double dist = calc_distance(input, neighbor_docid);
+            if (dist < nearest.distance) {
+                nearest = HnswCandidate(neighbor_docid, dist);
+                keep_searching = true;
+            }
+        }
+    }
+    return nearest;
+}
+
+void
+HnswIndex::search_layer(const TypedCells& input, uint32_t neighbors_to_find, FurthestPriQ& best_neighbors, uint32_t level) const
+{
+    NearestPriQ candidates;
+    uint32_t doc_id_limit = _graph.node_refs.size();
+    auto visited = _visited_set_pool.get(doc_id_limit);
+    for (const auto &entry : best_neighbors.peek()) {
+        assert(entry.docid < doc_id_limit);
+        candidates.push(entry);
+        visited.mark(entry.docid);
+    }
+    double limit_dist = std::numeric_limits<double>::max();
+
+    while (!candidates.empty()) {
+        auto cand = candidates.top();
+        if (cand.distance > limit_dist) {
+            break;
+        }
+        candidates.pop();
+        for (uint32_t neighbor_docid : _graph.get_link_array(cand.docid, level)) {
+            if ((neighbor_docid >= doc_id_limit) || visited.is_marked(neighbor_docid)) {
+                continue;
+            }
+            visited.mark(neighbor_docid);
+            double dist_to_input = calc_distance(input, neighbor_docid);
+            if (dist_to_input < limit_dist) {
+                candidates.emplace(neighbor_docid, dist_to_input);
+                best_neighbors.emplace(neighbor_docid, dist_to_input);
+                if (best_neighbors.size() > neighbors_to_find) {
+                    best_neighbors.pop();
+                    limit_dist = best_neighbors.top().distance;
+                }
+            }
+        }
+    }
+}
+
+HnswIndex::HnswIndex(const DocVectorAccess& vectors, DistanceFunction::UP distance_func,
+                     RandomLevelGenerator::UP level_generator, const Config& cfg)
+    :
+      _graph(),
+      _vectors(vectors),
+      _distance_func(std::move(distance_func)),
+      _level_generator(std::move(level_generator)),
+      _cfg(cfg)
+{
+}
+
+HnswIndex::~HnswIndex() = default;
+
+void
+HnswIndex::add_document(uint32_t docid)
+{
+    auto input = get_vector(docid);
+    // TODO: Add capping on num_levels
+    int level = _level_generator->max_level();
+    _graph.make_node_for_document(docid, level + 1);
+    uint32_t entry_docid = get_entry_docid();
+    if (entry_docid == 0) {
+        _graph.set_entry_node(docid, level);
+        return;
+    }
+
+    int search_level = get_entry_level();
+    double entry_dist = calc_distance(input, entry_docid);
+    HnswCandidate entry_point(entry_docid, entry_dist);
+    while (search_level > level) {
+        entry_point = find_nearest_in_layer(input, entry_point, search_level);
+        --search_level;
+    }
+
+    FurthestPriQ best_neighbors;
+    best_neighbors.push(entry_point);
+    search_level = std::min(level, search_level);
+
+    // Insert the added document in each level it should exist in.
+    while (search_level >= 0) {
+        // TODO: Rename to search_level?
+        search_layer(input, _cfg.neighbors_to_explore_at_construction(), best_neighbors, search_level);
+        auto neighbors = select_neighbors(best_neighbors.peek(), _cfg.max_links_on_inserts());
+        connect_new_node(docid, neighbors.used, search_level);
+        --search_level;
+    }
+    if (level > get_entry_level()) {
+        _graph.set_entry_node(docid, level);
+    }
+}
+
+void
+HnswIndex::mutual_reconnect(const LinkArrayRef &cluster, uint32_t level)
+{
+    std::vector<PairDist> pairs;
+    for (uint32_t i = 0; i + 1 < cluster.size(); ++i) {
+        uint32_t n_id_1 = cluster[i];
+        LinkArrayRef n_list_1 = _graph.get_link_array(n_id_1, level);
+        for (uint32_t j = i + 1; j < cluster.size(); ++j) {
+            uint32_t n_id_2 = cluster[j];
+            if (has_link_to(n_list_1, n_id_2)) continue;
+            pairs.emplace_back(n_id_1, n_id_2, calc_distance(n_id_1, n_id_2));
+        }
+    }
+    std::sort(pairs.begin(), pairs.end());
+    for (const PairDist & pair : pairs) {
+        LinkArrayRef old_links_1 = _graph.get_link_array(pair.id_first, level);
+        if (old_links_1.size() >= _cfg.max_links_on_inserts()) continue;
+
+        LinkArrayRef old_links_2 = _graph.get_link_array(pair.id_second, level);
+        if (old_links_2.size() >= _cfg.max_links_on_inserts()) continue;
+
+        add_link_to(pair.id_first, level, old_links_1, pair.id_second);
+        add_link_to(pair.id_second, level, old_links_2, pair.id_first);
+    }
+}
+
+void
+HnswIndex::remove_document(uint32_t docid)
+{
+    bool need_new_entrypoint = (docid == get_entry_docid());
+    LinkArray empty;
+    LevelArrayRef node_levels = _graph.get_level_array(docid);
+    for (int level = node_levels.size(); level-- > 0; ) {
+        LinkArrayRef my_links = _graph.get_link_array(docid, level);
+        for (uint32_t neighbor_id : my_links) {
+            if (need_new_entrypoint) {
+                _graph.set_entry_node(neighbor_id, level);
+                need_new_entrypoint = false;
+            }
+            remove_link_to(neighbor_id, docid, level);
+        }
+        mutual_reconnect(my_links, level);
+        _graph.set_link_array(docid, level, empty);
+    }
+    if (need_new_entrypoint) {
+        _graph.set_entry_node(0, -1);
+    }
+    _graph.remove_node_for_document(docid);
+}
+
+void
+HnswIndex::transfer_hold_lists(generation_t current_gen)
+{
+    // Note: RcuVector transfers hold lists as part of reallocation based on current generation.
+    //       We need to set the next generation here, as it is incremented on a higher level right after this call.
+    _graph.node_refs.setGeneration(current_gen + 1);
+    _graph.nodes.transferHoldLists(current_gen);
+    _graph.links.transferHoldLists(current_gen);
+}
+
+void
+HnswIndex::trim_hold_lists(generation_t first_used_gen)
+{
+    _graph.node_refs.removeOldGenerations(first_used_gen);
+    _graph.nodes.trimHoldLists(first_used_gen);
+    _graph.links.trimHoldLists(first_used_gen);
+}
+
+vespalib::MemoryUsage
+HnswIndex::memory_usage() const
+{
+    vespalib::MemoryUsage result;
+    result.merge(_graph.node_refs.getMemoryUsage());
+    result.merge(_graph.nodes.getMemoryUsage());
+    result.merge(_graph.links.getMemoryUsage());
+    result.merge(_visited_set_pool.memory_usage());
+    return result;
+}
+
+void
+HnswIndex::get_state(const vespalib::slime::Inserter& inserter) const
+{
+    auto& object = inserter.insertObject();
+    StateExplorerUtils::memory_usage_to_slime(memory_usage(), object.setObject("memory_usage"));
+    object.setLong("nodes", _graph.size());
+    auto& histogram_array = object.setArray("level_histogram");
+    auto& links_hst_array = object.setArray("level_0_links_histogram");
+    auto histograms = _graph.histograms();
+    uint32_t valid_nodes = 0;
+    for (uint32_t hist_val : histograms.level_histogram) {
+        histogram_array.addLong(hist_val);
+        valid_nodes += hist_val;
+    }
+    object.setLong("valid_nodes", valid_nodes);
+    for (uint32_t hist_val : histograms.links_histogram) {
+        links_hst_array.addLong(hist_val);
+    }
+    uint32_t reachable = count_reachable_nodes();
+    uint32_t unreachable = valid_nodes - reachable;
+    object.setLong("unreachable_nodes", unreachable);
+    object.setLong("entry_docid", _graph.entry_docid);
+    object.setLong("entry_level", _graph.entry_level);
+    auto& cfgObj = object.setObject("cfg");
+    cfgObj.setLong("max_links_at_level_0", _cfg.max_links_at_level_0());
+    cfgObj.setLong("max_links_on_inserts", _cfg.max_links_on_inserts());
+    cfgObj.setLong("neighbors_to_explore_at_construction",
+                   _cfg.neighbors_to_explore_at_construction());
+}
+
+std::unique_ptr<NearestNeighborIndexSaver>
+HnswIndex::make_saver() const
+{
+    return std::make_unique<HnswIndexSaver>(_graph);
+}
+
+bool
+HnswIndex::load(const fileutil::LoadedBuffer& buf)
+{
+    assert(get_entry_docid() == 0); // cannot load after index has data
+    HnswIndexLoader loader(_graph);
+    return loader.load(buf);
+}
+
+struct NeighborsByDocId {
+    bool operator() (const NearestNeighborIndex::Neighbor &lhs,
+                     const NearestNeighborIndex::Neighbor &rhs)
+    {
+        return (lhs.docid < rhs.docid);
+    }
+};
+
+std::vector<NearestNeighborIndex::Neighbor>
+HnswIndex::find_top_k(uint32_t k, TypedCells vector, uint32_t explore_k) const
+{
+    std::vector<Neighbor> result;
+    FurthestPriQ candidates = top_k_candidates(vector, std::max(k, explore_k));
+    while (candidates.size() > k) {
+        candidates.pop();
+    }
+    result.reserve(candidates.size());
+    for (const HnswCandidate & hit : candidates.peek()) {
+        result.emplace_back(hit.docid, hit.distance);
+    }
+    std::sort(result.begin(), result.end(), NeighborsByDocId());
+    return result;
+}
+
+FurthestPriQ
+HnswIndex::top_k_candidates(const TypedCells &vector, uint32_t k) const
+{
+    FurthestPriQ best_neighbors;
+    if (get_entry_level() < 0) {
+        return best_neighbors;
+    }
+    uint32_t entry_docid = get_entry_docid();
+    int search_level = get_entry_level();
+    double entry_dist = calc_distance(vector, entry_docid);
+    HnswCandidate entry_point(entry_docid, entry_dist);
+    while (search_level > 0) {
+        entry_point = find_nearest_in_layer(vector, entry_point, search_level);
+        --search_level;
+    }
+    best_neighbors.push(entry_point);
+    search_layer(vector, k, best_neighbors, 0);
+    return best_neighbors;
+}
+
+HnswNode
+HnswIndex::get_node(uint32_t docid) const
+{
+    auto node_ref = _graph.node_refs[docid].load_acquire();
+    if (!node_ref.valid()) {
+        return HnswNode();
+    }
+    auto levels = _graph.nodes.get(node_ref);
+    HnswNode::LevelArray result;
+    for (const auto& links_ref : levels) {
+        auto links = _graph.links.get(links_ref.load_acquire());
+        HnswNode::LinkArray result_links(links.begin(), links.end());
+        std::sort(result_links.begin(), result_links.end());
+        result.push_back(result_links);
+    }
+    return HnswNode(result);
+}
+
+void
+HnswIndex::set_node(uint32_t docid, const HnswNode &node)
+{
+    size_t num_levels = node.size();
+    assert(num_levels > 0);
+    _graph.make_node_for_document(docid, num_levels);
+    for (size_t level = 0; level < num_levels; ++level) {
+        connect_new_node(docid, node.level(level), level);
+    }
+    int max_level = num_levels - 1;
+    if (get_entry_level() < max_level) {
+        _graph.set_entry_node(docid, max_level);
+    }
+}
+
+bool
+HnswIndex::check_link_symmetry() const
+{
+    bool all_sym = true;
+    for (size_t docid = 0; docid < _graph.node_refs.size(); ++docid) {
+        auto node_ref = _graph.node_refs[docid].load_acquire();
+        if (node_ref.valid()) {
+            auto levels = _graph.nodes.get(node_ref);
+            uint32_t level = 0;
+            for (const auto& links_ref : levels) {
+                auto links = _graph.links.get(links_ref.load_acquire());
+                for (auto neighbor_docid : links) {
+                    auto neighbor_links = _graph.get_link_array(neighbor_docid, level);
+                    if (! has_link_to(neighbor_links, docid)) {
+                        all_sym = false;
+                    }
+                }
+                ++level;
+            }
+        }
+    }
+    return all_sym;
+}
+
+uint32_t
+HnswIndex::count_reachable_nodes() const
+{
+    int search_level = get_entry_level();
+    if (search_level < 0) {
+        return 0;
+    }
+    auto visited = _visited_set_pool.get(_graph.size());
+    uint32_t entry_id = get_entry_docid();
+    LinkArray found_links;
+    found_links.push_back(entry_id);
+    visited.mark(entry_id);
+    while (search_level >= 0) {
+        for (uint32_t idx = 0; idx < found_links.size(); ++idx) {
+            uint32_t docid = found_links[idx];
+            auto neighbors = _graph.get_link_array(docid, search_level);
+            for (uint32_t neighbor : neighbors) {
+                if (visited.is_marked(neighbor)) continue;
+                visited.mark(neighbor);
+                found_links.push_back(neighbor);
+            }
+        }
+        --search_level;
+    }
+    return found_links.size();
+}
+
+} // namespace
diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index.h b/searchlib/src/vespa/searchlib/tensor/hnsw_index.h
new file mode 100644
index 00000000000..95001853710
--- /dev/null
+++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index.h
@@ -0,0 +1,157 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "distance_function.h"
+#include "doc_vector_access.h"
+#include "hnsw_index_utils.h"
+#include "hnsw_node.h"
+#include "nearest_neighbor_index.h"
+#include "random_level_generator.h"
+#include "hnsw_graph.h"
+#include <vespa/eval/tensor/dense/typed_cells.h>
+#include <vespa/searchlib/common/bitvector.h>
+#include <vespa/vespalib/datastore/array_store.h>
+#include <vespa/vespalib/datastore/atomic_entry_ref.h>
+#include <vespa/vespalib/datastore/entryref.h>
+#include <vespa/vespalib/util/rcuvector.h>
+#include <vespa/vespalib/util/reusable_set_pool.h>
+
+namespace search::tensor {
+
+/**
+ * Implementation of a hierarchical navigable small world graph (HNSW)
+ * that is used for approximate K-nearest neighbor search.
+ *
+ * The implementation supports 1 write thread and multiple search threads without the use of mutexes.
+ * This is achieved by using data stores that use generation tracking and associated memory management.
+ *
+ * The implementation is mainly based on the algorithms described in
+ * "Efficient and robust approximate nearest neighbor search using Hierarchical Navigable Small World graphs" (Yu. A. Malkov, D. A. Yashunin),
+ * but some adjustments are made to support proper removes.
+ *
+ * TODO: Add details on how to handle removes.
+ */
+class HnswIndex : public NearestNeighborIndex {
+public:
+    class Config {
+    private:
+        uint32_t _max_links_at_level_0;
+        uint32_t _max_links_on_inserts;
+        uint32_t _neighbors_to_explore_at_construction;
+        bool _heuristic_select_neighbors;
+
+    public:
+        Config(uint32_t max_links_at_level_0_in,
+               uint32_t max_links_on_inserts_in,
+               uint32_t neighbors_to_explore_at_construction_in,
+               bool heuristic_select_neighbors_in)
+            : _max_links_at_level_0(max_links_at_level_0_in),
+              _max_links_on_inserts(max_links_on_inserts_in),
+              _neighbors_to_explore_at_construction(neighbors_to_explore_at_construction_in),
+              _heuristic_select_neighbors(heuristic_select_neighbors_in)
+        {}
+        uint32_t max_links_at_level_0() const { return _max_links_at_level_0; }
+        uint32_t max_links_on_inserts() const { return _max_links_on_inserts; }
+        uint32_t neighbors_to_explore_at_construction() const { return _neighbors_to_explore_at_construction; }
+        bool heuristic_select_neighbors() const { return _heuristic_select_neighbors; }
+    };
+
+protected:
+    using AtomicEntryRef = HnswGraph::AtomicEntryRef;
+    using NodeStore = HnswGraph::NodeStore;
+
+    using LinkStore = HnswGraph::LinkStore;
+    using LinkArrayRef = HnswGraph::LinkArrayRef;
+    using LinkArray = vespalib::Array<uint32_t>;
+
+    using LevelArrayRef = HnswGraph::LevelArrayRef;
+    using LevelArray = vespalib::Array<AtomicEntryRef>;
+
+    using TypedCells = vespalib::tensor::TypedCells;
+
+    HnswGraph _graph;
+    const DocVectorAccess& _vectors;
+    DistanceFunction::UP _distance_func;
+    RandomLevelGenerator::UP _level_generator;
+    Config _cfg;
+    mutable vespalib::ReusableSetPool _visited_set_pool;
+
+    uint32_t max_links_for_level(uint32_t level) const;
+    void add_link_to(uint32_t docid, uint32_t level, const LinkArrayRef& old_links, uint32_t new_link) {
+        LinkArray new_links(old_links.begin(), old_links.end());
+        new_links.push_back(new_link);
+        _graph.set_link_array(docid, level, new_links);
+    }
+
+    /**
+     * Returns true if the distance between the candidate and a node in the current result
+     * is less than the distance between the candidate and the node we want to add to the graph.
+     * In this case the candidate should be discarded as we already are connected to the space
+     * where the candidate is located.
+     * Used by select_neighbors_heuristic().
+     */
+    bool have_closer_distance(HnswCandidate candidate, const LinkArrayRef& curr_result) const;
+    struct SelectResult {
+        LinkArray used;
+        LinkArray unused;
+    };
+    SelectResult select_neighbors_heuristic(const HnswCandidateVector& neighbors, uint32_t max_links) const;
+    SelectResult select_neighbors_simple(const HnswCandidateVector& neighbors, uint32_t max_links) const;
+    SelectResult select_neighbors(const HnswCandidateVector& neighbors, uint32_t max_links) const;
+    void shrink_if_needed(uint32_t docid, uint32_t level);
+    void connect_new_node(uint32_t docid, const LinkArrayRef &neighbors, uint32_t level);
+    void mutual_reconnect(const LinkArrayRef &cluster, uint32_t level);
+    void remove_link_to(uint32_t remove_from, uint32_t remove_id, uint32_t level);
+
+    inline TypedCells get_vector(uint32_t docid) const {
+        return _vectors.get_vector(docid);
+    }
+
+    double calc_distance(uint32_t lhs_docid, uint32_t rhs_docid) const;
+    double calc_distance(const TypedCells& lhs, uint32_t rhs_docid) const;
+
+    /**
+     * Performs a greedy search in the given layer to find the candidate that is nearest the input vector.
+     */
+    HnswCandidate find_nearest_in_layer(const TypedCells& input, const HnswCandidate& entry_point, uint32_t level) const;
+    void search_layer(const TypedCells& input, uint32_t neighbors_to_find, FurthestPriQ& found_neighbors, uint32_t level) const;
+
+public:
+    HnswIndex(const DocVectorAccess& vectors, DistanceFunction::UP distance_func,
+              RandomLevelGenerator::UP level_generator, const Config& cfg);
+    ~HnswIndex() override;
+
+    const Config& config() const { return _cfg; }
+
+    // Implements NearestNeighborIndex
+    void add_document(uint32_t docid) override;
+    void remove_document(uint32_t docid) override;
+    void transfer_hold_lists(generation_t current_gen) override;
+    void trim_hold_lists(generation_t first_used_gen) override;
+    vespalib::MemoryUsage memory_usage() const override;
+    void get_state(const vespalib::slime::Inserter& inserter) const override;
+
+    std::unique_ptr<NearestNeighborIndexSaver> make_saver() const override;
+    bool load(const fileutil::LoadedBuffer& buf) override;
+
+    std::vector<Neighbor> find_top_k(uint32_t k, TypedCells vector, uint32_t explore_k) const override;
+    const DistanceFunction *distance_function() const override { return _distance_func.get(); }
+
+    FurthestPriQ top_k_candidates(const TypedCells &vector, uint32_t k) const;
+
+    uint32_t get_entry_docid() const { return _graph.entry_docid; }
+    int32_t get_entry_level() const { return _graph.entry_level; }
+
+    // Should only be used by unit tests.
+    HnswNode get_node(uint32_t docid) const;
+    void set_node(uint32_t docid, const HnswNode &node);
+    bool check_link_symmetry() const;
+    uint32_t count_reachable_nodes() const;
+
+    static search::datastore::ArrayStoreConfig make_default_node_store_config();
+    static search::datastore::ArrayStoreConfig make_default_link_store_config();
+};
+
+}
+
diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index_loader.cpp b/searchlib/src/vespa/searchlib/tensor/hnsw_index_loader.cpp
new file mode 100644
index 00000000000..f02ead86a8d
--- /dev/null
+++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index_loader.cpp
@@ -0,0 +1,47 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "hnsw_index_loader.h"
+#include "hnsw_graph.h"
+#include <vespa/searchlib/util/fileutil.h>
+
+namespace search::tensor {
+
+HnswIndexLoader::~HnswIndexLoader() {}
+
+HnswIndexLoader::HnswIndexLoader(HnswGraph &graph)
+    : _graph(graph), _ptr(nullptr), _end(nullptr), _failed(false)
+{
+}
+
+bool
+HnswIndexLoader::load(const fileutil::LoadedBuffer& buf)
+{
+    size_t num_readable = buf.size(sizeof(uint32_t));
+    _ptr = static_cast<const uint32_t *>(buf.buffer());
+    _end = _ptr + num_readable;
+    uint32_t entry_docid = next_int();
+    int32_t entry_level = next_int();
+    uint32_t num_nodes = next_int();
+    std::vector<uint32_t> link_array;
+    for (uint32_t docid = 0; docid < num_nodes; ++docid) {
+        uint32_t num_levels = next_int();
+        if (num_levels > 0) {
+            _graph.make_node_for_document(docid, num_levels);
+            for (uint32_t level = 0; level < num_levels; ++level) {
+                uint32_t num_links = next_int();
+                link_array.clear();
+                while (num_links-- > 0) {
+                    link_array.push_back(next_int());
+                }
+                _graph.set_link_array(docid, level, link_array);
+            }
+        }
+    }
+    if (_failed) return false;
+    _graph.node_refs.ensure_size(num_nodes);
+    _graph.set_entry_node(entry_docid, entry_level);
+    return true;
+}
+
+
+}
diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index_loader.h b/searchlib/src/vespa/searchlib/tensor/hnsw_index_loader.h
new file mode 100644
index 00000000000..9f5ae66011f
--- /dev/null
+++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index_loader.h
@@ -0,0 +1,35 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <cstdint>
+
+namespace search::fileutil { class LoadedBuffer; }
+
+namespace search::tensor {
+
+struct HnswGraph;
+
+/**
+ * Implements loading of HNSW graph structure from binary format.
+ **/
+class HnswIndexLoader {
+public:
+    HnswIndexLoader(HnswGraph &graph);
+    ~HnswIndexLoader();
+    bool load(const fileutil::LoadedBuffer& buf);
+private:
+    HnswGraph &_graph;
+    const uint32_t *_ptr;
+    const uint32_t *_end;
+    bool _failed;
+    uint32_t next_int() {
+        if (__builtin_expect((_ptr == _end), false)) {
+            _failed = true;
+            return 0;
+        }
+        return *_ptr++;
+    }
+};
+
+}
diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index_saver.cpp b/searchlib/src/vespa/searchlib/tensor/hnsw_index_saver.cpp
new file mode 100644
index 00000000000..acff30f8cbf
--- /dev/null
+++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index_saver.cpp
@@ -0,0 +1,57 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "hnsw_index_saver.h"
+#include "hnsw_graph.h"
+#include <vespa/vespalib/util/bufferwriter.h>
+
+namespace search::tensor {
+
+HnswIndexSaver::~HnswIndexSaver() {}
+
+HnswIndexSaver::HnswIndexSaver(const HnswGraph &graph)
+    : _graph_links(graph.links), _meta_data()
+{
+    _meta_data.entry_docid = graph.entry_docid;
+    _meta_data.entry_level = graph.entry_level;
+    size_t num_nodes = graph.node_refs.size();
+    _meta_data.nodes.reserve(num_nodes);
+    for (size_t i = 0; i < num_nodes; ++i) {
+        LevelVector node;
+        auto node_ref = graph.node_refs[i].load_acquire();
+        if (node_ref.valid()) {
+            auto levels = graph.nodes.get(node_ref);
+            for (const auto& links_ref : levels) {
+                auto level = links_ref.load_acquire();
+                node.push_back(level);
+            }
+        }
+        _meta_data.nodes.emplace_back(std::move(node));
+    }
+}
+
+void
+HnswIndexSaver::save(BufferWriter& writer) const
+{
+    writer.write(&_meta_data.entry_docid, sizeof(uint32_t));
+    writer.write(&_meta_data.entry_level, sizeof(int32_t));
+    uint32_t num_nodes = _meta_data.nodes.size();
+    writer.write(&num_nodes, sizeof(uint32_t));
+    for (const auto &node : _meta_data.nodes) {
+        uint32_t num_levels = node.size();
+        writer.write(&num_levels, sizeof(uint32_t));
+        for (auto links_ref : node) {
+            if (links_ref.valid()) {
+                vespalib::ConstArrayRef<uint32_t> link_array = _graph_links.get(links_ref);
+                uint32_t num_links = link_array.size();
+                writer.write(&num_links, sizeof(uint32_t));
+                writer.write(link_array.cbegin(), sizeof(uint32_t)*num_links);
+            } else {
+                uint32_t num_links = 0;
+                writer.write(&num_links, sizeof(uint32_t));
+            }
+        }
+    }
+    writer.flush();
+}
+
+}
diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index_saver.h b/searchlib/src/vespa/searchlib/tensor/hnsw_index_saver.h
new file mode 100644
index 00000000000..d1d8e0db19d
--- /dev/null
+++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index_saver.h
@@ -0,0 +1,37 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "nearest_neighbor_index_saver.h"
+#include "hnsw_graph.h"
+#include <vespa/vespalib/datastore/entryref.h>
+#include <vector>
+
+namespace search::tensor {
+
+/**
+ * Implements saving of HNSW graph structure in binary format.
+ * The constructor takes a snapshot of all meta-data, but
+ * the links will be fetched from the graph in the save()
+ * method.
+ **/
+class HnswIndexSaver : public NearestNeighborIndexSaver {
+public:
+    using LevelVector = std::vector<search::datastore::EntryRef>;
+
+    HnswIndexSaver(const HnswGraph &graph);
+    ~HnswIndexSaver();
+    void save(BufferWriter& writer) const override;
+
+private:
+    struct MetaData {
+        uint32_t entry_docid;
+        int32_t  entry_level;
+        std::vector<LevelVector> nodes;
+        MetaData() : entry_docid(0), entry_level(-1), nodes() {}
+    };
+    const HnswGraph::LinkStore &_graph_links;
+    MetaData _meta_data;
+};
+
+}
diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index_utils.h b/searchlib/src/vespa/searchlib/tensor/hnsw_index_utils.h
new file mode 100644
index 00000000000..b11d3f36a7a
--- /dev/null
+++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index_utils.h
@@ -0,0 +1,48 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <cstdint>
+#include <queue>
+#include <vector>
+
+namespace search::tensor {
+
+/**
+ * Represents a candidate node with its distance to another point in space.
+ */
+struct HnswCandidate {
+    uint32_t docid;
+    double distance;
+    HnswCandidate(uint32_t docid_in, double distance_in) : docid(docid_in), distance(distance_in) {}
+};
+
+struct GreaterDistance {
+    bool operator() (const HnswCandidate& lhs, const HnswCandidate& rhs) const {
+        return (rhs.distance < lhs.distance);
+    }
+};
+
+struct LesserDistance {
+    bool operator() (const HnswCandidate& lhs, const HnswCandidate& rhs) const {
+        return (lhs.distance < rhs.distance);
+    }
+};
+
+using HnswCandidateVector = std::vector<HnswCandidate>;
+
+/**
+ * Priority queue that keeps the candidate node that is nearest a point in space on top.
+ */
+using NearestPriQ = std::priority_queue<HnswCandidate, HnswCandidateVector, GreaterDistance>;
+
+/**
+ * Priority queue that keeps the candidate node that is furthest away a point in space on top.
+ */
+class FurthestPriQ : public std::priority_queue<HnswCandidate, HnswCandidateVector, LesserDistance> {
+public:
+    const HnswCandidateVector& peek() const { return c; }
+};
+
+}
+
diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_node.h b/searchlib/src/vespa/searchlib/tensor/hnsw_node.h
new file mode 100644
index 00000000000..8a6187ffbcc
--- /dev/null
+++ b/searchlib/src/vespa/searchlib/tensor/hnsw_node.h
@@ -0,0 +1,35 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vector>
+
+namespace search::tensor {
+
+/**
+ * Represents a snapshot of a graph node with all its levels and links.
+ * Should only be used by unit tests.
+ */
+class HnswNode {
+public:
+    using LinkArray = std::vector<uint32_t>;
+    using LevelArray = std::vector<LinkArray>;
+
+private:
+    LevelArray _levels;
+
+public:
+    HnswNode() : _levels() {}
+    HnswNode(const LinkArray& level_0) : _levels() { _levels.push_back(level_0); }
+    HnswNode(const LevelArray& levels_in) : _levels(levels_in) {}
+    bool empty() const { return _levels.empty(); }
+    size_t size() const { return _levels.size(); }
+    const LevelArray& levels() const { return _levels; }
+    const LinkArray& level(size_t idx) const { return _levels[idx]; }
+    bool operator==(const HnswNode& rhs) {
+        return _levels == rhs._levels;
+    }
+};
+
+}
+
diff --git a/searchlib/src/vespa/searchlib/tensor/i_tensor_attribute.h b/searchlib/src/vespa/searchlib/tensor/i_tensor_attribute.h
index 6c83d3caae9..520abb13d06 100644
--- a/searchlib/src/vespa/searchlib/tensor/i_tensor_attribute.h
+++ b/searchlib/src/vespa/searchlib/tensor/i_tensor_attribute.h
@@ -9,6 +9,7 @@ class MutableDenseTensorView;
 class Tensor;
 }
 namespace vespalib::eval { class ValueType; }
+namespace vespalib::slime { struct Inserter; }
 
 namespace search::tensor {
 
@@ -25,6 +26,12 @@ public:
     virtual std::unique_ptr<Tensor> getEmptyTensor() const = 0;
     virtual void getTensor(uint32_t docId, vespalib::tensor::MutableDenseTensorView &tensor) const = 0;
     virtual vespalib::eval::ValueType getTensorType() const = 0;
+
+    /**
+     * Gets custom state for this tensor attribute by inserting it into the given Slime inserter.
+     * This function is only called by the writer thread or when the writer thread is blocked.
+     */
+    virtual void get_state(const vespalib::slime::Inserter& inserter) const = 0;
 };
 
 }  // namespace search::tensor
diff --git a/searchlib/src/vespa/searchlib/tensor/imported_tensor_attribute_vector_read_guard.cpp b/searchlib/src/vespa/searchlib/tensor/imported_tensor_attribute_vector_read_guard.cpp
index 8b60f0c3e8e..593330b7bd4 100644
--- a/searchlib/src/vespa/searchlib/tensor/imported_tensor_attribute_vector_read_guard.cpp
+++ b/searchlib/src/vespa/searchlib/tensor/imported_tensor_attribute_vector_read_guard.cpp
@@ -62,4 +62,10 @@ ImportedTensorAttributeVectorReadGuard::getTensorType() const
     return _target_tensor_attribute.getTensorType();
 }
 
+void
+ImportedTensorAttributeVectorReadGuard::get_state(const vespalib::slime::Inserter& inserter) const
+{
+    _target_tensor_attribute.get_state(inserter);
+}
+
 }
diff --git a/searchlib/src/vespa/searchlib/tensor/imported_tensor_attribute_vector_read_guard.h b/searchlib/src/vespa/searchlib/tensor/imported_tensor_attribute_vector_read_guard.h
index fce744503cd..74c5cd9a685 100644
--- a/searchlib/src/vespa/searchlib/tensor/imported_tensor_attribute_vector_read_guard.h
+++ b/searchlib/src/vespa/searchlib/tensor/imported_tensor_attribute_vector_read_guard.h
@@ -34,6 +34,7 @@ public:
     virtual std::unique_ptr<Tensor> getEmptyTensor() const override;
     virtual void getTensor(uint32_t docId, vespalib::tensor::MutableDenseTensorView &tensor) const override;
     virtual vespalib::eval::ValueType getTensorType() const override;
+    virtual void get_state(const vespalib::slime::Inserter& inserter) const override;
 };
 
 }
diff --git a/searchlib/src/vespa/searchlib/tensor/inv_log_level_generator.cpp b/searchlib/src/vespa/searchlib/tensor/inv_log_level_generator.cpp
new file mode 100644
index 00000000000..540edc5e664
--- /dev/null
+++ b/searchlib/src/vespa/searchlib/tensor/inv_log_level_generator.cpp
@@ -0,0 +1,3 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "inv_log_level_generator.h"
diff --git a/searchlib/src/vespa/searchlib/tensor/inv_log_level_generator.h b/searchlib/src/vespa/searchlib/tensor/inv_log_level_generator.h
new file mode 100644
index 00000000000..2f7f9f4445e
--- /dev/null
+++ b/searchlib/src/vespa/searchlib/tensor/inv_log_level_generator.h
@@ -0,0 +1,35 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "random_level_generator.h"
+#include <random>
+
+namespace search::tensor {
+
+/**
+ * Geometric distribution for level selection in HnswIndex.
+ * Pr(level=k) is (1/M)^k * (1 - 1/M)
+ * Note that the level is theoretically unbounded, but in
+ * practice less than 30.
+ * Generated using floor(ln(U)/ln(1-p)), see
+ * https://en.wikipedia.org/wiki/Geometric_distribution#Related_distributions
+ **/
+
+class InvLogLevelGenerator : public RandomLevelGenerator {
+    std::mt19937_64 _rng;
+    std::uniform_real_distribution<double> _uniform;
+    double _levelMultiplier;
+public:
+    InvLogLevelGenerator(uint32_t m)
+      : _rng(0x1234deadbeef5678uLL),
+        _uniform(0.0, 1.0),
+        _levelMultiplier(1.0 / log(1.0 * m))
+    {}
+
+    uint32_t max_level() override {
+        double unif = _uniform(_rng);
+        double r = -log(1.0-unif) * _levelMultiplier;
+        return (uint32_t) r;
+    }
+};
+
+}
diff --git a/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index.cpp b/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index.cpp
new file mode 100644
index 00000000000..f31230af381
--- /dev/null
+++ b/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index.cpp
@@ -0,0 +1,3 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "nearest_neighbor_index.h"
diff --git a/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index.h b/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index.h
new file mode 100644
index 00000000000..aca2ce2af66
--- /dev/null
+++ b/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index.h
@@ -0,0 +1,59 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "distance_function.h"
+#include <vespa/eval/tensor/dense/typed_cells.h>
+#include <vespa/vespalib/util/generationhandler.h>
+#include <vespa/vespalib/util/memoryusage.h>
+#include <cstdint>
+#include <memory>
+#include <vector>
+
+namespace vespalib::slime { struct Inserter; }
+
+namespace search::fileutil { class LoadedBuffer; }
+
+namespace search::tensor {
+
+class NearestNeighborIndexSaver;
+
+/**
+ * Interface for an index that is used for (approximate) nearest neighbor search.
+ */
+class NearestNeighborIndex {
+public:
+    using generation_t = vespalib::GenerationHandler::generation_t;
+    struct Neighbor {
+        uint32_t docid;
+        double distance;
+        Neighbor(uint32_t id, double dist)
+          : docid(id), distance(dist)
+        {}
+        Neighbor() : docid(0), distance(0.0) {}
+    };
+    virtual ~NearestNeighborIndex() {}
+    virtual void add_document(uint32_t docid) = 0;
+    virtual void remove_document(uint32_t docid) = 0;
+    virtual void transfer_hold_lists(generation_t current_gen) = 0;
+    virtual void trim_hold_lists(generation_t first_used_gen) = 0;
+    virtual vespalib::MemoryUsage memory_usage() const = 0;
+    virtual void get_state(const vespalib::slime::Inserter& inserter) const = 0;
+
+    /**
+     * Creates a saver that is used to save the index to binary form.
+     *
+     * This function is always called by the attribute write thread,
+     * and the caller ensures that an attribute read guard is held during the lifetime of the saver.
+     */
+    virtual std::unique_ptr<NearestNeighborIndexSaver> make_saver() const = 0;
+    virtual bool load(const fileutil::LoadedBuffer& buf) = 0;
+
+    virtual std::vector<Neighbor> find_top_k(uint32_t k,
+                                             vespalib::tensor::TypedCells vector,
+                                             uint32_t explore_k) const = 0;
+
+    virtual const DistanceFunction *distance_function() const = 0;
+};
+
+}
diff --git a/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index_factory.h b/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index_factory.h
new file mode 100644
index 00000000000..089119944a7
--- /dev/null
+++ b/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index_factory.h
@@ -0,0 +1,27 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/eval/eval/value_type.h>
+#include <memory>
+
+namespace search::attribute { class HnswIndexParams; }
+
+namespace search::tensor {
+
+class DocVectorAccess;
+class NearestNeighborIndex;
+
+/**
+ * Factory interface used to instantiate an index used for (approximate) nearest neighbor search.
+ */
+class NearestNeighborIndexFactory {
+public:
+    virtual ~NearestNeighborIndexFactory() {}
+    virtual std::unique_ptr<NearestNeighborIndex> make(const DocVectorAccess& vectors,
+                                                       size_t vector_size,
+                                                       vespalib::eval::ValueType::CellType cell_type,
+                                                       const search::attribute::HnswIndexParams& params) const = 0;
+};
+
+}
diff --git a/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index_saver.cpp b/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index_saver.cpp
new file mode 100644
index 00000000000..4b293488737
--- /dev/null
+++ b/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index_saver.cpp
@@ -0,0 +1,3 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "nearest_neighbor_index_saver.h"
diff --git a/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index_saver.h b/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index_saver.h
new file mode 100644
index 00000000000..99d8960ae10
--- /dev/null
+++ b/searchlib/src/vespa/searchlib/tensor/nearest_neighbor_index_saver.h
@@ -0,0 +1,33 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+namespace search { class BufferWriter; }
+
+namespace search::tensor {
+
+/**
+ * Interface that is used to save a nearest neighbor index to binary form.
+ *
+ * An instance of this interface must hold a snapshot of the index from the
+ * point in time the instance was created, and then save this to binary form in the save() function.
+ *
+ * The instance is always created by the attribute write thread,
+ * and the caller ensures that an attribute read guard is held during the lifetime of the saver.
+ * Data that might change later must be copied in the constructor.
+ *
+ * A flush thread is calling save() at a later point in time.
+ */
+class NearestNeighborIndexSaver {
+public:
+    virtual ~NearestNeighborIndexSaver() {}
+
+    /**
+     * Saves the index in binary form using the given writer.
+     *
+     * It is the responsibility of the implementer to call BufferWriter::flush() at the end.
+     */
+    virtual void save(BufferWriter& writer) const = 0;
+};
+
+}
diff --git a/searchlib/src/vespa/searchlib/tensor/random_level_generator.h b/searchlib/src/vespa/searchlib/tensor/random_level_generator.h
new file mode 100644
index 00000000000..0f4c7c34445
--- /dev/null
+++ b/searchlib/src/vespa/searchlib/tensor/random_level_generator.h
@@ -0,0 +1,19 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <memory>
+
+namespace search::tensor {
+
+/**
+ * Interface used to randomly draw the max level a new hnsw node should exist in.
+ */
+class RandomLevelGenerator {
+public:
+    using UP = std::unique_ptr<RandomLevelGenerator>;
+    virtual ~RandomLevelGenerator() {}
+    virtual uint32_t max_level() = 0;
+};
+
+}
diff --git a/searchlib/src/vespa/searchlib/tensor/tensor_attribute.cpp b/searchlib/src/vespa/searchlib/tensor/tensor_attribute.cpp
index 89b8e77e136..95af9f0471b 100644
--- a/searchlib/src/vespa/searchlib/tensor/tensor_attribute.cpp
+++ b/searchlib/src/vespa/searchlib/tensor/tensor_attribute.cpp
@@ -7,21 +7,23 @@
 #include <vespa/eval/tensor/dense/typed_dense_tensor_builder.h>
 #include <vespa/eval/tensor/sparse/sparse_tensor.h>
 #include <vespa/eval/tensor/wrapped_simple_tensor.h>
+#include <vespa/searchlib/util/state_explorer_utils.h>
+#include <vespa/vespalib/data/slime/cursor.h>
+#include <vespa/vespalib/data/slime/inserter.h>
 #include <vespa/vespalib/util/rcuvector.hpp>
 
+using document::TensorDataType;
+using document::WrongTensorTypeException;
 using vespalib::eval::SimpleTensor;
 using vespalib::eval::ValueType;
+using vespalib::tensor::SparseTensor;
 using vespalib::tensor::Tensor;
 using vespalib::tensor::TypedDenseTensorBuilder;
-using vespalib::tensor::dispatch_0;
-using vespalib::tensor::SparseTensor;
 using vespalib::tensor::WrappedSimpleTensor;
-using document::TensorDataType;
-using document::WrongTensorTypeException;
-
-namespace search {
+using vespalib::tensor::dispatch_0;
+using search::StateExplorerUtils;
 
-namespace tensor {
+namespace search::tensor {
 
 namespace {
 
@@ -71,7 +73,6 @@ TensorAttribute::TensorAttribute(vespalib::stringref name, const Config &cfg, Te
 {
 }
 
-
 TensorAttribute::~TensorAttribute() = default;
 
 const ITensorAttribute *
@@ -93,7 +94,6 @@ TensorAttribute::clearDoc(DocId docId)
     return 0u;
 }
 
-
 void
 TensorAttribute::onCommit()
 {
@@ -110,14 +110,10 @@ TensorAttribute::onCommit()
     }
 }
 
-
 void
 TensorAttribute::onUpdateStat()
 {
-    // update statistics
-    vespalib::MemoryUsage total = _refVector.getMemoryUsage();
-    total.merge(_tensorStore.getMemoryUsage());
-    total.mergeGenerationHeldBytes(getGenerationHolder().getHeldBytes());
+    vespalib::MemoryUsage total = memory_usage();
     this->updateStatistics(_refVector.size(),
                            _refVector.size(),
                            total.allocatedBytes(),
@@ -126,7 +122,6 @@ TensorAttribute::onUpdateStat()
                            total.allocatedBytesOnHold());
 }
 
-
 void
 TensorAttribute::removeOldGenerations(generation_t firstUsed)
 {
@@ -141,7 +136,6 @@ TensorAttribute::onGenerationChange(generation_t generation)
     _tensorStore.transferHoldLists(generation - 1);
 }
 
-
 bool
 TensorAttribute::addDoc(DocId &docId)
 {
@@ -183,6 +177,25 @@ TensorAttribute::setTensorRef(DocId docId, EntryRef ref)
     }
 }
 
+vespalib::MemoryUsage
+TensorAttribute::memory_usage() const
+{
+    vespalib::MemoryUsage result = _refVector.getMemoryUsage();
+    result.merge(_tensorStore.getMemoryUsage());
+    result.mergeGenerationHeldBytes(getGenerationHolder().getHeldBytes());
+    return result;
+}
+
+void
+TensorAttribute::populate_state(vespalib::slime::Cursor& object) const
+{
+    object.setLong("compact_generation", _compactGeneration);
+    StateExplorerUtils::memory_usage_to_slime(_refVector.getMemoryUsage(),
+                                              object.setObject("ref_vector").setObject("memory_usage"));
+    StateExplorerUtils::memory_usage_to_slime(_tensorStore.getMemoryUsage(),
+                                              object.setObject("tensor_store").setObject("memory_usage"));
+}
+
 Tensor::UP
 TensorAttribute::getEmptyTensor() const
 {
@@ -196,6 +209,13 @@ TensorAttribute::getTensorType() const
 }
 
 void
+TensorAttribute::get_state(const vespalib::slime::Inserter& inserter) const
+{
+    auto& object = inserter.insertObject();
+    populate_state(object);
+}
+
+void
 TensorAttribute::clearDocs(DocId lidLow, DocId lidLimit)
 {
     assert(lidLow <= lidLimit);
@@ -209,7 +229,6 @@ TensorAttribute::clearDocs(DocId lidLow, DocId lidLimit)
     }
 }
 
-
 void
 TensorAttribute::onShrinkLidSpace()
 {
@@ -220,14 +239,12 @@ TensorAttribute::onShrinkLidSpace()
     setNumDocs(committedDocIdLimit);
 }
 
-
 uint32_t
 TensorAttribute::getVersion() const
 {
     return TENSOR_ATTRIBUTE_VERSION;
 }
 
-
 TensorAttribute::RefCopyVector
 TensorAttribute::getRefCopy() const
 {
@@ -238,6 +255,4 @@ TensorAttribute::getRefCopy() const
 
 IMPLEMENT_IDENTIFIABLE_ABSTRACT(TensorAttribute, AttributeVector);
 
-}  // namespace search::tensor
-
-}  // namespace search
+}
diff --git a/searchlib/src/vespa/searchlib/tensor/tensor_attribute.h b/searchlib/src/vespa/searchlib/tensor/tensor_attribute.h
index 64f978a31d7..e8efd2170c9 100644
--- a/searchlib/src/vespa/searchlib/tensor/tensor_attribute.h
+++ b/searchlib/src/vespa/searchlib/tensor/tensor_attribute.h
@@ -27,6 +27,9 @@ protected:
     void doCompactWorst();
     void checkTensorType(const Tensor &tensor);
     void setTensorRef(DocId docId, EntryRef ref);
+    virtual vespalib::MemoryUsage memory_usage() const;
+    void populate_state(vespalib::slime::Cursor& object) const;
+
 public:
     DECLARE_IDENTIFIABLE_ABSTRACT(TensorAttribute);
     using RefCopyVector = vespalib::Array<EntryRef>;
@@ -42,6 +45,7 @@ public:
     bool addDoc(DocId &docId) override;
     std::unique_ptr<Tensor> getEmptyTensor() const override;
     vespalib::eval::ValueType getTensorType() const override;
+    void get_state(const vespalib::slime::Inserter& inserter) const override;
     void clearDocs(DocId lidLow, DocId lidLimit) override;
     void onShrinkLidSpace() override;
     uint32_t getVersion() const override;
diff --git a/searchlib/src/vespa/searchlib/test/fakedata/fakeegcompr64filterocc.cpp b/searchlib/src/vespa/searchlib/test/fakedata/fakeegcompr64filterocc.cpp
index 0de200ae32c..04aa790e4cb 100644
--- a/searchlib/src/vespa/searchlib/test/fakedata/fakeegcompr64filterocc.cpp
+++ b/searchlib/src/vespa/searchlib/test/fakedata/fakeegcompr64filterocc.cpp
@@ -1282,7 +1282,7 @@ FakeFilterOccEGCompressed64SkipArrayIterator<doSkip>::doSeek(uint32_t docId)
                     s1CacheInt,
                     K_VALUE_FILTEROCC_L1SKIPDELTA_BITPOS, EC,
                     _l1SkipDocIdBitsOffset += 1 +);
-            assert(docIdBitsOffset = _l1SkipDocIdBitsOffset);
+            assert(docIdBitsOffset == _l1SkipDocIdBitsOffset);
             if (__builtin_expect(oDocId >= _l2SkipDocId, false)) {
                 // Validate L2 Skip information
                 assert(oDocId == _l2SkipDocId);
diff --git a/searchlib/src/vespa/searchlib/test/mock_attribute_manager.cpp b/searchlib/src/vespa/searchlib/test/mock_attribute_manager.cpp
index 5ef9b6cb2d4..4c80c6f869c 100644
--- a/searchlib/src/vespa/searchlib/test/mock_attribute_manager.cpp
+++ b/searchlib/src/vespa/searchlib/test/mock_attribute_manager.cpp
@@ -61,4 +61,9 @@ MockAttributeManager::addAttribute(const AttributeVector::SP &attr) {
     addAttribute(attr->getName(), attr);
 }
 
+std::shared_ptr<attribute::ReadableAttributeVector>
+MockAttributeManager::readable_attribute_vector(const string& name) const {
+    return findAttribute(name);
+}
+
 }
diff --git a/searchlib/src/vespa/searchlib/test/mock_attribute_manager.h b/searchlib/src/vespa/searchlib/test/mock_attribute_manager.h
index dbf84a40e84..a8ec686433a 100644
--- a/searchlib/src/vespa/searchlib/test/mock_attribute_manager.h
+++ b/searchlib/src/vespa/searchlib/test/mock_attribute_manager.h
@@ -28,6 +28,7 @@ public:
     IAttributeContext::UP createContext() const override;
     void addAttribute(const vespalib::string &name, const AttributeVector::SP &attr);
     void addAttribute(const AttributeVector::SP &attr);
+    std::shared_ptr<attribute::ReadableAttributeVector> readable_attribute_vector(const string& name) const override;
 };
 
 }
diff --git a/searchlib/src/vespa/searchlib/test/statestring.cpp b/searchlib/src/vespa/searchlib/test/statestring.cpp
index 33e47b3d961..480e6bbc275 100644
--- a/searchlib/src/vespa/searchlib/test/statestring.cpp
+++ b/searchlib/src/vespa/searchlib/test/statestring.cpp
@@ -7,7 +7,7 @@ namespace search::test::statestring {
 bool
 testStartPos(vespalib::string &s, size_t pos)
 {
-    return (pos < s.size() && (pos == 0 || s[pos - 1] == ' '));
+    return ((pos >= s.size()) || (pos == 0) || (s[pos - 1] == ' '));
 }
 
 size_t
diff --git a/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp b/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp
index 8a6e833bd1f..91599f8218a 100644
--- a/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp
+++ b/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp
@@ -315,6 +315,8 @@ DomainPart::DomainPart(const string & name, const string & baseDir, SerialNum s,
     handleSync(*_transLog);
     _writtenSerial = _range.to();
     _syncedSerial = _writtenSerial;
+    assert(int64_t(byteSize()) == _transLog->GetSize());
+    assert(int64_t(byteSize()) == _transLog->GetPosition());
 }
 
 DomainPart::~DomainPart()
@@ -402,7 +404,7 @@ DomainPart::erase(SerialNum to)
 void
 DomainPart::commit(SerialNum firstSerial, const Packet &packet)
 {
-    int64_t firstPos(_transLog->GetPosition());
+    int64_t firstPos(byteSize());
     nbostream_longlivedbuf h(packet.getHandle().data(), packet.getHandle().size());
     if (_range.from() == 0) {
         _range.from(firstSerial);
@@ -576,7 +578,7 @@ DomainPart::visit(FastOS_FileInterface &file, SerialNumRange &r, Packet &packet)
 void
 DomainPart::write(FastOS_FileInterface &file, const Packet::Entry &entry)
 {
-    int64_t lastKnownGoodPos(file.GetPosition());
+    int64_t lastKnownGoodPos(byteSize());
     int32_t crc(0);
     uint32_t len(entry.serializedSize() + sizeof(crc));
     nbostream os;
diff --git a/searchlib/src/vespa/searchlib/util/CMakeLists.txt b/searchlib/src/vespa/searchlib/util/CMakeLists.txt
index 873f1824d04..ac5fc5f54fc 100644
--- a/searchlib/src/vespa/searchlib/util/CMakeLists.txt
+++ b/searchlib/src/vespa/searchlib/util/CMakeLists.txt
@@ -16,6 +16,7 @@ vespa_add_library(searchlib_util OBJECT
     rawbuf.cpp
     sigbushandler.cpp
     slime_output_raw_buf_adapter.cpp
+    state_explorer_utils.cpp
     statebuf.cpp
     statefile.cpp
     stringenum.cpp
diff --git a/searchlib/src/vespa/searchlib/util/fileutil.cpp b/searchlib/src/vespa/searchlib/util/fileutil.cpp
index 559042a8c7a..e38763f9ba8 100644
--- a/searchlib/src/vespa/searchlib/util/fileutil.cpp
+++ b/searchlib/src/vespa/searchlib/util/fileutil.cpp
@@ -8,6 +8,7 @@
 #include <fcntl.h>
 #include <sys/mman.h>
 #include <sys/stat.h>
+#include <stdexcept>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".searchlib.util.fileutil");
diff --git a/searchlib/src/vespa/searchlib/util/state_explorer_utils.cpp b/searchlib/src/vespa/searchlib/util/state_explorer_utils.cpp
new file mode 100644
index 00000000000..307fc91a605
--- /dev/null
+++ b/searchlib/src/vespa/searchlib/util/state_explorer_utils.cpp
@@ -0,0 +1,19 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "state_explorer_utils.h"
+#include <vespa/vespalib/data/slime/cursor.h>
+#include <vespa/vespalib/util/memoryusage.h>
+
+namespace search {
+
+void
+StateExplorerUtils::memory_usage_to_slime(const vespalib::MemoryUsage& usage, vespalib::slime::Cursor& object)
+{
+    object.setLong("allocated", usage.allocatedBytes());
+    object.setLong("used", usage.usedBytes());
+    object.setLong("dead", usage.deadBytes());
+    object.setLong("onHold", usage.allocatedBytesOnHold());
+}
+
+}
+
diff --git a/searchlib/src/vespa/searchlib/util/state_explorer_utils.h b/searchlib/src/vespa/searchlib/util/state_explorer_utils.h
new file mode 100644
index 00000000000..58867b426a1
--- /dev/null
+++ b/searchlib/src/vespa/searchlib/util/state_explorer_utils.h
@@ -0,0 +1,19 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+namespace vespalib { class MemoryUsage; }
+namespace vespalib::slime { struct Cursor; }
+
+namespace search {
+
+/**
+ * Utility functions for state explorers to convert objects to slime.
+ */
+class StateExplorerUtils {
+public:
+    static void memory_usage_to_slime(const vespalib::MemoryUsage& usage, vespalib::slime::Cursor& object);
+};
+
+}
+
diff --git a/searchlib/src/vespa/searchlib/util/url.cpp b/searchlib/src/vespa/searchlib/util/url.cpp
index 496a19d153f..2e1a808c493 100644
--- a/searchlib/src/vespa/searchlib/util/url.cpp
+++ b/searchlib/src/vespa/searchlib/util/url.cpp
@@ -2,6 +2,7 @@
 
 #include "url.h"
 #include <algorithm>
+#include <cstdio>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".searchlib.util.url");
diff --git a/searchsummary/src/tests/docsummary/matched_elements_filter/matched_elements_filter_test.cpp b/searchsummary/src/tests/docsummary/matched_elements_filter/matched_elements_filter_test.cpp
index 9019a212f3f..dff3acc5b89 100644
--- a/searchsummary/src/tests/docsummary/matched_elements_filter/matched_elements_filter_test.cpp
+++ b/searchsummary/src/tests/docsummary/matched_elements_filter/matched_elements_filter_test.cpp
@@ -246,6 +246,7 @@ TEST_F(MatchedElementsFilterTest, filters_elements_in_array_field_value_when_inp
     expect_filtered("array_in_doc", {0, 1, 2}, "[{'name':'a','weight':3},"
                                                "{'name':'b','weight':5},"
                                                "{'name':'c','weight':7}]");
+    expect_filtered("array_in_doc", {0, 1, 100}, "[]");
 }
 
 TEST_F(MatchedElementsFilterTest, struct_field_mapper_is_setup_for_array_field_value)
@@ -276,6 +277,7 @@ TEST_F(MatchedElementsFilterTest, filters_elements_in_map_field_value_when_input
     expect_filtered("map_in_doc", {0, 1, 2}, "[{'key':'a','value':{'name':'a','weight':3}},"
                                              "{'key':'b','value':{'name':'b','weight':5}},"
                                              "{'key':'c','value':{'name':'c','weight':7}}]");
+    expect_filtered("map_in_doc", {0, 1, 100}, "[]");
 }
 
 TEST_F(MatchedElementsFilterTest, struct_field_mapper_is_setup_for_map_field_value)
diff --git a/searchsummary/src/tests/docsummary/positionsdfw_test.cpp b/searchsummary/src/tests/docsummary/positionsdfw_test.cpp
index b161958fc43..4cad98a8e01 100644
--- a/searchsummary/src/tests/docsummary/positionsdfw_test.cpp
+++ b/searchsummary/src/tests/docsummary/positionsdfw_test.cpp
@@ -100,6 +100,10 @@ public:
     IAttributeContext::UP createContext() const override {
         return IAttributeContext::UP(new MyAttributeContext(_attr));
     }
+
+    std::shared_ptr<attribute::ReadableAttributeVector> readable_attribute_vector(const string&) const override {
+        LOG_ABORT("should not be reached");
+    }
 };
 
 struct MyGetDocsumsStateCallback : GetDocsumsStateCallback {
diff --git a/searchsummary/src/vespa/searchsummary/docsummary/attributedfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/attributedfw.cpp
index 1ca1a336d2d..f1b12d8a227 100644
--- a/searchsummary/src/vespa/searchsummary/docsummary/attributedfw.cpp
+++ b/searchsummary/src/vespa/searchsummary/docsummary/attributedfw.cpp
@@ -5,7 +5,6 @@
 #include "docsumstate.h"
 #include <vespa/searchlib/attribute/stringbase.h>
 #include <vespa/searchlib/attribute/integerbase.h>
-#include <vespa/searchlib/attribute/floatbase.h>
 #include <vespa/searchlib/attribute/iattributemanager.h>
 #include <vespa/searchlib/tensor/i_tensor_attribute.h>
 #include <vespa/searchcommon/attribute/iattributecontext.h>
@@ -40,7 +39,7 @@ AttrDFW::vec(const GetDocsumsState & s) const {
 class SingleAttrDFW : public AttrDFW
 {
 public:
-    SingleAttrDFW(const vespalib::string & attrName) :
+    explicit SingleAttrDFW(const vespalib::string & attrName) :
         AttrDFW(attrName)
     { }
     void insertField(uint32_t docid, GetDocsumsState *state, ResType type, Inserter &target) override;
@@ -55,7 +54,6 @@ bool SingleAttrDFW::isDefaultValue(uint32_t docid, const GetDocsumsState * state
 void
 SingleAttrDFW::insertField(uint32_t docid, GetDocsumsState * state, ResType type, Inserter &target)
 {
-    const char *s="";
     const IAttributeVector & v = vec(*state);
     switch (type) {
     case RES_INT: {
@@ -116,13 +114,13 @@ SingleAttrDFW::insertField(uint32_t docid, GetDocsumsState * state, ResType type
     case RES_FEATUREDATA:
     case RES_LONG_STRING:
     case RES_STRING: {
-        s = v.getString(docid, nullptr, 0); // no need to pass in a buffer, this attribute has a string storage.
+        const char *s = v.getString(docid, nullptr, 0); // no need to pass in a buffer, this attribute has a string storage.
         target.insertString(vespalib::Memory(s));
         break;
     }
     case RES_LONG_DATA:
     case RES_DATA: {
-        s = v.getString(docid, nullptr, 0); // no need to pass in a buffer, this attribute has a string storage.
+        const char *s = v.getString(docid, nullptr, 0); // no need to pass in a buffer, this attribute has a string storage.
         target.insertData(vespalib::Memory(s));
         break;
     }
@@ -138,7 +136,7 @@ SingleAttrDFW::insertField(uint32_t docid, GetDocsumsState * state, ResType type
 class MultiAttrDFW : public AttrDFW
 {
 public:
-    MultiAttrDFW(const vespalib::string & attrName) : AttrDFW(attrName) {}
+    explicit MultiAttrDFW(const vespalib::string & attrName) : AttrDFW(attrName) {}
     void insertField(uint32_t docid, GetDocsumsState *state, ResType type, Inserter &target) override;
 
 };
diff --git a/searchsummary/src/vespa/searchsummary/docsummary/dynamicteaserdfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/dynamicteaserdfw.cpp
index 43eaf4b57b8..4872a183358 100644
--- a/searchsummary/src/vespa/searchsummary/docsummary/dynamicteaserdfw.cpp
+++ b/searchsummary/src/vespa/searchsummary/docsummary/dynamicteaserdfw.cpp
@@ -257,11 +257,11 @@ JuniperQueryAdapter::Traverse(juniper::IQueryVisitor *v) const
                 rc = SkipItem(&iterator);
             break;
         case search::ParseItem::ITEM_NEAR:
-            if (!v->VisitNEAR(&item, iterator.getArity(),iterator.getArg1()))
+            if (!v->VisitNEAR(&item, iterator.getArity(),iterator.getNearDistance()))
                 rc = SkipItem(&iterator);
             break;
         case search::ParseItem::ITEM_ONEAR:
-            if (!v->VisitWITHIN(&item, iterator.getArity(),iterator.getArg1()))
+            if (!v->VisitWITHIN(&item, iterator.getArity(),iterator.getNearDistance()))
                 rc = SkipItem(&iterator);
             break;
         // Unhandled items are just ignored by juniper
diff --git a/searchsummary/src/vespa/searchsummary/docsummary/general_result.cpp b/searchsummary/src/vespa/searchsummary/docsummary/general_result.cpp
index a6a5263f5ac..030cf7e82c1 100644
--- a/searchsummary/src/vespa/searchsummary/docsummary/general_result.cpp
+++ b/searchsummary/src/vespa/searchsummary/docsummary/general_result.cpp
@@ -3,6 +3,7 @@
 #include "general_result.h"
 #include "resultconfig.h"
 #include <vespa/document/fieldvalue/document.h>
+#include <vespa/document/datatype/datatype.h>
 #include <zlib.h>
 #include <cassert>
 
@@ -96,7 +97,13 @@ std::unique_ptr<document::FieldValue>
 GeneralResult::get_field_value(const vespalib::string& field_name) const
 {
     if (_document != nullptr) {
-        return _document->getValue(field_name);
+        const document::Field & field = _document->getField(field_name);
+        auto value(field.getDataType().createFieldValue());
+        if (value) {
+            if (_document->getValue(field, *value)) {
+                return value;
+            }
+        }
     }
     return std::unique_ptr<document::FieldValue>();
 }
diff --git a/searchsummary/src/vespa/searchsummary/docsummary/summaryfieldconverter.cpp b/searchsummary/src/vespa/searchsummary/docsummary/summaryfieldconverter.cpp
index 7368a199569..ada14bf17f5 100644
--- a/searchsummary/src/vespa/searchsummary/docsummary/summaryfieldconverter.cpp
+++ b/searchsummary/src/vespa/searchsummary/docsummary/summaryfieldconverter.cpp
@@ -389,9 +389,11 @@ private:
         MapFieldValueInserter map_inserter(_inserter, _tokenize);
         if (filter_matching_elements()) {
             assert(v.has_no_erased_keys());
-            for (uint32_t id_to_keep : (*_matching_elems)) {
-                auto entry = v[id_to_keep];
-                map_inserter.insert_entry(*entry.first, *entry.second);
+            if (!_matching_elems->empty() && _matching_elems->back() < v.size()) {
+                for (uint32_t id_to_keep : (*_matching_elems)) {
+                    auto entry = v[id_to_keep];
+                    map_inserter.insert_entry(*entry.first, *entry.second);
+                }
             }
         } else {
             for (const auto &entry : v) {
@@ -406,8 +408,10 @@ private:
             ArrayInserter ai(a);
             SlimeFiller conv(ai, _tokenize);
             if (filter_matching_elements()) {
-                for (uint32_t id_to_keep : (*_matching_elems)) {
-                    value[id_to_keep].accept(conv);
+                if (!_matching_elems->empty() && _matching_elems->back() < value.size()) {
+                    for (uint32_t id_to_keep : (*_matching_elems)) {
+                        value[id_to_keep].accept(conv);
+                    }
                 }
             } else {
                 for (const FieldValue &fv : value) {
diff --git a/security-tools/pom.xml b/security-tools/pom.xml
index 38b14ce957f..195e2d06311 100644
--- a/security-tools/pom.xml
+++ b/security-tools/pom.xml
@@ -57,6 +57,7 @@
                 <exclude>META-INF/*.SF</exclude>
                 <exclude>META-INF/*.DSA</exclude>
                 <exclude>META-INF/*.RSA</exclude>
+                <exclude>META-INF/versions/*/module-info.class</exclude>
               </excludes>
             </filter>
           </filters>
diff --git a/security-tools/src/main/java/com/yahoo/vespa/security/tool/securityenv/Main.java b/security-tools/src/main/java/com/yahoo/vespa/security/tool/securityenv/Main.java
index 367d7b9dd83..c314d17e018 100644
--- a/security-tools/src/main/java/com/yahoo/vespa/security/tool/securityenv/Main.java
+++ b/security-tools/src/main/java/com/yahoo/vespa/security/tool/securityenv/Main.java
@@ -54,6 +54,9 @@ public class Main {
             MixedMode mixedMode = TransportSecurityUtils.getInsecureMixedMode(envVars);
             if (options.isPresent() && mixedMode != MixedMode.PLAINTEXT_CLIENT_MIXED_SERVER) {
                 outputVariables.put(OutputVariable.TLS_ENABLED, "1");
+                if (options.get().isHostnameValidationDisabled()) {
+                    outputVariables.put(OutputVariable.DISABLE_HOSTNAME_VALIDATION, "1");
+                }
                 options.get().getCaCertificatesFile()
                         .ifPresent(caCertFile -> outputVariables.put(OutputVariable.CA_CERTIFICATE, caCertFile.toString()));
                 options.get().getCertificatesFile()
diff --git a/security-tools/src/main/java/com/yahoo/vespa/security/tool/securityenv/OutputVariable.java b/security-tools/src/main/java/com/yahoo/vespa/security/tool/securityenv/OutputVariable.java
index dd248d05aac..9a90a145f30 100644
--- a/security-tools/src/main/java/com/yahoo/vespa/security/tool/securityenv/OutputVariable.java
+++ b/security-tools/src/main/java/com/yahoo/vespa/security/tool/securityenv/OutputVariable.java
@@ -10,7 +10,8 @@ enum OutputVariable {
     TLS_ENABLED("VESPA_TLS_ENABLED", "Set to '1' if TLS is enabled in Vespa"),
     CA_CERTIFICATE("VESPA_TLS_CA_CERT", "Path to CA certificates file"),
     CERTIFICATE("VESPA_TLS_CERT", "Path to certificate file"),
-    PRIVATE_KEY("VESPA_TLS_PRIVATE_KEY", "Path to private key file");
+    PRIVATE_KEY("VESPA_TLS_PRIVATE_KEY", "Path to private key file"),
+    DISABLE_HOSTNAME_VALIDATION("VESPA_TLS_HOSTNAME_VALIDATION_DISABLED", "Set to '1' if TLS hostname validation is disabled");
 
     private final String variableName;
     private final String description;
diff --git a/security-tools/src/main/sh/vespa-curl-wrapper b/security-tools/src/main/sh/vespa-curl-wrapper
index e286e121f64..b4fd9224a8a 100755
--- a/security-tools/src/main/sh/vespa-curl-wrapper
+++ b/security-tools/src/main/sh/vespa-curl-wrapper
@@ -88,6 +88,11 @@ then
   CURL_PARAMETERS=("${CURL_PARAMETERS[@]/http:/https:}")
 fi
 
+if [ -n "${VESPA_TLS_HOSTNAME_VALIDATION_DISABLED}" ]
+then
+  CURL_PARAMETERS=("--insecure" "${CURL_PARAMETERS[@]}")
+fi
+
 if [ -n "${VESPA_TLS_CA_CERT}" ]
 then
   CURL_PARAMETERS=("--cacert" "${VESPA_TLS_CA_CERT}" "${CURL_PARAMETERS[@]}")
diff --git a/security-tools/src/test/java/com/yahoo/vespa/security/tool/securityenv/MainTest.java b/security-tools/src/test/java/com/yahoo/vespa/security/tool/securityenv/MainTest.java
index b563ebd14f4..45626820f4d 100644
--- a/security-tools/src/test/java/com/yahoo/vespa/security/tool/securityenv/MainTest.java
+++ b/security-tools/src/test/java/com/yahoo/vespa/security/tool/securityenv/MainTest.java
@@ -106,6 +106,7 @@ public class MainTest {
         TransportSecurityOptions options = new TransportSecurityOptions.Builder()
                 .withCertificates(Paths.get("/path/to/certificate"), Paths.get("/path/to/key"))
                 .withCaCertificates(Paths.get("/path/to/cacerts"))
+                .withHostnameValidationDisabled(true)
                 .build();
         Path configFile = tmpFolder.newFile().toPath();
         options.toJsonFile(configFile);
diff --git a/security-tools/src/test/resources/bash-output.txt b/security-tools/src/test/resources/bash-output.txt
index c07c667af47..182dc177d42 100644
--- a/security-tools/src/test/resources/bash-output.txt
+++ b/security-tools/src/test/resources/bash-output.txt
@@ -2,3 +2,4 @@ VESPA_TLS_ENABLED="1"; export VESPA_TLS_ENABLED;
 VESPA_TLS_CA_CERT="/path/to/cacerts"; export VESPA_TLS_CA_CERT;
 VESPA_TLS_CERT="/path/to/certificate"; export VESPA_TLS_CERT;
 VESPA_TLS_PRIVATE_KEY="/path/to/key"; export VESPA_TLS_PRIVATE_KEY;
+VESPA_TLS_HOSTNAME_VALIDATION_DISABLED="1"; export VESPA_TLS_HOSTNAME_VALIDATION_DISABLED;
diff --git a/security-tools/src/test/resources/csh-output.txt b/security-tools/src/test/resources/csh-output.txt
index 2b6716de92b..2e6cd886c26 100644
--- a/security-tools/src/test/resources/csh-output.txt
+++ b/security-tools/src/test/resources/csh-output.txt
@@ -2,3 +2,4 @@ setenv VESPA_TLS_ENABLED "1";
 setenv VESPA_TLS_CA_CERT "/path/to/cacerts";
 setenv VESPA_TLS_CERT "/path/to/certificate";
 setenv VESPA_TLS_PRIVATE_KEY "/path/to/key";
+setenv VESPA_TLS_HOSTNAME_VALIDATION_DISABLED "1";
diff --git a/security-tools/src/test/resources/expected-help-output.txt b/security-tools/src/test/resources/expected-help-output.txt
index 7d125fe15a2..33ad3b6d232 100644
--- a/security-tools/src/test/resources/expected-help-output.txt
+++ b/security-tools/src/test/resources/expected-help-output.txt
@@ -9,3 +9,5 @@ The output may include the following variables:
  - 'VESPA_TLS_CA_CERT': Path to CA certificates file
  - 'VESPA_TLS_CERT': Path to certificate file
  - 'VESPA_TLS_PRIVATE_KEY': Path to private key file
+ - 'VESPA_TLS_HOSTNAME_VALIDATION_DISABLED': Set to '1' if TLS hostname
+validation is disabled
diff --git a/security-tools/src/test/resources/no-security-output.txt b/security-tools/src/test/resources/no-security-output.txt
index 3467f1316b5..257a2747ee2 100644
--- a/security-tools/src/test/resources/no-security-output.txt
+++ b/security-tools/src/test/resources/no-security-output.txt
@@ -2,3 +2,4 @@ unset VESPA_TLS_ENABLED;
 unset VESPA_TLS_CA_CERT;
 unset VESPA_TLS_CERT;
 unset VESPA_TLS_PRIVATE_KEY;
+unset VESPA_TLS_HOSTNAME_VALIDATION_DISABLED;
diff --git a/security-utils/src/main/java/com/yahoo/security/SslContextBuilder.java b/security-utils/src/main/java/com/yahoo/security/SslContextBuilder.java
index d2b98fd20d9..f3932c84a17 100644
--- a/security-utils/src/main/java/com/yahoo/security/SslContextBuilder.java
+++ b/security-utils/src/main/java/com/yahoo/security/SslContextBuilder.java
@@ -35,6 +35,7 @@ public class SslContextBuilder {
     private TrustManagerFactory trustManagerFactory = TrustManagerUtils::createDefaultX509TrustManager;
     private KeyManagerFactory keyManagerFactory = KeyManagerUtils::createDefaultX509KeyManager;
     private X509ExtendedKeyManager keyManager;
+    private X509ExtendedTrustManager trustManager;
 
     public SslContextBuilder() {}
 
@@ -121,15 +122,25 @@ public class SslContextBuilder {
         return this;
     }
 
+    /**
+     * Note: Callee is responsible for configuring the trust manager.
+     *       Any truststore configured by {@link #withTrustStore(KeyStore)} or the other overloads will be ignored.
+     */
+    public SslContextBuilder withTrustManager(X509ExtendedTrustManager trustManager) {
+        this.trustManager = trustManager;
+        return this;
+    }
+
     public SSLContext build() {
         try {
             SSLContext sslContext = SSLContext.getInstance(TlsContext.SSL_CONTEXT_VERSION);
-            TrustManager[] trustManagers = new TrustManager[] { trustManagerFactory.createTrustManager(trustStoreSupplier.get()) };
+            X509ExtendedTrustManager trustManager = this.trustManager != null
+                    ? this.trustManager
+                    : trustManagerFactory.createTrustManager(trustStoreSupplier.get());
             X509ExtendedKeyManager keyManager = this.keyManager != null
                     ? this.keyManager
                     : keyManagerFactory.createKeyManager(keyStoreSupplier.get(), keyStorePassword);
-            KeyManager[] keyManagers = new KeyManager[] {keyManager};
-            sslContext.init(keyManagers, trustManagers, null);
+            sslContext.init(new KeyManager[] {keyManager}, new TrustManager[] {trustManager}, null);
             return sslContext;
         } catch (GeneralSecurityException e) {
             throw new RuntimeException(e);
diff --git a/security-utils/src/main/java/com/yahoo/security/tls/ConfigFileBasedTlsContext.java b/security-utils/src/main/java/com/yahoo/security/tls/ConfigFileBasedTlsContext.java
index f746480b126..28854c59b2c 100644
--- a/security-utils/src/main/java/com/yahoo/security/tls/ConfigFileBasedTlsContext.java
+++ b/security-utils/src/main/java/com/yahoo/security/tls/ConfigFileBasedTlsContext.java
@@ -12,11 +12,9 @@ import com.yahoo.security.tls.policy.AuthorizedPeers;
 import javax.net.ssl.SSLContext;
 import javax.net.ssl.SSLEngine;
 import javax.net.ssl.SSLParameters;
-import javax.net.ssl.X509ExtendedTrustManager;
 import java.io.IOException;
 import java.io.UncheckedIOException;
 import java.lang.ref.WeakReference;
-import java.nio.file.Files;
 import java.nio.file.Path;
 import java.security.KeyStore;
 import java.time.Duration;
@@ -110,12 +108,14 @@ public class ConfigFileBasedTlsContext implements TlsContext {
                                                              MutableX509TrustManager mutableTrustManager,
                                                              MutableX509KeyManager mutableKeyManager,
                                                              PeerAuthentication peerAuthentication) {
+
+        HostnameVerification hostnameVerification = options.isHostnameValidationDisabled() ? HostnameVerification.DISABLED : HostnameVerification.ENABLED;
+        PeerAuthorizerTrustManager authorizerTrustManager = options.getAuthorizedPeers()
+                .map(authorizedPeers -> new PeerAuthorizerTrustManager(authorizedPeers, mode, hostnameVerification, mutableTrustManager))
+                .orElseGet(() -> new PeerAuthorizerTrustManager(new AuthorizedPeers(com.yahoo.vespa.jdk8compat.Set.of()), AuthorizationMode.DISABLE, hostnameVerification, mutableTrustManager));
         SSLContext sslContext = new SslContextBuilder()
                 .withKeyManager(mutableKeyManager)
-                .withTrustManagerFactory(
-                        ignoredTruststore -> options.getAuthorizedPeers()
-                                .map(authorizedPeers -> (X509ExtendedTrustManager) new PeerAuthorizerTrustManager(authorizedPeers, mode, mutableTrustManager))
-                                .orElseGet(() -> new PeerAuthorizerTrustManager(new AuthorizedPeers(com.yahoo.vespa.jdk8compat.Set.of()), AuthorizationMode.DISABLE, mutableTrustManager)))
+                .withTrustManager(authorizerTrustManager)
                 .build();
         List<String> acceptedCiphers = options.getAcceptedCiphers();
         Set<String> ciphers = acceptedCiphers.isEmpty() ? TlsContext.ALLOWED_CIPHER_SUITES : new HashSet<>(acceptedCiphers);
diff --git a/security-utils/src/main/java/com/yahoo/security/tls/DefaultTlsContext.java b/security-utils/src/main/java/com/yahoo/security/tls/DefaultTlsContext.java
index c3f10a464a5..def3e49be4d 100644
--- a/security-utils/src/main/java/com/yahoo/security/tls/DefaultTlsContext.java
+++ b/security-utils/src/main/java/com/yahoo/security/tls/DefaultTlsContext.java
@@ -34,8 +34,9 @@ public class DefaultTlsContext implements TlsContext {
                              List<X509Certificate> caCertificates,
                              AuthorizedPeers authorizedPeers,
                              AuthorizationMode mode,
-                             PeerAuthentication peerAuthentication) {
-        this(createSslContext(certificates, privateKey, caCertificates, authorizedPeers, mode), peerAuthentication);
+                             PeerAuthentication peerAuthentication,
+                             HostnameVerification hostnameVerification) {
+        this(createSslContext(certificates, privateKey, caCertificates, authorizedPeers, mode, hostnameVerification), peerAuthentication);
     }
 
     public DefaultTlsContext(SSLContext sslContext, PeerAuthentication peerAuthentication) {
@@ -120,7 +121,8 @@ public class DefaultTlsContext implements TlsContext {
                                                PrivateKey privateKey,
                                                List<X509Certificate> caCertificates,
                                                AuthorizedPeers authorizedPeers,
-                                               AuthorizationMode mode) {
+                                               AuthorizationMode mode,
+                                               HostnameVerification hostnameVerification) {
         SslContextBuilder builder = new SslContextBuilder();
         if (!certificates.isEmpty()) {
             builder.withKeyStore(privateKey, certificates);
@@ -129,12 +131,12 @@ public class DefaultTlsContext implements TlsContext {
             builder.withTrustStore(caCertificates);
         }
         if (authorizedPeers != null) {
-            builder.withTrustManagerFactory(truststore -> new PeerAuthorizerTrustManager(authorizedPeers, mode, truststore));
+            builder.withTrustManagerFactory(truststore -> new PeerAuthorizerTrustManager(authorizedPeers, mode, hostnameVerification, truststore));
         } else {
-            builder.withTrustManagerFactory(truststore -> new PeerAuthorizerTrustManager(new AuthorizedPeers(com.yahoo.vespa.jdk8compat.Set.of()), AuthorizationMode.DISABLE, truststore));
+            builder.withTrustManagerFactory(truststore -> new PeerAuthorizerTrustManager(
+                    new AuthorizedPeers(com.yahoo.vespa.jdk8compat.Set.of()), AuthorizationMode.DISABLE, hostnameVerification, truststore));
         }
         return builder.build();
     }
 
-
 }
diff --git a/security-utils/src/main/java/com/yahoo/security/tls/HostnameVerification.java b/security-utils/src/main/java/com/yahoo/security/tls/HostnameVerification.java
new file mode 100644
index 00000000000..a41edc6dc44
--- /dev/null
+++ b/security-utils/src/main/java/com/yahoo/security/tls/HostnameVerification.java
@@ -0,0 +1,7 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.security.tls;
+
+/**
+ * @author bjorncs
+ */
+public enum HostnameVerification { ENABLED, DISABLED }
diff --git a/security-utils/src/main/java/com/yahoo/security/tls/TlsContext.java b/security-utils/src/main/java/com/yahoo/security/tls/TlsContext.java
index e12ea3cf47d..886cf3e886b 100644
--- a/security-utils/src/main/java/com/yahoo/security/tls/TlsContext.java
+++ b/security-utils/src/main/java/com/yahoo/security/tls/TlsContext.java
@@ -35,7 +35,7 @@ public interface TlsContext extends AutoCloseable {
             "TLS_CHACHA20_POLY1305_SHA256"); // TLSv1.3, Java 12
 
     Set<String> ALLOWED_PROTOCOLS = com.yahoo.vespa.jdk8compat.Set.of("TLSv1.2"); // TODO Enable TLSv1.3
-    String SSL_CONTEXT_VERSION = "TLSv1.2"; // TODO Enable TLSv1.3
+    String SSL_CONTEXT_VERSION = "TLS"; // Use SSLContext implementations that supports all TLS versions
 
     /**
      * @return the allowed cipher suites supported by the provided context instance
diff --git a/security-utils/src/main/java/com/yahoo/security/tls/TransportSecurityOptions.java b/security-utils/src/main/java/com/yahoo/security/tls/TransportSecurityOptions.java
index c0e9e1053c3..5db6d551193 100644
--- a/security-utils/src/main/java/com/yahoo/security/tls/TransportSecurityOptions.java
+++ b/security-utils/src/main/java/com/yahoo/security/tls/TransportSecurityOptions.java
@@ -30,6 +30,7 @@ public class TransportSecurityOptions {
     private final Path caCertificatesFile;
     private final AuthorizedPeers authorizedPeers;
     private final List<String> acceptedCiphers;
+    private final boolean isHostnameValidationDisabled;
 
     private TransportSecurityOptions(Builder builder) {
         this.privateKeyFile = builder.privateKeyFile;
@@ -37,6 +38,7 @@ public class TransportSecurityOptions {
         this.caCertificatesFile = builder.caCertificatesFile;
         this.authorizedPeers = builder.authorizedPeers;
         this.acceptedCiphers = builder.acceptedCiphers;
+        this.isHostnameValidationDisabled = builder.isHostnameValidationDisabled;
     }
 
     public Optional<Path> getPrivateKeyFile() {
@@ -57,6 +59,8 @@ public class TransportSecurityOptions {
 
     public List<String> getAcceptedCiphers() { return acceptedCiphers; }
 
+    public boolean isHostnameValidationDisabled() { return isHostnameValidationDisabled; }
+
     public static TransportSecurityOptions fromJsonFile(Path file) {
         try (InputStream in = Files.newInputStream(file)) {
             return new TransportSecurityOptionsJsonSerializer().deserialize(in);
@@ -90,6 +94,7 @@ public class TransportSecurityOptions {
         private Path caCertificatesFile;
         private AuthorizedPeers authorizedPeers;
         private List<String> acceptedCiphers = new ArrayList<>();
+        private boolean isHostnameValidationDisabled;
 
         public Builder() {}
 
@@ -114,6 +119,11 @@ public class TransportSecurityOptions {
             return this;
         }
 
+        public Builder withHostnameValidationDisabled(boolean isDisabled) {
+            this.isHostnameValidationDisabled = isDisabled;
+            return this;
+        }
+
         public TransportSecurityOptions build() {
             return new TransportSecurityOptions(this);
         }
@@ -127,6 +137,7 @@ public class TransportSecurityOptions {
                 ", caCertificatesFile=" + caCertificatesFile +
                 ", authorizedPeers=" + authorizedPeers +
                 ", acceptedCiphers=" + acceptedCiphers +
+                ", isHostnameValidationDisabled=" + isHostnameValidationDisabled +
                 '}';
     }
 
@@ -135,7 +146,8 @@ public class TransportSecurityOptions {
         if (this == o) return true;
         if (o == null || getClass() != o.getClass()) return false;
         TransportSecurityOptions that = (TransportSecurityOptions) o;
-        return Objects.equals(privateKeyFile, that.privateKeyFile) &&
+        return isHostnameValidationDisabled == that.isHostnameValidationDisabled &&
+                Objects.equals(privateKeyFile, that.privateKeyFile) &&
                 Objects.equals(certificatesFile, that.certificatesFile) &&
                 Objects.equals(caCertificatesFile, that.caCertificatesFile) &&
                 Objects.equals(authorizedPeers, that.authorizedPeers) &&
@@ -144,6 +156,6 @@ public class TransportSecurityOptions {
 
     @Override
     public int hashCode() {
-        return Objects.hash(privateKeyFile, certificatesFile, caCertificatesFile, authorizedPeers, acceptedCiphers);
+        return Objects.hash(privateKeyFile, certificatesFile, caCertificatesFile, authorizedPeers, acceptedCiphers, isHostnameValidationDisabled);
     }
 }
\ No newline at end of file
diff --git a/security-utils/src/main/java/com/yahoo/security/tls/authz/PeerAuthorizerTrustManager.java b/security-utils/src/main/java/com/yahoo/security/tls/authz/PeerAuthorizerTrustManager.java
index 3ddd0861f39..03358190e8a 100644
--- a/security-utils/src/main/java/com/yahoo/security/tls/authz/PeerAuthorizerTrustManager.java
+++ b/security-utils/src/main/java/com/yahoo/security/tls/authz/PeerAuthorizerTrustManager.java
@@ -3,6 +3,7 @@ package com.yahoo.security.tls.authz;
 
 import com.yahoo.security.X509CertificateUtils;
 import com.yahoo.security.tls.AuthorizationMode;
+import com.yahoo.security.tls.HostnameVerification;
 import com.yahoo.security.tls.TrustManagerUtils;
 import com.yahoo.security.tls.policy.AuthorizedPeers;
 
@@ -14,7 +15,6 @@ import java.net.Socket;
 import java.security.KeyStore;
 import java.security.cert.CertificateException;
 import java.security.cert.X509Certificate;
-import java.util.Objects;
 import java.util.Optional;
 import java.util.logging.Logger;
 
@@ -33,15 +33,23 @@ public class PeerAuthorizerTrustManager extends X509ExtendedTrustManager {
     private final PeerAuthorizer authorizer;
     private final X509ExtendedTrustManager defaultTrustManager;
     private final AuthorizationMode mode;
+    private final HostnameVerification hostnameVerification;
 
-    public PeerAuthorizerTrustManager(AuthorizedPeers authorizedPeers, AuthorizationMode mode, X509ExtendedTrustManager defaultTrustManager) {
+    public PeerAuthorizerTrustManager(AuthorizedPeers authorizedPeers,
+                                      AuthorizationMode mode,
+                                      HostnameVerification hostnameVerification,
+                                      X509ExtendedTrustManager defaultTrustManager) {
         this.authorizer = new PeerAuthorizer(authorizedPeers);
         this.mode = mode;
+        this.hostnameVerification = hostnameVerification;
         this.defaultTrustManager = defaultTrustManager;
     }
 
-    public PeerAuthorizerTrustManager(AuthorizedPeers authorizedPeers, AuthorizationMode mode, KeyStore truststore) {
-        this(authorizedPeers, mode, TrustManagerUtils.createDefaultX509TrustManager(truststore));
+    public PeerAuthorizerTrustManager(AuthorizedPeers authorizedPeers,
+                                      AuthorizationMode mode,
+                                      HostnameVerification hostnameVerification,
+                                      KeyStore truststore) {
+        this(authorizedPeers, mode, hostnameVerification, TrustManagerUtils.createDefaultX509TrustManager(truststore));
     }
 
     @Override
@@ -58,28 +66,26 @@ public class PeerAuthorizerTrustManager extends X509ExtendedTrustManager {
 
     @Override
     public void checkClientTrusted(X509Certificate[] chain, String authType, Socket socket) throws CertificateException {
-        overrideHostnameVerification(socket);
         defaultTrustManager.checkClientTrusted(chain, authType, socket);
         authorizePeer(chain[0], authType, true, null);
     }
 
     @Override
     public void checkServerTrusted(X509Certificate[] chain, String authType, Socket socket) throws CertificateException {
-        overrideHostnameVerification(socket);
+        overrideHostnameVerificationForClient(socket);
         defaultTrustManager.checkServerTrusted(chain, authType, socket);
         authorizePeer(chain[0], authType, false, null);
     }
 
     @Override
     public void checkClientTrusted(X509Certificate[] chain, String authType, SSLEngine sslEngine) throws CertificateException {
-        overrideHostnameVerification(sslEngine);
         defaultTrustManager.checkClientTrusted(chain, authType, sslEngine);
         authorizePeer(chain[0], authType, true, sslEngine);
     }
 
     @Override
     public void checkServerTrusted(X509Certificate[] chain, String authType, SSLEngine sslEngine) throws CertificateException {
-        overrideHostnameVerification(sslEngine);
+        overrideHostnameVerificationForClient(sslEngine);
         defaultTrustManager.checkServerTrusted(chain, authType, sslEngine);
         authorizePeer(chain[0], authType, false, sslEngine);
     }
@@ -121,31 +127,44 @@ public class PeerAuthorizerTrustManager extends X509ExtendedTrustManager {
                              certificate.getSubjectX500Principal(), X509CertificateUtils.getSubjectAlternativeNames(certificate), authType, isVerifyingClient);
     }
 
-    private static void overrideHostnameVerification(SSLEngine engine) {
+    private void overrideHostnameVerificationForClient(SSLEngine engine) {
         SSLParameters params = engine.getSSLParameters();
-        if (overrideHostnameVerification(params)) {
+        if (overrideHostnameVerificationForClient(params)) {
             engine.setSSLParameters(params);
         }
     }
 
-    private static void overrideHostnameVerification(Socket socket) {
+    private void overrideHostnameVerificationForClient(Socket socket) {
         if (socket instanceof SSLSocket) {
             SSLSocket sslSocket = (SSLSocket) socket;
             SSLParameters params = sslSocket.getSSLParameters();
-            if (overrideHostnameVerification(params)) {
+            if (overrideHostnameVerificationForClient(params)) {
                 sslSocket.setSSLParameters(params);
             }
         }
     }
 
-    // Disable the default hostname verification that is performed by underlying trust manager when 'HTTPS' is used as endpoint identification algorithm.
-    // Some http clients, notably the new http client in Java 11, does not allow user configuration of the endpoint algorithm or custom HostnameVerifier.
-    private static boolean overrideHostnameVerification(SSLParameters params) {
-        if (Objects.equals("HTTPS", params.getEndpointIdentificationAlgorithm())) {
-            params.setEndpointIdentificationAlgorithm("");
-            return true;
+    // Overrides the endpoint identification algorithm specified in the ssl parameters of the ssl engine/socket.
+    // The underlying trust manager will perform hostname verification if endpoint identification algorithm is set to 'HTTPS'.
+    // Returns true if the parameter instance was modified
+    private boolean overrideHostnameVerificationForClient(SSLParameters params) {
+        String configuredAlgorithm = params.getEndpointIdentificationAlgorithm();
+        switch (hostnameVerification) {
+            case ENABLED:
+                if (!"HTTPS".equals(configuredAlgorithm)) {
+                    params.setEndpointIdentificationAlgorithm("HTTPS");
+                    return true;
+                }
+                return false;
+            case DISABLED:
+                if (configuredAlgorithm != null && !configuredAlgorithm.isEmpty()) {
+                    params.setEndpointIdentificationAlgorithm(""); // disable any configured endpoint identification algorithm
+                    return true;
+                }
+                return false;
+            default:
+                throw new IllegalStateException("Unknown host verification type: " + hostnameVerification);
         }
-        return false;
     }
 
 }
diff --git a/security-utils/src/main/java/com/yahoo/security/tls/json/TransportSecurityOptionsEntity.java b/security-utils/src/main/java/com/yahoo/security/tls/json/TransportSecurityOptionsEntity.java
index 6594fa84255..2b001ca2ca0 100644
--- a/security-utils/src/main/java/com/yahoo/security/tls/json/TransportSecurityOptionsEntity.java
+++ b/security-utils/src/main/java/com/yahoo/security/tls/json/TransportSecurityOptionsEntity.java
@@ -8,6 +8,7 @@ import com.fasterxml.jackson.annotation.JsonProperty;
 import java.util.List;
 
 import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_EMPTY;
+import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL;
 
 /**
  * Jackson bindings for transport security options
@@ -20,6 +21,7 @@ class TransportSecurityOptionsEntity {
     @JsonProperty("files") Files files;
     @JsonProperty("authorized-peers") @JsonInclude(NON_EMPTY) List<AuthorizedPeer> authorizedPeers;
     @JsonProperty("accepted-ciphers") @JsonInclude(NON_EMPTY) List<String> acceptedCiphers;
+    @JsonProperty("disable-hostname-validation") @JsonInclude(NON_NULL) Boolean isHostnameValidationDisabled;
 
     static class Files {
         @JsonProperty("private-key") String privateKeyFile;
diff --git a/security-utils/src/main/java/com/yahoo/security/tls/json/TransportSecurityOptionsJsonSerializer.java b/security-utils/src/main/java/com/yahoo/security/tls/json/TransportSecurityOptionsJsonSerializer.java
index 5487bad24e7..3cba434912c 100644
--- a/security-utils/src/main/java/com/yahoo/security/tls/json/TransportSecurityOptionsJsonSerializer.java
+++ b/security-utils/src/main/java/com/yahoo/security/tls/json/TransportSecurityOptionsJsonSerializer.java
@@ -77,6 +77,9 @@ public class TransportSecurityOptionsJsonSerializer {
             }
             builder.withAcceptedCiphers(entity.acceptedCiphers);
         }
+        if (entity.isHostnameValidationDisabled != null) {
+            builder.withHostnameValidationDisabled(entity.isHostnameValidationDisabled);
+        }
         return builder.build();
     }
 
@@ -158,6 +161,9 @@ public class TransportSecurityOptionsJsonSerializer {
         if (!options.getAcceptedCiphers().isEmpty()) {
             entity.acceptedCiphers = options.getAcceptedCiphers();
         }
+        if (options.isHostnameValidationDisabled()) {
+            entity.isHostnameValidationDisabled = true;
+        }
         return entity;
     }
 
diff --git a/security-utils/src/test/java/com/yahoo/security/tls/DefaultTlsContextTest.java b/security-utils/src/test/java/com/yahoo/security/tls/DefaultTlsContextTest.java
index 727a64ae934..00928187f55 100644
--- a/security-utils/src/test/java/com/yahoo/security/tls/DefaultTlsContextTest.java
+++ b/security-utils/src/test/java/com/yahoo/security/tls/DefaultTlsContextTest.java
@@ -46,7 +46,9 @@ public class DefaultTlsContextTest {
                                 singletonList(new RequiredPeerCredential(RequiredPeerCredential.Field.CN, new HostGlobPattern("dummy"))))));
 
         DefaultTlsContext tlsContext =
-                new DefaultTlsContext(singletonList(certificate), keyPair.getPrivate(), singletonList(certificate), authorizedPeers, AuthorizationMode.ENFORCE, PeerAuthentication.NEED);
+                new DefaultTlsContext(
+                        singletonList(certificate), keyPair.getPrivate(), singletonList(certificate), authorizedPeers,
+                        AuthorizationMode.ENFORCE, PeerAuthentication.NEED, HostnameVerification.ENABLED);
 
         SSLEngine sslEngine = tlsContext.createSslEngine();
         assertThat(sslEngine).isNotNull();
diff --git a/security-utils/src/test/java/com/yahoo/security/tls/TransportSecurityOptionsTest.java b/security-utils/src/test/java/com/yahoo/security/tls/TransportSecurityOptionsTest.java
index 28dc10d31d5..f2d2b932cd0 100644
--- a/security-utils/src/test/java/com/yahoo/security/tls/TransportSecurityOptionsTest.java
+++ b/security-utils/src/test/java/com/yahoo/security/tls/TransportSecurityOptionsTest.java
@@ -21,6 +21,7 @@ public class TransportSecurityOptionsTest {
             .withCertificates(Paths.get("certs.pem"), Paths.get("myhost.key"))
             .withCaCertificates(Paths.get("my_cas.pem"))
             .withAcceptedCiphers(com.yahoo.vespa.jdk8compat.List.of("TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384" , "TLS_AES_256_GCM_SHA384"))
+            .withHostnameValidationDisabled(true)
             .build();
 
     @Test
diff --git a/security-utils/src/test/java/com/yahoo/security/tls/json/TransportSecurityOptionsJsonSerializerTest.java b/security-utils/src/test/java/com/yahoo/security/tls/json/TransportSecurityOptionsJsonSerializerTest.java
index 078aa58c948..0dec75fa711 100644
--- a/security-utils/src/test/java/com/yahoo/security/tls/json/TransportSecurityOptionsJsonSerializerTest.java
+++ b/security-utils/src/test/java/com/yahoo/security/tls/json/TransportSecurityOptionsJsonSerializerTest.java
@@ -22,7 +22,6 @@ import java.nio.file.Paths;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashSet;
-import java.util.List;
 
 import static com.yahoo.security.tls.policy.RequiredPeerCredential.Field.CN;
 import static com.yahoo.security.tls.policy.RequiredPeerCredential.Field.SAN_DNS;
@@ -44,6 +43,7 @@ public class TransportSecurityOptionsJsonSerializerTest {
         TransportSecurityOptions options = new TransportSecurityOptions.Builder()
                 .withCaCertificates(Paths.get("/path/to/ca-certs.pem"))
                 .withCertificates(Paths.get("/path/to/cert.pem"), Paths.get("/path/to/key.pem"))
+                .withHostnameValidationDisabled(false)
                 .withAuthorizedPeers(
                         new AuthorizedPeers(
                                 new HashSet<>(Arrays.asList(
@@ -66,6 +66,7 @@ public class TransportSecurityOptionsJsonSerializerTest {
                 .withCertificates(Paths.get("certs.pem"), Paths.get("myhost.key"))
                 .withCaCertificates(Paths.get("my_cas.pem"))
                 .withAcceptedCiphers(com.yahoo.vespa.jdk8compat.List.of("TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384" , "TLS_AES_256_GCM_SHA384"))
+                .withHostnameValidationDisabled(true)
                 .build();
         File outputFile = tempDirectory.newFile();
         try (OutputStream out = Files.newOutputStream(outputFile.toPath())) {
@@ -76,4 +77,22 @@ public class TransportSecurityOptionsJsonSerializerTest {
         assertJsonEquals(expectedOutput, actualOutput);
     }
 
+    @Test
+    public void disable_hostname_validation_is_not_serialized_if_false() throws IOException {
+        TransportSecurityOptions options = new TransportSecurityOptions.Builder()
+                .withCertificates(Paths.get("certs.pem"), Paths.get("myhost.key"))
+                .withCaCertificates(Paths.get("my_cas.pem"))
+                .withHostnameValidationDisabled(false)
+                .build();
+        File outputFile = tempDirectory.newFile();
+        try (OutputStream out = Files.newOutputStream(outputFile.toPath())) {
+            new TransportSecurityOptionsJsonSerializer().serialize(out, options);
+        }
+
+        String expectedOutput = new String(Files.readAllBytes(
+                Paths.get("src/test/resources/transport-security-options-with-disable-hostname-validation-set-to-false.json")));
+        String actualOutput = new String(Files.readAllBytes(outputFile.toPath()));
+        assertJsonEquals(expectedOutput, actualOutput);
+    }
+
 }
diff --git a/security-utils/src/test/resources/transport-security-options-with-disable-hostname-validation-set-to-false.json b/security-utils/src/test/resources/transport-security-options-with-disable-hostname-validation-set-to-false.json
new file mode 100644
index 00000000000..0506c130722
--- /dev/null
+++ b/security-utils/src/test/resources/transport-security-options-with-disable-hostname-validation-set-to-false.json
@@ -0,0 +1,7 @@
+{
+  "files": {
+    "private-key": "myhost.key",
+    "ca-certificates": "my_cas.pem",
+    "certificates": "certs.pem"
+  }
+}
\ No newline at end of file
diff --git a/security-utils/src/test/resources/transport-security-options.json b/security-utils/src/test/resources/transport-security-options.json
index 2e55c8fd931..7983982f644 100644
--- a/security-utils/src/test/resources/transport-security-options.json
+++ b/security-utils/src/test/resources/transport-security-options.json
@@ -1,8 +1,9 @@
 {
+  "disable-hostname-validation": true,
   "files": {
     "private-key": "myhost.key",
     "ca-certificates": "my_cas.pem",
     "certificates": "certs.pem"
-    },
-    "accepted-ciphers": ["TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384", "TLS_AES_256_GCM_SHA384"]
+  },
+  "accepted-ciphers": ["TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384", "TLS_AES_256_GCM_SHA384"]
 }
\ No newline at end of file
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/CriticalRegionChecker.java b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/CriticalRegionChecker.java
new file mode 100644
index 00000000000..238b70a7310
--- /dev/null
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/CriticalRegionChecker.java
@@ -0,0 +1,63 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.service.duper;
+
+import com.yahoo.vespa.service.monitor.CriticalRegion;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * To detect and throw an {@link IllegalStateException} if the execution of the current thread has
+ * reached code point P, some time after the start of a critical region R and before the end of it:
+ *
+ * <ol>
+ *     <li>Declare a static final instance of {@link CriticalRegionChecker}.</li>
+ *     <li>Invoke {@link #startCriticalRegion(String)} when entering region R, and close
+ *     the returned {@link CriticalRegion} when leaving it.</li>
+ *     <li>Invoke {@link #assertOutsideCriticalRegions(String)} at code point P.</li>
+ * </ol>
+ *
+ * @author hakonhall
+ */
+public class CriticalRegionChecker {
+
+    private final ThreadLocalDescriptions threadLocalDescriptions = new ThreadLocalDescriptions();
+    private final String name;
+
+    public CriticalRegionChecker(String name) {
+        this.name = name;
+    }
+
+    /** Start of the critical region, within which {@link #assertOutsideCriticalRegions(String)} will throw. */
+    public CriticalRegion startCriticalRegion(String regionDescription) {
+        List<String> regionDescriptions = threadLocalDescriptions.get();
+        regionDescriptions.add(regionDescription);
+        Thread threadAtStart = Thread.currentThread();
+
+        return () -> {
+            regionDescriptions.remove(regionDescription);
+
+            Thread treadAtClose = Thread.currentThread();
+            if (threadAtStart != treadAtClose) {
+                throw new IllegalStateException(name + ": A critical region cannot cross threads: " +
+                        regionDescription);
+            }
+        };
+    }
+
+    /** @throws IllegalStateException if within one or more critical regions. */
+    public void assertOutsideCriticalRegions(String codePointDescription) throws IllegalStateException {
+        List<String> regionDescriptions = threadLocalDescriptions.get();
+        if (regionDescriptions.size() > 0) {
+            throw new IllegalStateException(name + ": Code point " + codePointDescription +
+                    " is within these critical regions: " + regionDescriptions);
+        }
+    }
+
+    private static class ThreadLocalDescriptions extends ThreadLocal<List<String>> {
+        @Override
+        protected List<String> initialValue() {
+            return new ArrayList<>();
+        }
+    }
+}
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModel.java b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModel.java
index f559e9336c8..104f41c39ed 100644
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModel.java
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModel.java
@@ -2,14 +2,19 @@
 package com.yahoo.vespa.service.duper;
 
 import com.yahoo.config.model.api.ApplicationInfo;
+import com.yahoo.config.model.api.HostInfo;
 import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.HostName;
 import com.yahoo.log.LogLevel;
+import com.yahoo.vespa.service.monitor.DuperModelListener;
 
 import java.util.ArrayList;
-import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
-import java.util.TreeMap;
+import java.util.Optional;
+import java.util.Set;
 import java.util.logging.Logger;
 
 /**
@@ -20,33 +25,128 @@ import java.util.logging.Logger;
 public class DuperModel {
     private static Logger logger = Logger.getLogger(DuperModel.class.getName());
 
-    private final Map<ApplicationId, ApplicationInfo> applications = new TreeMap<>();
+    private final Map<ApplicationId, ApplicationInfo> applicationsById = new HashMap<>();
+    private final Map<HostName, ApplicationId> idsByHostname = new HashMap<>();
+    private final Map<ApplicationId, HashSet<HostName>> hostnamesById = new HashMap<>();
+
     private final List<DuperModelListener> listeners = new ArrayList<>();
+    private boolean isComplete = false;
 
     public void registerListener(DuperModelListener listener) {
-        applications.values().forEach(listener::applicationActivated);
+        applicationsById.values().forEach(listener::applicationActivated);
+        if (isComplete) {
+            listener.bootstrapComplete();
+        }
         listeners.add(listener);
     }
 
+    void setComplete() {
+        if (!isComplete) {
+            logger.log(LogLevel.DEBUG, "All applications have been activated: duper model is complete");
+            isComplete = true;
+
+            listeners.forEach(DuperModelListener::bootstrapComplete);
+        }
+    }
+
+    public boolean isComplete() { return isComplete; }
+
+    public int numberOfApplications() {
+        return applicationsById.size();
+    }
+
+    public int numberOfHosts() {
+        return idsByHostname.size();
+    }
+
     public boolean contains(ApplicationId applicationId) {
-        return applications.containsKey(applicationId);
+        return applicationsById.containsKey(applicationId);
+    }
+
+    public Optional<ApplicationInfo> getApplicationInfo(ApplicationId applicationId) {
+        return Optional.ofNullable(applicationsById.get(applicationId));
+    }
+
+    public Optional<ApplicationInfo> getApplicationInfo(HostName hostName) {
+        return Optional.ofNullable(idsByHostname.get(hostName)).map(applicationsById::get);
+    }
+
+    public List<ApplicationInfo> getApplicationInfos() {
+        return List.copyOf(applicationsById.values());
+    }
+
+    /** Note: Returns an empty set for unknown application. */
+    public Set<HostName> getHostnames(ApplicationId applicationId) {
+        HashSet<HostName> set = hostnamesById.get(applicationId);
+        return set == null ? Set.of() : Set.copyOf(set);
+    }
+
+    public Optional<ApplicationId> getApplicationId(HostName hostname) {
+        return Optional.ofNullable(idsByHostname.get(hostname));
     }
 
     public void add(ApplicationInfo applicationInfo) {
-        applications.put(applicationInfo.getApplicationId(), applicationInfo);
-        logger.log(LogLevel.DEBUG, "Added " + applicationInfo.getApplicationId());
+        ApplicationId id = applicationInfo.getApplicationId();
+        ApplicationInfo oldApplicationInfo = applicationsById.put(id, applicationInfo);
+
+        final String logPrefix;
+        if (oldApplicationInfo == null) {
+            logPrefix = isComplete ? "New application " : "Bootstrapped application ";
+        } else {
+            logPrefix = isComplete ? "Reactivated application " : "Rebootstrapped application ";
+        }
+        logger.log(LogLevel.DEBUG, logPrefix + id.toFullString());
+
+        updateHostnameVsIdMaps(applicationInfo, id);
+
         listeners.forEach(listener -> listener.applicationActivated(applicationInfo));
     }
 
     public void remove(ApplicationId applicationId) {
-        if (applications.remove(applicationId) != null) {
-            logger.log(LogLevel.DEBUG, "Removed " + applicationId);
+        Set<HostName> hostnames = hostnamesById.remove(applicationId);
+        if (hostnames != null) {
+            hostnames.forEach(idsByHostname::remove);
+        }
+
+        ApplicationInfo application = applicationsById.remove(applicationId);
+        if (application != null) {
+            logger.log(LogLevel.DEBUG, "Removed application " + applicationId.toFullString());
             listeners.forEach(listener -> listener.applicationRemoved(applicationId));
         }
     }
 
-    public List<ApplicationInfo> getApplicationInfos() {
-        logger.log(LogLevel.DEBUG, "Applications in duper model: " + applications.values().size());
-        return Collections.unmodifiableList(new ArrayList<>(applications.values()));
+    /** Update hostnamesById and idsByHostname based on a new applicationInfo. */
+    private void updateHostnameVsIdMaps(ApplicationInfo applicationInfo, ApplicationId id) {
+        Set<HostName> removedHosts = new HashSet<>(hostnamesById.computeIfAbsent(id, k -> new HashSet<>()));
+
+        applicationInfo.getModel().getHosts().stream()
+                .map(HostInfo::getHostname)
+                .map(HostName::from)
+                .forEach(hostname -> {
+                    if (!removedHosts.remove(hostname)) {
+                        // hostname has been added
+                        hostnamesById.get(id).add(hostname);
+                        ApplicationId previousId = idsByHostname.put(hostname, id);
+
+                        if (previousId != null && !previousId.equals(id)) {
+                            // If an activation contains a host that is currently assigned to a
+                            // different application we will patch up our data structures to remain
+                            // internally consistent. But listeners may be fooled.
+                            logger.log(LogLevel.WARNING, hostname + " has been reassigned from " +
+                                    previousId.toFullString() + " to " + id.toFullString());
+
+                            Set<HostName> previousHostnames = hostnamesById.get(previousId);
+                            if (previousHostnames != null) {
+                                previousHostnames.remove(hostname);
+                            }
+                        }
+                    }
+                });
+
+        removedHosts.forEach(hostname -> {
+            // hostname has been removed
+            idsByHostname.remove(hostname);
+            hostnamesById.get(id).remove(hostname);
+        });
     }
 }
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java
index 885368810a8..cb1c6d2d95f 100644
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java
@@ -10,8 +10,12 @@ import com.yahoo.config.model.api.SuperModelProvider;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.HostName;
 import com.yahoo.config.provision.SystemName;
+import com.yahoo.log.LogLevel;
 import com.yahoo.vespa.flags.FlagSource;
+import com.yahoo.vespa.service.monitor.CriticalRegion;
 import com.yahoo.vespa.service.monitor.DuperModelInfraApi;
+import com.yahoo.vespa.service.monitor.DuperModelListener;
+import com.yahoo.vespa.service.monitor.DuperModelProvider;
 import com.yahoo.vespa.service.monitor.InfraApplicationApi;
 
 import java.util.ArrayList;
@@ -20,14 +24,19 @@ import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
+import java.util.concurrent.locks.ReentrantLock;
 import java.util.function.Function;
+import java.util.logging.Logger;
+import java.util.function.Supplier;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
 /**
  * @author hakonhall
  */
-public class DuperModelManager implements DuperModelInfraApi {
+public class DuperModelManager implements DuperModelProvider, DuperModelInfraApi {
+
+    private static final Logger logger = Logger.getLogger(DuperModelManager.class.getName());
 
     // Infrastructure applications
     static final ControllerHostApplication controllerHostApplication = new ControllerHostApplication();
@@ -40,11 +49,16 @@ public class DuperModelManager implements DuperModelInfraApi {
 
     private final Map<ApplicationId, InfraApplication> supportedInfraApplications;
 
-    private final Object monitor = new Object();
+    private static CriticalRegionChecker disallowedDuperModeLockAcquisitionRegions =
+            new CriticalRegionChecker("duper model deadlock detection");
+
+    private final ReentrantLock lock = new ReentrantLock(true);
     private final DuperModel duperModel;
     // The set of active infrastructure ApplicationInfo. Not all are necessarily in the DuperModel for historical reasons.
     private final Set<ApplicationId> activeInfraInfos = new HashSet<>(10);
 
+    private boolean superModelIsComplete = false;
+    private boolean infraApplicationsIsComplete = false;
 
     @Inject
     public DuperModelManager(ConfigserverConfig configServerConfig, FlagSource flagSource, SuperModelProvider superModelProvider) {
@@ -53,7 +67,7 @@ public class DuperModelManager implements DuperModelInfraApi {
              superModelProvider, new DuperModel(), flagSource, SystemName.from(configServerConfig.system()));
     }
 
-    /** For testing */
+    /** Non-private for testing */
     DuperModelManager(boolean multitenant, boolean isController, SuperModelProvider superModelProvider, DuperModel duperModel, FlagSource flagSource, SystemName system) {
         this.duperModel = duperModel;
 
@@ -75,16 +89,23 @@ public class DuperModelManager implements DuperModelInfraApi {
         superModelProvider.registerListener(new SuperModelListener() {
             @Override
             public void applicationActivated(SuperModel superModel, ApplicationInfo application) {
-                synchronized (monitor) {
-                    duperModel.add(application);
-                }
+                lockedRunnable(() -> duperModel.add(application));
             }
 
             @Override
             public void applicationRemoved(SuperModel superModel, ApplicationId applicationId) {
-                synchronized (monitor) {
-                    duperModel.remove(applicationId);
-                }
+                lockedRunnable(() -> duperModel.remove(applicationId));
+            }
+
+            @Override
+            public void notifyOfCompleteness(SuperModel superModel) {
+                lockedRunnable(() -> {
+                    if (!superModelIsComplete) {
+                        superModelIsComplete = true;
+                        logger.log(LogLevel.INFO, "All bootstrap tenant applications have been activated");
+                        maybeSetDuperModelAsComplete();
+                    }
+                });
             }
         });
     }
@@ -93,10 +114,9 @@ public class DuperModelManager implements DuperModelInfraApi {
      * Synchronously call {@link DuperModelListener#applicationActivated(ApplicationInfo) listener.applicationActivated()}
      * for each currently active application, and forward future changes.
      */
+    @Override
     public void registerListener(DuperModelListener listener) {
-        synchronized (monitor) {
-            duperModel.registerListener(listener);
-        }
+        lockedRunnable(() -> duperModel.registerListener(listener));
     }
 
     @Override
@@ -118,9 +138,7 @@ public class DuperModelManager implements DuperModelInfraApi {
 
     @Override
     public boolean infraApplicationIsActive(ApplicationId applicationId) {
-        synchronized (monitor) {
-            return activeInfraInfos.contains(applicationId);
-        }
+        return lockedSupplier(() -> activeInfraInfos.contains(applicationId));
     }
 
     @Override
@@ -130,10 +148,10 @@ public class DuperModelManager implements DuperModelInfraApi {
             throw new IllegalArgumentException("There is no infrastructure application with ID '" + applicationId + "'");
         }
 
-        synchronized (monitor) {
+        lockedRunnable(() -> {
             activeInfraInfos.add(applicationId);
             duperModel.add(application.makeApplicationInfo(hostnames));
-        }
+        });
     }
 
     @Override
@@ -142,15 +160,78 @@ public class DuperModelManager implements DuperModelInfraApi {
             throw new IllegalArgumentException("There is no infrastructure application with ID '" + applicationId + "'");
         }
 
-        synchronized (monitor) {
+        lockedRunnable(() -> {
             activeInfraInfos.remove(applicationId);
             duperModel.remove(applicationId);
-        }
+        });
+    }
+
+    @Override
+    public void infraApplicationsIsNowComplete() {
+        lockedRunnable(() -> {
+            if (!infraApplicationsIsComplete) {
+                infraApplicationsIsComplete = true;
+                logger.log(LogLevel.INFO, "All infrastructure applications have been activated");
+                maybeSetDuperModelAsComplete();
+            }
+        });
+    }
+
+    public Optional<ApplicationInfo> getApplicationInfo(ApplicationId applicationId) {
+        return lockedSupplier(() -> duperModel.getApplicationInfo(applicationId));
+    }
+
+    public Optional<ApplicationInfo> getApplicationInfo(HostName hostname) {
+        return lockedSupplier(() -> duperModel.getApplicationInfo(hostname));
     }
 
     public List<ApplicationInfo> getApplicationInfos() {
-        synchronized (monitor) {
-            return duperModel.getApplicationInfos();
+        return lockedSupplier(() -> duperModel.getApplicationInfos());
+    }
+
+    /**
+     * Within the region, trying to accesss the duper model (without already having the lock)
+     * will cause an IllegalStateException to be thrown.
+     */
+    public CriticalRegion disallowDuperModelLockAcquisition(String regionDescription) {
+        return disallowedDuperModeLockAcquisitionRegions.startCriticalRegion(regionDescription);
+    }
+
+    private void maybeSetDuperModelAsComplete() {
+        if (superModelIsComplete && infraApplicationsIsComplete) {
+            duperModel.setComplete();
+        }
+    }
+
+    private <T> T lockedSupplier(Supplier<T> supplier) {
+        if (lock.isHeldByCurrentThread()) {
+            return supplier.get();
+        }
+
+        disallowedDuperModeLockAcquisitionRegions
+                .assertOutsideCriticalRegions("acquiring duper model lock");
+
+        lock.lock();
+        try {
+            return supplier.get();
+        } finally {
+            lock.unlock();
+        }
+    }
+
+    private void lockedRunnable(Runnable runnable) {
+        if (lock.isHeldByCurrentThread()) {
+            runnable.run();
+        }
+
+        disallowedDuperModeLockAcquisitionRegions
+                .assertOutsideCriticalRegions("acquiring duper model lock");
+
+        lock.lock();
+        try {
+            runnable.run();
+        } finally {
+            lock.unlock();
         }
     }
 }
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/InfraApplication.java b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/InfraApplication.java
index 09140423010..d113b34f3c6 100644
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/InfraApplication.java
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/InfraApplication.java
@@ -32,6 +32,7 @@ import java.util.stream.Collectors;
  * @author freva
  */
 public abstract class InfraApplication implements InfraApplicationApi {
+
     private static final TenantName TENANT_NAME = TenantName.from("hosted-vespa");
 
     private final ApplicationId applicationId;
@@ -74,7 +75,7 @@ public abstract class InfraApplication implements InfraApplicationApi {
 
     @Override
     public ClusterSpec getClusterSpecWithVersion(Version version) {
-        return ClusterSpec.request(clusterSpecType, clusterSpecId, version, true);
+        return ClusterSpec.request(clusterSpecType, clusterSpecId).vespaVersion(version).exclusive(true).build();
     }
 
     public ClusterSpec.Type getClusterSpecType() {
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthMonitorManager.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthMonitorManager.java
index 3cc7010e209..d6e15f6af4e 100644
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthMonitorManager.java
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/health/HealthMonitorManager.java
@@ -92,6 +92,10 @@ public class HealthMonitorManager implements MonitorManager, HealthMonitorApi {
     }
 
     @Override
+    public void bootstrapComplete() {
+    }
+
+    @Override
     public ServiceStatusInfo getStatus(ApplicationId applicationId,
                                        ClusterId clusterId,
                                        ServiceType serviceType,
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/manager/MonitorManager.java b/service-monitor/src/main/java/com/yahoo/vespa/service/manager/MonitorManager.java
index dd781a02cef..a7579d3f0da 100644
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/manager/MonitorManager.java
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/manager/MonitorManager.java
@@ -1,7 +1,7 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.service.manager;
 
-import com.yahoo.vespa.service.duper.DuperModelListener;
+import com.yahoo.vespa.service.monitor.DuperModelListener;
 import com.yahoo.vespa.service.monitor.ServiceStatusProvider;
 
 /**
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/manager/UnionMonitorManager.java b/service-monitor/src/main/java/com/yahoo/vespa/service/manager/UnionMonitorManager.java
index 3490ad4a5d2..2aacc3eadac 100644
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/manager/UnionMonitorManager.java
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/manager/UnionMonitorManager.java
@@ -51,4 +51,8 @@ public class UnionMonitorManager implements MonitorManager {
         slobrokMonitorManager.applicationRemoved(id);
         healthMonitorManager.applicationRemoved(id);
     }
+
+    @Override
+    public void bootstrapComplete() {
+    }
 }
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/model/ApplicationInstanceGenerator.java b/service-monitor/src/main/java/com/yahoo/vespa/service/model/ApplicationInstanceGenerator.java
index 5cc2d538c24..d0ecad5f27a 100644
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/model/ApplicationInstanceGenerator.java
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/model/ApplicationInstanceGenerator.java
@@ -5,9 +5,13 @@ import com.yahoo.config.model.api.ApplicationInfo;
 import com.yahoo.config.model.api.HostInfo;
 import com.yahoo.config.model.api.ServiceInfo;
 import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.ApplicationName;
+import com.yahoo.config.provision.InstanceName;
+import com.yahoo.config.provision.TenantName;
 import com.yahoo.config.provision.Zone;
 import com.yahoo.vespa.applicationmodel.ApplicationInstance;
 import com.yahoo.vespa.applicationmodel.ApplicationInstanceId;
+import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference;
 import com.yahoo.vespa.applicationmodel.ClusterId;
 import com.yahoo.vespa.applicationmodel.ConfigId;
 import com.yahoo.vespa.applicationmodel.HostName;
@@ -24,7 +28,9 @@ import com.yahoo.vespa.service.monitor.ServiceStatusProvider;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Set;
+import java.util.function.Predicate;
 import java.util.stream.Collectors;
 
 /**
@@ -37,23 +43,77 @@ public class ApplicationInstanceGenerator {
 
     private final ApplicationInfo applicationInfo;
     private final Zone zone;
-    private ApplicationId configServerApplicationId;
+
+    // This is cheating a bit, but we don't expect DuperModel's config server application ID to be different.
+    // We do this to avoid passing through the ID through multiple levels.
+    private static final ApplicationId configServerApplicationId = new ConfigServerApplication().getApplicationId();
 
     public ApplicationInstanceGenerator(ApplicationInfo applicationInfo, Zone zone) {
         this.applicationInfo = applicationInfo;
         this.zone = zone;
-
-        // This is cheating a bit, but we don't expect DuperModel's config server application ID to be different.
-        // We do this to avoid passing through the ID through multiple levels.
-        this.configServerApplicationId = new ConfigServerApplication().getApplicationId();
     }
 
     public ApplicationInstance makeApplicationInstance(ServiceStatusProvider serviceStatusProvider) {
+        return makeApplicationInstanceLimitedToHosts(serviceStatusProvider, hostname -> true);
+    }
+
+    public ApplicationInstanceReference toApplicationInstanceReference() {
+        return toApplicationInstanceReference(applicationInfo.getApplicationId(), zone);
+    }
+
+    static ApplicationInstanceReference toApplicationInstanceReference(ApplicationId applicationId, Zone zone) {
+        TenantId tenantId = new TenantId(applicationId.tenant().toString());
+        ApplicationInstanceId applicationInstanceId = toApplicationInstanceId(applicationId, zone);
+        return new ApplicationInstanceReference(tenantId, applicationInstanceId);
+    }
+
+    public boolean containsHostname(HostName hostname) {
+        return applicationInfo.getModel().getHosts().stream()
+                .map(HostInfo::getHostname)
+                .anyMatch(hostnameString -> Objects.equals(hostnameString, hostname.s()));
+    }
+
+    public ApplicationInstance makeApplicationInstanceLimitedTo(
+            HostName hostname, ServiceStatusProvider serviceStatusProvider) {
+        return makeApplicationInstanceLimitedToHosts(
+                serviceStatusProvider, candidateHostname -> candidateHostname.equals(hostname));
+    }
+
+    /** Reverse of toApplicationInstanceId, put in this file because it its inverse is. */
+    public static ApplicationId toApplicationId(ApplicationInstanceReference reference) {
+
+        String appNameStr = reference.asString();
+        String[] appNameParts = appNameStr.split(":");
+
+        // Env, region and instance seems to be optional due to the hardcoded config server app
+        // Assume here that first two are tenant and application name.
+        if (appNameParts.length == 2) {
+            return ApplicationId.from(TenantName.from(appNameParts[0]),
+                    ApplicationName.from(appNameParts[1]),
+                    InstanceName.defaultName());
+        }
+
+        // Other normal application should have 5 parts.
+        if (appNameParts.length != 5)  {
+            throw new IllegalArgumentException("Application reference not valid (not 5 parts): " + reference);
+        }
+
+        return ApplicationId.from(TenantName.from(appNameParts[0]),
+                ApplicationName.from(appNameParts[1]),
+                InstanceName.from(appNameParts[4]));
+    }
+
+    private ApplicationInstance makeApplicationInstanceLimitedToHosts(ServiceStatusProvider serviceStatusProvider,
+                                                                      Predicate<HostName> includeHostPredicate) {
         Map<ServiceClusterKey, Set<ServiceInstance>> groupedServiceInstances = new HashMap<>();
 
         for (HostInfo host : applicationInfo.getModel().getHosts()) {
             HostName hostName = new HostName(host.getHostname());
 
+            if (!includeHostPredicate.test(hostName)) {
+                continue;
+            }
+
             for (ServiceInfo serviceInfo : host.getServices()) {
                 ServiceClusterKey serviceClusterKey = toServiceClusterKey(serviceInfo);
 
@@ -77,10 +137,8 @@ public class ApplicationInstanceGenerator {
                         entry.getValue()))
                 .collect(Collectors.toSet());
 
-        ApplicationInstance applicationInstance = new ApplicationInstance(
-                new TenantId(applicationInfo.getApplicationId().tenant().toString()),
-                toApplicationInstanceId(applicationInfo, zone),
-                serviceClusters);
+        ApplicationInstanceReference reference = toApplicationInstanceReference();
+        ApplicationInstance applicationInstance = new ApplicationInstance(reference, serviceClusters);
 
         // Fill back-references
         for (ServiceCluster serviceCluster : applicationInstance.serviceClusters()) {
@@ -106,18 +164,18 @@ public class ApplicationInstanceGenerator {
         return new ServiceInstance(configId, hostName, status);
     }
 
-    private ApplicationInstanceId toApplicationInstanceId(ApplicationInfo applicationInfo, Zone zone) {
-        if (applicationInfo.getApplicationId().equals(configServerApplicationId)) {
+    private static ApplicationInstanceId toApplicationInstanceId(ApplicationId applicationId, Zone zone) {
+        if (applicationId.equals(configServerApplicationId)) {
             // Removing this historical discrepancy would break orchestration during rollout.
             // An alternative may be to use a feature flag and flip it between releases,
             // once that's available.
-            return new ApplicationInstanceId(applicationInfo.getApplicationId().application().value());
+            return new ApplicationInstanceId(applicationId.application().value());
         } else {
             return new ApplicationInstanceId(String.format("%s:%s:%s:%s",
-                    applicationInfo.getApplicationId().application().value(),
+                    applicationId.application().value(),
                     zone.environment().value(),
                     zone.region().value(),
-                    applicationInfo.getApplicationId().instance().value()));
+                    applicationId.instance().value()));
         }
     }
 
@@ -129,7 +187,7 @@ public class ApplicationInstanceGenerator {
                 toConfigId(serviceInfo));
     }
 
-    public static ClusterId getClusterId(ServiceInfo serviceInfo) {
+    private static ClusterId getClusterId(ServiceInfo serviceInfo) {
         return new ClusterId(serviceInfo.getProperty(CLUSTER_ID_PROPERTY_NAME).orElse(""));
     }
 
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/model/ModelGenerator.java b/service-monitor/src/main/java/com/yahoo/vespa/service/model/ModelGenerator.java
index a6bf41d6c9b..ad57e5e3565 100644
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/model/ModelGenerator.java
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/model/ModelGenerator.java
@@ -2,14 +2,17 @@
 package com.yahoo.vespa.service.model;
 
 import com.yahoo.config.model.api.ApplicationInfo;
+import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.Zone;
 import com.yahoo.vespa.applicationmodel.ApplicationInstance;
 import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference;
+import com.yahoo.vespa.applicationmodel.HostName;
 import com.yahoo.vespa.service.monitor.ServiceModel;
 import com.yahoo.vespa.service.monitor.ServiceStatusProvider;
 
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
@@ -19,13 +22,18 @@ import java.util.stream.Collectors;
 public class ModelGenerator {
     public static final String CLUSTER_ID_PROPERTY_NAME = "clustername";
 
+    private final Zone zone;
+
+    public ModelGenerator(Zone zone) {
+        this.zone = zone;
+    }
+
     /**
      * Create service model based primarily on super model.
      *
      * If the configServerhosts is non-empty, a config server application is added.
      */
     public ServiceModel toServiceModel(List<ApplicationInfo> allApplicationInfos,
-                                       Zone zone,
                                        ServiceStatusProvider serviceStatusProvider) {
         Map<ApplicationInstanceReference, ApplicationInstance> applicationInstances =
                 allApplicationInfos.stream()
@@ -36,4 +44,31 @@ public class ModelGenerator {
         return new ServiceModel(applicationInstances);
     }
 
+    public Set<ApplicationInstanceReference> toApplicationInstanceReferenceSet(List<ApplicationInfo> infos) {
+        return infos.stream()
+                .map(info -> new ApplicationInstanceGenerator(info, zone).toApplicationInstanceReference())
+                .collect(Collectors.toSet());
+    }
+
+    public ApplicationInstance toApplicationInstance(ApplicationInfo applicationInfo,
+                                                     ServiceStatusProvider serviceStatusProvider) {
+        var generator = new ApplicationInstanceGenerator(applicationInfo, zone);
+        return generator.makeApplicationInstance(serviceStatusProvider);
+    }
+
+    /**
+     * Make an application instance that contains all services and clusters present on the host,
+     * but lacking other services and hosts. This is an optimization over
+     * {@link #toApplicationInstance(ApplicationInfo, ServiceStatusProvider)}.
+     */
+    public ApplicationInstance toApplicationNarrowedToHost(ApplicationInfo applicationInfo,
+                                                           HostName hostname,
+                                                           ServiceStatusProvider serviceStatusProvider) {
+        var generator = new ApplicationInstanceGenerator(applicationInfo, zone);
+        return generator.makeApplicationInstanceLimitedTo(hostname, serviceStatusProvider);
+    }
+
+    public ApplicationInstanceReference toApplicationInstanceReference(ApplicationId applicationId) {
+        return ApplicationInstanceGenerator.toApplicationInstanceReference(applicationId, zone);
+    }
 }
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceHostListenerAdapter.java b/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceHostListenerAdapter.java
new file mode 100644
index 00000000000..73e2573fc58
--- /dev/null
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceHostListenerAdapter.java
@@ -0,0 +1,73 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+package com.yahoo.vespa.service.model;
+
+import com.yahoo.config.model.api.ApplicationInfo;
+import com.yahoo.config.model.api.HostInfo;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference;
+import com.yahoo.vespa.applicationmodel.HostName;
+import com.yahoo.vespa.service.monitor.DuperModelListener;
+import com.yahoo.vespa.service.monitor.ServiceHostListener;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * Allows a {@link ServiceHostListener} to register with the duper model as a {@link DuperModelListener}.
+ *
+ * <p>This class is not thread-safe: As with the DuperModelListener, events from the duper model
+ * happens within an exclusive duper model lock.</p>
+ */
+public class ServiceHostListenerAdapter implements DuperModelListener {
+    private final ServiceHostListener listener;
+    private final ModelGenerator modelGenerator;
+
+    private final Map<ApplicationInstanceReference, Set<HostName>> hostnamesByReference = new HashMap<>();
+
+    public static ServiceHostListenerAdapter asDuperModelListener(ServiceHostListener listener,
+                                                                  ModelGenerator generator) {
+        return new ServiceHostListenerAdapter(listener, generator);
+    }
+
+    ServiceHostListenerAdapter(ServiceHostListener listener, ModelGenerator generator) {
+        this.listener = listener;
+        this.modelGenerator = generator;
+    }
+
+    @Override
+    public void applicationActivated(ApplicationInfo application) {
+        Set<HostName> newHostnames = application.getModel().getHosts().stream()
+                .map(HostInfo::getHostname)
+                .map(HostName::new)
+                .collect(Collectors.toSet());
+
+        var reference = toApplicationInstanceReference(application.getApplicationId());
+
+        Set<HostName> oldHostnames = hostnamesByReference.get(reference);
+        if (!Objects.equals(newHostnames, oldHostnames)) {
+            hostnamesByReference.put(reference, newHostnames);
+            listener.onApplicationActivate(reference, newHostnames);
+        }
+    }
+
+    @Override
+    public void applicationRemoved(ApplicationId applicationId) {
+        var reference = toApplicationInstanceReference(applicationId);
+
+        if (hostnamesByReference.remove(reference) != null) {
+            listener.onApplicationRemove(reference);
+        }
+    }
+
+    @Override
+    public void bootstrapComplete() {
+    }
+
+    private ApplicationInstanceReference toApplicationInstanceReference(ApplicationId applicationId) {
+        return modelGenerator.toApplicationInstanceReference(applicationId);
+    }
+}
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceModelCache.java b/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceModelCache.java
deleted file mode 100644
index c50f5e6c2d5..00000000000
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceModelCache.java
+++ /dev/null
@@ -1,69 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-package com.yahoo.vespa.service.model;
-
-import com.yahoo.jdisc.Timer;
-import com.yahoo.vespa.service.monitor.ServiceModel;
-
-import java.util.function.Supplier;
-
-/**
- * Adds caching of a supplier of ServiceModel.
- *
- * @author hakonhall
- */
-public class ServiceModelCache implements Supplier<ServiceModel> {
-    public static final long EXPIRY_MILLIS = 10000;
-
-    private final Supplier<ServiceModel> expensiveSupplier;
-    private final Timer timer;
-
-    private volatile ServiceModel snapshot;
-    private boolean updatePossiblyInProgress = false;
-
-    private final Object updateMonitor = new Object();
-    private long snapshotMillis;
-
-    public ServiceModelCache(Supplier<ServiceModel> expensiveSupplier, Timer timer) {
-        this.expensiveSupplier = expensiveSupplier;
-        this.timer = timer;
-    }
-
-    @Override
-    public ServiceModel get() {
-        if (snapshot == null) {
-            synchronized (updateMonitor) {
-                if (snapshot == null) {
-                    takeSnapshot();
-                }
-            }
-        } else if (expired()) {
-            synchronized (updateMonitor) {
-                if (updatePossiblyInProgress) {
-                    return snapshot;
-                }
-
-                updatePossiblyInProgress = true;
-            }
-
-            try {
-                takeSnapshot();
-            } finally {
-                synchronized (updateMonitor) {
-                    updatePossiblyInProgress = false;
-                }
-            }
-        }
-
-        return snapshot;
-    }
-
-    private void takeSnapshot() {
-        snapshot = expensiveSupplier.get();
-        snapshotMillis = timer.currentTimeMillis();
-    }
-
-    private boolean expired() {
-        return timer.currentTimeMillis() - snapshotMillis >= EXPIRY_MILLIS;
-    }
-}
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceModelProvider.java b/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceModelProvider.java
deleted file mode 100644
index c6947810fa0..00000000000
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceModelProvider.java
+++ /dev/null
@@ -1,48 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.service.model;
-
-import com.yahoo.config.model.api.ApplicationInfo;
-import com.yahoo.config.provision.Zone;
-import com.yahoo.vespa.service.monitor.ServiceModel;
-import com.yahoo.vespa.service.monitor.ServiceStatusProvider;
-import com.yahoo.vespa.service.manager.MonitorManager;
-import com.yahoo.vespa.service.duper.DuperModelManager;
-
-import java.util.List;
-import java.util.function.Supplier;
-
-/**
- * An uncached supplier of ServiceModel based on the DuperModel and MonitorManager.
- *
- * @author hakonhall
- */
-public class ServiceModelProvider implements Supplier<ServiceModel> {
-    private final ServiceMonitorMetrics metrics;
-    private final DuperModelManager duperModelManager;
-    private final ModelGenerator modelGenerator;
-    private final Zone zone;
-    private final MonitorManager monitorManager;
-
-    public ServiceModelProvider(MonitorManager monitorManager,
-                                ServiceMonitorMetrics metrics,
-                                DuperModelManager duperModelManager,
-                                ModelGenerator modelGenerator,
-                                Zone zone) {
-        this.monitorManager = monitorManager;
-        this.metrics = metrics;
-        this.duperModelManager = duperModelManager;
-        this.modelGenerator = modelGenerator;
-        this.zone = zone;
-    }
-
-    @Override
-    public ServiceModel get() {
-        try (LatencyMeasurement measurement = metrics.startServiceModelSnapshotLatencyMeasurement()) {
-            // WARNING: The monitor manager may be out-of-sync with duper model (no locking)
-            List<ApplicationInfo> applicationInfos = duperModelManager.getApplicationInfos();
-
-            return modelGenerator.toServiceModel(applicationInfos, zone, (ServiceStatusProvider) monitorManager);
-        }
-    }
-
-}
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceMonitorImpl.java b/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceMonitorImpl.java
index 0a40555036c..e6e6e85c710 100644
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceMonitorImpl.java
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceMonitorImpl.java
@@ -2,23 +2,33 @@
 package com.yahoo.vespa.service.model;
 
 import com.google.inject.Inject;
+import com.yahoo.config.model.api.ApplicationInfo;
+import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.Zone;
 import com.yahoo.jdisc.Metric;
 import com.yahoo.jdisc.Timer;
 import com.yahoo.vespa.applicationmodel.ApplicationInstance;
 import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference;
+import com.yahoo.vespa.applicationmodel.HostName;
 import com.yahoo.vespa.service.duper.DuperModelManager;
-import com.yahoo.vespa.service.health.HealthMonitorManager;
+import com.yahoo.vespa.service.manager.MonitorManager;
 import com.yahoo.vespa.service.manager.UnionMonitorManager;
+import com.yahoo.vespa.service.monitor.AntiServiceMonitor;
+import com.yahoo.vespa.service.monitor.CriticalRegion;
+import com.yahoo.vespa.service.monitor.ServiceHostListener;
 import com.yahoo.vespa.service.monitor.ServiceModel;
 import com.yahoo.vespa.service.monitor.ServiceMonitor;
-import com.yahoo.vespa.service.slobrok.SlobrokMonitorManagerImpl;
+import com.yahoo.vespa.service.monitor.ServiceStatusProvider;
 
-import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
 
-public class ServiceMonitorImpl implements ServiceMonitor {
+public class ServiceMonitorImpl implements ServiceMonitor, AntiServiceMonitor {
 
-    private final ServiceModelCache serviceModelProvider;
+    private final ServiceMonitorMetrics metrics;
+    private final DuperModelManager duperModelManager;
+    private final ModelGenerator modelGenerator;
+    private final ServiceStatusProvider serviceStatusProvider;
 
     @Inject
     public ServiceMonitorImpl(DuperModelManager duperModelManager,
@@ -26,20 +36,88 @@ public class ServiceMonitorImpl implements ServiceMonitor {
                               Metric metric,
                               Timer timer,
                               Zone zone) {
-        duperModelManager.registerListener(monitorManager);
-
-        ServiceModelProvider uncachedServiceModelProvider = new ServiceModelProvider(
-                monitorManager,
+        this(monitorManager,
                 new ServiceMonitorMetrics(metric, timer),
                 duperModelManager,
-                new ModelGenerator(),
-                zone);
-        serviceModelProvider = new ServiceModelCache(uncachedServiceModelProvider, timer);
+                new ModelGenerator(zone)
+        );
     }
 
+    ServiceMonitorImpl(MonitorManager monitorManager,
+                       ServiceMonitorMetrics metrics,
+                       DuperModelManager duperModelManager,
+                       ModelGenerator modelGenerator) {
+        this.serviceStatusProvider = monitorManager;
+        this.metrics = metrics;
+        this.duperModelManager = duperModelManager;
+        this.modelGenerator = modelGenerator;
+
+        duperModelManager.registerListener(monitorManager);
+    }
     @Override
     public ServiceModel getServiceModelSnapshot() {
-        return serviceModelProvider.get();
+        try (LatencyMeasurement measurement = metrics.startServiceModelSnapshotLatencyMeasurement()) {
+            return modelGenerator.toServiceModel(duperModelManager.getApplicationInfos(), serviceStatusProvider);
+        }
+    }
+
+    @Override
+    public Set<ApplicationInstanceReference> getAllApplicationInstanceReferences() {
+        return modelGenerator.toApplicationInstanceReferenceSet(duperModelManager.getApplicationInfos());
+    }
+
+    @Override
+    public Optional<ApplicationInstanceReference> getApplicationInstanceReference(HostName hostname) {
+        return duperModelManager.getApplicationInfo(toConfigProvisionHostName(hostname))
+                .map(ApplicationInfo::getApplicationId)
+                .map(modelGenerator::toApplicationInstanceReference);
+    }
+
+    @Override
+    public Optional<ApplicationInstance> getApplication(HostName hostname) {
+        return getApplicationInfo(hostname)
+                .map(applicationInfo -> modelGenerator.toApplicationInstance(applicationInfo, serviceStatusProvider));
+    }
+
+    @Override
+    public Optional<ApplicationInstance> getApplication(ApplicationInstanceReference reference) {
+        return getApplicationInfo(reference)
+                .map(applicationInfo -> modelGenerator.toApplicationInstance(applicationInfo, serviceStatusProvider));
+    }
+
+    @Override
+    public Optional<ApplicationInstance> getApplicationNarrowedTo(HostName hostname) {
+        Optional<ApplicationInfo> applicationInfo = getApplicationInfo(hostname);
+        if (applicationInfo.isEmpty()) {
+            return Optional.empty();
+        }
+
+        return Optional.of(modelGenerator.toApplicationNarrowedToHost(
+                applicationInfo.get(), hostname, serviceStatusProvider));
+    }
+
+    @Override
+    public void registerListener(ServiceHostListener listener) {
+        var duperModelListener = ServiceHostListenerAdapter.asDuperModelListener(listener, modelGenerator);
+        duperModelManager.registerListener(duperModelListener);
     }
 
+    @Override
+    public CriticalRegion disallowDuperModelLockAcquisition(String regionDescription) {
+        return duperModelManager.disallowDuperModelLockAcquisition(regionDescription);
+    }
+
+    private Optional<ApplicationInfo> getApplicationInfo(ApplicationInstanceReference reference) {
+        ApplicationId applicationId = ApplicationInstanceGenerator.toApplicationId(reference);
+        return duperModelManager.getApplicationInfo(applicationId);
+    }
+
+    private Optional<ApplicationInfo> getApplicationInfo(HostName hostname) {
+        return duperModelManager.getApplicationInfo(toConfigProvisionHostName(hostname));
+    }
+
+    /** The duper model uses HostName from config.provision. */
+    private static com.yahoo.config.provision.HostName toConfigProvisionHostName(HostName hostname) {
+        return com.yahoo.config.provision.HostName.from(hostname.s());
+    }
 }
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/AntiServiceMonitor.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/AntiServiceMonitor.java
new file mode 100644
index 00000000000..1c0460c3797
--- /dev/null
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/AntiServiceMonitor.java
@@ -0,0 +1,25 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.service.monitor;
+
+import com.yahoo.vespa.service.duper.DuperModelManager;
+
+/**
+ * Interface that allows a thread to declare it must NOT access certain parts of the ServiceMonitor.
+ *
+ * @author hakonhall
+ */
+public interface AntiServiceMonitor {
+    /**
+     * Disallow the current thread to acquire the "duper model lock" (see {@link DuperModelManager}),
+     * necessarily acquired by most of the {@link ServiceMonitor} methods, starting from now and
+     * up until the returned region is closed.
+     *
+     * <p>For instance, if an application is activated the duper model is notified:  The duper model
+     * will acquire a lock to update the model atomically, and while having that lock notify
+     * the status service.  The status service will typically acquire an application lock and prune
+     * hosts no longer part of the application.  If a thread were to try to acquire these locks
+     * in the reverse order, it might become deadlocked. This method allows one to detect this
+     * and throw an exception, causing it to be caught earlier or at least easier to debug.</p>
+     */
+    CriticalRegion disallowDuperModelLockAcquisition(String regionDescription);
+}
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/CriticalRegion.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/CriticalRegion.java
new file mode 100644
index 00000000000..ec674f133a1
--- /dev/null
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/CriticalRegion.java
@@ -0,0 +1,18 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.service.monitor;
+
+/**
+ * Represents a region of execution, e.g. the block of a try-with-resources.
+ *
+ * @author hakonhall
+ */
+@FunctionalInterface
+public interface CriticalRegion extends AutoCloseable {
+    /**
+     * Ends the critical region.
+     *
+     * @throws IllegalStateException if the current thread is different from the one that started
+     *         the critical region.
+     */
+    @Override void close() throws IllegalStateException;
+}
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/DuperModelInfraApi.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/DuperModelInfraApi.java
index d08bba2bd3d..f9e47b6b80a 100644
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/DuperModelInfraApi.java
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/DuperModelInfraApi.java
@@ -27,4 +27,7 @@ public interface DuperModelInfraApi {
 
     /** Update the DuperModel: A supported infrastructure application has been removed or is not active. */
     void infraApplicationRemoved(ApplicationId applicationId);
+
+    /** All infra applications that are supposed to activate on config server bootstrap has been activated. */
+    void infraApplicationsIsNowComplete();
 }
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelListener.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/DuperModelListener.java
index a969b6c3f40..f664e5246ca 100644
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelListener.java
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/DuperModelListener.java
@@ -1,34 +1,45 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.service.duper;
+package com.yahoo.vespa.service.monitor;
 
 import com.yahoo.config.model.api.ApplicationInfo;
 import com.yahoo.config.model.api.SuperModel;
 import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.vespa.service.duper.DuperModel;
 
 /**
  * Interface for listening for changes to the {@link DuperModel}.
  *
- * @author hakon
+ * @author hakonhall
  */
 public interface DuperModelListener {
     /**
      * An application has been activated:
      *
      * <ul>
-     *     <li>A synthetic application like the config server application has been added/"activated"
+     *     <li>A synthetic application like the config server application has been added/activated
      *     <li>A super model application has been activated (see
      *     {@link com.yahoo.config.model.api.SuperModelListener#applicationActivated(SuperModel, ApplicationInfo)
      *     SuperModelListener}
      * </ul>
      *
-     * No other threads will concurrently call any methods on this interface.
+     * <p>No other threads will concurrently call any methods on this interface.</p>
      */
     void applicationActivated(ApplicationInfo application);
 
     /**
      * Application has been removed.
      *
-     * No other threads will concurrently call any methods on this interface.
+     * <p>No other threads will concurrently call any methods on this interface.</p>
      */
     void applicationRemoved(ApplicationId id);
+
+    /**
+     * During bootstrap of the config server, a number of applications are activated before
+     * resuming normal operations: The normal "tenant" application (making the super model) and
+     * the relevant infrastructure applications. Once all of these have been activated, this method
+     * will be invoked.
+     *
+     * <p>No other threads will concurrently call any methods on this interface.</p>
+     */
+    void bootstrapComplete();
 }
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/DuperModelProvider.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/DuperModelProvider.java
new file mode 100644
index 00000000000..a90fa418054
--- /dev/null
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/DuperModelProvider.java
@@ -0,0 +1,6 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.service.monitor;
+
+public interface DuperModelProvider {
+    void registerListener(DuperModelListener listener);
+}
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/InfraApplicationApi.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/InfraApplicationApi.java
index dbb326b3c50..4e7a557ffc7 100644
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/InfraApplicationApi.java
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/InfraApplicationApi.java
@@ -6,13 +6,17 @@ import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.Capacity;
 import com.yahoo.config.provision.ClusterSpec;
 
+import java.util.Optional;
+
 /**
  * API of infrastructure application that is accessible via DuperModelInfraApi.
  *
  * @author hakonhall
  */
 public interface InfraApplicationApi {
+
     ApplicationId getApplicationId();
     Capacity getCapacity();
     ClusterSpec getClusterSpecWithVersion(Version version);
+
 }
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceHostListener.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceHostListener.java
new file mode 100644
index 00000000000..17d6fcf0d75
--- /dev/null
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceHostListener.java
@@ -0,0 +1,20 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+package com.yahoo.vespa.service.monitor;
+
+import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference;
+import com.yahoo.vespa.applicationmodel.HostName;
+
+import java.util.Set;
+
+/**
+ * Interface for listening to changes to the set of applications, or the set of hosts
+ * assigned to each application, in the service model.
+ *
+ * <p>This is equivalent to listening to the duper model, since no health information leaks through
+ * from the service model, but the exposed types are those of the service model.</p>
+ */
+public interface ServiceHostListener {
+    void onApplicationActivate(ApplicationInstanceReference reference, Set<HostName> hostnames);
+    void onApplicationRemove(ApplicationInstanceReference reference);
+}
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceModel.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceModel.java
index ed72893400a..c8f491a0fc7 100644
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceModel.java
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceModel.java
@@ -1,66 +1,91 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.service.monitor;
 
+import com.yahoo.config.model.api.ApplicationInfo;
 import com.yahoo.vespa.applicationmodel.ApplicationInstance;
 import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference;
 import com.yahoo.vespa.applicationmodel.HostName;
 import com.yahoo.vespa.applicationmodel.ServiceCluster;
 import com.yahoo.vespa.applicationmodel.ServiceInstance;
 
+import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
-import java.util.stream.Collectors;
 
 /**
- * The ServiceModel is almost a mirror of the SuperModel, except that it
- * also gives ServiceStatus on each service, and there may be
- * artificial applications like the config server "application".
+ * The service model is the union of the duper model and the service monitor, and presented
+ * as classes from the {@code application-model} module.
+ *
+ * <p>The duper model contains the latest {@link ApplicationInfo} of both tenant and infrastructure
+ * applications. The service monitor provides Slobrok or {@code /state/v1/health} information
+ * on (most) services. The application model presents an application as a set of clusters, and
+ * each cluster a set of services, and each service is associated with a particular host
+ * and has a health status.</p>
+ *
+ * @author hakonhall
  */
 public class ServiceModel {
 
-    private final Map<ApplicationInstanceReference, ApplicationInstance> applications;
-    private final Map<HostName, ApplicationInstance> applicationsByHostName;
+    private final Map<ApplicationInstanceReference, ApplicationInstance> applicationsByReference;
 
-    public ServiceModel(Map<ApplicationInstanceReference, ApplicationInstance> applications) {
-        this.applications = Collections.unmodifiableMap(applications);
-        this.applicationsByHostName = Collections.unmodifiableMap(applicationsByHostNames(applications.values()));
+    private Map<HostName, ApplicationInstance> applicationsByHostName = null;
+    private Map<HostName, List<ServiceInstance>> servicesByHostName = null;
+
+    public ServiceModel(Map<ApplicationInstanceReference, ApplicationInstance> applicationsByReference) {
+        this.applicationsByReference = Collections.unmodifiableMap(Map.copyOf(applicationsByReference));
     }
 
     public Map<ApplicationInstanceReference, ApplicationInstance> getAllApplicationInstances() {
-        return applications;
+        return applicationsByReference;
     }
 
     public Optional<ApplicationInstance> getApplicationInstance(ApplicationInstanceReference reference) {
-        return Optional.ofNullable(applications.get(reference));
+        return Optional.ofNullable(applicationsByReference.get(reference));
     }
 
-    public Map<HostName, ApplicationInstance> getApplicationsByHostName() {
-        return applicationsByHostName;
+    public Optional<ApplicationInstance> getApplication(HostName hostname) {
+        if (applicationsByHostName == null) {
+            fillMaps();
+        }
+
+        return Optional.ofNullable(applicationsByHostName.get(hostname));
     }
 
     public Map<HostName, List<ServiceInstance>> getServiceInstancesByHostName() {
-        return applications.values().stream()
-                .flatMap(application -> application.serviceClusters().stream())
-                .flatMap(cluster -> cluster.serviceInstances().stream())
-                .collect(Collectors.groupingBy(ServiceInstance::hostName, Collectors.toList()));
+        if (servicesByHostName == null) {
+            fillMaps();
+        }
+
+        return servicesByHostName;
     }
 
-    private static Map<HostName, ApplicationInstance> applicationsByHostNames(Collection<ApplicationInstance> applications) {
-        Map<HostName, ApplicationInstance> hosts = new HashMap<>();
-        for (ApplicationInstance application : applications)
-            for (ServiceCluster cluster : application.serviceClusters())
+    private void fillMaps() {
+        Map<HostName, ApplicationInstance> applicationInstances = new HashMap<>();
+        Map<HostName, List<ServiceInstance>> serviceInstances = new HashMap<>();
+
+        for (ApplicationInstance application : applicationsByReference.values()) {
+            for (ServiceCluster cluster : application.serviceClusters()) {
                 for (ServiceInstance instance : cluster.serviceInstances()) {
-                    ApplicationInstance previous = hosts.put(instance.hostName(), application);
-                    if (previous != null && ! previous.equals(application))
+
+                    ApplicationInstance previous = applicationInstances.put(instance.hostName(), application);
+                    if (previous != null && !previous.equals(application)) {
                         throw new IllegalStateException("Major assumption broken: Multiple application instances contain host " +
-                                                        instance.hostName().s() + ": " + Arrays.asList(previous, application));
+                                instance.hostName().s() + ": " + Arrays.asList(previous, application));
+                    }
+
+                    serviceInstances
+                            .computeIfAbsent(instance.hostName(), key -> new ArrayList<>())
+                            .add(instance);
                 }
-        return hosts;
+            }
+        }
+
+        applicationsByHostName = Collections.unmodifiableMap(applicationInstances);
+        servicesByHostName = Collections.unmodifiableMap(serviceInstances);
     }
 
 }
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceMonitor.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceMonitor.java
index 49539c61e5d..4f52d7d229c 100644
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceMonitor.java
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/ServiceMonitor.java
@@ -1,6 +1,13 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.service.monitor;
 
+import com.yahoo.vespa.applicationmodel.ApplicationInstance;
+import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference;
+import com.yahoo.vespa.applicationmodel.HostName;
+
+import java.util.Optional;
+import java.util.Set;
+
 /**
  * The service monitor interface. A service monitor provides up to date information about the liveness status
  * (up, down or not known) of each service instance in a Vespa zone
@@ -12,7 +19,39 @@ public interface ServiceMonitor {
     /**
      * Returns a ServiceModel which contains the current liveness status (up, down or unknown) of all instances
      * of all services of all clusters of all applications in a zone.
+     *
+     * <p>Please use the more specific methods below to avoid the cost of this method.</p>
      */
     ServiceModel getServiceModelSnapshot();
 
+    default Set<ApplicationInstanceReference> getAllApplicationInstanceReferences() {
+        return getServiceModelSnapshot().getAllApplicationInstances().keySet();
+    }
+
+    default Optional<ApplicationInstanceReference> getApplicationInstanceReference(HostName hostname) {
+        return getApplication(hostname).map(ApplicationInstance::reference);
+    }
+
+    default Optional<ApplicationInstance> getApplication(HostName hostname) {
+        return getServiceModelSnapshot().getApplication(hostname);
+    }
+
+    default Optional<ApplicationInstance> getApplication(ApplicationInstanceReference reference) {
+        return getServiceModelSnapshot().getApplicationInstance(reference);
+    }
+
+    default Optional<ApplicationInstance> getApplicationNarrowedTo(HostName hostname) {
+        return getApplication(hostname);
+    }
+
+    /**
+     * Get notified of changes to the set of applications, or set of hosts assigned to an application.
+     *
+     * <p>When notified of model changes, the new model can be accessed through this interface
+     * by the listener. The model changes are visible to other threads strictly after the listener
+     * has been notified.</p>
+     *
+     * <p>WARNING: Methods on the listener may be invoked before returning from this method.</p>
+     */
+    default void registerListener(ServiceHostListener listener) { }
 }
diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImpl.java b/service-monitor/src/main/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImpl.java
index e3ea48ca9fe..e7a8d33ea14 100644
--- a/service-monitor/src/main/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImpl.java
+++ b/service-monitor/src/main/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImpl.java
@@ -73,6 +73,10 @@ public class SlobrokMonitorManagerImpl implements SlobrokApi, MonitorManager {
     }
 
     @Override
+    public void bootstrapComplete() {
+    }
+
+    @Override
     public List<Mirror.Entry> lookup(ApplicationId id, String pattern) {
         synchronized (monitor) {
             SlobrokMonitor slobrokMonitor = slobrokMonitors.get(id);
diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/duper/DuperModelManagerTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/duper/DuperModelManagerTest.java
index 67508f14e5a..3448ce46ff9 100644
--- a/service-monitor/src/test/java/com/yahoo/vespa/service/duper/DuperModelManagerTest.java
+++ b/service-monitor/src/test/java/com/yahoo/vespa/service/duper/DuperModelManagerTest.java
@@ -71,7 +71,6 @@ public class DuperModelManagerTest {
         verify(duperModel, times(0)).add(any());
         manager.infraApplicationActivated(id, proxyHostHosts);
         verify(duperModel, times(1)).add(any());
-        when(duperModel.contains(id)).thenReturn(true);
 
         verify(duperModel, times(0)).remove(any());
         manager.infraApplicationRemoved(id);
@@ -98,7 +97,6 @@ public class DuperModelManagerTest {
         List<HostName> hostnames1 = Stream.of("node11", "node12").map(HostName::from).collect(Collectors.toList());
         manager.infraApplicationActivated(firstId, hostnames1);
         verify(duperModel, times(1)).add(any());
-        when(duperModel.contains(firstId)).thenReturn(true);
 
         // Adding the second config server like application will be ignored
         List<HostName> hostnames2 = Stream.of("node21", "node22").map(HostName::from).collect(Collectors.toList());
diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/duper/DuperModelTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/duper/DuperModelTest.java
index 31fd266649a..2e38283a091 100644
--- a/service-monitor/src/test/java/com/yahoo/vespa/service/duper/DuperModelTest.java
+++ b/service-monitor/src/test/java/com/yahoo/vespa/service/duper/DuperModelTest.java
@@ -2,14 +2,23 @@
 package com.yahoo.vespa.service.duper;
 
 import com.yahoo.config.model.api.ApplicationInfo;
+import com.yahoo.config.model.api.HostInfo;
+import com.yahoo.config.model.api.Model;
 import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.HostName;
+import com.yahoo.vespa.service.monitor.DuperModelListener;
 import org.junit.Before;
 import org.junit.Test;
 
 import java.util.Arrays;
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertSame;
 import static org.junit.Assert.assertTrue;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.times;
@@ -22,37 +31,60 @@ import static org.mockito.Mockito.when;
  */
 public class DuperModelTest {
     private final DuperModel duperModel = new DuperModel();
-    private final ApplicationInfo application1 = mock(ApplicationInfo.class);
+
     private final ApplicationId id1 = ApplicationId.fromSerializedForm("tenant:app1:default");
+    private final ApplicationInfo application1 = mock(ApplicationInfo.class);
+    private final HostName hostname1_1 = HostName.from("hostname1-1");
+    private final HostName hostname1_2 = HostName.from("hostname1-2");
+
     private final ApplicationId id2 = ApplicationId.fromSerializedForm("tenant:app2:default");
     private final ApplicationInfo application2 = mock(ApplicationInfo.class);
+    private final HostName hostname2_1 = HostName.from("hostname2-1");
+
     private final DuperModelListener listener1 = mock(DuperModelListener.class);
 
     @Before
     public void setUp() {
-        when(application1.getApplicationId()).thenReturn(id1);
-        when(application2.getApplicationId()).thenReturn(id2);
+        setUpApplication(id1, application1, hostname1_1, hostname1_2);
+        setUpApplication(id2, application2, hostname2_1);
+    }
+
+    private void setUpApplication(ApplicationId id, ApplicationInfo info, HostName... hostnames) {
+        when(info.getApplicationId()).thenReturn(id);
+
+        Model model = mock(Model.class);
+        when(info.getModel()).thenReturn(model);
+
+        List<HostInfo> hostInfos = Arrays.stream(hostnames)
+                .map(hostname -> new HostInfo(hostname.value(), List.of()))
+                .collect(Collectors.toList());
+        when(model.getHosts()).thenReturn(hostInfos);
     }
 
     @Test
-    public void test() {
+    public void testListeners() {
+        assertEquals(0, duperModel.numberOfApplications());
+
         duperModel.add(application1);
-        assertTrue(duperModel.contains(id1));
+        assertEquals(Optional.of(application1), duperModel.getApplicationInfo(id1));
         assertEquals(Arrays.asList(application1), duperModel.getApplicationInfos());
+        assertEquals(1, duperModel.numberOfApplications());
 
         duperModel.registerListener(listener1);
         verify(listener1, times(1)).applicationActivated(application1);
         verifyNoMoreInteractions(listener1);
 
         duperModel.remove(id2);
+        assertEquals(1, duperModel.numberOfApplications());
         verifyNoMoreInteractions(listener1);
 
         duperModel.add(application2);
+        assertEquals(2, duperModel.numberOfApplications());
         verify(listener1, times(1)).applicationActivated(application2);
         verifyNoMoreInteractions(listener1);
 
         duperModel.remove(id1);
-        assertFalse(duperModel.contains(id1));
+        assertEquals(Optional.empty(), duperModel.getApplicationInfo(id1));
         verify(listener1, times(1)).applicationRemoved(id1);
         verifyNoMoreInteractions(listener1);
         assertEquals(Arrays.asList(application2), duperModel.getApplicationInfos());
@@ -60,4 +92,63 @@ public class DuperModelTest {
         duperModel.remove(id1);
         verifyNoMoreInteractions(listener1);
     }
+
+    @Test
+    public void hostIndices() {
+        assertEquals(0, duperModel.numberOfHosts());
+
+        duperModel.add(application1);
+        assertEquals(2, duperModel.numberOfHosts());
+        assertEquals(Optional.of(application1), duperModel.getApplicationInfo(hostname1_1));
+        assertEquals(Optional.empty(), duperModel.getApplicationInfo(hostname2_1));
+
+        duperModel.add(application2);
+        assertEquals(3, duperModel.numberOfHosts());
+        assertEquals(Optional.of(application1), duperModel.getApplicationInfo(hostname1_1));
+        assertEquals(Optional.of(application2), duperModel.getApplicationInfo(hostname2_1));
+
+        duperModel.remove(application1.getApplicationId());
+        assertEquals(1, duperModel.numberOfHosts());
+        assertEquals(Optional.empty(), duperModel.getApplicationInfo(hostname1_1));
+        assertEquals(Optional.of(application2), duperModel.getApplicationInfo(hostname2_1));
+
+        // Remove hostname2_1 and add hostname1_1 added to id2
+        setUpApplication(id2, application2, hostname1_1);
+        duperModel.add(application2);
+        assertEquals(1, duperModel.numberOfHosts());
+        assertEquals(Optional.of(application2), duperModel.getApplicationInfo(hostname1_1));
+        assertEquals(Optional.empty(), duperModel.getApplicationInfo(hostname2_1));
+    }
+
+    @Test
+    public void hostIndicesForOneApplication() {
+        assertEquals(0, duperModel.numberOfApplications());
+        assertEquals(0, duperModel.numberOfHosts());
+        assertEquals(Set.of(), duperModel.getHostnames(id1));
+
+        addAndVerifyApplication1("host1");
+        addAndVerifyApplication1("host1", "host2");
+        addAndVerifyApplication1("host2", "host3");
+        assertEquals(Optional.empty(), duperModel.getApplicationId(HostName.from("host1")));
+
+        duperModel.remove(id1);
+        assertEquals(0, duperModel.numberOfApplications());
+        assertEquals(0, duperModel.numberOfHosts());
+        assertEquals(Set.of(), duperModel.getHostnames(id1));
+    }
+
+    private void addAndVerifyApplication1(String... hostnameStrings) {
+        HostName[] hostnameArray = Stream.of(hostnameStrings).map(HostName::from).toArray(HostName[]::new);
+        setUpApplication(id1, application1, hostnameArray);
+        duperModel.add(application1);
+
+        assertEquals(1, duperModel.numberOfApplications());
+        Optional<ApplicationInfo> applicationInfo = duperModel.getApplicationInfo(id1);
+        assertTrue(applicationInfo.isPresent());
+        assertSame(application1, applicationInfo.get());
+
+        assertEquals(hostnameArray.length, duperModel.numberOfHosts());
+        assertEquals(Set.of(hostnameArray), duperModel.getHostnames(id1));
+        Stream.of(hostnameArray).forEach(hostname -> assertEquals(Optional.of(id1), duperModel.getApplicationId(hostname)));
+    }
 }
\ No newline at end of file
diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/model/ExampleModel.java b/service-monitor/src/test/java/com/yahoo/vespa/service/model/ExampleModel.java
index 0f7c0dde357..3fb10f1f24e 100644
--- a/service-monitor/src/test/java/com/yahoo/vespa/service/model/ExampleModel.java
+++ b/service-monitor/src/test/java/com/yahoo/vespa/service/model/ExampleModel.java
@@ -47,7 +47,7 @@ public class ExampleModel {
 
         Map<ApplicationId, ApplicationInfo> applicationInfos = new HashMap<>();
         applicationInfos.put(applicationInfo.getApplicationId(), applicationInfo);
-        return new SuperModel(applicationInfos);
+        return new SuperModel(applicationInfos, true);
     }
 
     public static ApplicationBuilder createApplication(String tenant,
diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/model/ModelGeneratorTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/model/ModelGeneratorTest.java
index ce13cf60082..7be4ff38a7f 100644
--- a/service-monitor/src/test/java/com/yahoo/vespa/service/model/ModelGeneratorTest.java
+++ b/service-monitor/src/test/java/com/yahoo/vespa/service/model/ModelGeneratorTest.java
@@ -38,9 +38,9 @@ public class ModelGeneratorTest {
 
     @Test
     public void toApplicationModel() throws Exception {
-        ModelGenerator modelGenerator = new ModelGenerator();
-
         Zone zone = new Zone(Environment.from(ENVIRONMENT), RegionName.from(REGION));
+        ModelGenerator modelGenerator = new ModelGenerator(zone);
+
 
         SlobrokMonitorManagerImpl slobrokMonitorManager = mock(SlobrokMonitorManagerImpl.class);
         when(slobrokMonitorManager.getStatus(any(), any(), any(), any()))
@@ -49,7 +49,6 @@ public class ModelGeneratorTest {
         ServiceModel serviceModel =
                 modelGenerator.toServiceModel(
                         getExampleApplicationInfos(),
-                        zone,
                         slobrokMonitorManager);
 
         Map<ApplicationInstanceReference,
diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/model/ServiceHostListenerAdapterTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/model/ServiceHostListenerAdapterTest.java
new file mode 100644
index 00000000000..6b1f0dfd5b6
--- /dev/null
+++ b/service-monitor/src/test/java/com/yahoo/vespa/service/model/ServiceHostListenerAdapterTest.java
@@ -0,0 +1,148 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.service.model;
+
+import com.yahoo.config.model.api.ApplicationInfo;
+import com.yahoo.config.model.api.HostInfo;
+import com.yahoo.config.model.api.Model;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.CloudName;
+import com.yahoo.config.provision.Environment;
+import com.yahoo.config.provision.RegionName;
+import com.yahoo.config.provision.SystemName;
+import com.yahoo.config.provision.Zone;
+import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference;
+import com.yahoo.vespa.applicationmodel.HostName;
+import com.yahoo.vespa.service.duper.DuperModel;
+import com.yahoo.vespa.service.monitor.ServiceHostListener;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
+import static org.mockito.Mockito.when;
+
+public class ServiceHostListenerAdapterTest {
+    private final Zone zone = new Zone(CloudName.from("AWS"), SystemName.cd, Environment.dev, RegionName.from("us-east-1"));
+    private final ModelGenerator generator = new ModelGenerator(zone);
+    private final ServiceHostListener listener = mock(ServiceHostListener.class);
+    private final ServiceHostListenerAdapter adapter = new ServiceHostListenerAdapter(listener, generator);
+    private final DuperModel duperModel = new DuperModel();
+
+    @Before
+    public void setUp() {
+        duperModel.registerListener(adapter);
+    }
+
+    @Test
+    public void test() {
+        var applicationId1 = ApplicationId.from("tnt", "app", "default");
+        var applicationId2 = ApplicationId.from("tnt2", "app2", "default2");
+
+        verifyNoMoreInteractions(listener);
+
+        activate(applicationId1, "host1", "host2");
+        verifyActivate(applicationId1, "host1", "host2");
+
+        activate(applicationId1, "host1", "host3");
+        verifyActivate(applicationId1, "host1", "host3");
+
+        activate(applicationId1, "host1", "host3");
+        verifyNoActivate();
+
+        activate(applicationId2, "host4");
+        verifyActivate(applicationId2, "host4");
+
+        activate(applicationId1, "host1", "host3");
+        verifyNoActivate();
+
+        removeAndVerify(applicationId1, true);
+
+        activate(applicationId1, "host1", "host5");
+        verifyActivate(applicationId1, "host1", "host5");
+    }
+
+    @Test
+    public void documentDuplicateHostnameStrangeness() {
+        var applicationId1 = ApplicationId.from("tnt", "app", "default");
+        var applicationInfo1 = makeApplicationInfo(applicationId1, "host1", "host2");
+        duperModel.add(applicationInfo1);
+        verifyActivate(applicationId1, "host1", "host2");
+
+        var applicationId2 = ApplicationId.from("tnt2", "app2", "default2");
+        var applicationInfo2 = makeApplicationInfo(applicationId2, "host2", "host3");
+        duperModel.add(applicationInfo2);
+        verifyActivate(applicationId2, "host2", "host3");
+
+        // Duplicate hosts doesn't affect the ServiceHostListener.
+
+        duperModel.add(applicationInfo1);
+        verifyNoMoreInteractions(listener);
+
+        duperModel.add(applicationInfo2);
+        verifyNoMoreInteractions(listener);
+
+        // But do affect host lookup in duper model.
+
+        assertEquals(Optional.of(applicationInfo1), getDuperModelApplicationInfo("host1"));
+        assertEquals(Optional.of(applicationInfo2), getDuperModelApplicationInfo("host2")); // <--
+        assertEquals(Optional.of(applicationInfo2), getDuperModelApplicationInfo("host3"));
+    }
+
+    private Optional<ApplicationInfo> getDuperModelApplicationInfo(String hostname) {
+        return duperModel.getApplicationInfo(com.yahoo.config.provision.HostName.from(hostname));
+    }
+
+    private void removeAndVerify(ApplicationId id, boolean listenerInvoked) {
+        duperModel.remove(id);
+
+        if (listenerInvoked) {
+            ApplicationInstanceReference reference = generator.toApplicationInstanceReference(id);
+            verify(listener, times(1)).onApplicationRemove(reference);
+        }
+
+        verifyNoMoreInteractions(listener);
+    }
+
+    private void verifyActivate(ApplicationId id, String... hostnames) {
+        Set<HostName> hostnameSet = Stream.of(hostnames)
+                .map(HostName::new)
+                .collect(Collectors.toSet());
+
+        ApplicationInstanceReference reference = generator.toApplicationInstanceReference(id);
+
+        verify(listener, times(1)).onApplicationActivate(reference, hostnameSet);
+        verifyNoMoreInteractions(listener);
+    }
+
+    private void verifyNoActivate() {
+        verifyNoMoreInteractions(listener);
+    }
+
+    private void activate(ApplicationId id, String... hostnames) {
+        duperModel.add(makeApplicationInfo(id, hostnames));
+    }
+
+    private ApplicationInfo makeApplicationInfo(ApplicationId applicationId, String... hostnames) {
+        var applicationInfo = mock(ApplicationInfo.class);
+        when(applicationInfo.getApplicationId()).thenReturn(applicationId);
+
+        var model = mock(Model.class);
+        when(applicationInfo.getModel()).thenReturn(model);
+
+        List<HostInfo> hostnameList = Stream.of(hostnames)
+                .map(hostname -> new HostInfo(hostname, List.of()))
+                .collect(Collectors.toList());
+        when(model.getHosts()).thenReturn(hostnameList);
+
+        return applicationInfo;
+    }
+}
\ No newline at end of file
diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/model/ServiceModelCacheTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/model/ServiceModelCacheTest.java
deleted file mode 100644
index 2d6921df374..00000000000
--- a/service-monitor/src/test/java/com/yahoo/vespa/service/model/ServiceModelCacheTest.java
+++ /dev/null
@@ -1,60 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-package com.yahoo.vespa.service.model;
-
-import com.yahoo.jdisc.Timer;
-import com.yahoo.vespa.service.monitor.ServiceModel;
-import org.junit.Test;
-
-import java.util.function.Supplier;
-
-import static org.junit.Assert.assertTrue;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
-
-public class ServiceModelCacheTest {
-    @SuppressWarnings("unchecked")
-    private final Supplier<ServiceModel> rawSupplier = mock(Supplier.class);
-    private final Timer timer = mock(Timer.class);
-    private final ServiceModelCache cache = new ServiceModelCache(rawSupplier, timer);
-
-    @Test
-    public void sanityCheck() {
-        ServiceModel serviceModel = mock(ServiceModel.class);
-        when(rawSupplier.get()).thenReturn(serviceModel);
-
-        long timeMillis = 0;
-        when(timer.currentTimeMillis()).thenReturn(timeMillis);
-
-        // Will always populate cache the first time
-        ServiceModel actualServiceModel = cache.get();
-        assertTrue(actualServiceModel == serviceModel);
-        verify(rawSupplier, times(1)).get();
-
-        // Cache hit
-        timeMillis += ServiceModelCache.EXPIRY_MILLIS / 2;
-        when(timer.currentTimeMillis()).thenReturn(timeMillis);
-        actualServiceModel = cache.get();
-        assertTrue(actualServiceModel == serviceModel);
-
-        // Cache expired
-        timeMillis += ServiceModelCache.EXPIRY_MILLIS + 1;
-        when(timer.currentTimeMillis()).thenReturn(timeMillis);
-
-        ServiceModel serviceModel2 = mock(ServiceModel.class);
-        when(rawSupplier.get()).thenReturn(serviceModel2);
-
-        actualServiceModel = cache.get();
-        assertTrue(actualServiceModel == serviceModel2);
-        // '2' because it's cumulative with '1' from the first times(1).
-        verify(rawSupplier, times(2)).get();
-
-        // Cache hit #2
-        timeMillis += 1;
-        when(timer.currentTimeMillis()).thenReturn(timeMillis);
-        actualServiceModel = cache.get();
-        assertTrue(actualServiceModel == serviceModel2);
-    }
-}
\ No newline at end of file
diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/model/ServiceModelProviderTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/model/ServiceMonitorImplTest.java
index 13f6da1534d..5b0855d0be5 100644
--- a/service-monitor/src/test/java/com/yahoo/vespa/service/model/ServiceModelProviderTest.java
+++ b/service-monitor/src/test/java/com/yahoo/vespa/service/model/ServiceMonitorImplTest.java
@@ -2,9 +2,8 @@
 package com.yahoo.vespa.service.model;
 
 import com.yahoo.config.model.api.ApplicationInfo;
-import com.yahoo.config.provision.Zone;
-import com.yahoo.vespa.service.monitor.ServiceModel;
 import com.yahoo.vespa.service.duper.DuperModelManager;
+import com.yahoo.vespa.service.monitor.ServiceModel;
 import com.yahoo.vespa.service.slobrok.SlobrokMonitorManagerImpl;
 import org.junit.Test;
 
@@ -17,19 +16,17 @@ import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
-public class ServiceModelProviderTest {
+public class ServiceMonitorImplTest {
     @Test
     public void sanityCheck() {
         SlobrokMonitorManagerImpl slobrokMonitorManager = mock(SlobrokMonitorManagerImpl.class);
         DuperModelManager duperModelManager = mock(DuperModelManager.class);
         ModelGenerator modelGenerator = mock(ModelGenerator.class);
-        Zone zone = mock(Zone.class);
-        ServiceModelProvider provider = new ServiceModelProvider(
+        ServiceMonitorImpl serviceMonitor = new ServiceMonitorImpl(
                 slobrokMonitorManager,
                 mock(ServiceMonitorMetrics.class),
                 duperModelManager,
-                modelGenerator,
-                zone);
+                modelGenerator);
 
         ApplicationInfo application1 = mock(ApplicationInfo.class);
         ApplicationInfo application2 = mock(ApplicationInfo.class);
@@ -37,8 +34,8 @@ public class ServiceModelProviderTest {
                 .collect(Collectors.toList());
         when(duperModelManager.getApplicationInfos()).thenReturn(applications);
 
-        ServiceModel serviceModel = provider.get();
+        ServiceModel serviceModel = serviceMonitor.getServiceModelSnapshot();
         verify(duperModelManager, times(1)).getApplicationInfos();
-        verify(modelGenerator).toServiceModel(applications, zone, slobrokMonitorManager);
+        verify(modelGenerator).toServiceModel(applications, slobrokMonitorManager);
     }
 }
\ No newline at end of file
diff --git a/slobrok/src/tests/configure/configure.cpp b/slobrok/src/tests/configure/configure.cpp
index fa509c17d0c..aa9826045ef 100644
--- a/slobrok/src/tests/configure/configure.cpp
+++ b/slobrok/src/tests/configure/configure.cpp
@@ -6,6 +6,7 @@
 #include <vespa/slobrok/server/slobrokserver.h>
 #include <vespa/config/config.h>
 #include <vespa/config-slobroks.h>
+#include <vespa/fnet/transport.h>
 #include <vespa/fnet/frt/supervisor.h>
 #include <vespa/vespalib/util/host_name.h>
 #include <sstream>
@@ -22,9 +23,6 @@ using slobrok::ConfigShim;
 using slobrok::SlobrokServer;
 using slobrok::ConfiguratorFactory;
 
-TEST_SETUP(Test);
-
-
 std::string
 createSpec(int port)
 {
@@ -92,10 +90,7 @@ compare(MirrorAPI &api, const char *pattern, SpecList expect)
     return false;
 }
 
-int
-Test::Main()
-{
-    TEST_INIT("configure_test");
+TEST("configure_test") {
 
     fnet::frt::StandaloneFRT orb1;
     fnet::frt::StandaloneFRT orb2;
@@ -213,5 +208,10 @@ Test::Main()
     serverOne.stop();
     serverTwo.stop();
 
-    TEST_DONE();
+    orb4.supervisor().GetTransport()->ShutDown(true);
+    orb3.supervisor().GetTransport()->ShutDown(true);
+    orb2.supervisor().GetTransport()->ShutDown(true);
+    orb1.supervisor().GetTransport()->ShutDown(true);
 }
+
+TEST_MAIN() { TEST_RUN_ALL(); }
\ No newline at end of file
diff --git a/staging_vespalib/CMakeLists.txt b/staging_vespalib/CMakeLists.txt
index 3c4e2a9f444..f2f8a41b68d 100644
--- a/staging_vespalib/CMakeLists.txt
+++ b/staging_vespalib/CMakeLists.txt
@@ -1,4 +1,9 @@
 # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+if(NOT CMAKE_HOST_SYSTEM_NAME STREQUAL "Darwin")
+set(STAGING_VESPALIB_DIRECTIO_TESTDIR src/tests/directio)
+set(STAGING_VESPALIB_PROCESS_MEMORY_STATS_TESTDIR src/tests/util/process_memory_stats)
+endif()
+
 vespa_define_module(
     DEPENDS
     fastos
@@ -12,7 +17,7 @@ vespa_define_module(
     src/tests/bits
     src/tests/clock
     src/tests/crc
-    src/tests/directio
+    ${STAGING_VESPALIB_DIRECTIO_TESTDIR}
     src/tests/encoding/base64
     src/tests/fileheader
     src/tests/floatingpointtype
@@ -30,8 +35,10 @@ vespa_define_module(
     src/tests/shutdownguard
     src/tests/state_server
     src/tests/stllike
+    src/tests/sequencedtaskexecutor
+    src/tests/singleexecutor
     src/tests/timer
-    src/tests/util/process_memory_stats
+    ${STAGING_VESPALIB_PROCESS_MEMORY_STATS_TESTDIR}
     src/tests/xmlserializable
 
     LIBS
diff --git a/staging_vespalib/src/tests/memorydatastore/memorydatastore.cpp b/staging_vespalib/src/tests/memorydatastore/memorydatastore.cpp
index 7d047e36566..6c71d0a2cf6 100644
--- a/staging_vespalib/src/tests/memorydatastore/memorydatastore.cpp
+++ b/staging_vespalib/src/tests/memorydatastore/memorydatastore.cpp
@@ -36,7 +36,7 @@ MemoryDataStoreTest::testMemoryDataStore()
 void
 MemoryDataStoreTest::testVariableSizeVector()
 {
-    VariableSizeVector v(256);
+    VariableSizeVector v(20000, 5*20000);
     for (size_t i(0); i < 10000; i++) {
         asciistream os;
         os << i;
diff --git a/staging_vespalib/src/tests/sequencedtaskexecutor/.gitignore b/staging_vespalib/src/tests/sequencedtaskexecutor/.gitignore
new file mode 100644
index 00000000000..523cfe5e3e1
--- /dev/null
+++ b/staging_vespalib/src/tests/sequencedtaskexecutor/.gitignore
@@ -0,0 +1,4 @@
+staging_vespalib_sequencedtaskexecutor_test_app
+staging_vespalib_sequencedtaskexecutor_benchmark_app
+staging_vespalib_adaptive_sequenced_executor_test_app
+staging_vespalib_foregroundtaskexecutor_test_app
diff --git a/staging_vespalib/src/tests/sequencedtaskexecutor/CMakeLists.txt b/staging_vespalib/src/tests/sequencedtaskexecutor/CMakeLists.txt
new file mode 100644
index 00000000000..6895eafd94a
--- /dev/null
+++ b/staging_vespalib/src/tests/sequencedtaskexecutor/CMakeLists.txt
@@ -0,0 +1,31 @@
+# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(staging_vespalib_sequencedtaskexecutor_benchmark_app TEST
+    SOURCES
+    sequencedtaskexecutor_benchmark.cpp
+    DEPENDS
+    staging_vespalib
+)
+
+vespa_add_executable(staging_vespalib_sequencedtaskexecutor_test_app TEST
+    SOURCES
+    sequencedtaskexecutor_test.cpp
+    DEPENDS
+    staging_vespalib
+)
+vespa_add_test(NAME staging_vespalib_sequencedtaskexecutor_test_app COMMAND staging_vespalib_sequencedtaskexecutor_test_app)
+
+vespa_add_executable(staging_vespalib_adaptive_sequenced_executor_test_app TEST
+    SOURCES
+    adaptive_sequenced_executor_test.cpp
+    DEPENDS
+    staging_vespalib
+)
+vespa_add_test(NAME staging_vespalib_adaptive_sequenced_executor_test_app COMMAND staging_vespalib_adaptive_sequenced_executor_test_app)
+
+vespa_add_executable(staging_vespalib_foregroundtaskexecutor_test_app TEST
+    SOURCES
+    foregroundtaskexecutor_test.cpp
+    DEPENDS
+    staging_vespalib
+)
+vespa_add_test(NAME staging_vespalib_foregroundtaskexecutor_test_app COMMAND staging_vespalib_foregroundtaskexecutor_test_app)
diff --git a/searchlib/src/tests/common/sequencedtaskexecutor/sequencedtaskexecutor_test.cpp b/staging_vespalib/src/tests/sequencedtaskexecutor/adaptive_sequenced_executor_test.cpp
index fcc7fd7300d..10f3f6089e3 100644
--- a/searchlib/src/tests/common/sequencedtaskexecutor/sequencedtaskexecutor_test.cpp
+++ b/staging_vespalib/src/tests/sequencedtaskexecutor/adaptive_sequenced_executor_test.cpp
@@ -1,25 +1,24 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
-#include <vespa/searchlib/common/sequencedtaskexecutor.h>
+#include <vespa/vespalib/util/adaptive_sequenced_executor.h>
 #include <vespa/vespalib/testkit/testapp.h>
 #include <vespa/vespalib/test/insertion_operators.h>
 
-#include <mutex>
 #include <condition_variable>
 #include <unistd.h>
 
 #include <vespa/log/log.h>
-LOG_SETUP("sequencedtaskexecutor_test");
+LOG_SETUP("adaptive_sequenced_executor_test");
 
-namespace search::common {
+namespace vespalib {
 
 
 class Fixture
 {
 public:
-    SequencedTaskExecutor _threads;
+    AdaptiveSequencedExecutor _threads;
 
-    Fixture() : _threads(2) { }
+    Fixture() : _threads(2, 2, 0, 1000) { }
 };
 
 
@@ -60,14 +59,14 @@ public:
     wait(int wantDone)
     {
         std::unique_lock<std::mutex> guard(_m);
-        _cv.wait(guard, [=] { return this->_done >= wantDone; });
+        _cv.wait(guard, [&] { return this->_done >= wantDone; });
     }
 };
 
 TEST_F("testExecute", Fixture) {
     std::shared_ptr<TestObj> tv(std::make_shared<TestObj>());
     EXPECT_EQUAL(0, tv->_val);
-    f._threads.execute(1, [=]() { tv->modify(0, 42); });
+    f._threads.execute(1, [&]() { tv->modify(0, 42); });
     tv->wait(1);
     EXPECT_EQUAL(0,  tv->_fail);
     EXPECT_EQUAL(42, tv->_val);
@@ -81,8 +80,8 @@ TEST_F("require that task with same component id are serialized", Fixture)
 {
     std::shared_ptr<TestObj> tv(std::make_shared<TestObj>());
     EXPECT_EQUAL(0, tv->_val);
-    f._threads.execute(0, [=]() { usleep(2000); tv->modify(0, 14); });
-    f._threads.execute(0, [=]() { tv->modify(14, 42); });
+    f._threads.execute(0, [&]() { usleep(2000); tv->modify(0, 14); });
+    f._threads.execute(0, [&]() { tv->modify(14, 42); });
     tv->wait(2);
     EXPECT_EQUAL(0,  tv->_fail);
     EXPECT_EQUAL(42, tv->_val);
@@ -97,8 +96,8 @@ TEST_F("require that task with different component ids are not serialized", Fixt
     for (tryCnt = 0; tryCnt < 100; ++tryCnt) {
         std::shared_ptr<TestObj> tv(std::make_shared<TestObj>());
         EXPECT_EQUAL(0, tv->_val);
-        f._threads.execute(0, [=]() { usleep(2000); tv->modify(0, 14); });
-        f._threads.execute(2, [=]() { tv->modify(14, 42); });
+        f._threads.execute(0, [&]() { usleep(2000); tv->modify(0, 14); });
+        f._threads.execute(2, [&]() { tv->modify(14, 42); });
         tv->wait(2);
         if (tv->_fail != 1) {
              continue;
@@ -118,8 +117,8 @@ TEST_F("require that task with same string component id are serialized", Fixture
 {
     std::shared_ptr<TestObj> tv(std::make_shared<TestObj>());
     EXPECT_EQUAL(0, tv->_val);
-    auto test2 = [=]() { tv->modify(14, 42); };
-    f._threads.execute(f._threads.getExecutorId("0"), [=]() { usleep(2000); tv->modify(0, 14); });
+    auto test2 = [&]() { tv->modify(14, 42); };
+    f._threads.execute(f._threads.getExecutorId("0"), [&]() { usleep(2000); tv->modify(0, 14); });
     f._threads.execute(f._threads.getExecutorId("0"), test2);
     tv->wait(2);
     EXPECT_EQUAL(0,  tv->_fail);
@@ -137,8 +136,8 @@ int detectSerializeFailure(Fixture &f, vespalib::stringref altComponentId, int t
     for (tryCnt = 0; tryCnt < tryLimit; ++tryCnt) {
         std::shared_ptr<TestObj> tv(std::make_shared<TestObj>());
         EXPECT_EQUAL(0, tv->_val);
-        f._threads.execute(f._threads.getExecutorId("0"), [=]() { usleep(2000); tv->modify(0, 14); });
-        f._threads.execute(f._threads.getExecutorId(altComponentId), [=]() { tv->modify(14, 42); });
+        f._threads.execute(f._threads.getExecutorId("0"), [&]() { usleep(2000); tv->modify(0, 14); });
+        f._threads.execute(f._threads.getExecutorId(altComponentId), [&]() { tv->modify(14, 42); });
         tv->wait(2);
         if (tv->_fail != 1) {
              continue;
@@ -230,12 +229,12 @@ TEST_F("require that executeLambda works", Fixture)
 }
 
 TEST("require that you get correct number of executors") {
-    SequencedTaskExecutor seven(7);
+    AdaptiveSequencedExecutor seven(7, 1, 0, 10);
     EXPECT_EQUAL(7u, seven.getNumExecutors());
 }
 
 TEST("require that you distribute well") {
-    SequencedTaskExecutor seven(7);
+    AdaptiveSequencedExecutor seven(7, 1, 0, 10);
     EXPECT_EQUAL(7u, seven.getNumExecutors());
     EXPECT_EQUAL(97u, seven.getComponentHashSize());
     EXPECT_EQUAL(0u, seven.getComponentEffectiveHashSize());
diff --git a/searchlib/src/tests/common/foregroundtaskexecutor/foregroundtaskexecutor_test.cpp b/staging_vespalib/src/tests/sequencedtaskexecutor/foregroundtaskexecutor_test.cpp
index 0cbd4bd9473..a2671bb81a7 100644
--- a/searchlib/src/tests/common/foregroundtaskexecutor/foregroundtaskexecutor_test.cpp
+++ b/staging_vespalib/src/tests/sequencedtaskexecutor/foregroundtaskexecutor_test.cpp
@@ -1,16 +1,15 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
-#include <vespa/searchlib/common/foregroundtaskexecutor.h>
+#include <vespa/vespalib/util/foregroundtaskexecutor.h>
 #include <vespa/vespalib/testkit/testapp.h>
 
-#include <mutex>
 #include <condition_variable>
 #include <unistd.h>
 
 #include <vespa/log/log.h>
 LOG_SETUP("foregroundtaskexecutor_test");
 
-namespace search::common {
+namespace vespalib {
 
 
 class Fixture
diff --git a/staging_vespalib/src/tests/sequencedtaskexecutor/sequencedtaskexecutor_benchmark.cpp b/staging_vespalib/src/tests/sequencedtaskexecutor/sequencedtaskexecutor_benchmark.cpp
new file mode 100644
index 00000000000..ba82651f1fc
--- /dev/null
+++ b/staging_vespalib/src/tests/sequencedtaskexecutor/sequencedtaskexecutor_benchmark.cpp
@@ -0,0 +1,71 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/vespalib/util/sequencedtaskexecutor.h>
+#include <vespa/vespalib/util/adaptive_sequenced_executor.h>
+#include <vespa/vespalib/util/lambdatask.h>
+#include <vespa/vespalib/util/time.h>
+#include <atomic>
+#include <cinttypes>
+
+using vespalib::ISequencedTaskExecutor;
+using vespalib::SequencedTaskExecutor;
+using vespalib::AdaptiveSequencedExecutor;
+using ExecutorId = vespalib::ISequencedTaskExecutor::ExecutorId;
+
+size_t do_work(size_t size) {
+    size_t ret = 0;
+    for (size_t i = 0; i < size; ++i) {
+        for (size_t j = 0; j < 128; ++j) {
+            ret = (ret + i) * j;
+        }
+    }
+    return ret;
+}
+
+struct SimpleParams {
+    int argc;
+    char **argv;
+    int idx;
+    SimpleParams(int argc_in, char **argv_in) : argc(argc_in), argv(argv_in), idx(0) {}
+    int next(const char *name, int fallback) {
+        ++idx;
+        int value = 0;
+        if (argc > idx) {
+            value = atoi(argv[idx]);
+        } else {
+            value = fallback;
+        }
+        fprintf(stderr, "param %s: %d\n", name, value);
+        return value;
+    }
+};
+
+int main(int argc, char **argv) {
+    SimpleParams params(argc, argv);
+    bool use_adaptive_executor = params.next("use_adaptive_executor", 0);
+    bool optimize_for_throughput = params.next("optimize_for_throughput", 0);
+    size_t num_tasks = params.next("num_tasks", 1000000);
+    size_t num_strands = params.next("num_strands", 4);
+    size_t task_limit = params.next("task_limit", 1000);
+    size_t num_threads = params.next("num_threads", num_strands);
+    size_t max_waiting = params.next("max_waiting", optimize_for_throughput ? 32 : 0);
+    size_t work_size = params.next("work_size", 0);
+    std::atomic<long> counter(0);
+    std::unique_ptr<ISequencedTaskExecutor> executor;
+    if (use_adaptive_executor) {
+        executor = std::make_unique<AdaptiveSequencedExecutor>(num_strands, num_threads, max_waiting, task_limit);
+    } else {
+        auto optimize = optimize_for_throughput
+                        ? vespalib::Executor::OptimizeFor::THROUGHPUT
+                        : vespalib::Executor::OptimizeFor::LATENCY;
+        executor = SequencedTaskExecutor::create(num_strands, task_limit, optimize);
+    }
+    vespalib::Timer timer;
+    for (size_t task_id = 0; task_id < num_tasks; ++task_id) {
+        executor->executeTask(ExecutorId(task_id % num_strands),
+                              vespalib::makeLambdaTask([&counter,work_size] { (void) do_work(work_size); counter++; }));
+    }
+    executor.reset();
+    fprintf(stderr, "\ntotal time: %" PRId64 " ms\n", vespalib::count_ms(timer.elapsed()));
+    return (size_t(counter) == num_tasks) ? 0 : 1;
+}
diff --git a/staging_vespalib/src/tests/sequencedtaskexecutor/sequencedtaskexecutor_test.cpp b/staging_vespalib/src/tests/sequencedtaskexecutor/sequencedtaskexecutor_test.cpp
new file mode 100644
index 00000000000..70d0f1c743d
--- /dev/null
+++ b/staging_vespalib/src/tests/sequencedtaskexecutor/sequencedtaskexecutor_test.cpp
@@ -0,0 +1,272 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/vespalib/util/sequencedtaskexecutor.h>
+#include <vespa/vespalib/util/adaptive_sequenced_executor.h>
+
+#include <vespa/vespalib/testkit/testapp.h>
+#include <vespa/vespalib/test/insertion_operators.h>
+
+#include <condition_variable>
+#include <unistd.h>
+
+#include <vespa/log/log.h>
+LOG_SETUP("sequencedtaskexecutor_test");
+
+namespace vespalib {
+
+
+class Fixture
+{
+public:
+    std::unique_ptr<ISequencedTaskExecutor> _threads;
+
+    Fixture() : _threads(SequencedTaskExecutor::create(2)) { }
+};
+
+
+class TestObj
+{
+public:
+    std::mutex _m;
+    std::condition_variable _cv;
+    int _done;
+    int _fail;
+    int _val;
+
+    TestObj()
+        : _m(),
+          _cv(),
+          _done(0),
+          _fail(0),
+          _val(0)
+    {
+    }
+
+    void
+    modify(int oldValue, int newValue)
+    {
+        {
+            std::lock_guard<std::mutex> guard(_m);
+            if (_val == oldValue) {
+                _val = newValue;
+            } else {
+                ++_fail;
+            }
+            ++_done;
+        }
+        _cv.notify_all();
+    }
+
+    void
+    wait(int wantDone)
+    {
+        std::unique_lock<std::mutex> guard(_m);
+        _cv.wait(guard, [=] { return this->_done >= wantDone; });
+    }
+};
+
+TEST_F("testExecute", Fixture) {
+    std::shared_ptr<TestObj> tv(std::make_shared<TestObj>());
+    EXPECT_EQUAL(0, tv->_val);
+    f._threads->execute(1, [=]() { tv->modify(0, 42); });
+    tv->wait(1);
+    EXPECT_EQUAL(0,  tv->_fail);
+    EXPECT_EQUAL(42, tv->_val);
+    f._threads->sync();
+    EXPECT_EQUAL(0,  tv->_fail);
+    EXPECT_EQUAL(42, tv->_val);
+}
+
+
+TEST_F("require that task with same component id are serialized", Fixture)
+{
+    std::shared_ptr<TestObj> tv(std::make_shared<TestObj>());
+    EXPECT_EQUAL(0, tv->_val);
+    f._threads->execute(0, [=]() { usleep(2000); tv->modify(0, 14); });
+    f._threads->execute(0, [=]() { tv->modify(14, 42); });
+    tv->wait(2);
+    EXPECT_EQUAL(0,  tv->_fail);
+    EXPECT_EQUAL(42, tv->_val);
+    f._threads->sync();
+    EXPECT_EQUAL(0,  tv->_fail);
+    EXPECT_EQUAL(42, tv->_val);
+}
+
+TEST_F("require that task with different component ids are not serialized", Fixture)
+{
+    int tryCnt = 0;
+    for (tryCnt = 0; tryCnt < 100; ++tryCnt) {
+        std::shared_ptr<TestObj> tv(std::make_shared<TestObj>());
+        EXPECT_EQUAL(0, tv->_val);
+        f._threads->execute(0, [=]() { usleep(2000); tv->modify(0, 14); });
+        f._threads->execute(2, [=]() { tv->modify(14, 42); });
+        tv->wait(2);
+        if (tv->_fail != 1) {
+             continue;
+        }
+        EXPECT_EQUAL(1,  tv->_fail);
+        EXPECT_EQUAL(14, tv->_val);
+        f._threads->sync();
+        EXPECT_EQUAL(1,  tv->_fail);
+        EXPECT_EQUAL(14, tv->_val);
+        break;
+    }
+    EXPECT_TRUE(tryCnt < 100);
+}
+
+
+TEST_F("require that task with same string component id are serialized", Fixture)
+{
+    std::shared_ptr<TestObj> tv(std::make_shared<TestObj>());
+    EXPECT_EQUAL(0, tv->_val);
+    auto test2 = [=]() { tv->modify(14, 42); };
+    f._threads->execute(f._threads->getExecutorId("0"), [=]() { usleep(2000); tv->modify(0, 14); });
+    f._threads->execute(f._threads->getExecutorId("0"), test2);
+    tv->wait(2);
+    EXPECT_EQUAL(0,  tv->_fail);
+    EXPECT_EQUAL(42, tv->_val);
+    f._threads->sync();
+    EXPECT_EQUAL(0,  tv->_fail);
+    EXPECT_EQUAL(42, tv->_val);
+}
+
+namespace {
+
+int detectSerializeFailure(Fixture &f, vespalib::stringref altComponentId, int tryLimit)
+{
+    int tryCnt = 0;
+    for (tryCnt = 0; tryCnt < tryLimit; ++tryCnt) {
+        std::shared_ptr<TestObj> tv(std::make_shared<TestObj>());
+        EXPECT_EQUAL(0, tv->_val);
+        f._threads->execute(f._threads->getExecutorId("0"), [=]() { usleep(2000); tv->modify(0, 14); });
+        f._threads->execute(f._threads->getExecutorId(altComponentId), [=]() { tv->modify(14, 42); });
+        tv->wait(2);
+        if (tv->_fail != 1) {
+             continue;
+        }
+        EXPECT_EQUAL(1,  tv->_fail);
+        EXPECT_EQUAL(14, tv->_val);
+        f._threads->sync();
+        EXPECT_EQUAL(1,  tv->_fail);
+        EXPECT_EQUAL(14, tv->_val);
+        break;
+    }
+    return tryCnt;
+}
+
+vespalib::string makeAltComponentId(Fixture &f)
+{
+    int tryCnt = 0;
+    char altComponentId[20];
+    ISequencedTaskExecutor::ExecutorId executorId0 = f._threads->getExecutorId("0");
+    for (tryCnt = 1; tryCnt < 100; ++tryCnt) {
+        sprintf(altComponentId, "%d", tryCnt);
+        if (f._threads->getExecutorId(altComponentId) == executorId0) {
+            break;
+        }
+    }
+    EXPECT_TRUE(tryCnt < 100);
+    return altComponentId;
+}
+
+}
+
+TEST_F("require that task with different string component ids are not serialized", Fixture)
+{
+    int tryCnt = detectSerializeFailure(f, "2", 100);
+    EXPECT_TRUE(tryCnt < 100);
+}
+
+
+TEST_F("require that task with different string component ids mapping to the same executor id are serialized",
+       Fixture)
+{
+    vespalib::string altComponentId = makeAltComponentId(f);
+    LOG(info, "second string component id is \"%s\"", altComponentId.c_str());
+    int tryCnt = detectSerializeFailure(f, altComponentId, 100);
+    EXPECT_TRUE(tryCnt == 100);
+}
+
+
+TEST_F("require that execute works with const lambda", Fixture)
+{
+    int i = 5;
+    std::vector<int> res;
+    const auto lambda = [i, &res]() mutable
+                        { res.push_back(i--); res.push_back(i--); };
+    f._threads->execute(0, lambda);
+    f._threads->execute(0, lambda);
+    f._threads->sync();
+    std::vector<int> exp({5, 4, 5, 4});
+    EXPECT_EQUAL(exp, res);
+    EXPECT_EQUAL(5, i);
+}
+
+TEST_F("require that execute works with reference to lambda", Fixture)
+{
+    int i = 5;
+    std::vector<int> res;
+    auto lambda = [i, &res]() mutable
+                  { res.push_back(i--); res.push_back(i--); };
+    auto &lambdaref = lambda;
+    f._threads->execute(0, lambdaref);
+    f._threads->execute(0, lambdaref);
+    f._threads->sync();
+    std::vector<int> exp({5, 4, 5, 4});
+    EXPECT_EQUAL(exp, res);
+    EXPECT_EQUAL(5, i);
+}
+
+TEST_F("require that executeLambda works", Fixture)
+{
+    int i = 5;
+    std::vector<int> res;
+    const auto lambda = [i, &res]() mutable
+                        { res.push_back(i--); res.push_back(i--); };
+    f._threads->executeLambda(ISequencedTaskExecutor::ExecutorId(0), lambda);
+    f._threads->sync();
+    std::vector<int> exp({5, 4});
+    EXPECT_EQUAL(exp, res);
+    EXPECT_EQUAL(5, i);
+}
+
+TEST("require that you get correct number of executors") {
+    auto seven = SequencedTaskExecutor::create(7);
+    EXPECT_EQUAL(7u, seven->getNumExecutors());
+}
+
+TEST("require that you distribute well") {
+    auto seven = SequencedTaskExecutor::create(7);
+    EXPECT_EQUAL(7u, seven->getNumExecutors());
+    EXPECT_EQUAL(97u, seven->getComponentHashSize());
+    EXPECT_EQUAL(0u, seven->getComponentEffectiveHashSize());
+    for (uint32_t id=0; id < 1000; id++) {
+        EXPECT_EQUAL((id%97)%7, seven->getExecutorId(id).getId());
+    }
+    EXPECT_EQUAL(97u, seven->getComponentHashSize());
+    EXPECT_EQUAL(97u, seven->getComponentEffectiveHashSize());
+}
+
+TEST("Test creation of different types") {
+    auto iseq = SequencedTaskExecutor::create(1);
+
+    EXPECT_EQUAL(1u, iseq->getNumExecutors());
+    auto * seq = dynamic_cast<SequencedTaskExecutor *>(iseq.get());
+    ASSERT_TRUE(seq != nullptr);
+
+    iseq = SequencedTaskExecutor::create(1, 1000, Executor::OptimizeFor::LATENCY);
+    seq = dynamic_cast<SequencedTaskExecutor *>(iseq.get());
+    ASSERT_TRUE(seq != nullptr);
+
+    iseq = SequencedTaskExecutor::create(1, 1000, Executor::OptimizeFor::THROUGHPUT);
+    seq = dynamic_cast<SequencedTaskExecutor *>(iseq.get());
+    ASSERT_TRUE(seq != nullptr);
+
+    iseq = SequencedTaskExecutor::create(1, 1000, Executor::OptimizeFor::ADAPTIVE, 17);
+    auto aseq = dynamic_cast<AdaptiveSequencedExecutor *>(iseq.get());
+    ASSERT_TRUE(aseq != nullptr);
+}
+
+}
+
+TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/staging_vespalib/src/tests/singleexecutor/CMakeLists.txt b/staging_vespalib/src/tests/singleexecutor/CMakeLists.txt
new file mode 100644
index 00000000000..c5d42d2c8c5
--- /dev/null
+++ b/staging_vespalib/src/tests/singleexecutor/CMakeLists.txt
@@ -0,0 +1,8 @@
+# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(staging_vespalib_singleexecutor_test_app TEST
+    SOURCES
+    singleexecutor_test.cpp
+    DEPENDS
+    staging_vespalib
+)
+vespa_add_test(NAME staging_vespalib_singleexecutor_test_app COMMAND staging_vespalib_singleexecutor_test_app)
diff --git a/staging_vespalib/src/tests/singleexecutor/singleexecutor_test.cpp b/staging_vespalib/src/tests/singleexecutor/singleexecutor_test.cpp
new file mode 100644
index 00000000000..5dacaa5d204
--- /dev/null
+++ b/staging_vespalib/src/tests/singleexecutor/singleexecutor_test.cpp
@@ -0,0 +1,80 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/vespalib/testkit/testapp.h>
+
+#include <vespa/vespalib/util/singleexecutor.h>
+#include <vespa/vespalib/util/lambdatask.h>
+#include <atomic>
+
+using namespace vespalib;
+
+TEST("test that all tasks are executed") {
+
+    std::atomic<uint64_t> counter(0);
+    SingleExecutor executor(10);
+
+    for (uint64_t i(0); i < 10; i++) {
+        executor.execute(makeLambdaTask([&counter] {counter++;}));
+    }
+    executor.sync();
+    EXPECT_EQUAL(10u, counter);
+
+    counter = 0;
+    for (uint64_t i(0); i < 10000; i++) {
+        executor.execute(makeLambdaTask([&counter] {counter++;}));
+    }
+    executor.sync();
+    EXPECT_EQUAL(10000u, counter);
+}
+
+void verifyResizeTaskLimit(bool up) {
+    Monitor lock;
+    std::atomic<uint64_t> started(0);
+    std::atomic<uint64_t> allowed(0);
+    SingleExecutor executor(10);
+
+    uint32_t targetTaskLimit = up ? 20 : 5;
+    uint32_t roundedTaskLimit = roundUp2inN(targetTaskLimit);
+    EXPECT_NOT_EQUAL(16u, roundedTaskLimit);
+
+    for (uint64_t i(0); i < 10; i++) {
+        executor.execute(makeLambdaTask([&lock, &started, &allowed] {
+            started++;
+            MonitorGuard guard(lock);
+            while (allowed < started) {
+                guard.wait(1ms);
+            }
+        }));
+    }
+    while (started < 1);
+    EXPECT_EQUAL(1u, started);
+    executor.setTaskLimit(targetTaskLimit);
+    EXPECT_EQUAL(16u, executor.getTaskLimit());
+    allowed = 5;
+    while (started < 6);
+    EXPECT_EQUAL(6u, started);
+    EXPECT_EQUAL(16u, executor.getTaskLimit());
+    allowed = 10;
+    while (started < 10);
+    EXPECT_EQUAL(10u, started);
+    EXPECT_EQUAL(16u, executor.getTaskLimit());
+    executor.execute(makeLambdaTask([&lock, &started, &allowed] {
+        started++;
+        MonitorGuard guard(lock);
+        while (allowed < started) {
+            guard.wait(1ms);
+        }
+    }));
+    while (started < 11);
+    EXPECT_EQUAL(11u, started);
+    EXPECT_EQUAL(roundedTaskLimit, executor.getTaskLimit());
+    allowed = 11;
+}
+TEST("test that resizing up and down works") {
+    TEST_DO(verifyResizeTaskLimit(true));
+    TEST_DO(verifyResizeTaskLimit(false));
+
+
+}
+
+TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/staging_vespalib/src/vespa/vespalib/objects/visit.cpp b/staging_vespalib/src/vespa/vespalib/objects/visit.cpp
index 10df89e3ae0..2672f61fdb6 100644
--- a/staging_vespalib/src/vespa/vespalib/objects/visit.cpp
+++ b/staging_vespalib/src/vespa/vespalib/objects/visit.cpp
@@ -19,39 +19,52 @@ void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, bool val
     self.visitBool(name, value);
 }
 
-void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, int8_t value)
+void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, char value)
 {
     self.visitInt(name, value);
 }
 
-void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, uint8_t value)
+void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, signed char value)
 {
     self.visitInt(name, value);
 }
 
-void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, int16_t value)
+void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, unsigned char value)
 {
     self.visitInt(name, value);
 }
 
-void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, uint16_t value)
+void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, short value)
 {
     self.visitInt(name, value);
 }
 
-void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, int32_t value) {
+void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, unsigned short value)
+{
+    self.visitInt(name, value);
+}
+
+void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, int value) {
+    self.visitInt(name, value);
+}
+
+void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, unsigned int value) {
+    self.visitInt(name, value);
+}
+
+void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, long value) {
     self.visitInt(name, value);
 }
 
-void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, uint32_t value) {
+void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, unsigned long value) {
     self.visitInt(name, value);
 }
 
-void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, int64_t value) {
+void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, long long value) {
     self.visitInt(name, value);
 }
 
-void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, uint64_t value) {
+void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, unsigned long long value) {
     self.visitInt(name, value);
 }
 
diff --git a/staging_vespalib/src/vespa/vespalib/objects/visit.h b/staging_vespalib/src/vespa/vespalib/objects/visit.h
index 22c73dceefd..dec994421b4 100644
--- a/staging_vespalib/src/vespa/vespalib/objects/visit.h
+++ b/staging_vespalib/src/vespa/vespalib/objects/visit.h
@@ -11,14 +11,17 @@ namespace vespalib {
 void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, const vespalib::Identifiable *obj);
 void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, const vespalib::Identifiable &obj);
 void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, bool value);
-void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, int8_t value);
-void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, uint8_t value);
-void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, int16_t value);
-void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, uint16_t value);
-void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, int32_t value);
-void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, uint32_t value);
-void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, int64_t value);
-void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, uint64_t value);
+void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, char value);
+void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, signed char value);
+void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, unsigned char value);
+void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, short value);
+void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, unsigned short value);
+void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, int value);
+void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, unsigned int value);
+void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, long value);
+void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, unsigned long value);
+void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, long long value);
+void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, unsigned long long value);
 void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, float value);
 void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, double value);
 void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, const vespalib::string &value);
diff --git a/staging_vespalib/src/vespa/vespalib/stllike/lrucache_map.h b/staging_vespalib/src/vespa/vespalib/stllike/lrucache_map.h
index 07137263cf6..ea5ccf0659b 100644
--- a/staging_vespalib/src/vespa/vespalib/stllike/lrucache_map.h
+++ b/staging_vespalib/src/vespa/vespalib/stllike/lrucache_map.h
@@ -28,32 +28,32 @@ struct LinkedValue : public LinkedValueBase
 template<typename K, typename V, typename H = vespalib::hash<K>, typename EQ = std::equal_to<K> >
 struct LruParam
 {
-    typedef LinkedValue<V> LV;
-    typedef std::pair< K, LV > value_type;
-    typedef vespalib::Select1st< value_type > select_key;
-    typedef K Key;
-    typedef V Value;
-    typedef H Hash;
-    typedef EQ Equal;
-    typedef hashtable< Key, value_type, Hash, Equal, select_key > HashTable;
+    using LV = LinkedValue<V>;
+    using value_type = std::pair< K, LV >;
+    using select_key = vespalib::Select1st< value_type >;
+    using Key = K;
+    using Value = V;
+    using Hash = H;
+    using Equal = EQ;
+    using HashTable = hashtable< Key, value_type, Hash, Equal, select_key >;
 };
 
 template< typename P >
 class lrucache_map : private P::HashTable
 {
 private:
-    typedef typename P::HashTable   HashTable;
-    typedef typename P::Value       V;
-    typedef typename P::Key         K;
-    typedef typename P::value_type  value_type;
-    typedef typename P::LV  LV;
-    typedef typename HashTable::iterator internal_iterator;
-    typedef typename HashTable::next_t next_t;
-    typedef typename HashTable::NodeStore NodeStore;
+    using HashTable = typename P::HashTable;
+    using V = typename P::Value;
+    using K = typename P::Key;
+    using value_type = typename P::value_type;
+    using LV = typename P::LV;
+    using internal_iterator = typename HashTable::iterator;
+    using next_t = typename HashTable::next_t;
+    using NodeStore = typename HashTable::NodeStore;
 protected:
     static constexpr size_t UNLIMITED = std::numeric_limits<size_t>::max();
 public:
-    typedef typename HashTable::insert_result insert_result;
+    using insert_result = typename HashTable::insert_result;
 
     class iterator {
     public:
@@ -108,7 +108,7 @@ public:
     /**
      * This fetches the object without modifying the lru list.
      */
-    const V & get(const K & key) { return HashTable::find(key)->second._value; }
+    const V & get(const K & key) const { return HashTable::find(key)->second._value; }
 
     /**
      * This simply erases the object.
@@ -133,13 +133,11 @@ public:
     insert_result insert(const K & key, V && value);
 
     /**
-     * Return the object with the given key. If it does not exist an empty one will be created.
-     * This can be used as an insert.
-     * Object is then put at head of LRU list.
+     * Return pointer to the object with the given key.
+     * Object is then put at head of LRU list if found.
+     * If not found nullptr is returned.
      */
-    const V & operator [] (const K & key) const {
-        return const_cast<lrucache_map<P> *>(this)->findAndRef(key).second._value;
-    }
+    V * findAndRef(const K & key);
 
     /**
      * Return the object with the given key. If it does not exist an empty one will be created.
@@ -177,13 +175,12 @@ public:
     void swap(lrucache_map & rhs);
 
 private:
-    typedef std::pair<uint32_t, uint32_t> MoveRecord;
-    typedef std::vector<MoveRecord> MoveRecords;
+    using MoveRecord = std::pair<uint32_t, uint32_t>;
+    using MoveRecords = std::vector<MoveRecord>;
     /**
      * Implements the resize of the hashtable
      */
     void move(NodeStore && oldStore) override;
-    internal_iterator findAndRef(const K & key);
     void ref(const internal_iterator & it);
     insert_result insert(value_type && value);
     void removeOld();
diff --git a/staging_vespalib/src/vespa/vespalib/stllike/lrucache_map.hpp b/staging_vespalib/src/vespa/vespalib/stllike/lrucache_map.hpp
index 61147229497..839a93cc5ca 100644
--- a/staging_vespalib/src/vespa/vespalib/stllike/lrucache_map.hpp
+++ b/staging_vespalib/src/vespa/vespalib/stllike/lrucache_map.hpp
@@ -74,7 +74,7 @@ lrucache_map<P>::lrucache_map(size_t maxElems) :
 { }
 
 template< typename P >
-lrucache_map<P>::~lrucache_map() { }
+lrucache_map<P>::~lrucache_map() = default;
 
 template< typename P >
 void
@@ -263,14 +263,17 @@ lrucache_map<P>::operator [] (const K & key)
 }
 
 template< typename P >
-typename lrucache_map<P>::internal_iterator
+typename P::Value *
 lrucache_map<P>::findAndRef(const K & key)
 {
     internal_iterator found = HashTable::find(key);
     if (found != HashTable::end()) {
-        ref(found);
+        if (size()*2 > capacity()) {
+            ref(found);
+        }
+        return &found->second._value;
     }
-    return found;
+    return nullptr;
 }
 
 }
diff --git a/staging_vespalib/src/vespa/vespalib/util/CMakeLists.txt b/staging_vespalib/src/vespa/vespalib/util/CMakeLists.txt
index 71364a813f6..586e06396e7 100644
--- a/staging_vespalib/src/vespa/vespalib/util/CMakeLists.txt
+++ b/staging_vespalib/src/vespa/vespalib/util/CMakeLists.txt
@@ -1,11 +1,14 @@
 # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 vespa_add_library(staging_vespalib_vespalib_util OBJECT
     SOURCES
+    adaptive_sequenced_executor.cpp
     bits.cpp
     clock.cpp
     crc.cpp
     doom.cpp
+    foregroundtaskexecutor.cpp
     growablebytebuffer.cpp
+    isequencedtaskexecutor.cpp
     jsonexception.cpp
     jsonstream.cpp
     jsonwriter.cpp
@@ -15,8 +18,11 @@ vespa_add_library(staging_vespalib_vespalib_util OBJECT
     programoptions_testutils.cpp
     document_runnable.cpp
     rusage.cpp
+    sequencedtaskexecutor.cpp
+    sequencedtaskexecutorobserver.cpp
     shutdownguard.cpp
     scheduledexecutor.cpp
+    singleexecutor.cpp
     xmlserializable.cpp
     xmlstream.cpp
     DEPENDS
diff --git a/staging_vespalib/src/vespa/vespalib/util/adaptive_sequenced_executor.cpp b/staging_vespalib/src/vespa/vespalib/util/adaptive_sequenced_executor.cpp
new file mode 100644
index 00000000000..50bc3b020a8
--- /dev/null
+++ b/staging_vespalib/src/vespa/vespalib/util/adaptive_sequenced_executor.cpp
@@ -0,0 +1,324 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "adaptive_sequenced_executor.h"
+
+namespace vespalib {
+
+//-----------------------------------------------------------------------------
+
+AdaptiveSequencedExecutor::Strand::Strand()
+    : state(State::IDLE),
+      queue()
+{
+}
+
+AdaptiveSequencedExecutor::Strand::~Strand()
+{
+    assert(queue.empty());
+}
+
+//-----------------------------------------------------------------------------
+
+AdaptiveSequencedExecutor::Worker::Worker()
+    : cond(),
+      state(State::RUNNING),
+      strand(nullptr)
+{
+}
+
+AdaptiveSequencedExecutor::Worker::~Worker()
+{
+    assert(state == State::DONE);
+    assert(strand == nullptr);
+}
+
+//-----------------------------------------------------------------------------
+
+AdaptiveSequencedExecutor::Self::Self()
+    : cond(),
+      state(State::OPEN),
+      waiting_tasks(0),
+      pending_tasks(0)
+{
+}
+
+AdaptiveSequencedExecutor::Self::~Self()
+{
+    assert(state == State::CLOSED);
+    assert(waiting_tasks == 0);
+    assert(pending_tasks == 0);
+}
+
+//-----------------------------------------------------------------------------
+
+AdaptiveSequencedExecutor::ThreadTools::ThreadTools(AdaptiveSequencedExecutor &parent_in)
+    : parent(parent_in),
+      pool(std::make_unique<FastOS_ThreadPool>(STACK_SIZE)),
+      allow_worker_exit()
+{
+}
+
+AdaptiveSequencedExecutor::ThreadTools::~ThreadTools()
+{
+    assert(pool->isClosed());
+}
+
+void
+AdaptiveSequencedExecutor::ThreadTools::Run(FastOS_ThreadInterface *, void *)
+{
+    parent.worker_main();
+}
+
+void
+AdaptiveSequencedExecutor::ThreadTools::start(size_t num_threads)
+{
+    for (size_t i = 0; i < num_threads; ++i) {
+        FastOS_ThreadInterface *thread = pool->NewThread(this);
+        assert(thread != nullptr);
+        (void)thread;
+    }
+}
+
+void
+AdaptiveSequencedExecutor::ThreadTools::close()
+{
+    allow_worker_exit.countDown();
+    pool->Close();
+}
+
+//-----------------------------------------------------------------------------
+
+void
+AdaptiveSequencedExecutor::maybe_block_self(std::unique_lock<std::mutex> &lock)
+{
+    while (_self.state == Self::State::BLOCKED) {
+        _self.cond.wait(lock);
+    }
+    while ((_self.state == Self::State::OPEN) && (_self.pending_tasks >= _cfg.max_pending)) {
+        _self.state = Self::State::BLOCKED;
+        while (_self.state == Self::State::BLOCKED) {
+            _self.cond.wait(lock);
+        }
+    }
+}
+
+bool
+AdaptiveSequencedExecutor::maybe_unblock_self(const std::unique_lock<std::mutex> &)
+{
+    if ((_self.state == Self::State::BLOCKED) && (_self.pending_tasks < _cfg.wakeup_limit)) {
+        _self.state = Self::State::OPEN;
+        return true;
+    }
+    return false;
+}
+
+AdaptiveSequencedExecutor::Worker *
+AdaptiveSequencedExecutor::get_worker_to_wake(const std::unique_lock<std::mutex> &)
+{
+    if ((_self.waiting_tasks > _cfg.max_waiting) && (!_worker_stack.empty())) {
+        assert(!_wait_queue.empty());
+        Worker *worker = _worker_stack.back();
+        _worker_stack.popBack();
+        assert(worker->state == Worker::State::BLOCKED);
+        assert(worker->strand == nullptr);
+        worker->state = Worker::State::RUNNING;
+        worker->strand = _wait_queue.front();
+        _wait_queue.pop();
+        assert(worker->strand->state == Strand::State::WAITING);
+        assert(!worker->strand->queue.empty());
+        worker->strand->state = Strand::State::ACTIVE;
+        assert(_self.waiting_tasks >= worker->strand->queue.size());
+        _self.waiting_tasks -= worker->strand->queue.size();
+        return worker;
+    }
+    return nullptr;
+}
+
+bool
+AdaptiveSequencedExecutor::obtain_strand(Worker &worker, std::unique_lock<std::mutex> &lock)
+{
+    assert(worker.strand == nullptr);
+    if (!_wait_queue.empty()) {
+        worker.strand = _wait_queue.front();
+        _wait_queue.pop();
+        assert(worker.strand->state == Strand::State::WAITING);
+        assert(!worker.strand->queue.empty());
+        worker.strand->state = Strand::State::ACTIVE;
+        assert(_self.waiting_tasks >= worker.strand->queue.size());
+        _self.waiting_tasks -= worker.strand->queue.size();
+    } else if (_self.state == Self::State::CLOSED) {
+        worker.state = Worker::State::DONE;
+    } else {
+        worker.state = Worker::State::BLOCKED;
+        _worker_stack.push(&worker);
+        while (worker.state == Worker::State::BLOCKED) {
+            worker.cond.wait(lock);
+        }
+    }
+    return (worker.state == Worker::State::RUNNING);
+}
+
+bool
+AdaptiveSequencedExecutor::exchange_strand(Worker &worker, std::unique_lock<std::mutex> &lock)
+{
+    if (worker.strand == nullptr) {
+        return obtain_strand(worker, lock);
+    }
+    if (worker.strand->queue.empty()) {
+        worker.strand->state = Strand::State::IDLE;
+        worker.strand = nullptr;
+        return obtain_strand(worker, lock);
+    }
+    if (!_wait_queue.empty()) {
+        worker.strand->state = Strand::State::WAITING;
+        _self.waiting_tasks += worker.strand->queue.size();
+        _wait_queue.push(worker.strand);
+        worker.strand = nullptr;
+        return obtain_strand(worker, lock);
+    }
+    return true;
+}
+
+AdaptiveSequencedExecutor::Task::UP
+AdaptiveSequencedExecutor::next_task(Worker &worker)
+{
+    Task::UP task;
+    Worker *worker_to_wake = nullptr;
+    auto guard = std::unique_lock(_mutex);
+    if (exchange_strand(worker, guard)) {
+        assert(worker.state == Worker::State::RUNNING);
+        assert(worker.strand != nullptr);
+        assert(!worker.strand->queue.empty());
+        task = std::move(worker.strand->queue.front());
+        worker.strand->queue.pop();
+        _stats.queueSize.add(--_self.pending_tasks);
+        worker_to_wake = get_worker_to_wake(guard);
+    } else {
+        assert(worker.state == Worker::State::DONE);
+        assert(worker.strand == nullptr);
+    }
+    bool signal_self = maybe_unblock_self(guard);
+    guard.unlock(); // UNLOCK
+    if (worker_to_wake != nullptr) {
+        worker_to_wake->cond.notify_one();
+    }
+    if (signal_self) {
+        _self.cond.notify_all();
+    }
+    return task;
+}
+
+void
+AdaptiveSequencedExecutor::worker_main()
+{
+    Worker worker;
+    while (Task::UP my_task = next_task(worker)) {
+        my_task->run();
+    }
+    _thread_tools->allow_worker_exit.await();
+}
+
+AdaptiveSequencedExecutor::AdaptiveSequencedExecutor(size_t num_strands, size_t num_threads,
+                                                     size_t max_waiting, size_t max_pending)
+    : ISequencedTaskExecutor(num_strands),
+      _thread_tools(std::make_unique<ThreadTools>(*this)),
+      _mutex(),
+      _strands(num_strands),
+      _wait_queue(num_strands),
+      _worker_stack(num_threads),
+      _self(),
+      _stats(),
+      _cfg(num_threads, max_waiting, max_pending)
+{
+    _stats.queueSize.add(_self.pending_tasks);
+    _thread_tools->start(num_threads);
+}
+
+AdaptiveSequencedExecutor::~AdaptiveSequencedExecutor()
+{
+    sync();
+    {
+        auto guard = std::unique_lock(_mutex);
+        assert(_self.state == Self::State::OPEN);
+        _self.state = Self::State::CLOSED;
+        while (!_worker_stack.empty()) {
+            Worker *worker = _worker_stack.back();
+            _worker_stack.popBack();
+            assert(worker->state == Worker::State::BLOCKED);
+            assert(worker->strand == nullptr);
+            worker->state = Worker::State::DONE;
+            worker->cond.notify_one();
+        }
+        _self.cond.notify_all();
+    }
+    _thread_tools->close();
+    assert(_wait_queue.empty());
+    assert(_worker_stack.empty());
+}
+
+void
+AdaptiveSequencedExecutor::executeTask(ExecutorId id, Task::UP task)
+{
+    assert(id.getId() < _strands.size());
+    Strand &strand = _strands[id.getId()];
+    auto guard = std::unique_lock(_mutex);
+    maybe_block_self(guard);
+    assert(_self.state != Self::State::CLOSED);
+    strand.queue.push(std::move(task));
+    _stats.queueSize.add(++_self.pending_tasks);
+    ++_stats.acceptedTasks;
+    if (strand.state == Strand::State::WAITING) {
+        ++_self.waiting_tasks;
+    } else if (strand.state == Strand::State::IDLE) {
+        if (_worker_stack.size() < _cfg.num_threads) {
+            strand.state = Strand::State::WAITING;
+            _wait_queue.push(&strand);
+            _self.waiting_tasks += strand.queue.size();
+        } else {
+            strand.state = Strand::State::ACTIVE;
+            assert(_wait_queue.empty());
+            Worker *worker = _worker_stack.back();
+            _worker_stack.popBack();
+            assert(worker->state == Worker::State::BLOCKED);
+            assert(worker->strand == nullptr);
+            worker->state = Worker::State::RUNNING;
+            worker->strand = &strand;
+            guard.unlock(); // UNLOCK
+            worker->cond.notify_one();
+        }
+    }
+}
+
+void
+AdaptiveSequencedExecutor::sync()
+{
+    vespalib::CountDownLatch latch(_strands.size());
+    for (size_t i = 0; i < _strands.size(); ++i) {
+        execute(ExecutorId(i), [&](){ latch.countDown(); });
+    }
+    latch.await();
+}
+
+void
+AdaptiveSequencedExecutor::setTaskLimit(uint32_t task_limit)
+{
+    auto guard = std::unique_lock(_mutex);   
+    _cfg.set_max_pending(task_limit);
+    bool signal_self = maybe_unblock_self(guard);
+    guard.unlock(); // UNLOCK
+    if (signal_self) {
+        _self.cond.notify_all();
+    }
+}
+
+AdaptiveSequencedExecutor::Stats
+AdaptiveSequencedExecutor::getStats()
+{
+    auto guard = std::lock_guard(_mutex);
+    Stats stats = _stats;
+    _stats = Stats();
+    _stats.queueSize.add(_self.pending_tasks);
+    return stats;
+}
+
+}
diff --git a/staging_vespalib/src/vespa/vespalib/util/adaptive_sequenced_executor.h b/staging_vespalib/src/vespa/vespalib/util/adaptive_sequenced_executor.h
new file mode 100644
index 00000000000..bc3457a72ef
--- /dev/null
+++ b/staging_vespalib/src/vespa/vespalib/util/adaptive_sequenced_executor.h
@@ -0,0 +1,126 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "isequencedtaskexecutor.h"
+#include <vespa/vespalib/util/arrayqueue.hpp>
+#include <vespa/vespalib/util/gate.h>
+#include <vespa/fastos/thread.h>
+#include <mutex>
+#include <condition_variable>
+#include <cassert>
+
+namespace vespalib {
+
+/**
+ * Sequenced executor that balances the number of active threads in
+ * order to optimize for throughput over latency by minimizing the
+ * number of critical-path wakeups.
+ **/
+class AdaptiveSequencedExecutor : public ISequencedTaskExecutor
+{
+private:
+    using Stats = vespalib::ExecutorStats;
+    using Task = vespalib::Executor::Task;
+
+    /**
+     * Values used to configure the executor.
+     **/
+    struct Config {
+        size_t num_threads;
+        size_t max_waiting;
+        size_t max_pending;
+        size_t wakeup_limit;
+        void set_max_pending(size_t max_pending_in) {
+            max_pending = std::max(1uL, max_pending_in);
+            wakeup_limit = std::max(1uL, size_t(max_pending * 0.9));
+            assert(wakeup_limit > 0);
+            assert(wakeup_limit <= max_pending);
+        }
+        Config(size_t num_threads_in, size_t max_waiting_in, size_t max_pending_in)
+            : num_threads(num_threads_in), max_waiting(max_waiting_in), max_pending(1000), wakeup_limit(900)
+        {
+            assert(num_threads > 0);
+            set_max_pending(max_pending_in);
+        }
+    };
+
+    /**
+     * Tasks that need to be sequenced are handled by a single strand.
+     **/
+    struct Strand {
+        enum class State { IDLE, WAITING, ACTIVE };
+        State state;
+        vespalib::ArrayQueue<Task::UP> queue;
+        Strand();
+        ~Strand();
+    };
+
+    /**
+     * The state of a single worker thread.
+     **/
+    struct Worker {
+        enum class State { RUNNING, BLOCKED, DONE };
+        std::condition_variable cond;
+        State state;
+        Strand *strand;
+        Worker();
+        ~Worker();
+    };
+
+    /**
+     * State related to the executor itself.
+     **/
+    struct Self {
+        enum class State { OPEN, BLOCKED, CLOSED };
+        std::condition_variable cond;
+        State state;
+        size_t waiting_tasks;
+        size_t pending_tasks;
+        Self();
+        ~Self();
+    };
+
+    /**
+     * Stuff related to worker thread startup and shutdown.
+     **/
+    struct ThreadTools : FastOS_Runnable {
+        static constexpr size_t STACK_SIZE = (256 * 1024);
+        AdaptiveSequencedExecutor &parent;
+        std::unique_ptr<FastOS_ThreadPool> pool;
+        vespalib::Gate allow_worker_exit;
+        ThreadTools(AdaptiveSequencedExecutor &parent_in);
+        ~ThreadTools();
+        void Run(FastOS_ThreadInterface *, void *) override;
+        void start(size_t num_threads);
+        void close();
+    };
+
+    std::unique_ptr<ThreadTools>       _thread_tools;
+    std::mutex                         _mutex;
+    std::vector<Strand>                _strands;
+    vespalib::ArrayQueue<Strand*>      _wait_queue;
+    vespalib::ArrayQueue<Worker*>      _worker_stack;
+    Self                               _self;
+    Stats                              _stats;
+    Config                             _cfg;
+
+    void maybe_block_self(std::unique_lock<std::mutex> &lock);
+    bool maybe_unblock_self(const std::unique_lock<std::mutex> &lock);
+
+    Worker *get_worker_to_wake(const std::unique_lock<std::mutex> &lock);
+    bool obtain_strand(Worker &worker, std::unique_lock<std::mutex> &lock);
+    bool exchange_strand(Worker &worker, std::unique_lock<std::mutex> &lock);
+    Task::UP next_task(Worker &worker);
+    void worker_main();
+public:
+    AdaptiveSequencedExecutor(size_t num_strands, size_t num_threads,
+                              size_t max_waiting, size_t max_pending);
+    ~AdaptiveSequencedExecutor() override;
+    void executeTask(ExecutorId id, Task::UP task) override;
+    void sync() override;
+    void setTaskLimit(uint32_t task_limit) override;
+    vespalib::ExecutorStats getStats() override;
+};
+
+}
diff --git a/searchlib/src/vespa/searchlib/common/foregroundtaskexecutor.cpp b/staging_vespalib/src/vespa/vespalib/util/foregroundtaskexecutor.cpp
index 513684d3fd5..b45ada1c58c 100644
--- a/searchlib/src/vespa/searchlib/common/foregroundtaskexecutor.cpp
+++ b/staging_vespalib/src/vespa/vespalib/util/foregroundtaskexecutor.cpp
@@ -1,12 +1,9 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include "foregroundtaskexecutor.h"
-#include <vespa/vespalib/util/threadstackexecutor.h>
-#include <vespa/vespalib/stllike/hash_map.hpp>
+#include <cassert>
 
-using vespalib::ThreadStackExecutor;
-
-namespace search {
+namespace vespalib {
 
 ForegroundTaskExecutor::ForegroundTaskExecutor()
     : ForegroundTaskExecutor(1)
@@ -14,7 +11,8 @@ ForegroundTaskExecutor::ForegroundTaskExecutor()
 }
 
 ForegroundTaskExecutor::ForegroundTaskExecutor(uint32_t threads)
-    : ISequencedTaskExecutor(threads)
+    : ISequencedTaskExecutor(threads),
+      _accepted(0)
 {
 }
 
@@ -25,6 +23,7 @@ ForegroundTaskExecutor::executeTask(ExecutorId id, vespalib::Executor::Task::UP
 {
     assert(id.getId() < getNumExecutors());
     task->run();
+    _accepted++;
 }
 
 void
@@ -32,4 +31,12 @@ ForegroundTaskExecutor::sync()
 {
 }
 
+void ForegroundTaskExecutor::setTaskLimit(uint32_t) {
+
+}
+
+vespalib::ExecutorStats ForegroundTaskExecutor::getStats() {
+    return vespalib::ExecutorStats(vespalib::ExecutorStats::QueueSizeT(0) , _accepted.load(std::memory_order_relaxed), 0);
+}
+
 } // namespace search
diff --git a/searchlib/src/vespa/searchlib/common/foregroundtaskexecutor.h b/staging_vespalib/src/vespa/vespalib/util/foregroundtaskexecutor.h
index 2074eda009b..d9a348ed012 100644
--- a/searchlib/src/vespa/searchlib/common/foregroundtaskexecutor.h
+++ b/staging_vespalib/src/vespa/vespalib/util/foregroundtaskexecutor.h
@@ -2,11 +2,9 @@
 #pragma once
 
 #include "isequencedtaskexecutor.h"
-#include <vespa/vespalib/stllike/hash_map.h>
+#include <atomic>
 
-namespace vespalib { class ThreadStackExecutorBase; }
-
-namespace search {
+namespace vespalib {
 
 /**
  * Class to run multiple tasks in parallel, but tasks with same
@@ -25,6 +23,12 @@ public:
 
     void executeTask(ExecutorId id, vespalib::Executor::Task::UP task) override;
     void sync() override;
+
+    void setTaskLimit(uint32_t taskLimit) override;
+
+    vespalib::ExecutorStats getStats() override;
+private:
+    std::atomic<uint64_t> _accepted;
 };
 
 } // namespace search
diff --git a/searchlib/src/vespa/searchlib/common/isequencedtaskexecutor.cpp b/staging_vespalib/src/vespa/vespalib/util/isequencedtaskexecutor.cpp
index 9d1bc99bebb..d05702cc85b 100644
--- a/searchlib/src/vespa/searchlib/common/isequencedtaskexecutor.cpp
+++ b/staging_vespalib/src/vespa/vespalib/util/isequencedtaskexecutor.cpp
@@ -5,10 +5,11 @@
 #include <vespa/vespalib/stllike/hashtable.h>
 #include <cassert>
 
-namespace search {
-    namespace {
-        constexpr uint8_t MAGIC = 255;
-    }
+namespace vespalib {
+
+namespace {
+    constexpr uint8_t MAGIC = 255;
+}
 
 ISequencedTaskExecutor::ISequencedTaskExecutor(uint32_t numExecutors)
     : _component2Id(vespalib::hashtable_base::getModuloStl(numExecutors*8), MAGIC),
diff --git a/searchlib/src/vespa/searchlib/common/isequencedtaskexecutor.h b/staging_vespalib/src/vespa/vespalib/util/isequencedtaskexecutor.h
index 109e8319148..cd2a6c6f0d8 100644
--- a/searchlib/src/vespa/searchlib/common/isequencedtaskexecutor.h
+++ b/staging_vespalib/src/vespa/vespalib/util/isequencedtaskexecutor.h
@@ -2,12 +2,13 @@
 #pragma once
 
 #include <vespa/vespalib/util/executor.h>
+#include <vespa/vespalib/util/executor_stats.h>
 #include <vespa/vespalib/stllike/string.h>
 #include <vespa/vespalib/util/lambdatask.h>
 #include <vector>
 #include <mutex>
 
-namespace search {
+namespace vespalib {
 
 /**
  * Interface class to run multiple tasks in parallel, but tasks with same
@@ -67,6 +68,10 @@ public:
      */
     virtual void sync() = 0;
 
+    virtual void setTaskLimit(uint32_t taskLimit) = 0;
+
+    virtual vespalib::ExecutorStats getStats() = 0;
+
     /**
      * Wrap lambda function into a task and schedule it to be run.
      * Caller must ensure that pointers and references are valid and
@@ -105,4 +110,4 @@ private:
     mutable uint32_t             _nextId;
 };
 
-} // namespace search
+}
diff --git a/staging_vespalib/src/vespa/vespalib/util/process_memory_stats.cpp b/staging_vespalib/src/vespa/vespalib/util/process_memory_stats.cpp
index ff1d7e1d6ec..a758ca1fbbe 100644
--- a/staging_vespalib/src/vespa/vespalib/util/process_memory_stats.cpp
+++ b/staging_vespalib/src/vespa/vespalib/util/process_memory_stats.cpp
@@ -196,6 +196,8 @@ ProcessMemoryStats::create(uint64_t sizeEpsilon)
             i, (samples.rbegin()+1)->toString().c_str(), samples.back().toString().c_str());
     }
     std::sort(samples.begin(), samples.end());
+    LOG(warning, "We failed to find 2 consecutive samples that where similar with epsilon of %" PRIu64 ".\nSmallest is '%s',\n median is '%s',\n largest is '%s'",
+                 sizeEpsilon, samples.front().toString().c_str(), samples[samples.size()/2].toString().c_str(), samples.back().toString().c_str());
     return samples[samples.size()/2];
 }
 
diff --git a/staging_vespalib/src/vespa/vespalib/util/sequencedtaskexecutor.cpp b/staging_vespalib/src/vespa/vespalib/util/sequencedtaskexecutor.cpp
new file mode 100644
index 00000000000..a0c2f0ac237
--- /dev/null
+++ b/staging_vespalib/src/vespa/vespalib/util/sequencedtaskexecutor.cpp
@@ -0,0 +1,89 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "sequencedtaskexecutor.h"
+#include "adaptive_sequenced_executor.h"
+#include "singleexecutor.h"
+#include <vespa/vespalib/util/blockingthreadstackexecutor.h>
+
+namespace vespalib {
+
+namespace {
+
+constexpr uint32_t stackSize = 128 * 1024;
+
+}
+
+
+std::unique_ptr<ISequencedTaskExecutor>
+SequencedTaskExecutor::create(uint32_t threads, uint32_t taskLimit, OptimizeFor optimize, uint32_t kindOfWatermark, duration reactionTime)
+{
+    if (optimize == OptimizeFor::ADAPTIVE) {
+        return std::make_unique<AdaptiveSequencedExecutor>(threads, threads, kindOfWatermark, taskLimit);
+    } else {
+        auto executors = std::make_unique<std::vector<std::unique_ptr<SyncableThreadExecutor>>>();
+        executors->reserve(threads);
+        for (uint32_t id = 0; id < threads; ++id) {
+            if (optimize == OptimizeFor::THROUGHPUT) {
+                uint32_t watermark = kindOfWatermark == 0 ? taskLimit / 10 : kindOfWatermark;
+                executors->push_back(std::make_unique<SingleExecutor>(taskLimit, watermark, reactionTime));
+            } else {
+                executors->push_back(std::make_unique<BlockingThreadStackExecutor>(1, stackSize, taskLimit));
+            }
+        }
+        return std::unique_ptr<ISequencedTaskExecutor>(new SequencedTaskExecutor(std::move(executors)));
+    }
+}
+
+SequencedTaskExecutor::~SequencedTaskExecutor()
+{
+    sync();
+}
+
+SequencedTaskExecutor::SequencedTaskExecutor(std::unique_ptr<std::vector<std::unique_ptr<vespalib::SyncableThreadExecutor>>> executors)
+    : ISequencedTaskExecutor(executors->size()),
+      _executors(std::move(executors))
+{
+}
+
+void
+SequencedTaskExecutor::setTaskLimit(uint32_t taskLimit)
+{
+    for (const auto &executor : *_executors) {
+        executor->setTaskLimit(taskLimit);
+    }
+}
+
+void
+SequencedTaskExecutor::executeTask(ExecutorId id, vespalib::Executor::Task::UP task)
+{
+    assert(id.getId() < _executors->size());
+    auto rejectedTask = (*_executors)[id.getId()]->execute(std::move(task));
+    assert(!rejectedTask);
+}
+
+void
+SequencedTaskExecutor::sync()
+{
+    for (auto &executor : *_executors) {
+        SingleExecutor * single = dynamic_cast<vespalib::SingleExecutor *>(executor.get());
+        if (single) {
+            //Enforce parallel wakeup of napping executors.
+            single->startSync();
+        }
+    }
+    for (auto &executor : *_executors) {
+        executor->sync();
+    }
+}
+
+SequencedTaskExecutor::Stats
+SequencedTaskExecutor::getStats()
+{
+    Stats accumulatedStats;
+    for (auto &executor :* _executors) {
+        accumulatedStats += executor->getStats();
+    }
+    return accumulatedStats;
+}
+
+} // namespace search
diff --git a/staging_vespalib/src/vespa/vespalib/util/sequencedtaskexecutor.h b/staging_vespalib/src/vespa/vespalib/util/sequencedtaskexecutor.h
new file mode 100644
index 00000000000..b3dd400478a
--- /dev/null
+++ b/staging_vespalib/src/vespa/vespalib/util/sequencedtaskexecutor.h
@@ -0,0 +1,40 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include "isequencedtaskexecutor.h"
+#include <vespa/vespalib/util/time.h>
+
+namespace vespalib {
+
+class SyncableThreadExecutor;
+
+/**
+ * Class to run multiple tasks in parallel, but tasks with same
+ * id has to be run in sequence.
+ */
+class SequencedTaskExecutor final : public ISequencedTaskExecutor
+{
+    using Stats = vespalib::ExecutorStats;
+    std::unique_ptr<std::vector<std::unique_ptr<vespalib::SyncableThreadExecutor>>> _executors;
+
+    SequencedTaskExecutor(std::unique_ptr<std::vector<std::unique_ptr<vespalib::SyncableThreadExecutor>>> executor);
+public:
+    using ISequencedTaskExecutor::getExecutorId;
+    using OptimizeFor = vespalib::Executor::OptimizeFor;
+
+    ~SequencedTaskExecutor();
+
+    void setTaskLimit(uint32_t taskLimit) override;
+    void executeTask(ExecutorId id, vespalib::Executor::Task::UP task) override;
+    void sync() override;
+    Stats getStats() override;
+
+    /*
+     * Note that if you choose Optimize::THROUGHPUT, you must ensure only a single producer, or synchronize on the outside.
+     *
+     */
+    static std::unique_ptr<ISequencedTaskExecutor>
+    create(uint32_t threads, uint32_t taskLimit = 1000, OptimizeFor optimize = OptimizeFor::LATENCY, uint32_t kindOfWatermark = 0, duration reactionTime = 10ms);
+};
+
+} // namespace search
diff --git a/searchlib/src/vespa/searchlib/common/sequencedtaskexecutorobserver.cpp b/staging_vespalib/src/vespa/vespalib/util/sequencedtaskexecutorobserver.cpp
index 04504086520..3d9ed4e21f4 100644
--- a/searchlib/src/vespa/searchlib/common/sequencedtaskexecutorobserver.cpp
+++ b/staging_vespalib/src/vespa/vespalib/util/sequencedtaskexecutorobserver.cpp
@@ -2,7 +2,7 @@
 
 #include "sequencedtaskexecutorobserver.h"
 
-namespace search {
+namespace vespalib {
 
 SequencedTaskExecutorObserver::SequencedTaskExecutorObserver(ISequencedTaskExecutor &executor)
     : ISequencedTaskExecutor(executor.getNumExecutors()),
@@ -41,4 +41,12 @@ SequencedTaskExecutorObserver::getExecuteHistory()
     return _executeHistory;
 }
 
+void SequencedTaskExecutorObserver::setTaskLimit(uint32_t taskLimit) {
+    _executor.setTaskLimit(taskLimit);
+}
+
+vespalib::ExecutorStats SequencedTaskExecutorObserver::getStats() {
+    return _executor.getStats();
+}
+
 } // namespace search
diff --git a/searchlib/src/vespa/searchlib/common/sequencedtaskexecutorobserver.h b/staging_vespalib/src/vespa/vespalib/util/sequencedtaskexecutorobserver.h
index b2de71f06b3..9307a7ddb37 100644
--- a/searchlib/src/vespa/searchlib/common/sequencedtaskexecutorobserver.h
+++ b/staging_vespalib/src/vespa/vespalib/util/sequencedtaskexecutorobserver.h
@@ -3,10 +3,8 @@
 
 #include "isequencedtaskexecutor.h"
 #include <atomic>
-#include <vector>
-#include <mutex>
 
-namespace search {
+namespace vespalib {
 
 /**
  * Observer class to observe class to run multiple tasks in parallel,
@@ -31,6 +29,10 @@ public:
     uint32_t getExecuteCnt() const { return _executeCnt; }
     uint32_t getSyncCnt() const { return _syncCnt; }
     std::vector<uint32_t> getExecuteHistory();
+
+    void setTaskLimit(uint32_t taskLimit) override;
+
+    vespalib::ExecutorStats getStats() override;
 };
 
 } // namespace search
diff --git a/staging_vespalib/src/vespa/vespalib/util/singleexecutor.cpp b/staging_vespalib/src/vespa/vespalib/util/singleexecutor.cpp
new file mode 100644
index 00000000000..a17037799a3
--- /dev/null
+++ b/staging_vespalib/src/vespa/vespalib/util/singleexecutor.cpp
@@ -0,0 +1,170 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "singleexecutor.h"
+#include <vespa/vespalib/util/time.h>
+
+namespace vespalib {
+
+SingleExecutor::SingleExecutor(uint32_t taskLimit)
+    : SingleExecutor(taskLimit, taskLimit/10, 5ms)
+{ } 
+
+SingleExecutor::SingleExecutor(uint32_t taskLimit, uint32_t watermark, duration reactionTime)
+    : _taskLimit(vespalib::roundUp2inN(taskLimit)),
+      _wantedTaskLimit(_taskLimit.load()),
+      _rp(0),
+      _tasks(std::make_unique<Task::UP[]>(_taskLimit)),
+      _mutex(),
+      _consumerCondition(),
+      _producerCondition(),
+      _thread(*this),
+      _lastAccepted(0),
+      _queueSize(),
+      _wakeupConsumerAt(0),
+      _producerNeedWakeupAt(0),
+      _wp(0),
+      _watermark(std::min(_taskLimit.load(), watermark)),
+      _reactionTime(reactionTime),
+      _closed(false)
+{
+    assert(taskLimit >= watermark);
+    _thread.start();
+}
+
+SingleExecutor::~SingleExecutor() {
+    shutdown();
+    sync();
+    _thread.stop().join();
+}
+
+size_t
+SingleExecutor::getNumThreads() const {
+    return 1;
+}
+
+void
+SingleExecutor::sleepProducer(Lock & lock, duration maxWaitTime, uint64_t wakeupAt) {
+    _producerNeedWakeupAt.store(wakeupAt, std::memory_order_relaxed);
+    _producerCondition.wait_for(lock, maxWaitTime);
+    _producerNeedWakeupAt.store(0, std::memory_order_relaxed);
+}
+
+Executor::Task::UP
+SingleExecutor::execute(Task::UP task) {
+    uint64_t wp;
+    {
+        Lock guard(_mutex);
+        if (_closed) {
+            return task;
+        }
+        wait_for_room(guard);
+        wp = _wp.load(std::memory_order_relaxed);
+        _tasks[index(wp)] = std::move(task);
+        _wp.store(wp + 1, std::memory_order_release);
+    }
+    if (wp == _wakeupConsumerAt.load(std::memory_order_relaxed)) {
+        _consumerCondition.notify_one();
+    }
+    return task;
+}
+
+void
+SingleExecutor::setTaskLimit(uint32_t taskLimit) {
+    _wantedTaskLimit = std::max(vespalib::roundUp2inN(taskLimit), size_t(_watermark));
+}
+
+void
+SingleExecutor::drain(Lock & lock) {
+    uint64_t wp = _wp.load(std::memory_order_relaxed);
+    while (numTasks() > 0) {
+        _consumerCondition.notify_one();
+        sleepProducer(lock, 100us, wp);
+    }
+}
+
+void
+SingleExecutor::startSync() {
+    _consumerCondition.notify_one();
+}
+
+SingleExecutor &
+SingleExecutor::sync() {
+    Lock lock(_mutex);
+    uint64_t wp = _wp.load(std::memory_order_relaxed);
+    while (wp > _rp.load(std::memory_order_acquire)) {
+        _consumerCondition.notify_one();
+        sleepProducer(lock, 100us, wp);
+    }
+    return *this;
+}
+
+SingleExecutor &
+SingleExecutor::shutdown() {
+    Lock lock(_mutex);
+    _closed = true;
+    return *this;
+}
+
+void
+SingleExecutor::run() {
+    while (!_thread.stopped()) {
+        drain_tasks();
+        _producerCondition.notify_all();
+        _wakeupConsumerAt.store(_wp.load(std::memory_order_relaxed) + _watermark, std::memory_order_relaxed);
+        Lock lock(_mutex);
+        if (numTasks() <= 0) {
+            _consumerCondition.wait_for(lock, _reactionTime);
+        }
+        _wakeupConsumerAt.store(0, std::memory_order_relaxed);
+    }
+}
+
+void
+SingleExecutor::drain_tasks() {
+    while (numTasks() > 0) {
+        run_tasks_till(_wp.load(std::memory_order_acquire));
+    }
+}
+
+void
+SingleExecutor::run_tasks_till(uint64_t available) {
+    uint64_t consumed = _rp.load(std::memory_order_relaxed);
+    uint64_t wakeupLimit = _producerNeedWakeupAt.load(std::memory_order_relaxed);
+    while (consumed  < available) {
+        Task::UP task = std::move(_tasks[index(consumed)]);
+        task->run();
+        _rp.store(++consumed, std::memory_order_release);
+        if (wakeupLimit == consumed) {
+            _producerCondition.notify_all();
+        }
+    }
+}
+
+void
+SingleExecutor::wait_for_room(Lock & lock) {
+    uint64_t wp = _wp.load(std::memory_order_relaxed);
+    uint64_t taskLimit = _taskLimit.load(std::memory_order_relaxed);
+    if (taskLimit != _wantedTaskLimit.load(std::memory_order_relaxed)) {
+        drain(lock);
+        _tasks = std::make_unique<Task::UP[]>(_wantedTaskLimit);
+        _taskLimit = _wantedTaskLimit.load();
+        taskLimit = _taskLimit;
+    }
+    _queueSize.add(numTasks());
+    while (numTasks() >= _taskLimit.load(std::memory_order_relaxed)) {
+        sleepProducer(lock, _reactionTime, wp - _watermark);
+    }
+}
+
+ThreadExecutor::Stats
+SingleExecutor::getStats() {
+    Lock lock(_mutex);
+    uint64_t accepted = _wp.load(std::memory_order_relaxed);
+    Stats stats(_queueSize, (accepted - _lastAccepted), 0);
+    _lastAccepted = accepted;
+    _queueSize = Stats::QueueSizeT() ;
+    return stats;
+}
+
+
+}
diff --git a/staging_vespalib/src/vespa/vespalib/util/singleexecutor.h b/staging_vespalib/src/vespa/vespalib/util/singleexecutor.h
new file mode 100644
index 00000000000..a58128c15aa
--- /dev/null
+++ b/staging_vespalib/src/vespa/vespalib/util/singleexecutor.h
@@ -0,0 +1,65 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/vespalib/util/threadexecutor.h>
+#include <vespa/vespalib/util/thread.h>
+#include <vespa/vespalib/util/time.h>
+#include <thread>
+#include <atomic>
+
+namespace vespalib {
+
+/**
+ * Has a single thread consuming tasks from a fixed size ringbuffer.
+ * Made for throughput where the producer has no interaction with the consumer and
+ * it is hence very cheap to produce a task. High and low watermark at 25%/75% is used
+ * to reduce ping-pong.
+ */
+class SingleExecutor final : public vespalib::SyncableThreadExecutor, vespalib::Runnable {
+public:
+    explicit SingleExecutor(uint32_t taskLimit);
+    SingleExecutor(uint32_t taskLimit, uint32_t watermark, duration reactionTime);
+    ~SingleExecutor() override;
+    Task::UP execute(Task::UP task) override;
+    void setTaskLimit(uint32_t taskLimit) override;
+    SingleExecutor & sync() override;
+    void startSync();
+    size_t getNumThreads() const override;
+    uint32_t getTaskLimit() const { return _taskLimit.load(std::memory_order_relaxed); }
+    Stats getStats() override;
+    SingleExecutor & shutdown() override;
+private:
+    using Lock = std::unique_lock<std::mutex>;
+    void drain(Lock & lock);
+    void run() override;
+    void drain_tasks();
+    void sleepProducer(Lock & guard, duration maxWaitTime, uint64_t wakeupAt);
+    void run_tasks_till(uint64_t available);
+    void wait_for_room(Lock & guard);
+    uint64_t index(uint64_t counter) const {
+        return counter & (_taskLimit.load(std::memory_order_relaxed) - 1);
+    }
+
+    uint64_t numTasks() const {
+        return _wp.load(std::memory_order_relaxed) - _rp.load(std::memory_order_acquire);
+    }
+    std::atomic<uint32_t>       _taskLimit;
+    std::atomic<uint32_t>       _wantedTaskLimit;
+    std::atomic<uint64_t>       _rp;
+    std::unique_ptr<Task::UP[]> _tasks;
+    std::mutex                  _mutex;
+    std::condition_variable     _consumerCondition;
+    std::condition_variable     _producerCondition;
+    vespalib::Thread            _thread;
+    uint64_t                    _lastAccepted;
+    Stats::QueueSizeT           _queueSize;
+    std::atomic<uint64_t>       _wakeupConsumerAt;
+    std::atomic<uint64_t>       _producerNeedWakeupAt;
+    std::atomic<uint64_t>       _wp;
+    const uint32_t              _watermark;
+    const duration              _reactionTime;
+    bool                        _closed;
+};
+
+}
diff --git a/standalone-container/CMakeLists.txt b/standalone-container/CMakeLists.txt
index 83c58e09945..6e8bf49d846 100644
--- a/standalone-container/CMakeLists.txt
+++ b/standalone-container/CMakeLists.txt
@@ -1,2 +1,3 @@
 # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 install_fat_java_artifact(standalone-container)
+install(PROGRAMS src/main/sh/standalone-container.sh DESTINATION libexec/vespa)
diff --git a/standalone-container/pom.xml b/standalone-container/pom.xml
index 744d6854ab2..b9062b9284c 100644
--- a/standalone-container/pom.xml
+++ b/standalone-container/pom.xml
@@ -90,8 +90,6 @@
             container-disc-jar-with-dependencies.jar,
             vespajlib.jar
           </discPreInstallBundle>
-          <bundleActivator>com.yahoo.container.standalone.StandaloneContainerActivator</bundleActivator>
-          <jdiscPrivilegedActivator>true</jdiscPrivilegedActivator>
         </configuration>
       </plugin>
       <plugin>
diff --git a/standalone-container/src/main/java/com/yahoo/container/standalone/StandaloneContainerActivator.java b/standalone-container/src/main/java/com/yahoo/container/standalone/StandaloneContainerActivator.java
deleted file mode 100644
index cfd5f753c4f..00000000000
--- a/standalone-container/src/main/java/com/yahoo/container/standalone/StandaloneContainerActivator.java
+++ /dev/null
@@ -1,162 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.container.standalone;
-
-import com.google.inject.Binder;
-import com.google.inject.Guice;
-import com.google.inject.Module;
-import com.google.inject.util.Modules;
-import com.yahoo.jdisc.application.ContainerActivator;
-import com.yahoo.jdisc.application.ContainerBuilder;
-import com.yahoo.jdisc.application.DeactivatedContainer;
-import com.yahoo.jdisc.application.OsgiFramework;
-import com.yahoo.jdisc.http.ConnectorConfig;
-import com.yahoo.vespa.model.VespaModel;
-import com.yahoo.vespa.model.container.Container;
-import com.yahoo.vespa.model.container.http.ConnectorFactory;
-import com.yahoo.vespa.model.container.http.Http;
-import com.yahoo.vespa.model.container.http.JettyHttpServer;
-import org.osgi.framework.Bundle;
-import org.osgi.framework.BundleActivator;
-import org.osgi.framework.BundleContext;
-
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.nio.channels.ServerSocketChannel;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Hashtable;
-import java.util.List;
-import java.util.stream.Collectors;
-
-/**
- * @author Einar M R Rosenvinge
- */
-public class StandaloneContainerActivator implements BundleActivator {
-
-    @Override
-    public void start(BundleContext bundleContext) throws Exception {
-        for (ConnectorConfig config: getConnectorConfigs(getContainer())) {
-            ServerSocketChannel socketChannel = bindChannel(config);
-            registerChannels(bundleContext, config.listenPort(), socketChannel);
-        }
-    }
-
-    static void registerChannels(BundleContext bundleContext, int listenPort, ServerSocketChannel boundChannel) {
-        Hashtable<String, Integer> properties = new Hashtable<>();
-        properties.put("port", listenPort);
-        bundleContext.registerService(ServerSocketChannel.class, boundChannel, properties);
-    }
-
-    static ServerSocketChannel bindChannel(ConnectorConfig channelInfo) throws IOException {
-        ServerSocketChannel serverChannel = ServerSocketChannel.open();
-        InetSocketAddress bindAddress = new InetSocketAddress(channelInfo.listenPort());
-        serverChannel.socket().bind(bindAddress, channelInfo.acceptQueueSize());
-        return serverChannel;
-    }
-
-    @Override
-    public void stop(BundleContext bundleContext) throws Exception { }
-
-    static Container getContainer(Module... modules) {
-        Module activatorModule = new ActivatorModule();
-        List<Module> allModules = new ArrayList<>();
-        allModules.addAll(Arrays.asList(modules));
-        allModules.add(activatorModule);
-
-        StandaloneContainerApplication app = new StandaloneContainerApplication(Guice.createInjector(Modules.combine(allModules)));
-        return app.container();
-    }
-
-    static List<ConnectorConfig> getConnectorConfigs(Container container) {
-        return getConnectorConfigs(container.getHttpServer());
-    }
-
-    private static List<ConnectorConfig> getConnectorConfigs(JettyHttpServer jettyHttpServer) {
-        if (jettyHttpServer == null)
-            return Collections.emptyList();
-
-        return jettyHttpServer.getConnectorFactories().stream().
-                map(StandaloneContainerActivator::getConnectorConfig).
-                collect(Collectors.toList());
-        }
-
-    private static ConnectorConfig getConnectorConfig(ConnectorFactory connectorFactory) {
-        return VespaModel.getConfig(ConnectorConfig.class, connectorFactory);
-    }
-
-
-    private static class ActivatorModule implements Module {
-        @Override
-        public void configure(Binder binder) {
-            binder.bind(OsgiFramework.class).toInstance(new DummyOsgiFramework());
-            binder.bind(ContainerActivator.class).toInstance(new DummyActivatorForStandaloneContainerApp());
-        }
-    }
-
-    private static class DummyActivatorForStandaloneContainerApp implements ContainerActivator {
-        @Override
-        public ContainerBuilder newContainerBuilder() {
-            return new ContainerBuilder(new ArrayList<Module>());
-        }
-
-        @Override
-        public DeactivatedContainer activateContainer(ContainerBuilder builder) {
-            return new DeactivatedContainer() {
-                @Override
-                public Object appContext() {
-                    return new Object();
-                }
-
-                @Override
-                public void notifyTermination(Runnable task) {
-                }
-            };
-        }
-    }
-
-    public static class DummyOsgiFramework implements OsgiFramework {
-        @Override
-        public List<Bundle> installBundle(String bundleLocation) {
-            throw new UnsupportedOperationException();
-        }
-
-        @Override
-        public void startBundles(List<Bundle> bundles, boolean privileged) {
-            throw new UnsupportedOperationException();
-        }
-
-        @Override
-        public void refreshPackages() {
-            throw new UnsupportedOperationException();
-        }
-
-        @Override
-        public BundleContext bundleContext() {
-            return null;
-        }
-
-        @Override
-        public List<Bundle> bundles() {
-            return Collections.emptyList();
-        }
-
-        @Override
-        public List<Bundle> getBundles(Bundle requestingBundle) {
-            return Collections.emptyList();
-        }
-
-        @Override
-        public void allowDuplicateBundles(Collection<Bundle> bundles) { }
-
-        @Override
-        public void start() {
-        }
-
-        @Override
-        public void stop() {
-        }
-    }
-
-}
diff --git a/standalone-container/src/main/sh/standalone-container.sh b/standalone-container/src/main/sh/standalone-container.sh
index b49744ebe49..b8025b9629b 100755
--- a/standalone-container/src/main/sh/standalone-container.sh
+++ b/standalone-container/src/main/sh/standalone-container.sh
@@ -112,13 +112,13 @@ StartCommand() {
 
     local service_regex='^[0-9a-zA-Z_-]+$'
     if ! [[ "$service" =~ $service_regex ]]; then
-	Fail "Service must match regex '$service_regex'"
+        Fail "Service must match regex '$service_regex'"
     fi
 
     local pidfile="$VESPA_HOME/var/run/$service.pid"
     if [ "$force" = false ] && test -r "$pidfile"; then
-	echo "$service is already running as PID $(< "$pidfile") according to $pidfile"
-	return
+        echo "$service is already running as PID $(< "$pidfile") according to $pidfile"
+        return
     fi
 
     # common setup
@@ -199,71 +199,71 @@ Kill() {
 
     local -i now
     if ! now=$(date +%s); then
-	Fail "Failed to get the current date in seconds since epoch"
+        Fail "Failed to get the current date in seconds since epoch"
     fi
     local -i timeout=$(( now + 300 ))
 
     local has_killed=false
 
     while true; do
-	local ps_output=""
-	if ! ps_output=$(ps -p "$pid" -o user= -o comm=); then
-	    # success
-	    return
-	fi
-
-	local user comm
-	read -r user comm <<< "$ps_output"
-
-	if test "$user" != "$expected_user"; then
-	    echo "Warning: Pid collision ($pid): Expected user $expected_user but found $user."
-	    echo "Will assume original process has died."
-	    return
-	fi
-
-	if test "$comm" != "$expected_comm"; then
-	    echo "Warning: Pid collision ($pid): Expected program $expected_comm but found $comm."
-	    echo "Will assume original process has died."
-	    return
-	fi
-
-	if ! "$has_killed"; then
-	    if $force; then
-		if ! kill -KILL "$pid"; then
-		    Fail "Failed to kill $pid"
-		fi
-	    else
-		if ! kill "$pid"; then
-		    Fail "Failed to kill $pid"
-		fi
-	    fi
-
-	    has_killed=true
-	fi
-
-	sleep 1
-
-	now=$(date +%s)
-	if (( now >= timeout )); then
-	    Fail "Process $pid still exists after $timeout seconds, giving up"
-	fi
+        local ps_output=""
+        if ! ps_output=$(ps -p "$pid" -o user= -o comm=); then
+            # success
+            return
+        fi
+
+        local user comm
+        read -r user comm <<< "$ps_output"
+
+        if test "$user" != "$expected_user"; then
+            echo "Warning: Pid collision ($pid): Expected user $expected_user but found $user."
+            echo "Will assume original process has died."
+            return
+        fi
+
+        if test "$comm" != "$expected_comm"; then
+            echo "Warning: Pid collision ($pid): Expected program $expected_comm but found $comm."
+            echo "Will assume original process has died."
+            return
+        fi
+
+        if ! "$has_killed"; then
+            if $force; then
+                if ! kill -KILL "$pid"; then
+                    Fail "Failed to kill $pid"
+                fi
+            else
+                if ! kill "$pid"; then
+                    Fail "Failed to kill $pid"
+                fi
+            fi
+
+            has_killed=true
+        fi
+
+        sleep 1
+
+        now=$(date +%s)
+        if (( now >= timeout )); then
+            Fail "Process $pid still exists after $timeout seconds, giving up"
+        fi
     done
 }
 
 StopCommand() {
     local user="$1"
-    local force="$2"
-    local service="$3"
+    local service="$2"
+    local force="$3"
 
     local pidfile="$VESPA_HOME/var/run/$service.pid"
     if ! test -r "$pidfile"; then
-	echo "$service is not running"
-	return
+        echo "$service is not running"
+        return
     fi
 
     local pid=$(< "$pidfile")
     if ! [[ "$pid" =~ ^[0-9]+$ ]]; then
-	Fail "Pid file '$pidfile' does not contain a valid pid: $pid"
+        Fail "Pid file '$pidfile' does not contain a valid pid: $pid"
     fi
 
     Kill "$force" "$user" java "$pid"
@@ -272,7 +272,7 @@ StopCommand() {
 
 Main() {
     if (( $# == 0 )); then
-	Usage
+        Usage
     fi
 
     local command="$1"
@@ -284,49 +284,49 @@ Main() {
     local -a jvm_arguments=()
 
     while (( $# > 0 )); do
-	case "$1" in
-	    --help|-h) Usage ;;
-	    --service|-s)
-		service="$2"
-		shift 2
-		;;
-	    --user|-u)
-		user="$2"
-		shift 2
-		;;
-	    --force|-f)
-		force=true
-		shift
-		;;
-	    --)
-		shift
-		jvm_arguments=("$@")
-		break
-		;;
-	    *) break ;;
-	esac
+        case "$1" in
+            --help|-h) Usage ;;
+            --service|-s)
+                service="$2"
+                shift 2
+                ;;
+            --user|-u)
+                user="$2"
+                shift 2
+                ;;
+            --force|-f)
+                force=true
+                shift
+                ;;
+            --)
+                shift
+                jvm_arguments=("$@")
+                break
+                ;;
+            *) break ;;
+        esac
     done
 
     # Service name will be included in paths and possibly environment variable
     # names, so be restrictive.
     local service_regex='^[a-zA-Z0-9_-]+$'
     if test -z "$service"; then
-	Fail "SERVICE not specified"
+        Fail "SERVICE not specified"
     elif ! [[ "$service" =~ $service_regex ]]; then
-	Fail "Service must math the regex '$service_regex'"
+        Fail "Service must math the regex '$service_regex'"
     fi
 
     if ! getent passwd "$user" &> /dev/null; then
-	Fail "Bad user ($user): not found in passwd"
+        Fail "Bad user ($user): not found in passwd"
     elif test "$(id -un)" != "$user"; then
-	Fail "${0##*/} must be started by $user"
+        Fail "${0##*/} must be started by $user"
     fi
 
     case "$command" in
-	help) Usage ;;
-	start) StartCommand "$service" "$force" "${jvm_arguments[@]}" ;;
-	stop) StopCommand "$user" "$service" "$force" "$@" ;;
-	*) Fail "Unknown command '$command'" ;;
+        help) Usage ;;
+        start) StartCommand "$service" "$force" "${jvm_arguments[@]}" ;;
+        stop) StopCommand "$user" "$service" "$force" "$@" ;;
+        *) Fail "Unknown command '$command'" ;;
     esac
 }
 
diff --git a/standalone-container/src/test/java/com/yahoo/container/standalone/StandaloneContainerActivatorTest.java b/standalone-container/src/test/java/com/yahoo/container/standalone/StandaloneContainerActivatorTest.java
deleted file mode 100644
index f1c02b0149f..00000000000
--- a/standalone-container/src/test/java/com/yahoo/container/standalone/StandaloneContainerActivatorTest.java
+++ /dev/null
@@ -1,106 +0,0 @@
-// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.container.standalone;
-
-import com.google.inject.Module;
-import com.yahoo.io.IOUtils;
-import com.yahoo.jdisc.http.ConnectorConfig;
-import com.yahoo.vespa.defaults.Defaults;
-import com.yahoo.vespa.model.container.Container;
-import org.junit.Test;
-import org.xml.sax.SAXException;
-
-import javax.xml.parsers.ParserConfigurationException;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.util.List;
-
-import static java.util.Arrays.asList;
-import static java.util.Collections.singletonList;
-import static java.util.stream.Collectors.toList;
-import static org.hamcrest.CoreMatchers.is;
-import static org.hamcrest.collection.IsEmptyCollection.empty;
-import static org.junit.Assert.assertThat;
-
-/**
- * @author Einar M R Rosenvinge
- */
-public class StandaloneContainerActivatorTest {
-
-    private static String getJdiscXml(String contents) throws ParserConfigurationException, IOException, SAXException {
-        return "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n" +
-                "<services>\n" +
-                "  <container version=\"1.0\" jetty=\"true\">\n" +
-                contents +
-                "  </container>\n" +
-                "</services>";
-    }
-
-    private static void writeApplicationPackage(String servicesXml, Path tmpDir) throws IOException {
-        FileWriter fw = new FileWriter(tmpDir.resolve("services.xml").toFile(), false);
-        fw.write(servicesXml);
-        fw.close();
-    }
-
-    @Test
-    public void requireThatPortsCanBeFoundBasic() throws IOException, ParserConfigurationException, SAXException {
-        final Path applicationDir = Files.createTempDirectory("application");
-        try {
-            writeApplicationPackage(getJdiscXml(""), applicationDir);
-            StandaloneContainerActivator activator = new StandaloneContainerActivator();
-            Container container = StandaloneContainerActivator.getContainer(newAppDirBinding(applicationDir));
-            List<Integer> ports = getPorts(activator, container);
-            assertThat(ports, is(singletonList(Defaults.getDefaults().vespaWebServicePort())));
-        } finally {
-            IOUtils.recursiveDeleteDir(applicationDir.toFile());
-        }
-    }
-
-    private static List<Integer> getPorts(StandaloneContainerActivator activator, Container container) {
-        return StandaloneContainerActivator.getConnectorConfigs(container).stream().
-                map(ConnectorConfig::listenPort).
-                collect(toList());
-    }
-
-    @Test
-    public void requireThatPortsCanBeFoundNoHttp() throws IOException, ParserConfigurationException, SAXException {
-        final Path applicationDir = Files.createTempDirectory("application");
-        try {
-            writeApplicationPackage(getJdiscXml("<http/>"), applicationDir);
-            StandaloneContainerActivator activator = new StandaloneContainerActivator();
-            Container container = StandaloneContainerActivator.getContainer(newAppDirBinding(applicationDir));
-            List<Integer> ports = getPorts(activator, container);
-            assertThat(ports, empty());
-        } finally {
-            IOUtils.recursiveDeleteDir(applicationDir.toFile());
-        }
-    }
-
-    @Test
-    public void requireThatPortsCanBeFoundHttpThreeServers() throws IOException, ParserConfigurationException, SAXException {
-        final Path applicationDir = Files.createTempDirectory("application");
-        try {
-            final String contents =
-                    "<http>\n" +
-                    "  <server id=\"a\" port=\"123\"/>\n" +
-                    "  <server id=\"b\" port=\"456\"/>\n" +
-                    "  <server id=\"c\" port=\"789\"/>\n" +
-                    "</http>\n";
-            writeApplicationPackage(getJdiscXml(contents), applicationDir);
-            StandaloneContainerActivator activator = new StandaloneContainerActivator();
-            Container container = StandaloneContainerActivator.getContainer(newAppDirBinding(applicationDir));
-            List<Integer> ports = getPorts(activator, container);
-            assertThat(ports, is(asList(123, 456, 789)));
-        } finally {
-            IOUtils.recursiveDeleteDir(applicationDir.toFile());
-        }
-    }
-
-    private static Module newAppDirBinding(final Path applicationDir) {
-        return binder -> binder.bind(Path.class)
-                .annotatedWith(StandaloneContainerApplication.APPLICATION_PATH_NAME)
-                .toInstance(applicationDir);
-    }
-
-}
diff --git a/standalone-container/vespa-standalone-container.spec b/standalone-container/vespa-standalone-container.spec
index 98df4f71406..532706eb36f 100644
--- a/standalone-container/vespa-standalone-container.spec
+++ b/standalone-container/vespa-standalone-container.spec
@@ -8,7 +8,7 @@
 
 Name:           vespa-standalone-container
 Version:        %version
-BuildArch:      noarch
+BuildArch:      x86_64
 Release:        1%{?dist}
 Summary:        Vespa standalone JDisc container
 Group:          Applications/Databases
@@ -72,7 +72,6 @@ cp vespajlib/target/vespajlib.jar "$jars_dir"
 
 # Copy from submodules, so must be done separately
 cp zookeeper-server/zookeeper-server-common/target/zookeeper-server-common-jar-with-dependencies.jar "$jars_dir"
-cp zookeeper-server/zookeeper-server-3.4/target/zookeeper-server-3.4-jar-with-dependencies.jar "$jars_dir"
 cp zookeeper-server/zookeeper-server-3.5/target/zookeeper-server-3.5-jar-with-dependencies.jar "$jars_dir"
 # Symlink to default version
 ln -s zookeeper-server-3.5-jar-with-dependencies.jar "$jars_dir"/zookeeper-server-jar-with-dependencies.jar
@@ -100,22 +99,6 @@ done
 %clean
 rm -rf %buildroot
 
-%pre
-getent group vespa >/dev/null || groupadd -r vespa
-getent passwd vespa >/dev/null || \
-    useradd -r -g vespa -d %_prefix -s /sbin/nologin \
-    -c "Create owner of all Vespa data files" vespa
-echo "pathmunge %_prefix/bin" > /etc/profile.d/vespa.sh
-echo "export VESPA_HOME=%_prefix" >> /etc/profile.d/vespa.sh
-chmod +x /etc/profile.d/vespa.sh
-
-%postun
-if [ $1 -eq 0 ]; then # this is an uninstallation
-    rm -f /etc/profile.d/vespa.sh
-    ! getent passwd vespa >/dev/null || userdel vespa
-    ! getent group vespa >/dev/null || groupdel vespa
-fi
-
 %files
 %defattr(-,vespa,vespa,-)
 %_prefix/*
diff --git a/storage/src/tests/distributor/btree_bucket_database_test.cpp b/storage/src/tests/distributor/btree_bucket_database_test.cpp
index 43d74ca2fb5..a2518272a7f 100644
--- a/storage/src/tests/distributor/btree_bucket_database_test.cpp
+++ b/storage/src/tests/distributor/btree_bucket_database_test.cpp
@@ -9,8 +9,8 @@ using namespace ::testing;
 
 namespace storage::distributor {
 
-INSTANTIATE_TEST_CASE_P(BTreeDatabase, BucketDatabaseTest,
-                        ::testing::Values(std::make_shared<BTreeBucketDatabase>()));
+VESPA_GTEST_INSTANTIATE_TEST_SUITE_P(BTreeDatabase, BucketDatabaseTest,
+                                     ::testing::Values(std::make_shared<BTreeBucketDatabase>()));
 
 using document::BucketId;
 
diff --git a/storage/src/tests/distributor/bucketdbmetricupdatertest.cpp b/storage/src/tests/distributor/bucketdbmetricupdatertest.cpp
index 1008d3ee4f2..e0c3cf161bb 100644
--- a/storage/src/tests/distributor/bucketdbmetricupdatertest.cpp
+++ b/storage/src/tests/distributor/bucketdbmetricupdatertest.cpp
@@ -4,6 +4,7 @@
 #include <vespa/storage/distributor/distributormetricsset.h>
 #include <vespa/storage/distributor/idealstatemetricsset.h>
 #include <vespa/storage/config/config-stor-distributormanager.h>
+#include <vespa/vespalib/util/memoryusage.h>
 #include <vespa/vespalib/gtest/gtest.h>
 #include <string>
 #include <sstream>
@@ -100,6 +101,40 @@ TEST_F(BucketDBMetricUpdaterTest, doc_and_byte_counts_are_updated) {
     EXPECT_EQ(34, dms.bytesStored.getLast());
 }
 
+TEST_F(BucketDBMetricUpdaterTest, bucket_db_memory_usage_metrics_are_updated) {
+    BucketDBMetricUpdater metric_updater;
+    IdealStateMetricSet ims;
+    DistributorMetricSet dms(_loadTypes);
+
+    vespalib::MemoryUsage mem_usage;
+    mem_usage.incAllocatedBytes(1000);
+    mem_usage.incDeadBytes(700);
+    metric_updater.update_db_memory_usage(mem_usage, true);
+
+    mem_usage.incAllocatedBytes(500);
+    mem_usage.incDeadBytes(100);
+    metric_updater.update_db_memory_usage(mem_usage, false);
+
+    metric_updater.completeRound(false);
+    metric_updater.getLastCompleteStats().propagateMetrics(ims, dms);
+
+    auto* m = dms.mutable_dbs.memory_usage.getMetric("allocated_bytes");
+    ASSERT_TRUE(m != nullptr);
+    EXPECT_EQ(m->getLongValue("last"), 1000);
+
+    m = dms.mutable_dbs.memory_usage.getMetric("dead_bytes");
+    ASSERT_TRUE(m != nullptr);
+    EXPECT_EQ(m->getLongValue("last"), 700);
+
+    m = dms.read_only_dbs.memory_usage.getMetric("allocated_bytes");
+    ASSERT_TRUE(m != nullptr);
+    EXPECT_EQ(m->getLongValue("last"), 1500);
+
+    m = dms.read_only_dbs.memory_usage.getMetric("dead_bytes");
+    ASSERT_TRUE(m != nullptr);
+    EXPECT_EQ(m->getLongValue("last"), 800);
+}
+
 TEST_F(BucketDBMetricUpdaterTest, buckets_with_too_few_and_too_many_copies) {
     BucketDBMetricUpdater metricUpdater;
     IdealStateMetricSet ims;
diff --git a/storage/src/tests/distributor/distributor_message_sender_stub.h b/storage/src/tests/distributor/distributor_message_sender_stub.h
index 7ebd4dee1ae..440dee70d48 100644
--- a/storage/src/tests/distributor/distributor_message_sender_stub.h
+++ b/storage/src/tests/distributor/distributor_message_sender_stub.h
@@ -5,6 +5,7 @@
 #include <vespa/storage/distributor/distributormessagesender.h>
 #include <tests/common/message_sender_stub.h>
 #include <cassert>
+#include <string>
 
 namespace storage {
 
diff --git a/storage/src/tests/distributor/distributortest.cpp b/storage/src/tests/distributor/distributortest.cpp
index afc3b254c64..421fd2cb4b6 100644
--- a/storage/src/tests/distributor/distributortest.cpp
+++ b/storage/src/tests/distributor/distributortest.cpp
@@ -12,6 +12,7 @@
 #include <vespa/document/test/make_bucket_space.h>
 #include <vespa/storage/config/config-stor-distributormanager.h>
 #include <vespa/storage/distributor/distributor.h>
+#include <vespa/storage/distributor/distributormetricsset.h>
 #include <vespa/vespalib/text/stringtokenizer.h>
 #include <vespa/vespalib/util/time.h>
 #include <thread>
@@ -193,6 +194,12 @@ struct DistributorTest : Test, DistributorTestUtil {
         configureDistributor(builder);
     }
 
+    void configure_metadata_update_phase_enabled(bool enabled) {
+        ConfigBuilder builder;
+        builder.enableMetadataOnlyFetchPhaseForInconsistentUpdates = enabled;
+        configureDistributor(builder);
+    }
+
     void configureMaxClusterClockSkew(int seconds);
     void sendDownClusterStateCommand();
     void replyToSingleRequestBucketInfoCommandWith1Bucket();
@@ -457,6 +464,51 @@ TEST_F(DistributorTest, metric_update_hook_updates_pending_maintenance_metrics)
     }
 }
 
+namespace {
+
+uint64_t db_sample_interval_sec(const Distributor& d) noexcept {
+    return std::chrono::duration_cast<std::chrono::seconds>(d.db_memory_sample_interval()).count();
+}
+
+}
+
+TEST_F(DistributorTest, bucket_db_memory_usage_metrics_only_updated_at_fixed_time_intervals) {
+    getClock().setAbsoluteTimeInSeconds(1000);
+
+    setupDistributor(Redundancy(2), NodeCount(2), "storage:2 distributor:1");
+    addNodesToBucketDB(document::BucketId(16, 1), "0=1/1/1/t/a,1=2/2/2");
+    tickDistributorNTimes(10);
+
+    vespalib::Monitor l;
+    distributor_metric_update_hook().updateMetrics(vespalib::MonitorGuard(l));
+    auto* m = getDistributor().getMetrics().mutable_dbs.memory_usage.getMetric("used_bytes");
+    ASSERT_TRUE(m != nullptr);
+    auto last_used = m->getLongValue("last");
+    EXPECT_GT(last_used, 0);
+
+    // Add another bucket to the DB. This should increase the underlying used number of
+    // bytes, but this should not be aggregated into the metrics until the sampling time
+    // interval has passed. Instead, old metric gauge values should be preserved.
+    addNodesToBucketDB(document::BucketId(16, 2), "0=1/1/1/t/a,1=2/2/2");
+
+    const auto sample_interval_sec = db_sample_interval_sec(getDistributor());
+    getClock().setAbsoluteTimeInSeconds(1000 + sample_interval_sec - 1); // Not there yet.
+    tickDistributorNTimes(50);
+    distributor_metric_update_hook().updateMetrics(vespalib::MonitorGuard(l));
+
+    m = getDistributor().getMetrics().mutable_dbs.memory_usage.getMetric("used_bytes");
+    auto now_used = m->getLongValue("last");
+    EXPECT_EQ(now_used, last_used);
+
+    getClock().setAbsoluteTimeInSeconds(1000 + sample_interval_sec + 1);
+    tickDistributorNTimes(10);
+    distributor_metric_update_hook().updateMetrics(vespalib::MonitorGuard(l));
+
+    m = getDistributor().getMetrics().mutable_dbs.memory_usage.getMetric("used_bytes");
+    now_used = m->getLongValue("last");
+    EXPECT_GT(now_used, last_used);
+}
+
 TEST_F(DistributorTest, priority_config_is_propagated_to_distributor_configuration) {
     using namespace vespa::config::content::core;
 
@@ -1044,6 +1096,17 @@ TEST_F(DistributorTest, merge_disabling_config_is_propagated_to_internal_config)
     EXPECT_FALSE(getConfig().merge_operations_disabled());
 }
 
+TEST_F(DistributorTest, metadata_update_phase_config_is_propagated_to_internal_config) {
+    createLinks(true);
+    setupDistributor(Redundancy(1), NodeCount(1), "distributor:1 storage:1");
+
+    configure_metadata_update_phase_enabled(true);
+    EXPECT_TRUE(getConfig().enable_metadata_only_fetch_phase_for_inconsistent_updates());
+
+    configure_metadata_update_phase_enabled(false);
+    EXPECT_FALSE(getConfig().enable_metadata_only_fetch_phase_for_inconsistent_updates());
+}
+
 TEST_F(DistributorTest, weak_internal_read_consistency_config_is_propagated_to_internal_configs) {
     createLinks(true);
     setupDistributor(Redundancy(1), NodeCount(1), "distributor:1 storage:1");
diff --git a/storage/src/tests/distributor/garbagecollectiontest.cpp b/storage/src/tests/distributor/garbagecollectiontest.cpp
index 65c1ac726b5..776cfc14d84 100644
--- a/storage/src/tests/distributor/garbagecollectiontest.cpp
+++ b/storage/src/tests/distributor/garbagecollectiontest.cpp
@@ -3,6 +3,7 @@
 #include <vespa/storageapi/message/removelocation.h>
 #include <vespa/storage/distributor/operations/idealstate/garbagecollectionoperation.h>
 #include <vespa/storage/distributor/idealstatemanager.h>
+#include <vespa/storage/distributor/idealstatemetricsset.h>
 #include <tests/distributor/distributortestutil.h>
 #include <vespa/storage/distributor/distributor.h>
 #include <vespa/document/test/make_document_bucket.h>
@@ -35,11 +36,13 @@ struct GarbageCollectionOperationTest : Test, DistributorTestUtil {
     }
 
     // FIXME fragile to assume that send order == node index, but that's the way it currently works
-    void reply_to_nth_request(GarbageCollectionOperation& op, size_t n, uint32_t bucket_info_checksum) {
+    void reply_to_nth_request(GarbageCollectionOperation& op, size_t n,
+                              uint32_t bucket_info_checksum, uint32_t n_docs_removed) {
         auto msg = _sender.command(n);
         assert(msg->getType() == api::MessageType::REMOVELOCATION);
         std::shared_ptr<api::StorageReply> reply(msg->makeReply());
         auto& gc_reply = dynamic_cast<api::RemoveLocationReply&>(*reply);
+        gc_reply.set_documents_removed(n_docs_removed);
         gc_reply.setBucketInfo(api::BucketInfo(bucket_info_checksum, 90, 500));
 
         op.receive(_sender, reply);
@@ -56,6 +59,13 @@ struct GarbageCollectionOperationTest : Test, DistributorTestUtil {
                     << entry->getNode(i)->getBucketInfo();
         }
     }
+
+    uint32_t gc_removed_documents_metric() {
+        auto metric_base = getIdealStateManager().getMetrics().operations[IdealStateOperation::GARBAGE_COLLECTION];
+        auto gc_metrics = std::dynamic_pointer_cast<GcMetricSet>(metric_base);
+        assert(gc_metrics);
+        return gc_metrics->documents_removed.getValue();
+    }
 };
 
 TEST_F(GarbageCollectionOperationTest, simple) {
@@ -63,29 +73,34 @@ TEST_F(GarbageCollectionOperationTest, simple) {
     op->start(_sender, framework::MilliSecTime(0));
 
     ASSERT_EQ(2, _sender.commands().size());
+    EXPECT_EQ(0u, gc_removed_documents_metric());
 
     for (uint32_t i = 0; i < 2; ++i) {
         std::shared_ptr<api::StorageCommand> msg = _sender.command(i);
         ASSERT_EQ(msg->getType(), api::MessageType::REMOVELOCATION);
         auto& tmp = dynamic_cast<api::RemoveLocationCommand&>(*msg);
         EXPECT_EQ("music.date < 34", tmp.getDocumentSelection());
-        reply_to_nth_request(*op, i, 777 + i);
+        reply_to_nth_request(*op, i, 777 + i, 50);
     }
     ASSERT_NO_FATAL_FAILURE(assert_bucket_db_contains({api::BucketInfo(777, 90, 500), api::BucketInfo(778, 90, 500)}, 34));
+    EXPECT_EQ(50u, gc_removed_documents_metric());
 }
 
 TEST_F(GarbageCollectionOperationTest, replica_bucket_info_not_added_to_db_until_all_replies_received) {
     auto op = create_op();
     op->start(_sender, framework::MilliSecTime(0));
     ASSERT_EQ(2, _sender.commands().size());
+    EXPECT_EQ(0u, gc_removed_documents_metric());
 
     // Respond to 1st request. Should _not_ cause bucket info to be merged into the database yet
-    reply_to_nth_request(*op, 0, 1234);
+    reply_to_nth_request(*op, 0, 1234, 70);
     ASSERT_NO_FATAL_FAILURE(assert_bucket_db_contains({api::BucketInfo(250, 50, 300), api::BucketInfo(250, 50, 300)}, 0));
 
     // Respond to 2nd request. This _should_ cause bucket info to be merged into the database.
-    reply_to_nth_request(*op, 1, 4567);
+    reply_to_nth_request(*op, 1, 4567, 60);
     ASSERT_NO_FATAL_FAILURE(assert_bucket_db_contains({api::BucketInfo(1234, 90, 500), api::BucketInfo(4567, 90, 500)}, 34));
+
+    EXPECT_EQ(70u, gc_removed_documents_metric()); // Use max of received metrics
 }
 
 TEST_F(GarbageCollectionOperationTest, gc_bucket_info_does_not_overwrite_later_sequenced_bucket_info_writes) {
@@ -93,10 +108,10 @@ TEST_F(GarbageCollectionOperationTest, gc_bucket_info_does_not_overwrite_later_s
     op->start(_sender, framework::MilliSecTime(0));
     ASSERT_EQ(2, _sender.commands().size());
 
-    reply_to_nth_request(*op, 0, 1234);
+    reply_to_nth_request(*op, 0, 1234, 0);
     // Change to replica on node 0 happens after GC op, but before GC info is merged into the DB. Must not be lost.
     insertBucketInfo(op->getBucketId(), 0, 7777, 100, 2000);
-    reply_to_nth_request(*op, 1, 4567);
+    reply_to_nth_request(*op, 1, 4567, 0);
     // Bucket info for node 0 is that of the later sequenced operation, _not_ from the earlier GC op.
     ASSERT_NO_FATAL_FAILURE(assert_bucket_db_contains({api::BucketInfo(7777, 100, 2000), api::BucketInfo(4567, 90, 500)}, 34));
 }
diff --git a/storage/src/tests/distributor/getoperationtest.cpp b/storage/src/tests/distributor/getoperationtest.cpp
index 0dbec8444cc..f14b78094d1 100644
--- a/storage/src/tests/distributor/getoperationtest.cpp
+++ b/storage/src/tests/distributor/getoperationtest.cpp
@@ -21,6 +21,7 @@ using config::ConfigGetter;
 using document::DocumenttypesConfig;
 using config::FileSpec;
 using document::test::makeDocumentBucket;
+using document::BucketId;
 using namespace ::testing;
 
 namespace storage::distributor {
@@ -29,8 +30,8 @@ struct GetOperationTest : Test, DistributorTestUtil {
 
     std::shared_ptr<const document::DocumentTypeRepo> _repo;
     document::DocumentId docId;
-    document::BucketId bucketId;
-    std::unique_ptr<Operation> op;
+    BucketId bucketId;
+    std::unique_ptr<GetOperation> op;
 
     GetOperationTest();
     ~GetOperationTest() override;
@@ -52,7 +53,7 @@ struct GetOperationTest : Test, DistributorTestUtil {
     }
 
     void sendGet(api::InternalReadConsistency consistency = api::InternalReadConsistency::Strong) {
-        auto msg = std::make_shared<api::GetCommand>(makeDocumentBucket(document::BucketId(0)), docId, "[all]");
+        auto msg = std::make_shared<api::GetCommand>(makeDocumentBucket(BucketId(0)), docId, "[all]");
         op = std::make_unique<GetOperation>(
                 getExternalOperationHandler(), getDistributorBucketSpace(),
                 getDistributorBucketSpace().getBucketDatabase().acquire_read_guard(),
@@ -135,6 +136,14 @@ struct GetOperationTest : Test, DistributorTestUtil {
 GetOperationTest::GetOperationTest() = default;
 GetOperationTest::~GetOperationTest() = default;
 
+namespace {
+
+NewestReplica replica_of(api::Timestamp ts, const document::BucketId& bucket_id, uint16_t node) {
+    return NewestReplica::of(ts, bucket_id, node);
+}
+
+}
+
 TEST_F(GetOperationTest, simple) {
     setClusterState("distributor:1 storage:2");
 
@@ -149,7 +158,10 @@ TEST_F(GetOperationTest, simple) {
     EXPECT_EQ("GetReply(BucketId(0x0000000000000000), id:ns:text/html::uri, "
               "timestamp 100) ReturnCode(NONE)",
               _sender.getLastReply());
+    EXPECT_FALSE(op->any_replicas_failed());
     EXPECT_TRUE(last_reply_had_consistent_replicas());
+    ASSERT_TRUE(op->newest_replica().has_value());
+    EXPECT_EQ(replica_of(api::Timestamp(100), bucketId, 0), *op->newest_replica());
 }
 
 TEST_F(GetOperationTest, ask_all_checksum_groups_if_inconsistent_even_if_trusted_replica_available) {
@@ -167,7 +179,10 @@ TEST_F(GetOperationTest, ask_all_checksum_groups_if_inconsistent_even_if_trusted
     EXPECT_EQ("GetReply(BucketId(0x0000000000000000), id:ns:text/html::uri, "
               "timestamp 2) ReturnCode(NONE)",
               _sender.getLastReply());
+    EXPECT_FALSE(op->any_replicas_failed());
     EXPECT_FALSE(last_reply_had_consistent_replicas());
+    ASSERT_TRUE(op->newest_replica().has_value());
+    EXPECT_EQ(replica_of(api::Timestamp(2), bucketId, 0), *op->newest_replica());
 }
 
 TEST_F(GetOperationTest, ask_all_nodes_if_bucket_is_inconsistent) {
@@ -179,15 +194,18 @@ TEST_F(GetOperationTest, ask_all_nodes_if_bucket_is_inconsistent) {
 
     ASSERT_EQ("Get => 0,Get => 1", _sender.getCommands(true));
 
-    ASSERT_NO_FATAL_FAILURE(sendReply(0, api::ReturnCode::OK, "newauthor", 2));
-    ASSERT_NO_FATAL_FAILURE(sendReply(1, api::ReturnCode::OK, "oldauthor", 1));
+    ASSERT_NO_FATAL_FAILURE(sendReply(1, api::ReturnCode::OK, "newauthor", 2));
+    ASSERT_NO_FATAL_FAILURE(sendReply(0, api::ReturnCode::OK, "oldauthor", 1));
 
     ASSERT_EQ("GetReply(BucketId(0x0000000000000000), id:ns:text/html::uri, "
               "timestamp 2) ReturnCode(NONE)",
               _sender.getLastReply());
-
     EXPECT_EQ("newauthor", getLastReplyAuthor());
+
+    EXPECT_FALSE(op->any_replicas_failed());
     EXPECT_FALSE(last_reply_had_consistent_replicas());
+    ASSERT_TRUE(op->newest_replica().has_value());
+    EXPECT_EQ(replica_of(api::Timestamp(2), bucketId, 1), *op->newest_replica());
 }
 
 TEST_F(GetOperationTest, send_to_all_invalid_copies) {
@@ -205,8 +223,9 @@ TEST_F(GetOperationTest, send_to_all_invalid_copies) {
     ASSERT_EQ("GetReply(BucketId(0x0000000000000000), id:ns:text/html::uri, "
               "timestamp 2) ReturnCode(NONE)",
               _sender.getLastReply());
-
     EXPECT_EQ("newauthor", getLastReplyAuthor());
+
+    EXPECT_FALSE(op->any_replicas_failed());
     EXPECT_FALSE(last_reply_had_consistent_replicas());
 }
 
@@ -235,8 +254,8 @@ TEST_F(GetOperationTest, send_to_all_invalid_nodes_when_inconsistent) {
 TEST_F(GetOperationTest, inconsistent_split) {
     setClusterState("distributor:1 storage:4");
 
-    addNodesToBucketDB(document::BucketId(16, 0x0593), "0=100");
-    addNodesToBucketDB(document::BucketId(17, 0x10593), "1=200");
+    addNodesToBucketDB(BucketId(16, 0x0593), "0=100");
+    addNodesToBucketDB(BucketId(17, 0x10593), "1=200");
 
     sendGet();
 
@@ -248,9 +267,13 @@ TEST_F(GetOperationTest, inconsistent_split) {
     ASSERT_EQ("GetReply(BucketId(0x0000000000000000), id:ns:text/html::uri, "
               "timestamp 2) ReturnCode(NONE)",
               _sender.getLastReply());
-
     EXPECT_EQ("newauthor", getLastReplyAuthor());
+
+    EXPECT_FALSE(op->any_replicas_failed());
     EXPECT_FALSE(last_reply_had_consistent_replicas());
+    // Bucket with highest timestamp should be returned. In this case it's the one on node 0.
+    ASSERT_TRUE(op->newest_replica().has_value());
+    EXPECT_EQ(replica_of(api::Timestamp(2), BucketId(16, 0x0593), 0), *op->newest_replica());
 }
 
 TEST_F(GetOperationTest, multi_inconsistent_bucket_not_found) {
@@ -268,6 +291,8 @@ TEST_F(GetOperationTest, multi_inconsistent_bucket_not_found) {
     ASSERT_EQ("GetReply(BucketId(0x0000000000000000), id:ns:text/html::uri, "
               "timestamp 2) ReturnCode(NONE)",
               _sender.getLastReply());
+
+    EXPECT_FALSE(op->any_replicas_failed());
     EXPECT_FALSE(last_reply_had_consistent_replicas());
 }
 
@@ -288,7 +313,11 @@ TEST_F(GetOperationTest, multi_inconsistent_bucket_not_found_deleted) {
     ASSERT_EQ("GetReply(BucketId(0x0000000000000000), id:ns:text/html::uri, "
               "timestamp 3) ReturnCode(NONE)",
               _sender.getLastReply());
+
+    EXPECT_FALSE(op->any_replicas_failed());
     EXPECT_FALSE(last_reply_had_consistent_replicas());
+    ASSERT_TRUE(op->newest_replica().has_value());
+    EXPECT_EQ(replica_of(api::Timestamp(3), bucketId, 1), *op->newest_replica());
 }
 
 TEST_F(GetOperationTest, multi_inconsistent_bucket) {
@@ -308,6 +337,8 @@ TEST_F(GetOperationTest, multi_inconsistent_bucket) {
               _sender.getLastReply());
 
     EXPECT_EQ("newauthor", getLastReplyAuthor());
+
+    EXPECT_FALSE(op->any_replicas_failed());
     EXPECT_FALSE(last_reply_had_consistent_replicas());
 }
 
@@ -331,7 +362,12 @@ TEST_F(GetOperationTest, multi_inconsistent_bucket_fail) {
     ASSERT_EQ("GetReply(BucketId(0x0000000000000000), id:ns:text/html::uri, "
               "timestamp 100) ReturnCode(NONE)",
               _sender.getLastReply());
+
+    EXPECT_TRUE(op->any_replicas_failed());
     EXPECT_FALSE(last_reply_had_consistent_replicas());
+    ASSERT_TRUE(op->newest_replica().has_value());
+    // First send to node 2 fails, second is to node 3 which returned the highest timestamp
+    EXPECT_EQ(replica_of(api::Timestamp(100), bucketId, 3), *op->newest_replica());
 }
 
 TEST_F(GetOperationTest, return_not_found_when_bucket_not_in_db) {
@@ -342,6 +378,8 @@ TEST_F(GetOperationTest, return_not_found_when_bucket_not_in_db) {
     ASSERT_EQ("GetReply(BucketId(0x0000000000000000), id:ns:text/html::uri, "
               "timestamp 0) ReturnCode(NONE)",
               _sender.getLastReply());
+
+    EXPECT_FALSE(op->any_replicas_failed());
     EXPECT_TRUE(last_reply_had_consistent_replicas()); // Nothing in the bucket, so nothing to be inconsistent with.
 }
 
@@ -363,7 +401,14 @@ TEST_F(GetOperationTest, not_found) {
 
     EXPECT_EQ(1, getDistributor().getMetrics().gets[documentapi::LoadType::DEFAULT].
                                  failures.notfound.getValue());
+    EXPECT_FALSE(op->any_replicas_failed()); // "Not found" is not a failure.
     EXPECT_TRUE(last_reply_had_consistent_replicas());
+    EXPECT_TRUE(op->newest_replica().has_value());
+    // "Not found" is still a success with a timestamp of 0. This is because
+    // the caller may want to perform special logic if all replicas are in sync
+    // but are missing the document.
+    // FIXME make sure all callers are aware of this!
+    EXPECT_EQ(replica_of(api::Timestamp(0), bucketId, 0), *op->newest_replica());
 }
 
 TEST_F(GetOperationTest, not_found_on_subset_of_replicas_marks_get_as_inconsistent) {
@@ -403,8 +448,12 @@ TEST_F(GetOperationTest, resend_on_storage_failure) {
     ASSERT_EQ("GetReply(BucketId(0x0000000000000000), id:ns:text/html::uri, "
               "timestamp 100) ReturnCode(NONE)",
               _sender.getLastReply());
+
+    EXPECT_TRUE(op->any_replicas_failed());
     // Replica had read failure, but they're still in sync. An immutable Get won't change that fact.
     EXPECT_TRUE(last_reply_had_consistent_replicas());
+    ASSERT_TRUE(op->newest_replica().has_value());
+    EXPECT_EQ(replica_of(api::Timestamp(100), bucketId, 2), *op->newest_replica());
 }
 
 TEST_F(GetOperationTest, storage_failure_of_out_of_sync_replica_is_tracked_as_inconsistent) {
@@ -417,7 +466,11 @@ TEST_F(GetOperationTest, storage_failure_of_out_of_sync_replica_is_tracked_as_in
     ASSERT_EQ("GetReply(BucketId(0x0000000000000000), id:ns:text/html::uri, "
               "timestamp 3) ReturnCode(NONE)",
               _sender.getLastReply());
+
+    EXPECT_TRUE(op->any_replicas_failed());
     EXPECT_FALSE(last_reply_had_consistent_replicas());
+    ASSERT_TRUE(op->newest_replica().has_value());
+    EXPECT_EQ(replica_of(api::Timestamp(3), bucketId, 2), *op->newest_replica());
 }
 
 TEST_F(GetOperationTest, resend_on_storage_failure_all_fail) {
@@ -442,7 +495,10 @@ TEST_F(GetOperationTest, resend_on_storage_failure_all_fail) {
     ASSERT_EQ("GetReply(BucketId(0x0000000000000000), id:ns:text/html::uri, "
               "timestamp 0) ReturnCode(IO_FAILURE)",
               _sender.getLastReply());
+
+    EXPECT_TRUE(op->any_replicas_failed());
     EXPECT_TRUE(last_reply_had_consistent_replicas()); // Doesn't really matter since operation itself failed
+    EXPECT_FALSE(op->newest_replica().has_value());
 }
 
 TEST_F(GetOperationTest, send_to_ideal_copy_if_bucket_in_sync) {
@@ -462,6 +518,8 @@ TEST_F(GetOperationTest, send_to_ideal_copy_if_bucket_in_sync) {
               "timestamp 100) ReturnCode(NONE)",
               _sender.getLastReply());
     EXPECT_TRUE(last_reply_had_consistent_replicas());
+    ASSERT_TRUE(op->newest_replica().has_value());
+    EXPECT_EQ(replica_of(api::Timestamp(100), bucketId, 1), *op->newest_replica());
 }
 
 TEST_F(GetOperationTest, multiple_copies_with_failure_on_local_node) {
@@ -469,7 +527,7 @@ TEST_F(GetOperationTest, multiple_copies_with_failure_on_local_node) {
 
     // Node 0 is local copy to distributor 0 and will be preferred when
     // sending initially.
-    addNodesToBucketDB(document::BucketId(16, 0x0593), "2=100,0=100");
+    addNodesToBucketDB(BucketId(16, 0x0593), "2=100,0=100");
 
     sendGet();
 
@@ -487,9 +545,12 @@ TEST_F(GetOperationTest, multiple_copies_with_failure_on_local_node) {
     ASSERT_EQ("GetReply(BucketId(0x0000000000000000), id:ns:text/html::uri, "
               "timestamp 3) ReturnCode(NONE)",
               _sender.getLastReply());
-
     EXPECT_EQ("newestauthor", getLastReplyAuthor());
+
+    EXPECT_TRUE(op->any_replicas_failed());
     EXPECT_TRUE(last_reply_had_consistent_replicas());
+    ASSERT_TRUE(op->newest_replica().has_value());
+    EXPECT_EQ(replica_of(api::Timestamp(3), BucketId(16, 0x0593), 2), *op->newest_replica());
 }
 
 TEST_F(GetOperationTest, can_get_documents_when_all_replica_nodes_retired) {
diff --git a/storage/src/tests/distributor/mapbucketdatabasetest.cpp b/storage/src/tests/distributor/mapbucketdatabasetest.cpp
index 0ae4a49530e..2c000f6b5db 100644
--- a/storage/src/tests/distributor/mapbucketdatabasetest.cpp
+++ b/storage/src/tests/distributor/mapbucketdatabasetest.cpp
@@ -5,7 +5,7 @@
 
 namespace storage::distributor {
 
-INSTANTIATE_TEST_CASE_P(MapDatabase, BucketDatabaseTest,
-                        ::testing::Values(std::make_shared<MapBucketDatabase>()));
+VESPA_GTEST_INSTANTIATE_TEST_SUITE_P(MapDatabase, BucketDatabaseTest,
+                                     ::testing::Values(std::make_shared<MapBucketDatabase>()));
 
 }
diff --git a/storage/src/tests/distributor/putoperationtest.cpp b/storage/src/tests/distributor/putoperationtest.cpp
index bd76b559490..f5bacfae252 100644
--- a/storage/src/tests/distributor/putoperationtest.cpp
+++ b/storage/src/tests/distributor/putoperationtest.cpp
@@ -182,7 +182,7 @@ TEST_F(PutOperationTest, do_not_send_inline_split_if_not_configured) {
               _sender.getCommands(true, true));
 }
 
-TEST_F(PutOperationTest, node_removed_on_reply) {
+TEST_F(PutOperationTest, return_success_if_op_acked_on_all_replicas_even_if_bucket_concurrently_removed_from_db) {
     setupDistributor(2, 2, "storage:2 distributor:1");
     createAndSendSampleDocument(TIMEOUT);
 
@@ -194,14 +194,19 @@ TEST_F(PutOperationTest, node_removed_on_reply) {
 
     getExternalOperationHandler().removeNodeFromDB(makeDocumentBucket(document::BucketId(16, 0x1dd4)), 0);
 
+    // If we get an ACK from the backend nodes, the operation has been persisted OK.
+    // Even if the bucket has been removed from the DB in the meantime (usually would
+    // happen due to ownership changes) there is no reason for us to trigger a client
+    // resend in this scenario.
+    // If a node goes down (as opposed to distributor ownership transfer) and therefore
+    // has its replicas removed from the DB, this by definition has happened-after
+    // the ACK was sent from the node, so returning OK here still maintains the
+    // backend persistence property.
     sendReply(0);
     sendReply(1);
 
     ASSERT_EQ("PutReply(id:test:testdoctype1::, BucketId(0x0000000000000000), "
-              "timestamp 100) ReturnCode(BUCKET_DELETED, "
-              "Bucket(BucketSpace(0x0000000000000001), BucketId(0x4000000000001dd4)) was deleted from nodes [0] "
-              "after message was sent but before it was done. "
-              "Sent to [0,1])",
+              "timestamp 100) ReturnCode(NONE)",
               _sender.getLastReply());
 }
 
diff --git a/storage/src/tests/distributor/twophaseupdateoperationtest.cpp b/storage/src/tests/distributor/twophaseupdateoperationtest.cpp
index f1900c40d56..962ce085cb0 100644
--- a/storage/src/tests/distributor/twophaseupdateoperationtest.cpp
+++ b/storage/src/tests/distributor/twophaseupdateoperationtest.cpp
@@ -32,9 +32,10 @@ struct TwoPhaseUpdateOperationTest : Test, DistributorTestUtil {
     document::TestDocRepo _testRepo;
     std::shared_ptr<const DocumentTypeRepo> _repo;
     const DocumentType* _doc_type;
+    DistributorMessageSenderStub _sender;
 
     TwoPhaseUpdateOperationTest();
-    ~TwoPhaseUpdateOperationTest();
+    ~TwoPhaseUpdateOperationTest() override;
 
     void checkMessageSettingsPropagatedTo(
         const api::StorageCommand::SP& msg) const;
@@ -81,6 +82,14 @@ struct TwoPhaseUpdateOperationTest : Test, DistributorTestUtil {
             api::ReturnCode::Result result = api::ReturnCode::OK,
             const std::string& traceMsg = "");
 
+    void reply_to_metadata_get(
+            Operation& callback,
+            DistributorMessageSenderStub& sender,
+            uint32_t index,
+            uint64_t old_timestamp,
+            api::ReturnCode::Result result = api::ReturnCode::OK,
+            const std::string& trace_msg = "");
+
     struct UpdateOptions {
         bool _makeInconsistentSplit;
         bool _createIfNonExistent;
@@ -130,6 +139,14 @@ struct TwoPhaseUpdateOperationTest : Test, DistributorTestUtil {
                                                                  Timestamp highest_get_timestamp,
                                                                  Timestamp expected_response_timestamp);
 
+    std::shared_ptr<TwoPhaseUpdateOperation> set_up_2_inconsistent_replicas_and_start_update(bool enable_3phase = true) {
+        setupDistributor(2, 2, "storage:2 distributor:1");
+        getConfig().set_enable_metadata_only_fetch_phase_for_inconsistent_updates(enable_3phase);
+        auto cb = sendUpdate("0=1/2/3,1=2/3/4"); // Inconsistent replicas.
+        cb->start(_sender, framework::MilliSecTime(0));
+        return cb;
+    }
+
 };
 
 TwoPhaseUpdateOperationTest::TwoPhaseUpdateOperationTest() = default;
@@ -218,6 +235,24 @@ TwoPhaseUpdateOperationTest::replyToGet(
     callback.receive(sender, reply);
 }
 
+void
+TwoPhaseUpdateOperationTest::reply_to_metadata_get(
+        Operation& callback,
+        DistributorMessageSenderStub& sender,
+        uint32_t index,
+        uint64_t old_timestamp,
+        api::ReturnCode::Result result,
+        const std::string& trace_msg)
+{
+    auto& get = dynamic_cast<const api::GetCommand&>(*sender.command(index));
+    auto reply = std::make_shared<api::GetReply>(get, std::shared_ptr<Document>(), old_timestamp);
+    reply->setResult(api::ReturnCode(result, ""));
+    if (!trace_msg.empty()) {
+        MBUS_TRACE(reply->getTrace(), 1, trace_msg);
+    }
+    callback.receive(sender, reply);
+}
+
 namespace {
 
 struct DummyTransportContext : api::TransportContext {
@@ -281,234 +316,212 @@ TwoPhaseUpdateOperationTest::sendUpdate(const std::string& bucketState,
 
 TEST_F(TwoPhaseUpdateOperationTest, simple) {
     setupDistributor(1, 1, "storage:1 distributor:1");
+    auto cb = sendUpdate("0=1/2/3");
+    cb->start(_sender, framework::MilliSecTime(0));
 
-    std::shared_ptr<TwoPhaseUpdateOperation> cb(sendUpdate("0=1/2/3"));
-    DistributorMessageSenderStub sender;
-    cb->start(sender, framework::MilliSecTime(0));
+    ASSERT_EQ("Update => 0", _sender.getCommands(true));
 
-    ASSERT_EQ("Update => 0", sender.getCommands(true));
-
-    replyToMessage(*cb, sender, 0, 90);
+    replyToMessage(*cb, _sender, 0, 90);
 
     EXPECT_EQ("UpdateReply(id:ns:testdoctype1::1, BucketId(0x0000000000000000), "
               "timestamp 0, timestamp of updated doc: 90) ReturnCode(NONE)",
-              sender.getLastReply(true));
+              _sender.getLastReply(true));
 }
 
 TEST_F(TwoPhaseUpdateOperationTest, non_existing) {
     setupDistributor(1, 1, "storage:1 distributor:1");
-
-    std::shared_ptr<TwoPhaseUpdateOperation> cb(sendUpdate(""));
-    DistributorMessageSenderStub sender;
-    cb->start(sender, framework::MilliSecTime(0));
+    auto cb = sendUpdate("");
+    cb->start(_sender, framework::MilliSecTime(0));
 
     EXPECT_EQ("UpdateReply(id:ns:testdoctype1::1, BucketId(0x0000000000000000), "
               "timestamp 0, timestamp of updated doc: 0) ReturnCode(NONE)",
-              sender.getLastReply(true));
+              _sender.getLastReply(true));
 }
 
 TEST_F(TwoPhaseUpdateOperationTest, update_failed) {
     setupDistributor(1, 1, "storage:1 distributor:1");
+    auto cb = sendUpdate("0=1/2/3");
+    cb->start(_sender, framework::MilliSecTime(0));
 
-    std::shared_ptr<TwoPhaseUpdateOperation> cb(sendUpdate("0=1/2/3"));
-    DistributorMessageSenderStub sender;
-    cb->start(sender, framework::MilliSecTime(0));
-
-    ASSERT_EQ("Update => 0", sender.getCommands(true));
+    ASSERT_EQ("Update => 0", _sender.getCommands(true));
 
-    replyToMessage(*cb, sender, 0, 90, api::ReturnCode::INTERNAL_FAILURE);
+    replyToMessage(*cb, _sender, 0, 90, api::ReturnCode::INTERNAL_FAILURE);
 
     EXPECT_EQ("UpdateReply(id:ns:testdoctype1::1, BucketId(0x0000000000000000), "
               "timestamp 0, timestamp of updated doc: 0) "
               "ReturnCode(INTERNAL_FAILURE)",
-              sender.getLastReply(true));
+              _sender.getLastReply(true));
 }
 
 TEST_F(TwoPhaseUpdateOperationTest, fast_path_inconsistent_timestamps) {
     setupDistributor(2, 2, "storage:2 distributor:1");
+    auto cb = sendUpdate("0=1/2/3,1=1/2/3");
+    cb->start(_sender, framework::MilliSecTime(0));
 
-    std::shared_ptr<TwoPhaseUpdateOperation> cb(sendUpdate("0=1/2/3,1=1/2/3"));
-    DistributorMessageSenderStub sender;
-    cb->start(sender, framework::MilliSecTime(0));
+    ASSERT_EQ("Update => 0,Update => 1", _sender.getCommands(true));
 
-    ASSERT_EQ("Update => 0,Update => 1", sender.getCommands(true));
+    replyToMessage(*cb, _sender, 0, 90);
+    replyToMessage(*cb, _sender, 1, 110);
 
-    replyToMessage(*cb, sender, 0, 90);
-    replyToMessage(*cb, sender, 1, 110);
+    ASSERT_EQ("Get(BucketId(0x400000000000cac4), id:ns:testdoctype1::1) => 1", _sender.getLastCommand(true));
 
-    ASSERT_EQ("Get(BucketId(0x400000000000cac4), id:ns:testdoctype1::1) => 1", sender.getLastCommand(true));
+    replyToGet(*cb, _sender, 2, 110);
 
-    replyToGet(*cb, sender, 2, 110);
+    ASSERT_EQ("Update => 0,Update => 1,Get => 1,Put => 1,Put => 0", _sender.getCommands(true));
 
-    ASSERT_EQ("Update => 0,Update => 1,Get => 1,Put => 1,Put => 0", sender.getCommands(true));
+    ASSERT_TRUE(_sender.replies().empty());
 
-    ASSERT_TRUE(sender.replies().empty());
-
-    replyToPut(*cb, sender, 3);
-    replyToPut(*cb, sender, 4);
+    replyToPut(*cb, _sender, 3);
+    replyToPut(*cb, _sender, 4);
 
     EXPECT_EQ("UpdateReply(id:ns:testdoctype1::1, BucketId(0x0000000000000000), "
               "timestamp 0, timestamp of updated doc: 110 Was inconsistent "
               "(best node 1)) ReturnCode(NONE)",
-              sender.getLastReply(true));
+              _sender.getLastReply(true));
 }
 
 TEST_F(TwoPhaseUpdateOperationTest, fast_path_inconsistent_timestamps_not_found) {
     setupDistributor(2, 2, "storage:2 distributor:1");
+    auto cb = sendUpdate("0=1/2/3,1=1/2/3");
+    cb->start(_sender, framework::MilliSecTime(0));
 
-    std::shared_ptr<TwoPhaseUpdateOperation> cb(sendUpdate("0=1/2/3,1=1/2/3"));
-    DistributorMessageSenderStub sender;
-    cb->start(sender, framework::MilliSecTime(0));
-
-    ASSERT_EQ("Update => 0,Update => 1", sender.getCommands(true));
+    ASSERT_EQ("Update => 0,Update => 1", _sender.getCommands(true));
 
-    replyToMessage(*cb, sender, 0, 90);
-    replyToMessage(*cb, sender, 1, 110);
+    replyToMessage(*cb, _sender, 0, 90);
+    replyToMessage(*cb, _sender, 1, 110);
 
-    ASSERT_EQ("Get(BucketId(0x400000000000cac4), id:ns:testdoctype1::1) => 1", sender.getLastCommand(true));
-    ASSERT_TRUE(sender.replies().empty());
+    ASSERT_EQ("Get(BucketId(0x400000000000cac4), id:ns:testdoctype1::1) => 1", _sender.getLastCommand(true));
+    ASSERT_TRUE(_sender.replies().empty());
 
-    replyToGet(*cb, sender, 2, 110, false);
+    replyToGet(*cb, _sender, 2, 110, false);
 
     EXPECT_EQ("UpdateReply(id:ns:testdoctype1::1, BucketId(0x0000000000000000), "
               "timestamp 0, timestamp of updated doc: 110 Was inconsistent "
               "(best node 1)) ReturnCode(INTERNAL_FAILURE)",
-              sender.getLastReply(true));
+              _sender.getLastReply(true));
 }
 
 TEST_F(TwoPhaseUpdateOperationTest, fast_path_inconsistent_timestamps_update_error) {
     setupDistributor(2, 2, "storage:2 distributor:1");
+    auto cb = sendUpdate("0=1/2/3,1=1/2/3");
+    cb->start(_sender, framework::MilliSecTime(0));
 
-    std::shared_ptr<TwoPhaseUpdateOperation> cb(sendUpdate("0=1/2/3,1=1/2/3"));
-    DistributorMessageSenderStub sender;
-    cb->start(sender, framework::MilliSecTime(0));
+    ASSERT_EQ("Update => 0,Update => 1", _sender.getCommands(true));
 
-    ASSERT_EQ("Update => 0,Update => 1", sender.getCommands(true));
-
-    replyToMessage(*cb, sender, 0, 90);
-    ASSERT_TRUE(sender.replies().empty());
-    replyToMessage(*cb, sender, 1, 110, api::ReturnCode::IO_FAILURE);
+    replyToMessage(*cb, _sender, 0, 90);
+    ASSERT_TRUE(_sender.replies().empty());
+    replyToMessage(*cb, _sender, 1, 110, api::ReturnCode::IO_FAILURE);
 
     EXPECT_EQ("UpdateReply(id:ns:testdoctype1::1, BucketId(0x0000000000000000), "
               "timestamp 0, timestamp of updated doc: 90) "
               "ReturnCode(IO_FAILURE)",
-              sender.getLastReply(true));
+              _sender.getLastReply(true));
 }
 
 TEST_F(TwoPhaseUpdateOperationTest, fast_path_inconsistent_timestamps_get_error) {
     setupDistributor(2, 2, "storage:2 distributor:1");
+    auto cb = sendUpdate("0=1/2/3,1=1/2/3");
+    cb->start(_sender, framework::MilliSecTime(0));
 
-    std::shared_ptr<TwoPhaseUpdateOperation> cb(sendUpdate("0=1/2/3,1=1/2/3"));
-    DistributorMessageSenderStub sender;
-    cb->start(sender, framework::MilliSecTime(0));
-
-    ASSERT_EQ("Update => 0,Update => 1", sender.getCommands(true));
+    ASSERT_EQ("Update => 0,Update => 1", _sender.getCommands(true));
 
-    replyToMessage(*cb, sender, 0, 90);
-    replyToMessage(*cb, sender, 1, 110);
+    replyToMessage(*cb, _sender, 0, 90);
+    replyToMessage(*cb, _sender, 1, 110);
 
     ASSERT_EQ("Get(BucketId(0x400000000000cac4), id:ns:testdoctype1::1) => 1",
-              sender.getLastCommand(true));
+              _sender.getLastCommand(true));
 
-    ASSERT_TRUE(sender.replies().empty());
-    replyToGet(*cb, sender, 2, 110, false, api::ReturnCode::IO_FAILURE);
+    ASSERT_TRUE(_sender.replies().empty());
+    replyToGet(*cb, _sender, 2, 110, false, api::ReturnCode::IO_FAILURE);
 
     EXPECT_EQ("UpdateReply(id:ns:testdoctype1::1, BucketId(0x0000000000000000), "
               "timestamp 0, timestamp of updated doc: 110 Was inconsistent "
               "(best node 1)) ReturnCode(IO_FAILURE)",
-              sender.getLastReply(true));
+              _sender.getLastReply(true));
 }
 
 TEST_F(TwoPhaseUpdateOperationTest, fast_path_inconsistent_timestamps_put_error) {
     setupDistributor(2, 2, "storage:2 distributor:1");
-    std::shared_ptr<TwoPhaseUpdateOperation> cb(sendUpdate("0=1/2/3,1=1/2/3"));
+    auto cb = sendUpdate("0=1/2/3,1=1/2/3");
+    cb->start(_sender, framework::MilliSecTime(0));
 
-    DistributorMessageSenderStub sender;
-    cb->start(sender, framework::MilliSecTime(0));
+    ASSERT_EQ("Update => 0,Update => 1", _sender.getCommands(true));
 
-    ASSERT_EQ("Update => 0,Update => 1", sender.getCommands(true));
-
-    replyToMessage(*cb, sender, 0, 90);
-    replyToMessage(*cb, sender, 1, 110);
+    replyToMessage(*cb, _sender, 0, 90);
+    replyToMessage(*cb, _sender, 1, 110);
 
     ASSERT_EQ("Get(BucketId(0x400000000000cac4), id:ns:testdoctype1::1) => 1",
-              sender.getLastCommand(true));
+              _sender.getLastCommand(true));
 
-    replyToGet(*cb, sender, 2, 110);
+    replyToGet(*cb, _sender, 2, 110);
 
     ASSERT_EQ("Update => 0,Update => 1,Get => 1,Put => 1,Put => 0",
-              sender.getCommands(true));
+              _sender.getCommands(true));
 
-    replyToPut(*cb, sender, 3, api::ReturnCode::IO_FAILURE);
-    ASSERT_TRUE(sender.replies().empty());
-    replyToPut(*cb, sender, 4);
+    replyToPut(*cb, _sender, 3, api::ReturnCode::IO_FAILURE);
+    ASSERT_TRUE(_sender.replies().empty());
+    replyToPut(*cb, _sender, 4);
 
     EXPECT_EQ("UpdateReply(id:ns:testdoctype1::1, BucketId(0x0000000000000000), "
               "timestamp 0, timestamp of updated doc: 110 Was inconsistent "
               "(best node 1)) ReturnCode(IO_FAILURE)",
-              sender.getLastReply(true));
+              _sender.getLastReply(true));
 }
 
 TEST_F(TwoPhaseUpdateOperationTest, fast_path_inconsistent_timestamps_put_not_started) {
     setupDistributor(2, 2, "storage:2 distributor:1");
-    std::shared_ptr<TwoPhaseUpdateOperation> cb(sendUpdate("0=1/2/3,1=1/2/3"));
-
-    DistributorMessageSenderStub sender;
-    cb->start(sender, framework::MilliSecTime(0));
+    auto cb = sendUpdate("0=1/2/3,1=1/2/3");
+    cb->start(_sender, framework::MilliSecTime(0));
 
-    ASSERT_EQ("Update => 0,Update => 1", sender.getCommands(true));
+    ASSERT_EQ("Update => 0,Update => 1", _sender.getCommands(true));
 
-    replyToMessage(*cb, sender, 0, 90);
-    replyToMessage(*cb, sender, 1, 110);
+    replyToMessage(*cb, _sender, 0, 90);
+    replyToMessage(*cb, _sender, 1, 110);
 
     ASSERT_EQ("Get(BucketId(0x400000000000cac4), id:ns:testdoctype1::1) => 1",
-              sender.getLastCommand(true));
-    checkMessageSettingsPropagatedTo(sender.commands().back());
+              _sender.getLastCommand(true));
+    checkMessageSettingsPropagatedTo(_sender.commands().back());
 
     enableDistributorClusterState("storage:0 distributor:1");
-    ASSERT_TRUE(sender.replies().empty());
-    replyToGet(*cb, sender, 2, 110);
+    ASSERT_TRUE(_sender.replies().empty());
+    replyToGet(*cb, _sender, 2, 110);
 
     EXPECT_EQ("UpdateReply(id:ns:testdoctype1::1, BucketId(0x0000000000000000), "
               "timestamp 0, timestamp of updated doc: 110 Was inconsistent "
               "(best node 1)) ReturnCode(NOT_CONNECTED, "
               "Can't store document: No storage nodes available)",
-              sender.getLastReply(true));
+              _sender.getLastReply(true));
 }
 
 TEST_F(TwoPhaseUpdateOperationTest, fast_path_inconsistent_timestamps_inconsistent_split) {
     setupDistributor(2, 2, "storage:2 distributor:1");
-
-    std::shared_ptr<TwoPhaseUpdateOperation> cb(
-            sendUpdate("0=1/2/3,1=1/2/3",
-                       UpdateOptions().makeInconsistentSplit(true)));
-    DistributorMessageSenderStub sender;
-    cb->start(sender, framework::MilliSecTime(0));
+    auto cb = sendUpdate("0=1/2/3,1=1/2/3", UpdateOptions().makeInconsistentSplit(true));
+    cb->start(_sender, framework::MilliSecTime(0));
 
     std::string wanted("Get(BucketId(0x400000000000cac4), id:ns:testdoctype1::1) => 0,"
                        "Get(BucketId(0x440000000000cac4), id:ns:testdoctype1::1) => 0");
 
-    std::string text = sender.getCommands(true, true);
+    std::string text = _sender.getCommands(true, true);
     ASSERT_EQ(wanted, text);
 
-    replyToGet(*cb, sender, 0, 90);
-    replyToGet(*cb, sender, 1, 120);
+    replyToGet(*cb, _sender, 0, 90);
+    replyToGet(*cb, _sender, 1, 120);
 
     ASSERT_EQ("Put(BucketId(0x440000000000cac4), id:ns:testdoctype1::1, "
               "timestamp 200000000, size 60) => 1,"
               "Put(BucketId(0x440000000000cac4), id:ns:testdoctype1::1, "
               "timestamp 200000000, size 60) => 0",
-              sender.getCommands(true, true, 2));
+              _sender.getCommands(true, true, 2));
 
-    replyToPut(*cb, sender, 2);
-    ASSERT_TRUE(sender.replies().empty());
-    replyToPut(*cb, sender, 3);
+    replyToPut(*cb, _sender, 2);
+    ASSERT_TRUE(_sender.replies().empty());
+    replyToPut(*cb, _sender, 3);
 
     EXPECT_EQ("UpdateReply(id:ns:testdoctype1::1, "
               "BucketId(0x0000000000000000), "
               "timestamp 0, timestamp of updated doc: 120) "
               "ReturnCode(NONE)",
-              sender.getLastReply(true));
+              _sender.getLastReply(true));
 }
 
 void
@@ -523,33 +536,30 @@ TwoPhaseUpdateOperationTest::checkMessageSettingsPropagatedTo(
 
 TEST_F(TwoPhaseUpdateOperationTest, fast_path_propagates_message_settings_to_update) {
     setupDistributor(1, 1, "storage:1 distributor:1");
-    std::shared_ptr<TwoPhaseUpdateOperation> cb(sendUpdate("0=1/2/3"));
-    DistributorMessageSenderStub sender;
-    cb->start(sender, framework::MilliSecTime(0));
+    auto cb = sendUpdate("0=1/2/3");
+    cb->start(_sender, framework::MilliSecTime(0));
 
-    ASSERT_EQ("Update => 0", sender.getCommands(true));
+    ASSERT_EQ("Update => 0", _sender.getCommands(true));
 
-    StorageCommand::SP msg(sender.commands().back());
+    StorageCommand::SP msg(_sender.commands().back());
     checkMessageSettingsPropagatedTo(msg);
 }
 
 TEST_F(TwoPhaseUpdateOperationTest, n_of_m) {
     setupDistributor(2, 2, "storage:2 distributor:1", 1);
+    auto cb = sendUpdate("0=1/2/3,1=1/2/3");
+    cb->start(_sender, framework::MilliSecTime(0));
 
-    std::shared_ptr<TwoPhaseUpdateOperation> cb(sendUpdate("0=1/2/3,1=1/2/3"));
-    DistributorMessageSenderStub sender;
-    cb->start(sender, framework::MilliSecTime(0));
+    ASSERT_EQ("Update => 0,Update => 1", _sender.getCommands(true));
 
-    ASSERT_EQ("Update => 0,Update => 1", sender.getCommands(true));
-
-    ASSERT_TRUE(sender.replies().empty());
-    replyToMessage(*cb, sender, 0, 90);
+    ASSERT_TRUE(_sender.replies().empty());
+    replyToMessage(*cb, _sender, 0, 90);
 
     EXPECT_EQ("UpdateReply(id:ns:testdoctype1::1, BucketId(0x0000000000000000), "
               "timestamp 0, timestamp of updated doc: 90) ReturnCode(NONE)",
-              sender.getLastReply(true));
+              _sender.getLastReply(true));
 
-    replyToMessage(*cb, sender, 1, 123);
+    replyToMessage(*cb, _sender, 1, 123);
 }
 
 std::string
@@ -565,132 +575,114 @@ TwoPhaseUpdateOperationTest::getUpdatedValueFromLastPut(
 TEST_F(TwoPhaseUpdateOperationTest, safe_path_updates_newest_received_document) {
     setupDistributor(3, 3, "storage:3 distributor:1");
     // 0,1 in sync. 2 out of sync.
-    std::shared_ptr<TwoPhaseUpdateOperation> cb(
-            sendUpdate("0=1/2/3,1=1/2/3,2=2/3/4"));
-
-    DistributorMessageSenderStub sender;
-    cb->start(sender, framework::MilliSecTime(0));
+    auto cb = sendUpdate("0=1/2/3,1=1/2/3,2=2/3/4");
+    cb->start(_sender, framework::MilliSecTime(0));
 
     ASSERT_EQ("Get(BucketId(0x400000000000cac4), id:ns:testdoctype1::1) => 0,"
               "Get(BucketId(0x400000000000cac4), id:ns:testdoctype1::1) => 2",
-              sender.getCommands(true, true));
-    replyToGet(*cb, sender, 0, 50);
-    replyToGet(*cb, sender, 1, 70);
+              _sender.getCommands(true, true));
+    replyToGet(*cb, _sender, 0, 50);
+    replyToGet(*cb, _sender, 1, 70);
 
     ASSERT_EQ("Put(BucketId(0x400000000000cac4), id:ns:testdoctype1::1, timestamp 200000000, size 60) => 1,"
               "Put(BucketId(0x400000000000cac4), id:ns:testdoctype1::1, timestamp 200000000, size 60) => 2,"
               "Put(BucketId(0x400000000000cac4), id:ns:testdoctype1::1, timestamp 200000000, size 60) => 0",
-              sender.getCommands(true, true, 2));
+              _sender.getCommands(true, true, 2));
     // Make sure Put contains an updated document (+10 arith. update on field
     // whose value equals gotten timestamp). In this case we want 70 -> 80.
-    ASSERT_EQ("80", getUpdatedValueFromLastPut(sender));
+    ASSERT_EQ("80", getUpdatedValueFromLastPut(_sender));
 
-    replyToPut(*cb, sender, 2);
-    replyToPut(*cb, sender, 3);
-    ASSERT_TRUE(sender.replies().empty());
-    replyToPut(*cb, sender, 4);
+    replyToPut(*cb, _sender, 2);
+    replyToPut(*cb, _sender, 3);
+    ASSERT_TRUE(_sender.replies().empty());
+    replyToPut(*cb, _sender, 4);
 
     EXPECT_EQ("UpdateReply(id:ns:testdoctype1::1, "
               "BucketId(0x0000000000000000), "
               "timestamp 0, timestamp of updated doc: 70) "
               "ReturnCode(NONE)",
-              sender.getLastReply(true));
+              _sender.getLastReply(true));
 }
 
 TEST_F(TwoPhaseUpdateOperationTest, create_if_non_existent_creates_document_if_all_empty_gets) {
     setupDistributor(3, 3, "storage:3 distributor:1");
-    std::shared_ptr<TwoPhaseUpdateOperation> cb(
-            sendUpdate("0=1/2/3,1=1/2/3,2=2/3/4",
-                       UpdateOptions().createIfNonExistent(true)));
+    auto cb = sendUpdate("0=1/2/3,1=1/2/3,2=2/3/4", UpdateOptions().createIfNonExistent(true));
+    cb->start(_sender, framework::MilliSecTime(0));
 
-    DistributorMessageSenderStub sender;
-    cb->start(sender, framework::MilliSecTime(0));
-
-    ASSERT_EQ("Get => 0,Get => 2", sender.getCommands(true));
-    replyToGet(*cb, sender, 0, 0, false);
-    replyToGet(*cb, sender, 1, 0, false);
+    ASSERT_EQ("Get => 0,Get => 2", _sender.getCommands(true));
+    replyToGet(*cb, _sender, 0, 0, false);
+    replyToGet(*cb, _sender, 1, 0, false);
     // Since create-if-non-existent is set, distributor should create doc from
     // scratch.
     ASSERT_EQ("Put(BucketId(0x400000000000cac4), id:ns:testdoctype1::1, timestamp 200000000, size 60) => 1,"
               "Put(BucketId(0x400000000000cac4), id:ns:testdoctype1::1, timestamp 200000000, size 60) => 2,"
               "Put(BucketId(0x400000000000cac4), id:ns:testdoctype1::1, timestamp 200000000, size 60) => 0",
-              sender.getCommands(true, true, 2));
+              _sender.getCommands(true, true, 2));
 
-    ASSERT_EQ("10", getUpdatedValueFromLastPut(sender));
+    ASSERT_EQ("10", getUpdatedValueFromLastPut(_sender));
 
-    replyToPut(*cb, sender, 2);
-    replyToPut(*cb, sender, 3);
-    ASSERT_TRUE(sender.replies().empty());
-    replyToPut(*cb, sender, 4);
+    replyToPut(*cb, _sender, 2);
+    replyToPut(*cb, _sender, 3);
+    ASSERT_TRUE(_sender.replies().empty());
+    replyToPut(*cb, _sender, 4);
 
     EXPECT_EQ("UpdateReply(id:ns:testdoctype1::1, "
               "BucketId(0x0000000000000000), "
               "timestamp 0, timestamp of updated doc: 200000000) "
               "ReturnCode(NONE)",
-              sender.getLastReply(true));
+              _sender.getLastReply(true));
 }
 
 TEST_F(TwoPhaseUpdateOperationTest, update_fails_if_safe_path_has_failed_put) {
     setupDistributor(3, 3, "storage:3 distributor:1");
-    std::shared_ptr<TwoPhaseUpdateOperation> cb(
-            sendUpdate("0=1/2/3,1=1/2/3,2=2/3/4",
-                       UpdateOptions().createIfNonExistent(true)));
-
-    DistributorMessageSenderStub sender;
-    cb->start(sender, framework::MilliSecTime(0));
+    auto cb = sendUpdate("0=1/2/3,1=1/2/3,2=2/3/4", UpdateOptions().createIfNonExistent(true));
+    cb->start(_sender, framework::MilliSecTime(0));
 
-    ASSERT_EQ("Get => 0,Get => 2", sender.getCommands(true));
-    replyToGet(*cb, sender, 0, 0, false);
-    replyToGet(*cb, sender, 1, 0, false);
+    ASSERT_EQ("Get => 0,Get => 2", _sender.getCommands(true));
+    replyToGet(*cb, _sender, 0, 0, false);
+    replyToGet(*cb, _sender, 1, 0, false);
     // Since create-if-non-existent is set, distributor should create doc from
     // scratch.
-    ASSERT_EQ("Put => 1,Put => 2,Put => 0", sender.getCommands(true, false, 2));
+    ASSERT_EQ("Put => 1,Put => 2,Put => 0", _sender.getCommands(true, false, 2));
 
-    replyToPut(*cb, sender, 2);
-    replyToPut(*cb, sender, 3);
-    ASSERT_TRUE(sender.replies().empty());
-    replyToPut(*cb, sender, 4, api::ReturnCode::IO_FAILURE);
+    replyToPut(*cb, _sender, 2);
+    replyToPut(*cb, _sender, 3);
+    ASSERT_TRUE(_sender.replies().empty());
+    replyToPut(*cb, _sender, 4, api::ReturnCode::IO_FAILURE);
 
     EXPECT_EQ("UpdateReply(id:ns:testdoctype1::1, "
               "BucketId(0x0000000000000000), "
               "timestamp 0, timestamp of updated doc: 200000000) "
               "ReturnCode(IO_FAILURE)",
-              sender.getLastReply(true));
+              _sender.getLastReply(true));
 }
 
 TEST_F(TwoPhaseUpdateOperationTest, update_fails_if_safe_path_gets_fail) {
     setupDistributor(2, 2, "storage:2 distributor:1");
-    std::shared_ptr<TwoPhaseUpdateOperation> cb(
-            sendUpdate("0=1/2/3,1=2/3/4",
-                       UpdateOptions().createIfNonExistent(true)));
+    auto cb = sendUpdate("0=1/2/3,1=2/3/4", UpdateOptions().createIfNonExistent(true));
+    cb->start(_sender, framework::MilliSecTime(0));
 
-    DistributorMessageSenderStub sender;
-    cb->start(sender, framework::MilliSecTime(0));
-
-    ASSERT_EQ("Get => 0,Get => 1", sender.getCommands(true));
-    replyToGet(*cb, sender, 0, 0, false, api::ReturnCode::IO_FAILURE);
-    ASSERT_TRUE(sender.replies().empty());
-    replyToGet(*cb, sender, 1, 0, false, api::ReturnCode::IO_FAILURE);
+    ASSERT_EQ("Get => 0,Get => 1", _sender.getCommands(true));
+    replyToGet(*cb, _sender, 0, 0, false, api::ReturnCode::IO_FAILURE);
+    ASSERT_TRUE(_sender.replies().empty());
+    replyToGet(*cb, _sender, 1, 0, false, api::ReturnCode::IO_FAILURE);
     EXPECT_EQ("UpdateReply(id:ns:testdoctype1::1, "
               "BucketId(0x0000000000000000), "
               "timestamp 0, timestamp of updated doc: 0) "
               "ReturnCode(IO_FAILURE)",
-              sender.getLastReply(true));
+              _sender.getLastReply(true));
 }
 
 TEST_F(TwoPhaseUpdateOperationTest, update_fails_if_apply_throws_exception) {
     setupDistributor(2, 2, "storage:2 distributor:1");
     // Create update for wrong doctype which will fail the update.
-    std::shared_ptr<TwoPhaseUpdateOperation> cb(
-            sendUpdate("0=1/2/3,1=2/3/4", UpdateOptions().withError()));
-
-    DistributorMessageSenderStub sender;
-    cb->start(sender, framework::MilliSecTime(0));
+    auto cb = sendUpdate("0=1/2/3,1=2/3/4", UpdateOptions().withError());
+    cb->start(_sender, framework::MilliSecTime(0));
 
-    ASSERT_EQ("Get => 0,Get => 1", sender.getCommands(true));
-    replyToGet(*cb, sender, 0, 50);
-    ASSERT_TRUE(sender.replies().empty());
-    replyToGet(*cb, sender, 1, 70);
+    ASSERT_EQ("Get => 0,Get => 1", _sender.getCommands(true));
+    replyToGet(*cb, _sender, 0, 50);
+    ASSERT_TRUE(_sender.replies().empty());
+    replyToGet(*cb, _sender, 1, 70);
 
     EXPECT_EQ("UpdateReply(id:ns:testdoctype1::1, "
               "BucketId(0x0000000000000000), "
@@ -698,95 +690,88 @@ TEST_F(TwoPhaseUpdateOperationTest, update_fails_if_apply_throws_exception) {
               "ReturnCode(INTERNAL_FAILURE, Can not apply a "
               "\"testdoctype2\" document update to a "
               "\"testdoctype1\" document.)",
-              sender.getLastReply(true));
+              _sender.getLastReply(true));
 }
 
 TEST_F(TwoPhaseUpdateOperationTest, non_existing_with_auto_create) {
     setupDistributor(1, 1, "storage:1 distributor:1");
 
-    std::shared_ptr<TwoPhaseUpdateOperation> cb(
-            sendUpdate("", UpdateOptions().createIfNonExistent(true)));
-    DistributorMessageSenderStub sender;
-    cb->start(sender, framework::MilliSecTime(0));
+    auto cb = sendUpdate("", UpdateOptions().createIfNonExistent(true));
+    cb->start(_sender, framework::MilliSecTime(0));
 
     ASSERT_EQ("CreateBucketCommand(BucketId(0x400000000000cac4), active) "
               "Reasons to start:  => 0,"
               "Put(BucketId(0x400000000000cac4), id:ns:testdoctype1::1, "
               "timestamp 200000000, size 60) => 0",
-              sender.getCommands(true, true));
+              _sender.getCommands(true, true));
 
-    ASSERT_EQ("10", getUpdatedValueFromLastPut(sender));
+    ASSERT_EQ("10", getUpdatedValueFromLastPut(_sender));
 
-    replyToCreateBucket(*cb, sender, 0);
-    ASSERT_TRUE(sender.replies().empty());
-    replyToPut(*cb, sender, 1);
+    replyToCreateBucket(*cb, _sender, 0);
+    ASSERT_TRUE(_sender.replies().empty());
+    replyToPut(*cb, _sender, 1);
 
     EXPECT_EQ("UpdateReply(id:ns:testdoctype1::1, "
               "BucketId(0x0000000000000000), "
               "timestamp 0, timestamp of updated doc: 200000000) "
               "ReturnCode(NONE)",
-              sender.getLastReply(true));
+              _sender.getLastReply(true));
 }
 
 TEST_F(TwoPhaseUpdateOperationTest, safe_path_fails_update_when_mismatching_timestamp_constraint) {
     setupDistributor(2, 2, "storage:2 distributor:1");
-    std::shared_ptr<TwoPhaseUpdateOperation> cb(
-            sendUpdate("0=1/2/3,1=2/3/4",
-                       UpdateOptions().timestampToUpdate(1234)));
+    auto cb = sendUpdate("0=1/2/3,1=2/3/4", UpdateOptions().timestampToUpdate(1234));
 
-    DistributorMessageSenderStub sender;
-    cb->start(sender, framework::MilliSecTime(0));
+    cb->start(_sender, framework::MilliSecTime(0));
 
-    ASSERT_EQ("Get => 0,Get => 1", sender.getCommands(true));
-    replyToGet(*cb, sender, 0, 100);
-    ASSERT_TRUE(sender.replies().empty());
-    replyToGet(*cb, sender, 1, 110);
+    ASSERT_EQ("Get => 0,Get => 1", _sender.getCommands(true));
+    replyToGet(*cb, _sender, 0, 100);
+    ASSERT_TRUE(_sender.replies().empty());
+    replyToGet(*cb, _sender, 1, 110);
     EXPECT_EQ("UpdateReply(id:ns:testdoctype1::1, "
               "BucketId(0x0000000000000000), "
               "timestamp 0, timestamp of updated doc: 0) "
               "ReturnCode(NONE, No document with requested "
                          "timestamp found)",
-              sender.getLastReply(true));
+              _sender.getLastReply(true));
 }
 
 TEST_F(TwoPhaseUpdateOperationTest, safe_path_update_propagates_message_settings_to_gets_and_puts) {
     setupDistributor(3, 3, "storage:3 distributor:1");
-    std::shared_ptr<TwoPhaseUpdateOperation> cb(sendUpdate("0=1/2/3,1=1/2/3,2=2/3/4"));
-    DistributorMessageSenderStub sender;
-    cb->start(sender, framework::MilliSecTime(0));
-
-    ASSERT_EQ("Get => 0,Get => 2", sender.getCommands(true));
-    checkMessageSettingsPropagatedTo(sender.command(0));
-    checkMessageSettingsPropagatedTo(sender.command(1));
-    replyToGet(*cb, sender, 0, 50);
-    replyToGet(*cb, sender, 1, 70);
-    ASSERT_EQ("Put => 1,Put => 2,Put => 0", sender.getCommands(true, false, 2));
-    checkMessageSettingsPropagatedTo(sender.command(2));
-    checkMessageSettingsPropagatedTo(sender.command(3));
-    checkMessageSettingsPropagatedTo(sender.command(4));
-    replyToPut(*cb, sender, 2);
-    replyToPut(*cb, sender, 3);
-    replyToPut(*cb, sender, 4);
+    auto cb = sendUpdate("0=1/2/3,1=1/2/3,2=2/3/4");
+    cb->start(_sender, framework::MilliSecTime(0));
+
+    ASSERT_EQ("Get => 0,Get => 2", _sender.getCommands(true));
+    checkMessageSettingsPropagatedTo(_sender.command(0));
+    checkMessageSettingsPropagatedTo(_sender.command(1));
+    replyToGet(*cb, _sender, 0, 50);
+    replyToGet(*cb, _sender, 1, 70);
+    ASSERT_EQ("Put => 1,Put => 2,Put => 0", _sender.getCommands(true, false, 2));
+    checkMessageSettingsPropagatedTo(_sender.command(2));
+    checkMessageSettingsPropagatedTo(_sender.command(3));
+    checkMessageSettingsPropagatedTo(_sender.command(4));
+    replyToPut(*cb, _sender, 2);
+    replyToPut(*cb, _sender, 3);
+    replyToPut(*cb, _sender, 4);
 }
 
 TEST_F(TwoPhaseUpdateOperationTest, safe_path_propagates_mbus_traces_from_replies) {
     setupDistributor(3, 3, "storage:3 distributor:1");
-    std::shared_ptr<TwoPhaseUpdateOperation> cb(sendUpdate("0=1/2/3,1=1/2/3,2=2/3/4"));
-    DistributorMessageSenderStub sender;
-    cb->start(sender, framework::MilliSecTime(0));
-
-    ASSERT_EQ("Get => 0,Get => 2", sender.getCommands(true));
-    replyToGet(*cb, sender, 0, 50, true, api::ReturnCode::OK, "hello earthlings");
-    replyToGet(*cb, sender, 1, 70);
-    ASSERT_EQ("Put => 1,Put => 2,Put => 0", sender.getCommands(true, false, 2));
-    replyToPut(*cb, sender, 2, api::ReturnCode::OK, "fooo");
-    replyToPut(*cb, sender, 3, api::ReturnCode::OK, "baaa");
-    ASSERT_TRUE(sender.replies().empty());
-    replyToPut(*cb, sender, 4);
+    auto cb = sendUpdate("0=1/2/3,1=1/2/3,2=2/3/4");
+    cb->start(_sender, framework::MilliSecTime(0));
+
+    ASSERT_EQ("Get => 0,Get => 2", _sender.getCommands(true));
+    replyToGet(*cb, _sender, 0, 50, true, api::ReturnCode::OK, "hello earthlings");
+    replyToGet(*cb, _sender, 1, 70);
+    ASSERT_EQ("Put => 1,Put => 2,Put => 0", _sender.getCommands(true, false, 2));
+    replyToPut(*cb, _sender, 2, api::ReturnCode::OK, "fooo");
+    replyToPut(*cb, _sender, 3, api::ReturnCode::OK, "baaa");
+    ASSERT_TRUE(_sender.replies().empty());
+    replyToPut(*cb, _sender, 4);
     
-    ASSERT_EQ("Update Reply", sender.getLastReply(false));
+    ASSERT_EQ("Update Reply", _sender.getLastReply(false));
 
-    std::string trace(sender.replies().back()->getTrace().toString());
+    std::string trace(_sender.replies().back()->getTrace().toString());
     ASSERT_THAT(trace, HasSubstr("hello earthlings"));
     ASSERT_THAT(trace, HasSubstr("fooo"));
     ASSERT_THAT(trace, HasSubstr("baaa"));
@@ -798,13 +783,11 @@ void TwoPhaseUpdateOperationTest::do_test_ownership_changed_between_gets_and_sec
         Timestamp expected_response_timestamp)
 {
     setupDistributor(2, 2, "storage:2 distributor:1");
-
     // Update towards inconsistent bucket invokes safe path.
-    std::shared_ptr<TwoPhaseUpdateOperation> cb(sendUpdate("0=1/2/3,1=2/3/4"));
-    DistributorMessageSenderStub sender;
-    cb->start(sender, framework::MilliSecTime(0));
+    auto cb = sendUpdate("0=1/2/3,1=2/3/4");
+    cb->start(_sender, framework::MilliSecTime(0));
 
-    ASSERT_EQ("Get => 0,Get => 1", sender.getCommands(true));
+    ASSERT_EQ("Get => 0,Get => 1", _sender.getCommands(true));
 
     // Alter cluster state so that distributor is now down (technically the
     // entire cluster is down in this state, but this should not matter). In
@@ -813,8 +796,8 @@ void TwoPhaseUpdateOperationTest::do_test_ownership_changed_between_gets_and_sec
     // to a bucket we no longer own.
     enableDistributorClusterState("storage:2 distributor:1 .0.s:d");
     getBucketDatabase().clear();
-    replyToGet(*cb, sender, 0, lowest_get_timestamp);
-    replyToGet(*cb, sender, 1, highest_get_timestamp);
+    replyToGet(*cb, _sender, 0, lowest_get_timestamp);
+    replyToGet(*cb, _sender, 1, highest_get_timestamp);
 
     // BUCKET_NOT_FOUND is a transient error code which should cause the client
     // to re-send the operation, presumably to the correct distributor the next
@@ -827,7 +810,7 @@ void TwoPhaseUpdateOperationTest::do_test_ownership_changed_between_gets_and_sec
               "ReturnCode(BUCKET_NOT_FOUND, Distributor lost "
               "ownership of bucket between executing the read "
               "and write phases of a two-phase update operation)",
-              sender.getLastReply(true));
+              _sender.getLastReply(true));
 }
 
 TEST_F(TwoPhaseUpdateOperationTest, update_fails_if_ownership_changes_between_get_and_put) {
@@ -843,46 +826,37 @@ TEST_F(TwoPhaseUpdateOperationTest, update_fails_if_ownership_changes_between_ge
 
 TEST_F(TwoPhaseUpdateOperationTest, safe_path_condition_mismatch_fails_with_tas_error) {
     setupDistributor(2, 2, "storage:2 distributor:1");
-    std::shared_ptr<TwoPhaseUpdateOperation> cb(
-            sendUpdate("0=1/2/3,1=2/3/4", UpdateOptions().condition(
-                            "testdoctype1.headerval==120")));
+    auto cb = sendUpdate("0=1/2/3,1=2/3/4", UpdateOptions().condition("testdoctype1.headerval==120"));
 
-    DistributorMessageSenderStub sender;
-    cb->start(sender, framework::MilliSecTime(0));
+    cb->start(_sender, framework::MilliSecTime(0));
     // Newest doc has headerval==110, not 120.
-    replyToGet(*cb, sender, 0, 100);
-    replyToGet(*cb, sender, 1, 110);
+    replyToGet(*cb, _sender, 0, 100);
+    replyToGet(*cb, _sender, 1, 110);
     EXPECT_EQ("UpdateReply(id:ns:testdoctype1::1, "
               "BucketId(0x0000000000000000), "
               "timestamp 0, timestamp of updated doc: 0) "
               "ReturnCode(TEST_AND_SET_CONDITION_FAILED, "
                          "Condition did not match document)",
-              sender.getLastReply(true));
+              _sender.getLastReply(true));
 }
 
 TEST_F(TwoPhaseUpdateOperationTest, safe_path_condition_match_sends_puts_with_updated_doc) {
     setupDistributor(2, 2, "storage:2 distributor:1");
-    std::shared_ptr<TwoPhaseUpdateOperation> cb(
-            sendUpdate("0=1/2/3,1=2/3/4", UpdateOptions().condition(
-                            "testdoctype1.headerval==110")));
-
-    DistributorMessageSenderStub sender;
-    cb->start(sender, framework::MilliSecTime(0));
-    replyToGet(*cb, sender, 0, 100);
-    replyToGet(*cb, sender, 1, 110);
-    ASSERT_EQ("Put => 1,Put => 0", sender.getCommands(true, false, 2));
+    auto cb = sendUpdate("0=1/2/3,1=2/3/4", UpdateOptions().condition("testdoctype1.headerval==110"));
+
+    cb->start(_sender, framework::MilliSecTime(0));
+    replyToGet(*cb, _sender, 0, 100);
+    replyToGet(*cb, _sender, 1, 110);
+    ASSERT_EQ("Put => 1,Put => 0", _sender.getCommands(true, false, 2));
 }
 
 TEST_F(TwoPhaseUpdateOperationTest, safe_path_condition_parse_failure_fails_with_illegal_params_error) {
     setupDistributor(2, 2, "storage:2 distributor:1");
-    std::shared_ptr<TwoPhaseUpdateOperation> cb(
-            sendUpdate("0=1/2/3,1=2/3/4", UpdateOptions().condition(
-                            "testdoctype1.san==fran...cisco")));
-
-    DistributorMessageSenderStub sender;
-    cb->start(sender, framework::MilliSecTime(0));
-    replyToGet(*cb, sender, 0, 100);
-    replyToGet(*cb, sender, 1, 110);
+    auto cb = sendUpdate("0=1/2/3,1=2/3/4", UpdateOptions().condition("testdoctype1.san==fran...cisco"));
+
+    cb->start(_sender, framework::MilliSecTime(0));
+    replyToGet(*cb, _sender, 0, 100);
+    replyToGet(*cb, _sender, 1, 110);
     // NOTE: condition is currently not attempted parsed until Gets have been
     // replied to. This may change in the future.
     // XXX reliance on parser/exception error message is very fragile.
@@ -893,19 +867,16 @@ TEST_F(TwoPhaseUpdateOperationTest, safe_path_condition_parse_failure_fails_with
                                       "Failed to parse test and set condition: "
                                       "syntax error, unexpected . at column 24 when "
                                       "parsing selection 'testdoctype1.san==fran...cisco')",
-              sender.getLastReply(true));
+              _sender.getLastReply(true));
 }
 
 TEST_F(TwoPhaseUpdateOperationTest, safe_path_condition_unknown_doc_type_fails_with_illegal_params_error) {
     setupDistributor(2, 2, "storage:2 distributor:1");
-    std::shared_ptr<TwoPhaseUpdateOperation> cb(
-            sendUpdate("0=1/2/3,1=2/3/4", UpdateOptions().condition(
-                            "langbein.headerval=1234")));
-
-    DistributorMessageSenderStub sender;
-    cb->start(sender, framework::MilliSecTime(0));
-    replyToGet(*cb, sender, 0, 100);
-    replyToGet(*cb, sender, 1, 110);
+    auto cb = sendUpdate("0=1/2/3,1=2/3/4", UpdateOptions().condition("langbein.headerval=1234"));
+
+    cb->start(_sender, framework::MilliSecTime(0));
+    replyToGet(*cb, _sender, 0, 100);
+    replyToGet(*cb, _sender, 1, 110);
     // NOTE: condition is currently not attempted parsed until Gets have been
     // replied to. This may change in the future.
     EXPECT_EQ("UpdateReply(id:ns:testdoctype1::1, "
@@ -915,40 +886,35 @@ TEST_F(TwoPhaseUpdateOperationTest, safe_path_condition_unknown_doc_type_fails_w
                                      "Failed to parse test and set condition: "
                                      "Document type 'langbein' not found at column 1 "
                                      "when parsing selection 'langbein.headerval=1234')",
-              sender.getLastReply(true));
+              _sender.getLastReply(true));
 }
 
 TEST_F(TwoPhaseUpdateOperationTest, safe_path_condition_with_missing_doc_and_no_auto_create_fails_with_tas_error) {
     setupDistributor(2, 2, "storage:2 distributor:1");
-    std::shared_ptr<TwoPhaseUpdateOperation> cb(
-            sendUpdate("0=1/2/3,1=2/3/4", UpdateOptions().condition(
-                            "testdoctype1.headerval==120")));
+    auto cb = sendUpdate("0=1/2/3,1=2/3/4", UpdateOptions().condition("testdoctype1.headerval==120"));
 
-    DistributorMessageSenderStub sender;
-    cb->start(sender, framework::MilliSecTime(0));
+    cb->start(_sender, framework::MilliSecTime(0));
     // Both Gets return nothing at all, nothing at all.
-    replyToGet(*cb, sender, 0, 100, false);
-    replyToGet(*cb, sender, 1, 110, false);
+    replyToGet(*cb, _sender, 0, 100, false);
+    replyToGet(*cb, _sender, 1, 110, false);
     EXPECT_EQ("UpdateReply(id:ns:testdoctype1::1, "
                           "BucketId(0x0000000000000000), "
                           "timestamp 0, timestamp of updated doc: 0) "
                           "ReturnCode(TEST_AND_SET_CONDITION_FAILED, "
                                      "Document did not exist)",
-              sender.getLastReply(true));
+              _sender.getLastReply(true));
 }
 
 TEST_F(TwoPhaseUpdateOperationTest, safe_path_condition_with_missing_doc_and_auto_create_sends_puts) {
     setupDistributor(2, 2, "storage:2 distributor:1");
-    std::shared_ptr<TwoPhaseUpdateOperation> cb(
-            sendUpdate("0=1/2/3,1=2/3/4", UpdateOptions()
+    auto cb = sendUpdate("0=1/2/3,1=2/3/4", UpdateOptions()
                     .condition("testdoctype1.headerval==120")
-                    .createIfNonExistent(true)));
+                    .createIfNonExistent(true));
 
-    DistributorMessageSenderStub sender;
-    cb->start(sender, framework::MilliSecTime(0));
-    replyToGet(*cb, sender, 0, 100, false);
-    replyToGet(*cb, sender, 1, 110, false);
-    ASSERT_EQ("Put => 1,Put => 0", sender.getCommands(true, false, 2));
+    cb->start(_sender, framework::MilliSecTime(0));
+    replyToGet(*cb, _sender, 0, 100, false);
+    replyToGet(*cb, _sender, 1, 110, false);
+    ASSERT_EQ("Put => 1,Put => 0", _sender.getCommands(true, false, 2));
 }
 
 void
@@ -966,11 +932,10 @@ TwoPhaseUpdateOperationTest::assertAbortedUpdateReplyWithContextPresent(
 TEST_F(TwoPhaseUpdateOperationTest, fast_path_close_edge_sends_correct_reply) {
     setupDistributor(1, 1, "storage:1 distributor:1");
     // Only 1 replica; consistent with itself by definition.
-    std::shared_ptr<TwoPhaseUpdateOperation> cb(sendUpdate("0=1/2/3"));
-    DistributorMessageSenderStub sender;
-    cb->start(sender, framework::MilliSecTime(0));
+    auto cb = sendUpdate("0=1/2/3");
+    cb->start(_sender, framework::MilliSecTime(0));
 
-    ASSERT_EQ("Update => 0", sender.getCommands(true));
+    ASSERT_EQ("Update => 0", _sender.getCommands(true));
     // Close the operation. This should generate a single reply that is
     // bound to the original command. We can identify rogue replies by these
     // not having a transport context, as these are unique_ptrs that are
@@ -985,11 +950,10 @@ TEST_F(TwoPhaseUpdateOperationTest, fast_path_close_edge_sends_correct_reply) {
 TEST_F(TwoPhaseUpdateOperationTest, safe_path_close_edge_sends_correct_reply) {
     setupDistributor(2, 2, "storage:2 distributor:1");
 
-    std::shared_ptr<TwoPhaseUpdateOperation> cb(sendUpdate("0=1/2/3,1=2/3/4")); // Inconsistent replicas.
-    DistributorMessageSenderStub sender;
-    cb->start(sender, framework::MilliSecTime(0));
+    auto cb = sendUpdate("0=1/2/3,1=2/3/4"); // Inconsistent replicas.
+    cb->start(_sender, framework::MilliSecTime(0));
 
-    ASSERT_EQ("Get => 0,Get => 1", sender.getCommands(true));
+    ASSERT_EQ("Get => 0,Get => 1", _sender.getCommands(true));
     // Closing the operation should now only return an ABORTED reply for
     // the UpdateCommand, _not_ from the nested, pending Get operation (which
     // will implicitly generate an ABORTED reply for the synthesized Get
@@ -1004,24 +968,23 @@ TEST_F(TwoPhaseUpdateOperationTest, safe_path_consistent_get_reply_timestamps_re
     setupDistributor(2, 2, "storage:2 distributor:1");
     getConfig().set_update_fast_path_restart_enabled(true);
 
-    std::shared_ptr<TwoPhaseUpdateOperation> cb(sendUpdate("0=1/2/3,1=2/3/4")); // Inconsistent replicas.
-    DistributorMessageSenderStub sender;
-    cb->start(sender, framework::MilliSecTime(0));
+    auto cb = sendUpdate("0=1/2/3,1=2/3/4"); // Inconsistent replicas.
+    cb->start(_sender, framework::MilliSecTime(0));
 
     Timestamp old_timestamp = 500;
-    ASSERT_EQ("Get => 0,Get => 1", sender.getCommands(true));
-    replyToGet(*cb, sender, 0, old_timestamp);
-    replyToGet(*cb, sender, 1, old_timestamp);
+    ASSERT_EQ("Get => 0,Get => 1", _sender.getCommands(true));
+    replyToGet(*cb, _sender, 0, old_timestamp);
+    replyToGet(*cb, _sender, 1, old_timestamp);
 
-    ASSERT_EQ("Update => 0,Update => 1", sender.getCommands(true, false, 2));
-    replyToMessage(*cb, sender, 2, old_timestamp);
-    replyToMessage(*cb, sender, 3, old_timestamp);
+    ASSERT_EQ("Update => 0,Update => 1", _sender.getCommands(true, false, 2));
+    replyToMessage(*cb, _sender, 2, old_timestamp);
+    replyToMessage(*cb, _sender, 3, old_timestamp);
 
     EXPECT_EQ("UpdateReply(id:ns:testdoctype1::1, "
               "BucketId(0x0000000000000000), "
               "timestamp 0, timestamp of updated doc: 500) "
               "ReturnCode(NONE)",
-              sender.getLastReply(true));
+              _sender.getLastReply(true));
 
     auto& metrics = getDistributor().getMetrics().updates[documentapi::LoadType::DEFAULT];
     EXPECT_EQ(1, metrics.fast_path_restarts.getValue());
@@ -1031,17 +994,16 @@ TEST_F(TwoPhaseUpdateOperationTest, safe_path_consistent_get_reply_timestamps_do
     setupDistributor(2, 2, "storage:2 distributor:1");
     getConfig().set_update_fast_path_restart_enabled(false);
 
-    std::shared_ptr<TwoPhaseUpdateOperation> cb(sendUpdate("0=1/2/3,1=2/3/4")); // Inconsistent replicas.
-    DistributorMessageSenderStub sender;
-    cb->start(sender, framework::MilliSecTime(0));
+    auto cb = sendUpdate("0=1/2/3,1=2/3/4"); // Inconsistent replicas.
+    cb->start(_sender, framework::MilliSecTime(0));
 
     Timestamp old_timestamp = 500;
-    ASSERT_EQ("Get => 0,Get => 1", sender.getCommands(true));
-    replyToGet(*cb, sender, 0, old_timestamp);
-    replyToGet(*cb, sender, 1, old_timestamp);
+    ASSERT_EQ("Get => 0,Get => 1", _sender.getCommands(true));
+    replyToGet(*cb, _sender, 0, old_timestamp);
+    replyToGet(*cb, _sender, 1, old_timestamp);
 
     // Should _not_ be restarted with fast path, as it has been config disabled
-    ASSERT_EQ("Put => 1,Put => 0", sender.getCommands(true, false, 2));
+    ASSERT_EQ("Put => 1,Put => 0", _sender.getCommands(true, false, 2));
 
     auto& metrics = getDistributor().getMetrics().updates[documentapi::LoadType::DEFAULT];
     EXPECT_EQ(0, metrics.fast_path_restarts.getValue());
@@ -1051,9 +1013,8 @@ TEST_F(TwoPhaseUpdateOperationTest, fast_path_not_restarted_if_replica_set_alter
     setupDistributor(3, 3, "storage:3 distributor:1");
     getConfig().set_update_fast_path_restart_enabled(true);
 
-    std::shared_ptr<TwoPhaseUpdateOperation> cb(sendUpdate("0=1/2/3,1=2/3/4")); // Inconsistent replicas.
-    DistributorMessageSenderStub sender;
-    cb->start(sender, framework::MilliSecTime(0));
+    auto cb = sendUpdate("0=1/2/3,1=2/3/4"); // Inconsistent replicas.
+    cb->start(_sender, framework::MilliSecTime(0));
 
     // Replica set changes between time of Get requests sent and
     // responses received. This may happen e.g. if concurrent mutations
@@ -1065,27 +1026,38 @@ TEST_F(TwoPhaseUpdateOperationTest, fast_path_not_restarted_if_replica_set_alter
     addNodesToBucketDB(bucket, "0=1/2/3,1=2/3/4,2=3/3/3");
 
     Timestamp old_timestamp = 500;
-    ASSERT_EQ("Get => 0,Get => 1", sender.getCommands(true));
-    replyToGet(*cb, sender, 0, old_timestamp);
-    replyToGet(*cb, sender, 1, old_timestamp);
+    ASSERT_EQ("Get => 0,Get => 1", _sender.getCommands(true));
+    replyToGet(*cb, _sender, 0, old_timestamp);
+    replyToGet(*cb, _sender, 1, old_timestamp);
 
-    ASSERT_EQ("Put => 1,Put => 2,Put => 0", sender.getCommands(true, false, 2));
+    ASSERT_EQ("Put => 1,Put => 2,Put => 0", _sender.getCommands(true, false, 2));
 }
 
 TEST_F(TwoPhaseUpdateOperationTest, fast_path_not_restarted_if_document_not_found_on_a_replica_node) {
     setupDistributor(2, 2, "storage:2 distributor:1");
     getConfig().set_update_fast_path_restart_enabled(true);
 
-    std::shared_ptr<TwoPhaseUpdateOperation> cb(sendUpdate("0=1/2/3,1=2/3/4")); // Inconsistent replicas.
-    DistributorMessageSenderStub sender;
-    cb->start(sender, framework::MilliSecTime(0));
+    auto cb = sendUpdate("0=1/2/3,1=2/3/4"); // Inconsistent replicas.
+    cb->start(_sender, framework::MilliSecTime(0));
 
-    ASSERT_EQ("Get => 0,Get => 1", sender.getCommands(true));
-    replyToGet(*cb, sender, 0, Timestamp(0), false);
-    replyToGet(*cb, sender, 1, Timestamp(500));
+    ASSERT_EQ("Get => 0,Get => 1", _sender.getCommands(true));
+    replyToGet(*cb, _sender, 0, Timestamp(0), false);
+    replyToGet(*cb, _sender, 1, Timestamp(500));
 
     // Should _not_ send Update operations!
-    ASSERT_EQ("Put => 1,Put => 0", sender.getCommands(true, false, 2));
+    ASSERT_EQ("Put => 1,Put => 0", _sender.getCommands(true, false, 2));
+}
+
+// Buckets must be created from scratch by Put operations, updates alone cannot do this.
+TEST_F(TwoPhaseUpdateOperationTest, fast_path_not_restarted_if_no_initial_replicas_exist) {
+    setupDistributor(2, 2, "storage:2 distributor:1");
+    getConfig().set_update_fast_path_restart_enabled(true);
+
+    // No replicas, technically consistent but cannot use fast path.
+    auto cb = sendUpdate("", UpdateOptions().createIfNonExistent(true));
+    cb->start(_sender, framework::MilliSecTime(0));
+    ASSERT_EQ("Create bucket => 1,Create bucket => 0,Put => 1,Put => 0",
+              _sender.getCommands(true));
 }
 
 // The weak consistency config _only_ applies to Get operations initiated directly
@@ -1095,15 +1067,256 @@ TEST_F(TwoPhaseUpdateOperationTest, update_gets_are_sent_with_strong_consistency
     setupDistributor(2, 2, "storage:2 distributor:1");
     getConfig().set_use_weak_internal_read_consistency_for_client_gets(true);
 
-    std::shared_ptr<TwoPhaseUpdateOperation> cb(sendUpdate("0=1/2/3,1=2/3/4")); // Inconsistent replicas.
-    DistributorMessageSenderStub sender;
-    cb->start(sender, framework::MilliSecTime(0));
+    auto cb = sendUpdate("0=1/2/3,1=2/3/4"); // Inconsistent replicas.
+    cb->start(_sender, framework::MilliSecTime(0));
 
-    ASSERT_EQ("Get => 0,Get => 1", sender.getCommands(true));
-    auto& get_cmd = dynamic_cast<const api::GetCommand&>(*sender.command(0));
+    ASSERT_EQ("Get => 0,Get => 1", _sender.getCommands(true));
+    auto& get_cmd = dynamic_cast<const api::GetCommand&>(*_sender.command(0));
     EXPECT_EQ(get_cmd.internal_read_consistency(), api::InternalReadConsistency::Strong);
 }
 
+struct ThreePhaseUpdateTest : TwoPhaseUpdateOperationTest {};
+
+TEST_F(ThreePhaseUpdateTest, metadata_only_gets_are_sent_if_3phase_update_enabled) {
+    auto cb = set_up_2_inconsistent_replicas_and_start_update();
+    ASSERT_EQ("Get => 0,Get => 1", _sender.getCommands(true));
+    {
+        auto& get_cmd = dynamic_cast<const api::GetCommand&>(*_sender.command(0));
+        EXPECT_EQ("[none]", get_cmd.getFieldSet());
+        EXPECT_EQ(get_cmd.internal_read_consistency(), api::InternalReadConsistency::Weak);
+        checkMessageSettingsPropagatedTo(_sender.command(0));
+    }
+    {
+        auto& get_cmd = dynamic_cast<const api::GetCommand&>(*_sender.command(1));
+        EXPECT_EQ("[none]", get_cmd.getFieldSet());
+        EXPECT_EQ(get_cmd.internal_read_consistency(), api::InternalReadConsistency::Weak);
+        checkMessageSettingsPropagatedTo(_sender.command(1));
+    }
+}
+
+TEST_F(ThreePhaseUpdateTest, full_document_get_sent_to_replica_with_highest_timestamp) {
+    auto cb = set_up_2_inconsistent_replicas_and_start_update();
+    ASSERT_EQ("Get => 0,Get => 1", _sender.getCommands(true));
+    reply_to_metadata_get(*cb, _sender, 0, 1000U);
+    reply_to_metadata_get(*cb, _sender, 1, 2000U);
+
+    auto& metrics = getDistributor().getMetrics().update_metadata_gets[documentapi::LoadType::DEFAULT];
+    EXPECT_EQ(1, metrics.ok.getValue()); // Technically tracks an entire operation covering multiple Gets.
+
+    // Node 1 has newest document version at ts=2000
+    ASSERT_EQ("Get => 1", _sender.getCommands(true, false, 2));
+    {
+        auto& get_cmd = dynamic_cast<const api::GetCommand&>(*_sender.command(2));
+        EXPECT_EQ("[all]", get_cmd.getFieldSet());
+        EXPECT_EQ(get_cmd.internal_read_consistency(), api::InternalReadConsistency::Strong);
+    }
+}
+
+TEST_F(ThreePhaseUpdateTest, puts_are_sent_after_receiving_full_document_get) {
+    auto cb = set_up_2_inconsistent_replicas_and_start_update();
+    ASSERT_EQ("Get => 0,Get => 1", _sender.getCommands(true));
+    reply_to_metadata_get(*cb, _sender, 0, 2000U);
+    reply_to_metadata_get(*cb, _sender, 1, 1000U);
+    ASSERT_EQ("Get => 0", _sender.getCommands(true, false, 2));
+    replyToGet(*cb, _sender, 2, 2000U);
+    ASSERT_EQ("Put => 1,Put => 0", _sender.getCommands(true, false, 3));
+
+    auto& metrics = getDistributor().getMetrics().update_gets[documentapi::LoadType::DEFAULT];
+    EXPECT_EQ(1, metrics.ok.getValue());
+}
+
+TEST_F(ThreePhaseUpdateTest, consistent_meta_get_timestamps_can_restart_in_fast_path) {
+    auto cb = set_up_2_inconsistent_replicas_and_start_update();
+    ASSERT_EQ("Get => 0,Get => 1", _sender.getCommands(true));
+    api::Timestamp old_timestamp(1500);
+    reply_to_metadata_get(*cb, _sender, 0, old_timestamp);
+    reply_to_metadata_get(*cb, _sender, 1, old_timestamp);
+
+    ASSERT_EQ("Update => 0,Update => 1", _sender.getCommands(true, false, 2));
+    replyToMessage(*cb, _sender, 2, old_timestamp);
+    replyToMessage(*cb, _sender, 3, old_timestamp);
+
+    EXPECT_EQ("UpdateReply(id:ns:testdoctype1::1, "
+              "BucketId(0x0000000000000000), "
+              "timestamp 0, timestamp of updated doc: 1500) "
+              "ReturnCode(NONE)",
+              _sender.getLastReply(true));
+
+    auto& metrics = getDistributor().getMetrics().updates[documentapi::LoadType::DEFAULT];
+    EXPECT_EQ(1, metrics.fast_path_restarts.getValue());
+}
+
+TEST_F(ThreePhaseUpdateTest, fast_path_not_restarted_if_document_not_found_subset_of_replicas) {
+    auto cb = set_up_2_inconsistent_replicas_and_start_update();
+    ASSERT_EQ("Get => 0,Get => 1", _sender.getCommands(true));
+    reply_to_metadata_get(*cb, _sender, 0, 0U);
+    reply_to_metadata_get(*cb, _sender, 1, 1000U);
+    ASSERT_EQ("Get => 1", _sender.getCommands(true, false, 2)); // Not sending updates.
+}
+
+TEST_F(ThreePhaseUpdateTest, no_document_found_on_any_replicas_is_considered_consistent) {
+    auto cb = set_up_2_inconsistent_replicas_and_start_update();
+    ASSERT_EQ("Get => 0,Get => 1", _sender.getCommands(true));
+    api::Timestamp no_document_timestamp(0);
+    reply_to_metadata_get(*cb, _sender, 0, no_document_timestamp);
+    reply_to_metadata_get(*cb, _sender, 1, no_document_timestamp);
+
+    ASSERT_EQ("Update => 0,Update => 1", _sender.getCommands(true, false, 2));
+    auto& metrics = getDistributor().getMetrics().updates[documentapi::LoadType::DEFAULT];
+    EXPECT_EQ(1, metrics.fast_path_restarts.getValue());
+}
+
+TEST_F(ThreePhaseUpdateTest, metadata_get_phase_fails_if_any_replicas_return_failure) {
+    auto cb = set_up_2_inconsistent_replicas_and_start_update();
+    ASSERT_EQ("Get => 0,Get => 1", _sender.getCommands(true));
+    reply_to_metadata_get(*cb, _sender, 1, 1000U);
+    reply_to_metadata_get(*cb, _sender, 0, 0U, api::ReturnCode::INTERNAL_FAILURE);
+    ASSERT_EQ("", _sender.getCommands(true, false, 2)); // No further requests sent.
+
+    EXPECT_EQ("UpdateReply(id:ns:testdoctype1::1, "
+              "BucketId(0x0000000000000000), "
+              "timestamp 0, timestamp of updated doc: 0) "
+              "ReturnCode(ABORTED, One or more metadata Get operations failed; aborting Update)",
+              _sender.getLastReply(true));
+}
+
+TEST_F(ThreePhaseUpdateTest, update_failed_with_transient_error_code_if_replica_set_changed_after_metadata_gets) {
+    setupDistributor(3, 3, "storage:3 distributor:1");
+    getConfig().set_enable_metadata_only_fetch_phase_for_inconsistent_updates(true);
+    auto cb = sendUpdate("0=1/2/3,1=2/3/4"); // 2 replicas, room for 1 more.
+    cb->start(_sender, framework::MilliSecTime(0));
+    // Add new replica to deterministic test bucket after gets have been sent
+    BucketId bucket(0x400000000000cac4); // Always the same in the test.
+    addNodesToBucketDB(bucket, "0=1/2/3,1=2/3/4,2=3/3/3");
+
+    Timestamp old_timestamp = 500;
+    ASSERT_EQ("Get => 0,Get => 1", _sender.getCommands(true));
+    reply_to_metadata_get(*cb, _sender, 0, old_timestamp);
+    reply_to_metadata_get(*cb, _sender, 1, old_timestamp);
+
+    EXPECT_EQ("UpdateReply(id:ns:testdoctype1::1, "
+              "BucketId(0x0000000000000000), "
+              "timestamp 0, timestamp of updated doc: 0) "
+              "ReturnCode(BUCKET_NOT_FOUND, Replica sets changed between update phases, client must retry)",
+              _sender.getLastReply(true));
+}
+
+TEST_F(ThreePhaseUpdateTest, single_full_get_cannot_restart_in_fast_path) {
+    setupDistributor(2, 2, "storage:2 distributor:1");
+    getConfig().set_enable_metadata_only_fetch_phase_for_inconsistent_updates(true);
+    getConfig().set_update_fast_path_restart_enabled(true);
+    auto cb = sendUpdate("0=1/2/3,1=2/3/4"); // Inconsistent replicas.
+    cb->start(_sender, framework::MilliSecTime(0));
+
+    ASSERT_EQ("Get => 0,Get => 1", _sender.getCommands(true));
+    reply_to_metadata_get(*cb, _sender, 0, 1000U);
+    reply_to_metadata_get(*cb, _sender, 1, 2000U);
+    ASSERT_EQ("Get => 1", _sender.getCommands(true, false, 2));
+    replyToGet(*cb, _sender, 2, 2000U);
+    ASSERT_EQ("Put => 1,Put => 0", _sender.getCommands(true, false, 3));
+}
+
+/*
+ * We unify checking for changed replica sets and changed bucket ownership by only
+ * checking for changed replica sets, thereby avoiding a relatively costly ideal
+ * state recomputation that is otherwise redundant. Rationale for why this shall
+ * always be safe:
+ * - for metadata gets to be sent at all, there must be at least one replica
+ *   under the target bucket subtree
+ *   - if there are no replicas, the bucket is implicitly considered inconsistent,
+ *     triggering safe path
+ *   - since there were no replicas initially, the safe path will _not_ restart in
+ *     fast path
+ *   - the safe path will perform the update locally and start a PutOperation,
+ *     implicitly creating new replicas
+ *     - this happens in the same execution context as starting the update operation itself,
+ *       consequently ownership in DB cannot have changed concurrently
+ * - when the a state transition happens where a distributor loses ownership of
+ *   a bucket, it will always immediately purge it from its DB
+ *   - this means that the replica set will inherently change
+ *
+ * It is technically possible to have an ABA situation where, in the course of
+ * an operation's lifetime, a distributor goes from owning a bucket to not
+ * owning it, back to owning it again. Although extremely unlikely to happen,
+ * it doesn't matter since the bucket info from the resulting mutations will
+ * be applied to the current state of the database anyway.
+ */
+TEST_F(ThreePhaseUpdateTest, update_aborted_if_ownership_changed_between_gets_and_fast_restart_update) {
+    auto cb = set_up_2_inconsistent_replicas_and_start_update();
+    ASSERT_EQ("Get => 0,Get => 1", _sender.getCommands(true));
+    // See do_test_ownership_changed_between_gets_and_second_phase() for more in-depth
+    // comments on why this particular cluster state is used.
+    enableDistributorClusterState("storage:2 distributor:1 .0.s:d");
+    getBucketDatabase().clear();
+    reply_to_metadata_get(*cb, _sender, 0, api::Timestamp(70));
+    reply_to_metadata_get(*cb, _sender, 1, api::Timestamp(71));
+
+    // As mentioned in the above comments, ownership changes trigger
+    // on the replicas changed test instead of an explicit ownership
+    // change test.
+    EXPECT_EQ("UpdateReply(id:ns:testdoctype1::1, "
+              "BucketId(0x0000000000000000), "
+              "timestamp 0, timestamp of updated doc: 0) "
+              "ReturnCode(BUCKET_NOT_FOUND, Replica sets changed between update phases, client must retry)",
+              _sender.getLastReply(true));
+}
+
+TEST_F(ThreePhaseUpdateTest, safe_mode_is_implicitly_triggered_if_no_replicas_exist) {
+    setupDistributor(1, 1, "storage:1 distributor:1");
+    getConfig().set_enable_metadata_only_fetch_phase_for_inconsistent_updates(true);
+    auto cb = sendUpdate("", UpdateOptions().createIfNonExistent(true));
+    cb->start(_sender, framework::MilliSecTime(0));
+
+    ASSERT_EQ("CreateBucketCommand(BucketId(0x400000000000cac4), active) "
+              "Reasons to start:  => 0,"
+              "Put(BucketId(0x400000000000cac4), id:ns:testdoctype1::1, "
+              "timestamp 200000000, size 60) => 0",
+              _sender.getCommands(true, true));
+}
+
+TEST_F(ThreePhaseUpdateTest, metadata_gets_propagate_mbus_trace_to_reply) {
+    auto cb = set_up_2_inconsistent_replicas_and_start_update();
+    ASSERT_EQ("Get => 0,Get => 1", _sender.getCommands(true));
+    reply_to_metadata_get(*cb, _sender, 1, 1000U);
+    reply_to_metadata_get(*cb, _sender, 0, 0U, api::ReturnCode::INTERNAL_FAILURE,
+                          "'ello 'ello what's all this then?");
+    ASSERT_EQ("", _sender.getCommands(true, false, 2));
+    ASSERT_EQ("Update Reply", _sender.getLastReply(false));
+
+    std::string trace(_sender.replies().back()->getTrace().toString());
+    ASSERT_THAT(trace, HasSubstr("'ello 'ello what's all this then?"));
+}
+
+TEST_F(ThreePhaseUpdateTest, single_get_mbus_trace_is_propagated_to_reply) {
+    auto cb = set_up_2_inconsistent_replicas_and_start_update();
+    ASSERT_EQ("Get => 0,Get => 1", _sender.getCommands(true));
+    reply_to_metadata_get(*cb, _sender, 0, 0U);
+    reply_to_metadata_get(*cb, _sender, 1, 1000U);
+    ASSERT_EQ("Get => 1", _sender.getCommands(true, false, 2));
+    replyToGet(*cb, _sender, 2, 2000U, false, api::ReturnCode::INTERNAL_FAILURE,
+               "it is me, Leclerc! *lifts glasses*");
+    ASSERT_EQ("Update Reply", _sender.getLastReply(false));
+
+    std::string trace(_sender.replies().back()->getTrace().toString());
+    ASSERT_THAT(trace, HasSubstr("it is me, Leclerc! *lifts glasses*"));
+}
+
+TEST_F(ThreePhaseUpdateTest, single_full_get_reply_received_after_close_is_no_op) {
+    auto cb = set_up_2_inconsistent_replicas_and_start_update();
+    ASSERT_EQ("Get => 0,Get => 1", _sender.getCommands(true));
+    reply_to_metadata_get(*cb, _sender, 0, 0U);
+    reply_to_metadata_get(*cb, _sender, 1, 1000U);
+    ASSERT_EQ("Get => 1", _sender.getCommands(true, false, 2));
+    cb->onClose(_sender);
+    ASSERT_EQ("Update Reply", _sender.getLastReply(false));
+    // Operation closed prior to receiving Get. Note that we should not really get
+    // into this situation since the owner of the operation itself should clear
+    // any mappings associating the reply with the operation, but ensure we handle
+    // it gracefully anyway.
+    replyToGet(*cb, _sender, 2, 2000U);
+    ASSERT_EQ("", _sender.getCommands(true, false, 3)); // Nothing new sent.
+}
+
 // XXX currently differs in behavior from content nodes in that updates for
 // document IDs without explicit doctypes will _not_ be auto-failed on the
 // distributor.
diff --git a/storage/src/tests/frameworkimpl/status/statustest.cpp b/storage/src/tests/frameworkimpl/status/statustest.cpp
index 81d91e2f08a..7c259f00899 100644
--- a/storage/src/tests/frameworkimpl/status/statustest.cpp
+++ b/storage/src/tests/frameworkimpl/status/statustest.cpp
@@ -19,7 +19,7 @@ vespalib::string fetch(int port, const vespalib::string &path) {
     auto crypto = vespalib::CryptoEngine::get_default();
     auto socket = vespalib::SocketSpec::from_port(port).client_address().connect();
     assert(socket.valid());
-    auto conn = vespalib::SyncCryptoSocket::create(*crypto, std::move(socket), false);
+    auto conn = vespalib::SyncCryptoSocket::create_client(*crypto, std::move(socket), vespalib::SocketSpec::from_host_port("localhost", port));
     vespalib::string http_req = vespalib::make_string("GET %s HTTP/1.1\r\n"
                                                       "Host: localhost:%d\r\n"
                                                       "\r\n", path.c_str(), port);
diff --git a/storage/src/tests/persistence/processalltest.cpp b/storage/src/tests/persistence/processalltest.cpp
index 8c0f8853d2d..83f243ed1b2 100644
--- a/storage/src/tests/persistence/processalltest.cpp
+++ b/storage/src/tests/persistence/processalltest.cpp
@@ -23,11 +23,15 @@ TEST_F(ProcessAllHandlerTest, remove_location) {
     api::RemoveLocationCommand removeLocation("id.user == 4", makeDocumentBucket(bucketId));
     ProcessAllHandler handler(getEnv(), getPersistenceProvider());
     spi::Context context(documentapi::LoadType::DEFAULT, 0, 0);
-    handler.handleRemoveLocation(removeLocation, context);
+    auto tracker = handler.handleRemoveLocation(removeLocation, context);
 
     EXPECT_EQ("DocEntry(1234, 1, id:mail:testdoctype1:n=4:3619.html)\n"
               "DocEntry(2345, 1, id:mail:testdoctype1:n=4:4008.html)\n",
               dumpBucket(bucketId));
+
+    auto reply = std::dynamic_pointer_cast<api::RemoveLocationReply>(tracker->getReply());
+    ASSERT_TRUE(reply.get() != nullptr);
+    EXPECT_EQ(2u, reply->documents_removed());
 }
 
 TEST_F(ProcessAllHandlerTest, remove_location_document_subset) {
@@ -44,7 +48,7 @@ TEST_F(ProcessAllHandlerTest, remove_location_document_subset) {
     api::RemoveLocationCommand
         removeLocation("testdoctype1.headerval % 2 == 0", makeDocumentBucket(bucketId));
     spi::Context context(documentapi::LoadType::DEFAULT, 0, 0);
-    handler.handleRemoveLocation(removeLocation, context);
+    auto tracker = handler.handleRemoveLocation(removeLocation, context);
 
     EXPECT_EQ("DocEntry(100, 1, id:mail:testdoctype1:n=4:3619.html)\n"
               "DocEntry(101, 0, Doc(id:mail:testdoctype1:n=4:33113.html))\n"
@@ -57,6 +61,10 @@ TEST_F(ProcessAllHandlerTest, remove_location_document_subset) {
               "DocEntry(108, 1, id:mail:testdoctype1:n=4:42967.html)\n"
               "DocEntry(109, 0, Doc(id:mail:testdoctype1:n=4:6925.html))\n",
               dumpBucket(bucketId));
+
+    auto reply = std::dynamic_pointer_cast<api::RemoveLocationReply>(tracker->getReply());
+    ASSERT_TRUE(reply.get() != nullptr);
+    EXPECT_EQ(5u, reply->documents_removed());
 }
 
 TEST_F(ProcessAllHandlerTest, remove_location_throws_exception_on_unknown_doc_type) {
diff --git a/storage/src/vespa/storage/bucketdb/btree_bucket_database.cpp b/storage/src/vespa/storage/bucketdb/btree_bucket_database.cpp
index 66d44a655e0..85f5c8c5be9 100644
--- a/storage/src/vespa/storage/bucketdb/btree_bucket_database.cpp
+++ b/storage/src/vespa/storage/bucketdb/btree_bucket_database.cpp
@@ -526,6 +526,12 @@ void BTreeBucketDatabase::print(std::ostream& out, bool verbose,
     (void)indent;
 }
 
+vespalib::MemoryUsage BTreeBucketDatabase::memory_usage() const noexcept {
+    auto mem_usage = _tree.getMemoryUsage();
+    mem_usage.merge(_store.getMemoryUsage());
+    return mem_usage;
+}
+
 BTreeBucketDatabase::ReadGuardImpl::ReadGuardImpl(const BTreeBucketDatabase& db)
     : _db(&db),
       _guard(_db->_generation_handler.takeGuard()),
diff --git a/storage/src/vespa/storage/bucketdb/btree_bucket_database.h b/storage/src/vespa/storage/bucketdb/btree_bucket_database.h
index 1f2b25814a8..8898b0c395a 100644
--- a/storage/src/vespa/storage/bucketdb/btree_bucket_database.h
+++ b/storage/src/vespa/storage/bucketdb/btree_bucket_database.h
@@ -91,6 +91,8 @@ public:
     std::unique_ptr<ReadGuard> acquire_read_guard() const override {
         return std::make_unique<ReadGuardImpl>(*this);
     }
+
+    vespalib::MemoryUsage memory_usage() const noexcept override;
 };
 
 }
diff --git a/storage/src/vespa/storage/bucketdb/bucketdatabase.h b/storage/src/vespa/storage/bucketdb/bucketdatabase.h
index 46aaaa997d9..2dbcdd194ef 100644
--- a/storage/src/vespa/storage/bucketdb/bucketdatabase.h
+++ b/storage/src/vespa/storage/bucketdb/bucketdatabase.h
@@ -7,6 +7,7 @@
 #include <vespa/vespalib/util/printable.h>
 #include <vespa/storage/bucketdb/bucketinfo.h>
 #include <vespa/document/bucket/bucketid.h>
+#include <vespa/vespalib/util/memoryusage.h>
 
 namespace storage {
 
@@ -250,6 +251,8 @@ public:
     virtual std::unique_ptr<ReadGuard> acquire_read_guard() const {
         return std::unique_ptr<ReadGuard>();
     }
+
+    [[nodiscard]] virtual vespalib::MemoryUsage memory_usage() const noexcept = 0;
 };
 
 template <typename BucketInfoType>
diff --git a/storage/src/vespa/storage/bucketdb/mapbucketdatabase.cpp b/storage/src/vespa/storage/bucketdb/mapbucketdatabase.cpp
index 463e4a4b8ce..edb808da294 100644
--- a/storage/src/vespa/storage/bucketdb/mapbucketdatabase.cpp
+++ b/storage/src/vespa/storage/bucketdb/mapbucketdatabase.cpp
@@ -594,4 +594,30 @@ void MapBucketDatabase::ReadGuardImpl::find_parents_and_self(const document::Buc
     _db->getParents(bucket, entries);
 }
 
+namespace {
+
+template <typename T>
+size_t allocated_by_vec(const T& vec) noexcept {
+    return (vec.capacity() * sizeof(typename T::value_type));
+}
+
+template <typename T>
+size_t used_by_vec(const T& vec) noexcept {
+    return (vec.size() * sizeof(typename T::value_type));
+}
+
+}
+
+vespalib::MemoryUsage MapBucketDatabase::memory_usage() const noexcept {
+    // We don't have a concept of hold lists here, nor do we know the exact size of the
+    // entries on our free list (these wrap a secondary replica vector allocation).
+    // So we fudge the numbers a bit, returning a lower bound approximation only.
+    // That's OK since this is a legacy database that's on the way out anyway.
+    vespalib::MemoryUsage mem_usage;
+    mem_usage.incAllocatedBytes(allocated_by_vec(_values) + allocated_by_vec(_db));
+    mem_usage.incUsedBytes(used_by_vec(_values) + used_by_vec(_db));
+    mem_usage.incDeadBytes(allocated_by_vec(_free) + allocated_by_vec(_freeValues));
+    return mem_usage;
+}
+
 } // storage
diff --git a/storage/src/vespa/storage/bucketdb/mapbucketdatabase.h b/storage/src/vespa/storage/bucketdb/mapbucketdatabase.h
index 9fe5e2d7740..e41b797a321 100644
--- a/storage/src/vespa/storage/bucketdb/mapbucketdatabase.h
+++ b/storage/src/vespa/storage/bucketdb/mapbucketdatabase.h
@@ -30,6 +30,7 @@ public:
     void print(std::ostream& out, bool verbose, const std::string& indent) const override;
 
     std::unique_ptr<ReadGuard> acquire_read_guard() const override;
+    vespalib::MemoryUsage memory_usage() const noexcept override;
 private:
     struct E {
         E() : value(-1), e_0(-1), e_1(-1) {};
diff --git a/storage/src/vespa/storage/config/distributorconfiguration.cpp b/storage/src/vespa/storage/config/distributorconfiguration.cpp
index 522561ee8a5..9f51d70ce60 100644
--- a/storage/src/vespa/storage/config/distributorconfiguration.cpp
+++ b/storage/src/vespa/storage/config/distributorconfiguration.cpp
@@ -42,6 +42,7 @@ DistributorConfiguration::DistributorConfiguration(StorageComponent& component)
       _update_fast_path_restart_enabled(false),
       _merge_operations_disabled(false),
       _use_weak_internal_read_consistency_for_client_gets(false),
+      _enable_metadata_only_fetch_phase_for_inconsistent_updates(false),
       _minimumReplicaCountingMode(ReplicaCountingMode::TRUSTED)
 { }
 
@@ -155,6 +156,7 @@ DistributorConfiguration::configure(const vespa::config::content::core::StorDist
     _update_fast_path_restart_enabled = config.restartWithFastUpdatePathIfAllGetTimestampsAreConsistent;
     _merge_operations_disabled = config.mergeOperationsDisabled;
     _use_weak_internal_read_consistency_for_client_gets = config.useWeakInternalReadConsistencyForClientGets;
+    _enable_metadata_only_fetch_phase_for_inconsistent_updates = config.enableMetadataOnlyFetchPhaseForInconsistentUpdates;
 
     _minimumReplicaCountingMode = config.minimumReplicaCountingMode;
 
diff --git a/storage/src/vespa/storage/config/distributorconfiguration.h b/storage/src/vespa/storage/config/distributorconfiguration.h
index 333d7073715..b8e99165d69 100644
--- a/storage/src/vespa/storage/config/distributorconfiguration.h
+++ b/storage/src/vespa/storage/config/distributorconfiguration.h
@@ -235,6 +235,13 @@ public:
         return _use_weak_internal_read_consistency_for_client_gets;
     }
 
+    void set_enable_metadata_only_fetch_phase_for_inconsistent_updates(bool enable) noexcept {
+        _enable_metadata_only_fetch_phase_for_inconsistent_updates = enable;
+    }
+    bool enable_metadata_only_fetch_phase_for_inconsistent_updates() const noexcept {
+        return _enable_metadata_only_fetch_phase_for_inconsistent_updates;
+    }
+
     bool containsTimeStatement(const std::string& documentSelection) const;
     
 private:
@@ -281,6 +288,7 @@ private:
     bool _update_fast_path_restart_enabled;
     bool _merge_operations_disabled;
     bool _use_weak_internal_read_consistency_for_client_gets;
+    bool _enable_metadata_only_fetch_phase_for_inconsistent_updates;
 
     DistrConfig::MinimumReplicaCountingMode _minimumReplicaCountingMode;
     
diff --git a/storage/src/vespa/storage/config/stor-communicationmanager.def b/storage/src/vespa/storage/config/stor-communicationmanager.def
index 2a2a840dd4e..c855a4e683a 100644
--- a/storage/src/vespa/storage/config/stor-communicationmanager.def
+++ b/storage/src/vespa/storage/config/stor-communicationmanager.def
@@ -33,6 +33,8 @@ mbus.rpctargetcache.ttl double default = 600
 ## Any value below 1 will be 1.
 mbus.num_threads int default=4
 
+mbus.optimize_for enum {LATENCY, THROUGHPUT, ADAPTIVE} default = LATENCY
+
 ## Enable to use above thread pool for encoding replies
 ## False will use network(fnet) thread
 mbus.dispatch_on_encode bool default=true
diff --git a/storage/src/vespa/storage/config/stor-distributormanager.def b/storage/src/vespa/storage/config/stor-distributormanager.def
index e4d3182ce7a..915b9b6b304 100644
--- a/storage/src/vespa/storage/config/stor-distributormanager.def
+++ b/storage/src/vespa/storage/config/stor-distributormanager.def
@@ -228,3 +228,12 @@ merge_operations_disabled bool default=false
 ## consistent view of fields across document versions.
 ## This is mostly useful in a system that is effectively read-only.
 use_weak_internal_read_consistency_for_client_gets bool default=false
+
+## If true, adds an initial metadata-only fetch phase to updates that touch buckets
+## with inconsistent replicas. Metadata timestamps are compared and a single full Get
+## is sent _only_ to one node with the highest timestamp. Without a metadata phase,
+## full gets would be sent to _all_ nodes.
+## Setting this option to true always implicitly enables the fast update restart
+## feature, so it's not required to set that config to true, nor will setting it
+## to false actually disable the feature.
+enable_metadata_only_fetch_phase_for_inconsistent_updates bool default=false
\ No newline at end of file
diff --git a/storage/src/vespa/storage/distributor/bucketdb/bucketdbmetricupdater.cpp b/storage/src/vespa/storage/distributor/bucketdb/bucketdbmetricupdater.cpp
index 61e67b40f44..51eda0f948b 100644
--- a/storage/src/vespa/storage/distributor/bucketdb/bucketdbmetricupdater.cpp
+++ b/storage/src/vespa/storage/distributor/bucketdb/bucketdbmetricupdater.cpp
@@ -4,8 +4,7 @@
 #include <vespa/storage/distributor/distributormetricsset.h>
 #include <vespa/storage/distributor/idealstatemetricsset.h>
 
-namespace storage {
-namespace distributor {
+namespace storage::distributor {
 
 BucketDBMetricUpdater::Stats::Stats()
     : _docCount(0),
@@ -27,9 +26,7 @@ BucketDBMetricUpdater::BucketDBMetricUpdater()
 {
 }
 
-BucketDBMetricUpdater::~BucketDBMetricUpdater()
-{
-}
+BucketDBMetricUpdater::~BucketDBMetricUpdater() = default;
 
 void
 BucketDBMetricUpdater::resetStats()
@@ -135,6 +132,8 @@ BucketDBMetricUpdater::Stats::propagateMetrics(
 {
     distributorMetrics.docsStored.set(_docCount);
     distributorMetrics.bytesStored.set(_byteCount);
+    distributorMetrics.mutable_dbs.memory_usage.update(_mutable_db_mem_usage);
+    distributorMetrics.read_only_dbs.memory_usage.update(_read_only_db_mem_usage);
 
     idealStateMetrics.buckets_toofewcopies.set(_tooFewCopies);
     idealStateMetrics.buckets_toomanycopies.set(_tooManyCopies);
@@ -148,5 +147,10 @@ BucketDBMetricUpdater::reset()
     resetStats();
 }
 
-} // distributor
-} // storage
+void BucketDBMetricUpdater::update_db_memory_usage(const vespalib::MemoryUsage& mem_usage, bool is_mutable_db) {
+    auto& target = (is_mutable_db ? _workingStats._mutable_db_mem_usage
+                                  : _workingStats._read_only_db_mem_usage);
+    target.merge(mem_usage);
+}
+
+} // storage::distributor
diff --git a/storage/src/vespa/storage/distributor/bucketdb/bucketdbmetricupdater.h b/storage/src/vespa/storage/distributor/bucketdb/bucketdbmetricupdater.h
index 6e15ee03d12..766307f49c2 100644
--- a/storage/src/vespa/storage/distributor/bucketdb/bucketdbmetricupdater.h
+++ b/storage/src/vespa/storage/distributor/bucketdb/bucketdbmetricupdater.h
@@ -4,15 +4,12 @@
 
 #include <vespa/storage/bucketdb/bucketdatabase.h>
 #include <vespa/storage/config/config-stor-distributormanager.h>
-
+#include <vespa/vespalib/util/memoryusage.h>
 #include <unordered_map>
 
-namespace storage {
+namespace storage::distributor {
 
 class DistributorMetricSet;
-
-namespace distributor {
-
 class IdealStateMetricSet;
 
 class BucketDBMetricUpdater {
@@ -25,10 +22,12 @@ public:
         uint64_t _tooManyCopies;
         uint64_t _noTrusted;
         uint64_t _totalBuckets;
+        vespalib::MemoryUsage _mutable_db_mem_usage;
+        vespalib::MemoryUsage _read_only_db_mem_usage;
 
         Stats();
         Stats(const Stats &rhs);
-        ~Stats() { }
+        ~Stats() = default;
 
         Stats &operator=(const Stats &rhs) = default;
 
@@ -66,7 +65,6 @@ private:
 
 public:
     BucketDBMetricUpdater();
-
     ~BucketDBMetricUpdater();
 
     void setMinimumReplicaCountingMode(ReplicaCountingMode mode) noexcept {
@@ -101,11 +99,12 @@ public:
         return _lastCompleteStats;
     }
 
+    void update_db_memory_usage(const vespalib::MemoryUsage& mem_usage, bool is_mutable_db);
+
 private:
     void updateMinReplicationStats(const BucketDatabase::Entry& entry, uint32_t trustedCopies);
 
     void resetStats();
 };
 
-} // distributor
-} // storage
+} // storage::distributor
diff --git a/storage/src/vespa/storage/distributor/distributor.cpp b/storage/src/vespa/storage/distributor/distributor.cpp
index 988af43a7be..cce3d2d1acb 100644
--- a/storage/src/vespa/storage/distributor/distributor.cpp
+++ b/storage/src/vespa/storage/distributor/distributor.cpp
@@ -13,10 +13,13 @@
 #include <vespa/storage/common/global_bucket_space_distribution_converter.h>
 #include <vespa/storageframework/generic/status/xmlstatusreporter.h>
 #include <vespa/document/bucket/fixed_bucket_spaces.h>
+#include <vespa/vespalib/util/memoryusage.h>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".distributor-main");
 
+using namespace std::chrono_literals;
+
 namespace storage::distributor {
 
 class Distributor::Status {
@@ -99,9 +102,9 @@ Distributor::Distributor(DistributorComponentRegister& compReg,
       _bucketSpacesStats(),
       _bucketDbStats(),
       _hostInfoReporter(*this, *this),
-      _ownershipSafeTimeCalc(
-            std::make_unique<OwnershipTransferSafeTimePointCalculator>(
-                std::chrono::seconds(0))), // Set by config later
+      _ownershipSafeTimeCalc(std::make_unique<OwnershipTransferSafeTimePointCalculator>(0s)), // Set by config later
+      _db_memory_sample_interval(30s),
+      _last_db_memory_sample_time_point(),
       _must_send_updated_host_info(false),
       _use_btree_database(use_btree_database)
 {
@@ -768,6 +771,24 @@ Distributor::updateInternalMetricsForCompletedScan()
         _must_send_updated_host_info = true;
     }
     _bucketSpacesStats = std::move(new_space_stats);
+    maybe_update_bucket_db_memory_usage_stats();
+}
+
+void Distributor::maybe_update_bucket_db_memory_usage_stats() {
+    auto now = _component.getClock().getMonotonicTime();
+    if ((now - _last_db_memory_sample_time_point) > _db_memory_sample_interval) {
+        for (auto& space : *_bucketSpaceRepo) {
+            _bucketDBMetricUpdater.update_db_memory_usage(space.second->getBucketDatabase().memory_usage(), true);
+        }
+        for (auto& space : *_readOnlyBucketSpaceRepo) {
+            _bucketDBMetricUpdater.update_db_memory_usage(space.second->getBucketDatabase().memory_usage(), false);
+        }
+        _last_db_memory_sample_time_point = now;
+    } else {
+        // Reuse previous memory statistics instead of sampling new.
+        _bucketDBMetricUpdater.update_db_memory_usage(_bucketDbStats._mutable_db_mem_usage, true);
+        _bucketDBMetricUpdater.update_db_memory_usage(_bucketDbStats._read_only_db_mem_usage, false);
+    }
 }
 
 void
diff --git a/storage/src/vespa/storage/distributor/distributor.h b/storage/src/vespa/storage/distributor/distributor.h
index ac6e306a4fb..bf780434edf 100644
--- a/storage/src/vespa/storage/distributor/distributor.h
+++ b/storage/src/vespa/storage/distributor/distributor.h
@@ -191,6 +191,10 @@ public:
         Distributor& _self;
     };
 
+    std::chrono::steady_clock::duration db_memory_sample_interval() const noexcept {
+        return _db_memory_sample_interval;
+    }
+
 private:
     friend struct DistributorTest;
     friend class BucketDBUpdaterTest;
@@ -226,6 +230,7 @@ private:
      * Takes metric lock.
      */
     void updateInternalMetricsForCompletedScan();
+    void maybe_update_bucket_db_memory_usage_stats();
     void scanAllBuckets();
     MaintenanceScanner::ScanResult scanNextBucket();
     void enableNextConfig();
@@ -329,6 +334,8 @@ private:
     BucketDBMetricUpdater::Stats _bucketDbStats;
     DistributorHostInfoReporter _hostInfoReporter;
     std::unique_ptr<OwnershipTransferSafeTimePointCalculator> _ownershipSafeTimeCalc;
+    std::chrono::steady_clock::duration _db_memory_sample_interval;
+    std::chrono::steady_clock::time_point _last_db_memory_sample_time_point;
     bool _must_send_updated_host_info;
     const bool _use_btree_database;
 };
diff --git a/storage/src/vespa/storage/distributor/distributorinterface.h b/storage/src/vespa/storage/distributor/distributorinterface.h
index aba58e112dc..b17bcd56d19 100644
--- a/storage/src/vespa/storage/distributor/distributorinterface.h
+++ b/storage/src/vespa/storage/distributor/distributorinterface.h
@@ -12,10 +12,10 @@ namespace storage::api { class MergeBucketReply; }
 namespace storage::lib { class ClusterStateBundle; }
 namespace storage {
    class DistributorConfiguration;
-   class DistributorMetricSet;
 }
 namespace storage::distributor {
 
+class DistributorMetricSet;
 class PendingMessageTracker;
 
 class DistributorInterface : public DistributorMessageSender
diff --git a/storage/src/vespa/storage/distributor/distributormetricsset.cpp b/storage/src/vespa/storage/distributor/distributormetricsset.cpp
index 244406ca6fb..70ab5229311 100644
--- a/storage/src/vespa/storage/distributor/distributormetricsset.cpp
+++ b/storage/src/vespa/storage/distributor/distributormetricsset.cpp
@@ -2,17 +2,26 @@
 #include "distributormetricsset.h"
 #include <vespa/metrics/loadmetric.hpp>
 #include <vespa/metrics/summetric.hpp>
+#include <vespa/vespalib/util/memoryusage.h>
 
-namespace storage {
+namespace storage::distributor {
 
 using metrics::MetricSet;
 
+BucketDbMetrics::BucketDbMetrics(const vespalib::string& db_type, metrics::MetricSet* owner)
+    : metrics::MetricSet("bucket_db", {{"bucket_db_type", db_type}}, "", owner),
+      memory_usage(this)
+{}
+
+BucketDbMetrics::~BucketDbMetrics() = default;
+
 DistributorMetricSet::DistributorMetricSet(const metrics::LoadTypeSet& lt)
     : MetricSet("distributor", {{"distributor"}}, ""),
       puts(lt, PersistenceOperationMetricSet("puts"), this),
       updates(lt, UpdateMetricSet(), this),
       update_puts(lt, PersistenceOperationMetricSet("update_puts"), this),
       update_gets(lt, PersistenceOperationMetricSet("update_gets"), this),
+      update_metadata_gets(lt, PersistenceOperationMetricSet("update_metadata_gets"), this),
       removes(lt, PersistenceOperationMetricSet("removes"), this),
       removelocations(lt, PersistenceOperationMetricSet("removelocations"), this),
       gets(lt, PersistenceOperationMetricSet("gets"), this),
@@ -40,7 +49,9 @@ DistributorMetricSet::DistributorMetricSet(const metrics::LoadTypeSet& lt)
       bytesStored("bytesstored",
               {{"logdefault"},{"yamasdefault"}},
               "Number of bytes stored in all buckets controlled by "
-              "this distributor", this)
+              "this distributor", this),
+      mutable_dbs("mutable", this),
+      read_only_dbs("read_only", this)
 {
     docsStored.logOnlyIfSet();
     bytesStored.logOnlyIfSet();
diff --git a/storage/src/vespa/storage/distributor/distributormetricsset.h b/storage/src/vespa/storage/distributor/distributormetricsset.h
index 1e4730b8de6..ce4025d8311 100644
--- a/storage/src/vespa/storage/distributor/distributormetricsset.h
+++ b/storage/src/vespa/storage/distributor/distributormetricsset.h
@@ -5,17 +5,27 @@
 #include "update_metric_set.h"
 #include "visitormetricsset.h"
 #include <vespa/metrics/metrics.h>
+#include <vespa/metrics/common/memory_usage_metrics.h>
 #include <vespa/documentapi/loadtypes/loadtypeset.h>
 
-namespace storage {
+namespace vespalib { class MemoryUsage; }
 
-class DistributorMetricSet : public metrics::MetricSet
-{
+namespace storage::distributor {
+
+struct BucketDbMetrics : metrics::MetricSet {
+    BucketDbMetrics(const vespalib::string& db_type, metrics::MetricSet* owner);
+    ~BucketDbMetrics() override;
+
+    metrics::MemoryUsageMetrics memory_usage;
+};
+
+class DistributorMetricSet : public metrics::MetricSet {
 public:
     metrics::LoadMetric<PersistenceOperationMetricSet> puts;
     metrics::LoadMetric<UpdateMetricSet> updates;
     metrics::LoadMetric<PersistenceOperationMetricSet> update_puts;
     metrics::LoadMetric<PersistenceOperationMetricSet> update_gets;
+    metrics::LoadMetric<PersistenceOperationMetricSet> update_metadata_gets;
     metrics::LoadMetric<PersistenceOperationMetricSet> removes;
     metrics::LoadMetric<PersistenceOperationMetricSet> removelocations;
     metrics::LoadMetric<PersistenceOperationMetricSet> gets;
@@ -28,6 +38,8 @@ public:
     metrics::DoubleAverageMetric recoveryModeTime;
     metrics::LongValueMetric docsStored;
     metrics::LongValueMetric bytesStored;
+    BucketDbMetrics mutable_dbs;
+    BucketDbMetrics read_only_dbs;
 
     explicit DistributorMetricSet(const metrics::LoadTypeSet& lt);
     ~DistributorMetricSet() override;
diff --git a/storage/src/vespa/storage/distributor/externaloperationhandler.cpp b/storage/src/vespa/storage/distributor/externaloperationhandler.cpp
index b946b788391..7e8eaaff15e 100644
--- a/storage/src/vespa/storage/distributor/externaloperationhandler.cpp
+++ b/storage/src/vespa/storage/distributor/externaloperationhandler.cpp
@@ -241,7 +241,8 @@ void ExternalOperationHandler::bounce_or_invoke_read_only_op(
 
 IMPL_MSG_COMMAND_H(ExternalOperationHandler, Put)
 {
-    auto& metrics = getMetrics().puts[cmd->getLoadType()];
+    const documentapi::LoadType & loadType = cmd->getLoadType();
+    auto& metrics = getMetrics().puts[loadType];
     if (!checkTimestampMutationPreconditions(*cmd, getBucketId(cmd->getDocumentId()), metrics)) {
         return true;
     }
@@ -252,9 +253,10 @@ IMPL_MSG_COMMAND_H(ExternalOperationHandler, Put)
 
     auto handle = _mutationSequencer.try_acquire(cmd->getDocumentId());
     if (allowMutation(handle)) {
+        document::BucketSpace bucketSpace = cmd->getBucket().getBucketSpace();
         _op = std::make_shared<PutOperation>(*this,
-                                             _bucketSpaceRepo.get(cmd->getBucket().getBucketSpace()),
-                                             cmd, getMetrics().puts[cmd->getLoadType()], std::move(handle));
+                                             _bucketSpaceRepo.get(bucketSpace),
+                                             std::move(cmd), getMetrics().puts[loadType], std::move(handle));
     } else {
         sendUp(makeConcurrentMutationRejectionReply(*cmd, cmd->getDocumentId(), metrics));
     }
@@ -265,7 +267,8 @@ IMPL_MSG_COMMAND_H(ExternalOperationHandler, Put)
 
 IMPL_MSG_COMMAND_H(ExternalOperationHandler, Update)
 {
-    auto& metrics = getMetrics().updates[cmd->getLoadType()];
+    const documentapi::LoadType & loadType = cmd->getLoadType();
+    auto& metrics = getMetrics().updates[loadType];
     if (!checkTimestampMutationPreconditions(*cmd, getBucketId(cmd->getDocumentId()), metrics)) {
         return true;
     }
@@ -275,9 +278,10 @@ IMPL_MSG_COMMAND_H(ExternalOperationHandler, Update)
     }
     auto handle = _mutationSequencer.try_acquire(cmd->getDocumentId());
     if (allowMutation(handle)) {
+        document::BucketSpace bucketSpace = cmd->getBucket().getBucketSpace();
         _op = std::make_shared<TwoPhaseUpdateOperation>(*this,
-                                                        _bucketSpaceRepo.get(cmd->getBucket().getBucketSpace()),
-                                                        cmd, getMetrics(), std::move(handle));
+                                                        _bucketSpaceRepo.get(bucketSpace),
+                                                        std::move(cmd), getMetrics(), std::move(handle));
     } else {
         sendUp(makeConcurrentMutationRejectionReply(*cmd, cmd->getDocumentId(), metrics));
     }
@@ -288,7 +292,8 @@ IMPL_MSG_COMMAND_H(ExternalOperationHandler, Update)
 
 IMPL_MSG_COMMAND_H(ExternalOperationHandler, Remove)
 {
-    auto& metrics = getMetrics().removes[cmd->getLoadType()];
+    const documentapi::LoadType & loadType = cmd->getLoadType();
+    auto& metrics = getMetrics().removes[loadType];
     if (!checkTimestampMutationPreconditions(*cmd, getBucketId(cmd->getDocumentId()), metrics)) {
         return true;
     }
@@ -299,8 +304,9 @@ IMPL_MSG_COMMAND_H(ExternalOperationHandler, Remove)
     auto handle = _mutationSequencer.try_acquire(cmd->getDocumentId());
     if (allowMutation(handle)) {
         auto &distributorBucketSpace(_bucketSpaceRepo.get(cmd->getBucket().getBucketSpace()));
-        _op = std::make_shared<RemoveOperation>(*this, distributorBucketSpace, cmd,
-                                                getMetrics().removes[cmd->getLoadType()], std::move(handle));
+
+        _op = std::make_shared<RemoveOperation>(*this, distributorBucketSpace, std::move(cmd),
+                                                getMetrics().removes[loadType], std::move(handle));
     } else {
         sendUp(makeConcurrentMutationRejectionReply(*cmd, cmd->getDocumentId(), metrics));
     }
@@ -320,7 +326,7 @@ IMPL_MSG_COMMAND_H(ExternalOperationHandler, RemoveLocation)
     }
 
     _op = std::make_shared<RemoveLocationOperation>(*this, _bucketSpaceRepo.get(cmd->getBucket().getBucketSpace()),
-                                                    cmd, getMetrics().removelocations[cmd->getLoadType()]);
+                                                    std::move(cmd), getMetrics().removelocations[cmd->getLoadType()]);
     return true;
 }
 
diff --git a/storage/src/vespa/storage/distributor/externaloperationhandler.h b/storage/src/vespa/storage/distributor/externaloperationhandler.h
index 96875a3644a..60cad15a791 100644
--- a/storage/src/vespa/storage/distributor/externaloperationhandler.h
+++ b/storage/src/vespa/storage/distributor/externaloperationhandler.h
@@ -13,11 +13,11 @@
 
 namespace storage {
 
-class DistributorMetricSet;
 class PersistenceOperationMetricSet;
 
 namespace distributor {
 
+class DistributorMetricSet;
 class Distributor;
 class MaintenanceOperationGenerator;
 class DirectDispatchSender;
diff --git a/storage/src/vespa/storage/distributor/idealstatemetricsset.cpp b/storage/src/vespa/storage/distributor/idealstatemetricsset.cpp
index d72f4a80ef4..fd193ad6fd8 100644
--- a/storage/src/vespa/storage/distributor/idealstatemetricsset.cpp
+++ b/storage/src/vespa/storage/distributor/idealstatemetricsset.cpp
@@ -6,7 +6,7 @@ namespace storage {
 namespace distributor {
 
 OperationMetricSet::OperationMetricSet(const std::string& name, metrics::Metric::Tags tags, const std::string& description, MetricSet* owner)
-    : MetricSet(name, tags, description, owner),
+    : MetricSet(name, std::move(tags), description, owner),
       pending("pending",
               {{"logdefault"},{"yamasdefault"}},
               "The number of operations pending", this),
@@ -16,14 +16,25 @@ OperationMetricSet::OperationMetricSet(const std::string& name, metrics::Metric:
       failed("done_failed",
              {{"logdefault"},{"yamasdefault"}},
              "The number of operations that failed", this)
-{ }
+{}
 
-OperationMetricSet::~OperationMetricSet() { }
+OperationMetricSet::~OperationMetricSet() = default;
+
+GcMetricSet::GcMetricSet(const std::string& name, metrics::Metric::Tags tags, const std::string& description, MetricSet* owner)
+        : OperationMetricSet(name, std::move(tags), description, owner),
+          documents_removed("documents_removed",
+                           {{"logdefault"},{"yamasdefault"}},
+                           "Number of documents removed by GC operations", this)
+{}
+
+GcMetricSet::~GcMetricSet() = default;
 
 void
 IdealStateMetricSet::createOperationMetrics() {
     typedef IdealStateOperation ISO;
     operations.resize(ISO::OPERATION_COUNT);
+    // Note: naked new is used instead of make_shared due to the latter not being
+    // able to properly transitively deduce the types for the tag initializer lists.
     operations[ISO::DELETE_BUCKET] = std::shared_ptr<OperationMetricSet>(
             new OperationMetricSet("delete_bucket",
                                    {{"logdefault"},{"yamasdefault"}},
@@ -45,9 +56,9 @@ IdealStateMetricSet::createOperationMetrics() {
                                    {{"logdefault"},{"yamasdefault"}},
                                    "Operations to set active/ready state for bucket copies", this));
     operations[ISO::GARBAGE_COLLECTION] = std::shared_ptr<OperationMetricSet>(
-            new OperationMetricSet("garbage_collection",
-                                   {{"logdefault"},{"yamasdefault"}},
-                                   "Operations to garbage collect data from buckets", this));
+            new GcMetricSet("garbage_collection",
+                            {{"logdefault"},{"yamasdefault"}},
+                            "Operations to garbage collect data from buckets", this));
 }
 
 IdealStateMetricSet::IdealStateMetricSet()
@@ -81,7 +92,7 @@ IdealStateMetricSet::IdealStateMetricSet()
     createOperationMetrics();
 }
 
-IdealStateMetricSet::~IdealStateMetricSet() { }
+IdealStateMetricSet::~IdealStateMetricSet() = default;
 
 void IdealStateMetricSet::setPendingOperations(const std::vector<uint64_t>& newMetrics) {
     for (uint32_t i = 0; i < IdealStateOperation::OPERATION_COUNT; i++) {
diff --git a/storage/src/vespa/storage/distributor/idealstatemetricsset.h b/storage/src/vespa/storage/distributor/idealstatemetricsset.h
index 7bb472b4a2c..2679da17598 100644
--- a/storage/src/vespa/storage/distributor/idealstatemetricsset.h
+++ b/storage/src/vespa/storage/distributor/idealstatemetricsset.h
@@ -16,13 +16,21 @@ public:
     metrics::LongCountMetric failed;
 
     OperationMetricSet(const std::string& name, metrics::Metric::Tags tags, const std::string& description, MetricSet* owner);
-    ~OperationMetricSet();
+    ~OperationMetricSet() override;
+};
+
+struct GcMetricSet : OperationMetricSet {
+    metrics::LongCountMetric documents_removed;
+
+    GcMetricSet(const std::string& name, metrics::Metric::Tags tags,
+                const std::string& description, MetricSet* owner);
+    ~GcMetricSet() override;
 };
 
 class IdealStateMetricSet : public metrics::MetricSet
 {
 public:
-    std::vector<std::shared_ptr<OperationMetricSet> > operations;
+    std::vector<std::shared_ptr<OperationMetricSet>> operations;
     metrics::LongValueMetric idealstate_diff;
     metrics::LongValueMetric buckets_toofewcopies;
     metrics::LongValueMetric buckets_toomanycopies;
@@ -35,7 +43,7 @@ public:
     void createOperationMetrics();
 
     IdealStateMetricSet();
-    ~IdealStateMetricSet();
+    ~IdealStateMetricSet() override;
 
     void setPendingOperations(const std::vector<uint64_t>& newMetrics);
 };
diff --git a/storage/src/vespa/storage/distributor/operations/external/CMakeLists.txt b/storage/src/vespa/storage/distributor/operations/external/CMakeLists.txt
index efef4e5e51d..998fc22b1d7 100644
--- a/storage/src/vespa/storage/distributor/operations/external/CMakeLists.txt
+++ b/storage/src/vespa/storage/distributor/operations/external/CMakeLists.txt
@@ -2,6 +2,7 @@
 vespa_add_library(storage_distributoroperationexternal OBJECT
     SOURCES
     getoperation.cpp
+    newest_replica.cpp
     putoperation.cpp
     removelocationoperation.cpp
     removeoperation.cpp
diff --git a/storage/src/vespa/storage/distributor/operations/external/getoperation.cpp b/storage/src/vespa/storage/distributor/operations/external/getoperation.cpp
index 9e66c212ac6..f800166a647 100644
--- a/storage/src/vespa/storage/distributor/operations/external/getoperation.cpp
+++ b/storage/src/vespa/storage/distributor/operations/external/getoperation.cpp
@@ -56,11 +56,12 @@ GetOperation::GetOperation(DistributorComponent& manager,
       _msg(std::move(msg)),
       _returnCode(api::ReturnCode::OK),
       _doc(),
-      _lastModified(),
+      _newest_replica(),
       _metric(metric),
       _operationTimer(manager.getClock()),
       _desired_read_consistency(desired_read_consistency),
-      _has_replica_inconsistency(false)
+      _has_replica_inconsistency(false),
+      _any_replicas_failed(false)
 {
     assignTargetNodeGroups(*read_guard);
 }
@@ -110,7 +111,9 @@ GetOperation::sendForChecksum(DistributorMessageSender& sender, const document::
 
         LOG(spam, "Sending %s to node %d", command->toString(true).c_str(), res[best].copy.getNode());
 
-        res[best].sent = sender.sendToNode(lib::NodeType::STORAGE, res[best].copy.getNode(), command);
+        const auto target_node = res[best].copy.getNode();
+        res[best].sent = sender.sendToNode(lib::NodeType::STORAGE, target_node, command);
+        res[best].to_node = target_node;
         return true;
     }
 
@@ -145,27 +148,31 @@ GetOperation::onReceive(DistributorMessageSender& sender, const std::shared_ptr<
     bool allDone = true;
     for (auto& response : _responses) {
         for (uint32_t i = 0; i < response.second.size(); i++) {
-            if (response.second[i].sent == getreply->getMsgId()) {
+            const auto& bucket_id = response.first.getBucketId();
+            auto& send_state = response.second[i];
+            if (send_state.sent == getreply->getMsgId()) {
                 LOG(debug, "Get on %s returned %s",
                     _msg->getDocumentId().toString().c_str(),
                     getreply->getResult().toString().c_str());
 
-                response.second[i].received = true;
-                response.second[i].returnCode = getreply->getResult();
+                send_state.received = true;
+                send_state.returnCode = getreply->getResult();
 
                 if (getreply->getResult().success()) {
-                    if (_lastModified.has_value() && (getreply->getLastModifiedTimestamp() != *_lastModified)) {
+                    if (_newest_replica.has_value() && (getreply->getLastModifiedTimestamp() != _newest_replica->timestamp)) {
                         // At least two document versions returned had different timestamps.
                         _has_replica_inconsistency = true; // This is a one-way toggle.
                     }
-                    if (!_lastModified.has_value() || getreply->getLastModifiedTimestamp() > *_lastModified) {
+                    if (!_newest_replica.has_value() || getreply->getLastModifiedTimestamp() > _newest_replica->timestamp) {
                         _returnCode = getreply->getResult();
-                        _lastModified = getreply->getLastModifiedTimestamp();
+                        assert(response.second[i].to_node != UINT16_MAX);
+                        _newest_replica = NewestReplica::of(getreply->getLastModifiedTimestamp(), bucket_id, send_state.to_node);
                         _doc = getreply->getDocument();
                     }
                 } else {
-                    if (!_lastModified.has_value()) {
-                        _returnCode = getreply->getResult();
+                    _any_replicas_failed = true;
+                    if (!_newest_replica.has_value()) {
+                        _returnCode = getreply->getResult(); // Don't overwrite if we have a good response.
                     }
                     if (!all_bucket_metadata_initially_consistent()) {
                         // If we're sending to more than a single group of replicas it means our replica set is
@@ -175,7 +182,7 @@ GetOperation::onReceive(DistributorMessageSender& sender, const std::shared_ptr<
                     }
 
                     // Try to send to another node in this checksum group.
-                    bool sent = sendForChecksum(sender, response.first.getBucketId(), response.second);
+                    bool sent = sendForChecksum(sender, bucket_id, response.second);
                     if (sent) {
                         allDone = false;
                     }
@@ -219,7 +226,8 @@ void
 GetOperation::sendReply(DistributorMessageSender& sender)
 {
     if (_msg.get()) {
-        auto repl = std::make_shared<api::GetReply>(*_msg, _doc, _lastModified.value_or(0), !_has_replica_inconsistency);
+        const auto timestamp = _newest_replica.value_or(NewestReplica::make_empty()).timestamp;
+        auto repl = std::make_shared<api::GetReply>(*_msg, _doc, timestamp, !_has_replica_inconsistency);
         repl->setResult(_returnCode);
         update_internal_metrics();
         sender.sendReply(repl);
@@ -250,9 +258,9 @@ GetOperation::assignTargetNodeGroups(const BucketDatabase::ReadGuard& read_guard
             _replicas_in_db.emplace_back(e.getBucketId(), copy.getNode());
 
             if (!copy.valid()) {
-                _responses[GroupId(e.getBucketId(), copy.getChecksum(), copy.getNode())].push_back(copy);
+                _responses[GroupId(e.getBucketId(), copy.getChecksum(), copy.getNode())].emplace_back(copy);
             } else if (!copy.empty()) {
-                _responses[GroupId(e.getBucketId(), copy.getChecksum(), -1)].push_back(copy);
+                _responses[GroupId(e.getBucketId(), copy.getChecksum(), -1)].emplace_back(copy);
             }
         }
     }
diff --git a/storage/src/vespa/storage/distributor/operations/external/getoperation.h b/storage/src/vespa/storage/distributor/operations/external/getoperation.h
index 1106968bcf7..57e878d9e40 100644
--- a/storage/src/vespa/storage/distributor/operations/external/getoperation.h
+++ b/storage/src/vespa/storage/distributor/operations/external/getoperation.h
@@ -1,6 +1,7 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 #pragma once
 
+#include "newest_replica.h"
 #include <vespa/storageapi/defs.h>
 #include <vespa/storage/distributor/operations/operation.h>
 #include <vespa/storage/bucketdb/bucketdatabase.h>
@@ -38,6 +39,9 @@ public:
     std::string getStatus() const override { return ""; }
 
     bool all_bucket_metadata_initially_consistent() const;
+    bool any_replicas_failed() const noexcept {
+        return _any_replicas_failed;
+    }
 
     // Exposed for unit testing. TODO feels a bit dirty :I
     const DistributorBucketSpace& bucketSpace() const noexcept { return _bucketSpace; }
@@ -50,6 +54,13 @@ public:
         return _desired_read_consistency;
     }
 
+    // Note: in the case the document could not be found on any replicas, but
+    // at least one node returned a non-error response, the returned value will
+    // have a timestamp of zero and the most recently asked node as its node.
+    const std::optional<NewestReplica>& newest_replica() const noexcept {
+        return _newest_replica;
+    }
+
 private:
     class GroupId {
     public:
@@ -66,20 +77,19 @@ private:
         int _node;
     };
 
-    class BucketChecksumGroup {
-    public:
-        BucketChecksumGroup(const BucketCopy& c) :
-            copy(c),
-            sent(0), received(false), returnCode(api::ReturnCode::OK)
+    struct BucketChecksumGroup {
+        explicit BucketChecksumGroup(const BucketCopy& c)
+            : copy(c), sent(0), returnCode(api::ReturnCode::OK), to_node(UINT16_MAX), received(false)
         {}
 
         BucketCopy copy;
         api::StorageMessage::Id sent;
-        bool received;
         api::ReturnCode returnCode;
+        uint16_t to_node;
+        bool received;
     };
 
-    typedef std::vector<BucketChecksumGroup> GroupVector;
+    using GroupVector = std::vector<BucketChecksumGroup>;
 
     // Organize the different copies by bucket/checksum pairs. We should
     // try to request GETs from each bucket and each different checksum
@@ -94,13 +104,14 @@ private:
     api::ReturnCode _returnCode;
     std::shared_ptr<document::Document> _doc;
 
-    std::optional<api::Timestamp> _lastModified;
+    std::optional<NewestReplica> _newest_replica;
 
     PersistenceOperationMetricSet& _metric;
     framework::MilliSecTimer _operationTimer;
     std::vector<std::pair<document::BucketId, uint16_t>> _replicas_in_db;
     api::InternalReadConsistency _desired_read_consistency;
     bool _has_replica_inconsistency;
+    bool _any_replicas_failed;
 
     void sendReply(DistributorMessageSender& sender);
     bool sendForChecksum(DistributorMessageSender& sender, const document::BucketId& id, GroupVector& res);
diff --git a/storage/src/vespa/storage/distributor/operations/external/newest_replica.cpp b/storage/src/vespa/storage/distributor/operations/external/newest_replica.cpp
new file mode 100644
index 00000000000..8ca3b9bf411
--- /dev/null
+++ b/storage/src/vespa/storage/distributor/operations/external/newest_replica.cpp
@@ -0,0 +1,14 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include "newest_replica.h"
+#include <ostream>
+
+namespace storage::distributor {
+
+std::ostream& operator<<(std::ostream& os, const NewestReplica& nr) {
+    os << "NewestReplica(timestamp " << nr.timestamp
+       << ", bucket_id " << nr.bucket_id
+       << ", node " << nr.node << ')';
+    return os;
+}
+
+}
diff --git a/storage/src/vespa/storage/distributor/operations/external/newest_replica.h b/storage/src/vespa/storage/distributor/operations/external/newest_replica.h
new file mode 100644
index 00000000000..9eb9c1b8bd0
--- /dev/null
+++ b/storage/src/vespa/storage/distributor/operations/external/newest_replica.h
@@ -0,0 +1,44 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/document/bucket/bucketid.h>
+#include <vespa/storageapi/defs.h>
+#include <iosfwd>
+#include <stddef.h>
+
+namespace storage::distributor {
+
+/*
+ * Tracks the information required to identify the location of the newest replica
+ * for any given document. Newest here means the replica containing the document
+ * version with the highest mutation timestamp.
+ */
+struct NewestReplica {
+    api::Timestamp timestamp {0};
+    document::BucketId bucket_id;
+    uint16_t node {UINT16_MAX};
+
+    static NewestReplica of(api::Timestamp timestamp,
+                            const document::BucketId& bucket_id,
+                            uint16_t node) noexcept {
+        return {timestamp, bucket_id, node};
+    }
+
+    static NewestReplica make_empty() {
+        return {api::Timestamp(0), document::BucketId(), 0};
+    }
+
+    bool operator==(const NewestReplica& rhs) const noexcept {
+        return ((timestamp == rhs.timestamp) &&
+                (bucket_id == rhs.bucket_id) &&
+                (node == rhs.node));
+    }
+    bool operator!=(const NewestReplica& rhs) const noexcept {
+        return !(*this == rhs);
+    }
+};
+
+
+std::ostream& operator<<(std::ostream&, const NewestReplica&);
+
+}
diff --git a/storage/src/vespa/storage/distributor/operations/external/removelocationoperation.cpp b/storage/src/vespa/storage/distributor/operations/external/removelocationoperation.cpp
index 71d8f1b17ea..ca1b6f266d6 100644
--- a/storage/src/vespa/storage/distributor/operations/external/removelocationoperation.cpp
+++ b/storage/src/vespa/storage/distributor/operations/external/removelocationoperation.cpp
@@ -18,30 +18,28 @@ using document::BucketSpace;
 RemoveLocationOperation::RemoveLocationOperation(
         DistributorComponent& manager,
         DistributorBucketSpace &bucketSpace,
-        const std::shared_ptr<api::RemoveLocationCommand> & msg,
+        std::shared_ptr<api::RemoveLocationCommand> msg,
         PersistenceOperationMetricSet& metric)
     : Operation(),
       _trackerInstance(metric,
-               std::shared_ptr<api::BucketInfoReply>(new api::RemoveLocationReply(*msg)),
+               std::make_shared<api::RemoveLocationReply>(*msg),
                manager,
                0),
       _tracker(_trackerInstance),
-      _msg(msg),
+      _msg(std::move(msg)),
       _manager(manager),
       _bucketSpace(bucketSpace)
 {}
 
-RemoveLocationOperation::~RemoveLocationOperation() {}
+RemoveLocationOperation::~RemoveLocationOperation() = default;
 
 int
 RemoveLocationOperation::getBucketId(
         DistributorComponent& manager,
         const api::RemoveLocationCommand& cmd, document::BucketId& bid)
 {
-        std::shared_ptr<const document::DocumentTypeRepo> repo =
-            manager.getTypeRepo();
-        document::select::Parser parser(
-                *repo, manager.getBucketIdFactory());
+        std::shared_ptr<const document::DocumentTypeRepo> repo = manager.getTypeRepo();
+        document::select::Parser parser(*repo, manager.getBucketIdFactory());
 
     document::BucketSelector bucketSel(manager.getBucketIdFactory());
     std::unique_ptr<document::BucketSelector::BucketVector> exprResult
diff --git a/storage/src/vespa/storage/distributor/operations/external/removelocationoperation.h b/storage/src/vespa/storage/distributor/operations/external/removelocationoperation.h
index 64aeb19bae9..bad55497fa1 100644
--- a/storage/src/vespa/storage/distributor/operations/external/removelocationoperation.h
+++ b/storage/src/vespa/storage/distributor/operations/external/removelocationoperation.h
@@ -17,9 +17,9 @@ class RemoveLocationOperation  : public Operation
 public:
     RemoveLocationOperation(DistributorComponent& manager,
                             DistributorBucketSpace &bucketSpace,
-                            const std::shared_ptr<api::RemoveLocationCommand> & msg,
+                            std::shared_ptr<api::RemoveLocationCommand> msg,
                             PersistenceOperationMetricSet& metric);
-    ~RemoveLocationOperation();
+    ~RemoveLocationOperation() override;
 
 
     static int getBucketId(DistributorComponent& manager,
diff --git a/storage/src/vespa/storage/distributor/operations/external/removeoperation.cpp b/storage/src/vespa/storage/distributor/operations/external/removeoperation.cpp
index 0ad3d282cc1..9211e75ca80 100644
--- a/storage/src/vespa/storage/distributor/operations/external/removeoperation.cpp
+++ b/storage/src/vespa/storage/distributor/operations/external/removeoperation.cpp
@@ -13,21 +13,21 @@ using document::BucketSpace;
 
 RemoveOperation::RemoveOperation(DistributorComponent& manager,
                                  DistributorBucketSpace &bucketSpace,
-                                 const std::shared_ptr<api::RemoveCommand> & msg,
+                                 std::shared_ptr<api::RemoveCommand> msg,
                                  PersistenceOperationMetricSet& metric,
                                  SequencingHandle sequencingHandle)
     : SequencedOperation(std::move(sequencingHandle)),
       _trackerInstance(metric,
-               std::shared_ptr<api::BucketInfoReply>(new api::RemoveReply(*msg)),
+               std::make_shared<api::RemoveReply>(*msg),
                manager, msg->getTimestamp()),
       _tracker(_trackerInstance),
-      _msg(msg),
+      _msg(std::move(msg)),
       _manager(manager),
       _bucketSpace(bucketSpace)
 {
 }
 
-RemoveOperation::~RemoveOperation() {}
+RemoveOperation::~RemoveOperation() = default;
 
 void
 RemoveOperation::onStart(DistributorMessageSender& sender)
@@ -43,22 +43,19 @@ RemoveOperation::onStart(DistributorMessageSender& sender)
 
     bool sent = false;
 
-    for (uint32_t j = 0; j < entries.size(); j++) {
-        const BucketDatabase::Entry& e = entries[j];
+    for (const BucketDatabase::Entry& e : entries) {
         std::vector<MessageTracker::ToSend> messages;
-
+        messages.reserve(e->getNodeCount());
         for (uint32_t i = 0; i < e->getNodeCount(); i++) {
-            std::shared_ptr<api::RemoveCommand> command(new api::RemoveCommand(
-                                                                  document::Bucket(_msg->getBucket().getBucketSpace(), e.getBucketId()),
-                                                                  _msg->getDocumentId(),
-                                                                  _msg->getTimestamp()));
+            auto command = std::make_shared<api::RemoveCommand>(document::Bucket(_msg->getBucket().getBucketSpace(), e.getBucketId()),
+                                                                _msg->getDocumentId(),
+                                                                _msg->getTimestamp());
 
             copyMessageSettings(*_msg, *command);
             command->getTrace().setLevel(_msg->getTrace().getLevel());
             command->setCondition(_msg->getCondition());
 
-            messages.push_back(
-                    MessageTracker::ToSend(command, e->getNodeRef(i).getNode()));
+            messages.emplace_back(std::move(command), e->getNodeRef(i).getNode());
             sent = true;
         }
 
diff --git a/storage/src/vespa/storage/distributor/operations/external/removeoperation.h b/storage/src/vespa/storage/distributor/operations/external/removeoperation.h
index 7794be73ac8..90e7ac94f0a 100644
--- a/storage/src/vespa/storage/distributor/operations/external/removeoperation.h
+++ b/storage/src/vespa/storage/distributor/operations/external/removeoperation.h
@@ -17,10 +17,10 @@ class RemoveOperation  : public SequencedOperation
 public:
     RemoveOperation(DistributorComponent& manager,
                     DistributorBucketSpace &bucketSpace,
-                    const std::shared_ptr<api::RemoveCommand> & msg,
+                    std::shared_ptr<api::RemoveCommand> msg,
                     PersistenceOperationMetricSet& metric,
                     SequencingHandle sequencingHandle = SequencingHandle());
-    ~RemoveOperation();
+    ~RemoveOperation() override;
 
     void onStart(DistributorMessageSender& sender) override;
     const char* getName() const override { return "remove"; };
diff --git a/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.cpp b/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.cpp
index 47cacafee80..4f49d89929f 100644
--- a/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.cpp
+++ b/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.cpp
@@ -22,20 +22,24 @@ namespace storage::distributor {
 TwoPhaseUpdateOperation::TwoPhaseUpdateOperation(
         DistributorComponent& manager,
         DistributorBucketSpace &bucketSpace,
-        const std::shared_ptr<api::UpdateCommand>& msg,
+        std::shared_ptr<api::UpdateCommand> msg,
         DistributorMetricSet& metrics,
         SequencingHandle sequencingHandle)
     : SequencedOperation(std::move(sequencingHandle)),
     _updateMetric(metrics.updates[msg->getLoadType()]),
     _putMetric(metrics.update_puts[msg->getLoadType()]),
     _getMetric(metrics.update_gets[msg->getLoadType()]),
-    _updateCmd(msg),
+    _metadata_get_metrics(metrics.update_metadata_gets[msg->getLoadType()]),
+    _updateCmd(std::move(msg)),
     _updateReply(),
     _manager(manager),
     _bucketSpace(bucketSpace),
     _sendState(SendState::NONE_SENT),
     _mode(Mode::FAST_PATH),
+    _single_get_latency_timer(),
     _fast_path_repair_source_node(0xffff),
+    _use_initial_cheap_metadata_fetch_phase(
+            _manager.getDistributor().getConfig().enable_metadata_only_fetch_phase_for_inconsistent_updates()),
     _replySent(false)
 {
     document::BucketIdFactory idFactory;
@@ -92,10 +96,12 @@ const char*
 TwoPhaseUpdateOperation::stateToString(SendState state)
 {
     switch (state) {
-    case SendState::NONE_SENT: return "NONE_SENT";
-    case SendState::UPDATES_SENT: return "UPDATES_SENT";
-    case SendState::GETS_SENT: return "GETS_SENT";
-    case SendState::PUTS_SENT: return "PUTS_SENT";
+    case SendState::NONE_SENT:          return "NONE_SENT";
+    case SendState::UPDATES_SENT:       return "UPDATES_SENT";
+    case SendState::METADATA_GETS_SENT: return "METADATA_GETS_SENT";
+    case SendState::SINGLE_GET_SENT:    return "SINGLE_GET_SENT";
+    case SendState::FULL_GETS_SENT:     return "FULL_GETS_SENT";
+    case SendState::PUTS_SENT:          return "PUTS_SENT";
     default:
         assert(!"Unknown state");
         return "";
@@ -159,7 +165,7 @@ void
 TwoPhaseUpdateOperation::startFastPathUpdate(DistributorMessageSender& sender)
 {
     _mode = Mode::FAST_PATH;
-    LOG(debug, "Update(%s) fast path: sending Update commands", _updateCmd->getDocumentId().toString().c_str());
+    LOG(debug, "Update(%s) fast path: sending Update commands", update_doc_id().c_str());
     auto updateOperation = std::make_shared<UpdateOperation>(_manager, _bucketSpace, _updateCmd, _updateMetric);
     UpdateOperation & op = *updateOperation;
     IntermediateMessageSender intermediate(_sentMessageMap, std::move(updateOperation), sender);
@@ -174,26 +180,51 @@ TwoPhaseUpdateOperation::startFastPathUpdate(DistributorMessageSender& sender)
 void
 TwoPhaseUpdateOperation::startSafePathUpdate(DistributorMessageSender& sender)
 {
-    LOG(debug, "Update(%s) safe path: sending Get commands", _updateCmd->getDocumentId().toString().c_str());
-
     _mode = Mode::SLOW_PATH;
-    document::Bucket bucket(_updateCmd->getBucket().getBucketSpace(), document::BucketId(0));
-    auto get = std::make_shared<api::GetCommand>(bucket, _updateCmd->getDocumentId(),"[all]");
-    copyMessageSettings(*_updateCmd, *get);
-    auto getOperation = std::make_shared<GetOperation>(
-            _manager, _bucketSpace, _bucketSpace.getBucketDatabase().acquire_read_guard(), get, _getMetric);
-    GetOperation & op = *getOperation;
-    IntermediateMessageSender intermediate(_sentMessageMap, std::move(getOperation), sender);
+    auto get_operation = create_initial_safe_path_get_operation();
+    GetOperation& op = *get_operation;
+    IntermediateMessageSender intermediate(_sentMessageMap, std::move(get_operation), sender);
     _replicas_at_get_send_time = op.replicas_in_db(); // Populated at construction time, not at start()-time
     op.start(intermediate, _manager.getClock().getTimeInMillis());
-    transitionTo(SendState::GETS_SENT);
+
+    transitionTo(_use_initial_cheap_metadata_fetch_phase
+                 ? SendState::METADATA_GETS_SENT
+                 : SendState::FULL_GETS_SENT);
 
     if (intermediate._reply.get()) {
         assert(intermediate._reply->getType() == api::MessageType::GET_REPLY);
+        // We always trigger the safe path Get reply handling here regardless of whether
+        // metadata-only or full Gets were sent. This is because we might get an early
+        // reply due to there being no replicas in existence at all for the target bucket.
+        // In this case, we rely on the safe path fallback to implicitly create the bucket
+        // by performing the update locally and sending CreateBucket+Put to the ideal nodes.
         handleSafePathReceivedGet(sender, static_cast<api::GetReply&>(*intermediate._reply));
     }
 }
 
+std::shared_ptr<GetOperation>
+TwoPhaseUpdateOperation::create_initial_safe_path_get_operation() {
+    document::Bucket bucket(_updateCmd->getBucket().getBucketSpace(), document::BucketId(0));
+    const char* field_set = _use_initial_cheap_metadata_fetch_phase ? "[none]" : "[all]";
+    auto get = std::make_shared<api::GetCommand>(bucket, _updateCmd->getDocumentId(), field_set);
+    copyMessageSettings(*_updateCmd, *get);
+    // Metadata-only Gets just look at the data in the meta-store, not any fields.
+    // The meta-store is always updated before any ACK is returned for a mutation,
+    // so all the information we need is guaranteed to be consistent even with a
+    // weak read. But since weak reads allow the Get operation to bypass commit
+    // queues, latency may be greatly reduced in contended situations.
+    auto read_consistency = (_use_initial_cheap_metadata_fetch_phase
+                             ? api::InternalReadConsistency::Weak
+                             : api::InternalReadConsistency::Strong);
+    LOG(debug, "Update(%s) safe path: sending Get commands with field set '%s' "
+               "and internal read consistency %s",
+               update_doc_id().c_str(), field_set, api::to_string(read_consistency));
+    auto& get_metric = (_use_initial_cheap_metadata_fetch_phase ? _metadata_get_metrics : _getMetric);
+    return std::make_shared<GetOperation>(
+            _manager, _bucketSpace, _bucketSpace.getBucketDatabase().acquire_read_guard(),
+            get, get_metric, read_consistency);
+}
+
 void
 TwoPhaseUpdateOperation::onStart(DistributorMessageSender& sender) {
     if (isFastPathPossible()) {
@@ -247,7 +278,7 @@ TwoPhaseUpdateOperation::schedulePutsWithUpdatedDocument(std::shared_ptr<documen
     transitionTo(SendState::PUTS_SENT);
 
     LOG(debug, "Update(%s): sending Put commands with doc %s",
-        _updateCmd->getDocumentId().toString().c_str(), doc->toString(true).c_str());
+        update_doc_id().c_str(), doc->toString(true).c_str());
 
     if (intermediate._reply.get()) {
         sendReplyWithResult(sender, intermediate._reply->getResult());
@@ -269,13 +300,12 @@ TwoPhaseUpdateOperation::handleFastPathReceive(DistributorMessageSender& sender,
                                                const std::shared_ptr<api::StorageReply>& msg)
 {
     if (msg->getType() == api::MessageType::GET_REPLY) {
-        assert(_sendState == SendState::GETS_SENT);
-        api::GetReply& getReply = static_cast<api::GetReply&> (*msg);
+        assert(_sendState == SendState::FULL_GETS_SENT);
+        auto& getReply = static_cast<api::GetReply&>(*msg);
         addTraceFromReply(getReply);
 
-        LOG(debug, "Update(%s) Get reply had result: %s",
-            _updateCmd->getDocumentId().toString().c_str(),
-            getReply.getResult().toString().c_str());
+        LOG(debug, "Update(%s) fast path: Get reply had result %s",
+            update_doc_id().c_str(), getReply.getResult().toString().c_str());
 
         if (!getReply.getResult().success()) {
             sendReplyWithResult(sender, getReply.getResult());
@@ -310,7 +340,7 @@ TwoPhaseUpdateOperation::handleFastPathReceive(DistributorMessageSender& sender,
                 // Failed or was consistent
                 sendReply(sender, intermediate._reply);
             } else {
-                LOG(debug, "Update(%s) fast path: was inconsistent!", _updateCmd->getDocumentId().toString().c_str());
+                LOG(debug, "Update(%s) fast path: was inconsistent!", update_doc_id().c_str());
 
                 _updateReply = intermediate._reply;
                 _fast_path_repair_source_node = bestNode.second;
@@ -319,7 +349,7 @@ TwoPhaseUpdateOperation::handleFastPathReceive(DistributorMessageSender& sender,
                 copyMessageSettings(*_updateCmd, *cmd);
 
                 sender.sendToNode(lib::NodeType::STORAGE, _fast_path_repair_source_node, cmd);
-                transitionTo(SendState::GETS_SENT);
+                transitionTo(SendState::FULL_GETS_SENT);
             }
         }
     } else {
@@ -328,7 +358,7 @@ TwoPhaseUpdateOperation::handleFastPathReceive(DistributorMessageSender& sender,
             addTraceFromReply(*intermediate._reply);
             sendReplyWithResult(sender, intermediate._reply->getResult());
             LOG(warning, "Forced convergence of '%s' using document from node %u",
-                _updateCmd->getDocumentId().toString().c_str(), _fast_path_repair_source_node);
+                update_doc_id().c_str(), _fast_path_repair_source_node);
         }
     }
 }
@@ -337,6 +367,13 @@ void
 TwoPhaseUpdateOperation::handleSafePathReceive(DistributorMessageSender& sender,
                                                const std::shared_ptr<api::StorageReply>& msg)
 {
+    // No explicit operation is associated with the direct replica Get operation,
+    // so we handle its reply separately.
+    if (_sendState == SendState::SINGLE_GET_SENT) {
+        assert(msg->getType() == api::MessageType::GET_REPLY);
+        handle_safe_path_received_single_full_get(sender, dynamic_cast<api::GetReply&>(*msg));
+        return;
+    }
     std::shared_ptr<Operation> callback = _sentMessageMap.pop(msg->getMsgId());
     assert(callback.get());
     Operation & callbackOp = *callback;
@@ -348,7 +385,12 @@ TwoPhaseUpdateOperation::handleSafePathReceive(DistributorMessageSender& sender,
         return; // Not enough replies received yet or we're draining callbacks.
     }
     addTraceFromReply(*intermediate._reply);
-    if (_sendState == SendState::GETS_SENT) {
+    if (_sendState == SendState::METADATA_GETS_SENT) {
+        assert(intermediate._reply->getType() == api::MessageType::GET_REPLY);
+        const auto& get_op = dynamic_cast<const GetOperation&>(*intermediate.callback);
+        handle_safe_path_received_metadata_get(sender, static_cast<api::GetReply&>(*intermediate._reply),
+                                               get_op.newest_replica(), get_op.any_replicas_failed());
+    } else if (_sendState == SendState::FULL_GETS_SENT) {
         assert(intermediate._reply->getType() == api::MessageType::GET_REPLY);
         handleSafePathReceivedGet(sender, static_cast<api::GetReply&>(*intermediate._reply));
     } else if (_sendState == SendState::PUTS_SENT) {
@@ -359,6 +401,80 @@ TwoPhaseUpdateOperation::handleSafePathReceive(DistributorMessageSender& sender,
     }
 }
 
+void TwoPhaseUpdateOperation::handle_safe_path_received_single_full_get(
+        DistributorMessageSender& sender,
+        api::GetReply& reply)
+{
+    LOG(spam, "Received single full Get reply for '%s'", update_doc_id().c_str());
+    if (_replySent) {
+        return; // Bail out; the operation has been concurrently closed.
+    }
+    addTraceFromReply(reply);
+    if (reply.getResult().success()) {
+        _getMetric.ok.inc();
+    } else {
+        _getMetric.failures.storagefailure.inc();
+    }
+    assert(_single_get_latency_timer.has_value());
+    _getMetric.latency.addValue(_single_get_latency_timer->getElapsedTimeAsDouble());
+    handleSafePathReceivedGet(sender, reply);
+}
+
+void TwoPhaseUpdateOperation::handle_safe_path_received_metadata_get(
+        DistributorMessageSender& sender, api::GetReply& reply,
+        const std::optional<NewestReplica>& newest_replica,
+        bool any_replicas_failed)
+{
+    LOG(debug, "Update(%s): got (metadata only) Get reply with result %s",
+        update_doc_id().c_str(), reply.getResult().toString().c_str());
+
+    if (!reply.getResult().success()) {
+        sendReplyWithResult(sender, reply.getResult());
+        return;
+    }
+    // It's possible for a single replica to fail during processing without the entire
+    // Get operation failing. Although we know a priori if replicas are out of sync,
+    // we don't know which one has the highest timestamp (it might have been the one
+    // on the node that the metadata Get just failed towards). To err on the side of
+    // caution we abort the update if this happens. If a simple metadata Get fails, it
+    // is highly likely that a full partial update or put operation would fail as well.
+    if (any_replicas_failed) {
+        LOG(debug, "Update(%s): had failed replicas, aborting update", update_doc_id().c_str());
+        sendReplyWithResult(sender, api::ReturnCode(api::ReturnCode::Result::ABORTED,
+                            "One or more metadata Get operations failed; aborting Update"));
+        return;
+    }
+    if (!replica_set_unchanged_after_get_operation()) {
+        // Use BUCKET_NOT_FOUND to trigger a silent retry.
+        LOG(debug, "Update(%s): replica set has changed after metadata get phase", update_doc_id().c_str());
+        sendReplyWithResult(sender, api::ReturnCode(api::ReturnCode::Result::BUCKET_NOT_FOUND,
+                                                    "Replica sets changed between update phases, client must retry"));
+        return;
+    }
+    if (reply.had_consistent_replicas()) {
+        LOG(debug, "Update(%s): metadata Gets consistent; restarting in fast path", update_doc_id().c_str());
+        restart_with_fast_path_due_to_consistent_get_timestamps(sender);
+        return;
+    }
+    // If we've gotten here, we must have had no Get failures and replicas must
+    // be somehow inconsistent. Replicas can only be inconsistent if their timestamps
+    // mismatch, so we must have observed at least one non-zero timestamp.
+    assert(newest_replica.has_value() && (newest_replica->timestamp != api::Timestamp(0)));
+    // Timestamps were not in sync, so we have to fetch the document from the highest
+    // timestamped replica, apply the update to it and then explicitly Put the result
+    // to all replicas.
+    _single_get_latency_timer.emplace(_manager.getClock());
+    document::Bucket bucket(_updateCmd->getBucket().getBucketSpace(), newest_replica->bucket_id);
+    LOG(debug, "Update(%s): sending single payload Get to %s on node %u (had timestamp %" PRIu64 ")",
+        update_doc_id().c_str(), bucket.toString().c_str(),
+        newest_replica->node, newest_replica->timestamp);
+    auto cmd = std::make_shared<api::GetCommand>(bucket, _updateCmd->getDocumentId(), "[all]");
+    copyMessageSettings(*_updateCmd, *cmd);
+    sender.sendToNode(lib::NodeType::STORAGE, newest_replica->node, cmd);
+
+    transitionTo(SendState::SINGLE_GET_SENT);
+}
+
 void
 TwoPhaseUpdateOperation::handleSafePathReceivedGet(DistributorMessageSender& sender, api::GetReply& reply)
 {
@@ -370,7 +486,9 @@ TwoPhaseUpdateOperation::handleSafePathReceivedGet(DistributorMessageSender& sen
         sendReplyWithResult(sender, reply.getResult());
         return;
     }
-    if (may_restart_with_fast_path(reply)) {
+    // Single Get could technically be considered consistent with itself, so make
+    // sure we never treat that as sufficient for restarting in the fast path.
+    if ((_sendState != SendState::SINGLE_GET_SENT) && may_restart_with_fast_path(reply)) {
         restart_with_fast_path_due_to_consistent_get_timestamps(sender);
         return;
     }
@@ -395,7 +513,7 @@ TwoPhaseUpdateOperation::handleSafePathReceivedGet(DistributorMessageSender& sen
         return;
     } else if (shouldCreateIfNonExistent()) {
         LOG(debug, "No existing documents found for %s, creating blank document to update",
-            _updateCmd->getUpdate()->getId().toString().c_str());
+            update_doc_id().c_str());
         docToUpdate = createBlankDocument();
         setUpdatedForTimestamp(putTimestamp);
     } else {
@@ -412,7 +530,7 @@ TwoPhaseUpdateOperation::handleSafePathReceivedGet(DistributorMessageSender& sen
 
 bool TwoPhaseUpdateOperation::may_restart_with_fast_path(const api::GetReply& reply) {
     return (_manager.getDistributor().getConfig().update_fast_path_restart_enabled() &&
-            reply.wasFound() &&
+            !_replicas_at_get_send_time.empty() && // To ensure we send CreateBucket+Put if no replicas exist.
             reply.had_consistent_replicas() &&
             replica_set_unchanged_after_get_operation());
 }
@@ -434,12 +552,15 @@ bool TwoPhaseUpdateOperation::replica_set_unchanged_after_get_operation() const
 
 void TwoPhaseUpdateOperation::restart_with_fast_path_due_to_consistent_get_timestamps(DistributorMessageSender& sender) {
     LOG(debug, "Update(%s): all Gets returned in initial safe path were consistent, restarting in fast path mode",
-               _updateCmd->getDocumentId().toString().c_str());
+               update_doc_id().c_str());
     if (lostBucketOwnershipBetweenPhases()) {
         sendLostOwnershipTransientErrorReply(sender);
         return;
     }
     _updateMetric.fast_path_restarts.inc();
+    // Must not be any other messages in flight, or we might mis-interpret them when we
+    // have switched back to fast-path mode.
+    assert(_sentMessageMap.empty());
     startFastPathUpdate(sender);
 }
 
@@ -553,4 +674,9 @@ TwoPhaseUpdateOperation::onClose(DistributorMessageSender& sender) {
     }
 }
 
+vespalib::string TwoPhaseUpdateOperation::update_doc_id() const {
+    assert(_updateCmd.get() != nullptr);
+    return _updateCmd->getDocumentId().toString();
+}
+
 }
diff --git a/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.h b/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.h
index f9e32c31a9c..2d8f3e8494d 100644
--- a/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.h
+++ b/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.h
@@ -1,11 +1,13 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 #pragma once
 
-#include <set>
+#include "newest_replica.h"
 #include <vespa/storageapi/messageapi/returncode.h>
 #include <vespa/storage/distributor/persistencemessagetracker.h>
 #include <vespa/storage/distributor/operations/sequenced_operation.h>
 #include <vespa/document/update/documentupdate.h>
+#include <set>
+#include <optional>
 
 namespace document {
 class Document;
@@ -23,6 +25,7 @@ class UpdateMetricSet;
 namespace distributor {
 
 class DistributorBucketSpace;
+class GetOperation;
 
 /*
  * General functional outline:
@@ -45,6 +48,7 @@ class DistributorBucketSpace;
  *
  * Note that the above case also implicitly handles the case in which a
  * bucket does not exist.
+ *
 */
 
 
@@ -53,10 +57,10 @@ class TwoPhaseUpdateOperation : public SequencedOperation
 public:
     TwoPhaseUpdateOperation(DistributorComponent& manager,
                             DistributorBucketSpace &bucketSpace,
-                            const std::shared_ptr<api::UpdateCommand> & msg,
+                            std::shared_ptr<api::UpdateCommand> msg,
                             DistributorMetricSet& metrics,
                             SequencingHandle sequencingHandle = SequencingHandle());
-    ~TwoPhaseUpdateOperation();
+    ~TwoPhaseUpdateOperation() override;
 
     void onStart(DistributorMessageSender& sender) override;
 
@@ -69,13 +73,13 @@ public:
 
     void onClose(DistributorMessageSender& sender) override;
 
-    bool canSendHeaderOnly() const;
-
 private:
     enum class SendState {
         NONE_SENT,
         UPDATES_SENT,
-        GETS_SENT,
+        METADATA_GETS_SENT,
+        SINGLE_GET_SENT,
+        FULL_GETS_SENT,
         PUTS_SENT,
     };
 
@@ -85,7 +89,7 @@ private:
     };
 
     void transitionTo(SendState newState);
-    const char* stateToString(SendState);
+    static const char* stateToString(SendState);
 
     void sendReply(DistributorMessageSender&,
                    std::shared_ptr<api::StorageReply>&);
@@ -108,10 +112,14 @@ private:
                                const std::shared_ptr<api::StorageReply>&);
     void handleSafePathReceive(DistributorMessageSender&,
                                const std::shared_ptr<api::StorageReply>&);
-    void handleSafePathReceivedGet(DistributorMessageSender&,
-                                   api::GetReply&);
-    void handleSafePathReceivedPut(DistributorMessageSender&,
-                                   const api::PutReply&);
+    std::shared_ptr<GetOperation> create_initial_safe_path_get_operation();
+    void handle_safe_path_received_metadata_get(DistributorMessageSender&,
+                                                api::GetReply&,
+                                                const std::optional<NewestReplica>&,
+                                                bool any_replicas_failed);
+    void handle_safe_path_received_single_full_get(DistributorMessageSender&, api::GetReply&);
+    void handleSafePathReceivedGet(DistributorMessageSender&, api::GetReply&);
+    void handleSafePathReceivedPut(DistributorMessageSender&, const api::PutReply&);
     bool shouldCreateIfNonExistent() const;
     bool processAndMatchTasCondition(
             DistributorMessageSender& sender,
@@ -124,10 +132,13 @@ private:
     bool may_restart_with_fast_path(const api::GetReply& reply);
     bool replica_set_unchanged_after_get_operation() const;
     void restart_with_fast_path_due_to_consistent_get_timestamps(DistributorMessageSender& sender);
+    // Precondition: reply has not yet been sent.
+    vespalib::string update_doc_id() const;
 
     UpdateMetricSet& _updateMetric;
     PersistenceOperationMetricSet& _putMetric;
     PersistenceOperationMetricSet& _getMetric;
+    PersistenceOperationMetricSet& _metadata_get_metrics;
     std::shared_ptr<api::UpdateCommand> _updateCmd;
     std::shared_ptr<api::StorageReply> _updateReply;
     DistributorComponent& _manager;
@@ -138,7 +149,9 @@ private:
     mbus::TraceNode _trace;
     document::BucketId _updateDocBucketId;
     std::vector<std::pair<document::BucketId, uint16_t>> _replicas_at_get_send_time;
+    std::optional<framework::MilliSecTimer> _single_get_latency_timer;
     uint16_t _fast_path_repair_source_node;
+    bool _use_initial_cheap_metadata_fetch_phase;
     bool _replySent;
 };
 
diff --git a/storage/src/vespa/storage/distributor/operations/idealstate/garbagecollectionoperation.cpp b/storage/src/vespa/storage/distributor/operations/idealstate/garbagecollectionoperation.cpp
index c674add80f7..fc127c2e0eb 100644
--- a/storage/src/vespa/storage/distributor/operations/idealstate/garbagecollectionoperation.cpp
+++ b/storage/src/vespa/storage/distributor/operations/idealstate/garbagecollectionoperation.cpp
@@ -2,6 +2,7 @@
 
 #include "garbagecollectionoperation.h"
 #include <vespa/storage/distributor/idealstatemanager.h>
+#include <vespa/storage/distributor/idealstatemetricsset.h>
 #include <vespa/storage/distributor/distributor.h>
 #include <vespa/storage/distributor/distributor_bucket_space.h>
 #include <vespa/storageapi/message/removelocation.h>
@@ -9,19 +10,18 @@
 #include <vespa/log/log.h>
 LOG_SETUP(".distributor.operation.idealstate.remove");
 
-using namespace storage::distributor;
+namespace storage::distributor {
 
 GarbageCollectionOperation::GarbageCollectionOperation(const std::string& clusterName, const BucketAndNodes& nodes)
     : IdealStateOperation(nodes),
       _tracker(clusterName),
-      _replica_info()
+      _replica_info(),
+      _max_documents_removed(0)
 {}
 
 GarbageCollectionOperation::~GarbageCollectionOperation() = default;
 
-void
-GarbageCollectionOperation::onStart(DistributorMessageSender& sender)
-{
+void GarbageCollectionOperation::onStart(DistributorMessageSender& sender) {
     BucketDatabase::Entry entry = _bucketSpace->getBucketDatabase().get(getBucketId());
     std::vector<uint16_t> nodes = entry->getNodes();
 
@@ -43,7 +43,7 @@ GarbageCollectionOperation::onStart(DistributorMessageSender& sender)
 
 void
 GarbageCollectionOperation::onReceive(DistributorMessageSender&,
-                           const std::shared_ptr<api::StorageReply>& reply)
+                                      const std::shared_ptr<api::StorageReply>& reply)
 {
     auto* rep = dynamic_cast<api::RemoveLocationReply*>(reply.get());
     assert(rep != nullptr);
@@ -53,6 +53,7 @@ GarbageCollectionOperation::onReceive(DistributorMessageSender&,
     if (!rep->getResult().failed()) {
         _replica_info.emplace_back(_manager->getDistributorComponent().getUniqueTimestamp(),
                                    node, rep->getBucketInfo());
+        _max_documents_removed = std::max(_max_documents_removed, rep->documents_removed());
     } else {
         _ok = false;
     }
@@ -61,6 +62,7 @@ GarbageCollectionOperation::onReceive(DistributorMessageSender&,
         if (_ok) {
             merge_received_bucket_info_into_db();
         }
+        update_gc_metrics();
         done();
     }
 }
@@ -76,8 +78,16 @@ void GarbageCollectionOperation::merge_received_bucket_info_into_db() {
     }
 }
 
+void GarbageCollectionOperation::update_gc_metrics() {
+    auto metric_base = _manager->getMetrics().operations[IdealStateOperation::GARBAGE_COLLECTION];
+    auto gc_metrics = std::dynamic_pointer_cast<GcMetricSet>(metric_base);
+    assert(gc_metrics);
+    gc_metrics->documents_removed.inc(_max_documents_removed);
+}
+
 bool
-GarbageCollectionOperation::shouldBlockThisOperation(uint32_t, uint8_t) const
-{
+GarbageCollectionOperation::shouldBlockThisOperation(uint32_t, uint8_t) const {
     return true;
 }
+
+}
diff --git a/storage/src/vespa/storage/distributor/operations/idealstate/garbagecollectionoperation.h b/storage/src/vespa/storage/distributor/operations/idealstate/garbagecollectionoperation.h
index 47ea11bb328..28de9592a63 100644
--- a/storage/src/vespa/storage/distributor/operations/idealstate/garbagecollectionoperation.h
+++ b/storage/src/vespa/storage/distributor/operations/idealstate/garbagecollectionoperation.h
@@ -26,8 +26,10 @@ protected:
     MessageTracker _tracker;
 private:
     std::vector<BucketCopy> _replica_info;
+    uint32_t _max_documents_removed;
 
     void merge_received_bucket_info_into_db();
+    void update_gc_metrics();
 };
 
 }
diff --git a/storage/src/vespa/storage/distributor/persistence_operation_metric_set.h b/storage/src/vespa/storage/distributor/persistence_operation_metric_set.h
index d951d7ceba2..1299fdad2ad 100644
--- a/storage/src/vespa/storage/distributor/persistence_operation_metric_set.h
+++ b/storage/src/vespa/storage/distributor/persistence_operation_metric_set.h
@@ -16,7 +16,7 @@ class PersistenceFailuresMetricSet : public metrics::MetricSet
 {
 public:
     explicit PersistenceFailuresMetricSet(metrics::MetricSet* owner);
-    ~PersistenceFailuresMetricSet();
+    ~PersistenceFailuresMetricSet() override;
 
     metrics::SumMetric<metrics::LongCountMetric> sum;
     metrics::LongCountMetric notready;
@@ -44,7 +44,7 @@ public:
     PersistenceFailuresMetricSet failures;
 
     PersistenceOperationMetricSet(const std::string& name, metrics::MetricSet* owner = nullptr);
-    ~PersistenceOperationMetricSet();
+    ~PersistenceOperationMetricSet() override;
 
     MetricSet * clone(std::vector<Metric::UP>& ownerList, CopyType copyType,
                       metrics::MetricSet* owner, bool includeUnused) const override;
diff --git a/storage/src/vespa/storage/distributor/persistencemessagetracker.cpp b/storage/src/vespa/storage/distributor/persistencemessagetracker.cpp
index 060eda2ffc5..d98a2ce6a74 100644
--- a/storage/src/vespa/storage/distributor/persistencemessagetracker.cpp
+++ b/storage/src/vespa/storage/distributor/persistencemessagetracker.cpp
@@ -20,33 +20,26 @@ PersistenceMessageTrackerImpl::PersistenceMessageTrackerImpl(
         api::Timestamp revertTimestamp)
     : MessageTracker(link.getClusterName()),
       _metric(metric),
-      _reply(reply),
+      _reply(std::move(reply)),
       _manager(link),
       _revertTimestamp(revertTimestamp),
       _requestTimer(link.getClock()),
-      _priority(reply->getPriority()),
+      _priority(_reply->getPriority()),
       _success(true)
 {
 }
 
-PersistenceMessageTrackerImpl::~PersistenceMessageTrackerImpl() {}
+PersistenceMessageTrackerImpl::~PersistenceMessageTrackerImpl() = default;
 
 void
 PersistenceMessageTrackerImpl::updateDB()
 {
-    for (BucketInfoMap::iterator iter = _bucketInfo.begin();
-         iter != _bucketInfo.end();
-         iter++)
-    {
-        _manager.updateBucketDatabase(iter->first, iter->second);
+    for (const auto & entry : _bucketInfo) {
+        _manager.updateBucketDatabase(entry.first, entry.second);
     }
 
-    for (BucketInfoMap::iterator iter = _remapBucketInfo.begin();
-         iter != _remapBucketInfo.end();
-         iter++)
-    {
-        _manager.updateBucketDatabase(iter->first, iter->second,
-                                      DatabaseUpdate::CREATE_IF_NONEXISTING);
+    for (const auto & entry :  _remapBucketInfo){
+        _manager.updateBucketDatabase(entry.first, entry.second,DatabaseUpdate::CREATE_IF_NONEXISTING);
     }
 }
 
@@ -94,11 +87,10 @@ PersistenceMessageTrackerImpl::revert(
         std::vector<api::Timestamp> reverts;
         reverts.push_back(_revertTimestamp);
 
-        for (uint32_t i = 0; i < revertNodes.size(); i++) {
-            std::shared_ptr<api::RevertCommand> toRevert(
-                    new api::RevertCommand(revertNodes[i].first, reverts));
+        for (const auto & revertNode : revertNodes) {
+            auto toRevert = std::make_shared<api::RevertCommand>(revertNode.first, reverts);
             toRevert->setPriority(_priority);
-            queueCommand(toRevert, revertNodes[i].second);
+            queueCommand(std::move(toRevert), revertNode.second);
         }
 
         flushQueue(sender);
@@ -107,14 +99,14 @@ PersistenceMessageTrackerImpl::revert(
 
 void
 PersistenceMessageTrackerImpl::queueMessageBatch(const std::vector<MessageTracker::ToSend>& messages) {
-    _messageBatches.push_back(MessageBatch());
-    for (uint32_t i = 0; i < messages.size(); i++) {
-        if (_reply.get()) {
-            messages[i]._msg->getTrace().setLevel(_reply->getTrace().getLevel());
+    _messageBatches.emplace_back();
+    for (const auto & message : messages) {
+        if (_reply) {
+            message._msg->getTrace().setLevel(_reply->getTrace().getLevel());
         }
 
-        _messageBatches.back().push_back(messages[i]._msg->getMsgId());
-        queueCommand(messages[i]._msg, messages[i]._target);
+        _messageBatches.back().push_back(message._msg->getMsgId());
+        queueCommand(message._msg, message._target);
     }
 }
 
@@ -134,13 +126,13 @@ PersistenceMessageTrackerImpl::canSendReplyEarly() const
         return false;
     }
 
-    for (uint32_t i = 0; i < _messageBatches.size(); i++) {
+    for (const MessageBatch & batch : _messageBatches) {
         uint32_t messagesDone = 0;
 
-        for (uint32_t j = 0; j < _messageBatches[i].size(); j++) {
-            if (_sentMessages.find(_messageBatches[i][j]) == _sentMessages.end()) {
+        for (uint32_t i = 0; i < batch.size(); i++) {
+            if (_sentMessages.find(batch[i]) == _sentMessages.end()) {
                 messagesDone++;
-            } else if (distribution.ensurePrimaryPersisted() && j == 0) {
+            } else if (distribution.ensurePrimaryPersisted() && i == 0) {
                 // Primary must always be written.
                 LOG(debug, "Not returning early because primary node wasn't done");
                 return false;
@@ -158,55 +150,6 @@ PersistenceMessageTrackerImpl::canSendReplyEarly() const
 }
 
 void
-PersistenceMessageTrackerImpl::checkCopiesDeleted()
-{
-    if (!_reply.get()) {
-        return;
-    }
-
-    // Don't check the buckets that have been remapped here, as we will
-    // create them.
-    const auto &bucketSpaceRepo(_manager.getBucketSpaceRepo());
-    for (BucketInfoMap::const_iterator iter = _bucketInfo.begin();
-         iter != _bucketInfo.end();
-         iter++)
-    {
-        const auto &bucketSpace(bucketSpaceRepo.get(iter->first.getBucketSpace()));
-        const auto &bucketDb(bucketSpace.getBucketDatabase());
-        BucketDatabase::Entry dbentry = bucketDb.get(iter->first.getBucketId());
-
-        if (!dbentry.valid()) {
-            continue;
-        }
-
-        std::vector<uint16_t> missing;
-        std::vector<uint16_t> total;
-
-        for (uint32_t i = 0; i < iter->second.size(); ++i) {
-            if (dbentry->getNode(iter->second[i].getNode()) == NULL) {
-                missing.push_back(iter->second[i].getNode());
-            }
-
-            total.push_back(iter->second[i].getNode());
-        }
-
-        if (!missing.empty()) {
-            std::ostringstream msg;
-            msg << iter->first.toString() << " was deleted from nodes ["
-                << commaSeparated(missing)
-                << "] after message was sent but before it was done. Sent to ["
-                << commaSeparated(total)
-                << "]";
-
-            LOG(debug, "%s", msg.str().c_str());
-            _reply->setResult(api::ReturnCode(api::ReturnCode::BUCKET_DELETED,
-                                              msg.str()));
-            break;
-        }
-    }
-}
-
-void
 PersistenceMessageTrackerImpl::addBucketInfoFromReply(
         uint16_t node,
         const api::BucketInfoReply& reply)
@@ -258,7 +201,7 @@ bool
 PersistenceMessageTrackerImpl::shouldRevert() const
 {
     return _manager.getDistributorConfig().enableRevert
-            && _revertNodes.size() && !_success && _reply.get();
+            &&  !_revertNodes.empty() && !_success && _reply;
 }
 
 void
@@ -266,7 +209,9 @@ PersistenceMessageTrackerImpl::sendReply(MessageSender& sender)
 {
     updateMetrics();
     _trace.setStrict(false);
-    _reply->getTrace().getRoot().addChild(_trace);
+    if ( ! _trace.isEmpty()) {
+        _reply->getTrace().getRoot().addChild(_trace);
+    }
     
     sender.sendReply(_reply);
     _reply = std::shared_ptr<api::BucketInfoReply>();
@@ -330,7 +275,9 @@ PersistenceMessageTrackerImpl::updateFromReply(
         api::BucketInfoReply& reply,
         uint16_t node)
 {
-    _trace.addChild(reply.getTrace().getRoot());
+    if ( ! reply.getTrace().getRoot().isEmpty()) {
+        _trace.addChild(reply.getTrace().getRoot());
+    }
 
     if (reply.getType() == api::MessageType::CREATEBUCKET_REPLY) {
         handleCreateBucketReply(reply, node);
@@ -341,7 +288,6 @@ PersistenceMessageTrackerImpl::updateFromReply(
     if (finished()) {
         bool doRevert(shouldRevert());
 
-        checkCopiesDeleted();
         updateDB();
 
         if (!hasSentReply()) {
@@ -352,7 +298,6 @@ PersistenceMessageTrackerImpl::updateFromReply(
         }
     } else if (canSendReplyEarly()) {
         LOG(debug, "Sending reply early because initial redundancy has been reached");
-        checkCopiesDeleted();
         sendReply(sender);
     }
 }
diff --git a/storage/src/vespa/storage/distributor/persistencemessagetracker.h b/storage/src/vespa/storage/distributor/persistencemessagetracker.h
index 2b4e7d8edc1..21977ddd881 100644
--- a/storage/src/vespa/storage/distributor/persistencemessagetracker.h
+++ b/storage/src/vespa/storage/distributor/persistencemessagetracker.h
@@ -84,13 +84,12 @@ private:
     bool hasSentReply() const { return _reply.get() == 0; }
     bool shouldRevert() const;
     void sendReply(MessageSender& sender);
-    void checkCopiesDeleted();
     void updateFailureResult(const api::BucketInfoReply& reply);
     void handleCreateBucketReply(api::BucketInfoReply& reply, uint16_t node);
     void handlePersistenceReply(api::BucketInfoReply& reply, uint16_t node);
 
     void queueCommand(std::shared_ptr<api::BucketCommand> msg, uint16_t target) override {
-        MessageTracker::queueCommand(msg, target);
+        MessageTracker::queueCommand(std::move(msg), target);
     }
     void flushQueue(MessageSender& s) override { MessageTracker::flushQueue(s); }
     uint16_t handleReply(api::BucketReply& r) override { return MessageTracker::handleReply(r); }
diff --git a/storage/src/vespa/storage/distributor/sentmessagemap.h b/storage/src/vespa/storage/distributor/sentmessagemap.h
index 3364c6606eb..405e59f7b31 100644
--- a/storage/src/vespa/storage/distributor/sentmessagemap.h
+++ b/storage/src/vespa/storage/distributor/sentmessagemap.h
@@ -16,26 +16,18 @@ class SentMessageMap
 {
 public:
     SentMessageMap();
-
     ~SentMessageMap();
 
     std::shared_ptr<Operation> pop(api::StorageMessage::Id id);
-
     std::shared_ptr<Operation> pop();
 
     void insert(api::StorageMessage::Id id, const std::shared_ptr<Operation> & msg);
-
     void clear();
-
     uint32_t size() const { return _map.size(); }
-
-    uint32_t empty() const { return _map.empty(); }
-
+    [[nodiscard]] bool empty() const noexcept { return _map.empty(); }
     std::string toString() const;
-
 private:
-    typedef std::map<api::StorageMessage::Id, std::shared_ptr<Operation> > Map;
-
+    using Map = std::map<api::StorageMessage::Id, std::shared_ptr<Operation>>;
     Map _map;
 };
 
diff --git a/storage/src/vespa/storage/persistence/bucketprocessor.cpp b/storage/src/vespa/storage/persistence/bucketprocessor.cpp
index d4a570ee062..d6c549ef6f4 100644
--- a/storage/src/vespa/storage/persistence/bucketprocessor.cpp
+++ b/storage/src/vespa/storage/persistence/bucketprocessor.cpp
@@ -4,6 +4,7 @@
 #include <vespa/document/fieldset/fieldsets.h>
 #include <vespa/vespalib/stllike/asciistream.h>
 #include <cassert>
+#include <stdexcept>
 
 namespace storage {
 
diff --git a/storage/src/vespa/storage/persistence/filestorage/filestorhandler.cpp b/storage/src/vespa/storage/persistence/filestorage/filestorhandler.cpp
index 350cdad791c..543accc80ae 100644
--- a/storage/src/vespa/storage/persistence/filestorage/filestorhandler.cpp
+++ b/storage/src/vespa/storage/persistence/filestorage/filestorhandler.cpp
@@ -6,22 +6,18 @@ namespace storage {
 
 FileStorHandler::FileStorHandler(MessageSender& sender, FileStorMetrics& metrics,
                                  const spi::PartitionStateList& partitions, ServiceLayerComponentRegister& compReg)
-        : _impl(new FileStorHandlerImpl(1, 1, sender, metrics, partitions, compReg))
+        : _impl(std::make_unique<FileStorHandlerImpl>(1, 1, sender, metrics, partitions, compReg))
 {
 }
 
 
 FileStorHandler::FileStorHandler(uint32_t numThreads, uint32_t numStripes, MessageSender& sender, FileStorMetrics& metrics,
                                  const spi::PartitionStateList& partitions, ServiceLayerComponentRegister& compReg)
-    : _impl(new FileStorHandlerImpl(numThreads, numStripes, sender, metrics, partitions, compReg))
+    : _impl(std::make_unique<FileStorHandlerImpl>(numThreads, numStripes, sender, metrics, partitions, compReg))
 {
 }
 
-FileStorHandler::~FileStorHandler()
-{
-    delete _impl;
-}
-
+FileStorHandler::~FileStorHandler() = default;
 void
 FileStorHandler::flush(bool flushMerges)
 {
diff --git a/storage/src/vespa/storage/persistence/filestorage/filestorhandler.h b/storage/src/vespa/storage/persistence/filestorage/filestorhandler.h
index ab3d03a5e9a..db294d7c39f 100644
--- a/storage/src/vespa/storage/persistence/filestorage/filestorhandler.h
+++ b/storage/src/vespa/storage/persistence/filestorage/filestorhandler.h
@@ -35,7 +35,6 @@ namespace framework {
 class FileStorHandlerImpl;
 struct FileStorMetrics;
 struct MessageSender;
-class MountPointList;
 struct ServiceLayerComponentRegister;
 class AbortBucketOperationsCommand;
 
@@ -259,7 +258,7 @@ public:
     std::string dumpQueue(uint16_t disk) const;
 
 private:
-    FileStorHandlerImpl* _impl;
+    std::unique_ptr<FileStorHandlerImpl> _impl;
 };
 
 } // storage
diff --git a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp
index f773ee774bb..bfd5233d017 100644
--- a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp
+++ b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp
@@ -24,15 +24,14 @@ namespace storage {
 
 namespace {
 
-uint32_t merge_soft_limit_from_thread_count(uint32_t num_threads) noexcept {
+uint32_t per_stripe_merge_limit(uint32_t num_threads, uint32_t num_stripes) noexcept {
     // Rationale: to avoid starving client ops we want to ensure that not all persistence
-    // threads can be blocked by processing merges all at the same time. We therefore allocate
-    // half of the threads to non-merge operations.
-    // This a _soft_ limit since the current operation locking design means there is a small
-    // window of time between when the limit is checked and when its updated. There are no
-    // correctness violations as a consequence of this, but non-merge liveness may be impacted.
-    // There must always be at least 1 thread that can process merges, or the system would stall.
-    return std::max(1u, num_threads / 2);
+    // threads in any given stripe can be blocked by processing merges all at the same time.
+    // We therefore allocate half of the per-stripe threads to non-merge operations.
+    // Note that if the _total_ number of threads is small and odd (e.g. 3 or 5), it's still
+    // possible to have a stripe where all threads are busy processing merges because there
+    // is only 1 thread in the stripe in total.
+    return std::max(1u, (num_threads / num_stripes) / 2);
 }
 
 }
@@ -46,8 +45,7 @@ FileStorHandlerImpl::FileStorHandlerImpl(uint32_t numThreads, uint32_t numStripe
       _messageSender(sender),
       _bucketIdFactory(_component.getBucketIdFactory()),
       _getNextMessageTimeout(100),
-      _activeMergesSoftLimit(merge_soft_limit_from_thread_count(numThreads)),
-      _activeMerges(0),
+      _max_active_merges_per_stripe(per_stripe_merge_limit(numThreads, numStripes)),
       _paused(false)
 {
     _diskInfo.reserve(_component.getDiskCount());
@@ -56,7 +54,7 @@ FileStorHandlerImpl::FileStorHandlerImpl(uint32_t numThreads, uint32_t numStripe
     }
     for (uint32_t i=0; i<_diskInfo.size(); ++i) {
         _diskInfo[i].metrics = metrics.disks[i].get();
-        assert(_diskInfo[i].metrics != 0);
+        assert(_diskInfo[i].metrics != nullptr);
         uint32_t j(0);
         for (Stripe & stripe : _diskInfo[i].getStripes()) {
             stripe.setMetrics(metrics.disks[i]->stripes[j++].get());
@@ -401,13 +399,30 @@ FileStorHandlerImpl::Stripe::lock(const document::Bucket &bucket, api::LockingRe
 
 namespace {
     struct MultiLockGuard {
-        std::map<uint16_t, vespalib::Monitor*> monitors;
+        struct DiskAndStripe {
+            uint16_t disk;
+            uint16_t stripe;
+
+            DiskAndStripe(uint16_t disk_, uint16_t stripe_) noexcept : disk(disk_), stripe(stripe_) {}
+
+            bool operator==(const DiskAndStripe& rhs) const noexcept {
+                return (disk == rhs.disk) && (stripe == rhs.stripe);
+            }
+            bool operator<(const DiskAndStripe& rhs) const noexcept {
+                if (disk != rhs.disk) {
+                    return disk < rhs.disk;
+                }
+                return stripe < rhs.stripe;
+            }
+        };
+
+        std::map<DiskAndStripe, vespalib::Monitor*> monitors;
         std::vector<std::shared_ptr<vespalib::MonitorGuard>> guards;
 
         MultiLockGuard() = default;
 
-        void addLock(vespalib::Monitor& monitor, uint16_t index) {
-            monitors[index] = &monitor;
+        void addLock(vespalib::Monitor& monitor, uint16_t disk_index, uint16_t stripe_index) {
+            monitors[DiskAndStripe(disk_index, stripe_index)] = &monitor;
         }
         void lock() {
             for (auto & entry : monitors) {
@@ -760,6 +775,9 @@ FileStorHandlerImpl::remapQueueNoLock(Disk& from, const RemapInfo& source,
         } else {
             entry._bucket = bucket;
             // Move to correct disk queue if needed
+            assert(bucket == source.bucket || std::find_if(targets.begin(), targets.end(), [bucket](auto* e){
+                return e->bucket == bucket;
+            }) != targets.end());
             _diskInfo[targetDisk].stripe(bucket).exposeQueue().emplace_back(std::move(entry));
         }
     }
@@ -774,11 +792,11 @@ FileStorHandlerImpl::remapQueue(const RemapInfo& source, RemapInfo& target, Oper
     MultiLockGuard guard;
 
     Disk& from(_diskInfo[source.diskIndex]);
-    guard.addLock(from.stripe(source.bucket).exposeLock(), source.diskIndex);
+    guard.addLock(from.stripe(source.bucket).exposeLock(), source.diskIndex, from.stripe_index(source.bucket));
 
     Disk& to1(_diskInfo[target.diskIndex]);
     if (target.bucket.getBucketId().getRawId() != 0) {
-        guard.addLock(to1.stripe(target.bucket).exposeLock(), target.diskIndex);
+        guard.addLock(to1.stripe(target.bucket).exposeLock(), target.diskIndex, to1.stripe_index(target.bucket));
     }
 
     std::vector<RemapInfo*> targets;
@@ -797,16 +815,16 @@ FileStorHandlerImpl::remapQueue(const RemapInfo& source, RemapInfo& target1, Rem
     MultiLockGuard guard;
 
     Disk& from(_diskInfo[source.diskIndex]);
-    guard.addLock(from.stripe(source.bucket).exposeLock(), source.diskIndex);
+    guard.addLock(from.stripe(source.bucket).exposeLock(), source.diskIndex, from.stripe_index(source.bucket));
 
     Disk& to1(_diskInfo[target1.diskIndex]);
     if (target1.bucket.getBucketId().getRawId() != 0) {
-        guard.addLock(to1.stripe(target1.bucket).exposeLock(), target1.diskIndex);
+        guard.addLock(to1.stripe(target1.bucket).exposeLock(), target1.diskIndex, to1.stripe_index(target1.bucket));
     }
 
     Disk& to2(_diskInfo[target2.diskIndex]);
     if (target2.bucket.getBucketId().getRawId() != 0) {
-        guard.addLock(to2.stripe(target2.bucket).exposeLock(), target2.diskIndex);
+        guard.addLock(to2.stripe(target2.bucket).exposeLock(), target2.diskIndex, to2.stripe_index(target2.bucket));
     }
 
     guard.lock();
@@ -879,15 +897,15 @@ FileStorHandlerImpl::MessageEntry::MessageEntry(MessageEntry && entry) noexcept
       _priority(entry._priority)
 { }
 
-FileStorHandlerImpl::MessageEntry::~MessageEntry() { }
+FileStorHandlerImpl::MessageEntry::~MessageEntry() = default;
 
-FileStorHandlerImpl::Disk::Disk(const FileStorHandlerImpl & owner, MessageSender & messageSender, uint32_t numThreads)
+FileStorHandlerImpl::Disk::Disk(const FileStorHandlerImpl & owner, MessageSender & messageSender, uint32_t numStripes)
     : metrics(0),
       _nextStripeId(0),
-      _stripes(numThreads, Stripe(owner, messageSender)),
+      _stripes(numStripes, Stripe(owner, messageSender)),
       state(FileStorHandler::AVAILABLE)
 {
-    assert(numThreads > 0);
+    assert(numStripes > 0);
 }
 
 FileStorHandlerImpl::Disk::Disk(Disk && rhs) noexcept
@@ -930,8 +948,10 @@ FileStorHandlerImpl::Disk::schedule(const std::shared_ptr<api::StorageMessage>&
 
 FileStorHandlerImpl::Stripe::Stripe(const FileStorHandlerImpl & owner, MessageSender & messageSender)
     : _owner(owner),
-      _messageSender(messageSender)
-{ }
+      _messageSender(messageSender),
+      _active_merges(0)
+{}
+
 FileStorHandler::LockedMessage
 FileStorHandlerImpl::Stripe::getNextMessage(uint32_t timeout, Disk & disk)
 {
@@ -1112,6 +1132,23 @@ FileStorHandlerImpl::Stripe::flush()
     }
 }
 
+namespace {
+
+bool message_type_is_merge_related(api::MessageType::Id msg_type_id) {
+    switch (msg_type_id) {
+    case api::MessageType::MERGEBUCKET_ID:
+    case api::MessageType::MERGEBUCKET_REPLY_ID:
+    case api::MessageType::GETBUCKETDIFF_ID:
+    case api::MessageType::GETBUCKETDIFF_REPLY_ID:
+    case api::MessageType::APPLYBUCKETDIFF_ID:
+    case api::MessageType::APPLYBUCKETDIFF_REPLY_ID:
+        return true;
+    default: return false;
+    }
+}
+
+}
+
 void FileStorHandlerImpl::Stripe::release(const document::Bucket & bucket,
                                           api::LockingRequirements reqOfReleasedLock,
                                           api::StorageMessage::Id lockMsgId) {
@@ -1123,9 +1160,9 @@ void FileStorHandlerImpl::Stripe::release(const document::Bucket & bucket,
     if (reqOfReleasedLock == api::LockingRequirements::Exclusive) {
         assert(entry._exclusiveLock);
         assert(entry._exclusiveLock->msgId == lockMsgId);
-        if (entry._exclusiveLock->msgType == api::MessageType::MERGEBUCKET_ID) {
-            auto before = _owner._activeMerges.fetch_sub(1, std::memory_order_relaxed);
-            assert(before > 0);
+        if (message_type_is_merge_related(entry._exclusiveLock->msgType)) {
+            assert(_active_merges > 0);
+            --_active_merges;
         }
         entry._exclusiveLock.reset();
     } else {
@@ -1147,8 +1184,8 @@ void FileStorHandlerImpl::Stripe::lock(const vespalib::MonitorGuard &, const doc
     assert(!entry._exclusiveLock);
     if (lockReq == api::LockingRequirements::Exclusive) {
         assert(entry._sharedLocks.empty());
-        if (lockEntry.msgType == api::MessageType::MERGEBUCKET_ID) {
-            _owner._activeMerges.fetch_add(1, std::memory_order_relaxed);
+        if (message_type_is_merge_related(lockEntry.msgType)) {
+            ++_active_merges;
         }
         entry._exclusiveLock = lockEntry;
     } else {
@@ -1183,8 +1220,8 @@ bool
 FileStorHandlerImpl::Stripe::operationIsInhibited(const vespalib::MonitorGuard& guard, const document::Bucket& bucket,
                                                   const api::StorageMessage& msg) const noexcept
 {
-    if ((msg.getType() == api::MessageType::MERGEBUCKET)
-        && (_owner._activeMerges.load(std::memory_order_relaxed) > _owner._activeMergesSoftLimit))
+    if (message_type_is_merge_related(msg.getType().getId())
+        && (_active_merges >= _owner._max_active_merges_per_stripe))
     {
         return true;
     }
diff --git a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.h b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.h
index 5fc592e11cb..da4d242a4c9 100644
--- a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.h
+++ b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.h
@@ -144,12 +144,13 @@ public:
         FileStorHandler::LockedMessage getMessage(vespalib::MonitorGuard & guard, PriorityIdx & idx,
                                                   PriorityIdx::iterator iter);
         using LockedBuckets = vespalib::hash_map<document::Bucket, MultiLockEntry, document::Bucket::hash>;
-        const FileStorHandlerImpl  &_owner;
-        MessageSender              &_messageSender;
-        FileStorStripeMetrics      *_metrics;
-        vespalib::Monitor           _lock;
-        PriorityQueue               _queue;
-        LockedBuckets               _lockedBuckets;
+        const FileStorHandlerImpl &_owner;
+        MessageSender             &_messageSender;
+        FileStorStripeMetrics     *_metrics;
+        vespalib::Monitor          _lock;
+        PriorityQueue             _queue;
+        LockedBuckets             _lockedBuckets;
+        uint32_t                  _active_merges;
     };
     struct Disk {
         FileStorDiskMetrics * metrics;
@@ -169,7 +170,7 @@ public:
             state.store(s, std::memory_order_relaxed);
         }
 
-        Disk(const FileStorHandlerImpl & owner, MessageSender & messageSender, uint32_t numThreads);
+        Disk(const FileStorHandlerImpl & owner, MessageSender & messageSender, uint32_t numStripes);
         Disk(Disk &&) noexcept;
         ~Disk();
 
@@ -206,8 +207,12 @@ public:
             // and the stripe an operation ends up on.
             return bucket.getBucketId().getId() * 1099511628211ULL;
         }
+        // We make a fairly reasonable assumption that there will be less than 64k stripes.
+        uint16_t stripe_index(const document::Bucket& bucket) const noexcept {
+            return static_cast<uint16_t>(dispersed_bucket_bits(bucket) % _stripes.size());
+        }
         Stripe & stripe(const document::Bucket & bucket) {
-            return _stripes[dispersed_bucket_bits(bucket) % _stripes.size()];
+            return _stripes[stripe_index(bucket)];
         }
         std::vector<Stripe> & getStripes() { return _stripes; }
     private:
@@ -222,7 +227,7 @@ public:
         BucketLock(const vespalib::MonitorGuard & guard, Stripe& disk, const document::Bucket &bucket,
                    uint8_t priority, api::MessageType::Id msgType, api::StorageMessage::Id,
                    api::LockingRequirements lockReq);
-        ~BucketLock();
+        ~BucketLock() override;
 
         const document::Bucket &getBucket() const override { return _bucket; }
         api::LockingRequirements lockingRequirements() const noexcept override { return _lockReq; }
@@ -287,20 +292,15 @@ public:
 
 private:
     ServiceLayerComponent _component;
-    std::vector<Disk> _diskInfo;
-    MessageSender& _messageSender;
+    std::vector<Disk>     _diskInfo;
+    MessageSender&        _messageSender;
     const document::BucketIdFactory& _bucketIdFactory;
-
-    vespalib::Lock _mergeStatesLock;
-
+    vespalib::Lock        _mergeStatesLock;
     std::map<document::Bucket, MergeStatus::SP> _mergeStates;
-
-    uint32_t _getNextMessageTimeout;
-
-    uint32_t _activeMergesSoftLimit;
-    mutable std::atomic<uint32_t> _activeMerges;
-    vespalib::Monitor _pauseMonitor;
-    std::atomic<bool> _paused;
+    uint32_t              _getNextMessageTimeout;
+    const uint32_t        _max_active_merges_per_stripe; // Read concurrently by stripes.
+    vespalib::Monitor     _pauseMonitor;
+    std::atomic<bool>     _paused;
 
     void reply(api::StorageMessage&, DiskState state) const;
 
diff --git a/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp b/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp
index baa6523cbb2..ea6f5e03a80 100644
--- a/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp
+++ b/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp
@@ -38,11 +38,11 @@ FileStorManager(const config::ConfigUri & configUri, const spi::PartitionStateLi
       _bucketIdFactory(_component.getBucketIdFactory()),
       _configUri(configUri),
       _disks(),
-      _bucketOwnershipNotifier(new BucketOwnershipNotifier(_component, *this)),
+      _bucketOwnershipNotifier(std::make_unique<BucketOwnershipNotifier>(_component, *this)),
       _configFetcher(_configUri.getContext()),
       _threadLockCheckInterval(60),
       _failDiskOnError(false),
-      _metrics(new FileStorMetrics(_component.getLoadTypes()->getMetricLoadTypes())),
+      _metrics(std::make_unique<FileStorMetrics>(_component.getLoadTypes()->getMetricLoadTypes())),
       _threadMonitor(),
       _closed(false)
 {
@@ -105,10 +105,10 @@ FileStorManager::configure(std::unique_ptr<vespa::config::content::StorFilestorC
         _config = std::move(config);
         _disks.resize(_component.getDiskCount());
         size_t numThreads = _config->numThreads;
-        size_t numStripes = std::min(2ul, numThreads);
+        size_t numStripes = std::max(size_t(1u), numThreads / 2);
         _metrics->initDiskMetrics(_disks.size(), _component.getLoadTypes()->getMetricLoadTypes(), numStripes, numThreads);
 
-        _filestorHandler.reset(new FileStorHandler(numThreads, numStripes, *this, *_metrics, _partitions, _compReg));
+        _filestorHandler = std::make_unique<FileStorHandler>(numThreads, numStripes, *this, *_metrics, _partitions, _compReg);
         for (uint32_t i=0; i<_component.getDiskCount(); ++i) {
             if (_partitions[i].isUp()) {
                 LOG(spam, "Setting up disk %u", i);
@@ -849,12 +849,6 @@ FileStorManager::reportHtmlStatus(std::ostream& out, const framework::HttpUrlPat
     _filestorHandler->getStatus(out, path);
 }
 
-bool
-FileStorManager::isMerging(const document::Bucket& bucket) const
-{
-    return _filestorHandler->isMerging(bucket);
-}
-
 namespace {
     struct Deactivator {
         StorBucketDatabase::Decision operator()(document::BucketId::Type, StorBucketDatabase::Entry& data)
diff --git a/storage/src/vespa/storage/persistence/filestorage/filestormanager.h b/storage/src/vespa/storage/persistence/filestorage/filestormanager.h
index 65d4035a3dd..433b9ddbd39 100644
--- a/storage/src/vespa/storage/persistence/filestorage/filestormanager.h
+++ b/storage/src/vespa/storage/persistence/filestorage/filestormanager.h
@@ -45,65 +45,43 @@ class FileStorManager : public StorageLinkQueued,
                         public framework::HtmlStatusReporter,
                         public StateListener,
                         private config::IFetcherCallback<vespa::config::content::StorFilestorConfig>,
-                        private MessageSender
+                        public MessageSender
 {
-    ServiceLayerComponentRegister& _compReg;
-    ServiceLayerComponent _component;
-    const spi::PartitionStateList& _partitions;
-    spi::PersistenceProvider& _providerCore;
-    ProviderErrorWrapper _providerErrorWrapper;
-    spi::PersistenceProvider* _provider;
+    ServiceLayerComponentRegister & _compReg;
+    ServiceLayerComponent          _component;
+    const spi::PartitionStateList & _partitions;
+    spi::PersistenceProvider      & _providerCore;
+    ProviderErrorWrapper            _providerErrorWrapper;
+    spi::PersistenceProvider      * _provider;
     
     const document::BucketIdFactory& _bucketIdFactory;
-    config::ConfigUri _configUri;
+    config::ConfigUri                _configUri;
 
     typedef std::vector<DiskThread::SP> DiskThreads;
-    std::vector<DiskThreads> _disks;
+    std::vector<DiskThreads>                 _disks;
     std::unique_ptr<BucketOwnershipNotifier> _bucketOwnershipNotifier;
 
     std::unique_ptr<vespa::config::content::StorFilestorConfig> _config;
     config::ConfigFetcher _configFetcher;
-    uint32_t _threadLockCheckInterval; // In seconds
-    bool _failDiskOnError;
-    int _killSignal;
+    uint32_t              _threadLockCheckInterval; // In seconds
+    bool                  _failDiskOnError;
     std::shared_ptr<FileStorMetrics> _metrics;
     std::unique_ptr<FileStorHandler> _filestorHandler;
-    lib::ClusterState _lastState;
 
-    struct ReplyHolder {
-        int refCount;
-        std::unique_ptr<api::StorageReply> reply;
-
-        ReplyHolder(int rc, std::unique_ptr<api::StorageReply> r)
-            : refCount(rc), reply(std::move(r)) {};
-    };
-
-    std::map<api::StorageMessage::Id,
-             std::shared_ptr<ReplyHolder> > _splitMessages;
-    vespalib::Lock _splitLock;
     mutable vespalib::Monitor _threadMonitor; // Notify to stop sleeping
-    bool _closed;
-
-    FileStorManager(const FileStorManager &);
-    FileStorManager& operator=(const FileStorManager &);
+    bool                      _closed;
 
-    std::vector<DiskThreads> getThreads() { return _disks; }
-
-    friend class BucketMergeTest;
     friend struct FileStorManagerTest;
-    friend class MessageTest;
 
 public:
-    explicit FileStorManager(const config::ConfigUri &,
-                             const spi::PartitionStateList&,
-                             spi::PersistenceProvider&,
-                             ServiceLayerComponentRegister&);
-    ~FileStorManager();
+    FileStorManager(const config::ConfigUri &, const spi::PartitionStateList&,
+                    spi::PersistenceProvider&, ServiceLayerComponentRegister&);
+    FileStorManager(const FileStorManager &) = delete;
+    FileStorManager& operator=(const FileStorManager &) = delete;
 
-    void print(std::ostream& out, bool verbose, const std::string& indent) const override;
+    ~FileStorManager() override;
 
-    // Return true if we are currently merging the given bucket.
-    bool isMerging(const document::Bucket& bucket) const;
+    void print(std::ostream& out, bool verbose, const std::string& indent) const override;
 
     FileStorHandler& getFileStorHandler() {
         return *_filestorHandler;
diff --git a/storage/src/vespa/storage/persistence/persistencethread.cpp b/storage/src/vespa/storage/persistence/persistencethread.cpp
index 7d0ce26b83d..4bcd92293d3 100644
--- a/storage/src/vespa/storage/persistence/persistencethread.cpp
+++ b/storage/src/vespa/storage/persistence/persistencethread.cpp
@@ -36,8 +36,8 @@ PersistenceThread::PersistenceThread(ServiceLayerComponentRegister& compReg,
 {
     std::ostringstream threadName;
     threadName << "Disk " << _env._partition << " thread " << _stripeId;
-    _component.reset(new ServiceLayerComponent(compReg, threadName.str()));
-    _bucketOwnershipNotifier.reset(new BucketOwnershipNotifier(*_component, filestorHandler));
+    _component = std::make_unique<ServiceLayerComponent>(compReg, threadName.str());
+    _bucketOwnershipNotifier = std::make_unique<BucketOwnershipNotifier>(*_component, filestorHandler);
     framework::MilliSecTime maxProcessingTime(60 * 1000);
     framework::MilliSecTime waitTime(1000);
     _thread = _component->startThread(*this, maxProcessingTime, waitTime);
@@ -473,11 +473,10 @@ PersistenceThread::handleSplitBucket(api::SplitBucketCommand& cmd)
         const document::Bucket &target(i == 0 ? target1 : target2);
         uint16_t disk(i == 0 ? lock1.disk : lock2.disk);
         assert(target.getBucketId().getRawId() != 0);
-        targets.push_back(TargetInfo(
-                _env.getBucketDatabase(target.getBucketSpace()).get(
+        targets.emplace_back(_env.getBucketDatabase(target.getBucketSpace()).get(
                     target.getBucketId(), "PersistenceThread::handleSplitBucket - Target",
                     StorBucketDatabase::CREATE_IF_NONEXISTING),
-                FileStorHandler::RemapInfo(target, disk)));
+                FileStorHandler::RemapInfo(target, disk));
         targets.back().first->setBucketInfo(_env.getBucketInfo(target, disk));
         targets.back().first->disk = disk;
     }
@@ -795,8 +794,8 @@ PersistenceThread::handleCommand(api::StorageCommand& msg)
 {
     _context = spi::Context(msg.getLoadType(), msg.getPriority(), msg.getTrace().getLevel());
     MessageTracker::UP mtracker(handleCommandSplitByType(msg));
-    if (mtracker.get() != 0) {
-        if (mtracker->getReply().get() != 0) {
+    if (mtracker && ! _context.getTrace().getRoot().isEmpty()) {
+        if (mtracker->getReply()) {
             mtracker->getReply()->getTrace().getRoot().addChild(_context.getTrace().getRoot());
         } else {
             msg.getTrace().getRoot().addChild(_context.getTrace().getRoot());
@@ -844,11 +843,11 @@ PersistenceThread::processMessage(api::StorageMessage& msg)
             LOG(debug, "Handling command: %s", msg.toString().c_str());
             LOG(spam, "Message content: %s", msg.toString(true).c_str());
             auto tracker(handleCommand(initiatingCommand));
-            if (!tracker.get()) {
+            if (!tracker) {
                 LOG(debug, "Received unsupported command %s", msg.getType().getName().c_str());
             } else {
                 tracker->generateReply(initiatingCommand);
-                if ((tracker->getReply().get()
+                if ((tracker->getReply()
                      && tracker->getReply()->getResult().failed())
                     || tracker->getResult().failed())
                 {
diff --git a/storage/src/vespa/storage/persistence/processallhandler.cpp b/storage/src/vespa/storage/persistence/processallhandler.cpp
index 8c951a9f50d..5b94a3da027 100644
--- a/storage/src/vespa/storage/persistence/processallhandler.cpp
+++ b/storage/src/vespa/storage/persistence/processallhandler.cpp
@@ -23,6 +23,7 @@ public:
     spi::PersistenceProvider& _provider;
     const spi::Bucket& _bucket;
     spi::Context& _context;
+    uint32_t _n_removed;
 
     UnrevertableRemoveEntryProcessor(
             spi::PersistenceProvider& provider,
@@ -30,7 +31,9 @@ public:
             spi::Context& context)
         : _provider(provider),
           _bucket(bucket),
-          _context(context) {}
+          _context(context),
+          _n_removed(0)
+    {}
 
     void process(spi::DocEntry& entry) override {
         spi::RemoveResult removeResult = _provider.remove(
@@ -45,13 +48,14 @@ public:
                << removeResult.getErrorMessage();
             throw std::runtime_error(ss.str());
         }
+        ++_n_removed;
     }
 };
 
 class StatEntryProcessor : public BucketProcessor::EntryProcessor {
 public:
     std::ostream& ost;
-    StatEntryProcessor(std::ostream& o)
+    explicit StatEntryProcessor(std::ostream& o)
         : ost(o) {};
 
     void process(spi::DocEntry& e) override {
@@ -97,7 +101,9 @@ ProcessAllHandler::handleRemoveLocation(api::RemoveLocationCommand& cmd,
                                 context);
     spi::Result result = _spi.flush(bucket, context);
     uint32_t code = _env.convertErrorCode(result);
-    if (code != 0) {
+    if (code == 0) {
+        tracker->setReply(std::make_shared<api::RemoveLocationReply>(cmd, processor._n_removed));
+    } else {
         tracker->fail(code, result.getErrorMessage());
     }
 
diff --git a/storage/src/vespa/storage/storageserver/CMakeLists.txt b/storage/src/vespa/storage/storageserver/CMakeLists.txt
index 73873e78032..1aafe58af6c 100644
--- a/storage/src/vespa/storage/storageserver/CMakeLists.txt
+++ b/storage/src/vespa/storage/storageserver/CMakeLists.txt
@@ -7,11 +7,12 @@ vespa_add_library(storage_storageserver
     changedbucketownershiphandler.cpp
     communicationmanager.cpp
     communicationmanagermetrics.cpp
-    configurable_bucket_resolver.cpp
     config_logging.cpp
+    configurable_bucket_resolver.cpp
     distributornode.cpp
     distributornodecontext.cpp
     documentapiconverter.cpp
+    fnet_metrics_wrapper.cpp
     fnetlistener.cpp
     mergethrottler.cpp
     messagesink.cpp
diff --git a/storage/src/vespa/storage/storageserver/communicationmanager.cpp b/storage/src/vespa/storage/storageserver/communicationmanager.cpp
index 978d434847e..fa2b0cda018 100644
--- a/storage/src/vespa/storage/storageserver/communicationmanager.cpp
+++ b/storage/src/vespa/storage/storageserver/communicationmanager.cpp
@@ -415,6 +415,7 @@ void CommunicationManager::configure(std::unique_ptr<CommunicationManagerConfig>
         params.setNumThreads(std::max(1, config->mbus.numThreads));
         params.setDispatchOnDecode(config->mbus.dispatchOnDecode);
         params.setDispatchOnEncode(config->mbus.dispatchOnEncode);
+        params.setTcpNoDelay(config->mbus.optimizeFor == CommunicationManagerConfig::Mbus::OptimizeFor::LATENCY);
 
         params.setIdentity(mbus::Identity(_component.getIdentity()));
         if (config->mbusport != -1) {
diff --git a/storage/src/vespa/storage/storageserver/fnet_metrics_wrapper.cpp b/storage/src/vespa/storage/storageserver/fnet_metrics_wrapper.cpp
new file mode 100644
index 00000000000..476ececd078
--- /dev/null
+++ b/storage/src/vespa/storage/storageserver/fnet_metrics_wrapper.cpp
@@ -0,0 +1,21 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "fnet_metrics_wrapper.h"
+
+namespace storage {
+
+FnetMetricsWrapper::FnetMetricsWrapper(metrics::MetricSet* owner)
+    : metrics::MetricSet("fnet", {}, "transport layer metrics", owner),
+      _num_connections("num-connections", {}, "total number of connection objects", this)
+{
+}
+
+FnetMetricsWrapper::~FnetMetricsWrapper() = default;
+
+void
+FnetMetricsWrapper::update_metrics()
+{
+    _num_connections.set(FNET_Connection::get_num_connections());
+}
+
+}
diff --git a/storage/src/vespa/storage/storageserver/fnet_metrics_wrapper.h b/storage/src/vespa/storage/storageserver/fnet_metrics_wrapper.h
new file mode 100644
index 00000000000..dacb9e9f52c
--- /dev/null
+++ b/storage/src/vespa/storage/storageserver/fnet_metrics_wrapper.h
@@ -0,0 +1,22 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/metrics/metrics.h>
+#include <vespa/fnet/connection.h>
+
+namespace storage {
+
+// Simple wrapper around low-level fnet network metrics
+class FnetMetricsWrapper : public metrics::MetricSet
+{
+private:
+    metrics::LongValueMetric _num_connections;
+
+public:
+    explicit FnetMetricsWrapper(metrics::MetricSet* owner);
+    ~FnetMetricsWrapper() override;
+    void update_metrics();
+};
+
+}
diff --git a/storage/src/vespa/storage/storageserver/storagemetricsset.cpp b/storage/src/vespa/storage/storageserver/storagemetricsset.cpp
index f3240f0663b..6ed5c1ab650 100644
--- a/storage/src/vespa/storage/storageserver/storagemetricsset.cpp
+++ b/storage/src/vespa/storage/storageserver/storagemetricsset.cpp
@@ -23,7 +23,8 @@ StorageMetricSet::StorageMetricSet()
       memoryUse_messages(this),
       memoryUse_visiting("memoryusage_visiting", {{"memory"}},
             "Message use from visiting", this),
-      tls_metrics(this)
+      tls_metrics(this),
+      fnet_metrics(this)
 {}
 
 StorageMetricSet::~StorageMetricSet() = default;
@@ -35,6 +36,7 @@ void StorageMetricSet::updateMetrics() {
     // be erased from history. This will no longer be a problem once we move to a
     // metrics system built around absolute (rather than derived) values.
     tls_metrics.update_metrics_with_snapshot_delta();
+    fnet_metrics.update_metrics();
 }
 
 } // storage
diff --git a/storage/src/vespa/storage/storageserver/storagemetricsset.h b/storage/src/vespa/storage/storageserver/storagemetricsset.h
index 49795c63324..a315e974c01 100644
--- a/storage/src/vespa/storage/storageserver/storagemetricsset.h
+++ b/storage/src/vespa/storage/storageserver/storagemetricsset.h
@@ -3,6 +3,7 @@
 #pragma once
 
 #include "tls_statistics_metrics_wrapper.h"
+#include "fnet_metrics_wrapper.h"
 #include <vespa/metrics/metrics.h>
 
 namespace storage {
@@ -27,6 +28,7 @@ struct StorageMetricSet : public metrics::MetricSet
     metrics::LongValueMetric memoryUse_visiting;
 
     TlsStatisticsMetricsWrapper tls_metrics;
+    FnetMetricsWrapper fnet_metrics;
 
     StorageMetricSet();
     ~StorageMetricSet() override;
diff --git a/storage/src/vespa/storage/tools/analyzedistribution.cpp b/storage/src/vespa/storage/tools/analyzedistribution.cpp
index 52c26c4bb17..472aa63fff6 100644
--- a/storage/src/vespa/storage/tools/analyzedistribution.cpp
+++ b/storage/src/vespa/storage/tools/analyzedistribution.cpp
@@ -132,7 +132,7 @@ Node::Node(const lib::NodeState& dstate, const lib::NodeState& sstate, uint32_t
         disks.push_back(Disk(storageState.getDiskState(i)));
     }
 }
-Node::~Node() {}
+Node::~Node() = default;
 
 struct Distribution {
     std::vector<Node> nodes;
diff --git a/storageapi/src/tests/mbusprot/storageprotocoltest.cpp b/storageapi/src/tests/mbusprot/storageprotocoltest.cpp
index 2f959e40e2a..0f628f59aac 100644
--- a/storageapi/src/tests/mbusprot/storageprotocoltest.cpp
+++ b/storageapi/src/tests/mbusprot/storageprotocoltest.cpp
@@ -20,7 +20,7 @@
 #include <iomanip>
 #include <sstream>
 
-#include <gtest/gtest.h>
+#include <vespa/vespalib/gtest/gtest.h>
 
 using namespace ::testing;
 
@@ -105,11 +105,10 @@ std::string version_as_gtest_string(TestParamInfo<vespalib::Version> info) {
 
 }
 
-// TODO replace with INSTANTIATE_TEST_SUITE_P on newer gtest versions
-INSTANTIATE_TEST_CASE_P(MultiVersionTest, StorageProtocolTest,
-                        Values(vespalib::Version(6, 240, 0),
-                               vespalib::Version(7, 41, 19)),
-                        version_as_gtest_string);
+VESPA_GTEST_INSTANTIATE_TEST_SUITE_P(MultiVersionTest, StorageProtocolTest,
+                                     Values(vespalib::Version(6, 240, 0),
+                                            vespalib::Version(7, 41, 19)),
+                                     version_as_gtest_string);
 
 namespace {
     mbus::Message::UP lastCommand;
@@ -522,8 +521,15 @@ TEST_P(StorageProtocolTest, remove_location) {
     EXPECT_EQ("id.group == \"mygroup\"", cmd2->getDocumentSelection());
     EXPECT_EQ(_bucket, cmd2->getBucket());
 
-    auto reply = std::make_shared<RemoveLocationReply>(*cmd2);
+    uint32_t n_docs_removed = 12345;
+    auto reply = std::make_shared<RemoveLocationReply>(*cmd2, n_docs_removed);
     auto reply2 = copyReply(reply);
+    if (GetParam().getMajor() == 7) {
+        // Statistics are only available for protobuf-enabled version.
+        EXPECT_EQ(n_docs_removed, reply2->documents_removed());
+    } else {
+        EXPECT_EQ(0, reply2->documents_removed());
+    }
 }
 
 TEST_P(StorageProtocolTest, create_visitor) {
diff --git a/storageapi/src/vespa/storageapi/mbusprot/protobuf/feed.proto b/storageapi/src/vespa/storageapi/mbusprot/protobuf/feed.proto
index 810f88f588f..12dbaf59146 100644
--- a/storageapi/src/vespa/storageapi/mbusprot/protobuf/feed.proto
+++ b/storageapi/src/vespa/storageapi/mbusprot/protobuf/feed.proto
@@ -90,7 +90,12 @@ message RemoveLocationRequest {
     bytes  document_selection = 2;
 }
 
+message RemoveLocationStats {
+    uint32 documents_removed = 1;
+}
+
 message RemoveLocationResponse {
     BucketInfo bucket_info        = 1;
     BucketId   remapped_bucket_id = 2;
+    RemoveLocationStats stats     = 3;
 }
diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization7.cpp b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization7.cpp
index 9751fd1be98..90c8d1c7d2a 100644
--- a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization7.cpp
+++ b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization7.cpp
@@ -643,7 +643,9 @@ void ProtocolSerialization7::onEncode(GBBuf& buf, const api::RemoveLocationComma
 }
 
 void ProtocolSerialization7::onEncode(GBBuf& buf, const api::RemoveLocationReply& msg) const {
-    encode_bucket_info_response<protobuf::RemoveLocationResponse>(buf, msg, no_op_encode);
+    encode_bucket_info_response<protobuf::RemoveLocationResponse>(buf, msg, [&](auto& res) {
+        res.mutable_stats()->set_documents_removed(msg.documents_removed());
+    });
 }
 
 api::StorageCommand::UP ProtocolSerialization7::onDecodeRemoveLocationCommand(BBuf& buf) const {
@@ -653,8 +655,11 @@ api::StorageCommand::UP ProtocolSerialization7::onDecodeRemoveLocationCommand(BB
 }
 
 api::StorageReply::UP ProtocolSerialization7::onDecodeRemoveLocationReply(const SCmd& cmd, BBuf& buf) const {
-    return decode_bucket_info_response<protobuf::RemoveLocationResponse>(buf, [&]([[maybe_unused]] auto& res) {
-        return std::make_unique<api::RemoveLocationReply>(static_cast<const api::RemoveLocationCommand&>(cmd));
+    return decode_bucket_info_response<protobuf::RemoveLocationResponse>(buf, [&](auto& res) {
+        uint32_t documents_removed = (res.has_stats() ? res.stats().documents_removed() : 0u);
+        return std::make_unique<api::RemoveLocationReply>(
+                static_cast<const api::RemoveLocationCommand&>(cmd),
+                documents_removed);
     });
 }
 
diff --git a/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.cpp b/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.cpp
index 33fcc29db11..b5107e68454 100644
--- a/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.cpp
+++ b/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.cpp
@@ -25,7 +25,7 @@ StorageProtocol::StorageProtocol(const std::shared_ptr<const document::DocumentT
 {
 }
 
-StorageProtocol::~StorageProtocol() {}
+StorageProtocol::~StorageProtocol() = default;
 
 mbus::IRoutingPolicy::UP
 StorageProtocol::createPolicy(const mbus::string&, const mbus::string&) const
diff --git a/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.h b/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.h
index 67ea121c340..40f2d26d833 100644
--- a/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.h
+++ b/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.h
@@ -17,13 +17,13 @@ public:
 
     StorageProtocol(const std::shared_ptr<const document::DocumentTypeRepo>,
                     const documentapi::LoadTypeSet& loadTypes);
-    ~StorageProtocol();
+    ~StorageProtocol() override;
 
     const mbus::string& getName() const override { return NAME; }
     mbus::IRoutingPolicy::UP createPolicy(const mbus::string& name, const mbus::string& param) const override;
     mbus::Blob encode(const vespalib::Version&, const mbus::Routable&) const override;
     mbus::Routable::UP decode(const vespalib::Version&, mbus::BlobRef) const override;
-    virtual bool requireSequencing() const override { return true; }
+    bool requireSequencing() const override { return true; }
 private:
     ProtocolSerialization5_0 _serializer5_0;
     ProtocolSerialization5_1 _serializer5_1;
diff --git a/storageapi/src/vespa/storageapi/message/removelocation.cpp b/storageapi/src/vespa/storageapi/message/removelocation.cpp
index b53584601ef..49c9d22f5ee 100644
--- a/storageapi/src/vespa/storageapi/message/removelocation.cpp
+++ b/storageapi/src/vespa/storageapi/message/removelocation.cpp
@@ -25,8 +25,9 @@ RemoveLocationCommand::print(std::ostream& out, bool verbose, const std::string&
     BucketInfoCommand::print(out, verbose, indent);
 }
 
-RemoveLocationReply::RemoveLocationReply(const RemoveLocationCommand& cmd)
-    : BucketInfoReply(cmd)
+RemoveLocationReply::RemoveLocationReply(const RemoveLocationCommand& cmd, uint32_t docs_removed)
+    : BucketInfoReply(cmd),
+      _documents_removed(docs_removed)
 {
 }
 
diff --git a/storageapi/src/vespa/storageapi/message/removelocation.h b/storageapi/src/vespa/storageapi/message/removelocation.h
index 46555497035..812cc8c413b 100644
--- a/storageapi/src/vespa/storageapi/message/removelocation.h
+++ b/storageapi/src/vespa/storageapi/message/removelocation.h
@@ -11,7 +11,7 @@ class RemoveLocationCommand : public BucketInfoCommand
 {
 public:
     RemoveLocationCommand(vespalib::stringref documentSelection, const document::Bucket &bucket);
-    ~RemoveLocationCommand();
+    ~RemoveLocationCommand() override;
 
     void print(std::ostream& out, bool verbose, const std::string& indent) const override;
     const vespalib::string& getDocumentSelection() const { return _documentSelection; }
@@ -22,8 +22,13 @@ private:
 
 class RemoveLocationReply : public BucketInfoReply
 {
+    uint32_t _documents_removed;
 public:
-    RemoveLocationReply(const RemoveLocationCommand& cmd);
+    explicit RemoveLocationReply(const RemoveLocationCommand& cmd, uint32_t docs_removed = 0);
+    void set_documents_removed(uint32_t docs_removed) noexcept {
+        _documents_removed = docs_removed;
+    }
+    uint32_t documents_removed() const noexcept { return _documents_removed; }
     DECLARE_STORAGEREPLY(RemoveLocationReply, onRemoveLocationReply)
 };
 
diff --git a/storageframework/src/vespa/storageframework/generic/status/httpurlpath.cpp b/storageframework/src/vespa/storageframework/generic/status/httpurlpath.cpp
index 337709cc591..10691d37b9e 100644
--- a/storageframework/src/vespa/storageframework/generic/status/httpurlpath.cpp
+++ b/storageframework/src/vespa/storageframework/generic/status/httpurlpath.cpp
@@ -77,7 +77,7 @@ HttpUrlPath::print(std::ostream& out, bool, const std::string&) const
     if (!_attributes.empty()) {
         out << "?";
         size_t cnt = 0;
-        for (const auto attr: _attributes) {
+        for (const auto &attr: _attributes) {
             if (cnt++ > 0) {
                 out << "&";
             }
diff --git a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp
index eb1cc7f0256..cc48ad09aa6 100644
--- a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp
+++ b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp
@@ -70,44 +70,41 @@ createMultiValueAttribute(const vespalib::string & name, const document::FieldVa
     }
     LOG(debug, "Create %s attribute '%s' with data type '%s' (%s)",
         arrayType ? "array" : "weighted set", name.c_str(), ndt->getName().c_str(), fv.getClass().name());
-    AttributeVector::SP attr;
     if (ndt->getId() == DataType::T_BYTE ||
         ndt->getId() == DataType::T_INT ||
         ndt->getId() == DataType::T_LONG)
     {
-        attr.reset(arrayType ? static_cast<AttributeVector *>(new search::MultiIntegerExtAttribute(name))
-                             : static_cast<AttributeVector *>(new search::WeightedSetIntegerExtAttribute(name)));
+        return arrayType ? std::make_shared<search::MultiIntegerExtAttribute>(name)
+                         : std::make_shared<search::WeightedSetIntegerExtAttribute>(name);
     } else if (ndt->getId() == DataType::T_DOUBLE ||
                ndt->getId() == DataType::T_FLOAT)
     {
-        attr.reset(arrayType ? static_cast<AttributeVector *>(new search::MultiFloatExtAttribute(name))
-                             : static_cast<AttributeVector *>(new search::WeightedSetFloatExtAttribute(name)));
+        return arrayType ? std::make_shared<search::MultiFloatExtAttribute>(name)
+                         : std::make_shared<search::WeightedSetFloatExtAttribute>(name);
     } else if (ndt->getId() == DataType::T_STRING) {
-        attr.reset(arrayType ? static_cast<AttributeVector *>(new search::MultiStringExtAttribute(name))
-                             : static_cast<AttributeVector *>(new search::WeightedSetStringExtAttribute(name)));
+        return arrayType ? std::make_shared<search::MultiStringExtAttribute>(name)
+                         : std::make_shared<search::WeightedSetStringExtAttribute>(name);
     } else {
         LOG(debug, "Can not make an multivalue attribute out of %s with data type '%s' (%s)",
             name.c_str(), ndt->getName().c_str(), fv.getClass().name());
     }
-    return attr;
+    return AttributeVector::SP();
 }
 
 AttributeVector::SP
 createAttribute(const vespalib::string & name, const document::FieldValue & fv)
 {
     LOG(debug, "Create single value attribute '%s' with value type '%s'", name.c_str(), fv.getClass().name());
-    AttributeVector::SP attr;
-
     if (fv.inherits(document::ByteFieldValue::classId) || fv.inherits(document::IntFieldValue::classId) || fv.inherits(document::LongFieldValue::classId)) {
-        attr.reset(new search::SingleIntegerExtAttribute(name));
+        return std::make_shared<search::SingleIntegerExtAttribute>(name);
     } else if (fv.inherits(document::DoubleFieldValue::classId) || fv.inherits(document::FloatFieldValue::classId)) {
-        attr.reset(new search::SingleFloatExtAttribute(name));
+        return std::make_shared<search::SingleFloatExtAttribute>(name);
     } else if (fv.inherits(document::StringFieldValue::classId)) {
-        attr.reset(new search::SingleStringExtAttribute(name));
+        return std::make_shared<search::SingleStringExtAttribute>(name);
     } else {
         LOG(debug, "Can not make an attribute out of %s of type '%s'.", name.c_str(), fv.getClass().name());
     }
-    return attr;
+    return AttributeVector::SP();
 }
 
 SearchVisitor::SummaryGenerator::SummaryGenerator() :
@@ -120,7 +117,7 @@ SearchVisitor::SummaryGenerator::SummaryGenerator() :
 {
 }
 
-SearchVisitor::SummaryGenerator::~SummaryGenerator() { }
+SearchVisitor::SummaryGenerator::~SummaryGenerator() = default;
 
 
 vespalib::ConstBufferRef
@@ -137,7 +134,7 @@ SearchVisitor::SummaryGenerator::fillSummary(AttributeVector::DocId lid, const H
 
 void SearchVisitor::HitsResultPreparator::execute(vespalib::Identifiable & obj)
 {
-    HitsAggregationResult & hitsAggr(static_cast<HitsAggregationResult &>(obj));
+    auto & hitsAggr(static_cast<HitsAggregationResult &>(obj));
     hitsAggr.setSummaryGenerator(_summaryGenerator);
     _numHitsAggregators++;
 }
@@ -154,7 +151,7 @@ SearchVisitor::GroupingEntry::GroupingEntry(Grouping * grouping) :
 {
 }
 
-SearchVisitor::GroupingEntry::~GroupingEntry() { }
+SearchVisitor::GroupingEntry::~GroupingEntry() = default;
 
 void SearchVisitor::GroupingEntry::aggregate(const document::Document & doc, search::HitRank rank)
 {
@@ -175,7 +172,7 @@ SearchVisitor::SearchVisitor(StorageComponent& component,
                              VisitorEnvironment& vEnv,
                              const Parameters& params) :
     Visitor(component),
-    _env(static_cast<SearchEnvironment &>(vEnv)),
+    _env(dynamic_cast<SearchEnvironment &>(vEnv)),
     _params(params),
     _vsmAdapter(nullptr),
     _docSearchedCount(0),
@@ -240,9 +237,9 @@ void SearchVisitor::init(const Parameters & params)
         LOG(debug, "QFLAG_DUMP_FEATURES: %s", _rankController.getDumpFeatures() ? "true" : "false");
     }
 
-    if (params.lookup("rankproperties", valueRef) && valueRef.size() > 0) {
+    if (params.lookup("rankproperties", valueRef) && ! valueRef.empty()) {
         LOG(spam, "Received rank properties of %zd bytes", valueRef.size());
-        uint32_t len = static_cast<uint32_t>(valueRef.size());
+        uint32_t len = valueRef.size();
         char * data = const_cast<char *>(valueRef.data());
         FNET_DataBuffer src(data, len);
         uint32_t cnt = src.ReadInt32();
@@ -403,7 +400,7 @@ SearchVisitor::PositionInserter::PositionInserter(AttributeVector & attribute, A
 {
 }
 
-SearchVisitor::PositionInserter::~PositionInserter() {}
+SearchVisitor::PositionInserter::~PositionInserter() = default;
 
 void
 SearchVisitor::PositionInserter::onPrimitive(uint32_t, const Content & c)
@@ -414,7 +411,7 @@ SearchVisitor::PositionInserter::onPrimitive(uint32_t, const Content & c)
 void
 SearchVisitor::PositionInserter::onStructStart(const Content & c)
 {
-    const document::StructuredFieldValue & value = static_cast<const document::StructuredFieldValue &>(c.getValue());
+    const auto & value = static_cast<const document::StructuredFieldValue &>(c.getValue());
     LOG(debug, "PositionInserter: Adding value '%s'(%d) to attribute '%s' for docid '%d'",
         value.toString().c_str(), c.getWeight(), _attribute.getName().c_str(), _docId);
 
@@ -445,7 +442,7 @@ SearchVisitor::RankController::processHintedAttributes(const IndexEnvironment &
                 AttributeGuard::UP attr(attrMan.getAttribute(name));
                 if (attr->valid()) {
                     LOG(debug, "Add attribute '%s' with field id '%u' to the list of needed attributes", name.c_str(), fid);
-                    attributeFields.push_back(AttrInfo(fid, std::move(attr)));
+                    attributeFields.emplace_back(fid, std::move(attr));
                 } else {
                     LOG(warning, "Cannot locate attribute '%s' in the attribute manager. "
                         "Ignore access hint about this attribute", name.c_str());
@@ -470,7 +467,7 @@ SearchVisitor::RankController::RankController() :
 {
 }
 
-SearchVisitor::RankController::~RankController() {}
+SearchVisitor::RankController::~RankController() = default;
 
 void
 SearchVisitor::RankController::setupRankProcessors(Query & query,
@@ -485,7 +482,7 @@ SearchVisitor::RankController::setupRankProcessors(Query & query,
     const IndexEnvironment & indexEnv = _rankManagerSnapshot->getIndexEnvironment(_rankProfile);
     processHintedAttributes(indexEnv, true, attrMan, attributeFields);
 
-    _rankProcessor.reset(new RankProcessor(_rankManagerSnapshot, _rankProfile, query, location, _queryProperties, &attrMan));
+    _rankProcessor = std::make_unique<RankProcessor>(_rankManagerSnapshot, _rankProfile, query, location, _queryProperties, &attrMan);
     LOG(debug, "Initialize rank processor");
     _rankProcessor->initForRanking(wantedHitCount);
 
@@ -493,7 +490,7 @@ SearchVisitor::RankController::setupRankProcessors(Query & query,
         // register attribute vectors needed for dumping
         processHintedAttributes(indexEnv, false, attrMan, attributeFields);
 
-        _dumpProcessor.reset(new RankProcessor(_rankManagerSnapshot, _rankProfile, query, location, _queryProperties, &attrMan));
+        _dumpProcessor = std::make_unique<RankProcessor>(_rankManagerSnapshot, _rankProfile, query, location, _queryProperties, &attrMan);
         LOG(debug, "Initialize dump processor");
         _dumpProcessor->initForDumping(wantedHitCount);
     }
@@ -620,14 +617,14 @@ SearchVisitor::registerAdditionalFields(const std::vector<vsm::DocsumTools::Fiel
         for (size_t j = 0; j < inputNames.size(); ++j) {
             fieldList.push_back(inputNames[j]);
             if (PositionDataType::isZCurveFieldName(inputNames[j])) {
-                fieldList.push_back(PositionDataType::cutZCurveFieldName(inputNames[j]));
+                fieldList.emplace_back(PositionDataType::cutZCurveFieldName(inputNames[j]));
             }
         }
     }
     // fields used during sorting
-    fieldList.push_back("[docid]");
-    fieldList.push_back("[rank]");
-    fieldList.push_back("documentid");
+    fieldList.emplace_back("[docid]");
+    fieldList.emplace_back("[rank]");
+    fieldList.emplace_back("documentid");
 }
 
 void
@@ -710,7 +707,7 @@ SearchVisitor::setupDocsumObjects()
                            }
                         }
                         if (index == _attributeFields.size()) {
-                            _attributeFields.push_back(AttrInfo(fid, std::move(attr)));
+                            _attributeFields.emplace_back(fid, std::move(attr));
                         }
                    } else {
                        LOG(warning, "Attribute '%s' is not valid", name.c_str());
@@ -764,7 +761,7 @@ void SearchVisitor::setupAttributeVector(const FieldPath &fieldPath) {
         attr = createAttribute(attrName, fv);
     }
 
-    if (attr.get()) {
+    if (attr) {
         LOG(debug, "Adding attribute '%s' for field '%s' with data type '%s' (%s)",
             attr->getName().c_str(), attrName.c_str(), fv.getDataType()->getName().c_str(), fv.getClass().name());
         if ( ! _attrMan.add(attr) ) {
@@ -796,7 +793,7 @@ SearchVisitor::setupAttributeVectorsForSorting(const search::common::SortSpec &
                             }
                         }
                         if (index == _attributeFields.size()) {
-                            _attributeFields.push_back(AttrInfo(fid, std::move(attr), sInfo._ascending, sInfo._converter.get()));
+                            _attributeFields.emplace_back(fid, std::move(attr), sInfo._ascending, sInfo._converter.get());
                         }
                         _sortList.push_back(index);
                     } else {
@@ -849,8 +846,8 @@ SearchVisitor::setupGrouping(const std::vector<char> & groupingBlob)
 class SingleDocumentStore : public vsm::IDocSumCache
 {
 public:
-    SingleDocumentStore(const StorageDocument & doc) : _doc(doc) { }
-    virtual const vsm::Document & getDocSum(const search::DocumentIdT & docId) const override {
+    explicit SingleDocumentStore(const StorageDocument & doc) : _doc(doc) { }
+    const vsm::Document & getDocSum(const search::DocumentIdT & docId) const override {
         (void) docId;
         return _doc;
     }
@@ -1001,7 +998,7 @@ SearchVisitor::fillAttributeVectors(const vespalib::string & documentId, const S
             fieldId = _fieldsUnion.find(org)->second;
         }
         const StorageDocument::SubDocument & subDoc = document.getComplexField(fieldId);
-        AttributeVector & attrV = const_cast<AttributeVector & >(*finfoGuard);
+        auto & attrV = const_cast<AttributeVector & >(*finfoGuard);
         AttributeVector::DocId docId(0);
         attrV.addDoc(docId);
         if (subDoc.getFieldValue() != nullptr) {
diff --git a/tenant-auth/src/main/java/ai/vespa/hosted/auth/ApiAuthenticator.java b/tenant-auth/src/main/java/ai/vespa/hosted/auth/ApiAuthenticator.java
index 82d859b08bd..55b3af93050 100644
--- a/tenant-auth/src/main/java/ai/vespa/hosted/auth/ApiAuthenticator.java
+++ b/tenant-auth/src/main/java/ai/vespa/hosted/auth/ApiAuthenticator.java
@@ -13,10 +13,12 @@ public class ApiAuthenticator implements ai.vespa.hosted.api.ApiAuthenticator {
                          .map(certificateFile -> ControllerHttpClient.withKeyAndCertificate(Properties.apiEndpoint(),
                                                                                             Properties.apiKeyFile(),
                                                                                             certificateFile))
-                         .orElseGet(() ->
-                                            ControllerHttpClient.withSignatureKey(Properties.apiEndpoint(),
-                                                                                  Properties.apiKeyFile(),
-                                                                                  Properties.application()));
+                         .or(() -> Properties.apiKey().map(apiKey -> ControllerHttpClient.withSignatureKey(Properties.apiEndpoint(),
+                                                                                                           apiKey,
+                                                                                                           Properties.application())))
+                         .orElseGet(() -> ControllerHttpClient.withSignatureKey(Properties.apiEndpoint(),
+                                                                                Properties.apiKeyFile(),
+                                                                                Properties.application()));
     }
 
 }
diff --git a/tenant-base/pom.xml b/tenant-base/pom.xml
index 22e1bb0e9ca..a3fac22a93d 100644
--- a/tenant-base/pom.xml
+++ b/tenant-base/pom.xml
@@ -253,6 +253,30 @@
                 </plugins>
             </build>
         </profile>
+
+        <profile> <!-- Alias vespaversion with a more descriptive vespa.compile.version -->
+            <id>set-vespa-compile-version</id>
+            <activation>
+                <property>
+                    <name>vespa.compile.version</name>
+                </property>
+            </activation>
+            <properties>
+                <vespaversion>${vespa.compile.version}</vespaversion>
+            </properties>
+        </profile>
+
+        <profile> <!-- Alias vespaVersion with a more descriptive vespa.runtime.version -->
+            <id>set-vespa-runtime-version</id>
+            <activation>
+                <property>
+                    <name>vespa.runtime.version</name>
+                </property>
+            </activation>
+            <properties>
+                <vespaVersion>${vespa.runtime.version}</vespaVersion>
+            </properties>
+        </profile>
     </profiles>
 
     <build>
diff --git a/tenant-cd/src/main/java/ai/vespa/hosted/cd/http/HttpEndpoint.java b/tenant-cd/src/main/java/ai/vespa/hosted/cd/http/HttpEndpoint.java
index 0c81b483ba0..74eb765dd0b 100644
--- a/tenant-cd/src/main/java/ai/vespa/hosted/cd/http/HttpEndpoint.java
+++ b/tenant-cd/src/main/java/ai/vespa/hosted/cd/http/HttpEndpoint.java
@@ -4,6 +4,7 @@ package ai.vespa.hosted.cd.http;
 import ai.vespa.hosted.api.EndpointAuthenticator;
 import ai.vespa.hosted.cd.Endpoint;
 
+import javax.net.ssl.SSLParameters;
 import java.io.IOException;
 import java.net.URI;
 import java.net.http.HttpClient;
@@ -31,10 +32,13 @@ public class HttpEndpoint implements Endpoint {
     public HttpEndpoint(URI endpoint, EndpointAuthenticator authenticator) {
         this.endpoint = requireNonNull(endpoint);
         this.authenticator = requireNonNull(authenticator);
+        SSLParameters sslParameters = new SSLParameters();
+        sslParameters.setProtocols(new String[] {"TLSv1.2" });
         this.client = HttpClient.newBuilder()
                                 .sslContext(authenticator.sslContext())
                                 .connectTimeout(Duration.ofSeconds(5))
                                 .version(HttpClient.Version.HTTP_1_1)
+                                .sslParameters(sslParameters)
                                 .build();
     }
 
@@ -58,7 +62,7 @@ public class HttpEndpoint implements Endpoint {
         return HttpRequest.newBuilder(endpoint.resolve(path +
                                                        properties.entrySet().stream()
                                                                  .map(entry -> encode(entry.getKey(), UTF_8) + "=" + encode(entry.getValue(), UTF_8))
-                                                                 .collect(Collectors.joining("&", "?", ""))));
+                                                                 .collect(Collectors.joining("&", path.contains("?") ? "&" : "?", ""))));
     }
 
 }
diff --git a/travis/travis-build-full.sh b/travis/travis-build-full.sh
index 1715c9e2288..f44e5e55d7a 100755
--- a/travis/travis-build-full.sh
+++ b/travis/travis-build-full.sh
@@ -6,12 +6,12 @@ export SOURCE_DIR=/source
 export NUM_THREADS=6
 export MALLOC_ARENA_MAX=1
 export MAVEN_OPTS="-Xss1m -Xms128m -Xmx2g"
-source /etc/profile.d/enable-devtoolset-8.sh
+source /etc/profile.d/enable-devtoolset-9.sh
 source /etc/profile.d/enable-rh-maven35.sh
 
 ccache --max-size=1250M
 ccache --set-config=compression=true
-ccache --print-config
+ccache -p
 
 cd ${SOURCE_DIR}
 env VESPA_MAVEN_EXTRA_OPTS="--no-snapshot-updates --batch-mode --threads ${NUM_THREADS}" sh ./bootstrap.sh java
diff --git a/travis/travis.sh b/travis/travis.sh
index 77bfebf2505..315771b43b9 100755
--- a/travis/travis.sh
+++ b/travis/travis.sh
@@ -10,7 +10,7 @@ function bell() {
   done
 }
 
-DOCKER_IMAGE=vespaengine/vespa-dev:latest
+DOCKER_IMAGE=vespaengine/vespa-build-centos7:latest
 
 bell &
 docker run --rm -v ${HOME}/.m2:/root/.m2 -v ${HOME}/.ccache:/root/.ccache -v $(pwd):/source \
diff --git a/vagrant/.gitignore b/vagrant/.gitignore
deleted file mode 100644
index d16c9dbeeb5..00000000000
--- a/vagrant/.gitignore
+++ /dev/null
@@ -1,2 +0,0 @@
-.vagrant/
-*.box
\ No newline at end of file
diff --git a/vagrant/README.md b/vagrant/README.md
deleted file mode 100644
index 71fa071cb46..00000000000
--- a/vagrant/README.md
+++ /dev/null
@@ -1,107 +0,0 @@
-<!-- Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
-
-# Create C++ / Java dev environment on CentOS using VirtualBox and Vagrant
-
-## Prerequisites
-* [Install VirtualBox](https://www.virtualbox.org/wiki/Downloads)
-* [Install Vagrant](https://www.vagrantup.com/downloads.html)
-
-## Create dev environment
-
-#### 1. Change working directory to &lt;vespa-source&gt;/vagrant
-
-    cd <vespa-source>/vagrant
-
-#### 2. Choose dev environment
-
-##### a. For a dev environment with plain centos/7 and no GUI:
-
-    export VESPA_VAGRANT_VM_BOX=centos/7
-    export VESPA_VAGRANT_DISABLE_GUI=true
-
-##### b. For a dev environment with GUI and CLion:
-
-Create centos7-desktop box:
-
-* Install packer by following guide at [packer.io](https://www.packer.io/intro/getting-started/install.html)
-
-* Clone boxcutter centos repo and build the box:
-```
-git clone https://github.com/boxcutter/centos.git
-./bin/box build centos7-desktop.json virtualbox
-```
-
-Example exports:
-
-    export VESPA_VAGRANT_VM_BOX="centos7-desktop"
-    export VESPA_VAGRANT_VM_BOX_URL="$HOME/git/boxcutter/centos/box/virtualbox/centos7-desktop-xx.yyyy.z.box"
-
-
-#### 3. Install Vagrant VirtualBox Guest Additions plugin
-This is required for mounting shared folders and get mouse pointer integration and seamless windows in the virtual CentOS desktop.
-
-    vagrant plugin install vagrant-vbguest
-
-#### 4. Start and provision the environment
-
-    vagrant up
-
-#### 5. Connect to machine via SSH
-SSH agent forwarding is enabled to ensure easy interaction with GitHub inside the machine.
-
-    vagrant ssh
-
-#### 6. Checkout vespa source inside virtual machine
-This is needed in order to compile and run tests fast on the local file system inside the virtual machine.
-
-    git clone git@github.com:vespa-engine/vespa.git
-
-## Build Java modules
-Please follow the build instructions described [here](../README.md#build-java-modules).
-
-
-## Build C++ modules
-Please follow the build instructions described [here](../README.md#build-c-modules).
-Skip these steps if doing development with CLion.
-
-
-## Build and Develop using CLion
-CLion is installed as part of the environment and is recommended for C++ development.
-
-#### 1. Bootstrap C++ building
-cd to the vespa/ directory created by git clone and execute:
-
-    ./bootstrap.sh java
-    ./bootstrap-cpp.sh . .
-
-#### 2. Start CLion
-Open a terminal inside the virtual CentOS desktop (password is "vagrant") and run:
-
-    clion
-
-When prompted, configure toolchains as follows:
-
-    CMake: /usr/bin/cmake3
-    Make: /usr/bin/make
-    C Compiler: /opt/rh/devtoolset-8/root/usr/bin/cc
-    C++ Compiler: /opt/rh/devtoolset-8/root/usr/bin/c++
-
-#### 3. Open the Vespa Project
-Go to *File* -> *Open* and choose &lt;vespa-source>&gt;/CMakeLists.txt.
-
-#### 4. Set compiler threads
-Go to *File* -> *Settings* -> *Build, Execution, Deployment* -> *CMake*.
-Under *Build Options* specify "-j 4" and click *Apply*.
-
-#### 5. Run bootstrap again
-
-    ./bootstrap-cpp.sh . .
-
-(Some of the changes made by it are undone by clion on the first startup.)
-
-#### 6. Build all modules
-Choose target **all_modules** from the set of build targets at the top right and click build.
-
-## Starting and stopping the Vagrant machine
-Use `vagrant suspend` to suspend the machine and then `vagrant resume` to resume it later on.
-Alternatively use `vagrant halt` + `vagrant up` to shutdown and reboot. Latter approach is slower but requires less disk space since RAM content is not persisted to host.
diff --git a/vagrant/Vagrantfile b/vagrant/Vagrantfile
deleted file mode 100644
index be0824dbd55..00000000000
--- a/vagrant/Vagrantfile
+++ /dev/null
@@ -1,93 +0,0 @@
-# -*- mode: ruby -*-
-# vi: set ft=ruby :
-
-disable_gui = ENV['VESPA_VAGRANT_DISABLE_GUI']
-
-def get_mandatory_env_value(name)
-  opt = ENV[name]
-  if opt.nil? or opt.empty?
-    raise Vagrant::Errors::VagrantError.new, "Environment variable #{name} must be set to a valid value before running vagrant"
-  end
-  return opt
-end
-
-def get_env_value(name, fallback)
-  opt = ENV[name]
-  if opt.nil? or opt.empty?
-    return fallback
-  end
-  return opt
-end
-
-vm_box = get_mandatory_env_value('VESPA_VAGRANT_VM_BOX')
-vm_memory = get_env_value('VESPA_VAGRANT_VM_MEMORY', "8192")
-vm_cpus = get_env_value('VESPA_VAGRANT_VM_CPUS', 4)
-
-unless disable_gui
-  vm_box_url = get_mandatory_env_value('VESPA_VAGRANT_VM_BOX_URL')
-end
-
-# For a complete reference, please see the online documentation at https://docs.vagrantup.com.
-Vagrant.configure("2") do |config|
-
-  config.vm.box = vm_box
-  config.vm.box_url = vm_box_url unless disable_gui
-
-  config.ssh.forward_agent = true
-
-  config.vm.synced_folder "../dist", "/vagrant/dist"
-
-  config.vm.provider "virtualbox" do |vb|
-    # Display the VirtualBox GUI when booting the machine
-    vb.gui = true unless disable_gui
-    vb.name = "vespa-dev"
-
-    vb.memory = vm_memory
-    vb.cpus = vm_cpus
-  end
-  
-  # Install required and nice-to-have packages
-  config.vm.provision "shell", inline: <<-SHELL
-    yum-config-manager --add-repo https://copr.fedorainfracloud.org/coprs/g/vespa/vespa/repo/epel-7/group_vespa-vespa-epel-7.repo
-    yum -y install epel-release
-    yum -y install centos-release-scl
-    yum -y install yum-utils
-    yum -y install git \
-        ccache \
-        maven \
-        rpm-build \
-        valgrind \
-        sudo \
-        firefox \
-        vim \
-        emacs
-    sed -e '/^BuildRequires:/d' -e 's/^Requires:/BuildRequires:/' /vagrant/dist/vespa.spec > /tmp/vesparun.spec
-    yum-builddep -y /vagrant/dist/vespa.spec /tmp/vesparun.spec
-    rm /tmp/vesparun.spec
-    echo -e "* soft nproc 409600\n* hard nproc 409600" > /etc/security/limits.d/99-nproc.conf
-    echo -e "* soft nofile 262144\n* hard nofile 262144" > /etc/security/limits.d/99-nofile.conf
-
-    unless disable_gui
-      echo -e "fs.inotify.max_user_watches = 524288" > /etc/sysctl.d/clion.conf
-      wget -q -O - https://download.jetbrains.com/cpp/CLion-2018.1.6.tar.gz | tar -C /opt -zx
-      ln -sf /opt/clion-2018.1.6/bin/clion.sh /usr/bin/clion
-    end
-
-    yum update -y
-    hostname localhost
-  SHELL
-
-  # Add settings for Vespa and dev tools as the default user, usually 'vagrant' (privileged: false)
-  # NOTE: adding these settings to .bashrc would break vagrant suspend/resume/provision
-  #       due to env vars modified by /opt/rh/devtoolset-8/enable.
-  config.vm.provision "shell", privileged: false, inline: <<-SCRIPT
-    grep -l VESPA_HOME ~/.bash_profile >/dev/null || (\
-      printf "%s\n" \
-        'export VESPA_HOME=$HOME/vespa' \
-        'export PATH=$PATH:$VESPA_HOME/bin' \
-        'source /opt/rh/rh-maven35/enable' \
-        'source /opt/rh/devtoolset-8/enable' \
-        >> ~/.bash_profile )
-  SCRIPT
-
-end
diff --git a/valgrind-suppressions.txt b/valgrind-suppressions.txt
index 13be6234a94..b40f0957454 100644
--- a/valgrind-suppressions.txt
+++ b/valgrind-suppressions.txt
@@ -305,3 +305,39 @@
    fun:__add_to_environ
    fun:setenv
 }
+{
+   RE2 sparse structures deliberately do not care about uninitialized memory (https://github.com/google/re2/issues/121)
+   Memcheck:Cond
+   ...
+   fun:_ZN3re28Compiler7CompileEPNS_6RegexpEbl
+}
+{
+   RE2 sparse structures deliberately do not care about uninitialized memory (https://github.com/google/re2/issues/121)
+   Memcheck:Value8
+   ...
+   fun:_ZN3re28Compiler7CompileEPNS_6RegexpEbl
+}
+{
+   RE2 sparse structures deliberately do not care about uninitialized memory (https://github.com/google/re2/issues/121)
+   Memcheck:Cond
+   ...
+   fun:_ZN3re23RE2C1ERKNS_11StringPieceERKNS0_7OptionsE
+}
+{
+   RE2 sparse structures deliberately do not care about uninitialized memory (https://github.com/google/re2/issues/121)
+   Memcheck:Value8
+   ...
+   fun:_ZN3re23RE2C1ERKNS_11StringPieceERKNS0_7OptionsE
+}
+{
+   RE2 sparse structures deliberately do not care about uninitialized memory (https://github.com/google/re2/issues/121)
+   Memcheck:Value8
+   ...
+   fun:_ZNK3re23RE25MatchERKNS_11StringPieceEmmNS0_6AnchorEPS1_i
+}
+{
+   RE2 sparse structures deliberately do not care about uninitialized memory (https://github.com/google/re2/issues/121)
+   Memcheck:Cond
+   ...
+   fun:_ZNK3re23RE25MatchERKNS_11StringPieceEmmNS0_6AnchorEPS1_i
+}
diff --git a/vbench/src/vbench/core/socket.cpp b/vbench/src/vbench/core/socket.cpp
index 822b96b2c07..0431b6889a8 100644
--- a/vbench/src/vbench/core/socket.cpp
+++ b/vbench/src/vbench/core/socket.cpp
@@ -29,7 +29,8 @@ Socket::Socket(SyncCryptoSocket::UP socket)
 }
 
 Socket::Socket(CryptoEngine &crypto, const string &host, int port)
-    : _socket(SyncCryptoSocket::create(crypto, connect(host, port), false)),
+    : _socket(SyncCryptoSocket::create_client(crypto, connect(host, port),
+                                              vespalib::SocketSpec::from_host_port(host, port))),
       _input(),
       _output(),
       _taint(),
diff --git a/vbench/src/vbench/core/socket.h b/vbench/src/vbench/core/socket.h
index 0e8848e8292..4b5721c0fb7 100644
--- a/vbench/src/vbench/core/socket.h
+++ b/vbench/src/vbench/core/socket.h
@@ -52,7 +52,7 @@ struct ServerSocket {
     Stream::UP accept(CryptoEngine &crypto) {
         vespalib::SocketHandle handle = server_socket.accept();
         if (handle.valid()) {
-            return std::make_unique<Socket>(SyncCryptoSocket::create(crypto, std::move(handle), true));
+            return std::make_unique<Socket>(SyncCryptoSocket::create_server(crypto, std::move(handle)));
         } else {
             return Stream::UP();
         }
diff --git a/vbench/src/vbench/vbench/vbench.cpp b/vbench/src/vbench/vbench/vbench.cpp
index 4f6efadfbdd..58854af705e 100644
--- a/vbench/src/vbench/vbench/vbench.cpp
+++ b/vbench/src/vbench/vbench/vbench.cpp
@@ -29,11 +29,13 @@ CryptoEngine::SP setup_crypto(const vespalib::slime::Inspector &tls) {
     if (!tls.valid()) {
         return std::make_shared<vespalib::NullCryptoEngine>();
     }
-    vespalib::net::tls::TransportSecurityOptions
-        tls_opts(maybe_load(tls["ca-certificates"]),
-                 maybe_load(tls["certificates"]),
-                 maybe_load(tls["private-key"]));
-    return std::make_shared<vespalib::TlsCryptoEngine>(tls_opts);
+    auto ts_builder = vespalib::net::tls::TransportSecurityOptions::Params().
+            ca_certs_pem(maybe_load(tls["ca-certificates"])).
+            cert_chain_pem(maybe_load(tls["certificates"])).
+            private_key_pem(maybe_load(tls["private-key"])).
+            authorized_peers(vespalib::net::tls::AuthorizedPeers::allow_all_authenticated()).
+            disable_hostname_validation(true); // TODO configurable or default false!
+    return std::make_shared<vespalib::TlsCryptoEngine>(vespalib::net::tls::TransportSecurityOptions(std::move(ts_builder)));
 }
 
 } // namespace vbench::<unnamed>
diff --git a/vespa-athenz/pom.xml b/vespa-athenz/pom.xml
index 0f23eaed964..7d2ad924ae3 100644
--- a/vespa-athenz/pom.xml
+++ b/vespa-athenz/pom.xml
@@ -131,7 +131,17 @@
                     <artifactId>jackson-annotations</artifactId>
                 </exclusion>
             </exclusions>
-    </dependency>
+        </dependency>
+        <dependency>
+            <groupId>com.auth0</groupId>
+            <artifactId>java-jwt</artifactId>
+            <exclusions>
+                <exclusion>
+                    <groupId>com.fasterxml.jackson.core</groupId>
+                    <artifactId>jackson-databind</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
     </dependencies>
 
     <build>
diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/api/AthenzAccessToken.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/api/AthenzAccessToken.java
new file mode 100644
index 00000000000..16b923382b3
--- /dev/null
+++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/api/AthenzAccessToken.java
@@ -0,0 +1,68 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.athenz.api;
+
+import com.auth0.jwt.JWT;
+import com.auth0.jwt.interfaces.DecodedJWT;
+
+import java.time.Instant;
+import java.util.Objects;
+
+/**
+ * Represents an Athenz Access Token
+ *
+ * @author bjorncs
+ */
+public class AthenzAccessToken {
+
+    public static final String HTTP_HEADER_NAME = "Authorization";
+
+    private static final String BEARER_TOKEN_PREFIX = "Bearer ";
+
+    private final String value;
+    private volatile DecodedJWT jwt;
+
+    public AthenzAccessToken(String value) {
+        this.value = stripBearerTokenPrefix(value);
+    }
+
+    private static String stripBearerTokenPrefix(String rawValue) {
+        String stripped = rawValue.strip();
+        String prefixRemoved = stripped.startsWith(BEARER_TOKEN_PREFIX)
+                ? stripped.substring(BEARER_TOKEN_PREFIX.length()).strip()
+                : stripped;
+        if (prefixRemoved.isBlank()) {
+            throw new IllegalArgumentException(String.format("Access token is blank: '%s'", prefixRemoved));
+        }
+        return prefixRemoved;
+    }
+
+    public String value() { return value; }
+    public String valueWithBearerPrefix() { return BEARER_TOKEN_PREFIX + value; }
+    public Instant getExpiryTime () {
+        return jwt().getExpiresAt().toInstant();
+    }
+
+    private DecodedJWT jwt() {
+        if (jwt == null) {
+            // Decoding a token is expensive and involves construction of at least one Jackson ObjectMapper instance
+            // TODO Cache encoder/decoder as static field in AthenzAccessToken
+            jwt = JWT.decode(this.value);
+        }
+        return jwt;
+    }
+
+    @Override public String toString() { return "AthenzAccessToken{value='" + value + "'}"; }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        AthenzAccessToken that = (AthenzAccessToken) o;
+        return Objects.equals(value, that.value);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(value);
+    }
+}
diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/api/AthenzRole.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/api/AthenzRole.java
index 3a81e4a5e17..4e432768298 100644
--- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/api/AthenzRole.java
+++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/api/AthenzRole.java
@@ -7,6 +7,8 @@ import java.util.Objects;
  * @author tokle
  */
 public class AthenzRole {
+    private static final String ROLE_RESOURCE_PREFIX = "role.";
+
     private final AthenzDomain domain;
     private final String roleName;
 
@@ -20,6 +22,19 @@ public class AthenzRole {
         this.roleName = roleName;
     }
 
+    public static AthenzRole fromResourceNameString(String string) {
+        return fromResourceName(AthenzResourceName.fromString(string));
+    }
+
+    public static AthenzRole fromResourceName(AthenzResourceName resourceName) {
+        String entityName = resourceName.getEntityName();
+        if (!entityName.startsWith(ROLE_RESOURCE_PREFIX)) {
+            throw new IllegalArgumentException("Not a valid role: " + resourceName.toResourceNameString());
+        }
+        String roleName = entityName.substring(ROLE_RESOURCE_PREFIX.length());
+        return new AthenzRole(resourceName.getDomain(), roleName);
+    }
+
     public AthenzDomain domain() {
         return domain;
     }
@@ -28,6 +43,10 @@ public class AthenzRole {
         return roleName;
     }
 
+    public String toResourceNameString() { return toResourceName().toResourceNameString(); }
+
+    public AthenzResourceName toResourceName() { return new AthenzResourceName(domain, ROLE_RESOURCE_PREFIX + roleName); }
+
     @Override
     public boolean equals(Object o) {
         if (this == o) return true;
diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/common/ClientBase.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/common/ClientBase.java
index 4cc92828b0e..c1ce45c35da 100644
--- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/common/ClientBase.java
+++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/common/ClientBase.java
@@ -2,6 +2,7 @@
 package com.yahoo.vespa.athenz.client.common;
 
 import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.JsonMappingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
 import com.yahoo.vespa.athenz.client.common.bindings.ErrorResponseEntity;
@@ -16,6 +17,7 @@ import org.apache.http.entity.StringEntity;
 import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.http.impl.client.DefaultHttpRequestRetryHandler;
 import org.apache.http.impl.client.HttpClientBuilder;
+import org.apache.http.util.EntityUtils;
 
 import javax.net.ssl.HostnameVerifier;
 import javax.net.ssl.SSLContext;
@@ -23,12 +25,16 @@ import java.io.IOException;
 import java.io.UncheckedIOException;
 import java.time.Duration;
 import java.util.function.Supplier;
+import java.util.logging.Level;
+import java.util.logging.Logger;
 
 /**
  * @author bjorncs
  */
 public abstract class ClientBase implements AutoCloseable {
 
+    private static final Logger logger = Logger.getLogger(ClientBase.class.getName());
+
     private static final ObjectMapper objectMapper = new ObjectMapper().registerModule(new JavaTimeModule());
 
     private final CloseableHttpClient client;
@@ -59,15 +65,22 @@ public abstract class ClientBase implements AutoCloseable {
     }
 
     protected <T> T readEntity(HttpResponse response, Class<T> entityType) throws IOException {
-        if (isSuccess(response.getStatusLine().getStatusCode())) {
+        int statusCode = response.getStatusLine().getStatusCode();
+        if (isSuccess(statusCode)) {
             if (entityType.equals(Void.class)) {
                 return null;
             } else {
                 return objectMapper.readValue(response.getEntity().getContent(), entityType);
             }
         } else {
-            ErrorResponseEntity errorEntity = objectMapper.readValue(response.getEntity().getContent(), ErrorResponseEntity.class);
-            throw exceptionFactory.createException(errorEntity.code, errorEntity.description);
+            byte[] entity = EntityUtils.toByteArray(response.getEntity());
+            try {
+                ErrorResponseEntity errorEntity = objectMapper.readValue(entity, ErrorResponseEntity.class);
+                throw exceptionFactory.createException(errorEntity.code, errorEntity.description);
+            } catch (JsonMappingException e) {
+                logger.log(Level.INFO, String.format("Response returned status %d, but error response not parseable: %s", statusCode, new String(entity)), e);
+                throw new RuntimeException("Non JSON response from Athenz.");
+            }
         }
     }
 
@@ -80,6 +93,7 @@ public abstract class ClientBase implements AutoCloseable {
                 .setRetryHandler(new DefaultHttpRequestRetryHandler(3, /*requestSentRetryEnabled*/true))
                 .setUserAgent(userAgent)
                 .setSSLSocketFactory(new SSLConnectionSocketFactory(new ServiceIdentitySslSocketFactory(sslContextSupplier), hostnameVerifier))
+                .setMaxConnPerRoute(8)
                 .setDefaultRequestConfig(RequestConfig.custom()
                                                  .setConnectTimeout((int) Duration.ofSeconds(10).toMillis())
                                                  .setConnectionRequestTimeout((int)Duration.ofSeconds(10).toMillis())
diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/DefaultZtsClient.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/DefaultZtsClient.java
index eec578ddc26..c05213c8008 100644
--- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/DefaultZtsClient.java
+++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/DefaultZtsClient.java
@@ -2,14 +2,17 @@
 package com.yahoo.vespa.athenz.client.zts;
 
 import com.yahoo.security.Pkcs10Csr;
+import com.yahoo.vespa.athenz.api.AthenzAccessToken;
 import com.yahoo.vespa.athenz.api.AthenzDomain;
 import com.yahoo.vespa.athenz.api.AthenzIdentity;
+import com.yahoo.vespa.athenz.api.AthenzResourceName;
 import com.yahoo.vespa.athenz.api.AthenzRole;
 import com.yahoo.vespa.athenz.api.AwsRole;
 import com.yahoo.vespa.athenz.api.AwsTemporaryCredentials;
 import com.yahoo.vespa.athenz.api.NToken;
 import com.yahoo.vespa.athenz.api.ZToken;
 import com.yahoo.vespa.athenz.client.common.ClientBase;
+import com.yahoo.vespa.athenz.client.zts.bindings.AccessTokenResponseEntity;
 import com.yahoo.vespa.athenz.client.zts.bindings.AwsTemporaryCredentialsResponseEntity;
 import com.yahoo.vespa.athenz.client.zts.bindings.IdentityRefreshRequestEntity;
 import com.yahoo.vespa.athenz.client.zts.bindings.IdentityResponseEntity;
@@ -36,6 +39,7 @@ import java.time.Duration;
 import java.util.List;
 import java.util.Optional;
 import java.util.function.Supplier;
+import java.util.stream.Collectors;
 
 import static java.util.stream.Collectors.toList;
 
@@ -147,6 +151,33 @@ public class DefaultZtsClient extends ClientBase implements ZtsClient {
     }
 
     @Override
+    public AthenzAccessToken getAccessToken(AthenzDomain domain) {
+        return this.getAccessTokenImpl(List.of(new AthenzResourceName(domain, "domain")));
+    }
+
+    @Override
+    public AthenzAccessToken getAccessToken(List<AthenzRole> athenzRole) {
+        List<AthenzResourceName> athenzResourceNames = athenzRole.stream()
+                .map(AthenzRole::toResourceName)
+                .collect(toList());
+        return this.getAccessTokenImpl(athenzResourceNames);
+    }
+
+    private AthenzAccessToken getAccessTokenImpl(List<AthenzResourceName> resources) {
+        URI uri = ztsUrl.resolve("oauth2/token");
+        RequestBuilder requestBuilder = RequestBuilder.post(uri)
+                .addHeader("Content-Type", "application/x-www-form-urlencoded")
+                .addParameter("grant_type", "client_credentials")
+                .addParameter("scope", resources.stream().map(AthenzResourceName::toResourceNameString).collect(Collectors.joining(" ")));
+
+        HttpUriRequest request = requestBuilder.build();
+        return execute(request, response -> {
+            AccessTokenResponseEntity accessTokenResponseEntity = readEntity(response, AccessTokenResponseEntity.class);
+            return accessTokenResponseEntity.accessToken();
+        });
+    }
+
+    @Override
     public X509Certificate getRoleCertificate(AthenzRole role, Pkcs10Csr csr, Duration expiry) {
         RoleCertificateRequestEntity requestEntity = new RoleCertificateRequestEntity(csr, expiry);
         URI uri = ztsUrl.resolve(String.format("domain/%s/role/%s/token", role.domain().getName(), role.roleName()));
diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/ZtsClient.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/ZtsClient.java
index c09ad8f48a0..baeb4271905 100644
--- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/ZtsClient.java
+++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/ZtsClient.java
@@ -2,6 +2,7 @@
 package com.yahoo.vespa.athenz.client.zts;
 
 import com.yahoo.security.Pkcs10Csr;
+import com.yahoo.vespa.athenz.api.AthenzAccessToken;
 import com.yahoo.vespa.athenz.api.AthenzDomain;
 import com.yahoo.vespa.athenz.api.AthenzIdentity;
 import com.yahoo.vespa.athenz.api.AthenzRole;
@@ -78,6 +79,22 @@ public interface ZtsClient extends AutoCloseable {
     ZToken getRoleToken(AthenzRole athenzRole);
 
     /**
+     * Fetch an access token for the target domain
+     *
+     * @param domain Target domain
+     * @return An Athenz access token
+     */
+    AthenzAccessToken getAccessToken(AthenzDomain domain);
+
+    /**
+     * Fetch an access token for the target roles
+     *
+     * @param athenzRole List of athenz roles to get access token for
+     * @return An Athenz access token
+     */
+    AthenzAccessToken getAccessToken(List<AthenzRole> athenzRole);
+
+    /**
      * Fetch role certificate for the target domain and role
      *
      * @param role Target role
diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/AccessTokenResponseEntity.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/AccessTokenResponseEntity.java
new file mode 100644
index 00000000000..edd423b52bd
--- /dev/null
+++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/AccessTokenResponseEntity.java
@@ -0,0 +1,49 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.athenz.client.zts.bindings;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.yahoo.vespa.athenz.api.AthenzAccessToken;
+import com.yahoo.vespa.athenz.api.AthenzResourceName;
+import com.yahoo.vespa.athenz.api.AthenzRole;
+
+import java.time.Instant;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+/**
+ * @author mortent
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class AccessTokenResponseEntity {
+    private final AthenzAccessToken accessToken;
+    private final Instant expiryTime;
+    private final List<AthenzRole> roles;
+
+    public AccessTokenResponseEntity(
+            @JsonProperty("access_token") String accessToken,
+            @JsonProperty("expires_in") int expiresIn,
+            @JsonProperty("scope") String roles) {
+
+        this.accessToken =  new AthenzAccessToken(accessToken);
+        // We do not know from when, so best we can do is assume now ...
+        this.expiryTime = Instant.now().plusSeconds(expiresIn);
+        this.roles = Stream.of(roles.split(" "))
+                .map(AthenzResourceName::fromString)
+                .map(AthenzRole::fromResourceName)
+                .collect(Collectors.toList());
+    }
+
+    public AthenzAccessToken accessToken() {
+        return accessToken;
+    }
+
+    public Instant expiryTime() {
+        return expiryTime;
+    }
+
+    public List<AthenzRole> roles() {
+        return roles;
+    }
+}
diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/utils/RoleCsrGenerator.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/utils/RoleCsrGenerator.java
new file mode 100644
index 00000000000..102bfd82646
--- /dev/null
+++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/utils/RoleCsrGenerator.java
@@ -0,0 +1,39 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.athenz.client.zts.utils;
+
+import com.yahoo.security.Pkcs10Csr;
+import com.yahoo.security.Pkcs10CsrBuilder;
+import com.yahoo.security.SubjectAlternativeName.Type;
+import com.yahoo.vespa.athenz.api.AthenzIdentity;
+import com.yahoo.vespa.athenz.api.AthenzRole;
+import com.yahoo.vespa.athenz.client.zts.ZtsClient;
+
+import javax.security.auth.x500.X500Principal;
+import java.security.KeyPair;
+
+import static com.yahoo.security.SignatureAlgorithm.SHA256_WITH_RSA;
+
+/**
+ * Generates a {@link Pkcs10Csr} instance for use with {@link ZtsClient#getRoleCertificate(AthenzRole, Pkcs10Csr)}.
+ *
+ * @author bjorncs
+ */
+public class RoleCsrGenerator {
+
+    private final String dnsSuffix;
+
+    public RoleCsrGenerator(String dnsSuffix) {
+        this.dnsSuffix = dnsSuffix;
+    }
+
+    public Pkcs10Csr generateCsr(AthenzIdentity identity, AthenzRole role, KeyPair keyPair) {
+        return Pkcs10CsrBuilder.fromKeypair(new X500Principal("CN=" + role.toResourceNameString()), keyPair, SHA256_WITH_RSA)
+                .addSubjectAlternativeName(
+                        Type.DNS_NAME,
+                        String.format("%s.%s.%s", identity.getName(), identity.getDomainName().replace(".", "-"), dnsSuffix))
+                .addSubjectAlternativeName(
+                        Type.RFC822_NAME,
+                        String.format("%s@%s", identity.getFullName(), dnsSuffix))
+                .build();
+    }
+}
diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImpl.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImpl.java
index bea9af458b4..144ffaea5b4 100644
--- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImpl.java
+++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImpl.java
@@ -16,6 +16,7 @@ import com.yahoo.security.KeyStoreType;
 import com.yahoo.security.Pkcs10Csr;
 import com.yahoo.security.SslContextBuilder;
 import com.yahoo.security.tls.MutableX509KeyManager;
+import com.yahoo.vespa.athenz.api.AthenzAccessToken;
 import com.yahoo.vespa.athenz.api.AthenzDomain;
 import com.yahoo.vespa.athenz.api.AthenzRole;
 import com.yahoo.vespa.athenz.api.AthenzService;
@@ -84,6 +85,8 @@ public final class AthenzIdentityProviderImpl extends AbstractComponent implemen
     private final LoadingCache<AthenzRole, SSLContext> roleSslContextCache;
     private final LoadingCache<AthenzRole, ZToken> roleSpecificRoleTokenCache;
     private final LoadingCache<AthenzDomain, ZToken> domainSpecificRoleTokenCache;
+    private final LoadingCache<AthenzDomain, AthenzAccessToken> domainSpecificAccessTokenCache;
+    private final LoadingCache<List<AthenzRole>, AthenzAccessToken> roleSpecificAccessTokenCache;
     private final CsrGenerator csrGenerator;
 
     @Inject
@@ -116,6 +119,8 @@ public final class AthenzIdentityProviderImpl extends AbstractComponent implemen
         roleSslContextCache = createCache(ROLE_SSL_CONTEXT_EXPIRY, this::createRoleSslContext);
         roleSpecificRoleTokenCache = createCache(ROLE_TOKEN_EXPIRY, this::createRoleToken);
         domainSpecificRoleTokenCache = createCache(ROLE_TOKEN_EXPIRY, this::createRoleToken);
+        domainSpecificAccessTokenCache = createCache(ROLE_TOKEN_EXPIRY, this::createAccessToken);
+        roleSpecificAccessTokenCache = createCache(ROLE_TOKEN_EXPIRY, this::createAccessToken);
         this.csrGenerator = new CsrGenerator(config.athenzDnsSuffix(), config.configserverIdentityName());
         this.identitySslContext = createIdentitySslContext(identityKeyManager, trustStore);
         registerInstance();
@@ -199,11 +204,26 @@ public final class AthenzIdentityProviderImpl extends AbstractComponent implemen
     }
 
     @Override
+    public String getAccessToken(String domain) {
+        return null;
+    }
+
+    @Override
+    public String getAccessToken(String domain, List<String> roles) {
+        return null;
+    }
+
+    @Override
     public PrivateKey getPrivateKey() {
         return credentials.getKeyPair().getPrivate();
     }
 
     @Override
+    public Path trustStorePath() {
+        return trustStore;
+    }
+
+    @Override
     public List<X509Certificate> getIdentityCertificate() {
         return Collections.singletonList(credentials.getCertificate());
     }
@@ -240,6 +260,18 @@ public final class AthenzIdentityProviderImpl extends AbstractComponent implemen
         }
     }
 
+    private AthenzAccessToken createAccessToken(AthenzDomain domain) {
+        try (ZtsClient client = createZtsClient()) {
+            return client.getAccessToken(domain);
+        }
+    }
+
+    private AthenzAccessToken createAccessToken(List<AthenzRole> roles) {
+        try (ZtsClient client = createZtsClient()) {
+            return client.getAccessToken(roles);
+        }
+    }
+
     private DefaultZtsClient createZtsClient() {
         return new DefaultZtsClient(ztsEndpoint, getIdentitySslContext());
     }
diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/tls/AthenzX509CertificateUtils.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/tls/AthenzX509CertificateUtils.java
index 33e5552eaf6..bec21a5b25f 100644
--- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/tls/AthenzX509CertificateUtils.java
+++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/tls/AthenzX509CertificateUtils.java
@@ -17,20 +17,8 @@ import static com.yahoo.security.SubjectAlternativeName.Type.RFC822_NAME;
  */
 public class AthenzX509CertificateUtils {
 
-    private static final String COMMON_NAME_ROLE_DELIMITER = ":role.";
-
     private AthenzX509CertificateUtils() {}
 
-    public static boolean isAthenzRoleCertificate(X509Certificate certificate) {
-        return isAthenzIssuedCertificate(certificate) &&
-                com.yahoo.security.X509CertificateUtils.getSubjectCommonNames(certificate).get(0).contains(COMMON_NAME_ROLE_DELIMITER);
-    }
-
-    public static boolean isAthenzIssuedCertificate(X509Certificate certificate) {
-        return com.yahoo.security.X509CertificateUtils.getIssuerCommonNames(certificate).stream()
-                .anyMatch(cn -> cn.equalsIgnoreCase("Yahoo Athenz CA") || cn.equalsIgnoreCase("Athenz AWS CA"));
-    }
-
     public static AthenzIdentity getIdentityFromRoleCertificate(X509Certificate certificate) {
         List<com.yahoo.security.SubjectAlternativeName> sans = com.yahoo.security.X509CertificateUtils.getSubjectAlternativeNames(certificate);
         return sans.stream()
@@ -43,10 +31,7 @@ public class AthenzX509CertificateUtils {
 
     public static AthenzRole getRolesFromRoleCertificate(X509Certificate certificate) {
         String commonName = com.yahoo.security.X509CertificateUtils.getSubjectCommonNames(certificate).get(0);
-        int delimiterIndex = commonName.indexOf(COMMON_NAME_ROLE_DELIMITER);
-        String domain = commonName.substring(0, delimiterIndex);
-        String roleName = commonName.substring(delimiterIndex + COMMON_NAME_ROLE_DELIMITER.length());
-        return new AthenzRole(domain, roleName);
+        return AthenzRole.fromResourceNameString(commonName);
     }
 
     private static AthenzIdentity getIdentityFromSanEmail(String email) {
diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/zpe/AuthorizationResult.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/zpe/AuthorizationResult.java
index faf05011af9..13991199a1f 100644
--- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/zpe/AuthorizationResult.java
+++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/zpe/AuthorizationResult.java
@@ -2,45 +2,88 @@
 package com.yahoo.vespa.athenz.zpe;
 
 import com.yahoo.athenz.zpe.AuthZpeClient.AccessCheckStatus;
+import com.yahoo.vespa.athenz.api.AthenzRole;
 
 import java.util.Arrays;
+import java.util.Objects;
+import java.util.Optional;
 
 /**
  * The various types of access control results.
  *
  * @author bjorncs
  */
-public enum AuthorizationResult {
-    ALLOW(AccessCheckStatus.ALLOW),
-    DENY(AccessCheckStatus.DENY),
-    DENY_NO_MATCH(AccessCheckStatus.DENY_NO_MATCH),
-    DENY_ROLETOKEN_EXPIRED(AccessCheckStatus.DENY_ROLETOKEN_EXPIRED),
-    DENY_ROLETOKEN_INVALID(AccessCheckStatus.DENY_ROLETOKEN_INVALID),
-    DENY_DOMAIN_MISMATCH(AccessCheckStatus.DENY_DOMAIN_MISMATCH),
-    DENY_DOMAIN_NOT_FOUND(AccessCheckStatus.DENY_DOMAIN_NOT_FOUND),
-    DENY_DOMAIN_EXPIRED(AccessCheckStatus.DENY_DOMAIN_EXPIRED),
-    DENY_DOMAIN_EMPTY(AccessCheckStatus.DENY_DOMAIN_EMPTY),
-    DENY_INVALID_PARAMETERS(AccessCheckStatus.DENY_INVALID_PARAMETERS),
-    DENY_CERT_MISMATCH_ISSUER(AccessCheckStatus.DENY_CERT_MISMATCH_ISSUER),
-    DENY_CERT_MISSING_SUBJECT(AccessCheckStatus.DENY_CERT_MISSING_SUBJECT),
-    DENY_CERT_MISSING_DOMAIN(AccessCheckStatus.DENY_CERT_MISSING_DOMAIN),
-    DENY_CERT_MISSING_ROLE_NAME(AccessCheckStatus.DENY_CERT_MISSING_ROLE_NAME);
-
-    private final AccessCheckStatus wrappedElement;
-
-    AuthorizationResult(AccessCheckStatus wrappedElement) {
-        this.wrappedElement = wrappedElement;
+public class AuthorizationResult {
+
+    private final Type type;
+    private final AthenzRole matchedRole;
+
+    public AuthorizationResult(Type type) {
+        this(type, null);
+    }
+
+    public AuthorizationResult(Type type, AthenzRole matchedRole) {
+        this.type = type;
+        this.matchedRole = matchedRole;
+    }
+
+    public Type type() { return type; }
+    public Optional<AthenzRole> matchedRole() { return Optional.ofNullable(matchedRole); }
+
+    public enum Type {
+        ALLOW(AccessCheckStatus.ALLOW),
+        DENY(AccessCheckStatus.DENY),
+        DENY_NO_MATCH(AccessCheckStatus.DENY_NO_MATCH),
+        DENY_ROLETOKEN_EXPIRED(AccessCheckStatus.DENY_ROLETOKEN_EXPIRED),
+        DENY_ROLETOKEN_INVALID(AccessCheckStatus.DENY_ROLETOKEN_INVALID),
+        DENY_DOMAIN_MISMATCH(AccessCheckStatus.DENY_DOMAIN_MISMATCH),
+        DENY_DOMAIN_NOT_FOUND(AccessCheckStatus.DENY_DOMAIN_NOT_FOUND),
+        DENY_DOMAIN_EXPIRED(AccessCheckStatus.DENY_DOMAIN_EXPIRED),
+        DENY_DOMAIN_EMPTY(AccessCheckStatus.DENY_DOMAIN_EMPTY),
+        DENY_INVALID_PARAMETERS(AccessCheckStatus.DENY_INVALID_PARAMETERS),
+        DENY_CERT_MISMATCH_ISSUER(AccessCheckStatus.DENY_CERT_MISMATCH_ISSUER),
+        DENY_CERT_MISSING_SUBJECT(AccessCheckStatus.DENY_CERT_MISSING_SUBJECT),
+        DENY_CERT_MISSING_DOMAIN(AccessCheckStatus.DENY_CERT_MISSING_DOMAIN),
+        DENY_CERT_MISSING_ROLE_NAME(AccessCheckStatus.DENY_CERT_MISSING_ROLE_NAME),
+        DENY_CERT_HASH_MISMATCH(AccessCheckStatus.DENY_CERT_HASH_MISMATCH);
+
+        private final AccessCheckStatus wrappedElement;
+
+        Type(AccessCheckStatus wrappedElement) {
+            this.wrappedElement = wrappedElement;
+        }
+
+        public String getDescription() {
+            return wrappedElement.toString();
+        }
+
+        static Type fromAccessCheckStatus(AccessCheckStatus accessCheckStatus) {
+            return Arrays.stream(values())
+                    .filter(value -> value.wrappedElement == accessCheckStatus)
+                    .findFirst()
+                    .orElseThrow(() -> new IllegalArgumentException("Unknown status: " + accessCheckStatus));
+        }
     }
 
-    public String getDescription() {
-        return wrappedElement.toString();
+    @Override
+    public String toString() {
+        return "AuthorizationResult{" +
+                "type=" + type +
+                ", matchedRole=" + matchedRole +
+                '}';
     }
 
-    static AuthorizationResult fromAccessCheckStatus(AccessCheckStatus accessCheckStatus) {
-        return Arrays.stream(values())
-                .filter(value -> value.wrappedElement == accessCheckStatus)
-                .findFirst()
-                .orElseThrow(() -> new IllegalArgumentException("Unknown status: " + accessCheckStatus));
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        AuthorizationResult that = (AuthorizationResult) o;
+        return type == that.type &&
+                Objects.equals(matchedRole, that.matchedRole);
     }
 
+    @Override
+    public int hashCode() {
+        return Objects.hash(type, matchedRole);
+    }
 }
diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/zpe/DefaultZpe.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/zpe/DefaultZpe.java
index 29044111ada..47ae45a69ca 100644
--- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/zpe/DefaultZpe.java
+++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/zpe/DefaultZpe.java
@@ -2,8 +2,11 @@
 package com.yahoo.vespa.athenz.zpe;
 
 import com.yahoo.athenz.zpe.AuthZpeClient;
+import com.yahoo.vespa.athenz.api.AthenzAccessToken;
 import com.yahoo.vespa.athenz.api.AthenzResourceName;
+import com.yahoo.vespa.athenz.api.AthenzRole;
 import com.yahoo.vespa.athenz.api.ZToken;
+import com.yahoo.vespa.athenz.zpe.AuthorizationResult.Type;
 
 import java.security.cert.X509Certificate;
 
@@ -21,14 +24,41 @@ public class DefaultZpe implements Zpe {
 
     @Override
     public AuthorizationResult checkAccessAllowed(ZToken roleToken, AthenzResourceName resourceName, String action) {
-        return AuthorizationResult.fromAccessCheckStatus(
-                AuthZpeClient.allowAccess(roleToken.getRawToken(), resourceName.toResourceNameString(), action));
+        StringBuilder returnedMatchedRole = new StringBuilder();
+        AuthZpeClient.AccessCheckStatus rawResult =
+                AuthZpeClient.allowAccess(roleToken.getRawToken(), resourceName.toResourceNameString(), action, returnedMatchedRole);
+        return createResult(returnedMatchedRole, rawResult, resourceName);
     }
 
     @Override
     public AuthorizationResult checkAccessAllowed(X509Certificate roleCertificate, AthenzResourceName resourceName, String action) {
-        return AuthorizationResult.fromAccessCheckStatus(
-                AuthZpeClient.allowAccess(roleCertificate, resourceName.toResourceNameString(), action));
+        StringBuilder returnedMatchedRole = new StringBuilder();
+        AuthZpeClient.AccessCheckStatus rawResult =
+                AuthZpeClient.allowAccess(roleCertificate, resourceName.toResourceNameString(), action, returnedMatchedRole);
+        return createResult(returnedMatchedRole, rawResult, resourceName);
+    }
+
+    @Override
+    public AuthorizationResult checkAccessAllowed(
+            AthenzAccessToken accessToken, X509Certificate identityCertificate, AthenzResourceName resourceName, String action) {
+        StringBuilder returnedMatchedRole = new StringBuilder();
+        AuthZpeClient.AccessCheckStatus rawResult =
+                AuthZpeClient.allowAccess(
+                        accessToken.value(), identityCertificate, /*certHash*/null, resourceName.toResourceNameString(), action, returnedMatchedRole);
+        return createResult(returnedMatchedRole, rawResult, resourceName);
+    }
+
+    private static AuthorizationResult createResult(
+            StringBuilder matchedRole, AuthZpeClient.AccessCheckStatus rawResult, AthenzResourceName resourceName) {
+        return new AuthorizationResult(Type.fromAccessCheckStatus(rawResult), toRole(matchedRole, resourceName));
+    }
+
+    private static AthenzRole toRole(StringBuilder rawRole, AthenzResourceName resourceName) {
+        if (rawRole.length() == 0) {
+            return null;
+        } else {
+            return new AthenzRole(resourceName.getDomain(), rawRole.toString());
+        }
     }
 
 }
diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/zpe/Zpe.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/zpe/Zpe.java
index e22e27f1508..51e5ee4dbb1 100644
--- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/zpe/Zpe.java
+++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/zpe/Zpe.java
@@ -1,6 +1,7 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.athenz.zpe;
 
+import com.yahoo.vespa.athenz.api.AthenzAccessToken;
 import com.yahoo.vespa.athenz.api.AthenzResourceName;
 import com.yahoo.vespa.athenz.api.ZToken;
 
@@ -14,4 +15,5 @@ import java.security.cert.X509Certificate;
 public interface Zpe {
     AuthorizationResult checkAccessAllowed(ZToken roleToken, AthenzResourceName resourceName, String action);
     AuthorizationResult checkAccessAllowed(X509Certificate roleCertificate, AthenzResourceName resourceName, String action);
+    AuthorizationResult checkAccessAllowed(AthenzAccessToken accessToken, X509Certificate identityCertificate, AthenzResourceName resourceName, String action);
 }
diff --git a/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java b/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java
index d95eddac57f..95b6528bd77 100644
--- a/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java
+++ b/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java
@@ -465,7 +465,8 @@ public class DocumentGenMojo extends AbstractMojo {
         exportStructTypeGetter(docType.getName()+".body", docType.allBody().getFields(), out, 1, "getBodyStructType", "com.yahoo.document.StructDataType");
 
         Collection<Field> allUniqueFields = getAllUniqueFields(multiExtends, docType.getAllFields());
-        exportExtendedStructTypeGetter(className, docType.getName(), allUniqueFields, docType.getFieldSets(),out, 1, "getDocumentType", "com.yahoo.document.DocumentType");
+        exportExtendedStructTypeGetter(className, docType.getName(), allUniqueFields, docType.getFieldSets(),
+                docType.getImportedFieldNames(), out, 1, "getDocumentType", "com.yahoo.document.DocumentType");
         exportCopyConstructor(className, out, 1, true);
 
         exportFieldsAndAccessors(className, "com.yahoo.document.Document".equals(superType) ? allUniqueFields : docType.getFields(), out, 1, true);
@@ -627,10 +628,21 @@ public class DocumentGenMojo extends AbstractMojo {
         }
         out.write(ind(ind) + "ret.addFieldSets(fieldSets);\n");
     }
+    private static void exportImportedFields(Set<String> importedFieldNames, Writer out, int ind) throws IOException {
+        out.write(ind(ind) + "java.util.Set<java.lang.String> importedFieldNames = new java.util.HashSet<>();\n");
+        for (String importedField : importedFieldNames) {
+            out.write(ind(ind) + "importedFieldNames.add(\"" + importedField + "\");\n");
+        }
+    }
     private static void exportExtendedStructTypeGetter(String className, String name, Collection<Field> fields, Set<FieldSet> fieldSets,
-                                                       Writer out, int ind, String methodName, String retType) throws IOException {
+                                                       Set<String> importedFieldNames, Writer out, int ind, String methodName, String retType) throws IOException {
         out.write(ind(ind)+"private static "+retType+" "+methodName+"() {\n");
-        out.write(ind(ind+1)+retType+" ret = new "+retType+"(\""+name+"\");\n");
+        if (importedFieldNames != null) {
+            exportImportedFields(importedFieldNames, out, ind + 1);
+            out.write(ind(ind+1)+retType+" ret = new "+retType+"(\"" + name + "\", importedFieldNames);\n");
+        } else {
+            out.write(ind(ind+1)+retType+" ret = new "+retType+"(\""+name+"\");\n");
+        }
         for (Field f : fields) {
             if (f.getDataType().equals(DataType.STRING)) {
                 addExtendedStringField(className, f, out, ind + 1);
@@ -783,7 +795,7 @@ public class DocumentGenMojo extends AbstractMojo {
                 ind(ind+2)+"super("+structClassName+".type);\n" +
                 ind(ind+1)+"}\n\n");
                 exportCopyConstructor(structClassName, out, ind+1, false);
-                exportExtendedStructTypeGetter(structClassName, structType.getName(), structType.getFields(), null, out, ind+1, "getStructType", "com.yahoo.document.StructDataType");
+                exportExtendedStructTypeGetter(structClassName, structType.getName(), structType.getFields(), null, null, out, ind+1, "getStructType", "com.yahoo.document.StructDataType");
                 exportAssign(structType, structClassName, out, ind+1);
                 exportFieldsAndAccessors(structClassName, structType.getFields(), out, ind+1, true);
 
diff --git a/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/pig/VespaDocumentOperation.java b/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/pig/VespaDocumentOperation.java
index fa463a77923..b6fe19f8cde 100644
--- a/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/pig/VespaDocumentOperation.java
+++ b/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/pig/VespaDocumentOperation.java
@@ -6,18 +6,21 @@ import com.fasterxml.jackson.core.JsonFactory;
 import com.fasterxml.jackson.core.JsonGenerator;
 import com.yahoo.vespa.hadoop.mapreduce.util.TupleTools;
 import com.yahoo.vespa.hadoop.mapreduce.util.VespaConfiguration;
-import org.apache.commons.lang.exception.ExceptionUtils;
 import org.apache.pig.EvalFunc;
 import org.apache.pig.PigWarning;
 import org.apache.pig.data.DataBag;
 import org.apache.pig.data.DataByteArray;
 import org.apache.pig.data.DataType;
 import org.apache.pig.data.Tuple;
+import org.apache.pig.tools.pigstats.PigStatusReporter;
 import org.apache.pig.impl.logicalLayer.schema.Schema;
 import org.joda.time.DateTime;
 
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
+import java.io.PrintWriter;
+import java.io.StringWriter;
+import java.io.UncheckedIOException;
 import java.math.BigDecimal;
 import java.math.BigInteger;
 import java.util.*;
@@ -64,35 +67,74 @@ public class VespaDocumentOperation extends EvalFunc<String> {
     private static final String PROPERTY_CREATE_IF_NON_EXISTENT = "create-if-non-existent";
     private static final String PROPERTY_ID_TEMPLATE = "docid";
     private static final String PROPERTY_OPERATION = "operation";
+    private static final String PROPERTY_VERBOSE = "verbose";
     private static final String BAG_AS_MAP_FIELDS = "bag-as-map-fields";
     private static final String SIMPLE_ARRAY_FIELDS = "simple-array-fields";
     private static final String SIMPLE_OBJECT_FIELDS = "simple-object-fields";
     private static final String CREATE_TENSOR_FIELDS = "create-tensor-fields";
+    private static final String REMOVE_TENSOR_FIELDS = "remove-tensor-fields";
+    private static final String UPDATE_TENSOR_FIELDS = "update-tensor-fields";
+    private static final String REMOVE_MAP_FIELDS = "remove-map-fields";
+    private static final String UPDATE_MAP_FIELDS = "update-map-fields";
     private static final String EXCLUDE_FIELDS = "exclude-fields";
-
+    private static final String TESTSET_CONDITION = "condition";
     private static final String PARTIAL_UPDATE_ASSIGN = "assign";
+    private static final String PARTIAL_UPDATE_ADD = "add";
+    private static final String PARTIAL_UPDATE_REMOVE = "remove";
+
+    private static Map<String, String> mapPartialOperationMap;
+
+    static {
+        mapPartialOperationMap = new HashMap<>();
+        mapPartialOperationMap.put(REMOVE_MAP_FIELDS, PARTIAL_UPDATE_REMOVE);
+        mapPartialOperationMap.put(UPDATE_MAP_FIELDS, PARTIAL_UPDATE_ASSIGN);
+    }
+
+    private static Map<String, String> partialOperationMap;
+
+    static {
+        partialOperationMap = new HashMap<>();
+        partialOperationMap.put(REMOVE_TENSOR_FIELDS, PARTIAL_UPDATE_REMOVE);
+        partialOperationMap.put(UPDATE_TENSOR_FIELDS, PARTIAL_UPDATE_ADD);
+        partialOperationMap.put(REMOVE_MAP_FIELDS, PARTIAL_UPDATE_REMOVE);
+        partialOperationMap.put(UPDATE_MAP_FIELDS, PARTIAL_UPDATE_ASSIGN);
+    }
 
+    private final boolean verbose;
     private final String template;
     private final Operation operation;
     private final Properties properties;
+    private PigStatusReporter statusReporter;
 
     public VespaDocumentOperation(String... params) {
+        statusReporter = PigStatusReporter.getInstance();
+        if (statusReporter != null) {
+            statusReporter.incrCounter("Vespa Document Operation Counters", "Document operation ok", 0);
+            statusReporter.incrCounter("Vespa Document Operation Counters", "Document operation failed", 0);
+        }
         properties = VespaConfiguration.loadProperties(params);
         template = properties.getProperty(PROPERTY_ID_TEMPLATE);
         operation = Operation.fromString(properties.getProperty(PROPERTY_OPERATION, "put"));
+        verbose = Boolean.parseBoolean(properties.getProperty(PROPERTY_VERBOSE, "false"));
     }
 
     @Override
     public String exec(Tuple tuple) throws IOException {
         if (tuple == null || tuple.size() == 0) {
+            if (statusReporter != null) {
+                statusReporter.incrCounter("Vespa Document Operation Counters", "Document operation failed", 1);
+            }
             return null;
         }
         if (template == null || template.length() == 0) {
-            warn("No valid document id template found. Skipping.", PigWarning.UDF_WARNING_1);
+            if (statusReporter != null) {
+                statusReporter.incrCounter("Vespa Document Operation Counters", "Document operation failed", 1);
+            }
+            warnLog("No valid document id template found. Skipping.", PigWarning.UDF_WARNING_1);
             return null;
         }
         if (operation == null) {
-            warn("No valid operation found. Skipping.", PigWarning.UDF_WARNING_1);
+            warnLog("No valid operation found. Skipping.", PigWarning.UDF_WARNING_2);
             return null;
         }
 
@@ -106,25 +148,32 @@ public class VespaDocumentOperation extends EvalFunc<String> {
             Schema inputSchema = getInputSchema();
             Map<String, Object> fields = TupleTools.tupleMap(inputSchema, tuple);
             String docId = TupleTools.toString(fields, template);
-
+            if (verbose) {
+                System.out.println("Processing docId: "+ docId);
+            }
             // create json
             json = create(operation, docId, fields, properties, inputSchema);
             if (json == null || json.length() == 0) {
-                warn("No valid document operation could be created.", PigWarning.UDF_WARNING_1);
+                warnLog("No valid document operation could be created.", PigWarning.UDF_WARNING_3);
                 return null;
             }
 
 
         } catch (Exception e) {
+            if (statusReporter != null) {
+                statusReporter.incrCounter("Vespa Document Operation Counters", "Document operation failed", 1);
+            }
             StringBuilder sb = new StringBuilder();
             sb.append("Caught exception processing input row: \n");
             sb.append(tuple.toString());
             sb.append("\nException: ");
-            sb.append(ExceptionUtils.getStackTrace(e));
-            warn(sb.toString(), PigWarning.UDF_WARNING_1);
+            sb.append(getStackTraceAsString(e));
+            warnLog(sb.toString(), PigWarning.UDF_WARNING_4);
             return null;
         }
-
+        if (statusReporter != null) {
+            statusReporter.incrCounter("Vespa Document Operation Counters", "Document operation ok", 1);
+        }
         return json;
     }
 
@@ -133,14 +182,14 @@ public class VespaDocumentOperation extends EvalFunc<String> {
      * Create a JSON Vespa document operation given the supplied fields,
      * operation and document id template.
      *
-     * @param op        Operation (put, remove, update)
-     * @param docId     Document id
-     * @param fields    Fields to put in document operation
-     * @return          A valid JSON Vespa document operation
+     * @param op     Operation (put, remove, update)
+     * @param docId  Document id
+     * @param fields Fields to put in document operation
+     * @return A valid JSON Vespa document operation
      * @throws IOException ...
      */
     public static String create(Operation op, String docId, Map<String, Object> fields, Properties properties,
-            Schema schema) throws IOException {
+                                Schema schema) throws IOException {
         if (op == null) {
             return null;
         }
@@ -162,7 +211,11 @@ public class VespaDocumentOperation extends EvalFunc<String> {
         if (op == Operation.UPDATE && createIfNonExistent) {
             writeField("create", true, DataType.BOOLEAN, g, properties, schema, op, 0);
         }
-
+        String testSetConditionTemplate = properties.getProperty(TESTSET_CONDITION);
+        if (testSetConditionTemplate != null) {
+            String testSetCondition = TupleTools.toString(fields, testSetConditionTemplate);
+            writeField(TESTSET_CONDITION, testSetCondition, DataType.CHARARRAY, g, properties, schema, op, 0);
+        }
         if (op != Operation.REMOVE) {
             writeField("fields", fields, DataType.MAP, g, properties, schema, op, 0);
         }
@@ -173,15 +226,72 @@ public class VespaDocumentOperation extends EvalFunc<String> {
         return out.toString();
     }
 
+    private static String getPartialOperation(Map<String, String> operationMap, String name, Properties properties) {
+        // This function checks if the property of the name falls into the map provided
+        // if yes, return the desired operation. if no, return null
+        // for example, input:
+        // operationMap map{"update-map-fields":"assign","remove-map-fields":"remove"}
+        // name date
+        // properties "update-map-fields":"date,month"
+        // output: assign
+        for (String label : operationMap.keySet()) {
+            if (properties.getProperty(label) != null) {
+                String[] p = properties.getProperty(label).split(",");
+                if (Arrays.asList(p).contains(name)) {
+                    return operationMap.get(label);
+                }
+            }
+        }
+        return null;
+    }
 
     @SuppressWarnings("unchecked")
     private static void writeField(String name, Object value, Byte type, JsonGenerator g, Properties properties, Schema schema, Operation op, int depth) throws IOException {
         if (shouldWriteField(name, properties, depth)) {
-            g.writeFieldName(name);
-            if (shouldWritePartialUpdate(op, depth)) {
-                writePartialUpdate(value, type, g, name, properties, schema, op, depth);
+            String operation = getPartialOperation(mapPartialOperationMap, name, properties);
+            // check if the name has the property update-map-fields/remove-map-fields
+            // if yes, we need special treatments here as we need to loop through the tuple
+            // be aware the the operation here is not vespa operation such as "put" and "update"
+            // operation here are the field name we wish use to such as "assign" and "remove"
+            if (operation != null) {
+                writePartialUpdateAndRemoveMap(name, value, g, properties, schema, op, depth, operation);
             } else {
-                writeValue(value, type, g, name, properties, schema, op, depth);
+                g.writeFieldName(name);
+                if (shouldWritePartialUpdate(op, depth)) {
+                    writePartialUpdate(value, type, g, name, properties, schema, op, depth);
+                } else {
+                    writeValue(value, type, g, name, properties, schema, op, depth);
+                }
+            }
+
+        }
+    }
+
+    private static void writePartialUpdateAndRemoveMap(String name, Object value, JsonGenerator g, Properties properties, Schema schema, Operation op, int depth, String operation) throws IOException {
+        schema = (schema != null) ? schema.getField(0).schema : null;
+        // extract the key of map and keys in map for writing json when partial updating maps
+        Schema valueSchema = (schema != null) ? schema.getField(1).schema : null;
+        // data format  { ( key; id, value: (abc,123,(123234,bbaa))) }
+        // the first element of each tuple in the bag will be the map to update
+        // the second element of each tuple in the bag will be the new value of the map
+        DataBag bag = (DataBag) value;
+        for (Tuple element : bag) {
+            if (element.size() != 2) {
+                continue;
+            }
+            String k = (String) element.get(0);
+            Object v = element.get(1);
+            Byte t = DataType.findType(v);
+            if (t == DataType.TUPLE) {
+                g.writeFieldName(name + "{" + k + "}");
+                if (operation.equals(PARTIAL_UPDATE_REMOVE)) {
+                    g.writeStartObject();
+                    g.writeFieldName(PARTIAL_UPDATE_REMOVE);
+                    g.writeNumber(0);
+                    g.writeEndObject();
+                } else {
+                    writePartialUpdate(v, t, g, name, properties, valueSchema, op, depth);
+                }
             }
         }
     }
@@ -230,14 +340,18 @@ public class VespaDocumentOperation extends EvalFunc<String> {
                 g.writeStartObject();
                 Map<Object, Object> map = (Map<Object, Object>) value;
                 if (shouldCreateTensor(map, name, properties)) {
-                    writeTensor(map, g);
+                    if (isRemoveTensor(name, properties)) {
+                        writeRemoveTensor(map, g);
+                    } else {
+                        writeTensor(map, g);
+                    }
                 } else {
                     for (Map.Entry<Object, Object> entry : map.entrySet()) {
                         String k = entry.getKey().toString();
                         Object v = entry.getValue();
-                        Byte   t = DataType.findType(v);
+                        Byte t = DataType.findType(v);
                         Schema fieldSchema = (schema != null) ? schema.getField(k).schema : null;
-                        writeField(k, v, t, g, properties, fieldSchema, op, depth+1);
+                        writeField(k, v, t, g, properties, fieldSchema, op, depth + 1);
                     }
                 }
                 g.writeEndObject();
@@ -264,7 +378,6 @@ public class VespaDocumentOperation extends EvalFunc<String> {
                 DataBag bag = (DataBag) value;
                 // get the schema of the tuple in bag
                 schema = (schema != null) ? schema.getField(0).schema : null;
-
                 if (shouldWriteBagAsMap(name, properties)) {
                     // when treating bag as map, the schema of bag should be {(key, val)....}
                     // the size of tuple in bag should be 2. 1st one is key. 2nd one is val.
@@ -280,9 +393,9 @@ public class VespaDocumentOperation extends EvalFunc<String> {
                         Byte t = DataType.findType(v);
                         if (t == DataType.TUPLE) {
                             Map<String, Object> fields = TupleTools.tupleMap(valueSchema, (Tuple) v);
-                            writeField(k, fields, DataType.MAP, g, properties, valueSchema, op, depth);
+                            writeField(k, fields, DataType.MAP, g, properties, valueSchema, op, depth + 1);
                         } else {
-                            writeField(k, v, t, g, properties, valueSchema, op, depth);
+                            writeField(k, v, t, g, properties, valueSchema, op, depth + 1);
                         }
                     }
                     g.writeEndObject();
@@ -304,7 +417,15 @@ public class VespaDocumentOperation extends EvalFunc<String> {
 
     private static void writePartialUpdate(Object value, Byte type, JsonGenerator g, String name, Properties properties, Schema schema, Operation op, int depth) throws IOException {
         g.writeStartObject();
-        g.writeFieldName(PARTIAL_UPDATE_ASSIGN); // TODO: lookup field name in a property to determine correct operation
+        // here we check if the operation falls into the four partial operations we do on map/tensor structure
+        // if no, we assume it's a update on the whole document and we write assign here
+        // if yes, we write the desired operation here
+        String operation = getPartialOperation(partialOperationMap, name, properties);
+        if (operation != null) {
+            g.writeFieldName(operation);
+        } else {
+            g.writeFieldName(PARTIAL_UPDATE_ASSIGN);
+        }
         writeValue(value, type, g, name, properties, schema, op, depth);
         g.writeEndObject();
     }
@@ -330,21 +451,38 @@ public class VespaDocumentOperation extends EvalFunc<String> {
     }
 
     private static boolean shouldWriteTupleAsMap(String name, Properties properties) {
+        // include UPDATE_MAP_FIELDS here because when updating the map
+        // the second element in each tuple should be written as a map
         if (properties == null) {
             return false;
         }
+        String addBagAsMapFields = properties.getProperty(UPDATE_MAP_FIELDS);
         String simpleObjectFields = properties.getProperty(SIMPLE_OBJECT_FIELDS);
-        if (simpleObjectFields == null) {
+        if (simpleObjectFields == null && addBagAsMapFields == null) {
             return false;
         }
-        if (simpleObjectFields.equals("*")) {
-            return true;
+        if (addBagAsMapFields != null) {
+            if (addBagAsMapFields.equals("*")) {
+                return true;
+            }
+            String[] fields = addBagAsMapFields.split(",");
+            for (String field : fields) {
+                if (field.trim().equalsIgnoreCase(name)) {
+                    return true;
+                }
+            }
+
         }
-        String[] fields = simpleObjectFields.split(",");
-        for (String field : fields) {
-            if (field.trim().equalsIgnoreCase(name)) {
+        if (simpleObjectFields != null) {
+            if (simpleObjectFields.equals("*")) {
                 return true;
             }
+            String[] fields = simpleObjectFields.split(",");
+            for (String field : fields) {
+                if (field.trim().equalsIgnoreCase(name)) {
+                    return true;
+                }
+            }
         }
         return false;
     }
@@ -373,11 +511,50 @@ public class VespaDocumentOperation extends EvalFunc<String> {
         if (properties == null) {
             return false;
         }
-        String tensorFields = properties.getProperty(CREATE_TENSOR_FIELDS);
-        if (tensorFields == null) {
+        String createTensorFields = properties.getProperty(CREATE_TENSOR_FIELDS);
+        String addTensorFields = properties.getProperty(UPDATE_TENSOR_FIELDS);
+        String removeTensorFields = properties.getProperty(REMOVE_TENSOR_FIELDS);
+
+        if (createTensorFields == null && addTensorFields == null && removeTensorFields == null) {
+            return false;
+        }
+        String[] fields;
+        if (createTensorFields != null) {
+            fields = createTensorFields.split(",");
+            for (String field : fields) {
+                if (field.trim().equalsIgnoreCase(name)) {
+                    return true;
+                }
+            }
+        }
+        if (addTensorFields != null) {
+            fields = addTensorFields.split(",");
+            for (String field : fields) {
+                if (field.trim().equalsIgnoreCase(name)) {
+                    return true;
+                }
+            }
+        }
+        if (removeTensorFields != null) {
+            fields = removeTensorFields.split(",");
+            for (String field : fields) {
+                if (field.trim().equalsIgnoreCase(name)) {
+                    return true;
+                }
+            }
+        }
+        return false;
+    }
+
+    private static boolean isRemoveTensor(String name, Properties properties) {
+        if (properties == null) {
+            return false;
+        }
+        String removeTensorFields = properties.getProperty(REMOVE_TENSOR_FIELDS);
+        if (removeTensorFields == null) {
             return false;
         }
-        String[] fields = tensorFields.split(",");
+        String[] fields = removeTensorFields.split(",");
         for (String field : fields) {
             if (field.trim().equalsIgnoreCase(name)) {
                 return true;
@@ -444,4 +621,49 @@ public class VespaDocumentOperation extends EvalFunc<String> {
         g.writeEndArray();
     }
 
+    private static void writeRemoveTensor(Map<Object, Object> map, JsonGenerator g) throws IOException {
+        g.writeFieldName("addresses");
+        g.writeStartArray();
+        for (Map.Entry<Object, Object> entry : map.entrySet()) {
+            String k = entry.getKey().toString();
+            String[] dimensions = k.split(",");
+            for (String dimension : dimensions) {
+                g.writeStartObject();
+                if (dimension == null || dimension.isEmpty()) {
+                    continue;
+                }
+                String[] address = dimension.split(":");
+                if (address.length != 2) {
+                    throw new IllegalArgumentException("Malformed cell address: " + dimension);
+                }
+                String dim = address[0];
+                String label = address[1];
+                if (dim == null || label == null || dim.isEmpty() || label.isEmpty()) {
+                    throw new IllegalArgumentException("Malformed cell address: " + dimension);
+                }
+                g.writeFieldName(dim.trim());
+                g.writeString(label.trim());
+                g.writeEndObject();
+                // Write address
+            }
+        }
+        g.writeEndArray();
+    }
+
+    // copied from vespajlib for reducing dependency and building with JDK 8
+    private static String getStackTraceAsString(Throwable throwable) {
+        try (StringWriter stringWriter = new StringWriter();
+             PrintWriter printWriter = new PrintWriter(stringWriter, true)) {
+            throwable.printStackTrace(printWriter);
+            return stringWriter.getBuffer().toString();
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+    }
+
+    // wrapper to emit logs
+    private void warnLog(String msg, PigWarning warning) {
+        warn(msg, warning);
+        System.err.println(msg);
+    }
 }
diff --git a/vespa-hadoop/src/test/java/com/yahoo/vespa/hadoop/pig/VespaDocumentOperationTest.java b/vespa-hadoop/src/test/java/com/yahoo/vespa/hadoop/pig/VespaDocumentOperationTest.java
index 7d0fe72fc64..e278026b00d 100644
--- a/vespa-hadoop/src/test/java/com/yahoo/vespa/hadoop/pig/VespaDocumentOperationTest.java
+++ b/vespa-hadoop/src/test/java/com/yahoo/vespa/hadoop/pig/VespaDocumentOperationTest.java
@@ -6,19 +6,37 @@ import org.apache.pig.impl.logicalLayer.FrontendException;
 import org.apache.pig.impl.logicalLayer.schema.Schema;
 import org.codehaus.jackson.JsonNode;
 import org.codehaus.jackson.map.ObjectMapper;
+import org.hamcrest.CoreMatchers;
+import org.junit.After;
+import org.junit.Before;
 import org.junit.Test;
 
+import java.io.ByteArrayOutputStream;
 import java.io.IOException;
+import java.io.PrintStream;
 import java.util.HashMap;
+import java.util.Iterator;
 import java.util.Map;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertThat;
 
 @SuppressWarnings("serial")
 public class VespaDocumentOperationTest {
+    private final ByteArrayOutputStream outContent = new ByteArrayOutputStream();
+    private final PrintStream originalOut = System.out;
 
+    @Before
+    public void setUpStreams() {
+        System.setOut(new PrintStream(outContent));
+    }
+
+    @After
+    public void restoreStreams() {
+        System.setOut(originalOut);
+    }
     @Test
     public void requireThatUDFReturnsCorrectJson() throws Exception {
         String json = getDocumentOperationJson("docid=id:<application>:metrics::<name>-<date>");
@@ -47,6 +65,19 @@ public class VespaDocumentOperationTest {
         assertEquals(3, fields.get("value").get("assign").getIntValue());
     }
 
+    @Test
+    public void requireThatUDFSupportsConditionalUpdateAssign() throws IOException {
+        String json = getDocumentOperationJson("docid=id:<application>:metrics::<name>-<date>", "operation=update", "condition=clicks < <value>");
+        ObjectMapper m = new ObjectMapper();
+        JsonNode root = m.readTree(json);
+        JsonNode fields = root.path("fields");
+
+        assertEquals("id:testapp:metrics::clicks-20160112", root.get("update").getTextValue());
+        assertEquals("clicks < 3", root.get("condition").getTextValue());
+        assertEquals("testapp", fields.get("application").get("assign").getTextValue());
+        assertEquals("clicks", fields.get("name").get("assign").getTextValue());
+        assertEquals(3, fields.get("value").get("assign").getIntValue());
+    }
 
     @Test
     public void requireThatUDFSupportsCreateIfNonExistent() throws IOException {
@@ -72,6 +103,189 @@ public class VespaDocumentOperationTest {
 
 
     @Test
+    public void requireThatUDFCorrectlyGeneratesRemoveBagAsMapOperation() throws Exception {
+        DataBag bag = BagFactory.getInstance().newDefaultBag();
+
+        Schema innerObjectSchema = new Schema();
+        Tuple innerObjectTuple = TupleFactory.getInstance().newTuple();
+        addToTuple("year", DataType.CHARARRAY, "2020", innerObjectSchema, innerObjectTuple);
+        addToTuple("month", DataType.INTEGER, 3, innerObjectSchema, innerObjectTuple);
+
+        Schema objectSchema = new Schema();
+        Tuple objectTuple = TupleFactory.getInstance().newTuple();
+        addToTuple("key", DataType.CHARARRAY, "234566", objectSchema, objectTuple);
+        addToTuple("value", DataType.TUPLE, innerObjectTuple,innerObjectSchema,objectSchema, objectTuple);
+
+        Schema bagSchema = new Schema();
+        addToBagWithSchema("firstLayerTuple",DataType.TUPLE,objectTuple,objectSchema,bagSchema,bag);
+
+        innerObjectSchema = new Schema();
+        innerObjectTuple = TupleFactory.getInstance().newTuple();
+        addToTuple("year", DataType.CHARARRAY, "2020", innerObjectSchema, innerObjectTuple);
+        addToTuple("month", DataType.INTEGER, 3, innerObjectSchema, innerObjectTuple);
+
+        objectSchema = new Schema();
+        objectTuple = TupleFactory.getInstance().newTuple();
+        addToTuple("key", DataType.CHARARRAY, "123456", objectSchema, objectTuple);
+        addToTuple("value", DataType.TUPLE, innerObjectTuple,innerObjectSchema,objectSchema, objectTuple);
+
+        addToBagWithSchema("firstLayerTuple",DataType.TUPLE,objectTuple,objectSchema,bagSchema,bag);
+
+        Schema schema = new Schema();
+        Tuple tuple = TupleFactory.getInstance().newTuple();
+        addToTuple("bag", DataType.BAG, bag, bagSchema, schema, tuple);
+        addToTuple("id", DataType.CHARARRAY, "123", schema, tuple);
+
+        VespaDocumentOperation docOp = new VespaDocumentOperation("docid=id", "remove-map-fields=bag","operation=update");
+        docOp.setInputSchema(schema);
+        String json = docOp.exec(tuple);
+
+        ObjectMapper m = new ObjectMapper();
+        JsonNode root = m.readTree(json);
+        JsonNode fields = root.get("fields");
+        assertEquals("{\"remove\":0}", fields.get("bag{123456}").toString());
+        assertEquals("{\"remove\":0}", fields.get("bag{234566}").toString());
+
+    }
+
+    @Test
+    public void requireThatUDFCorrectlyGeneratesAddBagAsMapOperation() throws Exception {
+
+        DataBag bag = BagFactory.getInstance().newDefaultBag();
+
+        Schema innerObjectSchema = new Schema();
+        Tuple innerObjectTuple = TupleFactory.getInstance().newTuple();
+        addToTuple("year", DataType.CHARARRAY, "2020", innerObjectSchema, innerObjectTuple);
+        addToTuple("month", DataType.INTEGER, 3, innerObjectSchema, innerObjectTuple);
+
+        Schema objectSchema = new Schema();
+        Tuple objectTuple = TupleFactory.getInstance().newTuple();
+        addToTuple("key", DataType.CHARARRAY, "123456", objectSchema, objectTuple);
+        addToTuple("value", DataType.TUPLE, innerObjectTuple,innerObjectSchema,objectSchema, objectTuple);
+
+        Schema bagSchema = new Schema();
+        addToBagWithSchema("firstLayerTuple",DataType.TUPLE,objectTuple,objectSchema,bagSchema,bag);
+
+        Schema schema = new Schema();
+        Tuple tuple = TupleFactory.getInstance().newTuple();
+        addToTuple("bag", DataType.BAG, bag, bagSchema, schema, tuple);
+        addToTuple("id", DataType.CHARARRAY, "123", schema, tuple);
+        VespaDocumentOperation docOp = new VespaDocumentOperation("docid=id", "update-map-fields=bag","operation=update");
+        docOp.setInputSchema(schema);
+        String json = docOp.exec(tuple);
+
+        ObjectMapper m = new ObjectMapper();
+        JsonNode root = m.readTree(json);
+
+        JsonNode fields = root.get("fields");
+        JsonNode value = fields.get("bag{123456}");
+        JsonNode assign = value.get("assign");
+        assertEquals("2020", assign.get("year").getTextValue());
+        assertEquals(3, assign.get("month").getIntValue());
+    }
+
+    @Test
+    public void requireThatUDFCorrectlyGeneratesAddTensorOperation() throws Exception {
+
+        Schema schema = new Schema();
+        Tuple tuple = TupleFactory.getInstance().newTuple();
+
+        // Please refer to the tensor format documentation
+
+        Map<String, Double> tensor = new HashMap<String, Double>() {{
+            put("x:label1,y:label2,z:label4", 2.0);
+            put("x:label3", 3.0);
+        }};
+
+        addToTuple("id", DataType.CHARARRAY, "123", schema, tuple);
+        addToTuple("tensor", DataType.MAP, tensor, schema, tuple);
+
+        VespaDocumentOperation docOp = new VespaDocumentOperation("docid=empty", "update-tensor-fields=tensor","operation=update");
+        docOp.setInputSchema(schema);
+        String json = docOp.exec(tuple);
+
+        ObjectMapper m = new ObjectMapper();
+        JsonNode root = m.readTree(json);
+        JsonNode fields = root.get("fields");
+        JsonNode tensorValue = fields.get("tensor");
+        JsonNode add = tensorValue.get("add");
+        JsonNode cells = add.get("cells");
+        Iterator<JsonNode> cellsIterator = cells.getElements();
+
+        JsonNode element = cellsIterator.next();
+        assertEquals("label1", element.get("address").get("x").getTextValue());
+        assertEquals("label2", element.get("address").get("y").getTextValue());
+        assertEquals("label4", element.get("address").get("z").getTextValue());
+        assertEquals("2.0", element.get("value").toString());
+
+        element = cellsIterator.next();
+        assertEquals("label3", element.get("address").get("x").getTextValue());
+        assertEquals("3.0", element.get("value").toString());
+    }
+
+    @Test
+    public void requireThatUDFCorrectlyGeneratesRemoveTensorOperation() throws Exception {
+
+        Schema schema = new Schema();
+        Tuple tuple = TupleFactory.getInstance().newTuple();
+
+        // Please refer to the tensor format documentation
+
+        Map<String, Double> tensor = new HashMap<String, Double>() {{
+            put("x:label1,y:label2,z:label4", 2.0);
+            put("x:label3", 3.0);
+        }};
+
+        addToTuple("id", DataType.CHARARRAY, "123", schema, tuple);
+        addToTuple("tensor", DataType.MAP, tensor, schema, tuple);
+
+        VespaDocumentOperation docOp = new VespaDocumentOperation("docid=empty", "remove-tensor-fields=tensor","operation=update");
+        docOp.setInputSchema(schema);
+        String json = docOp.exec(tuple);
+
+        ObjectMapper m = new ObjectMapper();
+        JsonNode root = m.readTree(json);
+        JsonNode fields = root.get("fields");
+        JsonNode tensorValue = fields.get("tensor");
+        JsonNode remove = tensorValue.get("remove");
+        JsonNode address = remove.get("addresses");
+
+        Iterator<JsonNode> addressIterator = address.getElements();
+
+        JsonNode element = addressIterator.next();
+        assertEquals("label1", element.get("x").getTextValue());
+
+        element = addressIterator.next();
+        assertEquals("label2", element.get("y").getTextValue());
+
+        element = addressIterator.next();
+        assertEquals("label4", element.get("z").getTextValue());
+
+        element = addressIterator.next();
+        assertEquals("label3", element.get("x").getTextValue());
+    }
+
+    @Test
+    public void requireThatUDFReturnsNullWhenExceptionHappens() throws IOException {
+        Schema schema = new Schema();
+        Tuple tuple = TupleFactory.getInstance().newTuple();
+
+        // broken DELTA format that would throw internally
+        Map<String, Double> tensor = new HashMap<String, Double>() {{
+            put("xlabel1", 2.0); // missing : between 'x' and 'label1'
+        }};
+
+        addToTuple("id", DataType.CHARARRAY, "123", schema, tuple);
+        addToTuple("tensor", DataType.MAP, tensor, schema, tuple);
+  
+        VespaDocumentOperation docOp = new VespaDocumentOperation("docid=empty", "create-tensor-fields=tensor");
+        docOp.setInputSchema(schema);
+        String json = docOp.exec(tuple);
+
+        assertNull(json);
+    }
+
+    @Test
     public void requireThatUDFCorrectlyGeneratesRemoveOperation() throws Exception {
         String json = getDocumentOperationJson("operation=remove", "docid=id:<application>:metrics::<name>-<date>");
         ObjectMapper m = new ObjectMapper();
@@ -343,6 +557,59 @@ public class VespaDocumentOperationTest {
         assertEquals(234567, bagNode.get("234567").asInt());
     }
 
+    @Test
+    public void requireThatUDFPrintIdWhenVerbose() throws IOException {
+        DataBag bag = BagFactory.getInstance().newDefaultBag();
+
+        Schema objectSchema = new Schema();
+        Tuple objectTuple = TupleFactory.getInstance().newTuple();
+        addToTuple("key", DataType.CHARARRAY, "123456", objectSchema, objectTuple);
+        addToTuple("value", DataType.INTEGER, 123456, objectSchema, objectTuple);
+        bag.add(objectTuple);
+
+        objectSchema = new Schema();
+        objectTuple = TupleFactory.getInstance().newTuple();
+        addToTuple("key", DataType.CHARARRAY, "234567", objectSchema, objectTuple);
+        addToTuple("value", DataType.INTEGER, 234567, objectSchema, objectTuple);
+        bag.add(objectTuple);
+
+        Schema schema = new Schema();
+        Tuple tuple = TupleFactory.getInstance().newTuple();
+        addToTuple("bag", DataType.BAG, bag, objectSchema, schema, tuple);
+
+        VespaDocumentOperation docOp = new VespaDocumentOperation("docid=7654321", "bag-as-map-fields=bag","verbose=true");
+        docOp.setInputSchema(schema);
+        String json = docOp.exec(tuple);
+
+        assertThat(outContent.toString(), CoreMatchers.containsString("Processing docId: 7654321"));
+    }
+
+    @Test
+    public void requireThatUDFVerboseSetToFalseByDefault() throws IOException {
+        DataBag bag = BagFactory.getInstance().newDefaultBag();
+
+        Schema objectSchema = new Schema();
+        Tuple objectTuple = TupleFactory.getInstance().newTuple();
+        addToTuple("key", DataType.CHARARRAY, "123456", objectSchema, objectTuple);
+        addToTuple("value", DataType.INTEGER, 123456, objectSchema, objectTuple);
+        bag.add(objectTuple);
+
+        objectSchema = new Schema();
+        objectTuple = TupleFactory.getInstance().newTuple();
+        addToTuple("key", DataType.CHARARRAY, "234567", objectSchema, objectTuple);
+        addToTuple("value", DataType.INTEGER, 234567, objectSchema, objectTuple);
+        bag.add(objectTuple);
+
+        Schema schema = new Schema();
+        Tuple tuple = TupleFactory.getInstance().newTuple();
+        addToTuple("bag", DataType.BAG, bag, objectSchema, schema, tuple);
+
+        VespaDocumentOperation docOp = new VespaDocumentOperation("docid=7654321", "bag-as-map-fields=bag");
+        docOp.setInputSchema(schema);
+        String json = docOp.exec(tuple);
+
+        assertEquals("", outContent.toString());
+    }
 
     private void addToTuple(String alias, byte type, Object value, Schema schema, Tuple tuple) {
         schema.add(new Schema.FieldSchema(alias, type));
@@ -355,4 +622,10 @@ public class VespaDocumentOperationTest {
         schema.add(new Schema.FieldSchema(alias, schemaInField, type));
         tuple.append(value);
     }
+
+    private void addToBagWithSchema(String alias, byte type, Tuple value, Schema schemaInField, Schema schema,DataBag bag)
+            throws FrontendException {
+        schema.add(new Schema.FieldSchema(alias, schemaInField, type));
+        bag.add(value);
+    }
 }
diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/FeedClient.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/FeedClient.java
index 8e820a89eb1..8cd22fe2e5c 100644
--- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/FeedClient.java
+++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/FeedClient.java
@@ -30,7 +30,7 @@ public interface FeedClient extends AutoCloseable {
      * produced faster than the can be handled. Transient failured are retried internally by this client.
      * Exactly one callback will always be received for each (completed) call to this.
      *
-     * @param documentId the document id of the document.
+     * @param documentId the document id of the document
      * @param documentData the document data as JSON or XML (as specified when using the factory to create the API)
      */
     default void stream(String documentId, CharSequence documentData) {
@@ -43,7 +43,7 @@ public interface FeedClient extends AutoCloseable {
      * produced faster than the can be handled. Transient failured are retried internally by this client.
      * Exactly one callback will always be received for each (completed) call to this.
      *
-     * @param documentId the document id of the document.
+     * @param documentId the document id of the document
      * @param documentData the document data as JSON or XML (as specified when using the factory to create the API)
      * @param context a context object which will be accessible in the result of the callback, or null if none
      */
@@ -57,7 +57,7 @@ public interface FeedClient extends AutoCloseable {
      * produced faster than the can be handled. Transient failured are retried internally by this client.
      * Exactly one callback will always be received for each (completed) call to this.
      *
-     * @param documentId the document id of the document.
+     * @param documentId the document id of the document
      * @param operationId the id to use for this operation, or null to let the client decide an operation id.
      *                    This id must be unique for every operation. Passing the operation id allows clients
      *                    to prepare to receive a response for it before issuing the operation to the client.
@@ -74,9 +74,9 @@ public interface FeedClient extends AutoCloseable {
     void close();
 
     /**
-     * Returns stats about the cluster.
+     * Returns stats about the cluster
      * 
-     * @return JSON string with information about cluster.
+     * @return JSON string with information about cluster
      */
     String getStatsAsJson();
 
@@ -132,7 +132,7 @@ public interface FeedClient extends AutoCloseable {
          * Document specific errors will be reported back through {@link #onCompletion(String, Result)}.
          *
          * @see FeedEndpointException
-         * @param exception An exception specifying endpoint and cause. See {@link FeedEndpointException} for details.
+         * @param exception an exception specifying endpoint and cause. See {@link FeedEndpointException} for details.
          */
         default void onEndpointException(FeedEndpointException exception) {}
 
diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/Result.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/Result.java
index 821e12b4140..473b9494ba4 100644
--- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/Result.java
+++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/Result.java
@@ -34,11 +34,7 @@ public class Result {
     public Result(Document document, Collection<Detail> values, StringBuilder localTrace) {
         this.document = document;
         this.details = Collections.unmodifiableList(new ArrayList<>(values));
-        boolean totalSuccess = true;
-        for (Detail d : details) {
-            if (d.getResultType() != ResultType.OPERATION_EXECUTED) {totalSuccess = false; }
-        }
-        this.success = totalSuccess;
+        this.success = details.stream().allMatch(d -> d.getResultType() == ResultType.OPERATION_EXECUTED);
         this.localTrace = localTrace == null ? null : localTrace.toString();
     }
 
@@ -70,18 +66,12 @@ public class Result {
 
     public List<Detail> getDetails() { return details; }
 
-    /**
-     * Checks if operation has been set up with local tracing.
-     *
-     * @return true if operation has local trace.
-     */
+    /** Returns whether the operation has been set up with local tracing */
     public boolean hasLocalTrace() {
         return localTrace != null;
     }
 
-    /**
-     * Information in a Result for a single operation sent to a single endpoint.
-     */
+    /** Information in a Result for a single operation sent to a single endpoint. */
     public static final class Detail {
 
         private final ResultType resultType;
@@ -105,43 +95,29 @@ public class Result {
         }
 
         /**
-         * Returns the endpoint from which the result was received.
-         *
-         * @return the endpoint from which the result was received.
+         * Returns the endpoint from which the result was received,
+         * or null if this failed before being assigned an endpoint
          */
         public Endpoint getEndpoint() {
             return endpoint;
         }
 
-        /**
-         * Check if operation was successful.
-         *
-         * @return true if the operation was successful.
-         */
+        /** Returns whether the operation was successful */
         public boolean isSuccess() {
             return resultType == ResultType.OPERATION_EXECUTED;
         }
 
-        /**
-         * Returns the result of the operation.
-         */
+        /** Returns the result of the operation */
         public ResultType getResultType() {
             return resultType;
         }
 
-        /**
-         * Returns any exception related to this Detail, if unsuccessful. Might be null.
-         *
-         * @return any exception related to this Detail, if unsuccessful. Might be null.
-         */
+        /** Returns any exception related to this Detail, if unsuccessful. Might be null. */
         public Exception getException() {
             return exception;
         }
 
-        /**
-         * Returns trace message if any from gateway.
-         * @return null or trace message.
-         */
+        /** Returns any trace message produces, or null if none */
         public String getTraceMessage() {
             return traceMessage;
         }
@@ -151,26 +127,21 @@ public class Result {
             StringBuilder b = new StringBuilder();
             b.append("Detail ");
             b.append("resultType=").append(resultType);
-            if (exception != null) {
+            if (exception != null)
                 b.append(" exception='").append(Exceptions.toMessageString(exception)).append("'");
-            }
-            if (traceMessage != null && ! traceMessage.isEmpty()) {
+            if (traceMessage != null && ! traceMessage.isEmpty())
                 b.append(" trace='").append(traceMessage).append("'");
-            }
-            b.append(" endpoint=").append(endpoint);
+            if (endpoint != null)
+                b.append(" endpoint=").append(endpoint);
             b.append(" resultTimeLocally=").append(timeStampMillis).append("\n");
             return b.toString();
         }
+
     }
 
     @Override
     public String toString() {
-        StringBuilder b = new StringBuilder();
-        b.append("Result for '").append(document.getDocumentId());
-        if (localTrace != null) {
-            b.append(localTrace);
-        }
-        return b.toString();
+        return "Result for " + document + " " + (localTrace != null ? localTrace : "");
     }
 
 }
diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/Cluster.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/Cluster.java
index 4707cdb705c..fccb79c77b4 100644
--- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/Cluster.java
+++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/Cluster.java
@@ -13,9 +13,7 @@ import java.util.List;
  */
 public final class Cluster {
 
-    /**
-     * Builder for {@link Cluster}.
-     */
+    /** Builder for {@link Cluster}. */
     public final static class Builder {
         private final List<Endpoint> endpoints = new LinkedList<>();
         private String route = null;
@@ -23,8 +21,8 @@ public final class Cluster {
         /**
          * Adds an Endpoint (a HTTP gateway) to this Cluster.
          *
-         * @param endpoint the Endpoint to add.
-         * @return this, for chaining.
+         * @param endpoint the Endpoint to add
+         * @return this, for chaining
          */
         public Builder addEndpoint(Endpoint endpoint) {
             endpoints.add(endpoint);
@@ -34,8 +32,8 @@ public final class Cluster {
         /**
          * Sets a route specific to this cluster, which overrides the route set in {@link com.yahoo.vespa.http.client.config.FeedParams#getRoute()}.
          *
-         * @param route a route specific to this cluster.
-         * @return this, for chaining.
+         * @param route a route specific to this cluster
+         * @return this, for chaining
          */
         public Builder setRoute(String route) {
             this.route = route;
@@ -68,7 +66,7 @@ public final class Cluster {
 
     @Override
     public String toString() {
-        return "Cluster " + endpoints + ", route " + route;
+        return "cluster with endpoints " + endpoints + " and route '" + route + "'";
     }
 
 }
diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/Document.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/Document.java
index f4da5d02012..bc38155d07a 100644
--- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/Document.java
+++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/Document.java
@@ -86,4 +86,7 @@ final public class Document {
         return operationId;
     }
 
+    @Override
+    public String toString() { return "document '" + documentId + "'"; }
+
 }
diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/Encoder.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/Encoder.java
index 135b2021a16..7bb65827b51 100644
--- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/Encoder.java
+++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/Encoder.java
@@ -15,10 +15,8 @@ public final class Encoder {
     /**
      * ISO 646.irv:1991 safe quoting into a StringBuilder instance.
      *
-     * @param input
-     *            the string to encode
-     * @param output
-     *            the destination buffer
+     * @param input the string to encode
+     * @param output the destination buffer
      * @return the destination buffer given as input
      */
     public static StringBuilder encode(String input, StringBuilder output) {
@@ -47,28 +45,20 @@ public final class Encoder {
     /**
      * ISO 646.irv:1991 safe unquoting into a StringBuilder instance.
      *
-     * @param input
-     *            the string to decode
-     * @param output
-     *            the destination buffer
+     * @param input the string to decode
+     * @param output the destination buffer
      * @return the destination buffer given as input
-     * @throws IllegalArgumentException
-     *             if the input string contains unexpected or invalid data
+     * @throws IllegalArgumentException if the input string contains unexpected or invalid data
      */
     public static StringBuilder decode(String input, StringBuilder output) {
         for (int i = 0; i < input.length(); i = input.offsetByCodePoints(i, 1)) {
             int c = input.codePointAt(i);
-            if (c > '~') {
-                throw new IllegalArgumentException("Input contained character above printable ASCII.");
-            }
-            switch (c) {
-                case '{':
-                    i = decode(input, i, output);
-                    break;
-                default:
-                    output.append((char) c);
-                    break;
-            }
+            if (c > '~')
+                throw new IllegalArgumentException("Input contained character above printable ASCII at position " + i);
+            if (c == '{')
+                i = decode(input, i, output);
+            else
+                output.append((char) c);
         }
         return output;
     }
diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ApacheGatewayConnection.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ApacheGatewayConnection.java
index f59d4a4bbba..d510ce4b7ea 100644
--- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ApacheGatewayConnection.java
+++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ApacheGatewayConnection.java
@@ -149,16 +149,12 @@ class ApacheGatewayConnection implements GatewayConnection {
 
     private InputStream write(List<Document> docs, boolean drain, boolean useCompression)
             throws ServerResponseException, IOException {
-        HttpPost httpPost = createPost(drain, useCompression, false /* this is not hanshake */);
+        HttpPost httpPost = createPost(drain, useCompression, false);
 
-        final ByteBuffer[] buffers = getDataWithStartAndEndOfFeed(docs, negotiatedVersion);
-        final InputStream inputStream = new ByteBufferInputStream(buffers);
-        final InputStreamEntity reqEntity;
-        if (useCompression ) {
-            reqEntity = zipAndCreateEntity(inputStream);
-        } else {
-            reqEntity = new InputStreamEntity(inputStream, -1);
-        }
+        ByteBuffer[] buffers = getDataWithStartAndEndOfFeed(docs, negotiatedVersion);
+        InputStream inputStream = new ByteBufferInputStream(buffers);
+        InputStreamEntity reqEntity = useCompression ? zipAndCreateEntity(inputStream)
+                                                     : new InputStreamEntity(inputStream, -1);
         reqEntity.setChunked(true);
         httpPost.setEntity(reqEntity);
         return executePost(httpPost);
diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ClusterConnection.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ClusterConnection.java
index 6e1f3419e8e..98755320d74 100644
--- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ClusterConnection.java
+++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ClusterConnection.java
@@ -24,10 +24,18 @@ import java.util.concurrent.TimeUnit;
  */
 public class ClusterConnection implements AutoCloseable {
 
+    private static final JsonFactory jsonFactory = new JsonFactory();
+    private static final ObjectMapper objectMapper = new ObjectMapper();
+
     private final List<IOThread> ioThreads = new ArrayList<>();
     private final int clusterId;
-    private static JsonFactory jsonFactory = new JsonFactory();
-    private static ObjectMapper objectMapper = new ObjectMapper();
+    private final ThreadGroup ioThreadGroup;
+
+    /** The shared queue of document operations the io threads will take from */
+    private final DocumentQueue documentQueue;
+
+    /** The single endpoint this sends to, or null if it will send to multiple endpoints */
+    private final Endpoint singleEndpoint;
 
     public ClusterConnection(OperationProcessor operationProcessor,
                              FeedParams feedParams,
@@ -37,16 +45,17 @@ public class ClusterConnection implements AutoCloseable {
                              int clientQueueSizePerCluster,
                              ScheduledThreadPoolExecutor timeoutExecutor) {
         if (cluster.getEndpoints().isEmpty())
-            throw new IllegalArgumentException("Cannot feed to empty cluster.");
+            throw new IllegalArgumentException("At least a single endpoint is required in " + cluster);
 
         this.clusterId = clusterId;
         int totalNumberOfEndpointsInThisCluster = cluster.getEndpoints().size() * connectionParams.getNumPersistentConnectionsPerEndpoint();
-        if (totalNumberOfEndpointsInThisCluster == 0) return;
-
-        // Lower than 1 does not make any sense.
+        if (totalNumberOfEndpointsInThisCluster == 0)
+            throw new IllegalArgumentException("At least 1 persistent connection per endpoint is required in " + cluster);
         int maxInFlightPerSession = Math.max(1, feedParams.getMaxInFlightRequests() / totalNumberOfEndpointsInThisCluster);
 
-        DocumentQueue documentQueue = null;
+        documentQueue = new DocumentQueue(clientQueueSizePerCluster);
+        ioThreadGroup = operationProcessor.getIoThreadGroup();
+        singleEndpoint = cluster.getEndpoints().size() == 1 ? cluster.getEndpoints().get(0) : null;
         for (Endpoint endpoint : cluster.getEndpoints()) {
             EndpointResultQueue endpointResultQueue = new EndpointResultQueue(operationProcessor,
                                                                               endpoint,
@@ -58,18 +67,14 @@ public class ClusterConnection implements AutoCloseable {
                 if (connectionParams.isDryRun()) {
                     gatewayConnection = new DryRunGatewayConnection(endpoint);
                 } else {
-                    gatewayConnection = new ApacheGatewayConnection(
-                            endpoint,
-                            feedParams,
-                            cluster.getRoute(),
-                            connectionParams,
-                            new ApacheGatewayConnection.HttpClientFactory(connectionParams, endpoint.isUseSsl()),
-                            operationProcessor.getClientId()
+                    gatewayConnection = new ApacheGatewayConnection(endpoint,
+                                                                    feedParams,
+                                                                    cluster.getRoute(),
+                                                                    connectionParams,
+                                                                    new ApacheGatewayConnection.HttpClientFactory(connectionParams, endpoint.isUseSsl()),
+                                                                    operationProcessor.getClientId()
                     );
                 }
-                if (documentQueue == null) {
-                    documentQueue = new DocumentQueue(clientQueueSizePerCluster);
-                }
                 IOThread ioThread = new IOThread(operationProcessor.getIoThreadGroup(),
                                                  endpointResultQueue,
                                                  gatewayConnection,
@@ -89,15 +94,10 @@ public class ClusterConnection implements AutoCloseable {
     }
 
     public void post(Document document) throws EndpointIOException {
-        String documentIdStr = document.getDocumentId();
-        // The same document ID must always go to the same destination
-        // In noHandshakeMode this has no effect as the documentQueue is shared between the IOThreads.
-        int hash = documentIdStr.hashCode() & 0x7FFFFFFF;  // Strip sign bit
-        IOThread ioThread = ioThreads.get(hash % ioThreads.size());
         try {
-            ioThread.post(document);
-        } catch (Throwable t) {
-            throw new EndpointIOException(ioThread.getEndpoint(), "While sending", t);
+            documentQueue.put(document, Thread.currentThread().getThreadGroup() == ioThreadGroup);
+        } catch (Throwable t) { // InterruptedException if shutting down, IllegalStateException if already shut down
+            throw new EndpointIOException(singleEndpoint, "While sending", t);
         }
     }
 
@@ -165,4 +165,5 @@ public class ClusterConnection implements AutoCloseable {
     public int hashCode() {
         return clusterId;
     }
+
 }
diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/EndpointIOException.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/EndpointIOException.java
index ae15f6ec22b..d8efeea93bc 100644
--- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/EndpointIOException.java
+++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/EndpointIOException.java
@@ -20,8 +20,7 @@ public class EndpointIOException extends IOException {
         this.endpoint = endpoint;
     }
 
-    public Endpoint getEndpoint() {
-        return endpoint;
-    }
+    /** Returns the endpoint, or null if the failure occurred before this was assigned to a unique endpoint */
+    public Endpoint getEndpoint() { return endpoint; }
 
 }
diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/IOThread.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/IOThread.java
index 77ed8464284..44799e598b0 100644
--- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/IOThread.java
+++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/IOThread.java
@@ -17,6 +17,7 @@ import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
 import java.util.Optional;
+import java.util.Random;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -36,7 +37,6 @@ class IOThread implements Runnable, AutoCloseable {
     private final DocumentQueue documentQueue;
     private final EndpointResultQueue resultQueue;
     private final Thread thread;
-    private final ThreadGroup ioThreadGroup;
     private final int clusterId;
     private final CountDownLatch running = new CountDownLatch(1);
     private final CountDownLatch stopSignal = new CountDownLatch(1);
@@ -44,6 +44,7 @@ class IOThread implements Runnable, AutoCloseable {
     private final int maxInFlightRequests;
     private final long localQueueTimeOut;
     private final GatewayThrottler gatewayThrottler;
+    private final Random random = new Random();
 
     private enum ThreadState { DISCONNECTED, CONNECTED, SESSION_SYNCED };
     private final AtomicInteger wrongSessionDetectedCounter = new AtomicInteger(0);
@@ -74,7 +75,6 @@ class IOThread implements Runnable, AutoCloseable {
         this.maxInFlightRequests = maxInFlightRequests;
         this.gatewayThrottler = new GatewayThrottler(maxSleepTimeMs);
         this.thread = new Thread(ioThreadGroup, this, "IOThread " + endpoint);
-        this.ioThreadGroup = ioThreadGroup;
         thread.setDaemon(true);
         this.localQueueTimeOut = localQueueTimeOut;
         thread.start();
@@ -165,8 +165,9 @@ class IOThread implements Runnable, AutoCloseable {
         log.fine("Session to " + endpoint + " closed.");
     }
 
+    /** For testing only */
     public void post(Document document) throws InterruptedException {
-        documentQueue.put(document, Thread.currentThread().getThreadGroup() == ioThreadGroup);
+        documentQueue.put(document, true);
     }
 
     @Override
@@ -174,7 +175,6 @@ class IOThread implements Runnable, AutoCloseable {
         return "I/O thread (for " + endpoint + ")";
     }
 
-
     List<Document> getNextDocsForFeeding(long maxWaitUnits, TimeUnit timeUnit) {
         List<Document> docsForSendChunk = new ArrayList<>();
         int chunkSizeBytes = 0;
@@ -186,26 +186,35 @@ class IOThread implements Runnable, AutoCloseable {
                 chunkSizeBytes = doc.size();
             }
         } catch (InterruptedException ie) {
-            log.fine("Got break signal while waiting for new documents to feed.");
+            log.fine("Got break signal while waiting for new documents to feed");
             return docsForSendChunk;
         }
         int pendingSize = 1 + resultQueue.getPendingSize();
+
         // see if we can get more documents without blocking
-        while (chunkSizeBytes < maxChunkSizeBytes && pendingSize < maxInFlightRequests) {
+        // slightly randomize how much is taken to avoid harmonic interactions leading
+        // to some threads consistently taking more than others
+        int thisMaxChunkSizeBytes = randomize(maxChunkSizeBytes);
+        int thisMaxInFlightRequests = randomize(maxInFlightRequests);
+        while (chunkSizeBytes < thisMaxChunkSizeBytes && pendingSize < thisMaxInFlightRequests) {
             drainFirstDocumentsInQueueIfOld();
-            Document d = documentQueue.poll();
-            if (d == null) {
-                break;
-            }
-            docsForSendChunk.add(d);
-            chunkSizeBytes += d.size();
+            Document document = documentQueue.poll();
+            if (document == null) break;
+            docsForSendChunk.add(document);
+            chunkSizeBytes += document.size();
             pendingSize++;
         }
-        log.finest("Chunk has " + docsForSendChunk.size() + " docs with a size " + chunkSizeBytes + " bytes.");
+        if (log.isLoggable(Level.FINEST))
+            log.finest("Chunk has " + docsForSendChunk.size() + " docs with a size " + chunkSizeBytes + " bytes");
         docsReceivedCounter.addAndGet(docsForSendChunk.size());
         return docsForSendChunk;
     }
 
+    private int randomize(int limit) {
+        double multiplier = 0.75 + 0.25 * random.nextDouble();
+        return Math.max(1, (int)(limit * multiplier));
+    }
+
     private void addDocumentsToResultQueue(List<Document> docs) {
         for (Document doc : docs) {
             resultQueue.operationSent(doc.getOperationId());
@@ -273,10 +282,9 @@ class IOThread implements Runnable, AutoCloseable {
         int pendingResultQueueSize = resultQueue.getPendingSize();
         pendingDocumentStatusCount.set(pendingResultQueueSize);
 
-        List<Document> nextDocsForFeeding =
-                (pendingResultQueueSize > maxInFlightRequests)
-              ? new ArrayList<>()       // The queue is full, will not send more documents.
-              : getNextDocsForFeeding(maxWaitTimeMs, TimeUnit.MILLISECONDS);
+        List<Document> nextDocsForFeeding = (pendingResultQueueSize > maxInFlightRequests)
+                                            ? new ArrayList<>() // The queue is full, will not send more documents
+                                            : getNextDocsForFeeding(maxWaitTimeMs, TimeUnit.MILLISECONDS);
 
         if (nextDocsForFeeding.isEmpty() && pendingResultQueueSize == 0) {
             //we have no unfinished business with the server now.
@@ -288,8 +296,7 @@ class IOThread implements Runnable, AutoCloseable {
 
         if (pendingResultQueueSize > maxInFlightRequests && processResponse.processResultsCount == 0) {
             try {
-                // Max outstanding document operations, no more results on server side, wait a bit
-                // before asking again.
+                // Max outstanding document operations, no more results on server side, wait a bit before asking again
                 Thread.sleep(300);
             } catch (InterruptedException e) {
                 // Ignore
diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/EndPointResultFactory.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/EndPointResultFactory.java
index 95df465c7ca..205153a7a00 100644
--- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/EndPointResultFactory.java
+++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/EndPointResultFactory.java
@@ -38,30 +38,28 @@ public final class EndPointResultFactory {
         return results;
     }
 
-    public static EndpointResult createError(
-            Endpoint endpoint, String operationId, Exception exception) {
-        return new EndpointResult(operationId, new Result.Detail(
-                endpoint,  Result.ResultType.FATAL_ERROR, null, exception));
+    public static EndpointResult createError(Endpoint endpoint, String operationId, Exception exception) {
+        return new EndpointResult(operationId, new Result.Detail(endpoint,
+                                                                 Result.ResultType.FATAL_ERROR,
+                                                                 null,
+                                                                 exception));
     }
 
-    public static EndpointResult createTransientError(
-            Endpoint endpoint, String operationId, Exception exception) {
-        return new EndpointResult(operationId, new Result.Detail(
-                endpoint, Result.ResultType.TRANSITIVE_ERROR, null, exception));
+    public static EndpointResult createTransientError(Endpoint endpoint, String operationId, Exception exception) {
+        return new EndpointResult(operationId, new Result.Detail(endpoint,
+                                                                 Result.ResultType.TRANSITIVE_ERROR,
+                                                                 null,
+                                                                 exception));
     }
 
     private static Result.ResultType replyToResultType(OperationStatus reply) {
-        final Result.ResultType resultType;
         // The ordering below is important, e.g. if success, it is never a transient error even if isTransient is true.
-        if (reply.errorCode.isSuccess()) {
+        if (reply.errorCode.isSuccess())
             return Result.ResultType.OPERATION_EXECUTED;
-        }
-        if (reply.isConditionNotMet) {
+        if (reply.isConditionNotMet)
             return Result.ResultType.CONDITION_NOT_MET;
-        }
-        if (reply.errorCode.isTransient()) {
+        if (reply.errorCode.isTransient())
             return Result.ResultType.TRANSITIVE_ERROR;
-        }
         return Result.ResultType.FATAL_ERROR;
     }
 
diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/CommandLineArguments.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/CommandLineArguments.java
index cff9e2fefb0..83113722814 100644
--- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/CommandLineArguments.java
+++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/CommandLineArguments.java
@@ -34,7 +34,7 @@ import java.util.concurrent.TimeUnit;
  * @author dybis
  */
 @Command(name = "vespa-http-client",
-        description = "This is a tool for feeding xml or json data to a Vespa application.")
+         description = "This is a tool for feeding xml or json data to a Vespa application.")
 public class CommandLineArguments {
 
     /**
@@ -44,7 +44,7 @@ public class CommandLineArguments {
      * @return null on failure or if help option is set to true.
      */
     static CommandLineArguments build(String[] args) {
-        final CommandLineArguments cmdArgs;
+        CommandLineArguments cmdArgs;
         try {
             cmdArgs =  SingleCommand.singleCommand(CommandLineArguments.class).parse(args);
         } catch (Exception e) {
@@ -115,7 +115,7 @@ public class CommandLineArguments {
                 return true;
             default:
                 System.err.println("Not valid value for priority. Allowed values are HIGHEST, VERY_HIGH, HIGH_[1-3], " +
-                        "NORMAL_[1-6], LOW_[1-3], VERY_LOW, and LOWEST.");
+                                   "NORMAL_[1-6], LOW_[1-3], VERY_LOW, and LOWEST.");
                 return false;
         }
     }
@@ -254,7 +254,7 @@ public class CommandLineArguments {
     public boolean getAddRootElementToXml() { return addRootElementToXml; }
 
     SessionParams createSessionParams(boolean useJson) {
-        final int minThrottleValue = useDynamicThrottlingArg ? 10 : 0;
+        int minThrottleValue = useDynamicThrottlingArg ? 10 : 0;
         Path privateKeyPath = Optional.ofNullable(this.privateKeyPath).map(Paths::get).orElse(null);
         Path certificatePath = Optional.ofNullable(this.certificatePath).map(Paths::get).orElse(null);
         Path caCertificatesPath = Optional.ofNullable(this.caCertificatesPath).map(Paths::get).orElse(null);
@@ -307,8 +307,7 @@ public class CommandLineArguments {
         else {
             Iterable<String> hosts = Splitter.on(',').trimResults().split(hostArg);
             for (String host : hosts) {
-                builder.addCluster(new Cluster.Builder()
-                        .addEndpoint(Endpoint.create(host, portArg, useTls))
+                builder.addCluster(new Cluster.Builder().addEndpoint(Endpoint.create(host, portArg, useTls))
                         .build());
             }
         }
diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/FormatInputStream.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/FormatInputStream.java
index 36cdf18e102..5bf48019785 100644
--- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/FormatInputStream.java
+++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/FormatInputStream.java
@@ -31,14 +31,14 @@ public class FormatInputStream {
      * IllegalArgumentException if unable to determine data format.
      *
      * @param stream              InputStream of the data if present
-     * @param inputFile           Path to file to use as input
-     * @param addRootElementToXml To add vespafeed root element around the input data stream
-     * @throws IOException        on errors.
+     * @param inputFile           path to file to use as input
+     * @param addRootElementToXml to add vespafeed root element around the input data stream
+     * @throws IOException        on errors
      */
     public FormatInputStream(InputStream stream, Optional<String> inputFile, boolean addRootElementToXml)
             throws IOException {
-        final DataFormatDetector dataFormatDetector = new DataFormatDetector(new JsonFactory(), new XmlFactory());
-        final DataFormatMatcher formatMatcher;
+        DataFormatDetector dataFormatDetector = new DataFormatDetector(new JsonFactory(), new XmlFactory());
+        DataFormatMatcher formatMatcher;
 
         if (inputFile.isPresent()) {
             try (FileInputStream fileInputStream = new FileInputStream(inputFile.get())) {
@@ -47,9 +47,8 @@ public class FormatInputStream {
             inputStream = new FileInputStream(inputFile.get());
 
         } else {
-            if (stream.available() == 0) {
+            if (stream.available() == 0)
                 System.out.println("No data in stream yet and no file specified, waiting for data.");
-            }
 
             inputStream = stream.markSupported() ? stream : new BufferedInputStream(stream);
             inputStream.mark(DataFormatDetector.DEFAULT_MAX_INPUT_LOOKAHEAD);
@@ -63,8 +62,8 @@ public class FormatInputStream {
             return;
         }
 
-        if (formatMatcher.getMatchStrength() == MatchStrength.INCONCLUSIVE ||
-                formatMatcher.getMatchStrength() == MatchStrength.NO_MATCH) {
+        if (formatMatcher.getMatchStrength() == MatchStrength.INCONCLUSIVE
+            || formatMatcher.getMatchStrength() == MatchStrength.NO_MATCH) {
             throw new IllegalArgumentException("Could not detect input format");
         }
 
@@ -72,11 +71,9 @@ public class FormatInputStream {
             case "json":
                 format = Format.JSON;
                 break;
-
             case "xml":
                 format = Format.XML;
                 break;
-
             default:
                 throw new IllegalArgumentException("Unknown data format");
         }
@@ -84,8 +81,7 @@ public class FormatInputStream {
 
     private static InputStream addVespafeedTag(InputStream inputStream) {
         return new SequenceInputStream(Collections.enumeration(Arrays.asList(
-                new ByteArrayInputStream("<vespafeed>".getBytes()),
-                inputStream,
+                new ByteArrayInputStream("<vespafeed>".getBytes()), inputStream,
                 new ByteArrayInputStream("</vespafeed>".getBytes())))
         );
     }
diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/Runner.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/Runner.java
index 0e202d1f348..7c034cab75f 100644
--- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/Runner.java
+++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/Runner.java
@@ -29,12 +29,15 @@ public class Runner {
      * @param verbose if true will print some information to stderr
      * @return send time in ms, not including validating
      */
-    public static long send(
-            FeedClient feedClient, InputStream inputStream, boolean isJson, AtomicInteger numSent, boolean verbose) {
+    public static long send(FeedClient feedClient,
+                            InputStream inputStream,
+                            boolean isJson,
+                            AtomicInteger numSent,
+                            boolean verbose) {
 
-        if (verbose) {
+        if (verbose)
             System.err.println("Now sending data.");
-        }
+
         long sendStartTime = System.currentTimeMillis();
         if (isJson) {
             JsonReader.read(inputStream, feedClient, numSent);
@@ -48,51 +51,52 @@ public class Runner {
 
         long sendTotalTime = System.currentTimeMillis() - sendStartTime;
 
-        if (verbose) {
+        if (verbose)
             System.err.println("Waiting for all results, sent " + numSent.get() + " docs.");
-        }
+
         feedClient.close();
-        if (verbose) {
+        if (verbose)
             System.err.println("Session closed.");
-        }
         return sendTotalTime;
     }
 
 
     public static void main(String[] args) throws IOException {
         CommandLineArguments commandLineArgs = CommandLineArguments.build(args);
-        if (commandLineArgs == null) {
+        if (commandLineArgs == null)
             System.exit(1);
-        }
-
-        FormatInputStream formatInputStream = new FormatInputStream(
-                System.in,
-                Optional.ofNullable(commandLineArgs.getFile()),
-                commandLineArgs.getAddRootElementToXml());
 
+        FormatInputStream formatInputStream = new FormatInputStream(System.in,
+                                                                    Optional.ofNullable(commandLineArgs.getFile()),
+                                                                    commandLineArgs.getAddRootElementToXml());
 
-        int intervalOfLogging = commandLineArgs.getVerbose()
+        int intervalOfLogging =
+                commandLineArgs.getVerbose()
                 ? commandLineArgs.getWhenVerboseEnabledPrintMessageForEveryXDocuments()
                 : Integer.MAX_VALUE;
         AtomicInteger numSent = new AtomicInteger(0);
         SimpleLoggerResultCallback callback = new SimpleLoggerResultCallback(numSent, intervalOfLogging);
 
-        FeedClient feedClient = FeedClientFactory.create(
-                commandLineArgs.createSessionParams(formatInputStream.getFormat()== FormatInputStream.Format.JSON), callback);
+        FeedClient feedClient = FeedClientFactory.create(commandLineArgs.createSessionParams(formatInputStream.getFormat()== FormatInputStream.Format.JSON),
+                                                         callback);
 
-        long sendTotalTimeMs = send(
-                feedClient, formatInputStream.getInputStream(),
-                formatInputStream.getFormat() == FormatInputStream.Format.JSON, numSent, commandLineArgs.getVerbose());
+        long sendTotalTimeMs = send(feedClient,
+                                    formatInputStream.getInputStream(),
+                                    formatInputStream.getFormat() == FormatInputStream.Format.JSON,
+                                    numSent,
+                                    commandLineArgs.getVerbose());
 
         if (commandLineArgs.getVerbose()) {
             System.err.println(feedClient.getStatsAsJson());
-            double fileSizeMb = ((double) new File(commandLineArgs.getFile()).length()) / 1024.0 / 1024.0;
             double transferTimeSec = ((double) sendTotalTimeMs) / 1000.0;
-            System.err.println("Sent " + fileSizeMb + " MB in " + transferTimeSec + " seconds.");
-            System.err.println("Speed: " + ((fileSizeMb / transferTimeSec) * 8.0) + " Mbits/sec, + HTTP overhead " +
-                               "(not taking compression into account)");
-            if (transferTimeSec > 0) {
-                System.err.printf("Docs/sec %.3f%n\n", numSent.get() / transferTimeSec);
+            if (transferTimeSec > 0)
+                System.err.printf("Docs/sec %.3f%n", numSent.get() / transferTimeSec);
+
+            if (commandLineArgs.getFile() != null) {
+                double fileSizeMb = ((double) new File(commandLineArgs.getFile()).length()) / 1024.0 / 1024.0;
+                System.err.println("Sent " + fileSizeMb + " MB in " + transferTimeSec + " seconds.");
+                System.err.println("Speed: " + ((fileSizeMb / transferTimeSec) * 8.0) + " Mbits/sec, + HTTP overhead " +
+                                   "(not taking compression into account)");
             }
         }
         callback.printProgress();
diff --git a/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/AbstractVespaDeploymentMojo.java b/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/AbstractVespaDeploymentMojo.java
index d9ac88c6ff4..f9c07d29ee1 100644
--- a/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/AbstractVespaDeploymentMojo.java
+++ b/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/AbstractVespaDeploymentMojo.java
@@ -12,6 +12,8 @@ import org.apache.maven.plugins.annotations.Parameter;
  */
 public abstract class AbstractVespaDeploymentMojo extends AbstractVespaMojo {
 
+    protected ZoneId zone;
+
     @Parameter(property = "environment")
     protected String environment;
 
@@ -20,13 +22,21 @@ public abstract class AbstractVespaDeploymentMojo extends AbstractVespaMojo {
 
     protected ZoneId zoneOf(String environment, String region) {
         if (region == null)
-            return controller.defaultZone(environment != null ? Environment.from(environment)
-                                                              : Environment.dev);
+            return zone = controller.defaultZone(environment != null ? Environment.from(environment)
+                                                                     : Environment.dev);
 
         if (environment == null)
             throw new IllegalArgumentException("Environment must be specified if region is specified");
 
-        return ZoneId.from(environment, region);
+        return zone = ZoneId.from(environment, region);
+    }
+
+    @Override
+    protected String name() {
+        return super.name() + "." + instance + " in " +
+               (zone != null ? zone.region() + " in " + zone.environment()
+                             : (region == null ? "default region" : region) + " in " +
+                               (environment == null ? "default environment (dev)" : environment));
     }
 
 }
diff --git a/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/AbstractVespaMojo.java b/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/AbstractVespaMojo.java
index 8602d89c90c..0adb139f2c1 100644
--- a/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/AbstractVespaMojo.java
+++ b/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/AbstractVespaMojo.java
@@ -63,13 +63,16 @@ public abstract class AbstractVespaMojo extends AbstractMojo {
             throw e;
         }
         catch (Exception e) {
-            throw new MojoExecutionException("Execution failed for application '" + id + "':", e);
+            throw new MojoExecutionException("Execution failed for application " + name() + ":", e);
         }
     }
 
     /** Override this in subclasses, instead of {@link #execute()}. */
     protected abstract void doExecute() throws Exception;
 
+    /** Return the name of the relevant entity, e.g., application with or without instance. */
+    protected String name() { return tenant + "." + application; }
+
     protected void setup() {
         tenant = firstNonBlank(tenant, project.getProperties().getProperty("tenant"));
         application = firstNonBlank(application, project.getProperties().getProperty("application"));
diff --git a/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/DeployMojo.java b/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/DeployMojo.java
index 851b1fa214c..a798c2ad6df 100644
--- a/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/DeployMojo.java
+++ b/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/DeployMojo.java
@@ -60,35 +60,18 @@ public class DeployMojo extends AbstractVespaDeploymentMojo {
     }
 
     private void tailLogs(ApplicationId id, ZoneId zone, long run) throws MojoFailureException, MojoExecutionException {
-        long last = -1;
-        DeploymentLog log;
-        while (true) {
-            log = controller.deploymentLog(id, zone, run, last);
-            for (DeploymentLog.Entry entry : log.entries())
-                print(entry);
-            last = log.last().orElse(last);
-
-            if ( ! log.isActive())
-                break;
-
-            try {
-                Thread.sleep(1000);
-            }
-            catch (InterruptedException e) {
-                Thread.currentThread().interrupt();
-                break;
-            }
-        }
+        DeploymentLog log = controller.followDeploymentUntilDone(id, zone, run, this::print);
         switch (log.status()) {
-            case success:            return;
-            case error:              throw new MojoExecutionException("Unexpected error during deployment; see log for details");
-            case aborted:            throw new MojoFailureException("Deployment was aborted, probably by a newer deployment");
-            case outOfCapacity:      throw new MojoFailureException("No capacity left in zone; please contact the Vespa team");
-            case deploymentFailed:   throw new MojoFailureException("Deployment failed; see log for details");
-            case installationFailed: throw new MojoFailureException("Installation failed; see Vespa log for details");
-            case running:            throw new MojoFailureException("Deployment not completed");
-            case testFailure:        throw new IllegalStateException("Unexpected status; tests are not run for manual deployments");
-            default:                 throw new IllegalArgumentException("Unexpected status '" + log.status() + "'");
+            case success:                    return;
+            case error:                      throw new MojoExecutionException("Unexpected error during deployment; see log for details");
+            case aborted:                    throw new MojoFailureException("Deployment was aborted, probably by a newer deployment");
+            case outOfCapacity:              throw new MojoFailureException("No capacity left in zone; please contact the Vespa team");
+            case deploymentFailed:           throw new MojoFailureException("Deployment failed; see log for details");
+            case installationFailed:         throw new MojoFailureException("Installation failed; see Vespa log for details");
+            case running:                    throw new MojoFailureException("Deployment not completed");
+            case endpointCertificateTimeout: throw new MojoFailureException("Endpoint certificate not ready in time; please contact Vespa team");
+            case testFailure:                throw new IllegalStateException("Unexpected status; tests are not run for manual deployments");
+            default:                         throw new IllegalArgumentException("Unexpected status '" + log.status() + "'");
         }
     }
 
diff --git a/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/EffectiveServicesMojo.java b/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/EffectiveServicesMojo.java
index 91c7809dc76..ea536ffa6b3 100644
--- a/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/EffectiveServicesMojo.java
+++ b/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/EffectiveServicesMojo.java
@@ -2,6 +2,7 @@
 package ai.vespa.hosted.plugin;
 
 import com.yahoo.config.application.XmlPreProcessor;
+import com.yahoo.config.provision.InstanceName;
 import com.yahoo.config.provision.zone.ZoneId;
 import org.apache.maven.plugins.annotations.Mojo;
 import org.apache.maven.plugins.annotations.Parameter;
@@ -43,12 +44,17 @@ public class EffectiveServicesMojo extends AbstractVespaDeploymentMojo {
 
         ZoneId zone = zoneOf(environment, region);
         Path output = Paths.get(outputDirectory).resolve("services-" + zone.environment().value() + "-" + zone.region().value() + ".xml");
-        Files.write(output, effectiveServices(services, zone).getBytes(StandardCharsets.UTF_8));
+        Files.write(output, effectiveServices(services, zone, InstanceName.from(instance)).getBytes(StandardCharsets.UTF_8));
         getLog().info("Effective services for " + zone + " written to " + output);
     }
 
-    static String effectiveServices(File servicesFile, ZoneId zone) throws Exception {
-        Document processedServicesXml = new XmlPreProcessor(servicesFile.getParentFile(), servicesFile, zone.environment(), zone.region()).run();
+    static String effectiveServices(File servicesFile, ZoneId zone, InstanceName instance) throws Exception {
+        Document processedServicesXml = new XmlPreProcessor(servicesFile.getParentFile(),
+                                                            servicesFile,
+                                                            instance,
+                                                            zone.environment(),
+                                                            zone.region())
+                .run();
         Transformer transformer = TransformerFactory.newInstance().newTransformer();
         transformer.setOutputProperty(OutputKeys.INDENT, "yes");
         Writer writer = new StringWriter();
diff --git a/vespa-maven-plugin/src/test/java/ai/vespa/hosted/plugin/EffectiveServicesMojoTest.java b/vespa-maven-plugin/src/test/java/ai/vespa/hosted/plugin/EffectiveServicesMojoTest.java
index 35f3425706d..5b472574efd 100644
--- a/vespa-maven-plugin/src/test/java/ai/vespa/hosted/plugin/EffectiveServicesMojoTest.java
+++ b/vespa-maven-plugin/src/test/java/ai/vespa/hosted/plugin/EffectiveServicesMojoTest.java
@@ -1,6 +1,7 @@
 // Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package ai.vespa.hosted.plugin;
 
+import com.yahoo.config.provision.InstanceName;
 import com.yahoo.config.provision.zone.ZoneId;
 import org.junit.jupiter.api.DisplayName;
 import org.junit.jupiter.api.Test;
@@ -9,6 +10,7 @@ import java.io.File;
 import java.nio.file.Files;
 import java.nio.file.Paths;
 
+import static ai.vespa.hosted.plugin.EffectiveServicesMojo.effectiveServices;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 
 /**
@@ -24,21 +26,21 @@ class EffectiveServicesMojoTest {
     @DisplayName("when zone matches environment-only directive")
     void devServices() throws Exception {
         assertEquals(Files.readString(Paths.get("src/test/resources/effective-services/dev.xml")),
-                     EffectiveServicesMojo.effectiveServices(servicesFile, ZoneId.from("dev", "us-east-3")));
+                     effectiveServices(servicesFile, ZoneId.from("dev", "us-east-3"), InstanceName.defaultName()));
     }
 
     @Test
     @DisplayName("when zone matches region-and-environment directive")
     void prodUsEast3() throws Exception {
         assertEquals(Files.readString(Paths.get("src/test/resources/effective-services/prod_us-east-3.xml")),
-                     EffectiveServicesMojo.effectiveServices(servicesFile, ZoneId.from("prod", "us-east-3")));
+                     effectiveServices(servicesFile, ZoneId.from("prod", "us-east-3"), InstanceName.defaultName()));
     }
 
     @Test
     @DisplayName("when zone doesn't match any directives")
     void prodUsWest1Services() throws Exception {
         assertEquals(Files.readString(Paths.get("src/test/resources/effective-services/prod_us-west-1.xml")),
-                     EffectiveServicesMojo.effectiveServices(servicesFile, ZoneId.from("prod", "us-west-1")));
+                     effectiveServices(servicesFile, ZoneId.from("prod", "us-west-1"), InstanceName.defaultName()));
     }
 
 }
diff --git a/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunnerHandler.java b/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunnerHandler.java
index 275d9f97dcf..90f7a76b356 100644
--- a/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunnerHandler.java
+++ b/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunnerHandler.java
@@ -62,9 +62,9 @@ public class TestRunnerHandler extends LoggingRequestHandler {
     private HttpResponse handleGET(HttpRequest request) {
         String path = request.getUri().getPath();
         if (path.equals("/tester/v1/log")) {
-            return new SlimeJsonResponse(toSlime(testRunner.getLog(request.hasProperty("after")
-                                                                           ? Long.parseLong(request.getProperty("after"))
-                                                                           : -1)));
+            return new SlimeJsonResponse(logToSlime(testRunner.getLog(request.hasProperty("after")
+                                                                               ? Long.parseLong(request.getProperty("after"))
+                                                                               : -1)));
         } else if (path.equals("/tester/v1/status")) {
             log.info("Responding with status " + testRunner.getStatus());
             return new Response(testRunner.getStatus().name());
@@ -72,7 +72,7 @@ public class TestRunnerHandler extends LoggingRequestHandler {
         return new Response(Status.NOT_FOUND, "Not found: " + request.getUri().getPath());
     }
 
-    private HttpResponse handlePOST(HttpRequest request) throws IOException, InterruptedException {
+    private HttpResponse handlePOST(HttpRequest request) throws IOException {
         final String path = request.getUri().getPath();
         if (path.startsWith("/tester/v1/run/")) {
             String type = lastElement(path);
@@ -90,12 +90,18 @@ public class TestRunnerHandler extends LoggingRequestHandler {
             path = path.substring(0, path.length() - 1);
         int lastSlash = path.lastIndexOf("/");
         if (lastSlash < 0) return path;
-        return path.substring(lastSlash + 1, path.length());
+        return path.substring(lastSlash + 1);
     }
 
-    static Slime toSlime(Collection<LogRecord> log) {
-        Slime root = new Slime();
-        Cursor recordArray = root.setArray();
+    static Slime logToSlime(Collection<LogRecord> log) {
+        Slime slime = new Slime();
+        Cursor root = slime.setObject();
+        Cursor recordArray = root.setArray("logRecords");
+        logArrayToSlime(recordArray, log);
+        return slime;
+    }
+
+    static void logArrayToSlime(Cursor recordArray, Collection<LogRecord> log) {
         log.forEach(record -> {
             Cursor recordObject = recordArray.addObject();
             recordObject.setLong("id", record.getSequenceNumber());
@@ -109,7 +115,6 @@ public class TestRunnerHandler extends LoggingRequestHandler {
             }
             recordObject.setString("message", message);
         });
-        return root;
     }
 
     public static String typeOf(Level level) {
@@ -120,7 +125,7 @@ public class TestRunnerHandler extends LoggingRequestHandler {
                 : "error";
     }
 
-    private class SlimeJsonResponse extends HttpResponse {
+    private static class SlimeJsonResponse extends HttpResponse {
         private final Slime slime;
 
         private SlimeJsonResponse(Slime slime) {
diff --git a/vespa-testrunner-components/src/test/java/com/yahoo/vespa/hosted/testrunner/TestRunnerHandlerTest.java b/vespa-testrunner-components/src/test/java/com/yahoo/vespa/hosted/testrunner/TestRunnerHandlerTest.java
index 3f6fecc8249..77de009571b 100644
--- a/vespa-testrunner-components/src/test/java/com/yahoo/vespa/hosted/testrunner/TestRunnerHandlerTest.java
+++ b/vespa-testrunner-components/src/test/java/com/yahoo/vespa/hosted/testrunner/TestRunnerHandlerTest.java
@@ -1,7 +1,7 @@
 // Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.testrunner;
 
-import com.yahoo.vespa.config.SlimeUtils;
+import com.yahoo.slime.SlimeUtils;
 import org.junit.Test;
 
 import java.io.ByteArrayOutputStream;
@@ -21,18 +21,43 @@ public class TestRunnerHandlerTest {
 
     @Test
     public void logSerialization() throws IOException {
-        LogRecord record = new LogRecord(Level.INFO, "Hello.");
-        record.setSequenceNumber(1);
-        record.setInstant(Instant.ofEpochMilli(2));
-        Exception exception = new RuntimeException();
-        record.setThrown(exception);
-        ByteArrayOutputStream buffer = new ByteArrayOutputStream();
-        exception.printStackTrace(new PrintStream(buffer));
-        String trace = buffer.toString()
-                             .replaceAll("\n", "\\\\n")
-                             .replaceAll("\t", "\\\\t");
-        assertEquals("[{\"id\":1,\"at\":2,\"type\":\"info\",\"message\":\"Hello.\\n" + trace + "\"}]",
-                     new String(SlimeUtils.toJsonBytes(TestRunnerHandler.toSlime(Collections.singletonList(record)))));
+        Log log = new Log();
+        LogRecord record = log.getLogRecord();
+        String trace = log.getTrace();
+        assertEquals("{\"logRecords\":[{\"id\":1,\"at\":2,\"type\":\"info\",\"message\":\"Hello.\\n" + trace + "\"}]}",
+                     new String(SlimeUtils.toJsonBytes(TestRunnerHandler.logToSlime(Collections.singletonList(record)))));
+    }
+
+    private static class Log {
+
+        private final LogRecord record;
+        private final String trace;
+
+        public Log() {
+            Exception exception = new RuntimeException();
+            record = createRecord(exception);
+            ByteArrayOutputStream buffer = new ByteArrayOutputStream();
+            exception.printStackTrace(new PrintStream(buffer));
+            trace = buffer.toString()
+                    .replaceAll("\n", "\\\\n")
+                    .replaceAll("\t", "\\\\t");
+        }
+
+        LogRecord getLogRecord() {
+            return record;
+        }
+
+        String getTrace() {
+            return trace;
+        }
+
+        private static LogRecord createRecord(Exception exception) {
+            LogRecord record = new LogRecord(Level.INFO, "Hello.");
+            record.setSequenceNumber(1);
+            record.setInstant(Instant.ofEpochMilli(2));
+            record.setThrown(exception);
+            return record;
+        }
     }
 
 }
diff --git a/vespa_feed_perf/src/main/java/com/yahoo/vespa/feed/perf/FeederParams.java b/vespa_feed_perf/src/main/java/com/yahoo/vespa/feed/perf/FeederParams.java
index 98394a56694..cad8bb7b312 100644
--- a/vespa_feed_perf/src/main/java/com/yahoo/vespa/feed/perf/FeederParams.java
+++ b/vespa_feed_perf/src/main/java/com/yahoo/vespa/feed/perf/FeederParams.java
@@ -34,7 +34,15 @@ class FeederParams {
     private boolean benchmarkMode = false;
     private int numDispatchThreads = 1;
     private int maxPending = 0;
-    private int numConnectionsPerTarget = 2;
+    private double timeout = 180.0;
+
+    private double windowSizeBackOff = 0.95;
+    private double windowDecrementFactor = 1.2;
+    private double windowResizeRate = 3;
+    private int windowIncrementSize = 20;
+
+    private int numConnectionsPerTarget = 1;
+    private long numMessagesToSend = Long.MAX_VALUE;
     private List<InputStream> inputStreams = new ArrayList<>();
 
     FeederParams() {
@@ -82,12 +90,29 @@ class FeederParams {
         this.configId = configId;
         return this;
     }
+    public double getWindowSizeBackOff() {
+        return windowSizeBackOff;
+    }
 
-    int getNumConnectionsPerTarget() { return numConnectionsPerTarget; }
-    FeederParams setNumConnectionsPerTarget(int numConnectionsPerTarget) {
-        this.numConnectionsPerTarget = numConnectionsPerTarget;
-        return this;
+    public double getWindowDecrementFactor() {
+        return windowDecrementFactor;
+    }
+
+    public double getWindowResizeRate() {
+        return windowResizeRate;
+    }
+    public double getTimeout() {
+        return timeout;
+    }
+
+    public int getWindowIncrementSize() {
+        return windowIncrementSize;
     }
+
+    int getNumConnectionsPerTarget() { return numConnectionsPerTarget; }
+
+    long getNumMessagesToSend() { return numMessagesToSend; }
+
     boolean isSerialTransferEnabled() {
         return maxPending == 1;
     }
@@ -116,6 +141,12 @@ class FeederParams {
         opts.addOption("b", "mode", true, "Mode for benchmarking.");
         opts.addOption("o", "output", true, "File to write to. Extensions gives format (.xml, .json, .vespa) json will be produced if no extension.");
         opts.addOption("c", "numconnections", true, "Number of connections per host.");
+        opts.addOption("t", "timeout", true, "Timeout for a message in seconds. default = " + timeout);
+        opts.addOption("l", "nummessages", true, "Number of messages to send (all is default).");
+        opts.addOption("wi", "window_incrementsize", true, "Dynamic window increment step size. default = " + windowIncrementSize);
+        opts.addOption("wd", "window_decrementfactor", true, "Dynamic window decrement step size factor. default = " + windowDecrementFactor);
+        opts.addOption("wb", "window_backoffactor", true, "Dynamic window backoff factor. default = " + windowSizeBackOff);
+        opts.addOption("wr", "window_resizerate", true, "Dynamic window resize rate. default = " + windowResizeRate);
 
         CommandLine cmd = new DefaultParser().parse(opts, args);
 
@@ -128,9 +159,24 @@ class FeederParams {
         if (cmd.hasOption('c')) {
             numConnectionsPerTarget = Integer.valueOf(cmd.getOptionValue('c').trim());
         }
+        if (cmd.hasOption("wi")) {
+            windowIncrementSize = Integer.valueOf(cmd.getOptionValue("wi").trim());
+        }
+        if (cmd.hasOption("wd")) {
+            windowDecrementFactor = Double.valueOf(cmd.getOptionValue("wd").trim());
+        }
+        if (cmd.hasOption("wb")) {
+            windowSizeBackOff = Double.valueOf(cmd.getOptionValue("wb").trim());
+        }
+        if (cmd.hasOption("wr")) {
+            windowResizeRate = Double.valueOf(cmd.getOptionValue("wr").trim());
+        }
         if (cmd.hasOption('r')) {
             route = Route.parse(cmd.getOptionValue('r').trim());
         }
+        if (cmd.hasOption("t")) {
+            timeout = Double.valueOf(cmd.getOptionValue("t").trim());
+        }
         benchmarkMode =  cmd.hasOption('b');
         if (cmd.hasOption('o')) {
             String fileName = cmd.getOptionValue('o').trim();
@@ -142,6 +188,9 @@ class FeederParams {
         if (cmd.hasOption('s')) {
             setSerialTransfer();
         }
+        if (cmd.hasOption('l')) {
+            numMessagesToSend = Long.valueOf(cmd.getOptionValue('l').trim());
+        }
 
         if ( !cmd.getArgList().isEmpty()) {
             inputStreams.clear();
diff --git a/vespa_feed_perf/src/main/java/com/yahoo/vespa/feed/perf/SimpleFeeder.java b/vespa_feed_perf/src/main/java/com/yahoo/vespa/feed/perf/SimpleFeeder.java
index 2925ea08de9..aac7ab750bd 100644
--- a/vespa_feed_perf/src/main/java/com/yahoo/vespa/feed/perf/SimpleFeeder.java
+++ b/vespa_feed_perf/src/main/java/com/yahoo/vespa/feed/perf/SimpleFeeder.java
@@ -20,6 +20,7 @@ import com.yahoo.documentapi.messagebus.protocol.PutDocumentMessage;
 import com.yahoo.documentapi.messagebus.protocol.RemoveDocumentMessage;
 import com.yahoo.documentapi.messagebus.protocol.UpdateDocumentMessage;
 import com.yahoo.io.GrowableByteBuffer;
+import com.yahoo.messagebus.DynamicThrottlePolicy;
 import com.yahoo.messagebus.Error;
 import com.yahoo.messagebus.Message;
 import com.yahoo.messagebus.MessageBusParams;
@@ -45,9 +46,10 @@ import java.io.PrintStream;
 import java.nio.ByteBuffer;
 import java.util.Arrays;
 import java.util.List;
+import java.util.concurrent.ArrayBlockingQueue;
 import java.util.concurrent.Executor;
 import java.util.concurrent.ExecutorService;
-import java.util.concurrent.SynchronousQueue;
+import java.util.concurrent.RejectedExecutionHandler;
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
@@ -65,6 +67,7 @@ public class SimpleFeeder implements ReplyHandler {
     private final RPCMessageBus mbus;
     private final SourceSession session;
     private final int numThreads;
+    private final long numMessagesToSend;
     private final Destination destination;
     private final boolean benchmarkMode;
     private final static long REPORT_INTERVAL = TimeUnit.SECONDS.toMillis(10);
@@ -81,18 +84,20 @@ public class SimpleFeeder implements ReplyHandler {
         private final Destination destination;
         private final FeedReader reader;
         private final Executor executor;
-        AtomicReference<Throwable> failure;
+        private final long messagesToSend;
+        private final AtomicReference<Throwable> failure;
 
-        Metrics(Destination destination, FeedReader reader, Executor executor, AtomicReference<Throwable> failure) {
+        Metrics(Destination destination, FeedReader reader, Executor executor, AtomicReference<Throwable> failure, long messagesToSend) {
             this.destination = destination;
             this.reader = reader;
             this.executor = executor;
+            this.messagesToSend = messagesToSend;
             this.failure = failure;
         }
 
         long feed() throws Throwable {
             long numMessages = 0;
-            while (failure.get() == null) {
+            while ((failure.get() == null) && (numMessages < messagesToSend)) {
                 FeedOperation op = reader.read();
                 if (op.getType() == FeedOperation.Type.INVALID) {
                     break;
@@ -125,11 +130,13 @@ public class SimpleFeeder implements ReplyHandler {
         private final PrintStream err;
         private final Route route;
         private final SourceSession session;
+        private final long timeoutMS;
         private final AtomicReference<Throwable> failure;
-        MbusDestination(SourceSession session, Route route, AtomicReference<Throwable> failure, PrintStream err) {
+        MbusDestination(SourceSession session, Route route, double timeoutS, AtomicReference<Throwable> failure, PrintStream err) {
             this.route = route;
             this.err = err;
             this.session = session;
+            this.timeoutMS = (long)(timeoutS * 1000.0);
             this.failure = failure;
         }
         public void send(FeedOperation op) {
@@ -138,6 +145,7 @@ public class SimpleFeeder implements ReplyHandler {
                 err.println("ignoring operation; " + op.getType());
                 return;
             }
+            msg.setTimeRemaining(timeoutMS);
             msg.setContext(System.currentTimeMillis());
             msg.setRoute(route);
             try {
@@ -147,7 +155,7 @@ public class SimpleFeeder implements ReplyHandler {
                 }
             } catch (InterruptedException e) {}
         }
-        public void close() throws Exception {
+        public void close() {
             session.destroy();
         }
     }
@@ -270,7 +278,7 @@ public class SimpleFeeder implements ReplyHandler {
             }
         }
 
-        class LazyDocumentOperation extends ConditionalFeedOperation {
+        static class LazyDocumentOperation extends ConditionalFeedOperation {
             private final DocumentDeserializer deserializer;
             LazyDocumentOperation(DocumentDeserializer deserializer, TestAndSetCondition condition) {
                 super(Type.DOCUMENT, condition);
@@ -282,7 +290,7 @@ public class SimpleFeeder implements ReplyHandler {
                 return new Document(deserializer);
             }
         }
-        class LazyUpdateOperation extends ConditionalFeedOperation {
+        static class LazyUpdateOperation extends ConditionalFeedOperation {
             private final DocumentDeserializer deserializer;
             LazyUpdateOperation(DocumentDeserializer deserializer, TestAndSetCondition condition) {
                 super(Type.UPDATE, condition);
@@ -341,13 +349,14 @@ public class SimpleFeeder implements ReplyHandler {
         inputStreams = params.getInputStreams();
         out = params.getStdOut();
         numThreads = params.getNumDispatchThreads();
+        numMessagesToSend = params.getNumMessagesToSend();
         mbus = newMessageBus(docTypeMgr, params);
-        session = newSession(mbus, this, params.getMaxPending());
+        session = newSession(mbus, this, params);
         docTypeMgr.configure(params.getConfigId());
         benchmarkMode = params.isBenchmarkMode();
         destination = (params.getDumpStream() != null)
                 ? createDumper(params)
-                : new MbusDestination(session, params.getRoute(), failure, params.getStdErr());
+                : new MbusDestination(session, params.getRoute(), params.getTimeout(), failure, params.getStdErr());
     }
 
     SourceSession getSourceSession() { return session; }
@@ -369,18 +378,27 @@ public class SimpleFeeder implements ReplyHandler {
     }
 
 
+    static class RetryExecutionhandler implements RejectedExecutionHandler {
+
+        @Override
+        public void rejectedExecution(Runnable r, ThreadPoolExecutor executor) {
+            try {
+                executor.getQueue().put(r);
+            } catch (InterruptedException e) {}
+        }
+    }
 
     SimpleFeeder run() throws Throwable {
         ExecutorService executor = (numThreads > 1)
                 ? new ThreadPoolExecutor(numThreads, numThreads, 0L, TimeUnit.SECONDS,
-                                         new SynchronousQueue<>(false),
+                                         new ArrayBlockingQueue<>(numThreads*100),
                                          ThreadFactoryFactory.getDaemonThreadFactory("perf-feeder"),
-                                         new ThreadPoolExecutor.CallerRunsPolicy())
+                                         new RetryExecutionhandler())
                 : null;
         printHeader(out);
         long numMessagesSent = 0;
         for (InputStream in : inputStreams) {
-            Metrics m = new Metrics(destination, createFeedReader(in), executor, failure);
+            Metrics m = new Metrics(destination, createFeedReader(in), executor, failure, numMessagesToSend);
             numMessagesSent += m.feed();
         }
         while (failure.get() == null && numReplies.get() < numMessagesSent) {
@@ -469,11 +487,18 @@ public class SimpleFeeder implements ReplyHandler {
                                  params.getConfigId());
     }
 
-    private static SourceSession newSession(RPCMessageBus mbus, ReplyHandler replyHandler, int maxPending) {
+    private static SourceSession newSession(RPCMessageBus mbus, ReplyHandler replyHandler, FeederParams feederParams ) {
         SourceSessionParams params = new SourceSessionParams();
         params.setReplyHandler(replyHandler);
-        if (maxPending > 0) {
-            params.setThrottlePolicy(new StaticThrottlePolicy().setMaxPendingCount(maxPending));
+        if (feederParams.getMaxPending() > 0) {
+            params.setThrottlePolicy(new StaticThrottlePolicy().setMaxPendingCount(feederParams.getMaxPending()));
+        } else {
+            DynamicThrottlePolicy throttlePolicy = new DynamicThrottlePolicy()
+                    .setWindowSizeIncrement(feederParams.getWindowIncrementSize())
+                    .setResizeRate(feederParams.getWindowResizeRate())
+                    .setWindowSizeDecrementFactor(feederParams.getWindowDecrementFactor())
+                    .setWindowSizeBackOff(feederParams.getWindowSizeBackOff());
+            params.setThrottlePolicy(throttlePolicy);
         }
         return mbus.getMessageBus().createSourceSession(params);
     }
diff --git a/vespa_feed_perf/src/main/sh/vespa-feed-perf b/vespa_feed_perf/src/main/sh/vespa-feed-perf
index 466cd2ee98c..d6ccf0e4fc5 100755
--- a/vespa_feed_perf/src/main/sh/vespa-feed-perf
+++ b/vespa_feed_perf/src/main/sh/vespa-feed-perf
@@ -74,4 +74,4 @@ findhost
 
 # END environment bootstrap section
 
-exec java -jar $VESPA_HOME/lib/jars/vespa_feed_perf-jar-with-dependencies.jar "$@"
+exec java -XX:+UseParallelGC -XX:ParallelGCThreads=4 -jar $VESPA_HOME/lib/jars/vespa_feed_perf-jar-with-dependencies.jar "$@"
diff --git a/vespa_feed_perf/src/test/java/com/yahoo/vespa/feed/perf/FeederParamsTest.java b/vespa_feed_perf/src/test/java/com/yahoo/vespa/feed/perf/FeederParamsTest.java
index 8682adc0935..6f575038f75 100644
--- a/vespa_feed_perf/src/test/java/com/yahoo/vespa/feed/perf/FeederParamsTest.java
+++ b/vespa_feed_perf/src/test/java/com/yahoo/vespa/feed/perf/FeederParamsTest.java
@@ -28,6 +28,8 @@ public class FeederParamsTest {
     private static final String TESTFILE_JSON = "test.json";
     private static final String TESTFILE_VESPA = "test.vespa";
     private static final String TESTFILE_UNKNOWN = "test.xyz";
+    private static final double EPSILON = 0.000000000001;
+
 
     @Test
     public void requireThatAccessorsWork() {
@@ -88,12 +90,50 @@ public class FeederParamsTest {
     }
     @Test
     public void requireThatNumConnectionsAreParsed() throws ParseException, FileNotFoundException {
-        assertEquals(2, new FeederParams().getNumConnectionsPerTarget());
-        assertEquals(17, new FeederParams().parseArgs("-c 17").getNumConnectionsPerTarget());
+        assertEquals(1, new FeederParams().getNumConnectionsPerTarget());
+        assertEquals(16, new FeederParams().parseArgs("-c 16").getNumConnectionsPerTarget());
         assertEquals(17, new FeederParams().parseArgs("--numconnections", "17").getNumConnectionsPerTarget());
     }
 
     @Test
+    public void requireThatTimeoutIsParsed() throws ParseException, FileNotFoundException {
+        assertEquals(180.0, new FeederParams().getTimeout(), EPSILON);
+        assertEquals(16.7, new FeederParams().parseArgs("-t 16.7").getTimeout(), EPSILON);
+        assertEquals(1700.9, new FeederParams().parseArgs("--timeout", "1700.9").getTimeout(), EPSILON);
+    }
+
+    @Test
+    public void requireThatNumMessagesToSendAreParsed() throws ParseException, FileNotFoundException {
+        assertEquals(Long.MAX_VALUE, new FeederParams().getNumMessagesToSend());
+        assertEquals(18, new FeederParams().parseArgs("-l 18").getNumMessagesToSend());
+        assertEquals(19, new FeederParams().parseArgs("--nummessages", "19").getNumMessagesToSend());
+    }
+
+    @Test
+    public void requireThatWindowSizeIncrementIsParsed() throws ParseException, FileNotFoundException {
+        assertEquals(20, new FeederParams().getWindowIncrementSize());
+        assertEquals(17, new FeederParams().parseArgs("--window_incrementsize", "17").getWindowIncrementSize());
+    }
+
+    @Test
+    public void requireThatWindowSizeDecrementFactorIsParsed() throws ParseException, FileNotFoundException {
+        assertEquals(1.2, new FeederParams().getWindowDecrementFactor(), EPSILON);
+        assertEquals(1.3, new FeederParams().parseArgs("--window_decrementfactor", "1.3").getWindowDecrementFactor(), EPSILON);
+    }
+
+    @Test
+    public void requireThatWindowResizeRateIsParsed() throws ParseException, FileNotFoundException {
+        assertEquals(3.0, new FeederParams().getWindowResizeRate(), EPSILON);
+        assertEquals(5.5, new FeederParams().parseArgs("--window_resizerate", "5.5").getWindowResizeRate(), EPSILON);
+    }
+
+    @Test
+    public void requireThatWindowBackOffIsParsed() throws ParseException, FileNotFoundException {
+        assertEquals(0.95, new FeederParams().getWindowSizeBackOff(), EPSILON);
+        assertEquals(0.97, new FeederParams().parseArgs("--window_backoff", "0.97").getWindowSizeBackOff(), EPSILON);
+    }
+
+    @Test
     public void requireThatDumpStreamAreParsed() throws ParseException, IOException {
         assertNull(new FeederParams().getDumpStream());
 
diff --git a/vespabase/src/common-env.sh b/vespabase/src/common-env.sh
index b970c5bed60..ad77965fef6 100755
--- a/vespabase/src/common-env.sh
+++ b/vespabase/src/common-env.sh
@@ -23,7 +23,7 @@ consider_fallback () {
         : $1 already has value $oldvariablevalue
     elif [ -z "${2}" ]; then
         : proposed value "${2}" is empty
-    elif [ `expr match "$2" ".*'"` != 0 ]; then
+    elif [ `expr "$2" : ".*'"` != 0 ]; then
         : proposed value "${2}" contains a single-quote
     else
         eval "${1}='${2}'"
diff --git a/vespabase/src/rhel-prestart.sh b/vespabase/src/rhel-prestart.sh
index 3a770cdd785..d84122898a3 100755
--- a/vespabase/src/rhel-prestart.sh
+++ b/vespabase/src/rhel-prestart.sh
@@ -96,32 +96,32 @@ fixdir () {
 
 # BEGIN directory fixups
 
-fixdir   root        wheel 1777  logs
-fixdir   root        wheel 1777  tmp
-fixdir   root        wheel 1777  var/run
-fixdir ${VESPA_USER} wheel 1777  var/crash
-fixdir ${VESPA_USER} wheel 1777  logs/vespa
-fixdir ${VESPA_USER} wheel 1777  tmp/vespa
-fixdir ${VESPA_USER} wheel  755  var
-fixdir ${VESPA_USER} wheel  755  libexec/vespa/plugins/qrs
-fixdir ${VESPA_USER} wheel  755  logs/vespa/configserver
-fixdir ${VESPA_USER} wheel  755  logs/vespa/qrs
-fixdir ${VESPA_USER} wheel  755  logs/vespa/search
-fixdir ${VESPA_USER} wheel  755  var/db/vespa
-fixdir ${VESPA_USER} wheel  755  var/db/vespa/tmp
-fixdir ${VESPA_USER} wheel  755  var/db/vespa/config_server
-fixdir ${VESPA_USER} wheel  755  var/db/vespa/config_server/serverdb
-fixdir ${VESPA_USER} wheel  755  var/db/vespa/config_server/serverdb/tenants
-fixdir ${VESPA_USER} wheel  755  var/db/vespa/filedistribution
-fixdir ${VESPA_USER} wheel  755  var/db/vespa/index
-fixdir ${VESPA_USER} wheel  755  var/db/vespa/logcontrol
-fixdir ${VESPA_USER} wheel  755  var/db/vespa/search
-fixdir ${VESPA_USER} wheel  755  var/jdisc_container
-fixdir ${VESPA_USER} wheel  755  var/vespa
-fixdir ${VESPA_USER} wheel  755  var/vespa/application
-fixdir ${VESPA_USER} wheel  755  var/vespa/bundlecache
-fixdir ${VESPA_USER} wheel  755  var/vespa/bundlecache/configserver
-fixdir ${VESPA_USER} wheel  755  var/vespa/cache/config/
+fixdir   root        root  1777  logs
+fixdir   root        root  1777  tmp
+fixdir   root        root  1777  var/run
+fixdir ${VESPA_USER} root  1777  var/crash
+fixdir ${VESPA_USER} root  1777  logs/vespa
+fixdir ${VESPA_USER} root  1777  tmp/vespa
+fixdir   root        root   755  var
+fixdir ${VESPA_USER} root   755  libexec/vespa/plugins/qrs
+fixdir ${VESPA_USER} root   755  logs/vespa/configserver
+fixdir ${VESPA_USER} root   755  logs/vespa/qrs
+fixdir ${VESPA_USER} root   755  logs/vespa/search
+fixdir ${VESPA_USER} root   755  var/db/vespa
+fixdir ${VESPA_USER} root   755  var/db/vespa/tmp
+fixdir ${VESPA_USER} root   755  var/db/vespa/config_server
+fixdir ${VESPA_USER} root   755  var/db/vespa/config_server/serverdb
+fixdir ${VESPA_USER} root   755  var/db/vespa/config_server/serverdb/tenants
+fixdir ${VESPA_USER} root   755  var/db/vespa/filedistribution
+fixdir ${VESPA_USER} root   755  var/db/vespa/index
+fixdir ${VESPA_USER} root   755  var/db/vespa/logcontrol
+fixdir ${VESPA_USER} root   755  var/db/vespa/search
+fixdir ${VESPA_USER} root   755  var/jdisc_container
+fixdir ${VESPA_USER} root   755  var/vespa
+fixdir ${VESPA_USER} root   755  var/vespa/application
+fixdir ${VESPA_USER} root   755  var/vespa/bundlecache
+fixdir ${VESPA_USER} root   755  var/vespa/bundlecache/configserver
+fixdir ${VESPA_USER} root   755  var/vespa/cache/config/
 
 if [ "${VESPA_UNPRIVILEGED}" != yes ]; then
   chown -hR ${VESPA_USER} logs/vespa
diff --git a/vespabase/vespa-base.spec b/vespabase/vespa-base.spec
index 7299974d1b0..e385d0b7367 100644
--- a/vespabase/vespa-base.spec
+++ b/vespabase/vespa-base.spec
@@ -6,7 +6,7 @@
 Name:           vespa-base
 Version:        %version
 Release:        1%{?dist}
-BuildArch:      noarch
+BuildArch:      x86_64
 Summary:        Vespa common files
 Group:          Applications/Databases
 License:        Commercial
diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/OperationHandlerImpl.java b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/OperationHandlerImpl.java
index 0485689b15d..1224e668bc0 100644
--- a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/OperationHandlerImpl.java
+++ b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/OperationHandlerImpl.java
@@ -35,6 +35,7 @@ import java.time.Instant;
 import java.util.List;
 import java.util.Optional;
 import java.util.Set;
+import java.util.stream.Collectors;
 
 /**
  * Sends operations to messagebus via document api.
@@ -352,12 +353,16 @@ public class OperationHandlerImpl implements OperationHandler {
     protected static ClusterDef resolveClusterDef(Optional<String> wantedCluster, List<ClusterDef> clusters) throws RestApiException {
         if (clusters.size() == 0) {
             throw new IllegalArgumentException("Your Vespa cluster does not have any content clusters " +
-                    "declared. Visiting feature is not available.");
+                                               "declared. Visiting feature is not available.");
         }
         if (! wantedCluster.isPresent()) {
             if (clusters.size() != 1) {
-                throw new RestApiException(Response.createErrorResponse(400, "Several clusters exist: " +
-                        clusterListToString(clusters) + " you must specify one. ", RestUri.apiErrorCodes.SEVERAL_CLUSTERS));
+                String message = "Several clusters exist: " +
+                                 clusters.stream().map(c -> "'" + c.getName() + "'").collect(Collectors.joining(", ")) +
+                                 ". You must specify one.";
+                throw new RestApiException(Response.createErrorResponse(400,
+                                                                        message,
+                                                                        RestUri.apiErrorCodes.SEVERAL_CLUSTERS));
             }
             return clusters.get(0);
         }
@@ -367,20 +372,18 @@ public class OperationHandlerImpl implements OperationHandler {
                 return clusterDef;
             }
         }
-        throw new RestApiException(Response.createErrorResponse(400, "Your vespa cluster contains the content clusters " +
-                clusterListToString(clusters) + " not " + wantedCluster.get() + ". Please select a valid vespa cluster.", RestUri.apiErrorCodes.MISSING_CLUSTER));
+        String message = "Your vespa cluster contains the content clusters " +
+                         clusters.stream().map(c -> "'" + c.getName() + "'").collect(Collectors.joining(", ")) +
+                         ", not '" + wantedCluster.get() + "'. Please select a valid vespa cluster.";
+        throw new RestApiException(Response.createErrorResponse(400,
+                                                                message,
+                                                                RestUri.apiErrorCodes.MISSING_CLUSTER));
     }
 
     protected static String clusterDefToRoute(ClusterDef clusterDef) {
         return "[Storage:cluster=" + clusterDef.getName() + ";clusterconfigid=" + clusterDef.getConfigId() + "]";
     }
 
-    private static String clusterListToString(List<ClusterDef> clusters) {
-        StringBuilder clusterListString = new StringBuilder();
-        clusters.forEach(x -> clusterListString.append(x.getName()).append(" (").append(x.getConfigId()).append("), "));
-        return clusterListString.toString();
-    }
-
     private static String buildAugmentedDocumentSelection(RestUri restUri, String  documentSelection) {
         if (restUri.isRootOnly()) {
             return documentSelection; // May be empty, that's fine.
diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientFeederV3.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientFeederV3.java
index ec5fc0cad07..e6bc2211bea 100644
--- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientFeederV3.java
+++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientFeederV3.java
@@ -53,12 +53,11 @@ class ClientFeederV3 {
     private final Metric metric;
     private Instant prevOpsPerSecTime = Instant.now();
     private double operationsForOpsPerSec = 0d;
-
     private final Object monitor = new Object();
     private final StreamReaderV3 streamReaderV3;
     private final AtomicInteger ongoingRequests = new AtomicInteger(0);
-    private String hostName;
-    private AtomicInteger threadsAvailableForFeeding;
+    private final String hostName;
+    private final AtomicInteger threadsAvailableForFeeding;
 
     ClientFeederV3(
             ReferencedResource<SharedSourceSession> sourceSession,
diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandlerV3.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandlerV3.java
index 37803d96714..bcd78c790fe 100644
--- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandlerV3.java
+++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandlerV3.java
@@ -19,6 +19,8 @@ import com.yahoo.messagebus.shared.SharedSourceSession;
 import com.yahoo.vespa.http.client.core.Headers;
 import com.yahoo.yolean.Exceptions;
 
+import java.time.Duration;
+import java.time.Instant;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.Map;
@@ -43,6 +45,9 @@ public class FeedHandlerV3 extends LoggingRequestHandler {
     protected final ReplyHandler feedReplyHandler;
     private final Metric metric;
     private final Object monitor = new Object();
+    private int remainingThreadsForFeedingAllowance;
+    private final Duration timeBetweenBumpingMaxThreads;
+    private Instant nextTimeToAllocateAnotherThread = Instant.now();
     private final AtomicInteger threadsAvailableForFeeding;
     private static final Logger log = Logger.getLogger(FeedHandlerV3.class.getName());
 
@@ -60,10 +65,20 @@ public class FeedHandlerV3 extends LoggingRequestHandler {
         this.metric = parentCtx.getMetric();
         // 40% of the threads can be blocking on feeding before we deny requests.
         if (threadpoolConfig != null) {
-            threadsAvailableForFeeding = new AtomicInteger(Math.max((int) (0.4 * threadpoolConfig.maxthreads()), 1));
+            remainingThreadsForFeedingAllowance = Math.max((int) (0.4 * threadpoolConfig.maxthreads()), 1);
+            if (threadpoolConfig.softStartSeconds() > 0.0) {
+                threadsAvailableForFeeding = new AtomicInteger(0);
+                timeBetweenBumpingMaxThreads = Duration.ofMillis((long)(threadpoolConfig.softStartSeconds() * 1000) / remainingThreadsForFeedingAllowance);
+            } else {
+                threadsAvailableForFeeding = new AtomicInteger(remainingThreadsForFeedingAllowance);
+                remainingThreadsForFeedingAllowance = 0;
+                timeBetweenBumpingMaxThreads = null;
+            }
         } else {
             log.warning("No config for threadpool, using 200 for max blocking threads for feeding.");
             threadsAvailableForFeeding = new AtomicInteger(200);
+            remainingThreadsForFeedingAllowance = 0;
+            timeBetweenBumpingMaxThreads = null;
         }
     }
 
@@ -78,6 +93,12 @@ public class FeedHandlerV3 extends LoggingRequestHandler {
         String clientId = clientId(request);
         ClientFeederV3 clientFeederV3;
         synchronized (monitor) {
+            Instant now = Instant.now();
+            if ((remainingThreadsForFeedingAllowance > 0) && (now.isAfter(nextTimeToAllocateAnotherThread))) {
+                threadsAvailableForFeeding.incrementAndGet();
+                remainingThreadsForFeedingAllowance --;
+                nextTimeToAllocateAnotherThread = now.plus(timeBetweenBumpingMaxThreads);
+            }
             if (! clientFeederByClientId.containsKey(clientId)) {
                 SourceSessionParams sourceSessionParams = sourceSessionParams(request);
                 clientFeederByClientId.put(clientId,
diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/OperationHandlerImplTest.java b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/OperationHandlerImplTest.java
index 4e5092cd416..bda49ecd3f5 100644
--- a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/OperationHandlerImplTest.java
+++ b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/OperationHandlerImplTest.java
@@ -94,8 +94,8 @@ public class OperationHandlerImplTest {
             assertThat(e.getResponse().getStatus(), is(400));
             String errorMsg = renderRestApiExceptionAsString(e);
             assertThat(errorMsg, is("{\"errors\":[{\"description\":" +
-                    "\"MISSING_CLUSTER Your vespa cluster contains the content clusters foo2 (configId2), foo (configId)," +
-                    " foo3 (configId2),  not wrong. Please select a valid vespa cluster.\",\"id\":-9}]}"));
+                    "\"MISSING_CLUSTER Your vespa cluster contains the content clusters 'foo2', 'foo'," +
+                    " 'foo3', not 'wrong'. Please select a valid vespa cluster.\",\"id\":-9}]}"));
             return;
         }
         fail("Expected exception");
diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/RestApiTest.java b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/RestApiTest.java
index aeddc762586..0661363477f 100644
--- a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/RestApiTest.java
+++ b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/RestApiTest.java
@@ -202,7 +202,7 @@ public class RestApiTest {
 
     // Get logs through some hackish fetch method. Logs is something the mocked backend write.
     String getLog() throws IOException {
-        // The mocked backend will throw a runtime exception wtih a log if delete is called three times..
+        // The mocked backend will throw a runtime exception with a log if delete is called three times..
         Request request = new Request("http://localhost:" + getFirstListenPort() + remove_test_uri);
         HttpDelete delete = new HttpDelete(request.getUri());
         doRest(delete);
diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/feedhandler/v3/FeedTesterV3.java b/vespaclient-container-plugin/src/test/java/com/yahoo/feedhandler/v3/FeedTesterV3.java
index 1677c95a54a..0bb42851347 100644
--- a/vespaclient-container-plugin/src/test/java/com/yahoo/feedhandler/v3/FeedTesterV3.java
+++ b/vespaclient-container-plugin/src/test/java/com/yahoo/feedhandler/v3/FeedTesterV3.java
@@ -2,6 +2,7 @@
 package com.yahoo.feedhandler.v3;
 
 import com.google.common.base.Splitter;
+import com.yahoo.container.handler.ThreadpoolConfig;
 import com.yahoo.container.jdisc.HttpRequest;
 import com.yahoo.container.jdisc.HttpResponse;
 import com.yahoo.container.jdisc.messagebus.SessionCache;
@@ -46,7 +47,7 @@ public class FeedTesterV3 {
 
     @Test
     public void feedOneDocument() throws Exception {
-        final FeedHandlerV3 feedHandlerV3 = setupFeederHandler();
+        final FeedHandlerV3 feedHandlerV3 = setupFeederHandler(null);
         HttpResponse httpResponse = feedHandlerV3.handle(createRequest(1));
         ByteArrayOutputStream outStream = new ByteArrayOutputStream();
         httpResponse.render(outStream);
@@ -56,7 +57,7 @@ public class FeedTesterV3 {
 
     @Test
     public void feedOneBrokenDocument() throws Exception {
-        final FeedHandlerV3 feedHandlerV3 = setupFeederHandler();
+        final FeedHandlerV3 feedHandlerV3 = setupFeederHandler(null);
         HttpResponse httpResponse = feedHandlerV3.handle(createBrokenRequest());
         ByteArrayOutputStream outStream = new ByteArrayOutputStream();
         httpResponse.render(outStream);
@@ -67,7 +68,7 @@ public class FeedTesterV3 {
 
     @Test
     public void feedManyDocument() throws Exception {
-        final FeedHandlerV3 feedHandlerV3 = setupFeederHandler();
+        final FeedHandlerV3 feedHandlerV3 = setupFeederHandler(null);
         HttpResponse httpResponse = feedHandlerV3.handle(createRequest(100));
         ByteArrayOutputStream outStream = new ByteArrayOutputStream();
         httpResponse.render(outStream);
@@ -76,6 +77,20 @@ public class FeedTesterV3 {
         assertThat(Splitter.on("\n").splitToList(result).size(), is(101));
     }
 
+    @Test
+    public void softRestart() throws Exception {
+        ThreadpoolConfig.Builder builder = new ThreadpoolConfig.Builder().softStartSeconds(5);
+        final FeedHandlerV3 feedHandlerV3 = setupFeederHandler(builder.build());
+        for (int i= 0; i < 100; i++) {
+            HttpResponse httpResponse = feedHandlerV3.handle(createRequest(100));
+            ByteArrayOutputStream outStream = new ByteArrayOutputStream();
+            httpResponse.render(outStream);
+            assertThat(httpResponse.getContentType(), is("text/plain"));
+            String result = Utf8.toString(outStream.toByteArray());
+            assertThat(Splitter.on("\n").splitToList(result).size(), is(101));
+        }
+    }
+
     private static DocumentTypeManager createDoctypeManager() {
         DocumentTypeManager docTypeManager = new DocumentTypeManager();
         DocumentType documentType = new DocumentType("testdocument");
@@ -115,14 +130,14 @@ public class FeedTesterV3 {
         return request;
     }
 
-    private FeedHandlerV3 setupFeederHandler() throws Exception {
+    private FeedHandlerV3 setupFeederHandler(ThreadpoolConfig threadPoolConfig) throws Exception {
         Executor threadPool = Executors.newCachedThreadPool();
         DocumentmanagerConfig docMan = new DocumentmanagerConfig(new DocumentmanagerConfig.Builder().enablecompression(true));
         FeedHandlerV3 feedHandlerV3 = new FeedHandlerV3(
                 new FeedHandlerV3.Context(threadPool, AccessLog.voidAccessLog(), metric),
                 docMan,
                 null /* session cache */,
-                null /* thread pool config */,
+                threadPoolConfig /* thread pool config */,
                 new DocumentApiMetrics(MetricReceiver.nullImplementation, "test")) {
             @Override
             protected ReferencedResource<SharedSourceSession> retainSource(
diff --git a/vespaclient-container-plugin/src/test/rest-api-application/services.xml b/vespaclient-container-plugin/src/test/rest-api-application/services.xml
index 346740bc815..ae1b87635a9 100644
--- a/vespaclient-container-plugin/src/test/rest-api-application/services.xml
+++ b/vespaclient-container-plugin/src/test/rest-api-application/services.xml
@@ -2,6 +2,8 @@
 <!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
 <container version="1.0" jetty="true">
 
+    <accesslog type="disabled"/>
+
     <handler id="com.yahoo.document.restapi.resource.RestApiWithTestDocumentHandler" bundle="integration-test">
         <binding>http://*/document/v1/*</binding>
     </handler>
diff --git a/vespaclient-core/src/main/java/com/yahoo/feedapi/MessagePropertyProcessor.java b/vespaclient-core/src/main/java/com/yahoo/feedapi/MessagePropertyProcessor.java
index 74a4f916c07..581a51ffc02 100644
--- a/vespaclient-core/src/main/java/com/yahoo/feedapi/MessagePropertyProcessor.java
+++ b/vespaclient-core/src/main/java/com/yahoo/feedapi/MessagePropertyProcessor.java
@@ -208,14 +208,6 @@ public class MessagePropertyProcessor implements ConfigSubscriber.SingleSubscrib
             this.route = route;
         }
 
-        public long getTimeout() {
-            return timeout;
-        }
-
-        public void setTimeout(long timeout) {
-            this.timeout = timeout;
-        }
-
         public DocumentProtocol.Priority getPriority() {
             return priority;
         }
diff --git a/vespaclient-java/src/main/java/com/yahoo/vespasummarybenchmark/VespaSummaryBenchmark.java b/vespaclient-java/src/main/java/com/yahoo/vespasummarybenchmark/VespaSummaryBenchmark.java
index 16f8198a8ac..c47ffb241d2 100644
--- a/vespaclient-java/src/main/java/com/yahoo/vespasummarybenchmark/VespaSummaryBenchmark.java
+++ b/vespaclient-java/src/main/java/com/yahoo/vespasummarybenchmark/VespaSummaryBenchmark.java
@@ -5,13 +5,29 @@ import com.yahoo.compress.CompressionType;
 import com.yahoo.document.GlobalId;
 import com.yahoo.document.idstring.IdString;
 import com.yahoo.document.serialization.DeserializationException;
-import com.yahoo.jrt.*;
+import com.yahoo.jrt.DataValue;
+import com.yahoo.jrt.Int32Value;
+import com.yahoo.jrt.Int8Value;
+import com.yahoo.jrt.Request;
+import com.yahoo.jrt.RequestWaiter;
+import com.yahoo.jrt.Spec;
+import com.yahoo.jrt.Supervisor;
+import com.yahoo.jrt.Target;
+import com.yahoo.jrt.Transport;
+import com.yahoo.jrt.Values;
 import com.yahoo.log.LogSetup;
-import com.yahoo.slime.*;
+import com.yahoo.slime.BinaryFormat;
+import com.yahoo.slime.Cursor;
+import com.yahoo.slime.JsonFormat;
+import com.yahoo.slime.Slime;
 import net.jpountz.lz4.LZ4Factory;
 import net.jpountz.lz4.LZ4FastDecompressor;
 
-import java.io.*;
+import java.io.BufferedReader;
+import java.io.DataInputStream;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
 import java.util.ArrayList;
 import java.util.List;
 
@@ -26,6 +42,7 @@ import java.util.List;
 public class VespaSummaryBenchmark {
 
     private final Supervisor supervisor = new Supervisor(new Transport());
+    private static final LZ4Factory lz4Factory = LZ4Factory.fastestInstance();
 
     private VespaSummaryBenchmark() { }
 
@@ -82,8 +99,7 @@ public class VespaSummaryBenchmark {
             int uncompressedSize = ret.get(1).asInt32();
             byte [] blob = ret.get(2).asData();
             if (type == CompressionType.LZ4) {
-                LZ4Factory factory = LZ4Factory.fastestInstance();
-                LZ4FastDecompressor decompressor = factory.fastDecompressor();
+                LZ4FastDecompressor decompressor = lz4Factory.fastDecompressor();
                 byte [] uncompressed = new byte [uncompressedSize];
                 int compressedLength = decompressor.decompress(blob, 0, uncompressed, 0, uncompressedSize);
                 if (compressedLength != blob.length) {
diff --git a/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisit.java b/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisit.java
index 307cf979c33..83199c76e5a 100644
--- a/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisit.java
+++ b/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisit.java
@@ -25,6 +25,7 @@ import org.apache.commons.cli.Options;
 import java.io.*;
 import java.nio.charset.Charset;
 import java.util.Map;
+import java.util.stream.Collectors;
 
 /**
  * Client using visiting, used by the vespa-visit command line tool.
@@ -572,18 +573,14 @@ public class VdsVisit {
     protected static String resolveClusterRoute(ClusterList clusters, String wantedCluster) {
         if (clusters.getStorageClusters().size() == 0) {
             throw new IllegalArgumentException("Your Vespa cluster does not have any content clusters " +
-                    "declared. Visiting feature is not available.");
+                                               "declared. Visiting feature is not available.");
         }
 
         ClusterDef found = null;
 
-        String names = "";
-        for (ClusterDef c : clusters.getStorageClusters()) {
-            if (!names.isEmpty()) {
-                names += ", ";
-            }
-            names += c.getName();
-        }
+        String names = clusters.getStorageClusters()
+                               .stream().map(c -> "'" + c.getName() + "'")
+                               .collect(Collectors.joining(", "));
         if (wantedCluster != null) {
             for (ClusterDef c : clusters.getStorageClusters()) {
                 if (c.getName().equals(wantedCluster)) {
@@ -592,13 +589,14 @@ public class VdsVisit {
             }
             if (found == null) {
                 throw new IllegalArgumentException("Your vespa cluster contains the content clusters " +
-                        names + ", not " + wantedCluster + ". Please select a valid vespa cluster.");
+                                                   names + ", not '" + wantedCluster +
+                                                   "'. Please select a valid vespa cluster.");
             }
         } else if (clusters.getStorageClusters().size() == 1) {
             found = clusters.getStorageClusters().get(0);
         } else {
             throw new IllegalArgumentException("Your vespa cluster contains the content clusters " +
-                    names + ". Please use the -c option to select one of them as a target for visiting.");
+                                               names + ". Please use the -c option to select one of them as a target for visiting.");
         }
 
         return "[Storage:cluster=" + found.getName() + ";clusterconfigid=" + found.getConfigId() + "]";
diff --git a/vespaclient-java/src/test/java/com/yahoo/vespavisit/VdsVisitTestCase.java b/vespaclient-java/src/test/java/com/yahoo/vespavisit/VdsVisitTestCase.java
index 20f3cbcfe23..d9efe2c5129 100644
--- a/vespaclient-java/src/test/java/com/yahoo/vespavisit/VdsVisitTestCase.java
+++ b/vespaclient-java/src/test/java/com/yahoo/vespavisit/VdsVisitTestCase.java
@@ -242,7 +242,9 @@ public class VdsVisitTestCase {
         try {
             VdsVisit.resolveClusterRoute(clusterList, "borkbork");
         } catch (IllegalArgumentException e) {
-            assertTrue(e.getMessage().contains("Your vespa cluster contains the content clusters storage, not borkbork."));
+            assertEquals("Your vespa cluster contains the content clusters 'storage', not 'borkbork'. " +
+                         "Please select a valid vespa cluster.",
+                         e.getMessage());
         }
     }
 
diff --git a/vespaclient/src/perl/lib/Yahoo/Vespa/Http.pm b/vespaclient/src/perl/lib/Yahoo/Vespa/Http.pm
index 2dbf475f2a7..d907e89fa54 100644
--- a/vespaclient/src/perl/lib/Yahoo/Vespa/Http.pm
+++ b/vespaclient/src/perl/lib/Yahoo/Vespa/Http.pm
@@ -100,7 +100,10 @@ sub initialize { # ()
     my $tls_enabled = $ENV{'VESPA_TLS_ENABLED'};
     if (defined $tls_enabled and $tls_enabled eq '1') {
         $BROWSER->ssl_opts( SSL_version => 'TLSv12');
-        $BROWSER->ssl_opts( verify_hostname => 0);
+        my $hostname_verification_disabled = $ENV{'VESPA_TLS_HOSTNAME_VALIDATION_DISABLED'};
+        if (defined $hostname_verification_disabled and $hostname_verification_disabled eq '1') {
+            $BROWSER->ssl_opts( verify_hostname => 0);
+        }
         $BROWSER->ssl_opts( SSL_cipher_list => 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-CHACHA20-POLY1305:TLS13-AES-128-GCM-SHA256:TLS13-AES-256-GCM-SHA384:TLS13-CHACHA20-POLY1305-SHA256' );
     }
     if (defined $ENV{'VESPA_TLS_CA_CERT'}) {
diff --git a/vespajlib/abi-spec.json b/vespajlib/abi-spec.json
index b8b6716d879..d9467a41f78 100644
--- a/vespajlib/abi-spec.json
+++ b/vespajlib/abi-spec.json
@@ -1538,7 +1538,9 @@
       "public"
     ],
     "methods": [
+      "public void <init>(com.yahoo.tensor.functions.TensorFunction)",
       "public void <init>(com.yahoo.tensor.functions.TensorFunction, java.lang.String)",
+      "public void <init>(com.yahoo.tensor.functions.TensorFunction, java.util.List)",
       "public java.util.List arguments()",
       "public com.yahoo.tensor.functions.TensorFunction withArguments(java.util.List)",
       "public com.yahoo.tensor.functions.PrimitiveTensorFunction toPrimitive()",
@@ -1553,7 +1555,9 @@
       "public"
     ],
     "methods": [
+      "public void <init>(com.yahoo.tensor.functions.TensorFunction)",
       "public void <init>(com.yahoo.tensor.functions.TensorFunction, java.lang.String)",
+      "public void <init>(com.yahoo.tensor.functions.TensorFunction, java.util.List)",
       "public java.util.List arguments()",
       "public com.yahoo.tensor.functions.TensorFunction withArguments(java.util.List)",
       "public com.yahoo.tensor.functions.PrimitiveTensorFunction toPrimitive()",
@@ -2859,7 +2863,8 @@
     ],
     "methods": [
       "public static java.lang.String encode(java.util.Map)",
-      "public static java.lang.String escape(java.lang.String)"
+      "public static java.lang.String escape(java.lang.String)",
+      "public static boolean equals(java.lang.String, java.lang.String)"
     ],
     "fields": []
   },
@@ -3022,7 +3027,8 @@
       "public static boolean isTextCharacter(int)",
       "public static java.util.OptionalInt validateTextString(java.lang.String)",
       "public static boolean isDisplayable(int)",
-      "public static java.lang.String stripInvalidCharacters(java.lang.String)"
+      "public static java.lang.String stripInvalidCharacters(java.lang.String)",
+      "public static java.lang.String truncate(java.lang.String, int)"
     ],
     "fields": []
   },
diff --git a/vespajlib/pom.xml b/vespajlib/pom.xml
index a8380e9513c..7631a2af0fb 100644
--- a/vespajlib/pom.xml
+++ b/vespajlib/pom.xml
@@ -20,8 +20,8 @@
 
     <!-- compile scope -->
     <dependency>
-      <groupId>net.jpountz.lz4</groupId>
-      <artifactId>lz4</artifactId>
+      <groupId>org.lz4</groupId>
+      <artifactId>lz4-java</artifactId>
     </dependency>
     <dependency>
       <groupId>org.apache.commons</groupId>
diff --git a/vespajlib/src/main/java/com/yahoo/collections/AbstractFilteringList.java b/vespajlib/src/main/java/com/yahoo/collections/AbstractFilteringList.java
index b7c6322d951..176e5044bc2 100644
--- a/vespajlib/src/main/java/com/yahoo/collections/AbstractFilteringList.java
+++ b/vespajlib/src/main/java/com/yahoo/collections/AbstractFilteringList.java
@@ -4,15 +4,14 @@ package com.yahoo.collections;
 import java.util.Collection;
 import java.util.Comparator;
 import java.util.HashSet;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Optional;
-import java.util.concurrent.atomic.AtomicLong;
 import java.util.function.BiFunction;
 import java.util.function.Function;
 import java.util.function.Predicate;
 import java.util.stream.Stream;
 
-import static java.util.stream.Collectors.reducing;
 import static java.util.stream.Collectors.toUnmodifiableList;
 
 /**
@@ -20,7 +19,7 @@ import static java.util.stream.Collectors.toUnmodifiableList;
  *
  * @author jonmv
  */
-public abstract class AbstractFilteringList<Type, ListType extends AbstractFilteringList<Type, ListType>> {
+public abstract class AbstractFilteringList<Type, ListType extends AbstractFilteringList<Type, ListType>> implements Iterable<Type> {
 
     private final List<Type> items;
     private final boolean negate;
@@ -63,7 +62,7 @@ public abstract class AbstractFilteringList<Type, ListType extends AbstractFilte
     }
 
     /** Returns the union of the two lists. */
-    public ListType and(ListType others) {
+    public ListType concat(ListType others) {
         return constructor.apply(Stream.concat(items.stream(), others.asList().stream()).collect(toUnmodifiableList()), false);
     }
 
@@ -84,4 +83,9 @@ public abstract class AbstractFilteringList<Type, ListType extends AbstractFilte
 
     public final int size() { return items.size(); }
 
+    @Override
+    public Iterator<Type> iterator() {
+        return items.iterator();
+    }
+
 }
diff --git a/vespajlib/src/main/java/com/yahoo/collections/Pair.java b/vespajlib/src/main/java/com/yahoo/collections/Pair.java
index 506ad10b98e..6587d1804f9 100644
--- a/vespajlib/src/main/java/com/yahoo/collections/Pair.java
+++ b/vespajlib/src/main/java/com/yahoo/collections/Pair.java
@@ -1,6 +1,8 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.collections;
 
+import java.util.Objects;
+
 /**
  * An immutable pair of objects. This implements equals and hashCode by delegating to the
  * pair objects.
@@ -33,19 +35,13 @@ public class Pair<F, S> {
     }
 
     @Override
-    public boolean equals(final Object o) {
+    public boolean equals(Object o) {
         if (o == this) return true;
         if (!(o instanceof Pair)) return false;
 
         @SuppressWarnings("rawtypes")
-        final Pair other = (Pair) o;
-        return equals(this.first, other.first)
-                && equals(this.second, other.second);
-    }
-
-    private static boolean equals(final Object a, final Object b) {
-        if (a == null) return b == null;
-        return a.equals(b);
+        Pair other = (Pair) o;
+        return Objects.equals(this.first, other.first) && Objects.equals(this.second, other.second);
     }
 
     @Override
diff --git a/vespajlib/src/main/java/com/yahoo/compress/Compressor.java b/vespajlib/src/main/java/com/yahoo/compress/Compressor.java
index 9e9fac936f4..fb5da192f36 100644
--- a/vespajlib/src/main/java/com/yahoo/compress/Compressor.java
+++ b/vespajlib/src/main/java/com/yahoo/compress/Compressor.java
@@ -3,8 +3,12 @@ package com.yahoo.compress;
 
 import net.jpountz.lz4.LZ4Compressor;
 import net.jpountz.lz4.LZ4Factory;
+import net.jpountz.lz4.LZ4FastDecompressor;
+import net.jpountz.lz4.LZ4SafeDecompressor;
+
 import java.util.Arrays;
 import java.util.Optional;
+import java.util.Random;
 
 /**
  * Compressor which can compress and decompress in various formats.
@@ -19,7 +23,7 @@ public class Compressor {
     private final double compressionThresholdFactor;
     private final int compressMinSizeBytes;
 
-    private final LZ4Factory factory = LZ4Factory.fastestInstance();
+    private static final LZ4Factory factory = LZ4Factory.fastestInstance();
 
     /** Creates a compressor with default settings. */
     public Compressor() {
@@ -31,6 +35,10 @@ public class Compressor {
         this(type, 9, 0.95, 0);
     }
 
+    public Compressor(CompressionType type, int level) {
+        this(type, level, 0.95, 0);
+    }
+
     /**
      * Creates a compressor.
      *
@@ -79,8 +87,7 @@ public class Compressor {
             case LZ4:
                 int dataSize = uncompressedSize.isPresent() ? uncompressedSize.get() : data.length;
                 if (dataSize < compressMinSizeBytes) return new Compression(CompressionType.INCOMPRESSIBLE, dataSize, data);
-                LZ4Compressor compressor = level < 7 ? factory.fastCompressor() : factory.highCompressor();
-                byte[] compressedData = compressor.compress(data, 0, dataSize);
+                byte[] compressedData = getCompressor().compress(data, 0, dataSize);
                 if (compressedData.length + 8 >= dataSize * compressionThresholdFactor)
                     return new Compression(CompressionType.INCOMPRESSIBLE, dataSize, data);
                 return new Compression(CompressionType.LZ4, dataSize, compressedData);
@@ -88,6 +95,9 @@ public class Compressor {
                 throw new IllegalArgumentException(requestedCompression + " is not supported");
         }
     }
+    private LZ4Compressor getCompressor() {
+        return level < 7 ? factory.fastCompressor() : factory.highCompressor();
+    }
     /** Compresses some data using the requested compression type */
     public Compression compress(CompressionType requestedCompression, byte[] data) { return compress(requestedCompression, data, Optional.empty()); }
     /** Compresses some data using the compression type of this compressor */
@@ -133,6 +143,39 @@ public class Compressor {
         return decompress(compression.type(), compression.data(), 0, compression.uncompressedSize(), Optional.empty());
     }
 
+    public byte[] compressUnconditionally(byte[] input) {
+        return getCompressor().compress(input);
+    }
+
+    public byte [] decompressUnconditionally(byte[] input, int srcOffset, int uncompressedLen) {
+        if (input.length > 0) {
+            return factory.fastDecompressor().decompress(input, srcOffset, uncompressedLen);
+        }
+        return new byte[0];
+    }
+
+    public long warmup(double seconds) {
+        byte [] input = new byte[0x4000];
+        new Random().nextBytes(input);
+        long timeDone = System.nanoTime() + (long)(seconds*1000000000);
+        long compressedBytes = 0;
+        byte [] decompressed = new byte [input.length];
+        LZ4FastDecompressor fastDecompressor = factory.fastDecompressor();
+        LZ4SafeDecompressor safeDecompressor = factory.safeDecompressor();
+        LZ4Compressor fastCompressor = factory.fastCompressor();
+        LZ4Compressor highCompressor = factory.highCompressor();
+        while (System.nanoTime() < timeDone) {
+            byte [] compressedFast = fastCompressor.compress(input);
+            byte [] compressedHigh = highCompressor.compress(input);
+            fastDecompressor.decompress(compressedFast, decompressed);
+            fastDecompressor.decompress(compressedHigh, decompressed);
+            safeDecompressor.decompress(compressedFast, decompressed);
+            safeDecompressor.decompress(compressedHigh, decompressed);
+            compressedBytes += compressedFast.length + compressedHigh.length;
+        }
+        return compressedBytes;
+    }
+
     public static class Compression {
 
         private final CompressionType compressionType;
diff --git a/vespajlib/src/main/java/com/yahoo/concurrent/CachedThreadPoolWithFallback.java b/vespajlib/src/main/java/com/yahoo/concurrent/CachedThreadPoolWithFallback.java
new file mode 100644
index 00000000000..42e86aad1ba
--- /dev/null
+++ b/vespajlib/src/main/java/com/yahoo/concurrent/CachedThreadPoolWithFallback.java
@@ -0,0 +1,63 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+package com.yahoo.concurrent;
+
+import java.util.concurrent.Executor;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.RejectedExecutionException;
+import java.util.concurrent.SynchronousQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * An executor that will first try a bounded cached threadpool before falling back to a unbounded
+ * single threaded threadpool that will take over dispatching to the primary pool.
+ *
+ */
+public class CachedThreadPoolWithFallback implements AutoCloseable, Executor {
+    private final ExecutorService primary;
+    private final ExecutorService secondary;
+    public CachedThreadPoolWithFallback(String baseName, int corePoolSize, int maximumPoolSize, long keepAlimeTime, TimeUnit timeUnit) {
+        primary = new ThreadPoolExecutor(corePoolSize, maximumPoolSize, keepAlimeTime, timeUnit,
+                new SynchronousQueue<>(), ThreadFactoryFactory.getDaemonThreadFactory(baseName + ".primary"));
+        secondary = Executors.newSingleThreadExecutor(ThreadFactoryFactory.getDaemonThreadFactory(baseName + ".secondary"));
+    }
+    @Override
+    public void execute(Runnable command) {
+        try {
+            primary.execute(command);
+        } catch (RejectedExecutionException e1) {
+            secondary.execute(() -> retryForever(command));
+        }
+    }
+    private void retryForever(Runnable command) {
+        while (true) {
+            try {
+                primary.execute(command);
+                return;
+            } catch (RejectedExecutionException rejected) {
+                try {
+                    Thread.sleep(1);
+                } catch (InterruptedException silenced) { }
+            }
+        }
+    }
+
+    @Override
+    public void close() {
+        secondary.shutdown();
+        join(secondary);
+        primary.shutdown();
+        join(primary);
+    }
+    private static void join(ExecutorService executor) {
+        while (true) {
+            try {
+                if (executor.awaitTermination(60, TimeUnit.SECONDS)) {
+                    return;
+                }
+            } catch (InterruptedException e) {}
+        }
+    }
+}
diff --git a/vespajlib/src/main/java/com/yahoo/slime/JsonDecoder.java b/vespajlib/src/main/java/com/yahoo/slime/JsonDecoder.java
index f199fefd185..f677ae23a45 100644
--- a/vespajlib/src/main/java/com/yahoo/slime/JsonDecoder.java
+++ b/vespajlib/src/main/java/com/yahoo/slime/JsonDecoder.java
@@ -3,10 +3,8 @@ package com.yahoo.slime;
 
 import com.yahoo.text.Text;
 import com.yahoo.text.Utf8;
-import org.w3c.dom.CharacterData;
 
 import java.io.ByteArrayOutputStream;
-import java.nio.charset.StandardCharsets;
 
 /**
  * A port of the C++ json decoder intended to be fast.
@@ -47,6 +45,17 @@ public class JsonDecoder {
         return slime;
     }
 
+    /** Decode bytes as a UTF-8 JSON into Slime, or throw {@link JsonParseException} on invalid JSON. */
+    public Slime decodeOrThrow(Slime slime, byte[] bytes) {
+        in = new BufferedInput(bytes);
+        next();
+        decodeValue(slimeInserter.adjust(slime));
+        if (in.failed()) {
+            throw new JsonParseException(in);
+        }
+        return slime;
+    }
+
     private void decodeValue(Inserter inserter) {
         skipWhiteSpace();
         switch (c) {
diff --git a/vespajlib/src/main/java/com/yahoo/slime/JsonParseException.java b/vespajlib/src/main/java/com/yahoo/slime/JsonParseException.java
new file mode 100644
index 00000000000..6c42f7d38c1
--- /dev/null
+++ b/vespajlib/src/main/java/com/yahoo/slime/JsonParseException.java
@@ -0,0 +1,23 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.slime;
+
+/**
+ * @author hakonhall
+ */
+public class JsonParseException extends RuntimeException {
+
+    private static final long serialVersionUID = 1586949558L;
+
+    private final BufferedInput input;
+
+    JsonParseException(BufferedInput input) {
+        super(input.getErrorMessage());
+        this.input = input;
+    }
+
+    public byte[] getOffendingBytes() {
+        // potentially expensive array copy
+        return input.getOffending();
+    }
+
+}
diff --git a/vespajlib/src/main/java/com/yahoo/slime/Slime.java b/vespajlib/src/main/java/com/yahoo/slime/Slime.java
index 8357e3035c0..83934e0c206 100644
--- a/vespajlib/src/main/java/com/yahoo/slime/Slime.java
+++ b/vespajlib/src/main/java/com/yahoo/slime/Slime.java
@@ -159,11 +159,10 @@ public final class Slime {
     }
 
     /**
-     * Tests whether this is equal to Inspector.
+     * Tests whether the two Inspectors are equal.
      *
-     * Since equality of two Inspectors is subtle, {@link Object#equals(Object)} is not used.
+     * <p>Since equality of two Inspectors is subtle, {@link Object#equals(Object)} is not used.</p>
      *
-     * @param that inspector.
      * @return true if they are equal.
      */
     public boolean equalTo(Slime that) {
diff --git a/config/src/main/java/com/yahoo/vespa/config/SlimeUtils.java b/vespajlib/src/main/java/com/yahoo/slime/SlimeUtils.java
index 80e93977980..2ed7331a60c 100644
--- a/config/src/main/java/com/yahoo/vespa/config/SlimeUtils.java
+++ b/vespajlib/src/main/java/com/yahoo/slime/SlimeUtils.java
@@ -1,7 +1,5 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.config;
-
-import com.yahoo.slime.*;
+package com.yahoo.slime;
 
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
@@ -9,8 +7,7 @@ import java.nio.charset.StandardCharsets;
 import java.util.Optional;
 
 /**
- * Extra utilities/operations on slime trees that we would like to have as part of slime in the future, but
- * which resides here until we have a better place to put it.
+ * Extra utilities/operations on slime trees.
  *
  * @author Ulf Lilleengen
  */
@@ -20,12 +17,7 @@ public class SlimeUtils {
         if (from.type() != Type.OBJECT) {
             throw new IllegalArgumentException("Cannot copy object: " + from);
         }
-        from.traverse(new ObjectTraverser() {
-            @Override
-            public void field(String name, Inspector inspector) {
-                setObjectEntry(inspector, name, to);
-            }
-        });
+        from.traverse((ObjectTraverser) (name, inspector) -> setObjectEntry(inspector, name, to));
 
     }
 
@@ -61,13 +53,7 @@ public class SlimeUtils {
     }
 
     private static void copyArray(Inspector from, final Cursor to) {
-        from.traverse(new ArrayTraverser() {
-            @Override
-            public void entry(int i, Inspector inspector) {
-                addValue(inspector, to);
-            }
-        });
-
+        from.traverse((ArrayTraverser) (i, inspector) -> addValue(inspector, to));
     }
 
     private static void addValue(Inspector from, Cursor to) {
@@ -118,6 +104,17 @@ public class SlimeUtils {
         return jsonToSlime(json.getBytes(StandardCharsets.UTF_8));
     }
 
+    /** Throws {@link JsonParseException} on invalid JSON. */
+    public static Slime jsonToSlimeOrThrow(String json) {
+        return jsonToSlimeOrThrow(json.getBytes(StandardCharsets.UTF_8));
+    }
+
+    public static Slime jsonToSlimeOrThrow(byte[] json) {
+        Slime slime = new Slime();
+        new JsonDecoder().decodeOrThrow(slime, json);
+        return slime;
+    }
+
     public static Optional<String> optionalString(Inspector inspector) {
         return Optional.of(inspector.asString()).filter(s -> !s.isEmpty());
     }
diff --git a/vespajlib/src/main/java/com/yahoo/system/ProcessExecuter.java b/vespajlib/src/main/java/com/yahoo/system/ProcessExecuter.java
index cceac7e84bb..c455929bf51 100644
--- a/vespajlib/src/main/java/com/yahoo/system/ProcessExecuter.java
+++ b/vespajlib/src/main/java/com/yahoo/system/ProcessExecuter.java
@@ -16,6 +16,14 @@ import com.yahoo.collections.Pair;
  */
 public class ProcessExecuter {
 
+    private final boolean override_log_control;
+    public ProcessExecuter(boolean override_log_control) {
+        this.override_log_control = override_log_control;
+    }
+    public ProcessExecuter() {
+        this(false);
+    }
+
     /**
      * Executes the given command synchronously without timeout.
      * 
@@ -39,6 +47,10 @@ public class ProcessExecuter {
         ProcessBuilder pb = new ProcessBuilder(command);        
         StringBuilder ret = new StringBuilder();
         pb.environment().remove("VESPA_LOG_TARGET");
+        if (override_log_control) {
+            pb.environment().remove("VESPA_LOG_CONTROL_FILE");
+            pb.environment().put("VESPA_SERVICE_NAME", "exec-" + command[0]);
+        }
         pb.redirectErrorStream(true);
         Process p = pb.start();
         InputStream is = p.getInputStream();
diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/Argmax.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/Argmax.java
index a365f0f4bdc..a4b68a662da 100644
--- a/vespajlib/src/main/java/com/yahoo/tensor/functions/Argmax.java
+++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/Argmax.java
@@ -1,10 +1,12 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.tensor.functions;
 
+import com.google.common.collect.ImmutableList;
 import com.yahoo.tensor.evaluation.Name;
 
 import java.util.Collections;
 import java.util.List;
+import java.util.Objects;
 
 /**
  * @author bratseth
@@ -12,11 +14,20 @@ import java.util.List;
 public class Argmax<NAMETYPE extends Name> extends CompositeTensorFunction<NAMETYPE> {
 
     private final TensorFunction<NAMETYPE> argument;
-    private final String dimension;
+    private final List<String> dimensions;
+
+    public Argmax(TensorFunction<NAMETYPE> argument) {
+        this(argument, Collections.emptyList());
+    }
 
     public Argmax(TensorFunction<NAMETYPE> argument, String dimension) {
+        this(argument, Collections.singletonList(dimension));
+    }
+
+    public Argmax(TensorFunction<NAMETYPE> argument, List<String> dimensions) {
+        Objects.requireNonNull(dimensions, "The dimensions cannot be null");
         this.argument = argument;
-        this.dimension = dimension;
+        this.dimensions = ImmutableList.copyOf(dimensions);
     }
 
     @Override
@@ -24,22 +35,21 @@ public class Argmax<NAMETYPE extends Name> extends CompositeTensorFunction<NAMET
 
     @Override
     public TensorFunction<NAMETYPE> withArguments(List<TensorFunction<NAMETYPE>> arguments) {
-        if ( arguments.size() != 1)
+        if (arguments.size() != 1)
             throw new IllegalArgumentException("Argmax must have 1 argument, got " + arguments.size());
-        return new Argmax<>(arguments.get(0), dimension);
+        return new Argmax<>(arguments.get(0), dimensions);
     }
 
     @Override
     public PrimitiveTensorFunction<NAMETYPE> toPrimitive() {
         TensorFunction<NAMETYPE> primitiveArgument = argument.toPrimitive();
-        return new Join<>(primitiveArgument,
-                          new Reduce<>(primitiveArgument, Reduce.Aggregator.max, dimension),
-                          ScalarFunctions.equal());
+        TensorFunction<NAMETYPE> reduce = new Reduce<>(primitiveArgument, Reduce.Aggregator.max, dimensions);
+        return new Join<>(primitiveArgument, reduce, ScalarFunctions.equal());
     }
 
     @Override
     public String toString(ToStringContext context) {
-        return "argmax(" + argument.toString(context) + ", " + dimension + ")";
+        return "argmax(" + argument.toString(context) + Reduce.commaSeparated(dimensions) + ")";
     }
 
 }
diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/Argmin.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/Argmin.java
index 32ccdf51336..ad14bc1f1f2 100644
--- a/vespajlib/src/main/java/com/yahoo/tensor/functions/Argmin.java
+++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/Argmin.java
@@ -1,10 +1,12 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.tensor.functions;
 
+import com.google.common.collect.ImmutableList;
 import com.yahoo.tensor.evaluation.Name;
 
 import java.util.Collections;
 import java.util.List;
+import java.util.Objects;
 
 /**
  * @author bratseth
@@ -12,11 +14,20 @@ import java.util.List;
 public class Argmin<NAMETYPE extends Name> extends CompositeTensorFunction<NAMETYPE> {
 
     private final TensorFunction<NAMETYPE> argument;
-    private final String dimension;
+    private final List<String> dimensions;
+
+    public Argmin(TensorFunction<NAMETYPE> argument) {
+        this(argument, Collections.emptyList());
+    }
 
     public Argmin(TensorFunction<NAMETYPE> argument, String dimension) {
+        this(argument, Collections.singletonList(dimension));
+    }
+
+    public Argmin(TensorFunction<NAMETYPE> argument, List<String> dimensions) {
+        Objects.requireNonNull(dimensions, "The dimensions cannot be null");
         this.argument = argument;
-        this.dimension = dimension;
+        this.dimensions = ImmutableList.copyOf(dimensions);
     }
 
     @Override
@@ -24,22 +35,21 @@ public class Argmin<NAMETYPE extends Name> extends CompositeTensorFunction<NAMET
 
     @Override
     public TensorFunction<NAMETYPE> withArguments(List<TensorFunction<NAMETYPE>> arguments) {
-        if ( arguments.size() != 1)
+        if (arguments.size() != 1)
             throw new IllegalArgumentException("Argmin must have 1 argument, got " + arguments.size());
-        return new Argmin<>(arguments.get(0), dimension);
+        return new Argmin<>(arguments.get(0), dimensions);
     }
 
     @Override
     public PrimitiveTensorFunction<NAMETYPE> toPrimitive() {
         TensorFunction<NAMETYPE> primitiveArgument = argument.toPrimitive();
-        return new Join<>(primitiveArgument,
-                          new Reduce<>(primitiveArgument, Reduce.Aggregator.min, dimension),
-                          ScalarFunctions.equal());
+        TensorFunction<NAMETYPE> reduce = new Reduce<>(primitiveArgument, Reduce.Aggregator.min, dimensions);
+        return new Join<>(primitiveArgument, reduce, ScalarFunctions.equal());
     }
 
     @Override
     public String toString(ToStringContext context) {
-        return "argmin(" + argument.toString(context) + ", " + dimension + ")";
+        return "argmin(" + argument.toString(context) + Reduce.commaSeparated(dimensions) + ")";
     }
 
 }
diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/Slice.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/Slice.java
index 4d3989b8782..bccd66acd31 100644
--- a/vespajlib/src/main/java/com/yahoo/tensor/functions/Slice.java
+++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/Slice.java
@@ -230,7 +230,7 @@ public class Slice<NAMETYPE extends Name> extends PrimitiveTensorFunction<NAMETY
 
         @Override
         public String toString() {
-            return toString(null);
+            return toString(ToStringContext.empty());
         }
 
         public String toString(ToStringContext context) {
diff --git a/vespajlib/src/main/java/com/yahoo/text/JSON.java b/vespajlib/src/main/java/com/yahoo/text/JSON.java
index cfff16c9aba..2757bd7945c 100644
--- a/vespajlib/src/main/java/com/yahoo/text/JSON.java
+++ b/vespajlib/src/main/java/com/yahoo/text/JSON.java
@@ -1,10 +1,13 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.text;
 
+import com.yahoo.slime.Slime;
+import com.yahoo.slime.SlimeUtils;
+
 import java.util.Map;
 
 /**
- * Static methods for working with the map textual format which is parsed by {@link MapParser}
+ * Static methods for working with JSON.
  *
  * @author bratseth
  */
@@ -56,4 +59,20 @@ public final class JSON {
         return b != null ? b.toString() : s;
     }
 
+    /**
+     * Test whether two JSON strings are equal, e.g. the order of fields in an object is irrelevant.
+     *
+     * <p>When comparing two numbers of the two JSON strings, the result is only guaranteed to be
+     * correct if (a) both are integers (without fraction and exponent) and each fits in a long, or
+     * (b) both are non-integers, fits in a double, and are syntactically identical. Examples
+     * of pairs that may not be equal: 1 and 1.0 (different types), 0.1 and 1e-1, 0.0 and 0.00.</p>
+     *
+     * @throws RuntimeException on invalid JSON
+     */
+    public static boolean equals(String left, String right) {
+        Slime leftSlime = SlimeUtils.jsonToSlimeOrThrow(left);
+        Slime rightSlime = SlimeUtils.jsonToSlimeOrThrow(right);
+        return leftSlime.equalTo(rightSlime);
+    }
+
 }
diff --git a/vespajlib/src/main/java/com/yahoo/text/Text.java b/vespajlib/src/main/java/com/yahoo/text/Text.java
index 706fd1583a3..85b28639d89 100644
--- a/vespajlib/src/main/java/com/yahoo/text/Text.java
+++ b/vespajlib/src/main/java/com/yahoo/text/Text.java
@@ -174,4 +174,16 @@ public final class Text {
         return stripped != null ? stripped.toString() : string;
     }
 
+    /**
+     * Returns a string which is never larger than the given number of characters.
+     * If the string is longer than the given length it will be truncated.
+     * If length is 4 or less the string will be truncated to length.
+     * If length is longer than 4, it will be truncated at length-4 with " ..." added at the end.
+     */
+    public static String truncate(String s, int length) {
+        if (s.length() <= length) return s;
+        if (length <= 4) return s.substring(0, length);
+        return s.substring(0, length - 4) + " ...";
+    }
+
 }
diff --git a/document/src/main/java/net/jpountz/lz4/package-info.java b/vespajlib/src/main/java/net/jpountz/lz4/package-info.java
index 25c41288d47..3536beff420 100644
--- a/document/src/main/java/net/jpountz/lz4/package-info.java
+++ b/vespajlib/src/main/java/net/jpountz/lz4/package-info.java
@@ -1,5 +1,5 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-@ExportPackage(version = @Version(major = 1, minor = 3, micro = 0))
+@ExportPackage(version = @Version(major = 1, minor = 7, micro = 1))
 package net.jpountz.lz4;
 import com.yahoo.osgi.annotation.ExportPackage;
 import com.yahoo.osgi.annotation.Version;
diff --git a/vespajlib/src/main/java/net/jpountz/util/package-info.java b/vespajlib/src/main/java/net/jpountz/util/package-info.java
new file mode 100644
index 00000000000..f09b2dde726
--- /dev/null
+++ b/vespajlib/src/main/java/net/jpountz/util/package-info.java
@@ -0,0 +1,5 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage(version = @Version(major = 1, minor = 7, micro = 1))
+package net.jpountz.util;
+import com.yahoo.osgi.annotation.ExportPackage;
+import com.yahoo.osgi.annotation.Version;
diff --git a/document/src/main/java/net/jpountz/xxhash/package-info.java b/vespajlib/src/main/java/net/jpountz/xxhash/package-info.java
index 7599a76f46c..e8ad7b05556 100644
--- a/document/src/main/java/net/jpountz/xxhash/package-info.java
+++ b/vespajlib/src/main/java/net/jpountz/xxhash/package-info.java
@@ -1,5 +1,5 @@
 // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-@ExportPackage(version = @Version(major = 1, minor = 3, micro = 0))
+@ExportPackage(version = @Version(major = 1, minor = 7, micro = 1))
 package net.jpountz.xxhash;
 import com.yahoo.osgi.annotation.ExportPackage;
 import com.yahoo.osgi.annotation.Version;
diff --git a/vespajlib/src/test/java/com/yahoo/collections/AbstractFilteringListTest.java b/vespajlib/src/test/java/com/yahoo/collections/AbstractFilteringListTest.java
index 9386bf7256f..3524f507701 100644
--- a/vespajlib/src/test/java/com/yahoo/collections/AbstractFilteringListTest.java
+++ b/vespajlib/src/test/java/com/yahoo/collections/AbstractFilteringListTest.java
@@ -48,8 +48,8 @@ public class AbstractFilteringListTest {
         assertEquals(List.of("abc", "cba", "bbb"),
                      list.not().in(MyList.of("ABC", "CBA")).asList());
 
-        assertEquals(List.of("ABC", "abc", "cba", "bbb", "ABC", "aaa"),
-                     list.and(MyList.of("aaa")).asList());
+        assertEquals(List.of("ABC", "abc", "cba", "bbb", "ABC", "aaa", "ABC"),
+                     list.concat(MyList.of("aaa", "ABC")).asList());
     }
 
     private static class MyList extends AbstractFilteringList<String, MyList> {
diff --git a/vespajlib/src/test/java/com/yahoo/concurrent/CachedThreadPoolWithFallbackTest.java b/vespajlib/src/test/java/com/yahoo/concurrent/CachedThreadPoolWithFallbackTest.java
new file mode 100644
index 00000000000..52e17631a34
--- /dev/null
+++ b/vespajlib/src/test/java/com/yahoo/concurrent/CachedThreadPoolWithFallbackTest.java
@@ -0,0 +1,43 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+package com.yahoo.concurrent;
+
+import org.junit.Test;
+
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+
+import static org.junit.Assert.assertEquals;
+
+public class CachedThreadPoolWithFallbackTest {
+    private static void countAndBlock(AtomicLong counter, long waitLimit) {
+        counter.incrementAndGet();
+        try {
+            synchronized (counter) {
+                while (counter.get() < waitLimit) {
+                    counter.wait();
+                }
+            }
+        } catch (InterruptedException e) {}
+    }
+
+    @Test
+    public void testThatTaskAreQueued() throws InterruptedException {
+        CachedThreadPoolWithFallback executor = new CachedThreadPoolWithFallback("test", 1, 30, 1, TimeUnit.SECONDS);
+        AtomicLong counter = new AtomicLong(0);
+        for (int i = 0; i < 1000; i++) {
+            executor.execute(() -> countAndBlock(counter, 100));
+        }
+        while (counter.get() < 30) {
+            Thread.sleep(1);
+        }
+        Thread.sleep(1);
+        assertEquals(30L, counter.get());
+        counter.set(100);
+        synchronized (counter) {
+            counter.notifyAll();
+        }
+        executor.close();
+        assertEquals(1070L, counter.get());
+    }
+}
diff --git a/config/src/test/java/com/yahoo/vespa/config/SlimeUtilsTest.java b/vespajlib/src/test/java/com/yahoo/slime/SlimeUtilsTest.java
index ff7b640630e..237b1575bfb 100644
--- a/config/src/test/java/com/yahoo/vespa/config/SlimeUtilsTest.java
+++ b/vespajlib/src/test/java/com/yahoo/slime/SlimeUtilsTest.java
@@ -1,22 +1,22 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.config;
+package com.yahoo.slime;
 
-import com.yahoo.slime.Cursor;
-import com.yahoo.slime.Slime;
 import com.yahoo.text.Utf8;
 import org.junit.Test;
 
 import java.io.IOException;
 
 import static org.hamcrest.core.Is.is;
+import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertThat;
 import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
 
 /**
  * @author Ulf Lilleengen
- * @since 5.8
  */
 public class SlimeUtilsTest {
+
     @Test
     public void test_copying_slime_types_into_cursor() {
         Slime slime = new Slime();
@@ -79,4 +79,22 @@ public class SlimeUtilsTest {
         assertThat(slime.get().field("foo").asString(), is("foobie"));
         assertTrue(slime.get().field("bar").valid());
     }
+
+    @Test
+    public void test_json_to_slime_or_throw() {
+        Slime slime = SlimeUtils.jsonToSlimeOrThrow("{\"foo\":\"foobie\",\"bar\":{}}");
+        assertThat(slime.get().field("foo").asString(), is("foobie"));
+        assertTrue(slime.get().field("bar").valid());
+    }
+
+    @Test
+    public void test_invalid_json() {
+        try {
+            SlimeUtils.jsonToSlimeOrThrow("foo");
+            fail();
+        } catch (RuntimeException e) {
+            assertEquals("Unexpected character 'o'", e.getMessage());
+        }
+    }
+
 }
diff --git a/vespajlib/src/test/java/com/yahoo/tensor/functions/TensorFunctionTestCase.java b/vespajlib/src/test/java/com/yahoo/tensor/functions/TensorFunctionTestCase.java
index 625d5d44b19..05f7d27907c 100644
--- a/vespajlib/src/test/java/com/yahoo/tensor/functions/TensorFunctionTestCase.java
+++ b/vespajlib/src/test/java/com/yahoo/tensor/functions/TensorFunctionTestCase.java
@@ -21,6 +21,8 @@ public class TensorFunctionTestCase {
                          new Diag<>(new TensorType.Builder().indexed("y",3).indexed("x",2).indexed("z",4).build()));
         assertTranslated("join(tensor(x{}):{1:1.0,3:5.0,9:3.0}, reduce(tensor(x{}):{1:1.0,3:5.0,9:3.0}, max, x), f(a,b)(a==b))",
                          new Argmax<>(new ConstantTensor<>("{ {x:1}:1, {x:3}:5, {x:9}:3 }"), "x"));
+        assertTranslated("join(tensor(x{}):{1:1.0,3:5.0,9:3.0}, reduce(tensor(x{}):{1:1.0,3:5.0,9:3.0}, max), f(a,b)(a==b))",
+                        new Argmax<>(new ConstantTensor<>("{ {x:1}:1, {x:3}:5, {x:9}:3 }")));
     }
 
     private void assertTranslated(String expectedTranslation, TensorFunction<Name> inputFunction) {
diff --git a/vespajlib/src/test/java/com/yahoo/text/JSONTest.java b/vespajlib/src/test/java/com/yahoo/text/JSONTest.java
index 22174761571..fbd0f9d0403 100644
--- a/vespajlib/src/test/java/com/yahoo/text/JSONTest.java
+++ b/vespajlib/src/test/java/com/yahoo/text/JSONTest.java
@@ -2,11 +2,16 @@
 package com.yahoo.text;
 
 import org.junit.Test;
-import static org.junit.Assert.assertEquals;
 
 import java.util.LinkedHashMap;
 import java.util.Map;
 
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
 /**
  * @author bratseth
  */
@@ -22,4 +27,88 @@ public class JSONTest {
         assertEquals("{\"a \\\"key\\\"\":3,\"key2\":\"value\",\"key3\":3.3}", JSON.encode(map));
     }
 
+    @Test
+    public void testEquals() {
+        assertTrue(JSON.equals("{}", "{}"));
+
+        // Whitespace is irrelevant
+        assertTrue(JSON.equals("{}", "\n{ }"));
+
+        // Order of fields in object is irrelevant
+        assertTrue(JSON.equals("{\"a\":0, \"c\":1}", "{\"c\":1, \"a\":0}"));
+
+        // Object equality is not using subset
+        assertFalse(JSON.equals("{\"a\":0}", "{\"a\":0, \"b\":0}"));
+        assertFalse(JSON.equals("{\"a\":0, \"b\":0}", "{\"a\":0}"));
+
+        // Order of elements of array is significant
+        assertFalse(JSON.equals("[\"a\",\"b\"]", "[\"b\",\"a\"]"));
+
+        // Verify null-valued fields are not ignored
+        assertFalse(JSON.equals("{\"a\":null}", "{}"));
+
+        // Current impl uses BigInteger if integer doesn't fit in a long.
+        assertEquals(9223372036854775807L, Long.MAX_VALUE);
+        assertTrue(JSON.equals("{\"a\": 9223372036854775807}", "{\"a\": 9223372036854775807}"));
+
+        // double 1.0 and int 1 are different
+        assertTrue(JSON.equals("{\"a\": 1}", "{\"a\": 1}"));
+        assertTrue(JSON.equals("{\"a\": 1.0}", "{\"a\": 1.0}"));
+        assertFalse(JSON.equals("{\"a\": 1.0}", "{\"a\": 1}"));
+
+        // Double-precision on numbers. Constant from Math.E.
+        assertTrue(JSON.equals("{\"e\": 2.71828182845904}", "{\"e\": 2.71828182845904}"));
+
+        // Double.MAX_VALUE is 1.7976931348623157e+308
+        assertTrue(JSON.equals("{\"e\": 1.7976931348623156e+308}", "{\"e\": 1.7976931348623156e+308}"));
+
+        // Justification of above float values
+        double e1 = 2.7182818284590452354;
+        double e2 = 2.718281828459045;
+        double e3 = 2.71828182845904;
+        assertEquals(e1, Math.E, -1);
+        assertEquals(e1, e2, -1);
+        assertNotEquals(e1, e3, -1);
+
+        // Invalid JSON throws RuntimeException
+        assertRuntimeException(() -> JSON.equals("", "{}"));
+        assertRuntimeException(() -> JSON.equals("{}", ""));
+        assertRuntimeException(() -> JSON.equals("{", "{}"));
+        assertRuntimeException(() -> JSON.equals("{}", "{"));
+    }
+
+    @Test
+    public void implementationSpecificEqualsBehavior() {
+        // Exception thrown if outside a long
+        assertTrue(                  JSON.equals("{\"a\": 9223372036854775807}", "{\"a\": 9223372036854775807}"));
+        assertRuntimeException(() -> JSON.equals("{\"a\": 9223372036854775808}", "{\"a\": 9223372036854775808}"));
+
+        // Infinity if floating point number outside of double, and hence equal
+        assertTrue(JSON.equals("{\"a\": 2.7976931348623158e+308}", "{\"a\": 2.7976931348623158e+308}"));
+
+        // Ignores extraneous precision
+        assertTrue(JSON.equals( "{\"e\": 2.7182818284590452354}",
+                                 "{\"e\": 2.7182818284590452354}"));
+        assertTrue(JSON.equals( "{\"e\": 2.7182818284590452354}",
+                                 "{\"e\": 2.7182818284590452355}"));
+        assertFalse(JSON.equals("{\"e\": 2.7182818284590452354}",
+                                 "{\"e\": 2.71828182845904}"));
+
+        // Comparing equal but syntactically different numbers
+        assertFalse(JSON.equals("{\"a\": 1.0}", "{\"a\":1}"));
+        assertTrue(JSON.equals("{\"a\": 1.0}", "{\"a\":1.00}"));
+        assertTrue(JSON.equals("{\"a\": 1.0}", "{\"a\":1.0000000000000000000000000000}"));
+        assertTrue(JSON.equals("{\"a\": 10.0}", "{\"a\":1e1}"));
+        assertTrue(JSON.equals("{\"a\": 1.2}", "{\"a\":12e-1}"));
+    }
+
+    private static void assertRuntimeException(Runnable runnable) {
+        try {
+            runnable.run();
+            fail("Expected RuntimeException to be thrown, but no exception was thrown");
+        } catch (RuntimeException e) {
+            // OK
+        }
+    }
+
 }
diff --git a/vespajlib/src/test/java/com/yahoo/text/TextTestCase.java b/vespajlib/src/test/java/com/yahoo/text/TextTestCase.java
index e733b838c39..8bb8b2aaad5 100644
--- a/vespajlib/src/test/java/com/yahoo/text/TextTestCase.java
+++ b/vespajlib/src/test/java/com/yahoo/text/TextTestCase.java
@@ -61,4 +61,15 @@ public class TextTestCase {
         assertFalse(Text.isDisplayable(0));
     }
 
+    @Test
+    public void testTruncate() {
+        assertEquals("ab", Text.truncate("ab", 5));
+        assertEquals("ab", Text.truncate("ab", 6));
+        assertEquals("ab", Text.truncate("ab", 2));
+        assertEquals("a",  Text.truncate("ab", 1));
+        assertEquals("",   Text.truncate("ab", 0));
+        assertEquals("ab c",  Text.truncate("ab cde", 4));
+        assertEquals("a ...", Text.truncate("ab cde", 5));
+    }
+
 }
diff --git a/vespalib/CMakeLists.txt b/vespalib/CMakeLists.txt
index 9339cdacea0..2675bc16bf2 100644
--- a/vespalib/CMakeLists.txt
+++ b/vespalib/CMakeLists.txt
@@ -30,6 +30,7 @@ vespa_define_module(
     src/tests/component
     src/tests/compress
     src/tests/compression
+    src/tests/crypto
     src/tests/data/databuffer
     src/tests/data/input_reader
     src/tests/data/lz4_encode_decode
@@ -57,6 +58,7 @@ vespa_define_module(
     src/tests/gencnt
     src/tests/guard
     src/tests/host_name
+    src/tests/hwaccelrated
     src/tests/io/fileutil
     src/tests/io/mapped_file_input
     src/tests/latch
@@ -130,6 +132,7 @@ vespa_define_module(
     src/tests/util/generationhandler_stress
     src/tests/util/md5
     src/tests/util/rcuvector
+    src/tests/util/reusable_set
     src/tests/valgrind
     src/tests/visit_ranges
     src/tests/websocket
@@ -139,6 +142,7 @@ vespa_define_module(
     src/vespa/vespalib
     src/vespa/vespalib/btree
     src/vespa/vespalib/component
+    src/vespa/vespalib/crypto
     src/vespa/vespalib/data
     src/vespa/vespalib/data/slime
     src/vespa/vespalib/datastore
@@ -151,6 +155,7 @@ vespa_define_module(
     src/vespa/vespalib/net/tls/impl
     src/vespa/vespalib/objects
     src/vespa/vespalib/portal
+    src/vespa/vespalib/regex
     src/vespa/vespalib/stllike
     src/vespa/vespalib/test
     src/vespa/vespalib/testkit
diff --git a/vespalib/src/tests/alloc/alloc_test.cpp b/vespalib/src/tests/alloc/alloc_test.cpp
index 4a569050f45..d46d2374dfc 100644
--- a/vespalib/src/tests/alloc/alloc_test.cpp
+++ b/vespalib/src/tests/alloc/alloc_test.cpp
@@ -155,7 +155,7 @@ void ensureRoomForExtension(const Alloc & buf, Alloc & reserved) {
     // So in order to verify this we first mmap a reserved area that we unmap
     // before we test extension.
     if (reserved.get() > buf.get()) {
-        EXPECT_EQUAL(reserved.get(), static_cast<const char *>(buf.get()) + buf.size());
+        EXPECT_EQUAL(reserved.get(), static_cast<const void *>(static_cast<const char *>(buf.get()) + buf.size()));
         {
             Alloc().swap(reserved);
         }
@@ -166,14 +166,15 @@ void verifyNoExtensionWhenNoRoom(Alloc & buf, Alloc & reserved, size_t sz) {
     if (reserved.get() > buf.get()) {
         // Normally mmapping starts at the top and grows down in address space.
         // Then there is no room to extend the last mapping.
-        EXPECT_EQUAL(reserved.get(), static_cast<const char *>(buf.get()) + buf.size());
+        EXPECT_EQUAL(reserved.get(), static_cast<const void *>(static_cast<const char *>(buf.get()) + buf.size()));
         TEST_DO(verifyExtension(buf, sz, sz));
     } else {
-        EXPECT_EQUAL(buf.get(), static_cast<const char *>(reserved.get()) + reserved.size());
+        EXPECT_EQUAL(buf.get(), static_cast<const void *>(static_cast<const char *>(reserved.get()) + reserved.size()));
         TEST_DO(verifyExtension(reserved, sz, sz));
     }
 }
 
+#ifdef __linux__
 TEST("auto alloced mmap alloc can be extended if room") {
     static constexpr size_t SZ = MemoryAllocator::HUGEPAGE_SIZE*2;
     Alloc reserved = Alloc::alloc(SZ);
@@ -207,6 +208,7 @@ TEST("mmap alloc can not be extended if no room") {
 
     TEST_DO(verifyNoExtensionWhenNoRoom(buf, reserved, 4096));
 }
+#endif
 
 TEST("heap alloc can not be shrinked") {
     Alloc buf = Alloc::allocHeap(101);
diff --git a/vespalib/src/tests/crypto/CMakeLists.txt b/vespalib/src/tests/crypto/CMakeLists.txt
new file mode 100644
index 00000000000..b930b5715b5
--- /dev/null
+++ b/vespalib/src/tests/crypto/CMakeLists.txt
@@ -0,0 +1,10 @@
+# Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(vespalib_crypto_crypto_test_app TEST
+    SOURCES
+    crypto_test.cpp
+    DEPENDS
+    vespalib
+    gtest
+)
+vespa_add_test(NAME vespalib_crypto_crypto_test_app COMMAND vespalib_crypto_crypto_test_app)
+
diff --git a/vespalib/src/tests/crypto/crypto_test.cpp b/vespalib/src/tests/crypto/crypto_test.cpp
new file mode 100644
index 00000000000..8daba954793
--- /dev/null
+++ b/vespalib/src/tests/crypto/crypto_test.cpp
@@ -0,0 +1,37 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/vespalib/crypto/private_key.h>
+#include <vespa/vespalib/crypto/x509_certificate.h>
+#include <vespa/vespalib/gtest/gtest.h>
+#include <gmock/gmock.h>
+
+using namespace ::testing;
+
+namespace vespalib::crypto {
+
+// FIXME these tests are very high level and simple since the current crypto utility API we provide
+// is extremely simple and does not support loading PEMs, signing or verifying.
+
+TEST(CryptoTest, generated_p256_ec_private_key_can_be_exported_to_pem_format) {
+    auto key = PrivateKey::generate_p256_ec_key();
+    auto pem = key->private_to_pem();
+    EXPECT_THAT(pem, StartsWith("-----BEGIN PRIVATE KEY-----"));
+}
+
+TEST(CryptoTest, generated_x509_certificate_can_be_exported_to_pem_format) {
+    auto dn = X509Certificate::DistinguishedName()
+            .country("NO").locality("Trondheim")
+            .organization("Cool Unit Test Writers")
+            .organizational_unit("Only the finest tests, yes")
+            .add_common_name("cooltests.example.com");
+    auto subject = X509Certificate::SubjectInfo(std::move(dn));
+    auto key = PrivateKey::generate_p256_ec_key();
+    auto params = X509Certificate::Params::self_signed(std::move(subject), key);
+    auto cert = X509Certificate::generate_from(std::move(params));
+    auto pem = cert->to_pem();
+    EXPECT_THAT(pem, StartsWith("-----BEGIN CERTIFICATE-----"));
+}
+
+}
+
+GTEST_MAIN_RUN_ALL_TESTS()
diff --git a/vespalib/src/tests/datastore/unique_store/unique_store_test.cpp b/vespalib/src/tests/datastore/unique_store/unique_store_test.cpp
index 88a5a05738b..e4631e28625 100644
--- a/vespalib/src/tests/datastore/unique_store/unique_store_test.cpp
+++ b/vespalib/src/tests/datastore/unique_store/unique_store_test.cpp
@@ -159,7 +159,7 @@ template <>
 std::vector<double> TestBase<DoubleUniqueStore>::values{ 10.0, 20.0, 30.0, 10.0 };
 
 using UniqueStoreTestTypes = ::testing::Types<NumberUniqueStore, StringUniqueStore, CStringUniqueStore, DoubleUniqueStore>;
-TYPED_TEST_CASE(TestBase, UniqueStoreTestTypes);
+VESPA_GTEST_TYPED_TEST_SUITE(TestBase, UniqueStoreTestTypes);
 
 // Disable warnings emitted by gtest generated files when using typed tests
 #pragma GCC diagnostic push
diff --git a/vespalib/src/tests/dotproduct/dotproductbenchmark.cpp b/vespalib/src/tests/dotproduct/dotproductbenchmark.cpp
index 3588e0ce239..d6e1aef9394 100644
--- a/vespalib/src/tests/dotproduct/dotproductbenchmark.cpp
+++ b/vespalib/src/tests/dotproduct/dotproductbenchmark.cpp
@@ -48,12 +48,12 @@ public:
     FullBenchmark(size_t numDocs, size_t numValue);
     ~FullBenchmark();
     void compute(size_t docId) const override {
-        _dp->dotProduct(&_query[0], &_values[docId * _query.size()], _query.size());
+        _dp.dotProduct(&_query[0], &_values[docId * _query.size()], _query.size());
     }
 private:
     std::vector<T> _values;
     std::vector<T> _query;
-    IAccelrated::UP _dp;
+    const IAccelrated & _dp;
 };
 
 template <typename T>
diff --git a/vespalib/src/tests/exception_classes/mmap.cpp b/vespalib/src/tests/exception_classes/mmap.cpp
index 2d4c4796473..2a6896bb0e0 100644
--- a/vespalib/src/tests/exception_classes/mmap.cpp
+++ b/vespalib/src/tests/exception_classes/mmap.cpp
@@ -2,7 +2,8 @@
 #include <vespa/vespalib/util/alloc.h>
 #include <vector>
 #include <cassert>
-#include <string.h>
+#include <cstring>
+#include <cstdlib>
 #include <sys/resource.h>
 
 using namespace vespalib::alloc;
diff --git a/vespalib/src/tests/exception_classes/silenceuncaught_test.cpp b/vespalib/src/tests/exception_classes/silenceuncaught_test.cpp
index 34ea78db2fc..f94a298d19f 100644
--- a/vespalib/src/tests/exception_classes/silenceuncaught_test.cpp
+++ b/vespalib/src/tests/exception_classes/silenceuncaught_test.cpp
@@ -30,11 +30,14 @@ TEST("that caught silenced exception causes exitcode 0") {
 }
 
 TEST("that mmap within limits are fine cause exitcode 0") {
-    SlaveProc proc("exec ./vespalib_mmap_app 100000000 10485760 1");
+    SlaveProc proc("exec ./vespalib_mmap_app 150000000 10485760 1");
     proc.wait();
     EXPECT_EQUAL(proc.getExitCode(), 0);
 }
 
+#ifdef __APPLE__
+// setrlimit with RLIMIT_AS is broken on Darwin
+#else
 TEST("that mmap beyond limits cause negative exitcode.") {
     SlaveProc proc("ulimit -c 0 && exec ./vespalib_mmap_app 100000000 10485760 10");
     proc.wait();
@@ -46,5 +49,6 @@ TEST("that mmap beyond limits with set VESPA_SILENCE_CORE_ON_OOM cause exitcode
     proc.wait();
     EXPECT_EQUAL(proc.getExitCode(), 66);
 }
+#endif
 
 TEST_MAIN_WITH_PROCESS_PROXY() { TEST_RUN_ALL(); }
diff --git a/vespalib/src/tests/executor/executor_test.cpp b/vespalib/src/tests/executor/executor_test.cpp
index 9015391beaa..942b425be72 100644
--- a/vespalib/src/tests/executor/executor_test.cpp
+++ b/vespalib/src/tests/executor/executor_test.cpp
@@ -3,6 +3,7 @@
 #include <vespa/vespalib/testkit/testapp.h>
 #include <vespa/vespalib/util/closuretask.h>
 #include <vespa/vespalib/util/lambdatask.h>
+#include <vespa/vespalib/util/executor_stats.h>
 
 using namespace vespalib;
 
@@ -24,4 +25,29 @@ TEST("require that lambdas can be wrapped as tasks") {
     EXPECT_TRUE(called);
 }
 
+template<typename T>
+void verify(const AggregatedAverage<T> & avg, size_t expCount, T expTotal, T expMin, T expMax, double expAvg) {
+    EXPECT_EQUAL(expCount, avg.count());
+    EXPECT_EQUAL(expTotal, avg.total());
+    EXPECT_EQUAL(expMin, avg.min());
+    EXPECT_EQUAL(expMax, avg.max());
+    EXPECT_EQUAL(expAvg, avg.average());
+}
+
+TEST("test that aggregated averages") {
+    TEST_DO(verify(AggregatedAverage<size_t>(), 0ul, 0ul, std::numeric_limits<size_t>::max(), std::numeric_limits<size_t>::min(), 0.0));
+    AggregatedAverage<size_t> avg;
+    avg.add(9);
+    TEST_DO(verify(avg, 1ul, 9ul, 9ul, 9ul, 9.0));
+    avg.add(8);
+    TEST_DO(verify(avg, 2ul, 17ul, 8ul, 9ul, 8.5));
+    avg.add(3, 17, 4,17);
+    TEST_DO(verify(avg, 5ul, 34ul, 4ul, 17ul, 6.8));
+    AggregatedAverage<size_t> avg2;
+    avg2.add(avg);
+    TEST_DO(verify(avg2, 5ul, 34ul, 4ul, 17ul, 6.8));
+    avg2 += avg;
+    TEST_DO(verify(avg2, 10ul, 68ul, 4ul, 17ul, 6.8));
+}
+
 TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/vespalib/src/tests/executor/threadstackexecutor_test.cpp b/vespalib/src/tests/executor/threadstackexecutor_test.cpp
index db3c4628d2f..9d69adcd96a 100644
--- a/vespalib/src/tests/executor/threadstackexecutor_test.cpp
+++ b/vespalib/src/tests/executor/threadstackexecutor_test.cpp
@@ -78,10 +78,10 @@ struct MyState {
         EXPECT_EQUAL(expect_rejected, stats.rejectedTasks);
         EXPECT_TRUE(!(gate.getCount() == 1) || (expect_deleted == 0));
         if (expect_deleted == 0) {
-            EXPECT_EQUAL(expect_queue + expect_running, stats.maxPendingTasks);
+            EXPECT_EQUAL(expect_queue + expect_running, stats.queueSize.max());
         }
         stats = executor.getStats();
-        EXPECT_EQUAL(expect_queue + expect_running, stats.maxPendingTasks);
+        EXPECT_EQUAL(expect_queue + expect_running, stats.queueSize.max());
         EXPECT_EQUAL(0u, stats.acceptedTasks);
         EXPECT_EQUAL(0u, stats.rejectedTasks);
         return *this;
@@ -186,4 +186,22 @@ TEST_F("require that executor thread stack tag can be set", ThreadStackExecutor(
     }
 }
 
+TEST("require that stats can be accumulated") {
+    ThreadStackExecutor::Stats stats(ThreadExecutor::Stats::QueueSizeT(1) ,2,3);
+    EXPECT_EQUAL(1u, stats.queueSize.max());
+    EXPECT_EQUAL(2u, stats.acceptedTasks);
+    EXPECT_EQUAL(3u, stats.rejectedTasks);
+    stats += ThreadStackExecutor::Stats(ThreadExecutor::Stats::QueueSizeT(7),8,9);
+    EXPECT_EQUAL(2u, stats.queueSize.count());
+    EXPECT_EQUAL(8u, stats.queueSize.total());
+    EXPECT_EQUAL(8u, stats.queueSize.max());
+    EXPECT_EQUAL(8u, stats.queueSize.min());
+    EXPECT_EQUAL(8u, stats.queueSize.max());
+    EXPECT_EQUAL(4.0, stats.queueSize.average());
+
+    EXPECT_EQUAL(10u, stats.acceptedTasks);
+    EXPECT_EQUAL(12u, stats.rejectedTasks);
+
+}
+
 TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/vespalib/src/tests/hwaccelrated/CMakeLists.txt b/vespalib/src/tests/hwaccelrated/CMakeLists.txt
new file mode 100644
index 00000000000..49501341098
--- /dev/null
+++ b/vespalib/src/tests/hwaccelrated/CMakeLists.txt
@@ -0,0 +1,8 @@
+# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(vespalib_hwaccelrated_test_app TEST
+    SOURCES
+    hwaccelrated_test.cpp
+    DEPENDS
+    vespalib
+)
+vespa_add_test(NAME vespalib_hwaccelrated_test_app COMMAND vespalib_hwaccelrated_test_app)
diff --git a/vespalib/src/tests/hwaccelrated/hwaccelrated_test.cpp b/vespalib/src/tests/hwaccelrated/hwaccelrated_test.cpp
new file mode 100644
index 00000000000..421f3f6f520
--- /dev/null
+++ b/vespalib/src/tests/hwaccelrated/hwaccelrated_test.cpp
@@ -0,0 +1,40 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/vespalib/testkit/test_kit.h>
+#include <vespa/vespalib/hwaccelrated/iaccelrated.h>
+#include <vespa/vespalib/hwaccelrated/generic.h>
+
+using namespace vespalib;
+
+template<typename T>
+std::vector<T> createAndFill(size_t sz) {
+    std::vector<T> v(sz);
+    for (size_t i(0); i < sz; i++) {
+        v[i] = rand()%500;
+    }
+    return v;
+}
+
+template<typename T>
+void verifyEuclideanDistance(const hwaccelrated::IAccelrated & accel) {
+    const size_t testLength(255);
+    srand(1);
+    std::vector<T> a = createAndFill<T>(testLength);
+    std::vector<T> b = createAndFill<T>(testLength);
+    for (size_t j(0); j < 0x20; j++) {
+        T sum(0);
+        for (size_t i(j); i < testLength; i++) {
+            sum += (a[i] - b[i]) * (a[i] - b[i]);
+        }
+        T hwComputedSum(accel.squaredEuclideanDistance(&a[j], &b[j], testLength - j));
+        EXPECT_EQUAL(sum, hwComputedSum);
+    }
+}
+
+TEST("test euclidean distance") {
+    hwaccelrated::GenericAccelrator genericAccelrator;
+    verifyEuclideanDistance<float>(genericAccelrator);
+    verifyEuclideanDistance<double >(genericAccelrator);
+}
+
+TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/vespalib/src/tests/net/crypto_socket/crypto_socket_test.cpp b/vespalib/src/tests/net/crypto_socket/crypto_socket_test.cpp
index 62bad716597..e938e15f4e6 100644
--- a/vespalib/src/tests/net/crypto_socket/crypto_socket_test.cpp
+++ b/vespalib/src/tests/net/crypto_socket/crypto_socket_test.cpp
@@ -17,6 +17,7 @@
 #include <fcntl.h>
 
 using namespace vespalib;
+using namespace vespalib::test;
 
 struct SocketPair {
     SocketHandle client;
@@ -204,7 +205,9 @@ void verify_crypto_socket(SocketPair &sockets, CryptoEngine &engine, bool is_ser
     SocketHandle &my_handle = is_server ? sockets.server : sockets.client;
     my_handle.set_blocking(false);
     SmartBuffer read_buffer(4096);
-    CryptoSocket::UP my_socket = engine.create_crypto_socket(std::move(my_handle), is_server);
+    CryptoSocket::UP my_socket = is_server
+                                 ? engine.create_server_crypto_socket(std::move(my_handle))
+                                 : engine.create_client_crypto_socket(std::move(my_handle), local_spec);
     TEST_DO(verify_handshake(*my_socket));
     drain(*my_socket, read_buffer);
     TEST_DO(verify_socket_io(*my_socket, read_buffer, is_server));
@@ -226,19 +229,19 @@ TEST_MT_FFF("require that encrypted async socket io works with XorCryptoEngine",
 }
 
 TEST_MT_FFF("require that encrypted async socket io works with TlsCryptoEngine",
-            2, SocketPair(), TlsCryptoEngine(vespalib::test::make_tls_options_for_testing()), TimeBomb(60))
+            2, SocketPair(), TlsCryptoEngine(make_tls_options_for_testing()), TimeBomb(60))
 {
     TEST_DO(verify_crypto_socket(f1, f2, (thread_id == 0)));
 }
 
 TEST_MT_FFF("require that encrypted async socket io works with MaybeTlsCryptoEngine(true)",
-            2, SocketPair(), MaybeTlsCryptoEngine(std::make_shared<TlsCryptoEngine>(vespalib::test::make_tls_options_for_testing()), true), TimeBomb(60))
+            2, SocketPair(), MaybeTlsCryptoEngine(std::make_shared<TlsCryptoEngine>(make_tls_options_for_testing()), true), TimeBomb(60))
 {
     TEST_DO(verify_crypto_socket(f1, f2, (thread_id == 0)));
 }
 
 TEST_MT_FFF("require that encrypted async socket io works with MaybeTlsCryptoEngine(false)",
-            2, SocketPair(), MaybeTlsCryptoEngine(std::make_shared<TlsCryptoEngine>(vespalib::test::make_tls_options_for_testing()), false), TimeBomb(60))
+            2, SocketPair(), MaybeTlsCryptoEngine(std::make_shared<TlsCryptoEngine>(make_tls_options_for_testing()), false), TimeBomb(60))
 {
     TEST_DO(verify_crypto_socket(f1, f2, (thread_id == 0)));
 }
diff --git a/vespalib/src/tests/net/socket/socket_test.cpp b/vespalib/src/tests/net/socket/socket_test.cpp
index 88146cd4fb2..08893c9273b 100644
--- a/vespalib/src/tests/net/socket/socket_test.cpp
+++ b/vespalib/src/tests/net/socket/socket_test.cpp
@@ -204,14 +204,6 @@ TEST_MT_FF("require that basic unix domain socket io works (path)", 2,
     TEST_DO(verify_socket_io(is_server, socket));
 }
 
-TEST_MT_FF("require that basic unix domain socket io works (name)", 2,
-           ServerSocket(make_string("ipc/name:my_socket-%d", int(getpid()))), TimeBomb(60))
-{
-    bool is_server = (thread_id == 0);
-    SocketHandle socket = connect_sockets(is_server, f1);
-    TEST_DO(verify_socket_io(is_server, socket));
-}
-
 TEST_MT_FF("require that server accept can be interrupted", 2, ServerSocket("tcp/0"), TimeBomb(60)) {
     bool is_server = (thread_id == 0);
     if (is_server) {
@@ -279,6 +271,15 @@ TEST("require that a server socket will remove an old socket file if it cannot b
     EXPECT_TRUE(!is_socket("my_socket"));
 }
 
+#ifdef __linux__
+TEST_MT_FF("require that basic unix domain socket io works (name)", 2,
+           ServerSocket(make_string("ipc/name:my_socket-%d", int(getpid()))), TimeBomb(60))
+{
+    bool is_server = (thread_id == 0);
+    SocketHandle socket = connect_sockets(is_server, f1);
+    TEST_DO(verify_socket_io(is_server, socket));
+}
+
 TEST("require that two server sockets cannot have the same abstract unix domain socket name") {
     vespalib::string spec = make_string("ipc/name:my_socket-%d", int(getpid()));
     ServerSocket server1(spec);
@@ -313,6 +314,7 @@ TEST_MT_FFF("require that abstract and file-based unix domain sockets are not in
     SocketHandle socket = connect_sockets(is_server, server_socket);
     TEST_DO(verify_socket_io(is_server, socket));
 }
+#endif
 
 TEST("require that sockets can be set blocking and non-blocking") {
     SocketHandle handle(socket(my_inet(), SOCK_STREAM, 0));
diff --git a/vespalib/src/tests/net/socket_spec/socket_spec_test.cpp b/vespalib/src/tests/net/socket_spec/socket_spec_test.cpp
index 0c2b92bbb53..f2da6b70bf3 100644
--- a/vespalib/src/tests/net/socket_spec/socket_spec_test.cpp
+++ b/vespalib/src/tests/net/socket_spec/socket_spec_test.cpp
@@ -123,4 +123,8 @@ TEST("require that replace_host gives invalid spec when used with less than 2 ho
     TEST_DO(verify_invalid(SocketSpec("ipc/name:my_socket").replace_host("foo")));
 }
 
+TEST("require that invalid socket spec is not valid") {
+    EXPECT_FALSE(SocketSpec::invalid.valid());
+}
+
 TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/vespalib/src/tests/net/sync_crypto_socket/sync_crypto_socket_test.cpp b/vespalib/src/tests/net/sync_crypto_socket/sync_crypto_socket_test.cpp
index 76ecd9453b6..56767051dad 100644
--- a/vespalib/src/tests/net/sync_crypto_socket/sync_crypto_socket_test.cpp
+++ b/vespalib/src/tests/net/sync_crypto_socket/sync_crypto_socket_test.cpp
@@ -17,6 +17,7 @@
 #include <fcntl.h>
 
 using namespace vespalib;
+using namespace vespalib::test;
 
 struct SocketPair {
     SocketHandle client;
@@ -97,7 +98,9 @@ void verify_socket_io(SyncCryptoSocket &socket, bool is_server) {
 void verify_crypto_socket(SocketPair &sockets, CryptoEngine &engine, bool is_server) {
     SocketHandle &my_handle = is_server ? sockets.server : sockets.client;
     my_handle.set_blocking(false);
-    SyncCryptoSocket::UP my_socket = SyncCryptoSocket::create(engine, std::move(my_handle), is_server);
+    SyncCryptoSocket::UP my_socket = is_server
+                                     ? SyncCryptoSocket::create_server(engine, std::move(my_handle))
+                                     : SyncCryptoSocket::create_client(engine, std::move(my_handle), local_spec);
     ASSERT_TRUE(my_socket);
     TEST_DO(verify_socket_io(*my_socket, is_server));
     TEST_DO(verify_graceful_shutdown(*my_socket, is_server));
@@ -118,19 +121,19 @@ TEST_MT_FFF("require that encrypted sync socket io works with XorCryptoEngine",
 }
 
 TEST_MT_FFF("require that encrypted sync socket io works with TlsCryptoEngine",
-            2, SocketPair(), TlsCryptoEngine(vespalib::test::make_tls_options_for_testing()), TimeBomb(60))
+            2, SocketPair(), TlsCryptoEngine(make_tls_options_for_testing()), TimeBomb(60))
 {
     TEST_DO(verify_crypto_socket(f1, f2, (thread_id == 0)));
 }
 
 TEST_MT_FFF("require that encrypted sync socket io works with MaybeTlsCryptoEngine(true)",
-            2, SocketPair(), MaybeTlsCryptoEngine(std::make_shared<TlsCryptoEngine>(vespalib::test::make_tls_options_for_testing()), true), TimeBomb(60))
+            2, SocketPair(), MaybeTlsCryptoEngine(std::make_shared<TlsCryptoEngine>(make_tls_options_for_testing()), true), TimeBomb(60))
 {
     TEST_DO(verify_crypto_socket(f1, f2, (thread_id == 0)));
 }
 
 TEST_MT_FFF("require that encrypted sync socket io works with MaybeTlsCryptoEngine(false)",
-            2, SocketPair(), MaybeTlsCryptoEngine(std::make_shared<TlsCryptoEngine>(vespalib::test::make_tls_options_for_testing()), false), TimeBomb(60))
+            2, SocketPair(), MaybeTlsCryptoEngine(std::make_shared<TlsCryptoEngine>(make_tls_options_for_testing()), false), TimeBomb(60))
 {
     TEST_DO(verify_crypto_socket(f1, f2, (thread_id == 0)));
 }
diff --git a/vespalib/src/tests/net/tls/direct_buffer_bio/direct_buffer_bio_test.cpp b/vespalib/src/tests/net/tls/direct_buffer_bio/direct_buffer_bio_test.cpp
index 134df9b17eb..a51cfb688c1 100644
--- a/vespalib/src/tests/net/tls/direct_buffer_bio/direct_buffer_bio_test.cpp
+++ b/vespalib/src/tests/net/tls/direct_buffer_bio/direct_buffer_bio_test.cpp
@@ -5,6 +5,7 @@
 #include <cassert>
 
 using namespace vespalib;
+using namespace vespalib::crypto;
 using namespace vespalib::net::tls::impl;
 
 struct Fixture {
diff --git a/vespalib/src/tests/net/tls/openssl_impl/CMakeLists.txt b/vespalib/src/tests/net/tls/openssl_impl/CMakeLists.txt
index e8f77d36e16..799e2291d7c 100644
--- a/vespalib/src/tests/net/tls/openssl_impl/CMakeLists.txt
+++ b/vespalib/src/tests/net/tls/openssl_impl/CMakeLists.txt
@@ -1,7 +1,6 @@
 # Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 vespa_add_executable(vespalib_net_tls_openssl_impl_test_app TEST
     SOURCES
-    crypto_utils.cpp
     openssl_impl_test.cpp
     DEPENDS
     vespalib
diff --git a/vespalib/src/tests/net/tls/openssl_impl/openssl_impl_test.cpp b/vespalib/src/tests/net/tls/openssl_impl/openssl_impl_test.cpp
index 78838ce2cd2..4586beef910 100644
--- a/vespalib/src/tests/net/tls/openssl_impl/openssl_impl_test.cpp
+++ b/vespalib/src/tests/net/tls/openssl_impl/openssl_impl_test.cpp
@@ -1,5 +1,6 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-#include "crypto_utils.h"
+#include <vespa/vespalib/crypto/private_key.h>
+#include <vespa/vespalib/crypto/x509_certificate.h>
 #include <vespa/vespalib/testkit/test_kit.h>
 #include <vespa/vespalib/data/smart_buffer.h>
 #include <vespa/vespalib/net/tls/authorization_mode.h>
@@ -11,11 +12,11 @@
 #include <vespa/vespalib/net/tls/impl/openssl_tls_context_impl.h>
 #include <vespa/vespalib/test/make_tls_options_for_testing.h>
 #include <vespa/vespalib/test/peer_policy_utils.h>
-#include <iostream>
 #include <stdexcept>
 #include <stdlib.h>
 
 using namespace vespalib;
+using namespace vespalib::crypto;
 using namespace vespalib::net::tls;
 using namespace vespalib::net::tls::impl;
 
@@ -57,11 +58,23 @@ void print_decode_result(const char* mode, const DecodeResult& res) {
             decode_state_to_str(res.state));
 }
 
+TransportSecurityOptions ts_from_pems(vespalib::stringref ca_certs_pem,
+                                      vespalib::stringref cert_chain_pem,
+                                      vespalib::stringref private_key_pem)
+{
+    auto ts_builder = TransportSecurityOptions::Params().
+            ca_certs_pem(ca_certs_pem).
+            cert_chain_pem(cert_chain_pem).
+            private_key_pem(private_key_pem).
+            authorized_peers(AuthorizedPeers::allow_all_authenticated());
+    return TransportSecurityOptions(std::move(ts_builder));
+}
+
 struct Fixture {
     TransportSecurityOptions tls_opts;
     std::shared_ptr<TlsContext> tls_ctx;
-    std::unique_ptr<CryptoCodec> client;
-    std::unique_ptr<CryptoCodec> server;
+    std::unique_ptr<OpenSslCryptoCodecImpl> client;
+    std::unique_ptr<OpenSslCryptoCodecImpl> server;
     SmartBuffer client_to_server;
     SmartBuffer server_to_client;
 
@@ -77,16 +90,21 @@ struct Fixture {
 
     static TransportSecurityOptions create_options_without_own_peer_cert() {
         auto source_opts = vespalib::test::make_tls_options_for_testing();
-        return TransportSecurityOptions(source_opts.ca_certs_pem(), "", "");
+        return ts_from_pems(source_opts.ca_certs_pem(), "", "");
     }
 
-    static std::unique_ptr<CryptoCodec> create_openssl_codec(
-            const TransportSecurityOptions& opts, CryptoCodec::Mode mode) {
+    static std::unique_ptr<OpenSslCryptoCodecImpl> create_openssl_codec(
+            const TransportSecurityOptions& opts, CryptoCodec::Mode mode, const SocketSpec& peer_spec) {
         auto ctx = TlsContext::create_default_context(opts, AuthorizationMode::Enforce);
-        return create_openssl_codec(ctx, mode);
+        return create_openssl_codec(ctx, mode, peer_spec);
+    }
+
+    static std::unique_ptr<OpenSslCryptoCodecImpl> create_openssl_codec(
+            const TransportSecurityOptions& opts, CryptoCodec::Mode mode) {
+        return create_openssl_codec(opts, mode, SocketSpec::invalid);
     }
 
-    static std::unique_ptr<CryptoCodec> create_openssl_codec(
+    static std::unique_ptr<OpenSslCryptoCodecImpl> create_openssl_codec(
             const TransportSecurityOptions& opts,
             std::shared_ptr<CertificateVerificationCallback> cert_verify_callback,
             CryptoCodec::Mode mode) {
@@ -94,21 +112,30 @@ struct Fixture {
         return create_openssl_codec(ctx, mode);
     }
 
-    static std::unique_ptr<CryptoCodec> create_openssl_codec(
-            const std::shared_ptr<TlsContext>& ctx, CryptoCodec::Mode mode) {
+    static std::unique_ptr<OpenSslCryptoCodecImpl> create_openssl_codec(
+            const std::shared_ptr<TlsContext>& ctx, CryptoCodec::Mode mode, const SocketSpec& peer_spec) {
         auto ctx_impl = std::dynamic_pointer_cast<impl::OpenSslTlsContextImpl>(ctx);
-        return std::make_unique<impl::OpenSslCryptoCodecImpl>(std::move(ctx_impl), SocketAddress(), mode);
+        if (mode == CryptoCodec::Mode::Client) {
+            return OpenSslCryptoCodecImpl::make_client_codec(std::move(ctx_impl), peer_spec, SocketAddress());
+        } else {
+            return OpenSslCryptoCodecImpl::make_server_codec(std::move(ctx_impl), SocketAddress());
+        }
     }
 
-    EncodeResult do_encode(CryptoCodec& codec, Output& buffer, vespalib::stringref plaintext) {
+    static std::unique_ptr<OpenSslCryptoCodecImpl> create_openssl_codec(
+            const std::shared_ptr<TlsContext>& ctx, CryptoCodec::Mode mode) {
+        return create_openssl_codec(ctx, mode, SocketSpec::invalid);
+    }
+
+    static EncodeResult do_encode(CryptoCodec& codec, Output& buffer, vespalib::stringref plaintext) {
         auto out = buffer.reserve(codec.min_encode_buffer_size());
         auto enc_res = codec.encode(plaintext.data(), plaintext.size(), out.data, out.size);
         buffer.commit(enc_res.bytes_produced);
         return enc_res;
     }
 
-    DecodeResult do_decode(CryptoCodec& codec, Input& buffer, vespalib::string& out,
-                           size_t max_bytes_produced, size_t max_bytes_consumed) {
+    static DecodeResult do_decode(CryptoCodec& codec, Input& buffer, vespalib::string& out,
+                                  size_t max_bytes_produced, size_t max_bytes_consumed) {
         auto in = buffer.obtain();
         out.resize(max_bytes_produced);
         auto to_consume = std::min(in.size, max_bytes_consumed);
@@ -382,13 +409,13 @@ l9pLv1vrujrPEC78cyIQe2x55wf3pRoaDg==
 -----END EC PRIVATE KEY-----)";
 
 TEST_F("client with certificate signed by untrusted CA is rejected by server", Fixture) {
-    TransportSecurityOptions client_opts(unknown_ca_pem, untrusted_host_cert_pem, untrusted_host_key_pem);
+    auto client_opts = ts_from_pems(unknown_ca_pem, untrusted_host_cert_pem, untrusted_host_key_pem);
     f.client = f.create_openssl_codec(client_opts, CryptoCodec::Mode::Client);
     EXPECT_FALSE(f.handshake());
 }
 
 TEST_F("server with certificate signed by untrusted CA is rejected by client", Fixture) {
-    TransportSecurityOptions server_opts(unknown_ca_pem, untrusted_host_cert_pem, untrusted_host_key_pem);
+    auto server_opts = ts_from_pems(unknown_ca_pem, untrusted_host_cert_pem, untrusted_host_key_pem);
     f.server = f.create_openssl_codec(server_opts, CryptoCodec::Mode::Server);
     EXPECT_FALSE(f.handshake());
 }
@@ -396,8 +423,8 @@ TEST_F("server with certificate signed by untrusted CA is rejected by client", F
 TEST_F("Can specify multiple trusted CA certs in transport options", Fixture) {
     auto& base_opts = f.tls_opts;
     auto multi_ca_pem = base_opts.ca_certs_pem() + "\n" + unknown_ca_pem;
-    TransportSecurityOptions multi_ca_using_ca_1(multi_ca_pem, untrusted_host_cert_pem, untrusted_host_key_pem);
-    TransportSecurityOptions multi_ca_using_ca_2(multi_ca_pem, base_opts.cert_chain_pem(), base_opts.private_key_pem());
+    auto multi_ca_using_ca_1 = ts_from_pems(multi_ca_pem, untrusted_host_cert_pem, untrusted_host_key_pem);
+    auto multi_ca_using_ca_2 = ts_from_pems(multi_ca_pem, base_opts.cert_chain_pem(), base_opts.private_key_pem());
     // Let client be signed by CA 1, server by CA 2. Both have the two CAs in their trust store
     // so this should allow for a successful handshake.
     f.client = f.create_openssl_codec(multi_ca_using_ca_1, CryptoCodec::Mode::Client);
@@ -446,7 +473,7 @@ struct CertFixture : Fixture {
         return {std::move(cert), std::move(key)};
     }
 
-    static std::unique_ptr<CryptoCodec> create_openssl_codec_with_authz_mode(
+    static std::unique_ptr<OpenSslCryptoCodecImpl> create_openssl_codec_with_authz_mode(
             const TransportSecurityOptions& opts,
             std::shared_ptr<CertificateVerificationCallback> cert_verify_callback,
             CryptoCodec::Mode codec_mode,
@@ -455,33 +482,52 @@ struct CertFixture : Fixture {
         return create_openssl_codec(ctx, codec_mode);
     }
 
+    TransportSecurityOptions::Params ts_builder_from(const CertKeyWrapper& ck) const {
+        return TransportSecurityOptions::Params().
+                ca_certs_pem(root_ca.cert->to_pem()).
+                cert_chain_pem(ck.cert->to_pem()).
+                private_key_pem(ck.key->private_to_pem());
+    }
+
     void reset_client_with_cert_opts(const CertKeyWrapper& ck, AuthorizedPeers authorized) {
-        TransportSecurityOptions client_opts(root_ca.cert->to_pem(), ck.cert->to_pem(),
-                                             ck.key->private_to_pem(), std::move(authorized));
-        client = create_openssl_codec(client_opts, CryptoCodec::Mode::Client);
+        auto ts_params = ts_builder_from(ck).authorized_peers(std::move(authorized));
+        client = create_openssl_codec(TransportSecurityOptions(std::move(ts_params)), CryptoCodec::Mode::Client);
     }
 
     void reset_client_with_cert_opts(const CertKeyWrapper& ck, std::shared_ptr<CertificateVerificationCallback> cert_cb) {
-        TransportSecurityOptions client_opts(root_ca.cert->to_pem(), ck.cert->to_pem(), ck.key->private_to_pem());
-        client = create_openssl_codec(client_opts, std::move(cert_cb), CryptoCodec::Mode::Client);
+        auto ts_params = ts_builder_from(ck).authorized_peers(AuthorizedPeers::allow_all_authenticated());
+        client = create_openssl_codec(TransportSecurityOptions(std::move(ts_params)),
+                                      std::move(cert_cb), CryptoCodec::Mode::Client);
     }
 
     void reset_server_with_cert_opts(const CertKeyWrapper& ck, AuthorizedPeers authorized) {
-        TransportSecurityOptions server_opts(root_ca.cert->to_pem(), ck.cert->to_pem(),
-                                             ck.key->private_to_pem(), std::move(authorized));
-        server = create_openssl_codec(server_opts, CryptoCodec::Mode::Server);
+        auto ts_params = ts_builder_from(ck).authorized_peers(std::move(authorized));
+        server = create_openssl_codec(TransportSecurityOptions(std::move(ts_params)), CryptoCodec::Mode::Server);
     }
 
     void reset_server_with_cert_opts(const CertKeyWrapper& ck, std::shared_ptr<CertificateVerificationCallback> cert_cb) {
-        TransportSecurityOptions server_opts(root_ca.cert->to_pem(), ck.cert->to_pem(), ck.key->private_to_pem());
-        server = create_openssl_codec(server_opts, std::move(cert_cb), CryptoCodec::Mode::Server);
+        auto ts_params = ts_builder_from(ck).authorized_peers(AuthorizedPeers::allow_all_authenticated());
+        server = create_openssl_codec(TransportSecurityOptions(std::move(ts_params)),
+                                      std::move(cert_cb), CryptoCodec::Mode::Server);
     }
 
     void reset_server_with_cert_opts(const CertKeyWrapper& ck,
                                      std::shared_ptr<CertificateVerificationCallback> cert_cb,
                                      AuthorizationMode authz_mode) {
-        TransportSecurityOptions server_opts(root_ca.cert->to_pem(), ck.cert->to_pem(), ck.key->private_to_pem());
-        server = create_openssl_codec_with_authz_mode(server_opts, std::move(cert_cb), CryptoCodec::Mode::Server, authz_mode);
+        auto ts_params = ts_builder_from(ck).authorized_peers(AuthorizedPeers::allow_all_authenticated());
+        server = create_openssl_codec_with_authz_mode(TransportSecurityOptions(std::move(ts_params)),
+                                                      std::move(cert_cb), CryptoCodec::Mode::Server, authz_mode);
+    }
+
+    void reset_client_with_peer_spec(const CertKeyWrapper& ck,
+                                     const SocketSpec& peer_spec,
+                                     bool disable_hostname_validation = false)
+    {
+        auto ts_params = ts_builder_from(ck).
+                authorized_peers(AuthorizedPeers::allow_all_authenticated()).
+                disable_hostname_validation(disable_hostname_validation);
+        client = create_openssl_codec(TransportSecurityOptions(std::move(ts_params)),
+                                      CryptoCodec::Mode::Client, peer_spec);
     }
 };
 
@@ -537,7 +583,7 @@ TEST_F("Exception during verification callback processing breaks handshake", Cer
     EXPECT_FALSE(f.handshake());
 }
 
-TEST_F("certificate verification callback observes CN and DNS SANs", CertFixture) {
+TEST_F("Certificate verification callback observes CN and DNS SANs", CertFixture) {
     auto ck = f.create_ca_issued_peer_cert(
             {{"rockets.wile.example.com"}},
             {{"DNS:crash.wile.example.com"}, {"DNS:burn.wile.example.com"}});
@@ -556,7 +602,7 @@ TEST_F("certificate verification callback observes CN and DNS SANs", CertFixture
     EXPECT_EQUAL("burn.wile.example.com", creds.dns_sans[1]);
 }
 
-TEST_F("last occurring CN is given to verification callback if multiple CNs are present", CertFixture) {
+TEST_F("Last occurring CN is given to verification callback if multiple CNs are present", CertFixture) {
     auto ck = f.create_ca_issued_peer_cert(
             {{"foo.wile.example.com"}, {"bar.wile.example.com"}, {"baz.wile.example.com"}}, {});
 
@@ -646,6 +692,51 @@ TEST_F("Disabled insecure authorization mode ignores verification result", CertF
     EXPECT_TRUE(f.handshake());
 }
 
+void reset_peers_with_client_peer_spec(CertFixture& f,
+                                       const SocketSpec& peer_spec,
+                                       bool disable_hostname_validation = false)
+{
+    auto client_ck = f.create_ca_issued_peer_cert({"hello.world.example.com"}, {});
+    f.reset_client_with_peer_spec(client_ck, peer_spec, disable_hostname_validation);
+    // Since hostname validation is enabled by default, providing a peer spec also
+    // means that we must have a valid server name to present back (or the handshake fails).
+    auto server_ck = f.create_ca_issued_peer_cert({}, {{"DNS:*.example.com"}});
+    f.reset_server_with_cert_opts(server_ck, AuthorizedPeers::allow_all_authenticated());
+}
+
+TEST_F("Client does not send SNI extension if hostname not provided in spec", CertFixture) {
+    reset_peers_with_client_peer_spec(f, SocketSpec::invalid);
+
+    ASSERT_TRUE(f.handshake());
+    auto maybe_sni = f.server->client_provided_sni_extension();
+    EXPECT_FALSE(maybe_sni.has_value());
+}
+
+TEST_F("Client sends SNI extension with hostname provided in spec", CertFixture) {
+    reset_peers_with_client_peer_spec(f, SocketSpec::from_host_port("sni-test.example.com", 12345));
+
+    ASSERT_TRUE(f.handshake());
+    auto maybe_sni = f.server->client_provided_sni_extension();
+    ASSERT_TRUE(maybe_sni.has_value());
+    EXPECT_EQUAL("sni-test.example.com", *maybe_sni);
+}
+
+TEST_F("Client hostname validation passes handshake if server hostname matches certificate", CertFixture) {
+    reset_peers_with_client_peer_spec(f, SocketSpec::from_host_port("server-must-be-under.example.com", 12345), false);
+    EXPECT_TRUE(f.handshake());
+}
+
+TEST_F("Client hostname validation fails handshake if server hostname mismatches certificate", CertFixture) {
+    // Wildcards only apply to a single level, so this should fail as the server only has a cert for *.example.com
+    reset_peers_with_client_peer_spec(f, SocketSpec::from_host_port("nested.name.example.com", 12345), false);
+    EXPECT_FALSE(f.handshake());
+}
+
+TEST_F("Mismatching server cert vs hostname does not fail if hostname validation is disabled", CertFixture) {
+    reset_peers_with_client_peer_spec(f, SocketSpec::from_host_port("a.very.nested.name.example.com", 12345), true);
+    EXPECT_TRUE(f.handshake());
+}
+
 TEST_F("Failure statistics are incremented on authorization failures", CertFixture) {
     reset_peers_with_server_authz_mode(f, AuthorizationMode::Enforce);
     auto server_before = ConnectionStatistics::get(true).snapshot();
diff --git a/vespalib/src/tests/net/tls/policy_checking_certificate_verifier/policy_checking_certificate_verifier_test.cpp b/vespalib/src/tests/net/tls/policy_checking_certificate_verifier/policy_checking_certificate_verifier_test.cpp
index ad45c217701..a9e823bf3ab 100644
--- a/vespalib/src/tests/net/tls/policy_checking_certificate_verifier/policy_checking_certificate_verifier_test.cpp
+++ b/vespalib/src/tests/net/tls/policy_checking_certificate_verifier/policy_checking_certificate_verifier_test.cpp
@@ -1,11 +1,8 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-#include <vespa/vespalib/io/fileutil.h>
 #include <vespa/vespalib/net/tls/transport_security_options.h>
-#include <vespa/vespalib/net/tls/transport_security_options_reading.h>
 #include <vespa/vespalib/net/tls/policy_checking_certificate_verifier.h>
 #include <vespa/vespalib/test/peer_policy_utils.h>
 #include <vespa/vespalib/testkit/test_kit.h>
-#include <vespa/vespalib/util/exceptions.h>
 
 using namespace vespalib;
 using namespace vespalib::net::tls;
diff --git a/vespalib/src/tests/net/tls/transport_options/transport_options_reading_test.cpp b/vespalib/src/tests/net/tls/transport_options/transport_options_reading_test.cpp
index a54e2f29aa1..00459a4e69c 100644
--- a/vespalib/src/tests/net/tls/transport_options/transport_options_reading_test.cpp
+++ b/vespalib/src/tests/net/tls/transport_options/transport_options_reading_test.cpp
@@ -155,6 +155,47 @@ TEST("accepted cipher list is populated if specified") {
     EXPECT_EQUAL("bar", ciphers[1]);
 }
 
+// FIXME this is temporary until we know enabling it by default won't break the world!
+TEST("hostname validation is DISABLED by default when creating options from config file") {
+    const char* json = R"({"files":{"private-key":"dummy_privkey.txt",
+                                    "certificates":"dummy_certs.txt",
+                                    "ca-certificates":"dummy_ca_certs.txt"}})";
+    EXPECT_TRUE(read_options_from_json_string(json)->disable_hostname_validation());
+}
+
+TEST("TransportSecurityOptions builder does not disable hostname validation by default") {
+    auto ts_builder = vespalib::net::tls::TransportSecurityOptions::Params().
+            ca_certs_pem("foo").
+            cert_chain_pem("bar").
+            private_key_pem("fantonald");
+    TransportSecurityOptions ts_opts(std::move(ts_builder));
+    EXPECT_FALSE(ts_opts.disable_hostname_validation());
+}
+
+TEST("hostname validation can be explicitly disabled") {
+    const char* json = R"({"files":{"private-key":"dummy_privkey.txt",
+                                    "certificates":"dummy_certs.txt",
+                                    "ca-certificates":"dummy_ca_certs.txt"},
+                           "disable-hostname-validation": true})";
+    EXPECT_TRUE(read_options_from_json_string(json)->disable_hostname_validation());
+}
+
+TEST("hostname validation can be explicitly enabled") {
+    const char* json = R"({"files":{"private-key":"dummy_privkey.txt",
+                                    "certificates":"dummy_certs.txt",
+                                    "ca-certificates":"dummy_ca_certs.txt"},
+                           "disable-hostname-validation": false})";
+    EXPECT_FALSE(read_options_from_json_string(json)->disable_hostname_validation());
+}
+
+TEST("unknown fields are ignored at parse-time") {
+    const char* json = R"({"files":{"private-key":"dummy_privkey.txt",
+                                    "certificates":"dummy_certs.txt",
+                                    "ca-certificates":"dummy_ca_certs.txt"},
+                           "flipper-the-dolphin": "*weird dolphin noises*"})";
+    EXPECT_TRUE(read_options_from_json_string(json).get() != nullptr); // And no exception thrown.
+}
+
 // TODO test parsing of multiple policies
 
 TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/vespalib/src/tests/portal/portal_test.cpp b/vespalib/src/tests/portal/portal_test.cpp
index e54700306fe..0bd029c0c3a 100644
--- a/vespalib/src/tests/portal/portal_test.cpp
+++ b/vespalib/src/tests/portal/portal_test.cpp
@@ -14,13 +14,14 @@
 #include <vespa/vespalib/util/latch.h>
 
 using namespace vespalib;
+using namespace vespalib::test;
 
 //-----------------------------------------------------------------------------
 
 vespalib::string do_http(int port, CryptoEngine::SP crypto, const vespalib::string &method, const vespalib::string &uri, bool send_host = true) {
     auto socket = SocketSpec::from_port(port).client_address().connect();
     ASSERT_TRUE(socket.valid());
-    auto conn = SyncCryptoSocket::create(*crypto, std::move(socket), false);
+    auto conn = SyncCryptoSocket::create_client(*crypto, std::move(socket), local_spec);
     vespalib::string http_req = vespalib::make_string("%s %s HTTP/1.1\r\n"
                                                       "My-Header: my value\r\n"
                                                       "%s"
@@ -75,7 +76,7 @@ Encryption::~Encryption() = default;
 
 auto null_crypto() { return std::make_shared<NullCryptoEngine>(); }
 auto xor_crypto() { return std::make_shared<XorCryptoEngine>(); }
-auto tls_crypto() { return std::make_shared<TlsCryptoEngine>(vespalib::test::make_tls_options_for_testing()); }
+auto tls_crypto() { return std::make_shared<TlsCryptoEngine>(make_tls_options_for_testing()); }
 auto maybe_tls_crypto(bool client_tls) { return std::make_shared<MaybeTlsCryptoEngine>(tls_crypto(), client_tls); }
 
 std::vector<Encryption> crypto_list = {{"no encryption", null_crypto()},
@@ -260,18 +261,18 @@ TEST("require that connection errors do not block shutdown by leaking resources"
         auto bound = portal->bind("/test", handler);
         { // close before sending anything
             auto socket = SocketSpec::from_port(portal->listen_port()).client_address().connect();
-            auto conn = SyncCryptoSocket::create(*crypto.engine, std::move(socket), false);
+            auto conn = SyncCryptoSocket::create_client(*crypto.engine, std::move(socket), local_spec);
         }
         { // send partial request then close connection
             auto socket = SocketSpec::from_port(portal->listen_port()).client_address().connect();
-            auto conn = SyncCryptoSocket::create(*crypto.engine, std::move(socket), false);
+            auto conn = SyncCryptoSocket::create_client(*crypto.engine, std::move(socket), local_spec);
             vespalib::string req = "GET /test HTTP/1.1\r\n"
                                    "Host: local";
             ASSERT_EQUAL(conn->write(req.data(), req.size()), ssize_t(req.size()));
         }
         { // send request then close without reading response
             auto socket = SocketSpec::from_port(portal->listen_port()).client_address().connect();
-            auto conn = SyncCryptoSocket::create(*crypto.engine, std::move(socket), false);
+            auto conn = SyncCryptoSocket::create_client(*crypto.engine, std::move(socket), local_spec);
             vespalib::string req = "GET /test HTTP/1.1\r\n"
                                    "Host: localhost\r\n"
                                    "\r\n";
diff --git a/vespalib/src/tests/regex/regex.cpp b/vespalib/src/tests/regex/regex.cpp
index d1b94daa7ba..7dc5a7f4aa9 100644
--- a/vespalib/src/tests/regex/regex.cpp
+++ b/vespalib/src/tests/regex/regex.cpp
@@ -1,70 +1,147 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 #include <vespa/vespalib/testkit/test_kit.h>
-
+#include <vespa/vespalib/regex/regex.h>
 #include <vespa/vespalib/util/regexp.h>
-#include <vespa/vespalib/util/exception.h>
-#include <regex>
+#include <string>
 
 using namespace vespalib;
 
 TEST("require that prefix detection works") {
-    EXPECT_EQUAL("", Regexp::get_prefix(""));
-    EXPECT_EQUAL("", Regexp::get_prefix("foo"));
-    EXPECT_EQUAL("foo", Regexp::get_prefix("^foo"));
-    EXPECT_EQUAL("", Regexp::get_prefix("^foo|bar"));
-    EXPECT_EQUAL("foo", Regexp::get_prefix("^foo$"));
-    EXPECT_EQUAL("foo", Regexp::get_prefix("^foo[a-z]"));
-    EXPECT_EQUAL("fo", Regexp::get_prefix("^foo{0,1}"));
-    EXPECT_EQUAL("foo", Regexp::get_prefix("^foo."));
-    EXPECT_EQUAL("fo", Regexp::get_prefix("^foo*"));
-    EXPECT_EQUAL("fo", Regexp::get_prefix("^foo?"));
-    EXPECT_EQUAL("foo", Regexp::get_prefix("^foo+"));
+    EXPECT_EQUAL("", RegexpUtil::get_prefix(""));
+    EXPECT_EQUAL("", RegexpUtil::get_prefix("foo"));
+    EXPECT_EQUAL("foo", RegexpUtil::get_prefix("^foo"));
+    EXPECT_EQUAL("", RegexpUtil::get_prefix("^foo|bar"));
+    EXPECT_EQUAL("foo", RegexpUtil::get_prefix("^foo$"));
+    EXPECT_EQUAL("foo", RegexpUtil::get_prefix("^foo[a-z]"));
+    EXPECT_EQUAL("fo", RegexpUtil::get_prefix("^foo{0,1}"));
+    EXPECT_EQUAL("foo", RegexpUtil::get_prefix("^foo."));
+    EXPECT_EQUAL("fo", RegexpUtil::get_prefix("^foo*"));
+    EXPECT_EQUAL("fo", RegexpUtil::get_prefix("^foo?"));
+    EXPECT_EQUAL("foo", RegexpUtil::get_prefix("^foo+"));
 }
 
 TEST("require that prefix detection sometimes underestimates the prefix size") {
-    EXPECT_EQUAL("", Regexp::get_prefix("^^foo"));
-    EXPECT_EQUAL("", Regexp::get_prefix("^foo(bar|baz)"));
-    EXPECT_EQUAL("fo", Regexp::get_prefix("^foo{1,2}"));
-    EXPECT_EQUAL("foo", Regexp::get_prefix("^foo\\."));
-    EXPECT_EQUAL("foo", Regexp::get_prefix("^foo(bar)"));
-    EXPECT_EQUAL("", Regexp::get_prefix("(^foo)"));
-    EXPECT_EQUAL("", Regexp::get_prefix("^(foo)"));
-    EXPECT_EQUAL("foo", Regexp::get_prefix("^foo[a]"));
-    EXPECT_EQUAL("", Regexp::get_prefix("^foo|^foobar"));
+    EXPECT_EQUAL("", RegexpUtil::get_prefix("^^foo"));
+    EXPECT_EQUAL("", RegexpUtil::get_prefix("^foo(bar|baz)"));
+    EXPECT_EQUAL("fo", RegexpUtil::get_prefix("^foo{1,2}"));
+    EXPECT_EQUAL("foo", RegexpUtil::get_prefix("^foo\\."));
+    EXPECT_EQUAL("foo", RegexpUtil::get_prefix("^foo(bar)"));
+    EXPECT_EQUAL("", RegexpUtil::get_prefix("(^foo)"));
+    EXPECT_EQUAL("", RegexpUtil::get_prefix("^(foo)"));
+    EXPECT_EQUAL("foo", RegexpUtil::get_prefix("^foo[a]"));
+    EXPECT_EQUAL("", RegexpUtil::get_prefix("^foo|^foobar"));
 }
 
-const vespalib::string special("^|()[]{}.*?+\\$");
+const std::string special("^|()[]{}.*?+\\$");
 
 struct ExprFixture {
-    std::vector<vespalib::string> expressions;
+    std::vector<std::string> expressions;
     ExprFixture() {
         expressions.push_back(special);
         for (char c: special) {
-            expressions.push_back(vespalib::string(&c, 1));
+            expressions.emplace_back(std::string(&c, 1));
         }
-        expressions.push_back("abc");
-        expressions.push_back("[:digit:]");
+        expressions.emplace_back("abc");
+        expressions.emplace_back("[:digit:]");
     }
 };
 
 TEST_F("require that regexp can be made from suffix string", ExprFixture()) {
-    for (vespalib::string str: f1.expressions) {
-        std::regex re(std::string(Regexp::make_from_suffix(str)));
-        EXPECT_TRUE(std::regex_search(std::string(str), re));
-        EXPECT_FALSE(std::regex_search(std::string(str + "foo"), re));
-        EXPECT_TRUE(std::regex_search(std::string("foo" + str), re));
-        EXPECT_FALSE(std::regex_search(std::string("foo" + str + "bar"), re));
+    for (const auto& str: f1.expressions) {
+        auto re = Regex::from_pattern(std::string(RegexpUtil::make_from_suffix(str)));
+        ASSERT_TRUE(re.parsed_ok());
+
+        EXPECT_TRUE(re.partial_match(str));
+        EXPECT_FALSE(re.partial_match(str + "foo"));
+        EXPECT_TRUE(re.partial_match("foo" + str));
+        EXPECT_FALSE(re.partial_match("foo" + str + "bar"));
     }
 }
 
 TEST_F("require that regexp can be made from substring string", ExprFixture()) {
-    for (vespalib::string str: f1.expressions) {
-        std::regex re(std::string(Regexp::make_from_substring(str)));
-        EXPECT_TRUE(std::regex_search(std::string(str), re));
-        EXPECT_TRUE(std::regex_search(std::string(str + "foo"), re));
-        EXPECT_TRUE(std::regex_search(std::string("foo" + str), re));
-        EXPECT_TRUE(std::regex_search(std::string("foo" + str + "bar"), re));
+    for (const auto& str: f1.expressions) {
+        auto re = Regex::from_pattern(std::string(RegexpUtil::make_from_substring(str)));
+        ASSERT_TRUE(re.parsed_ok());
+
+        EXPECT_TRUE(re.partial_match(str));
+        EXPECT_TRUE(re.partial_match(str + "foo"));
+        EXPECT_TRUE(re.partial_match("foo" + str));
+        EXPECT_TRUE(re.partial_match("foo" + str + "bar"));
     }
 }
 
+TEST("full_match requires expression to match entire input string") {
+    std::string pattern = "[Aa][Bb][Cc]";
+    auto re = Regex::from_pattern(pattern);
+    ASSERT_TRUE(re.parsed_ok());
+
+    EXPECT_TRUE(re.full_match("abc"));
+    EXPECT_TRUE(re.full_match("ABC"));
+    EXPECT_FALSE(re.full_match("abcd"));
+    EXPECT_FALSE(re.full_match("aabc"));
+    EXPECT_FALSE(re.full_match("aabcc"));
+
+    EXPECT_TRUE(Regex::full_match("abc", pattern));
+    EXPECT_TRUE(Regex::full_match("ABC", pattern));
+    EXPECT_FALSE(Regex::full_match("abcd", pattern));
+    EXPECT_FALSE(Regex::full_match("aabc", pattern));
+    EXPECT_FALSE(Regex::full_match("aabcc", pattern));
+}
+
+TEST("partial_match requires expression to match substring of input string") {
+    std::string pattern = "[Aa][Bb][Cc]";
+    auto re = Regex::from_pattern(pattern);
+    ASSERT_TRUE(re.parsed_ok());
+
+    EXPECT_TRUE(re.partial_match("abc"));
+    EXPECT_TRUE(re.partial_match("ABC"));
+    EXPECT_TRUE(re.partial_match("abcd"));
+    EXPECT_TRUE(re.partial_match("aabc"));
+    EXPECT_TRUE(re.partial_match("aabcc"));
+    EXPECT_FALSE(re.partial_match("abd"));
+
+    EXPECT_TRUE(Regex::partial_match("abc", pattern));
+    EXPECT_TRUE(Regex::partial_match("ABC", pattern));
+    EXPECT_TRUE(Regex::partial_match("abcd", pattern));
+    EXPECT_TRUE(Regex::partial_match("aabc", pattern));
+    EXPECT_TRUE(Regex::partial_match("aabcc", pattern));
+    EXPECT_FALSE(Regex::partial_match("abd", pattern));
+}
+
+TEST("partial_match can be explicitly anchored") {
+    EXPECT_TRUE(Regex::partial_match("abcc", "^abc"));
+    EXPECT_FALSE(Regex::partial_match("aabc", "^abc"));
+    EXPECT_TRUE(Regex::partial_match("aabc", "abc$"));
+    EXPECT_FALSE(Regex::partial_match("abcc", "abc$"));
+    EXPECT_TRUE(Regex::partial_match("abc", "^abc$"));
+    EXPECT_FALSE(Regex::partial_match("aabc", "^abc$"));
+    EXPECT_FALSE(Regex::partial_match("abcc", "^abc$"));
+}
+
+TEST("Regex instance returns parsed_ok() == false upon parse failure") {
+    auto re = Regex::from_pattern("[a-z"); // Unterminated set
+    EXPECT_FALSE(re.parsed_ok());
+}
+
+TEST("Regex that has failed parsing immediately returns false for matches") {
+    auto re = Regex::from_pattern("[a-z");
+    EXPECT_FALSE(re.parsed_ok());
+    EXPECT_FALSE(re.partial_match("a"));
+    EXPECT_FALSE(re.full_match("b"));
+}
+
+TEST("can create case-insensitive regex matcher") {
+    auto re = Regex::from_pattern("hello", Regex::Options::IgnoreCase);
+    ASSERT_TRUE(re.parsed_ok());
+    EXPECT_TRUE(re.partial_match("HelLo world"));
+    EXPECT_TRUE(re.full_match("HELLO"));
+}
+
+TEST("regex is case sensitive by default") {
+    auto re = Regex::from_pattern("hello");
+    ASSERT_TRUE(re.parsed_ok());
+    EXPECT_FALSE(re.partial_match("HelLo world"));
+    EXPECT_FALSE(re.full_match("HELLO"));
+}
+
 TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/vespalib/src/tests/stllike/asciistream_test.cpp b/vespalib/src/tests/stllike/asciistream_test.cpp
index fd362d9c49a..c9435edea45 100644
--- a/vespalib/src/tests/stllike/asciistream_test.cpp
+++ b/vespalib/src/tests/stllike/asciistream_test.cpp
@@ -387,8 +387,10 @@ AsciistreamTest::testCreateFromFile()
     EXPECT_EQUAL("", s);
     EXPECT_TRUE(is.eof());
 
+#ifdef __linux__
     is = asciistream::createFromDevice("/proc/stat");
     EXPECT_FALSE(is.eof());
+#endif
 }
 
 void
diff --git a/vespalib/src/tests/stllike/hash_test.cpp b/vespalib/src/tests/stllike/hash_test.cpp
index 017a16ee7b6..d23c2c6b68c 100644
--- a/vespalib/src/tests/stllike/hash_test.cpp
+++ b/vespalib/src/tests/stllike/hash_test.cpp
@@ -32,7 +32,7 @@ namespace {
 TEST("test that hashValue gives expected response")
 {
     const char * s("abcdefghi");
-    EXPECT_EQUAL(7045194595191919248ul, vespalib::hashValue(s));
+    EXPECT_EQUAL(2878261200250560019ul, vespalib::hashValue(s));
     EXPECT_EQUAL(vespalib::hashValue(s), vespalib::hashValue(s, strlen(s)));
     EXPECT_NOT_EQUAL(vespalib::hashValue(s), vespalib::hashValue(s, strlen(s)-1));
 }
@@ -356,6 +356,9 @@ TEST("test hash set find")
     EXPECT_TRUE(*set.find(S(1)) == S(1));
     auto cit = set.find<uint32_t>(7);
     EXPECT_TRUE(*cit == S(7));
+
+    EXPECT_EQUAL(1u, set.count(S(7)));
+    EXPECT_EQUAL(0u, set.count(S(10007)));
 }
 
 TEST("test hash set range constructor")
diff --git a/vespalib/src/tests/stllike/lookup_benchmark.cpp b/vespalib/src/tests/stllike/lookup_benchmark.cpp
index 8ac895a3499..b3ce8c29a18 100644
--- a/vespalib/src/tests/stllike/lookup_benchmark.cpp
+++ b/vespalib/src/tests/stllike/lookup_benchmark.cpp
@@ -2,11 +2,10 @@
 #include <cstddef>
 #include <cstdlib>
 #include <cstdio>
-#include <map>
 #include <set>
 #include <unordered_set>
 #include <vector>
-#include <algorithm>
+#include <xxhash.h>
 #include <vespa/vespalib/stllike/hash_set.hpp>
 #include <vespa/vespalib/stllike/hash_map.hpp>
 
@@ -95,6 +94,41 @@ size_t benchHashMapVespaLib2(size_t sz, size_t numLookups)
     return benchM(set, sz, numLookups);
 }
 
+std::unique_ptr<char []> createData(size_t sz) {
+    auto data = std::make_unique<char []>(sz);
+    for (size_t i(0); i < sz; i++) {
+        data.get()[i] = i + '0';
+    }
+    return data;
+}
+
+size_t benchXXHash32(size_t sz, size_t numLookups) {
+    auto data = createData(sz);
+    size_t sum(0);
+    for (size_t i(0); i < numLookups; i++) {
+        sum += XXH32(data.get(), sz, 0);
+    }
+    return sum;
+}
+
+size_t benchXXHash64(size_t sz, size_t numLookups) {
+    auto data = createData(sz);
+    size_t sum(0);
+    for (size_t i(0); i < numLookups; i++) {
+        sum += XXH64(data.get(), sz, 0);
+    }
+    return sum;
+}
+
+size_t benchLegacyHash(size_t sz, size_t numLookups) {
+    auto data = createData(sz);
+    size_t sum(0);
+    for (size_t i(0); i < numLookups; i++) {
+        sum += vespalib::hashValue(data.get(), sz);
+    }
+    return sum;
+}
+
 int main(int argc, char *argv[])
 {
     size_t count(1000);
@@ -116,23 +150,27 @@ int main(int argc, char *argv[])
     description['G'] = "vespalib::hash_set with simple and modulator.";
     description['k'] = "vespalib::hash_map";
     description['K'] = "vespalib::hash_map with simple and modulator.";
+    description['x'] = "xxhash32";
+    description['X'] = "xxhash64";
+    description['l'] = "legacy";
     size_t found(0);
     switch (type) {
-    case 'm': found = benchMap(count, rep); break;
-    case 'h': found = benchHashStl(count, rep); break;
-    case 'g': found = benchHashVespaLib(count, rep); break;
-    case 'G': found = benchHashVespaLib2(count, rep); break;
-    case 'k': found = benchHashMapVespaLib(count, rep); break;
-    case 'K': found = benchHashMapVespaLib2(count, rep); break;
-    default:
-        printf("'m' = %s\n", description[type]);
-        printf("'h' = %s\n", description[type]);
-        printf("'g' = %s\n", description[type]);
-        printf("'G' = %s\n", description[type]);
-        printf("Unspecified type %c. Running map lookup benchmark\n", type);
-        exit(1);
-        break;
+        case 'm': found = benchMap(count, rep); break;
+        case 'h': found = benchHashStl(count, rep); break;
+        case 'g': found = benchHashVespaLib(count, rep); break;
+        case 'G': found = benchHashVespaLib2(count, rep); break;
+        case 'k': found = benchHashMapVespaLib(count, rep); break;
+        case 'K': found = benchHashMapVespaLib2(count, rep); break;
+        case 'x': found = benchXXHash32(count, rep); break;
+        case 'X': found = benchXXHash64(count, rep); break;
+        case 'l': found = benchLegacyHash(count, rep); break;
+        default:
+            for (char c : "mhgGkKxXl") {
+                printf("'%c' = %s\n", c, description[c]);
+            }
+            return 1;
     }
     printf("Running test '%c' = %s, result = %ld found values\n", type, description[type], found);
+    return 0;
 }
 
diff --git a/vespalib/src/tests/stringfmt/fmt.cpp b/vespalib/src/tests/stringfmt/fmt.cpp
index f9bd0ae12fb..e941c185e2f 100644
--- a/vespalib/src/tests/stringfmt/fmt.cpp
+++ b/vespalib/src/tests/stringfmt/fmt.cpp
@@ -4,6 +4,7 @@
 #include <vespa/vespalib/testkit/test_kit.h>
 
 using vespalib::make_string;
+using vespalib::make_string_short::fmt;
 
 TEST("test that make_string formats as one can expect.")
 {
@@ -39,4 +40,8 @@ TEST("test that make_string formats as one can expect.")
     EXPECT_TRUE(tst == make_string("%s", p));
 }
 
+TEST("require that short form make string can be used") {
+    EXPECT_EQUAL(fmt("format: %d", 123), vespalib::string("format: 123"));
+}
+
 TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/vespalib/src/tests/util/reusable_set/CMakeLists.txt b/vespalib/src/tests/util/reusable_set/CMakeLists.txt
new file mode 100644
index 00000000000..9c46b5ba61e
--- /dev/null
+++ b/vespalib/src/tests/util/reusable_set/CMakeLists.txt
@@ -0,0 +1,9 @@
+# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(vespalib_reusable_set_test_app TEST
+    SOURCES
+    reusable_set_test.cpp
+    DEPENDS
+    vespalib
+    gtest
+)
+vespa_add_test(NAME vespalib_reusable_set_test_app COMMAND vespalib_reusable_set_test_app)
diff --git a/vespalib/src/tests/util/reusable_set/reusable_set_test.cpp b/vespalib/src/tests/util/reusable_set/reusable_set_test.cpp
new file mode 100644
index 00000000000..26c3345c71e
--- /dev/null
+++ b/vespalib/src/tests/util/reusable_set/reusable_set_test.cpp
@@ -0,0 +1,139 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/vespalib/util/reusable_set_pool.h>
+#include <vespa/vespalib/gtest/gtest.h>
+
+using namespace vespalib;
+
+using Mark = ReusableSet::Mark;
+
+void verify_set(const ReusableSet &set, size_t sz, Mark val, size_t marked) {
+    EXPECT_EQ(sz, set.capacity());
+    EXPECT_EQ(val, set.generation());
+    size_t count = 0;
+    for (size_t i = 0; i < set.capacity(); ++i) {
+        if (set.is_marked(i)) ++count;
+    }
+    EXPECT_EQ(marked, count);
+}
+
+void verify_handle(const ReusableSetHandle &set, size_t sz, Mark val, size_t marked) {
+    EXPECT_EQ(sz, set.capacity());
+    EXPECT_EQ(val, set.generation());
+    size_t count = 0;
+    for (size_t i = 0; i < set.capacity(); ++i) {
+        if (set.is_marked(i)) ++count;
+    }
+    EXPECT_EQ(marked, count);
+}
+
+class Pool : public ::testing::Test {
+public:
+    ReusableSetPool pool;
+    Pool() : pool() {}
+    ~Pool() {}
+    void exercise(ReusableSetHandle &set) {
+        size_t sz = set.capacity();
+        size_t count = 0;
+        for (size_t i = 0; i < sz; ++i) {
+            if (set.is_marked(i)) ++count;
+        }
+        EXPECT_EQ(0, count);
+        for (int i = 0; i < 17; ++i) {
+            set.mark((i * 711) % sz);
+        }
+        count = 0;
+        for (size_t i = 0; i < sz; ++i) {
+            if (set.is_marked(i)) ++count;
+        }
+        EXPECT_EQ(17, count);
+        for (int i = 0; i < 17; ++i) {
+            set.mark((i * 711) % sz);
+        }
+        count = 0;
+        for (size_t i = 0; i < sz; ++i) {
+            if (set.is_marked(i)) ++count;
+        }
+        EXPECT_EQ(17, count);
+   }
+};
+
+
+TEST(ReusableSetTest, simple_usage)
+{
+    ReusableSet visited(7);
+    verify_set(visited, 7, 1, 0);
+    visited.mark(1);
+    visited.mark(2);
+    visited.mark(4);
+    EXPECT_EQ(false, visited.is_marked(0));
+    EXPECT_EQ(true, visited.is_marked(1));
+    EXPECT_EQ(true, visited.is_marked(2));
+    EXPECT_EQ(false, visited.is_marked(3));
+    verify_set(visited, 7, 1, 3);
+    visited.mark(4);
+    visited.mark(1);
+    visited.mark(2);
+    verify_set(visited, 7, 1, 3);
+    EXPECT_EQ(false, visited.is_marked(0));
+    EXPECT_EQ(true, visited.is_marked(1));
+    EXPECT_EQ(true, visited.is_marked(2));
+    EXPECT_EQ(false, visited.is_marked(3));
+    visited.clear();
+    verify_set(visited, 7, 2, 0);
+    visited.clear();
+    verify_set(visited, 7, 3, 0);
+}
+
+TEST_F(Pool, reuse_works)
+{
+    for (int i = 0; i < 65535; ++i) {
+        auto handle = pool.get(7);
+        EXPECT_EQ(i, pool.reuse_count());
+        EXPECT_EQ(1, pool.create_count());
+        verify_handle(handle, 248, i+1, 0);
+        exercise(handle);
+    }
+    EXPECT_LT(500, pool.memory_usage().allocatedBytes());
+    EXPECT_GT(1000, pool.memory_usage().allocatedBytes());
+    for (int i = 0; i < 5; ++i) {
+        auto handle = pool.get(7);
+        EXPECT_EQ(65535+i, pool.reuse_count());
+        EXPECT_EQ(1, pool.create_count());
+        verify_handle(handle, 248, i+1, 0);
+        exercise(handle);
+    }
+    auto handle3 = pool.get(260);
+    EXPECT_EQ(2, pool.create_count());
+    verify_handle(handle3, 297, 1, 0);
+    exercise(handle3);
+    {
+        auto handle4 = pool.get(400);
+        EXPECT_EQ(3, pool.create_count());
+        verify_handle(handle4, 400, 1, 0);
+        exercise(handle4);
+        EXPECT_LT(1000, pool.memory_usage().usedBytes());
+        EXPECT_GT(2000, pool.memory_usage().usedBytes());
+    }
+    EXPECT_LT(500, pool.memory_usage().usedBytes());
+    EXPECT_GT(1000, pool.memory_usage().usedBytes());
+    auto handle7 = pool.get(401);
+    EXPECT_EQ(4, pool.create_count());
+    verify_handle(handle7, 480, 1, 0);
+    exercise(handle7);
+    EXPECT_LT(1000, pool.memory_usage().allocatedBytes());
+    EXPECT_GT(3000, pool.memory_usage().allocatedBytes());
+    {
+        auto handle8 = pool.get(2500);
+        auto handle9 = pool.get(2500);
+        EXPECT_LT(11000, pool.memory_usage().allocatedBytes());
+        EXPECT_GT(13000, pool.memory_usage().allocatedBytes());
+        auto handleA = pool.get(25000);
+        auto handleB = pool.get(25000);
+        EXPECT_LT(111000, pool.memory_usage().usedBytes());
+        EXPECT_GT(113000, pool.memory_usage().usedBytes());
+    }
+    EXPECT_GT(3000, pool.memory_usage().usedBytes());
+}
+
+GTEST_MAIN_RUN_ALL_TESTS()
diff --git a/vespalib/src/vespa/vespalib/CMakeLists.txt b/vespalib/src/vespa/vespalib/CMakeLists.txt
index 4249f6333a4..92149d3f0ea 100644
--- a/vespalib/src/vespa/vespalib/CMakeLists.txt
+++ b/vespalib/src/vespa/vespalib/CMakeLists.txt
@@ -3,6 +3,7 @@ vespa_add_library(vespalib
     SOURCES
     $<TARGET_OBJECTS:vespalib_vespalib_btree>
     $<TARGET_OBJECTS:vespalib_vespalib_component>
+    $<TARGET_OBJECTS:vespalib_vespalib_crypto>
     $<TARGET_OBJECTS:vespalib_vespalib_data>
     $<TARGET_OBJECTS:vespalib_vespalib_data_slime>
     $<TARGET_OBJECTS:vespalib_vespalib_datastore>
@@ -15,6 +16,7 @@ vespa_add_library(vespalib
     $<TARGET_OBJECTS:vespalib_vespalib_net_tls_impl>
     $<TARGET_OBJECTS:vespalib_vespalib_objects>
     $<TARGET_OBJECTS:vespalib_vespalib_portal>
+    $<TARGET_OBJECTS:vespalib_vespalib_regex>
     $<TARGET_OBJECTS:vespalib_vespalib_stllike>
     $<TARGET_OBJECTS:vespalib_vespalib_test>
     $<TARGET_OBJECTS:vespalib_vespalib_testkit>
@@ -28,4 +30,8 @@ vespa_add_library(vespalib
     ${VESPA_GCC_LIB}
 )
 
+set(BLA_VENDOR OpenBLAS)
+vespa_add_target_package_dependency(vespalib BLAS)
 vespa_add_target_package_dependency(vespalib OpenSSL)
+vespa_add_target_package_dependency(vespalib RE2)
+
diff --git a/vespalib/src/vespa/vespalib/crypto/CMakeLists.txt b/vespalib/src/vespa/vespalib/crypto/CMakeLists.txt
new file mode 100644
index 00000000000..6000156fcfa
--- /dev/null
+++ b/vespalib/src/vespa/vespalib/crypto/CMakeLists.txt
@@ -0,0 +1,11 @@
+# Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_library(vespalib_vespalib_crypto OBJECT
+    SOURCES
+    crypto_exception.cpp
+    openssl_crypto_impl.cpp
+    private_key.cpp
+    x509_certificate.cpp
+    DEPENDS
+)
+find_package(OpenSSL)
+target_include_directories(vespalib_vespalib_crypto PUBLIC ${OPENSSL_INCLUDE_DIR})
diff --git a/vespalib/src/vespa/vespalib/net/tls/crypto_exception.cpp b/vespalib/src/vespa/vespalib/crypto/crypto_exception.cpp
index 41bb2060c04..226d8664de6 100644
--- a/vespalib/src/vespa/vespalib/net/tls/crypto_exception.cpp
+++ b/vespalib/src/vespa/vespalib/crypto/crypto_exception.cpp
@@ -2,7 +2,7 @@
 
 #include "crypto_exception.h"
 
-namespace vespalib::net::tls {
+namespace vespalib::crypto {
 
 VESPA_IMPLEMENT_EXCEPTION(CryptoException, Exception);
 
diff --git a/vespalib/src/vespa/vespalib/net/tls/crypto_exception.h b/vespalib/src/vespa/vespalib/crypto/crypto_exception.h
index 696a158e058..0d0dcc8ceec 100644
--- a/vespalib/src/vespa/vespalib/net/tls/crypto_exception.h
+++ b/vespalib/src/vespa/vespalib/crypto/crypto_exception.h
@@ -3,7 +3,7 @@
 
 #include <vespa/vespalib/util/exception.h>
 
-namespace vespalib::net::tls {
+namespace vespalib::crypto {
 
 VESPA_DEFINE_EXCEPTION(CryptoException, Exception);
 
diff --git a/vespalib/src/tests/net/tls/openssl_impl/crypto_utils.cpp b/vespalib/src/vespa/vespalib/crypto/openssl_crypto_impl.cpp
index 14755360b51..72efbb841c2 100644
--- a/vespalib/src/tests/net/tls/openssl_impl/crypto_utils.cpp
+++ b/vespalib/src/vespa/vespalib/crypto/openssl_crypto_impl.cpp
@@ -1,13 +1,12 @@
-// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#include "crypto_utils.h"
-#include <vespa/vespalib/net/tls/crypto_exception.h>
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include "openssl_crypto_impl.h"
+#include <vespa/vespalib/crypto/crypto_exception.h>
 #include <cassert>
 #include <openssl/bn.h>
 #include <openssl/rand.h>
 #include <openssl/x509v3.h>
 
-namespace vespalib::net::tls::impl {
+namespace vespalib::crypto::openssl_impl {
 
 namespace {
 
@@ -61,6 +60,77 @@ BioPtr new_memory_bio() {
     return bio;
 }
 
+} // anonymous namespace
+
+vespalib::string PrivateKeyImpl::private_to_pem() const {
+    BioPtr bio = new_memory_bio();
+    // TODO this API is const-broken even on 1.1.1, revisit in the future...
+    auto* mutable_pkey = const_cast<::EVP_PKEY*>(_pkey.get());
+    if (::PEM_write_bio_PrivateKey(bio.get(), mutable_pkey, nullptr, nullptr,
+                                   0, nullptr, nullptr) != 1) {
+        throw CryptoException("PEM_write_bio_PrivateKey");
+    }
+    return bio_to_string(*bio);
+}
+
+std::shared_ptr<PrivateKeyImpl> PrivateKeyImpl::generate_openssl_p256_ec_key() {
+    // We first have to generate an EVP context for the keygen _parameters_...
+    EvpPkeyCtxPtr params_ctx(::EVP_PKEY_CTX_new_id(EVP_PKEY_EC, nullptr));
+    if (!params_ctx) {
+        throw CryptoException("EVP_PKEY_CTX_new_id");
+    }
+    if (::EVP_PKEY_paramgen_init(params_ctx.get()) != 1) {
+        throw CryptoException("EVP_PKEY_paramgen_init");
+    }
+    // Set EC keygen parameters to use prime256v1, which is the same as P-256
+    if (EVP_PKEY_CTX_set_ec_paramgen_curve_nid(params_ctx.get(), NID_X9_62_prime256v1) <= 0) {
+        throw CryptoException("EVP_PKEY_CTX_set_ec_paramgen_curve_nid");
+    }
+#if (OPENSSL_VERSION_NUMBER >= 0x10100000L)
+    // Must tag _explicitly_ as a named curve or many won't accept our pretty keys.
+    // If we don't do this, explicit curve parameters are included with the key,
+    // and this is not widely supported nor needed since we're generating a key on
+    // a standardized curve.
+    if (EVP_PKEY_CTX_set_ec_param_enc(params_ctx.get(), OPENSSL_EC_NAMED_CURVE) <= 0) {
+        throw CryptoException("EVP_PKEY_CTX_set_ec_param_enc");
+    }
+#endif
+    // Note: despite being an EVP_PKEY this is not an actual key, just key parameters!
+    ::EVP_PKEY* params_raw = nullptr;
+    if (::EVP_PKEY_paramgen(params_ctx.get(), &params_raw) != 1) {
+        throw CryptoException("EVP_PKEY_paramgen");
+    }
+    EvpPkeyPtr params(params_raw);
+    // Now we can create a context for the proper key generation
+    EvpPkeyCtxPtr key_ctx(::EVP_PKEY_CTX_new(params.get(), nullptr));
+    if (!params_ctx) {
+        throw CryptoException("EVP_PKEY_CTX_new");
+    }
+    if (::EVP_PKEY_keygen_init(key_ctx.get()) != 1) {
+        throw CryptoException("EVP_PKEY_keygen_init");
+    }
+    // Finally, it's time to generate the key pair itself.
+    ::EVP_PKEY* pkey_raw = nullptr;
+    if (::EVP_PKEY_keygen(key_ctx.get(), &pkey_raw) != 1) {
+        throw CryptoException("EVP_PKEY_keygen");
+    }
+    EvpPkeyPtr generated_key(pkey_raw);
+#if (OPENSSL_VERSION_NUMBER < 0x10100000L)
+    // On OpenSSL versions prior to 1.1.0, we must set the named curve ASN1 flag
+    // directly on the EC_KEY, as the EVP_PKEY wrapper doesn't exist (this is a
+    // half truth, as it exists on 1.0.2 stable, but not necessarily on all 1.0.2
+    // versions, and certainly not on 1.0.1).
+    EcKeyPtr ec_key(::EVP_PKEY_get1_EC_KEY(generated_key.get())); // Bumps ref count, needs free
+    if (!ec_key) {
+        throw CryptoException("EVP_PKEY_get1_EC_KEY");
+    }
+    ::EC_KEY_set_asn1_flag(ec_key.get(), OPENSSL_EC_NAMED_CURVE);
+#endif
+    return std::make_shared<PrivateKeyImpl>(std::move(generated_key), Type::EC);
+}
+
+namespace {
+
 void assign_random_positive_serial_number(::X509& cert) {
     /*
      * From RFC3280, section 4.1.2.2:
@@ -165,88 +235,23 @@ void add_any_subject_alternate_names(::X509& subject, ::X509& issuer,
     }
 }
 
-} // anon ns
-
-vespalib::string PrivateKey::private_to_pem() const {
-    BioPtr bio = new_memory_bio();
-    // TODO this API is const-broken even on 1.1.1, revisit in the future...
-    auto* mutable_pkey = const_cast<::EVP_PKEY*>(_pkey.get());
-    if (::PEM_write_bio_PrivateKey(bio.get(), mutable_pkey, nullptr, nullptr,
-                                   0, nullptr, nullptr) != 1) {
-        throw CryptoException("PEM_write_bio_PrivateKey");
-    }
-    return bio_to_string(*bio);
-}
-
-std::shared_ptr<PrivateKey> PrivateKey::generate_p256_ec_key() {
-    // We first have to generate an EVP context for the keygen _parameters_...
-    EvpPkeyCtxPtr params_ctx(::EVP_PKEY_CTX_new_id(EVP_PKEY_EC, nullptr));
-    if (!params_ctx) {
-        throw CryptoException("EVP_PKEY_CTX_new_id");
-    }
-    if (::EVP_PKEY_paramgen_init(params_ctx.get()) != 1) {
-        throw CryptoException("EVP_PKEY_paramgen_init");
-    }
-    // Set EC keygen parameters to use prime256v1, which is the same as P-256
-    if (EVP_PKEY_CTX_set_ec_paramgen_curve_nid(params_ctx.get(), NID_X9_62_prime256v1) <= 0) {
-        throw CryptoException("EVP_PKEY_CTX_set_ec_paramgen_curve_nid");
-    }
-#if (OPENSSL_VERSION_NUMBER >= 0x10100000L)
-    // Must tag _explicitly_ as a named curve or many won't accept our pretty keys.
-    // If we don't do this, explicit curve parameters are included with the key,
-    // and this is not widely supported nor needed since we're generating a key on
-    // a standardized curve.
-    if (EVP_PKEY_CTX_set_ec_param_enc(params_ctx.get(), OPENSSL_EC_NAMED_CURVE) <= 0) {
-        throw CryptoException("EVP_PKEY_CTX_set_ec_param_enc");
-    }
-#endif
-    // Note: despite being an EVP_PKEY this is not an actual key, just key parameters!
-    ::EVP_PKEY* params_raw = nullptr;
-    if (::EVP_PKEY_paramgen(params_ctx.get(), &params_raw) != 1) {
-        throw CryptoException("EVP_PKEY_paramgen");
-    }
-    EvpPkeyPtr params(params_raw);
-    // Now we can create a context for the proper key generation
-    EvpPkeyCtxPtr key_ctx(::EVP_PKEY_CTX_new(params.get(), nullptr));
-    if (!params_ctx) {
-        throw CryptoException("EVP_PKEY_CTX_new");
-    }
-    if (::EVP_PKEY_keygen_init(key_ctx.get()) != 1) {
-        throw CryptoException("EVP_PKEY_keygen_init");
-    }
-    // Finally, it's time to generate the key pair itself.
-    ::EVP_PKEY* pkey_raw = nullptr;
-    if (::EVP_PKEY_keygen(key_ctx.get(), &pkey_raw) != 1) {
-        throw CryptoException("EVP_PKEY_keygen");
-    }
-    EvpPkeyPtr generated_key(pkey_raw);
-#if (OPENSSL_VERSION_NUMBER < 0x10100000L)
-    // On OpenSSL versions prior to 1.1.0, we must set the named curve ASN1 flag
-    // directly on the EC_KEY, as the EVP_PKEY wrapper doesn't exist (this is a
-    // half truth, as it exists on 1.0.2 stable, but not necessarily on all 1.0.2
-    // versions, and certainly not on 1.0.1).
-    EcKeyPtr ec_key(::EVP_PKEY_get1_EC_KEY(generated_key.get())); // Bumps ref count, needs free
-    if (!ec_key) {
-        throw CryptoException("EVP_PKEY_get1_EC_KEY");
-    }
-    ::EC_KEY_set_asn1_flag(ec_key.get(), OPENSSL_EC_NAMED_CURVE);
-#endif
-    return std::make_shared<PrivateKey>(std::move(generated_key), Type::EC);
-}
+} // anonymous namespace
 
 // Some references:
 // https://stackoverflow.com/questions/256405/programmatically-create-x509-certificate-using-openssl
 // https://opensource.apple.com/source/OpenSSL/OpenSSL-22/openssl/demos/x509/mkcert.c
-std::shared_ptr<X509Certificate> X509Certificate::generate_from(Params params) {
+std::shared_ptr<X509CertificateImpl> X509CertificateImpl::generate_openssl_x509_from(Params params) {
     X509Ptr cert(::X509_new());
     if (!cert) {
         throw CryptoException("X509_new");
     }
+    // FIXME make this const, currently is not due to OpenSSL API const issues (ugh).
+    auto& subject_key_impl = dynamic_cast<PrivateKeyImpl&>(*params.subject_key);
     ::X509_set_version(cert.get(), 2); // 2 actually means v3 :)
     assign_random_positive_serial_number(*cert);
     set_certificate_expires_from_now(*cert, params.valid_for);
     // Internal key copy; does not take ownership
-    if (::X509_set_pubkey(cert.get(), params.subject_key->native_key()) != 1) {
+    if (::X509_set_pubkey(cert.get(), subject_key_impl.native_key()) != 1) {
         throw CryptoException("X509_set_pubkey");
     }
     // The "subject" is the target entity the certificate is intended to, well, certify.
@@ -259,13 +264,14 @@ std::shared_ptr<X509Certificate> X509Certificate::generate_from(Params params) {
     // If we _do_ have an issuer, we'll record its Subject name as our Issuer name.
     // Note that it's legal to have a self-signed non-CA certificate, though it obviously
     // cannot be used to sign any subordinate certificates.
-    ::X509_NAME* issuer_name = (params.issuer
-                                ? ::X509_get_subject_name(params.issuer->native_cert())
+    auto* issuer_cert_impl = dynamic_cast<X509CertificateImpl*>(params.issuer.get()); // May be nullptr.
+    ::X509_NAME* issuer_name = (issuer_cert_impl
+                                ? ::X509_get_subject_name(issuer_cert_impl->native_cert())
                                 : subj_name);
     if (::X509_set_issuer_name(cert.get(), issuer_name) != 1) { // Makes internal copy
         throw CryptoException("X509_set_issuer_name");
     }
-    ::X509& issuer_cert = params.issuer ? *params.issuer->native_cert() : *cert;
+    ::X509& issuer_cert = issuer_cert_impl ? *issuer_cert_impl->native_cert() : *cert;
 
     const char* basic_constraints = params.is_ca ? "critical,CA:TRUE" : "critical,CA:FALSE";
     const char* key_usage = params.is_ca ? "critical,keyCertSign,digitalSignature"
@@ -278,13 +284,14 @@ std::shared_ptr<X509Certificate> X509Certificate::generate_from(Params params) {
     add_v3_ext(*cert, issuer_cert, NID_authority_key_identifier, "keyid:always");
     add_any_subject_alternate_names(*cert, issuer_cert, params.subject_info.subject_alt_names);
 
-    if (::X509_sign(cert.get(), params.issuer_key->native_key(), ::EVP_sha256()) == 0) {
+    auto& issuer_key_impl = dynamic_cast<PrivateKeyImpl&>(*params.issuer_key);
+    if (::X509_sign(cert.get(), issuer_key_impl.native_key(), ::EVP_sha256()) == 0) {
         throw CryptoException("X509_sign");
     }
-    return std::make_shared<X509Certificate>(std::move(cert));
+    return std::make_shared<X509CertificateImpl>(std::move(cert));
 }
 
-vespalib::string X509Certificate::to_pem() const {
+vespalib::string X509CertificateImpl::to_pem() const {
     BioPtr bio = new_memory_bio();
     // TODO this API is const-broken, revisit in the future...
     auto* mutable_cert = const_cast<::X509*>(_cert.get());
@@ -295,47 +302,4 @@ vespalib::string X509Certificate::to_pem() const {
 }
 
 
-X509Certificate::DistinguishedName::DistinguishedName() = default;
-X509Certificate::DistinguishedName::DistinguishedName(const DistinguishedName&) = default;
-X509Certificate::DistinguishedName& X509Certificate::DistinguishedName::operator=(const DistinguishedName&) = default;
-X509Certificate::DistinguishedName::DistinguishedName(DistinguishedName&&) = default;
-X509Certificate::DistinguishedName& X509Certificate::DistinguishedName::operator=(DistinguishedName&&) = default;
-X509Certificate::DistinguishedName::~DistinguishedName() = default;
-
-X509Certificate::Params::Params() = default;
-X509Certificate::Params::~Params() = default;
-
-X509Certificate::Params
-X509Certificate::Params::self_signed(SubjectInfo subject,
-                                     std::shared_ptr<PrivateKey> key) {
-    Params params;
-    params.subject_info = std::move(subject);
-    params.subject_key = key;
-    params.issuer_key = std::move(key); // self-signed, subject == issuer
-    params.is_ca = true;
-    return params;
-}
-
-X509Certificate::Params
-X509Certificate::Params::issued_by(SubjectInfo subject,
-                                   std::shared_ptr<PrivateKey> subject_key,
-                                   std::shared_ptr<X509Certificate> issuer,
-                                   std::shared_ptr<PrivateKey> issuer_key) {
-    Params params;
-    params.subject_info = std::move(subject);
-    params.issuer = std::move(issuer);
-    params.subject_key = std::move(subject_key);
-    params.issuer_key = std::move(issuer_key);
-    params.is_ca = false; // By default, caller can change for intermediate CAs
-    return params;
-}
-
-CertKeyWrapper::CertKeyWrapper(std::shared_ptr<X509Certificate> cert_,
-                               std::shared_ptr<PrivateKey> key_)
-    : cert(std::move(cert_)),
-      key(std::move(key_))
-{}
-
-CertKeyWrapper::~CertKeyWrapper() = default;
-
 }
diff --git a/vespalib/src/vespa/vespalib/crypto/openssl_crypto_impl.h b/vespalib/src/vespa/vespalib/crypto/openssl_crypto_impl.h
new file mode 100644
index 00000000000..87e4ee14e65
--- /dev/null
+++ b/vespalib/src/vespa/vespalib/crypto/openssl_crypto_impl.h
@@ -0,0 +1,44 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/vespalib/crypto/openssl_typedefs.h>
+#include "private_key.h"
+#include "x509_certificate.h"
+
+namespace vespalib::crypto::openssl_impl {
+
+class PrivateKeyImpl : public PrivateKey {
+    EvpPkeyPtr _pkey;
+    Type _type;
+public:
+    PrivateKeyImpl(EvpPkeyPtr pkey, Type type)
+        : _pkey(std::move(pkey)),
+          _type(type)
+    {}
+    ~PrivateKeyImpl() override = default;
+
+    ::EVP_PKEY* native_key() noexcept { return _pkey.get(); }
+    const ::EVP_PKEY* native_key() const noexcept { return _pkey.get(); }
+
+    Type type() const noexcept override { return _type; }
+    vespalib::string private_to_pem() const override;
+
+    static std::shared_ptr<PrivateKeyImpl> generate_openssl_p256_ec_key();
+};
+
+class X509CertificateImpl : public X509Certificate {
+    X509Ptr _cert;
+public:
+    explicit X509CertificateImpl(X509Ptr cert) : _cert(std::move(cert)) {}
+    ~X509CertificateImpl() = default;
+
+    ::X509* native_cert() noexcept { return _cert.get(); }
+    const ::X509* native_cert() const noexcept { return _cert.get(); }
+
+    vespalib::string to_pem() const override;
+
+    // Generates an X509 certificate using a SHA-256 digest
+    static std::shared_ptr<X509CertificateImpl> generate_openssl_x509_from(Params params);
+};
+
+}
diff --git a/vespalib/src/vespa/vespalib/net/tls/impl/openssl_typedefs.h b/vespalib/src/vespa/vespalib/crypto/openssl_typedefs.h
index afafe556338..2986a4515f7 100644
--- a/vespalib/src/vespa/vespalib/net/tls/impl/openssl_typedefs.h
+++ b/vespalib/src/vespa/vespalib/crypto/openssl_typedefs.h
@@ -6,7 +6,7 @@
 #include <openssl/crypto.h>
 #include <openssl/x509.h>
 
-namespace vespalib::net::tls::impl {
+namespace vespalib::crypto {
 
 struct BioDeleter {
     void operator()(::BIO* bio) const noexcept {
diff --git a/vespalib/src/vespa/vespalib/crypto/private_key.cpp b/vespalib/src/vespa/vespalib/crypto/private_key.cpp
new file mode 100644
index 00000000000..7ece9418bef
--- /dev/null
+++ b/vespalib/src/vespa/vespalib/crypto/private_key.cpp
@@ -0,0 +1,11 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include "private_key.h"
+#include "openssl_crypto_impl.h"
+
+namespace vespalib::crypto {
+
+std::shared_ptr<PrivateKey> PrivateKey::generate_p256_ec_key() {
+    return openssl_impl::PrivateKeyImpl::generate_openssl_p256_ec_key();
+}
+
+}
diff --git a/vespalib/src/vespa/vespalib/crypto/private_key.h b/vespalib/src/vespa/vespalib/crypto/private_key.h
new file mode 100644
index 00000000000..7ac5c31502c
--- /dev/null
+++ b/vespalib/src/vespa/vespalib/crypto/private_key.h
@@ -0,0 +1,34 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <vespa/vespalib/stllike/string.h>
+#include <memory>
+
+namespace vespalib::crypto {
+
+/*
+ * Represents an asymmetric cryptographic private key.
+ *
+ * Can only be used for private/public key crypto, not for secret key (e.g. AES) crypto.
+ * Currently only supports generating EC keys on the standard P-256 curve.
+ */
+class PrivateKey {
+public:
+    enum class Type {
+        EC,
+        RSA // TODO implement support..!
+    };
+
+    virtual ~PrivateKey() = default;
+
+    virtual Type type() const noexcept = 0;
+    // TODO should have a wrapper for this that takes care to securely erase
+    // string memory on destruction.
+    virtual vespalib::string private_to_pem() const = 0;
+
+    static std::shared_ptr<PrivateKey> generate_p256_ec_key();
+protected:
+    PrivateKey() = default;
+};
+
+}
diff --git a/vespalib/src/vespa/vespalib/crypto/x509_certificate.cpp b/vespalib/src/vespa/vespalib/crypto/x509_certificate.cpp
new file mode 100644
index 00000000000..ecd061e573a
--- /dev/null
+++ b/vespalib/src/vespa/vespalib/crypto/x509_certificate.cpp
@@ -0,0 +1,62 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "x509_certificate.h"
+#include "openssl_crypto_impl.h"
+
+namespace vespalib::crypto {
+
+X509Certificate::DistinguishedName::DistinguishedName() = default;
+X509Certificate::DistinguishedName::DistinguishedName(const DistinguishedName&) = default;
+X509Certificate::DistinguishedName& X509Certificate::DistinguishedName::operator=(const DistinguishedName&) = default;
+X509Certificate::DistinguishedName::DistinguishedName(DistinguishedName&&) noexcept = default;
+X509Certificate::DistinguishedName& X509Certificate::DistinguishedName::operator=(DistinguishedName&&) noexcept = default;
+X509Certificate::DistinguishedName::~DistinguishedName() = default;
+
+X509Certificate::Params::Params() = default;
+X509Certificate::Params::~Params() = default;
+
+X509Certificate::Params::Params(const Params&) = default;
+X509Certificate::Params& X509Certificate::Params::operator=(const Params&) = default;
+X509Certificate::Params::Params(Params&&) noexcept = default;
+X509Certificate::Params& X509Certificate::Params::operator=(Params&&) noexcept = default;
+
+X509Certificate::Params
+X509Certificate::Params::self_signed(SubjectInfo subject,
+                                     std::shared_ptr<PrivateKey> key)
+{
+    Params params;
+    params.subject_info = std::move(subject);
+    params.subject_key = key;
+    params.issuer_key = std::move(key); // self-signed, subject == issuer
+    params.is_ca = true;
+    return params;
+}
+
+X509Certificate::Params
+X509Certificate::Params::issued_by(SubjectInfo subject,
+                                   std::shared_ptr<PrivateKey> subject_key,
+                                   std::shared_ptr<X509Certificate> issuer,
+                                   std::shared_ptr<PrivateKey> issuer_key)
+{
+    Params params;
+    params.subject_info = std::move(subject);
+    params.issuer = std::move(issuer);
+    params.subject_key = std::move(subject_key);
+    params.issuer_key = std::move(issuer_key);
+    params.is_ca = false; // By default, caller can change for intermediate CAs
+    return params;
+}
+
+std::shared_ptr<X509Certificate> X509Certificate::generate_from(Params params) {
+    return openssl_impl::X509CertificateImpl::generate_openssl_x509_from(std::move(params));
+}
+
+CertKeyWrapper::CertKeyWrapper(std::shared_ptr<X509Certificate> cert_,
+                               std::shared_ptr<PrivateKey> key_)
+        : cert(std::move(cert_)),
+          key(std::move(key_))
+{}
+
+CertKeyWrapper::~CertKeyWrapper() = default;
+
+}
diff --git a/vespalib/src/tests/net/tls/openssl_impl/crypto_utils.h b/vespalib/src/vespa/vespalib/crypto/x509_certificate.h
index 017dfbdbc12..9eea423c5a0 100644
--- a/vespalib/src/tests/net/tls/openssl_impl/crypto_utils.h
+++ b/vespalib/src/vespa/vespalib/crypto/x509_certificate.h
@@ -1,52 +1,32 @@
-// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 #pragma once
 
+#include "private_key.h"
 #include <vespa/vespalib/stllike/string.h>
-#include <vespa/vespalib/net/tls/impl/openssl_typedefs.h>
 #include <chrono>
 #include <memory>
 #include <vector>
 
-// TODOs
-//  - add unit testing
-//  - extend interfaces (separate PublicKey etc)
-//  - hide all OpenSSL details from header
-//  - move to appropriate new namespace/directory somewhere under vespalib
-
-namespace vespalib::net::tls::impl {
-
-class PrivateKey {
-public:
-    enum class Type {
-        EC,
-        RSA // TODO implement support..!
-    };
-private:
-    EvpPkeyPtr _pkey;
-    Type _type;
-public:
-    PrivateKey(EvpPkeyPtr pkey, Type type)
-            : _pkey(std::move(pkey)),
-              _type(type)
-    {}
-
-    ::EVP_PKEY* native_key() noexcept { return _pkey.get(); }
-    const ::EVP_PKEY* native_key() const noexcept { return _pkey.get(); }
-
-    Type type() const noexcept { return _type; }
-    vespalib::string private_to_pem() const;
-
-    static std::shared_ptr<PrivateKey> generate_p256_ec_key();
-};
-
-
+namespace vespalib::crypto {
+
+/**
+ * Represents an X509 certificate instance and provides utility methods
+ * for generating new certificates on the fly. Certificates can be created
+ * for both Certificate Authorities and regular hosts (leaves).
+ *
+ * This implementation aims to ensure that best cryptographic practices are
+ * followed automatically. In particular:
+ *   - The certificate digest is always SHA-256, never SHA-1 or MD5
+ *   - The certificate serial number is a 160-bit secure random sequence
+ *     (technically 159 bits since the MSB is always zero) rather than a
+ *     collision-prone or predictable sequence number.
+ *
+ */
 class X509Certificate {
-    X509Ptr _cert;
 public:
-    explicit X509Certificate(X509Ptr cert) : _cert(std::move(cert)) {}
+    virtual ~X509Certificate() = default;
 
-    ::X509* native_cert() noexcept { return _cert.get(); }
-    const ::X509* native_cert() const noexcept { return _cert.get(); }
+    virtual vespalib::string to_pem() const = 0;
 
     struct DistinguishedName {
         vespalib::string _country;             // "C"
@@ -61,9 +41,8 @@ public:
         DistinguishedName();
         DistinguishedName(const DistinguishedName&);
         DistinguishedName& operator=(const DistinguishedName&);
-        // TODO make these noexcept once vespalib::string has noexcept move.. or move at all!
-        DistinguishedName(DistinguishedName&&);
-        DistinguishedName& operator=(DistinguishedName&&);
+        DistinguishedName(DistinguishedName&&) noexcept;
+        DistinguishedName& operator=(DistinguishedName&&) noexcept;
         ~DistinguishedName();
 
         // TODO could add rvalue overloads as well...
@@ -101,8 +80,13 @@ public:
         Params();
         ~Params();
 
+        Params(const Params&);
+        Params& operator=(const Params&);
+        Params(Params&&) noexcept;
+        Params& operator=(Params&&) noexcept;
+
         SubjectInfo subject_info;
-        // TODO make public key, but private key has both and this is currently just for testing.
+        // TODO make public key, but private key has both.
         std::shared_ptr<PrivateKey> subject_key;
         std::shared_ptr<X509Certificate> issuer; // May be nullptr for self-signed certs
         std::shared_ptr<PrivateKey> issuer_key;
@@ -119,9 +103,14 @@ public:
 
     // Generates an X509 certificate using a SHA-256 digest
     static std::shared_ptr<X509Certificate> generate_from(Params params);
-    vespalib::string to_pem() const;
+protected:
+    X509Certificate() = default;
 };
 
+/*
+ * Simple wrapper for storing both a X509 certificate and the private key
+ * that signed it. Useful for testing.
+ */
 struct CertKeyWrapper {
     std::shared_ptr<X509Certificate> cert;
     std::shared_ptr<PrivateKey> key;
diff --git a/vespalib/src/vespa/vespalib/data/databuffer.cpp b/vespalib/src/vespa/vespalib/data/databuffer.cpp
index 758922aec6d..39c0f3f7482 100644
--- a/vespalib/src/vespa/vespalib/data/databuffer.cpp
+++ b/vespalib/src/vespa/vespalib/data/databuffer.cpp
@@ -1,6 +1,7 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 #include "databuffer.h"
 #include <algorithm>
+#include <cstdio>
 
 namespace vespalib {
 
diff --git a/vespalib/src/vespa/vespalib/data/memorydatastore.cpp b/vespalib/src/vespa/vespalib/data/memorydatastore.cpp
index 866fb7d4929..df1b68cff12 100644
--- a/vespalib/src/vespa/vespalib/data/memorydatastore.cpp
+++ b/vespalib/src/vespa/vespalib/data/memorydatastore.cpp
@@ -15,9 +15,7 @@ MemoryDataStore::MemoryDataStore(Alloc && initialAlloc, Lock * lock) :
     _buffers.emplace_back(std::move(initialAlloc));
 }
 
-MemoryDataStore::~MemoryDataStore()
-{
-}
+MemoryDataStore::~MemoryDataStore() = default;
 
 MemoryDataStore::Reference
 MemoryDataStore::push_back(const void * data, const size_t sz)
@@ -40,15 +38,14 @@ MemoryDataStore::push_back(const void * data, const size_t sz)
     return ref;
 }
 
-VariableSizeVector::VariableSizeVector(size_t initialSize) :
+VariableSizeVector::VariableSizeVector(size_t initialCount, size_t initialBufferSize) :
     _vector(),
-    _store(Alloc::alloc(initialSize))
+    _store(Alloc::alloc(initialBufferSize))
 {
+    _vector.reserve(initialCount);
 }
 
-VariableSizeVector::~VariableSizeVector()
-{
-}
+VariableSizeVector::~VariableSizeVector() = default;
 
 VariableSizeVector::Reference
 VariableSizeVector::push_back(const void * data, const size_t sz)
diff --git a/vespalib/src/vespa/vespalib/data/memorydatastore.h b/vespalib/src/vespa/vespalib/data/memorydatastore.h
index 3ef0e3ee986..2d02bfb107c 100644
--- a/vespalib/src/vespa/vespalib/data/memorydatastore.h
+++ b/vespalib/src/vespa/vespalib/data/memorydatastore.h
@@ -99,7 +99,7 @@ public:
     };
     VariableSizeVector(const VariableSizeVector &) = delete;
     VariableSizeVector & operator = (const VariableSizeVector &) = delete;
-    VariableSizeVector(size_t initialSize=256);
+    VariableSizeVector(size_t initialCount, size_t initialBufferSize);
     ~VariableSizeVector();
     iterator begin() { return iterator(_vector, 0); }
     iterator end() { return iterator(_vector, size()); }
diff --git a/vespalib/src/vespa/vespalib/data/slime/binary_format.cpp b/vespalib/src/vespa/vespalib/data/slime/binary_format.cpp
index 2febe3f4405..1956dde689c 100644
--- a/vespalib/src/vespa/vespalib/data/slime/binary_format.cpp
+++ b/vespalib/src/vespa/vespalib/data/slime/binary_format.cpp
@@ -7,10 +7,7 @@
 #include <vespa/log/log.h>
 LOG_SETUP(".vespalib.data.slime.binary_format");
 
-namespace vespalib {
-namespace slime {
-
-namespace binary_format {
+namespace vespalib::slime::binary_format {
 
 struct BinaryEncoder : public ArrayTraverser,
                        public ObjectSymbolTraverser
@@ -242,32 +239,6 @@ size_t decode(const Memory &memory, Slime &slime, const Inserter &inserter) {
     return input.failed() ? 0 : input.get_offset();
 }
 
-} // namespace vespalib::slime::binary_format
-
-void
-BinaryFormat::encode(const Slime &slime, Output &output)
-{
-    size_t chunk_size = 8000;
-    OutputWriter out(output, chunk_size);
-    binary_format::BinaryEncoder encoder(out);
-    encoder.encodeSymbolTable(slime);
-    encoder.encodeValue(slime.get());
-}
-
-size_t
-BinaryFormat::decode(const Memory &memory, Slime &slime)
-{
-    return binary_format::decode<false>(memory, slime, SlimeInserter(slime));
-}
-
-size_t
-BinaryFormat::decode_into(const Memory &memory, Slime &slime, const Inserter &inserter)
-{
-    return binary_format::decode<true>(memory, slime, inserter);
-}
-
-namespace binary_format {
-
 void
 write_cmpr_ulong(OutputWriter &out, uint64_t value) {
     out.commit(encode_cmpr_ulong(out.reserve(10), value));
@@ -288,5 +259,25 @@ write_type_and_size(OutputWriter &out, uint32_t type, uint64_t size) {
 
 }
 
-} // namespace vespalib::slime
-} // namespace vespalib
+namespace vespalib::slime {
+
+void
+BinaryFormat::encode(const Slime &slime, Output &output) {
+    size_t chunk_size = 8000;
+    OutputWriter out(output, chunk_size);
+    binary_format::BinaryEncoder encoder(out);
+    encoder.encodeSymbolTable(slime);
+    encoder.encodeValue(slime.get());
+}
+
+size_t
+BinaryFormat::decode(const Memory &memory, Slime &slime) {
+    return binary_format::decode<false>(memory, slime, SlimeInserter(slime));
+}
+
+size_t
+BinaryFormat::decode_into(const Memory &memory, Slime &slime, const Inserter &inserter) {
+    return binary_format::decode<true>(memory, slime, inserter);
+}
+
+}
diff --git a/vespalib/src/vespa/vespalib/data/slime/object_value.h b/vespalib/src/vespa/vespalib/data/slime/object_value.h
index 377bf5bd37f..651f3a156d2 100644
--- a/vespalib/src/vespa/vespalib/data/slime/object_value.h
+++ b/vespalib/src/vespa/vespalib/data/slime/object_value.h
@@ -12,8 +12,7 @@
 #include <vespa/vespalib/stllike/vector_map.h>
 #include <vespa/vespalib/util/stash.h>
 
-namespace vespalib {
-namespace slime {
+namespace vespalib::slime {
 
 /**
  * Class representing a collection of unordered values that can be
@@ -32,7 +31,7 @@ private:
 
     Cursor &setIfUnset(SymbolInserter &symbol, const ValueFactory &input) {
         Value *&pos = _fields[symbol.insert()];
-        if (pos != 0) {
+        if (pos != nullptr) {
             return *NixValue::invalid();
         }
         pos = input.create(_stash);
@@ -40,7 +39,7 @@ private:
     }
 
     Value *lookup(const SymbolLookup &symbol) const {
-        SymbolValueMap::const_iterator pos = _fields.find(symbol.lookup());
+        auto pos = _fields.find(symbol.lookup());
         if (pos == _fields.end()) {
             return NixValue::invalid();
         }
@@ -81,9 +80,7 @@ public:
     Cursor &setObject(Memory name) override;
     Symbol resolve(Memory symbol_name) override;
 
-    ~ObjectValue() { }
+    ~ObjectValue() override = default;
 };
 
-} // namespace vespalib::slime
-} // namespace vespalib
-
+}
diff --git a/vespalib/src/vespa/vespalib/data/slime/symbol_table.cpp b/vespalib/src/vespa/vespalib/data/slime/symbol_table.cpp
index 643e23b5887..a7de28932f3 100644
--- a/vespalib/src/vespa/vespalib/data/slime/symbol_table.cpp
+++ b/vespalib/src/vespa/vespalib/data/slime/symbol_table.cpp
@@ -3,15 +3,14 @@
 #include "symbol_table.h"
 #include <vespa/vespalib/stllike/hash_map.hpp>
 
-namespace vespalib {
-namespace slime {
+namespace vespalib::slime {
 
 SymbolTable::SymbolTable(size_t expectedNumSymbols) :
     _symbols(3*expectedNumSymbols),
-    _names()
+    _names(expectedNumSymbols, expectedNumSymbols*16)
 { }
 
-SymbolTable::~SymbolTable() { }
+SymbolTable::~SymbolTable() = default;
 
 void
 SymbolTable::clear() {
@@ -39,5 +38,4 @@ SymbolTable::lookup(const Memory &name) const {
     return pos->second;
 }
 
-} // namespace vespalib::slime
-} // namespace vespalib
+}
diff --git a/vespalib/src/vespa/vespalib/data/slime/symbol_table.h b/vespalib/src/vespa/vespalib/data/slime/symbol_table.h
index c726da0319b..30599d76597 100644
--- a/vespalib/src/vespa/vespalib/data/slime/symbol_table.h
+++ b/vespalib/src/vespa/vespalib/data/slime/symbol_table.h
@@ -7,8 +7,7 @@
 #include <vespa/vespalib/stllike/hash_map.h>
 #include <vespa/vespalib/data/memorydatastore.h>
 
-namespace vespalib {
-namespace slime {
+namespace vespalib::slime {
 
 /**
  * Maps between strings and symbols.
@@ -43,6 +42,5 @@ public:
     void clear();
 };
 
-} // namespace vespalib::slime
-} // namespace vespalib
+}
 
diff --git a/vespalib/src/vespa/vespalib/datastore/array_store.h b/vespalib/src/vespa/vespalib/datastore/array_store.h
index d5fef404a43..4c289c04564 100644
--- a/vespalib/src/vespa/vespalib/datastore/array_store.h
+++ b/vespalib/src/vespa/vespalib/datastore/array_store.h
@@ -25,6 +25,7 @@ template <typename EntryT, typename RefT = EntryRefT<19> >
 class ArrayStore
 {
 public:
+    using ArrayRef = vespalib::ArrayRef<EntryT>;
     using ConstArrayRef = vespalib::ConstArrayRef<EntryT>;
     using DataStoreType  = DataStoreT<RefT>;
     using SmallArrayType = BufferType<EntryT>;
@@ -82,6 +83,17 @@ public:
             return getLargeArray(internalRef);
         }
     }
+
+    /**
+     * Returns a writeable reference to the given array.
+     *
+     * NOTE: Use with care if reader threads are accessing arrays at the same time.
+     *       If so, replacing an element in the array should be an atomic operation.
+     */
+    ArrayRef get_writable(EntryRef ref) {
+        return vespalib::unconstify(get(ref));
+    }
+
     void remove(EntryRef ref);
     ICompactionContext::UP compactWorst(bool compactMemory, bool compactAddressSpace);
     vespalib::MemoryUsage getMemoryUsage() const { return _store.getMemoryUsage(); }
diff --git a/vespalib/src/vespa/vespalib/datastore/atomic_entry_ref.h b/vespalib/src/vespa/vespalib/datastore/atomic_entry_ref.h
new file mode 100644
index 00000000000..154c9669550
--- /dev/null
+++ b/vespalib/src/vespa/vespalib/datastore/atomic_entry_ref.h
@@ -0,0 +1,42 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "entryref.h"
+#include <atomic>
+
+namespace search::datastore {
+
+/**
+ * A wrapper for std::atomic of type EntryRef that supports copy and move constructors and assignment operator,
+ * and uses Release-Acquire ordering for store and load.
+ *
+ * Use this class when entry refs are stored in data stores or rcu vectors,
+ * where copy and move constructors and assignment operator are needed when resizing underlying buffers.
+ * In this case synchronization between the writer thread and reader threads
+ * is handled as part of the buffer switch.
+ */
+class AtomicEntryRef {
+private:
+    std::atomic<uint32_t> _ref;
+
+public:
+    AtomicEntryRef() noexcept : _ref() {}
+    explicit AtomicEntryRef(EntryRef ref) noexcept : _ref(ref.ref()) {}
+    AtomicEntryRef(const AtomicEntryRef& rhs) noexcept : _ref(rhs._ref.load(std::memory_order_relaxed)) {}
+    AtomicEntryRef(AtomicEntryRef&& rhs) noexcept : _ref(rhs._ref.load(std::memory_order_relaxed)) {}
+    AtomicEntryRef& operator=(const AtomicEntryRef& rhs) noexcept {
+        uint32_t ref = rhs._ref.load(std::memory_order_relaxed);
+        _ref.store(ref, std::memory_order_relaxed);
+        return *this;
+    }
+
+    void store_release(EntryRef ref) noexcept {
+        _ref.store(ref.ref(), std::memory_order_release);
+    }
+    EntryRef load_acquire() const noexcept {
+        return EntryRef(_ref.load(std::memory_order_acquire));
+    }
+};
+
+}
diff --git a/vespalib/src/vespa/vespalib/datastore/entryref.h b/vespalib/src/vespa/vespalib/datastore/entryref.h
index c65e788637f..d49b1ab9071 100644
--- a/vespalib/src/vespa/vespalib/datastore/entryref.h
+++ b/vespalib/src/vespa/vespalib/datastore/entryref.h
@@ -32,9 +32,8 @@ public:
     EntryRefT() : EntryRef() {}
     EntryRefT(size_t offset_, uint32_t bufferId_);
     EntryRefT(const EntryRef & ref_) : EntryRef(ref_.ref()) {}
-    uint32_t hash() const { return offset() + (bufferId() << OffsetBits); }
-    size_t offset() const { return _ref >> BufferBits; }
-    uint32_t bufferId() const { return _ref & (numBuffers() - 1); }
+    size_t offset() const { return _ref & (offsetSize() - 1); }
+    uint32_t bufferId() const { return _ref >> OffsetBits; }
     static size_t offsetSize() { return 1ul << OffsetBits; }
     static uint32_t numBuffers() { return 1 << BufferBits; } 
     static size_t align(size_t val) { return val; }
diff --git a/vespalib/src/vespa/vespalib/datastore/entryref.hpp b/vespalib/src/vespa/vespalib/datastore/entryref.hpp
index 6e8b94f8989..b7aafb30fdd 100644
--- a/vespalib/src/vespa/vespalib/datastore/entryref.hpp
+++ b/vespalib/src/vespa/vespalib/datastore/entryref.hpp
@@ -9,7 +9,7 @@ namespace search::datastore {
 
 template <uint32_t OffsetBits, uint32_t BufferBits>
 EntryRefT<OffsetBits, BufferBits>::EntryRefT(size_t offset_, uint32_t bufferId_) :
-    EntryRef((offset_ << BufferBits) + bufferId_)
+    EntryRef((bufferId_ << OffsetBits) + offset_)
 {
     ASSERT_ONCE_OR_LOG(offset_ < offsetSize(), "EntryRefT.offset_overflow", 10000);
     ASSERT_ONCE_OR_LOG(bufferId_ < numBuffers(), "EntryRefT.bufferId_overflow", 10000);
diff --git a/vespalib/src/vespa/vespalib/geo/zcurve.cpp b/vespalib/src/vespa/vespalib/geo/zcurve.cpp
index b2dc02759dc..00ce7ee18c6 100644
--- a/vespalib/src/vespa/vespalib/geo/zcurve.cpp
+++ b/vespalib/src/vespa/vespalib/geo/zcurve.cpp
@@ -4,8 +4,7 @@
 #include <vespa/vespalib/util/priority_queue.h>
 #include <vespa/vespalib/util/fiddle.h>
 
-namespace vespalib {
-namespace geo {
+namespace vespalib::geo {
 
 namespace {
 
@@ -182,4 +181,3 @@ ZCurve::decodeSlow(int64_t enc, int32_t *xp, int32_t *yp)
 }
 
 }
-}
diff --git a/vespalib/src/vespa/vespalib/geo/zcurve.h b/vespalib/src/vespa/vespalib/geo/zcurve.h
index 2a05c4c7744..bd76b78ea23 100644
--- a/vespalib/src/vespa/vespalib/geo/zcurve.h
+++ b/vespalib/src/vespa/vespalib/geo/zcurve.h
@@ -6,8 +6,7 @@
 #include <cassert>
 #include <vector>
 
-namespace vespalib {
-namespace geo {
+namespace vespalib::geo {
 
 /**
  * @brief Utility methods for a Z-curve (Morton-order) encoder and decoder.
@@ -31,7 +30,7 @@ public:
     public:
         BoundingBox(int32_t minx, int32_t maxx, int32_t miny, int32_t maxy);
 
-        ~BoundingBox() { }
+        ~BoundingBox() = default;
 
         int64_t getzMinx() const { return _zMinx; } 
         int64_t getzMaxx() const { return _zMaxx; } 
@@ -221,5 +220,3 @@ public:
 };
 
 }
-}
-
diff --git a/vespalib/src/vespa/vespalib/gtest/gtest.h b/vespalib/src/vespa/vespalib/gtest/gtest.h
index e5bfcf2ae55..87362687103 100644
--- a/vespalib/src/vespa/vespalib/gtest/gtest.h
+++ b/vespalib/src/vespa/vespalib/gtest/gtest.h
@@ -14,3 +14,15 @@ main(int argc, char* argv[])                \
     ::testing::InitGoogleTest(&argc, argv); \
     return RUN_ALL_TESTS();                 \
 }
+
+#ifdef INSTANTIATE_TEST_SUITE_P
+#define VESPA_GTEST_INSTANTIATE_TEST_SUITE_P INSTANTIATE_TEST_SUITE_P
+#else
+#define VESPA_GTEST_INSTANTIATE_TEST_SUITE_P INSTANTIATE_TEST_CASE_P
+#endif
+
+#ifdef TYPED_TEST_SUITE
+#define VESPA_GTEST_TYPED_TEST_SUITE TYPED_TEST_SUITE
+#else
+#define VESPA_GTEST_TYPED_TEST_SUITE TYPED_TEST_CASE
+#endif
diff --git a/vespalib/src/vespa/vespalib/hwaccelrated/CMakeLists.txt b/vespalib/src/vespa/vespalib/hwaccelrated/CMakeLists.txt
index f6b38028471..ac9d8d76074 100644
--- a/vespalib/src/vespa/vespalib/hwaccelrated/CMakeLists.txt
+++ b/vespalib/src/vespa/vespalib/hwaccelrated/CMakeLists.txt
@@ -3,8 +3,6 @@ vespa_add_library(vespalib_vespalib_hwaccelrated OBJECT
     SOURCES
     iaccelrated.cpp
     generic.cpp
-    sse2.cpp
-    avx.cpp
     avx2.cpp
     avx512.cpp
     DEPENDS
diff --git a/vespalib/src/vespa/vespalib/hwaccelrated/avx.cpp b/vespalib/src/vespa/vespalib/hwaccelrated/avx.cpp
deleted file mode 100644
index 39ea0d2d73b..00000000000
--- a/vespalib/src/vespa/vespalib/hwaccelrated/avx.cpp
+++ /dev/null
@@ -1,25 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#include "avx.h"
-#include "avxprivate.hpp"
-
-namespace vespalib::hwaccelrated {
-
-float
-AvxAccelrator::dotProduct(const float * af, const float * bf, size_t sz) const
-{
-    return avx::dotProductSelectAlignment<float, 32>(af, bf, sz);
-}
-
-double
-AvxAccelrator::dotProduct(const double * af, const double * bf, size_t sz) const
-{
-    return avx::dotProductSelectAlignment<double, 32>(af, bf, sz);
-}
-
-size_t
-AvxAccelrator::populationCount(const uint64_t *a, size_t sz) const {
-    return helper::populationCount(a, sz);
-}
-
-}
diff --git a/vespalib/src/vespa/vespalib/hwaccelrated/avx.h b/vespalib/src/vespa/vespalib/hwaccelrated/avx.h
deleted file mode 100644
index 624531a9ca5..00000000000
--- a/vespalib/src/vespa/vespalib/hwaccelrated/avx.h
+++ /dev/null
@@ -1,20 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#pragma once
-
-#include "sse2.h"
-
-namespace vespalib::hwaccelrated {
-
-/**
- * Avx-256 implementation.
- */
-class AvxAccelrator : public Sse2Accelrator
-{
-public:
-    float dotProduct(const float * a, const float * b, size_t sz) const override;
-    double dotProduct(const double * a, const double * b, size_t sz) const override;
-    size_t populationCount(const uint64_t *a, size_t sz) const override;
-};
-
-}
diff --git a/vespalib/src/vespa/vespalib/hwaccelrated/avx2.cpp b/vespalib/src/vespa/vespalib/hwaccelrated/avx2.cpp
index ea8a3ead538..7ff393c87f8 100644
--- a/vespalib/src/vespa/vespalib/hwaccelrated/avx2.cpp
+++ b/vespalib/src/vespa/vespalib/hwaccelrated/avx2.cpp
@@ -5,21 +5,19 @@
 
 namespace vespalib::hwaccelrated {
 
-float
-Avx2Accelrator::dotProduct(const float * af, const float * bf, size_t sz) const
-{
-    return avx::dotProductSelectAlignment<float, 32>(af, bf, sz);
+size_t
+Avx2Accelrator::populationCount(const uint64_t *a, size_t sz) const {
+    return helper::populationCount(a, sz);
 }
 
 double
-Avx2Accelrator::dotProduct(const double * af, const double * bf, size_t sz) const
-{
-    return avx::dotProductSelectAlignment<double, 32>(af, bf, sz);
+Avx2Accelrator::squaredEuclideanDistance(const float * a, const float * b, size_t sz) const {
+    return avx::euclideanDistanceSelectAlignment<float, 32>(a, b, sz);
 }
 
-size_t
-Avx2Accelrator::populationCount(const uint64_t *a, size_t sz) const {
-    return helper::populationCount(a, sz);
+double
+Avx2Accelrator::squaredEuclideanDistance(const double * a, const double * b, size_t sz) const {
+    return avx::euclideanDistanceSelectAlignment<double, 32>(a, b, sz);
 }
 
 }
diff --git a/vespalib/src/vespa/vespalib/hwaccelrated/avx2.h b/vespalib/src/vespa/vespalib/hwaccelrated/avx2.h
index cf91bc81cfd..3e0dbb28110 100644
--- a/vespalib/src/vespa/vespalib/hwaccelrated/avx2.h
+++ b/vespalib/src/vespa/vespalib/hwaccelrated/avx2.h
@@ -2,19 +2,19 @@
 
 #pragma once
 
-#include "avx.h"
+#include "generic.h"
 
 namespace vespalib::hwaccelrated {
 
 /**
  * Avx-512 implementation.
  */
-class Avx2Accelrator : public AvxAccelrator
+class Avx2Accelrator : public GenericAccelrator
 {
 public:
-    float dotProduct(const float * a, const float * b, size_t sz) const override;
-    double dotProduct(const double * a, const double * b, size_t sz) const override;
     size_t populationCount(const uint64_t *a, size_t sz) const override;
+    double squaredEuclideanDistance(const float * a, const float * b, size_t sz) const override;
+    double squaredEuclideanDistance(const double * a, const double * b, size_t sz) const override;
 };
 
 }
diff --git a/vespalib/src/vespa/vespalib/hwaccelrated/avx512.cpp b/vespalib/src/vespa/vespalib/hwaccelrated/avx512.cpp
index 1abf6b270cf..0941e6d6ad8 100644
--- a/vespalib/src/vespa/vespalib/hwaccelrated/avx512.cpp
+++ b/vespalib/src/vespa/vespalib/hwaccelrated/avx512.cpp
@@ -22,4 +22,14 @@ Avx512Accelrator::populationCount(const uint64_t *a, size_t sz) const {
     return helper::populationCount(a, sz);
 }
 
+double
+Avx512Accelrator::squaredEuclideanDistance(const float * a, const float * b, size_t sz) const {
+    return avx::euclideanDistanceSelectAlignment<float, 64>(a, b, sz);
+}
+
+double
+Avx512Accelrator::squaredEuclideanDistance(const double * a, const double * b, size_t sz) const {
+    return avx::euclideanDistanceSelectAlignment<double, 64>(a, b, sz);
+}
+
 }
diff --git a/vespalib/src/vespa/vespalib/hwaccelrated/avx512.h b/vespalib/src/vespa/vespalib/hwaccelrated/avx512.h
index eac8c96832b..209ec06c857 100644
--- a/vespalib/src/vespa/vespalib/hwaccelrated/avx512.h
+++ b/vespalib/src/vespa/vespalib/hwaccelrated/avx512.h
@@ -15,6 +15,8 @@ public:
     float dotProduct(const float * a, const float * b, size_t sz) const override;
     double dotProduct(const double * a, const double * b, size_t sz) const override;
     size_t populationCount(const uint64_t *a, size_t sz) const override;
+    double squaredEuclideanDistance(const float * a, const float * b, size_t sz) const override;
+    double squaredEuclideanDistance(const double * a, const double * b, size_t sz) const override;
 };
 
 }
diff --git a/vespalib/src/vespa/vespalib/hwaccelrated/avxprivate.hpp b/vespalib/src/vespa/vespalib/hwaccelrated/avxprivate.hpp
index 9e6a6d8817f..087729a23b2 100644
--- a/vespalib/src/vespa/vespalib/hwaccelrated/avxprivate.hpp
+++ b/vespalib/src/vespa/vespalib/hwaccelrated/avxprivate.hpp
@@ -4,7 +4,6 @@
 
 #include "private_helpers.hpp"
 #include <vespa/fastos/dynamiclibrary.h>
-#include <cstring>
 
 namespace vespalib::hwaccelrated::avx {
 
@@ -87,4 +86,52 @@ T dotProductSelectAlignment(const T * af, const T * bf, size_t sz)
     }
 }
 
+template <typename T, unsigned VLEN, unsigned AlignA, unsigned AlignB>
+double
+euclideanDistanceT(const T * af, const T * bf, size_t sz)
+{
+    constexpr unsigned VectorsPerChunk = 4;
+    constexpr unsigned ChunkSize = VLEN*VectorsPerChunk/sizeof(T);
+    typedef T V __attribute__ ((vector_size (VLEN)));
+    typedef T A __attribute__ ((vector_size (VLEN), aligned(AlignA)));
+    typedef T B __attribute__ ((vector_size (VLEN), aligned(AlignB)));
+    V partial[VectorsPerChunk];
+    memset(partial, 0, sizeof(partial));
+    const A * a = reinterpret_cast<const A *>(af);
+    const B * b = reinterpret_cast<const B *>(bf);
+
+    const size_t numChunks(sz/ChunkSize);
+    for (size_t i(0); i < numChunks; i++) {
+        for (size_t j(0); j < VectorsPerChunk; j++) {
+            partial[j] += (a[VectorsPerChunk*i+j] - b[VectorsPerChunk*i+j]) * (a[VectorsPerChunk*i+j] - b[VectorsPerChunk*i+j]);
+        }
+    }
+    double sum(0);
+    for (size_t i(numChunks*ChunkSize); i < sz; i++) {
+        sum += (af[i] - bf[i]) * (af[i] - bf[i]);
+    }
+    partial[0] = sumR<V, VectorsPerChunk>(partial);
+
+    return sum + sumT<T, V>(partial[0]);
+}
+
+template <typename T, unsigned VLEN>
+double euclideanDistanceSelectAlignment(const T * af, const T * bf, size_t sz)
+{
+    constexpr unsigned ALIGN = 32;
+    if (validAlignment(af, ALIGN)) {
+        if (validAlignment(bf, ALIGN)) {
+            return euclideanDistanceT<T, VLEN, ALIGN, ALIGN>(af, bf, sz);
+        } else {
+            return euclideanDistanceT<T, ALIGN, ALIGN, 1>(af, bf, sz);
+        }
+    } else {
+        if (validAlignment(bf, ALIGN)) {
+            return euclideanDistanceT<T, VLEN, 1, ALIGN>(af, bf, sz);
+        } else {
+            return euclideanDistanceT<T, VLEN, 1, 1>(af, bf, sz);
+        }
+    }
+}
+
 }
diff --git a/vespalib/src/vespa/vespalib/hwaccelrated/generic.cpp b/vespalib/src/vespa/vespalib/hwaccelrated/generic.cpp
index b70ebb4051a..f9684e88c63 100644
--- a/vespalib/src/vespa/vespalib/hwaccelrated/generic.cpp
+++ b/vespalib/src/vespa/vespalib/hwaccelrated/generic.cpp
@@ -2,6 +2,7 @@
 
 #include "generic.h"
 #include "private_helpers.hpp"
+#include <cblas.h>
 
 namespace vespalib::hwaccelrated {
 
@@ -31,14 +32,38 @@ multiplyAdd(const T * a, const T * b, size_t sz)
     return sum;
 }
 
+template <typename T, size_t UNROLL>
+double
+euclideanDistanceT(const T * a, const T * b, size_t sz)
+{
+    T partial[UNROLL];
+    for (size_t i(0); i < UNROLL; i++) {
+        partial[i] = 0;
+    }
+    size_t i(0);
+    for (; i + UNROLL <= sz; i += UNROLL) {
+        for (size_t j(0); j < UNROLL; j++) {
+            partial[j] += (a[i+j] - b[i+j]) * (a[i+j] - b[i+j]);
+        }
+    }
+    for (;i < sz; i++) {
+        partial[i%UNROLL] += (a[i] - b[i]) * (a[i] - b[i]);
+    }
+    double sum(0);
+    for (size_t j(0); j < UNROLL; j++) {
+        sum += partial[j];
+    }
+    return sum;
+}
+
 template<size_t UNROLL, typename Operation>
 void
 bitOperation(Operation operation, void * aOrg, const void * bOrg, size_t bytes) {
 
     const size_t sz(bytes/sizeof(uint64_t));
     {
-        uint64_t *a(static_cast<uint64_t *>(aOrg));
-        const uint64_t *b(static_cast<const uint64_t *>(bOrg));
+        auto a(static_cast<uint64_t *>(aOrg));
+        auto b(static_cast<const uint64_t *>(bOrg));
         size_t i(0);
         for (; i + UNROLL <= sz; i += UNROLL) {
             for (size_t j(0); j < UNROLL; j++) {
@@ -50,8 +75,8 @@ bitOperation(Operation operation, void * aOrg, const void * bOrg, size_t bytes)
         }
     }
 
-    uint8_t *a(static_cast<uint8_t *>(aOrg));
-    const uint8_t *b(static_cast<const uint8_t *>(bOrg));
+    auto a(static_cast<uint8_t *>(aOrg));
+    auto b(static_cast<const uint8_t *>(bOrg));
     for (size_t i(sz*sizeof(uint64_t)); i < bytes; i++) {
         a[i] = operation(a[i], b[i]);
     }
@@ -62,36 +87,36 @@ bitOperation(Operation operation, void * aOrg, const void * bOrg, size_t bytes)
 float
 GenericAccelrator::dotProduct(const float * a, const float * b, size_t sz) const
 {
-    return multiplyAdd<float, float, 4>(a, b, sz);
+    return cblas_sdot(sz, a, 1, b, 1);
 }
 
 double
 GenericAccelrator::dotProduct(const double * a, const double * b, size_t sz) const
 {
-    return multiplyAdd<double, double, 4>(a, b, sz);
+    return cblas_ddot(sz, a, 1, b, 1);
 }
 
 int64_t
 GenericAccelrator::dotProduct(const int8_t * a, const int8_t * b, size_t sz) const
 {
-    return multiplyAdd<int64_t, int8_t, 4>(a, b, sz);
+    return multiplyAdd<int64_t, int8_t, 8>(a, b, sz);
 }
 
 int64_t
 GenericAccelrator::dotProduct(const int16_t * a, const int16_t * b, size_t sz) const
 {
-    return multiplyAdd<int64_t, int16_t, 4>(a, b, sz);
+    return multiplyAdd<int64_t, int16_t, 8>(a, b, sz);
 }
 int64_t
 GenericAccelrator::dotProduct(const int32_t * a, const int32_t * b, size_t sz) const
 {
-    return multiplyAdd<int64_t, int32_t, 4>(a, b, sz);
+    return multiplyAdd<int64_t, int32_t, 8>(a, b, sz);
 }
 
 long long
 GenericAccelrator::dotProduct(const int64_t * a, const int64_t * b, size_t sz) const
 {
-    return multiplyAdd<long long, int64_t, 4>(a, b, sz);
+    return multiplyAdd<long long, int64_t, 8>(a, b, sz);
 }
 
 void
@@ -114,12 +139,12 @@ GenericAccelrator::andNotBit(void * aOrg, const void * bOrg, size_t bytes) const
 void
 GenericAccelrator::notBit(void * aOrg, size_t bytes) const
 {
-    uint64_t *a(static_cast<uint64_t *>(aOrg));
+    auto a(static_cast<uint64_t *>(aOrg));
     const size_t sz(bytes/sizeof(uint64_t));
     for (size_t i(0); i < sz; i++) {
         a[i] = ~a[i];
     }
-    uint8_t *ac(static_cast<uint8_t *>(aOrg));
+    auto ac(static_cast<uint8_t *>(aOrg));
     for (size_t i(sz*sizeof(uint64_t)); i < bytes; i++) {
         ac[i] = ~ac[i];
     }
@@ -130,4 +155,14 @@ GenericAccelrator::populationCount(const uint64_t *a, size_t sz) const {
     return helper::populationCount(a, sz);
 }
 
+double
+GenericAccelrator::squaredEuclideanDistance(const float * a, const float * b, size_t sz) const {
+    return euclideanDistanceT<float, 8>(a, b, sz);
+}
+
+double
+GenericAccelrator::squaredEuclideanDistance(const double * a, const double * b, size_t sz) const {
+    return euclideanDistanceT<double, 4>(a, b, sz);
+}
+
 }
diff --git a/vespalib/src/vespa/vespalib/hwaccelrated/generic.h b/vespalib/src/vespa/vespalib/hwaccelrated/generic.h
index d76d0728bdd..50a3d59d49d 100644
--- a/vespalib/src/vespa/vespalib/hwaccelrated/generic.h
+++ b/vespalib/src/vespa/vespalib/hwaccelrated/generic.h
@@ -23,6 +23,8 @@ public:
     void andNotBit(void * a, const void * b, size_t bytes) const override;
     void notBit(void * a, size_t bytes) const override;
     size_t populationCount(const uint64_t *a, size_t sz) const override;
+    double squaredEuclideanDistance(const float * a, const float * b, size_t sz) const override;
+    double squaredEuclideanDistance(const double * a, const double * b, size_t sz) const override;
 };
 
 }
diff --git a/vespalib/src/vespa/vespalib/hwaccelrated/iaccelrated.cpp b/vespalib/src/vespa/vespalib/hwaccelrated/iaccelrated.cpp
index 4006897dce5..bb132165e53 100644
--- a/vespalib/src/vespa/vespalib/hwaccelrated/iaccelrated.cpp
+++ b/vespalib/src/vespa/vespalib/hwaccelrated/iaccelrated.cpp
@@ -2,11 +2,11 @@
 
 #include "iaccelrated.h"
 #include "generic.h"
-#include "sse2.h"
-#include "avx.h"
 #include "avx2.h"
 #include "avx512.h"
 #include <vespa/vespalib/util/memory.h>
+#include <cstdio>
+#include <vector>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".vespalib.hwaccelrated");
@@ -17,7 +17,7 @@ namespace {
 
 class Factory {
 public:
-    virtual ~Factory() { }
+    virtual ~Factory() = default;
     virtual IAccelrated::UP create() const = 0;
 };
 
@@ -26,16 +26,6 @@ public:
     IAccelrated::UP create() const override { return std::make_unique<GenericAccelrator>(); }
 };
 
-class Sse2Factory :public Factory{
-public:
-    IAccelrated::UP create() const override { return std::make_unique<Sse2Accelrator>(); }
-};
-
-class AvxFactory :public Factory{
-public:
-    IAccelrated::UP create() const override { return std::make_unique<AvxAccelrator>(); }
-};
-
 class Avx2Factory :public Factory{
 public:
     IAccelrated::UP create() const override { return std::make_unique<Avx2Accelrator>(); }
@@ -47,16 +37,25 @@ public:
 };
 
 template<typename T>
-void verifyAccelrator(const IAccelrated & accel)
+std::vector<T> createAndFill(size_t sz) {
+    std::vector<T> v(sz);
+    for (size_t i(0); i < sz; i++) {
+        v[i] = rand()%100;
+    }
+    return v;
+}
+
+template<typename T>
+void verifyDotproduct(const IAccelrated & accel)
 {
-    const size_t testLength(127);
-    T * a = new T[testLength];
-    T * b = new T[testLength];
+    const size_t testLength(255);
+    srand(1);
+    std::vector<T> a = createAndFill<T>(testLength);
+    std::vector<T> b = createAndFill<T>(testLength);
     for (size_t j(0); j < 0x20; j++) {
         T sum(0);
         for (size_t i(j); i < testLength; i++) {
-            a[i] = b[i] = i;
-            sum += i*i;
+            sum += a[i]*b[i];
         }
         T hwComputedSum(accel.dotProduct(&a[j], &b[j], testLength - j));
         if (sum != hwComputedSum) {
@@ -64,8 +63,25 @@ void verifyAccelrator(const IAccelrated & accel)
             LOG_ABORT("should not be reached");
         }
     }
-    delete [] a;
-    delete [] b;
+}
+
+template<typename T>
+void verifyEuclideanDistance(const IAccelrated & accel) {
+    const size_t testLength(255);
+    srand(1);
+    std::vector<T> a = createAndFill<T>(testLength);
+    std::vector<T> b = createAndFill<T>(testLength);
+    for (size_t j(0); j < 0x20; j++) {
+        T sum(0);
+        for (size_t i(j); i < testLength; i++) {
+            sum += (a[i] - b[i]) * (a[i] - b[i]);
+        }
+        T hwComputedSum(accel.squaredEuclideanDistance(&a[j], &b[j], testLength - j));
+        if (sum != hwComputedSum) {
+            fprintf(stderr, "Accelrator is not computing euclidean distance correctly.\n");
+            LOG_ABORT("should not be reached");
+        }
+    }
 }
 
 void verifyPopulationCount(const IAccelrated & accel)
@@ -89,23 +105,25 @@ class RuntimeVerificator
 {
 public:
     RuntimeVerificator();
+private:
+    void verify(const IAccelrated & accelrated) {
+        verifyDotproduct<float>(accelrated);
+        verifyDotproduct<double>(accelrated);
+        verifyDotproduct<int32_t>(accelrated);
+        verifyDotproduct<int64_t>(accelrated);
+        verifyEuclideanDistance<float>(accelrated);
+        verifyEuclideanDistance<double>(accelrated);
+        verifyPopulationCount(accelrated);
+    }
 };
 
 RuntimeVerificator::RuntimeVerificator()
 {
-   GenericAccelrator generic;
-   verifyAccelrator<float>(generic); 
-   verifyAccelrator<double>(generic); 
-   verifyAccelrator<int32_t>(generic); 
-   verifyAccelrator<int64_t>(generic);
-   verifyPopulationCount(generic);
-
-   IAccelrated::UP thisCpu(IAccelrated::getAccelrator());
-   verifyAccelrator<float>(*thisCpu); 
-   verifyAccelrator<double>(*thisCpu); 
-   verifyAccelrator<int32_t>(*thisCpu); 
-   verifyAccelrator<int64_t>(*thisCpu); 
-   
+    GenericAccelrator generic;
+    verify(generic);
+
+    const IAccelrated & thisCpu(IAccelrated::getAccelrator());
+    verify(thisCpu);
 }
 
 class Selector
@@ -118,17 +136,15 @@ private:
 };
 
 Selector::Selector() :
-    _factory(new GenericFactory())
+    _factory()
 {
     __builtin_cpu_init ();
     if (__builtin_cpu_supports("avx512f")) {
-        _factory.reset(new Avx512Factory());
+        _factory = std::make_unique<Avx512Factory>();
     } else if (__builtin_cpu_supports("avx2")) {
-        _factory.reset(new Avx2Factory());
-    } else if (__builtin_cpu_supports("avx")) {
-        _factory.reset(new AvxFactory());
-    } else if (__builtin_cpu_supports("sse2")) {
-        _factory.reset(new Sse2Factory());
+        _factory = std::make_unique<Avx2Factory>();
+    } else {
+        _factory = std::make_unique<GenericFactory>();
     }
 }
 
@@ -138,11 +154,11 @@ static Selector _G_selector;
 
 RuntimeVerificator _G_verifyAccelrator;
 
-
-IAccelrated::UP
+const IAccelrated &
 IAccelrated::getAccelrator()
 {
-    return _G_selector.create();
+    static IAccelrated::UP accelrator = _G_selector.create();
+    return *accelrator;
 }
 
 }
diff --git a/vespalib/src/vespa/vespalib/hwaccelrated/iaccelrated.h b/vespalib/src/vespa/vespalib/hwaccelrated/iaccelrated.h
index 4031169c44d..0292ad14643 100644
--- a/vespalib/src/vespa/vespalib/hwaccelrated/iaccelrated.h
+++ b/vespalib/src/vespa/vespalib/hwaccelrated/iaccelrated.h
@@ -27,8 +27,10 @@ public:
     virtual void andNotBit(void * a, const void * b, size_t bytes) const = 0;
     virtual void notBit(void * a, size_t bytes) const = 0;
     virtual size_t populationCount(const uint64_t *a, size_t sz) const = 0;
+    virtual double squaredEuclideanDistance(const float * a, const float * b, size_t sz) const = 0;
+    virtual double squaredEuclideanDistance(const double * a, const double * b, size_t sz) const = 0;
 
-    static IAccelrated::UP getAccelrator() __attribute__((noinline));
+    static const IAccelrated & getAccelrator() __attribute__((noinline));
 };
 
 }
diff --git a/vespalib/src/vespa/vespalib/hwaccelrated/private_helpers.hpp b/vespalib/src/vespa/vespalib/hwaccelrated/private_helpers.hpp
index 8eba313d5f1..f5daf2b9081 100644
--- a/vespalib/src/vespa/vespalib/hwaccelrated/private_helpers.hpp
+++ b/vespalib/src/vespa/vespalib/hwaccelrated/private_helpers.hpp
@@ -3,6 +3,7 @@
 #pragma once
 
 #include <vespa/vespalib/util/optimized.h>
+#include <cstring>
 
 namespace vespalib::hwaccelrated::helper {
 namespace {
@@ -24,4 +25,4 @@ populationCount(const uint64_t *a, size_t sz) {
 }
 
 }
-}
\ No newline at end of file
+}
diff --git a/vespalib/src/vespa/vespalib/hwaccelrated/sse2.cpp b/vespalib/src/vespa/vespalib/hwaccelrated/sse2.cpp
deleted file mode 100644
index a0f584f8a9f..00000000000
--- a/vespalib/src/vespa/vespalib/hwaccelrated/sse2.cpp
+++ /dev/null
@@ -1,85 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#include "sse2.h"
-#include "private_helpers.hpp"
-
-namespace vespalib::hwaccelrated {
-
-namespace {
-
-bool validAlignment16(const void * p) {
-    return (reinterpret_cast<uint64_t>(p) & 0xful) == 0;
-}
-
-bool validAlignment16(const void * a, const void * b) {
-    return validAlignment16(a) && validAlignment16(b);
-}
-
-}
-
-float
-Sse2Accelrator::dotProduct(const float * af, const float * bf, size_t sz) const
-{
-    if ( ! validAlignment16(af, bf)) {
-        return GenericAccelrator::dotProduct(af, bf, sz);
-    }
-    typedef float v4sf __attribute__ ((vector_size (16)));
-    const size_t ChunkSize(16);
-    const size_t VectorsPerChunk(ChunkSize/4);
-    v4sf partial[VectorsPerChunk] = { {0.0, 0.0, 0.0, 0.0}, {0.0, 0.0, 0.0, 0.0}, {0.0, 0.0, 0.0, 0.0}, {0.0, 0.0, 0.0, 0.0} };
-    const v4sf * a = reinterpret_cast<const v4sf *>(af);
-    const v4sf * b = reinterpret_cast<const v4sf *>(bf);
-
-    const size_t numChunks(sz/ChunkSize);
-    for (size_t i(0); i < numChunks; i++) {
-        for (size_t j(0); j < VectorsPerChunk; j++) {
-            partial[j] += a[VectorsPerChunk*i+j] * b[VectorsPerChunk*i+j];
-        }
-    }
-    float sum(0);
-    for (size_t i(numChunks*ChunkSize); i < sz; i++) {
-        sum += af[i] * bf[i];
-    }
-    for (size_t i(1); i < VectorsPerChunk; i++) {
-        partial[0] += partial[i];
-    }
-    sum += partial[0][0] + partial[0][1] + partial[0][2] + partial[0][3];
-    return sum; 
-}
-
-double
-Sse2Accelrator::dotProduct(const double * af, const double * bf, size_t sz) const
-{
-    if ( ! validAlignment16(af, bf)) {
-        return GenericAccelrator::dotProduct(af, bf, sz);
-    }
-    typedef double v2sd __attribute__ ((vector_size (16)));
-    const size_t ChunkSize(8);
-    const size_t VectorsPerChunk(ChunkSize/2);
-    v2sd partial[VectorsPerChunk] = { {0.0, 0.0}, {0.0, 0.0}, {0.0, 0.0}, {0.0, 0.0} };
-    const v2sd * a = reinterpret_cast<const v2sd *>(af);
-    const v2sd * b = reinterpret_cast<const v2sd *>(bf);
-
-    const size_t numChunks(sz/ChunkSize);
-    for (size_t i(0); i < numChunks; i++) {
-        for (size_t j(0); j < VectorsPerChunk; j++) {
-            partial[j] += a[VectorsPerChunk*i+j] * b[VectorsPerChunk*i+j];
-        }
-    }
-    double sum(0);
-    for (size_t i(numChunks*ChunkSize); i < sz; i++) {
-        sum += af[i] * bf[i];
-    }
-    for (size_t i(1); i < VectorsPerChunk; i++) {
-        partial[0] += partial[i];
-    }
-    sum += partial[0][0] + partial[0][1];
-    return sum; 
-}
-
-size_t
-Sse2Accelrator::populationCount(const uint64_t *a, size_t sz) const {
-    return helper::populationCount(a, sz);
-}
-
-}
diff --git a/vespalib/src/vespa/vespalib/hwaccelrated/sse2.h b/vespalib/src/vespa/vespalib/hwaccelrated/sse2.h
deleted file mode 100644
index d0fbefe5f03..00000000000
--- a/vespalib/src/vespa/vespalib/hwaccelrated/sse2.h
+++ /dev/null
@@ -1,21 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#pragma once
-
-#include "generic.h"
-
-namespace vespalib::hwaccelrated {
-
-/**
- * Generic cpu agnostic implementation.
- */
-class Sse2Accelrator : public GenericAccelrator
-{
-public:
-    float dotProduct(const float * a, const float * b, size_t sz) const override;
-    double dotProduct(const double * a, const double * b, size_t sz) const override;
-
-    size_t populationCount(const uint64_t *a, size_t sz) const override;
-};
-
-}
diff --git a/vespalib/src/vespa/vespalib/net/crypto_engine.cpp b/vespalib/src/vespa/vespalib/net/crypto_engine.cpp
index a52d4eeb690..7bbc4b7523c 100644
--- a/vespalib/src/vespa/vespalib/net/crypto_engine.cpp
+++ b/vespalib/src/vespa/vespalib/net/crypto_engine.cpp
@@ -2,9 +2,9 @@
 
 #include "crypto_engine.h"
 #include <vespa/vespalib/data/smart_buffer.h>
+#include <vespa/vespalib/crypto/crypto_exception.h>
 #include <vespa/vespalib/net/tls/authorization_mode.h>
 #include <vespa/vespalib/net/tls/auto_reloading_tls_crypto_engine.h>
-#include <vespa/vespalib/net/tls/crypto_exception.h>
 #include <vespa/vespalib/net/tls/maybe_tls_crypto_engine.h>
 #include <vespa/vespalib/net/tls/statistics.h>
 #include <vespa/vespalib/net/tls/tls_crypto_engine.h>
@@ -232,7 +232,7 @@ CryptoEngine::SP create_default_crypto_engine() {
 CryptoEngine::SP try_create_default_crypto_engine() {
     try {
         return create_default_crypto_engine();
-    } catch (net::tls::CryptoException &e) {
+    } catch (crypto::CryptoException &e) {
         LOG(error, "failed to create default crypto engine: %s", e.what());
         std::_Exit(78);
     }
@@ -250,16 +250,29 @@ CryptoEngine::get_default()
 }
 
 CryptoSocket::UP
-NullCryptoEngine::create_crypto_socket(SocketHandle socket, bool is_server)
+NullCryptoEngine::create_client_crypto_socket(SocketHandle socket, const SocketSpec &)
 {
-    net::tls::ConnectionStatistics::get(is_server).inc_insecure_connections();
+    net::tls::ConnectionStatistics::get(false).inc_insecure_connections();
     return std::make_unique<NullCryptoSocket>(std::move(socket));
 }
 
 CryptoSocket::UP
-XorCryptoEngine::create_crypto_socket(SocketHandle socket, bool is_server)
+NullCryptoEngine::create_server_crypto_socket(SocketHandle socket)
 {
-    return std::make_unique<XorCryptoSocket>(std::move(socket), is_server);
+    net::tls::ConnectionStatistics::get(true).inc_insecure_connections();
+    return std::make_unique<NullCryptoSocket>(std::move(socket));
+}
+
+CryptoSocket::UP
+XorCryptoEngine::create_client_crypto_socket(SocketHandle socket, const SocketSpec &)
+{
+    return std::make_unique<XorCryptoSocket>(std::move(socket), false);
+}
+
+CryptoSocket::UP
+XorCryptoEngine::create_server_crypto_socket(SocketHandle socket)
+{
+    return std::make_unique<XorCryptoSocket>(std::move(socket), true);
 }
 
 } // namespace vespalib
diff --git a/vespalib/src/vespa/vespalib/net/crypto_engine.h b/vespalib/src/vespa/vespalib/net/crypto_engine.h
index 1cb1305e039..71511b8a552 100644
--- a/vespalib/src/vespa/vespalib/net/crypto_engine.h
+++ b/vespalib/src/vespa/vespalib/net/crypto_engine.h
@@ -9,6 +9,8 @@
 
 namespace vespalib {
 
+class SocketSpec;
+
 /**
  * Component responsible for wrapping low-level sockets into
  * appropriate CryptoSocket instances. This is the top-level interface
@@ -17,7 +19,10 @@ namespace vespalib {
  **/
 struct CryptoEngine {
     using SP = std::shared_ptr<CryptoEngine>;
-    virtual CryptoSocket::UP create_crypto_socket(SocketHandle socket, bool is_server) = 0;
+    virtual bool use_tls_when_client() const = 0;
+    virtual bool always_use_tls_when_server() const = 0;
+    virtual CryptoSocket::UP create_client_crypto_socket(SocketHandle socket, const SocketSpec &spec) = 0;
+    virtual CryptoSocket::UP create_server_crypto_socket(SocketHandle socket) = 0;
     virtual ~CryptoEngine();
     static CryptoEngine::SP get_default();
 };
@@ -26,7 +31,10 @@ struct CryptoEngine {
  * Crypto engine without encryption.
  **/
 struct NullCryptoEngine : public CryptoEngine {
-    CryptoSocket::UP create_crypto_socket(SocketHandle socket, bool is_server) override;
+    bool use_tls_when_client() const override { return false; }
+    bool always_use_tls_when_server() const override { return false; }
+    CryptoSocket::UP create_client_crypto_socket(SocketHandle socket, const SocketSpec &spec) override;
+    CryptoSocket::UP create_server_crypto_socket(SocketHandle socket) override;
 };
 
 /**
@@ -35,7 +43,10 @@ struct NullCryptoEngine : public CryptoEngine {
  * from TLS.
  **/
 struct XorCryptoEngine : public CryptoEngine {
-    CryptoSocket::UP create_crypto_socket(SocketHandle socket, bool is_server) override;
+    bool use_tls_when_client() const override { return false; }
+    bool always_use_tls_when_server() const override { return false; }
+    CryptoSocket::UP create_client_crypto_socket(SocketHandle socket, const SocketSpec &spec) override;
+    CryptoSocket::UP create_server_crypto_socket(SocketHandle socket) override;
 };
 
 } // namespace vespalib
diff --git a/vespalib/src/vespa/vespalib/net/crypto_socket.h b/vespalib/src/vespa/vespalib/net/crypto_socket.h
index c409e5a0e83..d51a97e2743 100644
--- a/vespalib/src/vespa/vespalib/net/crypto_socket.h
+++ b/vespalib/src/vespa/vespalib/net/crypto_socket.h
@@ -3,6 +3,7 @@
 #pragma once
 
 #include <memory>
+#include <cstdlib>
 
 namespace vespalib {
 
diff --git a/vespalib/src/vespa/vespalib/net/socket_spec.cpp b/vespalib/src/vespa/vespalib/net/socket_spec.cpp
index d1376ce1dd7..d1dd81a454a 100644
--- a/vespalib/src/vespa/vespalib/net/socket_spec.cpp
+++ b/vespalib/src/vespa/vespalib/net/socket_spec.cpp
@@ -41,6 +41,8 @@ SocketSpec::address(bool server) const
     return SocketAddress();
 }
 
+const SocketSpec SocketSpec::invalid;
+
 SocketSpec::SocketSpec(const vespalib::string &spec)
     : SocketSpec()
 {
diff --git a/vespalib/src/vespa/vespalib/net/socket_spec.h b/vespalib/src/vespa/vespalib/net/socket_spec.h
index f28b14573ac..4e3dddf6814 100644
--- a/vespalib/src/vespa/vespalib/net/socket_spec.h
+++ b/vespalib/src/vespa/vespalib/net/socket_spec.h
@@ -24,6 +24,7 @@ private:
         : _type(type), _node(node), _port(port) {}
     SocketAddress address(bool server) const;
 public:
+    static const SocketSpec invalid;
     explicit SocketSpec(const vespalib::string &spec);
     vespalib::string spec() const;
     SocketSpec replace_host(const vespalib::string &new_host) const;
diff --git a/vespalib/src/vespa/vespalib/net/sync_crypto_socket.cpp b/vespalib/src/vespa/vespalib/net/sync_crypto_socket.cpp
index 29388035bda..3aa2d3b0683 100644
--- a/vespalib/src/vespa/vespalib/net/sync_crypto_socket.cpp
+++ b/vespalib/src/vespa/vespalib/net/sync_crypto_socket.cpp
@@ -29,6 +29,25 @@ void set_blocking(int fd) {
 
 } // namespace vespalib::<unnamed>
 
+SyncCryptoSocket::UP
+SyncCryptoSocket::create(CryptoSocket::UP socket)
+{
+    set_blocking(socket->get_fd());
+    for (;;) {
+        switch (socket->handshake()) {
+        case CryptoSocket::HandshakeResult::FAIL:
+            return std::unique_ptr<SyncCryptoSocket>(nullptr);
+        case CryptoSocket::HandshakeResult::DONE:
+            return UP(new SyncCryptoSocket(std::move(socket)));
+        case CryptoSocket::HandshakeResult::NEED_READ:
+        case CryptoSocket::HandshakeResult::NEED_WRITE:
+            break;
+        case CryptoSocket::HandshakeResult::NEED_WORK:
+            socket->do_handshake_work();
+        }
+    }
+}
+
 SyncCryptoSocket::~SyncCryptoSocket() = default;
 
 ssize_t
@@ -90,23 +109,15 @@ SyncCryptoSocket::half_close()
 }
 
 SyncCryptoSocket::UP
-SyncCryptoSocket::create(CryptoEngine &engine, SocketHandle socket, bool is_server)
+SyncCryptoSocket::create_client(CryptoEngine &engine, SocketHandle socket, const SocketSpec &spec)
 {
-    auto crypto_socket = engine.create_crypto_socket(std::move(socket), is_server);
-    set_blocking(crypto_socket->get_fd());
-    for (;;) {
-        switch (crypto_socket->handshake()) {
-        case CryptoSocket::HandshakeResult::FAIL:
-            return std::unique_ptr<SyncCryptoSocket>(nullptr);
-        case CryptoSocket::HandshakeResult::DONE:
-            return UP(new SyncCryptoSocket(std::move(crypto_socket)));
-        case CryptoSocket::HandshakeResult::NEED_READ:
-        case CryptoSocket::HandshakeResult::NEED_WRITE:
-            break;
-        case CryptoSocket::HandshakeResult::NEED_WORK:
-            crypto_socket->do_handshake_work();
-        }
-    }
+    return create(engine.create_client_crypto_socket(std::move(socket), spec));
+}
+
+SyncCryptoSocket::UP
+SyncCryptoSocket::create_server(CryptoEngine &engine, SocketHandle socket)
+{
+    return create(engine.create_server_crypto_socket(std::move(socket)));
 }
 
 } // namespace vespalib
diff --git a/vespalib/src/vespa/vespalib/net/sync_crypto_socket.h b/vespalib/src/vespa/vespalib/net/sync_crypto_socket.h
index 00d6cbca0db..36fcfe12ed9 100644
--- a/vespalib/src/vespa/vespalib/net/sync_crypto_socket.h
+++ b/vespalib/src/vespa/vespalib/net/sync_crypto_socket.h
@@ -19,17 +19,20 @@ namespace vespalib {
  **/
 class SyncCryptoSocket
 {
+public:
+    using UP = std::unique_ptr<SyncCryptoSocket>;
 private:
     CryptoSocket::UP _socket;
     SmartBuffer _buffer;
     SyncCryptoSocket(CryptoSocket::UP socket) : _socket(std::move(socket)), _buffer(0) {}
+    static UP create(CryptoSocket::UP socket);
 public:
-    using UP = std::unique_ptr<SyncCryptoSocket>;
     ~SyncCryptoSocket();
     ssize_t read(char *buf, size_t len);
     ssize_t write(const char *buf, size_t len);
     ssize_t half_close();
-    static UP create(CryptoEngine &engine, SocketHandle socket, bool is_server);
+    static UP create_client(CryptoEngine &engine, SocketHandle socket, const SocketSpec &spec);
+    static UP create_server(CryptoEngine &engine, SocketHandle socket);
 };
 
 } // namespace vespalib
diff --git a/vespalib/src/vespa/vespalib/net/tls/CMakeLists.txt b/vespalib/src/vespa/vespalib/net/tls/CMakeLists.txt
index 6dc48db68e4..b7801f40959 100644
--- a/vespalib/src/vespa/vespalib/net/tls/CMakeLists.txt
+++ b/vespalib/src/vespa/vespalib/net/tls/CMakeLists.txt
@@ -5,7 +5,6 @@ vespa_add_library(vespalib_vespalib_net_tls OBJECT
     auto_reloading_tls_crypto_engine.cpp
     crypto_codec.cpp
     crypto_codec_adapter.cpp
-    crypto_exception.cpp
     maybe_tls_crypto_engine.cpp
     maybe_tls_crypto_socket.cpp
     peer_credentials.cpp
diff --git a/vespalib/src/vespa/vespalib/net/tls/auto_reloading_tls_crypto_engine.cpp b/vespalib/src/vespa/vespalib/net/tls/auto_reloading_tls_crypto_engine.cpp
index 5f20280e0e2..bdb2402adbc 100644
--- a/vespalib/src/vespa/vespalib/net/tls/auto_reloading_tls_crypto_engine.cpp
+++ b/vespalib/src/vespa/vespalib/net/tls/auto_reloading_tls_crypto_engine.cpp
@@ -91,13 +91,34 @@ AutoReloadingTlsCryptoEngine::EngineSP AutoReloadingTlsCryptoEngine::acquire_cur
     return _current_engine;
 }
 
-CryptoSocket::UP AutoReloadingTlsCryptoEngine::create_crypto_socket(SocketHandle socket, bool is_server) {
-    return acquire_current_engine()->create_crypto_socket(std::move(socket), is_server);
+CryptoSocket::UP AutoReloadingTlsCryptoEngine::create_client_crypto_socket(SocketHandle socket, const SocketSpec &spec) {
+    return acquire_current_engine()->create_client_crypto_socket(std::move(socket), spec);
+}
+
+CryptoSocket::UP AutoReloadingTlsCryptoEngine::create_server_crypto_socket(SocketHandle socket) {
+    return acquire_current_engine()->create_server_crypto_socket(std::move(socket));
+}
+
+bool
+AutoReloadingTlsCryptoEngine::use_tls_when_client() const
+{
+    return acquire_current_engine()->use_tls_when_client();
+}
+
+bool
+AutoReloadingTlsCryptoEngine::always_use_tls_when_server() const
+{
+    return acquire_current_engine()->always_use_tls_when_server();
+}
+
+std::unique_ptr<TlsCryptoSocket>
+AutoReloadingTlsCryptoEngine::create_tls_client_crypto_socket(SocketHandle socket, const SocketSpec &spec) {
+    return acquire_current_engine()->create_tls_client_crypto_socket(std::move(socket), spec);
 }
 
 std::unique_ptr<TlsCryptoSocket>
-AutoReloadingTlsCryptoEngine::create_tls_crypto_socket(SocketHandle socket, bool is_server) {
-    return acquire_current_engine()->create_tls_crypto_socket(std::move(socket), is_server);
+AutoReloadingTlsCryptoEngine::create_tls_server_crypto_socket(SocketHandle socket) {
+    return acquire_current_engine()->create_tls_server_crypto_socket(std::move(socket));
 }
 
 }
diff --git a/vespalib/src/vespa/vespalib/net/tls/auto_reloading_tls_crypto_engine.h b/vespalib/src/vespa/vespalib/net/tls/auto_reloading_tls_crypto_engine.h
index 6287fdd4f63..1b80b782daf 100644
--- a/vespalib/src/vespa/vespalib/net/tls/auto_reloading_tls_crypto_engine.h
+++ b/vespalib/src/vespa/vespalib/net/tls/auto_reloading_tls_crypto_engine.h
@@ -45,8 +45,12 @@ public:
 
     EngineSP acquire_current_engine() const;
 
-    CryptoSocket::UP create_crypto_socket(SocketHandle socket, bool is_server) override;
-    std::unique_ptr<TlsCryptoSocket> create_tls_crypto_socket(SocketHandle socket, bool is_server) override;
+    CryptoSocket::UP create_client_crypto_socket(SocketHandle socket, const SocketSpec &spec) override;
+    CryptoSocket::UP create_server_crypto_socket(SocketHandle socket) override;
+    bool use_tls_when_client() const override;
+    bool always_use_tls_when_server() const override;
+    std::unique_ptr<TlsCryptoSocket> create_tls_client_crypto_socket(SocketHandle socket, const SocketSpec &spec) override;
+    std::unique_ptr<TlsCryptoSocket> create_tls_server_crypto_socket(SocketHandle socket) override;
 };
 
 }
diff --git a/vespalib/src/vespa/vespalib/net/tls/crypto_codec.cpp b/vespalib/src/vespa/vespalib/net/tls/crypto_codec.cpp
index c54990b3782..d3ac975d90a 100644
--- a/vespalib/src/vespa/vespalib/net/tls/crypto_codec.cpp
+++ b/vespalib/src/vespa/vespalib/net/tls/crypto_codec.cpp
@@ -6,12 +6,23 @@
 
 namespace vespalib::net::tls {
 
-std::unique_ptr<CryptoCodec> CryptoCodec::create_default_codec(
-        std::shared_ptr<TlsContext> ctx, const SocketAddress& peer_address, Mode mode)
+std::unique_ptr<CryptoCodec>
+CryptoCodec::create_default_client_codec(std::shared_ptr<TlsContext> ctx,
+                                         const SocketSpec& peer_spec,
+                                         const SocketAddress& peer_address)
 {
     auto ctx_impl = std::dynamic_pointer_cast<impl::OpenSslTlsContextImpl>(ctx); // only takes by const ref
     assert(ctx_impl);
-    return std::make_unique<impl::OpenSslCryptoCodecImpl>(std::move(ctx_impl), peer_address, mode);
+    return impl::OpenSslCryptoCodecImpl::make_client_codec(std::move(ctx_impl), peer_spec, peer_address);
+}
+
+std::unique_ptr<CryptoCodec>
+CryptoCodec::create_default_server_codec(std::shared_ptr<TlsContext> ctx,
+                                         const SocketAddress& peer_address)
+{
+    auto ctx_impl = std::dynamic_pointer_cast<impl::OpenSslTlsContextImpl>(ctx); // only takes by const ref
+    assert(ctx_impl);
+    return impl::OpenSslCryptoCodecImpl::make_server_codec(std::move(ctx_impl), peer_address);
 }
 
 }
diff --git a/vespalib/src/vespa/vespalib/net/tls/crypto_codec.h b/vespalib/src/vespa/vespalib/net/tls/crypto_codec.h
index 5d9684461d7..787485b47be 100644
--- a/vespalib/src/vespa/vespalib/net/tls/crypto_codec.h
+++ b/vespalib/src/vespa/vespalib/net/tls/crypto_codec.h
@@ -4,6 +4,8 @@
 #include <vespa/vespalib/net/socket_address.h>
 #include <memory>
 
+namespace vespalib { class SocketSpec; }
+
 namespace vespalib::net::tls {
 
 struct HandshakeResult {
@@ -179,9 +181,13 @@ public:
      *
      * Throws CryptoException if resources cannot be allocated for the codec.
      */
-    static std::unique_ptr<CryptoCodec> create_default_codec(std::shared_ptr<TlsContext> ctx,
-                                                             const SocketAddress& peer_address,
-                                                             Mode mode);
+    static std::unique_ptr<CryptoCodec>
+    create_default_client_codec(std::shared_ptr<TlsContext> ctx,
+                                const SocketSpec& peer_spec,
+                                const SocketAddress& peer_address);
+    static std::unique_ptr<CryptoCodec>
+    create_default_server_codec(std::shared_ptr<TlsContext> ctx,
+                                const SocketAddress& peer_address);
 };
 
 }
diff --git a/vespalib/src/vespa/vespalib/net/tls/impl/direct_buffer_bio.cpp b/vespalib/src/vespa/vespalib/net/tls/impl/direct_buffer_bio.cpp
index 614722a9769..d7d02534242 100644
--- a/vespalib/src/vespa/vespalib/net/tls/impl/direct_buffer_bio.cpp
+++ b/vespalib/src/vespa/vespalib/net/tls/impl/direct_buffer_bio.cpp
@@ -1,11 +1,10 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 #include "direct_buffer_bio.h"
-#include <vespa/vespalib/net/tls/crypto_exception.h>
+#include <vespa/vespalib/crypto/crypto_exception.h>
+#include <vespa/vespalib/util/backtrace.h>
 #include <utility>
 #include <cassert>
 
-#include <vespa/vespalib/util/backtrace.h>
-
 #include <vespa/log/log.h>
 LOG_SETUP(".vespalib.net.tls.impl.direct_buffer_bio");
 
@@ -19,6 +18,8 @@ LOG_SETUP(".vespalib.net.tls.impl.direct_buffer_bio");
  *  - https://github.com/indutny/uv_ssl_t/blob/master/src/bio.c
  */
 
+using namespace vespalib::crypto;
+
 namespace vespalib::net::tls::impl {
 
 namespace {
diff --git a/vespalib/src/vespa/vespalib/net/tls/impl/direct_buffer_bio.h b/vespalib/src/vespa/vespalib/net/tls/impl/direct_buffer_bio.h
index 581d43d6f29..8492bf2c436 100644
--- a/vespalib/src/vespa/vespalib/net/tls/impl/direct_buffer_bio.h
+++ b/vespalib/src/vespa/vespalib/net/tls/impl/direct_buffer_bio.h
@@ -1,7 +1,7 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 #pragma once
 
-#include "openssl_typedefs.h"
+#include <vespa/vespalib/crypto/openssl_typedefs.h>
 #include <openssl/bio.h>
 
 /*
@@ -22,8 +22,8 @@
 
 namespace vespalib::net::tls::impl {
 
-BioPtr new_mutable_direct_buffer_bio();
-BioPtr new_const_direct_buffer_bio();
+crypto::BioPtr new_mutable_direct_buffer_bio();
+crypto::BioPtr new_const_direct_buffer_bio();
 
 struct MutableBufferView {
     // Could use a pointer pair instead (or just modify the ptr), but being explicit is good for readability.
diff --git a/vespalib/src/vespa/vespalib/net/tls/impl/openssl_crypto_codec_impl.cpp b/vespalib/src/vespa/vespalib/net/tls/impl/openssl_crypto_codec_impl.cpp
index 5315754d53a..1d87a50190e 100644
--- a/vespalib/src/vespa/vespalib/net/tls/impl/openssl_crypto_codec_impl.cpp
+++ b/vespalib/src/vespa/vespalib/net/tls/impl/openssl_crypto_codec_impl.cpp
@@ -3,8 +3,8 @@
 #include "openssl_tls_context_impl.h"
 #include "direct_buffer_bio.h"
 
+#include <vespa/vespalib/crypto/crypto_exception.h>
 #include <vespa/vespalib/net/tls/crypto_codec.h>
-#include <vespa/vespalib/net/tls/crypto_exception.h>
 #include <vespa/vespalib/net/tls/statistics.h>
 
 #include <mutex>
@@ -36,6 +36,8 @@ LOG_SETUP(".vespalib.net.tls.openssl_crypto_codec_impl");
  * light fades and turns to all-enveloping darkness.
  */
 
+using namespace vespalib::crypto;
+
 namespace vespalib::net::tls::impl {
 
 namespace {
@@ -172,9 +174,11 @@ void log_ssl_error(const char* source, const SocketAddress& peer_address, int ss
 } // anon ns
 
 OpenSslCryptoCodecImpl::OpenSslCryptoCodecImpl(std::shared_ptr<OpenSslTlsContextImpl> ctx,
+                                               const SocketSpec& peer_spec,
                                                const SocketAddress& peer_address,
                                                Mode mode)
     : _ctx(std::move(ctx)),
+      _peer_spec(peer_spec),
       _peer_address(peer_address),
       _ssl(::SSL_new(_ctx->native_context())),
       _mode(mode),
@@ -219,6 +223,8 @@ OpenSslCryptoCodecImpl::OpenSslCryptoCodecImpl(std::shared_ptr<OpenSslTlsContext
     _output_bio = tmp_output_bio.release();
     if (_mode == Mode::Client) {
         ::SSL_set_connect_state(_ssl.get());
+        enable_hostname_validation_if_requested();
+        set_server_name_indication_extension();
     } else {
         ::SSL_set_accept_state(_ssl.get());
     }
@@ -230,6 +236,59 @@ OpenSslCryptoCodecImpl::OpenSslCryptoCodecImpl(std::shared_ptr<OpenSslTlsContext
 
 OpenSslCryptoCodecImpl::~OpenSslCryptoCodecImpl() = default;
 
+std::unique_ptr<OpenSslCryptoCodecImpl>
+OpenSslCryptoCodecImpl::make_client_codec(std::shared_ptr<OpenSslTlsContextImpl> ctx,
+                                          const SocketSpec& peer_spec,
+                                          const SocketAddress& peer_address)
+{
+    // Naked new due to private ctor
+    return std::unique_ptr<OpenSslCryptoCodecImpl>(
+            new OpenSslCryptoCodecImpl(std::move(ctx), peer_spec, peer_address, Mode::Client));
+}
+std::unique_ptr<OpenSslCryptoCodecImpl>
+OpenSslCryptoCodecImpl::make_server_codec(std::shared_ptr<OpenSslTlsContextImpl> ctx,
+                                          const SocketAddress& peer_address)
+{
+    // Naked new due to private ctor
+    return std::unique_ptr<OpenSslCryptoCodecImpl>(
+            new OpenSslCryptoCodecImpl(std::move(ctx), SocketSpec::invalid, peer_address, Mode::Server));
+}
+
+void OpenSslCryptoCodecImpl::enable_hostname_validation_if_requested() {
+    if (_peer_spec.valid() && !_ctx->transport_security_options().disable_hostname_validation()) {
+        auto* verify_param = SSL_get0_param(_ssl.get()); // Internal ptr, no refcount bump or alloc. We must not free.
+        LOG_ASSERT(verify_param != nullptr);
+        vespalib::string host = _peer_spec.host();
+        if (X509_VERIFY_PARAM_set1_host(verify_param, host.c_str(), host.size()) != 1) {
+            throw CryptoException("X509_VERIFY_PARAM_set1_host() failed");
+        }
+        // TODO should we set expected IP based on peer address as well?
+    }
+}
+
+void OpenSslCryptoCodecImpl::set_server_name_indication_extension() {
+    if (_peer_spec.valid()) {
+        vespalib::string host = _peer_spec.host();
+        // OpenSSL tries to cast const char* to void* in a macro, even on 1.1.1. GCC is not overly impressed,
+        // so to satiate OpenSSL's quirks we pre-cast away the constness.
+        auto* host_cstr_that_trusts_openssl_not_to_mess_up = const_cast<char*>(host.c_str());
+        if (SSL_set_tlsext_host_name(_ssl.get(), host_cstr_that_trusts_openssl_not_to_mess_up) != 1) {
+            throw CryptoException("SSL_set_tlsext_host_name() failed");
+        }
+    }
+}
+
+std::optional<vespalib::string> OpenSslCryptoCodecImpl::client_provided_sni_extension() const {
+    if ((_mode != Mode::Server) || (SSL_get_servername_type(_ssl.get()) != TLSEXT_NAMETYPE_host_name)) {
+        return {};
+    }
+    const char* sni_host_raw = SSL_get_servername(_ssl.get(), TLSEXT_NAMETYPE_host_name);
+    if (sni_host_raw == nullptr) {
+        return {};
+    }
+    return vespalib::string(sni_host_raw);
+}
+
 HandshakeResult OpenSslCryptoCodecImpl::handshake(const char* from_peer, size_t from_peer_buf_size,
                                                   char* to_peer, size_t to_peer_buf_size) noexcept
 {
@@ -428,3 +487,5 @@ EncodeResult OpenSslCryptoCodecImpl::half_close(char* ciphertext, size_t ciphert
 // External references:
 //  [0] http://openssl.6102.n7.nabble.com/nonblocking-implementation-question-tp1728p1732.html
 //  [1] https://github.com/grpc/grpc/blob/master/src/core/tsi/ssl_transport_security.cc
+//  [2] https://wiki.openssl.org/index.php/Hostname_validation
+//  [3] https://wiki.openssl.org/index.php/SSL/TLS_Client
diff --git a/vespalib/src/vespa/vespalib/net/tls/impl/openssl_crypto_codec_impl.h b/vespalib/src/vespa/vespalib/net/tls/impl/openssl_crypto_codec_impl.h
index 14200de449a..80f3e12786a 100644
--- a/vespalib/src/vespa/vespalib/net/tls/impl/openssl_crypto_codec_impl.h
+++ b/vespalib/src/vespa/vespalib/net/tls/impl/openssl_crypto_codec_impl.h
@@ -1,8 +1,9 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 #pragma once
 
-#include "openssl_typedefs.h"
+#include <vespa/vespalib/crypto/openssl_typedefs.h>
 #include <vespa/vespalib/net/socket_address.h>
+#include <vespa/vespalib/net/socket_spec.h>
 #include <vespa/vespalib/net/tls/transport_security_options.h>
 #include <vespa/vespalib/net/tls/crypto_codec.h>
 #include <memory>
@@ -46,17 +47,23 @@ class OpenSslCryptoCodecImpl : public CryptoCodec {
     // The context maintains shared verification callback state, so it must be
     // kept alive explictly for at least as long as any codecs.
     std::shared_ptr<OpenSslTlsContextImpl> _ctx;
-    SocketAddress _peer_address;
-    SslPtr _ssl;
-    ::BIO* _input_bio;  // Owned by _ssl
-    ::BIO* _output_bio; // Owned by _ssl
-    Mode _mode;
+    SocketSpec     _peer_spec;
+    SocketAddress  _peer_address;
+    crypto::SslPtr _ssl;
+    ::BIO*         _input_bio;  // Owned by _ssl
+    ::BIO*         _output_bio; // Owned by _ssl
+    Mode           _mode;
     std::optional<DeferredHandshakeParams> _deferred_handshake_params;
-    std::optional<HandshakeResult> _deferred_handshake_result;
+    std::optional<HandshakeResult>         _deferred_handshake_result;
 public:
-    OpenSslCryptoCodecImpl(std::shared_ptr<OpenSslTlsContextImpl> ctx, const SocketAddress& peer_address, Mode mode);
     ~OpenSslCryptoCodecImpl() override;
 
+    static std::unique_ptr<OpenSslCryptoCodecImpl> make_client_codec(std::shared_ptr<OpenSslTlsContextImpl> ctx,
+                                                                     const SocketSpec& peer_spec,
+                                                                     const SocketAddress& peer_address);
+    static std::unique_ptr<OpenSslCryptoCodecImpl> make_server_codec(std::shared_ptr<OpenSslTlsContextImpl> ctx,
+                                                                     const SocketAddress& peer_address);
+
     /*
      * From RFC 8449 (Record Size Limit Extension for TLS), section 1:
      *   "TLS versions 1.2 [RFC5246] and earlier permit senders to
@@ -89,7 +96,20 @@ public:
     EncodeResult half_close(char* ciphertext, size_t ciphertext_size) noexcept override;
 
     const SocketAddress& peer_address() const noexcept { return _peer_address; }
+    /*
+     * If a client has sent a SNI extension field as part of the handshake,
+     * returns the raw string representation of this. It only makes sense to
+     * call this for codecs in server mode.
+     */
+    std::optional<vespalib::string> client_provided_sni_extension() const;
 private:
+    OpenSslCryptoCodecImpl(std::shared_ptr<OpenSslTlsContextImpl> ctx,
+                           const SocketSpec& peer_spec,
+                           const SocketAddress& peer_address,
+                           Mode mode);
+
+    void enable_hostname_validation_if_requested();
+    void set_server_name_indication_extension();
     HandshakeResult do_handshake_and_consume_peer_input_bytes() noexcept;
     DecodeResult drain_and_produce_plaintext_from_ssl(char* plaintext, size_t plaintext_size) noexcept;
     // Precondition: read_result < 0
diff --git a/vespalib/src/vespa/vespalib/net/tls/impl/openssl_tls_context_impl.cpp b/vespalib/src/vespa/vespalib/net/tls/impl/openssl_tls_context_impl.cpp
index 98675ec6b0b..e66baf87999 100644
--- a/vespalib/src/vespa/vespalib/net/tls/impl/openssl_tls_context_impl.cpp
+++ b/vespalib/src/vespa/vespalib/net/tls/impl/openssl_tls_context_impl.cpp
@@ -1,9 +1,9 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 #include "iana_cipher_map.h"
-#include "openssl_typedefs.h"
 #include "openssl_tls_context_impl.h"
 #include "openssl_crypto_codec_impl.h"
-#include <vespa/vespalib/net/tls/crypto_exception.h>
+#include <vespa/vespalib/crypto/crypto_exception.h>
+#include <vespa/vespalib/crypto/openssl_typedefs.h>
 #include <vespa/vespalib/net/tls/statistics.h>
 #include <vespa/vespalib/net/tls/transport_security_options.h>
 #include <vespa/vespalib/util/stringfmt.h>
@@ -26,6 +26,8 @@ LOG_SETUP(".vespalib.net.tls.openssl_tls_context_impl");
 #  error "Provided OpenSSL version is too darn old, need at least 1.0.0"
 #endif
 
+using namespace vespalib::crypto;
+
 namespace vespalib::net::tls::impl {
 
 namespace {
diff --git a/vespalib/src/vespa/vespalib/net/tls/impl/openssl_tls_context_impl.h b/vespalib/src/vespa/vespalib/net/tls/impl/openssl_tls_context_impl.h
index c519b1ae874..badfe8306d1 100644
--- a/vespalib/src/vespa/vespalib/net/tls/impl/openssl_tls_context_impl.h
+++ b/vespalib/src/vespa/vespalib/net/tls/impl/openssl_tls_context_impl.h
@@ -1,7 +1,7 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 #pragma once
 
-#include "openssl_typedefs.h"
+#include <vespa/vespalib/crypto/openssl_typedefs.h>
 #include <vespa/vespalib/net/socket_address.h>
 #include <vespa/vespalib/net/tls/tls_context.h>
 #include <vespa/vespalib/net/tls/transport_security_options.h>
@@ -13,7 +13,7 @@
 namespace vespalib::net::tls::impl {
 
 class OpenSslTlsContextImpl : public TlsContext {
-    SslCtxPtr _ctx;
+    crypto::SslCtxPtr _ctx;
     AuthorizationMode _authorization_mode;
     std::shared_ptr<CertificateVerificationCallback> _cert_verify_callback;
     TransportSecurityOptions _redacted_transport_options;
diff --git a/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_engine.cpp b/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_engine.cpp
index 891f8cdab23..f7f0284bded 100644
--- a/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_engine.cpp
+++ b/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_engine.cpp
@@ -6,15 +6,19 @@
 namespace vespalib {
 
 CryptoSocket::UP
-MaybeTlsCryptoEngine::create_crypto_socket(SocketHandle socket, bool is_server)
+MaybeTlsCryptoEngine::create_client_crypto_socket(SocketHandle socket, const SocketSpec &spec)
 {
-    if (is_server) {
-        return std::make_unique<MaybeTlsCryptoSocket>(std::move(socket), _tls_engine);
-    } else if (_use_tls_when_client) {
-        return _tls_engine->create_crypto_socket(std::move(socket), false);
+    if (_use_tls_when_client) {
+        return _tls_engine->create_client_crypto_socket(std::move(socket), spec);
     } else {
-        return _null_engine->create_crypto_socket(std::move(socket), false);
+        return _null_engine->create_client_crypto_socket(std::move(socket), spec);
     }
 }
 
+CryptoSocket::UP
+MaybeTlsCryptoEngine::create_server_crypto_socket(SocketHandle socket)
+{
+    return std::make_unique<MaybeTlsCryptoSocket>(std::move(socket), _tls_engine);
+}
+
 } // namespace vespalib
diff --git a/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_engine.h b/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_engine.h
index 29909fa115d..ece7d094c54 100644
--- a/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_engine.h
+++ b/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_engine.h
@@ -28,7 +28,10 @@ public:
         : _null_engine(std::make_shared<NullCryptoEngine>()),
           _tls_engine(std::move(tls_engine)),
           _use_tls_when_client(use_tls_when_client) {}
-    CryptoSocket::UP create_crypto_socket(SocketHandle socket, bool is_server) override;
+    bool use_tls_when_client() const override { return _use_tls_when_client; }
+    bool always_use_tls_when_server() const override { return false; }
+    CryptoSocket::UP create_client_crypto_socket(SocketHandle socket, const SocketSpec &spec) override;
+    CryptoSocket::UP create_server_crypto_socket(SocketHandle socket) override;
 };
 
 } // namespace vespalib
diff --git a/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_socket.cpp b/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_socket.cpp
index 9af6703acbc..8ab6adad2e5 100644
--- a/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_socket.cpp
+++ b/vespalib/src/vespa/vespalib/net/tls/maybe_tls_crypto_socket.cpp
@@ -50,7 +50,7 @@ public:
             }
             if (looksLikeTlsToMe(src.data)) {
                 CryptoSocket::UP &self = _self; // need copy due to self destruction
-                auto tls_socket = _factory->create_tls_crypto_socket(std::move(_socket), true);
+                auto tls_socket = _factory->create_tls_server_crypto_socket(std::move(_socket));
                 tls_socket->inject_read_data(src.data, src.size);
                 self = std::move(tls_socket);
                 return self->handshake();
diff --git a/vespalib/src/vespa/vespalib/net/tls/peer_policies.cpp b/vespalib/src/vespa/vespalib/net/tls/peer_policies.cpp
index 8d2fb04d853..27a11b3f0f1 100644
--- a/vespalib/src/vespa/vespalib/net/tls/peer_policies.cpp
+++ b/vespalib/src/vespa/vespalib/net/tls/peer_policies.cpp
@@ -1,28 +1,34 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include "peer_policies.h"
+#include <vespa/vespalib/regex/regex.h>
 #include <iostream>
-#include <regex>
 
 namespace vespalib::net::tls {
 
 namespace {
 
-// Note: this is for basix regexp only, _not_ extended regexp
-bool is_basic_regex_special_char(char c) noexcept {
+bool is_regex_special_char(char c) noexcept {
     switch (c) {
-        case '^':
-        case '$':
-        case '.':
-        case '[':
-        case '\\':
-            return true;
-        default:
-            return false;
+    case '^':
+    case '$':
+    case '|':
+    case '{':
+    case '}':
+    case '(':
+    case ')':
+    case '[':
+    case ']':
+    case '\\':
+    case '+':
+    case '.':
+        return true;
+    default:
+        return false;
     }
 }
 
-std::string glob_to_basic_regex(vespalib::stringref glob) {
+std::string dot_separated_glob_to_regex(vespalib::stringref glob) {
     std::string ret = "^";
     ret.reserve(glob.size() + 2);
     for (auto c : glob) {
@@ -34,7 +40,7 @@ std::string glob_to_basic_regex(vespalib::stringref glob) {
             // Same applies for single chars; they should only match _within_ a dot boundary.
             ret += "[^.]";
         } else {
-            if (is_basic_regex_special_char(c)) {
+            if (is_regex_special_char(c)) {
                 ret += '\\';
             }
             ret += c;
@@ -45,16 +51,16 @@ std::string glob_to_basic_regex(vespalib::stringref glob) {
 }
 
 class RegexHostMatchPattern : public HostGlobPattern {
-    std::regex _pattern_as_regex;
+    Regex _pattern_as_regex;
 public:
     explicit RegexHostMatchPattern(vespalib::stringref glob_pattern)
-        : _pattern_as_regex(glob_to_basic_regex(glob_pattern), std::regex_constants::basic)
+        : _pattern_as_regex(Regex::from_pattern(dot_separated_glob_to_regex(glob_pattern)))
     {
     }
     ~RegexHostMatchPattern() override = default;
 
-    bool matches(vespalib::stringref str) const override {
-        return std::regex_match(str.begin(), str.end(), _pattern_as_regex);
+    [[nodiscard]] bool matches(vespalib::stringref str) const override {
+        return _pattern_as_regex.full_match(std::string_view(str.data(), str.size()));
     }
 };
 
diff --git a/vespalib/src/vespa/vespalib/net/tls/peer_policies.h b/vespalib/src/vespa/vespalib/net/tls/peer_policies.h
index 44cbe70fd92..9d34b62415f 100644
--- a/vespalib/src/vespa/vespalib/net/tls/peer_policies.h
+++ b/vespalib/src/vespa/vespalib/net/tls/peer_policies.h
@@ -10,7 +10,7 @@ namespace vespalib::net::tls {
 
 struct HostGlobPattern {
     virtual ~HostGlobPattern() = default;
-    virtual bool matches(vespalib::stringref str) const = 0;
+    [[nodiscard]] virtual bool matches(vespalib::stringref str) const = 0;
 
     static std::shared_ptr<const HostGlobPattern> create_from_glob(vespalib::stringref pattern);
 };
@@ -36,7 +36,7 @@ public:
                 && (_original_pattern == rhs._original_pattern));
     }
 
-    bool matches(vespalib::stringref str) const {
+    [[nodiscard]] bool matches(vespalib::stringref str) const {
         return (_match_pattern && _match_pattern->matches(str));
     }
 
@@ -64,19 +64,24 @@ public:
 class AuthorizedPeers {
     // A peer will be authorized iff it matches _one or more_ policies.
     std::vector<PeerPolicy> _peer_policies;
-    bool _allow_all_if_empty = false;
+    bool _allow_all_if_empty;
 
     explicit AuthorizedPeers(bool allow_all_if_empty)
         : _peer_policies(),
           _allow_all_if_empty(allow_all_if_empty)
     {}
 public:
-    AuthorizedPeers() = default;
+    AuthorizedPeers() : _peer_policies(), _allow_all_if_empty(false) {}
     explicit AuthorizedPeers(std::vector<PeerPolicy> peer_policies_)
         : _peer_policies(std::move(peer_policies_)),
           _allow_all_if_empty(false)
     {}
 
+    AuthorizedPeers(const AuthorizedPeers&) = default;
+    AuthorizedPeers& operator=(const AuthorizedPeers&) = default;
+    AuthorizedPeers(AuthorizedPeers&&) noexcept = default;
+    AuthorizedPeers& operator=(AuthorizedPeers&&) noexcept = default;
+
     static AuthorizedPeers allow_all_authenticated() {
         return AuthorizedPeers(true);
     }
@@ -84,7 +89,7 @@ public:
     bool operator==(const AuthorizedPeers& rhs) const {
         return (_peer_policies == rhs._peer_policies);
     }
-    bool allows_all_authenticated() const noexcept {
+    [[nodiscard]] bool allows_all_authenticated() const noexcept {
         return _allow_all_if_empty;
     }
     const std::vector<PeerPolicy>& peer_policies() const noexcept { return _peer_policies; }
diff --git a/vespalib/src/vespa/vespalib/net/tls/tls_crypto_engine.cpp b/vespalib/src/vespa/vespalib/net/tls/tls_crypto_engine.cpp
index 58d99cc7108..99862e83dbf 100644
--- a/vespalib/src/vespa/vespalib/net/tls/tls_crypto_engine.cpp
+++ b/vespalib/src/vespa/vespalib/net/tls/tls_crypto_engine.cpp
@@ -12,10 +12,16 @@ TlsCryptoEngine::TlsCryptoEngine(net::tls::TransportSecurityOptions tls_opts, ne
 }
 
 std::unique_ptr<TlsCryptoSocket>
-TlsCryptoEngine::create_tls_crypto_socket(SocketHandle socket, bool is_server)
+TlsCryptoEngine::create_tls_client_crypto_socket(SocketHandle socket, const SocketSpec &peer_spec)
 {
-    auto mode = is_server ? net::tls::CryptoCodec::Mode::Server : net::tls::CryptoCodec::Mode::Client;
-    auto codec = net::tls::CryptoCodec::create_default_codec(_tls_ctx, SocketAddress::peer_address(socket.get()), mode);
+    auto codec = net::tls::CryptoCodec::create_default_client_codec(_tls_ctx, peer_spec, SocketAddress::peer_address(socket.get()));
+    return std::make_unique<net::tls::CryptoCodecAdapter>(std::move(socket), std::move(codec));
+}
+
+std::unique_ptr<TlsCryptoSocket>
+TlsCryptoEngine::create_tls_server_crypto_socket(SocketHandle socket)
+{
+    auto codec = net::tls::CryptoCodec::create_default_server_codec(_tls_ctx, SocketAddress::peer_address(socket.get()));
     return std::make_unique<net::tls::CryptoCodecAdapter>(std::move(socket), std::move(codec));
 }
 
diff --git a/vespalib/src/vespa/vespalib/net/tls/tls_crypto_engine.h b/vespalib/src/vespa/vespalib/net/tls/tls_crypto_engine.h
index dc7d7eaf9ce..444a817b357 100644
--- a/vespalib/src/vespa/vespalib/net/tls/tls_crypto_engine.h
+++ b/vespalib/src/vespa/vespalib/net/tls/tls_crypto_engine.h
@@ -11,7 +11,8 @@ namespace vespalib {
 
 class AbstractTlsCryptoEngine : public CryptoEngine {
 public:
-    virtual std::unique_ptr<TlsCryptoSocket> create_tls_crypto_socket(SocketHandle socket, bool is_server) = 0;
+    virtual std::unique_ptr<TlsCryptoSocket> create_tls_client_crypto_socket(SocketHandle socket, const SocketSpec &spec) = 0;
+    virtual std::unique_ptr<TlsCryptoSocket> create_tls_server_crypto_socket(SocketHandle socket) = 0;
 };
 
 /**
@@ -24,9 +25,15 @@ private:
 public:
     explicit TlsCryptoEngine(net::tls::TransportSecurityOptions tls_opts,
                              net::tls::AuthorizationMode authz_mode = net::tls::AuthorizationMode::Enforce);
-    std::unique_ptr<TlsCryptoSocket> create_tls_crypto_socket(SocketHandle socket, bool is_server) override;
-    CryptoSocket::UP create_crypto_socket(SocketHandle socket, bool is_server) override {
-        return create_tls_crypto_socket(std::move(socket), is_server);
+    std::unique_ptr<TlsCryptoSocket> create_tls_client_crypto_socket(SocketHandle socket, const SocketSpec &spec) override;
+    std::unique_ptr<TlsCryptoSocket> create_tls_server_crypto_socket(SocketHandle socket) override;
+    bool use_tls_when_client() const override { return true; }
+    bool always_use_tls_when_server() const override { return true; }
+    CryptoSocket::UP create_client_crypto_socket(SocketHandle socket, const SocketSpec &spec) override {
+        return create_tls_client_crypto_socket(std::move(socket), spec);
+    }
+    CryptoSocket::UP create_server_crypto_socket(SocketHandle socket) override {
+        return create_tls_server_crypto_socket(std::move(socket));
     }
 
     std::shared_ptr<net::tls::TlsContext> tls_context() const noexcept { return _tls_ctx; };
diff --git a/vespalib/src/vespa/vespalib/net/tls/transport_security_options.cpp b/vespalib/src/vespa/vespalib/net/tls/transport_security_options.cpp
index c9a5fa5a6e9..47b0e1e0a43 100644
--- a/vespalib/src/vespa/vespalib/net/tls/transport_security_options.cpp
+++ b/vespalib/src/vespa/vespalib/net/tls/transport_security_options.cpp
@@ -11,41 +11,57 @@ TransportSecurityOptions::TransportSecurityOptions(Params params)
       _cert_chain_pem(std::move(params._cert_chain_pem)),
       _private_key_pem(std::move(params._private_key_pem)),
       _authorized_peers(std::move(params._authorized_peers)),
-      _accepted_ciphers(std::move(params._accepted_ciphers))
+      _accepted_ciphers(std::move(params._accepted_ciphers)),
+      _disable_hostname_validation(params._disable_hostname_validation)
 {
 }
 
 TransportSecurityOptions::TransportSecurityOptions(vespalib::string ca_certs_pem,
                                                    vespalib::string cert_chain_pem,
-                                                   vespalib::string private_key_pem)
+                                                   vespalib::string private_key_pem,
+                                                   AuthorizedPeers authorized_peers,
+                                                   bool disable_hostname_validation)
     : _ca_certs_pem(std::move(ca_certs_pem)),
       _cert_chain_pem(std::move(cert_chain_pem)),
       _private_key_pem(std::move(private_key_pem)),
-      _authorized_peers(AuthorizedPeers::allow_all_authenticated())
+      _authorized_peers(std::move(authorized_peers)),
+      _disable_hostname_validation(disable_hostname_validation)
 {
 }
 
-TransportSecurityOptions::TransportSecurityOptions(vespalib::string ca_certs_pem,
-                                                   vespalib::string cert_chain_pem,
-                                                   vespalib::string private_key_pem,
-                                                   AuthorizedPeers authorized_peers)
-    : _ca_certs_pem(std::move(ca_certs_pem)),
-      _cert_chain_pem(std::move(cert_chain_pem)),
-      _private_key_pem(std::move(private_key_pem)),
-      _authorized_peers(std::move(authorized_peers))
-{
+TransportSecurityOptions TransportSecurityOptions::copy_without_private_key() const {
+    return TransportSecurityOptions(_ca_certs_pem, _cert_chain_pem, "",
+                                    _authorized_peers, _disable_hostname_validation);
 }
 
 void secure_memzero(void* buf, size_t size) noexcept {
     OPENSSL_cleanse(buf, size);
 }
 
-TransportSecurityOptions::Params::Params() = default;
+TransportSecurityOptions::Params::Params()
+    : _ca_certs_pem(),
+      _cert_chain_pem(),
+      _private_key_pem(),
+      _authorized_peers(),
+      _accepted_ciphers(),
+      _disable_hostname_validation(false)
+{
+}
 
 TransportSecurityOptions::Params::~Params() {
     secure_memzero(&_private_key_pem[0], _private_key_pem.size());
 }
 
+TransportSecurityOptions::Params::Params(const Params&) = default;
+
+TransportSecurityOptions::Params&
+TransportSecurityOptions::Params::operator=(const TransportSecurityOptions::Params&) = default;
+
+TransportSecurityOptions::Params::Params(Params&&) noexcept = default;
+
+TransportSecurityOptions::Params&
+TransportSecurityOptions::Params::operator=(TransportSecurityOptions::Params&&) noexcept = default;
+
 TransportSecurityOptions::~TransportSecurityOptions() {
     secure_memzero(&_private_key_pem[0], _private_key_pem.size());
 }
diff --git a/vespalib/src/vespa/vespalib/net/tls/transport_security_options.h b/vespalib/src/vespa/vespalib/net/tls/transport_security_options.h
index 39abb1ce4de..e0a48fc6cf5 100644
--- a/vespalib/src/vespa/vespalib/net/tls/transport_security_options.h
+++ b/vespalib/src/vespa/vespalib/net/tls/transport_security_options.h
@@ -14,18 +14,22 @@ class TransportSecurityOptions {
     vespalib::string _private_key_pem;
     AuthorizedPeers  _authorized_peers;
     std::vector<vespalib::string> _accepted_ciphers;
+    bool _disable_hostname_validation;
 public:
-    TransportSecurityOptions() = default;
-
     struct Params {
         vespalib::string _ca_certs_pem;
         vespalib::string _cert_chain_pem;
         vespalib::string _private_key_pem;
         AuthorizedPeers  _authorized_peers;
         std::vector<vespalib::string> _accepted_ciphers;
+        bool _disable_hostname_validation;
 
         Params();
         ~Params();
+        Params(const Params&);
+        Params& operator=(const Params&);
+        Params(Params&&) noexcept;
+        Params& operator=(Params&&) noexcept;
 
         Params& ca_certs_pem(vespalib::stringref pem) { _ca_certs_pem = pem; return *this; }
         Params& cert_chain_pem(vespalib::stringref pem) { _cert_chain_pem = pem; return *this; }
@@ -35,19 +39,14 @@ public:
             _accepted_ciphers = std::move(ciphers);
             return *this;
         }
+        Params& disable_hostname_validation(bool disable) {
+            _disable_hostname_validation = disable;
+            return *this;
+        }
     };
 
     explicit TransportSecurityOptions(Params params);
 
-    TransportSecurityOptions(vespalib::string ca_certs_pem,
-                             vespalib::string cert_chain_pem,
-                             vespalib::string private_key_pem);
-
-    TransportSecurityOptions(vespalib::string ca_certs_pem,
-                             vespalib::string cert_chain_pem,
-                             vespalib::string private_key_pem,
-                             AuthorizedPeers authorized_peers);
-
     ~TransportSecurityOptions();
 
     const vespalib::string& ca_certs_pem() const noexcept { return _ca_certs_pem; }
@@ -55,10 +54,16 @@ public:
     const vespalib::string& private_key_pem() const noexcept { return _private_key_pem; }
     const AuthorizedPeers& authorized_peers() const noexcept { return _authorized_peers; }
 
-    TransportSecurityOptions copy_without_private_key() const {
-        return TransportSecurityOptions(_ca_certs_pem, _cert_chain_pem, "", _authorized_peers);
-    }
+    TransportSecurityOptions copy_without_private_key() const;
     const std::vector<vespalib::string>& accepted_ciphers() const noexcept { return _accepted_ciphers; }
+    bool disable_hostname_validation() const noexcept { return _disable_hostname_validation; }
+
+private:
+    TransportSecurityOptions(vespalib::string ca_certs_pem,
+                             vespalib::string cert_chain_pem,
+                             vespalib::string private_key_pem,
+                             AuthorizedPeers authorized_peers,
+                             bool disable_hostname_validation);
 };
 
 // Zeroes out `size` bytes in `buf` in a way that shall never be optimized
diff --git a/vespalib/src/vespa/vespalib/net/tls/transport_security_options_reading.cpp b/vespalib/src/vespa/vespalib/net/tls/transport_security_options_reading.cpp
index 6b6e65bef8a..80caa15e8b2 100644
--- a/vespalib/src/vespa/vespalib/net/tls/transport_security_options_reading.cpp
+++ b/vespalib/src/vespa/vespalib/net/tls/transport_security_options_reading.cpp
@@ -123,6 +123,12 @@ std::unique_ptr<TransportSecurityOptions> load_from_input(Input& input) {
     auto priv_key = load_file_referenced_by_field(files, "private-key");
     auto authorized_peers = parse_authorized_peers(root["authorized-peers"]);
     auto accepted_ciphers = parse_accepted_ciphers(root["accepted-ciphers"]);
+    // FIXME this is temporary until we know it won't break a bunch of things!
+    // It's still possible to explicitly enable hostname validation by setting this to false.
+    bool disable_hostname_validation = true;
+    if (root["disable-hostname-validation"].valid()) {
+        disable_hostname_validation = root["disable-hostname-validation"].asBool();
+    }
 
     auto options = std::make_unique<TransportSecurityOptions>(
             TransportSecurityOptions::Params()
@@ -130,7 +136,8 @@ std::unique_ptr<TransportSecurityOptions> load_from_input(Input& input) {
                 .cert_chain_pem(certs)
                 .private_key_pem(priv_key)
                 .authorized_peers(std::move(authorized_peers))
-                .accepted_ciphers(std::move(accepted_ciphers)));
+                .accepted_ciphers(std::move(accepted_ciphers))
+                .disable_hostname_validation(disable_hostname_validation));
     secure_memzero(&priv_key[0], priv_key.size());
     return options;
 }
diff --git a/vespalib/src/vespa/vespalib/portal/portal.cpp b/vespalib/src/vespa/vespalib/portal/portal.cpp
index 47719ea4c69..a6d44348e5e 100644
--- a/vespalib/src/vespa/vespalib/portal/portal.cpp
+++ b/vespalib/src/vespa/vespalib/portal/portal.cpp
@@ -143,7 +143,7 @@ Portal::handle_accept(portal::HandleGuard guard, SocketHandle socket)
 {
     socket.set_blocking(false);
     socket.set_keepalive(true);
-    new HttpConnection(std::move(guard), _reactor, _crypto->create_crypto_socket(std::move(socket), true),
+    new HttpConnection(std::move(guard), _reactor, _crypto->create_server_crypto_socket(std::move(socket)),
                        [this](HttpConnection *conn)
                        {
                            handle_http(conn);
diff --git a/vespalib/src/vespa/vespalib/regex/CMakeLists.txt b/vespalib/src/vespa/vespalib/regex/CMakeLists.txt
new file mode 100644
index 00000000000..1034dbf6086
--- /dev/null
+++ b/vespalib/src/vespa/vespalib/regex/CMakeLists.txt
@@ -0,0 +1,10 @@
+# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_library(vespalib_vespalib_regex OBJECT
+    SOURCES
+    regex.cpp
+    DEPENDS
+)
+
+find_package(RE2 REQUIRED)
+# TODO can this be PRIVATE since we don't expose it transitively?
+target_include_directories(vespalib_vespalib_regex PUBLIC ${RE2_INCLUDE_DIR})
diff --git a/vespalib/src/vespa/vespalib/regex/regex.cpp b/vespalib/src/vespa/vespalib/regex/regex.cpp
new file mode 100644
index 00000000000..3229365b753
--- /dev/null
+++ b/vespalib/src/vespa/vespalib/regex/regex.cpp
@@ -0,0 +1,88 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "regex.h"
+#include <re2/re2.h>
+#include <cassert>
+#include <cstdint>
+
+namespace vespalib {
+
+using re2::StringPiece;
+
+// All RE2 instances use a Quiet option to prevent the library from
+// complaining to stderr if pattern compilation fails.
+
+Regex::Regex(std::shared_ptr<const Impl> impl)
+    : _impl(std::move(impl))
+{}
+
+Regex::Regex(const Regex&) = default;
+Regex& Regex::operator=(const Regex&) = default;
+Regex::Regex(Regex&&) noexcept = default;
+Regex& Regex::operator=(Regex&&) noexcept = default;
+
+Regex::~Regex() = default;
+
+class Regex::Impl {
+    RE2 _regex;
+public:
+    Impl(std::string_view pattern, const re2::RE2::Options& opts)
+        : _regex(StringPiece(pattern.data(), pattern.size()), opts)
+    {}
+
+    bool parsed_ok() const noexcept {
+        return _regex.ok();
+    }
+
+    bool partial_match(std::string_view input) const noexcept {
+        assert(input.size() <= INT32_MAX);
+        if (!_regex.ok()) {
+            return false;
+        }
+        return RE2::PartialMatch(StringPiece(input.data(), input.size()), _regex);
+    }
+
+    bool full_match(std::string_view input) const noexcept {
+        assert(input.size() <= INT32_MAX);
+        if (!_regex.ok()) {
+            return false;
+        }
+        return RE2::FullMatch(StringPiece(input.data(), input.size()), _regex);
+    }
+};
+
+Regex Regex::from_pattern(std::string_view pattern, uint32_t opt_mask) {
+    assert(pattern.size() <= INT32_MAX); // StringPiece limitation
+    RE2::Options opts;
+    opts.set_log_errors(false);
+    if ((opt_mask & Options::IgnoreCase) != 0) {
+        opts.set_case_sensitive(false);
+    }
+    return Regex(std::make_shared<const Impl>(pattern, opts));
+}
+
+bool Regex::parsed_ok() const noexcept {
+    return _impl->parsed_ok();
+}
+
+bool Regex::partial_match(std::string_view input) const noexcept {
+    return _impl->partial_match(input);
+}
+
+bool Regex::full_match(std::string_view input) const noexcept {
+    return _impl->full_match(input);
+}
+
+bool Regex::partial_match(std::string_view input, std::string_view pattern) noexcept {
+    assert(pattern.size() <= INT32_MAX);
+    Impl impl(pattern, RE2::Quiet);
+    return impl.partial_match(input);
+}
+
+bool Regex::full_match(std::string_view input, std::string_view pattern) noexcept {
+    assert(pattern.size() <= INT32_MAX);
+    Impl impl(pattern, RE2::Quiet);
+    return impl.full_match(input);
+}
+
+}
diff --git a/vespalib/src/vespa/vespalib/regex/regex.h b/vespalib/src/vespa/vespalib/regex/regex.h
new file mode 100644
index 00000000000..4382d057252
--- /dev/null
+++ b/vespalib/src/vespa/vespalib/regex/regex.h
@@ -0,0 +1,69 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+#include <memory>
+#include <string>
+#include <string_view>
+
+namespace vespalib {
+
+/**
+ * A simple Regex library wrapper which provides for both just-in-time
+ * pattern evaluation as well as pattern precompilation and reuse.
+ *
+ * Robustness and input safety:
+ * The underlying regex engine implementation must ensure that pattern
+ * parsing and input processing is safe to be run on _untrusted_ inputs.
+ * This means the underlying implementation shall provide upper bounds
+ * on both memory and CPU time and may never crash or corrupt the process.
+ *
+ * We currently use Google RE2 under the hood to achieve this.
+ *
+ * Note: due to underlying RE2 limitations, string lengths may
+ * not be longer than INT_MAX.
+ *
+ * Thread safety:
+ * A Regex object is safe to be used from multiple threads.
+ *
+ * Exception safety:
+ * Exceptions shall never be thrown from the regex code itself, neither
+ * at parse time nor at match time (ancillary exceptions _could_ be thrown
+ * from memory allocation failures etc, but we assume that the caller
+ * is running vespamalloc which terminates the process instead, making
+ * the whole thing effectively noexcept).
+ *
+ * If the provided regular expression pattern is malformed, parsing
+ * fails silently; all match functions will return false immediately.
+ */
+class Regex {
+    class Impl;
+    std::shared_ptr<const Impl> _impl; // shared_ptr to allow for cheap copying.
+
+    explicit Regex(std::shared_ptr<const Impl> impl);
+public:
+    // TODO consider using type-safe parameter instead.
+    enum Options {
+        None       = 0,
+        IgnoreCase = 1
+    };
+
+    ~Regex();
+    Regex(const Regex&);
+    Regex& operator=(const Regex&);
+    Regex(Regex&&) noexcept;
+    Regex& operator=(Regex&&) noexcept;
+
+    [[nodiscard]] bool parsed_ok() const noexcept;
+
+    [[nodiscard]] bool partial_match(std::string_view input) const noexcept;
+    [[nodiscard]] bool full_match(std::string_view input) const noexcept;
+
+    static Regex from_pattern(std::string_view pattern, uint32_t opt_flags = Options::None);
+
+    // Utility matchers for non-precompiled expressions.
+    [[nodiscard]] static bool partial_match(std::string_view input, std::string_view pattern) noexcept;
+    [[nodiscard]] static bool full_match(std::string_view input, std::string_view pattern) noexcept;
+};
+
+}
+
diff --git a/vespalib/src/vespa/vespalib/stllike/hash_fun.cpp b/vespalib/src/vespa/vespalib/stllike/hash_fun.cpp
index d8c6c87ecda..5f4fee06c4a 100644
--- a/vespalib/src/vespa/vespalib/stllike/hash_fun.cpp
+++ b/vespalib/src/vespa/vespalib/stllike/hash_fun.cpp
@@ -1,25 +1,20 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include "hash_fun.h"
+#include <xxhash.h>
 
 namespace vespalib {
 
 size_t
 hashValue(const char *str)
 {
-    size_t res = 0;
-    unsigned const char *pt = (unsigned const char *) str;
-    while (*pt != 0) {
-        res = (res << 7) + (res >> 25) + *pt++;
-    }
-    return res;
+    return hashValue(str, strlen(str));
 }
 
 /**
  * @brief Calculate hash value.
  *
- * This is the hash function used by the HashMap class.
- * The hash function is inherited from Fastserver4 / FastLib / pandora.
+ * The hash function XXH64 from xxhash library.
  * @param buf input buffer
  * @param sz input buffer size
  * @return hash value of input
@@ -27,12 +22,7 @@ hashValue(const char *str)
 size_t
 hashValue(const void * buf, size_t sz)
 {
-    size_t res = 0;
-    unsigned const char *pt = (unsigned const char *) buf;
-    for (size_t i(0); i < sz; i++) {
-        res = (res << 7) + (res >> 25) + pt[i];
-    }
-    return res;
+    return XXH64(buf, sz, 0);
 }
 
 }
diff --git a/vespalib/src/vespa/vespalib/stllike/hash_map.h b/vespalib/src/vespa/vespalib/stllike/hash_map.h
index 3dc5de65285..29a5ef01a9f 100644
--- a/vespalib/src/vespa/vespalib/stllike/hash_map.h
+++ b/vespalib/src/vespa/vespalib/stllike/hash_map.h
@@ -7,7 +7,7 @@
 
 namespace vespalib {
 
-template< typename K, typename V, typename H = vespalib::hash<K>, typename EQ = std::equal_to<>, typename M=hashtable_base::prime_modulator >
+template< typename K, typename V, typename H = vespalib::hash<K>, typename EQ = std::equal_to<>, typename M=hashtable_base::and_modulator >
 class hash_map
 {
 public:
diff --git a/vespalib/src/vespa/vespalib/stllike/hash_map.hpp b/vespalib/src/vespa/vespalib/stllike/hash_map.hpp
index 08edcf3c837..61789f6e2be 100644
--- a/vespalib/src/vespa/vespalib/stllike/hash_map.hpp
+++ b/vespalib/src/vespa/vespalib/stllike/hash_map.hpp
@@ -73,11 +73,12 @@ hash_map<K, V, H, EQ, M>::getMemoryUsed() const
     template vespalib::hashtable<K, std::pair<K,V>, H, E, vespalib::Select1st<std::pair<K,V>>, M>::insert_result \
              vespalib::hashtable<K, std::pair<K,V>, H, E, vespalib::Select1st<std::pair<K,V>>, M>::insert(std::pair<K,V> &&); \
     template vespalib::hashtable<K, std::pair<K,V>, H, E, vespalib::Select1st<std::pair<K,V>>, M>::insert_result \
-             vespalib::hashtable<K, std::pair<K,V>, H, E, vespalib::Select1st<std::pair<K,V>>, M>::insertInternal(std::pair<K,V> &&); \
-    template class vespalib::Array<vespalib::hash_node<std::pair<K,V>>>;
+             vespalib::hashtable<K, std::pair<K,V>, H, E, vespalib::Select1st<std::pair<K,V>>, M>::insertInternal(std::pair<K,V> &&);
 
 #define VESPALIB_HASH_MAP_INSTANTIATE_H_E(K, V, H, E) \
-    VESPALIB_HASH_MAP_INSTANTIATE_H_E_M(K, V, H, E, vespalib::hashtable_base::prime_modulator)
+    template class vespalib::Array<vespalib::hash_node<std::pair<K,V>>>; \
+    VESPALIB_HASH_MAP_INSTANTIATE_H_E_M(K, V, H, E, vespalib::hashtable_base::prime_modulator) \
+    VESPALIB_HASH_MAP_INSTANTIATE_H_E_M(K, V, H, E, vespalib::hashtable_base::and_modulator)
 
 #define VESPALIB_HASH_MAP_INSTANTIATE_H(K, V, H) VESPALIB_HASH_MAP_INSTANTIATE_H_E(K, V, H, std::equal_to<>)
 
diff --git a/vespalib/src/vespa/vespalib/stllike/hash_set.h b/vespalib/src/vespa/vespalib/stllike/hash_set.h
index 8d315ebfd07..0c3f2dcb220 100644
--- a/vespalib/src/vespa/vespalib/stllike/hash_set.h
+++ b/vespalib/src/vespa/vespalib/stllike/hash_set.h
@@ -8,7 +8,7 @@
 
 namespace vespalib {
 
-template< typename K, typename H = vespalib::hash<K>, typename EQ = std::equal_to<>, typename M=hashtable_base::prime_modulator>
+template< typename K, typename H = vespalib::hash<K>, typename EQ = std::equal_to<>, typename M=hashtable_base::and_modulator>
 class hash_set
 {
 private:
@@ -42,6 +42,7 @@ public:
     template<typename InputIt>
     void insert(InputIt first, InputIt last);
     void erase(const K & key);
+    size_t count(const K & key) const        { return _ht.find(key) != end() ? 1 : 0; }
     iterator find(const K & key)             { return _ht.find(key); }
     const_iterator find(const K & key) const { return _ht.find(key); }
 
diff --git a/vespalib/src/vespa/vespalib/stllike/hash_set.hpp b/vespalib/src/vespa/vespalib/stllike/hash_set.hpp
index 19114798806..3e48e62f2c7 100644
--- a/vespalib/src/vespa/vespalib/stllike/hash_set.hpp
+++ b/vespalib/src/vespa/vespalib/stllike/hash_set.hpp
@@ -85,11 +85,11 @@ hash_set<K, H, EQ, M>::insert(K &&value) {
 
 #define VESPALIB_HASH_SET_INSTANTIATE(K) \
     template class vespalib::hash_set<K>; \
-    template class vespalib::hashtable<K, K, vespalib::hash<K>, std::equal_to<>, vespalib::Identity>; \
+    template class vespalib::hashtable<K, K, vespalib::hash<K>, std::equal_to<>, vespalib::Identity, vespalib::hashtable_base::and_modulator>; \
     template class vespalib::Array<vespalib::hash_node<K>>;
 
 #define VESPALIB_HASH_SET_INSTANTIATE_H(K, H) \
     template class vespalib::hash_set<K, H>; \
-    template class vespalib::hashtable<K, K, H, std::equal_to<>, vespalib::Identity>; \
+    template class vespalib::hashtable<K, K, H, std::equal_to<>, vespalib::Identity, vespalib::hashtable_base::and_modulator>; \
     template class vespalib::Array<vespalib::hash_node<K>>;
 
diff --git a/vespalib/src/vespa/vespalib/test/make_tls_options_for_testing.cpp b/vespalib/src/vespa/vespalib/test/make_tls_options_for_testing.cpp
index c685bffc23e..ad46e7272cb 100644
--- a/vespalib/src/vespa/vespalib/test/make_tls_options_for_testing.cpp
+++ b/vespalib/src/vespa/vespalib/test/make_tls_options_for_testing.cpp
@@ -1,74 +1,77 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include "make_tls_options_for_testing.h"
+#include <vespa/vespalib/crypto/private_key.h>
+#include <vespa/vespalib/crypto/x509_certificate.h>
 
-/*
- * Generated with the following commands:
- *
- * openssl ecparam -name prime256v1 -genkey -noout -out ca.key
- *
- * openssl req -new -x509 -nodes -key ca.key \
- *    -sha256 -out ca.pem \
- *    -subj '/C=US/L=LooneyVille/O=ACME/OU=ACME test CA/CN=acme.example.com' \
- *    -days 10000
- *
- * openssl ecparam -name prime256v1 -genkey -noout -out host.key
- *
- * openssl req -new -key host.key -out host.csr \
- *    -subj '/C=US/L=LooneyVille/O=Wile. E. Coyote, Ltd./CN=wile.example.com' \
- *    -sha256
- *
- * openssl x509 -req -in host.csr \
- *   -CA ca.pem \
- *   -CAkey ca.key \
- *   -CAcreateserial \
- *   -out host.pem \
- *   -days 10000 \
- *   -sha256
- *
- * TODO generate keypairs and certs at test-time to avoid any hard-coding
- * There certs are valid until 2046, so that buys us some time..!
- */
-
-// ca.pem
-constexpr const char* ca_pem = R"(-----BEGIN CERTIFICATE-----
-MIIBuDCCAV4CCQDpVjQIixTxvDAKBggqhkjOPQQDAjBkMQswCQYDVQQGEwJVUzEU
-MBIGA1UEBwwLTG9vbmV5VmlsbGUxDTALBgNVBAoMBEFDTUUxFTATBgNVBAsMDEFD
-TUUgdGVzdCBDQTEZMBcGA1UEAwwQYWNtZS5leGFtcGxlLmNvbTAeFw0xODA4MzEx
-MDU3NDVaFw00NjAxMTYxMDU3NDVaMGQxCzAJBgNVBAYTAlVTMRQwEgYDVQQHDAtM
-b29uZXlWaWxsZTENMAsGA1UECgwEQUNNRTEVMBMGA1UECwwMQUNNRSB0ZXN0IENB
-MRkwFwYDVQQDDBBhY21lLmV4YW1wbGUuY29tMFkwEwYHKoZIzj0CAQYIKoZIzj0D
-AQcDQgAE1L7IzCN5pbyVnBATIHieuxq+hf9kWyn5yfjkXMhD52T5ITz1huq4nbiN
-YtRoRP7XmipI60R/uiCHzERcsVz4rDAKBggqhkjOPQQDAgNIADBFAiEA6wmZDBca
-y0aJ6ABtjbjx/vlmVDxdkaSZSgO8h2CkvIECIFktCkbZhDFfSvbqUScPOGuwkdGQ
-L/EW2Bxp+1BPcYoZ
------END CERTIFICATE-----)";
-
-// host.pem
-constexpr const char* cert_pem = R"(-----BEGIN CERTIFICATE-----
-MIIBsTCCAVgCCQD6GfDh0ltpsjAKBggqhkjOPQQDAjBkMQswCQYDVQQGEwJVUzEU
-MBIGA1UEBwwLTG9vbmV5VmlsbGUxDTALBgNVBAoMBEFDTUUxFTATBgNVBAsMDEFD
-TUUgdGVzdCBDQTEZMBcGA1UEAwwQYWNtZS5leGFtcGxlLmNvbTAeFw0xODA4MzEx
-MDU3NDVaFw00NjAxMTYxMDU3NDVaMF4xCzAJBgNVBAYTAlVTMRQwEgYDVQQHDAtM
-b29uZXlWaWxsZTEeMBwGA1UECgwVV2lsZS4gRS4gQ295b3RlLCBMdGQuMRkwFwYD
-VQQDDBB3aWxlLmV4YW1wbGUuY29tMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE
-e+Y4hxt66em0STviGUj6ZDbxzoLoubXWRml8JDFrEc2S2433KWw2npxYSKVCyo3a
-/Vo33V8/H0WgOXioKEZJxDAKBggqhkjOPQQDAgNHADBEAiAN+87hQuGv3z0Ja2BV
-b8PHq2vp3BJHjeMuxWu4BFPn0QIgYlvIHikspgGatXRNMZ1gPC0oCccsJFcie+Cw
-zL06UPI=
------END CERTIFICATE-----)";
-
-// host.key
-constexpr const char* key_pem = R"(-----BEGIN EC PRIVATE KEY-----
-MHcCAQEEID6di2PFYn8hPrxPbkFDGkSqF+K8L520In7nx3g0jwzOoAoGCCqGSM49
-AwEHoUQDQgAEe+Y4hxt66em0STviGUj6ZDbxzoLoubXWRml8JDFrEc2S2433KWw2
-npxYSKVCyo3a/Vo33V8/H0WgOXioKEZJxA==
------END EC PRIVATE KEY-----)";
+namespace {
+
+using namespace vespalib::crypto;
+
+struct TransientCryptoCredentials {
+    CertKeyWrapper root_ca;
+    CertKeyWrapper host_creds;
+    vespalib::net::tls::TransportSecurityOptions cached_transport_options;
+
+    TransientCryptoCredentials();
+    ~TransientCryptoCredentials();
+
+    static CertKeyWrapper make_root_ca() {
+        auto dn = X509Certificate::DistinguishedName()
+                .country("US").state("CA").locality("Sunnyvale")
+                .organization("ACME, Inc.")
+                .organizational_unit("ACME Root CA")
+                .add_common_name("acme.example.com");
+        auto subject = X509Certificate::SubjectInfo(std::move(dn));
+        auto key = PrivateKey::generate_p256_ec_key();
+        auto params = X509Certificate::Params::self_signed(std::move(subject), key);
+        auto cert = X509Certificate::generate_from(std::move(params));
+        return {std::move(cert), std::move(key)};
+    }
+
+    static CertKeyWrapper make_host_creds(const CertKeyWrapper& root_ca_creds) {
+        auto dn = X509Certificate::DistinguishedName()
+                .country("US").state("CA").locality("Sunnyvale")
+                .organization("Wile E. Coyote, Ltd.")
+                .organizational_unit("Unit Testing and Anvil Dropping Division")
+                .add_common_name("localhost"); // Should technically not be needed, but including it anyway.
+        auto subject = X509Certificate::SubjectInfo(std::move(dn));
+        subject.add_subject_alt_name("DNS:localhost");
+        auto key = PrivateKey::generate_p256_ec_key();
+        auto params = X509Certificate::Params::issued_by(std::move(subject), key, root_ca_creds.cert, root_ca_creds.key);
+        params.valid_for = std::chrono::hours(1);
+        auto cert = X509Certificate::generate_from(std::move(params));
+        return {std::move(cert), std::move(key)};
+    }
+
+    static const TransientCryptoCredentials& instance();
+};
+
+TransientCryptoCredentials::TransientCryptoCredentials()
+    : root_ca(make_root_ca()),
+      host_creds(make_host_creds(root_ca)),
+      cached_transport_options(vespalib::net::tls::TransportSecurityOptions::Params().
+            ca_certs_pem(root_ca.cert->to_pem()).
+            cert_chain_pem(host_creds.cert->to_pem()).
+            private_key_pem(host_creds.key->private_to_pem()).
+            authorized_peers(vespalib::net::tls::AuthorizedPeers::allow_all_authenticated()))
+{}
+
+TransientCryptoCredentials::~TransientCryptoCredentials() = default;
+
+const TransientCryptoCredentials& TransientCryptoCredentials::instance() {
+    static TransientCryptoCredentials test_creds;
+    return test_creds;
+}
+
+}
 
 namespace vespalib::test {
 
+SocketSpec local_spec("tcp/localhost:123");
+
 vespalib::net::tls::TransportSecurityOptions make_tls_options_for_testing() {
-    return vespalib::net::tls::TransportSecurityOptions(ca_pem, cert_pem, key_pem);
+    return TransientCryptoCredentials::instance().cached_transport_options;
 }
 
 } // namespace vespalib::test
diff --git a/vespalib/src/vespa/vespalib/test/make_tls_options_for_testing.h b/vespalib/src/vespa/vespalib/test/make_tls_options_for_testing.h
index a1f1d5958f9..41e5d7cc86d 100644
--- a/vespalib/src/vespa/vespalib/test/make_tls_options_for_testing.h
+++ b/vespalib/src/vespa/vespalib/test/make_tls_options_for_testing.h
@@ -2,11 +2,19 @@
 
 #pragma once
 
+#include <vespa/vespalib/net/socket_spec.h>
 #include <vespa/vespalib/net/tls/transport_security_options.h>
 
 namespace vespalib::test {
 
 /**
+ * A socket spec representing "tcp/localhost:123". Used by unit tests
+ * performing hostname verification against the tls options created
+ * below.
+ **/
+extern SocketSpec local_spec;
+
+/**
  * Make security options allowing you to talk to yourself using
  * TLS. This is intended for testing purposes only.
  **/
diff --git a/vespalib/src/vespa/vespalib/test/socket_options_verifier.h b/vespalib/src/vespa/vespalib/test/socket_options_verifier.h
index 72694f643e2..8f9901e8acc 100644
--- a/vespalib/src/vespa/vespalib/test/socket_options_verifier.h
+++ b/vespalib/src/vespa/vespalib/test/socket_options_verifier.h
@@ -18,7 +18,7 @@ void verify_bool_opt(int fd, int level, int name, bool expect) {
     socklen_t size = sizeof(data);
     EXPECT_EQUAL(getsockopt(fd, level, name, &data, &size), 0);
     EXPECT_EQUAL(size, sizeof(data));
-    EXPECT_EQUAL(data, int(expect));
+    EXPECT_EQUAL(data != 0, expect);
 }
 
 } // namespace vespalib::test::<unnamed>
diff --git a/vespalib/src/vespa/vespalib/util/CMakeLists.txt b/vespalib/src/vespa/vespalib/util/CMakeLists.txt
index da81556bb61..e3397b54740 100644
--- a/vespalib/src/vespa/vespalib/util/CMakeLists.txt
+++ b/vespalib/src/vespa/vespalib/util/CMakeLists.txt
@@ -36,6 +36,9 @@ vespa_add_library(vespalib_vespalib_util OBJECT
     random.cpp
     rcuvector.cpp
     regexp.cpp
+    reusable_set.cpp
+    reusable_set_handle.cpp
+    reusable_set_pool.cpp
     runnable.cpp
     runnable_pair.cpp
     rwlock.cpp
diff --git a/vespalib/src/vespa/vespalib/util/alloc.cpp b/vespalib/src/vespa/vespalib/util/alloc.cpp
index 6aa96992254..4c0675198bb 100644
--- a/vespalib/src/vespa/vespalib/util/alloc.cpp
+++ b/vespalib/src/vespa/vespalib/util/alloc.cpp
@@ -66,7 +66,11 @@ readOptionalEnvironmentVar(const char * name, size_t defaultValue) {
 
 void initializeEnvironment()
 {
+#ifdef __linux__
     _G_HugeFlags = (getenv("VESPA_USE_HUGEPAGES") != nullptr) ? MAP_HUGETLB : 0;
+#else
+    _G_HugeFlags = 0;
+#endif
     _G_SilenceCoreOnOOM = (getenv("VESPA_SILENCE_CORE_ON_OOM") != nullptr) ? true : false;
     _G_MMapLogLimit = readOptionalEnvironmentVar("VESPA_MMAP_LOG_LIMIT", std::numeric_limits<size_t>::max());
     _G_MMapNoCoreLimit = readOptionalEnvironmentVar("VESPA_MMAP_NOCORE_LIMIT", std::numeric_limits<size_t>::max());
@@ -350,11 +354,13 @@ MMapAllocator::salloc(size_t sz, void * wantedAddress)
                 _G_hasHugePageFailureJustHappened = false;
             }
         }
+#ifdef __linux__
         if (sz >= _G_MMapNoCoreLimit) {
             if (madvise(buf, sz, MADV_DONTDUMP) != 0) {
                 LOG(warning, "Failed madvise(%p, %ld, MADV_DONTDUMP) = '%s'", buf, sz, FastOS_FileInterface::getLastErrorString().c_str());
             }
         }
+#endif
         if (sz >= _G_MMapLogLimit) {
             LockGuard guard(_G_lock);
             _G_HugeMappings[buf] = MMapInfo(mmapId, sz, stackTrace);
diff --git a/vespalib/src/vespa/vespalib/util/blockingthreadstackexecutor.cpp b/vespalib/src/vespa/vespalib/util/blockingthreadstackexecutor.cpp
index 33d6e46a244..b5cf16b4f80 100644
--- a/vespalib/src/vespa/vespalib/util/blockingthreadstackexecutor.cpp
+++ b/vespalib/src/vespa/vespalib/util/blockingthreadstackexecutor.cpp
@@ -39,10 +39,4 @@ BlockingThreadStackExecutor::~BlockingThreadStackExecutor()
     cleanup();
 }
 
-void
-BlockingThreadStackExecutor::setTaskLimit(uint32_t taskLimit)
-{
-    internalSetTaskLimit(taskLimit);
-}
-
 } // namespace vespalib
diff --git a/vespalib/src/vespa/vespalib/util/blockingthreadstackexecutor.h b/vespalib/src/vespa/vespalib/util/blockingthreadstackexecutor.h
index 73a142ae42c..df3f9408e0a 100644
--- a/vespalib/src/vespa/vespalib/util/blockingthreadstackexecutor.h
+++ b/vespalib/src/vespa/vespalib/util/blockingthreadstackexecutor.h
@@ -33,11 +33,6 @@ public:
                                 init_fun_t init_function);
 
     ~BlockingThreadStackExecutor();
-
-    /**
-     * Sets a new upper limit for accepted number of tasks.
-     */
-    void setTaskLimit(uint32_t taskLimit);
 };
 
 } // namespace vespalib
diff --git a/vespalib/src/vespa/vespalib/util/compressor.cpp b/vespalib/src/vespa/vespalib/util/compressor.cpp
index 8dfdac5ecc7..56533a77643 100644
--- a/vespalib/src/vespa/vespalib/util/compressor.cpp
+++ b/vespalib/src/vespa/vespalib/util/compressor.cpp
@@ -5,6 +5,7 @@
 #include <vespa/vespalib/util/memory.h>
 #include <vespa/vespalib/util/stringfmt.h>
 #include <vespa/vespalib/data/databuffer.h>
+#include <stdexcept>
 
 using vespalib::alloc::Alloc;
 
diff --git a/vespalib/src/vespa/vespalib/util/executor.h b/vespalib/src/vespa/vespalib/util/executor.h
index ef5cb4b84fa..a412832ecaf 100644
--- a/vespalib/src/vespa/vespalib/util/executor.h
+++ b/vespalib/src/vespa/vespalib/util/executor.h
@@ -23,6 +23,8 @@ public:
         virtual ~Task() {}
     };
 
+    enum class OptimizeFor {LATENCY, THROUGHPUT, ADAPTIVE};
+
     /**
      * Execute the given task using one of the internal threads some
      * time in the future. The task may also be rejected in which case
diff --git a/vespalib/src/vespa/vespalib/util/executor_stats.h b/vespalib/src/vespa/vespalib/util/executor_stats.h
index e8adfc62807..9b941095c27 100644
--- a/vespalib/src/vespa/vespalib/util/executor_stats.h
+++ b/vespalib/src/vespa/vespalib/util/executor_stats.h
@@ -2,16 +2,73 @@
 
 #pragma once
 
+#include <limits>
+
 namespace vespalib {
 
 /**
+ * Used for aggregating values, preserving min, max, sum and count.
+ */
+template <typename T>
+class AggregatedAverage {
+public:
+    AggregatedAverage() : AggregatedAverage(0ul, T(0), std::numeric_limits<T>::max(), std::numeric_limits<T>::min()) { }
+    explicit AggregatedAverage(T value) : AggregatedAverage(1, value, value, value) { }
+    AggregatedAverage(size_t count_in, T total_in, T min_in, T max_in)
+            : _count(count_in),
+              _total(total_in),
+              _min(min_in),
+              _max(max_in)
+    { }
+    AggregatedAverage & operator += (const AggregatedAverage & rhs) {
+        add(rhs);
+        return *this;
+    }
+    void add(const AggregatedAverage & rhs) {
+        add(rhs._count, rhs._total, rhs._min, rhs._max);
+    }
+    void add(T value) {
+        add(1, value, value, value);
+    }
+    void add(size_t count_in, T total_in, T min_in, T max_in) {
+        _count += count_in;
+        _total += total_in;
+        if (min_in < _min) _min = min_in;
+        if (max_in > _max) _max = max_in;
+    }
+    size_t count() const { return _count; }
+    T total() const { return _total; }
+    T min() const { return _min; }
+    T max() const { return _max; }
+    double average() const { return (_count > 0) ? (double(_total) / _count) : 0; }
+private:
+    size_t _count;
+    T      _total;
+    T      _min;
+    T      _max;
+};
+
+/**
  * Struct representing stats for an executor.
  **/
 struct ExecutorStats {
-    size_t maxPendingTasks;
+    using QueueSizeT = AggregatedAverage<size_t>;
+    QueueSizeT queueSize;
     size_t acceptedTasks;
     size_t rejectedTasks;
-    ExecutorStats() : maxPendingTasks(0), acceptedTasks(0), rejectedTasks(0) {}
+    ExecutorStats() : ExecutorStats(QueueSizeT(), 0, 0) {}
+    ExecutorStats(QueueSizeT queueSize_in, size_t accepted, size_t rejected)
+        : queueSize(queueSize_in), acceptedTasks(accepted), rejectedTasks(rejected)
+    {}
+    ExecutorStats & operator += (const ExecutorStats & rhs) {
+        queueSize = QueueSizeT(queueSize.count() + rhs.queueSize.count(),
+                               queueSize.total() + rhs.queueSize.total(),
+                               queueSize.min() + rhs.queueSize.min(),
+                               queueSize.max() + rhs.queueSize.max());
+        acceptedTasks += rhs.acceptedTasks;
+        rejectedTasks += rhs.rejectedTasks;
+        return *this;
+    }
 };
 
 }
diff --git a/vespalib/src/vespa/vespalib/util/gencnt.cpp b/vespalib/src/vespa/vespalib/util/gencnt.cpp
index 5adbf14a757..ad82cf2e67c 100644
--- a/vespalib/src/vespa/vespalib/util/gencnt.cpp
+++ b/vespalib/src/vespa/vespalib/util/gencnt.cpp
@@ -58,7 +58,7 @@ GenCnt::distance(const GenCnt &other) const
 GenCnt &
 GenCnt::operator=(const GenCnt &src)
 {
-    _val = src._val;
+    _val = src.getAsInt();
     return *this;
 }
 
diff --git a/vespalib/src/vespa/vespalib/util/gencnt.h b/vespalib/src/vespa/vespalib/util/gencnt.h
index 7bfc5a7e49b..cac868a8adb 100644
--- a/vespalib/src/vespa/vespalib/util/gencnt.h
+++ b/vespalib/src/vespa/vespalib/util/gencnt.h
@@ -2,6 +2,7 @@
 #pragma once
 
 #include <cstdint>
+#include <atomic>
 
 namespace vespalib {
 
@@ -16,7 +17,7 @@ namespace vespalib {
 class GenCnt
 {
 private:
-    uint32_t _val;
+    std::atomic<uint32_t> _val;
 public:
 
     /**
@@ -31,12 +32,12 @@ public:
      **/
     GenCnt(uint32_t val) : _val(val) {}
 
-    GenCnt(const GenCnt &rhs) = default;
+    GenCnt(const GenCnt &rhs) : _val(rhs.getAsInt()) {}
 
     /**
      * @brief empty destructor
      **/
-    ~GenCnt() {}
+    ~GenCnt() = default;
 
     /**
      * @brief Increase the generation count held by this object
@@ -95,7 +96,7 @@ public:
      *
      * @return generation counter
      **/
-    uint32_t getAsInt() const { return _val; }
+    uint32_t getAsInt() const { return _val.load(std::memory_order_relaxed); }
 
     /**
      * @brief Set the generation counter from an integer
diff --git a/vespalib/src/vespa/vespalib/util/memoryusage.h b/vespalib/src/vespa/vespalib/util/memoryusage.h
index 84bf39fde10..0ab339d1262 100644
--- a/vespalib/src/vespa/vespalib/util/memoryusage.h
+++ b/vespalib/src/vespa/vespalib/util/memoryusage.h
@@ -35,6 +35,7 @@ public:
     void incAllocatedBytes(size_t inc) { _allocatedBytes += inc; }
     void decAllocatedBytes(size_t dec) { _allocatedBytes -= dec; }
     void incUsedBytes(size_t inc) { _usedBytes += inc; }
+    void decUsedBytes(size_t dec) { _usedBytes -= dec; }
     void incDeadBytes(size_t inc) { _deadBytes += inc; }
     void incAllocatedBytesOnHold(size_t inc) { _allocatedBytesOnHold += inc; }
     void decAllocatedBytesOnHold(size_t inc) { _allocatedBytesOnHold -= inc; }
diff --git a/vespalib/src/vespa/vespalib/util/regexp.cpp b/vespalib/src/vespa/vespalib/util/regexp.cpp
index b3cad06382e..0d0c7b69b12 100644
--- a/vespalib/src/vespa/vespalib/util/regexp.cpp
+++ b/vespalib/src/vespa/vespalib/util/regexp.cpp
@@ -41,7 +41,7 @@ vespalib::string escape(vespalib::stringref str) {
 } // namespace vespalib::<unnamed>
 
 vespalib::string
-Regexp::get_prefix(vespalib::stringref re)
+RegexpUtil::get_prefix(vespalib::stringref re)
 {
     vespalib::string prefix;
     if ((re.size() > 0) && (re.data()[0] == '^') && !has_option(re)) {
@@ -58,13 +58,13 @@ Regexp::get_prefix(vespalib::stringref re)
 }
 
 vespalib::string
-Regexp::make_from_suffix(vespalib::stringref suffix)
+RegexpUtil::make_from_suffix(vespalib::stringref suffix)
 {
     return escape(suffix) + "$";
 }
 
 vespalib::string
-Regexp::make_from_substring(vespalib::stringref substring)
+RegexpUtil::make_from_substring(vespalib::stringref substring)
 {
     return escape(substring);
 }
diff --git a/vespalib/src/vespa/vespalib/util/regexp.h b/vespalib/src/vespa/vespalib/util/regexp.h
index 9897b488aff..74a69fee361 100644
--- a/vespalib/src/vespa/vespalib/util/regexp.h
+++ b/vespalib/src/vespa/vespalib/util/regexp.h
@@ -8,7 +8,7 @@ namespace vespalib {
 /**
  * Utility class inspecting and generating regular expression strings.
  **/
-class Regexp
+class RegexpUtil
 {
 public:
     /**
diff --git a/vespalib/src/vespa/vespalib/util/reusable_set.cpp b/vespalib/src/vespa/vespalib/util/reusable_set.cpp
new file mode 100644
index 00000000000..395fbf505ba
--- /dev/null
+++ b/vespalib/src/vespa/vespalib/util/reusable_set.cpp
@@ -0,0 +1,3 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "reusable_set.h"
diff --git a/vespalib/src/vespa/vespalib/util/reusable_set.h b/vespalib/src/vespa/vespalib/util/reusable_set.h
new file mode 100644
index 00000000000..92a68a68bd6
--- /dev/null
+++ b/vespalib/src/vespa/vespalib/util/reusable_set.h
@@ -0,0 +1,67 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "array.h"
+#include "array.hpp"
+#include <memory>
+#include <cstring>
+
+
+namespace vespalib {
+
+/**
+ * Generational marker implementation of a vector of boolean values.
+ * Limited API, used for marking "seen" nodes when exploring a graph.
+ **/
+class ReusableSet
+
+{
+public:
+    using Mark = unsigned short;
+
+    explicit ReusableSet(size_t size)
+      : _array(size),
+        _curval(-1),
+        _sz(size)
+    {
+        clear();
+    }
+
+    ~ReusableSet() {
+    }
+
+    /**
+     * Increments the generation value, only
+     * initializing the underlying memory when it wraps
+     **/
+    void clear() {
+        if (++_curval == 0) {
+            memset(bits(), 0, _sz * sizeof(Mark));
+            ++_curval;
+        }
+    }
+
+    void mark(size_t id) {
+        _array[id] = _curval;
+    }
+
+    bool is_marked(size_t id) const {
+        return (_array[id] == _curval);
+    }
+
+    Mark *bits() { return _array.begin(); }
+    Mark generation() const { return _curval; }
+    size_t capacity() const { return _sz; }
+
+    size_t memory_usage() const {
+        return (_sz * sizeof(Mark)) + sizeof(ReusableSet);
+    }
+
+private:
+    Array<Mark> _array;
+    Mark _curval;
+    size_t _sz;
+};
+
+} // namespace
diff --git a/vespalib/src/vespa/vespalib/util/reusable_set_handle.cpp b/vespalib/src/vespa/vespalib/util/reusable_set_handle.cpp
new file mode 100644
index 00000000000..e69fc1b8abd
--- /dev/null
+++ b/vespalib/src/vespa/vespalib/util/reusable_set_handle.cpp
@@ -0,0 +1,13 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "reusable_set_handle.h"
+#include "reusable_set_pool.h"
+
+namespace vespalib {
+
+ReusableSetHandle::~ReusableSetHandle()
+{
+    _pool.reuse(std::move(_owned));
+}
+
+} // namespace
diff --git a/vespalib/src/vespa/vespalib/util/reusable_set_handle.h b/vespalib/src/vespa/vespalib/util/reusable_set_handle.h
new file mode 100644
index 00000000000..be230fdc60d
--- /dev/null
+++ b/vespalib/src/vespa/vespalib/util/reusable_set_handle.h
@@ -0,0 +1,52 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "reusable_set.h"
+
+namespace vespalib {
+
+class ReusableSetPool;
+
+/**
+ * Wraps a ReusableSet allocated from a ReusableSetPool.
+ * Note that the handle returns the wrapped set to the pool
+ * on destruction.
+ **/
+class ReusableSetHandle
+{
+private:
+    using Mark = ReusableSet::Mark;
+    using RSUP = std::unique_ptr<ReusableSet>;
+
+    Mark *_bits;
+    Mark _curval;
+    RSUP _owned;
+    ReusableSetPool &_pool;
+
+public:
+    ReusableSetHandle(RSUP backing, ReusableSetPool& owner)
+      : _bits(backing->bits()),
+        _curval(backing->generation()),
+        _owned(std::move(backing)),
+        _pool(owner)
+    {}
+
+    ~ReusableSetHandle();
+
+    /** mark an ID */
+    void mark(size_t id) {
+        _bits[id] = _curval;
+    }
+
+    /** check if an ID has been marked */
+    bool is_marked(size_t id) const {
+        return (_bits[id] == _curval);
+    }
+
+    // for unit tests and statistics
+    size_t capacity() const { return _owned->capacity(); }
+    Mark generation() const { return _curval; }
+};
+
+} // namespace
diff --git a/vespalib/src/vespa/vespalib/util/reusable_set_pool.cpp b/vespalib/src/vespa/vespalib/util/reusable_set_pool.cpp
new file mode 100644
index 00000000000..ed4c8a4af46
--- /dev/null
+++ b/vespalib/src/vespa/vespalib/util/reusable_set_pool.cpp
@@ -0,0 +1,3 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "reusable_set_pool.h"
diff --git a/vespalib/src/vespa/vespalib/util/reusable_set_pool.h b/vespalib/src/vespa/vespalib/util/reusable_set_pool.h
new file mode 100644
index 00000000000..1af2b76dedc
--- /dev/null
+++ b/vespalib/src/vespa/vespalib/util/reusable_set_pool.h
@@ -0,0 +1,86 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "reusable_set.h"
+#include "reusable_set_handle.h"
+#include "memoryusage.h"
+
+#include <vector>
+#include <mutex>
+
+namespace vespalib {
+
+/**
+ * A resource pool for ReusableSet instances.
+ * Note that the pool should have a guaranteed lifetime
+ * that is longer than any Handle retrieved from the pool.
+ **/
+class ReusableSetPool
+{
+    using RSUP = std::unique_ptr<ReusableSet>;
+    using Guard = std::lock_guard<std::mutex>;
+    std::vector<RSUP> _lru_stack;
+    mutable std::mutex _lock;
+    size_t _reuse_count;
+    size_t _create_count;
+    MemoryUsage _total_memory;
+    const size_t _min_size;
+    const size_t _grow_percent;
+
+    ReusableSetPool(const ReusableSetPool &) = delete;
+    ReusableSetPool& operator= (const ReusableSetPool &) = delete;
+
+public:
+    ReusableSetPool()
+      : _lru_stack(), _lock(),
+        _reuse_count(0), _create_count(0),
+        _total_memory(),
+        _min_size(248), _grow_percent(20)
+    {
+        _total_memory.incAllocatedBytes(sizeof(ReusableSetPool));
+    }
+
+    /** Create or re-use a set with (at least) the given size. */
+    ReusableSetHandle get(size_t size) {
+        Guard guard(_lock);
+        size_t last_used_size = 0;
+        while (! _lru_stack.empty()) {
+            RSUP r = std::move(_lru_stack.back());
+            _lru_stack.pop_back();
+            if (r->capacity() >= size) {
+                r->clear();
+                ++_reuse_count;
+                _total_memory.incUsedBytes(r->memory_usage());
+                return ReusableSetHandle(std::move(r), *this);
+            }
+            _total_memory.decAllocatedBytes(r->memory_usage());
+            last_used_size = std::max(last_used_size, r->capacity());
+        }
+        double grow_factor = (1.0 + _grow_percent / 100.0);
+        last_used_size *= grow_factor;
+        size_t at_least_size = std::max(_min_size, last_used_size);
+        RSUP r = std::make_unique<ReusableSet>(std::max(at_least_size, size));
+        _total_memory.incAllocatedBytes(r->memory_usage());
+        ++_create_count;
+        _total_memory.incUsedBytes(r->memory_usage());
+        return ReusableSetHandle(std::move(r), *this);
+    }
+
+    /** Return a ReusableSet to the pool. */
+    void reuse(RSUP used) {
+        Guard guard(_lock);
+        _total_memory.decUsedBytes(used->memory_usage());
+        _lru_stack.push_back(std::move(used));
+    }
+
+    // for unit testing and statistics
+    size_t reuse_count() const { return _reuse_count; }
+    size_t create_count() const { return _create_count; }
+    MemoryUsage memory_usage() const {
+        Guard guard(_lock);
+        return _total_memory;
+    }
+};
+
+} // namespace
diff --git a/vespalib/src/vespa/vespalib/util/signalhandler.cpp b/vespalib/src/vespa/vespalib/util/signalhandler.cpp
index 21543ef10d8..c4fb7cfa517 100644
--- a/vespalib/src/vespa/vespalib/util/signalhandler.cpp
+++ b/vespalib/src/vespa/vespalib/util/signalhandler.cpp
@@ -2,6 +2,11 @@
 
 #include "signalhandler.h"
 #include <cassert>
+#include <atomic>
+#include <chrono>
+#include <thread>
+
+using namespace std::chrono_literals;
 
 namespace vespalib {
 
@@ -9,6 +14,9 @@ std::vector<SignalHandler*> SignalHandler::_handlers;
 
 namespace {
 
+// 31 bit concurrency counter, 1 (lsb) bit indicating shutdown
+std::atomic<int> signal_counter;
+
 class Shutdown
 {
 public:
@@ -19,9 +27,6 @@ public:
 
 }
 
-// Clear SignalHandler::_handlers in a slightly less unsafe manner.
-Shutdown shutdown;
-
 SignalHandler SignalHandler::HUP(SIGHUP);
 SignalHandler SignalHandler::INT(SIGINT);
 SignalHandler SignalHandler::TERM(SIGTERM);
@@ -36,12 +41,19 @@ SignalHandler SignalHandler::FPE(SIGFPE);
 SignalHandler SignalHandler::QUIT(SIGQUIT);
 SignalHandler SignalHandler::USR1(SIGUSR1);
 
+// Clear SignalHandler::_handlers in a slightly less unsafe manner.
+Shutdown shutdown;
+
 void
 SignalHandler::handleSignal(int signal)
 {
-    if ((((size_t)signal) < _handlers.size()) && (_handlers[signal] != 0)) {
-        _handlers[signal]->gotSignal();
+    static_assert(std::atomic<int>::is_always_lock_free, "signal_counter must be lock free");
+    if ((signal_counter.fetch_add(2) & 1) == 0) {
+        if ((((size_t)signal) < _handlers.size()) && (_handlers[signal] != 0)) {
+            _handlers[signal]->gotSignal();
+        }
     }
+    signal_counter.fetch_sub(2);
 }
 
 void
@@ -108,12 +120,21 @@ SignalHandler::unhook()
 void
 SignalHandler::shutdown()
 {
+    while ((signal_counter.fetch_or(1) & ~1) != 0) {
+        std::this_thread::sleep_for(10ms);
+    }
     for (std::vector<SignalHandler*>::iterator
              it = _handlers.begin(), ite = _handlers.end();
          it != ite;
          ++it) {
-        if (*it != nullptr)
-            (*it)->unhook();
+        if (*it != nullptr) {
+            // Ignore SIGTERM at shutdown in case valgrind is used.
+            if ((*it)->_signal == SIGTERM) {
+                (*it)->ignore();
+            } else {
+                (*it)->unhook();
+            }
+        }
     }
     std::vector<SignalHandler *>().swap(_handlers);
 }
diff --git a/vespalib/src/vespa/vespalib/util/stash.cpp b/vespalib/src/vespa/vespalib/util/stash.cpp
index 18ed0e56e27..836654c2fb2 100644
--- a/vespalib/src/vespa/vespalib/util/stash.cpp
+++ b/vespalib/src/vespa/vespalib/util/stash.cpp
@@ -59,14 +59,14 @@ Stash::do_alloc(size_t size)
     }
 }
 
-Stash::Stash(size_t chunk_size)
+Stash::Stash(size_t chunk_size) noexcept
     : _chunks(nullptr),
       _cleanup(nullptr),
       _chunk_size(std::max(size_t(4096), chunk_size))
 {
 }
 
-Stash::Stash(Stash &&rhs)
+Stash::Stash(Stash &&rhs) noexcept
     : _chunks(rhs._chunks),
       _cleanup(rhs._cleanup),
       _chunk_size(rhs._chunk_size)
@@ -76,7 +76,7 @@ Stash::Stash(Stash &&rhs)
 }
 
 Stash &
-Stash::operator=(Stash &&rhs)
+Stash::operator=(Stash &&rhs) noexcept
 {
     stash::run_cleanup(_cleanup);
     stash::free_chunks(_chunks);
diff --git a/vespalib/src/vespa/vespalib/util/stash.h b/vespalib/src/vespa/vespalib/util/stash.h
index 6804b096ee3..c5e8631ca9e 100644
--- a/vespalib/src/vespa/vespalib/util/stash.h
+++ b/vespalib/src/vespa/vespalib/util/stash.h
@@ -4,6 +4,7 @@
 
 #include "traits.h"
 #include "arrayref.h"
+#include <cstdlib>
 
 namespace vespalib {
 namespace stash {
@@ -13,19 +14,19 @@ struct Cleanup {
     explicit Cleanup(Cleanup *next_in) noexcept : next(next_in) {}
     virtual void cleanup() = 0;
 protected:
-    virtual ~Cleanup() {}
+    virtual ~Cleanup() = default;
 };
 
 // used as header for memory allocated outside the stash
 struct DeleteMemory : public Cleanup {
     explicit DeleteMemory(Cleanup *next_in) noexcept : Cleanup(next_in) {}
-    virtual void cleanup() override { free((void*)this); }
+    void cleanup() override { free((void*)this); }
 };
 
 // used as prefix for objects to be destructed
 template<typename T> struct DestructObject : public Cleanup {
     explicit DestructObject(Cleanup *next_in) noexcept : Cleanup(next_in) {}
-    virtual void cleanup() override { reinterpret_cast<T*>(this + 1)->~T(); }
+    void cleanup() override { reinterpret_cast<T*>(this + 1)->~T(); }
 };
 
 // used as prefix for arrays to be destructed
@@ -33,7 +34,7 @@ template<typename T> struct DestructArray : public Cleanup {
     size_t size;
     explicit DestructArray(Cleanup *next_in, size_t size_in) noexcept
         : Cleanup(next_in), size(size_in) {}
-    virtual void cleanup() override {
+    void cleanup() override {
         T *array = reinterpret_cast<T*>(this + 1);
         for (size_t i = size; i-- > 0;) {
             array[i].~T();
@@ -45,7 +46,7 @@ struct Chunk {
     Chunk  *next;
     size_t  used;
     Chunk(const Chunk &) = delete;
-    Chunk(Chunk *next_in) : next(next_in), used(sizeof(Chunk)) {}
+    explicit Chunk(Chunk *next_in) : next(next_in), used(sizeof(Chunk)) {}
     void clear() { used = sizeof(Chunk); }
     char *alloc(size_t size, size_t chunk_size) {
         size_t aligned_size = ((size + (sizeof(char *) - 1))
@@ -123,14 +124,14 @@ public:
     };
 
     typedef std::unique_ptr<Stash> UP;
-    explicit Stash(size_t chunk_size);
-    Stash() : Stash(4096) {}
-    Stash(Stash &&rhs);
+    explicit Stash(size_t chunk_size) noexcept ;
+    Stash() noexcept : Stash(4096) {}
+    Stash(Stash &&rhs) noexcept;
     Stash(const Stash &) = delete;
     Stash & operator = (const Stash &) = delete;
     ~Stash();
 
-    Stash &operator=(Stash &&rhs);
+    Stash &operator=(Stash &&rhs) noexcept;
 
     void clear();
 
diff --git a/vespalib/src/vespa/vespalib/util/stringfmt.cpp b/vespalib/src/vespa/vespalib/util/stringfmt.cpp
index 266fb60714d..3c5ebce355c 100644
--- a/vespalib/src/vespa/vespalib/util/stringfmt.cpp
+++ b/vespalib/src/vespa/vespalib/util/stringfmt.cpp
@@ -50,4 +50,15 @@ vespalib::string make_string(const char *fmt, ...)
     return ret;
 }
 
+namespace make_string_short {
+vespalib::string fmt(const char *format, ...)
+{
+    va_list ap;
+    va_start(ap, format);
+    vespalib::string ret = make_string_va(format, ap);
+    va_end(ap);
+    return ret;
+}
+} // namespace vespalib::make_string_short
+
 } // namespace vespalib
diff --git a/vespalib/src/vespa/vespalib/util/stringfmt.h b/vespalib/src/vespa/vespalib/util/stringfmt.h
index e9ec1c433bb..1009f3d9275 100644
--- a/vespalib/src/vespa/vespalib/util/stringfmt.h
+++ b/vespalib/src/vespa/vespalib/util/stringfmt.h
@@ -16,5 +16,13 @@ extern vespalib::string make_string(const char *fmt, ...)
 #endif
     ;
 
-} // namespace vespalib
+namespace make_string_short {
+extern vespalib::string fmt(const char *format, ...)
+#ifdef __GNUC__
+        // Add printf format checks with gcc
+        __attribute__ ((format (printf,1,2)))
+#endif
+    ;
+} // namespace vespalib::make_string_short
 
+} // namespace vespalib
diff --git a/vespalib/src/vespa/vespalib/util/threadexecutor.h b/vespalib/src/vespa/vespalib/util/threadexecutor.h
index 2dcbb595bb3..61a5d9d5ac7 100644
--- a/vespalib/src/vespa/vespalib/util/threadexecutor.h
+++ b/vespalib/src/vespa/vespalib/util/threadexecutor.h
@@ -4,6 +4,7 @@
 
 #include "executor.h"
 #include "syncable.h"
+#include "executor_stats.h"
 
 namespace vespalib {
 
@@ -11,10 +12,26 @@ class ThreadExecutor : public Executor
 {
 public:
     /**
+     * Internal stats that we want to observe externally. Note that
+     * all stats are reset each time they are observed.
+     **/
+    using Stats = ExecutorStats;
+    /**
      * Get number of threads in the executor pool.
      * @return number of threads in the pool
      */
     virtual size_t getNumThreads() const = 0;
+
+    /**
+     * Observe and reset stats for this object.
+     * @return stats
+     **/
+    virtual Stats getStats() = 0;
+
+    /**
+     * Sets a new upper limit for accepted number of tasks.
+     */
+    virtual void setTaskLimit(uint32_t taskLimit) = 0;
 };
 
 /**
@@ -23,6 +40,7 @@ public:
 class SyncableThreadExecutor : public ThreadExecutor, public Syncable
 {
 public:
+    virtual SyncableThreadExecutor & shutdown() = 0;
 };
 
 } // namespace vespalib
diff --git a/vespalib/src/vespa/vespalib/util/threadstackexecutorbase.cpp b/vespalib/src/vespa/vespalib/util/threadstackexecutorbase.cpp
index e933477bfb9..efb1dbf4054 100644
--- a/vespalib/src/vespa/vespalib/util/threadstackexecutorbase.cpp
+++ b/vespalib/src/vespa/vespalib/util/threadstackexecutorbase.cpp
@@ -168,6 +168,12 @@ size_t ThreadStackExecutorBase::getNumThreads() const {
 }
 
 void
+ThreadStackExecutorBase::setTaskLimit(uint32_t taskLimit)
+{
+    internalSetTaskLimit(taskLimit);
+}
+
+void
 ThreadStackExecutorBase::internalSetTaskLimit(uint32_t taskLimit)
 {
     MonitorGuard monitor(_monitor);
@@ -190,7 +196,7 @@ ThreadStackExecutorBase::getStats()
     LockGuard lock(_monitor);
     Stats stats = _stats;
     _stats = Stats();
-    _stats.maxPendingTasks = _taskCount;
+    _stats.queueSize.add(_taskCount);
     return stats;
 }
 
@@ -202,8 +208,7 @@ ThreadStackExecutorBase::execute(Task::UP task)
         TaggedTask taggedTask(std::move(task), _barrier.startEvent());
         ++_taskCount;
         ++_stats.acceptedTasks;
-        _stats.maxPendingTasks = (_taskCount > _stats.maxPendingTasks)
-                                 ?_taskCount : _stats.maxPendingTasks;
+        _stats.queueSize.add(_taskCount);
         if (!_workers.empty()) {
             Worker *worker = _workers.back();
             _workers.popBack();
diff --git a/vespalib/src/vespa/vespalib/util/threadstackexecutorbase.h b/vespalib/src/vespa/vespalib/util/threadstackexecutorbase.h
index 8718b04d2d3..6333a8fc66e 100644
--- a/vespalib/src/vespa/vespalib/util/threadstackexecutorbase.h
+++ b/vespalib/src/vespa/vespalib/util/threadstackexecutorbase.h
@@ -8,10 +8,8 @@
 #include "sync.h"
 #include "gate.h"
 #include "runnable.h"
-#include <memory>
 #include <vector>
 #include <functional>
-#include "executor_stats.h"
 
 class FastOS_ThreadPool;
 
@@ -36,12 +34,6 @@ class ThreadStackExecutorBase : public SyncableThreadExecutor,
                                 public Runnable
 {
 public:
-    /**
-     * Internal stats that we want to observe externally. Note that
-     * all stats are reset each time they are observed.
-     **/
-    using Stats = ExecutorStats;
-
     using init_fun_t = std::function<int(Runnable&)>;
 
 private:
@@ -204,14 +196,8 @@ public:
      **/
     size_t num_idle_workers() const;
 
-    /**
-     * Observe and reset stats for this object.
-     *
-     * @return stats
-     **/
-    Stats getStats();
+    Stats getStats() override;
 
-    // inherited from Executor
     Task::UP execute(Task::UP task) override;
 
     /**
@@ -232,6 +218,7 @@ public:
     void wait_for_task_count(uint32_t task_count);
 
     size_t getNumThreads() const override;
+    void setTaskLimit(uint32_t taskLimit) override;
 
     /**
      * Shut down this executor. This will make this executor reject
@@ -239,7 +226,7 @@ public:
      *
      * @return this object; for chaining
      **/
-    ThreadStackExecutorBase &shutdown();
+    ThreadStackExecutorBase &shutdown() override;
 
     /**
      * Will invoke shutdown then sync.
diff --git a/vespalog/abi-spec.json b/vespalog/abi-spec.json
index 09ac3fa75d3..996cc0259a0 100644
--- a/vespalog/abi-spec.json
+++ b/vespalog/abi-spec.json
@@ -186,19 +186,6 @@
     ],
     "fields": []
   },
-  "com.yahoo.log.MappedLevelControllerRepo": {
-    "superClass": "java.lang.Object",
-    "interfaces": [],
-    "attributes": [
-      "public"
-    ],
-    "methods": [
-      "public void <init>(java.nio.MappedByteBuffer, int, int, java.lang.String)",
-      "public com.yahoo.log.LevelController getLevelController(java.lang.String)",
-      "public void checkBack()"
-    ],
-    "fields": []
-  },
   "com.yahoo.log.RejectFilter": {
     "superClass": "java.lang.Object",
     "interfaces": [],
@@ -300,15 +287,10 @@
       "public"
     ],
     "methods": [
-      "public void <init>(java.lang.String, java.lang.String, java.lang.String)",
-      "public com.yahoo.log.LevelController getLevelControl(java.lang.String)",
       "public com.yahoo.log.LevelController getLevelController(java.lang.String)",
       "public void close()"
     ],
-    "fields": [
-      "public static final int controlFileHeaderLength",
-      "public static final int numLevels"
-    ]
+    "fields": []
   },
   "com.yahoo.log.event.Collection": {
     "superClass": "com.yahoo.log.event.Event",
diff --git a/vespalog/src/main/java/com/yahoo/log/LevelController.java b/vespalog/src/main/java/com/yahoo/log/LevelController.java
index ccd18f126d6..0efe0d4e7c1 100644
--- a/vespalog/src/main/java/com/yahoo/log/LevelController.java
+++ b/vespalog/src/main/java/com/yahoo/log/LevelController.java
@@ -1,4 +1,8 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.log;
+
+import java.util.logging.Level;
+
 /**
  * This is the interface for controlling the log level of a
  * component logger.  This hides the actual controlling
@@ -7,32 +11,24 @@
  * @author arnej27959
  *
  */
-
-/**
- * @author arnej27959
- **/
-package com.yahoo.log;
-
-import java.util.logging.Level;
-
 public interface LevelController {
 
     /**
      * should we actually publish a log message with the given Level now?
-     **/
-    public boolean shouldLog(Level level);
+     */
+    boolean shouldLog(Level level);
 
     /**
      * return a string suitable for printing in a logctl file.
      * the string must be be 4 * 8 characters, where each group
      * of 4 characters is either "  ON" or " OFF".
-     **/
-    public String getOnOffString();
+     */
+    String getOnOffString();
 
     /**
      * check the current state of logging and reflect it into the
      * associated Logger instance, if available.
-     **/
-    public void checkBack();
-    public Level getLevelLimit();
+     */
+    void checkBack();
+    Level getLevelLimit();
 }
diff --git a/vespalog/src/main/java/com/yahoo/log/MappedLevelControllerRepo.java b/vespalog/src/main/java/com/yahoo/log/MappedLevelControllerRepo.java
index f02d8793b23..53f4de4f264 100644
--- a/vespalog/src/main/java/com/yahoo/log/MappedLevelControllerRepo.java
+++ b/vespalog/src/main/java/com/yahoo/log/MappedLevelControllerRepo.java
@@ -13,14 +13,14 @@ import java.util.Map;
  * @author Ulf Lilleengen
  * @since 5.1
  */
-public class MappedLevelControllerRepo {
+class MappedLevelControllerRepo {
     private final Map<String, LevelController> levelControllerMap = new HashMap<>();
     private final MappedByteBuffer mapBuf;
     private final int controlFileHeaderLength;
     private final int numLevels;
     private final String logControlFilename;
 
-    public MappedLevelControllerRepo(MappedByteBuffer mapBuf, int controlFileHeaderLength, int numLevels, String logControlFilename) {
+    MappedLevelControllerRepo(MappedByteBuffer mapBuf, int controlFileHeaderLength, int numLevels, String logControlFilename) {
         this.mapBuf = mapBuf;
         this.controlFileHeaderLength = controlFileHeaderLength;
         this.numLevels = numLevels;
@@ -101,12 +101,12 @@ public class MappedLevelControllerRepo {
         return MappedLevelController.checkOnOff(mapBuf, levstart);
     }
 
-    public LevelController getLevelController(String suffix) {
+    LevelController getLevelController(String suffix) {
 
         return levelControllerMap.get(suffix);
     }
 
-    public void checkBack() {
+    void checkBack() {
         for (LevelController ctrl : levelControllerMap.values()) {
             ctrl.checkBack();
         }
diff --git a/vespalog/src/main/java/com/yahoo/log/VespaLevelControllerRepo.java b/vespalog/src/main/java/com/yahoo/log/VespaLevelControllerRepo.java
index 85d92075827..8526d9cc502 100644
--- a/vespalog/src/main/java/com/yahoo/log/VespaLevelControllerRepo.java
+++ b/vespalog/src/main/java/com/yahoo/log/VespaLevelControllerRepo.java
@@ -4,6 +4,7 @@ package com.yahoo.log;
 import com.yahoo.text.Utf8;
 
 import java.io.RandomAccessFile;
+import java.io.FileOutputStream;
 import java.nio.MappedByteBuffer;
 import java.nio.channels.FileChannel;
 import java.util.Enumeration;
@@ -11,6 +12,8 @@ import java.util.TimerTask;
 import java.util.logging.LogManager;
 import java.util.logging.Logger;
 
+import static java.nio.charset.StandardCharsets.US_ASCII;
+
 /**
  * @author Ulf Lilleengen
  * @since 5.1
@@ -18,6 +21,7 @@ import java.util.logging.Logger;
 public class VespaLevelControllerRepo implements LevelControllerRepo {
 
     private RandomAccessFile ctlFile;
+    private FileOutputStream ctlFileAppender;
     private MappedByteBuffer mapBuf;
     private MappedLevelControllerRepo levelControllerRepo;
     private final String logControlFilename;
@@ -30,12 +34,12 @@ public class VespaLevelControllerRepo implements LevelControllerRepo {
     /**
      * length of fixed header content of a control file, constant:
      **/
-    public static final int controlFileHeaderLength;
+    static final int controlFileHeaderLength;
     /**
      * number of distinctly controlled levels (in logctl files),
      * must be compatible with C++ Vespa logging
      **/
-    public static final int numLevels = 8;
+    static final int numLevels = 8;
 
     static {
         controlFileHeaderLength = CFHEADER.length()
@@ -50,7 +54,7 @@ public class VespaLevelControllerRepo implements LevelControllerRepo {
      **/
     private LevelController defaultLevelCtrl;
 
-    public VespaLevelControllerRepo(String logCtlFn, String logLevel, String applicationPrefix) {
+    VespaLevelControllerRepo(String logCtlFn, String logLevel, String applicationPrefix) {
         this.logControlFilename = logCtlFn;
         this.appPrefix = applicationPrefix;
         defaultLevelCtrl = new DefaultLevelController(logLevel);
@@ -66,6 +70,12 @@ public class VespaLevelControllerRepo implements LevelControllerRepo {
             }
         } catch (java.io.IOException ign) {}
         ctlFile = null;
+        try {
+            if (ctlFileAppender != null) {
+                ctlFileAppender.close();
+            }
+        } catch (java.io.IOException ign) {}
+        ctlFileAppender = null;
         mapBuf = null;
         levelControllerRepo = null;
     }
@@ -84,6 +94,7 @@ public class VespaLevelControllerRepo implements LevelControllerRepo {
 
         try {
             ctlFile = new RandomAccessFile(logControlFilename, "rw");
+            ctlFileAppender = new FileOutputStream(logControlFilename, true);
             ensureHeader();
             extendMapping();
 
@@ -107,26 +118,26 @@ public class VespaLevelControllerRepo implements LevelControllerRepo {
         if (l != hbytes.length
             || !java.util.Arrays.equals(hbytes, rbytes))
         {
-            ctlFile.seek(0);
-            ctlFile.write(hbytes);
-            ctlFile.writeBytes(CFPREPRE);
+            StringBuilder sb = new StringBuilder();
+            sb.append(CFHEADER);
+            sb.append(CFPREPRE);
             int appLen = 0;
             if (appPrefix != null) {
                 appLen = appPrefix.length();
-                ctlFile.writeBytes(appPrefix);
+                sb.append(appPrefix);
             }
-            ctlFile.writeBytes("\n");
-            for (int i = appLen; i < maxPrefix; i++) {
-                byte space = ' ';
-                ctlFile.write(space);
+            sb.append('\n');
+            for (int i = appLen; i < maxPrefix + 2; i++) {
+                sb.append(' ');
             }
-            ctlFile.writeBytes("\n");
+            sb.append('\n');
+            ctlFile.write(sb.toString().getBytes(US_ASCII));
             ctlFile.setLength(ctlFile.getFilePointer());
-            if (ctlFile.getFilePointer() != controlFileHeaderLength) {
+            if (ctlFile.getFilePointer() != (controlFileHeaderLength + 2)) {
                 System.err.println("internal error, bad header length: "
                                    + ctlFile.getFilePointer()
                                    + " (should have been: "
-                                   + controlFileHeaderLength
+                                   + (controlFileHeaderLength + 2)
                                    + ")");
             }
         }
@@ -142,7 +153,7 @@ public class VespaLevelControllerRepo implements LevelControllerRepo {
         levelControllerRepo = new MappedLevelControllerRepo(mapBuf, controlFileHeaderLength, numLevels, logControlFilename);
     }
 
-    public LevelController getLevelControl(String suffix) {
+    private LevelController getLevelControl(String suffix) {
         LevelController ctrl = null;
         if (levelControllerRepo != null) {
             if (suffix == null || suffix.equals("default")) {
@@ -181,8 +192,10 @@ public class VespaLevelControllerRepo implements LevelControllerRepo {
                         sb.append(" ");
                     }
                     sb.append(inherit.getOnOffString()).append("\n");
+                    byte[] lineBytes = sb.toString().getBytes(US_ASCII);
+                    ctlFileAppender.write(lineBytes);
+                    ctlFileAppender.flush();
                     ctlFile.seek(ctlFile.length());
-                    ctlFile.writeBytes(sb.toString());
                     extendMapping();
                     ctrl = levelControllerRepo.getLevelController(suffix);
                 } catch(java.nio.channels.ClosedByInterruptException e) {
diff --git a/vespalog/src/main/java/com/yahoo/log/VespaLogHandler.java b/vespalog/src/main/java/com/yahoo/log/VespaLogHandler.java
index 331780f226b..32b1003c20c 100644
--- a/vespalog/src/main/java/com/yahoo/log/VespaLogHandler.java
+++ b/vespalog/src/main/java/com/yahoo/log/VespaLogHandler.java
@@ -44,7 +44,7 @@ class VespaLogHandler extends StreamHandler {
     /**
      * Publish a log record into the Vespa log target.
      */
-    public synchronized void publish (LogRecord record) {
+    public synchronized void publish(LogRecord record) {
         Level level = record.getLevel();
         String component = record.getLoggerName();
 
diff --git a/vespalog/src/test/java/com/yahoo/log/VespaLevelControllerRepoTest.java b/vespalog/src/test/java/com/yahoo/log/VespaLevelControllerRepoTest.java
index 3bd4de03f4e..9d04e079f55 100644
--- a/vespalog/src/test/java/com/yahoo/log/VespaLevelControllerRepoTest.java
+++ b/vespalog/src/test/java/com/yahoo/log/VespaLevelControllerRepoTest.java
@@ -62,12 +62,14 @@ public class VespaLevelControllerRepoTest {
 
             RandomAccessFile lcfile = new RandomAccessFile(lcf, "rw");
 
-            lcfile.seek(VespaLevelControllerRepo.controlFileHeaderLength);
+            lcfile.seek(VespaLevelControllerRepo.controlFileHeaderLength+1);
+            assertEquals(lcfile.readByte(), '\n');
+            lcfile.seek(VespaLevelControllerRepo.controlFileHeaderLength+2);
             assertEquals(lcfile.readByte(), 'd');
-            lcfile.seek(VespaLevelControllerRepo.controlFileHeaderLength + 7);
+            lcfile.seek(VespaLevelControllerRepo.controlFileHeaderLength+2 + 7);
             assertEquals(lcfile.readByte(), ':');
-            assertEquals(0, (VespaLevelControllerRepo.controlFileHeaderLength+9) % 4);
-            lcfile.seek(VespaLevelControllerRepo.controlFileHeaderLength + 9);
+            assertEquals(0, (VespaLevelControllerRepo.controlFileHeaderLength+13) % 4);
+            lcfile.seek(VespaLevelControllerRepo.controlFileHeaderLength + 13);
             assertEquals(0x20204f4e, lcfile.readInt());
 
             int off = findControlString(lcfile, "com.yahoo.log.test");
diff --git a/vespalog/src/test/java/com/yahoo/log/VespaLogHandlerTestCase.java b/vespalog/src/test/java/com/yahoo/log/VespaLogHandlerTestCase.java
index c0dd856b634..220e5e9271e 100644
--- a/vespalog/src/test/java/com/yahoo/log/VespaLogHandlerTestCase.java
+++ b/vespalog/src/test/java/com/yahoo/log/VespaLogHandlerTestCase.java
@@ -13,6 +13,7 @@ import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.InputStreamReader;
 import java.io.OutputStream;
+import java.nio.charset.StandardCharsets;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.concurrent.BrokenBarrierException;
@@ -32,20 +33,20 @@ import static org.junit.Assert.fail;
  * @author  Bjorn Borud
  */
 public class VespaLogHandlerTestCase {
-    protected static String hostname;
-    protected static String pid;
+    private static String hostname;
+    private static String pid;
 
-    protected static LogRecord record1;
-    protected static String record1String;
+    static LogRecord record1;
+    static String record1String;
 
-    protected static LogRecord record2;
-    protected static String record2String;
+    static LogRecord record2;
+    private static String record2String;
 
-    protected static LogRecord record3;
-    protected static String record3String;
+    private static LogRecord record3;
+    private static String record3String;
 
-    protected static LogRecord record4;
-    protected static String record4String;
+    private static LogRecord record4;
+    private static String record4String;
 
     static {
         hostname = Util.getHostName();
@@ -139,7 +140,7 @@ public class VespaLogHandlerTestCase {
     }
 
     @Test
-    public void testFallback() throws FileNotFoundException {
+    public void testFallback() {
         File file = new File("mydir2");
         file.delete();
         assertTrue(file.mkdir());
@@ -157,7 +158,7 @@ public class VespaLogHandlerTestCase {
      * Perform simple test
      */
     @Test
-    public void testLogCtl () throws InterruptedException, FileNotFoundException {
+    public void testLogCtl () {
         MockLevelController ctl = new MockLevelController();
         MockLevelControllerRepo ctlRepo = new MockLevelControllerRepo(ctl);
         MockLogTarget target = new MockLogTarget();
@@ -203,7 +204,7 @@ public class VespaLogHandlerTestCase {
     @Test
     public void testRotate () throws IOException {
         // Doesn't work in Windows. TODO: Fix the logging stuff
-        if (System.getProperty("os.name").toLowerCase().indexOf("win")>=0)
+        if (System.getProperty("os.name").toLowerCase().contains("win"))
             return;
         try {
             VespaLogHandler h
@@ -269,10 +270,8 @@ public class VespaLogHandlerTestCase {
                 );
 
             class LogRacer implements Runnable {
-				private int n;
 
-                public LogRacer (int n) {
-                    this.n = n;
+                private LogRacer() {
                 }
 
                 public void run () {
@@ -285,7 +284,7 @@ public class VespaLogHandlerTestCase {
                     }
                 }
 
-                public void logLikeCrazy () {
+                void logLikeCrazy() {
                     for (int j = 0; j < numLogEntries; j++) {
                         try {
                             h.publish(record1);
@@ -299,7 +298,7 @@ public class VespaLogHandlerTestCase {
             }
 
             for (int i = 0; i < numThreads; i++) {
-                t[i] = new Thread(new LogRacer(i));
+                t[i] = new Thread(new LogRacer());
                 t[i].start();
             }
 
@@ -361,35 +360,23 @@ public class VespaLogHandlerTestCase {
      *
      */
     protected static String[] readFile (String fileName) {
-        BufferedReader br = null;
-        List<String> lines = new LinkedList<String>();
-        try {
-            br = new BufferedReader(
-                     new InputStreamReader(new FileInputStream(new File(fileName)), "UTF-8"));
+        List<String> lines = new LinkedList<>();
+        try (BufferedReader br = new BufferedReader(
+                new InputStreamReader(new FileInputStream(new File(fileName)), StandardCharsets.UTF_8))) {
             for (String line = br.readLine();
                  line != null;
-                 line = br.readLine())
-            {
+                 line = br.readLine()) {
                 lines.add(line);
             }
-            return lines.toArray(new String[lines.size()]);
-        }
-        catch (Throwable e) {
+            return lines.toArray(new String[0]);
+        } catch (Throwable e) {
             return new String[0];
         }
-        finally {
-            if (br != null) {
-                try {
-                    br.close();
-                }
-                catch (IOException e) {}
-            }
-        }
     }
 
     private static class MockLevelControllerRepo implements LevelControllerRepo {
         private LevelController levelController;
-        public MockLevelControllerRepo(LevelController controller) {
+        MockLevelControllerRepo(LevelController controller) {
             this.levelController = controller;
         }
 
@@ -411,7 +398,7 @@ public class VespaLogHandlerTestCase {
             return (level.equals(logLevel));
         }
 
-        public void setShouldLog(Level level) {
+        void setShouldLog(Level level) {
             this.logLevel = level;
         }
 
@@ -431,7 +418,7 @@ public class VespaLogHandlerTestCase {
     private static class MockLogTarget implements LogTarget {
         private final ByteArrayOutputStream baos = new ByteArrayOutputStream();
 
-        public String[] getLines() {
+        String[] getLines() {
             return baos.toString().split("\n");
         }
         @Override
diff --git a/vespalog/src/vespa/log/control-file.cpp b/vespalog/src/vespa/log/control-file.cpp
index 4a4fd36e0ac..aead7c456ef 100644
--- a/vespalog/src/vespa/log/control-file.cpp
+++ b/vespalog/src/vespa/log/control-file.cpp
@@ -58,28 +58,26 @@ ControlFile::ensureHeader()
     int wantsLen = strlen(fileHeader);
     int len = read(fd, &buf, wantsLen);
     if (len != wantsLen || memcmp(fileHeader, buf, wantsLen) != 0) {
-        if (len) {
-	    if (ftruncate(fd, 0) != 0) {
-		perror("log::ControlFile ftruncate failed");
-	    }
+        if (ftruncate(fd, 0) != 0) {
+            perror("log::ControlFile ftruncate failed");
         }
         lseek(fd, 0, SEEK_SET);
         ssize_t nbw = write(fd, fileHeader, wantsLen);
-	if (nbw != wantsLen) {
-	    perror("log::ControlFile write(A) failed");
-	}
+        if (nbw != wantsLen) {
+            perror("log::ControlFile write(A) failed");
+        }
 
-        char spaces[_maxPrefix + 1];
+        char spaces[_maxPrefix + 3];
         memset(spaces, ' ', sizeof spaces);
         spaces[sizeof(spaces) - 1] = '\0';
 
         char buf2[sizeof(spaces) + 100];
         snprintf(buf2, sizeof buf2, "Prefix: \n%s\n", spaces);
-	wantsLen = strlen(buf2);
+        wantsLen = strlen(buf2);
         nbw = write(fd, buf2, wantsLen);
-	if (nbw != wantsLen) {
-	    perror("log::ControlFile write(B) failed");
-	}
+        if (nbw != wantsLen) {
+            perror("log::ControlFile write(B) failed");
+        }
 
     }
 }
@@ -88,10 +86,10 @@ void
 ControlFile::flush()
 {
     if (_mapBase != NULL) {
-	if (msync(_mapBase, 0, MS_SYNC) != 0) {
-	    LOG(warning, "msync of log control file failed: %s",
-		strerror(errno));
-	}
+        if (msync(_mapBase, 0, MS_SYNC) != 0) {
+            LOG(warning, "msync of log control file failed: %s",
+                strerror(errno));
+        }
     }
 }
 
@@ -259,32 +257,36 @@ ControlFile::getLevels(const char *name)
 
     //  Append whatever is in buf, excluding the initial newline, and
     //  up to 3 more spaces to get the entire file length to be aligned.
-    int fileLength = _fileBacking.size();
+    int oldFileLength = _fileBacking.size();
     char *appendedString = buf + 1;
-    int newLength = fileLength + strlen(appendedString);
+    int newLength = oldFileLength + strlen(appendedString);
     unsigned int padding = static_cast<unsigned int>(-newLength) & 3u;
     strcat(appendedString, &padSpaces[3 - padding]);
+    int prefix_len = strlen(appendedString);
 
-    char *baseAddr = _mapBase + fileLength + strlen(appendedString);
-
+#pragma GCC diagnostic push
+#ifndef __clang__
+#pragma GCC diagnostic ignored "-Wstringop-truncation"
+#endif
     strncat(appendedString, inheritLevels, Logger::NUM_LOGLEVELS*sizeof(int));
+#pragma GCC diagnostic pop
     strcat(appendedString, "\n");
 
     int len = strlen(appendedString);
-
-    int fd = _fileBacking.fd();
-    lseek(fd, (off_t)fileLength, SEEK_SET);
-    int wlen = write(_fileBacking.fd(), appendedString, len);
+    int fd = open(_fileName, O_WRONLY | O_APPEND);
+    int wlen = write(fd, appendedString, len);
+    oldFileLength = lseek(fd, (off_t)0, SEEK_CUR) - wlen;
+    close(fd);
     if (wlen != len) {
         _fileBacking.unlock();
         LOG(error, "Writing to control file '%s' fails (%d/%d bytes): %s",
             _fileName, wlen, len, strerror(errno));
         return reinterpret_cast<unsigned int *>(inheritLevels);
     } else {
-        _fileSize += wlen;
+        _fileSize = _fileBacking.size();
     }
 
-    if (fileLength + wlen > _mappedSize) {
+    if (_fileSize > _mappedSize) {
         if (!extendMapping()) {
             _fileBacking.unlock(); // just for sure
             LOG(error, "Failed to extend mapping of '%s', losing runtime "
@@ -292,9 +294,8 @@ ControlFile::getLevels(const char *name)
             return defaultLevels();
         }
     }
-
+    char *baseAddr = _mapBase + oldFileLength + prefix_len;
     _fileBacking.unlock();
-
     return reinterpret_cast<unsigned int *>(baseAddr);
 }
 
diff --git a/vespalog/src/vespa/log/llparser.cpp b/vespalog/src/vespa/log/llparser.cpp
index 8e44f36c7ae..ae1af3e6416 100644
--- a/vespalog/src/vespa/log/llparser.cpp
+++ b/vespalog/src/vespa/log/llparser.cpp
@@ -361,8 +361,7 @@ LLParser::makeMessage(const char *tmf, const char *hsf, const char *pdf,
         if ((c == '\\' && src[0] == 't')
             || (c >= 32 && c < '\\')
             || (c > '\\' && c < 128)
-            || c == 0
-            || c > 160)
+            || c == 0)
 	{
             *dst++ = static_cast<char>(c);
         } else {
diff --git a/vespalog/src/vespa/log/log-assert.cpp b/vespalog/src/vespa/log/log-assert.cpp
index 51ab7a019ad..28fac730189 100644
--- a/vespalog/src/vespa/log/log-assert.cpp
+++ b/vespalog/src/vespa/log/log-assert.cpp
@@ -1,6 +1,7 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include "log.h"
+#include <cstdio>
 LOG_SETUP("");
 
 namespace ns_log {
diff --git a/vespalog/src/vespa/log/log.cpp b/vespalog/src/vespa/log/log.cpp
index a43c6dd0416..47f7377ac4d 100644
--- a/vespalog/src/vespa/log/log.cpp
+++ b/vespalog/src/vespa/log/log.cpp
@@ -39,6 +39,22 @@ char Logger::_hostname[1024] = { '\0'};
 char Logger::_serviceName[1024] = {'\0' };
 ControlFile *Logger::_controlFile = 0;
 
+namespace {
+
+class GetTid {
+public:
+    unsigned long operator()(const void *tid) const {
+        return reinterpret_cast<uint64_t>(tid) >> 3;
+    }
+    unsigned long operator()(unsigned long tid) const {
+        return tid;
+    }
+};
+
+GetTid gettid;
+
+}
+
 void
 Logger::ensureControlName()
 {
@@ -233,11 +249,10 @@ void Logger::doLogCore(uint64_t timestamp, LogLevel level,
 {
     const size_t sizeofEscapedPayload(msgSize*4+1);
     const size_t sizeofTotalMessage(sizeofEscapedPayload + 1000);
-    char * bigBuffer(new char[sizeofEscapedPayload+sizeofTotalMessage]);
-    char * escapedPayload(bigBuffer);
-    char * totalMessage(escapedPayload + sizeofEscapedPayload);
+    auto escapedPayload = std::make_unique<char[]>(sizeofEscapedPayload);
+    auto totalMessage = std::make_unique<char[]>(sizeofTotalMessage);
 
-    char *dst = escapedPayload;
+    char *dst = escapedPayload.get();
     for (size_t i(0); (i < msgSize) && msg[i]; i++) {
          unsigned char c = static_cast<unsigned char>(msg[i]);
         if ((c >= 32) && (c != '\\') && (c != 127)) {
@@ -266,7 +281,7 @@ void Logger::doLogCore(uint64_t timestamp, LogLevel level,
         // threads, only showing the least significant bits will hopefully
         // distinguish between all threads in your application. Alter later if
         // found to be too inaccurate.
-    int32_t tid = (fakePid ? -1 : pthread_self() % 0xffff);
+    int32_t tid = (fakePid ? -1 : gettid(pthread_self()) % 0xffff);
 
     if (_target->makeHumanReadable()) {
         time_t secs = static_cast<time_t>(timestamp / 1000000);
@@ -274,14 +289,14 @@ void Logger::doLogCore(uint64_t timestamp, LogLevel level,
         localtime_r(&secs, &tmbuf);
         char timebuf[100];
         strftime(timebuf, 100, "%Y-%m-%d %H:%M:%S", &tmbuf);
-        snprintf(totalMessage, sizeofTotalMessage,
+        snprintf(totalMessage.get(), sizeofTotalMessage,
                  "[%s.%06u] %d/%d (%s%s) %s: %s\n",
                  timebuf, static_cast<unsigned int>(timestamp % 1000000),
                  fakePid ? -1 : getpid(), tid,
                  _prefix, _appendix,
                  levelName(level), msg);
     } else if (level == debug || level == spam) {
-        snprintf(totalMessage, sizeofTotalMessage,
+        snprintf(totalMessage.get(), sizeofTotalMessage,
                  "%u.%06u\t%s\t%d/%d\t%s\t%s%s\t%s\t%s:%d %s%s\n",
                  static_cast<unsigned int>(timestamp / 1000000),
                  static_cast<unsigned int>(timestamp % 1000000),
@@ -289,19 +304,18 @@ void Logger::doLogCore(uint64_t timestamp, LogLevel level,
                  _serviceName, _prefix,
                  _appendix, levelName(level), file, line,
                  _rcsId,
-                 escapedPayload);
+                 escapedPayload.get());
     } else {
-        snprintf(totalMessage, sizeofTotalMessage,
+        snprintf(totalMessage.get(), sizeofTotalMessage,
                  "%u.%06u\t%s\t%d/%d\t%s\t%s%s\t%s\t%s\n",
                  static_cast<unsigned int>(timestamp / 1000000),
                  static_cast<unsigned int>(timestamp % 1000000),
                  _hostname, fakePid ? -1 : getpid(), tid,
                  _serviceName, _prefix,
-                 _appendix, levelName(level), escapedPayload);
+                 _appendix, levelName(level), escapedPayload.get());
     }
 
-    _target->write(totalMessage, strlen(totalMessage));
-    delete [] bigBuffer;
+    _target->write(totalMessage.get(), strlen(totalMessage.get()));
 }
 
 const char *
diff --git a/vespalog/src/vespa/log/log.h b/vespalog/src/vespa/log/log.h
index ba2408240f3..b4dfc612890 100644
--- a/vespalog/src/vespa/log/log.h
+++ b/vespalog/src/vespa/log/log.h
@@ -7,6 +7,7 @@
 #include <new>         // for placement new
 #include <cstdlib>     // for malloc
 #include <cstring>     // for memset
+#include <cstdarg>     // for va_list
 #include <cinttypes>
 
 /**
diff --git a/vespalog/vespa-log-utils.spec b/vespalog/vespa-log-utils.spec
index 8c7ca7f1cdd..2bde45bd075 100644
--- a/vespalog/vespa-log-utils.spec
+++ b/vespalog/vespa-log-utils.spec
@@ -6,7 +6,7 @@
 Name:           vespa-log-utils
 Version:        %version
 Release:        1%{?dist}
-BuildArch:      noarch
+BuildArch:      x86_64
 Summary:        Vespa log utilities
 Group:          Applications/Databases
 License:        Commercial
diff --git a/vespamalloc/src/tests/overwrite/overwrite.cpp b/vespamalloc/src/tests/overwrite/overwrite.cpp
index b016c13ab61..84f96fbbb3e 100644
--- a/vespamalloc/src/tests/overwrite/overwrite.cpp
+++ b/vespamalloc/src/tests/overwrite/overwrite.cpp
@@ -10,6 +10,13 @@ void check_ptr_real(void *ptr)
 
 void (*check_ptr)(void *ptr) = check_ptr_real;
 
+void overwrite_memory_real(char *ptr, int offset)
+{
+    *(ptr + offset) = 0;
+}
+
+void (*overwrite_memory)(char *ptr, int offset) = overwrite_memory_real;
+
 class Test : public TestApp
 {
 public:
@@ -63,16 +70,14 @@ void Test::testFillValue(char *a)
 void Test::verifyPreWriteDetection()
 {
     char * a = new char[8];
-    *(a-1) = 0;
-    check_ptr(a);
+    overwrite_memory(a, -1);
     delete [] a;
 }
 
 void Test::verifyPostWriteDetection()
 {
     char * a = new char[8];
-    a[8] = 0;
-    check_ptr(a);
+    overwrite_memory(a, 8);
     delete [] a;
 }
 
diff --git a/vespamalloc/src/tests/test2/testgraph.cpp b/vespamalloc/src/tests/test2/testgraph.cpp
index 9f25e45b7e9..ae0c468b185 100644
--- a/vespamalloc/src/tests/test2/testgraph.cpp
+++ b/vespamalloc/src/tests/test2/testgraph.cpp
@@ -3,6 +3,7 @@
 #include <vespamalloc/util/callgraph.h>
 #include <vespamalloc/util/callstack.h>
 #include <vespamalloc/util/traceutil.h>
+#include <string>
 
 using namespace vespamalloc;
 
diff --git a/vespamalloc/src/vespamalloc/malloc/allocchunk.cpp b/vespamalloc/src/vespamalloc/malloc/allocchunk.cpp
index 41165244e5c..fb95018e94e 100644
--- a/vespamalloc/src/vespamalloc/malloc/allocchunk.cpp
+++ b/vespamalloc/src/vespamalloc/malloc/allocchunk.cpp
@@ -7,7 +7,7 @@ namespace vespamalloc {
 void AFListBase::linkInList(AtomicHeadPtr & head, AFListBase * list)
 {
     AFListBase * tail;
-    for (tail = list; tail->_next != NULL ;tail = tail->_next) { }
+    for (tail = list; tail->_next != nullptr ;tail = tail->_next) { }
     linkIn(head, list, tail);
 }
 
@@ -16,7 +16,7 @@ void AFListBase::linkIn(AtomicHeadPtr & head, AFListBase * csl, AFListBase * tai
     HeadPtr oldHead = head.load(std::memory_order_relaxed);
     HeadPtr newHead(csl, oldHead._tag + 1);
     tail->_next = static_cast<AFListBase *>(oldHead._ptr);
-    while ( ! head.compare_exchange_weak(oldHead, newHead, std::memory_order_release, std::memory_order_relaxed) ) {
+    while ( __builtin_expect(! head.compare_exchange_weak(oldHead, newHead, std::memory_order_release, std::memory_order_relaxed), false) ) {
         newHead._tag =  oldHead._tag + 1;
         tail->_next = static_cast<AFListBase *>(oldHead._ptr);
     }
@@ -26,19 +26,19 @@ AFListBase * AFListBase::linkOut(AtomicHeadPtr & head)
 {
     HeadPtr oldHead = head.load(std::memory_order_relaxed);
     AFListBase *csl = static_cast<AFListBase *>(oldHead._ptr);
-    if (csl == NULL) {
-        return NULL;
+    if (csl == nullptr) {
+        return nullptr;
     }
     HeadPtr newHead(csl->_next, oldHead._tag + 1);
-    while ( ! head.compare_exchange_weak(oldHead, newHead, std::memory_order_acquire, std::memory_order_relaxed) ) {
+    while ( __builtin_expect(! head.compare_exchange_weak(oldHead, newHead, std::memory_order_acquire, std::memory_order_relaxed), false) ) {
         csl = static_cast<AFListBase *>(oldHead._ptr);
-        if (csl == NULL) {
-            return NULL;
+        if (csl == nullptr) {
+            return nullptr;
         }
         newHead._ptr = csl->_next;
         newHead._tag = oldHead._tag + 1;
     }
-    csl->_next = NULL;
+    csl->_next = nullptr;
     return csl;
 }
 
diff --git a/vespamalloc/src/vespamalloc/malloc/allocchunk.h b/vespamalloc/src/vespamalloc/malloc/allocchunk.h
index 8b6a2ce3fc8..ed0b52c7b73 100644
--- a/vespamalloc/src/vespamalloc/malloc/allocchunk.h
+++ b/vespamalloc/src/vespamalloc/malloc/allocchunk.h
@@ -24,12 +24,55 @@ struct TaggedPtr {
     size_t _tag;
 };
 
+#if defined(__x86_64__)
+struct AtomicTaggedPtr {
+    AtomicTaggedPtr() noexcept : _ptr(nullptr), _tag(0) { }
+    AtomicTaggedPtr(void *h, size_t t) noexcept : _ptr(h), _tag(t) {}
+
+    AtomicTaggedPtr load(std::memory_order = std::memory_order_seq_cst) {
+        // Note that this is NOT an atomic load. The current use as the initial load
+        // in a compare_exchange loop is safe as a teared load will just give a retry.
+        return *this;
+    }
+    void store(AtomicTaggedPtr ptr) {
+        // Note that this is NOT an atomic store. The current use is in a unit test as an initial
+        // store before any threads are started. Just done so to keep api compatible with std::atomic as
+        // that is the preferred implementation..
+        *this = ptr;
+    }
+    bool
+    compare_exchange_weak(AtomicTaggedPtr & oldPtr, AtomicTaggedPtr newPtr, std::memory_order, std::memory_order) {
+        char result;
+        __asm__ volatile (
+        "lock ;"
+        "cmpxchg16b %6;"
+        "setz %1;"
+        : "+m" (*this),
+          "=q" (result),
+          "+a" (oldPtr._ptr),
+          "+d" (oldPtr._tag)
+        : "b" (newPtr._ptr),
+          "c" (newPtr._tag)
+        : "cc", "memory"
+        );
+        return result;
+    }
+
+    void *_ptr;
+    size_t _tag;
+} __attribute__ ((aligned (16)));
+#else
+    using AtomicTaggedPtr = TaggedPtr;
+#endif
+
+
 class AFListBase
 {
 public:
-    using HeadPtr = TaggedPtr;
-    using AtomicHeadPtr = std::atomic<HeadPtr>;
-    AFListBase() : _next(NULL) { }
+    using HeadPtr = std::conditional<std::atomic<TaggedPtr>::is_always_lock_free, TaggedPtr, AtomicTaggedPtr>::type;
+    using AtomicHeadPtr = std::conditional<std::atomic<TaggedPtr>::is_always_lock_free, std::atomic<TaggedPtr>, AtomicTaggedPtr>::type;
+
+    AFListBase() : _next(nullptr) { }
     void setNext(AFListBase * csl)           { _next = csl; }
     static void init();
     static void linkInList(AtomicHeadPtr & head, AFListBase * list);
diff --git a/vespamalloc/src/vespamalloc/malloc/globalpool.hpp b/vespamalloc/src/vespamalloc/malloc/globalpool.hpp
index c954c1aae26..a3930c7cca1 100644
--- a/vespamalloc/src/vespamalloc/malloc/globalpool.hpp
+++ b/vespamalloc/src/vespamalloc/malloc/globalpool.hpp
@@ -15,12 +15,13 @@ size_t AllocPoolT<MemBlockPtrT>::_alwaysReuseLimit __attribute__((visibility("hi
 template <typename MemBlockPtrT>
 AllocPoolT<MemBlockPtrT>::AllocPoolT(DataSegment<MemBlockPtrT> & ds)
     : _chunkPool(NULL),
+      _scList(),
       _dataSegment(ds),
       _getChunks(0),
       _getChunksSum(0),
-      _allocChunkList(0)
+      _allocChunkList(0),
+      _stat()
 {
-    memset(_scList, 0, sizeof(_scList));
 }
 
 template <typename MemBlockPtrT>
diff --git a/vsm/src/tests/textutil/textutil.cpp b/vsm/src/tests/textutil/textutil.cpp
index e71b95f22f9..581419fc80e 100644
--- a/vsm/src/tests/textutil/textutil.cpp
+++ b/vsm/src/tests/textutil/textutil.cpp
@@ -60,10 +60,10 @@ void
 TextUtilTest::assertSkipSeparators(const char * input, size_t len, const UCS4V & expdstbuf, const SizeV & expoffsets)
 {
     const byte * srcbuf = reinterpret_cast<const byte *>(input);
-    ucs4_t dstbuf[len];
-    size_t offsets[len];
+    auto dstbuf = std::make_unique<ucs4_t[]>(len + 1);
+    auto offsets = std::make_unique<size_t[]>(len + 1);
     UTF8StrChrFieldSearcher fs;
-    BW bw(dstbuf, offsets);
+    BW bw(dstbuf.get(), offsets.get());
     size_t dstlen = fs.skipSeparators(srcbuf, len, bw);
     EXPECT_EQUAL(dstlen, expdstbuf.size());
     ASSERT_TRUE(dstlen == expdstbuf.size());
diff --git a/vtag.cmake b/vtag.cmake
index eefbb638ab2..a7aaf433024 100644
--- a/vtag.cmake
+++ b/vtag.cmake
@@ -1,10 +1,10 @@
 # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-if(NOT EXISTS "${CMAKE_SOURCE_DIR}/dist/vtag.map")
+if(NOT EXISTS "${CMAKE_CURRENT_LIST_DIR}/dist/vtag.map")
     message(FATAL_ERROR "dist/vtag.map does not exist, please run bootstrap.sh before configuring cmake" )
 endif()
 
 function(get_vtag_define KEY)
-    file(STRINGS dist/vtag.map VALUE REGEX "${KEY}")
+    file(STRINGS ${CMAKE_CURRENT_LIST_DIR}/dist/vtag.map VALUE REGEX "${KEY}")
     list(GET VALUE 0 LINE)
     separate_arguments(DATA UNIX_COMMAND "${LINE}")
     list(GET DATA 1 VALUE)
diff --git a/zkfacade/abi-spec.json b/zkfacade/abi-spec.json
index 05fb985dbaf..f4ad1ab4372 100644
--- a/zkfacade/abi-spec.json
+++ b/zkfacade/abi-spec.json
@@ -85,6 +85,7 @@
       "public java.util.List getChildren(com.yahoo.path.Path)",
       "public java.util.Optional getData(com.yahoo.path.Path)",
       "public java.util.Optional getStat(com.yahoo.path.Path)",
+      "public com.yahoo.vespa.curator.Lock lock(com.yahoo.path.Path, java.time.Duration)",
       "public org.apache.curator.framework.CuratorFramework framework()",
       "public void close()",
       "public java.lang.String zooKeeperEnsembleConnectionSpec()",
diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/Curator.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/Curator.java
index 7c5b1ae319a..6d4f6beece1 100644
--- a/zkfacade/src/main/java/com/yahoo/vespa/curator/Curator.java
+++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/Curator.java
@@ -32,6 +32,7 @@ import java.time.Duration;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Optional;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ExecutorService;
 import java.util.function.Function;
 import java.util.logging.Logger;
@@ -48,27 +49,26 @@ import java.util.logging.Logger;
  */
 public class Curator implements AutoCloseable {
 
-    private static final Logger logger = Logger.getLogger(Curator.class.getName());
-
-    private static final int ZK_SESSION_TIMEOUT = 30000;
-    private static final int ZK_CONNECTION_TIMEOUT = 30000;
-
-    private static final int BASE_SLEEP_TIME = 1000; //ms
+    private static final Logger LOG = Logger.getLogger(Curator.class.getName());
+    private static final File ZK_CLIENT_CONFIG_FILE = new File(Defaults.getDefaults().underVespaHome("conf/zookeeper/zookeeper-client.cfg"));
+    private static final Duration ZK_SESSION_TIMEOUT = Duration.ofSeconds(30);
+    private static final Duration ZK_CONNECTION_TIMEOUT = Duration.ofSeconds(30);
+    private static final Duration BASE_SLEEP_TIME = Duration.ofSeconds(1);
     private static final int MAX_RETRIES = 10;
 
-    private static final File zkClientConfigFile = new File(Defaults.getDefaults().underVespaHome("conf/zookeeper/zookeeper-client.cfg"));
-
     protected final RetryPolicy retryPolicy;
 
     private final CuratorFramework curatorFramework;
     private final String connectionSpec; // May be a subset of the servers in the ensemble
-
     private final String zooKeeperEnsembleConnectionSpec;
     private final int zooKeeperEnsembleCount;
 
+    // All lock keys, to allow re-entrancy. This will grow forever, but this should be too slow to be a problem
+    private final ConcurrentHashMap<Path, Lock> locks = new ConcurrentHashMap<>();
+
     /** Creates a curator instance from a comma-separated string of ZooKeeper host:port strings */
     public static Curator create(String connectionSpec) {
-        return new Curator(connectionSpec, connectionSpec, Optional.of(zkClientConfigFile));
+        return new Curator(connectionSpec, connectionSpec, Optional.of(ZK_CLIENT_CONFIG_FILE));
     }
 
     // For testing only, use Optional.empty for clientConfigFile parameter to create default zookeeper client config
@@ -80,7 +80,7 @@ public class Curator implements AutoCloseable {
     // TODO: Move zookeeperserver config out of configserverconfig (requires update of controller services.xml as well)
     @Inject
     public Curator(ConfigserverConfig configserverConfig, VespaZooKeeperServer server) {
-        this(configserverConfig, Optional.of(zkClientConfigFile));
+        this(configserverConfig, Optional.of(ZK_CLIENT_CONFIG_FILE));
     }
 
     Curator(ConfigserverConfig configserverConfig, Optional<File> clientConfigFile) {
@@ -93,8 +93,8 @@ public class Curator implements AutoCloseable {
                 (retryPolicy) -> CuratorFrameworkFactory
                         .builder()
                         .retryPolicy(retryPolicy)
-                        .sessionTimeoutMs(ZK_SESSION_TIMEOUT)
-                        .connectionTimeoutMs(ZK_CONNECTION_TIMEOUT)
+                        .sessionTimeoutMs((int) ZK_SESSION_TIMEOUT.toMillis())
+                        .connectionTimeoutMs((int) ZK_CONNECTION_TIMEOUT.toMillis())
                         .connectString(connectionSpec)
                         .zookeeperFactory(new VespaZooKeeperFactory(createClientConfig(clientConfigFile)))
                         .dontUseContainerParents() // TODO: Remove when we know ZooKeeper 3.5 works fine, consider waiting until Vespa 8
@@ -105,7 +105,7 @@ public class Curator implements AutoCloseable {
                       String zooKeeperEnsembleConnectionSpec,
                       Function<RetryPolicy, CuratorFramework> curatorFactory) {
         this(connectionSpec, zooKeeperEnsembleConnectionSpec, curatorFactory,
-                new ExponentialBackoffRetry(BASE_SLEEP_TIME, MAX_RETRIES));
+                new ExponentialBackoffRetry((int) BASE_SLEEP_TIME.toMillis(), MAX_RETRIES));
     }
 
     private Curator(String connectionSpec,
@@ -193,7 +193,7 @@ public class Curator implements AutoCloseable {
 
     /** For internal use; prefer creating a {@link CuratorCounter} */
     public DistributedAtomicLong createAtomicCounter(String path) {
-        return new DistributedAtomicLong(curatorFramework, path, new ExponentialBackoffRetry(BASE_SLEEP_TIME, MAX_RETRIES));
+        return new DistributedAtomicLong(curatorFramework, path, new ExponentialBackoffRetry((int) BASE_SLEEP_TIME.toMillis(), MAX_RETRIES));
     }
 
     /** For internal use; prefer creating a {@link com.yahoo.vespa.curator.Lock} */
@@ -204,9 +204,9 @@ public class Curator implements AutoCloseable {
     private void addLoggingListener() {
         curatorFramework.getConnectionStateListenable().addListener((curatorFramework, connectionState) -> {
             switch (connectionState) {
-                case SUSPENDED: logger.info("ZK connection state change: SUSPENDED"); break;
-                case RECONNECTED: logger.info("ZK connection state change: RECONNECTED"); break;
-                case LOST: logger.warning("ZK connection state change: LOST"); break;
+                case SUSPENDED: LOG.info("ZK connection state change: SUSPENDED"); break;
+                case RECONNECTED: LOG.info("ZK connection state change: RECONNECTED"); break;
+                case LOST: LOG.warning("ZK connection state change: LOST"); break;
             }
         });
     }
@@ -351,6 +351,14 @@ public class Curator implements AutoCloseable {
         }
     }
 
+    /** Create and acquire a re-entrant lock in given path */
+    public Lock lock(Path path, Duration timeout) {
+        create(path);
+        Lock lock = locks.computeIfAbsent(path, (pathArg) -> new Lock(pathArg.getAbsolute(), this));
+        lock.acquire(timeout);
+        return lock;
+    }
+
     /** Returns the curator framework API */
     public CuratorFramework framework() {
         return curatorFramework;
diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/Lock.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/Lock.java
index 30af89d0ea8..d97d8f5ed71 100644
--- a/zkfacade/src/main/java/com/yahoo/vespa/curator/Lock.java
+++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/Lock.java
@@ -2,45 +2,65 @@
 package com.yahoo.vespa.curator;
 
 import com.google.common.util.concurrent.UncheckedTimeoutException;
+import com.yahoo.path.Path;
 import com.yahoo.transaction.Mutex;
 import org.apache.curator.framework.recipes.locks.InterProcessLock;
 
 import java.time.Duration;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.ReentrantLock;
 
 /**
- * A cluster-wide re-entrant mutex which is released on (the last symmetric) close
+ * A cluster-wide re-entrant mutex which is released on (the last symmetric) close.
+ *
+ * Re-entrancy is limited to the instance of this. To ensure re-entrancy callers should access the lock through
+ * {@link Curator#lock(Path, Duration)} instead of constructing this directly.
  *
  * @author bratseth
  */
 public class Lock implements Mutex {
 
+    private final ReentrantLock lock;
     private final InterProcessLock mutex;
     private final String lockPath;
 
     public Lock(String lockPath, Curator curator) {
         this.lockPath = lockPath;
+        this.lock = new ReentrantLock(true);
         mutex = curator.createMutex(lockPath);
     }
 
     /** Take the lock with the given timeout. This may be called multiple times from the same thread - each matched by a close */
     public void acquire(Duration timeout) throws UncheckedTimeoutException {
-        boolean acquired;
         try {
-            acquired = mutex.acquire(timeout.toMillis(), TimeUnit.MILLISECONDS);
+            if ( ! mutex.acquire(timeout.toMillis(), TimeUnit.MILLISECONDS))
+                throw new UncheckedTimeoutException("Timed out after waiting " + timeout +
+                                                    " to acquire lock '" + lockPath + "'");
+            if ( ! lock.tryLock()) { // Should be available to only this thread, while holding the above mutex.
+                release();
+                throw new IllegalStateException("InterProcessMutex acquired, but guarded lock held by someone else, for lock '" + lockPath + "'");
+            }
+        }
+        catch (UncheckedTimeoutException | IllegalStateException e) {
+            throw e;
         }
         catch (Exception e) {
             throw new RuntimeException("Exception acquiring lock '" + lockPath + "'", e);
         }
-
-        if (! acquired)
-            throw new UncheckedTimeoutException("Timed out after waiting " + timeout +
-                                                " to acquire lock '" + lockPath + "'");
     }
 
     @Override
     public void close() {
         try {
+            lock.unlock();
+        }
+        finally {
+            release();
+        }
+    }
+
+    private void release() {
+        try {
             mutex.release();
         }
         catch (Exception e) {
diff --git a/zookeeper-command-line-client/src/main/sh/vespa-zkcli b/zookeeper-command-line-client/src/main/sh/vespa-zkcli
index 5b23226dc5b..abfe9ff75cf 100755
--- a/zookeeper-command-line-client/src/main/sh/vespa-zkcli
+++ b/zookeeper-command-line-client/src/main/sh/vespa-zkcli
@@ -83,7 +83,7 @@ usage() {
     echo "-nosudo                do not use sudo when running command"
 }
 
-sudo="sudo -u ${VESPA_USER}"
+sudo="/usr/bin/sudo -u ${VESPA_USER}"
 while [ $# -gt 0 ]; do
     case $1 in
         -h|-help)        usage; exit 0;;
diff --git a/zookeeper-server/CMakeLists.txt b/zookeeper-server/CMakeLists.txt
index 2d8620f1028..6e8c82bc66e 100644
--- a/zookeeper-server/CMakeLists.txt
+++ b/zookeeper-server/CMakeLists.txt
@@ -1,4 +1,3 @@
 # Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 add_subdirectory(zookeeper-server-common)
-add_subdirectory(zookeeper-server-3.4)
 add_subdirectory(zookeeper-server-3.5)
diff --git a/zookeeper-server/pom.xml b/zookeeper-server/pom.xml
index fe182645681..edfbdbad02e 100644
--- a/zookeeper-server/pom.xml
+++ b/zookeeper-server/pom.xml
@@ -13,7 +13,6 @@
   <version>7-SNAPSHOT</version>
   <modules>
     <module>zookeeper-server-common</module>
-    <module>zookeeper-server-3.4</module>
     <module>zookeeper-server-3.5</module>
   </modules>
   <dependencies>
diff --git a/zookeeper-server/zookeeper-server-3.4/pom.xml b/zookeeper-server/zookeeper-server-3.4/pom.xml
deleted file mode 100644
index 756e2332c15..00000000000
--- a/zookeeper-server/zookeeper-server-3.4/pom.xml
+++ /dev/null
@@ -1,74 +0,0 @@
-<?xml version="1.0"?>
-<!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
-  <modelVersion>4.0.0</modelVersion>
-  <parent>
-    <groupId>com.yahoo.vespa</groupId>
-    <artifactId>zookeeper-server</artifactId>
-    <version>7-SNAPSHOT</version>
-    <relativePath>../pom.xml</relativePath>
-  </parent>
-  <artifactId>zookeeper-server-3.4</artifactId>
-  <packaging>container-plugin</packaging>
-  <version>7-SNAPSHOT</version>
-  <dependencies>
-    <dependency>
-      <groupId>com.yahoo.vespa</groupId>
-      <artifactId>zookeeper-server-common</artifactId>
-      <version>${project.version}</version>
-    </dependency>
-    <dependency>
-      <groupId>org.apache.zookeeper</groupId>
-      <artifactId>zookeeper</artifactId>
-      <version>3.4.14</version>
-    </dependency>
-    <dependency>
-      <groupId>org.slf4j</groupId>
-      <artifactId>slf4j-log4j12</artifactId>
-      <version>1.7.5</version>
-    </dependency>
-    <dependency>
-      <groupId>junit</groupId>
-      <artifactId>junit</artifactId>
-      <scope>test</scope>
-    </dependency>
-  </dependencies>
-  <build>
-    <plugins>
-      <plugin>
-        <groupId>org.apache.maven.plugins</groupId>
-        <artifactId>maven-compiler-plugin</artifactId>
-        <configuration>
-          <compilerArgs>
-            <arg>-Xlint:all</arg>
-            <arg>-Werror</arg>
-          </compilerArgs>
-        </configuration>
-      </plugin>
-      <plugin>
-        <groupId>org.apache.maven.plugins</groupId>
-        <artifactId>maven-surefire-plugin</artifactId>
-        <configuration>
-          <redirectTestOutputToFile>${test.hide}</redirectTestOutputToFile>
-          <forkMode>once</forkMode>
-        </configuration>
-      </plugin>
-      <plugin>
-        <groupId>org.apache.maven.plugins</groupId>
-        <artifactId>maven-install-plugin</artifactId>
-        <configuration>
-          <updateReleaseInfo>true</updateReleaseInfo>
-        </configuration>
-      </plugin>
-      <plugin>
-        <groupId>com.yahoo.vespa</groupId>
-        <artifactId>bundle-plugin</artifactId>
-        <extensions>true</extensions>
-        <configuration>
-          <importPackage>com.sun.management</importPackage>
-          <bundleSymbolicName>zookeeper-server</bundleSymbolicName>
-        </configuration>
-      </plugin>
-    </plugins>
-  </build>
-</project>
diff --git a/zookeeper-server/zookeeper-server-3.4/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java b/zookeeper-server/zookeeper-server-3.4/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java
deleted file mode 100644
index 5b4c0c11e80..00000000000
--- a/zookeeper-server/zookeeper-server-3.4/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java
+++ /dev/null
@@ -1,133 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.zookeeper;
-
-import com.google.inject.Inject;
-import com.yahoo.cloud.config.ZookeeperServerConfig;
-import com.yahoo.component.AbstractComponent;
-import com.yahoo.log.LogLevel;
-import static com.yahoo.vespa.defaults.Defaults.getDefaults;
-
-import java.io.File;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.util.List;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-/**
- * Writes zookeeper config and starts zookeeper server.
- *
- * @author Ulf Lilleengen
- * @author Harald Musum
- */
-public class VespaZooKeeperServerImpl extends AbstractComponent implements Runnable, VespaZooKeeperServer {
-
-    private static final java.util.logging.Logger log = java.util.logging.Logger.getLogger(VespaZooKeeperServerImpl.class.getName());
-    private static final String ZOOKEEPER_JMX_LOG4J_DISABLE = "zookeeper.jmx.log4j.disable";
-    static final String ZOOKEEPER_JUTE_MAX_BUFFER = "jute.maxbuffer";
-    private final Thread zkServerThread;
-    private final ZookeeperServerConfig zookeeperServerConfig;
-
-    VespaZooKeeperServerImpl(ZookeeperServerConfig zookeeperServerConfig, boolean startServer) {
-        this.zookeeperServerConfig = zookeeperServerConfig;
-        System.setProperty("zookeeper.jmx.log4j.disable", "true");
-        System.setProperty("zookeeper.snapshot.trust.empty", Boolean.valueOf(zookeeperServerConfig.trustEmptySnapshot()).toString());
-        System.setProperty(ZOOKEEPER_JUTE_MAX_BUFFER, Integer.valueOf(zookeeperServerConfig.juteMaxBuffer()).toString());
-
-        writeConfigToDisk(zookeeperServerConfig);
-        zkServerThread = new Thread(this, "zookeeper server");
-        if (startServer) {
-            zkServerThread.start();
-        }
-    }
-
-    @Inject
-    public VespaZooKeeperServerImpl(ZookeeperServerConfig zookeeperServerConfig) {
-        this(zookeeperServerConfig, true);
-    }
-
-    private void writeConfigToDisk(ZookeeperServerConfig config) {
-       String configFilePath = getDefaults().underVespaHome(config.zooKeeperConfigFile());
-       new File(configFilePath).getParentFile().mkdirs();
-       try (FileWriter writer = new FileWriter(configFilePath)) {
-           writer.write(transformConfigToString(config));
-           writeMyIdFile(config);
-       } catch (IOException e) {
-           throw new RuntimeException("Error writing zookeeper config", e);
-       }
-   }
-
-    private String transformConfigToString(ZookeeperServerConfig config) {
-        StringBuilder sb = new StringBuilder();
-        sb.append("tickTime=").append(config.tickTime()).append("\n");
-        sb.append("initLimit=").append(config.initLimit()).append("\n");
-        sb.append("syncLimit=").append(config.syncLimit()).append("\n");
-        sb.append("maxClientCnxns=").append(config.maxClientConnections()).append("\n");
-        sb.append("snapCount=").append(config.snapshotCount()).append("\n");
-        sb.append("dataDir=").append(getDefaults().underVespaHome(config.dataDir())).append("\n");
-        sb.append("clientPort=").append(config.clientPort()).append("\n");
-        sb.append("autopurge.purgeInterval=").append(config.autopurge().purgeInterval()).append("\n");
-        sb.append("autopurge.snapRetainCount=").append(config.autopurge().snapRetainCount()).append("\n");
-        // See http://zookeeper.apache.org/doc/r3.4.13/zookeeperAdmin.html#sc_zkCommands
-        // Includes all available commands in 3.4, except 'wchc' and 'wchp'
-        // Mandatory when using ZooKeeper 3.5
-        sb.append("4lw.commands.whitelist=conf,cons,crst,dump,envi,mntr,ruok,srst,srvr,stat,wchs").append("\n");
-        ensureThisServerIsRepresented(config.myid(), config.server());
-        config.server().forEach(server -> addServerToCfg(sb, server));
-        return sb.toString();
-    }
-
-    private void writeMyIdFile(ZookeeperServerConfig config) throws IOException {
-        if (config.server().size() > 1) {
-            try (FileWriter writer = new FileWriter(getDefaults().underVespaHome(config.myidFile()))) {
-                writer.write(config.myid() + "\n");
-            }
-        }
-    }
-
-    private void ensureThisServerIsRepresented(int myid, List<ZookeeperServerConfig.Server> servers) {
-        boolean found = false;
-        for (ZookeeperServerConfig.Server server : servers) {
-            if (myid == server.id()) {
-                found = true;
-                break;
-            }
-        }
-        if (!found) {
-            throw new RuntimeException("No id in zookeeper server list that corresponds to my id(" + myid + ")");
-        }
-    }
-
-    private void addServerToCfg(StringBuilder sb, ZookeeperServerConfig.Server server) {
-        sb.append("server.").append(server.id()).append("=").append(server.hostname()).append(":").append(server.quorumPort()).append(":").append(server.electionPort()).append("\n");
-    }
-
-    private void shutdown() {
-        zkServerThread.interrupt();
-        try {
-            zkServerThread.join();
-        } catch (InterruptedException e) {
-            log.log(LogLevel.WARNING, "Error joining server thread on shutdown", e);
-        }
-    }
-
-    @Override
-    public void run() {
-        System.setProperty(ZOOKEEPER_JMX_LOG4J_DISABLE, "true");
-        String[] args = new String[]{getDefaults().underVespaHome(zookeeperServerConfig.zooKeeperConfigFile())};
-        log.log(LogLevel.DEBUG, "Starting ZooKeeper server with config file " + args[0]);
-        log.log(LogLevel.INFO, "Trying to establish ZooKeeper quorum (from " + zookeeperServerHostnames(zookeeperServerConfig) + ")");
-        org.apache.zookeeper.server.quorum.QuorumPeerMain.main(args);
-    }
-
-    @Override
-    public void deconstruct() {
-        shutdown();
-        super.deconstruct();
-    }
-
-    private static Set<String> zookeeperServerHostnames(ZookeeperServerConfig zookeeperServerConfig) {
-        return zookeeperServerConfig.server().stream().map(ZookeeperServerConfig.Server::hostname).collect(Collectors.toSet());
-    }
-
-}
diff --git a/zookeeper-server/zookeeper-server-3.4/src/test/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImplTest.java b/zookeeper-server/zookeeper-server-3.4/src/test/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImplTest.java
deleted file mode 100644
index 1081c5fda61..00000000000
--- a/zookeeper-server/zookeeper-server-3.4/src/test/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImplTest.java
+++ /dev/null
@@ -1,140 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.zookeeper;
-
-import com.yahoo.cloud.config.ZookeeperServerConfig;
-import com.yahoo.io.IOUtils;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-
-import java.io.File;
-import java.io.IOException;
-
-import static org.hamcrest.CoreMatchers.is;
-import static org.junit.Assert.assertThat;
-import static com.yahoo.vespa.defaults.Defaults.getDefaults;
-
-/**
- * Tests the zookeeper server.
- */
-public class VespaZooKeeperServerImplTest {
-
-    @Rule
-    public TemporaryFolder folder = new TemporaryFolder();
-
-    @Test
-    public void config_is_written_correctly_when_one_server() throws IOException {
-        File cfgFile = folder.newFile();
-        File idFile = folder.newFile();
-        ZookeeperServerConfig.Builder builder = new ZookeeperServerConfig.Builder();
-        builder.zooKeeperConfigFile(cfgFile.getAbsolutePath());
-        builder.myidFile(idFile.getAbsolutePath());
-        builder.server(newServer(0, "foo", 123, 321));
-        builder.myid(0);
-        createServer(builder);
-        validateConfigFileSingleHost(cfgFile);
-        validateIdFile(idFile, "");
-    }
-
-    @Test
-    public void config_is_written_correctly_when_multiple_servers() throws IOException {
-        File cfgFile = folder.newFile();
-        File idFile = folder.newFile();
-        ZookeeperServerConfig.Builder builder = new ZookeeperServerConfig.Builder();
-        builder.zooKeeperConfigFile(cfgFile.getAbsolutePath());
-        builder.server(newServer(0, "foo", 123, 321));
-        builder.server(newServer(1, "bar", 234, 432));
-        builder.server(newServer(2, "baz", 345, 543));
-        builder.myidFile(idFile.getAbsolutePath());
-        builder.myid(1);
-        createServer(builder);
-        validateConfigFileMultipleHosts(cfgFile);
-        validateIdFile(idFile, "1\n");
-    }
-
-    private void createServer(ZookeeperServerConfig.Builder builder) {
-        new VespaZooKeeperServerImpl(new ZookeeperServerConfig(builder), false);
-    }
-
-    @Test(expected = RuntimeException.class)
-    public void require_that_this_id_must_be_present_amongst_servers() {
-        ZookeeperServerConfig.Builder builder = new ZookeeperServerConfig.Builder();
-        builder.server(newServer(1, "bar", 234, 432));
-        builder.server(newServer(2, "baz", 345, 543));
-        builder.myid(0);
-        createServer(builder);
-    }
-
-    @Test
-    public void juteMaxBufferCanBeSet() throws IOException {
-        ZookeeperServerConfig.Builder builder = new ZookeeperServerConfig.Builder();
-        builder.myid(0);
-        File idFile = folder.newFile();
-        File cfgFile = folder.newFile();
-
-        builder.server(new ZookeeperServerConfig.Server.Builder().id(0).hostname("testhost"));
-        builder.zooKeeperConfigFile(cfgFile.getAbsolutePath());
-        builder.myidFile(idFile.getAbsolutePath());
-
-        createServer(builder);
-        assertThat(System.getProperty(VespaZooKeeperServerImpl.ZOOKEEPER_JUTE_MAX_BUFFER), is("" + new ZookeeperServerConfig(builder).juteMaxBuffer()));
-
-        final int max_buffer = 1;
-        builder.juteMaxBuffer(max_buffer);
-        createServer(builder);
-        assertThat(System.getProperty(VespaZooKeeperServerImpl.ZOOKEEPER_JUTE_MAX_BUFFER), is("" + max_buffer));
-    }
-
-    private ZookeeperServerConfig.Server.Builder newServer(int id, String hostName, int electionPort, int quorumPort) {
-        ZookeeperServerConfig.Server.Builder builder = new ZookeeperServerConfig.Server.Builder();
-        builder.id(id);
-        builder.hostname(hostName);
-        builder.electionPort(electionPort);
-        builder.quorumPort(quorumPort);
-        return builder;
-    }
-
-    private void validateIdFile(File idFile, String expected) throws IOException {
-        String actual = IOUtils.readFile(idFile);
-        assertThat(actual, is(expected));
-    }
-
-    private void validateConfigFileSingleHost(File cfgFile) throws IOException {
-        String expected =
-            "tickTime=2000\n" +
-            "initLimit=20\n" +
-            "syncLimit=15\n" +
-            "maxClientCnxns=0\n" +
-            "snapCount=50000\n" +
-            "dataDir=" + getDefaults().underVespaHome("var/zookeeper") + "\n" +
-            "clientPort=2181\n" +
-            "autopurge.purgeInterval=1\n" +
-            "autopurge.snapRetainCount=15\n" +
-            "4lw.commands.whitelist=conf,cons,crst,dump,envi,mntr,ruok,srst,srvr,stat,wchs\n" +
-            "server.0=foo:321:123\n";
-        validateConfigFile(cfgFile, expected);
-    }
-
-    private void validateConfigFileMultipleHosts(File cfgFile) throws IOException {
-        String expected =
-                "tickTime=2000\n" +
-                        "initLimit=20\n" +
-                        "syncLimit=15\n" +
-                        "maxClientCnxns=0\n" +
-                        "snapCount=50000\n" +
-                        "dataDir=" + getDefaults().underVespaHome("var/zookeeper") + "\n" +
-                        "clientPort=2181\n" +
-                        "autopurge.purgeInterval=1\n" +
-                        "autopurge.snapRetainCount=15\n" +
-                        "4lw.commands.whitelist=conf,cons,crst,dump,envi,mntr,ruok,srst,srvr,stat,wchs\n" +
-                        "server.0=foo:321:123\n" +
-                        "server.1=bar:432:234\n" +
-                        "server.2=baz:543:345\n";
-        validateConfigFile(cfgFile, expected);
-    }
-
-    private void validateConfigFile(File cfgFile, String expected) throws IOException {
-        String actual = IOUtils.readFile(cfgFile);
-        assertThat(actual, is(expected));
-    }
-}
diff --git a/zookeeper-server/zookeeper-server-3.5/src/test/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImplTest.java b/zookeeper-server/zookeeper-server-3.5/src/test/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImplTest.java
index 0981ae615df..863d2dba708 100644
--- a/zookeeper-server/zookeeper-server-3.5/src/test/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImplTest.java
+++ b/zookeeper-server/zookeeper-server-3.5/src/test/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImplTest.java
@@ -221,7 +221,7 @@ public class VespaZooKeeperServerImplTest {
                "ssl.quorum.clientAuth=NEED\n" +
                "ssl.quorum.ciphersuites=TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384\n" +
                "ssl.quorum.enabledProtocols=TLSv1.2\n" +
-               "ssl.quorum.protocol=TLSv1.2\n";
+               "ssl.quorum.protocol=TLS\n";
     }
 
     private String commonTlsClientServerConfig() {
@@ -229,7 +229,7 @@ public class VespaZooKeeperServerImplTest {
                "ssl.clientAuth=NEED\n" +
                "ssl.ciphersuites=TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384\n" +
                "ssl.enabledProtocols=TLSv1.2\n" +
-               "ssl.protocol=TLSv1.2\n";
+               "ssl.protocol=TLS\n";
     }
 
     private void validateConfigFileMultipleHosts(File cfgFile) throws IOException {