Create one Dispatcher component per search cluster

This avoids creating an excessive number of connections
to search clusters when the application (incorrectly) creates
many local provider chains to the same search cluster.

18 files changed, 241 insertions, 184 deletions

diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java
index 922c6488659..0bdc3fbef9a 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java
@@ -46,8 +46,7 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat
         RankProfilesConfig.Producer,
         RankingConstantsConfig.Producer,
         ServletPathsConfig.Producer,
-        ContainerMbusConfig.Producer
-{
+        ContainerMbusConfig.Producer {
 
     private final Set<FileReference> applicationBundles = new LinkedHashSet<>();
 
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java
index e4f5791b6b3..f5eb128e7f3 100755
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java
@@ -99,9 +99,7 @@ public abstract class ContainerCluster<CONTAINER extends Container>
         DocprocConfig.Producer,
         ClusterInfoConfig.Producer,
         RoutingProviderConfig.Producer,
-        ConfigserverConfig.Producer
-
-{
+        ConfigserverConfig.Producer {
 
     /**
      * URI prefix used for internal, usually programmatic, APIs. URIs using this
@@ -268,6 +266,7 @@ public abstract class ContainerCluster<CONTAINER extends Container>
      *
      * @return the removed component, or null if it was not present
      */
+    @SuppressWarnings("unused") // Used from other repositories
     public Component removeComponent(ComponentId componentId) {
         return componentGroup.removeComponent(componentId);
     }
@@ -298,7 +297,7 @@ public abstract class ContainerCluster<CONTAINER extends Container>
     public void addContainer(CONTAINER container) {
         container.setClusterName(name);
         container.setProp("clustername", name)
-                .setProp("index", this.containers.size());
+                 .setProp("index", this.containers.size());
         containers.add(container);
     }
 
@@ -330,7 +329,7 @@ public abstract class ContainerCluster<CONTAINER extends Container>
     public SearchChains getSearchChains() {
         if (containerSearch == null)
             throw new IllegalStateException("Search components not found in container cluster '" + getSubId() +
-                                                    "': Add <search/> to the cluster in services.xml");
+                                            "': Add <search/> to the cluster in services.xml");
         return containerSearch.getChains();
     }
 
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModel.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModel.java
index f02878fe545..67870dc049d 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModel.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModel.java
@@ -48,9 +48,8 @@ public class ContainerModel extends ConfigModel {
         List<AbstractSearchCluster> searchClusters = Content.getSearchClusters(configModelRepo);
 
         Map<String, AbstractSearchCluster> searchClustersByName = new TreeMap<>();
-        for (AbstractSearchCluster c : searchClusters) {
+        for (AbstractSearchCluster c : searchClusters)
             searchClustersByName.put(c.getClusterName(), c);
-        }
 
         getCluster().initialize(searchClustersByName);
     }
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/ConfigProducerGroup.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/ConfigProducerGroup.java
index 66294d3fcef..c671749cff0 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/ConfigProducerGroup.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/ConfigProducerGroup.java
@@ -55,9 +55,7 @@ public class ConfigProducerGroup<CHILD extends AbstractConfigProducer<?>> extend
         return Collections.unmodifiableCollection(result);
     }
 
-    /**
-     * @return A map of all components in this group, with (local) component ID as key.
-     */
+    /** Returns a map of all components in this group, with (local) component ID as key. */
     public Map<ComponentId, CHILD> getComponentMap() {
         return Collections.unmodifiableMap(producerById);
     }
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/Handler.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/Handler.java
index 62aae85b55a..e07c3216850 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/Handler.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/Handler.java
@@ -11,17 +11,13 @@ import java.util.Collections;
 import java.util.List;
 
 /**
- * <p>
  * Models a jdisc RequestHandler (including ClientProvider).
  * RequestHandlers always have at least one server binding,
  * while ClientProviders have at least one client binding.
- * </p>
  * <p>
  * Note that this is also used to model vespa handlers (which do not have any bindings)
- * </p>
  *
  * @author gjoranv
- * @since 5.1.6
  */
 public class Handler<CHILD extends AbstractConfigProducer<?>> extends Component<CHILD, ComponentModel> {
 
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/SimpleComponent.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/SimpleComponent.java
index c4e988e6749..46ee0d0bd37 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/SimpleComponent.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/SimpleComponent.java
@@ -9,7 +9,6 @@ import com.yahoo.config.model.producer.AbstractConfigProducer;
  * A component that only needs a simple ComponentModel.
  *
  * @author gjoranv
- * @since 5.1.9
  */
 public class SimpleComponent extends Component<AbstractConfigProducer<?>, ComponentModel> {
 
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/chain/Chain.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/chain/Chain.java
index 6b4f8c8f8b5..97a09aa48d8 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/chain/Chain.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/chain/Chain.java
@@ -26,7 +26,6 @@ public class Chain<T extends ChainedComponent<?>> extends AbstractConfigProducer
     private final ComponentGroup<T> innerComponentsGroup;
     private static final Type.Enum TYPE = Type.SEARCH;
 
-
     public Chain(ChainSpecification specWithoutInnerComponents) {
         super(specWithoutInnerComponents.componentId.stringValue());
 
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java
index 5266fc46f6e..e19d81e7fb2 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java
@@ -15,6 +15,7 @@ import com.yahoo.vespa.model.search.AbstractSearchCluster;
 import com.yahoo.vespa.model.search.IndexedSearchCluster;
 import com.yahoo.vespa.model.search.StreamingSearchCluster;
 
+import java.util.Collection;
 import java.util.LinkedHashMap;
 import java.util.LinkedList;
 import java.util.List;
@@ -33,7 +34,8 @@ public class ContainerSearch extends ContainerSubsystem<SearchChains>
         SemanticRulesConfig.Producer,
     	PageTemplatesConfig.Producer {
 
-    private final List<AbstractSearchCluster> systems = new LinkedList<>();
+    private ApplicationContainerCluster owningCluster;
+    private final List<AbstractSearchCluster> searchClusters = new LinkedList<>();
     private final Options options;
 
     private QueryProfiles queryProfiles;
@@ -42,26 +44,36 @@ public class ContainerSearch extends ContainerSubsystem<SearchChains>
 
     public ContainerSearch(ApplicationContainerCluster cluster, SearchChains chains, Options options) {
         super(chains);
+        this.owningCluster = cluster;
         this.options = options;
     }
 
     public void connectSearchClusters(Map<String, AbstractSearchCluster> searchClusters) {
-        systems.addAll(searchClusters.values());
+        this.searchClusters.addAll(searchClusters.values());
+        initializeDispatchers(searchClusters.values());
         initializeSearchChains(searchClusters);
     }
 
+    /** Adds a Dispatcher component to the owning container cluster for each search cluster */
+    private void initializeDispatchers(Collection<AbstractSearchCluster> searchClusters) {
+        for (AbstractSearchCluster searchCluster : searchClusters) {
+            if ( ! ( searchCluster instanceof IndexedSearchCluster)) continue;
+            owningCluster.addComponent(new DispatcherComponent((IndexedSearchCluster)searchCluster));
+        }
+    }
+
     // public for testing
     public void initializeSearchChains(Map<String, ? extends AbstractSearchCluster> searchClusters) {
         getChains().initialize(searchClusters);
 
         QrsCache defaultCacheOptions = getOptions().cacheSettings.get("");
         if (defaultCacheOptions != null) {
-            for (LocalProvider localProvider: getChains().localProviders()) {
+            for (LocalProvider localProvider : getChains().localProviders()) {
                 localProvider.setCacheSize(defaultCacheOptions.size);
             }
         }
 
-        for (LocalProvider localProvider: getChains().localProviders()) {
+        for (LocalProvider localProvider : getChains().localProviders()) {
             QrsCache cacheOptions = getOptions().cacheSettings.get(localProvider.getClusterName());
             if (cacheOptions != null) {
                 localProvider.setCacheSize(cacheOptions.size);
@@ -83,39 +95,39 @@ public class ContainerSearch extends ContainerSubsystem<SearchChains>
 
     @Override
     public void getConfig(QueryProfilesConfig.Builder builder) {
-        if (queryProfiles!=null) {
+        if (queryProfiles != null) {
             queryProfiles.getConfig(builder);
         }
     }
 
     @Override
     public void getConfig(SemanticRulesConfig.Builder builder) {
-        if (semanticRules!=null) semanticRules.getConfig(builder);
+        if (semanticRules != null) semanticRules.getConfig(builder);
     }
 
     @Override
     public void getConfig(PageTemplatesConfig.Builder builder) {
-        if (pageTemplates!=null) pageTemplates.getConfig(builder);
+        if (pageTemplates != null) pageTemplates.getConfig(builder);
     }
 
     @Override
     public void getConfig(IndexInfoConfig.Builder builder) {
-        for (AbstractSearchCluster sc : systems) {
+        for (AbstractSearchCluster sc : searchClusters) {
             sc.getConfig(builder);
         }
     }
 
     @Override
     public void getConfig(IlscriptsConfig.Builder builder) {
-        for (AbstractSearchCluster sc : systems) {
+        for (AbstractSearchCluster sc : searchClusters) {
             sc.getConfig(builder);
         }
     }
 
     @Override
     public void getConfig(QrSearchersConfig.Builder builder) {
-        for (int i = 0; i < systems.size(); i++) {
-    	    AbstractSearchCluster sys = findClusterWithId(systems, i);
+        for (int i = 0; i < searchClusters.size(); i++) {
+    	    AbstractSearchCluster sys = findClusterWithId(searchClusters, i);
     		QrSearchersConfig.Searchcluster.Builder scB = new QrSearchersConfig.Searchcluster.Builder().
     				name(sys.getClusterName());
     		for (AbstractSearchCluster.SearchDefinitionSpec spec : sys.getLocalSDS()) {
@@ -133,9 +145,8 @@ public class ContainerSearch extends ContainerSubsystem<SearchChains>
 
     private static AbstractSearchCluster findClusterWithId(List<AbstractSearchCluster> clusters, int index) {
         for (AbstractSearchCluster sys : clusters) {
-            if (sys.getClusterIndex() == index) {
+            if (sys.getClusterIndex() == index)
                 return sys;
-            }
         }
         throw new IllegalArgumentException("No search cluster with index " + index + " exists");
     }
@@ -144,10 +155,11 @@ public class ContainerSearch extends ContainerSubsystem<SearchChains>
         return options;
     }
 
-    /**
-     * Struct that encapsulates qrserver options.
-     */
+    /** Encapsulates qrserver options. */
     public static class Options {
+
         Map<String, QrsCache> cacheSettings = new LinkedHashMap<>();
+
     }
+
 }
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/DispatcherComponent.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/DispatcherComponent.java
new file mode 100644
index 00000000000..7f51efcd092
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/DispatcherComponent.java
@@ -0,0 +1,37 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.container.search;
+
+import com.yahoo.osgi.provider.model.ComponentModel;
+import com.yahoo.vespa.config.search.DispatchConfig;
+import com.yahoo.vespa.model.container.component.Component;
+import com.yahoo.vespa.model.container.xml.BundleMapper;
+import com.yahoo.vespa.model.search.IndexedSearchCluster;
+
+/**
+ * Represents a dispatcher component - an instance of a dispatcher in a search container cluster
+ * knows how to communicate with one indexed search cluster and owns the connections to it.
+ */
+public class DispatcherComponent extends Component<DispatcherComponent, ComponentModel>
+        implements DispatchConfig.Producer {
+
+    private final IndexedSearchCluster indexedSearchCluster;
+
+    public DispatcherComponent(IndexedSearchCluster indexedSearchCluster) {
+        super(toComponentModel(indexedSearchCluster));
+        this.indexedSearchCluster = indexedSearchCluster;
+    }
+
+    private static ComponentModel toComponentModel(IndexedSearchCluster indexedSearchCluster) {
+        String dispatcherComponentId = "dispatcher." + indexedSearchCluster.getClusterName(); // used by ClusterSearcher
+        return new ComponentModel(dispatcherComponentId,
+                                  "com.yahoo.search.dispatch.Dispatcher",
+                                  BundleMapper.searchAndDocprocBundle,
+                                  null);
+    }
+
+    @Override
+    public void getConfig(DispatchConfig.Builder builder) {
+        indexedSearchCluster.getConfig(builder);
+    }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/GenericTarget.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/GenericTarget.java
index 875de1465c6..660ee2628b8 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/GenericTarget.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/GenericTarget.java
@@ -6,6 +6,7 @@ import com.yahoo.search.searchchain.model.federation.FederationOptions;
 
 /**
  * A search chain that is intended to be used for federation (i.e. providers, sources)
+ *
  * @author Tony Vaagenes
  */
 abstract public class GenericTarget extends SearchChain {
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/LocalProvider.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/LocalProvider.java
index aa820d1d898..4d02082217c 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/LocalProvider.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/LocalProvider.java
@@ -18,7 +18,11 @@ import com.yahoo.vespa.model.search.AbstractSearchCluster;
 import com.yahoo.vespa.model.search.IndexedSearchCluster;
 import com.yahoo.vespa.model.search.SearchNode;
 
-import java.util.*;
+import java.util.ArrayList;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Set;
+
 
 /**
  * Config producer for search chain responsible for sending queries to a local cluster.
@@ -31,8 +35,7 @@ public class LocalProvider extends Provider implements
         AttributesConfig.Producer,
         QrMonitorConfig.Producer,
         RankProfilesConfig.Producer,
-        SearchNodesConfig.Producer,
-        DispatchConfig.Producer {
+        SearchNodesConfig.Producer {
 
     private final LocalProviderSpec providerSpec;
     private volatile AbstractSearchCluster searchCluster;
@@ -122,7 +125,6 @@ public class LocalProvider extends Provider implements
     }
 
     public void setSearchCluster(AbstractSearchCluster searchCluster) {
-        assert (this.searchCluster == null);
         this.searchCluster = searchCluster;
     }
 
@@ -176,13 +178,4 @@ public class LocalProvider extends Provider implements
         return (visibilityDelay < 1.0d) ? 0.0d : visibilityDelay;
     }
 
-    @Override
-    public void getConfig(DispatchConfig.Builder builder) {
-        if (!(searchCluster instanceof IndexedSearchCluster)) {
-            log.warning("Could not build DispatchConfig: Only supported for IndexedSearchCluster, got "
-                        + searchCluster.getClass().getCanonicalName());
-            return;
-        }
-        ((IndexedSearchCluster) searchCluster).getConfig(builder);
-    }
 }
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/SearchChains.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/SearchChains.java
index 24ff885f131..1520dffb695 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/SearchChains.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/SearchChains.java
@@ -29,7 +29,7 @@ public class SearchChains extends Chains<SearchChain> {
         LocalClustersCreator.addDefaultLocalProviders(this, searchClustersByName.keySet());
         VespaSearchChainsCreator.addVespaSearchChains(this);
 
-        validateSourceGroups(); //must be done before initializing searchers since they are used by FederationSearchers.
+        validateSourceGroups(); //must be done before initializing searchers since they are used by FederationSearchers
         initializeComponents(searchClustersByName);
     }
 
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleMapper.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleMapper.java
index d8e678a5cd3..aa7cb5eb539 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleMapper.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleMapper.java
@@ -28,7 +28,6 @@ public class BundleMapper {
 
     public static final Path LIBRARY_PATH = Paths.get(Defaults.getDefaults().underVespaHome("lib/jars"));
 
-
     public static final String searchAndDocprocBundle = "container-search-and-docproc";
 
     private static final Map<String, String> bundleFromClass;
@@ -111,7 +110,6 @@ public class BundleMapper {
         bundleFromClass.put("com.yahoo.search.pagetemplates.engine.resolvers.DeterministicResolver", searchAndDocprocBundle);
         bundleFromClass.put("com.yahoo.search.pagetemplates.engine.resolvers.RandomResolver", searchAndDocprocBundle);
         bundleFromClass.put("com.yahoo.search.pagetemplates.model.Renderer", searchAndDocprocBundle);
-        bundleFromClass.put("com.yahoo.search.pagetemplates.model.Renderer", searchAndDocprocBundle);
         bundleFromClass.put("com.yahoo.search.query.rewrite.QueryRewriteSearcher", searchAndDocprocBundle);
         bundleFromClass.put("com.yahoo.search.query.rewrite.SearchChainDispatcherSearcher", searchAndDocprocBundle);
         bundleFromClass.put("com.yahoo.search.query.rewrite.rewriters.GenericExpansionRewriter", searchAndDocprocBundle);
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/AbstractSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/AbstractSearchCluster.java
index fbbf029d5f1..85663a59556 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/search/AbstractSearchCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/search/AbstractSearchCluster.java
@@ -29,51 +29,7 @@ public abstract class AbstractSearchCluster extends AbstractConfigProducer
     protected int index;
     private Double visibilityDelay = 0.0;
     private List<String> documentNames = new ArrayList<>();
-
-    protected List<SearchDefinitionSpec> localSDS = new LinkedList<>();
-
-    public void prepareToDistributeFiles(List<SearchNode> backends) {
-        for (SearchDefinitionSpec sds : localSDS)
-            sds.getSearchDefinition().getSearch().rankingConstants().sendTo(backends);
-    }
-
-    public static final class IndexingMode {
-
-        public static final IndexingMode REALTIME  = new IndexingMode("REALTIME");
-        public static final IndexingMode STREAMING = new IndexingMode("STREAMING");
-
-        private final String name;
-
-        private IndexingMode(String name) {
-            this.name = name;
-        }
-
-        public String getName() { return name; }
-
-        public String toString() {
-            return "indexingmode: " + name;
-        }
-    }
-
-    public static final class SearchDefinitionSpec {
-
-        private final SearchDefinition searchDefinition;
-        private final UserConfigRepo userConfigRepo;
-
-        public SearchDefinitionSpec(SearchDefinition searchDefinition,
-                                    UserConfigRepo userConfigRepo) {
-            this.searchDefinition = searchDefinition;
-            this.userConfigRepo = userConfigRepo;
-        }
-
-        public SearchDefinition getSearchDefinition() {
-            return searchDefinition;
-        }
-
-        public UserConfigRepo getUserConfigs() {
-            return userConfigRepo;
-        }
-    }
+    private List<SearchDefinitionSpec> localSDS = new LinkedList<>();
 
     public AbstractSearchCluster(AbstractConfigProducer parent, String clusterName, int index) {
         super(parent, "cluster." + clusterName);
@@ -81,6 +37,11 @@ public abstract class AbstractSearchCluster extends AbstractConfigProducer
         this.index = index;
     }
 
+    public void prepareToDistributeFiles(List<SearchNode> backends) {
+        for (SearchDefinitionSpec sds : localSDS)
+            sds.getSearchDefinition().getSearch().rankingConstants().sendTo(backends);
+    }
+
     public void addDocumentNames(SearchDefinition searchDefinition) {
         String dName = searchDefinition.getSearch().getDocument().getDocumentName().getName();
         documentNames.add(dName);
@@ -97,14 +58,17 @@ public abstract class AbstractSearchCluster extends AbstractConfigProducer
     public final String getIndexingModeName()   { return getIndexingMode().getName(); }
     public final boolean isRealtime()           { return getIndexingMode() == IndexingMode.REALTIME; }
     public final boolean isStreaming()          { return getIndexingMode() == IndexingMode.STREAMING; }
+
     public final AbstractSearchCluster setQueryTimeout(Double to) {
         this.queryTimeout=to;
         return this;
     }
+
     public final AbstractSearchCluster setVisibilityDelay(double delay) {
         this.visibilityDelay=delay;
         return this;
     }
+
     protected abstract IndexingMode getIndexingMode();
     public final Double getVisibilityDelay() { return visibilityDelay; }
     public final Double getQueryTimeout() { return queryTimeout; }
@@ -122,4 +86,42 @@ public abstract class AbstractSearchCluster extends AbstractConfigProducer
     public abstract void getConfig(RankProfilesConfig.Builder builder);
     public abstract void getConfig(AttributesConfig.Builder builder);
 
+    public static final class IndexingMode {
+
+        public static final IndexingMode REALTIME  = new IndexingMode("REALTIME");
+        public static final IndexingMode STREAMING = new IndexingMode("STREAMING");
+
+        private final String name;
+
+        private IndexingMode(String name) {
+            this.name = name;
+        }
+
+        public String getName() { return name; }
+
+        public String toString() {
+            return "indexingmode: " + name;
+        }
+    }
+
+    public static final class SearchDefinitionSpec {
+
+        private final SearchDefinition searchDefinition;
+        private final UserConfigRepo userConfigRepo;
+
+        public SearchDefinitionSpec(SearchDefinition searchDefinition,
+                                    UserConfigRepo userConfigRepo) {
+            this.searchDefinition = searchDefinition;
+            this.userConfigRepo = userConfigRepo;
+        }
+
+        public SearchDefinition getSearchDefinition() {
+            return searchDefinition;
+        }
+
+        public UserConfigRepo getUserConfigs() {
+            return userConfigRepo;
+        }
+    }
+
 }
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java
index 40cc09938e2..3ea36dc800e 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java
@@ -30,53 +30,9 @@ import java.util.List;
 public class IndexedSearchCluster extends SearchCluster
     implements
         DocumentdbInfoConfig.Producer,
-        // TODO consider removing, these only produced by UnionConfiguration and DocumentDatabase?
         IndexInfoConfig.Producer,
         IlscriptsConfig.Producer,
-        DispatchConfig.Producer
-{
-
-    /**
-     * Class used to retrieve combined configuration from multiple document databases.
-     * It is not a {@link com.yahoo.config.ConfigInstance.Producer} of those configs,
-     * that is handled (by delegating to this) by the {@link IndexedSearchCluster}
-     * which is the parent to this. This avoids building the config multiple times.
-     */
-    public static class UnionConfiguration
-        extends AbstractConfigProducer
-        implements AttributesConfig.Producer {
-        private final List<DocumentDatabase> docDbs;
-
-        public void getConfig(IndexInfoConfig.Builder builder) {
-            for (DocumentDatabase docDb : docDbs) {
-                docDb.getConfig(builder);
-            }
-        }
-
-        public void getConfig(IlscriptsConfig.Builder builder) {
-            for (DocumentDatabase docDb : docDbs) {
-                docDb.getConfig(builder);
-            }
-        }
-
-        @Override
-        public void getConfig(AttributesConfig.Builder builder) {
-            for (DocumentDatabase docDb : docDbs) {
-                docDb.getConfig(builder);
-            }
-        }
-
-        public void getConfig(RankProfilesConfig.Builder builder) {
-            for (DocumentDatabase docDb : docDbs) {
-                docDb.getConfig(builder);
-            }
-        }
-
-        private UnionConfiguration(AbstractConfigProducer parent, List<DocumentDatabase> docDbs) {
-            super(parent, "union");
-            this.docDbs = docDbs;
-        }
-    }
+        DispatchConfig.Producer {
 
     private String indexingClusterName = null; // The name of the docproc cluster to run indexing, by config.
     private String indexingChainName = null;
@@ -388,4 +344,46 @@ public class IndexedSearchCluster extends SearchCluster
     @Override
     public int getRowBits() { return 8; }
 
+    /**
+     * Class used to retrieve combined configuration from multiple document databases.
+     * It is not a {@link com.yahoo.config.ConfigInstance.Producer} of those configs,
+     * that is handled (by delegating to this) by the {@link IndexedSearchCluster}
+     * which is the parent to this. This avoids building the config multiple times.
+     */
+    public static class UnionConfiguration
+            extends AbstractConfigProducer
+            implements AttributesConfig.Producer {
+        private final List<DocumentDatabase> docDbs;
+
+        public void getConfig(IndexInfoConfig.Builder builder) {
+            for (DocumentDatabase docDb : docDbs) {
+                docDb.getConfig(builder);
+            }
+        }
+
+        public void getConfig(IlscriptsConfig.Builder builder) {
+            for (DocumentDatabase docDb : docDbs) {
+                docDb.getConfig(builder);
+            }
+        }
+
+        @Override
+        public void getConfig(AttributesConfig.Builder builder) {
+            for (DocumentDatabase docDb : docDbs) {
+                docDb.getConfig(builder);
+            }
+        }
+
+        public void getConfig(RankProfilesConfig.Builder builder) {
+            for (DocumentDatabase docDb : docDbs) {
+                docDb.getConfig(builder);
+            }
+        }
+
+        private UnionConfiguration(AbstractConfigProducer parent, List<DocumentDatabase> docDbs) {
+            super(parent, "union");
+            this.docDbs = docDbs;
+        }
+    }
+
 }
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedTest.java
index 62cdcf5a0c6..ecf8f100288 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedTest.java
@@ -166,8 +166,7 @@ public class IndexedTest extends ContentBaseTest {
     }
 
     @Test
-    public void requireProtonStreamingOnly()
-    {
+    public void requireProtonStreamingOnly() {
         VespaModel model = getStreamingVespaModel();
         // TODO
         // HostResource h = model.getHostSystem().getHosts().get(0);
@@ -185,8 +184,7 @@ public class IndexedTest extends ContentBaseTest {
     }
 
     @Test
-    public void requireCorrectClusterList()
-    {
+    public void requireCorrectClusterList() {
         VespaModel model = getStreamingVespaModel();
         ContentCluster s = model.getContentClusters().get("test");
         assertNotNull(s);
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java
index 4291c68eff8..54ded910bfa 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java
@@ -24,7 +24,8 @@ import static org.junit.Assert.assertTrue;
  */
 public class ClusterTest {
 
-    static final double DELTA = 1E-12;
+    private static final double DELTA = 1E-12;
+
     @Test
     public void requireThatContentSearchIsApplied() {
         ContentCluster cluster = newContentCluster(joinLines("<search>",
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchClusterTest.java
index 007dc5d7266..1c4e005cb67 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchClusterTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchClusterTest.java
@@ -1,6 +1,7 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.model.search.test;
 
+import com.yahoo.component.ComponentId;
 import com.yahoo.container.QrSearchersConfig;
 import com.yahoo.document.DataType;
 import com.yahoo.search.config.ClusterConfig;
@@ -9,20 +10,22 @@ import com.yahoo.searchdefinition.SearchBuilder;
 import com.yahoo.searchdefinition.document.Attribute;
 import com.yahoo.searchdefinition.document.SDDocumentType;
 import com.yahoo.searchdefinition.document.SDField;
+import com.yahoo.vespa.config.search.DispatchConfig;
 import com.yahoo.vespa.indexinglanguage.expressions.AttributeExpression;
 import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression;
 import com.yahoo.vespa.indexinglanguage.expressions.StatementExpression;
 import com.yahoo.vespa.model.VespaModel;
 import com.yahoo.vespa.model.container.ContainerCluster;
+import com.yahoo.vespa.model.container.component.Component;
+import com.yahoo.vespa.model.search.AbstractSearchCluster;
+import com.yahoo.vespa.model.search.SearchCluster;
 import com.yahoo.vespa.model.test.utils.ApplicationPackageUtils;
 import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithMockPkg;
 import org.junit.Test;
 
-import static org.hamcrest.CoreMatchers.is;
 import static org.junit.Assert.*;
 
 /**
- *
  * Unit tests for SearchCluster. Please use this instead of SearchModelTestCase if possible and
  * write _unit_ tests. Thanks.
  *
@@ -30,32 +33,19 @@ import static org.junit.Assert.*;
  */
 public class SearchClusterTest {
 
-    private String vespaHosts = "<?xml version=\"1.0\" encoding=\"utf-8\" ?>" +
-            "<hosts>  " +
-            "<host name=\"foo\">" +
-            "<alias>node0</alias>" +
-            "</host>" +
-            "<host name=\"bar\">" +
-            "<alias>node1</alias>" +
-            "</host>" +
-            "<host name=\"baz\">" +
-            "<alias>node2</alias>" +
-            "</host>" +
-            "</hosts>";
-
     @Test
     public void testSdConfigLogical() {
         // sd1
-        SDDocumentType sdt1=new SDDocumentType("s1");
+        SDDocumentType sdt1 = new SDDocumentType("s1");
         Search search1 = new Search("s1", null);
-        SDField f1=new SDField("f1", DataType.STRING);
+        SDField f1 = new SDField("f1", DataType.STRING);
         f1.addAttribute(new Attribute("f1", DataType.STRING));
         f1.setIndexingScript(new ScriptExpression(new StatementExpression(new AttributeExpression("f1"))));
         sdt1.addField(f1);
         search1.addDocument(sdt1);
 
         // sd2
-        SDDocumentType sdt2=new SDDocumentType("s2");
+        SDDocumentType sdt2 = new SDDocumentType("s2");
         Search search2 = new Search("s2", null);
         SDField f2=new SDField("f2", DataType.STRING);
         f2.addAttribute(new Attribute("f2", DataType.STRING));
@@ -71,7 +61,21 @@ public class SearchClusterTest {
 
     @Test
     public void search_model_is_connected_to_container_clusters_two_content_clusters() {
-        String services = "<?xml version=\"1.0\" encoding=\"utf-8\" ?>" +
+        String vespaHosts = "<?xml version='1.0' encoding='utf-8' ?>" +
+                            "<hosts>" +
+                            "  <host name='node0host'>" +
+                            "    <alias>node0</alias>" +
+                            "  </host>" +
+                            "  <host name='node1host'>" +
+                            "    <alias>node1</alias>" +
+                            "  </host>" +
+                            "  <host name='node2host'>" +
+                            "    <alias>node2</alias>" +
+                            "  </host>" +
+                            "</hosts>";
+
+        String services =
+                "<?xml version=\"1.0\" encoding=\"utf-8\" ?>" +
                 "<services version=\"1.0\">" +
                 "  <admin version='2.0'>" +
                 "    <adminserver hostalias='node0' />" +
@@ -105,50 +109,74 @@ public class SearchClusterTest {
                 "     <documents>" +
                 "       <document mode='index' type=\"music\" />" +
                 "     </documents>" +
-                "         <nodes>" +
-                "            <node hostalias=\"node0\" distribution-key=\"0\" />" +
-                "         </nodes>" +
+                "     <nodes>" +
+                "       <node hostalias=\"node0\" distribution-key=\"0\" />" +
+                "     </nodes>" +
                 "  </content>" +
                 "  <content id=\"normal\" version='1.0'>" +
                 "     <redundancy>2</redundancy>" +
                 "     <documents>" +
                 "       <document mode='index' type=\"music\" />" +
                 "     </documents>" +
-                "         <nodes>" +
-                "            <node hostalias=\"node2\" distribution-key=\"0\" />" +
-                "         </nodes>" +
+                "     <nodes>" +
+                "       <node hostalias=\"node2\" distribution-key=\"0\" />" +
+                "     </nodes>" +
                 "  </content>" +
                 "</services>";
 
         VespaModel model = new VespaModelCreatorWithMockPkg(vespaHosts, services, ApplicationPackageUtils.generateSearchDefinitions("music")).create();
 
-        ContainerCluster cluster1 = (ContainerCluster)model.getConfigProducer("j1").get();
-        assertFalse(cluster1.getSearch().getChains().localProviders().isEmpty());
+        ContainerCluster containerCluster1 = (ContainerCluster)model.getConfigProducer("j1").get();
+        assertFalse(containerCluster1.getSearch().getChains().localProviders().isEmpty());
 
-        ContainerCluster cluster2 = (ContainerCluster)model.getConfigProducer("j2").get();
-        assertFalse(cluster2.getSearch().getChains().localProviders().isEmpty());
+        ContainerCluster containerCluster2 = (ContainerCluster)model.getConfigProducer("j2").get();
+        assertFalse(containerCluster2.getSearch().getChains().localProviders().isEmpty());
 
         QrSearchersConfig.Builder builder = new QrSearchersConfig.Builder();
-        cluster1.getConfig(builder);
+        containerCluster1.getConfig(builder);
         QrSearchersConfig config = new QrSearchersConfig(builder);
 
-        assertThat(config.searchcluster().size(), is(2));
-        int normalId = 0;
-        int bulkId = 1;
-        assertThat(config.searchcluster().get(normalId).name(), is("normal"));
-        assertThat(config.searchcluster().get(bulkId).name(), is("xbulk"));
+        assertEquals(2, config.searchcluster().size());
+        int normalIndex = 0;
+        int xbulkIndex = 1;
+        assertEquals("normal", config.searchcluster().get(normalIndex).name());
+        assertEquals("xbulk", config.searchcluster().get(xbulkIndex).name());
 
         ClusterConfig.Builder clusterConfigBuilder = new ClusterConfig.Builder();
         model.getConfig(clusterConfigBuilder, "j1/searchchains/chain/normal/component/com.yahoo.prelude.cluster.ClusterSearcher");
         ClusterConfig clusterConfig = new ClusterConfig(clusterConfigBuilder);
-        assertThat(clusterConfig.clusterId(), is(normalId));
-        assertThat(clusterConfig.clusterName(), is("normal"));
+        assertEquals(normalIndex, clusterConfig.clusterId());
+        assertEquals("normal", clusterConfig.clusterName());
 
         ClusterConfig.Builder clusterConfigBuilder2 = new ClusterConfig.Builder();
         model.getConfig(clusterConfigBuilder2, "j2/searchchains/chain/xbulk/component/com.yahoo.prelude.cluster.ClusterSearcher");
         ClusterConfig clusterConfig2 = new ClusterConfig(clusterConfigBuilder2);
-        assertThat(clusterConfig2.clusterId(), is(bulkId));
-        assertThat(clusterConfig2.clusterName(), is("xbulk"));
+        assertEquals(xbulkIndex, clusterConfig2.clusterId());
+        assertEquals("xbulk", clusterConfig2.clusterName());
+
+        AbstractSearchCluster searchCluster1 = model.getSearchClusters().get(normalIndex);
+        assertEquals("normal", searchCluster1.getClusterName());
+        assertEquals("normal/search/cluster.normal", searchCluster1.getConfigId());
+        AbstractSearchCluster searchCluster2 = model.getSearchClusters().get(xbulkIndex);
+        assertEquals("xbulk", searchCluster2.getClusterName());
+
+        Component<?,?> normalDispatcher = (Component<?, ?>)containerCluster1.getComponentsMap().get(new ComponentId("dispatcher.normal"));
+        assertNotNull(normalDispatcher);
+        assertEquals("dispatcher.normal", normalDispatcher.getComponentId().stringValue());
+        assertEquals("com.yahoo.search.dispatch.Dispatcher", normalDispatcher.getClassId().stringValue());
+        assertEquals("j1/component/dispatcher.normal", normalDispatcher.getConfigId());
+        DispatchConfig.Builder normalDispatchConfigBuilder = new DispatchConfig.Builder();
+        model.getConfig(normalDispatchConfigBuilder, "j1/component/dispatcher.normal");
+        assertEquals("node2host", normalDispatchConfigBuilder.build().node(0).host());
+
+        Component<?,?> xbulkDispatcher = (Component<?, ?>)containerCluster1.getComponentsMap().get(new ComponentId("dispatcher.xbulk"));
+        assertNotNull(xbulkDispatcher);
+        assertEquals("dispatcher.xbulk", xbulkDispatcher.getComponentId().stringValue());
+        assertEquals("com.yahoo.search.dispatch.Dispatcher", xbulkDispatcher.getClassId().stringValue());
+        assertEquals("j1/component/dispatcher.xbulk", xbulkDispatcher.getConfigId());
+        DispatchConfig.Builder xbulkDispatchConfigBuilder = new DispatchConfig.Builder();
+        model.getConfig(xbulkDispatchConfigBuilder, "j1/component/dispatcher.xbulk");
+        assertEquals("node0host", xbulkDispatchConfigBuilder.build().node(0).host());
     }
 
 }