Always add an indexing docproc, also for streaming search.

10 files changed, 48 insertions, 77 deletions

diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java b/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java
index 4e56d1d1d5a..1254f8e110a 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java
@@ -27,7 +27,6 @@ import com.yahoo.vespa.model.container.docproc.ContainerDocproc;
 import com.yahoo.vespa.model.container.docproc.DocprocChain;
 import com.yahoo.vespa.model.container.docproc.DocprocChains;
 import com.yahoo.vespa.model.content.cluster.ContentCluster;
-import com.yahoo.vespa.model.search.IndexedSearchCluster;
 import com.yahoo.vespa.model.search.IndexingDocproc;
 import com.yahoo.vespa.model.search.IndexingDocprocChain;
 import com.yahoo.vespa.model.search.SearchCluster;
@@ -215,25 +214,21 @@ public class Content extends ConfigModel {
         private void buildIndexingClusters(Content content, ConfigModelContext modelContext,
                                            ApplicationConfigProducerRoot root) {
             var search = content.getCluster().getSearch();
-            if (!search.getIndexingDocproc().isPresent()) {
-                return;
-            }
-            var indexingDocproc = search.getIndexingDocproc().get();
+            var indexingDocproc = search.getIndexingDocproc();
             if (indexingDocproc.hasExplicitCluster()) {
                 setExistingIndexingCluster(content, indexingDocproc, content.containers);
             } else {
-                if (search.hasIndexedCluster()) {
-                    setContainerAsIndexingCluster(search.getIndexed(), content, modelContext, root);
-                }
+                setContainerAsIndexingCluster(search.getSearchNodes(), indexingDocproc, content, modelContext, root);
             }
         }
 
-        private void setContainerAsIndexingCluster(IndexedSearchCluster indexedSearchCluster,
+        private void setContainerAsIndexingCluster(List<SearchNode> cluster,
+                                                   IndexingDocproc indexingDocproc,
                                                    Content content,
                                                    ConfigModelContext modelContext,
                                                    ApplicationConfigProducerRoot root) {
             if (content.containers.isEmpty()) {
-                createImplicitIndexingCluster(indexedSearchCluster, content, modelContext, root);
+                createImplicitIndexingCluster(cluster, indexingDocproc, content, modelContext, root);
             } else {
                 ContainerCluster<?> targetCluster = getContainerWithDocproc(content.containers);
                 if (targetCluster == null)
@@ -242,7 +237,6 @@ public class Content extends ConfigModel {
                     targetCluster = content.containers.iterator().next().getCluster();
 
                 addDocproc(targetCluster);
-                var indexingDocproc = indexedSearchCluster.getIndexingDocproc();
                 indexingDocproc.setClusterName(targetCluster.getName());
                 addIndexingChainsTo(targetCluster, content, indexingDocproc);
             }
@@ -303,11 +297,12 @@ public class Content extends ConfigModel {
         }
 
         /** Create a new container cluster for indexing and add it to the Vespa model */
-        private void createImplicitIndexingCluster(IndexedSearchCluster cluster,
+        private void createImplicitIndexingCluster(List<SearchNode> cluster,
+                                                   IndexingDocproc indexingDocproc,
                                                    Content content,
                                                    ConfigModelContext modelContext,
                                                    ApplicationConfigProducerRoot root) {
-            String indexerName = cluster.getIndexingDocproc().getClusterName(content.getCluster().getName());
+            String indexerName = indexingDocproc.getClusterName(content.getCluster().getName());
             TreeConfigProducer<AnyConfigProducer> parent = getDocProc(root);
             ApplicationContainerCluster indexingCluster = new ApplicationContainerCluster(parent, "cluster." + indexerName, indexerName, modelContext.getDeployState());
             ContainerModel indexingClusterModel = new ContainerModel(modelContext.withParent(parent).withId(indexingCluster.getSubId()));
@@ -323,7 +318,7 @@ public class Content extends ConfigModel {
             List<ApplicationContainer> nodes = new ArrayList<>();
             int index = 0;
             Set<HostResource> processedHosts = new LinkedHashSet<>();
-            for (SearchNode searchNode : cluster.getSearchNodes()) {
+            for (SearchNode searchNode : cluster) {
                 HostResource host = searchNode.getHostResource();
                 if (!processedHosts.contains(host)) {
                     String containerName = String.valueOf(searchNode.getDistributionKey());
@@ -340,7 +335,7 @@ public class Content extends ConfigModel {
             indexingCluster.addContainers(nodes);
 
             addIndexingChain(indexingCluster);
-            cluster.getIndexingDocproc().setChain(indexingCluster.getDocprocChains().allChains().getComponent(IndexingDocprocChain.NAME));
+            indexingDocproc.setChain(indexingCluster.getDocprocChains().allChains().getComponent(IndexingDocprocChain.NAME));
         }
 
         private ContainerCluster<?> getContainerWithDocproc(Collection<ContainerModel> containers) {
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java
index 1420cd82247..0027a9ca45c 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java
@@ -58,7 +58,7 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer>
 
     /** The single, indexed search cluster this sets up (supporting multiple document types), or null if none */
     private IndexedSearchCluster indexedCluster;
-    private Optional<IndexingDocproc> indexingDocproc;
+    private final IndexingDocproc indexingDocproc;
     private Redundancy redundancy;
 
     private final String clusterName;
@@ -208,7 +208,7 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer>
                                  double fractionOfMemoryReserved)
     {
         super(parent, "search");
-        this.indexingDocproc = Optional.empty();
+        this.indexingDocproc = new IndexingDocproc();
         this.clusterName = clusterName;
         this.documentDefinitions = documentDefinitions;
         this.globallyDistributedDocuments = globallyDistributedDocuments;
@@ -262,10 +262,6 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer>
                 throw new IllegalArgumentException("Duplicate indexed cluster '" + indexedCluster.getClusterName() + "'");
             }
             indexedCluster = (IndexedSearchCluster)sc;
-            if (indexingDocproc.isPresent()) {
-                throw new IllegalArgumentException("Indexing docproc has previously been setup for streaming search");
-            }
-            indexingDocproc = Optional.of(indexedCluster.getIndexingDocproc());
         }
         clusters.put(sc.getClusterName(), sc);
     }
@@ -483,12 +479,7 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer>
     public Map<String, SearchCluster> getClusters() { return clusters; }
     public IndexedSearchCluster getIndexed() { return indexedCluster; }
     public boolean hasIndexedCluster()       { return indexedCluster != null; }
-    public Optional<IndexingDocproc> getIndexingDocproc() { return indexingDocproc; }
-    public void setupStreamingSearchIndexingDocProc() {
-        if (indexingDocproc.isEmpty()) {
-            indexingDocproc = Optional.of(new IndexingDocproc());
-        }
-    }
+    public IndexingDocproc getIndexingDocproc() { return indexingDocproc; }
     public String getClusterName() { return clusterName; }
 
     @Override
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java
index 7c48ec11729..791faa401ed 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java
@@ -209,20 +209,10 @@ public class ContentCluster extends TreeConfigProducer<AnyConfigProducer> implem
                 docprocChain = docprocChain.trim();
             }
             if (docprocCluster != null && !docprocCluster.isEmpty()) {
-                if (!c.getSearch().hasIndexedCluster() && c.getSearch().getIndexingDocproc().isEmpty() &&
-                        docprocChain != null && !docprocChain.isEmpty()) {
-                    c.getSearch().setupStreamingSearchIndexingDocProc();
-                }
-                var indexingDocproc = c.getSearch().getIndexingDocproc();
-                if (indexingDocproc.isPresent()) {
-                    indexingDocproc.get().setClusterName(docprocCluster);
-                }
+                c.getSearch().getIndexingDocproc().setClusterName(docprocCluster);
             }
             if (docprocChain != null && !docprocChain.isEmpty()) {
-                var indexingDocproc = c.getSearch().getIndexingDocproc();
-                if (indexingDocproc.isPresent()) {
-                    indexingDocproc.get().setChainName(docprocChain);
-                }
+                c.getSearch().getIndexingDocproc().setChainName(docprocChain);
             }
         }
 
@@ -301,10 +291,7 @@ public class ContentCluster extends TreeConfigProducer<AnyConfigProducer> implem
                     Objects.requireNonNull(admin.getLogserver(), "logserver cannot be null");
                     List<HostResource> host = List.of(admin.getLogserver().getHostResource());
                     admin.setClusterControllers(createClusterControllers(new ClusterControllerCluster(admin, "standalone", deployState),
-                                                                         host,
-                                                                         clusterName,
-                                                                         true,
-                                                                         deployState),
+                                                                         host, clusterName, true, deployState),
                                                 deployState);
                 }
                 clusterControllers = admin.getClusterControllers();
@@ -457,7 +444,6 @@ public class ContentCluster extends TreeConfigProducer<AnyConfigProducer> implem
 
     @Override
     public void getConfig(MessagetyperouteselectorpolicyConfig.Builder builder) {
-        if (getSearch().getIndexingDocproc().isEmpty()) return;
         DocumentProtocol.getConfig(builder, getConfigId());
     }
 
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/routing/DocumentProtocol.java b/config-model/src/main/java/com/yahoo/vespa/model/routing/DocumentProtocol.java
index d555d13c09c..1c8567b4079 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/routing/DocumentProtocol.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/routing/DocumentProtocol.java
@@ -110,11 +110,7 @@ public final class DocumentProtocol implements Protocol,
         for (ContentCluster cluster : Content.getContentClusters(repo)) {
             DocumentProtocolPoliciesConfig.Cluster.Builder clusterBuilder = new DocumentProtocolPoliciesConfig.Cluster.Builder();
             addSelector(cluster.getConfigId(), cluster.getRoutingSelector(), clusterBuilder);
-            if (cluster.getSearch().getIndexingDocproc().isPresent())
-                addRoutes(getDirectRouteName(cluster.getConfigId()), getIndexedRouteName(cluster.getConfigId()), clusterBuilder);
-            else
-                clusterBuilder.defaultRoute(cluster.getConfigId());
-
+            addRoutes(getDirectRouteName(cluster.getConfigId()), getIndexedRouteName(cluster.getConfigId()), clusterBuilder);
             builder.cluster(cluster.getConfigId(), clusterBuilder);
         }
     }
@@ -226,18 +222,13 @@ public final class DocumentProtocol implements Protocol,
     private static void addContentRouting(List<ContentCluster> content, RoutingTableSpec table) {
         for (ContentCluster cluster : content) {
             RouteSpec spec = new RouteSpec(cluster.getConfigId());
-
-            if (cluster.getSearch().getIndexingDocproc().isPresent()) {
-                var indexingDocproc = cluster.getSearch().getIndexingDocproc().get();
-                table.addRoute(spec.addHop("[MessageType:" + cluster.getConfigId() + "]"));
-                table.addRoute(new RouteSpec(getIndexedRouteName(cluster.getConfigId()))
-                                       .addHop(indexingDocproc.getServiceName())
-                                       .addHop("[Content:cluster=" + cluster.getName() + "]"));
-                table.addRoute(new RouteSpec(getDirectRouteName(cluster.getConfigId()))
-                                       .addHop("[Content:cluster=" + cluster.getName() + "]"));
-            } else {
-                table.addRoute(spec.addHop("[Content:cluster=" + cluster.getName() + "]"));
-            }
+            var indexingDocproc = cluster.getSearch().getIndexingDocproc();
+            table.addRoute(spec.addHop("[MessageType:" + cluster.getConfigId() + "]"));
+            table.addRoute(new RouteSpec(getIndexedRouteName(cluster.getConfigId()))
+                                   .addHop(indexingDocproc.getServiceName())
+                                   .addHop("[Content:cluster=" + cluster.getName() + "]"));
+            table.addRoute(new RouteSpec(getDirectRouteName(cluster.getConfigId()))
+                                   .addHop("[Content:cluster=" + cluster.getName() + "]"));
             table.addRoute(new RouteSpec("storage/cluster." + cluster.getName())
                                    .addHop("route:" + cluster.getConfigId()));
         }
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java
index a79bc14db52..77c6f8f99c4 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java
@@ -41,7 +41,6 @@ public class IndexedSearchCluster extends SearchCluster
         DispatchNodesConfig.Producer,
         ConfigInstance.Producer {
 
-    private final IndexingDocproc indexingDocproc;
     private Tuning tuning;
     private SearchCoverage searchCoverage;
 
@@ -68,7 +67,6 @@ public class IndexedSearchCluster extends SearchCluster
 
     public IndexedSearchCluster(TreeConfigProducer<AnyConfigProducer> parent, String clusterName, int index, ModelContext.FeatureFlags featureFlags) {
         super(parent, clusterName, index);
-        indexingDocproc = new IndexingDocproc();
         documentDbsConfigProducer = new MultipleDocumentDatabasesConfigProducer(this, documentDbs);
         defaultDispatchPolicy = DispatchTuning.Builder.toDispatchPolicy(featureFlags.queryDispatchPolicy());
         dispatchWarmup = featureFlags.queryDispatchWarmup();
@@ -78,9 +76,6 @@ public class IndexedSearchCluster extends SearchCluster
     @Override
     protected IndexingMode getIndexingMode() { return IndexingMode.REALTIME; }
 
-    public IndexingDocproc getIndexingDocproc() { return indexingDocproc; }
-
-
     public void addSearcher(SearchNode searcher) {
         searchNodes.add(searcher);
     }
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexingDocproc.java b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexingDocproc.java
index 4f301f6df9a..2032720db96 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexingDocproc.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexingDocproc.java
@@ -3,6 +3,8 @@ package com.yahoo.vespa.model.search;
 
 import com.yahoo.vespa.model.container.docproc.DocprocChain;
 
+import java.util.Objects;
+
 /**
  * Utility class to track configuration for which indexing docproc to use by a search cluster.
  */
@@ -51,7 +53,7 @@ public class IndexingDocproc {
     }
 
     public String getServiceName() {
-        return chain.getServiceName();
+        return Objects.requireNonNull(chain).getServiceName();
     }
 
     /**
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java
index af825ca544a..97c349aeef2 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java
@@ -136,7 +136,7 @@ public class MetricsProxyContainerTest {
     @Test
     void vespa_services_config_has_all_services() {
         VespaServicesConfig vespaServicesConfig = getVespaServicesConfig(hostedServicesWithContent());
-        assertEquals(9, vespaServicesConfig.service().size());
+        assertEquals(10, vespaServicesConfig.service().size());
 
         for (var service : vespaServicesConfig.service()) {
             if (service.configId().equals("admin/cluster-controllers/0")) {
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java
index 43ea0191ca5..f8adb18a2c3 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java
@@ -213,7 +213,7 @@ public class ContentBuilderTest extends DomBuilderTest {
         assertEquals(1, cluster.getRoot().hostSystem().getHosts().size());
         HostResource h = cluster.getRoot().hostSystem().getHost("mockhost");
         String [] expectedServices = {
-                "logd", "configproxy", "config-sentinel", "configserver", "logserver",
+                "logd", "configproxy", "config-sentinel", "configserver", "container", "logserver",
                 "slobrok", "storagenode", "distributor", "searchnode", "transactionlogserver",
                 CLUSTERCONTROLLER_CONTAINER.serviceName, METRICS_PROXY_CONTAINER.serviceName
         };
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java
index d4087c0acf9..354f2d9cf8d 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java
@@ -1440,7 +1440,7 @@ public class ContentClusterTest extends ContentBaseTest {
         assertGroupsAllowedDown(2, 1, 2);
     }
 
-    private void assertIndexingDocprocEnabled(boolean indexed, boolean force, boolean expEnabled) {
+    private void assertIndexingDocprocEnabled(boolean indexed, boolean force) {
         String services = "<?xml version='1.0' encoding='UTF-8' ?>" +
                 "<services version='1.0'>" +
                 "  <container id='default' version='1.0'>" +
@@ -1456,25 +1456,32 @@ public class ContentClusterTest extends ContentBaseTest {
                 "</services>";
         VespaModel model = createEnd2EndOneNode(new TestProperties(), services);
         var searchCluster = model.getContentClusters().get("search").getSearch();
-        assertEquals(expEnabled, searchCluster.getIndexingDocproc().isPresent());
+        if (force) {
+            assertEquals("default", searchCluster.getIndexingDocproc().getClusterName("search"));
+            assertEquals("default", searchCluster.getIndexingDocproc().getClusterName("search"));
+        } else if (indexed) {
+            assertEquals("default", searchCluster.getIndexingDocproc().getClusterName("search"));
+        } else {
+            assertEquals("default", searchCluster.getIndexingDocproc().getClusterName("search"));
+        }
     }
 
     @Test
     void testIndexingDocprocEnabledWhenIndexMode()
     {
-        assertIndexingDocprocEnabled(true, false, true);
+        assertIndexingDocprocEnabled(true, false);
     }
 
     @Test
     void testIndexingDocprocNotEnabledWhenStreamingMode()
     {
-        assertIndexingDocprocEnabled(false, false, false);
+        assertIndexingDocprocEnabled(false, false);
     }
 
     @Test
     void testIndexingDocprocEnabledWhenStreamingModeAndForced()
     {
-        assertIndexingDocprocEnabled(false, true, true);
+        assertIndexingDocprocEnabled(false, true);
     }
 
     private void assertGroupsAllowedDown(int groupCount, double groupsAllowedDown, int expectedGroupsAllowedDown) {
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageContentTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageContentTest.java
index e37999ded12..3defaad549c 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageContentTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageContentTest.java
@@ -65,9 +65,13 @@ public class StorageContentTest extends ContentBaseTest {
         DocumentProtocol protocol = (DocumentProtocol) routing.getProtocols().get(0);
 
         RoutingTableSpec spec = protocol.getRoutingTableSpec();
-        assertEquals(1, spec.getNumHops());
-        assertEquals("indexing", spec.getHop(0).getName());
-        assertEquals("[DocumentRouteSelector]", spec.getHop(0).getSelector());
+        assertEquals(3, spec.getNumHops());
+        assertEquals("docproc/cluster.bar.indexing/chain.indexing", spec.getHop(0).getName());
+        assertEquals("[LoadBalancer:cluster=docproc/cluster.bar.indexing;session=chain.indexing]", spec.getHop(0).getSelector());
+        assertEquals("docproc/cluster.zoo.indexing/chain.indexing", spec.getHop(1).getName());
+        assertEquals("[LoadBalancer:cluster=docproc/cluster.zoo.indexing;session=chain.indexing]", spec.getHop(1).getSelector());
+        assertEquals("indexing", spec.getHop(2).getName());
+        assertEquals("[DocumentRouteSelector]", spec.getHop(2).getSelector());
 
         Map<String, RouteSpec> routes = new TreeMap<>();