diff options
author | Henning Baldersheim <balder@yahoo-inc.com> | 2024-01-15 00:41:25 +0100 |
---|---|---|
committer | Henning Baldersheim <balder@yahoo-inc.com> | 2024-01-15 00:41:25 +0100 |
commit | a2f7d4a67d84d1fe2ebc633ca727409d56d81e0c (patch) | |
tree | 0b06ed6f8c7688ba8cb29c8681804ac839824429 /config-model/src | |
parent | ae9a732e3e16d46457da372a4da594064442be04 (diff) |
Always add an indexing docproc, also for streaming search.
Diffstat (limited to 'config-model/src')
10 files changed, 48 insertions, 77 deletions
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java b/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java index 4e56d1d1d5a..1254f8e110a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java @@ -27,7 +27,6 @@ import com.yahoo.vespa.model.container.docproc.ContainerDocproc; import com.yahoo.vespa.model.container.docproc.DocprocChain; import com.yahoo.vespa.model.container.docproc.DocprocChains; import com.yahoo.vespa.model.content.cluster.ContentCluster; -import com.yahoo.vespa.model.search.IndexedSearchCluster; import com.yahoo.vespa.model.search.IndexingDocproc; import com.yahoo.vespa.model.search.IndexingDocprocChain; import com.yahoo.vespa.model.search.SearchCluster; @@ -215,25 +214,21 @@ public class Content extends ConfigModel { private void buildIndexingClusters(Content content, ConfigModelContext modelContext, ApplicationConfigProducerRoot root) { var search = content.getCluster().getSearch(); - if (!search.getIndexingDocproc().isPresent()) { - return; - } - var indexingDocproc = search.getIndexingDocproc().get(); + var indexingDocproc = search.getIndexingDocproc(); if (indexingDocproc.hasExplicitCluster()) { setExistingIndexingCluster(content, indexingDocproc, content.containers); } else { - if (search.hasIndexedCluster()) { - setContainerAsIndexingCluster(search.getIndexed(), content, modelContext, root); - } + setContainerAsIndexingCluster(search.getSearchNodes(), indexingDocproc, content, modelContext, root); } } - private void setContainerAsIndexingCluster(IndexedSearchCluster indexedSearchCluster, + private void setContainerAsIndexingCluster(List<SearchNode> cluster, + IndexingDocproc indexingDocproc, Content content, ConfigModelContext modelContext, ApplicationConfigProducerRoot root) { if (content.containers.isEmpty()) { - createImplicitIndexingCluster(indexedSearchCluster, content, modelContext, root); + createImplicitIndexingCluster(cluster, indexingDocproc, content, modelContext, root); } else { ContainerCluster<?> targetCluster = getContainerWithDocproc(content.containers); if (targetCluster == null) @@ -242,7 +237,6 @@ public class Content extends ConfigModel { targetCluster = content.containers.iterator().next().getCluster(); addDocproc(targetCluster); - var indexingDocproc = indexedSearchCluster.getIndexingDocproc(); indexingDocproc.setClusterName(targetCluster.getName()); addIndexingChainsTo(targetCluster, content, indexingDocproc); } @@ -303,11 +297,12 @@ public class Content extends ConfigModel { } /** Create a new container cluster for indexing and add it to the Vespa model */ - private void createImplicitIndexingCluster(IndexedSearchCluster cluster, + private void createImplicitIndexingCluster(List<SearchNode> cluster, + IndexingDocproc indexingDocproc, Content content, ConfigModelContext modelContext, ApplicationConfigProducerRoot root) { - String indexerName = cluster.getIndexingDocproc().getClusterName(content.getCluster().getName()); + String indexerName = indexingDocproc.getClusterName(content.getCluster().getName()); TreeConfigProducer<AnyConfigProducer> parent = getDocProc(root); ApplicationContainerCluster indexingCluster = new ApplicationContainerCluster(parent, "cluster." + indexerName, indexerName, modelContext.getDeployState()); ContainerModel indexingClusterModel = new ContainerModel(modelContext.withParent(parent).withId(indexingCluster.getSubId())); @@ -323,7 +318,7 @@ public class Content extends ConfigModel { List<ApplicationContainer> nodes = new ArrayList<>(); int index = 0; Set<HostResource> processedHosts = new LinkedHashSet<>(); - for (SearchNode searchNode : cluster.getSearchNodes()) { + for (SearchNode searchNode : cluster) { HostResource host = searchNode.getHostResource(); if (!processedHosts.contains(host)) { String containerName = String.valueOf(searchNode.getDistributionKey()); @@ -340,7 +335,7 @@ public class Content extends ConfigModel { indexingCluster.addContainers(nodes); addIndexingChain(indexingCluster); - cluster.getIndexingDocproc().setChain(indexingCluster.getDocprocChains().allChains().getComponent(IndexingDocprocChain.NAME)); + indexingDocproc.setChain(indexingCluster.getDocprocChains().allChains().getComponent(IndexingDocprocChain.NAME)); } private ContainerCluster<?> getContainerWithDocproc(Collection<ContainerModel> containers) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java index 1420cd82247..0027a9ca45c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java @@ -58,7 +58,7 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> /** The single, indexed search cluster this sets up (supporting multiple document types), or null if none */ private IndexedSearchCluster indexedCluster; - private Optional<IndexingDocproc> indexingDocproc; + private final IndexingDocproc indexingDocproc; private Redundancy redundancy; private final String clusterName; @@ -208,7 +208,7 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> double fractionOfMemoryReserved) { super(parent, "search"); - this.indexingDocproc = Optional.empty(); + this.indexingDocproc = new IndexingDocproc(); this.clusterName = clusterName; this.documentDefinitions = documentDefinitions; this.globallyDistributedDocuments = globallyDistributedDocuments; @@ -262,10 +262,6 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> throw new IllegalArgumentException("Duplicate indexed cluster '" + indexedCluster.getClusterName() + "'"); } indexedCluster = (IndexedSearchCluster)sc; - if (indexingDocproc.isPresent()) { - throw new IllegalArgumentException("Indexing docproc has previously been setup for streaming search"); - } - indexingDocproc = Optional.of(indexedCluster.getIndexingDocproc()); } clusters.put(sc.getClusterName(), sc); } @@ -483,12 +479,7 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> public Map<String, SearchCluster> getClusters() { return clusters; } public IndexedSearchCluster getIndexed() { return indexedCluster; } public boolean hasIndexedCluster() { return indexedCluster != null; } - public Optional<IndexingDocproc> getIndexingDocproc() { return indexingDocproc; } - public void setupStreamingSearchIndexingDocProc() { - if (indexingDocproc.isEmpty()) { - indexingDocproc = Optional.of(new IndexingDocproc()); - } - } + public IndexingDocproc getIndexingDocproc() { return indexingDocproc; } public String getClusterName() { return clusterName; } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java index 7c48ec11729..791faa401ed 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java @@ -209,20 +209,10 @@ public class ContentCluster extends TreeConfigProducer<AnyConfigProducer> implem docprocChain = docprocChain.trim(); } if (docprocCluster != null && !docprocCluster.isEmpty()) { - if (!c.getSearch().hasIndexedCluster() && c.getSearch().getIndexingDocproc().isEmpty() && - docprocChain != null && !docprocChain.isEmpty()) { - c.getSearch().setupStreamingSearchIndexingDocProc(); - } - var indexingDocproc = c.getSearch().getIndexingDocproc(); - if (indexingDocproc.isPresent()) { - indexingDocproc.get().setClusterName(docprocCluster); - } + c.getSearch().getIndexingDocproc().setClusterName(docprocCluster); } if (docprocChain != null && !docprocChain.isEmpty()) { - var indexingDocproc = c.getSearch().getIndexingDocproc(); - if (indexingDocproc.isPresent()) { - indexingDocproc.get().setChainName(docprocChain); - } + c.getSearch().getIndexingDocproc().setChainName(docprocChain); } } @@ -301,10 +291,7 @@ public class ContentCluster extends TreeConfigProducer<AnyConfigProducer> implem Objects.requireNonNull(admin.getLogserver(), "logserver cannot be null"); List<HostResource> host = List.of(admin.getLogserver().getHostResource()); admin.setClusterControllers(createClusterControllers(new ClusterControllerCluster(admin, "standalone", deployState), - host, - clusterName, - true, - deployState), + host, clusterName, true, deployState), deployState); } clusterControllers = admin.getClusterControllers(); @@ -457,7 +444,6 @@ public class ContentCluster extends TreeConfigProducer<AnyConfigProducer> implem @Override public void getConfig(MessagetyperouteselectorpolicyConfig.Builder builder) { - if (getSearch().getIndexingDocproc().isEmpty()) return; DocumentProtocol.getConfig(builder, getConfigId()); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/routing/DocumentProtocol.java b/config-model/src/main/java/com/yahoo/vespa/model/routing/DocumentProtocol.java index d555d13c09c..1c8567b4079 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/routing/DocumentProtocol.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/routing/DocumentProtocol.java @@ -110,11 +110,7 @@ public final class DocumentProtocol implements Protocol, for (ContentCluster cluster : Content.getContentClusters(repo)) { DocumentProtocolPoliciesConfig.Cluster.Builder clusterBuilder = new DocumentProtocolPoliciesConfig.Cluster.Builder(); addSelector(cluster.getConfigId(), cluster.getRoutingSelector(), clusterBuilder); - if (cluster.getSearch().getIndexingDocproc().isPresent()) - addRoutes(getDirectRouteName(cluster.getConfigId()), getIndexedRouteName(cluster.getConfigId()), clusterBuilder); - else - clusterBuilder.defaultRoute(cluster.getConfigId()); - + addRoutes(getDirectRouteName(cluster.getConfigId()), getIndexedRouteName(cluster.getConfigId()), clusterBuilder); builder.cluster(cluster.getConfigId(), clusterBuilder); } } @@ -226,18 +222,13 @@ public final class DocumentProtocol implements Protocol, private static void addContentRouting(List<ContentCluster> content, RoutingTableSpec table) { for (ContentCluster cluster : content) { RouteSpec spec = new RouteSpec(cluster.getConfigId()); - - if (cluster.getSearch().getIndexingDocproc().isPresent()) { - var indexingDocproc = cluster.getSearch().getIndexingDocproc().get(); - table.addRoute(spec.addHop("[MessageType:" + cluster.getConfigId() + "]")); - table.addRoute(new RouteSpec(getIndexedRouteName(cluster.getConfigId())) - .addHop(indexingDocproc.getServiceName()) - .addHop("[Content:cluster=" + cluster.getName() + "]")); - table.addRoute(new RouteSpec(getDirectRouteName(cluster.getConfigId())) - .addHop("[Content:cluster=" + cluster.getName() + "]")); - } else { - table.addRoute(spec.addHop("[Content:cluster=" + cluster.getName() + "]")); - } + var indexingDocproc = cluster.getSearch().getIndexingDocproc(); + table.addRoute(spec.addHop("[MessageType:" + cluster.getConfigId() + "]")); + table.addRoute(new RouteSpec(getIndexedRouteName(cluster.getConfigId())) + .addHop(indexingDocproc.getServiceName()) + .addHop("[Content:cluster=" + cluster.getName() + "]")); + table.addRoute(new RouteSpec(getDirectRouteName(cluster.getConfigId())) + .addHop("[Content:cluster=" + cluster.getName() + "]")); table.addRoute(new RouteSpec("storage/cluster." + cluster.getName()) .addHop("route:" + cluster.getConfigId())); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java index a79bc14db52..77c6f8f99c4 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java @@ -41,7 +41,6 @@ public class IndexedSearchCluster extends SearchCluster DispatchNodesConfig.Producer, ConfigInstance.Producer { - private final IndexingDocproc indexingDocproc; private Tuning tuning; private SearchCoverage searchCoverage; @@ -68,7 +67,6 @@ public class IndexedSearchCluster extends SearchCluster public IndexedSearchCluster(TreeConfigProducer<AnyConfigProducer> parent, String clusterName, int index, ModelContext.FeatureFlags featureFlags) { super(parent, clusterName, index); - indexingDocproc = new IndexingDocproc(); documentDbsConfigProducer = new MultipleDocumentDatabasesConfigProducer(this, documentDbs); defaultDispatchPolicy = DispatchTuning.Builder.toDispatchPolicy(featureFlags.queryDispatchPolicy()); dispatchWarmup = featureFlags.queryDispatchWarmup(); @@ -78,9 +76,6 @@ public class IndexedSearchCluster extends SearchCluster @Override protected IndexingMode getIndexingMode() { return IndexingMode.REALTIME; } - public IndexingDocproc getIndexingDocproc() { return indexingDocproc; } - - public void addSearcher(SearchNode searcher) { searchNodes.add(searcher); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexingDocproc.java b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexingDocproc.java index 4f301f6df9a..2032720db96 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexingDocproc.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexingDocproc.java @@ -3,6 +3,8 @@ package com.yahoo.vespa.model.search; import com.yahoo.vespa.model.container.docproc.DocprocChain; +import java.util.Objects; + /** * Utility class to track configuration for which indexing docproc to use by a search cluster. */ @@ -51,7 +53,7 @@ public class IndexingDocproc { } public String getServiceName() { - return chain.getServiceName(); + return Objects.requireNonNull(chain).getServiceName(); } /** diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java index af825ca544a..97c349aeef2 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java @@ -136,7 +136,7 @@ public class MetricsProxyContainerTest { @Test void vespa_services_config_has_all_services() { VespaServicesConfig vespaServicesConfig = getVespaServicesConfig(hostedServicesWithContent()); - assertEquals(9, vespaServicesConfig.service().size()); + assertEquals(10, vespaServicesConfig.service().size()); for (var service : vespaServicesConfig.service()) { if (service.configId().equals("admin/cluster-controllers/0")) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java index 43ea0191ca5..f8adb18a2c3 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java @@ -213,7 +213,7 @@ public class ContentBuilderTest extends DomBuilderTest { assertEquals(1, cluster.getRoot().hostSystem().getHosts().size()); HostResource h = cluster.getRoot().hostSystem().getHost("mockhost"); String [] expectedServices = { - "logd", "configproxy", "config-sentinel", "configserver", "logserver", + "logd", "configproxy", "config-sentinel", "configserver", "container", "logserver", "slobrok", "storagenode", "distributor", "searchnode", "transactionlogserver", CLUSTERCONTROLLER_CONTAINER.serviceName, METRICS_PROXY_CONTAINER.serviceName }; diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java index d4087c0acf9..354f2d9cf8d 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java @@ -1440,7 +1440,7 @@ public class ContentClusterTest extends ContentBaseTest { assertGroupsAllowedDown(2, 1, 2); } - private void assertIndexingDocprocEnabled(boolean indexed, boolean force, boolean expEnabled) { + private void assertIndexingDocprocEnabled(boolean indexed, boolean force) { String services = "<?xml version='1.0' encoding='UTF-8' ?>" + "<services version='1.0'>" + " <container id='default' version='1.0'>" + @@ -1456,25 +1456,32 @@ public class ContentClusterTest extends ContentBaseTest { "</services>"; VespaModel model = createEnd2EndOneNode(new TestProperties(), services); var searchCluster = model.getContentClusters().get("search").getSearch(); - assertEquals(expEnabled, searchCluster.getIndexingDocproc().isPresent()); + if (force) { + assertEquals("default", searchCluster.getIndexingDocproc().getClusterName("search")); + assertEquals("default", searchCluster.getIndexingDocproc().getClusterName("search")); + } else if (indexed) { + assertEquals("default", searchCluster.getIndexingDocproc().getClusterName("search")); + } else { + assertEquals("default", searchCluster.getIndexingDocproc().getClusterName("search")); + } } @Test void testIndexingDocprocEnabledWhenIndexMode() { - assertIndexingDocprocEnabled(true, false, true); + assertIndexingDocprocEnabled(true, false); } @Test void testIndexingDocprocNotEnabledWhenStreamingMode() { - assertIndexingDocprocEnabled(false, false, false); + assertIndexingDocprocEnabled(false, false); } @Test void testIndexingDocprocEnabledWhenStreamingModeAndForced() { - assertIndexingDocprocEnabled(false, true, true); + assertIndexingDocprocEnabled(false, true); } private void assertGroupsAllowedDown(int groupCount, double groupsAllowedDown, int expectedGroupsAllowedDown) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageContentTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageContentTest.java index e37999ded12..3defaad549c 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageContentTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageContentTest.java @@ -65,9 +65,13 @@ public class StorageContentTest extends ContentBaseTest { DocumentProtocol protocol = (DocumentProtocol) routing.getProtocols().get(0); RoutingTableSpec spec = protocol.getRoutingTableSpec(); - assertEquals(1, spec.getNumHops()); - assertEquals("indexing", spec.getHop(0).getName()); - assertEquals("[DocumentRouteSelector]", spec.getHop(0).getSelector()); + assertEquals(3, spec.getNumHops()); + assertEquals("docproc/cluster.bar.indexing/chain.indexing", spec.getHop(0).getName()); + assertEquals("[LoadBalancer:cluster=docproc/cluster.bar.indexing;session=chain.indexing]", spec.getHop(0).getSelector()); + assertEquals("docproc/cluster.zoo.indexing/chain.indexing", spec.getHop(1).getName()); + assertEquals("[LoadBalancer:cluster=docproc/cluster.zoo.indexing;session=chain.indexing]", spec.getHop(1).getSelector()); + assertEquals("indexing", spec.getHop(2).getName()); + assertEquals("[DocumentRouteSelector]", spec.getHop(2).getSelector()); Map<String, RouteSpec> routes = new TreeMap<>(); |