aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorHenning Baldersheim <balder@yahoo-inc.com>2022-11-21 17:56:57 +0100
committerHenning Baldersheim <balder@yahoo-inc.com>2022-11-21 17:56:57 +0100
commit3e955a564d65de056c87b706432a4a12ff8d1d10 (patch)
tree5f4346c521fb07e3c2dc17b9747ffc30400274c9
parentaaaa5efdde6562a44e3fa4e6168783029d0d5a98 (diff)
Splitt nodes static and dynamic config in DispatchConfig to prepare for seamless cluster changes.
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/search/DispatcherComponent.java12
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java15
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java9
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java39
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaClusterTest.java5
-rw-r--r--configdefinitions/src/vespa/CMakeLists.txt2
-rw-r--r--configdefinitions/src/vespa/dispatch.def12
-rw-r--r--container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java4
-rw-r--r--container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcResourcePool.java5
-rw-r--r--container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java12
-rw-r--r--container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java5
-rw-r--r--container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java7
-rw-r--r--container-search/src/test/java/com/yahoo/prelude/fastsearch/test/MockDispatcher.java19
-rw-r--r--container-search/src/test/java/com/yahoo/search/dispatch/InterleavedSearchInvokerTest.java15
-rw-r--r--container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java9
-rw-r--r--container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java29
-rw-r--r--container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTest.java2
17 files changed, 119 insertions, 82 deletions
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/DispatcherComponent.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/DispatcherComponent.java
index 44c60cf0619..634b1ae3275 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/DispatcherComponent.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/DispatcherComponent.java
@@ -4,6 +4,7 @@ package com.yahoo.vespa.model.container.search;
import com.yahoo.config.model.producer.AbstractConfigProducer;
import com.yahoo.osgi.provider.model.ComponentModel;
import com.yahoo.vespa.config.search.DispatchConfig;
+import com.yahoo.vespa.config.search.DispatchNodesConfig;
import com.yahoo.vespa.model.container.component.Component;
import com.yahoo.vespa.model.container.PlatformBundles;
import com.yahoo.vespa.model.search.IndexedSearchCluster;
@@ -14,8 +15,10 @@ import com.yahoo.vespa.model.search.IndexedSearchCluster;
*
* @author bratseth
*/
-public class DispatcherComponent extends Component<AbstractConfigProducer<?>, ComponentModel>
- implements DispatchConfig.Producer {
+public class DispatcherComponent extends Component<AbstractConfigProducer<?>, ComponentModel> implements
+ DispatchConfig.Producer,
+ DispatchNodesConfig.Producer
+{
private final IndexedSearchCluster indexedSearchCluster;
@@ -40,4 +43,9 @@ public class DispatcherComponent extends Component<AbstractConfigProducer<?>, Co
indexedSearchCluster.getConfig(builder);
}
+ @Override
+ public void getConfig(DispatchNodesConfig.Builder builder) {
+ indexedSearchCluster.getConfig(builder);
+ }
+
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java
index 877ea1d59e7..9379e69e278 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java
@@ -8,6 +8,7 @@ import com.yahoo.documentmodel.NewDocumentType;
import com.yahoo.schema.Schema;
import com.yahoo.schema.derived.SchemaInfo;
import com.yahoo.vespa.config.search.DispatchConfig;
+import com.yahoo.vespa.config.search.DispatchNodesConfig;
import com.yahoo.vespa.config.search.core.ProtonConfig;
import com.yahoo.vespa.model.builder.xml.dom.DomSearchTuningBuilder;
import com.yahoo.vespa.model.builder.xml.dom.ModelElement;
@@ -40,7 +41,11 @@ import static java.util.stream.Collectors.toList;
* Encapsulates the various options for search in a content model.
* Wraps a search cluster from com.yahoo.vespa.model.search.
*/
-public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> implements ProtonConfig.Producer, DispatchConfig.Producer {
+public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> implements
+ ProtonConfig.Producer,
+ DispatchNodesConfig.Producer,
+ DispatchConfig.Producer
+{
private static final int DEFAULT_DOC_STORE_COMPRESSION_LEVEL = 3;
private static final double DEFAULT_DISK_BLOAT = 0.25;
@@ -429,12 +434,18 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster>
}
@Override
- public void getConfig(DispatchConfig.Builder builder) {
+ public void getConfig(DispatchNodesConfig.Builder builder) {
if (hasIndexedCluster()) {
getIndexed().getConfig(builder);
}
}
+ @Override
+ public void getConfig(DispatchConfig.Builder builder) {
+ if (hasIndexedCluster()) {
+ getIndexed().getConfig(builder);
+ }
+ }
public Map<String, SearchCluster> getClusters() { return clusters; }
public IndexedSearchCluster getIndexed() { return indexedCluster; }
public boolean hasIndexedCluster() { return indexedCluster != null; }
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java
index c2218730d08..8e71de136e9 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java
@@ -14,6 +14,7 @@ import com.yahoo.schema.derived.SchemaInfo;
import com.yahoo.vespa.config.search.AttributesConfig;
import com.yahoo.vespa.config.search.DispatchConfig;
import com.yahoo.vespa.config.search.DispatchConfig.DistributionPolicy;
+import com.yahoo.vespa.config.search.DispatchNodesConfig;
import com.yahoo.vespa.config.search.RankProfilesConfig;
import com.yahoo.vespa.config.search.core.ProtonConfig;
import com.yahoo.vespa.configdefinition.IlscriptsConfig;
@@ -38,6 +39,7 @@ public class IndexedSearchCluster extends SearchCluster
SchemaInfoConfig.Producer,
IlscriptsConfig.Producer,
DispatchConfig.Producer,
+ DispatchNodesConfig.Producer,
ConfigInstance.Producer {
private String indexingClusterName = null; // The name of the docproc cluster to run indexing, by config.
@@ -297,15 +299,18 @@ public class IndexedSearchCluster extends SearchCluster
};
}
@Override
- public void getConfig(DispatchConfig.Builder builder) {
+ public void getConfig(DispatchNodesConfig.Builder builder) {
for (SearchNode node : getSearchNodes()) {
- DispatchConfig.Node.Builder nodeBuilder = new DispatchConfig.Node.Builder();
+ DispatchNodesConfig.Node.Builder nodeBuilder = new DispatchNodesConfig.Node.Builder();
nodeBuilder.key(node.getDistributionKey());
nodeBuilder.group(node.getNodeSpec().groupIndex());
nodeBuilder.host(node.getHostName());
nodeBuilder.port(node.getRpcPort());
builder.node(nodeBuilder);
}
+ }
+ @Override
+ public void getConfig(DispatchConfig.Builder builder) {
if (tuning.dispatch.getTopkProbability() != null) {
builder.topKProbability(tuning.dispatch.getTopkProbability());
}
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java
index 6b6a61823ab..a6ea6cb8132 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java
@@ -5,6 +5,7 @@ import com.yahoo.config.application.api.ApplicationPackage;
import com.yahoo.config.model.test.MockApplicationPackage;
import com.yahoo.config.model.test.TestDriver;
import com.yahoo.vespa.config.search.DispatchConfig;
+import com.yahoo.vespa.config.search.DispatchNodesConfig;
import com.yahoo.vespa.config.search.core.ProtonConfig;
import com.yahoo.vespa.model.content.Content;
import com.yahoo.vespa.model.search.IndexedSearchCluster;
@@ -90,8 +91,11 @@ public class ClusterTest {
ContentCluster cluster = newContentCluster(joinLines("<search>", "</search>"),
joinLines("<tuning>", "</tuning>"));
DispatchConfig.Builder builder = new DispatchConfig.Builder();
+ DispatchNodesConfig.Builder nodesBuilder = new DispatchNodesConfig.Builder();
cluster.getSearch().getConfig(builder);
- DispatchConfig config = new DispatchConfig(builder);
+ cluster.getSearch().getConfig(nodesBuilder);
+ DispatchConfig config = builder.build();
+ DispatchNodesConfig nodesConfig = nodesBuilder.build();
assertEquals(3, config.redundancy());
assertEquals(DispatchConfig.DistributionPolicy.ADAPTIVE, config.distributionPolicy());
assertEquals(1.0, config.maxWaitAfterCoverageFactor(), DELTA);
@@ -101,22 +105,23 @@ public class ClusterTest {
assertEquals(100.0, config.minSearchCoverage(), DELTA);
assertEquals(97.0, config.minActivedocsPercentage(), DELTA);
assertEquals(0.9999, config.topKProbability(), DELTA);
- assertEquals(3, config.node().size());
- assertEquals(0, config.node(0).key());
- assertEquals(1, config.node(1).key());
- assertEquals(2, config.node(2).key());
-
- assertEquals(19106, config.node(0).port());
- assertEquals(19118, config.node(1).port());
- assertEquals(19130, config.node(2).port());
-
- assertEquals(0, config.node(0).group());
- assertEquals(0, config.node(1).group());
- assertEquals(0, config.node(2).group());
-
- assertEquals("localhost", config.node(0).host());
- assertEquals("localhost", config.node(1).host());
- assertEquals("localhost", config.node(2).host());
+
+ assertEquals(3, nodesConfig.node().size());
+ assertEquals(0, nodesConfig.node(0).key());
+ assertEquals(1, nodesConfig.node(1).key());
+ assertEquals(2, nodesConfig.node(2).key());
+
+ assertEquals(19106, nodesConfig.node(0).port());
+ assertEquals(19118, nodesConfig.node(1).port());
+ assertEquals(19130, nodesConfig.node(2).port());
+
+ assertEquals(0, nodesConfig.node(0).group());
+ assertEquals(0, nodesConfig.node(1).group());
+ assertEquals(0, nodesConfig.node(2).group());
+
+ assertEquals("localhost", nodesConfig.node(0).host());
+ assertEquals("localhost", nodesConfig.node(1).host());
+ assertEquals("localhost", nodesConfig.node(2).host());
}
private static ContentCluster newContentCluster(String contentSearchXml, String searchNodeTuningXml) {
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaClusterTest.java
index 3e5a473c1ca..9166de2f648 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaClusterTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaClusterTest.java
@@ -12,6 +12,7 @@ import com.yahoo.schema.document.Attribute;
import com.yahoo.schema.document.SDDocumentType;
import com.yahoo.schema.document.SDField;
import com.yahoo.vespa.config.search.DispatchConfig;
+import com.yahoo.vespa.config.search.DispatchNodesConfig;
import com.yahoo.vespa.indexinglanguage.expressions.AttributeExpression;
import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression;
import com.yahoo.vespa.indexinglanguage.expressions.StatementExpression;
@@ -171,7 +172,7 @@ public class SchemaClusterTest {
assertEquals("dispatcher." + cluster, dispatcher.getComponentId().stringValue());
assertEquals("com.yahoo.search.dispatch.Dispatcher", dispatcher.getClassId().stringValue());
assertEquals("j1/component/dispatcher." + cluster, dispatcher.getConfigId());
- DispatchConfig.Builder dispatchConfigBuilder = new DispatchConfig.Builder();
+ DispatchNodesConfig.Builder dispatchConfigBuilder = new DispatchNodesConfig.Builder();
model.getConfig(dispatchConfigBuilder, dispatcher.getConfigId());
assertEquals(host, dispatchConfigBuilder.build().node(0).host());
@@ -182,7 +183,7 @@ public class SchemaClusterTest {
assertEquals("rpcresourcepool." + cluster, rpcResourcePool.getComponentId().stringValue());
assertEquals("com.yahoo.search.dispatch.rpc.RpcResourcePool", rpcResourcePool.getClassId().stringValue());
assertEquals("j1/component/dispatcher." + cluster + "/rpcresourcepool." + cluster, rpcResourcePool.getConfigId());
- dispatchConfigBuilder = new DispatchConfig.Builder();
+ dispatchConfigBuilder = new DispatchNodesConfig.Builder();
model.getConfig(dispatchConfigBuilder, rpcResourcePool.getConfigId());
assertEquals(host, dispatchConfigBuilder.build().node(0).host());
}
diff --git a/configdefinitions/src/vespa/CMakeLists.txt b/configdefinitions/src/vespa/CMakeLists.txt
index f7d70450ba3..2894c8c38b1 100644
--- a/configdefinitions/src/vespa/CMakeLists.txt
+++ b/configdefinitions/src/vespa/CMakeLists.txt
@@ -20,6 +20,8 @@ vespa_generate_config(configdefinitions curator.def)
install_config_definition(curator.def cloud.config.curator.def)
vespa_generate_config(configdefinitions dispatch.def)
install_config_definition(dispatch.def vespa.config.search.dispatch.def)
+vespa_generate_config(configdefinitions dispatch-nodes.def)
+install_config_definition(dispatch-nodes.def vespa.config.search.dispatch-nodes.def)
vespa_generate_config(configdefinitions fleetcontroller.def)
install_config_definition(fleetcontroller.def vespa.config.content.fleetcontroller.def)
vespa_generate_config(configdefinitions ilscripts.def)
diff --git a/configdefinitions/src/vespa/dispatch.def b/configdefinitions/src/vespa/dispatch.def
index e219ea25612..936b1400053 100644
--- a/configdefinitions/src/vespa/dispatch.def
+++ b/configdefinitions/src/vespa/dispatch.def
@@ -47,15 +47,3 @@ numJrtConnectionsPerNode int default=8
# Number of seconds to spend warming up code to prevent JIT cold start issues.
warmuptime double default=0.1
-
-# The unique key of a search node
-node[].key int
-
-# The index of the group this search node belongs to
-node[].group int default=0
-
-# The host name of this search node
-node[].host string
-
-# The rpc port of this search node
-node[].port int
diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java
index 2a95908ba05..e4147f6ba14 100644
--- a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java
+++ b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java
@@ -23,6 +23,7 @@ import com.yahoo.search.query.profile.types.FieldType;
import com.yahoo.search.query.profile.types.QueryProfileType;
import com.yahoo.search.result.ErrorMessage;
import com.yahoo.vespa.config.search.DispatchConfig;
+import com.yahoo.vespa.config.search.DispatchNodesConfig;
import java.time.Duration;
import java.util.HashSet;
@@ -74,8 +75,9 @@ public class Dispatcher extends AbstractComponent {
public Dispatcher(RpcResourcePool resourcePool,
ComponentId clusterId,
DispatchConfig dispatchConfig,
+ DispatchNodesConfig nodesConfig,
VipStatus vipStatus) {
- this(resourcePool, new SearchCluster(clusterId.stringValue(), dispatchConfig,
+ this(resourcePool, new SearchCluster(clusterId.stringValue(), dispatchConfig, nodesConfig,
vipStatus, new RpcPingFactory(resourcePool)),
dispatchConfig);
diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcResourcePool.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcResourcePool.java
index 09628db1284..eb937e00b17 100644
--- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcResourcePool.java
+++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcResourcePool.java
@@ -12,6 +12,7 @@ import com.yahoo.search.Query;
import com.yahoo.search.dispatch.FillInvoker;
import com.yahoo.search.dispatch.rpc.Client.NodeConnection;
import com.yahoo.vespa.config.search.DispatchConfig;
+import com.yahoo.vespa.config.search.DispatchNodesConfig;
import java.util.ArrayList;
import java.util.Collections;
@@ -45,14 +46,14 @@ public class RpcResourcePool extends AbstractComponent {
}
@Inject
- public RpcResourcePool(DispatchConfig dispatchConfig) {
+ public RpcResourcePool(DispatchConfig dispatchConfig, DispatchNodesConfig nodesConfig) {
super();
client = new RpcClient("dispatch-client", dispatchConfig.numJrtTransportThreads());
// Create rpc node connection pools indexed by the node distribution key
var builder = new ImmutableMap.Builder<Integer, NodeConnectionPool>();
var numConnections = dispatchConfig.numJrtConnectionsPerNode();
- for (var node : dispatchConfig.node()) {
+ for (var node : nodesConfig.node()) {
var connections = new ArrayList<NodeConnection>(numConnections);
for (int i = 0; i < numConnections; i++) {
connections.add(client.createConnection(node.host(), node.port()));
diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java
index 25617e48aa1..54a3e42b9ab 100644
--- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java
+++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java
@@ -10,6 +10,7 @@ import com.yahoo.search.cluster.ClusterMonitor;
import com.yahoo.search.cluster.NodeManager;
import com.yahoo.search.dispatch.TopKEstimator;
import com.yahoo.vespa.config.search.DispatchConfig;
+import com.yahoo.vespa.config.search.DispatchNodesConfig;
import java.util.LinkedHashMap;
import java.util.List;
@@ -50,13 +51,14 @@ public class SearchCluster implements NodeManager<Node> {
private final Optional<Node> localCorpusDispatchTarget;
public SearchCluster(String clusterId, DispatchConfig dispatchConfig,
+ DispatchNodesConfig nodesConfig,
VipStatus vipStatus, PingFactory pingFactory) {
this.clusterId = clusterId;
this.dispatchConfig = dispatchConfig;
this.vipStatus = vipStatus;
this.pingFactory = pingFactory;
- this.nodes = toNodes(dispatchConfig);
+ this.nodes = toNodes(nodesConfig);
// Create groups
ImmutableMap.Builder<Integer, Group> groupsBuilder = new ImmutableMap.Builder<>();
@@ -92,7 +94,7 @@ public class SearchCluster implements NodeManager<Node> {
// The search cluster to be searched has at least as many nodes as the container cluster we're running in.
List<Node> localSearchNodes = nodes.stream()
.filter(node -> node.hostname().equals(selfHostname))
- .collect(Collectors.toList());
+ .toList();
// Only use direct dispatch if we have exactly 1 search node on the same machine:
if (localSearchNodes.size() != 1) return Optional.empty();
@@ -105,10 +107,10 @@ public class SearchCluster implements NodeManager<Node> {
return Optional.of(localSearchNode);
}
- private static List<Node> toNodes(DispatchConfig dispatchConfig) {
- return dispatchConfig.node().stream()
+ private static List<Node> toNodes(DispatchNodesConfig nodesConfig) {
+ return nodesConfig.node().stream()
.map(n -> new Node(n.key(), n.host(), n.group()))
- .collect(Collectors.toUnmodifiableList());
+ .toList();
}
public DispatchConfig dispatchConfig() {
diff --git a/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java
index 095b3ef57ac..978da396f3e 100644
--- a/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java
+++ b/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java
@@ -24,6 +24,7 @@ import com.yahoo.search.schema.Schema;
import com.yahoo.search.schema.SchemaInfo;
import com.yahoo.search.searchchain.Execution;
import com.yahoo.vespa.config.search.DispatchConfig;
+import com.yahoo.vespa.config.search.DispatchNodesConfig;
import org.junit.jupiter.api.Test;
@@ -449,9 +450,11 @@ public class ClusterSearcherTestCase {
var schema = new Schema.Builder("type1");
DispatchConfig dispatchConfig = new DispatchConfig.Builder().build();
- Dispatcher dispatcher = new Dispatcher(new RpcResourcePool(dispatchConfig),
+ DispatchNodesConfig nodesConfig = new DispatchNodesConfig.Builder().build();
+ Dispatcher dispatcher = new Dispatcher(new RpcResourcePool(dispatchConfig, nodesConfig),
ComponentId.createAnonymousComponentId("test-id"),
dispatchConfig,
+ nodesConfig,
vipStatus);
ComponentRegistry<Dispatcher> dispatchers = new ComponentRegistry<>();
dispatchers.register(new ComponentId("dispatcher." + clusterName), dispatcher);
diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java
index dbc827ea5c2..2f960add4a8 100644
--- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java
+++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java
@@ -3,27 +3,22 @@ package com.yahoo.prelude.fastsearch.test;
import com.google.common.collect.ImmutableList;
import com.yahoo.component.chain.Chain;
-import com.yahoo.config.subscription.ConfigGetter;
import com.yahoo.container.QrSearchersConfig;
import com.yahoo.container.handler.VipStatus;
import com.yahoo.container.protect.Error;
-import com.yahoo.language.simple.SimpleLinguistics;
import com.yahoo.prelude.fastsearch.ClusterParams;
import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig;
import com.yahoo.prelude.fastsearch.FastSearcher;
import com.yahoo.prelude.fastsearch.SummaryParameters;
-import com.yahoo.prelude.fastsearch.VespaBackEndSearcher;
import com.yahoo.search.Query;
import com.yahoo.search.Result;
import com.yahoo.search.Searcher;
-import com.yahoo.search.config.SchemaInfoConfig;
import com.yahoo.search.dispatch.rpc.RpcResourcePool;
import com.yahoo.search.dispatch.searchcluster.Node;
import com.yahoo.search.grouping.GroupingRequest;
import com.yahoo.search.grouping.request.AllOperation;
import com.yahoo.search.grouping.request.EachOperation;
import com.yahoo.search.grouping.request.GroupingOperation;
-import com.yahoo.search.rendering.RendererRegistry;
import com.yahoo.search.result.ErrorMessage;
import com.yahoo.search.schema.DocumentSummary;
import com.yahoo.search.schema.RankProfile;
@@ -190,7 +185,7 @@ public class FastSearcherTestCase {
b.searchcluster(searchClusterB);
VipStatus vipStatus = new VipStatus(b.build());
List<Node> nodes_1 = ImmutableList.of(new Node(0, "host0", 0));
- RpcResourcePool rpcPool_1 = new RpcResourcePool(MockDispatcher.toDispatchConfig(nodes_1));
+ RpcResourcePool rpcPool_1 = new RpcResourcePool(MockDispatcher.toDispatchConfig(), MockDispatcher.toNodesConfig(nodes_1));
MockDispatcher dispatch_1 = MockDispatcher.create(nodes_1, rpcPool_1, vipStatus);
dispatch_1.clusterMonitor.shutdown();
vipStatus.addToRotation(clusterName);
diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/MockDispatcher.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/MockDispatcher.java
index 6d2f7efec38..95ecd0e5a40 100644
--- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/MockDispatcher.java
+++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/MockDispatcher.java
@@ -10,6 +10,7 @@ import com.yahoo.search.dispatch.rpc.RpcResourcePool;
import com.yahoo.search.dispatch.searchcluster.Node;
import com.yahoo.search.dispatch.searchcluster.SearchCluster;
import com.yahoo.vespa.config.search.DispatchConfig;
+import com.yahoo.vespa.config.search.DispatchNodesConfig;
import java.util.List;
@@ -18,14 +19,15 @@ class MockDispatcher extends Dispatcher {
public final ClusterMonitor clusterMonitor;
public static MockDispatcher create(List<Node> nodes) {
- var rpcResourcePool = new RpcResourcePool(toDispatchConfig(nodes));
+ var rpcResourcePool = new RpcResourcePool(toDispatchConfig(), toNodesConfig(nodes));
return create(nodes, rpcResourcePool, new VipStatus());
}
public static MockDispatcher create(List<Node> nodes, RpcResourcePool rpcResourcePool, VipStatus vipStatus) {
- var dispatchConfig = toDispatchConfig(nodes);
- var searchCluster = new SearchCluster("a", dispatchConfig, vipStatus, new RpcPingFactory(rpcResourcePool));
+ var dispatchConfig = toDispatchConfig();
+ var nodesConfig = toNodesConfig(nodes);
+ var searchCluster = new SearchCluster("a", dispatchConfig, nodesConfig, vipStatus, new RpcPingFactory(rpcResourcePool));
return new MockDispatcher(new ClusterMonitor<>(searchCluster, true), searchCluster, dispatchConfig, rpcResourcePool);
}
@@ -38,18 +40,21 @@ class MockDispatcher extends Dispatcher {
this.clusterMonitor = clusterMonitor;
}
- static DispatchConfig toDispatchConfig(List<Node> nodes) {
- DispatchConfig.Builder dispatchConfigBuilder = new DispatchConfig.Builder();
+ static DispatchConfig toDispatchConfig() {
+ return new DispatchConfig.Builder().build();
+ }
+ static DispatchNodesConfig toNodesConfig(List<Node> nodes) {
+ DispatchNodesConfig.Builder dispatchConfigBuilder = new DispatchNodesConfig.Builder();
int key = 0;
for (Node node : nodes) {
- DispatchConfig.Node.Builder dispatchConfigNodeBuilder = new DispatchConfig.Node.Builder();
+ DispatchNodesConfig.Node.Builder dispatchConfigNodeBuilder = new DispatchNodesConfig.Node.Builder();
dispatchConfigNodeBuilder.host(node.hostname());
dispatchConfigNodeBuilder.port(0); // Mandatory, but currently not used here
dispatchConfigNodeBuilder.group(node.group());
dispatchConfigNodeBuilder.key(key++); // not used
dispatchConfigBuilder.node(dispatchConfigNodeBuilder);
}
- return new DispatchConfig(dispatchConfigBuilder);
+ return dispatchConfigBuilder.build();
}
}
diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/InterleavedSearchInvokerTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/InterleavedSearchInvokerTest.java
index 7a6a64fac58..15656ffb457 100644
--- a/container-search/src/test/java/com/yahoo/search/dispatch/InterleavedSearchInvokerTest.java
+++ b/container-search/src/test/java/com/yahoo/search/dispatch/InterleavedSearchInvokerTest.java
@@ -22,6 +22,7 @@ import com.yahoo.searchlib.expression.IntegerResultNode;
import com.yahoo.searchlib.expression.StringResultNode;
import com.yahoo.test.ManualClock;
import com.yahoo.vespa.config.search.DispatchConfig;
+import com.yahoo.vespa.config.search.DispatchNodesConfig;
import org.junit.jupiter.api.Test;
import java.io.IOException;
@@ -40,6 +41,7 @@ import java.util.stream.StreamSupport;
import static com.yahoo.container.handler.Coverage.DEGRADED_BY_MATCH_PHASE;
import static com.yahoo.container.handler.Coverage.DEGRADED_BY_TIMEOUT;
import static com.yahoo.search.dispatch.MockSearchCluster.createDispatchConfig;
+import static com.yahoo.search.dispatch.MockSearchCluster.createNodesConfig;
import static org.junit.jupiter.api.Assertions.*;
/**
@@ -54,7 +56,7 @@ public class InterleavedSearchInvokerTest {
@Test
void requireThatAdaptiveTimeoutsAreNotUsedWithFullCoverageRequirement() throws IOException {
- SearchCluster cluster = new MockSearchCluster("!", createDispatchConfig(100.0), 1, 3);
+ SearchCluster cluster = new MockSearchCluster("!", createDispatchConfig(100.0), createNodesConfig(), 1, 3);
try (SearchInvoker invoker = createInterleavedInvoker(cluster, new Group(0, List.of()), 3)) {
expectedEvents.add(new Event(5000, 100, 0));
@@ -69,7 +71,7 @@ public class InterleavedSearchInvokerTest {
@Test
void requireThatTimeoutsAreNotMarkedAsAdaptive() throws IOException {
- SearchCluster cluster = new MockSearchCluster("!", createDispatchConfig(100.0), 1, 3);
+ SearchCluster cluster = new MockSearchCluster("!", createDispatchConfig(100.0), createNodesConfig(), 1, 3);
try (SearchInvoker invoker = createInterleavedInvoker(cluster, new Group(0, List.of()), 3)) {
expectedEvents.add(new Event(5000, 300, 0));
@@ -88,7 +90,7 @@ public class InterleavedSearchInvokerTest {
@Test
void requireThatAdaptiveTimeoutDecreasesTimeoutWhenCoverageIsReached() throws IOException {
- SearchCluster cluster = new MockSearchCluster("!", createDispatchConfig(50.0), 1, 4);
+ SearchCluster cluster = new MockSearchCluster("!", createDispatchConfig(50.0), createNodesConfig(), 1, 4);
try (SearchInvoker invoker = createInterleavedInvoker(cluster, new Group(0, List.of()), 4)) {
expectedEvents.add(new Event(5000, 100, 0));
@@ -400,8 +402,8 @@ public class InterleavedSearchInvokerTest {
return hits;
}
- void verifyCorrectCoverageCalculationWhenDegradedCoverageIsExpected(DispatchConfig dispatchConfig, int expectedCoverage) throws IOException {
- SearchCluster cluster = new MockSearchCluster("!", dispatchConfig, 1, 2);
+ void verifyCorrectCoverageCalculationWhenDegradedCoverageIsExpected(DispatchConfig dispatchConfig, DispatchNodesConfig nodesConfig, int expectedCoverage) throws IOException {
+ SearchCluster cluster = new MockSearchCluster("!", dispatchConfig, nodesConfig, 1, 2);
invokers.add(new MockInvoker(0, createCoverage(50155, 50155, 60000, 1, 1, 0)));
Coverage errorCoverage = new Coverage(0, 0, 0);
errorCoverage.setNodesTried(1);
@@ -427,9 +429,10 @@ public class InterleavedSearchInvokerTest {
@Test
void requireCorrectCoverageCalculationWhenDegradedCoverageIsExpectedUsingTargetActiveDocs() throws IOException {
- verifyCorrectCoverageCalculationWhenDegradedCoverageIsExpected(MockSearchCluster.createDispatchConfig(100.0, List.of())
+ verifyCorrectCoverageCalculationWhenDegradedCoverageIsExpected(MockSearchCluster.createDispatchConfigBuilder(100.0)
.redundancy(1)
.build(),
+ MockSearchCluster.createNodesConfig(),
42);
}
diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java
index ce3876e59c1..427062a1c4c 100644
--- a/container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java
+++ b/container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java
@@ -19,6 +19,7 @@ import java.util.Optional;
import java.util.Random;
import static com.yahoo.search.dispatch.MockSearchCluster.createDispatchConfig;
+import static com.yahoo.search.dispatch.MockSearchCluster.createNodesConfig;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -31,7 +32,7 @@ public class LoadBalancerTest {
@Test
void requireThatLoadBalancerServesSingleNodeSetups() {
Node n1 = new Node(0, "test-node1", 0);
- SearchCluster cluster = new SearchCluster("a", createDispatchConfig(n1), null, null);
+ SearchCluster cluster = new SearchCluster("a", createDispatchConfig(), createNodesConfig(n1), null, null);
LoadBalancer lb = new LoadBalancer(cluster, LoadBalancer.Policy.ROUNDROBIN);
Optional<Group> grp = lb.takeGroup(null);
@@ -45,7 +46,7 @@ public class LoadBalancerTest {
void requireThatLoadBalancerServesMultiGroupSetups() {
Node n1 = new Node(0, "test-node1", 0);
Node n2 = new Node(1, "test-node2", 1);
- SearchCluster cluster = new SearchCluster("a", createDispatchConfig(n1, n2), null, null);
+ SearchCluster cluster = new SearchCluster("a", createDispatchConfig(), createNodesConfig(n1, n2), null, null);
LoadBalancer lb = new LoadBalancer(cluster, LoadBalancer.Policy.ROUNDROBIN);
Optional<Group> grp = lb.takeGroup(null);
@@ -61,7 +62,7 @@ public class LoadBalancerTest {
Node n2 = new Node(1, "test-node2", 0);
Node n3 = new Node(0, "test-node3", 1);
Node n4 = new Node(1, "test-node4", 1);
- SearchCluster cluster = new SearchCluster("a", createDispatchConfig(n1, n2, n3, n4), null, null);
+ SearchCluster cluster = new SearchCluster("a", createDispatchConfig(), createNodesConfig(n1, n2, n3, n4), null, null);
LoadBalancer lb = new LoadBalancer(cluster, LoadBalancer.Policy.ROUNDROBIN);
Optional<Group> grp = lb.takeGroup(null);
@@ -72,7 +73,7 @@ public class LoadBalancerTest {
void requireThatLoadBalancerReturnsDifferentGroups() {
Node n1 = new Node(0, "test-node1", 0);
Node n2 = new Node(1, "test-node2", 1);
- SearchCluster cluster = new SearchCluster("a", createDispatchConfig(n1, n2), null, null);
+ SearchCluster cluster = new SearchCluster("a", createDispatchConfig(), createNodesConfig(n1, n2), null, null);
LoadBalancer lb = new LoadBalancer(cluster, LoadBalancer.Policy.ROUNDROBIN);
// get first group
diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java b/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java
index 7a11e906293..c90153e8008 100644
--- a/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java
+++ b/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java
@@ -8,9 +8,9 @@ import com.yahoo.search.dispatch.searchcluster.Group;
import com.yahoo.search.dispatch.searchcluster.Node;
import com.yahoo.search.dispatch.searchcluster.SearchCluster;
import com.yahoo.vespa.config.search.DispatchConfig;
+import com.yahoo.vespa.config.search.DispatchNodesConfig;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.List;
import java.util.Optional;
@@ -26,11 +26,11 @@ public class MockSearchCluster extends SearchCluster {
private final List<Node> nodes;
public MockSearchCluster(String clusterId, int groups, int nodesPerGroup) {
- this(clusterId, createDispatchConfig(), groups, nodesPerGroup);
+ this(clusterId, createDispatchConfig(), createNodesConfig(), groups, nodesPerGroup);
}
- public MockSearchCluster(String clusterId, DispatchConfig dispatchConfig, int groups, int nodesPerGroup) {
- super(clusterId, dispatchConfig, null, null);
+ public MockSearchCluster(String clusterId, DispatchConfig dispatchConfig, DispatchNodesConfig nodesConfig, int groups, int nodesPerGroup) {
+ super(clusterId, dispatchConfig, nodesConfig, null, null);
ImmutableList.Builder<Group> orderedGroupBuilder = ImmutableList.builder();
ImmutableMap.Builder<Integer, Group> groupBuilder = ImmutableMap.builder();
@@ -101,18 +101,18 @@ public class MockSearchCluster extends SearchCluster {
node.setWorking(false);
}
- public static DispatchConfig createDispatchConfig(Node... nodes) {
- return createDispatchConfig(100.0, nodes);
+ public static DispatchConfig createDispatchConfig() {
+ return createDispatchConfig(100.0);
}
- public static DispatchConfig createDispatchConfig(List<Node> nodes) {
- return createDispatchConfig(100.0, nodes).build();
+ public static DispatchNodesConfig createNodesConfig(Node... nodes) {
+ return createNodesConfig(List.of(nodes)).build();
}
- public static DispatchConfig createDispatchConfig(double minSearchCoverage, Node... nodes) {
- return createDispatchConfig(minSearchCoverage, Arrays.asList(nodes)).build();
+ public static DispatchConfig createDispatchConfig(double minSearchCoverage) {
+ return createDispatchConfigBuilder(minSearchCoverage).build();
}
- public static DispatchConfig.Builder createDispatchConfig(double minSearchCoverage, List<Node> nodes) {
+ public static DispatchConfig.Builder createDispatchConfigBuilder(double minSearchCoverage) {
DispatchConfig.Builder builder = new DispatchConfig.Builder();
builder.minActivedocsPercentage(88.0);
builder.minSearchCoverage(minSearchCoverage);
@@ -121,9 +121,14 @@ public class MockSearchCluster extends SearchCluster {
builder.minWaitAfterCoverageFactor(0);
builder.maxWaitAfterCoverageFactor(0.5);
}
+ return builder;
+ }
+
+ public static DispatchNodesConfig.Builder createNodesConfig(List<Node> nodes) {
+ DispatchNodesConfig.Builder builder = new DispatchNodesConfig.Builder();
int port = 10000;
for (Node n : nodes) {
- builder.node(new DispatchConfig.Node.Builder().key(n.key()).host(n.hostname()).port(port++).group(n.group()));
+ builder.node(new DispatchNodesConfig.Node.Builder().key(n.key()).host(n.hostname()).port(port++).group(n.group()));
}
return builder;
}
diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTest.java
index 6c46b2a492f..283ef29c878 100644
--- a/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTest.java
+++ b/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTest.java
@@ -56,7 +56,7 @@ public class SearchClusterTest {
numDocsPerNode.add(new AtomicInteger(1));
pingCounts.add(new AtomicInteger(0));
}
- searchCluster = new SearchCluster(clusterId, MockSearchCluster.createDispatchConfig(nodes),
+ searchCluster = new SearchCluster(clusterId, MockSearchCluster.createDispatchConfig(), MockSearchCluster.createNodesConfig(nodes).build(),
vipStatus, new Factory(nodesPerGroup, numDocsPerNode, pingCounts));
clusterMonitor = new ClusterMonitor(searchCluster, false);
searchCluster.addMonitoring(clusterMonitor);