diff options
author | Harald Musum <musum@verizonmedia.com> | 2020-02-14 14:39:51 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2020-02-14 14:39:51 +0100 |
commit | 5d042d27898fd7905da8d43ea79999576c949309 (patch) | |
tree | 7c585ca6f3292bd959a41c2040a925e40d785b0c /container-search | |
parent | 61cfe162e5b1944e038b44b8621aee51b2151891 (diff) |
Revert "Create a resourcepool so that we do not need to reconnect to content …"
Diffstat (limited to 'container-search')
11 files changed, 39 insertions, 34 deletions
diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java index 423de07f8c4..24ecea9c3ee 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java @@ -18,6 +18,7 @@ import com.yahoo.search.dispatch.rpc.RpcPingFactory; import com.yahoo.search.dispatch.rpc.RpcResourcePool; import com.yahoo.search.dispatch.searchcluster.Group; import com.yahoo.search.dispatch.searchcluster.Node; +import com.yahoo.search.dispatch.searchcluster.PingFactory; import com.yahoo.search.dispatch.searchcluster.SearchCluster; import com.yahoo.search.query.profile.types.FieldDescription; import com.yahoo.search.query.profile.types.FieldType; @@ -84,13 +85,21 @@ public class Dispatcher extends AbstractComponent { public static QueryProfileType getArgumentType() { return argumentType; } @Inject - public Dispatcher(RpcResourcePool resourcePool, - ComponentId clusterId, + public Dispatcher(ComponentId clusterId, DispatchConfig dispatchConfig, ClusterInfoConfig clusterInfoConfig, VipStatus vipStatus, Metric metric) { - this(resourcePool, new SearchCluster(clusterId.stringValue(), dispatchConfig,clusterInfoConfig.nodeCount(), + this(new RpcResourcePool(dispatchConfig), clusterId, dispatchConfig, clusterInfoConfig, vipStatus, metric); + } + + private Dispatcher(RpcResourcePool resourcePool, + ComponentId clusterId, + DispatchConfig dispatchConfig, + ClusterInfoConfig clusterInfoConfig, + VipStatus vipStatus, + Metric metric) { + this(resourcePool, new SearchCluster(clusterId.stringValue(), dispatchConfig, clusterInfoConfig.nodeCount(), vipStatus, new RpcPingFactory(resourcePool)), dispatchConfig, metric); diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcInvokerFactory.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcInvokerFactory.java index 74bc9e8bfbb..a45ec59c3ee 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcInvokerFactory.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcInvokerFactory.java @@ -1,19 +1,25 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.dispatch.rpc; +import com.yahoo.prelude.Pong; import com.yahoo.prelude.fastsearch.DocumentDatabase; import com.yahoo.prelude.fastsearch.VespaBackEndSearcher; import com.yahoo.processing.request.CompoundName; import com.yahoo.search.Query; import com.yahoo.search.Result; +import com.yahoo.search.cluster.ClusterMonitor; import com.yahoo.search.dispatch.Dispatcher; import com.yahoo.search.dispatch.FillInvoker; import com.yahoo.search.dispatch.InvokerFactory; import com.yahoo.search.dispatch.SearchInvoker; import com.yahoo.search.dispatch.searchcluster.Node; +import com.yahoo.search.dispatch.searchcluster.PingFactory; +import com.yahoo.search.dispatch.searchcluster.Pinger; +import com.yahoo.search.dispatch.searchcluster.PongHandler; import com.yahoo.search.dispatch.searchcluster.SearchCluster; import java.util.Optional; +import java.util.concurrent.Callable; /** * @author ollivir @@ -56,4 +62,7 @@ public class RpcInvokerFactory extends InvokerFactory { return new RpcFillInvoker(rpcResourcePool, documentDb); } + public void release() { + rpcResourcePool.release(); + } } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcResourcePool.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcResourcePool.java index 065489ef9a0..ca2a0c9bfb0 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcResourcePool.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcResourcePool.java @@ -2,9 +2,6 @@ package com.yahoo.search.dispatch.rpc; import com.google.common.collect.ImmutableMap; -import com.google.inject.Inject; -import com.yahoo.component.AbstractComponent; -import com.yahoo.component.ComponentId; import com.yahoo.compress.CompressionType; import com.yahoo.compress.Compressor; import com.yahoo.compress.Compressor.Compression; @@ -26,7 +23,7 @@ import java.util.Random; * * @author ollivir */ -public class RpcResourcePool extends AbstractComponent { +public class RpcResourcePool { /** The compression method which will be used with rpc dispatch. "lz4" (default) and "none" is supported. */ public final static CompoundName dispatchCompression = new CompoundName("dispatch.compression"); @@ -36,15 +33,13 @@ public class RpcResourcePool extends AbstractComponent { /** Connections to the search nodes this talks to, indexed by node id ("partid") */ private final ImmutableMap<Integer, NodeConnectionPool> nodeConnectionPools; - RpcResourcePool(Map<Integer, NodeConnection> nodeConnections) { + public RpcResourcePool(Map<Integer, NodeConnection> nodeConnections) { var builder = new ImmutableMap.Builder<Integer, NodeConnectionPool>(); nodeConnections.forEach((key, connection) -> builder.put(key, new NodeConnectionPool(Collections.singletonList(connection)))); this.nodeConnectionPools = builder.build(); } - @Inject public RpcResourcePool(DispatchConfig dispatchConfig) { - super(); var client = new RpcClient(dispatchConfig.numJrtTransportThreads()); // Create rpc node connection pools indexed by the node distribution key @@ -78,9 +73,7 @@ public class RpcResourcePool extends AbstractComponent { } } - @Override - public void deconstruct() { - super.deconstruct(); + public void release() { nodeConnectionPools.values().forEach(NodeConnectionPool::release); } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcSearchInvoker.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcSearchInvoker.java index 76240e55c98..07d8439ff46 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcSearchInvoker.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcSearchInvoker.java @@ -5,6 +5,7 @@ import com.yahoo.compress.CompressionType; import com.yahoo.compress.Compressor; import com.yahoo.prelude.fastsearch.VespaBackEndSearcher; import com.yahoo.search.Query; +import com.yahoo.search.Result; import com.yahoo.search.dispatch.InvokerResult; import com.yahoo.search.dispatch.SearchInvoker; import com.yahoo.search.dispatch.rpc.Client.ProtobufResponse; diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java index 7619cb34b77..1da13f6c082 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java @@ -83,13 +83,6 @@ public class SearchCluster implements NodeManager<Node> { nodesByHost, groups); } - - /* Testing only */ - public SearchCluster(String clusterId, DispatchConfig dispatchConfig, - VipStatus vipStatus, PingFactory pingFactory) { - this(clusterId, dispatchConfig, 1, vipStatus, pingFactory); - } - public void addMonitoring(ClusterMonitor clusterMonitor) { for (var group : orderedGroups) { for (var node : group.nodes()) diff --git a/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java index 8b7a57c38e7..5b6a4b68930 100644 --- a/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java @@ -22,7 +22,6 @@ import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.config.ClusterConfig; import com.yahoo.search.dispatch.Dispatcher; -import com.yahoo.search.dispatch.rpc.RpcResourcePool; import com.yahoo.search.result.Hit; import com.yahoo.search.searchchain.Execution; import com.yahoo.vespa.config.search.DispatchConfig; @@ -515,10 +514,8 @@ public class ClusterSearcherTestCase { DocumentdbInfoConfig.Builder documentDbConfig = new DocumentdbInfoConfig.Builder(); documentDbConfig.documentdb(new DocumentdbInfoConfig.Documentdb.Builder().name("type1")); - DispatchConfig dispatchConfig = new DispatchConfig.Builder().build(); - Dispatcher dispatcher = new Dispatcher(new RpcResourcePool(dispatchConfig), - ComponentId.createAnonymousComponentId("test-id"), - dispatchConfig, + Dispatcher dispatcher = new Dispatcher(new ComponentId("test-id"), + new DispatchConfig.Builder().build(), createClusterInfoConfig(), vipStatus, new MockMetric()); diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java index 63475c9c189..0f49f6029ad 100644 --- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java @@ -14,6 +14,7 @@ import com.yahoo.prelude.fastsearch.SummaryParameters; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.Searcher; +import com.yahoo.search.dispatch.Dispatcher; import com.yahoo.search.dispatch.rpc.RpcResourcePool; import com.yahoo.search.dispatch.searchcluster.Node; import com.yahoo.search.grouping.GroupingRequest; @@ -150,7 +151,7 @@ public class FastSearcherTestCase { VipStatus vipStatus = new VipStatus(b.build()); List<Node> nodes_1 = ImmutableList.of(new Node(0, "host0", 0)); RpcResourcePool rpcPool_1 = new RpcResourcePool(MockDispatcher.toDispatchConfig(nodes_1)); - MockDispatcher dispatch_1 = MockDispatcher.create(nodes_1, rpcPool_1, vipStatus); + MockDispatcher dispatch_1 = MockDispatcher.create(nodes_1, rpcPool_1, 1, vipStatus); dispatch_1.clusterMonitor.shutdown(); vipStatus.addToRotation(clusterName); assertTrue(vipStatus.isInRotation()); diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/MockDispatcher.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/MockDispatcher.java index 2a2c8410b2c..707b051c03b 100644 --- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/MockDispatcher.java +++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/MockDispatcher.java @@ -20,12 +20,13 @@ class MockDispatcher extends Dispatcher { public static MockDispatcher create(List<Node> nodes) { var rpcResourcePool = new RpcResourcePool(toDispatchConfig(nodes)); - return create(nodes, rpcResourcePool, new VipStatus()); + return create(nodes, rpcResourcePool, 1, new VipStatus()); } - public static MockDispatcher create(List<Node> nodes, RpcResourcePool rpcResourcePool, VipStatus vipStatus) { + public static MockDispatcher create(List<Node> nodes, RpcResourcePool rpcResourcePool, + int containerClusterSize, VipStatus vipStatus) { var dispatchConfig = toDispatchConfig(nodes); - var searchCluster = new SearchCluster("a", dispatchConfig, vipStatus, new RpcPingFactory(rpcResourcePool)); + var searchCluster = new SearchCluster("a", dispatchConfig, containerClusterSize, vipStatus, new RpcPingFactory(rpcResourcePool)); return new MockDispatcher(new ClusterMonitor<>(searchCluster, true), searchCluster, dispatchConfig, rpcResourcePool); } diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java index 36b476e2936..6eedb8239a9 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java @@ -29,7 +29,7 @@ public class LoadBalancerTest { @Test public void requireThatLoadBalancerServesSingleNodeSetups() { Node n1 = new Node(0, "test-node1", 0); - SearchCluster cluster = new SearchCluster("a", createDispatchConfig(n1), null, null); + SearchCluster cluster = new SearchCluster("a", createDispatchConfig(n1), 1, null, null); LoadBalancer lb = new LoadBalancer(cluster, true); Optional<Group> grp = lb.takeGroup(null); @@ -43,7 +43,7 @@ public class LoadBalancerTest { public void requireThatLoadBalancerServesMultiGroupSetups() { Node n1 = new Node(0, "test-node1", 0); Node n2 = new Node(1, "test-node2", 1); - SearchCluster cluster = new SearchCluster("a", createDispatchConfig(n1, n2), null, null); + SearchCluster cluster = new SearchCluster("a", createDispatchConfig(n1, n2), 1, null, null); LoadBalancer lb = new LoadBalancer(cluster, true); Optional<Group> grp = lb.takeGroup(null); @@ -59,7 +59,7 @@ public class LoadBalancerTest { Node n2 = new Node(1, "test-node2", 0); Node n3 = new Node(0, "test-node3", 1); Node n4 = new Node(1, "test-node4", 1); - SearchCluster cluster = new SearchCluster("a", createDispatchConfig(n1, n2, n3, n4), null, null); + SearchCluster cluster = new SearchCluster("a", createDispatchConfig(n1, n2, n3, n4), 2, null, null); LoadBalancer lb = new LoadBalancer(cluster, true); Optional<Group> grp = lb.takeGroup(null); @@ -70,7 +70,7 @@ public class LoadBalancerTest { public void requireThatLoadBalancerReturnsDifferentGroups() { Node n1 = new Node(0, "test-node1", 0); Node n2 = new Node(1, "test-node2", 1); - SearchCluster cluster = new SearchCluster("a", createDispatchConfig(n1, n2), null,null); + SearchCluster cluster = new SearchCluster("a", createDispatchConfig(n1, n2), 1, null,null); LoadBalancer lb = new LoadBalancer(cluster, true); // get first group diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java b/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java index 32c6738fc3b..3b4d58cdfc2 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java @@ -30,7 +30,7 @@ public class MockSearchCluster extends SearchCluster { } public MockSearchCluster(String clusterId, DispatchConfig dispatchConfig, int groups, int nodesPerGroup) { - super(clusterId, dispatchConfig, null, null); + super(clusterId, dispatchConfig, 1, null, null); ImmutableList.Builder<Group> orderedGroupBuilder = ImmutableList.builder(); ImmutableMap.Builder<Integer, Group> groupBuilder = ImmutableMap.builder(); diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTest.java index cf90a1c6d81..766f9ea6c2d 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTest.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTest.java @@ -14,6 +14,7 @@ import org.junit.Test; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.concurrent.Callable; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; |