From 51fc998599a4751bf0cbfa1c34971338714e5c85 Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Wed, 19 Feb 2020 10:29:32 +0100 Subject: Revert "Revert "Revert "Revert "Create a resourcepool so that we do not need to reconnect to content …"""" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../main/java/com/yahoo/search/dispatch/Dispatcher.java | 15 +++------------ .../com/yahoo/search/dispatch/rpc/RpcInvokerFactory.java | 9 --------- .../com/yahoo/search/dispatch/rpc/RpcResourcePool.java | 13 ++++++++++--- .../com/yahoo/search/dispatch/rpc/RpcSearchInvoker.java | 1 - .../search/dispatch/searchcluster/SearchCluster.java | 7 +++++++ .../yahoo/prelude/cluster/ClusterSearcherTestCase.java | 7 +++++-- .../prelude/fastsearch/test/FastSearcherTestCase.java | 3 +-- .../com/yahoo/prelude/fastsearch/test/MockDispatcher.java | 7 +++---- .../java/com/yahoo/search/dispatch/LoadBalancerTest.java | 8 ++++---- .../java/com/yahoo/search/dispatch/MockSearchCluster.java | 2 +- .../search/dispatch/searchcluster/SearchClusterTest.java | 1 - 11 files changed, 34 insertions(+), 39 deletions(-) (limited to 'container-search') diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java index 24ecea9c3ee..423de07f8c4 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java @@ -18,7 +18,6 @@ import com.yahoo.search.dispatch.rpc.RpcPingFactory; import com.yahoo.search.dispatch.rpc.RpcResourcePool; import com.yahoo.search.dispatch.searchcluster.Group; import com.yahoo.search.dispatch.searchcluster.Node; -import com.yahoo.search.dispatch.searchcluster.PingFactory; import com.yahoo.search.dispatch.searchcluster.SearchCluster; import com.yahoo.search.query.profile.types.FieldDescription; import com.yahoo.search.query.profile.types.FieldType; @@ -85,21 +84,13 @@ public class Dispatcher extends AbstractComponent { public static QueryProfileType getArgumentType() { return argumentType; } @Inject - public Dispatcher(ComponentId clusterId, + public Dispatcher(RpcResourcePool resourcePool, + ComponentId clusterId, DispatchConfig dispatchConfig, ClusterInfoConfig clusterInfoConfig, VipStatus vipStatus, Metric metric) { - this(new RpcResourcePool(dispatchConfig), clusterId, dispatchConfig, clusterInfoConfig, vipStatus, metric); - } - - private Dispatcher(RpcResourcePool resourcePool, - ComponentId clusterId, - DispatchConfig dispatchConfig, - ClusterInfoConfig clusterInfoConfig, - VipStatus vipStatus, - Metric metric) { - this(resourcePool, new SearchCluster(clusterId.stringValue(), dispatchConfig, clusterInfoConfig.nodeCount(), + this(resourcePool, new SearchCluster(clusterId.stringValue(), dispatchConfig,clusterInfoConfig.nodeCount(), vipStatus, new RpcPingFactory(resourcePool)), dispatchConfig, metric); diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcInvokerFactory.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcInvokerFactory.java index a45ec59c3ee..74bc9e8bfbb 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcInvokerFactory.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcInvokerFactory.java @@ -1,25 +1,19 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.dispatch.rpc; -import com.yahoo.prelude.Pong; import com.yahoo.prelude.fastsearch.DocumentDatabase; import com.yahoo.prelude.fastsearch.VespaBackEndSearcher; import com.yahoo.processing.request.CompoundName; import com.yahoo.search.Query; import com.yahoo.search.Result; -import com.yahoo.search.cluster.ClusterMonitor; import com.yahoo.search.dispatch.Dispatcher; import com.yahoo.search.dispatch.FillInvoker; import com.yahoo.search.dispatch.InvokerFactory; import com.yahoo.search.dispatch.SearchInvoker; import com.yahoo.search.dispatch.searchcluster.Node; -import com.yahoo.search.dispatch.searchcluster.PingFactory; -import com.yahoo.search.dispatch.searchcluster.Pinger; -import com.yahoo.search.dispatch.searchcluster.PongHandler; import com.yahoo.search.dispatch.searchcluster.SearchCluster; import java.util.Optional; -import java.util.concurrent.Callable; /** * @author ollivir @@ -62,7 +56,4 @@ public class RpcInvokerFactory extends InvokerFactory { return new RpcFillInvoker(rpcResourcePool, documentDb); } - public void release() { - rpcResourcePool.release(); - } } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcResourcePool.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcResourcePool.java index ca2a0c9bfb0..065489ef9a0 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcResourcePool.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcResourcePool.java @@ -2,6 +2,9 @@ package com.yahoo.search.dispatch.rpc; import com.google.common.collect.ImmutableMap; +import com.google.inject.Inject; +import com.yahoo.component.AbstractComponent; +import com.yahoo.component.ComponentId; import com.yahoo.compress.CompressionType; import com.yahoo.compress.Compressor; import com.yahoo.compress.Compressor.Compression; @@ -23,7 +26,7 @@ import java.util.Random; * * @author ollivir */ -public class RpcResourcePool { +public class RpcResourcePool extends AbstractComponent { /** The compression method which will be used with rpc dispatch. "lz4" (default) and "none" is supported. */ public final static CompoundName dispatchCompression = new CompoundName("dispatch.compression"); @@ -33,13 +36,15 @@ public class RpcResourcePool { /** Connections to the search nodes this talks to, indexed by node id ("partid") */ private final ImmutableMap nodeConnectionPools; - public RpcResourcePool(Map nodeConnections) { + RpcResourcePool(Map nodeConnections) { var builder = new ImmutableMap.Builder(); nodeConnections.forEach((key, connection) -> builder.put(key, new NodeConnectionPool(Collections.singletonList(connection)))); this.nodeConnectionPools = builder.build(); } + @Inject public RpcResourcePool(DispatchConfig dispatchConfig) { + super(); var client = new RpcClient(dispatchConfig.numJrtTransportThreads()); // Create rpc node connection pools indexed by the node distribution key @@ -73,7 +78,9 @@ public class RpcResourcePool { } } - public void release() { + @Override + public void deconstruct() { + super.deconstruct(); nodeConnectionPools.values().forEach(NodeConnectionPool::release); } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcSearchInvoker.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcSearchInvoker.java index 07d8439ff46..76240e55c98 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcSearchInvoker.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcSearchInvoker.java @@ -5,7 +5,6 @@ import com.yahoo.compress.CompressionType; import com.yahoo.compress.Compressor; import com.yahoo.prelude.fastsearch.VespaBackEndSearcher; import com.yahoo.search.Query; -import com.yahoo.search.Result; import com.yahoo.search.dispatch.InvokerResult; import com.yahoo.search.dispatch.SearchInvoker; import com.yahoo.search.dispatch.rpc.Client.ProtobufResponse; diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java index 1da13f6c082..7619cb34b77 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java @@ -83,6 +83,13 @@ public class SearchCluster implements NodeManager { nodesByHost, groups); } + + /* Testing only */ + public SearchCluster(String clusterId, DispatchConfig dispatchConfig, + VipStatus vipStatus, PingFactory pingFactory) { + this(clusterId, dispatchConfig, 1, vipStatus, pingFactory); + } + public void addMonitoring(ClusterMonitor clusterMonitor) { for (var group : orderedGroups) { for (var node : group.nodes()) diff --git a/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java index 5b6a4b68930..8b7a57c38e7 100644 --- a/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java @@ -22,6 +22,7 @@ import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.config.ClusterConfig; import com.yahoo.search.dispatch.Dispatcher; +import com.yahoo.search.dispatch.rpc.RpcResourcePool; import com.yahoo.search.result.Hit; import com.yahoo.search.searchchain.Execution; import com.yahoo.vespa.config.search.DispatchConfig; @@ -514,8 +515,10 @@ public class ClusterSearcherTestCase { DocumentdbInfoConfig.Builder documentDbConfig = new DocumentdbInfoConfig.Builder(); documentDbConfig.documentdb(new DocumentdbInfoConfig.Documentdb.Builder().name("type1")); - Dispatcher dispatcher = new Dispatcher(new ComponentId("test-id"), - new DispatchConfig.Builder().build(), + DispatchConfig dispatchConfig = new DispatchConfig.Builder().build(); + Dispatcher dispatcher = new Dispatcher(new RpcResourcePool(dispatchConfig), + ComponentId.createAnonymousComponentId("test-id"), + dispatchConfig, createClusterInfoConfig(), vipStatus, new MockMetric()); diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java index 0f49f6029ad..63475c9c189 100644 --- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java @@ -14,7 +14,6 @@ import com.yahoo.prelude.fastsearch.SummaryParameters; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.Searcher; -import com.yahoo.search.dispatch.Dispatcher; import com.yahoo.search.dispatch.rpc.RpcResourcePool; import com.yahoo.search.dispatch.searchcluster.Node; import com.yahoo.search.grouping.GroupingRequest; @@ -151,7 +150,7 @@ public class FastSearcherTestCase { VipStatus vipStatus = new VipStatus(b.build()); List nodes_1 = ImmutableList.of(new Node(0, "host0", 0)); RpcResourcePool rpcPool_1 = new RpcResourcePool(MockDispatcher.toDispatchConfig(nodes_1)); - MockDispatcher dispatch_1 = MockDispatcher.create(nodes_1, rpcPool_1, 1, vipStatus); + MockDispatcher dispatch_1 = MockDispatcher.create(nodes_1, rpcPool_1, vipStatus); dispatch_1.clusterMonitor.shutdown(); vipStatus.addToRotation(clusterName); assertTrue(vipStatus.isInRotation()); diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/MockDispatcher.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/MockDispatcher.java index 707b051c03b..2a2c8410b2c 100644 --- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/MockDispatcher.java +++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/MockDispatcher.java @@ -20,13 +20,12 @@ class MockDispatcher extends Dispatcher { public static MockDispatcher create(List nodes) { var rpcResourcePool = new RpcResourcePool(toDispatchConfig(nodes)); - return create(nodes, rpcResourcePool, 1, new VipStatus()); + return create(nodes, rpcResourcePool, new VipStatus()); } - public static MockDispatcher create(List nodes, RpcResourcePool rpcResourcePool, - int containerClusterSize, VipStatus vipStatus) { + public static MockDispatcher create(List nodes, RpcResourcePool rpcResourcePool, VipStatus vipStatus) { var dispatchConfig = toDispatchConfig(nodes); - var searchCluster = new SearchCluster("a", dispatchConfig, containerClusterSize, vipStatus, new RpcPingFactory(rpcResourcePool)); + var searchCluster = new SearchCluster("a", dispatchConfig, vipStatus, new RpcPingFactory(rpcResourcePool)); return new MockDispatcher(new ClusterMonitor<>(searchCluster, true), searchCluster, dispatchConfig, rpcResourcePool); } diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java index 6eedb8239a9..36b476e2936 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java @@ -29,7 +29,7 @@ public class LoadBalancerTest { @Test public void requireThatLoadBalancerServesSingleNodeSetups() { Node n1 = new Node(0, "test-node1", 0); - SearchCluster cluster = new SearchCluster("a", createDispatchConfig(n1), 1, null, null); + SearchCluster cluster = new SearchCluster("a", createDispatchConfig(n1), null, null); LoadBalancer lb = new LoadBalancer(cluster, true); Optional grp = lb.takeGroup(null); @@ -43,7 +43,7 @@ public class LoadBalancerTest { public void requireThatLoadBalancerServesMultiGroupSetups() { Node n1 = new Node(0, "test-node1", 0); Node n2 = new Node(1, "test-node2", 1); - SearchCluster cluster = new SearchCluster("a", createDispatchConfig(n1, n2), 1, null, null); + SearchCluster cluster = new SearchCluster("a", createDispatchConfig(n1, n2), null, null); LoadBalancer lb = new LoadBalancer(cluster, true); Optional grp = lb.takeGroup(null); @@ -59,7 +59,7 @@ public class LoadBalancerTest { Node n2 = new Node(1, "test-node2", 0); Node n3 = new Node(0, "test-node3", 1); Node n4 = new Node(1, "test-node4", 1); - SearchCluster cluster = new SearchCluster("a", createDispatchConfig(n1, n2, n3, n4), 2, null, null); + SearchCluster cluster = new SearchCluster("a", createDispatchConfig(n1, n2, n3, n4), null, null); LoadBalancer lb = new LoadBalancer(cluster, true); Optional grp = lb.takeGroup(null); @@ -70,7 +70,7 @@ public class LoadBalancerTest { public void requireThatLoadBalancerReturnsDifferentGroups() { Node n1 = new Node(0, "test-node1", 0); Node n2 = new Node(1, "test-node2", 1); - SearchCluster cluster = new SearchCluster("a", createDispatchConfig(n1, n2), 1, null,null); + SearchCluster cluster = new SearchCluster("a", createDispatchConfig(n1, n2), null,null); LoadBalancer lb = new LoadBalancer(cluster, true); // get first group diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java b/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java index 3b4d58cdfc2..32c6738fc3b 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java @@ -30,7 +30,7 @@ public class MockSearchCluster extends SearchCluster { } public MockSearchCluster(String clusterId, DispatchConfig dispatchConfig, int groups, int nodesPerGroup) { - super(clusterId, dispatchConfig, 1, null, null); + super(clusterId, dispatchConfig, null, null); ImmutableList.Builder orderedGroupBuilder = ImmutableList.builder(); ImmutableMap.Builder groupBuilder = ImmutableMap.builder(); diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTest.java index 766f9ea6c2d..cf90a1c6d81 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTest.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTest.java @@ -14,7 +14,6 @@ import org.junit.Test; import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import java.util.concurrent.Callable; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; -- cgit v1.2.3