diff options
author | Henning Baldersheim <balder@yahoo-inc.com> | 2019-09-19 12:18:30 +0200 |
---|---|---|
committer | Henning Baldersheim <balder@yahoo-inc.com> | 2019-09-19 14:14:55 +0200 |
commit | db5569d9cf4f31ab196293a03ced3d8047a20570 (patch) | |
tree | 69f9f00d79870e724bd64d47f7142152e29b773f /container-search | |
parent | d082531b8c6244de5bc99ed887f706be3a1084df (diff) |
Add test for in and out of vip and fix bug.
Diffstat (limited to 'container-search')
3 files changed, 253 insertions, 5 deletions
diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java index 2bcac6e2ce4..6f775e7218e 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java @@ -239,9 +239,7 @@ public class SearchCluster implements NodeManager<Node> { else vipStatus.removeFromRotation(clusterId); } - else { - if ( ! hasInformationAboutAllNodes()) return; - + else if (localCorpusDispatchTarget.isEmpty() && hasInformationAboutAllNodes()) { if (hasWorkingNodes()) vipStatus.addToRotation(clusterId); else @@ -257,8 +255,8 @@ public class SearchCluster implements NodeManager<Node> { else vipStatus.removeFromRotation(clusterId); } - else { - if ( ! isInRotation && sufficientCoverage) + else if ( localCorpusDispatchTarget.isEmpty()) { + if (isInRotation && sufficientCoverage) vipStatus.addToRotation(clusterId); } } diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java b/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java index f4bfa766328..e3ff54102d4 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java @@ -10,6 +10,7 @@ import com.yahoo.search.dispatch.searchcluster.SearchCluster; import com.yahoo.vespa.config.search.DispatchConfig; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.Optional; @@ -106,8 +107,14 @@ public class MockSearchCluster extends SearchCluster { public static DispatchConfig createDispatchConfig(Node... nodes) { return createDispatchConfig(100.0, nodes); } + public static DispatchConfig createDispatchConfig(List<Node> nodes) { + return createDispatchConfig(100.0, nodes); + } public static DispatchConfig createDispatchConfig(double minSearchCoverage, Node... nodes) { + return createDispatchConfig(minSearchCoverage, Arrays.asList(nodes)); + } + public static DispatchConfig createDispatchConfig(double minSearchCoverage, List<Node> nodes) { DispatchConfig.Builder builder = new DispatchConfig.Builder(); builder.minActivedocsPercentage(88.0); builder.minGroupCoverage(99.0); diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTest.java new file mode 100644 index 00000000000..bbaf512534a --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTest.java @@ -0,0 +1,243 @@ +package com.yahoo.search.dispatch.searchcluster; + +import com.yahoo.container.QrSearchersConfig; +import com.yahoo.container.handler.ClustersStatus; +import com.yahoo.container.handler.VipStatus; +import com.yahoo.net.HostName; +import com.yahoo.prelude.Pong; +import com.yahoo.search.cluster.ClusterMonitor; +import com.yahoo.search.dispatch.MockSearchCluster; +import com.yahoo.search.result.ErrorMessage; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.Callable; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class SearchClusterTest { + + static class State { + final String clusterId; + final int nodesPerGroup; + final VipStatus vipStatus; + final SearchCluster sc; + final List<AtomicInteger> numDocsPerNode; + List<AtomicInteger> pingCounts; + State(String clusterId, int nodesPergroup, String ... nodeNames) { + this(clusterId, nodesPergroup, Arrays.asList(nodeNames)); + } + State(String clusterId, int nodesPergroup, List<String> nodeNames) { + this.clusterId = clusterId; + this.nodesPerGroup = nodesPergroup; + vipStatus = new VipStatus(new QrSearchersConfig.Builder().searchcluster(new QrSearchersConfig.Searchcluster.Builder().name(clusterId)).build(), new ClustersStatus()); + assertFalse(vipStatus.isInRotation()); + vipStatus.addToRotation(clusterId); + assertTrue(vipStatus.isInRotation()); + numDocsPerNode = new ArrayList<>(nodeNames.size()); + pingCounts = new ArrayList<>(nodeNames.size()); + List<Node> nodes = new ArrayList<>(nodeNames.size()); + + for (String name : nodeNames) { + int key = nodes.size() % nodesPergroup; + int group = nodes.size() / nodesPergroup; + nodes.add(new Node(key, name, 13333, group)); + numDocsPerNode.add(new AtomicInteger(1)); + pingCounts.add(new AtomicInteger(0)); + } + sc = new SearchCluster(clusterId, MockSearchCluster.createDispatchConfig(nodes),nodes.size() / nodesPergroup, vipStatus); + } + void startMonitoring() { + sc.startClusterMonitoring(new Factory(nodesPerGroup, numDocsPerNode, pingCounts)); + } + private static int getMaxValue(List<AtomicInteger> list) { + int max = list.get(0).get(); + for (AtomicInteger v : list) { + if (v.get() > max) { + max = v.get(); + } + } + return max; + } + private static int getMinValue(List<AtomicInteger> list) { + int min = list.get(0).get(); + for (AtomicInteger v : list) { + if (v.get() < min) { + min = v.get(); + } + } + return min; + } + private static void waitAtLeast(int atLeast, List<AtomicInteger> list) { + while (getMinValue(list) < atLeast) { + try { + Thread.sleep(100); + } catch (InterruptedException e) {} + } + } + void waitOneFullPingRound() { + waitAtLeast(getMaxValue(pingCounts) + 1, pingCounts); + } + static class Factory implements PingFactory { + static class Pinger implements Callable<Pong> { + private final AtomicInteger numDocs; + private final AtomicInteger pingCount; + Pinger(AtomicInteger numDocs, AtomicInteger pingCount) { + this.numDocs = numDocs; + this.pingCount = pingCount; + } + @Override + public Pong call() throws Exception { + int docs = numDocs.get(); + pingCount.incrementAndGet(); + return (docs < 0) + ? new Pong(ErrorMessage.createBackendCommunicationError("Negative numDocs = " + docs)) + : new Pong(docs); + } + } + + private final List<AtomicInteger> activeDocs; + private final List<AtomicInteger> pingCounts; + private final int numPerGroup; + Factory(int numPerGroup, List<AtomicInteger> activeDocs, List<AtomicInteger> pingCounts) { + this.numPerGroup = numPerGroup; + this.activeDocs = activeDocs; + this.pingCounts = pingCounts; + } + + @Override + public Callable<Pong> createPinger(Node node, ClusterMonitor<Node> monitor) { + int index = node.group*numPerGroup + node.key(); + return new Pinger(activeDocs.get(index), pingCounts.get(index)); + } + } + } + + @Test + public void requireThatVipStatusIsDefaultUp() { + State test = new State("cluster.1", 2, "a", "b"); + assertTrue(test.vipStatus.isInRotation()); + assertTrue(test.sc.localCorpusDispatchTarget().isEmpty()); + } + + @Test + public void requireThatZeroDocsAreFine() { + State test = new State("cluster.1", 2,"a", "b"); + assertTrue(test.vipStatus.isInRotation()); + assertTrue(test.sc.localCorpusDispatchTarget().isEmpty()); + + test.startMonitoring(); + test.numDocsPerNode.get(0).set(-1); + test.numDocsPerNode.get(1).set(-1); + test.waitOneFullPingRound(); + assertFalse(test.vipStatus.isInRotation()); + test.numDocsPerNode.get(0).set(0); + test.waitOneFullPingRound(); + assertTrue(test.vipStatus.isInRotation()); + } + + @Test + public void requireThatVipStatusIsDefaultUpWithLocalDispatch() { + State test = new State("cluster.1", 1, HostName.getLocalhost(), "b"); + assertTrue(test.vipStatus.isInRotation()); + assertTrue(test.sc.localCorpusDispatchTarget().isPresent()); + } + + @Test + public void requireThatVipStatusDownWhenLocalIsDown() { + State test = new State("cluster.1",1,HostName.getLocalhost(), "b"); + assertTrue(test.vipStatus.isInRotation()); + assertTrue(test.sc.localCorpusDispatchTarget().isPresent()); + + test.startMonitoring(); + test.waitOneFullPingRound(); + assertTrue(test.vipStatus.isInRotation()); + test.numDocsPerNode.get(0).set(-1); + test.waitOneFullPingRound(); + assertFalse(test.vipStatus.isInRotation()); + + test.numDocsPerNode.get(0).set(1); + test.waitOneFullPingRound(); + assertTrue(test.vipStatus.isInRotation()); + + test.numDocsPerNode.get(1).set(-1); + test.waitOneFullPingRound(); + assertTrue(test.vipStatus.isInRotation()); + + test.numDocsPerNode.get(0).set(-1); + test.numDocsPerNode.get(1).set(-1); + test.waitOneFullPingRound(); + assertFalse(test.vipStatus.isInRotation()); + test.numDocsPerNode.get(1).set(1); + test.waitOneFullPingRound(); + assertFalse(test.vipStatus.isInRotation()); + test.numDocsPerNode.get(0).set(1); + test.waitOneFullPingRound(); + assertTrue(test.vipStatus.isInRotation()); + } + + private void verifyThatVipStatusDownRequireAllNodesDown(int numGroups, int nodesPerGroup) { + List<String> nodeNames = generateNodeNames(numGroups, nodesPerGroup); + State test = new State("cluster.1", nodesPerGroup, nodeNames); + assertTrue(test.vipStatus.isInRotation()); + assertTrue(test.sc.localCorpusDispatchTarget().isEmpty()); + + test.startMonitoring(); + test.waitOneFullPingRound(); + assertTrue(test.vipStatus.isInRotation()); + + for (int i=0; i < test.numDocsPerNode.size()-1; i++) { + test.numDocsPerNode.get(i).set(-1); + } + test.waitOneFullPingRound(); + assertTrue(test.vipStatus.isInRotation()); + test.numDocsPerNode.get(test.numDocsPerNode.size()-1).set(-1); + test.waitOneFullPingRound(); + assertFalse(test.vipStatus.isInRotation()); + } + @Test + public void requireThatVipStatusDownRequireAllNodesDown() { + verifyThatVipStatusDownRequireAllNodesDown(1,2); + verifyThatVipStatusDownRequireAllNodesDown(3, 3); + } + + static private List<String> generateNodeNames(int numGroups, int nodesPerGroup) { + List<String> nodeNames = new ArrayList<>(numGroups*nodesPerGroup); + for (int g = 0; g < numGroups; g++) { + for (int n=0; n < nodesPerGroup; n++) { + nodeNames.add(new StringBuilder("node.").append(g).append('.').append(n).toString()); + } + } + return nodeNames; + } + private void verifyThatVipStatusUpRequireOnlyOneOnlineNode(int numGroups, int nodesPerGroup) { + List<String> nodeNames = generateNodeNames(numGroups, nodesPerGroup); + State test = new State("cluster.1", nodesPerGroup, nodeNames); + assertTrue(test.vipStatus.isInRotation()); + assertTrue(test.sc.localCorpusDispatchTarget().isEmpty()); + + test.startMonitoring(); + for (int i=0; i < test.numDocsPerNode.size()-1; i++) { + test.numDocsPerNode.get(i).set(-1); + } + test.waitOneFullPingRound(); + assertTrue(test.vipStatus.isInRotation()); + test.numDocsPerNode.get(test.numDocsPerNode.size()-1).set(-1); + test.waitOneFullPingRound(); + assertFalse(test.vipStatus.isInRotation()); + + test.numDocsPerNode.get(0).set(0); + test.waitOneFullPingRound(); + assertTrue(test.vipStatus.isInRotation()); + } + @Test + public void requireThatVipStatusUpRequireOnlyOneOnlineNode() { + verifyThatVipStatusUpRequireOnlyOneOnlineNode(1, 2); + verifyThatVipStatusUpRequireOnlyOneOnlineNode(3, 3); + } + +} |