diff options
author | Olli Virtanen <olli.virtanen@oath.com> | 2019-01-03 15:11:03 +0100 |
---|---|---|
committer | Olli Virtanen <olli.virtanen@oath.com> | 2019-01-03 15:11:03 +0100 |
commit | 033182288ab6f4f404a23d1456001d31cb930fc9 (patch) | |
tree | 6d89922ce1505766261b5df7b2656f7528485f42 | |
parent | b48a97a0814a9a5b6e42ef963f5cb9e2f61d5395 (diff) |
Increased java dispatcher test coverage
6 files changed, 335 insertions, 34 deletions
diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FS4InvokerFactory.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FS4InvokerFactory.java index 58203981039..612941ece7a 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FS4InvokerFactory.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FS4InvokerFactory.java @@ -115,16 +115,13 @@ public class FS4InvokerFactory { private SearchInvoker createCoverageErrorInvoker(List<Node> nodes, Set<Integer> failed) { StringBuilder down = new StringBuilder("Connection failure on nodes with distribution-keys: "); - Integer key = null; int count = 0; for (Node node : nodes) { if (failed.contains(node.key())) { - count++; - if (key == null) { - key = node.key(); - } else { + if (count > 0) { down.append(", "); } + count++; down.append(node.key()); } } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java index baf401a2536..e98fc830a06 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java @@ -49,7 +49,11 @@ public class Dispatcher extends AbstractComponent { private final boolean multilevelDispatch; public Dispatcher(String clusterId, DispatchConfig dispatchConfig, FS4ResourcePool fs4ResourcePool, int containerClusterSize, VipStatus vipStatus) { - this.searchCluster = new SearchCluster(clusterId, dispatchConfig, fs4ResourcePool, containerClusterSize, vipStatus); + this(new SearchCluster(clusterId, dispatchConfig, fs4ResourcePool, containerClusterSize, vipStatus), dispatchConfig); + } + + public Dispatcher(SearchCluster searchCluster, DispatchConfig dispatchConfig) { + this.searchCluster = searchCluster; this.loadBalancer = new LoadBalancer(searchCluster, dispatchConfig.distributionPolicy() == DispatchConfig.DistributionPolicy.ROUNDROBIN); this.rpcResourcePool = new RpcResourcePool(dispatchConfig); diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java new file mode 100644 index 00000000000..fae659b0d70 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java @@ -0,0 +1,132 @@ +// Copyright 2019 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.dispatch; + +import com.yahoo.prelude.fastsearch.FS4InvokerFactory; +import com.yahoo.processing.request.CompoundName; +import com.yahoo.search.Query; +import com.yahoo.search.dispatch.searchcluster.Node; +import com.yahoo.search.dispatch.searchcluster.SearchCluster; +import com.yahoo.vespa.config.search.DispatchConfig; +import org.junit.Test; + +import java.util.List; +import java.util.Optional; + +import static com.yahoo.search.dispatch.MockSearchCluster.createDispatchConfig; +import static org.hamcrest.Matchers.is; +import static org.junit.Assert.assertThat; + +/** + * @author ollivir + */ +public class DispatcherTest { + private static final CompoundName internalDispatch = new CompoundName("dispatch.internal"); + + private static Query query() { + Query q = new Query(); + q.properties().set(internalDispatch, "true"); + return q; + } + + @Test + public void requireDispatcherToIgnoreMultilevelConfigurations() { + SearchCluster cl = new MockSearchCluster("1", 2, 2); + DispatchConfig.Builder builder = new DispatchConfig.Builder(); + builder.useMultilevelDispatch(true); + DispatchConfig dc = new DispatchConfig(builder); + + Dispatcher disp = new Dispatcher(cl, dc); + assertThat(disp.getSearchInvoker(query(), null).isPresent(), is(false)); + } + + @Test + public void requireThatDispatcherSupportsSearchPath() { + SearchCluster cl = new MockSearchCluster("1", 2, 2); + Query q = query(); + q.getModel().setSearchPath("1/0"); // second node in first group + Dispatcher disp = new Dispatcher(cl, createDispatchConfig()); + MockFS4InvokerFactory invokerFactory = new MockFS4InvokerFactory(cl, (nodes, a) -> { + assertThat(nodes.size(), is(1)); + assertThat(nodes.get(0).key(), is(2)); + return true; + }); + Optional<SearchInvoker> invoker = disp.getSearchInvoker(q, invokerFactory); + assertThat(invoker.isPresent(), is(true)); + invokerFactory.verifyAllEventsProcessed(); + } + + @Test + public void requireThatDispatcherSupportsSingleNodeDirectDispatch() { + SearchCluster cl = new MockSearchCluster("1", 0, 0) { + @Override + public Optional<Node> directDispatchTarget() { + return Optional.of(new Node(1, "test", 123, 1)); + } + }; + Dispatcher disp = new Dispatcher(cl, createDispatchConfig()); + MockFS4InvokerFactory invokerFactory = new MockFS4InvokerFactory(cl, (n, a) -> true); + Optional<SearchInvoker> invoker = disp.getSearchInvoker(query(), invokerFactory); + assertThat(invoker.isPresent(), is(true)); + invokerFactory.verifyAllEventsProcessed(); + } + + @Test + public void requireThatInvokerConstructionIsRetriedAndLastAcceptsAnyCoverage() { + SearchCluster cl = new MockSearchCluster("1", 2, 1); + + Dispatcher disp = new Dispatcher(cl, createDispatchConfig()); + MockFS4InvokerFactory invokerFactory = new MockFS4InvokerFactory(cl, (n, acceptIncompleteCoverage) -> { + assertThat(acceptIncompleteCoverage, is(false)); + return false; + }, (n, acceptIncompleteCoverage) -> { + assertThat(acceptIncompleteCoverage, is(true)); + return true; + }); + Optional<SearchInvoker> invoker = disp.getSearchInvoker(query(), invokerFactory); + assertThat(invoker.isPresent(), is(true)); + invokerFactory.verifyAllEventsProcessed(); + } + + @Test + public void requireThatInvokerConstructionDoesNotRepeatGroups() { + SearchCluster cl = new MockSearchCluster("1", 2, 1); + + Dispatcher disp = new Dispatcher(cl, createDispatchConfig()); + MockFS4InvokerFactory invokerFactory = new MockFS4InvokerFactory(cl, (n, a) -> false, (n, a) -> false); + Optional<SearchInvoker> invoker = disp.getSearchInvoker(query(), invokerFactory); + assertThat(invoker.isPresent(), is(false)); + invokerFactory.verifyAllEventsProcessed(); + } + + interface FactoryStep { + public boolean returnInvoker(List<Node> nodes, boolean acceptIncompleteCoverage); + } + + private static class MockFS4InvokerFactory extends FS4InvokerFactory { + private final FactoryStep[] events; + private int step = 0; + + public MockFS4InvokerFactory(SearchCluster cl, FactoryStep... events) { + super(null, cl, null); + this.events = events; + } + + @Override + public Optional<SearchInvoker> getSearchInvoker(Query query, int groupId, List<Node> nodes, boolean acceptIncompleteCoverage) { + if (step >= events.length) { + throw new RuntimeException("Was not expecting more calls to getSearchInvoker"); + } + boolean nonEmpty = events[step].returnInvoker(nodes, acceptIncompleteCoverage); + step++; + if (nonEmpty) { + return Optional.of(new MockInvoker(1)); + } else { + return Optional.empty(); + } + } + + void verifyAllEventsProcessed() { + assertThat(step, is(events.length)); + } + } +} diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/InterleavedSearchInvokerTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/InterleavedSearchInvokerTest.java index 99694f22254..a4270ea0ae4 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/InterleavedSearchInvokerTest.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/InterleavedSearchInvokerTest.java @@ -1,13 +1,11 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.dispatch; -import com.yahoo.fs4.QueryPacket; -import com.yahoo.prelude.fastsearch.CacheKey; import com.yahoo.search.Query; import com.yahoo.search.Result; -import com.yahoo.search.dispatch.searchcluster.Node; import com.yahoo.search.dispatch.searchcluster.SearchCluster; -import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.result.Coverage; +import com.yahoo.search.result.ErrorMessage; import com.yahoo.test.ManualClock; import org.junit.Test; @@ -17,11 +15,14 @@ import java.time.Instant; import java.util.ArrayList; import java.util.LinkedList; import java.util.List; -import java.util.Optional; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.TimeUnit; +import static com.yahoo.container.handler.Coverage.DEGRADED_BY_MATCH_PHASE; +import static com.yahoo.container.handler.Coverage.DEGRADED_BY_TIMEOUT; import static com.yahoo.search.dispatch.MockSearchCluster.createDispatchConfig; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.is; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; @@ -84,9 +85,122 @@ public class InterleavedSearchInvokerTest { assertTrue("Degradataion reason is an adaptive timeout", result.getCoverage(false).isDegradedByAdapativeTimeout()); } + @Test + public void requireCorrectCoverageCalculationWhenAllNodesOk() throws IOException { + SearchCluster cluster = new MockSearchCluster("!", 1, 2); + invokers.add(new MockInvoker(0, createCoverage(50155, 50155, 50155, 1, 1, 0))); + invokers.add(new MockInvoker(1, createCoverage(49845, 49845, 49845, 1, 1, 0))); + SearchInvoker invoker = createInterleavedInvoker(cluster, 0); + + expectedEvents.add(new Event(null, 100, 0)); + expectedEvents.add(new Event(null, 200, 1)); + + Result result = invoker.search(query, null, null, null); + + Coverage cov = result.getCoverage(true); + assertThat(cov.getDocs(), is(100000L)); + assertThat(cov.getNodes(), is(2)); + assertThat(cov.getFull(), is(true)); + assertThat(cov.getResultPercentage(), is(100)); + assertThat(cov.getResultSets(), is(1)); + assertThat(cov.getFullResultSets(), is(1)); + } + + @Test + public void requireCorrectCoverageCalculationWhenResultsAreLimitedByMatchPhase() throws IOException { + SearchCluster cluster = new MockSearchCluster("!", 1, 2); + invokers.add(new MockInvoker(0, createCoverage(10101, 50155, 50155, 1, 1, DEGRADED_BY_MATCH_PHASE))); + invokers.add(new MockInvoker(1, createCoverage(13319, 49845, 49845, 1, 1, DEGRADED_BY_MATCH_PHASE))); + SearchInvoker invoker = createInterleavedInvoker(cluster, 0); + + expectedEvents.add(new Event(null, 100, 0)); + expectedEvents.add(new Event(null, 200, 1)); + + Result result = invoker.search(query, null, null, null); + + Coverage cov = result.getCoverage(true); + assertThat(cov.getDocs(), is(23420L)); + assertThat(cov.getNodes(), is(2)); + assertThat(cov.getFull(), is(false)); + assertThat(cov.getResultPercentage(), is(23)); + assertThat(cov.getResultSets(), is(1)); + assertThat(cov.getFullResultSets(), is(0)); + assertThat(cov.isDegradedByMatchPhase(), is(true)); + } + + @Test + public void requireCorrectCoverageCalculationWhenResultsAreLimitedBySoftTimeout() throws IOException { + SearchCluster cluster = new MockSearchCluster("!", 1, 2); + invokers.add(new MockInvoker(0, createCoverage(5000, 50155, 50155, 1, 1, DEGRADED_BY_TIMEOUT))); + invokers.add(new MockInvoker(1, createCoverage(4900, 49845, 49845, 1, 1, DEGRADED_BY_TIMEOUT))); + SearchInvoker invoker = createInterleavedInvoker(cluster, 0); + + expectedEvents.add(new Event(null, 100, 0)); + expectedEvents.add(new Event(null, 200, 1)); + + Result result = invoker.search(query, null, null, null); + + Coverage cov = result.getCoverage(true); + assertThat(cov.getDocs(), is(9900L)); + assertThat(cov.getNodes(), is(2)); + assertThat(cov.getFull(), is(false)); + assertThat(cov.getResultPercentage(), is(10)); + assertThat(cov.getResultSets(), is(1)); + assertThat(cov.getFullResultSets(), is(0)); + assertThat(cov.isDegradedByTimeout(), is(true)); + } + + @Test + public void requireCorrectCoverageCalculationWhenOneNodeIsUnexpectedlyDown() throws IOException { + SearchCluster cluster = new MockSearchCluster("!", 1, 2); + invokers.add(new MockInvoker(0, createCoverage(50155, 50155, 50155, 1, 1, 0))); + invokers.add(new MockInvoker(1, createCoverage(49845, 49845, 49845, 1, 1, 0))); + SearchInvoker invoker = createInterleavedInvoker(cluster, 0); + + expectedEvents.add(new Event(null, 100, 0)); + expectedEvents.add(null); + + Result result = invoker.search(query, null, null, null); + + Coverage cov = result.getCoverage(true); + assertThat(cov.getDocs(), is(50155L)); + assertThat(cov.getNodes(), is(1)); + assertThat(cov.getNodesTried(), is(2)); + assertThat(cov.getFull(), is(false)); + assertThat(cov.getResultPercentage(), is(50)); + assertThat(cov.getResultSets(), is(1)); + assertThat(cov.getFullResultSets(), is(0)); + assertThat(cov.isDegradedByTimeout(), is(true)); + } + + @Test + public void requireCorrectCoverageCalculationWhenDegradedCoverageIsExpected() throws IOException { + SearchCluster cluster = new MockSearchCluster("!", 1, 2); + invokers.add(new MockInvoker(0, createCoverage(50155, 50155, 50155, 1, 1, 0))); + Coverage errorCoverage = new Coverage(0, 0, 0); + errorCoverage.setNodesTried(1); + invokers.add(new SearchErrorInvoker(ErrorMessage.createBackendCommunicationError("node is down"), errorCoverage)); + SearchInvoker invoker = createInterleavedInvoker(cluster, 0); + + expectedEvents.add(new Event(null, 1, 1)); + expectedEvents.add(new Event(null, 100, 0)); + + Result result = invoker.search(query, null, null, null); + + Coverage cov = result.getCoverage(true); + assertThat(cov.getDocs(), is(50155L)); + assertThat(cov.getNodes(), is(1)); + assertThat(cov.getNodesTried(), is(2)); + assertThat(cov.getFull(), is(false)); + assertThat(cov.getResultPercentage(), is(50)); + assertThat(cov.getResultSets(), is(1)); + assertThat(cov.getFullResultSets(), is(0)); + assertThat(cov.isDegradedByTimeout(), is(true)); + } + private InterleavedSearchInvoker createInterleavedInvoker(SearchCluster searchCluster, int numInvokers) { for (int i = 0; i < numInvokers; i++) { - invokers.add(new TestInvoker()); + invokers.add(new MockInvoker(i)); } return new InterleavedSearchInvoker(invokers, null, searchCluster) { @@ -113,13 +227,23 @@ public class InterleavedSearchInvokerTest { }; } + private static Coverage createCoverage(int docs, int activeDocs, int soonActiveDocs, int nodes, int nodesTried, int degradedReason) { + Coverage coverage = new Coverage(docs, activeDocs, nodes); + coverage.setSoonActive(soonActiveDocs); + coverage.setNodesTried(nodesTried); + coverage.setDegradedReason(degradedReason); + return coverage; + } + private class Event { Long expectedTimeout; long delay; Integer invokerIndex; public Event(Integer expectedTimeout, int delay, Integer invokerIndex) { - this.expectedTimeout = (long) expectedTimeout; + if (expectedTimeout != null) { + this.expectedTimeout = (long) expectedTimeout; + } this.delay = delay; this.invokerIndex = invokerIndex; } @@ -140,25 +264,6 @@ public class InterleavedSearchInvokerTest { } } - private class TestInvoker extends SearchInvoker { - protected TestInvoker() { - super(Optional.of(new Node(42, "?", 0, 0))); - } - - @Override - protected void sendSearchRequest(Query query, QueryPacket queryPacket) throws IOException { - } - - @Override - protected Result getSearchResult(CacheKey cacheKey, Execution execution) throws IOException { - return new Result(query); - } - - @Override - protected void release() { - } - } - public class TestQuery extends Query { private long start = clock.millis(); diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/MockInvoker.java b/container-search/src/test/java/com/yahoo/search/dispatch/MockInvoker.java new file mode 100644 index 00000000000..92cc79573c8 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/dispatch/MockInvoker.java @@ -0,0 +1,45 @@ +// Copyright 2019 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.dispatch; + +import com.yahoo.fs4.QueryPacket; +import com.yahoo.prelude.fastsearch.CacheKey; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.dispatch.searchcluster.Node; +import com.yahoo.search.result.Coverage; +import com.yahoo.search.searchchain.Execution; + +import java.io.IOException; +import java.util.Optional; + +class MockInvoker extends SearchInvoker { + private final Coverage coverage; + private Query query; + + protected MockInvoker(int key, Coverage coverage) { + super(Optional.of(new Node(key, "?", 0, 0))); + this.coverage = coverage; + } + + protected MockInvoker(int key) { + this(key, null); + } + + @Override + protected void sendSearchRequest(Query query, QueryPacket queryPacket) throws IOException { + this.query = query; + } + + @Override + protected Result getSearchResult(CacheKey cacheKey, Execution execution) throws IOException { + Result ret = new Result(query); + if (coverage != null) { + ret.setCoverage(coverage); + } + return ret; + } + + @Override + protected void release() { + } +}
\ No newline at end of file diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java b/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java index f7b92419b52..2f970a9c007 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java @@ -1,6 +1,7 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.dispatch; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableMultimap; import com.yahoo.search.dispatch.searchcluster.Group; @@ -18,6 +19,7 @@ import java.util.Optional; public class MockSearchCluster extends SearchCluster { private final int numGroups; private final int numNodesPerGroup; + private final ImmutableList<Group> orderedGroups; private final ImmutableMap<Integer, Group> groups; private final ImmutableMultimap<String, Node> nodesByHost; @@ -28,6 +30,7 @@ public class MockSearchCluster extends SearchCluster { public MockSearchCluster(String clusterId, DispatchConfig dispatchConfig, int groups, int nodesPerGroup) { super(clusterId, dispatchConfig, null, 1, null); + ImmutableList.Builder<Group> orderedGroupBuilder = ImmutableList.builder(); ImmutableMap.Builder<Integer, Group> groupBuilder = ImmutableMap.builder(); ImmutableMultimap.Builder<String, Node> hostBuilder = ImmutableMultimap.builder(); int dk = 1; @@ -40,8 +43,11 @@ public class MockSearchCluster extends SearchCluster { hostBuilder.put(n.hostname(), n); dk++; } - groupBuilder.put(group, new Group(group, nodes)); + Group g = new Group(group, nodes); + groupBuilder.put(group, g); + orderedGroupBuilder.add(g); } + this.orderedGroups = orderedGroupBuilder.build(); this.groups = groupBuilder.build(); this.nodesByHost = hostBuilder.build(); this.numGroups = groups; @@ -49,18 +55,26 @@ public class MockSearchCluster extends SearchCluster { } @Override + public ImmutableList<Group> orderedGroups() { + return orderedGroups; + } + + @Override public int size() { return numGroups * numNodesPerGroup; } + @Override public ImmutableMap<Integer, Group> groups() { return groups; } + @Override public int groupSize() { return numNodesPerGroup; } + @Override public Optional<Group> group(int n) { if (n < numGroups) { return Optional.of(groups.get(n)); @@ -69,18 +83,22 @@ public class MockSearchCluster extends SearchCluster { } } + @Override public ImmutableMultimap<String, Node> nodesByHost() { return nodesByHost; } + @Override public Optional<Node> directDispatchTarget() { return Optional.empty(); } + @Override public void working(Node node) { node.setWorking(true); } + @Override public void failed(Node node) { node.setWorking(false); } @@ -95,7 +113,7 @@ public class MockSearchCluster extends SearchCluster { builder.minGroupCoverage(99.0); builder.maxNodesDownPerGroup(0); builder.minSearchCoverage(minSearchCoverage); - if(minSearchCoverage < 100.0) { + if (minSearchCoverage < 100.0) { builder.minWaitAfterCoverageFactor(0); builder.maxWaitAfterCoverageFactor(0.5); } |