Increased java dispatcher test coverage

4 files changed, 328 insertions, 28 deletions

diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java
new file mode 100644
index 00000000000..fae659b0d70
--- /dev/null
+++ b/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java
@@ -0,0 +1,132 @@
+// Copyright 2019 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.dispatch;
+
+import com.yahoo.prelude.fastsearch.FS4InvokerFactory;
+import com.yahoo.processing.request.CompoundName;
+import com.yahoo.search.Query;
+import com.yahoo.search.dispatch.searchcluster.Node;
+import com.yahoo.search.dispatch.searchcluster.SearchCluster;
+import com.yahoo.vespa.config.search.DispatchConfig;
+import org.junit.Test;
+
+import java.util.List;
+import java.util.Optional;
+
+import static com.yahoo.search.dispatch.MockSearchCluster.createDispatchConfig;
+import static org.hamcrest.Matchers.is;
+import static org.junit.Assert.assertThat;
+
+/**
+ * @author ollivir
+ */
+public class DispatcherTest {
+    private static final CompoundName internalDispatch = new CompoundName("dispatch.internal");
+
+    private static Query query() {
+        Query q = new Query();
+        q.properties().set(internalDispatch, "true");
+        return q;
+    }
+
+    @Test
+    public void requireDispatcherToIgnoreMultilevelConfigurations() {
+        SearchCluster cl = new MockSearchCluster("1", 2, 2);
+        DispatchConfig.Builder builder = new DispatchConfig.Builder();
+        builder.useMultilevelDispatch(true);
+        DispatchConfig dc = new DispatchConfig(builder);
+
+        Dispatcher disp = new Dispatcher(cl, dc);
+        assertThat(disp.getSearchInvoker(query(), null).isPresent(), is(false));
+    }
+
+    @Test
+    public void requireThatDispatcherSupportsSearchPath() {
+        SearchCluster cl = new MockSearchCluster("1", 2, 2);
+        Query q = query();
+        q.getModel().setSearchPath("1/0"); // second node in first group
+        Dispatcher disp = new Dispatcher(cl, createDispatchConfig());
+        MockFS4InvokerFactory invokerFactory = new MockFS4InvokerFactory(cl, (nodes, a) -> {
+            assertThat(nodes.size(), is(1));
+            assertThat(nodes.get(0).key(), is(2));
+            return true;
+        });
+        Optional<SearchInvoker> invoker = disp.getSearchInvoker(q, invokerFactory);
+        assertThat(invoker.isPresent(), is(true));
+        invokerFactory.verifyAllEventsProcessed();
+    }
+
+    @Test
+    public void requireThatDispatcherSupportsSingleNodeDirectDispatch() {
+        SearchCluster cl = new MockSearchCluster("1", 0, 0) {
+            @Override
+            public Optional<Node> directDispatchTarget() {
+                return Optional.of(new Node(1, "test", 123, 1));
+            }
+        };
+        Dispatcher disp = new Dispatcher(cl, createDispatchConfig());
+        MockFS4InvokerFactory invokerFactory = new MockFS4InvokerFactory(cl, (n, a) -> true);
+        Optional<SearchInvoker> invoker = disp.getSearchInvoker(query(), invokerFactory);
+        assertThat(invoker.isPresent(), is(true));
+        invokerFactory.verifyAllEventsProcessed();
+    }
+
+    @Test
+    public void requireThatInvokerConstructionIsRetriedAndLastAcceptsAnyCoverage() {
+        SearchCluster cl = new MockSearchCluster("1", 2, 1);
+
+        Dispatcher disp = new Dispatcher(cl, createDispatchConfig());
+        MockFS4InvokerFactory invokerFactory = new MockFS4InvokerFactory(cl, (n, acceptIncompleteCoverage) -> {
+            assertThat(acceptIncompleteCoverage, is(false));
+            return false;
+        }, (n, acceptIncompleteCoverage) -> {
+            assertThat(acceptIncompleteCoverage, is(true));
+            return true;
+        });
+        Optional<SearchInvoker> invoker = disp.getSearchInvoker(query(), invokerFactory);
+        assertThat(invoker.isPresent(), is(true));
+        invokerFactory.verifyAllEventsProcessed();
+    }
+
+    @Test
+    public void requireThatInvokerConstructionDoesNotRepeatGroups() {
+        SearchCluster cl = new MockSearchCluster("1", 2, 1);
+
+        Dispatcher disp = new Dispatcher(cl, createDispatchConfig());
+        MockFS4InvokerFactory invokerFactory = new MockFS4InvokerFactory(cl, (n, a) -> false, (n, a) -> false);
+        Optional<SearchInvoker> invoker = disp.getSearchInvoker(query(), invokerFactory);
+        assertThat(invoker.isPresent(), is(false));
+        invokerFactory.verifyAllEventsProcessed();
+    }
+
+    interface FactoryStep {
+        public boolean returnInvoker(List<Node> nodes, boolean acceptIncompleteCoverage);
+    }
+
+    private static class MockFS4InvokerFactory extends FS4InvokerFactory {
+        private final FactoryStep[] events;
+        private int step = 0;
+
+        public MockFS4InvokerFactory(SearchCluster cl, FactoryStep... events) {
+            super(null, cl, null);
+            this.events = events;
+        }
+
+        @Override
+        public Optional<SearchInvoker> getSearchInvoker(Query query, int groupId, List<Node> nodes, boolean acceptIncompleteCoverage) {
+            if (step >= events.length) {
+                throw new RuntimeException("Was not expecting more calls to getSearchInvoker");
+            }
+            boolean nonEmpty = events[step].returnInvoker(nodes, acceptIncompleteCoverage);
+            step++;
+            if (nonEmpty) {
+                return Optional.of(new MockInvoker(1));
+            } else {
+                return Optional.empty();
+            }
+        }
+
+        void verifyAllEventsProcessed() {
+            assertThat(step, is(events.length));
+        }
+    }
+}
diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/InterleavedSearchInvokerTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/InterleavedSearchInvokerTest.java
index 99694f22254..a4270ea0ae4 100644
--- a/container-search/src/test/java/com/yahoo/search/dispatch/InterleavedSearchInvokerTest.java
+++ b/container-search/src/test/java/com/yahoo/search/dispatch/InterleavedSearchInvokerTest.java
@@ -1,13 +1,11 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.search.dispatch;
 
-import com.yahoo.fs4.QueryPacket;
-import com.yahoo.prelude.fastsearch.CacheKey;
 import com.yahoo.search.Query;
 import com.yahoo.search.Result;
-import com.yahoo.search.dispatch.searchcluster.Node;
 import com.yahoo.search.dispatch.searchcluster.SearchCluster;
-import com.yahoo.search.searchchain.Execution;
+import com.yahoo.search.result.Coverage;
+import com.yahoo.search.result.ErrorMessage;
 import com.yahoo.test.ManualClock;
 import org.junit.Test;
 
@@ -17,11 +15,14 @@ import java.time.Instant;
 import java.util.ArrayList;
 import java.util.LinkedList;
 import java.util.List;
-import java.util.Optional;
 import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.TimeUnit;
 
+import static com.yahoo.container.handler.Coverage.DEGRADED_BY_MATCH_PHASE;
+import static com.yahoo.container.handler.Coverage.DEGRADED_BY_TIMEOUT;
 import static com.yahoo.search.dispatch.MockSearchCluster.createDispatchConfig;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.is;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotNull;
@@ -84,9 +85,122 @@ public class InterleavedSearchInvokerTest {
         assertTrue("Degradataion reason is an adaptive timeout", result.getCoverage(false).isDegradedByAdapativeTimeout());
     }
 
+    @Test
+    public void requireCorrectCoverageCalculationWhenAllNodesOk() throws IOException {
+        SearchCluster cluster = new MockSearchCluster("!", 1, 2);
+        invokers.add(new MockInvoker(0, createCoverage(50155, 50155, 50155, 1, 1, 0)));
+        invokers.add(new MockInvoker(1, createCoverage(49845, 49845, 49845, 1, 1, 0)));
+        SearchInvoker invoker = createInterleavedInvoker(cluster, 0);
+
+        expectedEvents.add(new Event(null, 100, 0));
+        expectedEvents.add(new Event(null, 200, 1));
+
+        Result result = invoker.search(query, null, null, null);
+
+        Coverage cov = result.getCoverage(true);
+        assertThat(cov.getDocs(), is(100000L));
+        assertThat(cov.getNodes(), is(2));
+        assertThat(cov.getFull(), is(true));
+        assertThat(cov.getResultPercentage(), is(100));
+        assertThat(cov.getResultSets(), is(1));
+        assertThat(cov.getFullResultSets(), is(1));
+    }
+
+    @Test
+    public void requireCorrectCoverageCalculationWhenResultsAreLimitedByMatchPhase() throws IOException {
+        SearchCluster cluster = new MockSearchCluster("!", 1, 2);
+        invokers.add(new MockInvoker(0, createCoverage(10101, 50155, 50155, 1, 1, DEGRADED_BY_MATCH_PHASE)));
+        invokers.add(new MockInvoker(1, createCoverage(13319, 49845, 49845, 1, 1, DEGRADED_BY_MATCH_PHASE)));
+        SearchInvoker invoker = createInterleavedInvoker(cluster, 0);
+
+        expectedEvents.add(new Event(null, 100, 0));
+        expectedEvents.add(new Event(null, 200, 1));
+
+        Result result = invoker.search(query, null, null, null);
+
+        Coverage cov = result.getCoverage(true);
+        assertThat(cov.getDocs(), is(23420L));
+        assertThat(cov.getNodes(), is(2));
+        assertThat(cov.getFull(), is(false));
+        assertThat(cov.getResultPercentage(), is(23));
+        assertThat(cov.getResultSets(), is(1));
+        assertThat(cov.getFullResultSets(), is(0));
+        assertThat(cov.isDegradedByMatchPhase(), is(true));
+    }
+
+    @Test
+    public void requireCorrectCoverageCalculationWhenResultsAreLimitedBySoftTimeout() throws IOException {
+        SearchCluster cluster = new MockSearchCluster("!", 1, 2);
+        invokers.add(new MockInvoker(0, createCoverage(5000, 50155, 50155, 1, 1, DEGRADED_BY_TIMEOUT)));
+        invokers.add(new MockInvoker(1, createCoverage(4900, 49845, 49845, 1, 1, DEGRADED_BY_TIMEOUT)));
+        SearchInvoker invoker = createInterleavedInvoker(cluster, 0);
+
+        expectedEvents.add(new Event(null, 100, 0));
+        expectedEvents.add(new Event(null, 200, 1));
+
+        Result result = invoker.search(query, null, null, null);
+
+        Coverage cov = result.getCoverage(true);
+        assertThat(cov.getDocs(), is(9900L));
+        assertThat(cov.getNodes(), is(2));
+        assertThat(cov.getFull(), is(false));
+        assertThat(cov.getResultPercentage(), is(10));
+        assertThat(cov.getResultSets(), is(1));
+        assertThat(cov.getFullResultSets(), is(0));
+        assertThat(cov.isDegradedByTimeout(), is(true));
+    }
+
+    @Test
+    public void requireCorrectCoverageCalculationWhenOneNodeIsUnexpectedlyDown() throws IOException {
+        SearchCluster cluster = new MockSearchCluster("!", 1, 2);
+        invokers.add(new MockInvoker(0, createCoverage(50155, 50155, 50155, 1, 1, 0)));
+        invokers.add(new MockInvoker(1, createCoverage(49845, 49845, 49845, 1, 1, 0)));
+        SearchInvoker invoker = createInterleavedInvoker(cluster, 0);
+
+        expectedEvents.add(new Event(null, 100, 0));
+        expectedEvents.add(null);
+
+        Result result = invoker.search(query, null, null, null);
+
+        Coverage cov = result.getCoverage(true);
+        assertThat(cov.getDocs(), is(50155L));
+        assertThat(cov.getNodes(), is(1));
+        assertThat(cov.getNodesTried(), is(2));
+        assertThat(cov.getFull(), is(false));
+        assertThat(cov.getResultPercentage(), is(50));
+        assertThat(cov.getResultSets(), is(1));
+        assertThat(cov.getFullResultSets(), is(0));
+        assertThat(cov.isDegradedByTimeout(), is(true));
+    }
+
+    @Test
+    public void requireCorrectCoverageCalculationWhenDegradedCoverageIsExpected() throws IOException {
+        SearchCluster cluster = new MockSearchCluster("!", 1, 2);
+        invokers.add(new MockInvoker(0, createCoverage(50155, 50155, 50155, 1, 1, 0)));
+        Coverage errorCoverage = new Coverage(0, 0, 0);
+        errorCoverage.setNodesTried(1);
+        invokers.add(new SearchErrorInvoker(ErrorMessage.createBackendCommunicationError("node is down"), errorCoverage));
+        SearchInvoker invoker = createInterleavedInvoker(cluster, 0);
+
+        expectedEvents.add(new Event(null,   1, 1));
+        expectedEvents.add(new Event(null, 100, 0));
+
+        Result result = invoker.search(query, null, null, null);
+
+        Coverage cov = result.getCoverage(true);
+        assertThat(cov.getDocs(), is(50155L));
+        assertThat(cov.getNodes(), is(1));
+        assertThat(cov.getNodesTried(), is(2));
+        assertThat(cov.getFull(), is(false));
+        assertThat(cov.getResultPercentage(), is(50));
+        assertThat(cov.getResultSets(), is(1));
+        assertThat(cov.getFullResultSets(), is(0));
+        assertThat(cov.isDegradedByTimeout(), is(true));
+    }
+
     private InterleavedSearchInvoker createInterleavedInvoker(SearchCluster searchCluster, int numInvokers) {
         for (int i = 0; i < numInvokers; i++) {
-            invokers.add(new TestInvoker());
+            invokers.add(new MockInvoker(i));
         }
 
         return new InterleavedSearchInvoker(invokers, null, searchCluster) {
@@ -113,13 +227,23 @@ public class InterleavedSearchInvokerTest {
         };
     }
 
+    private static Coverage createCoverage(int docs, int activeDocs, int soonActiveDocs, int nodes, int nodesTried, int degradedReason) {
+        Coverage coverage = new Coverage(docs, activeDocs, nodes);
+        coverage.setSoonActive(soonActiveDocs);
+        coverage.setNodesTried(nodesTried);
+        coverage.setDegradedReason(degradedReason);
+        return coverage;
+    }
+
     private class Event {
         Long expectedTimeout;
         long delay;
         Integer invokerIndex;
 
         public Event(Integer expectedTimeout, int delay, Integer invokerIndex) {
-            this.expectedTimeout = (long) expectedTimeout;
+            if (expectedTimeout != null) {
+                this.expectedTimeout = (long) expectedTimeout;
+            }
             this.delay = delay;
             this.invokerIndex = invokerIndex;
         }
@@ -140,25 +264,6 @@ public class InterleavedSearchInvokerTest {
         }
     }
 
-    private class TestInvoker extends SearchInvoker {
-        protected TestInvoker() {
-            super(Optional.of(new Node(42, "?", 0, 0)));
-        }
-
-        @Override
-        protected void sendSearchRequest(Query query, QueryPacket queryPacket) throws IOException {
-        }
-
-        @Override
-        protected Result getSearchResult(CacheKey cacheKey, Execution execution) throws IOException {
-            return new Result(query);
-        }
-
-        @Override
-        protected void release() {
-        }
-    }
-
     public class TestQuery extends Query {
         private long start = clock.millis();
 
diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/MockInvoker.java b/container-search/src/test/java/com/yahoo/search/dispatch/MockInvoker.java
new file mode 100644
index 00000000000..92cc79573c8
--- /dev/null
+++ b/container-search/src/test/java/com/yahoo/search/dispatch/MockInvoker.java
@@ -0,0 +1,45 @@
+// Copyright 2019 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.dispatch;
+
+import com.yahoo.fs4.QueryPacket;
+import com.yahoo.prelude.fastsearch.CacheKey;
+import com.yahoo.search.Query;
+import com.yahoo.search.Result;
+import com.yahoo.search.dispatch.searchcluster.Node;
+import com.yahoo.search.result.Coverage;
+import com.yahoo.search.searchchain.Execution;
+
+import java.io.IOException;
+import java.util.Optional;
+
+class MockInvoker extends SearchInvoker {
+    private final Coverage coverage;
+    private Query query;
+
+    protected MockInvoker(int key, Coverage coverage) {
+        super(Optional.of(new Node(key, "?", 0, 0)));
+        this.coverage = coverage;
+    }
+
+    protected MockInvoker(int key) {
+        this(key, null);
+    }
+
+    @Override
+    protected void sendSearchRequest(Query query, QueryPacket queryPacket) throws IOException {
+        this.query = query;
+    }
+
+    @Override
+    protected Result getSearchResult(CacheKey cacheKey, Execution execution) throws IOException {
+        Result ret = new Result(query);
+        if (coverage != null) {
+            ret.setCoverage(coverage);
+        }
+        return ret;
+    }
+
+    @Override
+    protected void release() {
+    }
+}
\ No newline at end of file
diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java b/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java
index f7b92419b52..2f970a9c007 100644
--- a/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java
+++ b/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java
@@ -1,6 +1,7 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.search.dispatch;
 
+import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableMultimap;
 import com.yahoo.search.dispatch.searchcluster.Group;
@@ -18,6 +19,7 @@ import java.util.Optional;
 public class MockSearchCluster extends SearchCluster {
     private final int numGroups;
     private final int numNodesPerGroup;
+    private final ImmutableList<Group> orderedGroups;
     private final ImmutableMap<Integer, Group> groups;
     private final ImmutableMultimap<String, Node> nodesByHost;
 
@@ -28,6 +30,7 @@ public class MockSearchCluster extends SearchCluster {
     public MockSearchCluster(String clusterId, DispatchConfig dispatchConfig, int groups, int nodesPerGroup) {
         super(clusterId, dispatchConfig, null, 1, null);
 
+        ImmutableList.Builder<Group> orderedGroupBuilder = ImmutableList.builder();
         ImmutableMap.Builder<Integer, Group> groupBuilder = ImmutableMap.builder();
         ImmutableMultimap.Builder<String, Node> hostBuilder = ImmutableMultimap.builder();
         int dk = 1;
@@ -40,8 +43,11 @@ public class MockSearchCluster extends SearchCluster {
                 hostBuilder.put(n.hostname(), n);
                 dk++;
             }
-            groupBuilder.put(group, new Group(group, nodes));
+            Group g = new Group(group, nodes);
+            groupBuilder.put(group, g);
+            orderedGroupBuilder.add(g);
         }
+        this.orderedGroups = orderedGroupBuilder.build();
         this.groups = groupBuilder.build();
         this.nodesByHost = hostBuilder.build();
         this.numGroups = groups;
@@ -49,18 +55,26 @@ public class MockSearchCluster extends SearchCluster {
     }
 
     @Override
+    public ImmutableList<Group> orderedGroups() {
+        return orderedGroups;
+    }
+
+    @Override
     public int size() {
         return numGroups * numNodesPerGroup;
     }
 
+    @Override
     public ImmutableMap<Integer, Group> groups() {
         return groups;
     }
 
+    @Override
     public int groupSize() {
         return numNodesPerGroup;
     }
 
+    @Override
     public Optional<Group> group(int n) {
         if (n < numGroups) {
             return Optional.of(groups.get(n));
@@ -69,18 +83,22 @@ public class MockSearchCluster extends SearchCluster {
         }
     }
 
+    @Override
     public ImmutableMultimap<String, Node> nodesByHost() {
         return nodesByHost;
     }
 
+    @Override
     public Optional<Node> directDispatchTarget() {
         return Optional.empty();
     }
 
+    @Override
     public void working(Node node) {
         node.setWorking(true);
     }
 
+    @Override
     public void failed(Node node) {
         node.setWorking(false);
     }
@@ -95,7 +113,7 @@ public class MockSearchCluster extends SearchCluster {
         builder.minGroupCoverage(99.0);
         builder.maxNodesDownPerGroup(0);
         builder.minSearchCoverage(minSearchCoverage);
-        if(minSearchCoverage < 100.0) {
+        if (minSearchCoverage < 100.0) {
             builder.minWaitAfterCoverageFactor(0);
             builder.maxWaitAfterCoverageFactor(0.5);
         }