Use median not average document count to determine group coverage

If a group has too many nodes, all others will have less than average.

8 files changed, 156 insertions, 8 deletions

diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java
index eaafb2d8b8a..943390cb10c 100644
--- a/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java
+++ b/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java
@@ -6,6 +6,7 @@ import com.yahoo.prelude.fastsearch.test.MockMetric;
 import com.yahoo.search.Query;
 import com.yahoo.search.Result;
 import com.yahoo.search.cluster.ClusterMonitor;
+import com.yahoo.search.dispatch.searchcluster.Group;
 import com.yahoo.search.dispatch.searchcluster.Node;
 import com.yahoo.search.dispatch.searchcluster.PingFactory;
 import com.yahoo.search.dispatch.searchcluster.Pinger;
@@ -166,7 +167,7 @@ public class DispatcherTest {
             boolean nonEmpty = events[step].returnInvoker(nodes, acceptIncompleteCoverage);
             step++;
             if (nonEmpty) {
-                return Optional.of(new MockInvoker(nodes.get(0).key()));
+                return Optional.of(new MockInvoker(nodes.get(0).key(), groupId));
             } else {
                 return Optional.empty();
             }
diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/InterleavedSearchInvokerTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/InterleavedSearchInvokerTest.java
index 8cab7884152..730aa0800e7 100644
--- a/container-search/src/test/java/com/yahoo/search/dispatch/InterleavedSearchInvokerTest.java
+++ b/container-search/src/test/java/com/yahoo/search/dispatch/InterleavedSearchInvokerTest.java
@@ -44,10 +44,11 @@ import static org.junit.Assert.fail;
  * @author ollivir
  */
 public class InterleavedSearchInvokerTest {
-    private ManualClock clock = new ManualClock(Instant.now());
-    private Query query = new TestQuery();
-    private LinkedList<Event> expectedEvents = new LinkedList<>();
-    private List<SearchInvoker> invokers = new ArrayList<>();
+
+    private final ManualClock clock = new ManualClock(Instant.now());
+    private final Query query = new TestQuery();
+    private final LinkedList<Event> expectedEvents = new LinkedList<>();
+    private final List<SearchInvoker> invokers = new ArrayList<>();
 
     @Test
     public void requireThatAdaptiveTimeoutsAreNotUsedWithFullCoverageRequirement() throws IOException {
diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java
index 36b476e2936..d0ba396ef94 100644
--- a/container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java
+++ b/container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java
@@ -26,6 +26,7 @@ import static org.junit.Assert.assertThat;
  * @author ollivir
  */
 public class LoadBalancerTest {
+
     @Test
     public void requireThatLoadBalancerServesSingleNodeSetups() {
         Node n1 = new Node(0, "test-node1", 0);
diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/MockInvoker.java b/container-search/src/test/java/com/yahoo/search/dispatch/MockInvoker.java
index 459dcc83ab0..53d1a2457d0 100644
--- a/container-search/src/test/java/com/yahoo/search/dispatch/MockInvoker.java
+++ b/container-search/src/test/java/com/yahoo/search/dispatch/MockInvoker.java
@@ -12,20 +12,32 @@ import com.yahoo.search.searchchain.Execution;
 import java.io.IOException;
 import java.util.List;
 import java.util.Optional;
+import java.util.OptionalInt;
 
 class MockInvoker extends SearchInvoker {
+
     private final Coverage coverage;
+    private final OptionalInt groupId;
     private Query query;
     private List<Hit> hits;
     int hitsRequested;
 
-    protected MockInvoker(int key, Coverage coverage) {
+    protected MockInvoker(int key, Coverage coverage, OptionalInt groupId) {
         super(Optional.of(new Node(key, "?", 0)));
         this.coverage = coverage;
+        this.groupId = groupId;
+    }
+
+    protected MockInvoker(int key, OptionalInt groupId) {
+        this(key, null, groupId);
+    }
+
+    protected MockInvoker(int key, Coverage coverage) {
+        this(key, coverage, OptionalInt.empty());
     }
 
     protected MockInvoker(int key) {
-        this(key, null);
+        this(key, null, OptionalInt.empty());
     }
 
     MockInvoker setHits(List<Hit> hits) {
@@ -33,6 +45,9 @@ class MockInvoker extends SearchInvoker {
         return this;
     }
 
+    /** Returns the group to be invoked, if known */
+    public OptionalInt groupId() { return groupId; }
+
     @Override
     protected Object sendSearchRequest(Query query, Object context) throws IOException {
         this.query = query;
@@ -62,4 +77,11 @@ class MockInvoker extends SearchInvoker {
     @Override
     protected void release() {
     }
+
+    @Override
+    public String toString() {
+        return "invoker with key " + distributionKey() +
+               (groupId().isPresent() ? " of group " + groupId().getAsInt() : "");
+    }
+
 }
\ No newline at end of file
diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java b/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java
index fe43658ee75..9d96b2302d7 100644
--- a/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java
+++ b/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java
@@ -114,6 +114,7 @@ public class MockSearchCluster extends SearchCluster {
     public static DispatchConfig createDispatchConfig(double minSearchCoverage, Node... nodes) {
         return createDispatchConfig(minSearchCoverage, Arrays.asList(nodes));
     }
+
     public static DispatchConfig createDispatchConfig(double minSearchCoverage, List<Node> nodes) {
         DispatchConfig.Builder builder = new DispatchConfig.Builder();
         builder.minActivedocsPercentage(88.0);
diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterCoverageTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterCoverageTest.java
new file mode 100644
index 00000000000..6338107d4b6
--- /dev/null
+++ b/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterCoverageTest.java
@@ -0,0 +1,89 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.dispatch.searchcluster;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * @author bratseth
+ */
+public class SearchClusterCoverageTest {
+
+    @Test
+    public void two_groups_equal_docs() {
+        var tester =  new SearchClusterTester(2, 3);
+
+        tester.setDocsPerNode(100, 0);
+        tester.setDocsPerNode(100, 1);
+        tester.pingIterationCompleted();
+        assertTrue(tester.group(0).hasSufficientCoverage());
+        assertTrue(tester.group(1).hasSufficientCoverage());
+    }
+
+    @Test
+    public void two_groups_one_missing_docs() {
+        var tester =  new SearchClusterTester(2, 3);
+
+        tester.setDocsPerNode(100, 0);
+        tester.setDocsPerNode( 70, 1);
+        tester.pingIterationCompleted();
+        assertTrue(tester.group(0).hasSufficientCoverage());
+        assertFalse(tester.group(1).hasSufficientCoverage());
+    }
+
+    @Test
+    public void three_groups_one_missing_docs() {
+        var tester =  new SearchClusterTester(3, 3);
+
+        tester.setDocsPerNode(100, 0);
+        tester.setDocsPerNode( 87, 1);  // min is set to 88 in MockSearchCluster
+        tester.setDocsPerNode(100, 2);
+        tester.pingIterationCompleted();
+        assertTrue(tester.group(0).hasSufficientCoverage());
+        assertFalse(tester.group(1).hasSufficientCoverage());
+        assertTrue(tester.group(2).hasSufficientCoverage());
+    }
+
+    @Test
+    public void three_groups_one_missing_docs_but_too_few() {
+        var tester =  new SearchClusterTester(3, 3);
+
+        tester.setDocsPerNode(100, 0);
+        tester.setDocsPerNode( 89, 1);  // min is set to 88 in MockSearchCluster
+        tester.setDocsPerNode(100, 2);
+        tester.pingIterationCompleted();
+        assertTrue(tester.group(0).hasSufficientCoverage());
+        assertTrue(tester.group(1).hasSufficientCoverage());
+        assertTrue(tester.group(2).hasSufficientCoverage());
+    }
+
+    @Test
+    public void three_groups_one_has_too_many_docs() {
+        var tester =  new SearchClusterTester(3, 3);
+
+        tester.setDocsPerNode(100, 0);
+        tester.setDocsPerNode(150, 1);
+        tester.setDocsPerNode(100, 2);
+        tester.pingIterationCompleted();
+        assertTrue(tester.group(0).hasSufficientCoverage());
+        assertTrue(tester.group(1).hasSufficientCoverage());
+        assertTrue(tester.group(2).hasSufficientCoverage());
+    }
+
+    @Test
+    public void three_groups_one_has_a_node_down() {
+        var tester =  new SearchClusterTester(3, 3);
+
+        tester.setDocsPerNode(100, 0);
+        tester.setDocsPerNode(150, 1);
+        tester.setDocsPerNode(100, 2);
+        tester.setWorking(1, 1, false);
+        tester.pingIterationCompleted();
+        assertTrue(tester.group(0).hasSufficientCoverage());
+        assertFalse(tester.group(1).hasSufficientCoverage());
+        assertTrue(tester.group(2).hasSufficientCoverage());
+    }
+
+}
diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTest.java
index c6fd48836fe..48134094faf 100644
--- a/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTest.java
+++ b/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTest.java
@@ -28,7 +28,7 @@ import static org.junit.Assert.assertTrue;
  */
 public class SearchClusterTest {
 
-    static class State implements AutoCloseable{
+    static class State implements AutoCloseable {
 
         final String clusterId;
         final int nodesPerGroup;
diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTester.java b/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTester.java
new file mode 100644
index 00000000000..5e7ecb854ff
--- /dev/null
+++ b/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTester.java
@@ -0,0 +1,33 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.dispatch.searchcluster;
+
+import com.yahoo.search.dispatch.MockSearchCluster;
+
+public class SearchClusterTester {
+
+    private final SearchCluster cluster;
+
+    public SearchClusterTester(int groups, int nodesPerGroup) {
+        cluster = new MockSearchCluster("1", groups, nodesPerGroup);
+    }
+
+    public void pingIterationCompleted() {
+        cluster.pingIterationCompleted();
+    }
+
+    public Group group(int id) {
+        return cluster.group(id).get();
+    }
+
+    public void setWorking(int group, int node, boolean working) {
+        cluster.group(group).get().nodes().get(node).setWorking(working);
+    }
+
+    public void setDocsPerNode(int docs, int groupId) {
+        for (Node node : cluster.groups().get(groupId).nodes()) {
+            node.setWorking(true);
+            node.setActiveDocuments(docs);
+        }
+    }
+
+}