uselocalnode support and fix to coverage estimation

7 files changed, 70 insertions, 21 deletions

diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java
index f99dde237bd..80bba10276d 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java
@@ -422,6 +422,7 @@ public class IndexedSearchCluster extends SearchCluster
         }
         builder.maxNodesDownPerGroup(rootDispatch.getMaxNodesDownPerFixedRow());
         builder.useMultilevelDispatch(useMultilevelDispatchSetup());
+        builder.useLocalNode(tuning.dispatch.useLocalNode);
         builder.searchableCopies(rootDispatch.getSearchableCopies());
         if (searchCoverage != null) {
             if (searchCoverage.getMinimum() != null)
diff --git a/configdefinitions/src/vespa/dispatch.def b/configdefinitions/src/vespa/dispatch.def
index 50989c3ef74..1108e40b3f1 100644
--- a/configdefinitions/src/vespa/dispatch.def
+++ b/configdefinitions/src/vespa/dispatch.def
@@ -1,4 +1,4 @@
-# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+# Copyright 2019 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 # Configuration of dispatch from container nodes to search clusters
 
 namespace=vespa.config.search
@@ -19,6 +19,9 @@ distributionPolicy enum { ROUNDROBIN, ADAPTIVE } default=ROUNDROBIN
 # Is multi-level dispatch configured for this cluster
 useMultilevelDispatch bool default=false
 
+# Dispatch only to local nodes
+useLocalNode bool default=false
+
 # Number of document copies
 searchableCopies long default=1
 
diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java
index f67641041ad..b613952b99e 100644
--- a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java
+++ b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java
@@ -135,7 +135,9 @@ public class Dispatcher extends AbstractComponent {
             return invokerFactory.supply(query, -1, Arrays.asList(node), true);
         }
 
-        int max = Integer.min(searchCluster.orderedGroups().size(), MAX_GROUP_SELECTION_ATTEMPTS);
+        int covered = searchCluster.groupsWithSufficientCoverage();
+        int groups = searchCluster.orderedGroups().size();
+        int max = Integer.min(Integer.min(covered + 1, groups), MAX_GROUP_SELECTION_ATTEMPTS);
         Set<Integer> rejected = null;
         for (int i = 0; i < max; i++) {
             Optional<Group> groupInCluster = loadBalancer.takeGroup(rejected);
diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/LoadBalancer.java b/container-search/src/main/java/com/yahoo/search/dispatch/LoadBalancer.java
index 3c7af447809..6a59f28ca4c 100644
--- a/container-search/src/main/java/com/yahoo/search/dispatch/LoadBalancer.java
+++ b/container-search/src/main/java/com/yahoo/search/dispatch/LoadBalancer.java
@@ -35,7 +35,7 @@ public class LoadBalancer {
         for (Group group : searchCluster.orderedGroups()) {
             scoreboard.add(new GroupStatus(group));
         }
-        if (roundRobin) {
+        if (roundRobin || scoreboard.size() == 1) {
             this.scheduler = new RoundRobinScheduler(scoreboard);
         } else {
             this.scheduler = new AdaptiveScheduler(new Random(), scoreboard);
diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Group.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Group.java
index 01cbc5cd307..146c62f0a16 100644
--- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Group.java
+++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Group.java
@@ -44,6 +44,16 @@ public class Group {
         hasSufficientCoverage.lazySet(sufficientCoverage);
     }
 
+    public int workingNodes() {
+        int nodesUp = 0;
+        for (Node node : nodes) {
+            if (node.isWorking()) {
+                nodesUp++;
+            }
+        }
+        return nodesUp;
+    }
+
     void aggregateActiveDocuments() {
         long activeDocumentsInGroup = 0;
         for (Node node : nodes) {
diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java
index 8e278f78d7a..e497ef6751b 100644
--- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java
+++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java
@@ -23,6 +23,7 @@ import java.util.concurrent.Executor;
 import java.util.concurrent.FutureTask;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
+import java.util.function.Predicate;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 import java.util.stream.Collectors;
@@ -102,7 +103,7 @@ public class SearchCluster implements NodeManager<Node> {
     private static Optional<Node> findDirectDispatchTarget(String selfHostname,
                                                            int searchClusterSize,
                                                            int containerClusterSize,
-                                                           ImmutableMultimap<String, Node>nodesByHost,
+                                                           ImmutableMultimap<String, Node> nodesByHost,
                                                            ImmutableMap<Integer, Group> groups) {
         // A search node in the search cluster in question is configured on the same host as the currently running container.
         // It has all the data <==> No other nodes in the search cluster have the same group id as this node.
@@ -129,8 +130,18 @@ public class SearchCluster implements NodeManager<Node> {
 
     private static ImmutableList<Node> toNodes(DispatchConfig dispatchConfig) {
         ImmutableList.Builder<Node> nodesBuilder = new ImmutableList.Builder<>();
-        for (DispatchConfig.Node node : dispatchConfig.node())
-            nodesBuilder.add(new Node(node.key(), node.host(), node.fs4port(), node.group()));
+        Predicate<DispatchConfig.Node> filter;
+        if (dispatchConfig.useLocalNode()) {
+            final String hostName = HostName.getLocalhost();
+            filter = node -> node.host().equals(hostName);
+        } else {
+            filter = node -> true;
+        }
+        for (DispatchConfig.Node node : dispatchConfig.node()) {
+            if (filter.test(node)) {
+                nodesBuilder.add(new Node(node.key(), node.host(), node.fs4port(), node.group()));
+            }
+        }
         return nodesBuilder.build();
     }
 
@@ -162,6 +173,16 @@ public class SearchCluster implements NodeManager<Node> {
         return size() / groups.size();
     }
 
+    public int groupsWithSufficientCoverage() {
+        int covered = 0;
+        for (Group g : orderedGroups) {
+            if (g.hasSufficientCoverage()) {
+                covered++;
+            }
+        }
+        return covered;
+    }
+
     /**
      * Returns the nodes of this cluster as an immutable map indexed by host.
      * One host may contain multiple nodes (on different ports), so this is a multi-map.
@@ -271,12 +292,13 @@ public class SearchCluster implements NodeManager<Node> {
             Group group = orderedGroups.get(i);
             long activeDocuments = activeDocumentsInGroup[i];
             long averageDocumentsInOtherGroups = (sumOfActiveDocuments - activeDocuments) / (numGroups - 1);
-            boolean sufficientCoverage = isGroupCoverageSufficient(group.nodes(), activeDocuments, averageDocumentsInOtherGroups);
+            boolean sufficientCoverage = isGroupCoverageSufficient(group.workingNodes(), group.nodes().size(), activeDocuments,
+                    averageDocumentsInOtherGroups);
             updateSufficientCoverage(group, sufficientCoverage);
         }
     }
 
-    private boolean isGroupCoverageSufficient(List<Node> nodes, long activeDocuments, long averageDocumentsInOtherGroups) {
+    private boolean isGroupCoverageSufficient(int workingNodes, int nodesInGroup, long activeDocuments, long averageDocumentsInOtherGroups) {
         boolean sufficientCoverage = true;
 
         if (averageDocumentsInOtherGroups > 0) {
@@ -284,22 +306,15 @@ public class SearchCluster implements NodeManager<Node> {
             sufficientCoverage = coverage >= dispatchConfig.minActivedocsPercentage();
         }
         if (sufficientCoverage) {
-            sufficientCoverage = isGroupNodeCoverageSufficient(nodes);
+            sufficientCoverage = isGroupNodeCoverageSufficient(workingNodes, nodesInGroup);
         }
         return sufficientCoverage;
     }
 
-    private boolean isGroupNodeCoverageSufficient(List<Node> nodes) {
-        int nodesUp = 0;
-        for (Node node : nodes) {
-            if (node.isWorking()) {
-                nodesUp++;
-            }
-        }
-        int numNodes = nodes.size();
+    private boolean isGroupNodeCoverageSufficient(int workingNodes, int nodesInGroup) {
         int nodesAllowedDown = dispatchConfig.maxNodesDownPerGroup()
-                + (int) (((double) numNodes * (100.0 - dispatchConfig.minGroupCoverage())) / 100.0);
-        return nodesUp + nodesAllowedDown >= numNodes;
+                + (int) (((double) nodesInGroup * (100.0 - dispatchConfig.minGroupCoverage())) / 100.0);
+        return workingNodes + nodesAllowedDown >= nodesInGroup;
     }
 
     private Pong getPong(FutureTask<Pong> futurePong, Node node) {
@@ -316,13 +331,24 @@ public class SearchCluster implements NodeManager<Node> {
         }
     }
 
+    private void logIfInsufficientCoverage(boolean sufficient, int groupId, int nodes) {
+        if (!sufficient) {
+            log.warning(() -> String.format("Coverage of group %s is only %d/%d (requires %d)", groupId, nodes, groupSize(),
+                    groupSize() - dispatchConfig.maxNodesDownPerGroup()));
+        }
+    }
+
     /**
      * Calculate whether a subset of nodes in a group has enough coverage
      */
     public boolean isPartialGroupCoverageSufficient(int groupId, List<Node> nodes) {
         if (orderedGroups.size() == 1) {
-            return nodes.size() >= groupSize() - dispatchConfig.maxNodesDownPerGroup();
+            boolean sufficient = nodes.size() >= groupSize() - dispatchConfig.maxNodesDownPerGroup();
+            logIfInsufficientCoverage(sufficient, groupId, nodes.size());
+            return sufficient;
         }
+
+        int nodesInGroup = groups.get(groupId).nodes().size();
         long sumOfActiveDocuments = 0;
         int otherGroups = 0;
         for (Group g : orderedGroups) {
@@ -336,6 +362,8 @@ public class SearchCluster implements NodeManager<Node> {
             activeDocuments += n.getActiveDocuments();
         }
         long averageDocumentsInOtherGroups = sumOfActiveDocuments / otherGroups;
-        return isGroupCoverageSufficient(nodes, activeDocuments, averageDocumentsInOtherGroups);
+        boolean sufficient = isGroupCoverageSufficient(nodes.size(), nodesInGroup, activeDocuments, averageDocumentsInOtherGroups);
+        logIfInsufficientCoverage(sufficient, groupId, nodes.size());
+        return sufficient;
     }
 }
diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java b/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java
index 2f970a9c007..3919bc26bdc 100644
--- a/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java
+++ b/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java
@@ -75,6 +75,11 @@ public class MockSearchCluster extends SearchCluster {
     }
 
     @Override
+    public int groupsWithSufficientCoverage() {
+        return numGroups;
+    }
+
+    @Override
     public Optional<Group> group(int n) {
         if (n < numGroups) {
             return Optional.of(groups.get(n));