Pull more configuration parameters from config model to internal dispatcher config

7 files changed, 114 insertions, 49 deletions

diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java
index 985b6e1e4b0..b8ec7e88354 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java
@@ -3,16 +3,17 @@ package com.yahoo.vespa.model.search;
 
 import com.yahoo.config.application.api.DeployLogger;
 import com.yahoo.config.model.deploy.DeployState;
-import com.yahoo.log.LogLevel;
-import com.yahoo.vespa.config.search.AttributesConfig;
-import com.yahoo.vespa.config.search.DispatchConfig;
-import com.yahoo.vespa.config.search.core.ProtonConfig;
-import com.yahoo.vespa.config.search.RankProfilesConfig;
 import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.log.LogLevel;
 import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig;
 import com.yahoo.search.config.IndexInfoConfig;
 import com.yahoo.searchdefinition.DocumentOnlySearch;
 import com.yahoo.searchdefinition.derived.DerivedConfiguration;
+import com.yahoo.vespa.config.search.AttributesConfig;
+import com.yahoo.vespa.config.search.DispatchConfig;
+import com.yahoo.vespa.config.search.DispatchConfig.DistributionPolicy;
+import com.yahoo.vespa.config.search.RankProfilesConfig;
+import com.yahoo.vespa.config.search.core.ProtonConfig;
 import com.yahoo.vespa.configdefinition.IlscriptsConfig;
 import com.yahoo.vespa.model.HostResource;
 import com.yahoo.vespa.model.SimpleConfigProducer;
@@ -23,8 +24,11 @@ import com.yahoo.vespa.model.content.DispatchSpec;
 import com.yahoo.vespa.model.content.SearchCoverage;
 
 import java.io.File;
-import java.io.IOException;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
 import java.util.logging.Logger;
 
 /**
@@ -316,17 +320,17 @@ public class IndexedSearchCluster extends SearchCluster
 
     @Override
     public DerivedConfiguration getSdConfig() { return null; }
-    
+
     @Override
     public void getConfig(IndexInfoConfig.Builder builder) {
         unionCfg.getConfig(builder);
     }
-    
+
     @Override
     public void getConfig(IlscriptsConfig.Builder builder) {
         unionCfg.getConfig(builder);
     }
-    
+
     @Override
     public void getConfig(AttributesConfig.Builder builder) {
         unionCfg.getConfig(builder);
@@ -402,6 +406,19 @@ public class IndexedSearchCluster extends SearchCluster
             nodeBuilder.fs4port(node.getDispatchPort());
             if (tuning.dispatch.minActiveDocsCoverage != null)
                 builder.minActivedocsPercentage(tuning.dispatch.minActiveDocsCoverage);
+            if (tuning.dispatch.minGroupCoverage != null)
+                builder.minGroupCoverage(tuning.dispatch.minGroupCoverage);
+            if (tuning.dispatch.policy != null) {
+                switch(tuning.dispatch.policy) {
+                case RANDOM:
+                    builder.distributionPolicy(DistributionPolicy.RANDOM);
+                    break;
+                case ROUNDROBIN:
+                    builder.distributionPolicy(DistributionPolicy.ROUNDROBIN);
+                    break;
+                }
+            }
+            builder.maxNodesDownPerGroup(rootDispatch.getMaxNodesDownPerFixedRow());
             builder.node(nodeBuilder);
         }
     }
diff --git a/configdefinitions/src/vespa/dispatch.def b/configdefinitions/src/vespa/dispatch.def
index d8ef600a33f..602d3b17a8e 100644
--- a/configdefinitions/src/vespa/dispatch.def
+++ b/configdefinitions/src/vespa/dispatch.def
@@ -7,6 +7,15 @@ namespace=vespa.config.search
 # for that group to be included in queries
 minActivedocsPercentage double default=97.0
 
+# Minimum coverage for allowing a group to be considered for serving
+minGroupCoverage double default=100
+
+# Maximum number of nodes allowed to be down for group to be considered for serving
+maxNodesDownPerGroup int default=0
+
+# Distribution policy for group selection
+distributionPolicy enum { ROUNDROBIN, RANDOM } default=ROUNDROBIN
+
 # The unique key of a search node
 node[].key int
 
diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java
index 2f67600522f..4638583a3f5 100644
--- a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java
+++ b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java
@@ -44,14 +44,15 @@ public class Dispatcher extends AbstractComponent {
 
     public Dispatcher(DispatchConfig dispatchConfig, FS4ResourcePool fs4ResourcePool, int containerClusterSize, VipStatus vipStatus) {
         this.searchCluster = new SearchCluster(dispatchConfig, fs4ResourcePool, containerClusterSize, vipStatus);
-        this.loadBalancer = new LoadBalancer(searchCluster);
+        this.loadBalancer = new LoadBalancer(searchCluster,
+                dispatchConfig.distributionPolicy() == DispatchConfig.DistributionPolicy.ROUNDROBIN);
         this.rpcResourcePool = new RpcResourcePool(dispatchConfig);
     }
 
     /** For testing */
     public Dispatcher(Map<Integer, Client.NodeConnection> nodeConnections, Client client) {
         this.searchCluster = null;
-        this.loadBalancer = new LoadBalancer(searchCluster);
+        this.loadBalancer = new LoadBalancer(searchCluster, true);
         this.rpcResourcePool = new RpcResourcePool(client, nodeConnections);
     }
 
diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/LoadBalancer.java b/container-search/src/main/java/com/yahoo/search/dispatch/LoadBalancer.java
index 4962746cdea..64e38a488ab 100644
--- a/container-search/src/main/java/com/yahoo/search/dispatch/LoadBalancer.java
+++ b/container-search/src/main/java/com/yahoo/search/dispatch/LoadBalancer.java
@@ -5,6 +5,7 @@ import com.yahoo.search.Query;
 import com.yahoo.search.dispatch.SearchCluster.Group;
 
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 import java.util.Optional;
 import java.util.logging.Level;
@@ -18,14 +19,13 @@ import java.util.logging.Logger;
  */
 public class LoadBalancer {
     // The implementation here is a simplistic least queries in flight + round-robin load balancer
-    // TODO: consider the options in com.yahoo.vespa.model.content.TuningDispatch
 
     private static final Logger log = Logger.getLogger(LoadBalancer.class.getName());
 
     private final List<GroupSchedule> scoreboard;
     private int needle = 0;
 
-    public LoadBalancer(SearchCluster searchCluster) {
+    public LoadBalancer(SearchCluster searchCluster, boolean roundRobin) {
         if (searchCluster == null) {
             this.scoreboard = null;
             return;
@@ -35,6 +35,11 @@ public class LoadBalancer {
         for (Group group : searchCluster.orderedGroups()) {
             scoreboard.add(new GroupSchedule(group));
         }
+
+        if(! roundRobin) {
+            // TODO - More randomness could be desirable
+            Collections.shuffle(scoreboard);
+        }
     }
 
     /**
diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/SearchCluster.java b/container-search/src/main/java/com/yahoo/search/dispatch/SearchCluster.java
index f806d4e685a..5dc63bc14d6 100644
--- a/container-search/src/main/java/com/yahoo/search/dispatch/SearchCluster.java
+++ b/container-search/src/main/java/com/yahoo/search/dispatch/SearchCluster.java
@@ -46,7 +46,9 @@ public class SearchCluster implements NodeManager<SearchCluster.Node> {
     private static final Logger log = Logger.getLogger(SearchCluster.class.getName());
 
     /** The min active docs a group must have to be considered up, as a % of the average active docs of the other groups */
-    private double minActivedocsCoveragePercentage;
+    public final double minActivedocsCoveragePercentage;
+    public final double minGroupCoverage;
+    public final int maxNodesDownPerGroup;
     private final int size;
     private final ImmutableMap<Integer, Group> groups;
     private final ImmutableMultimap<String, Node> nodesByHost;
@@ -67,15 +69,16 @@ public class SearchCluster implements NodeManager<SearchCluster.Node> {
     // Only needed until query requests are moved to rpc
     private final FS4ResourcePool fs4ResourcePool;
 
-    public SearchCluster(DispatchConfig dispatchConfig, FS4ResourcePool fs4ResourcePool,
-                         int containerClusterSize, VipStatus vipStatus) {
-        this(dispatchConfig.minActivedocsPercentage(), toNodes(dispatchConfig), fs4ResourcePool,
-             containerClusterSize, vipStatus);
+    public SearchCluster(DispatchConfig dispatchConfig, FS4ResourcePool fs4ResourcePool, int containerClusterSize, VipStatus vipStatus) {
+        this(dispatchConfig.minActivedocsPercentage(), dispatchConfig.minGroupCoverage(), dispatchConfig.maxNodesDownPerGroup(),
+                toNodes(dispatchConfig), fs4ResourcePool, containerClusterSize, vipStatus);
     }
 
-    public SearchCluster(double minActivedocsCoverage, List<Node> nodes, FS4ResourcePool fs4ResourcePool,
-                         int containerClusterSize, VipStatus vipStatus) {
+    public SearchCluster(double minActivedocsCoverage, double minGroupCoverage, int maxNodesDownPerGroup, List<Node> nodes, FS4ResourcePool fs4ResourcePool,
+            int containerClusterSize, VipStatus vipStatus) {
         this.minActivedocsCoveragePercentage = minActivedocsCoverage;
+        this.minGroupCoverage = minGroupCoverage;
+        this.maxNodesDownPerGroup = maxNodesDownPerGroup;
         this.size = nodes.size();
         this.fs4ResourcePool = fs4ResourcePool;
         this.vipStatus = vipStatus;
@@ -257,27 +260,54 @@ public class SearchCluster implements NodeManager<SearchCluster.Node> {
      */
     @Override
     public void pingIterationCompleted() {
+        int numGroups = orderedGroups.size();
+        if (numGroups == 1) {
+            Group group = groups.values().iterator().next();
+            group.aggregateActiveDocuments();
+            updateSufficientCoverage(group, true); // by definition
+            return;
+        }
+
         // Update active documents per group and use it to decide if the group should be active
-        for (Group group : groups.values())
+
+        long[] activeDocumentsInGroup = new long[numGroups];
+        long sumOfActiveDocuments = 0;
+        for(int i = 0; i < numGroups; i++) {
+            Group group = orderedGroups.get(i);
             group.aggregateActiveDocuments();
-        if (groups.size() == 1) {
-            updateSufficientCoverage(groups.values().iterator().next(), true); // by definition
-        } else {
-            for (Group currentGroup : groups.values()) {
-                long sumOfAactiveDocumentsInOtherGroups = 0;
-                for (Group otherGroup : groups.values())
-                    if (otherGroup != currentGroup)
-                        sumOfAactiveDocumentsInOtherGroups += otherGroup.getActiveDocuments();
-                long averageDocumentsInOtherGroups = sumOfAactiveDocumentsInOtherGroups / (groups.size() - 1);
-                if (averageDocumentsInOtherGroups == 0)
-                    updateSufficientCoverage(currentGroup, true); // no information about any group; assume coverage
-                else
-                    updateSufficientCoverage(currentGroup,
-                                             100 * (double) currentGroup.getActiveDocuments() / averageDocumentsInOtherGroups > minActivedocsCoveragePercentage);
+            activeDocumentsInGroup[i] = group.getActiveDocuments();
+            sumOfActiveDocuments += activeDocumentsInGroup[i];
+        }
+
+        for (int i = 0; i < numGroups; i++) {
+            Group group = orderedGroups.get(i);
+            long activeDocuments = activeDocumentsInGroup[i];
+            long averageDocumentsInOtherGroups = (sumOfActiveDocuments - activeDocuments) / (numGroups - 1);
+            boolean sufficientCoverage = true;
+
+            if (averageDocumentsInOtherGroups > 0) {
+                double coverage = 100.0 * (double) activeDocuments / averageDocumentsInOtherGroups;
+                sufficientCoverage = coverage >= minActivedocsCoveragePercentage;
             }
+            if (sufficientCoverage) {
+                sufficientCoverage = isNodeCoverageSufficient(group);
+            }
+            updateSufficientCoverage(group, sufficientCoverage);
         }
     }
 
+    private boolean isNodeCoverageSufficient(Group group) {
+        int nodesUp = 0;
+        for (Node node : group.nodes()) {
+            if (node.isWorking()) {
+                nodesUp++;
+            }
+        }
+        int nodes = group.nodes().size();
+        int nodesAllowedDown = maxNodesDownPerGroup + (int) (((double) nodes * (100.0 - minGroupCoverage)) / 100.0);
+        return nodesUp + nodesAllowedDown >= nodes;
+    }
+
     private Pong getPong(FutureTask<Pong> futurePong, Node node) {
         try {
             return futurePong.get(clusterMonitor.getConfiguration().getFailLimit(), TimeUnit.MILLISECONDS);
@@ -349,8 +379,11 @@ public class SearchCluster implements NodeManager<SearchCluster.Node> {
 
         void aggregateActiveDocuments() {
             long activeDocumentsInGroup = 0;
-            for (Node node : nodes)
-                activeDocumentsInGroup += node.getActiveDocuments();
+            for (Node node : nodes) {
+                if (node.isWorking()) {
+                    activeDocumentsInGroup += node.getActiveDocuments();
+                }
+            }
             activeDocuments.set(activeDocumentsInGroup);
 
         }
diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java
index b08a3a73a01..9311ddab3c6 100644
--- a/container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java
+++ b/container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java
@@ -22,8 +22,8 @@ public class LoadBalancerTest {
     @Test
     public void requreThatLoadBalancerServesSingleNodeSetups() {
         Node n1 = new SearchCluster.Node(0, "test-node1", 0, 0);
-        SearchCluster cluster = new SearchCluster(88.0, Arrays.asList(n1), null, 1, null);
-        LoadBalancer lb = new LoadBalancer(cluster);
+        SearchCluster cluster = new SearchCluster(88.0, 99.0, 0, Arrays.asList(n1), null, 1, null);
+        LoadBalancer lb = new LoadBalancer(cluster, true);
 
         Optional<Group> grp = lb.takeGroupForQuery(new Query());
         Group group = grp.orElseGet(() -> {
@@ -36,8 +36,8 @@ public class LoadBalancerTest {
     public void requreThatLoadBalancerServesMultiGroupSetups() {
         Node n1 = new SearchCluster.Node(0, "test-node1", 0, 0);
         Node n2 = new SearchCluster.Node(1, "test-node2", 1, 1);
-        SearchCluster cluster = new SearchCluster(88.0, Arrays.asList(n1, n2), null, 1, null);
-        LoadBalancer lb = new LoadBalancer(cluster);
+        SearchCluster cluster = new SearchCluster(88.0, 99.0, 0, Arrays.asList(n1, n2), null, 1, null);
+        LoadBalancer lb = new LoadBalancer(cluster, true);
 
         Optional<Group> grp = lb.takeGroupForQuery(new Query());
         Group group = grp.orElseGet(() -> {
@@ -52,8 +52,8 @@ public class LoadBalancerTest {
         Node n2 = new SearchCluster.Node(1, "test-node2", 1, 0);
         Node n3 = new SearchCluster.Node(0, "test-node3", 0, 1);
         Node n4 = new SearchCluster.Node(1, "test-node4", 1, 1);
-        SearchCluster cluster = new SearchCluster(88.0, Arrays.asList(n1, n2, n3, n4), null, 2, null);
-        LoadBalancer lb = new LoadBalancer(cluster);
+        SearchCluster cluster = new SearchCluster(88.0, 99.0, 0, Arrays.asList(n1, n2, n3, n4), null, 2, null);
+        LoadBalancer lb = new LoadBalancer(cluster, true);
 
         Optional<Group> grp = lb.takeGroupForQuery(new Query());
         assertThat(grp.isPresent(), is(true));
@@ -63,8 +63,8 @@ public class LoadBalancerTest {
     public void requreThatLoadBalancerReturnsDifferentGroups() {
         Node n1 = new SearchCluster.Node(0, "test-node1", 0, 0);
         Node n2 = new SearchCluster.Node(1, "test-node2", 1, 1);
-        SearchCluster cluster = new SearchCluster(88.0, Arrays.asList(n1, n2), null, 1, null);
-        LoadBalancer lb = new LoadBalancer(cluster);
+        SearchCluster cluster = new SearchCluster(88.0, 99.0, 0, Arrays.asList(n1, n2), null, 1, null);
+        LoadBalancer lb = new LoadBalancer(cluster, true);
 
         // get first group
         Optional<Group> grp = lb.takeGroupForQuery(new Query());
@@ -83,8 +83,8 @@ public class LoadBalancerTest {
     public void requreThatLoadBalancerReturnsGroupWithShortestQueue() {
         Node n1 = new SearchCluster.Node(0, "test-node1", 0, 0);
         Node n2 = new SearchCluster.Node(1, "test-node2", 1, 1);
-        SearchCluster cluster = new SearchCluster(88.0, Arrays.asList(n1, n2), null, 1, null);
-        LoadBalancer lb = new LoadBalancer(cluster);
+        SearchCluster cluster = new SearchCluster(88.0, 99.0, 0, Arrays.asList(n1, n2), null, 1, null);
+        LoadBalancer lb = new LoadBalancer(cluster, true);
 
         // get first group
         Optional<Group> grp = lb.takeGroupForQuery(new Query());
diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java b/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java
index 89b416f3293..ee903fd3fa0 100644
--- a/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java
+++ b/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java
@@ -19,7 +19,7 @@ public class MockSearchCluster extends SearchCluster {
     private final ImmutableMultimap<String, Node> nodesByHost;
 
     public MockSearchCluster(int groups, int nodesPerGroup) {
-        super(100, Collections.emptyList(), null, 1, null);
+        super(100, 100, 0, Collections.emptyList(), null, 1, null);
 
         ImmutableMap.Builder<Integer, Group> groupBuilder = ImmutableMap.builder();
         ImmutableMultimap.Builder<String, Node> hostBuilder = ImmutableMultimap.builder();