diff options
7 files changed, 114 insertions, 49 deletions
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java index 985b6e1e4b0..b8ec7e88354 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java @@ -3,16 +3,17 @@ package com.yahoo.vespa.model.search; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.deploy.DeployState; -import com.yahoo.log.LogLevel; -import com.yahoo.vespa.config.search.AttributesConfig; -import com.yahoo.vespa.config.search.DispatchConfig; -import com.yahoo.vespa.config.search.core.ProtonConfig; -import com.yahoo.vespa.config.search.RankProfilesConfig; import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.log.LogLevel; import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; import com.yahoo.search.config.IndexInfoConfig; import com.yahoo.searchdefinition.DocumentOnlySearch; import com.yahoo.searchdefinition.derived.DerivedConfiguration; +import com.yahoo.vespa.config.search.AttributesConfig; +import com.yahoo.vespa.config.search.DispatchConfig; +import com.yahoo.vespa.config.search.DispatchConfig.DistributionPolicy; +import com.yahoo.vespa.config.search.RankProfilesConfig; +import com.yahoo.vespa.config.search.core.ProtonConfig; import com.yahoo.vespa.configdefinition.IlscriptsConfig; import com.yahoo.vespa.model.HostResource; import com.yahoo.vespa.model.SimpleConfigProducer; @@ -23,8 +24,11 @@ import com.yahoo.vespa.model.content.DispatchSpec; import com.yahoo.vespa.model.content.SearchCoverage; import java.io.File; -import java.io.IOException; -import java.util.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; import java.util.logging.Logger; /** @@ -316,17 +320,17 @@ public class IndexedSearchCluster extends SearchCluster @Override public DerivedConfiguration getSdConfig() { return null; } - + @Override public void getConfig(IndexInfoConfig.Builder builder) { unionCfg.getConfig(builder); } - + @Override public void getConfig(IlscriptsConfig.Builder builder) { unionCfg.getConfig(builder); } - + @Override public void getConfig(AttributesConfig.Builder builder) { unionCfg.getConfig(builder); @@ -402,6 +406,19 @@ public class IndexedSearchCluster extends SearchCluster nodeBuilder.fs4port(node.getDispatchPort()); if (tuning.dispatch.minActiveDocsCoverage != null) builder.minActivedocsPercentage(tuning.dispatch.minActiveDocsCoverage); + if (tuning.dispatch.minGroupCoverage != null) + builder.minGroupCoverage(tuning.dispatch.minGroupCoverage); + if (tuning.dispatch.policy != null) { + switch(tuning.dispatch.policy) { + case RANDOM: + builder.distributionPolicy(DistributionPolicy.RANDOM); + break; + case ROUNDROBIN: + builder.distributionPolicy(DistributionPolicy.ROUNDROBIN); + break; + } + } + builder.maxNodesDownPerGroup(rootDispatch.getMaxNodesDownPerFixedRow()); builder.node(nodeBuilder); } } diff --git a/configdefinitions/src/vespa/dispatch.def b/configdefinitions/src/vespa/dispatch.def index d8ef600a33f..602d3b17a8e 100644 --- a/configdefinitions/src/vespa/dispatch.def +++ b/configdefinitions/src/vespa/dispatch.def @@ -7,6 +7,15 @@ namespace=vespa.config.search # for that group to be included in queries minActivedocsPercentage double default=97.0 +# Minimum coverage for allowing a group to be considered for serving +minGroupCoverage double default=100 + +# Maximum number of nodes allowed to be down for group to be considered for serving +maxNodesDownPerGroup int default=0 + +# Distribution policy for group selection +distributionPolicy enum { ROUNDROBIN, RANDOM } default=ROUNDROBIN + # The unique key of a search node node[].key int diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java index 2f67600522f..4638583a3f5 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java @@ -44,14 +44,15 @@ public class Dispatcher extends AbstractComponent { public Dispatcher(DispatchConfig dispatchConfig, FS4ResourcePool fs4ResourcePool, int containerClusterSize, VipStatus vipStatus) { this.searchCluster = new SearchCluster(dispatchConfig, fs4ResourcePool, containerClusterSize, vipStatus); - this.loadBalancer = new LoadBalancer(searchCluster); + this.loadBalancer = new LoadBalancer(searchCluster, + dispatchConfig.distributionPolicy() == DispatchConfig.DistributionPolicy.ROUNDROBIN); this.rpcResourcePool = new RpcResourcePool(dispatchConfig); } /** For testing */ public Dispatcher(Map<Integer, Client.NodeConnection> nodeConnections, Client client) { this.searchCluster = null; - this.loadBalancer = new LoadBalancer(searchCluster); + this.loadBalancer = new LoadBalancer(searchCluster, true); this.rpcResourcePool = new RpcResourcePool(client, nodeConnections); } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/LoadBalancer.java b/container-search/src/main/java/com/yahoo/search/dispatch/LoadBalancer.java index 4962746cdea..64e38a488ab 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/LoadBalancer.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/LoadBalancer.java @@ -5,6 +5,7 @@ import com.yahoo.search.Query; import com.yahoo.search.dispatch.SearchCluster.Group; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Optional; import java.util.logging.Level; @@ -18,14 +19,13 @@ import java.util.logging.Logger; */ public class LoadBalancer { // The implementation here is a simplistic least queries in flight + round-robin load balancer - // TODO: consider the options in com.yahoo.vespa.model.content.TuningDispatch private static final Logger log = Logger.getLogger(LoadBalancer.class.getName()); private final List<GroupSchedule> scoreboard; private int needle = 0; - public LoadBalancer(SearchCluster searchCluster) { + public LoadBalancer(SearchCluster searchCluster, boolean roundRobin) { if (searchCluster == null) { this.scoreboard = null; return; @@ -35,6 +35,11 @@ public class LoadBalancer { for (Group group : searchCluster.orderedGroups()) { scoreboard.add(new GroupSchedule(group)); } + + if(! roundRobin) { + // TODO - More randomness could be desirable + Collections.shuffle(scoreboard); + } } /** diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/SearchCluster.java b/container-search/src/main/java/com/yahoo/search/dispatch/SearchCluster.java index f806d4e685a..5dc63bc14d6 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/SearchCluster.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/SearchCluster.java @@ -46,7 +46,9 @@ public class SearchCluster implements NodeManager<SearchCluster.Node> { private static final Logger log = Logger.getLogger(SearchCluster.class.getName()); /** The min active docs a group must have to be considered up, as a % of the average active docs of the other groups */ - private double minActivedocsCoveragePercentage; + public final double minActivedocsCoveragePercentage; + public final double minGroupCoverage; + public final int maxNodesDownPerGroup; private final int size; private final ImmutableMap<Integer, Group> groups; private final ImmutableMultimap<String, Node> nodesByHost; @@ -67,15 +69,16 @@ public class SearchCluster implements NodeManager<SearchCluster.Node> { // Only needed until query requests are moved to rpc private final FS4ResourcePool fs4ResourcePool; - public SearchCluster(DispatchConfig dispatchConfig, FS4ResourcePool fs4ResourcePool, - int containerClusterSize, VipStatus vipStatus) { - this(dispatchConfig.minActivedocsPercentage(), toNodes(dispatchConfig), fs4ResourcePool, - containerClusterSize, vipStatus); + public SearchCluster(DispatchConfig dispatchConfig, FS4ResourcePool fs4ResourcePool, int containerClusterSize, VipStatus vipStatus) { + this(dispatchConfig.minActivedocsPercentage(), dispatchConfig.minGroupCoverage(), dispatchConfig.maxNodesDownPerGroup(), + toNodes(dispatchConfig), fs4ResourcePool, containerClusterSize, vipStatus); } - public SearchCluster(double minActivedocsCoverage, List<Node> nodes, FS4ResourcePool fs4ResourcePool, - int containerClusterSize, VipStatus vipStatus) { + public SearchCluster(double minActivedocsCoverage, double minGroupCoverage, int maxNodesDownPerGroup, List<Node> nodes, FS4ResourcePool fs4ResourcePool, + int containerClusterSize, VipStatus vipStatus) { this.minActivedocsCoveragePercentage = minActivedocsCoverage; + this.minGroupCoverage = minGroupCoverage; + this.maxNodesDownPerGroup = maxNodesDownPerGroup; this.size = nodes.size(); this.fs4ResourcePool = fs4ResourcePool; this.vipStatus = vipStatus; @@ -257,27 +260,54 @@ public class SearchCluster implements NodeManager<SearchCluster.Node> { */ @Override public void pingIterationCompleted() { + int numGroups = orderedGroups.size(); + if (numGroups == 1) { + Group group = groups.values().iterator().next(); + group.aggregateActiveDocuments(); + updateSufficientCoverage(group, true); // by definition + return; + } + // Update active documents per group and use it to decide if the group should be active - for (Group group : groups.values()) + + long[] activeDocumentsInGroup = new long[numGroups]; + long sumOfActiveDocuments = 0; + for(int i = 0; i < numGroups; i++) { + Group group = orderedGroups.get(i); group.aggregateActiveDocuments(); - if (groups.size() == 1) { - updateSufficientCoverage(groups.values().iterator().next(), true); // by definition - } else { - for (Group currentGroup : groups.values()) { - long sumOfAactiveDocumentsInOtherGroups = 0; - for (Group otherGroup : groups.values()) - if (otherGroup != currentGroup) - sumOfAactiveDocumentsInOtherGroups += otherGroup.getActiveDocuments(); - long averageDocumentsInOtherGroups = sumOfAactiveDocumentsInOtherGroups / (groups.size() - 1); - if (averageDocumentsInOtherGroups == 0) - updateSufficientCoverage(currentGroup, true); // no information about any group; assume coverage - else - updateSufficientCoverage(currentGroup, - 100 * (double) currentGroup.getActiveDocuments() / averageDocumentsInOtherGroups > minActivedocsCoveragePercentage); + activeDocumentsInGroup[i] = group.getActiveDocuments(); + sumOfActiveDocuments += activeDocumentsInGroup[i]; + } + + for (int i = 0; i < numGroups; i++) { + Group group = orderedGroups.get(i); + long activeDocuments = activeDocumentsInGroup[i]; + long averageDocumentsInOtherGroups = (sumOfActiveDocuments - activeDocuments) / (numGroups - 1); + boolean sufficientCoverage = true; + + if (averageDocumentsInOtherGroups > 0) { + double coverage = 100.0 * (double) activeDocuments / averageDocumentsInOtherGroups; + sufficientCoverage = coverage >= minActivedocsCoveragePercentage; } + if (sufficientCoverage) { + sufficientCoverage = isNodeCoverageSufficient(group); + } + updateSufficientCoverage(group, sufficientCoverage); } } + private boolean isNodeCoverageSufficient(Group group) { + int nodesUp = 0; + for (Node node : group.nodes()) { + if (node.isWorking()) { + nodesUp++; + } + } + int nodes = group.nodes().size(); + int nodesAllowedDown = maxNodesDownPerGroup + (int) (((double) nodes * (100.0 - minGroupCoverage)) / 100.0); + return nodesUp + nodesAllowedDown >= nodes; + } + private Pong getPong(FutureTask<Pong> futurePong, Node node) { try { return futurePong.get(clusterMonitor.getConfiguration().getFailLimit(), TimeUnit.MILLISECONDS); @@ -349,8 +379,11 @@ public class SearchCluster implements NodeManager<SearchCluster.Node> { void aggregateActiveDocuments() { long activeDocumentsInGroup = 0; - for (Node node : nodes) - activeDocumentsInGroup += node.getActiveDocuments(); + for (Node node : nodes) { + if (node.isWorking()) { + activeDocumentsInGroup += node.getActiveDocuments(); + } + } activeDocuments.set(activeDocumentsInGroup); } diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java index b08a3a73a01..9311ddab3c6 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java @@ -22,8 +22,8 @@ public class LoadBalancerTest { @Test public void requreThatLoadBalancerServesSingleNodeSetups() { Node n1 = new SearchCluster.Node(0, "test-node1", 0, 0); - SearchCluster cluster = new SearchCluster(88.0, Arrays.asList(n1), null, 1, null); - LoadBalancer lb = new LoadBalancer(cluster); + SearchCluster cluster = new SearchCluster(88.0, 99.0, 0, Arrays.asList(n1), null, 1, null); + LoadBalancer lb = new LoadBalancer(cluster, true); Optional<Group> grp = lb.takeGroupForQuery(new Query()); Group group = grp.orElseGet(() -> { @@ -36,8 +36,8 @@ public class LoadBalancerTest { public void requreThatLoadBalancerServesMultiGroupSetups() { Node n1 = new SearchCluster.Node(0, "test-node1", 0, 0); Node n2 = new SearchCluster.Node(1, "test-node2", 1, 1); - SearchCluster cluster = new SearchCluster(88.0, Arrays.asList(n1, n2), null, 1, null); - LoadBalancer lb = new LoadBalancer(cluster); + SearchCluster cluster = new SearchCluster(88.0, 99.0, 0, Arrays.asList(n1, n2), null, 1, null); + LoadBalancer lb = new LoadBalancer(cluster, true); Optional<Group> grp = lb.takeGroupForQuery(new Query()); Group group = grp.orElseGet(() -> { @@ -52,8 +52,8 @@ public class LoadBalancerTest { Node n2 = new SearchCluster.Node(1, "test-node2", 1, 0); Node n3 = new SearchCluster.Node(0, "test-node3", 0, 1); Node n4 = new SearchCluster.Node(1, "test-node4", 1, 1); - SearchCluster cluster = new SearchCluster(88.0, Arrays.asList(n1, n2, n3, n4), null, 2, null); - LoadBalancer lb = new LoadBalancer(cluster); + SearchCluster cluster = new SearchCluster(88.0, 99.0, 0, Arrays.asList(n1, n2, n3, n4), null, 2, null); + LoadBalancer lb = new LoadBalancer(cluster, true); Optional<Group> grp = lb.takeGroupForQuery(new Query()); assertThat(grp.isPresent(), is(true)); @@ -63,8 +63,8 @@ public class LoadBalancerTest { public void requreThatLoadBalancerReturnsDifferentGroups() { Node n1 = new SearchCluster.Node(0, "test-node1", 0, 0); Node n2 = new SearchCluster.Node(1, "test-node2", 1, 1); - SearchCluster cluster = new SearchCluster(88.0, Arrays.asList(n1, n2), null, 1, null); - LoadBalancer lb = new LoadBalancer(cluster); + SearchCluster cluster = new SearchCluster(88.0, 99.0, 0, Arrays.asList(n1, n2), null, 1, null); + LoadBalancer lb = new LoadBalancer(cluster, true); // get first group Optional<Group> grp = lb.takeGroupForQuery(new Query()); @@ -83,8 +83,8 @@ public class LoadBalancerTest { public void requreThatLoadBalancerReturnsGroupWithShortestQueue() { Node n1 = new SearchCluster.Node(0, "test-node1", 0, 0); Node n2 = new SearchCluster.Node(1, "test-node2", 1, 1); - SearchCluster cluster = new SearchCluster(88.0, Arrays.asList(n1, n2), null, 1, null); - LoadBalancer lb = new LoadBalancer(cluster); + SearchCluster cluster = new SearchCluster(88.0, 99.0, 0, Arrays.asList(n1, n2), null, 1, null); + LoadBalancer lb = new LoadBalancer(cluster, true); // get first group Optional<Group> grp = lb.takeGroupForQuery(new Query()); diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java b/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java index 89b416f3293..ee903fd3fa0 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java @@ -19,7 +19,7 @@ public class MockSearchCluster extends SearchCluster { private final ImmutableMultimap<String, Node> nodesByHost; public MockSearchCluster(int groups, int nodesPerGroup) { - super(100, Collections.emptyList(), null, 1, null); + super(100, 100, 0, Collections.emptyList(), null, 1, null); ImmutableMap.Builder<Integer, Group> groupBuilder = ImmutableMap.builder(); ImmutableMultimap.Builder<String, Node> hostBuilder = ImmutableMultimap.builder(); |