diff options
7 files changed, 70 insertions, 21 deletions
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java index f99dde237bd..80bba10276d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java @@ -422,6 +422,7 @@ public class IndexedSearchCluster extends SearchCluster } builder.maxNodesDownPerGroup(rootDispatch.getMaxNodesDownPerFixedRow()); builder.useMultilevelDispatch(useMultilevelDispatchSetup()); + builder.useLocalNode(tuning.dispatch.useLocalNode); builder.searchableCopies(rootDispatch.getSearchableCopies()); if (searchCoverage != null) { if (searchCoverage.getMinimum() != null) diff --git a/configdefinitions/src/vespa/dispatch.def b/configdefinitions/src/vespa/dispatch.def index 50989c3ef74..1108e40b3f1 100644 --- a/configdefinitions/src/vespa/dispatch.def +++ b/configdefinitions/src/vespa/dispatch.def @@ -1,4 +1,4 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# Copyright 2019 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. # Configuration of dispatch from container nodes to search clusters namespace=vespa.config.search @@ -19,6 +19,9 @@ distributionPolicy enum { ROUNDROBIN, ADAPTIVE } default=ROUNDROBIN # Is multi-level dispatch configured for this cluster useMultilevelDispatch bool default=false +# Dispatch only to local nodes +useLocalNode bool default=false + # Number of document copies searchableCopies long default=1 diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java index f67641041ad..b613952b99e 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java @@ -135,7 +135,9 @@ public class Dispatcher extends AbstractComponent { return invokerFactory.supply(query, -1, Arrays.asList(node), true); } - int max = Integer.min(searchCluster.orderedGroups().size(), MAX_GROUP_SELECTION_ATTEMPTS); + int covered = searchCluster.groupsWithSufficientCoverage(); + int groups = searchCluster.orderedGroups().size(); + int max = Integer.min(Integer.min(covered + 1, groups), MAX_GROUP_SELECTION_ATTEMPTS); Set<Integer> rejected = null; for (int i = 0; i < max; i++) { Optional<Group> groupInCluster = loadBalancer.takeGroup(rejected); diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/LoadBalancer.java b/container-search/src/main/java/com/yahoo/search/dispatch/LoadBalancer.java index 3c7af447809..6a59f28ca4c 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/LoadBalancer.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/LoadBalancer.java @@ -35,7 +35,7 @@ public class LoadBalancer { for (Group group : searchCluster.orderedGroups()) { scoreboard.add(new GroupStatus(group)); } - if (roundRobin) { + if (roundRobin || scoreboard.size() == 1) { this.scheduler = new RoundRobinScheduler(scoreboard); } else { this.scheduler = new AdaptiveScheduler(new Random(), scoreboard); diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Group.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Group.java index 01cbc5cd307..146c62f0a16 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Group.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Group.java @@ -44,6 +44,16 @@ public class Group { hasSufficientCoverage.lazySet(sufficientCoverage); } + public int workingNodes() { + int nodesUp = 0; + for (Node node : nodes) { + if (node.isWorking()) { + nodesUp++; + } + } + return nodesUp; + } + void aggregateActiveDocuments() { long activeDocumentsInGroup = 0; for (Node node : nodes) { diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java index 8e278f78d7a..e497ef6751b 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java @@ -23,6 +23,7 @@ import java.util.concurrent.Executor; import java.util.concurrent.FutureTask; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import java.util.function.Predicate; import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -102,7 +103,7 @@ public class SearchCluster implements NodeManager<Node> { private static Optional<Node> findDirectDispatchTarget(String selfHostname, int searchClusterSize, int containerClusterSize, - ImmutableMultimap<String, Node>nodesByHost, + ImmutableMultimap<String, Node> nodesByHost, ImmutableMap<Integer, Group> groups) { // A search node in the search cluster in question is configured on the same host as the currently running container. // It has all the data <==> No other nodes in the search cluster have the same group id as this node. @@ -129,8 +130,18 @@ public class SearchCluster implements NodeManager<Node> { private static ImmutableList<Node> toNodes(DispatchConfig dispatchConfig) { ImmutableList.Builder<Node> nodesBuilder = new ImmutableList.Builder<>(); - for (DispatchConfig.Node node : dispatchConfig.node()) - nodesBuilder.add(new Node(node.key(), node.host(), node.fs4port(), node.group())); + Predicate<DispatchConfig.Node> filter; + if (dispatchConfig.useLocalNode()) { + final String hostName = HostName.getLocalhost(); + filter = node -> node.host().equals(hostName); + } else { + filter = node -> true; + } + for (DispatchConfig.Node node : dispatchConfig.node()) { + if (filter.test(node)) { + nodesBuilder.add(new Node(node.key(), node.host(), node.fs4port(), node.group())); + } + } return nodesBuilder.build(); } @@ -162,6 +173,16 @@ public class SearchCluster implements NodeManager<Node> { return size() / groups.size(); } + public int groupsWithSufficientCoverage() { + int covered = 0; + for (Group g : orderedGroups) { + if (g.hasSufficientCoverage()) { + covered++; + } + } + return covered; + } + /** * Returns the nodes of this cluster as an immutable map indexed by host. * One host may contain multiple nodes (on different ports), so this is a multi-map. @@ -271,12 +292,13 @@ public class SearchCluster implements NodeManager<Node> { Group group = orderedGroups.get(i); long activeDocuments = activeDocumentsInGroup[i]; long averageDocumentsInOtherGroups = (sumOfActiveDocuments - activeDocuments) / (numGroups - 1); - boolean sufficientCoverage = isGroupCoverageSufficient(group.nodes(), activeDocuments, averageDocumentsInOtherGroups); + boolean sufficientCoverage = isGroupCoverageSufficient(group.workingNodes(), group.nodes().size(), activeDocuments, + averageDocumentsInOtherGroups); updateSufficientCoverage(group, sufficientCoverage); } } - private boolean isGroupCoverageSufficient(List<Node> nodes, long activeDocuments, long averageDocumentsInOtherGroups) { + private boolean isGroupCoverageSufficient(int workingNodes, int nodesInGroup, long activeDocuments, long averageDocumentsInOtherGroups) { boolean sufficientCoverage = true; if (averageDocumentsInOtherGroups > 0) { @@ -284,22 +306,15 @@ public class SearchCluster implements NodeManager<Node> { sufficientCoverage = coverage >= dispatchConfig.minActivedocsPercentage(); } if (sufficientCoverage) { - sufficientCoverage = isGroupNodeCoverageSufficient(nodes); + sufficientCoverage = isGroupNodeCoverageSufficient(workingNodes, nodesInGroup); } return sufficientCoverage; } - private boolean isGroupNodeCoverageSufficient(List<Node> nodes) { - int nodesUp = 0; - for (Node node : nodes) { - if (node.isWorking()) { - nodesUp++; - } - } - int numNodes = nodes.size(); + private boolean isGroupNodeCoverageSufficient(int workingNodes, int nodesInGroup) { int nodesAllowedDown = dispatchConfig.maxNodesDownPerGroup() - + (int) (((double) numNodes * (100.0 - dispatchConfig.minGroupCoverage())) / 100.0); - return nodesUp + nodesAllowedDown >= numNodes; + + (int) (((double) nodesInGroup * (100.0 - dispatchConfig.minGroupCoverage())) / 100.0); + return workingNodes + nodesAllowedDown >= nodesInGroup; } private Pong getPong(FutureTask<Pong> futurePong, Node node) { @@ -316,13 +331,24 @@ public class SearchCluster implements NodeManager<Node> { } } + private void logIfInsufficientCoverage(boolean sufficient, int groupId, int nodes) { + if (!sufficient) { + log.warning(() -> String.format("Coverage of group %s is only %d/%d (requires %d)", groupId, nodes, groupSize(), + groupSize() - dispatchConfig.maxNodesDownPerGroup())); + } + } + /** * Calculate whether a subset of nodes in a group has enough coverage */ public boolean isPartialGroupCoverageSufficient(int groupId, List<Node> nodes) { if (orderedGroups.size() == 1) { - return nodes.size() >= groupSize() - dispatchConfig.maxNodesDownPerGroup(); + boolean sufficient = nodes.size() >= groupSize() - dispatchConfig.maxNodesDownPerGroup(); + logIfInsufficientCoverage(sufficient, groupId, nodes.size()); + return sufficient; } + + int nodesInGroup = groups.get(groupId).nodes().size(); long sumOfActiveDocuments = 0; int otherGroups = 0; for (Group g : orderedGroups) { @@ -336,6 +362,8 @@ public class SearchCluster implements NodeManager<Node> { activeDocuments += n.getActiveDocuments(); } long averageDocumentsInOtherGroups = sumOfActiveDocuments / otherGroups; - return isGroupCoverageSufficient(nodes, activeDocuments, averageDocumentsInOtherGroups); + boolean sufficient = isGroupCoverageSufficient(nodes.size(), nodesInGroup, activeDocuments, averageDocumentsInOtherGroups); + logIfInsufficientCoverage(sufficient, groupId, nodes.size()); + return sufficient; } } diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java b/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java index 2f970a9c007..3919bc26bdc 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java @@ -75,6 +75,11 @@ public class MockSearchCluster extends SearchCluster { } @Override + public int groupsWithSufficientCoverage() { + return numGroups; + } + + @Override public Optional<Group> group(int n) { if (n < numGroups) { return Optional.of(groups.get(n)); |