From dbbe6e49c55399464fc33b67cb93e6ccb10eec88 Mon Sep 17 00:00:00 2001 From: Harald Musum Date: Fri, 20 Sep 2019 06:48:05 +0200 Subject: Revert "Bratseth/vip logic take 2" --- .../com/yahoo/search/cluster/BaseNodeMonitor.java | 16 +- .../com/yahoo/search/cluster/ClusterMonitor.java | 23 +- .../yahoo/search/cluster/TrafficNodeMonitor.java | 10 +- .../java/com/yahoo/search/dispatch/Dispatcher.java | 5 +- .../com/yahoo/search/dispatch/InvokerFactory.java | 13 +- .../yahoo/search/dispatch/searchcluster/Group.java | 13 +- .../yahoo/search/dispatch/searchcluster/Node.java | 9 +- .../search/dispatch/searchcluster/PingFactory.java | 2 - .../dispatch/searchcluster/SearchCluster.java | 134 +++++----- .../cluster/test/ClusteredConnectionTestCase.java | 2 +- .../com/yahoo/search/dispatch/DispatcherTest.java | 2 +- .../yahoo/search/dispatch/MockSearchCluster.java | 9 +- .../dispatch/searchcluster/SearchClusterTest.java | 279 --------------------- 13 files changed, 101 insertions(+), 416 deletions(-) delete mode 100644 container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTest.java (limited to 'container-search/src') diff --git a/container-search/src/main/java/com/yahoo/search/cluster/BaseNodeMonitor.java b/container-search/src/main/java/com/yahoo/search/cluster/BaseNodeMonitor.java index dd01d895963..59674d25402 100644 --- a/container-search/src/main/java/com/yahoo/search/cluster/BaseNodeMonitor.java +++ b/container-search/src/main/java/com/yahoo/search/cluster/BaseNodeMonitor.java @@ -26,25 +26,25 @@ public abstract class BaseNodeMonitor { /** The object representing the monitored node */ protected T node; - protected boolean isWorking = true; + protected boolean isWorking=true; /** Whether this node is quarantined for unstability */ - protected boolean isQuarantined = false; + protected boolean isQuarantined=false; /** The last time this node failed, in ms */ - protected long failedAt = 0; + protected long failedAt=0; /** The last time this node responded (failed or succeeded), in ms */ - protected long respondedAt = 0; + protected long respondedAt=0; /** The last time this node responded successfully */ - protected long succeededAt = 0; + protected long succeededAt=0; /** The configuration of this monitor */ protected MonitorConfiguration configuration; /** Is the node we monitor part of an internal Vespa cluster or not */ - private boolean internal; + private boolean internal=false; public BaseNodeMonitor(boolean internal) { this.internal=internal; @@ -54,12 +54,10 @@ public abstract class BaseNodeMonitor { /** * Returns whether this node is currently in a state suitable - * for receiving traffic (default true) + * for receiving traffic. As far as we know, that is */ public boolean isWorking() { return isWorking; } - /** @deprecated Not used */ - @Deprecated // TODO: Remove on Vespa 8 public boolean isQuarantined() { return isQuarantined; } /** diff --git a/container-search/src/main/java/com/yahoo/search/cluster/ClusterMonitor.java b/container-search/src/main/java/com/yahoo/search/cluster/ClusterMonitor.java index 22c7f59872c..ac0c8375f04 100644 --- a/container-search/src/main/java/com/yahoo/search/cluster/ClusterMonitor.java +++ b/container-search/src/main/java/com/yahoo/search/cluster/ClusterMonitor.java @@ -22,9 +22,9 @@ import java.util.logging.Logger; */ public class ClusterMonitor { - private final MonitorConfiguration configuration = new MonitorConfiguration(); + private MonitorConfiguration configuration = new MonitorConfiguration(); - private static Logger log = Logger.getLogger(ClusterMonitor.class.getName()); + private static Logger log=Logger.getLogger(ClusterMonitor.class.getName()); private NodeManager nodeManager; @@ -33,7 +33,7 @@ public class ClusterMonitor { private volatile boolean shutdown = false; /** A map from Node to corresponding MonitoredNode */ - private final Map> nodeMonitors = Collections.synchronizedMap(new java.util.LinkedHashMap<>()); + private final Map> nodeMonitors = Collections.synchronizedMap(new java.util.LinkedHashMap<>()); public ClusterMonitor(NodeManager manager) { nodeManager = manager; @@ -56,7 +56,8 @@ public class ClusterMonitor { * @param internal whether or not this node is internal to this cluster */ public void add(T node, boolean internal) { - nodeMonitors.put(node, new TrafficNodeMonitor<>(node, configuration, internal)); + BaseNodeMonitor monitor = new TrafficNodeMonitor<>(node, configuration, internal); + nodeMonitors.put(node, monitor); } /** @@ -68,20 +69,22 @@ public class ClusterMonitor { /** Called from ClusterSearcher/NodeManager when a node failed */ public synchronized void failed(T node, ErrorMessage error) { - TrafficNodeMonitor monitor = nodeMonitors.get(node); - Boolean wasWorking = monitor.isKnownWorking(); + BaseNodeMonitor monitor = nodeMonitors.get(node); + boolean wasWorking = monitor.isWorking(); monitor.failed(error); - if (wasWorking != monitor.isKnownWorking()) + if (wasWorking && !monitor.isWorking()) { nodeManager.failed(node); + } } /** Called when a node responded */ public synchronized void responded(T node) { - TrafficNodeMonitor monitor = nodeMonitors.get(node); - Boolean wasWorking = monitor.isKnownWorking(); + BaseNodeMonitor monitor = nodeMonitors.get(node); + boolean wasFailing =! monitor.isWorking(); monitor.responded(); - if (wasWorking != monitor.isKnownWorking()) + if (wasFailing && monitor.isWorking()) { nodeManager.working(monitor.getNode()); + } } /** diff --git a/container-search/src/main/java/com/yahoo/search/cluster/TrafficNodeMonitor.java b/container-search/src/main/java/com/yahoo/search/cluster/TrafficNodeMonitor.java index 830014bca46..ea881ad8b48 100644 --- a/container-search/src/main/java/com/yahoo/search/cluster/TrafficNodeMonitor.java +++ b/container-search/src/main/java/com/yahoo/search/cluster/TrafficNodeMonitor.java @@ -36,7 +36,6 @@ public class TrafficNodeMonitor extends BaseNodeMonitor { @Override public void failed(ErrorMessage error) { respondedAt = now(); - atStartUp = false; if (error.getCode() == Error.BACKEND_COMMUNICATION_ERROR.code) { setWorking(false, "Connection failure: " + error.toString()); @@ -61,14 +60,9 @@ public class TrafficNodeMonitor extends BaseNodeMonitor { setWorking(true,"Responds correctly"); } - /** - * Returns whether this node is currently is a state suitable for receiving traffic, or null if not known - */ - public Boolean isKnownWorking() { return atStartUp ? null : isWorking; } - /** Thread-safely changes the state of this node if required */ - protected synchronized void setWorking(boolean working, String explanation) { - if (this.isWorking == working) return; // Old news + protected synchronized void setWorking(boolean working,String explanation) { + if (this.isWorking==working) return; // Old news if (explanation==null) { explanation=""; diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java index af34fc3e106..58f73ea52cc 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java @@ -139,7 +139,7 @@ public class Dispatcher extends AbstractComponent { return invoker; } - /** Builds an invoker based on searchpath */ + // build invoker based on searchpath private Optional getSearchPathInvoker(Query query, VespaBackEndSearcher searcher) { String searchPath = query.getModel().getSearchPath(); if (searchPath == null) return Optional.empty(); @@ -156,7 +156,7 @@ public class Dispatcher extends AbstractComponent { } private Optional getInternalInvoker(Query query, VespaBackEndSearcher searcher) { - Optional directNode = searchCluster.localCorpusDispatchTarget(); + Optional directNode = searchCluster.directDispatchTarget(); if (directNode.isPresent()) { Node node = directNode.get(); query.trace(false, 2, "Dispatching directly to ", node); @@ -202,5 +202,4 @@ public class Dispatcher extends AbstractComponent { metric.add(INTERNAL_METRIC, 1, metricContext); } } - } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/InvokerFactory.java b/container-search/src/main/java/com/yahoo/search/dispatch/InvokerFactory.java index 78c50254a84..31af74a39b2 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/InvokerFactory.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/InvokerFactory.java @@ -48,24 +48,21 @@ public abstract class InvokerFactory { * @return Optional containing the SearchInvoker or empty if some node in the * list is invalid and the remaining coverage is not sufficient */ - public Optional createSearchInvoker(VespaBackEndSearcher searcher, - Query query, - OptionalInt groupId, - List nodes, - boolean acceptIncompleteCoverage) { + public Optional createSearchInvoker(VespaBackEndSearcher searcher, Query query, OptionalInt groupId, List nodes, + boolean acceptIncompleteCoverage) { List invokers = new ArrayList<>(nodes.size()); Set failed = null; for (Node node : nodes) { boolean nodeAdded = false; - if (node.isWorking() != Boolean.FALSE) { + if (node.isWorking()) { Optional invoker = createNodeSearchInvoker(searcher, query, node); - if (invoker.isPresent()) { + if(invoker.isPresent()) { invokers.add(invoker.get()); nodeAdded = true; } } - if ( ! nodeAdded) { + if (!nodeAdded) { if (failed == null) { failed = new HashSet<>(); } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Group.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Group.java index 0e4e87b9a6a..bafc72b9b43 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Group.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Group.java @@ -54,7 +54,7 @@ public class Group { public int workingNodes() { int nodesUp = 0; for (Node node : nodes) { - if (node.isWorking() == Boolean.TRUE) { + if (node.isWorking()) { nodesUp++; } } @@ -64,7 +64,7 @@ public class Group { void aggregateActiveDocuments() { long activeDocumentsInGroup = 0; for (Node node : nodes) { - if (node.isWorking() == Boolean.TRUE) { + if (node.isWorking()) { activeDocumentsInGroup += node.getActiveDocuments(); } } @@ -77,11 +77,6 @@ public class Group { return this.activeDocuments.get(); } - public boolean isFullCoverageStatusChanged(boolean hasFullCoverageNow) { - boolean previousState = hasFullCoverage.getAndSet(hasFullCoverageNow); - return previousState != hasFullCoverageNow; - } - @Override public String toString() { return "search group " + id; } @@ -95,4 +90,8 @@ public class Group { return ((Group) other).id == this.id; } + public boolean isFullCoverageStatusChanged(boolean hasFullCoverageNow) { + boolean previousState = hasFullCoverage.getAndSet(hasFullCoverageNow); + return previousState != hasFullCoverageNow; + } } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Node.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Node.java index b47f2fefa5b..a71ce0354f9 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Node.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Node.java @@ -19,7 +19,6 @@ public class Node { private final int fs4port; final int group; - private final AtomicBoolean statusIsKnown = new AtomicBoolean(false); private final AtomicBoolean working = new AtomicBoolean(true); private final AtomicLong activeDocuments = new AtomicLong(0); @@ -47,14 +46,11 @@ public class Node { public int group() { return group; } public void setWorking(boolean working) { - this.statusIsKnown.lazySet(true); this.working.lazySet(working); } - /** Returns whether this node is currently responding to requests, or null if status is not known */ - public Boolean isWorking() { - return statusIsKnown.get() ? working.get() : null; - } + /** Returns whether this node is currently responding to requests */ + public boolean isWorking() { return working.get(); } /** Updates the active documents on this node */ public void setActiveDocuments(long activeDocuments) { @@ -81,5 +77,4 @@ public class Node { @Override public String toString() { return "search node " + hostname + ":" + fs4port + " in group " + group; } - } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/PingFactory.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/PingFactory.java index 285a1fcd57e..c9f722ef79b 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/PingFactory.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/PingFactory.java @@ -6,7 +6,5 @@ import com.yahoo.search.cluster.ClusterMonitor; import java.util.concurrent.Callable; public interface PingFactory { - Callable createPinger(Node node, ClusterMonitor monitor); - } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java index c17212b2481..c091e211aca 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java @@ -55,7 +55,7 @@ public class SearchCluster implements NodeManager { * if it only queries this cluster when the local node cannot be used, to avoid unnecessary * cross-node network traffic. */ - private final Optional localCorpusDispatchTarget; + private final Optional directDispatchTarget; public SearchCluster(String clusterId, DispatchConfig dispatchConfig, int containerClusterSize, VipStatus vipStatus) { this.clusterId = clusterId; @@ -82,11 +82,8 @@ public class SearchCluster implements NodeManager { nodesByHostBuilder.put(node.hostname(), node); this.nodesByHost = nodesByHostBuilder.build(); - this.localCorpusDispatchTarget = findLocalCorpusDispatchTarget(HostName.getLocalhost(), - size, - containerClusterSize, - nodesByHost, - groups); + this.directDispatchTarget = findDirectDispatchTarget(HostName.getLocalhost(), size, containerClusterSize, + nodesByHost, groups); this.clusterMonitor = new ClusterMonitor<>(this); } @@ -95,18 +92,21 @@ public class SearchCluster implements NodeManager { this.pingFactory = pingFactory; for (var group : orderedGroups) { - for (var node : group.nodes()) + for (var node : group.nodes()) { + // cluster monitor will only call working() when the + // node transitions from down to up, so we need to + // register the initial (working) state here: + working(node); clusterMonitor.add(node, true); + } } } - ClusterMonitor clusterMonitor() { return clusterMonitor; } - - private static Optional findLocalCorpusDispatchTarget(String selfHostname, - int searchClusterSize, - int containerClusterSize, - ImmutableMultimap nodesByHost, - ImmutableMap groups) { + private static Optional findDirectDispatchTarget(String selfHostname, + int searchClusterSize, + int containerClusterSize, + ImmutableMultimap nodesByHost, + ImmutableMap groups) { // A search node in the search cluster in question is configured on the same host as the currently running container. // It has all the data <==> No other nodes in the search cluster have the same group id as this node. // That local search node responds. @@ -186,90 +186,70 @@ public class SearchCluster implements NodeManager { } /** - * Returns the single, local node we should dispatch queries directly to, + * Returns the recipient we should dispatch queries directly to (bypassing fdispatch), * or empty if we should not dispatch directly. */ - public Optional localCorpusDispatchTarget() { - if ( localCorpusDispatchTarget.isEmpty()) return Optional.empty(); + public Optional directDispatchTarget() { + if ( directDispatchTarget.isEmpty()) return Optional.empty(); // Only use direct dispatch if the local group has sufficient coverage - Group localSearchGroup = groups.get(localCorpusDispatchTarget.get().group()); + Group localSearchGroup = groups.get(directDispatchTarget.get().group()); if ( ! localSearchGroup.hasSufficientCoverage()) return Optional.empty(); - // Only use direct dispatch if the local search node is not down - if ( localCorpusDispatchTarget.get().isWorking() == Boolean.FALSE) return Optional.empty(); + // Only use direct dispatch if the local search node is up + if ( ! directDispatchTarget.get().isWorking()) return Optional.empty(); - return localCorpusDispatchTarget; + return directDispatchTarget; } - /** Called by the cluster monitor when node state changes to working */ + /** Used by the cluster monitor to manage node status */ @Override public void working(Node node) { node.setWorking(true); - updateVipStatusOnNodeChange(node, true); + + if (usesDirectDispatchTo(node)) + vipStatus.addToRotation(clusterId); } - /** Called by the cluster monitor when node state changes to failed */ + /** Used by the cluster monitor to manage node status */ @Override public void failed(Node node) { node.setWorking(false); - updateVipStatusOnNodeChange(node, true); + + // Take ourselves out if we usually dispatch only to our own host + if (usesDirectDispatchTo(node)) + vipStatus.removeFromRotation(clusterId); } private void updateSufficientCoverage(Group group, boolean sufficientCoverage) { - if (sufficientCoverage == group.hasSufficientCoverage()) return; // no change - + // update VIP status if we direct dispatch to this group and coverage status changed + boolean isInRotation = vipStatus.isInRotation(); + boolean hasChanged = sufficientCoverage != group.hasSufficientCoverage(); + boolean isDirectDispatchGroupAndChange = usesDirectDispatchTo(group) && hasChanged; group.setHasSufficientCoverage(sufficientCoverage); - updateVipStatusOnCoverageChange(group, sufficientCoverage); - } - - private void updateVipStatusOnNodeChange(Node node, boolean nodeIsWorking) { - if (localCorpusDispatchTarget.isEmpty()) { // consider entire cluster - if (hasInformationAboutAllNodes()) - setInRotationOnlyIf(hasWorkingNodes()); - } - else if (usesLocalCorpusIn(node)) { // follow the status of this node - setInRotationOnlyIf(nodeIsWorking); - } - } - - private void updateVipStatusOnCoverageChange(Group group, boolean sufficientCoverage) { - if ( localCorpusDispatchTarget.isEmpty()) { // consider entire cluster - // VIP status does not depend on coverage - } - else if (usesLocalCorpusIn(group)) { // follow the status of this group - setInRotationOnlyIf(sufficientCoverage); - } - } - - private void setInRotationOnlyIf(boolean inRotation) { - if (inRotation) + if ((!isInRotation || isDirectDispatchGroupAndChange) && sufficientCoverage) { + // We will set this cluster in rotation if + // - not already in rotation and one group has sufficient coverage. vipStatus.addToRotation(clusterId); - else + } else if (isDirectDispatchGroupAndChange) { + // We will take it out of rotation if the group is mandatory (direct dispatch to this group) vipStatus.removeFromRotation(clusterId); + } } - private boolean hasInformationAboutAllNodes() { - return nodesByHost.values().stream().allMatch(node -> node.isWorking() != null); - } - - private boolean hasWorkingNodes() { - return nodesByHost.values().stream().anyMatch(node -> node.isWorking() != Boolean.FALSE ); - } - - private boolean usesLocalCorpusIn(Node node) { - return localCorpusDispatchTarget.isPresent() && localCorpusDispatchTarget.get().equals(node); + private boolean usesDirectDispatchTo(Node node) { + return directDispatchTarget.isPresent() && directDispatchTarget.get().equals(node); } - private boolean usesLocalCorpusIn(Group group) { - return localCorpusDispatchTarget.isPresent() && localCorpusDispatchTarget.get().group() == group.id(); + private boolean usesDirectDispatchTo(Group group) { + return directDispatchTarget.isPresent() && directDispatchTarget.get().group() == group.id(); } /** Used by the cluster monitor to manage node status */ @Override public void ping(Node node, Executor executor) { - if (pingFactory == null) return; // not initialized yet - + if (pingFactory == null) // not initialized yet + return; FutureTask futurePong = new FutureTask<>(pingFactory.createPinger(node, clusterMonitor)); executor.execute(futurePong); Pong pong = getPong(futurePong, node); @@ -292,10 +272,9 @@ public class SearchCluster implements NodeManager { // group will always be marked sufficient for use. updateSufficientCoverage(group, true); boolean fullCoverage = isGroupCoverageSufficient(group.workingNodes(), group.nodes().size(), group.getActiveDocuments(), - group.getActiveDocuments()); + group.getActiveDocuments()); trackGroupCoverageChanges(0, group, fullCoverage, group.getActiveDocuments()); } - private void pingIterationCompletedMultipleGroups() { int numGroups = orderedGroups.size(); // Update active documents per group and use it to decide if the group should be active @@ -320,7 +299,13 @@ public class SearchCluster implements NodeManager { trackGroupCoverageChanges(i, group, sufficientCoverage, averageDocumentsInOtherGroups); } } - + private boolean areAllNodesDownInAllgroups() { + for(int i = 0; i < groups.size(); i++) { + Group group = orderedGroups.get(i); + if (group.workingNodes() > 0) return false; + } + return true; + } /** * Update statistics after a round of issuing pings. * Note that this doesn't wait for pings to return, so it will typically accumulate data from @@ -334,6 +319,9 @@ public class SearchCluster implements NodeManager { } else { pingIterationCompletedMultipleGroups(); } + if ( areAllNodesDownInAllgroups() ) { + vipStatus.removeFromRotation(clusterId); + } } private boolean isGroupCoverageSufficient(int workingNodes, int nodesInGroup, long activeDocuments, long averageDocumentsInOtherGroups) { @@ -409,11 +397,11 @@ public class SearchCluster implements NodeManager { if (changed) { int requiredNodes = groupSize() - dispatchConfig.maxNodesDownPerGroup(); if (fullCoverage) { - log.info(() -> String.format("Group %d is now good again (%d/%d active docs, coverage %d/%d)", - index, group.getActiveDocuments(), averageDocuments, group.workingNodes(), groupSize())); + log.info(() -> String.format("Group %d is now good again (%d/%d active docs, coverage %d/%d)", index, + group.getActiveDocuments(), averageDocuments, group.workingNodes(), groupSize())); } else { - log.warning(() -> String.format("Coverage of group %d is only %d/%d (requires %d)", - index, group.workingNodes(), groupSize(), requiredNodes)); + log.warning(() -> String.format("Coverage of group %d is only %d/%d (requires %d)", index, group.workingNodes(), groupSize(), + requiredNodes)); } } } diff --git a/container-search/src/test/java/com/yahoo/search/cluster/test/ClusteredConnectionTestCase.java b/container-search/src/test/java/com/yahoo/search/cluster/test/ClusteredConnectionTestCase.java index aa57e7903f9..84c10991293 100644 --- a/container-search/src/test/java/com/yahoo/search/cluster/test/ClusteredConnectionTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/cluster/test/ClusteredConnectionTestCase.java @@ -114,7 +114,7 @@ public class ClusteredConnectionTestCase { connection0.setInService(false); forcePing(myBackend); r=new Execution(myBackend, Execution.Context.createContextStub()).search(new SimpleQuery(0)); - assertEquals("No backends in service. Try later", r.hits().getError().getMessage()); + assertEquals("No backends in service. Try later",r.hits().getError().getMessage()); connection2.setInService(true); connection1.setInService(true); diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java index 310f536f961..03a417c9bbb 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java @@ -69,7 +69,7 @@ public class DispatcherTest { public void requireThatDispatcherSupportsSingleNodeDirectDispatch() { SearchCluster cl = new MockSearchCluster("1", 0, 0) { @Override - public Optional localCorpusDispatchTarget() { + public Optional directDispatchTarget() { return Optional.of(new Node(1, "test", 123, 1)); } }; diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java b/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java index e3ff54102d4..7ee62ae9978 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java @@ -10,7 +10,6 @@ import com.yahoo.search.dispatch.searchcluster.SearchCluster; import com.yahoo.vespa.config.search.DispatchConfig; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import java.util.Optional; @@ -90,7 +89,7 @@ public class MockSearchCluster extends SearchCluster { } @Override - public Optional localCorpusDispatchTarget() { + public Optional directDispatchTarget() { return Optional.empty(); } @@ -107,14 +106,8 @@ public class MockSearchCluster extends SearchCluster { public static DispatchConfig createDispatchConfig(Node... nodes) { return createDispatchConfig(100.0, nodes); } - public static DispatchConfig createDispatchConfig(List nodes) { - return createDispatchConfig(100.0, nodes); - } public static DispatchConfig createDispatchConfig(double minSearchCoverage, Node... nodes) { - return createDispatchConfig(minSearchCoverage, Arrays.asList(nodes)); - } - public static DispatchConfig createDispatchConfig(double minSearchCoverage, List nodes) { DispatchConfig.Builder builder = new DispatchConfig.Builder(); builder.minActivedocsPercentage(88.0); builder.minGroupCoverage(99.0); diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTest.java deleted file mode 100644 index bde0a3c6c02..00000000000 --- a/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTest.java +++ /dev/null @@ -1,279 +0,0 @@ -package com.yahoo.search.dispatch.searchcluster; - -import com.yahoo.container.QrSearchersConfig; -import com.yahoo.container.handler.ClustersStatus; -import com.yahoo.container.handler.VipStatus; -import com.yahoo.net.HostName; -import com.yahoo.prelude.Pong; -import com.yahoo.search.cluster.ClusterMonitor; -import com.yahoo.search.dispatch.MockSearchCluster; -import com.yahoo.search.result.ErrorMessage; -import org.jetbrains.annotations.NotNull; -import org.junit.Test; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.concurrent.Callable; -import java.util.concurrent.Executor; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; - -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -/** - * @author baldersheim - */ -public class SearchClusterTest { - - static class State { - class MyExecutor implements Executor { - private final List list = new ArrayList<>(); - @Override - public void execute(@NotNull Runnable command) { - list.add(command); - } - void run() { - for (Runnable runnable : list) { - runnable.run(); - } - list.clear(); - } - } - final String clusterId; - final int nodesPerGroup; - final VipStatus vipStatus; - final SearchCluster searchCluster; - final List numDocsPerNode; - List pingCounts; - State(String clusterId, int nodesPergroup, String ... nodeNames) { - this(clusterId, nodesPergroup, Arrays.asList(nodeNames)); - } - State(String clusterId, int nodesPergroup, List nodeNames) { - this.clusterId = clusterId; - this.nodesPerGroup = nodesPergroup; - vipStatus = new VipStatus(new QrSearchersConfig.Builder().searchcluster(new QrSearchersConfig.Searchcluster.Builder().name(clusterId)).build(), new ClustersStatus()); - numDocsPerNode = new ArrayList<>(nodeNames.size()); - pingCounts = new ArrayList<>(nodeNames.size()); - List nodes = new ArrayList<>(nodeNames.size()); - - for (String name : nodeNames) { - int key = nodes.size() % nodesPergroup; - int group = nodes.size() / nodesPergroup; - nodes.add(new Node(key, name, 13333, group)); - numDocsPerNode.add(new AtomicInteger(1)); - pingCounts.add(new AtomicInteger(0)); - } - searchCluster = new SearchCluster(clusterId, MockSearchCluster.createDispatchConfig(nodes), nodes.size() / nodesPergroup, vipStatus); - } - void startMonitoring() { - searchCluster.startClusterMonitoring(new Factory(nodesPerGroup, numDocsPerNode, pingCounts)); - } - static private int getMaxValue(List list) { - int max = list.get(0).get(); - for (AtomicInteger v : list) { - if (v.get() > max) { - max = v.get(); - } - } - return max; - } - private static int getMinValue(List list) { - int min = list.get(0).get(); - for (AtomicInteger v : list) { - if (v.get() < min) { - min = v.get(); - } - } - return min; - } - private void waitAtLeast(int atLeast, List list) { - while (getMinValue(list) < atLeast) { - ExecutorService executor = Executors.newCachedThreadPool(); - searchCluster.clusterMonitor().ping(executor); - executor.shutdown(); - try { - executor.awaitTermination(60, TimeUnit.SECONDS); - } catch (InterruptedException e) {} - } - } - void waitOneFullPingRound() { - waitAtLeast(getMaxValue(pingCounts) + 1, pingCounts); - } - static class Factory implements PingFactory { - static class Pinger implements Callable { - private final AtomicInteger numDocs; - private final AtomicInteger pingCount; - Pinger(AtomicInteger numDocs, AtomicInteger pingCount) { - this.numDocs = numDocs; - this.pingCount = pingCount; - } - @Override - public Pong call() throws Exception { - int docs = numDocs.get(); - pingCount.incrementAndGet(); - return (docs < 0) - ? new Pong(ErrorMessage.createBackendCommunicationError("Negative numDocs = " + docs)) - : new Pong(docs); - } - } - - private final List activeDocs; - private final List pingCounts; - private final int numPerGroup; - Factory(int numPerGroup, List activeDocs, List pingCounts) { - this.numPerGroup = numPerGroup; - this.activeDocs = activeDocs; - this.pingCounts = pingCounts; - } - - @Override - public Callable createPinger(Node node, ClusterMonitor monitor) { - int index = node.group*numPerGroup + node.key(); - return new Pinger(activeDocs.get(index), pingCounts.get(index)); - } - } - } - - @Test - public void requireThatVipStatusIsDefaultDownButComesUpAfterPinging() { - State test = new State("cluster.1", 2, "a", "b"); - assertTrue(test.searchCluster.localCorpusDispatchTarget().isEmpty()); - - assertFalse(test.vipStatus.isInRotation()); - test.startMonitoring(); - test.waitOneFullPingRound(); - assertTrue(test.vipStatus.isInRotation()); - } - - @Test - public void requireThatZeroDocsAreFine() { - State test = new State("cluster.1", 2,"a", "b"); - test.startMonitoring(); - test.waitOneFullPingRound(); - - assertTrue(test.vipStatus.isInRotation()); - assertTrue(test.searchCluster.localCorpusDispatchTarget().isEmpty()); - - test.numDocsPerNode.get(0).set(-1); - test.numDocsPerNode.get(1).set(-1); - test.waitOneFullPingRound(); - assertFalse(test.vipStatus.isInRotation()); - test.numDocsPerNode.get(0).set(0); - test.waitOneFullPingRound(); - assertTrue(test.vipStatus.isInRotation()); - } - - @Test - public void requireThatVipStatusIsDefaultDownWithLocalDispatch() { - State test = new State("cluster.1", 1, HostName.getLocalhost(), "b"); - assertTrue(test.searchCluster.localCorpusDispatchTarget().isPresent()); - - assertFalse(test.vipStatus.isInRotation()); - test.startMonitoring(); - test.waitOneFullPingRound(); - assertTrue(test.vipStatus.isInRotation()); - } - - @Test - public void requireThatVipStatusDownWhenLocalIsDown() { - State test = new State("cluster.1",1,HostName.getLocalhost(), "b"); - - test.startMonitoring(); - test.waitOneFullPingRound(); - assertTrue(test.vipStatus.isInRotation()); - assertTrue(test.searchCluster.localCorpusDispatchTarget().isPresent()); - - test.waitOneFullPingRound(); - assertTrue(test.vipStatus.isInRotation()); - test.numDocsPerNode.get(0).set(-1); - test.waitOneFullPingRound(); - assertFalse(test.vipStatus.isInRotation()); - - test.numDocsPerNode.get(0).set(1); - test.waitOneFullPingRound(); - assertTrue(test.vipStatus.isInRotation()); - - test.numDocsPerNode.get(1).set(-1); - test.waitOneFullPingRound(); - assertTrue(test.vipStatus.isInRotation()); - - test.numDocsPerNode.get(0).set(-1); - test.numDocsPerNode.get(1).set(-1); - test.waitOneFullPingRound(); - assertFalse(test.vipStatus.isInRotation()); - test.numDocsPerNode.get(1).set(1); - test.waitOneFullPingRound(); - assertFalse(test.vipStatus.isInRotation()); - test.numDocsPerNode.get(0).set(1); - test.waitOneFullPingRound(); - assertTrue(test.vipStatus.isInRotation()); - } - - private void verifyThatVipStatusDownRequireAllNodesDown(int numGroups, int nodesPerGroup) { - List nodeNames = generateNodeNames(numGroups, nodesPerGroup); - State test = new State("cluster.1", nodesPerGroup, nodeNames); - test.startMonitoring(); - test.waitOneFullPingRound(); - assertTrue(test.vipStatus.isInRotation()); - assertTrue(test.searchCluster.localCorpusDispatchTarget().isEmpty()); - - test.waitOneFullPingRound(); - assertTrue(test.vipStatus.isInRotation()); - - for (int i=0; i < test.numDocsPerNode.size()-1; i++) { - test.numDocsPerNode.get(i).set(-1); - } - test.waitOneFullPingRound(); - assertTrue(test.vipStatus.isInRotation()); - test.numDocsPerNode.get(test.numDocsPerNode.size()-1).set(-1); - test.waitOneFullPingRound(); - assertFalse(test.vipStatus.isInRotation()); - } - @Test - public void requireThatVipStatusDownRequireAllNodesDown() { - verifyThatVipStatusDownRequireAllNodesDown(1,2); - verifyThatVipStatusDownRequireAllNodesDown(3, 3); - } - - static private List generateNodeNames(int numGroups, int nodesPerGroup) { - List nodeNames = new ArrayList<>(numGroups*nodesPerGroup); - for (int g = 0; g < numGroups; g++) { - for (int n=0; n < nodesPerGroup; n++) { - nodeNames.add(new StringBuilder("node.").append(g).append('.').append(n).toString()); - } - } - return nodeNames; - } - - private void verifyThatVipStatusUpRequireOnlyOneOnlineNode(int numGroups, int nodesPerGroup) { - List nodeNames = generateNodeNames(numGroups, nodesPerGroup); - State test = new State("cluster.1", nodesPerGroup, nodeNames); - test.startMonitoring(); - test.waitOneFullPingRound(); - assertTrue(test.vipStatus.isInRotation()); - assertTrue(test.searchCluster.localCorpusDispatchTarget().isEmpty()); - - for (int i=0; i < test.numDocsPerNode.size()-1; i++) { - test.numDocsPerNode.get(i).set(-1); - } - test.waitOneFullPingRound(); - assertTrue(test.vipStatus.isInRotation()); - test.numDocsPerNode.get(test.numDocsPerNode.size()-1).set(-1); - test.waitOneFullPingRound(); - assertFalse(test.vipStatus.isInRotation()); - - test.numDocsPerNode.get(0).set(0); - test.waitOneFullPingRound(); - assertTrue(test.vipStatus.isInRotation()); - } - @Test - public void requireThatVipStatusUpRequireOnlyOneOnlineNode() { - verifyThatVipStatusUpRequireOnlyOneOnlineNode(1, 2); - verifyThatVipStatusUpRequireOnlyOneOnlineNode(3, 3); - } - -} -- cgit v1.2.3