diff options
Diffstat (limited to 'container-search/src/main/java/com')
7 files changed, 63 insertions, 28 deletions
diff --git a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java index 1f621eb926c..0c8e564578b 100644 --- a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java @@ -7,7 +7,6 @@ import com.yahoo.component.chain.dependencies.After; import com.yahoo.container.QrSearchersConfig; import com.yahoo.container.handler.VipStatus; import com.yahoo.jdisc.Metric; -import com.yahoo.net.HostName; import com.yahoo.prelude.IndexFacts; import com.yahoo.prelude.fastsearch.ClusterParams; import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; @@ -27,8 +26,6 @@ import com.yahoo.vespa.config.search.DispatchConfig; import com.yahoo.vespa.streamingvisitors.VdsStreamingSearcher; import org.apache.commons.lang.StringUtils; -import java.net.InetAddress; -import java.net.UnknownHostException; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -371,6 +368,10 @@ public class ClusterSearcher extends Searcher { } @Override - public void deconstruct() { } + public void deconstruct() { + if (server != null) { + server.shutDown(); + } + } } diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java index b0b3a7800e9..9a4913b3840 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastSearcher.java @@ -173,4 +173,9 @@ public class FastSearcher extends VespaBackEndSearcher { return getLogger().isLoggable(Level.FINE); } + @Override + public void shutDown() { + super.shutDown(); + dispatcher.shutDown(); + } } diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java index 8f4b49ac71e..bc3ac6cdef1 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java @@ -392,4 +392,6 @@ public abstract class VespaBackEndSearcher extends PingableSearcher { return getLogger().isLoggable(Level.FINE); } + public void shutDown() { } + } diff --git a/container-search/src/main/java/com/yahoo/search/cluster/ClusterMonitor.java b/container-search/src/main/java/com/yahoo/search/cluster/ClusterMonitor.java index 22c7f59872c..6308c0cb8db 100644 --- a/container-search/src/main/java/com/yahoo/search/cluster/ClusterMonitor.java +++ b/container-search/src/main/java/com/yahoo/search/cluster/ClusterMonitor.java @@ -9,7 +9,9 @@ import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.concurrent.Executor; +import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import java.util.logging.Level; import java.util.logging.Logger; @@ -26,9 +28,9 @@ public class ClusterMonitor<T> { private static Logger log = Logger.getLogger(ClusterMonitor.class.getName()); - private NodeManager<T> nodeManager; + private final NodeManager<T> nodeManager; - private MonitorThread monitorThread; + private final MonitorThread monitorThread; private volatile boolean shutdown = false; @@ -119,28 +121,36 @@ public class ClusterMonitor<T> { } public void run() { - log.fine("Starting cluster monitor thread"); + log.info("Starting cluster monitor thread " + getName()); // Pings must happen in a separate thread from this to handle timeouts // By using a cached thread pool we ensured that 1) a single thread will be used // for all pings when there are no problems (important because it ensures that // any thread local connections are reused) 2) a new thread will be started to execute // new pings when a ping is not responding - Executor pingExecutor=Executors.newCachedThreadPool(ThreadFactoryFactory.getDaemonThreadFactory("search.ping")); + ExecutorService pingExecutor=Executors.newCachedThreadPool(ThreadFactoryFactory.getDaemonThreadFactory("search.ping")); while (!isInterrupted()) { try { Thread.sleep(configuration.getCheckInterval()); log.finest("Activating ping"); ping(pingExecutor); } - catch (Exception e) { + catch (Throwable e) { if (shutdown && e instanceof InterruptedException) { break; } else { - log.log(Level.WARNING,"Error in monitor thread",e); + log.log(Level.WARNING,"Exception in monitor thread", e); + if ( ! (e instanceof Exception) ) { + log.log(Level.WARNING,"Error in monitor thread, will quit", e); + break; + } } } } - log.fine("Stopped cluster monitor thread"); + pingExecutor.shutdown(); + try { + pingExecutor.awaitTermination(10, TimeUnit.SECONDS); + } catch (InterruptedException e) { } + log.info("Stopped cluster monitor thread " + getName()); } } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java index 7369b33e82d..ddd319b7bcb 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java @@ -195,6 +195,10 @@ public class Dispatcher extends AbstractComponent { return Optional.empty(); } + public void shutDown() { + searchCluster.shutDown(); + } + private void emitDispatchMetric(Optional<SearchInvoker> invoker) { if (invoker.isEmpty()) { metric.add(FDISPATCH_METRIC, 1, metricContext); diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Node.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Node.java index b47f2fefa5b..53ecd59d991 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Node.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/Node.java @@ -16,17 +16,15 @@ public class Node { private final int key; private int pathIndex; private final String hostname; - private final int fs4port; - final int group; + private final int group; private final AtomicBoolean statusIsKnown = new AtomicBoolean(false); private final AtomicBoolean working = new AtomicBoolean(true); private final AtomicLong activeDocuments = new AtomicLong(0); - public Node(int key, String hostname, int fs4port, int group) { + public Node(int key, String hostname, int group) { this.key = key; this.hostname = hostname; - this.fs4port = fs4port; this.group = group; } @@ -41,14 +39,15 @@ public class Node { public String hostname() { return hostname; } - public int fs4port() { return fs4port; } - /** Returns the id of this group this node belongs to */ public int group() { return group; } public void setWorking(boolean working) { this.statusIsKnown.lazySet(true); this.working.lazySet(working); + if ( ! working ) { + activeDocuments.set(0); + } } /** Returns whether this node is currently responding to requests, or null if status is not known */ @@ -57,17 +56,17 @@ public class Node { } /** Updates the active documents on this node */ - public void setActiveDocuments(long activeDocuments) { + void setActiveDocuments(long activeDocuments) { this.activeDocuments.set(activeDocuments); } /** Returns the active documents on this node. If unknown, 0 is returned. */ - public long getActiveDocuments() { - return this.activeDocuments.get(); + long getActiveDocuments() { + return activeDocuments.get(); } @Override - public int hashCode() { return Objects.hash(hostname, fs4port); } + public int hashCode() { return Objects.hash(hostname, key, pathIndex, group); } @Override public boolean equals(Object o) { @@ -75,11 +74,15 @@ public class Node { if ( ! (o instanceof Node)) return false; Node other = (Node)o; if ( ! Objects.equals(this.hostname, other.hostname)) return false; - if ( ! Objects.equals(this.fs4port, other.fs4port)) return false; + if ( ! Objects.equals(this.key, other.key)) return false; + if ( ! Objects.equals(this.pathIndex, other.pathIndex)) return false; + if ( ! Objects.equals(this.group, other.group)) return false; + return true; } @Override - public String toString() { return "search node " + hostname + ":" + fs4port + " in group " + group; } + public String toString() { return "search node key = " + key + " hostname = "+ hostname + " path = " + pathIndex + " in group " + group + + " statusIsKnown = " + statusIsKnown.get() + " working" + working.get() + " activeDocs = " + activeDocuments.get(); } } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java index a55a970e8ff..7380140ac4a 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java @@ -73,7 +73,7 @@ public class SearchCluster implements NodeManager<Node> { } this.groups = groupsBuilder.build(); LinkedHashMap<Integer, Group> groupIntroductionOrder = new LinkedHashMap<>(); - nodes.forEach(node -> groupIntroductionOrder.put(node.group(), groups.get(node.group))); + nodes.forEach(node -> groupIntroductionOrder.put(node.group(), groups.get(node.group()))); this.orderedGroups = ImmutableList.<Group>builder().addAll(groupIntroductionOrder.values()).build(); // Index nodes by host @@ -91,6 +91,10 @@ public class SearchCluster implements NodeManager<Node> { this.clusterMonitor = new ClusterMonitor<>(this); } + public void shutDown() { + clusterMonitor.shutdown(); + } + public void startClusterMonitoring(PingFactory pingFactory) { this.pingFactory = pingFactory; @@ -141,7 +145,7 @@ public class SearchCluster implements NodeManager<Node> { } for (DispatchConfig.Node node : dispatchConfig.node()) { if (filter.test(node)) { - nodesBuilder.add(new Node(node.key(), node.host(), node.fs4port(), node.group())); + nodesBuilder.add(new Node(node.key(), node.host(), node.group())); } } return nodesBuilder.build(); @@ -409,14 +413,20 @@ public class SearchCluster implements NodeManager<Node> { private void trackGroupCoverageChanges(int index, Group group, boolean fullCoverage, long averageDocuments) { boolean changed = group.isFullCoverageStatusChanged(fullCoverage); - if (changed) { + if (changed || !fullCoverage) { int requiredNodes = groupSize() - dispatchConfig.maxNodesDownPerGroup(); if (fullCoverage) { log.info(() -> String.format("Group %d is now good again (%d/%d active docs, coverage %d/%d)", index, group.getActiveDocuments(), averageDocuments, group.workingNodes(), groupSize())); } else { - log.warning(() -> String.format("Coverage of group %d is only %d/%d (requires %d)", - index, group.workingNodes(), groupSize(), requiredNodes)); + StringBuilder missing = new StringBuilder(); + for (var node : group.nodes()) { + if (node.isWorking() != Boolean.TRUE) { + missing.append('\n').append(node.toString()); + } + } + log.warning(() -> String.format("Coverage of group %d is only %d/%d (requires %d) (%d/%d active docs)\nFailed nodes are:\n%s", + index, group.workingNodes(), groupSize(), requiredNodes, group.getActiveDocuments(), averageDocuments, missing.toString())); } } } |