diff options
3 files changed, 32 insertions, 26 deletions
diff --git a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterMonitor.java b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterMonitor.java index 0312db914df..bb5accd9022 100644 --- a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterMonitor.java +++ b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterMonitor.java @@ -89,7 +89,7 @@ public class ClusterMonitor implements Runnable, Freezable { monitor.responded(hasSearchNodesOnline); if (wasFailing && monitor.isWorking()) { log.info("Failed node '" + node + "' started working again."); - nodeManager.working(monitor.getNode()); + nodeManager.working(node); } updateVipStatus(); } diff --git a/container-search/src/main/java/com/yahoo/prelude/cluster/Hasher.java b/container-search/src/main/java/com/yahoo/prelude/cluster/Hasher.java index eefa90f63f0..0c0da390616 100644 --- a/container-search/src/main/java/com/yahoo/prelude/cluster/Hasher.java +++ b/container-search/src/main/java/com/yahoo/prelude/cluster/Hasher.java @@ -39,6 +39,10 @@ public class Hasher { } static private VespaBackEndSearcher[] addNode(VespaBackEndSearcher node, VespaBackEndSearcher[] oldNodes) { + assert node != null; + for (VespaBackEndSearcher n : oldNodes) { + if (n == node) return oldNodes; // already present + } VespaBackEndSearcher[] newNodes = new VespaBackEndSearcher[oldNodes.length + 1]; System.arraycopy(oldNodes, 0, newNodes, 0, oldNodes.length); newNodes[oldNodes.length] = node; @@ -56,25 +60,25 @@ public class Hasher { } } - private VespaBackEndSearcher[] removeNode(VespaBackEndSearcher node, VespaBackEndSearcher[] nodes) { - VespaBackEndSearcher[] newNodes = null; - for (VespaBackEndSearcher n : nodes) { + private VespaBackEndSearcher[] removeNode(VespaBackEndSearcher node, VespaBackEndSearcher[] oldNodes) { + int newLen = oldNodes.length; + for (VespaBackEndSearcher n : oldNodes) { if (n == node) { - newNodes = new VespaBackEndSearcher[nodes.length - 1]; - break; + --newLen; } } - if (newNodes != null) { - int numToKeep = 0; - - for (VespaBackEndSearcher n : nodes) { - if (n != node) { - newNodes[numToKeep++] = n; - } + if (newLen == oldNodes.length) { + return oldNodes; + } + VespaBackEndSearcher[] newNodes = new VespaBackEndSearcher[newLen]; + int idx = 0; + for (VespaBackEndSearcher n : oldNodes) { + if (n != node) { + newNodes[idx++] = n; } - return newNodes; } - return nodes; + assert idx == newLen; + return newNodes; } /** Removes a node */ @@ -113,18 +117,19 @@ public class Hasher { */ public VespaBackEndSearcher select(int trynum) { VespaBackEndSearcher[] nodes = allNodes; - if (nodes.length == 0) return null; if (localNodes.length > 0) { nodes = localNodes; - if (localNodes.length == 1) { - return nodes[0]; - } else { - return nodes[Math.abs(avoidAllQrsHitSameTld.incrementAndGet() % nodes.length)]; - } - } else { - return nodes[Math.abs(avoidAllQrsHitSameTld.incrementAndGet() % nodes.length)]; } + if (nodes.length == 0) { + return null; + } + int idx = 0; + if (nodes.length > 1) { + idx = Math.abs(avoidAllQrsHitSameTld.incrementAndGet() % nodes.length); + } + assert nodes[idx] != null; + return nodes[idx]; } } diff --git a/container-search/src/main/java/com/yahoo/prelude/cluster/NodeMonitor.java b/container-search/src/main/java/com/yahoo/prelude/cluster/NodeMonitor.java index ab52bab08a0..ba2c693a0fe 100644 --- a/container-search/src/main/java/com/yahoo/prelude/cluster/NodeMonitor.java +++ b/container-search/src/main/java/com/yahoo/prelude/cluster/NodeMonitor.java @@ -28,7 +28,7 @@ public class NodeMonitor { /** The object representing the monitored node */ private final VespaBackEndSearcher node; - private boolean isWorking = false; + private boolean isWorking = true; /** The last time this node responded successfully */ private long succeededAt = 0; @@ -72,6 +72,7 @@ public class NodeMonitor { // Only count not being able to talk to backend at all // as errors we care about if ((respondedAt - succeededAt) > 10000) { + this.searchNodesOnline = false; setWorking(false, "Not working for 10 s: " + error.toString()); } } else { @@ -95,9 +96,9 @@ public class NodeMonitor { if (isWorking == working) return; // Old news if (working && ! atStartUp) - log.info("Putting " + node + " in service:" + explanation); + log.info("Putting " + node + " in service: " + explanation); else if (! atStartUp) - log.info("Taking " + node + " out of service:" + explanation); + log.info("Taking " + node + " out of service: " + explanation); isWorking = working; } |