diff options
Diffstat (limited to 'clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core')
9 files changed, 34 insertions, 33 deletions
diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStateGenerator.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStateGenerator.java index 403c5c6089b..75c6dbe6cec 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStateGenerator.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStateGenerator.java @@ -41,6 +41,7 @@ public class ClusterStateGenerator { Params() { } + // FIXME de-dupe static Map<NodeType, Integer> buildTransitionTimeMap(int distributorTransitionTimeMs, int storageTransitionTimeMs) { Map<com.yahoo.vdslib.state.NodeType, java.lang.Integer> maxTransitionTime = new TreeMap<>(); maxTransitionTime.put(com.yahoo.vdslib.state.NodeType.DISTRIBUTOR, distributorTransitionTimeMs); @@ -64,7 +65,7 @@ public class ClusterStateGenerator { this.transitionTimes = timesMs; return this; } - Params currentTimeInMillis(long currentTimeMs) { + Params currentTimeInMilllis(long currentTimeMs) { this.currentTimeInMillis = currentTimeMs; return this; } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java index 3f3bf62bf4d..29887666a1b 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java @@ -1006,7 +1006,7 @@ public class FleetController implements NodeListener, SlobrokListener, SystemSta private AnnotatedClusterState computeCurrentAnnotatedState() { ClusterStateGenerator.Params params = ClusterStateGenerator.Params.fromOptions(options); - params.currentTimeInMillis(timer.getCurrentTimeInMillis()) + params.currentTimeInMilllis(timer.getCurrentTimeInMillis()) .cluster(cluster) .lowestObservedDistributionBitCount(stateVersionTracker.getLowestObservedDistributionBits()); return ClusterStateGenerator.generatedStateFrom(params); @@ -1200,7 +1200,7 @@ public class FleetController implements NodeListener, SlobrokListener, SystemSta while (true) { int distCount = 0, storCount = 0; for (NodeInfo info : cluster.getNodeInfos()) { - if (info.isInSlobrok()) { + if (!info.isRpcAddressOutdated()) { if (info.isDistributor()) ++distCount; else ++storCount; } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeInfo.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeInfo.java index 746432f1d38..2993784dba4 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeInfo.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeInfo.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.clustercontroller.core; import com.yahoo.collections.Pair; import com.yahoo.jrt.Target; +import java.util.logging.Level; import com.yahoo.vdslib.distribution.Distribution; import com.yahoo.vdslib.distribution.Group; import com.yahoo.vdslib.state.ClusterState; @@ -12,12 +13,12 @@ import com.yahoo.vdslib.state.NodeType; import com.yahoo.vdslib.state.State; import com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo; import com.yahoo.vespa.clustercontroller.core.rpc.RPCCommunicator; + import java.io.PrintWriter; import java.io.StringWriter; import java.util.LinkedList; import java.util.List; import java.util.TreeMap; -import java.util.logging.Level; import java.util.logging.Logger; /** @@ -243,13 +244,11 @@ abstract public class NodeInfo implements Comparable<NodeInfo> { public ContentCluster getCluster() { return cluster; } - /** Returns true if the node is registered in slobrok */ - public boolean isInSlobrok() { return lastSeenInSlobrok == null; } - - /** Returns true if the node is NOT registered in slobrok */ - public boolean isNotInSlobrok() { return ! isInSlobrok(); } + /** Returns true if the node is currently registered in slobrok */ + // FIXME why is this called "isRpcAddressOutdated" then??? + public boolean isRpcAddressOutdated() { return lastSeenInSlobrok != null; } - public Long lastSeenInSlobrok() { return lastSeenInSlobrok; } + public Long getRpcAddressOutdatedTimestamp() { return lastSeenInSlobrok; } public void abortCurrentNodeStateRequests() { for(Pair<GetNodeStateRequest, Long> it : pendingNodeStateRequests) { @@ -276,7 +275,7 @@ abstract public class NodeInfo implements Comparable<NodeInfo> { return wantedState; } - /** Returns the wanted state set directly by a user (i.e. not configured) */ + /** Returns the wanted state set directly by a user (i.e not configured) */ public NodeState getUserWantedState() { return wantedState; } public long getTimeOfFirstFailingConnectionAttempt() { diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeStateGatherer.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeStateGatherer.java index 68e46414c22..69e97de84f9 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeStateGatherer.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeStateGatherer.java @@ -3,13 +3,14 @@ package com.yahoo.vespa.clustercontroller.core; import com.yahoo.jrt.ErrorCode; import com.yahoo.jrt.Target; +import java.util.logging.Level; import com.yahoo.vdslib.state.NodeState; import com.yahoo.vdslib.state.State; import com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo; import com.yahoo.vespa.clustercontroller.core.listeners.NodeListener; + import java.util.LinkedList; import java.util.List; -import java.util.logging.Level; import java.util.logging.Logger; /** @@ -62,10 +63,10 @@ public class NodeStateGatherer { if (requestTime != null && (currentTime - requestTime < nodeStateRequestTimeoutMS)) continue; // pending request if (info.getTimeForNextStateRequestAttempt() > currentTime) continue; // too early - if (info.getRpcAddress() == null || info.isNotInSlobrok()) { // Cannot query state of node without RPC address or not in slobrok + if (info.getRpcAddress() == null || info.isRpcAddressOutdated()) { // Cannot query state of node without RPC address log.log(Level.FINE, () -> "Not sending getNodeState request to node " + info.getNode() + ": Not in slobrok"); NodeState reportedState = info.getReportedState().clone(); - if (( ! reportedState.getState().equals(State.DOWN) && currentTime - info.lastSeenInSlobrok() > maxSlobrokDisconnectGracePeriod) + if (( ! reportedState.getState().equals(State.DOWN) && currentTime - info.getRpcAddressOutdatedTimestamp() > maxSlobrokDisconnectGracePeriod) || reportedState.getState().equals(State.STOPPING)) // Don't wait for grace period if we expect node to be stopping { log.log(Level.FINE, () -> "Setting reported state to DOWN " @@ -74,8 +75,8 @@ public class NodeStateGatherer { : "as node has been out of slobrok longer than " + maxSlobrokDisconnectGracePeriod + ".")); if (reportedState.getState().oneOf("iur") || ! reportedState.hasDescription()) { StringBuilder sb = new StringBuilder().append("Set node down as it has been out of slobrok for ") - .append(currentTime - info.lastSeenInSlobrok()).append(" ms which is more than the max limit of ") - .append(maxSlobrokDisconnectGracePeriod).append(" ms."); + .append(currentTime - info.getRpcAddressOutdatedTimestamp()).append(" ms which is more than the max limit of ") + .append(maxSlobrokDisconnectGracePeriod).append(" ms."); reportedState.setDescription(sb.toString()); } reportedState.setState(State.DOWN); @@ -180,7 +181,7 @@ public class NodeStateGatherer { newState.setState(State.DOWN); } else if (msg.equals("jrt: Connection closed by peer") || msg.equals("Connection reset by peer")) { msg = "Connection error: Closed at other end. (Node or switch likely shut down)"; - if (info.isNotInSlobrok()) { + if (info.isRpcAddressOutdated()) { msg += " Node is no longer in slobrok."; } if (info.getReportedState().getState().oneOf("ui")) { diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StateChangeHandler.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StateChangeHandler.java index 9897e3cf04c..4c832592422 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StateChangeHandler.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StateChangeHandler.java @@ -329,7 +329,7 @@ public class StateChangeHandler { { return currentStateInSystem.getState().equals(State.MAINTENANCE) && node.getWantedState().above(new NodeState(node.getNode().getType(), State.DOWN)) - && (lastReportedState.getState().equals(State.DOWN) || node.isNotInSlobrok()) + && (lastReportedState.getState().equals(State.DOWN) || node.isRpcAddressOutdated()) && node.getTransitionTime() + maxTransitionTime.get(node.getNode().getType()) < currentTime; } @@ -339,14 +339,14 @@ public class StateChangeHandler { NodeInfo node, NodeState lastReportedState) { - if (node.isNotInSlobrok() + if (node.isRpcAddressOutdated() && !lastReportedState.getState().equals(State.DOWN) - && node.lastSeenInSlobrok() + maxSlobrokDisconnectGracePeriod <= currentTime) + && node.getRpcAddressOutdatedTimestamp() + maxSlobrokDisconnectGracePeriod <= currentTime) { final String desc = String.format( "Set node down as it has been out of slobrok for %d ms which " + "is more than the max limit of %d ms.", - currentTime - node.lastSeenInSlobrok(), + currentTime - node.getRpcAddressOutdatedTimestamp(), maxSlobrokDisconnectGracePeriod); node.abortCurrentNodeStateRequests(); NodeState state = lastReportedState.clone(); diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/SystemStateBroadcaster.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/SystemStateBroadcaster.java index 004cefe1e3c..2359e4d8389 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/SystemStateBroadcaster.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/SystemStateBroadcaster.java @@ -161,7 +161,7 @@ public class SystemStateBroadcaster { } private static boolean nodeIsReachable(NodeInfo node) { - if (node.getRpcAddress() == null || node.isNotInSlobrok()) { + if (node.getRpcAddress() == null || node.isRpcAddressOutdated()) { return false; // Can't set state on nodes we don't know where are } if (node.getReportedState().getState() == State.MAINTENANCE || diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/RpcServer.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/RpcServer.java index 06f6777ab80..dc21693dcdb 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/RpcServer.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/RpcServer.java @@ -199,7 +199,7 @@ public class RpcServer { if (!e.getMessage().equals(lastConnectError) || time - lastConnectErrorTime > 60 * 1000) { lastConnectError = e.getMessage(); lastConnectErrorTime = time; - log.log(Level.WARNING, "Failed to initialize RPC server socket: " + e.getMessage()); + log.log(Level.WARNING, "Failed to initailize RPC server socket: " + e.getMessage()); } } } @@ -227,8 +227,8 @@ public class RpcServer { } if (req.methodName().equals("getNodeList")) { log.log(Level.FINE, "Resolving RPC getNodeList request"); - List<String> slobrok = new ArrayList<>(); - List<String> rpc = new ArrayList<>(); + List<String> slobrok = new ArrayList<String>(); + List<String> rpc = new ArrayList<String>(); for(NodeInfo node : cluster.getNodeInfos()) { String s1 = node.getSlobrokAddress(); String s2 = node.getRpcAddress(); @@ -236,8 +236,8 @@ public class RpcServer { slobrok.add(s1); rpc.add(s2 == null ? "" : s2); } - req.returnValues().add(new StringArray(slobrok.toArray(new String[0]))); - req.returnValues().add(new StringArray(rpc.toArray(new String[0]))); + req.returnValues().add(new StringArray(slobrok.toArray(new String[slobrok.size()]))); + req.returnValues().add(new StringArray(rpc.toArray(new String[rpc.size()]))); req.returnRequest(); } else if (req.methodName().equals("getSystemState")) { log.log(Level.FINE, "Resolving RPC getSystemState request"); @@ -280,7 +280,7 @@ public class RpcServer { NodeState oldState = node.getUserWantedState(); String message = (nodeState.getState().equals(State.UP) ? "Clearing wanted nodeState for node " + node - : "New wantedstate '" + nodeState + "' stored for node " + node); + : "New wantedstate '" + nodeState.toString() + "' stored for node " + node); if (!oldState.equals(nodeState) || !oldState.getDescription().equals(nodeState.getDescription())) { if (!nodeState.getState().validWantedNodeState(nodeType)) { throw new IllegalStateException("State " + nodeState.getState() @@ -289,7 +289,7 @@ public class RpcServer { node.setWantedState(nodeState); changeListener.handleNewWantedNodeState(node, nodeState); } else { - message = "Node " + node + " already had wanted state " + nodeState; + message = "Node " + node + " already had wanted state " + nodeState.toString(); log.log(Level.FINE, message); } req.returnValues().add(new StringValue(message)); diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/SlobrokClient.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/SlobrokClient.java index 8393e776fc2..c88bf71af09 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/SlobrokClient.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/SlobrokClient.java @@ -150,7 +150,7 @@ public class SlobrokClient implements NodeLookup { } cluster.setSlobrokGenerationCount(mirrorVersion); for (NodeInfo nodeInfo : cluster.getNodeInfos()) { - if (slobrokNodes.containsKey(nodeInfo.getNode()) && nodeInfo.isNotInSlobrok()) { + if (slobrokNodes.containsKey(nodeInfo.getNode()) && nodeInfo.isRpcAddressOutdated()) { context.log(log, Level.WARNING, "Node " + nodeInfo @@ -187,7 +187,7 @@ public class SlobrokClient implements NodeLookup { newNext = null; } else if (newNext == null || newNext.node.compareTo(oldNext.getNode()) > 0) { assert(slobrokNodes.get(oldNext.getNode()) == null); - if (oldNext.isInSlobrok() && oldNext.getRpcAddress() != null) { + if (!oldNext.isRpcAddressOutdated() && oldNext.getRpcAddress() != null) { missingNodeInfos.add(oldNext); } oldNext = null; @@ -195,7 +195,7 @@ public class SlobrokClient implements NodeLookup { assert(newNext.rpcAddress != null); if (oldNext.getRpcAddress() == null || !oldNext.getRpcAddress().equals(newNext.rpcAddress)) { alteredRpcAddress.add(newNext); - } else if (oldNext.isNotInSlobrok()) { + } else if (oldNext.isRpcAddressOutdated()) { returningRpcAddressNodeInfos.add(oldNext); } oldNext = null; diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/VdsClusterHtmlRenderer.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/VdsClusterHtmlRenderer.java index d840529d361..1081f3e77cd 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/VdsClusterHtmlRenderer.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/VdsClusterHtmlRenderer.java @@ -243,7 +243,7 @@ public class VdsClusterHtmlRenderer { row.addCell(new HtmlTable.Cell("-").addProperties(ERROR_PROPERTY)); } else { row.addCell(new HtmlTable.Cell(HtmlTable.escape(nodeInfo.getRpcAddress()))); - if (nodeInfo.isNotInSlobrok()) { + if (nodeInfo.isRpcAddressOutdated()) { row.getLastCell().addProperties(WARNING_PROPERTY); } } |