diff options
author | Tor Brede Vekterli <vekterli@verizonmedia.com> | 2021-02-26 16:09:15 +0100 |
---|---|---|
committer | Tor Brede Vekterli <vekterli@verizonmedia.com> | 2021-02-26 16:09:15 +0100 |
commit | 45fec1d181ae0a85067c18f6f7d38a8b4b55716e (patch) | |
tree | 59866936d321c31e3b7cd31d70c9b54283f37b7e | |
parent | b1b406f68aeee9867e69cedd29b5a78de1c750ba (diff) |
Only compute feed blocked state from available nodes
Available nodes here mean nodes that are reported as Up/Initializing
and where the wanted state is Up/Retired.
3 files changed, 45 insertions, 2 deletions
diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ResourceExhaustionCalculator.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ResourceExhaustionCalculator.java index 00edd767ad6..3801c17ef1e 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ResourceExhaustionCalculator.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ResourceExhaustionCalculator.java @@ -127,10 +127,16 @@ public class ResourceExhaustionCalculator { return resourceExhaustionsFromHostInfo(nodeInfo, nodeInfo.getHostInfo()); } + private static boolean nodeMayContributeToFeedBlocked(NodeInfo info) { + return (info.getWantedState().getState().oneOf("ur") && + info.getReportedState().getState().oneOf("ui")); + } + // Returns 0-n entries per content node in the cluster, where n is the number of exhausted // resource types on any given node. public Set<NodeResourceExhaustion> enumerateNodeResourceExhaustionsAcrossAllNodes(Collection<NodeInfo> nodeInfos) { return nodeInfos.stream() + .filter(info -> nodeMayContributeToFeedBlocked(info)) .flatMap(info -> enumerateNodeResourceExhaustions(info).stream()) .collect(Collectors.toCollection(LinkedHashSet::new)); } diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterFeedBlockTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterFeedBlockTest.java index 4c3af7376ac..56d567db035 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterFeedBlockTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterFeedBlockTest.java @@ -95,12 +95,16 @@ public class ClusterFeedBlockTest extends FleetControllerTest { return createOptions(feedBlockLimits, 0.0); } - private void reportResourceUsageFromNode(int nodeIndex, Set<FeedBlockUtil.UsageDetails> resourceUsages) throws Exception { + private void reportResourceUsageFromNode(int nodeIndex, State nodeState, Set<FeedBlockUtil.UsageDetails> resourceUsages) throws Exception { String hostInfo = createResourceUsageJson(resourceUsages); - communicator.setNodeState(new Node(NodeType.STORAGE, nodeIndex), new NodeState(NodeType.STORAGE, State.UP), hostInfo); + communicator.setNodeState(new Node(NodeType.STORAGE, nodeIndex), new NodeState(NodeType.STORAGE, nodeState), hostInfo); ctrl.tick(); } + private void reportResourceUsageFromNode(int nodeIndex, Set<FeedBlockUtil.UsageDetails> resourceUsages) throws Exception { + reportResourceUsageFromNode(nodeIndex, State.UP, resourceUsages); + } + @Test public void cluster_feed_can_be_blocked_and_unblocked_by_single_node() throws Exception { initialize(createOptions(mapOf(usage("cheese", 0.7), usage("wine", 0.4)))); @@ -212,6 +216,16 @@ public class ClusterFeedBlockTest extends FleetControllerTest { assertFalse(bundle.clusterFeedIsBlocked()); } + @Test + public void unavailable_nodes_are_not_considered_when_computing_feed_blocked_state() throws Exception { + initialize(createOptions(mapOf(usage("cheese", 0.7), usage("wine", 0.4)), 0.1)); + assertFalse(ctrl.getClusterStateBundle().clusterFeedIsBlocked()); + + reportResourceUsageFromNode(1, State.DOWN, setOf(usage("cheese", 0.8), usage("wine", 0.5))); + // Not blocked, node with exhaustion is marked as Down + assertFalse(ctrl.getClusterStateBundle().clusterFeedIsBlocked()); + } + // FIXME implicit changes in limits due to hysteresis adds spurious exhaustion remove+add node event pair } diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ResourceExhaustionCalculatorTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ResourceExhaustionCalculatorTest.java index 55cf173aa25..cf9b2bc222d 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ResourceExhaustionCalculatorTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ResourceExhaustionCalculatorTest.java @@ -1,6 +1,7 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.clustercontroller.core; +import com.yahoo.vdslib.state.State; import org.junit.Test; import static com.yahoo.vespa.clustercontroller.core.ClusterFixture.storageNode; @@ -141,4 +142,26 @@ public class ResourceExhaustionCalculatorTest { assertNull(feedBlock); } + @Test + public void node_must_be_available_in_reported_state_to_trigger_feed_block() { + var calc = new ResourceExhaustionCalculator(true, mapOf(usage("disk", 0.5), usage("memory", 0.8))); + var cf = createFixtureWithReportedUsages(forNode(1, usage("disk", 0.51), usage("memory", 0.79)), + forNode(2, usage("disk", 0.6), usage("memory", 0.6))); + cf.reportStorageNodeState(1, State.DOWN); + cf.reportStorageNodeState(2, State.DOWN); + var feedBlock = calc.inferContentClusterFeedBlockOrNull(cf.cluster().getNodeInfo()); + assertNull(feedBlock); + } + + @Test + public void node_must_be_available_in_wanted_state_to_trigger_feed_block() { + var calc = new ResourceExhaustionCalculator(true, mapOf(usage("disk", 0.5), usage("memory", 0.8))); + var cf = createFixtureWithReportedUsages(forNode(1, usage("disk", 0.51), usage("memory", 0.79)), + forNode(2, usage("disk", 0.6), usage("memory", 0.6))); + cf.proposeStorageNodeWantedState(1, State.DOWN); + cf.proposeStorageNodeWantedState(2, State.MAINTENANCE); + var feedBlock = calc.inferContentClusterFeedBlockOrNull(cf.cluster().getNodeInfo()); + assertNull(feedBlock); + } + } |