diff options
author | Tor Brede Vekterli <vekterli@verizonmedia.com> | 2021-03-01 10:02:23 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2021-03-01 10:02:23 +0100 |
commit | 215e3de034a9026b6775d29d6b3ba162f4de2627 (patch) | |
tree | 6f52e5e1e511ff9ca9caf2e59380b65078aa190f | |
parent | 46dc77351d188d53f7bb960a114b80c678efa6f5 (diff) | |
parent | 45fec1d181ae0a85067c18f6f7d38a8b4b55716e (diff) |
Merge pull request #16699 from vespa-engine/vekterli/only-compute-feed-blocked-from-available-nodes
Only compute feed blocked state from available nodes
3 files changed, 45 insertions, 2 deletions
diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ResourceExhaustionCalculator.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ResourceExhaustionCalculator.java index 00edd767ad6..3801c17ef1e 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ResourceExhaustionCalculator.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ResourceExhaustionCalculator.java @@ -127,10 +127,16 @@ public class ResourceExhaustionCalculator { return resourceExhaustionsFromHostInfo(nodeInfo, nodeInfo.getHostInfo()); } + private static boolean nodeMayContributeToFeedBlocked(NodeInfo info) { + return (info.getWantedState().getState().oneOf("ur") && + info.getReportedState().getState().oneOf("ui")); + } + // Returns 0-n entries per content node in the cluster, where n is the number of exhausted // resource types on any given node. public Set<NodeResourceExhaustion> enumerateNodeResourceExhaustionsAcrossAllNodes(Collection<NodeInfo> nodeInfos) { return nodeInfos.stream() + .filter(info -> nodeMayContributeToFeedBlocked(info)) .flatMap(info -> enumerateNodeResourceExhaustions(info).stream()) .collect(Collectors.toCollection(LinkedHashSet::new)); } diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterFeedBlockTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterFeedBlockTest.java index 4c3af7376ac..56d567db035 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterFeedBlockTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterFeedBlockTest.java @@ -95,12 +95,16 @@ public class ClusterFeedBlockTest extends FleetControllerTest { return createOptions(feedBlockLimits, 0.0); } - private void reportResourceUsageFromNode(int nodeIndex, Set<FeedBlockUtil.UsageDetails> resourceUsages) throws Exception { + private void reportResourceUsageFromNode(int nodeIndex, State nodeState, Set<FeedBlockUtil.UsageDetails> resourceUsages) throws Exception { String hostInfo = createResourceUsageJson(resourceUsages); - communicator.setNodeState(new Node(NodeType.STORAGE, nodeIndex), new NodeState(NodeType.STORAGE, State.UP), hostInfo); + communicator.setNodeState(new Node(NodeType.STORAGE, nodeIndex), new NodeState(NodeType.STORAGE, nodeState), hostInfo); ctrl.tick(); } + private void reportResourceUsageFromNode(int nodeIndex, Set<FeedBlockUtil.UsageDetails> resourceUsages) throws Exception { + reportResourceUsageFromNode(nodeIndex, State.UP, resourceUsages); + } + @Test public void cluster_feed_can_be_blocked_and_unblocked_by_single_node() throws Exception { initialize(createOptions(mapOf(usage("cheese", 0.7), usage("wine", 0.4)))); @@ -212,6 +216,16 @@ public class ClusterFeedBlockTest extends FleetControllerTest { assertFalse(bundle.clusterFeedIsBlocked()); } + @Test + public void unavailable_nodes_are_not_considered_when_computing_feed_blocked_state() throws Exception { + initialize(createOptions(mapOf(usage("cheese", 0.7), usage("wine", 0.4)), 0.1)); + assertFalse(ctrl.getClusterStateBundle().clusterFeedIsBlocked()); + + reportResourceUsageFromNode(1, State.DOWN, setOf(usage("cheese", 0.8), usage("wine", 0.5))); + // Not blocked, node with exhaustion is marked as Down + assertFalse(ctrl.getClusterStateBundle().clusterFeedIsBlocked()); + } + // FIXME implicit changes in limits due to hysteresis adds spurious exhaustion remove+add node event pair } diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ResourceExhaustionCalculatorTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ResourceExhaustionCalculatorTest.java index 55cf173aa25..cf9b2bc222d 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ResourceExhaustionCalculatorTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ResourceExhaustionCalculatorTest.java @@ -1,6 +1,7 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.clustercontroller.core; +import com.yahoo.vdslib.state.State; import org.junit.Test; import static com.yahoo.vespa.clustercontroller.core.ClusterFixture.storageNode; @@ -141,4 +142,26 @@ public class ResourceExhaustionCalculatorTest { assertNull(feedBlock); } + @Test + public void node_must_be_available_in_reported_state_to_trigger_feed_block() { + var calc = new ResourceExhaustionCalculator(true, mapOf(usage("disk", 0.5), usage("memory", 0.8))); + var cf = createFixtureWithReportedUsages(forNode(1, usage("disk", 0.51), usage("memory", 0.79)), + forNode(2, usage("disk", 0.6), usage("memory", 0.6))); + cf.reportStorageNodeState(1, State.DOWN); + cf.reportStorageNodeState(2, State.DOWN); + var feedBlock = calc.inferContentClusterFeedBlockOrNull(cf.cluster().getNodeInfo()); + assertNull(feedBlock); + } + + @Test + public void node_must_be_available_in_wanted_state_to_trigger_feed_block() { + var calc = new ResourceExhaustionCalculator(true, mapOf(usage("disk", 0.5), usage("memory", 0.8))); + var cf = createFixtureWithReportedUsages(forNode(1, usage("disk", 0.51), usage("memory", 0.79)), + forNode(2, usage("disk", 0.6), usage("memory", 0.6))); + cf.proposeStorageNodeWantedState(1, State.DOWN); + cf.proposeStorageNodeWantedState(2, State.MAINTENANCE); + var feedBlock = calc.inferContentClusterFeedBlockOrNull(cf.cluster().getNodeInfo()); + assertNull(feedBlock); + } + } |