diff options
author | Tor Brede Vekterli <vekterli@verizonmedia.com> | 2021-02-03 13:50:35 +0100 |
---|---|---|
committer | Tor Brede Vekterli <vekterli@verizonmedia.com> | 2021-02-03 13:50:35 +0100 |
commit | 3dd9dbd0f0f7f83ae6793acc66906376cc2be165 (patch) | |
tree | a96fa241bc9d2f84e61812b3e9e023f2ef2b5643 /clustercontroller-core | |
parent | 21d9cda9a3a50ab3186eecbd9e9ac129762640a2 (diff) |
Emit node-level events when resource exhaustion set changes
Diffstat (limited to 'clustercontroller-core')
9 files changed, 138 insertions, 44 deletions
diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/EventDiffCalculator.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/EventDiffCalculator.java index 6e8bfbd4a0c..f4975ee4ee4 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/EventDiffCalculator.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/EventDiffCalculator.java @@ -9,9 +9,11 @@ import com.yahoo.vdslib.state.NodeType; import com.yahoo.vdslib.state.State; import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; import java.util.List; -import java.util.Map; import java.util.Optional; +import java.util.Set; /** * Responsible for inferring the difference between two cluster states and their @@ -160,6 +162,29 @@ public class EventDiffCalculator { emitSingleNodeEvents(params, events, cluster, fromState, toState, n); } } + emitNodeResourceExhaustionEvents(params, events, cluster); + } + + // Returns a - b as a set operation + private static <T> Set<T> setSubtraction(Set<T> a, Set<T> b) { + var ret = new HashSet<>(a); + ret.removeAll(b); + return ret; + } + + private static void emitNodeResourceExhaustionEvents(PerStateParams params, List<Event> events, ContentCluster cluster) { + // Feed block events are not ordered by node + Set<NodeResourceExhaustion> fromBlockSet = params.feedBlockFrom != null ? params.feedBlockFrom.getConcreteExhaustions() : Collections.emptySet(); + Set<NodeResourceExhaustion> toBlockSet = params.feedBlockTo != null ? params.feedBlockTo.getConcreteExhaustions() : Collections.emptySet(); + + for (var ex : setSubtraction(toBlockSet, fromBlockSet)) { + var info = cluster.getNodeInfo(ex.node); + events.add(createNodeEvent(info, String.format("Added resource exhaustion: %s", ex.toExhaustionAddedDescription()), params)); + } + for (var ex : setSubtraction(fromBlockSet, toBlockSet)) { + var info = cluster.getNodeInfo(ex.node); + events.add(createNodeEvent(info, String.format("Removed resource exhaustion: %s", ex.toExhaustionRemovedDescription()), params)); + } } private static void emitSingleNodeEvents(PerStateParams params, List<Event> events, ContentCluster cluster, ClusterState fromState, ClusterState toState, Node n) { diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeResourceExhaustion.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeResourceExhaustion.java index 79f04627073..c730350310c 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeResourceExhaustion.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeResourceExhaustion.java @@ -1,6 +1,7 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.clustercontroller.core; +import com.yahoo.jrt.Spec; import com.yahoo.vdslib.state.Node; import com.yahoo.vespa.clustercontroller.core.hostinfo.ResourceUsage; @@ -43,4 +44,32 @@ public class NodeResourceExhaustion { public int hashCode() { return Objects.hash(node, resourceType, resourceUsage, limit, rpcAddress); } + + public String toExhaustionAddedDescription() { + return String.format("%s (%.3g > %.3g)", makeDescriptionPrefix(), resourceUsage.getUsage(), limit); + } + + public String toExhaustionRemovedDescription() { + return String.format("%s (<= %.3g)", makeDescriptionPrefix(), limit); + } + + private String makeDescriptionPrefix() { + return String.format("%s%s on node %d [%s]", + resourceType, + (resourceUsage.getName() != null ? ":" + resourceUsage.getName() : ""), + node.getIndex(), + inferHostnameFromRpcAddress(rpcAddress)); + } + + private static String inferHostnameFromRpcAddress(String rpcAddress) { + if (rpcAddress == null) { + return "unknown hostname"; + } + var spec = new Spec(rpcAddress); + if (spec.malformed()) { + return "unknown hostname"; + } + return spec.host(); + } + } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ResourceExhaustionCalculator.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ResourceExhaustionCalculator.java index e2e61eb8ed0..231d9f95bdb 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ResourceExhaustionCalculator.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ResourceExhaustionCalculator.java @@ -1,15 +1,11 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.clustercontroller.core; -import com.yahoo.jrt.Spec; -import com.yahoo.vdslib.state.Node; import com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo; -import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.LinkedHashSet; -import java.util.List; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; @@ -40,7 +36,7 @@ public class ResourceExhaustionCalculator { int maxDescriptions = 3; String description = exhaustions.stream() .limit(maxDescriptions) - .map(ResourceExhaustionCalculator::formatNodeResourceExhaustion) + .map(NodeResourceExhaustion::toExhaustionAddedDescription) .collect(Collectors.joining(", ")); if (exhaustions.size() > maxDescriptions) { description += String.format(" (... and %d more)", exhaustions.size() - maxDescriptions); @@ -51,26 +47,6 @@ public class ResourceExhaustionCalculator { return ClusterStateBundle.FeedBlock.blockedWith(description, exhaustions); } - private static String formatNodeResourceExhaustion(NodeResourceExhaustion n) { - return String.format("%s%s on node %d [%s] (%.3g > %.3g)", - n.resourceType, - (n.resourceUsage.getName() != null ? ":" + n.resourceUsage.getName() : ""), - n.node.getIndex(), - inferHostnameFromRpcAddress(n.rpcAddress), - n.resourceUsage.getUsage(), n.limit); - } - - private static String inferHostnameFromRpcAddress(String rpcAddress) { - if (rpcAddress == null) { - return "unknown hostname"; - } - var spec = new Spec(rpcAddress); - if (spec.malformed()) { - return "unknown hostname"; - } - return spec.host(); - } - public Set<NodeResourceExhaustion> resourceExhaustionsFromHostInfo(NodeInfo nodeInfo, HostInfo hostInfo) { Set<NodeResourceExhaustion> exceedingLimit = null; for (var usage : hostInfo.getContentNode().getResourceUsage().entrySet()) { diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterFeedBlockTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterFeedBlockTest.java index 8dd2a9ca55c..e7477e2289c 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterFeedBlockTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterFeedBlockTest.java @@ -163,7 +163,7 @@ public class ClusterFeedBlockTest extends FleetControllerTest { assertEquals("cheese on node 1 [unknown hostname] (0.800 > 0.700)", bundle.getFeedBlock().get().getDescription()); // 80% -> 90%, should not trigger new state. - reportResourceUsageFromNode(1, setOf(usage("cheese", 0.9), usage("wine", 0.4))); + reportResourceUsageFromNode(1, setOf(usage("cheese", 0.9), usage("wine", 0.3))); bundle = ctrl.getClusterStateBundle(); assertTrue(bundle.clusterFeedIsBlocked()); assertEquals("cheese on node 1 [unknown hostname] (0.800 > 0.700)", bundle.getFeedBlock().get().getDescription()); diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterStateBundleTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterStateBundleTest.java index ceddf7cdcf3..3fb2fd534dd 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterStateBundleTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterStateBundleTest.java @@ -6,15 +6,13 @@ import com.yahoo.vdslib.state.Node; import com.yahoo.vdslib.state.NodeState; import com.yahoo.vdslib.state.NodeType; import com.yahoo.vdslib.state.State; -import com.yahoo.vespa.clustercontroller.core.hostinfo.ResourceUsage; import org.junit.Test; -import java.util.Arrays; -import java.util.LinkedHashSet; import java.util.Set; import java.util.function.Function; -import java.util.stream.Collectors; +import static com.yahoo.vespa.clustercontroller.core.FeedBlockUtil.exhaustion; +import static com.yahoo.vespa.clustercontroller.core.FeedBlockUtil.setOf; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.Assert.assertEquals; @@ -120,21 +118,11 @@ public class ClusterStateBundleTest { assertTrue(blockingBundle.similarTo(blockingBundleWithOtherDesc)); } - static NodeResourceExhaustion createDummyExhaustion(String type) { - return new NodeResourceExhaustion(new Node(NodeType.STORAGE, 1), type, new ResourceUsage(0.8, null), 0.7, "foo"); - } - - static Set<NodeResourceExhaustion> exhaustionsOf(String... types) { - return Arrays.stream(types) - .map(t -> createDummyExhaustion(t)) - .collect(Collectors.toCollection(LinkedHashSet::new)); - } - @Test public void similarity_test_considers_cluster_feed_block_concrete_exhaustion_set() { - var blockingBundleNoSet = createTestBundleWithFeedBlock("foo"); - var blockingBundleWithSet = createTestBundleWithFeedBlock("bar", exhaustionsOf("beer", "wine")); - var blockingBundleWithOtherSet = createTestBundleWithFeedBlock("bar", exhaustionsOf("beer", "soda")); + var blockingBundleNoSet = createTestBundleWithFeedBlock("foo"); + var blockingBundleWithSet = createTestBundleWithFeedBlock("bar", setOf(exhaustion(1, "beer"), exhaustion(1, "wine"))); + var blockingBundleWithOtherSet = createTestBundleWithFeedBlock("bar", setOf(exhaustion(1, "beer"), exhaustion(1, "soda"))); assertTrue(blockingBundleNoSet.similarTo(blockingBundleNoSet)); assertTrue(blockingBundleWithSet.similarTo(blockingBundleWithSet)); diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/EventDiffCalculatorTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/EventDiffCalculatorTest.java index fe913e177ca..ee65183c16c 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/EventDiffCalculatorTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/EventDiffCalculatorTest.java @@ -1,6 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.clustercontroller.core; +import static com.yahoo.vespa.clustercontroller.core.FeedBlockUtil.exhaustion; +import static com.yahoo.vespa.clustercontroller.core.FeedBlockUtil.setOf; import static com.yahoo.vespa.clustercontroller.core.matchers.EventForNode.eventForNode; import static com.yahoo.vespa.clustercontroller.core.matchers.NodeEventForBucketSpace.nodeEventForBucketSpace; import static com.yahoo.vespa.clustercontroller.core.matchers.NodeEventForBucketSpace.nodeEventForBaseline; @@ -493,4 +495,59 @@ public class EventDiffCalculatorTest { assertThat(events.size(), equalTo(0)); } + @Test + public void feed_block_engage_edge_with_node_exhaustion_info_emits_cluster_and_node_events() { + final EventFixture fixture = EventFixture.createForNodes(3) + .clusterStateBefore("distributor:3 storage:3") + .feedBlockBefore(null) + .clusterStateAfter("distributor:3 storage:3") + .feedBlockAfter(ClusterStateBundle.FeedBlock.blockedWith( + "we're closed", setOf(exhaustion(1, "oil")))); + + final List<Event> events = fixture.computeEventDiff(); + assertThat(events.size(), equalTo(2)); + assertThat(events, hasItem(allOf( + eventForNode(storageNode(1)), + nodeEventWithDescription("Added resource exhaustion: oil on node 1 [unknown hostname] (0.800 > 0.700)"), + nodeEventForBaseline()))); + assertThat(events, hasItem( + clusterEventWithDescription("Cluster feed blocked due to resource exhaustion: we're closed"))); + } + + @Test + public void added_exhaustion_in_feed_block_resource_set_emits_node_event() { + final EventFixture fixture = EventFixture.createForNodes(3) + .clusterStateBefore("distributor:3 storage:3") + .feedBlockBefore(ClusterStateBundle.FeedBlock.blockedWith( + "we're closed", setOf(exhaustion(1, "oil")))) + .clusterStateAfter("distributor:3 storage:3") + .feedBlockAfter(ClusterStateBundle.FeedBlock.blockedWith( + "we're still closed", setOf(exhaustion(1, "oil"), exhaustion(1, "cpu_brake_fluid")))); + + final List<Event> events = fixture.computeEventDiff(); + assertThat(events.size(), equalTo(1)); + assertThat(events, hasItem(allOf( + eventForNode(storageNode(1)), + nodeEventWithDescription("Added resource exhaustion: cpu_brake_fluid on node 1 [unknown hostname] (0.800 > 0.700)"), + nodeEventForBaseline()))); + } + + @Test + public void removed_exhaustion_in_feed_block_resource_set_emits_node_event() { + final EventFixture fixture = EventFixture.createForNodes(3) + .clusterStateBefore("distributor:3 storage:3") + .feedBlockBefore(ClusterStateBundle.FeedBlock.blockedWith( + "we're closed", setOf(exhaustion(1, "oil"), exhaustion(2, "cpu_brake_fluid")))) + .clusterStateAfter("distributor:3 storage:3") + .feedBlockAfter(ClusterStateBundle.FeedBlock.blockedWith( + "we're still closed", setOf(exhaustion(1, "oil")))); + + final List<Event> events = fixture.computeEventDiff(); + assertThat(events.size(), equalTo(1)); + assertThat(events, hasItem(allOf( + eventForNode(storageNode(2)), + nodeEventWithDescription("Removed resource exhaustion: cpu_brake_fluid on node 2 [unknown hostname] (<= 0.700)"), + nodeEventForBaseline()))); + } + } diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FeedBlockUtil.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FeedBlockUtil.java index a26f8791cda..650e4dc7888 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FeedBlockUtil.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FeedBlockUtil.java @@ -1,6 +1,10 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.clustercontroller.core; +import com.yahoo.vdslib.state.Node; +import com.yahoo.vdslib.state.NodeType; +import com.yahoo.vespa.clustercontroller.core.hostinfo.ResourceUsage; + import java.util.Arrays; import java.util.LinkedHashSet; import java.util.Map; @@ -78,4 +82,13 @@ public class FeedBlockUtil { return String.format("{\"content-node\":{\"resource-usage\":{%s}}}", usageInnerJson); } + static NodeResourceExhaustion exhaustion(int index, String type) { + return new NodeResourceExhaustion(new Node(NodeType.STORAGE, index), type, new ResourceUsage(0.8, null), 0.7, "foo"); + } + + static Set<NodeResourceExhaustion> setOf(NodeResourceExhaustion... exhaustions) { + return Arrays.stream(exhaustions).collect(Collectors.toCollection(LinkedHashSet::new)); + } + + } diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/matchers/ClusterEventWithDescription.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/matchers/ClusterEventWithDescription.java index 03df92318c6..9b20f139f96 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/matchers/ClusterEventWithDescription.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/matchers/ClusterEventWithDescription.java @@ -28,6 +28,9 @@ public class ClusterEventWithDescription extends BaseMatcher<ClusterEvent> { @Override public void describeMismatch(Object item, Description description) { + if (!(item instanceof ClusterEvent)) { + return; + } ClusterEvent other = (ClusterEvent)item; description.appendText(String.format("got description '%s'", other.getDescription())); } diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/matchers/NodeEventWithDescription.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/matchers/NodeEventWithDescription.java index 0b3630622bd..3823812b8c3 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/matchers/NodeEventWithDescription.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/matchers/NodeEventWithDescription.java @@ -28,6 +28,9 @@ public class NodeEventWithDescription extends BaseMatcher<NodeEvent> { @Override public void describeMismatch(Object item, Description description) { + if (!(item instanceof NodeEvent)) { + return; + } NodeEvent other = (NodeEvent)item; description.appendText(String.format("got description '%s'", other.getDescription())); } |