Emit node-level events when resource exhaustion set changes

9 files changed, 138 insertions, 44 deletions

diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/EventDiffCalculator.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/EventDiffCalculator.java
index 6e8bfbd4a0c..f4975ee4ee4 100644
--- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/EventDiffCalculator.java
+++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/EventDiffCalculator.java
@@ -9,9 +9,11 @@ import com.yahoo.vdslib.state.NodeType;
 import com.yahoo.vdslib.state.State;
 
 import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
 import java.util.List;
-import java.util.Map;
 import java.util.Optional;
+import java.util.Set;
 
 /**
  * Responsible for inferring the difference between two cluster states and their
@@ -160,6 +162,29 @@ public class EventDiffCalculator {
                 emitSingleNodeEvents(params, events, cluster, fromState, toState, n);
             }
         }
+        emitNodeResourceExhaustionEvents(params, events, cluster);
+    }
+
+    // Returns a - b as a set operation
+    private static <T> Set<T> setSubtraction(Set<T> a, Set<T> b) {
+        var ret = new HashSet<>(a);
+        ret.removeAll(b);
+        return ret;
+    }
+
+    private static void emitNodeResourceExhaustionEvents(PerStateParams params, List<Event> events, ContentCluster cluster) {
+        // Feed block events are not ordered by node
+        Set<NodeResourceExhaustion> fromBlockSet = params.feedBlockFrom != null ? params.feedBlockFrom.getConcreteExhaustions() : Collections.emptySet();
+        Set<NodeResourceExhaustion> toBlockSet   = params.feedBlockTo   != null ? params.feedBlockTo.getConcreteExhaustions()   : Collections.emptySet();
+
+        for (var ex : setSubtraction(toBlockSet, fromBlockSet)) {
+            var info = cluster.getNodeInfo(ex.node);
+            events.add(createNodeEvent(info, String.format("Added resource exhaustion: %s", ex.toExhaustionAddedDescription()), params));
+        }
+        for (var ex : setSubtraction(fromBlockSet, toBlockSet)) {
+            var info = cluster.getNodeInfo(ex.node);
+            events.add(createNodeEvent(info, String.format("Removed resource exhaustion: %s", ex.toExhaustionRemovedDescription()), params));
+        }
     }
 
     private static void emitSingleNodeEvents(PerStateParams params, List<Event> events, ContentCluster cluster, ClusterState fromState, ClusterState toState, Node n) {
diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeResourceExhaustion.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeResourceExhaustion.java
index 79f04627073..c730350310c 100644
--- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeResourceExhaustion.java
+++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeResourceExhaustion.java
@@ -1,6 +1,7 @@
 // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.clustercontroller.core;
 
+import com.yahoo.jrt.Spec;
 import com.yahoo.vdslib.state.Node;
 import com.yahoo.vespa.clustercontroller.core.hostinfo.ResourceUsage;
 
@@ -43,4 +44,32 @@ public class NodeResourceExhaustion {
     public int hashCode() {
         return Objects.hash(node, resourceType, resourceUsage, limit, rpcAddress);
     }
+
+    public String toExhaustionAddedDescription() {
+        return String.format("%s (%.3g > %.3g)", makeDescriptionPrefix(), resourceUsage.getUsage(), limit);
+    }
+
+    public String toExhaustionRemovedDescription() {
+        return String.format("%s (<= %.3g)", makeDescriptionPrefix(), limit);
+    }
+
+    private String makeDescriptionPrefix() {
+        return String.format("%s%s on node %d [%s]",
+                resourceType,
+                (resourceUsage.getName() != null ? ":" + resourceUsage.getName() : ""),
+                node.getIndex(),
+                inferHostnameFromRpcAddress(rpcAddress));
+    }
+
+    private static String inferHostnameFromRpcAddress(String rpcAddress) {
+        if (rpcAddress == null) {
+            return "unknown hostname";
+        }
+        var spec = new Spec(rpcAddress);
+        if (spec.malformed()) {
+            return "unknown hostname";
+        }
+        return spec.host();
+    }
+
 }
diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ResourceExhaustionCalculator.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ResourceExhaustionCalculator.java
index e2e61eb8ed0..231d9f95bdb 100644
--- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ResourceExhaustionCalculator.java
+++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ResourceExhaustionCalculator.java
@@ -1,15 +1,11 @@
 // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.clustercontroller.core;
 
-import com.yahoo.jrt.Spec;
-import com.yahoo.vdslib.state.Node;
 import com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo;
 
-import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.LinkedHashSet;
-import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;
@@ -40,7 +36,7 @@ public class ResourceExhaustionCalculator {
         int maxDescriptions = 3;
         String description = exhaustions.stream()
                 .limit(maxDescriptions)
-                .map(ResourceExhaustionCalculator::formatNodeResourceExhaustion)
+                .map(NodeResourceExhaustion::toExhaustionAddedDescription)
                 .collect(Collectors.joining(", "));
         if (exhaustions.size() > maxDescriptions) {
             description += String.format(" (... and %d more)", exhaustions.size() - maxDescriptions);
@@ -51,26 +47,6 @@ public class ResourceExhaustionCalculator {
         return ClusterStateBundle.FeedBlock.blockedWith(description, exhaustions);
     }
 
-    private static String formatNodeResourceExhaustion(NodeResourceExhaustion n) {
-        return String.format("%s%s on node %d [%s] (%.3g > %.3g)",
-                             n.resourceType,
-                             (n.resourceUsage.getName() != null ? ":" + n.resourceUsage.getName() : ""),
-                             n.node.getIndex(),
-                             inferHostnameFromRpcAddress(n.rpcAddress),
-                             n.resourceUsage.getUsage(), n.limit);
-    }
-
-    private static String inferHostnameFromRpcAddress(String rpcAddress) {
-        if (rpcAddress == null) {
-            return "unknown hostname";
-        }
-        var spec = new Spec(rpcAddress);
-        if (spec.malformed()) {
-            return "unknown hostname";
-        }
-        return spec.host();
-    }
-
     public Set<NodeResourceExhaustion> resourceExhaustionsFromHostInfo(NodeInfo nodeInfo, HostInfo hostInfo) {
         Set<NodeResourceExhaustion> exceedingLimit = null;
         for (var usage : hostInfo.getContentNode().getResourceUsage().entrySet()) {
diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterFeedBlockTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterFeedBlockTest.java
index 8dd2a9ca55c..e7477e2289c 100644
--- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterFeedBlockTest.java
+++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterFeedBlockTest.java
@@ -163,7 +163,7 @@ public class ClusterFeedBlockTest extends FleetControllerTest {
         assertEquals("cheese on node 1 [unknown hostname] (0.800 > 0.700)", bundle.getFeedBlock().get().getDescription());
 
         // 80% -> 90%, should not trigger new state.
-        reportResourceUsageFromNode(1, setOf(usage("cheese", 0.9), usage("wine", 0.4)));
+        reportResourceUsageFromNode(1, setOf(usage("cheese", 0.9), usage("wine", 0.3)));
         bundle = ctrl.getClusterStateBundle();
         assertTrue(bundle.clusterFeedIsBlocked());
         assertEquals("cheese on node 1 [unknown hostname] (0.800 > 0.700)", bundle.getFeedBlock().get().getDescription());
diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterStateBundleTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterStateBundleTest.java
index ceddf7cdcf3..3fb2fd534dd 100644
--- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterStateBundleTest.java
+++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterStateBundleTest.java
@@ -6,15 +6,13 @@ import com.yahoo.vdslib.state.Node;
 import com.yahoo.vdslib.state.NodeState;
 import com.yahoo.vdslib.state.NodeType;
 import com.yahoo.vdslib.state.State;
-import com.yahoo.vespa.clustercontroller.core.hostinfo.ResourceUsage;
 import org.junit.Test;
 
-import java.util.Arrays;
-import java.util.LinkedHashSet;
 import java.util.Set;
 import java.util.function.Function;
-import java.util.stream.Collectors;
 
+import static com.yahoo.vespa.clustercontroller.core.FeedBlockUtil.exhaustion;
+import static com.yahoo.vespa.clustercontroller.core.FeedBlockUtil.setOf;
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.junit.Assert.assertEquals;
@@ -120,21 +118,11 @@ public class ClusterStateBundleTest {
         assertTrue(blockingBundle.similarTo(blockingBundleWithOtherDesc));
     }
 
-    static NodeResourceExhaustion createDummyExhaustion(String type) {
-        return new NodeResourceExhaustion(new Node(NodeType.STORAGE, 1), type, new ResourceUsage(0.8, null), 0.7, "foo");
-    }
-
-    static Set<NodeResourceExhaustion> exhaustionsOf(String... types) {
-        return Arrays.stream(types)
-                .map(t -> createDummyExhaustion(t))
-                .collect(Collectors.toCollection(LinkedHashSet::new));
-    }
-
     @Test
     public void similarity_test_considers_cluster_feed_block_concrete_exhaustion_set() {
-        var blockingBundleNoSet = createTestBundleWithFeedBlock("foo");
-        var blockingBundleWithSet = createTestBundleWithFeedBlock("bar", exhaustionsOf("beer", "wine"));
-        var blockingBundleWithOtherSet = createTestBundleWithFeedBlock("bar", exhaustionsOf("beer", "soda"));
+        var blockingBundleNoSet        = createTestBundleWithFeedBlock("foo");
+        var blockingBundleWithSet      = createTestBundleWithFeedBlock("bar", setOf(exhaustion(1, "beer"), exhaustion(1, "wine")));
+        var blockingBundleWithOtherSet = createTestBundleWithFeedBlock("bar", setOf(exhaustion(1, "beer"), exhaustion(1, "soda")));
 
         assertTrue(blockingBundleNoSet.similarTo(blockingBundleNoSet));
         assertTrue(blockingBundleWithSet.similarTo(blockingBundleWithSet));
diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/EventDiffCalculatorTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/EventDiffCalculatorTest.java
index fe913e177ca..ee65183c16c 100644
--- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/EventDiffCalculatorTest.java
+++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/EventDiffCalculatorTest.java
@@ -1,6 +1,8 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.clustercontroller.core;
 
+import static com.yahoo.vespa.clustercontroller.core.FeedBlockUtil.exhaustion;
+import static com.yahoo.vespa.clustercontroller.core.FeedBlockUtil.setOf;
 import static com.yahoo.vespa.clustercontroller.core.matchers.EventForNode.eventForNode;
 import static com.yahoo.vespa.clustercontroller.core.matchers.NodeEventForBucketSpace.nodeEventForBucketSpace;
 import static com.yahoo.vespa.clustercontroller.core.matchers.NodeEventForBucketSpace.nodeEventForBaseline;
@@ -493,4 +495,59 @@ public class EventDiffCalculatorTest {
         assertThat(events.size(), equalTo(0));
     }
 
+    @Test
+    public void feed_block_engage_edge_with_node_exhaustion_info_emits_cluster_and_node_events() {
+        final EventFixture fixture = EventFixture.createForNodes(3)
+                .clusterStateBefore("distributor:3 storage:3")
+                .feedBlockBefore(null)
+                .clusterStateAfter("distributor:3 storage:3")
+                .feedBlockAfter(ClusterStateBundle.FeedBlock.blockedWith(
+                        "we're closed", setOf(exhaustion(1, "oil"))));
+
+        final List<Event> events = fixture.computeEventDiff();
+        assertThat(events.size(), equalTo(2));
+        assertThat(events, hasItem(allOf(
+                eventForNode(storageNode(1)),
+                nodeEventWithDescription("Added resource exhaustion: oil on node 1 [unknown hostname] (0.800 > 0.700)"),
+                nodeEventForBaseline())));
+        assertThat(events, hasItem(
+                clusterEventWithDescription("Cluster feed blocked due to resource exhaustion: we're closed")));
+    }
+
+    @Test
+    public void added_exhaustion_in_feed_block_resource_set_emits_node_event() {
+        final EventFixture fixture = EventFixture.createForNodes(3)
+                .clusterStateBefore("distributor:3 storage:3")
+                .feedBlockBefore(ClusterStateBundle.FeedBlock.blockedWith(
+                        "we're closed", setOf(exhaustion(1, "oil"))))
+                .clusterStateAfter("distributor:3 storage:3")
+                .feedBlockAfter(ClusterStateBundle.FeedBlock.blockedWith(
+                        "we're still closed", setOf(exhaustion(1, "oil"), exhaustion(1, "cpu_brake_fluid"))));
+
+        final List<Event> events = fixture.computeEventDiff();
+        assertThat(events.size(), equalTo(1));
+        assertThat(events, hasItem(allOf(
+                eventForNode(storageNode(1)),
+                nodeEventWithDescription("Added resource exhaustion: cpu_brake_fluid on node 1 [unknown hostname] (0.800 > 0.700)"),
+                nodeEventForBaseline())));
+    }
+
+    @Test
+    public void removed_exhaustion_in_feed_block_resource_set_emits_node_event() {
+        final EventFixture fixture = EventFixture.createForNodes(3)
+                .clusterStateBefore("distributor:3 storage:3")
+                .feedBlockBefore(ClusterStateBundle.FeedBlock.blockedWith(
+                        "we're closed", setOf(exhaustion(1, "oil"), exhaustion(2, "cpu_brake_fluid"))))
+                .clusterStateAfter("distributor:3 storage:3")
+                .feedBlockAfter(ClusterStateBundle.FeedBlock.blockedWith(
+                        "we're still closed", setOf(exhaustion(1, "oil"))));
+
+        final List<Event> events = fixture.computeEventDiff();
+        assertThat(events.size(), equalTo(1));
+        assertThat(events, hasItem(allOf(
+                eventForNode(storageNode(2)),
+                nodeEventWithDescription("Removed resource exhaustion: cpu_brake_fluid on node 2 [unknown hostname] (<= 0.700)"),
+                nodeEventForBaseline())));
+    }
+
 }
diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FeedBlockUtil.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FeedBlockUtil.java
index a26f8791cda..650e4dc7888 100644
--- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FeedBlockUtil.java
+++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FeedBlockUtil.java
@@ -1,6 +1,10 @@
 // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.clustercontroller.core;
 
+import com.yahoo.vdslib.state.Node;
+import com.yahoo.vdslib.state.NodeType;
+import com.yahoo.vespa.clustercontroller.core.hostinfo.ResourceUsage;
+
 import java.util.Arrays;
 import java.util.LinkedHashSet;
 import java.util.Map;
@@ -78,4 +82,13 @@ public class FeedBlockUtil {
         return String.format("{\"content-node\":{\"resource-usage\":{%s}}}", usageInnerJson);
     }
 
+    static NodeResourceExhaustion exhaustion(int index, String type) {
+        return new NodeResourceExhaustion(new Node(NodeType.STORAGE, index), type, new ResourceUsage(0.8, null), 0.7, "foo");
+    }
+
+    static Set<NodeResourceExhaustion> setOf(NodeResourceExhaustion... exhaustions) {
+        return Arrays.stream(exhaustions).collect(Collectors.toCollection(LinkedHashSet::new));
+    }
+
+
 }
diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/matchers/ClusterEventWithDescription.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/matchers/ClusterEventWithDescription.java
index 03df92318c6..9b20f139f96 100644
--- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/matchers/ClusterEventWithDescription.java
+++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/matchers/ClusterEventWithDescription.java
@@ -28,6 +28,9 @@ public class ClusterEventWithDescription extends BaseMatcher<ClusterEvent> {
 
     @Override
     public void describeMismatch(Object item, Description description) {
+        if (!(item instanceof ClusterEvent)) {
+            return;
+        }
         ClusterEvent other = (ClusterEvent)item;
         description.appendText(String.format("got description '%s'", other.getDescription()));
     }
diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/matchers/NodeEventWithDescription.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/matchers/NodeEventWithDescription.java
index 0b3630622bd..3823812b8c3 100644
--- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/matchers/NodeEventWithDescription.java
+++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/matchers/NodeEventWithDescription.java
@@ -28,6 +28,9 @@ public class NodeEventWithDescription extends BaseMatcher<NodeEvent> {
 
     @Override
     public void describeMismatch(Object item, Description description) {
+        if (!(item instanceof NodeEvent)) {
+            return;
+        }
         NodeEvent other = (NodeEvent)item;
         description.appendText(String.format("got description '%s'", other.getDescription()));
     }