Extend EventDiffCalculator to emit events for changes in derived bucket space cluster states.

8 files changed, 324 insertions, 80 deletions

diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/AnnotatedClusterState.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/AnnotatedClusterState.java
index 32cbcf33d8e..aad94f78aef 100644
--- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/AnnotatedClusterState.java
+++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/AnnotatedClusterState.java
@@ -4,10 +4,7 @@ package com.yahoo.vespa.clustercontroller.core;
 import com.yahoo.vdslib.state.ClusterState;
 import com.yahoo.vdslib.state.Node;
 
-import java.util.Collections;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Optional;
+import java.util.*;
 
 public class AnnotatedClusterState implements Cloneable {
 
@@ -15,6 +12,31 @@ public class AnnotatedClusterState implements Cloneable {
     private final Map<Node, NodeStateReason> nodeStateReasons;
     private final Optional<ClusterStateReason> clusterStateReason;
 
+    public static class Builder {
+        private ClusterState clusterState = ClusterState.emptyState();
+        private Optional<ClusterStateReason> clusterReason = Optional.empty();
+        private Map<Node, NodeStateReason> nodeStateReasons = new HashMap<>();
+
+        public Builder clusterState(String stateStr) {
+            clusterState = ClusterState.stateFromString(stateStr);
+            return this;
+        }
+
+        public Builder clusterReason(ClusterStateReason reason) {
+            clusterReason = Optional.of(reason);
+            return this;
+        }
+
+        public Builder storageNodeReason(int nodeIndex, NodeStateReason reason) {
+            nodeStateReasons.put(Node.ofStorage(nodeIndex), reason);
+            return this;
+        }
+
+        AnnotatedClusterState build() {
+            return new AnnotatedClusterState(clusterState, clusterReason, nodeStateReasons);
+        }
+    }
+
     public AnnotatedClusterState(ClusterState clusterState,
                                  Optional<ClusterStateReason> clusterStateReason,
                                  Map<Node, NodeStateReason> nodeStateReasons)
diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/EventDiffCalculator.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/EventDiffCalculator.java
index 93742e3a539..e92a7b6986a 100644
--- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/EventDiffCalculator.java
+++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/EventDiffCalculator.java
@@ -10,6 +10,7 @@ import com.yahoo.vdslib.state.State;
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Map;
 import java.util.Optional;
 
 /**
@@ -27,20 +28,20 @@ public class EventDiffCalculator {
 
     static class Params {
         ContentCluster cluster;
-        AnnotatedClusterState fromState;
-        AnnotatedClusterState toState;
+        ClusterStateBundle fromState;
+        ClusterStateBundle toState;
         long currentTime;
 
         public Params cluster(ContentCluster cluster) {
             this.cluster = cluster;
             return this;
         }
-        public Params fromState(AnnotatedClusterState clusterState) {
-            this.fromState = clusterState;
+        public Params fromState(ClusterStateBundle bundle) {
+            this.fromState = bundle;
             return this;
         }
-        public Params toState(AnnotatedClusterState clusterState) {
-            this.toState = clusterState;
+        public Params toState(ClusterStateBundle bundle) {
+            this.toState = bundle;
             return this;
         }
         public Params currentTimeMs(long time) {
@@ -49,24 +50,46 @@ public class EventDiffCalculator {
         }
     }
 
+    public static Params params() { return new Params(); }
+
+    private static class PerStateParams {
+        final ContentCluster cluster;
+        final Optional<String> bucketSpace;
+        final AnnotatedClusterState fromState;
+        final AnnotatedClusterState toState;
+        final long currentTime;
+
+        PerStateParams(ContentCluster cluster,
+                       Optional<String> bucketSpace,
+                       AnnotatedClusterState fromState,
+                       AnnotatedClusterState toState,
+                       long currentTime) {
+            this.cluster = cluster;
+            this.bucketSpace = bucketSpace;
+            this.fromState = fromState;
+            this.toState = toState;
+            this.currentTime = currentTime;
+        }
+    }
+
     public static List<Event> computeEventDiff(final Params params) {
         final List<Event> events = new ArrayList<>();
 
-        emitPerNodeDiffEvents(params, events);
-        emitWholeClusterDiffEvent(params, events);
+        emitPerNodeDiffEvents(createBaselineParams(params), events);
+        emitWholeClusterDiffEvent(createBaselineParams(params), events);
+        emitDerivedBucketSpaceStatesDiffEvents(params, events);
         return events;
     }
 
-    private static ClusterEvent createClusterEvent(String description, Params params) {
-        return new ClusterEvent(ClusterEvent.Type.SYSTEMSTATE, description, params.currentTime);
+    private static PerStateParams createBaselineParams(Params params) {
+        return new PerStateParams(params.cluster,
+                Optional.empty(),
+                params.fromState.getBaselineAnnotatedState(),
+                params.toState.getBaselineAnnotatedState(),
+                params.currentTime);
     }
 
-    private static boolean clusterDownBecause(final Params params, ClusterStateReason wantedReason) {
-        final Optional<ClusterStateReason> actualReason = params.toState.getClusterStateReason();
-        return actualReason.isPresent() && actualReason.get().equals(wantedReason);
-    }
-
-    private static void emitWholeClusterDiffEvent(final Params params, final List<Event> events) {
+    private static void emitWholeClusterDiffEvent(final PerStateParams params, final List<Event> events) {
         final ClusterState fromState = params.fromState.getClusterState();
         final ClusterState toState = params.toState.getClusterState();
 
@@ -87,11 +110,16 @@ public class EventDiffCalculator {
         }
     }
 
-    private static NodeEvent createNodeEvent(NodeInfo nodeInfo, String description, Params params) {
-        return new NodeEvent(nodeInfo, description, NodeEvent.Type.CURRENT, params.currentTime);
+    private static ClusterEvent createClusterEvent(String description, PerStateParams params) {
+        return new ClusterEvent(ClusterEvent.Type.SYSTEMSTATE, description, params.currentTime);
+    }
+
+    private static boolean clusterDownBecause(final PerStateParams params, ClusterStateReason wantedReason) {
+        final Optional<ClusterStateReason> actualReason = params.toState.getClusterStateReason();
+        return actualReason.isPresent() && actualReason.get().equals(wantedReason);
     }
 
-    private static void emitPerNodeDiffEvents(final Params params, final List<Event> events) {
+    private static void emitPerNodeDiffEvents(final PerStateParams params, final List<Event> events) {
         final ContentCluster cluster = params.cluster;
         final ClusterState fromState = params.fromState.getClusterState();
         final ClusterState toState = params.toState.getClusterState();
@@ -104,7 +132,7 @@ public class EventDiffCalculator {
         }
     }
 
-    private static void emitSingleNodeEvents(Params params, List<Event> events, ContentCluster cluster, ClusterState fromState, ClusterState toState, Node n) {
+    private static void emitSingleNodeEvents(PerStateParams params, List<Event> events, ContentCluster cluster, ClusterState fromState, ClusterState toState, Node n) {
         final NodeState nodeFrom = fromState.getNodeState(n);
         final NodeState nodeTo = toState.getNodeState(n);
         if (!nodeTo.equals(nodeFrom)) {
@@ -118,10 +146,22 @@ public class EventDiffCalculator {
                 events.add(createNodeEvent(info, "Group node availability is below configured threshold", params));
             } else if (isGroupUpEdge(prevReason, currReason)) {
                 events.add(createNodeEvent(info, "Group node availability has been restored", params));
+            } else if (isMayHaveMergesPendingUpEdge(prevReason, currReason)) {
+                events.add(createNodeEvent(info, "Node may have merges pending", params));
+            } else if (isMayHaveMergesPendingDownEdge(prevReason, currReason)) {
+                events.add(createNodeEvent(info, "Node no longer have merges pending", params));
             }
         }
     }
 
+    private static NodeEvent createNodeEvent(NodeInfo nodeInfo, String description, PerStateParams params) {
+        if (params.bucketSpace.isPresent()) {
+            return new NodeEvent(nodeInfo, params.bucketSpace.get(), description, NodeEvent.Type.CURRENT, params.currentTime);
+        } else {
+            return new NodeEvent(nodeInfo, description, NodeEvent.Type.CURRENT, params.currentTime);
+        }
+    }
+
     private static boolean isGroupUpEdge(NodeStateReason prevReason, NodeStateReason currReason) {
         return prevReason == NodeStateReason.GROUP_IS_DOWN && currReason != NodeStateReason.GROUP_IS_DOWN;
     }
@@ -130,6 +170,14 @@ public class EventDiffCalculator {
         return prevReason != NodeStateReason.GROUP_IS_DOWN && currReason == NodeStateReason.GROUP_IS_DOWN;
     }
 
+    private static boolean isMayHaveMergesPendingUpEdge(NodeStateReason prevReason, NodeStateReason currReason) {
+        return prevReason != NodeStateReason.MAY_HAVE_MERGES_PENDING && currReason == NodeStateReason.MAY_HAVE_MERGES_PENDING;
+    }
+
+    private static boolean isMayHaveMergesPendingDownEdge(NodeStateReason prevReason, NodeStateReason currReason) {
+        return prevReason == NodeStateReason.MAY_HAVE_MERGES_PENDING && currReason != NodeStateReason.MAY_HAVE_MERGES_PENDING;
+    }
+
     private static boolean clusterHasTransitionedToUpState(ClusterState prevState, ClusterState currentState) {
         return prevState.getClusterState() != State.UP && currentState.getClusterState() == State.UP;
     }
@@ -138,6 +186,28 @@ public class EventDiffCalculator {
         return prevState.getClusterState() != State.DOWN && currentState.getClusterState() == State.DOWN;
     }
 
-    public static Params params() { return new Params(); }
+    private static void emitDerivedBucketSpaceStatesDiffEvents(Params params, List<Event> events) {
+        params.toState.getDerivedBucketSpaceStates().entrySet().forEach(toEntry -> {
+            String toBucketSpace = toEntry.getKey();
+            AnnotatedClusterState toDerivedState = toEntry.getValue();
+            AnnotatedClusterState fromDerivedState = params.fromState.getDerivedBucketSpaceStates().get(toBucketSpace);
+            if (fromDerivedState != null && shouldConsiderDerivedStates(params, fromDerivedState, toDerivedState)) {
+                emitPerNodeDiffEvents(createDerivedParams(params, toBucketSpace, fromDerivedState, toDerivedState), events);
+            }
+        });
+    }
+
+    private static boolean shouldConsiderDerivedStates(Params params, AnnotatedClusterState fromDerivedState, AnnotatedClusterState toDerivedState) {
+        return (!fromDerivedState.getClusterState().equals(params.fromState.getBaselineClusterState())) ||
+                (!toDerivedState.getClusterState().equals(params.toState.getBaselineClusterState()));
+    }
+
+    private static PerStateParams createDerivedParams(Params params, String bucketSpace, AnnotatedClusterState fromDerivedState, AnnotatedClusterState toDerivedState) {
+        return new PerStateParams(params.cluster,
+                Optional.of(bucketSpace),
+                fromDerivedState,
+                toDerivedState,
+                params.currentTime);
+    }
 
 }
diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java
index 5c345b6f8a0..9fe5a64cfe6 100644
--- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java
+++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java
@@ -803,11 +803,11 @@ public class FleetController implements NodeStateOrHostInfoChangeHandler, NodeAd
                     || stateVersionTracker.hasReceivedNewVersionFromZooKeeper())
             {
                 final long timeNowMs = timer.getCurrentTimeInMillis();
-                final AnnotatedClusterState before = stateVersionTracker.getAnnotatedVersionedClusterState();
+                final ClusterStateBundle before = stateVersionTracker.getVersionedClusterStateBundle();
 
                 stateVersionTracker.promoteCandidateToVersionedState(timeNowMs);
                 // TODO also emit derived state edges events
-                emitEventsForAlteredStateEdges(before, stateVersionTracker.getAnnotatedVersionedClusterState(), timeNowMs);
+                emitEventsForAlteredStateEdges(before, stateVersionTracker.getVersionedClusterStateBundle(), timeNowMs);
                 handleNewSystemState(stateVersionTracker.getVersionedClusterStateBundle());
                 stateWasChanged = true;
             }
@@ -852,8 +852,8 @@ public class FleetController implements NodeStateOrHostInfoChangeHandler, NodeAd
         return ClusterStateGenerator.generatedStateFrom(params);
     }
 
-    private void emitEventsForAlteredStateEdges(final AnnotatedClusterState fromState,
-                                                final AnnotatedClusterState toState,
+    private void emitEventsForAlteredStateEdges(final ClusterStateBundle fromState,
+                                                final ClusterStateBundle toState,
                                                 final long timeNowMs) {
         final List<Event> deltaEvents = EventDiffCalculator.computeEventDiff(
                 EventDiffCalculator.params()
@@ -865,7 +865,7 @@ public class FleetController implements NodeStateOrHostInfoChangeHandler, NodeAd
             eventLog.add(event, isMaster);
         }
 
-        emitStateAppliedEvents(timeNowMs, fromState.getClusterState(), toState.getClusterState());
+        emitStateAppliedEvents(timeNowMs, fromState.getBaselineClusterState(), toState.getBaselineClusterState());
     }
 
     private void emitStateAppliedEvents(long timeNowMs, ClusterState fromClusterState, ClusterState toClusterState) {
diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeEvent.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeEvent.java
index 676f4228405..e6ff41a014f 100644
--- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeEvent.java
+++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeEvent.java
@@ -1,11 +1,14 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.clustercontroller.core;
 
+import java.util.Optional;
+
 public class NodeEvent implements Event {
 
     private final NodeInfo node;
     private final String description;
     private final long eventTime;
+    private final Optional<String> bucketSpace;
 
     public enum Type {
         REPORTED,
@@ -20,6 +23,15 @@ public class NodeEvent implements Event {
         this.description = description;
         this.eventTime = currentTime;
         this.type = type;
+        this.bucketSpace = Optional.empty();
+    }
+
+    public NodeEvent(NodeInfo node, String bucketSpace, String description, Type type, long currentTime) {
+        this.node = node;
+        this.description = description;
+        this.eventTime = currentTime;
+        this.type = type;
+        this.bucketSpace = Optional.of(bucketSpace);
     }
 
     public NodeInfo getNode() {
@@ -38,7 +50,14 @@ public class NodeEvent implements Event {
 
     @Override
     public String toString() {
-        return "Event: " + node.getNode() + ": " + description;
+        return "Event: " + getNodeBucketSpaceDescription() + ": " + description;
+    }
+
+    private String getNodeBucketSpaceDescription() {
+        if (bucketSpace.isPresent()) {
+            return node.getNode() + " (" + bucketSpace.get() + ")";
+        }
+        return node.getNode().toString();
     }
 
     @Override
@@ -50,4 +69,8 @@ public class NodeEvent implements Event {
         return type;
     }
 
+    public Optional<String> getBucketSpace() {
+        return bucketSpace;
+    }
+
 }
diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/EventDiffCalculatorTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/EventDiffCalculatorTest.java
index a09be817e1c..00e55af7287 100644
--- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/EventDiffCalculatorTest.java
+++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/EventDiffCalculatorTest.java
@@ -2,6 +2,8 @@
 package com.yahoo.vespa.clustercontroller.core;
 
 import static com.yahoo.vespa.clustercontroller.core.matchers.EventForNode.eventForNode;
+import static com.yahoo.vespa.clustercontroller.core.matchers.NodeEventForBucketSpace.nodeEventForBucketSpace;
+import static com.yahoo.vespa.clustercontroller.core.matchers.NodeEventForBucketSpace.nodeEventForNullBucketSpace;
 import static com.yahoo.vespa.clustercontroller.core.matchers.NodeEventWithDescription.nodeEventWithDescription;
 import static com.yahoo.vespa.clustercontroller.core.matchers.ClusterEventWithDescription.clusterEventWithDescription;
 import static com.yahoo.vespa.clustercontroller.core.matchers.EventTypeIs.eventTypeIs;
@@ -14,31 +16,21 @@ import static org.hamcrest.CoreMatchers.hasItem;
 import static com.yahoo.vespa.clustercontroller.core.ClusterFixture.storageNode;
 import static com.yahoo.vespa.clustercontroller.core.ClusterFixture.distributorNode;
 
-import com.yahoo.vdslib.state.ClusterState;
-import com.yahoo.vdslib.state.Node;
 import org.junit.Test;
 
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.Optional;
+import java.util.stream.Collectors;
 
 public class EventDiffCalculatorTest {
 
-    private static Map<Node, NodeStateReason> emptyNodeStateReasons() {
-        return Collections.emptyMap();
-    }
-
     private static class EventFixture {
         final ClusterFixture clusterFixture;
-        // TODO could reasonably put shared state into a common class to avoid dupes for both before/after
-        Optional<ClusterStateReason> clusterReasonBefore = Optional.empty();
-        Optional<ClusterStateReason> clusterReasonAfter = Optional.empty();
-        ClusterState clusterStateBefore = ClusterState.emptyState();
-        ClusterState clusterStateAfter = ClusterState.emptyState();
-        final Map<Node, NodeStateReason> nodeReasonsBefore = new HashMap<>();
-        final Map<Node, NodeStateReason> nodeReasonsAfter = new HashMap<>();
+        AnnotatedClusterState.Builder baselineBefore = new AnnotatedClusterState.Builder();
+        AnnotatedClusterState.Builder baselineAfter = new AnnotatedClusterState.Builder();
+        Map<String, AnnotatedClusterState.Builder> derivedBefore = new HashMap<>();
+        Map<String, AnnotatedClusterState.Builder> derivedAfter = new HashMap<>();
         long currentTimeMs = 0;
 
         EventFixture(int nodeCount) {
@@ -46,48 +38,72 @@ public class EventDiffCalculatorTest {
         }
 
         EventFixture clusterStateBefore(String stateStr) {
-            clusterStateBefore = ClusterState.stateFromString(stateStr);
+            baselineBefore.clusterState(stateStr);
             return this;
         }
         EventFixture clusterStateAfter(String stateStr) {
-            clusterStateAfter = ClusterState.stateFromString(stateStr);
+            baselineAfter.clusterState(stateStr);
             return this;
         }
-        EventFixture storageNodeReasonBefore(int index, NodeStateReason reason) {
-            nodeReasonsBefore.put(storageNode(index), reason);
+        EventFixture storageNodeReasonBefore(int nodeIndex, NodeStateReason reason) {
+            baselineBefore.storageNodeReason(nodeIndex, reason);
             return this;
         }
-        EventFixture storageNodeReasonAfter(int index, NodeStateReason reason) {
-            nodeReasonsAfter.put(storageNode(index), reason);
+        EventFixture storageNodeReasonAfter(int nodeIndex, NodeStateReason reason) {
+            baselineAfter.storageNodeReason(nodeIndex, reason);
             return this;
         }
         EventFixture clusterReasonBefore(ClusterStateReason reason) {
-            this.clusterReasonBefore = Optional.of(reason);
+            baselineBefore.clusterReason(reason);
             return this;
         }
         EventFixture clusterReasonAfter(ClusterStateReason reason) {
-            this.clusterReasonAfter = Optional.of(reason);
+            baselineAfter.clusterReason(reason);
             return this;
         }
         EventFixture currentTimeMs(long timeMs) {
             this.currentTimeMs = timeMs;
             return this;
         }
+        EventFixture derivedClusterStateBefore(String bucketSpace, String stateStr) {
+            getBuilder(derivedBefore, bucketSpace).clusterState(stateStr);
+            return this;
+        }
+        EventFixture derivedClusterStateAfter(String bucketSpace, String stateStr) {
+            getBuilder(derivedAfter, bucketSpace).clusterState(stateStr);
+            return this;
+        }
+        EventFixture derivedStorageNodeReasonBefore(String bucketSpace, int nodeIndex, NodeStateReason reason) {
+            getBuilder(derivedBefore, bucketSpace).storageNodeReason(nodeIndex, reason);
+            return this;
+        }
+        EventFixture derivedStorageNodeReasonAfter(String bucketSpace, int nodeIndex, NodeStateReason reason) {
+            getBuilder(derivedAfter, bucketSpace).storageNodeReason(nodeIndex, reason);
+            return this;
+        }
+        private static AnnotatedClusterState.Builder getBuilder(Map<String, AnnotatedClusterState.Builder> derivedStates, String bucketSpace) {
+            AnnotatedClusterState.Builder result = derivedStates.get(bucketSpace);
+            if (result == null) {
+                result = new AnnotatedClusterState.Builder();
+                derivedStates.put(bucketSpace, result);
+            }
+            return result;
+        }
 
         List<Event> computeEventDiff() {
-            final AnnotatedClusterState stateBefore = new AnnotatedClusterState(
-                    clusterStateBefore, clusterReasonBefore, nodeReasonsBefore);
-            final AnnotatedClusterState stateAfter = new AnnotatedClusterState(
-                    clusterStateAfter, clusterReasonAfter, nodeReasonsAfter);
-
             return EventDiffCalculator.computeEventDiff(
                     EventDiffCalculator.params()
                             .cluster(clusterFixture.cluster())
-                            .fromState(stateBefore)
-                            .toState(stateAfter)
+                            .fromState(ClusterStateBundle.of(baselineBefore.build(), toDerivedStates(derivedBefore)))
+                            .toState(ClusterStateBundle.of(baselineAfter.build(), toDerivedStates(derivedAfter)))
                             .currentTimeMs(currentTimeMs));
         }
 
+        private static Map<String, AnnotatedClusterState> toDerivedStates(Map<String, AnnotatedClusterState.Builder> derivedBuilders) {
+            return derivedBuilders.entrySet().stream()
+                    .collect(Collectors.toMap(entry -> entry.getKey(), entry -> entry.getValue().build()));
+        }
+
         static EventFixture createForNodes(int nodeCount) {
             return new EventFixture(nodeCount);
         }
@@ -316,4 +332,84 @@ public class EventDiffCalculatorTest {
                 clusterEventWithDescription("Too low ratio of available distributor nodes. Setting cluster state down")));
     }
 
+    @Test
+    public void may_have_merges_pending_up_edge_event_emitted_if_derived_bucket_space_state_differs_from_baseline() {
+        EventFixture f = EventFixture.createForNodes(3)
+                .clusterStateBefore("distributor:3 storage:3")
+                .derivedClusterStateBefore("default", "distributor:3 storage:3")
+                .clusterStateAfter("distributor:3 storage:3")
+                .derivedClusterStateAfter("default", "distributor:3 storage:3 .1.s:m")
+                .derivedStorageNodeReasonAfter("default", 1, NodeStateReason.MAY_HAVE_MERGES_PENDING);
+
+        List<Event> events = f.computeEventDiff();
+        assertThat(events.size(), equalTo(2));
+        assertThat(events, hasItem(allOf(
+                eventForNode(storageNode(1)),
+                nodeEventForBucketSpace("default"),
+                nodeEventWithDescription("Altered node state in cluster state from 'U' to 'M'"))));
+        assertThat(events, hasItem(allOf(
+                eventForNode(storageNode(1)),
+                nodeEventForBucketSpace("default"),
+                nodeEventWithDescription("Node may have merges pending"))));
+    }
+
+    @Test
+    public void may_have_merges_pending_down_edge_event_emitted_if_derived_bucket_space_state_differs_from_baseline() {
+        EventFixture f = EventFixture.createForNodes(3)
+                .clusterStateBefore("distributor:3 storage:3")
+                .derivedClusterStateBefore("default", "distributor:3 storage:3 .1.s:m")
+                .derivedStorageNodeReasonBefore("default", 1, NodeStateReason.MAY_HAVE_MERGES_PENDING)
+                .clusterStateAfter("distributor:3 storage:3")
+                .derivedClusterStateAfter("default", "distributor:3 storage:3");
+
+        List<Event> events = f.computeEventDiff();
+        assertThat(events.size(), equalTo(2));
+        assertThat(events, hasItem(allOf(
+                eventForNode(storageNode(1)),
+                nodeEventForBucketSpace("default"),
+                nodeEventWithDescription("Altered node state in cluster state from 'M' to 'U'"))));
+        assertThat(events, hasItem(allOf(
+                eventForNode(storageNode(1)),
+                nodeEventForBucketSpace("default"),
+                nodeEventWithDescription("Node no longer have merges pending"))));
+    }
+
+    @Test
+    public void both_baseline_and_derived_bucket_space_state_events_are_emitted() {
+        EventFixture f = EventFixture.createForNodes(3)
+                .clusterStateBefore("distributor:3 storage:3")
+                .derivedClusterStateBefore("default", "distributor:3 storage:3")
+                .clusterStateAfter("distributor:3 storage:3 .0.s:m")
+                .derivedClusterStateAfter("default", "distributor:3 storage:3 .1.s:m");
+
+        List<Event> events = f.computeEventDiff();
+        assertThat(events.size(), equalTo(2));
+        assertThat(events, hasItem(allOf(
+                eventForNode(storageNode(0)),
+                nodeEventForNullBucketSpace(),
+                nodeEventWithDescription("Altered node state in cluster state from 'U' to 'M'"))));
+        assertThat(events, hasItem(allOf(
+                eventForNode(storageNode(1)),
+                nodeEventForBucketSpace("default"),
+                nodeEventWithDescription("Altered node state in cluster state from 'U' to 'M'"))));
+    }
+
+    @Test
+    public void derived_bucket_space_state_events_are_not_emitted_if_similar_to_baseline() {
+        EventFixture f = EventFixture.createForNodes(3)
+                .clusterStateBefore("distributor:3 storage:3")
+                .derivedClusterStateBefore("default", "distributor:3 storage:3")
+                .derivedClusterStateBefore("global", "distributor:3 storage:3")
+                .clusterStateAfter("distributor:3 storage:3 .0.s:m")
+                .derivedClusterStateAfter("default", "distributor:3 storage:3 .0.s:m")
+                .derivedClusterStateAfter("global", "distributor:3 storage:3 .0.s:m");
+
+        List<Event> events = f.computeEventDiff();
+        assertThat(events.size(), equalTo(1));
+        assertThat(events, hasItem(allOf(
+                eventForNode(storageNode(0)),
+                nodeEventForNullBucketSpace(),
+                nodeEventWithDescription("Altered node state in cluster state from 'U' to 'M'"))));
+    }
+
 }
diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/GroupAutoTakedownTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/GroupAutoTakedownTest.java
index 95b6eca88f0..41c5922c932 100644
--- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/GroupAutoTakedownTest.java
+++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/GroupAutoTakedownTest.java
@@ -202,8 +202,8 @@ public class GroupAutoTakedownTest {
 
         final List<Event> events = EventDiffCalculator.computeEventDiff(EventDiffCalculator.params()
                 .cluster(fixture.cluster)
-                .fromState(fixture.annotatedGeneratedClusterState())
-                .toState(annotatedStateAfterStorageTransition(fixture, 5, State.DOWN)));
+                .fromState(ClusterStateBundle.ofBaselineOnly(fixture.annotatedGeneratedClusterState()))
+                .toState(ClusterStateBundle.ofBaselineOnly(annotatedStateAfterStorageTransition(fixture, 5, State.DOWN))));
 
         assertThat(events, hasItem(allOf(
                 nodeEventWithDescription("Group node availability is below configured threshold"),
@@ -220,8 +220,8 @@ public class GroupAutoTakedownTest {
 
         final List<Event> events = EventDiffCalculator.computeEventDiff(EventDiffCalculator.params()
                 .cluster(fixture.cluster)
-                .fromState(fixture.annotatedGeneratedClusterState())
-                .toState(annotatedStateAfterStorageTransition(fixture, 5, State.UP)));
+                .fromState(ClusterStateBundle.ofBaselineOnly(fixture.annotatedGeneratedClusterState()))
+                .toState(ClusterStateBundle.ofBaselineOnly(annotatedStateAfterStorageTransition(fixture, 5, State.UP))));
 
         assertThat(events, hasItem(allOf(
                 nodeEventWithDescription("Group node availability has been restored"),
diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/MaintenanceWhenPendingGlobalMergesTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/MaintenanceWhenPendingGlobalMergesTest.java
index 46b23346c34..a2e661aa161 100644
--- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/MaintenanceWhenPendingGlobalMergesTest.java
+++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/MaintenanceWhenPendingGlobalMergesTest.java
@@ -3,7 +3,6 @@ package com.yahoo.vespa.clustercontroller.core;
 
 import com.yahoo.document.FixedBucketSpaces;
 import com.yahoo.vdslib.state.ClusterState;
-import com.yahoo.vdslib.state.Node;
 import org.junit.Test;
 
 import java.util.*;
@@ -36,27 +35,16 @@ public class MaintenanceWhenPendingGlobalMergesTest {
         return AnnotatedClusterState.withoutAnnotations(ClusterState.stateFromString(stateStr));
     }
 
-    private static class AnnotatedClusterStateBuilder {
-        private ClusterState clusterState;
-        private Map<Node, NodeStateReason> nodeStateReasons = new HashMap<>();
-
-        private AnnotatedClusterStateBuilder(String stateStr) {
-            clusterState = ClusterState.stateFromString(stateStr);
-        }
+    private static class AnnotatedClusterStateBuilder extends AnnotatedClusterState.Builder {
 
         public static AnnotatedClusterStateBuilder ofState(String stateStr) {
-            return new AnnotatedClusterStateBuilder(stateStr);
+            return (AnnotatedClusterStateBuilder) new AnnotatedClusterStateBuilder().clusterState(stateStr);
         }
 
         public AnnotatedClusterStateBuilder reason(NodeStateReason reason, Integer... nodeIndices) {
-            Arrays.stream(nodeIndices).forEach(nodeIndex -> nodeStateReasons.put(Node.ofStorage(nodeIndex), reason));
+            Arrays.stream(nodeIndices).forEach(nodeIndex -> storageNodeReason(nodeIndex, reason));
             return this;
         }
-
-        public AnnotatedClusterState build() {
-            return new AnnotatedClusterState(clusterState, Optional.empty(), nodeStateReasons);
-        }
-
     }
 
     @Test
diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/matchers/NodeEventForBucketSpace.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/matchers/NodeEventForBucketSpace.java
new file mode 100644
index 00000000000..d543ae73ca9
--- /dev/null
+++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/matchers/NodeEventForBucketSpace.java
@@ -0,0 +1,45 @@
+package com.yahoo.vespa.clustercontroller.core.matchers;
+
+import com.yahoo.vespa.clustercontroller.core.NodeEvent;
+import org.hamcrest.Description;
+import org.hamcrest.Factory;
+import org.mockito.ArgumentMatcher;
+
+import java.util.Optional;
+
+public class NodeEventForBucketSpace extends ArgumentMatcher<NodeEvent> {
+    private final Optional<String> bucketSpace;
+
+    public NodeEventForBucketSpace(Optional<String> bucketSpace) {
+        this.bucketSpace = bucketSpace;
+    }
+
+    @Override
+    public boolean matches(Object o) {
+        if (!(o instanceof NodeEvent)) {
+            return false;
+        }
+        return bucketSpace.equals(((NodeEvent) o).getBucketSpace());
+    }
+
+    @Override
+    public void describeTo(Description description) {
+        description.appendText(String.format("NodeEvent for bucket space '%s'", bucketSpace.orElse("null")));
+    }
+
+    @Override
+    public void describeMismatch(Object item, Description description) {
+        NodeEvent other = (NodeEvent)item;
+        description.appendText(String.format("got bucket space '%s'", other.getBucketSpace().orElse("null")));
+    }
+
+    @Factory
+    public static NodeEventForBucketSpace nodeEventForBucketSpace(String bucketSpace) {
+        return new NodeEventForBucketSpace(Optional.of(bucketSpace));
+    }
+
+    @Factory
+    public static NodeEventForBucketSpace nodeEventForNullBucketSpace() {
+        return new NodeEventForBucketSpace(Optional.empty());
+    }
+}