Immediately send GetNodeState reply on "no more merges" pending edge

Lets the cluster controller update the derived bucket space states
as quickly as possible when merges are done for the global space,
without having to wait for the normal reply timeout period.

1 files changed, 52 insertions, 0 deletions

diff --git a/storage/src/tests/distributor/distributortest.cpp b/storage/src/tests/distributor/distributortest.cpp
index 33549226b98..28fccf438ff 100644
--- a/storage/src/tests/distributor/distributortest.cpp
+++ b/storage/src/tests/distributor/distributortest.cpp
@@ -20,6 +20,9 @@
 using document::test::makeDocumentBucket;
 using document::test::makeBucketSpace;
 using document::FixedBucketSpaces;
+using document::BucketSpace;
+using document::Bucket;
+using document::BucketId;
 
 namespace storage {
 
@@ -60,6 +63,8 @@ class Distributor_Test : public CppUnit::TestFixture,
     CPPUNIT_TEST(closing_aborts_priority_queued_client_requests);
     CPPUNIT_TEST(entering_recovery_mode_resets_bucket_space_stats);
     CPPUNIT_TEST(leaving_recovery_mode_immediately_sends_getnodestate_replies);
+    CPPUNIT_TEST(pending_to_no_pending_default_merges_edge_immediately_sends_getnodestate_replies);
+    CPPUNIT_TEST(pending_to_no_pending_global_merges_edge_immediately_sends_getnodestate_replies);
     CPPUNIT_TEST_SUITE_END();
 
 public:
@@ -97,6 +102,10 @@ protected:
     void closing_aborts_priority_queued_client_requests();
     void entering_recovery_mode_resets_bucket_space_stats();
     void leaving_recovery_mode_immediately_sends_getnodestate_replies();
+    void pending_to_no_pending_default_merges_edge_immediately_sends_getnodestate_replies();
+    void pending_to_no_pending_global_merges_edge_immediately_sends_getnodestate_replies();
+    // TODO handle edge case for window between getnodestate reply already
+    // sent and new request not yet received
 
     void assertBucketSpaceStats(size_t expBucketPending, size_t expBucketTotal, uint16_t node, const vespalib::string &bucketSpace,
                                 const BucketSpacesStatsProvider::PerNodeBucketSpacesStats &stats);
@@ -211,6 +220,7 @@ private:
     void assertNoMessageBounced();
     void configure_mutation_sequencing(bool enabled);
     void configure_merge_busy_inhibit_duration(int seconds);
+    void do_test_pending_merge_getnodestate_reply_edge(BucketSpace space);
 };
 
 CPPUNIT_TEST_SUITE_REGISTRATION(Distributor_Test);
@@ -1067,6 +1077,48 @@ void Distributor_Test::leaving_recovery_mode_immediately_sends_getnodestate_repl
     CPPUNIT_ASSERT_EQUAL(size_t(1), explicit_node_state_reply_send_invocations());
 }
 
+void Distributor_Test::do_test_pending_merge_getnodestate_reply_edge(BucketSpace space) {
+    setupDistributor(Redundancy(2), NodeCount(2), "version:1 distributor:1 storage:2");
+    // 2 buckets with missing replicas triggering merge pending stats
+    addNodesToBucketDB(Bucket(space, BucketId(16, 1)), "0=1/1/1/t/a");
+    addNodesToBucketDB(Bucket(space, BucketId(16, 2)), "0=1/1/1/t/a");
+    tickDistributorNTimes(3);
+    CPPUNIT_ASSERT(!_distributor->isInRecoveryMode());
+    const auto space_name = FixedBucketSpaces::to_string(space);
+    assertBucketSpaceStats(2, 0, 1, space_name, _distributor->getBucketSpacesStats());
+    CPPUNIT_ASSERT_EQUAL(size_t(0), explicit_node_state_reply_send_invocations());
+
+    // Edge not triggered when 1 bucket with missing replica left
+    addNodesToBucketDB(Bucket(space, BucketId(16, 1)), "0=1/1/1/t/a,1=1/1/1/t");
+    tickDistributorNTimes(3);
+    assertBucketSpaceStats(1, 1, 1, space_name, _distributor->getBucketSpacesStats());
+    CPPUNIT_ASSERT_EQUAL(size_t(0), explicit_node_state_reply_send_invocations());
+
+    // Edge triggered when no more buckets with requiring merge
+    addNodesToBucketDB(Bucket(space, BucketId(16, 2)), "0=1/1/1/t/a,1=1/1/1/t");
+    tickDistributorNTimes(3);
+    assertBucketSpaceStats(0, 2, 1, space_name, _distributor->getBucketSpacesStats());
+    CPPUNIT_ASSERT_EQUAL(size_t(1), explicit_node_state_reply_send_invocations());
+
+    // Should only send when edge happens, not in subsequent DB iterations
+    tickDistributorNTimes(10);
+    CPPUNIT_ASSERT_EQUAL(size_t(1), explicit_node_state_reply_send_invocations());
+
+    // Going back to merges pending should _not_ send a getnodestate reply (at least for now)
+    addNodesToBucketDB(Bucket(space, BucketId(16, 1)), "0=1/1/1/t/a");
+    tickDistributorNTimes(3);
+    assertBucketSpaceStats(1, 1, 1, space_name, _distributor->getBucketSpacesStats());
+    CPPUNIT_ASSERT_EQUAL(size_t(1), explicit_node_state_reply_send_invocations());
+}
+
+void Distributor_Test::pending_to_no_pending_default_merges_edge_immediately_sends_getnodestate_replies() {
+    do_test_pending_merge_getnodestate_reply_edge(FixedBucketSpaces::default_space());
+}
+
+void Distributor_Test::pending_to_no_pending_global_merges_edge_immediately_sends_getnodestate_replies() {
+    do_test_pending_merge_getnodestate_reply_edge(FixedBucketSpaces::global_space());
+}
+
 }
 
 }