Reduce default bucket delete pri and fix priority inheritance for merge deletion

Since deletes are now async in the backend, make them FIFO-order with
client feed by default to avoid swamping the queues with deletes.
Also, explicitly inherit priority for bucket deletion triggered by
bucket merging. This was actually missing previously and meant such
deletes got the default, very low priority.

4 files changed, 14 insertions, 2 deletions

diff --git a/storage/src/tests/distributor/mergeoperationtest.cpp b/storage/src/tests/distributor/mergeoperationtest.cpp
index 54bd06c98e0..e2b733c0a29 100644
--- a/storage/src/tests/distributor/mergeoperationtest.cpp
+++ b/storage/src/tests/distributor/mergeoperationtest.cpp
@@ -68,6 +68,7 @@ MergeOperationTest::setup_simple_merge_op(const std::vector<uint16_t>& nodes)
 
     auto op = std::make_shared<MergeOperation>(BucketAndNodes(makeDocumentBucket(document::BucketId(16, 1)), nodes));
     op->setIdealStateManager(&getIdealStateManager());
+    op->setPriority(api::StorageMessage::Priority(125));
     op->start(_sender, framework::MilliSecTime(0));
     return op;
 }
@@ -603,4 +604,14 @@ TEST_F(MergeOperationTest, unordered_merges_only_sent_iff_config_enabled_and_all
               _sender.getLastCommand(true));
 }
 
+TEST_F(MergeOperationTest, delete_bucket_inherits_merge_priority) {
+    auto op = setup_simple_merge_op();
+    ASSERT_NO_FATAL_FAILURE(assert_simple_merge_bucket_command());
+    sendReply(*op);
+    ASSERT_NO_FATAL_FAILURE(assert_simple_delete_bucket_command());
+    auto del_cmd = std::dynamic_pointer_cast<api::DeleteBucketCommand>(_sender.commands().back());
+    ASSERT_TRUE(del_cmd);
+    EXPECT_EQ(int(del_cmd->getPriority()), int(op->getPriority()));
+}
+
 } // storage::distributor
diff --git a/storage/src/tests/distributor/statecheckerstest.cpp b/storage/src/tests/distributor/statecheckerstest.cpp
index f5531a134d0..d481370b2c1 100644
--- a/storage/src/tests/distributor/statecheckerstest.cpp
+++ b/storage/src/tests/distributor/statecheckerstest.cpp
@@ -858,7 +858,7 @@ TEST_F(StateCheckersTest, delete_extra_copies) {
 
     EXPECT_EQ("[Removing all copies since bucket is empty:node(idx=0,crc=0x0,"
               "docs=0/0,bytes=0/0,trusted=false,active=false,ready=false)]"
-              " (pri 100)",
+              " (pri 120)",
               testDeleteExtraCopies("0=0", 2, PendingMessage(), "", true)) << "Remove empty buckets";
 
     EXPECT_EQ("[Removing redundant in-sync copy from node 2]",
diff --git a/storage/src/vespa/storage/config/stor-distributormanager.def b/storage/src/vespa/storage/config/stor-distributormanager.def
index 8021075faa3..5162e337f24 100644
--- a/storage/src/vespa/storage/config/stor-distributormanager.def
+++ b/storage/src/vespa/storage/config/stor-distributormanager.def
@@ -103,7 +103,7 @@ priority_activate_no_existing_active int default=100
 priority_activate_with_existing_active int default=100
 
 ## Deletion of bucket copy.
-priority_delete_bucket_copy int default=100
+priority_delete_bucket_copy int default=120
 
 ## Joining caused by bucket siblings getting sufficiently small to fit into a
 ## single bucket.
diff --git a/storage/src/vespa/storage/distributor/operations/idealstate/mergeoperation.cpp b/storage/src/vespa/storage/distributor/operations/idealstate/mergeoperation.cpp
index d220a71966f..6624ca7c8c6 100644
--- a/storage/src/vespa/storage/distributor/operations/idealstate/mergeoperation.cpp
+++ b/storage/src/vespa/storage/distributor/operations/idealstate/mergeoperation.cpp
@@ -253,6 +253,7 @@ MergeOperation::deleteSourceOnlyNodes(
             return;
         }
         _removeOperation->setIdealStateManager(_manager);
+        _removeOperation->setPriority(getPriority());
         
         if (_removeOperation->onStartInternal(sender)) {
             _ok = _removeOperation->ok();