Add count metric for number of documents garbage collected

New distributor metric available as:
```
vds.idealstate.garbage_collection.documents_removed
```
Add documents removed statistics to `RemoveLocation` responses,
which is what GC is currently built around. Could technically have
been implemented as a diff of before/after BucketInfo, but GC is
very low priority so many other mutating ops may have changed the
bucket document set in the time span between sending the GC ops
and receiving the replies.

This relates to issue #12139

5 files changed, 31 insertions, 8 deletions

diff --git a/storageapi/src/tests/mbusprot/storageprotocoltest.cpp b/storageapi/src/tests/mbusprot/storageprotocoltest.cpp
index 2f959e40e2a..2e5eb115844 100644
--- a/storageapi/src/tests/mbusprot/storageprotocoltest.cpp
+++ b/storageapi/src/tests/mbusprot/storageprotocoltest.cpp
@@ -522,8 +522,15 @@ TEST_P(StorageProtocolTest, remove_location) {
     EXPECT_EQ("id.group == \"mygroup\"", cmd2->getDocumentSelection());
     EXPECT_EQ(_bucket, cmd2->getBucket());
 
-    auto reply = std::make_shared<RemoveLocationReply>(*cmd2);
+    uint32_t n_docs_removed = 12345;
+    auto reply = std::make_shared<RemoveLocationReply>(*cmd2, n_docs_removed);
     auto reply2 = copyReply(reply);
+    if (GetParam().getMajor() == 7) {
+        // Statistics are only available for protobuf-enabled version.
+        EXPECT_EQ(n_docs_removed, reply2->documents_removed());
+    } else {
+        EXPECT_EQ(0, reply2->documents_removed());
+    }
 }
 
 TEST_P(StorageProtocolTest, create_visitor) {
diff --git a/storageapi/src/vespa/storageapi/mbusprot/protobuf/feed.proto b/storageapi/src/vespa/storageapi/mbusprot/protobuf/feed.proto
index 810f88f588f..12dbaf59146 100644
--- a/storageapi/src/vespa/storageapi/mbusprot/protobuf/feed.proto
+++ b/storageapi/src/vespa/storageapi/mbusprot/protobuf/feed.proto
@@ -90,7 +90,12 @@ message RemoveLocationRequest {
     bytes  document_selection = 2;
 }
 
+message RemoveLocationStats {
+    uint32 documents_removed = 1;
+}
+
 message RemoveLocationResponse {
     BucketInfo bucket_info        = 1;
     BucketId   remapped_bucket_id = 2;
+    RemoveLocationStats stats     = 3;
 }
diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization7.cpp b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization7.cpp
index 9751fd1be98..90c8d1c7d2a 100644
--- a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization7.cpp
+++ b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization7.cpp
@@ -643,7 +643,9 @@ void ProtocolSerialization7::onEncode(GBBuf& buf, const api::RemoveLocationComma
 }
 
 void ProtocolSerialization7::onEncode(GBBuf& buf, const api::RemoveLocationReply& msg) const {
-    encode_bucket_info_response<protobuf::RemoveLocationResponse>(buf, msg, no_op_encode);
+    encode_bucket_info_response<protobuf::RemoveLocationResponse>(buf, msg, [&](auto& res) {
+        res.mutable_stats()->set_documents_removed(msg.documents_removed());
+    });
 }
 
 api::StorageCommand::UP ProtocolSerialization7::onDecodeRemoveLocationCommand(BBuf& buf) const {
@@ -653,8 +655,11 @@ api::StorageCommand::UP ProtocolSerialization7::onDecodeRemoveLocationCommand(BB
 }
 
 api::StorageReply::UP ProtocolSerialization7::onDecodeRemoveLocationReply(const SCmd& cmd, BBuf& buf) const {
-    return decode_bucket_info_response<protobuf::RemoveLocationResponse>(buf, [&]([[maybe_unused]] auto& res) {
-        return std::make_unique<api::RemoveLocationReply>(static_cast<const api::RemoveLocationCommand&>(cmd));
+    return decode_bucket_info_response<protobuf::RemoveLocationResponse>(buf, [&](auto& res) {
+        uint32_t documents_removed = (res.has_stats() ? res.stats().documents_removed() : 0u);
+        return std::make_unique<api::RemoveLocationReply>(
+                static_cast<const api::RemoveLocationCommand&>(cmd),
+                documents_removed);
     });
 }
 
diff --git a/storageapi/src/vespa/storageapi/message/removelocation.cpp b/storageapi/src/vespa/storageapi/message/removelocation.cpp
index b53584601ef..49c9d22f5ee 100644
--- a/storageapi/src/vespa/storageapi/message/removelocation.cpp
+++ b/storageapi/src/vespa/storageapi/message/removelocation.cpp
@@ -25,8 +25,9 @@ RemoveLocationCommand::print(std::ostream& out, bool verbose, const std::string&
     BucketInfoCommand::print(out, verbose, indent);
 }
 
-RemoveLocationReply::RemoveLocationReply(const RemoveLocationCommand& cmd)
-    : BucketInfoReply(cmd)
+RemoveLocationReply::RemoveLocationReply(const RemoveLocationCommand& cmd, uint32_t docs_removed)
+    : BucketInfoReply(cmd),
+      _documents_removed(docs_removed)
 {
 }
 
diff --git a/storageapi/src/vespa/storageapi/message/removelocation.h b/storageapi/src/vespa/storageapi/message/removelocation.h
index 46555497035..812cc8c413b 100644
--- a/storageapi/src/vespa/storageapi/message/removelocation.h
+++ b/storageapi/src/vespa/storageapi/message/removelocation.h
@@ -11,7 +11,7 @@ class RemoveLocationCommand : public BucketInfoCommand
 {
 public:
     RemoveLocationCommand(vespalib::stringref documentSelection, const document::Bucket &bucket);
-    ~RemoveLocationCommand();
+    ~RemoveLocationCommand() override;
 
     void print(std::ostream& out, bool verbose, const std::string& indent) const override;
     const vespalib::string& getDocumentSelection() const { return _documentSelection; }
@@ -22,8 +22,13 @@ private:
 
 class RemoveLocationReply : public BucketInfoReply
 {
+    uint32_t _documents_removed;
 public:
-    RemoveLocationReply(const RemoveLocationCommand& cmd);
+    explicit RemoveLocationReply(const RemoveLocationCommand& cmd, uint32_t docs_removed = 0);
+    void set_documents_removed(uint32_t docs_removed) noexcept {
+        _documents_removed = docs_removed;
+    }
+    uint32_t documents_removed() const noexcept { return _documents_removed; }
     DECLARE_STORAGEREPLY(RemoveLocationReply, onRemoveLocationReply)
 };