Expose content node entry+document counts via host info

This adds two new fields to the host info payload sent to the
cluster controller; entry count (documents + tombstones) and
visible document count (i.e. sans tombstones).

To preserve symmetry, entry count has also been added to the
metric set, as the host info fields originally referred to raw
metric names.

4 files changed, 24 insertions, 7 deletions

diff --git a/storage/src/tests/bucketdb/bucketmanagertest.cpp b/storage/src/tests/bucketdb/bucketmanagertest.cpp
index 91e901c7254..45d8fab7061 100644
--- a/storage/src/tests/bucketdb/bucketmanagertest.cpp
+++ b/storage/src/tests/bucketdb/bucketmanagertest.cpp
@@ -453,7 +453,8 @@ TEST_F(BucketManagerTest, metrics_are_tracked_per_bucket_space) {
     auto& repo = _node->getComponentRegister().getBucketSpaceRepo();
     {
         bucketdb::StorageBucketInfo entry;
-        api::BucketInfo info(50, 100, 200);
+        // checksum, doc count, doc size, meta count, total bucket size (incl meta)
+        api::BucketInfo info(50, 100, 200, 101, 211);
         info.setReady(true);
         entry.setBucketInfo(info);
         repo.get(document::FixedBucketSpaces::default_space()).bucketDatabase()
@@ -461,7 +462,7 @@ TEST_F(BucketManagerTest, metrics_are_tracked_per_bucket_space) {
     }
     {
         bucketdb::StorageBucketInfo entry;
-        api::BucketInfo info(60, 150, 300);
+        api::BucketInfo info(60, 150, 300, 153, 307);
         info.setActive(true);
         entry.setBucketInfo(info);
         repo.get(document::FixedBucketSpaces::global_space()).bucketDatabase()
@@ -475,6 +476,7 @@ TEST_F(BucketManagerTest, metrics_are_tracked_per_bucket_space) {
     auto default_m = spaces.find(document::FixedBucketSpaces::default_space());
     ASSERT_TRUE(default_m != spaces.end());
     EXPECT_EQ(1,   default_m->second->buckets_total.getLast());
+    EXPECT_EQ(101, default_m->second->entries.getLast());
     EXPECT_EQ(100, default_m->second->docs.getLast());
     EXPECT_EQ(200, default_m->second->bytes.getLast());
     EXPECT_EQ(0,   default_m->second->active_buckets.getLast());
@@ -485,6 +487,7 @@ TEST_F(BucketManagerTest, metrics_are_tracked_per_bucket_space) {
     auto global_m = spaces.find(document::FixedBucketSpaces::global_space());
     ASSERT_TRUE(global_m != spaces.end());
     EXPECT_EQ(1,   global_m->second->buckets_total.getLast());
+    EXPECT_EQ(153, global_m->second->entries.getLast());
     EXPECT_EQ(150, global_m->second->docs.getLast());
     EXPECT_EQ(300, global_m->second->bytes.getLast());
     EXPECT_EQ(1,   global_m->second->active_buckets.getLast());
@@ -499,7 +502,11 @@ TEST_F(BucketManagerTest, metrics_are_tracked_per_bucket_space) {
     jsonStream << End();
     EXPECT_EQ(std::string("{\"values\":["
               "{\"name\":\"vds.datastored.bucket_space.buckets_total\",\"values\":{\"last\":1},\"dimensions\":{\"bucketSpace\":\"global\"}},"
+              "{\"name\":\"vds.datastored.bucket_space.entries\",\"values\":{\"last\":153},\"dimensions\":{\"bucketSpace\":\"global\"}},"
+              "{\"name\":\"vds.datastored.bucket_space.docs\",\"values\":{\"last\":150},\"dimensions\":{\"bucketSpace\":\"global\"}},"
               "{\"name\":\"vds.datastored.bucket_space.buckets_total\",\"values\":{\"last\":1},\"dimensions\":{\"bucketSpace\":\"default\"}},"
+              "{\"name\":\"vds.datastored.bucket_space.entries\",\"values\":{\"last\":101},\"dimensions\":{\"bucketSpace\":\"default\"}},"
+              "{\"name\":\"vds.datastored.bucket_space.docs\",\"values\":{\"last\":100},\"dimensions\":{\"bucketSpace\":\"default\"}},"
               "{\"name\":\"vds.datastored.alldisks.docs\",\"values\":{\"last\":250}},"
               "{\"name\":\"vds.datastored.alldisks.bytes\",\"values\":{\"last\":500}},"
               "{\"name\":\"vds.datastored.alldisks.buckets\",\"values\":{\"last\":2}}"
diff --git a/storage/src/vespa/storage/bucketdb/bucketmanager.cpp b/storage/src/vespa/storage/bucketdb/bucketmanager.cpp
index d12a9f72ac1..5337be6d79f 100644
--- a/storage/src/vespa/storage/bucketdb/bucketmanager.cpp
+++ b/storage/src/vespa/storage/bucketdb/bucketmanager.cpp
@@ -148,12 +148,13 @@ DistributorInfoGatherer::operator()(uint64_t bucketId, const StorBucketDatabase:
 struct MetricsUpdater {
     struct Count {
         uint64_t docs;
+        uint64_t entries; // docs + tombstones
         uint64_t bytes;
         uint64_t buckets;
         uint64_t active;
         uint64_t ready;
 
-        constexpr Count() noexcept : docs(0), bytes(0), buckets(0), active(0), ready(0) {}
+        constexpr Count() noexcept : docs(0), entries(0), bytes(0), buckets(0), active(0), ready(0) {}
     };
     Count    count;
     uint32_t lowestUsedBit;
@@ -174,8 +175,9 @@ struct MetricsUpdater {
             if (data.getBucketInfo().isReady()) {
                 ++count.ready;
             }
-            count.docs += data.getBucketInfo().getDocumentCount();
-            count.bytes += data.getBucketInfo().getTotalDocumentSize();
+            count.docs    += data.getBucketInfo().getDocumentCount();
+            count.entries += data.getBucketInfo().getMetaCount();
+            count.bytes   += data.getBucketInfo().getTotalDocumentSize();
 
             if (bucket.getUsedBits() < lowestUsedBit) {
                 lowestUsedBit = bucket.getUsedBits();
@@ -188,6 +190,7 @@ struct MetricsUpdater {
         const auto& s = rhs.count;
         d.buckets += s.buckets;
         d.docs    += s.docs;
+        d.entries += s.entries;
         d.bytes   += s.bytes;
         d.ready   += s.ready;
         d.active  += s.active;
@@ -234,11 +237,15 @@ BucketManager::report(vespalib::JsonStream & json) const {
         MetricsUpdater m = getMetrics(space.second->bucketDatabase());
         output(json, "vds.datastored.bucket_space.buckets_total", m.count.buckets,
                document::FixedBucketSpaces::to_string(space.first));
+        output(json, "vds.datastored.bucket_space.entries", m.count.entries,
+               document::FixedBucketSpaces::to_string(space.first));
+        output(json, "vds.datastored.bucket_space.docs", m.count.docs,
+               document::FixedBucketSpaces::to_string(space.first));
         total.add(m);
     }
     const auto & src = total.count;
-    output(json, "vds.datastored.alldisks.docs", src.docs);
-    output(json, "vds.datastored.alldisks.bytes", src.bytes);
+    output(json, "vds.datastored.alldisks.docs",    src.docs);
+    output(json, "vds.datastored.alldisks.bytes",   src.bytes);
     output(json, "vds.datastored.alldisks.buckets", src.buckets);
 }
 
@@ -258,6 +265,7 @@ BucketManager::updateMetrics() const
         auto bm = _metrics->bucket_spaces.find(space.first);
         assert(bm != _metrics->bucket_spaces.end());
         bm->second->buckets_total.set(m.count.buckets);
+        bm->second->entries.set(m.count.entries);
         bm->second->docs.set(m.count.docs);
         bm->second->bytes.set(m.count.bytes);
         bm->second->active_buckets.set(m.count.active);
diff --git a/storage/src/vespa/storage/bucketdb/bucketmanagermetrics.cpp b/storage/src/vespa/storage/bucketdb/bucketmanagermetrics.cpp
index ca9e556f83c..d2b019cc50d 100644
--- a/storage/src/vespa/storage/bucketdb/bucketmanagermetrics.cpp
+++ b/storage/src/vespa/storage/bucketdb/bucketmanagermetrics.cpp
@@ -31,6 +31,7 @@ ContentBucketDbMetrics::~ContentBucketDbMetrics() = default;
 BucketSpaceMetrics::BucketSpaceMetrics(const vespalib::string& space_name, metrics::MetricSet* owner)
     : metrics::MetricSet("bucket_space", {{"bucketSpace", space_name}}, "", owner),
       buckets_total("buckets_total", {}, "Total number buckets present in the bucket space (ready + not ready)", this),
+      entries("entries", {}, "Number of entries (documents + tombstones) stored in the bucket space", this),
       docs("docs", {}, "Documents stored in the bucket space", this),
       bytes("bytes", {}, "Bytes stored across all documents in the bucket space", this),
       active_buckets("active_buckets", {}, "Number of active buckets in the bucket space", this),
diff --git a/storage/src/vespa/storage/bucketdb/bucketmanagermetrics.h b/storage/src/vespa/storage/bucketdb/bucketmanagermetrics.h
index a73bb676526..cab3a397c54 100644
--- a/storage/src/vespa/storage/bucketdb/bucketmanagermetrics.h
+++ b/storage/src/vespa/storage/bucketdb/bucketmanagermetrics.h
@@ -34,6 +34,7 @@ struct ContentBucketDbMetrics : metrics::MetricSet {
 struct BucketSpaceMetrics : metrics::MetricSet {
     // Superficially very similar to DataStoredMetrics, but metric naming and dimensions differ
     metrics::LongValueMetric buckets_total;
+    metrics::LongValueMetric entries;
     metrics::LongValueMetric docs;
     metrics::LongValueMetric bytes;
     metrics::LongValueMetric active_buckets;