Produce per node bucket spaces stats from the distributor based on node maintenance stats.

4 files changed, 60 insertions, 6 deletions

diff --git a/storage/src/tests/distributor/distributortest.cpp b/storage/src/tests/distributor/distributortest.cpp
index 4d08e4adb71..776f4e200e8 100644
--- a/storage/src/tests/distributor/distributortest.cpp
+++ b/storage/src/tests/distributor/distributortest.cpp
@@ -94,6 +94,8 @@ protected:
     void internal_messages_are_started_in_fifo_order_batch();
     void closing_aborts_priority_queued_client_requests();
 
+    void assertBucketSpaceStats(size_t expBucketPending, uint16_t node, const vespalib::string &bucketSpace,
+                                const BucketSpacesStatsProvider::PerNodeBucketSpacesStats &stats);
     std::vector<document::BucketSpace> _bucketSpaces;
 
 public:
@@ -643,6 +645,25 @@ Distributor_Test::mergeStatsAreAccumulatedDuringDatabaseIteration()
         wanted.copyingIn = 2;
         CPPUNIT_ASSERT_EQUAL(wanted, stats.perNodeStats.forNode(2, makeBucketSpace()));
     }
+    auto bucketStats = _distributor->getBucketSpacesStats();
+    CPPUNIT_ASSERT_EQUAL(static_cast<size_t>(3), bucketStats.size());
+    assertBucketSpaceStats(1, 0, "default", bucketStats);
+    assertBucketSpaceStats(0, 1, "default", bucketStats);
+    assertBucketSpaceStats(3, 2, "default", bucketStats);
+}
+
+void
+Distributor_Test::assertBucketSpaceStats(size_t expBucketPending, uint16_t node, const vespalib::string &bucketSpace,
+                                         const BucketSpacesStatsProvider::PerNodeBucketSpacesStats &stats)
+{
+    auto nodeItr = stats.find(node);
+    CPPUNIT_ASSERT(nodeItr != stats.end());
+    CPPUNIT_ASSERT_EQUAL(static_cast<size_t>(1), nodeItr->second.size());
+    auto bucketSpaceItr = nodeItr->second.find(bucketSpace);
+    CPPUNIT_ASSERT(bucketSpaceItr != nodeItr->second.end());
+    CPPUNIT_ASSERT(bucketSpaceItr->second.valid());
+    CPPUNIT_ASSERT_EQUAL(static_cast<size_t>(0), bucketSpaceItr->second.bucketsTotal());
+    CPPUNIT_ASSERT_EQUAL(expBucketPending, bucketSpaceItr->second.bucketsPending());
 }
 
 /**
diff --git a/storage/src/vespa/storage/distributor/distributor.cpp b/storage/src/vespa/storage/distributor/distributor.cpp
index 32554d02397..39658912a2e 100644
--- a/storage/src/vespa/storage/distributor/distributor.cpp
+++ b/storage/src/vespa/storage/distributor/distributor.cpp
@@ -93,6 +93,7 @@ Distributor::Distributor(DistributorComponentRegister& compReg,
       _metricUpdateHook(*this),
       _metricLock(),
       _maintenanceStats(),
+      _bucketSpacesStats(),
       _bucketDbStats(),
       _hostInfoReporter(_pendingMessageTracker.getLatencyStatisticsProvider(), *this, *this),
       _ownershipSafeTimeCalc(
@@ -610,6 +611,13 @@ Distributor::getMinReplica() const
     return _bucketDbStats._minBucketReplica;
 }
 
+BucketSpacesStatsProvider::PerNodeBucketSpacesStats
+Distributor::getBucketSpacesStats() const
+{
+    vespalib::LockGuard guard(_metricLock);
+    return _bucketSpacesStats;
+}
+
 void
 Distributor::propagateInternalScanMetricsToExternal()
 {
@@ -624,6 +632,29 @@ Distributor::propagateInternalScanMetricsToExternal()
     }
 }
 
+namespace {
+
+BucketSpaceStats
+toBucketSpaceStats(const NodeMaintenanceStats &stats)
+{
+    return BucketSpaceStats(0, stats.syncing + stats.copyingIn);
+}
+
+BucketSpacesStatsProvider::PerNodeBucketSpacesStats
+toBucketSpacesStats(const NodeMaintenanceStatsTracker &maintenanceStats)
+{
+    BucketSpacesStatsProvider::PerNodeBucketSpacesStats result;
+    for (const auto &nodeEntry : maintenanceStats.perNodeStats()) {
+        for (const auto &bucketSpaceEntry : nodeEntry.second) {
+            auto bucketSpace = document::FixedBucketSpaces::to_string(bucketSpaceEntry.first);
+            result[nodeEntry.first][bucketSpace] = toBucketSpaceStats(bucketSpaceEntry.second);
+        }
+    }
+    return result;
+}
+
+}
+
 void
 Distributor::updateInternalMetricsForCompletedScan()
 {
@@ -632,7 +663,7 @@ Distributor::updateInternalMetricsForCompletedScan()
     _bucketDBMetricUpdater.completeRound();
     _bucketDbStats = _bucketDBMetricUpdater.getLastCompleteStats();
     _maintenanceStats = _scanner->getPendingMaintenanceStats();
-
+    _bucketSpacesStats = toBucketSpacesStats(_maintenanceStats.perNodeStats);
 }
 
 void
diff --git a/storage/src/vespa/storage/distributor/distributor.h b/storage/src/vespa/storage/distributor/distributor.h
index 1cf0a4f1866..b4b235838c5 100644
--- a/storage/src/vespa/storage/distributor/distributor.h
+++ b/storage/src/vespa/storage/distributor/distributor.h
@@ -198,10 +198,7 @@ private:
      */
     std::unordered_map<uint16_t, uint32_t> getMinReplica() const override;
 
-    PerNodeBucketSpacesStats getBucketSpacesStats() const override {
-        // TODO: implement
-        return BucketSpacesStatsProvider::PerNodeBucketSpacesStats();
-    }
+    PerNodeBucketSpacesStats getBucketSpacesStats() const override;
 
     /**
      * Atomically publish internal metrics to external ideal state metrics.
@@ -305,6 +302,7 @@ private:
      * manager thread but written by distributor thread.
      */
     SimpleMaintenanceScanner::PendingMaintenanceStats _maintenanceStats;
+    BucketSpacesStatsProvider::PerNodeBucketSpacesStats _bucketSpacesStats;
     BucketDBMetricUpdater::Stats _bucketDbStats;
     DistributorHostInfoReporter _hostInfoReporter;
     std::unique_ptr<OwnershipTransferSafeTimePointCalculator> _ownershipSafeTimeCalc;
diff --git a/storage/src/vespa/storage/distributor/maintenance/node_maintenance_stats_tracker.h b/storage/src/vespa/storage/distributor/maintenance/node_maintenance_stats_tracker.h
index 7cf65e4a7d8..891e6fd7fa7 100644
--- a/storage/src/vespa/storage/distributor/maintenance/node_maintenance_stats_tracker.h
+++ b/storage/src/vespa/storage/distributor/maintenance/node_maintenance_stats_tracker.h
@@ -38,9 +38,10 @@ class NodeMaintenanceStatsTracker
 {
 public:
     using BucketSpacesStats = std::unordered_map<document::BucketSpace, NodeMaintenanceStats, document::BucketSpace::hash>;
+    using PerNodeStats = std::unordered_map<uint16_t, BucketSpacesStats>;
 
 private:
-    std::unordered_map<uint16_t, BucketSpacesStats> _stats;
+    PerNodeStats _stats;
     static const NodeMaintenanceStats _emptyNodeMaintenanceStats;
 
 public:
@@ -77,6 +78,9 @@ public:
         return _emptyNodeMaintenanceStats;
     }
 
+    const PerNodeStats& perNodeStats() const {
+        return _stats;
+    }
 };
 
 } // distributor