Add config override for simulating bucket info request processing latency

Simulates added request latency caused by the BucketManager computing bucket
ownership for a very large number of buckets.

Fetched at BucketManager init only, so not a dynamic config. This is only
meant for internal testing so should not have any practical consequences.

4 files changed, 22 insertions, 10 deletions

diff --git a/storage/src/tests/bucketdb/bucketmanagertest.cpp b/storage/src/tests/bucketdb/bucketmanagertest.cpp
index ad46d494d11..181bac4c6dd 100644
--- a/storage/src/tests/bucketdb/bucketmanagertest.cpp
+++ b/storage/src/tests/bucketdb/bucketmanagertest.cpp
@@ -156,7 +156,7 @@ void BucketManagerTest::setupTestEnvironment(bool fakePersistenceLayer,
     _node->setTypeRepo(repo);
     _node->setupDummyPersistence();
     // Set up the 3 links
-    auto manager = std::make_unique<BucketManager>("", _node->getComponentRegister());
+    auto manager = std::make_unique<BucketManager>(config.getConfigId(), _node->getComponentRegister());
     _manager = manager.get();
     _top->push_back(std::move(manager));
     if (fakePersistenceLayer) {
diff --git a/storage/src/vespa/storage/bucketdb/bucketmanager.cpp b/storage/src/vespa/storage/bucketdb/bucketmanager.cpp
index 19012cfab18..c8cdaaaaa23 100644
--- a/storage/src/vespa/storage/bucketdb/bucketmanager.cpp
+++ b/storage/src/vespa/storage/bucketdb/bucketmanager.cpp
@@ -8,6 +8,7 @@
 #include <vespa/storage/common/nodestateupdater.h>
 #include <vespa/storage/common/global_bucket_space_distribution_converter.h>
 #include <vespa/vdslib/state/cluster_state_bundle.h>
+#include <vespa/storage/config/config-stor-server.h>
 #include <vespa/storage/storageutil/distributorstatecache.h>
 #include <vespa/storageframework/generic/status/htmlstatusreporter.h>
 #include <vespa/storageframework/generic/status/xmlstatusreporter.h>
@@ -18,8 +19,8 @@
 #include <vespa/storageapi/message/stat.h>
 #include <vespa/document/bucket/fixed_bucket_spaces.h>
 #include <vespa/vespalib/util/stringfmt.h>
-#include <vespa/vespalib/stllike/hash_map.hpp>
 #include <vespa/config/config.h>
+#include <vespa/config/helper/configgetter.hpp>
 #include <chrono>
 
 #include <vespa/log/bufferedlogger.h>
@@ -46,7 +47,8 @@ BucketManager::BucketManager(const config::ConfigUri & configUri,
       _doneInitialized(false),
       _requestsCurrentlyProcessing(0),
       _component(compReg, "bucketmanager"),
-      _metrics(std::make_shared<BucketManagerMetrics>(_component.getBucketSpaceRepo()))
+      _metrics(std::make_shared<BucketManagerMetrics>(_component.getBucketSpaceRepo())),
+      _simulated_processing_delay(0)
 {
     _metrics->setDisks(_component.getDiskCount());
     _component.registerStatusPage(*this);
@@ -60,6 +62,10 @@ BucketManager::BucketManager(const config::ConfigUri & configUri,
             *_component.getStateUpdater().getReportedNodeState());
     ns.setMinUsedBits(58);
     _component.getStateUpdater().setReportedNodeState(ns);
+
+    auto server_config = config::ConfigGetter<vespa::config::content::core::StorServerConfig>::getConfig(
+            configUri.getConfigId(), configUri.getContext());
+    _simulated_processing_delay = std::chrono::milliseconds(std::max(0, server_config->simulatedBucketRequestLatencyMsec));
 }
 
 BucketManager::~BucketManager()
@@ -632,6 +638,11 @@ BucketManager::processRequestBucketInfoCommands(document::BucketSpace bucketSpac
         }
     }
 
+    if (_simulated_processing_delay.count() > 0) {
+        LOG(info, "Simulating bucket processing delay for %" PRIu64 " ms", _simulated_processing_delay.count());
+        std::this_thread::sleep_for(_simulated_processing_delay);
+    }
+
     _metrics->fullBucketInfoRequestSize.addValue(requests.size());
     LOG(debug, "Processing %zu bucket info requests for "
                "distributors %s, using system state %s",
diff --git a/storage/src/vespa/storage/bucketdb/bucketmanager.h b/storage/src/vespa/storage/bucketdb/bucketmanager.h
index 0503cb33d49..cb534ad4002 100644
--- a/storage/src/vespa/storage/bucketdb/bucketmanager.h
+++ b/storage/src/vespa/storage/bucketdb/bucketmanager.h
@@ -1,13 +1,8 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 /**
- * @class storage::BucketManager
- * @ingroup bucketdb
+ * Storage link handling requests concerning buckets.
  *
- * @brief Storage link handling requests concerning buckets.
- *
- * @author H�kon Humberset
- * @date 2006-01-16
- * @version $Id$
+ * @author Håkon Humberset
  */
 
 #pragma once
@@ -23,6 +18,7 @@
 #include <vespa/storageframework/generic/metric/metricupdatehook.h>
 #include <vespa/storageframework/generic/status/statusreporter.h>
 
+#include <chrono>
 #include <list>
 #include <unordered_map>
 #include <unordered_set>
@@ -84,6 +80,7 @@ private:
     ServiceLayerComponent _component;
     std::shared_ptr<BucketManagerMetrics> _metrics;
     framework::Thread::UP _thread;
+    std::chrono::milliseconds _simulated_processing_delay;
 
     BucketManager(const BucketManager&);
     BucketManager& operator=(const BucketManager&);
diff --git a/storage/src/vespa/storage/config/stor-server.def b/storage/src/vespa/storage/config/stor-server.def
index 36ca88dd7de..5d303b59d81 100644
--- a/storage/src/vespa/storage/config/stor-server.def
+++ b/storage/src/vespa/storage/config/stor-server.def
@@ -80,3 +80,7 @@ switch_new_meta_data_flow bool default=false restart
 ## operations.
 bucket_rechecking_chunk_size int default=100
 
+## If greater than zero, simulates added latency caused by CPU processing during
+## full bucket info requests. The latency is added per batch of operations processed.
+## Only useful for testing!
+simulated_bucket_request_latency_msec int default=0