Merge pull request #8229 from vespa-engine/balder/multiple-threads-during-shutdown-of-subdbs

Use multiple threads during shutdown of documentdbs to speed up restart

4 files changed, 52 insertions, 50 deletions

diff --git a/searchcore/src/vespa/searchcore/proton/common/hw_info.h b/searchcore/src/vespa/searchcore/proton/common/hw_info.h
index 9129aa43483..d742a300d22 100644
--- a/searchcore/src/vespa/searchcore/proton/common/hw_info.h
+++ b/searchcore/src/vespa/searchcore/proton/common/hw_info.h
@@ -3,6 +3,7 @@
 #pragma once
 
 #include <cstdint>
+#include <algorithm>
 
 namespace proton {
 
@@ -41,7 +42,7 @@ public:
     private:
         uint32_t _cores;
     public:
-        Cpu(uint32_t cores_) : _cores(cores_) {}
+        Cpu(uint32_t cores_) : _cores(std::max(1u, cores_)) { }
         uint32_t cores() const { return _cores; }
         bool operator == (const Cpu & rhs) const { return _cores == rhs._cores; }
     };
@@ -55,7 +56,7 @@ public:
     HwInfo()
         : _disk(0, false, false),
           _memory(0),
-          _cpu(0)
+          _cpu(1)
     {
     }
 
diff --git a/searchcore/src/vespa/searchcore/proton/common/hw_info_sampler.cpp b/searchcore/src/vespa/searchcore/proton/common/hw_info_sampler.cpp
index 190e904ad68..c94a3048298 100644
--- a/searchcore/src/vespa/searchcore/proton/common/hw_info_sampler.cpp
+++ b/searchcore/src/vespa/searchcore/proton/common/hw_info_sampler.cpp
@@ -121,9 +121,7 @@ HwInfoSampler::HwInfoSampler(const vespalib::string &path,
           HwInfo::Cpu(sampleCpuCores(config)));
 }
 
-HwInfoSampler::~HwInfoSampler()
-{
-}
+HwInfoSampler::~HwInfoSampler() = default;
 
 void
 HwInfoSampler::setup(const HwInfo::Disk &disk, const HwInfo::Memory &memory, const HwInfo::Cpu &cpu)
diff --git a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp
index 22b50a65a41..a3cd4222a06 100644
--- a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp
@@ -8,12 +8,14 @@
 #include "maintenancecontroller.h"
 #include "searchabledocsubdb.h"
 #include <vespa/searchcore/proton/metrics/documentdb_tagged_metrics.h>
+#include <vespa/vespalib/util/lambdatask.h>
 
 using proton::matching::SessionManager;
 using search::GrowStrategy;
 using search::SerialNum;
 using search::index::Schema;
 using searchcorespi::IFlushTarget;
+using vespalib::makeLambdaTask;
 
 namespace proton {
 
@@ -52,7 +54,8 @@ DocumentSubDBCollection::DocumentSubDBCollection(
       _retrievers(),
       _reprocessingRunner(),
       _bucketDB(),
-      _bucketDBHandler()
+      _bucketDBHandler(),
+      _hwInfo(hwInfo)
 {
     _bucketDB = std::make_shared<BucketDBOwner>();
     _bucketDBHandler = std::make_unique<bucketdb::BucketDBHandler>(*_bucketDB);
@@ -60,59 +63,57 @@ DocumentSubDBCollection::DocumentSubDBCollection(
     StoreOnlyDocSubDB::Context context(owner, tlSyncer, getSerialNum, fileHeaderContext, writeService,
                                        sharedExecutor, _bucketDB, *_bucketDBHandler, metrics, configMutex, hwInfo);
     _subDBs.push_back
-        (new SearchableDocSubDB
-            (SearchableDocSubDB::Config(FastAccessDocSubDB::Config
-                (StoreOnlyDocSubDB::Config(docTypeName,
-                        "0.ready",
-                        baseDir,
-                        cfg.getReadyGrowth(),
-                        cfg.getFixedAttributeTotalSkew(),
-                        _readySubDbId,
-                        SubDbType::READY),
-                        true,
-                        true,
-                        false),
-                        cfg.getNumSearchThreads()),
-                SearchableDocSubDB::Context(FastAccessDocSubDB::Context
-                        (context,
-                         metrics.ready.attributes,
-                         metricsWireService),
-                         queryLimiter,
-                         clock,
-                         warmupExecutor)));
+        (new SearchableDocSubDB(
+                SearchableDocSubDB::Config(
+                    FastAccessDocSubDB::Config(
+                            StoreOnlyDocSubDB::Config(docTypeName, "0.ready", baseDir,
+                                    cfg.getReadyGrowth(), cfg.getFixedAttributeTotalSkew(),
+                                    _readySubDbId, SubDbType::READY),
+                            true, true, false),
+                    cfg.getNumSearchThreads()),
+                SearchableDocSubDB::Context(
+                        FastAccessDocSubDB::Context(context, metrics.ready.attributes, metricsWireService),
+                        queryLimiter, clock, warmupExecutor)));
+
     _subDBs.push_back
-        (new StoreOnlyDocSubDB(StoreOnlyDocSubDB::Config(docTypeName,
-                                                     "1.removed",
-                                                     baseDir,
-                                                     cfg.getRemovedGrowth(),
-                                                     cfg.getFixedAttributeTotalSkew(),
-                                                     _remSubDbId,
-                                                     SubDbType::REMOVED),
-                             context));
+        (new StoreOnlyDocSubDB(
+                StoreOnlyDocSubDB::Config(docTypeName, "1.removed", baseDir, cfg.getRemovedGrowth(),
+                        cfg.getFixedAttributeTotalSkew(), _remSubDbId, SubDbType::REMOVED),
+                context));
+
     _subDBs.push_back
-        (new FastAccessDocSubDB(FastAccessDocSubDB::Config
-                (StoreOnlyDocSubDB::Config(docTypeName,
-                        "2.notready",
-                        baseDir,
-                        cfg.getNotReadyGrowth(),
-                        cfg.getFixedAttributeTotalSkew(),
-                        _notReadySubDbId,
-                        SubDbType::NOTREADY),
-                        true,
-                        true,
-                        true),
-                FastAccessDocSubDB::Context(context,
-                        metrics.notReady.attributes,
-                        metricsWireService)));
+        (new FastAccessDocSubDB(
+                FastAccessDocSubDB::Config(
+                        StoreOnlyDocSubDB::Config(docTypeName, "2.notready", baseDir,
+                                cfg.getNotReadyGrowth(), cfg.getFixedAttributeTotalSkew(),
+                                _notReadySubDbId, SubDbType::NOTREADY),
+                        true, true, true),
+                FastAccessDocSubDB::Context(context, metrics.notReady.attributes, metricsWireService)));
 }
 
 
 DocumentSubDBCollection::~DocumentSubDBCollection()
 {
-    for (auto subDb : _subDBs) {
-        delete subDb;
+    size_t numThreads = std::min(_subDBs.size(), static_cast<size_t>(_hwInfo.cpu().cores()));
+    vespalib::ThreadStackExecutor closePool(numThreads, 0x20000);
+    while (!_subDBs.empty()) {
+        closePool.execute(makeLambdaTask([subDB=_subDBs.back()]() { delete subDB; }));
+        _subDBs.pop_back();
     }
+    closePool.sync();
+
     _bucketDB.reset();
+
+    RetrieversSP retrievers = _retrievers.get();
+    _retrievers.clear();
+    if (retrievers) {
+        while (!retrievers->empty()) {
+            auto retriever = std::move(retrievers->back());
+            retrievers->pop_back();
+            closePool.execute(makeLambdaTask([r = std::move(retriever)]() mutable { r.reset(); }));
+        }
+    }
+    closePool.sync();
 }
 
 void
diff --git a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h
index 68f790307ce..4f0e00c6ae5 100644
--- a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h
+++ b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h
@@ -3,6 +3,7 @@
 
 #include <vespa/searchcore/proton/reprocessing/reprocessingrunner.h>
 #include <vespa/searchcore/proton/bucketdb/bucketdbhandler.h>
+#include <vespa/searchcore/proton/common/hw_info.h>
 #include <vespa/searchcommon/common/growstrategy.h>
 #include <vespa/searchlib/common/serialnum.h>
 #include <vespa/vespalib/util/varholder.h>
@@ -90,6 +91,7 @@ private:
     ReprocessingRunner _reprocessingRunner;
     std::shared_ptr<BucketDBOwner> _bucketDB;
     std::unique_ptr<bucketdb::BucketDBHandler> _bucketDBHandler;
+    HwInfo _hwInfo;
 
 public:
     DocumentSubDBCollection(