diff options
author | Henning Baldersheim <balder@yahoo-inc.com> | 2023-10-05 21:48:26 +0000 |
---|---|---|
committer | Henning Baldersheim <balder@yahoo-inc.com> | 2023-10-05 21:48:26 +0000 |
commit | 881ee33e47aa8f2f3188e6178c11883f5bec05c8 (patch) | |
tree | ba159a2eab59657b9a2a346d79f8df31569da98a /searchlib/src | |
parent | 4a8f5727a3b7d123f0e62279e109156f1ec53bf9 (diff) |
- Avoid holding a bucketizer guard. Just get it everytime you need it.
- Max hold time is often above 2-3 seconds. This makes it very likely that a sudden
buildup might add l ot of memory to onhold.
Diffstat (limited to 'searchlib/src')
-rw-r--r-- | searchlib/src/vespa/searchlib/docstore/compacter.cpp | 24 | ||||
-rw-r--r-- | searchlib/src/vespa/searchlib/docstore/compacter.h | 4 |
2 files changed, 3 insertions, 25 deletions
diff --git a/searchlib/src/vespa/searchlib/docstore/compacter.cpp b/searchlib/src/vespa/searchlib/docstore/compacter.cpp index 29c4325d068..803b916b67d 100644 --- a/searchlib/src/vespa/searchlib/docstore/compacter.cpp +++ b/searchlib/src/vespa/searchlib/docstore/compacter.cpp @@ -4,7 +4,6 @@ #include "logdatastore.h" #include <vespa/vespalib/util/size_literals.h> #include <vespa/vespalib/util/array.hpp> -#include <cinttypes> #include <vespa/log/log.h> LOG_SETUP(".searchlib.docstore.compacter"); @@ -31,14 +30,10 @@ BucketCompacter::BucketCompacter(size_t maxSignificantBucketBits, CompressionCon _destinationFileId(destination), _ds(ds), _bucketizer(bucketizer), - _writeCount(0), - _maxBucketGuardDuration(vespalib::duration::zero()), - _lastSample(vespalib::steady_clock::now()), _lock(), _backingMemory(Alloc::alloc(INITIAL_BACKING_BUFFER_SIZE), &_lock), _tmpStore(), _lidGuard(ds.getLidReadGuard()), - _bucketizerGuard(), _stat() { for (auto & partition : _tmpStore) { @@ -54,27 +49,15 @@ BucketCompacter::getDestinationId(const LockGuard & guard) const { void BucketCompacter::write(LockGuard guard, uint32_t chunkId, uint32_t lid, ConstBufferRef data) { - if (_writeCount++ == 0) { - _bucketizerGuard = _bucketizer.getGuard(); - _lastSample = vespalib::steady_clock::now(); - } guard.unlock(); - BucketId bucketId = (data.size() > 0) ? _bucketizer.getBucketOf(_bucketizerGuard, lid) : BucketId(); + BucketId bucketId = (data.size() > 0) ? _bucketizer.getBucketOf(_bucketizer.getGuard(), lid) : BucketId(); uint64_t sortableBucketId = bucketId.toKey(); _tmpStore[(sortableBucketId >> _unSignificantBucketBits) % _tmpStore.size()]->add(bucketId, chunkId, lid, data); - if ((_writeCount % 1000) == 0) { - _bucketizerGuard = _bucketizer.getGuard(); - vespalib::steady_time now = vespalib::steady_clock::now(); - _maxBucketGuardDuration = std::max(_maxBucketGuardDuration, now - _lastSample); - _lastSample = now; - } } void BucketCompacter::close() { - _bucketizerGuard = GenerationHandler::Guard(); - vespalib::duration lastBucketGuardDuration = vespalib::steady_clock::now() - _lastSample; size_t lidCount1(0); size_t bucketCount(0); size_t chunkCount(0); @@ -84,9 +67,8 @@ BucketCompacter::close() bucketCount += store->getBucketCount(); chunkCount += store->getChunkCount(); } - LOG(info, "Have read %ld lids and placed them in %ld buckets. Temporary compressed in %ld chunks." - " Max bucket guard held for %" PRId64 " us, and last before close for %" PRId64 " us", - lidCount1, bucketCount, chunkCount, vespalib::count_us(_maxBucketGuardDuration), vespalib::count_us(lastBucketGuardDuration)); + LOG(info, "Have read %ld lids and placed them in %ld buckets. Temporary compressed in %ld chunks.", + lidCount1, bucketCount, chunkCount); for (auto & store_ref : _tmpStore) { auto store = std::move(store_ref); diff --git a/searchlib/src/vespa/searchlib/docstore/compacter.h b/searchlib/src/vespa/searchlib/docstore/compacter.h index 3760729b40f..354ca24ede9 100644 --- a/searchlib/src/vespa/searchlib/docstore/compacter.h +++ b/searchlib/src/vespa/searchlib/docstore/compacter.h @@ -51,14 +51,10 @@ private: FileId _destinationFileId; LogDataStore & _ds; const IBucketizer & _bucketizer; - uint64_t _writeCount; - vespalib::duration _maxBucketGuardDuration; - vespalib::steady_time _lastSample; std::mutex _lock; vespalib::MemoryDataStore _backingMemory; Partitions _tmpStore; GenerationHandler::Guard _lidGuard; - GenerationHandler::Guard _bucketizerGuard; vespalib::hash_map<uint64_t, uint32_t> _stat; }; |