diff options
6 files changed, 24 insertions, 11 deletions
diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp index e289b71a447..5c96173c678 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp @@ -1064,10 +1064,8 @@ DocumentMetaStore::getEstimatedShrinkLidSpaceGain() const BucketId DocumentMetaStore::getBucketOf(const vespalib::GenerationHandler::Guard &, uint32_t lid) const { - if (__builtin_expect(lid < getCommittedDocIdLimit(), true)) { - if (__builtin_expect(validLidFast(lid), true)) { - return getRawMetaData(lid).getBucketId(); - } + if (__builtin_expect(validLidFastSafe(lid, getCommittedDocIdLimit()), true)) { + return getRawMetaData(lid).getBucketId(); } return BucketId(); } diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.h b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.h index b5a95b8cd34..51f91241d3e 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.h +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.h @@ -182,6 +182,7 @@ public: void move(DocId fromLid, DocId toLid, uint64_t prepare_serial_num) override; bool validButMaybeUnusedLid(DocId lid) const { return _lidAlloc.validButMaybeUnusedLid(lid); } bool validLidFast(DocId lid) const { return _lidAlloc.validLid(lid); } + bool validLidFastSafe(DocId lid, uint32_t limit) const { return _lidAlloc.validLidSafe(lid, limit); } bool validLid(DocId lid) const override { return validLidFast(lid); } void removeBatch(const std::vector<DocId> &lidsToRemove, const DocId docIdLimit) override; const RawDocumentMetaData & getRawMetaData(DocId lid) const override { return _metaDataStore.acquire_elem_ref(lid); } diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.h b/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.h index 08fa4a6489a..e6327163aae 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.h +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.h @@ -70,6 +70,9 @@ public: bool validLid(DocId lid) const { return (lid < _usedLids.size() && _usedLids.testBit(lid)); } + bool validLidSafe(DocId lid, uint32_t limit) const { + return (lid < limit && _usedLids.testBitSafe(lid)); + } DocId getLowestFreeLid() const { return _freeLids.getLowest(); } diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/lidstatevector.h b/searchcore/src/vespa/searchcore/proton/documentmetastore/lidstatevector.h index 74851635124..c0db4e99fa5 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/lidstatevector.h +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/lidstatevector.h @@ -45,6 +45,7 @@ public: void consider_clear_bits(const std::vector<uint32_t>& idxs); void clear_bits(const std::vector<uint32_t>& idxs); bool testBit(unsigned int idx) const { return _bv.testBit(idx); } + bool testBitSafe(unsigned int idx) const { return _bv.testBitSafe(idx); } unsigned int size() const { return _bv.size(); } unsigned int byteSize() const { return _bv.extraByteSize() + sizeof(LidStateVector); diff --git a/searchlib/src/vespa/searchlib/common/bitvector.cpp b/searchlib/src/vespa/searchlib/common/bitvector.cpp index d2e6d9027ea..7eef1e7f8f2 100644 --- a/searchlib/src/vespa/searchlib/common/bitvector.cpp +++ b/searchlib/src/vespa/searchlib/common/bitvector.cpp @@ -106,11 +106,13 @@ BitVector::clearIntervalNoInvalidation(Range range_in) Index endw = wordNum(last); if (endw > startw) { - _words[startw++] &= startBits(range.start()); - memset(_words+startw, 0, sizeof(*_words)*(endw-startw)); - _words[endw] &= endBits(last); + vespalib::atomic::store_ref_relaxed(_words[startw], _words[startw] & startBits(range.start())); + for (Index i = startw + 1; i < endw; ++i) { + vespalib::atomic::store_ref_relaxed(_words[i], 0); + } + vespalib::atomic::store_ref_relaxed(_words[endw], _words[endw] & endBits(last)); } else { - _words[startw] &= (startBits(range.start()) | endBits(last)); + vespalib::atomic::store_ref_relaxed(_words[startw], _words[startw] & (startBits(range.start()) | endBits(last))); } } diff --git a/searchlib/src/vespa/searchlib/common/bitvector.h b/searchlib/src/vespa/searchlib/common/bitvector.h index 02f51fe8758..779e9887daf 100644 --- a/searchlib/src/vespa/searchlib/common/bitvector.h +++ b/searchlib/src/vespa/searchlib/common/bitvector.h @@ -5,6 +5,7 @@ #include "bitword.h" #include <memory> #include <vespa/vespalib/util/alloc.h> +#include <vespa/vespalib/util/atomic.h> #include <vespa/vespalib/util/generationholder.h> #include <vespa/fastos/types.h> @@ -46,6 +47,11 @@ public: bool testBit(Index idx) const { return ((_words[wordNum(idx)] & mask(idx)) != 0); } + bool testBitSafe(Index idx) const { + auto my_words = vespalib::atomic::load_ref_acquire(_words); + auto my_word = vespalib::atomic::load_ref_acquire(my_words[wordNum(idx)]); + return (my_word & mask(idx)) != 0; + } bool hasTrueBits() const { return isValidCount() ? (countTrueBits() != 0) @@ -136,10 +142,10 @@ public: _sz = sz; } void setBit(Index idx) { - _words[wordNum(idx)] |= mask(idx); + vespalib::atomic::store_ref_relaxed(_words[wordNum(idx)], _words[wordNum(idx)] | mask(idx)); } void clearBit(Index idx) { - _words[wordNum(idx)] &= ~ mask(idx); + vespalib::atomic::store_ref_relaxed(_words[wordNum(idx)], _words[wordNum(idx)] & ~ mask(idx)); } void flipBit(Index idx) { _words[wordNum(idx)] ^= mask(idx); @@ -200,7 +206,9 @@ public: } void swap(BitVector & rhs) { - std::swap(_words, rhs._words); + auto my_words = _words; + vespalib::atomic::store_ref_release(_words, rhs._words); + vespalib::atomic::store_ref_release(rhs._words, my_words); std::swap(_startOffset, rhs._startOffset); std::swap(_sz, rhs._sz); Index tmp = rhs._numTrueBits; |