diff options
author | Håvard Pettersen <havardpe@yahooinc.com> | 2022-05-11 13:21:16 +0000 |
---|---|---|
committer | Håvard Pettersen <havardpe@yahooinc.com> | 2022-05-11 14:14:53 +0000 |
commit | 530f4786b3ba73f2bb4a7e64132b7234e041d597 (patch) | |
tree | aaef5a32162c36b5f582a5f6c13de7884861839b /searchlib | |
parent | dfd60ee97e0531f1df053eaf6f506560eccbe89d (diff) |
make new bitvector when growing
Diffstat (limited to 'searchlib')
18 files changed, 187 insertions, 151 deletions
diff --git a/searchlib/src/tests/common/bitvector/bitvector_test.cpp b/searchlib/src/tests/common/bitvector/bitvector_test.cpp index a9850756ac6..79af28d20be 100644 --- a/searchlib/src/tests/common/bitvector/bitvector_test.cpp +++ b/searchlib/src/tests/common/bitvector/bitvector_test.cpp @@ -604,55 +604,55 @@ TEST("requireThatGrowWorks") { vespalib::GenerationHolder g; GrowableBitVector v(200, 200, g); - EXPECT_EQUAL(0u, v.countTrueBits()); + EXPECT_EQUAL(0u, v.writer().countTrueBits()); - v.setBitAndMaintainCount(7); - v.setBitAndMaintainCount(39); - v.setBitAndMaintainCount(71); - v.setBitAndMaintainCount(103); - EXPECT_EQUAL(4u, v.countTrueBits()); - - EXPECT_EQUAL(200u, v.size()); - EXPECT_EQUAL(1023u, v.capacity()); - EXPECT_TRUE(assertBV("[7,39,71,103]", v)); - EXPECT_EQUAL(4u, v.countTrueBits()); + v.writer().setBitAndMaintainCount(7); + v.writer().setBitAndMaintainCount(39); + v.writer().setBitAndMaintainCount(71); + v.writer().setBitAndMaintainCount(103); + EXPECT_EQUAL(4u, v.writer().countTrueBits()); + + EXPECT_EQUAL(200u, v.reader().size()); + EXPECT_EQUAL(1023u, v.writer().capacity()); + EXPECT_TRUE(assertBV("[7,39,71,103]", v.reader())); + EXPECT_EQUAL(4u, v.writer().countTrueBits()); EXPECT_TRUE(v.reserve(1024)); - EXPECT_EQUAL(200u, v.size()); - EXPECT_EQUAL(2047u, v.capacity()); - EXPECT_TRUE(assertBV("[7,39,71,103]", v)); - EXPECT_EQUAL(4u, v.countTrueBits()); + EXPECT_EQUAL(200u, v.reader().size()); + EXPECT_EQUAL(2047u, v.writer().capacity()); + EXPECT_TRUE(assertBV("[7,39,71,103]", v.reader())); + EXPECT_EQUAL(4u, v.writer().countTrueBits()); EXPECT_FALSE(v.extend(202)); - EXPECT_EQUAL(202u, v.size()); - EXPECT_EQUAL(2047u, v.capacity()); - EXPECT_TRUE(assertBV("[7,39,71,103]", v)); - EXPECT_EQUAL(4u, v.countTrueBits()); + EXPECT_EQUAL(202u, v.reader().size()); + EXPECT_EQUAL(2047u, v.writer().capacity()); + EXPECT_TRUE(assertBV("[7,39,71,103]", v.reader())); + EXPECT_EQUAL(4u, v.writer().countTrueBits()); EXPECT_FALSE(v.shrink(200)); - EXPECT_EQUAL(200u, v.size()); - EXPECT_EQUAL(2047u, v.capacity()); - EXPECT_TRUE(assertBV("[7,39,71,103]", v)); - EXPECT_EQUAL(4u, v.countTrueBits()); + EXPECT_EQUAL(200u, v.reader().size()); + EXPECT_EQUAL(2047u, v.writer().capacity()); + EXPECT_TRUE(assertBV("[7,39,71,103]", v.reader())); + EXPECT_EQUAL(4u, v.writer().countTrueBits()); EXPECT_FALSE(v.reserve(2047)); - EXPECT_EQUAL(200u, v.size()); - EXPECT_EQUAL(2047u, v.capacity()); - EXPECT_TRUE(assertBV("[7,39,71,103]", v)); - EXPECT_EQUAL(4u, v.countTrueBits()); + EXPECT_EQUAL(200u, v.reader().size()); + EXPECT_EQUAL(2047u, v.writer().capacity()); + EXPECT_TRUE(assertBV("[7,39,71,103]", v.reader())); + EXPECT_EQUAL(4u, v.writer().countTrueBits()); EXPECT_FALSE(v.shrink(202)); - EXPECT_EQUAL(202u, v.size()); - EXPECT_EQUAL(2047u, v.capacity()); - EXPECT_TRUE(assertBV("[7,39,71,103]", v)); - EXPECT_EQUAL(4u, v.countTrueBits()); + EXPECT_EQUAL(202u, v.reader().size()); + EXPECT_EQUAL(2047u, v.writer().capacity()); + EXPECT_TRUE(assertBV("[7,39,71,103]", v.reader())); + EXPECT_EQUAL(4u, v.writer().countTrueBits()); EXPECT_FALSE(v.shrink(100)); - EXPECT_EQUAL(100u, v.size()); - EXPECT_EQUAL(2047u, v.capacity()); - EXPECT_TRUE(assertBV("[7,39,71]", v)); - EXPECT_EQUAL(3u, v.countTrueBits()); + EXPECT_EQUAL(100u, v.reader().size()); + EXPECT_EQUAL(2047u, v.writer().capacity()); + EXPECT_TRUE(assertBV("[7,39,71]", v.reader())); + EXPECT_EQUAL(3u, v.writer().countTrueBits()); - v.invalidateCachedCount(); + v.writer().invalidateCachedCount(); EXPECT_TRUE(v.reserve(3100)); - EXPECT_EQUAL(100u, v.size()); - EXPECT_EQUAL(4095u, v.capacity()); - EXPECT_EQUAL(3u, v.countTrueBits()); + EXPECT_EQUAL(100u, v.reader().size()); + EXPECT_EQUAL(4095u, v.writer().capacity()); + EXPECT_EQUAL(3u, v.writer().countTrueBits()); g.transferHoldLists(1); g.trimHoldLists(2); @@ -666,7 +666,7 @@ TEST("require that growable bit vectors keeps memory allocator") vespalib::GenerationHolder g; GrowableBitVector v(200, 200, g, &init_alloc); EXPECT_EQUAL(AllocStats(1, 0), stats); - v.resize(1); + v.writer().resize(1); // DO NOT TRY THIS AT HOME EXPECT_EQUAL(AllocStats(2, 1), stats); v.reserve(2000); EXPECT_EQUAL(AllocStats(3, 1), stats); @@ -674,7 +674,7 @@ TEST("require that growable bit vectors keeps memory allocator") EXPECT_EQUAL(AllocStats(4, 1), stats); v.shrink(200); EXPECT_EQUAL(AllocStats(4, 1), stats); - v.resize(1); + v.writer().resize(1); // DO NOT TRY THIS AT HOME EXPECT_EQUAL(AllocStats(5, 2), stats); g.transferHoldLists(1); g.trimHoldLists(2); diff --git a/searchlib/src/vespa/searchlib/attribute/flagattribute.cpp b/searchlib/src/vespa/searchlib/attribute/flagattribute.cpp index d6aa136e15f..6e19d35a966 100644 --- a/searchlib/src/vespa/searchlib/attribute/flagattribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/flagattribute.cpp @@ -62,7 +62,7 @@ void FlagAttributeT<B>::clearOldValues(DocId doc) { const typename B::WType * values(nullptr); for (uint32_t i(0), m(this->get(doc, values)); i < m; i++) { - BitVector * bv = _bitVectors[getOffset(multivalue::get_value(values[i]))]; + BitVector * bv = _bitVectors[getOffset(multivalue::get_value(values[i]))].load_relaxed(); if (bv != nullptr) { bv->clearBitAndMaintainCount(doc); } @@ -103,7 +103,7 @@ bool FlagAttributeT<B>::onLoad(vespalib::Executor * executor) { for (size_t i(0), m(_bitVectors.size()); i < m; i++) { _bitVectorStore[i].reset(); - _bitVectors[i] = nullptr; + _bitVectors[i].store_relaxed(nullptr); } _bitVectorSize = 0; return B::onLoad(executor); @@ -119,12 +119,12 @@ void FlagAttributeT<B>::setNewValues(DocId doc, const std::vector<typename B::WT for (uint32_t i(0), m(values.size()); i < m; i++) { typename B::WType value = values[i]; uint32_t offset = getOffset(value); - BitVector * bv = _bitVectors[offset]; + BitVector * bv = _bitVectors[offset].load_relaxed(); if (bv == nullptr) { assert(_bitVectorSize >= this->getNumDocs()); _bitVectorStore[offset] = std::make_shared<GrowableBitVector>(_bitVectorSize, _bitVectorSize, _bitVectorHolder); - _bitVectors[offset] = _bitVectorStore[offset].get(); - bv = _bitVectors[offset]; + _bitVectors[offset].store_release(&_bitVectorStore[offset]->writer()); + bv = _bitVectors[offset].load_relaxed(); ensureGuardBit(*bv); } bv->setBitAndMaintainCount(doc); @@ -136,12 +136,12 @@ void FlagAttributeT<B>::setNewBVValue(DocId doc, multivalue::ValueType_t<typename B::WType> value) { uint32_t offset = getOffset(value); - BitVector * bv = _bitVectors[offset]; + BitVector * bv = _bitVectors[offset].load_relaxed(); if (bv == nullptr) { assert(_bitVectorSize >= this->getNumDocs()); _bitVectorStore[offset] = std::make_shared<GrowableBitVector>(_bitVectorSize, _bitVectorSize, _bitVectorHolder); - _bitVectors[offset] = _bitVectorStore[offset].get(); - bv = _bitVectors[offset]; + _bitVectors[offset].store_release(&_bitVectorStore[offset]->writer()); + bv = _bitVectors[offset].load_relaxed(); ensureGuardBit(*bv); } bv->setBitAndMaintainCount(doc); @@ -186,7 +186,8 @@ template <typename B> void FlagAttributeT<B>::ensureGuardBit() { - for (BitVector * bv : _bitVectors) { + for (const auto &wrapper: _bitVectors) { + BitVector *bv = wrapper.load_relaxed(); if (bv != nullptr) { ensureGuardBit(*bv); } @@ -197,7 +198,8 @@ template <typename B> void FlagAttributeT<B>::clearGuardBit(DocId doc) { - for (BitVector * bv : _bitVectors) { + for (const auto &wrapper: _bitVectors) { + BitVector *bv = wrapper.load_relaxed(); if (bv != nullptr) { bv->clearBit(doc); // clear guard bit and start using this doc id } @@ -211,9 +213,12 @@ FlagAttributeT<B>::resizeBitVectors(uint32_t neededSize) const GrowStrategy & gs = this->getConfig().getGrowStrategy(); uint32_t newSize = neededSize + (neededSize * gs.getDocsGrowFactor()) + gs.getDocsGrowDelta(); for (size_t i(0), m(_bitVectors.size()); i < m; i++) { - BitVector *bv = _bitVectors[i]; + BitVector *bv = _bitVectors[i].load_relaxed(); if (bv != nullptr) { - _bitVectorStore[i]->extend(newSize); + if (_bitVectorStore[i]->extend(newSize)) { + _bitVectors[i].store_release(&_bitVectorStore[i]->writer()); + bv = _bitVectors[i].load_relaxed(); + } ensureGuardBit(*bv); } } diff --git a/searchlib/src/vespa/searchlib/attribute/flagattribute.h b/searchlib/src/vespa/searchlib/attribute/flagattribute.h index c4f4980e372..796c1493cc9 100644 --- a/searchlib/src/vespa/searchlib/attribute/flagattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/flagattribute.h @@ -4,6 +4,7 @@ #include "multinumericattribute.h" #include "multi_numeric_search_context.h" #include <vespa/searchlib/common/growablebitvector.h> +#include <vespa/vespalib/datastore/atomic_value_wrapper.h> namespace search { @@ -35,9 +36,10 @@ private: void removeOldGenerations(vespalib::GenerationHandler::generation_t firstUsed) override; uint32_t getOffset(int8_t value) const { return value + 128; } + using AtomicBitVectorPtr = vespalib::datastore::AtomicValueWrapper<BitVector *>; vespalib::GenerationHolder _bitVectorHolder; std::vector<std::shared_ptr<GrowableBitVector> > _bitVectorStore; - std::vector<BitVector *> _bitVectors; + std::vector<AtomicBitVectorPtr> _bitVectors; uint32_t _bitVectorSize; }; diff --git a/searchlib/src/vespa/searchlib/attribute/multi_numeric_flag_search_context.cpp b/searchlib/src/vespa/searchlib/attribute/multi_numeric_flag_search_context.cpp index 6a0be483a55..045d80bc6c4 100644 --- a/searchlib/src/vespa/searchlib/attribute/multi_numeric_flag_search_context.cpp +++ b/searchlib/src/vespa/searchlib/attribute/multi_numeric_flag_search_context.cpp @@ -13,7 +13,7 @@ namespace search::attribute { using queryeval::SearchIterator; template <typename T, typename M> -MultiNumericFlagSearchContext<T, M>::MultiNumericFlagSearchContext(std::unique_ptr<QueryTermSimple> qTerm, const AttributeVector& toBeSearched, MultiValueMappingReadView<M> mv_mapping_read_view, vespalib::ConstArrayRef<BitVector *> bit_vectors) +MultiNumericFlagSearchContext<T, M>::MultiNumericFlagSearchContext(std::unique_ptr<QueryTermSimple> qTerm, const AttributeVector& toBeSearched, MultiValueMappingReadView<M> mv_mapping_read_view, AtomicBitVectorsRef bit_vectors) : MultiNumericSearchContext<T, M>(std::move(qTerm), toBeSearched, mv_mapping_read_view), _bit_vectors(bit_vectors), _zeroHits(false) diff --git a/searchlib/src/vespa/searchlib/attribute/multi_numeric_flag_search_context.h b/searchlib/src/vespa/searchlib/attribute/multi_numeric_flag_search_context.h index 00f7077b2d0..dd88ee4202c 100644 --- a/searchlib/src/vespa/searchlib/attribute/multi_numeric_flag_search_context.h +++ b/searchlib/src/vespa/searchlib/attribute/multi_numeric_flag_search_context.h @@ -3,6 +3,7 @@ #pragma once #include "multi_numeric_search_context.h" +#include <vespa/vespalib/datastore/atomic_value_wrapper.h> namespace search { class BitVector; @@ -20,16 +21,19 @@ template <typename T, typename M> class MultiNumericFlagSearchContext : public MultiNumericSearchContext<T, M> { public: - MultiNumericFlagSearchContext(std::unique_ptr<QueryTermSimple> qTerm, const AttributeVector& toBeSearched, MultiValueMappingReadView<M> mv_mapping_read_view, vespalib::ConstArrayRef<BitVector *> bit_vectors); + using AtomicBitVectorsRef = vespalib::ConstArrayRef<vespalib::datastore::AtomicValueWrapper<BitVector *>>; + + MultiNumericFlagSearchContext(std::unique_ptr<QueryTermSimple> qTerm, const AttributeVector& toBeSearched, MultiValueMappingReadView<M> mv_mapping_read_view, + AtomicBitVectorsRef bit_vectors); std::unique_ptr<queryeval::SearchIterator> createIterator(fef::TermFieldMatchData * matchData, bool strict) override; private: - vespalib::ConstArrayRef<BitVector *> _bit_vectors; + AtomicBitVectorsRef _bit_vectors; bool _zeroHits; const BitVector* get_bit_vector(T value) const { static_assert(std::is_same_v<T, int8_t>, "Flag attribute search context is only supported for int8_t data type"); - return _bit_vectors[value + 128]; + return _bit_vectors[value + 128].load_acquire(); } template <class SC> friend class ::search::FlagAttributeIteratorT; diff --git a/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.cpp b/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.cpp index 8accb1123af..c17627a5026 100644 --- a/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.cpp +++ b/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.cpp @@ -33,7 +33,7 @@ PostingListSearchContext(const IEnumStoreDictionary& dictionary, _FSTC(0.0), _PLSTC(0.0), _minBvDocFreq(minBvDocFreq), - _gbv(nullptr), + _bv(nullptr), _baseSearchCtx(baseSearchCtx) { } diff --git a/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.h b/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.h index 9ee56a27a85..5cc7c13c6bd 100644 --- a/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.h +++ b/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.h @@ -45,7 +45,7 @@ protected: float _FSTC; // Filtering Search Time Constant float _PLSTC; // Posting List Search Time Constant uint32_t _minBvDocFreq; - const GrowableBitVector *_gbv; // bitvector if _useBitVector has been set + const BitVector *_bv; // bitvector if _useBitVector has been set const ISearchContext &_baseSearchCtx; diff --git a/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.hpp b/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.hpp index cd783ec2fe8..14e0cf041f2 100644 --- a/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.hpp +++ b/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.hpp @@ -46,7 +46,7 @@ PostingListSearchContextT<DataT>::lookupSingle() const BitVectorEntry *bve = _postingList.getBitVectorEntry(_pidx); const GrowableBitVector *bv = bve->_bv.get(); if (_useBitVector) { - _gbv = bv; + _bv = &bv->reader(); } else { _pidx = bve->_tree; if (_pidx.valid()) { @@ -56,7 +56,7 @@ PostingListSearchContextT<DataT>::lookupSingle() _pidx = vespalib::datastore::EntryRef(); } } else { - _gbv = bv; + _bv = &bv->reader(); } } } else { @@ -179,8 +179,8 @@ createPostingIterator(fef::TermFieldMatchData *matchData, bool strict) return search::BitVectorIterator::create(bv, bv->size(), *matchData, strict); } if (_uniqueValues == 1) { - if (_gbv != nullptr) { - return BitVectorIterator::create(_gbv, std::min(_gbv->size(), _docIdLimit), *matchData, strict); + if (_bv != nullptr) { + return BitVectorIterator::create(_bv, std::min(_bv->size(), _docIdLimit), *matchData, strict); } if (!_pidx.valid()) { return std::make_unique<EmptySearch>(); @@ -217,9 +217,9 @@ template <typename DataT> unsigned int PostingListSearchContextT<DataT>::singleHits() const { - if (_gbv) { + if (_bv) { // Some inaccuracy is expected, data changes underfeet - return _gbv->countTrueBits(); + return _bv->countTrueBits(); } if (!_pidx.valid()) { return 0u; diff --git a/searchlib/src/vespa/searchlib/attribute/postingstore.cpp b/searchlib/src/vespa/searchlib/attribute/postingstore.cpp index df016b050af..a942e70085a 100644 --- a/searchlib/src/vespa/searchlib/attribute/postingstore.cpp +++ b/searchlib/src/vespa/searchlib/attribute/postingstore.cpp @@ -101,7 +101,7 @@ PostingStore<DataT>::removeSparseBitVectors() assert(isBitVector(typeId)); BitVectorEntry *bve = getWBitVectorEntry(iRef); GrowableBitVector &bv = *bve->_bv.get(); - uint32_t docFreq = bv.countTrueBits(); + uint32_t docFreq = bv.writer().countTrueBits(); if (bve->_tree.valid()) { RefType iRef2(bve->_tree); assert(isBTree(iRef2)); @@ -111,19 +111,19 @@ PostingStore<DataT>::removeSparseBitVectors() } if (docFreq < _minBvDocFreq) needscan = true; - unsigned int oldExtraSize = bv.extraByteSize(); - if (bv.size() > _bvSize) { + unsigned int oldExtraSize = bv.writer().extraByteSize(); + if (bv.writer().size() > _bvSize) { bv.shrink(_bvSize); res = true; } - if (bv.capacity() < _bvCapacity) { + if (bv.writer().capacity() < _bvCapacity) { bv.reserve(_bvCapacity); res = true; } - if (bv.size() < _bvSize) { + if (bv.writer().size() < _bvSize) { bv.extend(_bvSize); } - unsigned int newExtraSize = bv.extraByteSize(); + unsigned int newExtraSize = bv.writer().extraByteSize(); if (oldExtraSize != newExtraSize) { _bvExtraBytes = _bvExtraBytes + newExtraSize - oldExtraSize; } @@ -149,7 +149,7 @@ PostingStore<DataT>::consider_remove_sparse_bitvector(std::vector<EntryRef>& ref assert(isBitVector(typeId)); assert(_bvs.find(iRef.ref()) != _bvs.end()); BitVectorEntry *bve = getWBitVectorEntry(iRef); - BitVector &bv = *bve->_bv.get(); + BitVector &bv = bve->_bv->writer(); uint32_t docFreq = bv.countTrueBits(); if (bve->_tree.valid()) { RefType iRef2(bve->_tree); @@ -226,12 +226,12 @@ PostingStore<DataT>::dropBitVector(EntryRef &ref) assert(isBitVector(typeId)); (void) typeId; BitVectorEntry *bve = getWBitVectorEntry(iRef); - AllocatedBitVector *bv = bve->_bv.get(); + GrowableBitVector *bv = bve->_bv.get(); assert(bv); - uint32_t docFreq = bv->countTrueBits(); + uint32_t docFreq = bv->writer().countTrueBits(); EntryRef ref2(bve->_tree); if (!ref2.valid()) { - makeDegradedTree(ref2, *bv); + makeDegradedTree(ref2, bv->writer()); } assert(ref2.valid()); assert(isBTree(ref2)); @@ -242,7 +242,7 @@ PostingStore<DataT>::dropBitVector(EntryRef &ref) _bvs.erase(ref.ref()); _store.holdElem(iRef, 1); _status.decBitVectors(); - _bvExtraBytes -= bv->extraByteSize(); + _bvExtraBytes -= bv->writer().extraByteSize(); ref = ref2; } @@ -258,7 +258,7 @@ PostingStore<DataT>::makeBitVector(EntryRef &ref) (void) typeId; vespalib::GenerationHolder &genHolder = _store.getGenerationHolder(); auto bvsp = std::make_shared<GrowableBitVector>(_bvSize, _bvCapacity, genHolder); - AllocatedBitVector &bv = *bvsp.get(); + BitVector &bv = bvsp->writer(); uint32_t docIdLimit = _bvSize; (void) docIdLimit; Iterator it = begin(ref); @@ -283,7 +283,7 @@ PostingStore<DataT>::makeBitVector(EntryRef &ref) bve->_bv = bvsp; _bvs.insert(bPair.ref.ref()); _status.incBitVectors(); - _bvExtraBytes += bv.extraByteSize(); + _bvExtraBytes += bvsp->writer().extraByteSize(); // barrier ? ref = bPair.ref; } @@ -299,7 +299,7 @@ PostingStore<DataT>::applyNewBitVector(EntryRef &ref, RefType iRef(ref); vespalib::GenerationHolder &genHolder = _store.getGenerationHolder(); auto bvsp = std::make_shared<GrowableBitVector>(_bvSize, _bvCapacity, genHolder); - AllocatedBitVector &bv = *bvsp.get(); + BitVector &bv = bvsp->writer(); uint32_t docIdLimit = _bvSize; (void) docIdLimit; uint32_t expDocFreq = ae - aOrg; @@ -319,7 +319,7 @@ PostingStore<DataT>::applyNewBitVector(EntryRef &ref, bve->_bv = bvsp; _bvs.insert(bPair.ref.ref()); _status.incBitVectors(); - _bvExtraBytes += bv.extraByteSize(); + _bvExtraBytes += bvsp->writer().extraByteSize(); // barrier ? ref = bPair.ref; } @@ -390,7 +390,7 @@ PostingStore<DataT>::apply(EntryRef &ref, BTreeType *tree = getWTreeEntry(iRef2); applyTree(tree, a, ae, r, re, CompareT()); } - BitVector *bv = bve->_bv.get(); + BitVector *bv = &bve->_bv->writer(); assert(bv); apply(*bv, a, ae, r, re); uint32_t docFreq = bv->countTrueBits(); @@ -433,7 +433,7 @@ PostingStore<DataT>::internalSize(uint32_t typeId, const RefType & iRef) const const BTreeType *tree = getTreeEntry(iRef2); return tree->size(_allocator); } else { - const BitVector *bv = bve->_bv.get(); + const BitVector *bv = &bve->_bv->writer(); return bv->countTrueBits(); } } else { @@ -456,7 +456,7 @@ PostingStore<DataT>::internalFrozenSize(uint32_t typeId, const RefType & iRef) c return tree->frozenSize(_allocator); } else { // Some inaccuracy is expected, data changes underfeet - return bve->_bv->countTrueBits(); + return bve->_bv->reader().countTrueBits(); } } else { const BTreeType *tree = getTreeEntry(iRef); @@ -608,7 +608,7 @@ PostingStore<DataT>::clear(const EntryRef ref) } _bvs.erase(ref.ref()); _status.decBitVectors(); - _bvExtraBytes -= bve->_bv->extraByteSize(); + _bvExtraBytes -= bve->_bv->writer().extraByteSize(); _store.holdElem(ref, 1); } else { BTreeType *tree = getWTreeEntry(iRef); diff --git a/searchlib/src/vespa/searchlib/attribute/postingstore.hpp b/searchlib/src/vespa/searchlib/attribute/postingstore.hpp index 1993714e49d..7dab24e996f 100644 --- a/searchlib/src/vespa/searchlib/attribute/postingstore.hpp +++ b/searchlib/src/vespa/searchlib/attribute/postingstore.hpp @@ -25,7 +25,7 @@ PostingStore<DataT>::foreach_frozen_key(EntryRef ref, FunctionType func) const { const BTreeType *tree = getTreeEntry(iRef2); _allocator.getNodeStore().foreach_key(tree->getFrozenRoot(), func); } else { - const BitVector *bv = bve->_bv.get(); + const BitVector *bv = &bve->_bv->reader(); uint32_t docIdLimit = bv->size(); uint32_t docId = bv->getFirstTrueBit(1); while (docId < docIdLimit) { @@ -67,7 +67,7 @@ PostingStore<DataT>::foreach_frozen(EntryRef ref, FunctionType func) const { const BTreeType *tree = getTreeEntry(iRef2); _allocator.getNodeStore().foreach(tree->getFrozenRoot(), func); } else { - const BitVector *bv = bve->_bv.get(); + const BitVector *bv = &bve->_bv->reader(); uint32_t docIdLimit = bv->size(); uint32_t docId = bv->getFirstTrueBit(1); while (docId < docIdLimit) { diff --git a/searchlib/src/vespa/searchlib/attribute/singleboolattribute.cpp b/searchlib/src/vespa/searchlib/attribute/singleboolattribute.cpp index 8db99c115a0..46fd47014d3 100644 --- a/searchlib/src/vespa/searchlib/attribute/singleboolattribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/singleboolattribute.cpp @@ -32,7 +32,7 @@ SingleBoolAttribute::~SingleBoolAttribute() void SingleBoolAttribute::ensureRoom(DocId docIdLimit) { - if (_bv.capacity() < docIdLimit) { + if (_bv.writer().capacity() < docIdLimit) { const GrowStrategy & gs = this->getConfig().getGrowStrategy(); uint32_t newSize = docIdLimit + (docIdLimit * gs.getDocsGrowFactor()) + gs.getDocsGrowDelta(); bool incGen = _bv.reserve(newSize); @@ -72,7 +72,7 @@ SingleBoolAttribute::onCommit() { setBit(change._doc, val != 0); } else if (change._type == ChangeBase::CLEARDOC) { std::atomic_thread_fence(std::memory_order_release); - _bv.clearBitAndMaintainCount(change._doc); + _bv.writer().clearBitAndMaintainCount(change._doc); } } } @@ -91,11 +91,11 @@ SingleBoolAttribute::onAddDocs(DocId docIdLimit) { void SingleBoolAttribute::onUpdateStat() { vespalib::MemoryUsage usage; - usage.setAllocatedBytes(_bv.extraByteSize()); - usage.setUsedBytes(_bv.sizeBytes()); + usage.setAllocatedBytes(_bv.writer().extraByteSize()); + usage.setUsedBytes(_bv.writer().sizeBytes()); usage.mergeGenerationHeldBytes(getGenerationHolder().getHeldBytes()); usage.merge(this->getChangeVectorMemoryUsage()); - this->updateStatistics(_bv.size(), _bv.size(), usage.allocatedBytes(), usage.usedBytes(), + this->updateStatistics(_bv.writer().size(), _bv.writer().size(), usage.allocatedBytes(), usage.usedBytes(), usage.deadBytes(), usage.allocatedBytesOnHold()); } @@ -188,13 +188,13 @@ SingleBoolAttribute::onLoad(vespalib::Executor *) if (ok) { setCreateSerialNum(attrReader.getCreateSerialNum()); getGenerationHolder().clearHoldLists(); - _bv.clear(); + _bv.writer().clear(); uint32_t numDocs = attrReader.getNextData(); _bv.extend(numDocs); - ssize_t bytesRead = attrReader.getReader().read(_bv.getStart(), _bv.sizeBytes()); - _bv.invalidateCachedCount(); - _bv.countTrueBits(); - assert(bytesRead == _bv.sizeBytes()); + ssize_t bytesRead = attrReader.getReader().read(_bv.writer().getStart(), _bv.writer().sizeBytes()); + _bv.writer().invalidateCachedCount(); + _bv.writer().countTrueBits(); + assert(bytesRead == _bv.writer().sizeBytes()); setNumDocs(numDocs); setCommittedDocIdLimit(numDocs); } @@ -207,7 +207,7 @@ SingleBoolAttribute::onSave(IAttributeSaveTarget &saveTarget) { assert(!saveTarget.getEnumerated()); const size_t numDocs(getCommittedDocIdLimit()); - const size_t sz(sizeof(uint32_t) + _bv.sizeBytes()); + const size_t sz(sizeof(uint32_t) + _bv.writer().sizeBytes()); IAttributeSaveTarget::Buffer buf(saveTarget.datWriter().allocBuf(sz)); char *p = buf->getFree(); @@ -215,8 +215,8 @@ SingleBoolAttribute::onSave(IAttributeSaveTarget &saveTarget) uint32_t numDocs2 = numDocs; memcpy(p, &numDocs2, sizeof(uint32_t)); p += sizeof(uint32_t); - memcpy(p, _bv.getStart(), _bv.sizeBytes()); - p += _bv.sizeBytes(); + memcpy(p, _bv.writer().getStart(), _bv.writer().sizeBytes()); + p += _bv.writer().sizeBytes(); assert(p == e); (void) e; buf->moveFreeToData(sz); @@ -249,7 +249,7 @@ uint64_t SingleBoolAttribute::getEstimatedSaveByteSize() const { constexpr uint64_t headerSize = FileSettings::DIRECTIO_ALIGNMENT + sizeof(uint32_t); - return headerSize + _bv.sizeBytes(); + return headerSize + _bv.reader().sizeBytes(); } void diff --git a/searchlib/src/vespa/searchlib/attribute/singleboolattribute.h b/searchlib/src/vespa/searchlib/attribute/singleboolattribute.h index 77c8b7e318f..3465269de8a 100644 --- a/searchlib/src/vespa/searchlib/attribute/singleboolattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/singleboolattribute.h @@ -34,7 +34,7 @@ public: getSearch(std::unique_ptr<QueryTermSimple> term, const attribute::SearchContextParams & params) const override; uint32_t getValueCount(DocId doc) const override { - return (doc >= _bv.size()) ? 0 : 1; + return (doc >= _bv.reader().size()) ? 0 : 1; } largeint_t getInt(DocId doc) const override { return static_cast<largeint_t>(getFast(doc)); @@ -81,12 +81,12 @@ public: int8_t get(DocId doc) const override { return getFast(doc); } - const BitVector & getBitVector() const { return _bv; } + const BitVector & getBitVector() const { return _bv.reader(); } void setBit(DocId doc, bool value) { if (value) { - _bv.setBitAndMaintainCount(doc); + _bv.writer().setBitAndMaintainCount(doc); } else { - _bv.clearBitAndMaintainCount(doc); + _bv.writer().clearBitAndMaintainCount(doc); } } protected: @@ -99,7 +99,7 @@ private: return 0; } int8_t getFast(DocId doc) const { - return _bv.testBit(doc) ? 1 : 0; + return _bv.reader().testBit(doc) ? 1 : 0; } vespalib::alloc::Alloc _init_alloc; GrowableBitVector _bv; diff --git a/searchlib/src/vespa/searchlib/common/bitvector.cpp b/searchlib/src/vespa/searchlib/common/bitvector.cpp index 36ebc6597fa..2d7fda32037 100644 --- a/searchlib/src/vespa/searchlib/common/bitvector.cpp +++ b/searchlib/src/vespa/searchlib/common/bitvector.cpp @@ -41,9 +41,6 @@ constexpr size_t MMAP_LIMIT = 256_Mi; namespace search { using vespalib::nbostream; -using vespalib::GenerationHeldBase; -using vespalib::GenerationHeldAlloc; -using vespalib::GenerationHolder; Alloc BitVector::allocatePaddedAndAligned(Index start, Index end, Index capacity, const Alloc* init_alloc) diff --git a/searchlib/src/vespa/searchlib/common/bitvector.h b/searchlib/src/vespa/searchlib/common/bitvector.h index 7fea95661d7..dfc3d0b280a 100644 --- a/searchlib/src/vespa/searchlib/common/bitvector.h +++ b/searchlib/src/vespa/searchlib/common/bitvector.h @@ -6,7 +6,6 @@ #include <memory> #include <vespa/vespalib/util/alloc.h> #include <vespa/vespalib/util/atomic.h> -#include <vespa/vespalib/util/generationholder.h> #include <vespa/fastos/types.h> namespace vespalib { @@ -24,8 +23,6 @@ class BitVector : protected BitWord { public: using Index = BitWord::Index; - using GenerationHolder = vespalib::GenerationHolder; - using GenerationHeldBase = vespalib::GenerationHeldBase; using UP = std::unique_ptr<BitVector>; class Range { public: @@ -50,12 +47,11 @@ public: bool testBit(Index idx) const { return ((load_word(wordNum(idx)) & mask(idx)) != 0); } - Index getSizeSafe() const { + Index getSizeAcquire() const { return vespalib::atomic::load_ref_acquire(_sz); } - bool testBitSafe(Index idx) const { - auto my_words = vespalib::atomic::load_ref_acquire(_words); - auto my_word = vespalib::atomic::load_ref_acquire(my_words[wordNum(idx)]); + bool testBitAcquire(Index idx) const { + auto my_word = vespalib::atomic::load_ref_acquire(_words[wordNum(idx)]); return (my_word & mask(idx)) != 0; } bool hasTrueBits() const { diff --git a/searchlib/src/vespa/searchlib/common/growablebitvector.cpp b/searchlib/src/vespa/searchlib/common/growablebitvector.cpp index 7a548b3e7ac..e3334be3fd9 100644 --- a/searchlib/src/vespa/searchlib/common/growablebitvector.cpp +++ b/searchlib/src/vespa/searchlib/common/growablebitvector.cpp @@ -7,52 +7,65 @@ namespace search { using vespalib::GenerationHeldBase; -using vespalib::GenerationHeldAlloc; using vespalib::GenerationHolder; +namespace { + +struct GenerationHeldAllocatedBitVector : public vespalib::GenerationHeldBase { + std::unique_ptr<AllocatedBitVector> vector; + GenerationHeldAllocatedBitVector(std::unique_ptr<AllocatedBitVector> vector_in) + : GenerationHeldBase(sizeof(AllocatedBitVector) + vector_in->extraByteSize()), + vector(std::move(vector_in)) {} +}; + +} + GenerationHeldBase::UP -GrowableBitVector::grow(Index newSize, Index newCapacity) +GrowableBitVector::grow(BitWord::Index newSize, BitWord::Index newCapacity) { + AllocatedBitVector &self = *_stored; assert(newCapacity >= newSize); - GenerationHeldBase::UP ret; - if (newCapacity != capacity()) { - AllocatedBitVector tbv(newSize, newCapacity, _alloc.get(), size(), &_alloc); - if (newSize > size()) { - tbv.clearBitAndMaintainCount(size()); // Clear old guard bit. + if (newCapacity != self.capacity()) { + auto tbv = std::make_unique<AllocatedBitVector>(newSize, newCapacity, self._alloc.get(), self.size(), &self._alloc); + if (newSize > self.size()) { + tbv->clearBitAndMaintainCount(self.size()); // Clear old guard bit. } - ret = std::make_unique<GenerationHeldAlloc<Alloc>>(_alloc); - swap(tbv); + auto to_hold = std::make_unique<GenerationHeldAllocatedBitVector>(std::move(_stored)); + _self.store(tbv.get(), std::memory_order_release); + _stored = std::move(tbv); + return to_hold; } else { - if (newSize > size()) { - Range clearRange(size(), newSize); - setSize(newSize); - clearIntervalNoInvalidation(clearRange); + if (newSize > self.size()) { + BitVector::Range clearRange(self.size(), newSize); + self.setSize(newSize); + self.clearIntervalNoInvalidation(clearRange); } else { - clearIntervalNoInvalidation(Range(newSize, size())); - setSize(newSize); - updateCount(); + self.clearIntervalNoInvalidation(BitVector::Range(newSize, self.size())); + self.setSize(newSize); + self.updateCount(); } } - return ret; + return {}; } -GrowableBitVector::GrowableBitVector(Index newSize, Index newCapacity, +GrowableBitVector::GrowableBitVector(BitWord::Index newSize, BitWord::Index newCapacity, GenerationHolder &generationHolder, - const Alloc* init_alloc) - : AllocatedBitVector(newSize, newCapacity, nullptr, 0, init_alloc), - _generationHolder(generationHolder) + const Alloc *init_alloc) + : _stored(std::make_unique<AllocatedBitVector>(newSize, newCapacity, nullptr, 0, init_alloc)), + _self(_stored.get()), + _generationHolder(generationHolder) { assert(newSize <= newCapacity); } bool -GrowableBitVector::reserve(Index newCapacity) +GrowableBitVector::reserve(BitWord::Index newCapacity) { - Index oldCapacity = capacity(); + BitWord::Index oldCapacity = _stored->capacity(); assert(newCapacity >= oldCapacity); if (newCapacity == oldCapacity) return false; - return hold(grow(size(), newCapacity)); + return hold(grow(_stored->size(), newCapacity)); } bool @@ -66,18 +79,18 @@ GrowableBitVector::hold(GenerationHeldBase::UP v) } bool -GrowableBitVector::shrink(Index newCapacity) +GrowableBitVector::shrink(BitWord::Index newCapacity) { - Index oldCapacity = capacity(); + BitWord::Index oldCapacity = _stored->capacity(); assert(newCapacity <= oldCapacity); (void) oldCapacity; - return hold(grow(newCapacity, std::max(capacity(), newCapacity))); + return hold(grow(newCapacity, std::max(_stored->capacity(), newCapacity))); } bool -GrowableBitVector::extend(Index newCapacity) +GrowableBitVector::extend(BitWord::Index newCapacity) { - return hold(grow(newCapacity, std::max(capacity(), newCapacity))); + return hold(grow(newCapacity, std::max(_stored->capacity(), newCapacity))); } } // namespace search diff --git a/searchlib/src/vespa/searchlib/common/growablebitvector.h b/searchlib/src/vespa/searchlib/common/growablebitvector.h index 8773f1573d6..e9443512040 100644 --- a/searchlib/src/vespa/searchlib/common/growablebitvector.h +++ b/searchlib/src/vespa/searchlib/common/growablebitvector.h @@ -3,22 +3,39 @@ #pragma once #include "allocatedbitvector.h" +#include <vespa/vespalib/util/atomic.h> +#include <vespa/vespalib/util/generationholder.h> namespace search { -class GrowableBitVector : public AllocatedBitVector +class GrowableBitVector { public: - GrowableBitVector(Index newSize, Index newCapacity, GenerationHolder &generationHolder, const Alloc* init_alloc = nullptr); + using Alloc = vespalib::alloc::Alloc; + using GenerationHolder = vespalib::GenerationHolder; + using GenerationHeldBase = vespalib::GenerationHeldBase; + GrowableBitVector(BitWord::Index newSize, BitWord::Index newCapacity, + GenerationHolder &generationHolder, const Alloc *init_alloc = nullptr); + + const BitVector &reader() const { return acquire_self(); } + AllocatedBitVector &writer() { return *_stored; } + + BitWord::Index extraByteSize() const { + return sizeof(AllocatedBitVector) + acquire_self().extraByteSize(); + } /** Will return true if a a buffer is held */ - bool reserve(Index newCapacity); - bool shrink(Index newCapacity); - bool extend(Index newCapacity); + bool reserve(BitWord::Index newCapacity); + bool shrink(BitWord::Index newCapacity); + bool extend(BitWord::Index newCapacity); private: - GenerationHeldBase::UP grow(Index newLength, Index newCapacity); + GenerationHeldBase::UP grow(BitWord::Index newLength, BitWord::Index newCapacity); + + AllocatedBitVector &acquire_self() const { return *(_self.load(std::memory_order_acquire)); } VESPA_DLL_LOCAL bool hold(GenerationHeldBase::UP v); + std::unique_ptr<AllocatedBitVector> _stored; + std::atomic<AllocatedBitVector *> _self; GenerationHolder &_generationHolder; }; diff --git a/searchlib/src/vespa/searchlib/diskindex/bitvectordictionary.h b/searchlib/src/vespa/searchlib/diskindex/bitvectordictionary.h index 7f44728e1be..b1e5ca55a15 100644 --- a/searchlib/src/vespa/searchlib/diskindex/bitvectordictionary.h +++ b/searchlib/src/vespa/searchlib/diskindex/bitvectordictionary.h @@ -6,6 +6,7 @@ #include <vespa/searchlib/index/bitvectorkeys.h> #include <vespa/searchlib/common/tunefileinfo.h> #include <vespa/vespalib/stllike/string.h> +#include <vector> namespace search::diskindex { diff --git a/searchlib/src/vespa/searchlib/diskindex/bitvectorfile.h b/searchlib/src/vespa/searchlib/diskindex/bitvectorfile.h index 007368babdd..162ca512cec 100644 --- a/searchlib/src/vespa/searchlib/diskindex/bitvectorfile.h +++ b/searchlib/src/vespa/searchlib/diskindex/bitvectorfile.h @@ -7,6 +7,7 @@ #include <vespa/searchlib/common/tunefileinfo.h> #include <vespa/vespalib/stllike/string.h> #include <vespa/vespalib/stllike/allocator.h> +#include <vector> class Fast_BufferedFile; |