diff options
author | Henning Baldersheim <balder@yahoo-inc.com> | 2023-12-15 18:54:18 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2023-12-15 18:54:18 +0100 |
commit | d42b67f0fe821d122548a345f27fda7f9c9c9d10 (patch) | |
tree | 30c7b914294d8a3f4a9f7867424eafe576ab0b33 /searchlib | |
parent | 02a9c704fd13806b9e17eb28e1e6425fc87504f3 (diff) | |
parent | 82b20e53fc6d1b949955f9cfe6f4048b7de94640 (diff) |
Merge pull request #29679 from vespa-engine/balder/leaner-hotpath
Balder/leaner hotpath
Diffstat (limited to 'searchlib')
-rw-r--r-- | searchlib/src/vespa/searchlib/queryeval/multibitvectoriterator.cpp | 54 | ||||
-rw-r--r-- | searchlib/src/vespa/searchlib/queryeval/multibitvectoriterator.h | 10 |
2 files changed, 36 insertions, 28 deletions
diff --git a/searchlib/src/vespa/searchlib/queryeval/multibitvectoriterator.cpp b/searchlib/src/vespa/searchlib/queryeval/multibitvectoriterator.cpp index fdf4ec950dd..66f505581c7 100644 --- a/searchlib/src/vespa/searchlib/queryeval/multibitvectoriterator.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/multibitvectoriterator.cpp @@ -20,7 +20,7 @@ struct And { void operator () (const IAccelrated & accel, size_t offset, const std::vector<Meta> & src, void *dest) noexcept { accel.and64(offset, src, dest); } - static bool isAnd() noexcept { return true; } + static constexpr bool isAnd() noexcept { return true; } }; struct Or { @@ -28,7 +28,7 @@ struct Or { void operator () (const IAccelrated & accel, size_t offset, const std::vector<Meta> & src, void *dest) noexcept { accel.or64(offset, src, dest); } - static bool isAnd() noexcept { return false; } + static constexpr bool isAnd() noexcept { return false; } }; } @@ -63,36 +63,40 @@ MultiBitVector<Update>::MultiBitVector(size_t reserved) template<typename Update> bool -MultiBitVector<Update>::updateLastValue(uint32_t docId) noexcept +MultiBitVector<Update>::updateLastValueCold(uint32_t docId) noexcept { - if (docId >= _lastMaxDocIdLimit) { - if (__builtin_expect(isAtEnd(docId), false)) { - return true; - } - const uint32_t index(BitWord::wordNum(docId)); - if (docId >= _lastMaxDocIdLimitRequireFetch) { - uint32_t baseIndex = index & ~(NumWordsInBatch - 1); - _update(_accel, baseIndex*sizeof(Word), _bvs, _lastWords); - _lastMaxDocIdLimitRequireFetch = (baseIndex + NumWordsInBatch) * BitWord::WordLen; - } - _lastValue = _lastWords[index % NumWordsInBatch]; - _lastMaxDocIdLimit = (index + 1) * BitWord::WordLen; + if (__builtin_expect(isAtEnd(docId), false)) { + return true; + } + const uint32_t index(BitWord::wordNum(docId)); + if (docId >= _lastMaxDocIdLimitRequireFetch) { + fetchChunk(index); } + _lastValue = _lastWords[index % NumWordsInBatch]; + _lastMaxDocIdLimit = (index + 1) * BitWord::WordLen; return false; } template<typename Update> +void +MultiBitVector<Update>::fetchChunk(uint32_t index) noexcept +{ + uint32_t baseIndex = index & ~(NumWordsInBatch - 1); + _update(_accel, baseIndex*sizeof(Word), _bvs, _lastWords); + _lastMaxDocIdLimitRequireFetch = (baseIndex + NumWordsInBatch) * BitWord::WordLen; +} + +template<typename Update> uint32_t MultiBitVector<Update>::strictSeek(uint32_t docId) noexcept { bool atEnd; for (atEnd = updateLastValue(docId), _lastValue = _lastValue & BitWord::checkTab(docId); - (_lastValue == 0) && __builtin_expect(! atEnd, true); + __builtin_expect(_lastValue == 0, Update::isAnd()) && __builtin_expect(! atEnd, true); // And is likely to have few bits, while Or has many. atEnd = updateLastValue(_lastMaxDocIdLimit)); - if (__builtin_expect(!atEnd, true)) { - return _lastMaxDocIdLimit - BitWord::WordLen + vespalib::Optimized::lsbIdx(_lastValue); - } - return _numDocs; + return (__builtin_expect(!atEnd, true)) + ? _lastMaxDocIdLimit - BitWord::WordLen + vespalib::Optimized::lsbIdx(_lastValue) + : _numDocs; } template<typename Update> @@ -100,12 +104,8 @@ bool MultiBitVector<Update>::seek(uint32_t docId) noexcept { bool atEnd = updateLastValue(docId); - if (__builtin_expect( ! atEnd, true)) { - if (_lastValue & BitWord::mask(docId)) { - return true; - } - } - return false; + return __builtin_expect( ! atEnd, true) && + __builtin_expect(_lastValue & BitWord::mask(docId), false); } namespace { @@ -160,7 +160,7 @@ template<typename Update> void MultiBitVectorIterator<Update>::doSeek(uint32_t docId) { - if (_mbv.seek(docId)) { + if (_mbv.seek(docId)) [[unlikely]] { setDocId(docId); } } diff --git a/searchlib/src/vespa/searchlib/queryeval/multibitvectoriterator.h b/searchlib/src/vespa/searchlib/queryeval/multibitvectoriterator.h index 2b4f90544ac..0d9e2c4f25f 100644 --- a/searchlib/src/vespa/searchlib/queryeval/multibitvectoriterator.h +++ b/searchlib/src/vespa/searchlib/queryeval/multibitvectoriterator.h @@ -37,7 +37,15 @@ public: bool seek(uint32_t docId) noexcept; bool acceptExtraFilter() const noexcept { return Update::isAnd(); } private: - bool updateLastValue(uint32_t docId) noexcept; + bool updateLastValue(uint32_t docId) noexcept { + if (docId >= _lastMaxDocIdLimit) { + return updateLastValueCold(docId); + } + return false; + } + VESPA_DLL_LOCAL bool updateLastValueCold(uint32_t docId) noexcept __attribute__((noinline)); + VESPA_DLL_LOCAL void fetchChunk(uint32_t docId) noexcept __attribute__((noinline)); + using IAccelrated = vespalib::hwaccelrated::IAccelrated; Update _update; |