diff options
author | Henning Baldersheim <balder@yahoo-inc.com> | 2020-05-10 21:50:51 +0000 |
---|---|---|
committer | Henning Baldersheim <balder@yahoo-inc.com> | 2020-05-10 21:50:51 +0000 |
commit | a5f5eb075e407c2243788d9fc1ad1fb170435fb3 (patch) | |
tree | 6da4e2f861c3a220d833c1c83f169b952bf1a04f | |
parent | d6e059759286443da0e30abb9212baf3b8c281ab (diff) |
Use mmapped allocation for longlived buffers.
9 files changed, 87 insertions, 104 deletions
diff --git a/searchlib/src/vespa/searchlib/test/fakedata/bitencode64.h b/searchlib/src/vespa/searchlib/test/fakedata/bitencode64.h index ffddc4d7be7..0e968f2f4e3 100644 --- a/searchlib/src/vespa/searchlib/test/fakedata/bitencode64.h +++ b/searchlib/src/vespa/searchlib/test/fakedata/bitencode64.h @@ -14,7 +14,7 @@ class BitEncode64 : public bitcompression::EncodeContext64<bigEndian> public: BitEncode64(); - ~BitEncode64(); + ~BitEncode64() override; typedef bitcompression::EncodeContext64<bigEndian> EC; @@ -28,10 +28,8 @@ public: } std::pair<uint64_t *, size_t> - grabComprBuffer(void *&comprBufMalloc) - { - std::pair<void *, size_t> tres = _cbuf.grabComprBuffer(comprBufMalloc); - return std::make_pair(static_cast<uint64_t *>(tres.first), tres.second); + grabComprBuffer(vespalib::alloc::Alloc & comprAlloc) { + return _cbuf.grabComprBuffer(comprAlloc); } }; diff --git a/searchlib/src/vespa/searchlib/test/fakedata/fakeegcompr64filterocc.cpp b/searchlib/src/vespa/searchlib/test/fakedata/fakeegcompr64filterocc.cpp index 795e25899c0..f2ce65480ac 100644 --- a/searchlib/src/vespa/searchlib/test/fakedata/fakeegcompr64filterocc.cpp +++ b/searchlib/src/vespa/searchlib/test/fakedata/fakeegcompr64filterocc.cpp @@ -62,16 +62,16 @@ init(std::make_pair("EGCompr64FilterOcc", FakeEGCompr64FilterOcc::FakeEGCompr64FilterOcc(const FakeWord &fw) : FakePosting(fw.getName() + ".egc64filterocc"), - _compressed(std::make_pair(static_cast<uint64_t *>(NULL), 0)), - _l1SkipCompressed(std::make_pair(static_cast<uint64_t *>(NULL), 0)), - _l2SkipCompressed(std::make_pair(static_cast<uint64_t *>(NULL), 0)), - _l3SkipCompressed(std::make_pair(static_cast<uint64_t *>(NULL), 0)), - _l4SkipCompressed(std::make_pair(static_cast<uint64_t *>(NULL), 0)), - _compressedMalloc(NULL), - _l1SkipCompressedMalloc(NULL), - _l2SkipCompressedMalloc(NULL), - _l3SkipCompressedMalloc(NULL), - _l4SkipCompressedMalloc(NULL), + _compressed(std::make_pair(static_cast<uint64_t *>(nullptr), 0)), + _l1SkipCompressed(std::make_pair(static_cast<uint64_t *>(nullptr), 0)), + _l2SkipCompressed(std::make_pair(static_cast<uint64_t *>(nullptr), 0)), + _l3SkipCompressed(std::make_pair(static_cast<uint64_t *>(nullptr), 0)), + _l4SkipCompressed(std::make_pair(static_cast<uint64_t *>(nullptr), 0)), + _compressedAlloc(), + _l1SkipCompressedAlloc(), + _l2SkipCompressedAlloc(), + _l3SkipCompressedAlloc(), + _l4SkipCompressedAlloc(), _docIdLimit(0), _hitDocs(0), _lastDocId(0u), @@ -90,16 +90,16 @@ FakeEGCompr64FilterOcc::FakeEGCompr64FilterOcc(const FakeWord &fw, bool bigEndian, const char *nameSuffix) : FakePosting(fw.getName() + nameSuffix), - _compressed(std::make_pair(static_cast<uint64_t *>(NULL), 0)), - _l1SkipCompressed(std::make_pair(static_cast<uint64_t *>(NULL), 0)), - _l2SkipCompressed(std::make_pair(static_cast<uint64_t *>(NULL), 0)), - _l3SkipCompressed(std::make_pair(static_cast<uint64_t *>(NULL), 0)), - _l4SkipCompressed(std::make_pair(static_cast<uint64_t *>(NULL), 0)), - _compressedMalloc(NULL), - _l1SkipCompressedMalloc(NULL), - _l2SkipCompressedMalloc(NULL), - _l3SkipCompressedMalloc(NULL), - _l4SkipCompressedMalloc(NULL), + _compressed(std::make_pair(static_cast<uint64_t *>(nullptr), 0)), + _l1SkipCompressed(std::make_pair(static_cast<uint64_t *>(nullptr), 0)), + _l2SkipCompressed(std::make_pair(static_cast<uint64_t *>(nullptr), 0)), + _l3SkipCompressed(std::make_pair(static_cast<uint64_t *>(nullptr), 0)), + _l4SkipCompressed(std::make_pair(static_cast<uint64_t *>(nullptr), 0)), + _compressedAlloc(), + _l1SkipCompressedAlloc(), + _l2SkipCompressedAlloc(), + _l3SkipCompressedAlloc(), + _l4SkipCompressedAlloc(), _docIdLimit(0), _hitDocs(0), _lastDocId(0u), @@ -391,24 +391,17 @@ setupT(const FakeWord &fw) l4SkipBits.writeBits(static_cast<uint64_t>(-1), 64); l4SkipBits.flush(); l4SkipBits.writeComprBuffer(); - _compressed = bits.grabComprBuffer(_compressedMalloc); - _l1SkipCompressed = l1SkipBits.grabComprBuffer(_l1SkipCompressedMalloc); - _l2SkipCompressed = l2SkipBits.grabComprBuffer(_l2SkipCompressedMalloc); - _l3SkipCompressed = l3SkipBits.grabComprBuffer(_l3SkipCompressedMalloc); - _l4SkipCompressed = l4SkipBits.grabComprBuffer(_l4SkipCompressedMalloc); + _compressed = bits.grabComprBuffer(_compressedAlloc); + _l1SkipCompressed = l1SkipBits.grabComprBuffer(_l1SkipCompressedAlloc); + _l2SkipCompressed = l2SkipBits.grabComprBuffer(_l2SkipCompressedAlloc); + _l3SkipCompressed = l3SkipBits.grabComprBuffer(_l3SkipCompressedAlloc); + _l4SkipCompressed = l4SkipBits.grabComprBuffer(_l4SkipCompressedAlloc); _docIdLimit = fw._docIdLimit; _lastDocId = lastDocId; } -FakeEGCompr64FilterOcc::~FakeEGCompr64FilterOcc() -{ - free(_compressedMalloc); - free(_l1SkipCompressedMalloc); - free(_l2SkipCompressedMalloc); - free(_l3SkipCompressedMalloc); - free(_l4SkipCompressedMalloc); -} +FakeEGCompr64FilterOcc::~FakeEGCompr64FilterOcc() = default; void diff --git a/searchlib/src/vespa/searchlib/test/fakedata/fakeegcompr64filterocc.h b/searchlib/src/vespa/searchlib/test/fakedata/fakeegcompr64filterocc.h index 1d84082e446..e17418b799b 100644 --- a/searchlib/src/vespa/searchlib/test/fakedata/fakeegcompr64filterocc.h +++ b/searchlib/src/vespa/searchlib/test/fakedata/fakeegcompr64filterocc.h @@ -12,25 +12,26 @@ namespace search::fakedata { class FakeEGCompr64FilterOcc : public FakePosting { protected: + using Alloc = vespalib::alloc::Alloc; std::pair<uint64_t *, size_t> _compressed; std::pair<uint64_t *, size_t> _l1SkipCompressed; std::pair<uint64_t *, size_t> _l2SkipCompressed; std::pair<uint64_t *, size_t> _l3SkipCompressed; std::pair<uint64_t *, size_t> _l4SkipCompressed; - void *_compressedMalloc; - void *_l1SkipCompressedMalloc; - void *_l2SkipCompressedMalloc; - void *_l3SkipCompressedMalloc; - void *_l4SkipCompressedMalloc; + Alloc _compressedAlloc; + Alloc _l1SkipCompressedAlloc; + Alloc _l2SkipCompressedAlloc; + Alloc _l3SkipCompressedAlloc; + Alloc _l4SkipCompressedAlloc; unsigned int _docIdLimit; unsigned int _hitDocs; - uint32_t _lastDocId; - size_t _bitSize; - size_t _l1SkipBitSize; - size_t _l2SkipBitSize; - size_t _l3SkipBitSize; - size_t _l4SkipBitSize; - bool _bigEndian; + uint32_t _lastDocId; + size_t _bitSize; + size_t _l1SkipBitSize; + size_t _l2SkipBitSize; + size_t _l3SkipBitSize; + size_t _l4SkipBitSize; + bool _bigEndian; private: void setup(const FakeWord &fw); @@ -39,10 +40,10 @@ private: void setupT(const FakeWord &fw); public: - FakeEGCompr64FilterOcc(const FakeWord &fw); + explicit FakeEGCompr64FilterOcc(const FakeWord &fw); FakeEGCompr64FilterOcc(const FakeWord &fw, bool bigEndian, const char *nameSuffix); - ~FakeEGCompr64FilterOcc(); + ~FakeEGCompr64FilterOcc() override; static void forceLink(); diff --git a/searchlib/src/vespa/searchlib/test/fakedata/fakezcfilterocc.cpp b/searchlib/src/vespa/searchlib/test/fakedata/fakezcfilterocc.cpp index 1da086fb046..c3b66079af5 100644 --- a/searchlib/src/vespa/searchlib/test/fakedata/fakezcfilterocc.cpp +++ b/searchlib/src/vespa/searchlib/test/fakedata/fakezcfilterocc.cpp @@ -96,7 +96,7 @@ FakeZcFilterOcc::FakeZcFilterOcc(const FakeWord &fw) _lastDocId(0u), _compressedBits(0), _compressed(std::make_pair(static_cast<uint64_t *>(nullptr), 0)), - _compressedMalloc(nullptr), + _compressedAlloc(), _featuresSize(0), _fieldsParams(fw.getFieldsParams()), _bigEndian(true), @@ -201,8 +201,7 @@ FakeZcFilterOcc::setupT(const FakeWord &fw) _lastDocId = fw._postings.back()._docId; writer.on_close(); - std::pair<void *, size_t> ectxData = writeContext.grabComprBuffer(_compressedMalloc); - _compressed = std::make_pair(static_cast<uint64_t *>(ectxData.first), ectxData.second); + _compressed = writeContext.grabComprBuffer(_compressedAlloc); read_header<bigEndian>(); } @@ -288,10 +287,7 @@ FakeZcFilterOcc::validate_read(const FakeWord &fw) const assert(static_cast<int32_t>(features.doc_id()) == -1); } -FakeZcFilterOcc::~FakeZcFilterOcc() -{ - free(_compressedMalloc); -} +FakeZcFilterOcc::~FakeZcFilterOcc() = default; void diff --git a/searchlib/src/vespa/searchlib/test/fakedata/fakezcfilterocc.h b/searchlib/src/vespa/searchlib/test/fakedata/fakezcfilterocc.h index 030c4b7e936..ade2efce0ef 100644 --- a/searchlib/src/vespa/searchlib/test/fakedata/fakezcfilterocc.h +++ b/searchlib/src/vespa/searchlib/test/fakedata/fakezcfilterocc.h @@ -7,9 +7,7 @@ #include <vespa/searchlib/bitcompression/posocccompression.h> #include <vespa/searchlib/diskindex/zc4_posting_params.h> -namespace search { - -namespace fakedata { +namespace search::fakedata { /* * YST style compression of docid list. @@ -17,21 +15,21 @@ namespace fakedata { class FakeZcFilterOcc : public FakePosting { protected: - size_t _docIdsSize; - size_t _l1SkipSize; - size_t _l2SkipSize; - size_t _l3SkipSize; - size_t _l4SkipSize; + size_t _docIdsSize; + size_t _l1SkipSize; + size_t _l2SkipSize; + size_t _l3SkipSize; + size_t _l4SkipSize; unsigned int _hitDocs; - uint32_t _lastDocId; + uint32_t _lastDocId; - uint64_t _compressedBits; + uint64_t _compressedBits; std::pair<uint64_t *, size_t> _compressed; - void *_compressedMalloc; - uint64_t _featuresSize; - const search::bitcompression::PosOccFieldsParams &_fieldsParams; - bool _bigEndian; - diskindex::Zc4PostingParams _posting_params; + vespalib::alloc::Alloc _compressedAlloc; + uint64_t _featuresSize; + const bitcompression::PosOccFieldsParams &_fieldsParams; + bool _bigEndian; + diskindex::Zc4PostingParams _posting_params; protected: void setup(const FakeWord &fw); @@ -46,7 +44,7 @@ protected: void validate_read(const FakeWord &fw) const; public: - FakeZcFilterOcc(const FakeWord &fw); + explicit FakeZcFilterOcc(const FakeWord &fw); FakeZcFilterOcc(const FakeWord &fw, bool bigEndian, const diskindex::Zc4PostingParams &posting_params, @@ -70,6 +68,4 @@ public: queryeval::SearchIterator *createIterator(const fef::TermFieldMatchDataArray &matchData) const override; }; -} // namespace fakedata - -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/util/comprbuffer.cpp b/searchlib/src/vespa/searchlib/util/comprbuffer.cpp index 9b122419267..cc969695061 100644 --- a/searchlib/src/vespa/searchlib/util/comprbuffer.cpp +++ b/searchlib/src/vespa/searchlib/util/comprbuffer.cpp @@ -12,7 +12,7 @@ ComprBuffer::ComprBuffer(uint32_t unitSize) _padBefore(false), _comprBuf(nullptr), _comprBufSize(0), - _comprBufMalloc(nullptr) + _comprAlloc() { } @@ -25,9 +25,7 @@ ComprBuffer::~ComprBuffer() void ComprBuffer::dropComprBuf() { - free(_comprBufMalloc); _comprBuf = nullptr; - _comprBufMalloc = nullptr; } @@ -59,6 +57,7 @@ ComprBuffer::allocComprBuf() */ size_t paddingAfter = minimumPadding() * _unitSize; size_t paddingBefore = 0; + size_t memalign = FastOS_File::getMaxDirectIOMemAlign(); if (_padBefore) { /* * Add padding before normal buffer, to allow last data at end of @@ -68,21 +67,21 @@ ComprBuffer::allocComprBuf() * buffers. */ paddingBefore = paddingAfter + 2 * _unitSize; - size_t memalign = FastOS_File::getMaxDirectIOMemAlign(); + if (paddingBefore < memalign) paddingBefore = memalign; } size_t fullpadding = paddingAfter + paddingBefore; size_t allocLen = _comprBufSize * _unitSize + fullpadding; - void *alignedBuf = FastOS_File::allocateGenericDirectIOBuffer(allocLen, - _comprBufMalloc); + _comprAlloc = Alloc::alloc(allocLen, vespalib::alloc::MemoryAllocator::HUGEPAGE_SIZE, memalign); + void *alignedBuf = _comprAlloc.get(); memset(alignedBuf, 0, allocLen); /* * Set pointer to the start of normal buffer, which should be properly * aligned in memory for direct IO. */ - _comprBuf = reinterpret_cast<void *> - (static_cast<char *>(alignedBuf) + paddingBefore); + _comprBuf = (static_cast<char *>(alignedBuf) + paddingBefore); + _comprBufSize = (_comprAlloc.size() - fullpadding) / _unitSize; } @@ -95,14 +94,14 @@ ComprBuffer::expandComprBuf(uint32_t overflowUnits) newSize = 16; size_t paddingAfter = minimumPadding() * _unitSize; assert(overflowUnits <= minimumPadding()); - void *newBuf = malloc(newSize * _unitSize + paddingAfter); - size_t oldLen = (static_cast<size_t>(_comprBufSize) + overflowUnits) * - _unitSize; - if (oldLen > 0) - memcpy(newBuf, _comprBuf, oldLen); - free(_comprBufMalloc); - _comprBuf = _comprBufMalloc = newBuf; - _comprBufSize = newSize; + Alloc newBuf = Alloc::alloc(newSize * _unitSize + paddingAfter); + size_t oldLen = (static_cast<size_t>(_comprBufSize) + overflowUnits) * _unitSize; + if (oldLen > 0) { + memcpy(newBuf.get(), _comprBuf, oldLen); + } + _comprAlloc = std::move(newBuf); + _comprBuf = _comprAlloc.get(); + _comprBufSize = (_comprAlloc.size() - paddingAfter) / _unitSize; } diff --git a/searchlib/src/vespa/searchlib/util/comprbuffer.h b/searchlib/src/vespa/searchlib/util/comprbuffer.h index 679c0dccdf0..6bddf322c38 100644 --- a/searchlib/src/vespa/searchlib/util/comprbuffer.h +++ b/searchlib/src/vespa/searchlib/util/comprbuffer.h @@ -3,18 +3,20 @@ #pragma once #include <vespa/searchlib/util/filealign.h> +#include <vespa/vespalib/util/alloc.h> namespace search { class ComprBuffer { private: + using Alloc = vespalib::alloc::Alloc; void allocComprBuf(); const uint32_t _unitSize; // Size of unit in bytes, doubles up as alignment bool _padBefore; void *_comprBuf; size_t _comprBufSize; - void *_comprBufMalloc; + Alloc _comprAlloc; FileAlign _aligner; public: @@ -44,11 +46,9 @@ public: } const FileAlign & getAligner() const { return _aligner; } - void * stealComprBuf() { - void * stolen = _comprBufMalloc; - _comprBufMalloc = nullptr; + Alloc stealComprBuf() { setComprBuf(nullptr, 0); - return stolen; + return std::move(_comprAlloc); } /* diff --git a/searchlib/src/vespa/searchlib/util/comprfile.cpp b/searchlib/src/vespa/searchlib/util/comprfile.cpp index b0099ad9074..dffcd639a53 100644 --- a/searchlib/src/vespa/searchlib/util/comprfile.cpp +++ b/searchlib/src/vespa/searchlib/util/comprfile.cpp @@ -434,13 +434,13 @@ ComprFileWriteContext::writeComprBuffer(bool flushSlack) _encodeContext->afterWrite(*this, chunkUsedUnits, 0); } -std::pair<void *, size_t> -ComprFileWriteContext::grabComprBuffer(void *&comprBufMalloc) +std::pair<uint64_t *, size_t> +ComprFileWriteContext::grabComprBuffer(vespalib::alloc::Alloc & comprAlloc) { assert(_file == nullptr); - std::pair<void *, size_t> res = + std::pair<uint64_t *, size_t> res = std::make_pair(getComprBuf(), _encodeContext->getUsedUnits(getComprBuf())); - comprBufMalloc = stealComprBuf(); + comprAlloc = stealComprBuf(); return res; } diff --git a/searchlib/src/vespa/searchlib/util/comprfile.h b/searchlib/src/vespa/searchlib/util/comprfile.h index a28e14b02ac..b713615bed1 100644 --- a/searchlib/src/vespa/searchlib/util/comprfile.h +++ b/searchlib/src/vespa/searchlib/util/comprfile.h @@ -204,7 +204,7 @@ public: * Grab compressed buffer from write context. This is only legal when * no file is attached. */ - std::pair<void *, size_t> grabComprBuffer(void *&comprBufMalloc); + std::pair<uint64_t *, size_t> grabComprBuffer(vespalib::alloc::Alloc & comprAlloc); }; } |