Use mmapped allocation for longlived buffers.

9 files changed, 87 insertions, 104 deletions

diff --git a/searchlib/src/vespa/searchlib/test/fakedata/bitencode64.h b/searchlib/src/vespa/searchlib/test/fakedata/bitencode64.h
index ffddc4d7be7..0e968f2f4e3 100644
--- a/searchlib/src/vespa/searchlib/test/fakedata/bitencode64.h
+++ b/searchlib/src/vespa/searchlib/test/fakedata/bitencode64.h
@@ -14,7 +14,7 @@ class BitEncode64 : public bitcompression::EncodeContext64<bigEndian>
 
 public:
     BitEncode64();
-    ~BitEncode64();
+    ~BitEncode64() override;
 
     typedef bitcompression::EncodeContext64<bigEndian> EC;
 
@@ -28,10 +28,8 @@ public:
     }
 
     std::pair<uint64_t *, size_t>
-    grabComprBuffer(void *&comprBufMalloc)
-    {
-        std::pair<void *, size_t> tres = _cbuf.grabComprBuffer(comprBufMalloc);
-        return std::make_pair(static_cast<uint64_t *>(tres.first), tres.second);
+    grabComprBuffer(vespalib::alloc::Alloc & comprAlloc) {
+        return _cbuf.grabComprBuffer(comprAlloc);
     }
 };
 
diff --git a/searchlib/src/vespa/searchlib/test/fakedata/fakeegcompr64filterocc.cpp b/searchlib/src/vespa/searchlib/test/fakedata/fakeegcompr64filterocc.cpp
index 795e25899c0..f2ce65480ac 100644
--- a/searchlib/src/vespa/searchlib/test/fakedata/fakeegcompr64filterocc.cpp
+++ b/searchlib/src/vespa/searchlib/test/fakedata/fakeegcompr64filterocc.cpp
@@ -62,16 +62,16 @@ init(std::make_pair("EGCompr64FilterOcc",
 
 FakeEGCompr64FilterOcc::FakeEGCompr64FilterOcc(const FakeWord &fw)
     : FakePosting(fw.getName() + ".egc64filterocc"),
-      _compressed(std::make_pair(static_cast<uint64_t *>(NULL), 0)),
-      _l1SkipCompressed(std::make_pair(static_cast<uint64_t *>(NULL), 0)),
-      _l2SkipCompressed(std::make_pair(static_cast<uint64_t *>(NULL), 0)),
-      _l3SkipCompressed(std::make_pair(static_cast<uint64_t *>(NULL), 0)),
-      _l4SkipCompressed(std::make_pair(static_cast<uint64_t *>(NULL), 0)),
-      _compressedMalloc(NULL),
-      _l1SkipCompressedMalloc(NULL),
-      _l2SkipCompressedMalloc(NULL),
-      _l3SkipCompressedMalloc(NULL),
-      _l4SkipCompressedMalloc(NULL),
+      _compressed(std::make_pair(static_cast<uint64_t *>(nullptr), 0)),
+      _l1SkipCompressed(std::make_pair(static_cast<uint64_t *>(nullptr), 0)),
+      _l2SkipCompressed(std::make_pair(static_cast<uint64_t *>(nullptr), 0)),
+      _l3SkipCompressed(std::make_pair(static_cast<uint64_t *>(nullptr), 0)),
+      _l4SkipCompressed(std::make_pair(static_cast<uint64_t *>(nullptr), 0)),
+      _compressedAlloc(),
+      _l1SkipCompressedAlloc(),
+      _l2SkipCompressedAlloc(),
+      _l3SkipCompressedAlloc(),
+      _l4SkipCompressedAlloc(),
       _docIdLimit(0),
       _hitDocs(0),
       _lastDocId(0u),
@@ -90,16 +90,16 @@ FakeEGCompr64FilterOcc::FakeEGCompr64FilterOcc(const FakeWord &fw,
         bool bigEndian,
         const char *nameSuffix)
     : FakePosting(fw.getName() + nameSuffix),
-      _compressed(std::make_pair(static_cast<uint64_t *>(NULL), 0)),
-      _l1SkipCompressed(std::make_pair(static_cast<uint64_t *>(NULL), 0)),
-      _l2SkipCompressed(std::make_pair(static_cast<uint64_t *>(NULL), 0)),
-      _l3SkipCompressed(std::make_pair(static_cast<uint64_t *>(NULL), 0)),
-      _l4SkipCompressed(std::make_pair(static_cast<uint64_t *>(NULL), 0)),
-      _compressedMalloc(NULL),
-      _l1SkipCompressedMalloc(NULL),
-      _l2SkipCompressedMalloc(NULL),
-      _l3SkipCompressedMalloc(NULL),
-      _l4SkipCompressedMalloc(NULL),
+      _compressed(std::make_pair(static_cast<uint64_t *>(nullptr), 0)),
+      _l1SkipCompressed(std::make_pair(static_cast<uint64_t *>(nullptr), 0)),
+      _l2SkipCompressed(std::make_pair(static_cast<uint64_t *>(nullptr), 0)),
+      _l3SkipCompressed(std::make_pair(static_cast<uint64_t *>(nullptr), 0)),
+      _l4SkipCompressed(std::make_pair(static_cast<uint64_t *>(nullptr), 0)),
+      _compressedAlloc(),
+      _l1SkipCompressedAlloc(),
+      _l2SkipCompressedAlloc(),
+      _l3SkipCompressedAlloc(),
+      _l4SkipCompressedAlloc(),
       _docIdLimit(0),
       _hitDocs(0),
       _lastDocId(0u),
@@ -391,24 +391,17 @@ setupT(const FakeWord &fw)
     l4SkipBits.writeBits(static_cast<uint64_t>(-1), 64);
     l4SkipBits.flush();
     l4SkipBits.writeComprBuffer();
-    _compressed = bits.grabComprBuffer(_compressedMalloc);
-    _l1SkipCompressed = l1SkipBits.grabComprBuffer(_l1SkipCompressedMalloc);
-    _l2SkipCompressed = l2SkipBits.grabComprBuffer(_l2SkipCompressedMalloc);
-    _l3SkipCompressed = l3SkipBits.grabComprBuffer(_l3SkipCompressedMalloc);
-    _l4SkipCompressed = l4SkipBits.grabComprBuffer(_l4SkipCompressedMalloc);
+    _compressed = bits.grabComprBuffer(_compressedAlloc);
+    _l1SkipCompressed = l1SkipBits.grabComprBuffer(_l1SkipCompressedAlloc);
+    _l2SkipCompressed = l2SkipBits.grabComprBuffer(_l2SkipCompressedAlloc);
+    _l3SkipCompressed = l3SkipBits.grabComprBuffer(_l3SkipCompressedAlloc);
+    _l4SkipCompressed = l4SkipBits.grabComprBuffer(_l4SkipCompressedAlloc);
     _docIdLimit = fw._docIdLimit;
     _lastDocId = lastDocId;
 }
 
 
-FakeEGCompr64FilterOcc::~FakeEGCompr64FilterOcc()
-{
-    free(_compressedMalloc);
-    free(_l1SkipCompressedMalloc);
-    free(_l2SkipCompressedMalloc);
-    free(_l3SkipCompressedMalloc);
-    free(_l4SkipCompressedMalloc);
-}
+FakeEGCompr64FilterOcc::~FakeEGCompr64FilterOcc() = default;
 
 
 void
diff --git a/searchlib/src/vespa/searchlib/test/fakedata/fakeegcompr64filterocc.h b/searchlib/src/vespa/searchlib/test/fakedata/fakeegcompr64filterocc.h
index 1d84082e446..e17418b799b 100644
--- a/searchlib/src/vespa/searchlib/test/fakedata/fakeegcompr64filterocc.h
+++ b/searchlib/src/vespa/searchlib/test/fakedata/fakeegcompr64filterocc.h
@@ -12,25 +12,26 @@ namespace search::fakedata {
 class FakeEGCompr64FilterOcc : public FakePosting
 {
 protected:
+    using Alloc = vespalib::alloc::Alloc;
     std::pair<uint64_t *, size_t> _compressed;
     std::pair<uint64_t *, size_t> _l1SkipCompressed;
     std::pair<uint64_t *, size_t> _l2SkipCompressed;
     std::pair<uint64_t *, size_t> _l3SkipCompressed;
     std::pair<uint64_t *, size_t> _l4SkipCompressed;
-    void *_compressedMalloc;
-    void *_l1SkipCompressedMalloc;
-    void *_l2SkipCompressedMalloc;
-    void *_l3SkipCompressedMalloc;
-    void *_l4SkipCompressedMalloc;
+    Alloc _compressedAlloc;
+    Alloc _l1SkipCompressedAlloc;
+    Alloc _l2SkipCompressedAlloc;
+    Alloc _l3SkipCompressedAlloc;
+    Alloc _l4SkipCompressedAlloc;
     unsigned int _docIdLimit;
     unsigned int _hitDocs;
-    uint32_t _lastDocId;
-    size_t _bitSize;
-    size_t _l1SkipBitSize;
-    size_t _l2SkipBitSize;
-    size_t _l3SkipBitSize;
-    size_t _l4SkipBitSize;
-    bool _bigEndian;
+    uint32_t     _lastDocId;
+    size_t       _bitSize;
+    size_t       _l1SkipBitSize;
+    size_t       _l2SkipBitSize;
+    size_t       _l3SkipBitSize;
+    size_t       _l4SkipBitSize;
+    bool         _bigEndian;
 
 private:
     void setup(const FakeWord &fw);
@@ -39,10 +40,10 @@ private:
     void setupT(const FakeWord &fw);
 
 public:
-    FakeEGCompr64FilterOcc(const FakeWord &fw);
+    explicit FakeEGCompr64FilterOcc(const FakeWord &fw);
     FakeEGCompr64FilterOcc(const FakeWord &fw, bool bigEndian, const char *nameSuffix);
 
-    ~FakeEGCompr64FilterOcc();
+    ~FakeEGCompr64FilterOcc() override;
 
     static void forceLink();
 
diff --git a/searchlib/src/vespa/searchlib/test/fakedata/fakezcfilterocc.cpp b/searchlib/src/vespa/searchlib/test/fakedata/fakezcfilterocc.cpp
index 1da086fb046..c3b66079af5 100644
--- a/searchlib/src/vespa/searchlib/test/fakedata/fakezcfilterocc.cpp
+++ b/searchlib/src/vespa/searchlib/test/fakedata/fakezcfilterocc.cpp
@@ -96,7 +96,7 @@ FakeZcFilterOcc::FakeZcFilterOcc(const FakeWord &fw)
       _lastDocId(0u),
       _compressedBits(0),
       _compressed(std::make_pair(static_cast<uint64_t *>(nullptr), 0)),
-      _compressedMalloc(nullptr),
+      _compressedAlloc(),
       _featuresSize(0),
       _fieldsParams(fw.getFieldsParams()),
       _bigEndian(true),
@@ -201,8 +201,7 @@ FakeZcFilterOcc::setupT(const FakeWord &fw)
     _lastDocId = fw._postings.back()._docId;
     writer.on_close();
 
-    std::pair<void *, size_t> ectxData = writeContext.grabComprBuffer(_compressedMalloc);
-    _compressed = std::make_pair(static_cast<uint64_t *>(ectxData.first), ectxData.second);
+    _compressed = writeContext.grabComprBuffer(_compressedAlloc);
     read_header<bigEndian>();
 }
 
@@ -288,10 +287,7 @@ FakeZcFilterOcc::validate_read(const FakeWord &fw) const
     assert(static_cast<int32_t>(features.doc_id()) == -1);
 }
 
-FakeZcFilterOcc::~FakeZcFilterOcc()
-{
-    free(_compressedMalloc);
-}
+FakeZcFilterOcc::~FakeZcFilterOcc() = default;
 
 
 void
diff --git a/searchlib/src/vespa/searchlib/test/fakedata/fakezcfilterocc.h b/searchlib/src/vespa/searchlib/test/fakedata/fakezcfilterocc.h
index 030c4b7e936..ade2efce0ef 100644
--- a/searchlib/src/vespa/searchlib/test/fakedata/fakezcfilterocc.h
+++ b/searchlib/src/vespa/searchlib/test/fakedata/fakezcfilterocc.h
@@ -7,9 +7,7 @@
 #include <vespa/searchlib/bitcompression/posocccompression.h>
 #include <vespa/searchlib/diskindex/zc4_posting_params.h>
 
-namespace search {
-
-namespace fakedata {
+namespace search::fakedata {
 
 /*
  * YST style compression of docid list.
@@ -17,21 +15,21 @@ namespace fakedata {
 class FakeZcFilterOcc : public FakePosting
 {
 protected:
-    size_t _docIdsSize;
-    size_t _l1SkipSize;
-    size_t _l2SkipSize;
-    size_t _l3SkipSize;
-    size_t _l4SkipSize;
+    size_t       _docIdsSize;
+    size_t       _l1SkipSize;
+    size_t       _l2SkipSize;
+    size_t       _l3SkipSize;
+    size_t       _l4SkipSize;
     unsigned int _hitDocs;
-    uint32_t _lastDocId;
+    uint32_t     _lastDocId;
 
-    uint64_t _compressedBits;
+    uint64_t                      _compressedBits;
     std::pair<uint64_t *, size_t> _compressed;
-    void *_compressedMalloc;
-    uint64_t _featuresSize;
-    const search::bitcompression::PosOccFieldsParams &_fieldsParams;
-    bool _bigEndian;
-    diskindex::Zc4PostingParams _posting_params;
+    vespalib::alloc::Alloc        _compressedAlloc;
+    uint64_t                      _featuresSize;
+    const bitcompression::PosOccFieldsParams &_fieldsParams;
+    bool                          _bigEndian;
+    diskindex::Zc4PostingParams   _posting_params;
 protected:
     void setup(const FakeWord &fw);
 
@@ -46,7 +44,7 @@ protected:
     void validate_read(const FakeWord &fw) const;
 
 public:
-    FakeZcFilterOcc(const FakeWord &fw);
+    explicit FakeZcFilterOcc(const FakeWord &fw);
     FakeZcFilterOcc(const FakeWord &fw,
                     bool bigEndian,
                     const diskindex::Zc4PostingParams &posting_params,
@@ -70,6 +68,4 @@ public:
     queryeval::SearchIterator *createIterator(const fef::TermFieldMatchDataArray &matchData) const override;
 };
 
-} // namespace fakedata
-
-} // namespace search
+}
diff --git a/searchlib/src/vespa/searchlib/util/comprbuffer.cpp b/searchlib/src/vespa/searchlib/util/comprbuffer.cpp
index 9b122419267..cc969695061 100644
--- a/searchlib/src/vespa/searchlib/util/comprbuffer.cpp
+++ b/searchlib/src/vespa/searchlib/util/comprbuffer.cpp
@@ -12,7 +12,7 @@ ComprBuffer::ComprBuffer(uint32_t unitSize)
       _padBefore(false),
       _comprBuf(nullptr),
       _comprBufSize(0),
-      _comprBufMalloc(nullptr)
+      _comprAlloc()
 { }
 
 
@@ -25,9 +25,7 @@ ComprBuffer::~ComprBuffer()
 void
 ComprBuffer::dropComprBuf()
 {
-    free(_comprBufMalloc);
     _comprBuf = nullptr;
-    _comprBufMalloc = nullptr;
 }
 
 
@@ -59,6 +57,7 @@ ComprBuffer::allocComprBuf()
      */
     size_t paddingAfter = minimumPadding() * _unitSize;
     size_t paddingBefore = 0;
+    size_t memalign = FastOS_File::getMaxDirectIOMemAlign();
     if (_padBefore) {
         /*
          * Add padding before normal buffer, to allow last data at end of
@@ -68,21 +67,21 @@ ComprBuffer::allocComprBuf()
          * buffers.
          */
         paddingBefore = paddingAfter + 2 * _unitSize;
-        size_t memalign = FastOS_File::getMaxDirectIOMemAlign();
+
         if (paddingBefore < memalign)
             paddingBefore = memalign;
     }
     size_t fullpadding = paddingAfter + paddingBefore;
     size_t allocLen = _comprBufSize * _unitSize + fullpadding;
-    void *alignedBuf = FastOS_File::allocateGenericDirectIOBuffer(allocLen,
-            _comprBufMalloc);
+    _comprAlloc = Alloc::alloc(allocLen, vespalib::alloc::MemoryAllocator::HUGEPAGE_SIZE, memalign);
+    void *alignedBuf = _comprAlloc.get();
     memset(alignedBuf, 0, allocLen);
     /*
      * Set pointer to the start of normal buffer, which should be properly
      * aligned in memory for direct IO.
      */
-    _comprBuf = reinterpret_cast<void *>
-                (static_cast<char *>(alignedBuf) + paddingBefore);
+    _comprBuf = (static_cast<char *>(alignedBuf) + paddingBefore);
+    _comprBufSize = (_comprAlloc.size() - fullpadding) / _unitSize;
 }
 
 
@@ -95,14 +94,14 @@ ComprBuffer::expandComprBuf(uint32_t overflowUnits)
         newSize = 16;
     size_t paddingAfter = minimumPadding() * _unitSize;
     assert(overflowUnits <= minimumPadding());
-    void *newBuf = malloc(newSize * _unitSize + paddingAfter);
-    size_t oldLen = (static_cast<size_t>(_comprBufSize) + overflowUnits) *
-                    _unitSize;
-    if (oldLen > 0)
-        memcpy(newBuf, _comprBuf, oldLen);
-    free(_comprBufMalloc);
-    _comprBuf = _comprBufMalloc = newBuf;
-    _comprBufSize = newSize;
+    Alloc newBuf = Alloc::alloc(newSize * _unitSize + paddingAfter);
+    size_t oldLen = (static_cast<size_t>(_comprBufSize) + overflowUnits) * _unitSize;
+    if (oldLen > 0) {
+        memcpy(newBuf.get(), _comprBuf, oldLen);
+    }
+    _comprAlloc = std::move(newBuf);
+    _comprBuf = _comprAlloc.get();
+    _comprBufSize = (_comprAlloc.size() - paddingAfter) / _unitSize;
 }
 
 
diff --git a/searchlib/src/vespa/searchlib/util/comprbuffer.h b/searchlib/src/vespa/searchlib/util/comprbuffer.h
index 679c0dccdf0..6bddf322c38 100644
--- a/searchlib/src/vespa/searchlib/util/comprbuffer.h
+++ b/searchlib/src/vespa/searchlib/util/comprbuffer.h
@@ -3,18 +3,20 @@
 #pragma once
 
 #include <vespa/searchlib/util/filealign.h>
+#include <vespa/vespalib/util/alloc.h>
 
 namespace search {
 
 class ComprBuffer
 {
 private:
+    using Alloc = vespalib::alloc::Alloc;
     void allocComprBuf();
     const uint32_t   _unitSize; // Size of unit in bytes, doubles up as alignment
     bool             _padBefore;
     void            *_comprBuf;
     size_t           _comprBufSize;
-    void            *_comprBufMalloc;
+    Alloc            _comprAlloc;
     FileAlign        _aligner;
 public:
 
@@ -44,11 +46,9 @@ public:
     }
     const FileAlign & getAligner() const { return _aligner; }
 
-    void * stealComprBuf() {
-        void * stolen = _comprBufMalloc;
-        _comprBufMalloc = nullptr;
+    Alloc stealComprBuf() {
         setComprBuf(nullptr, 0);
-        return stolen;
+        return std::move(_comprAlloc);
     }
 
     /*
diff --git a/searchlib/src/vespa/searchlib/util/comprfile.cpp b/searchlib/src/vespa/searchlib/util/comprfile.cpp
index b0099ad9074..dffcd639a53 100644
--- a/searchlib/src/vespa/searchlib/util/comprfile.cpp
+++ b/searchlib/src/vespa/searchlib/util/comprfile.cpp
@@ -434,13 +434,13 @@ ComprFileWriteContext::writeComprBuffer(bool flushSlack)
     _encodeContext->afterWrite(*this, chunkUsedUnits, 0);
 }
 
-std::pair<void *, size_t>
-ComprFileWriteContext::grabComprBuffer(void *&comprBufMalloc)
+std::pair<uint64_t *, size_t>
+ComprFileWriteContext::grabComprBuffer(vespalib::alloc::Alloc & comprAlloc)
 {
     assert(_file == nullptr);
-    std::pair<void *, size_t> res =
+    std::pair<uint64_t *, size_t> res =
         std::make_pair(getComprBuf(), _encodeContext->getUsedUnits(getComprBuf()));
-    comprBufMalloc = stealComprBuf();
+    comprAlloc = stealComprBuf();
     return res;
 }
 
diff --git a/searchlib/src/vespa/searchlib/util/comprfile.h b/searchlib/src/vespa/searchlib/util/comprfile.h
index a28e14b02ac..b713615bed1 100644
--- a/searchlib/src/vespa/searchlib/util/comprfile.h
+++ b/searchlib/src/vespa/searchlib/util/comprfile.h
@@ -204,7 +204,7 @@ public:
      * Grab compressed buffer from write context.  This is only legal when
      * no file is attached.
      */
-    std::pair<void *, size_t> grabComprBuffer(void *&comprBufMalloc);
+    std::pair<uint64_t *, size_t> grabComprBuffer(vespalib::alloc::Alloc & comprAlloc);
 };
 
 }