From e908a53c916842018f91c769d3979ea516bf1c64 Mon Sep 17 00:00:00 2001
From: Henning Baldersheim <balder@yahoo-inc.com>
Date: Thu, 17 Dec 2020 18:21:47 +0000
Subject: Reduce amount of memory used by not keeping the _packets around
 anymore. Visiting is now always done from file and _skipList.

---
 .../vespa/searchlib/transactionlog/domainpart.cpp  | 34 ++++------------------
 .../vespa/searchlib/transactionlog/domainpart.h    | 23 +++++++--------
 2 files changed, 16 insertions(+), 41 deletions(-)

(limited to 'searchlib')

diff --git a/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp b/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp
index b2db271062d..9f7f39b7d5c 100644
--- a/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp
+++ b/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp
@@ -236,11 +236,8 @@ DomainPart::buildPacketMapping(bool allowTruncate)
                 _range.from(firstSerial);
             }
             _range.to(packet.range().to());
-            _packets.insert(std::make_pair(firstSerial, std::move(packet)));
-            {
-                std::lock_guard guard(_lock);
-                _skipList.push_back(SkipInfo(firstSerial, firstPos));
-            }
+            // Called only from constructor so no need to hold lock
+            _skipList.emplace_back(firstSerial, firstPos);
         } else {
             fSize = transLog.GetSize();
         }
@@ -259,7 +256,6 @@ DomainPart::DomainPart(const string & name, const string & baseDir, SerialNum s,
       _range(s),
       _sz(0),
       _byteSize(0),
-      _packets(),
       _fileName(fmt("%s/%s-%016" PRIu64, baseDir.c_str(), name.c_str(), s)),
       _transLog(std::make_unique<FastOS_File>(_fileName.c_str())),
       _skipList(),
@@ -345,10 +341,6 @@ DomainPart::close()
         throw runtime_error(fmt("Failed closing file '%s' of size %" PRId64 ".",
                                 _transLog->GetFileName(), _transLog->GetSize()));
     }
-    {
-        std::lock_guard guard(_lock);
-        _packets.clear();
-    }
     return retval;
 }
 
@@ -364,11 +356,9 @@ DomainPart::openAndFind(FastOS_FileInterface &file, const SerialNum &from)
     if (retval) {
         int64_t pos(_headerLen);
         std::lock_guard guard(_lock);
-        for(SkipList::const_iterator it(_skipList.begin()), mt(_skipList.end());
-            (it < mt) && (it->id() <= from);
-            it++)
-        {
-            pos = it->filePos();
+        for (const auto & skipInfo : _skipList) {
+            if (skipInfo.id() > from) break;
+            pos = skipInfo.filePos();
         }
         retval = file.SetPosition(pos);
     }
@@ -419,20 +409,8 @@ DomainPart::commit(SerialNum firstSerial, const Packet &packet)
     if ( ! chunk->getEntries().empty()) {
         write(*_transLog, *chunk);
     }
-
-    bool merged(false);
     std::lock_guard guard(_lock);
-    if ( ! _packets.empty() ) {
-        Packet & lastPacket = _packets.rbegin()->second;
-        if (lastPacket.sizeBytes() < 0xf000) {
-            lastPacket.merge(packet);
-            merged = true;
-        }
-    }
-    if (! merged ) {
-        _packets.insert(std::make_pair(firstSerial, std::move(packet)));
-        _skipList.push_back(SkipInfo(firstSerial, firstPos));
-    }
+    _skipList.emplace_back(firstSerial, firstPos);
 }
 
 void
diff --git a/searchlib/src/vespa/searchlib/transactionlog/domainpart.h b/searchlib/src/vespa/searchlib/transactionlog/domainpart.h
index a4285dc509c..f5a10cafe3e 100644
--- a/searchlib/src/vespa/searchlib/transactionlog/domainpart.h
+++ b/searchlib/src/vespa/searchlib/transactionlog/domainpart.h
@@ -55,22 +55,20 @@ private:
     class SkipInfo
     {
     public:
-        SkipInfo(SerialNum s, uint64_t p) : _id(s), _pos(p) {}
+        SkipInfo(SerialNum s, uint64_t p) noexcept : _id(s), _pos(p) {}
 
-        bool operator ==(const SkipInfo &b) const { return cmp(b) == 0; }
-        bool operator  <(const SkipInfo &b) const { return cmp(b) < 0; }
-        bool operator  >(const SkipInfo &b) const { return cmp(b) > 0; }
-        bool operator <=(const SkipInfo &b) const { return cmp(b) <= 0; }
-        bool operator >=(const SkipInfo &b) const { return cmp(b) >= 0; }
-        int64_t   filePos() const { return _pos; }
-        SerialNum      id() const { return _id; }
+        bool operator ==(const SkipInfo &b) const noexcept { return cmp(b) == 0; }
+        bool operator  <(const SkipInfo &b) const noexcept { return cmp(b) < 0; }
+        bool operator  >(const SkipInfo &b) const noexcept { return cmp(b) > 0; }
+        bool operator <=(const SkipInfo &b) const noexcept { return cmp(b) <= 0; }
+        bool operator >=(const SkipInfo &b) const noexcept { return cmp(b) >= 0; }
+        int64_t   filePos() const noexcept { return _pos; }
+        SerialNum      id() const noexcept { return _id; }
     private:
-        int64_t cmp(const SkipInfo & b) const { return _id - b._id; }
+        int64_t cmp(const SkipInfo & b) const noexcept { return _id - b._id; }
         SerialNum _id;
         uint64_t  _pos;
     };
-    typedef std::vector<SkipInfo> SkipList;
-    typedef std::map<SerialNum, Packet> PacketList;
     const Encoding        _encoding;
     const uint8_t         _compressionLevel;
     std::mutex            _lock;
@@ -78,10 +76,9 @@ private:
     SerialNumRange        _range;
     size_t                _sz;
     std::atomic<uint64_t> _byteSize;
-    PacketList            _packets;
     vespalib::string      _fileName;
     std::unique_ptr<FastOS_FileInterface> _transLog;
-    SkipList              _skipList;
+    std::vector<SkipInfo> _skipList;
     uint32_t              _headerLen;
     mutable std::mutex    _writeLock;
     // Protected by _writeLock
-- 
cgit v1.2.3