GC unused and non computed return value.

Refactor to prepare for streaming read.

4 files changed, 53 insertions, 46 deletions

diff --git a/searchlib/src/vespa/searchlib/docstore/filechunk.cpp b/searchlib/src/vespa/searchlib/docstore/filechunk.cpp
index 1881be4a0bb..a8b84fbeac4 100644
--- a/searchlib/src/vespa/searchlib/docstore/filechunk.cpp
+++ b/searchlib/src/vespa/searchlib/docstore/filechunk.cpp
@@ -12,8 +12,8 @@
 #include <vespa/vespalib/stllike/asciistream.h>
 #include <vespa/vespalib/objects/nbostream.h>
 #include <vespa/vespalib/util/executor.h>
+#include <vespa/vespalib/util/small_vector.h>
 #include <vespa/vespalib/util/arrayqueue.hpp>
-#include <vespa/vespalib/util/array.hpp>
 #include <vespa/fastos/file.h>
 #include <filesystem>
 #include <future>
@@ -117,25 +117,16 @@ FileChunk::addNumBuckets(size_t numBucketsInChunk)
     }
 }
 
-class TmpChunkMeta : public ChunkMeta,
-                     public std::vector<LidMeta>
-{
-public:
-    void fill(vespalib::nbostream & is) {
-        resize(getNumEntries());
-        for (LidMeta & lm : *this) {
-            lm.deserialize(is);
-        }
+void
+FileChunk::TmpChunkMeta::fill(vespalib::nbostream & is) {
+    resize(getNumEntries());
+    for (LidMeta & lm : *this) {
+        lm.deserialize(is);
     }
-};
-
-using TmpChunkMetaV = std::vector<TmpChunkMeta, vespalib::allocator_large<TmpChunkMeta>>;
-static_assert(sizeof(TmpChunkMeta) == 48);
-
-namespace {
+}
 
 void
-verifyOrAssert(const TmpChunkMetaV & v)
+FileChunk::verifyOrAssert(const TmpChunkMetaV & v)
 {
     for (auto prev(v.begin()), it(prev); it != v.end(); ++it) {
         assert(prev->getLastSerial() <= it->getLastSerial());
@@ -143,8 +134,6 @@ verifyOrAssert(const TmpChunkMetaV & v)
     }
 }
 
-}
-
 void
 FileChunk::erase()
 {
@@ -153,10 +142,9 @@ FileChunk::erase()
     std::filesystem::remove(std::filesystem::path(_dataFileName));
 }
 
-size_t
+void
 FileChunk::updateLidMap(const unique_lock &guard, ISetLid &ds, uint64_t serialNum, uint32_t docIdLimit)
 {
-    size_t sz(0);
     assert(_chunkInfo.empty());
 
     FastOS_File idxFile(_idxFileName.c_str());
@@ -212,25 +200,7 @@ FileChunk::updateLidMap(const unique_lock &guard, ISetLid &ds, uint64_t serialNu
                 vespalib::GenerationHandler::Guard bucketizerGuard = globalBucketMap.getGuard();
                 for (const TmpChunkMeta & chunkMeta : tempVector) {
                     assert(serialNum <= chunkMeta.getLastSerial());
-                    BucketDensityComputer bucketMap(_bucketizer);
-                    for (size_t i(0), m(chunkMeta.getNumEntries()); i < m; i++) {
-                        const LidMeta & lidMeta(chunkMeta[i]);
-                        if (lidMeta.getLid() < docIdLimit) {
-                            if (_bucketizer && (lidMeta.size() > 0)) {
-                                document::BucketId bucketId = _bucketizer->getBucketOf(bucketizerGuard, lidMeta.getLid());
-                                bucketMap.recordLid(bucketId);
-                                globalBucketMap.recordLid(bucketId);
-                            }
-                            ds.setLid(guard, lidMeta.getLid(), LidInfo(getFileId().getId(), _chunkInfo.size(), lidMeta.size()));
-                            _numLids++;
-                        } else {
-                            remove(lidMeta.getLid(), lidMeta.size());
-                        }
-                        _addedBytes += adjustSize(lidMeta.size());
-                    }
-                    serialNum = chunkMeta.getLastSerial();
-                    addNumBuckets(bucketMap.getNumBuckets());
-                    _chunkInfo.emplace_back(chunkMeta.getOffset(), chunkMeta.getSize(), chunkMeta.getLastSerial());
+                    serialNum = handleChunk(guard, ds, docIdLimit, bucketizerGuard, globalBucketMap, chunkMeta);
                     assert(serialNum >= _lastPersistedSerialNum.load(std::memory_order_relaxed));
                     _lastPersistedSerialNum.store(serialNum, std::memory_order_relaxed);
                 }
@@ -242,9 +212,35 @@ FileChunk::updateLidMap(const unique_lock &guard, ISetLid &ds, uint64_t serialNu
     } else {
         LOG_ABORT("should not reach here");
     }
-    return sz;
 }
 
+uint64_t
+FileChunk::handleChunk(const unique_lock &guard, ISetLid &ds, uint32_t docIdLimit,
+                       const vespalib::GenerationHandler::Guard & bucketizerGuard, BucketDensityComputer &globalBucketMap,
+                       const TmpChunkMeta & chunkMeta) {
+    BucketDensityComputer bucketMap(_bucketizer);
+    for (size_t i(0), m(chunkMeta.getNumEntries()); i < m; i++) {
+        const LidMeta & lidMeta(chunkMeta[i]);
+        if (lidMeta.getLid() < docIdLimit) {
+            if (_bucketizer && (lidMeta.size() > 0)) {
+                document::BucketId bucketId = _bucketizer->getBucketOf(bucketizerGuard, lidMeta.getLid());
+                bucketMap.recordLid(bucketId);
+                globalBucketMap.recordLid(bucketId);
+            }
+            ds.setLid(guard, lidMeta.getLid(), LidInfo(getFileId().getId(), _chunkInfo.size(), lidMeta.size()));
+            _numLids++;
+        } else {
+            remove(lidMeta.getLid(), lidMeta.size());
+        }
+        _addedBytes += adjustSize(lidMeta.size());
+    }
+    uint64_t serialNum = chunkMeta.getLastSerial();
+    addNumBuckets(bucketMap.getNumBuckets());
+    _chunkInfo.emplace_back(chunkMeta.getOffset(), chunkMeta.getSize(), chunkMeta.getLastSerial());
+    return serialNum;
+}
+
+
 void
 FileChunk::enableRead()
 {
diff --git a/searchlib/src/vespa/searchlib/docstore/filechunk.h b/searchlib/src/vespa/searchlib/docstore/filechunk.h
index 3664da3dfd9..b87dd819ac9 100644
--- a/searchlib/src/vespa/searchlib/docstore/filechunk.h
+++ b/searchlib/src/vespa/searchlib/docstore/filechunk.h
@@ -109,7 +109,7 @@ public:
               const IBucketizer *bucketizer);
     virtual ~FileChunk();
 
-    virtual size_t updateLidMap(const unique_lock &guard, ISetLid &lidMap, uint64_t serialNum, uint32_t docIdLimit);
+    virtual void updateLidMap(const unique_lock &guard, ISetLid &lidMap, uint64_t serialNum, uint32_t docIdLimit);
     virtual ssize_t read(uint32_t lid, SubChunkId chunk, vespalib::DataBuffer & buffer) const;
     virtual void read(LidInfoWithLidV::const_iterator begin, size_t count, IBufferVisitor & visitor) const;
     void remove(uint32_t lid, uint32_t size);
@@ -199,6 +199,18 @@ public:
     static vespalib::string createIdxFileName(const vespalib::string & name);
     static vespalib::string createDatFileName(const vespalib::string & name);
 private:
+    class TmpChunkMeta : public ChunkMeta,
+                         public std::vector<LidMeta>
+    {
+    public:
+        void fill(vespalib::nbostream & is);
+    };
+    using TmpChunkMetaV = std::vector<TmpChunkMeta, vespalib::allocator_large<TmpChunkMeta>>;
+    using BucketizerGuard = vespalib::GenerationHandler::Guard;
+    static void verifyOrAssert(const TmpChunkMetaV & v);
+    uint64_t handleChunk(const unique_lock &guard, ISetLid &lidMap, uint32_t docIdLimit,
+                         const BucketizerGuard & bucketizerGuard, BucketDensityComputer & global,
+                         const TmpChunkMeta & chunkMeta);
     using File = std::unique_ptr<FileRandRead>;
     const FileId           _fileId;
     const NameId           _nameId;
diff --git a/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.cpp b/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.cpp
index 7102b80d7d0..973287fc7bd 100644
--- a/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.cpp
+++ b/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.cpp
@@ -174,16 +174,15 @@ WriteableFileChunk::~WriteableFileChunk()
     }
 }
 
-size_t
+void
 WriteableFileChunk::updateLidMap(const unique_lock &guard, ISetLid &ds, uint64_t serialNum, uint32_t docIdLimit)
 {
-    size_t sz = FileChunk::updateLidMap(guard, ds, serialNum, docIdLimit);
+    FileChunk::updateLidMap(guard, ds, serialNum, docIdLimit);
     _nextChunkId = _chunkInfo.size();
     _active = std::make_unique<Chunk>(_nextChunkId++, Chunk::Config(_config.getMaxChunkBytes()));
     _serialNum = getLastPersistedSerialNum();
     _firstChunkIdToBeWritten = _active->getId();
     setDiskFootprint(0);
-    return sz;
 }
 
 void
diff --git a/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.h b/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.h
index b5a52dc83f7..028915d28e0 100644
--- a/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.h
+++ b/searchlib/src/vespa/searchlib/docstore/writeablefilechunk.h
@@ -64,7 +64,7 @@ public:
     size_t getMemoryFootprint() const override;
     size_t getMemoryMetaFootprint() const override;
     vespalib::MemoryUsage getMemoryUsage() const override;
-    size_t updateLidMap(const unique_lock &guard, ISetLid &lidMap, uint64_t serialNum, uint32_t docIdLimit) override;
+    void updateLidMap(const unique_lock &guard, ISetLid &lidMap, uint64_t serialNum, uint32_t docIdLimit) override;
     void waitForDiskToCatchUpToNow() const;
     void flushPendingChunks(uint64_t serialNum);
     DataStoreFileChunkStats getStats() const override;