From c7d77f0b0f5318e24156fa246b0bc318f137b8ab Mon Sep 17 00:00:00 2001
From: Henning Baldersheim <balder@yahoo-inc.com>
Date: Thu, 8 Feb 2024 16:39:51 +0000
Subject: - Complete dumping of 1 index field before progressing to the next. -
 This prevents allocating memory buffers, and file descriptors for all fields
 concurrently. - It will reduce memory footprint during flush if there are
 many fields.

---
 .../feed_and_search/feed_and_search_test.cpp       | 10 ++++++----
 .../src/tests/proton/index/fusionrunner_test.cpp   | 23 +++++++++++-----------
 .../searchcore/proton/index/memoryindexwrapper.cpp |  5 ++---
 3 files changed, 19 insertions(+), 19 deletions(-)

(limited to 'searchcore')

diff --git a/searchcore/src/tests/proton/feed_and_search/feed_and_search_test.cpp b/searchcore/src/tests/proton/feed_and_search/feed_and_search_test.cpp
index c773abef69f..0936f05b222 100644
--- a/searchcore/src/tests/proton/feed_and_search/feed_and_search_test.cpp
+++ b/searchcore/src/tests/proton/feed_and_search/feed_and_search_test.cpp
@@ -176,12 +176,14 @@ void Test::requireThatMemoryIndexCanBeDumpedAndSearched() {
     const string index_dir = "test_index";
     const uint32_t docIdLimit = memory_index.getDocIdLimit();
     const uint64_t num_words = memory_index.getNumWords();
-    IndexBuilder index_builder(schema, index_dir, docIdLimit);
     search::TuneFileIndexing tuneFileIndexing;
     DummyFileHeaderContext fileHeaderContext;
-    index_builder.open(num_words, MockFieldLengthInspector(), tuneFileIndexing, fileHeaderContext);
-    memory_index.dump(index_builder);
-    index_builder.close();
+    {
+        MockFieldLengthInspector fieldLengthInspector;
+        IndexBuilder index_builder(schema, index_dir, docIdLimit, num_words,
+                                   fieldLengthInspector, tuneFileIndexing, fileHeaderContext);
+        memory_index.dump(index_builder);
+    }
 
     // Fusion test.  Keep all documents to get an "indentical" copy.
     const string index_dir2 = "test_index2";
diff --git a/searchcore/src/tests/proton/index/fusionrunner_test.cpp b/searchcore/src/tests/proton/index/fusionrunner_test.cpp
index ac02c644885..a58d7540d7c 100644
--- a/searchcore/src/tests/proton/index/fusionrunner_test.cpp
+++ b/searchcore/src/tests/proton/index/fusionrunner_test.cpp
@@ -23,7 +23,6 @@
 #include <vespa/vespalib/util/gate.h>
 #include <vespa/vespalib/util/destructor_callbacks.h>
 #include <vespa/vespalib/testkit/testapp.h>
-#include <vespa/vespalib/util/size_literals.h>
 #include <vespa/vespalib/stllike/asciistream.h>
 #include <filesystem>
 #include <set>
@@ -96,7 +95,7 @@ class Test : public vespalib::TestApp {
 
 public:
     Test();
-    ~Test();
+    ~Test() override;
     int Main() override;
 };
 
@@ -145,17 +144,15 @@ getSchema()
 
 void Test::setUp() {
     std::filesystem::remove_all(std::filesystem::path(base_dir));
-    _fusion_runner.reset(new FusionRunner(base_dir, getSchema(),
-                                 TuneFileAttributes(),
-                                 _fileHeaderContext));
+    _fusion_runner = std::make_unique<FusionRunner>(base_dir, getSchema(), TuneFileAttributes(), _fileHeaderContext);
     const string selector_base = base_dir + "/index.flush.0/selector";
-    _selector.reset(new FixedSourceSelector(0, selector_base));
+    _selector = std::make_unique<FixedSourceSelector>(0, selector_base);
     _fusion_spec = FusionSpec();
 }
 
 void Test::tearDown() {
     std::filesystem::remove_all(std::filesystem::path(base_dir));
-    _selector.reset(0);
+    _selector.reset(nullptr);
 }
 
 Document::UP buildDocument(DocBuilder & doc_builder, int id, const string &word) {
@@ -200,12 +197,14 @@ void Test::createIndex(const string &dir, uint32_t id, bool fusion) {
     addDocument(doc_builder, memory_index, *_selector, id, id + 3, "qux");
 
     const uint32_t docIdLimit = std::min(memory_index.getDocIdLimit(), _selector->getDocIdLimit());
-    IndexBuilder index_builder(schema, index_dir, docIdLimit);
-    TuneFileIndexing tuneFileIndexing;
     TuneFileAttributes tuneFileAttributes;
-    index_builder.open(memory_index.getNumWords(), MockFieldLengthInspector(), tuneFileIndexing, _fileHeaderContext);
-    memory_index.dump(index_builder);
-    index_builder.close();
+    {
+        TuneFileIndexing tuneFileIndexing;
+        MockFieldLengthInspector fieldLengthInspector;
+        IndexBuilder index_builder(schema, index_dir, docIdLimit, memory_index.getNumWords(), fieldLengthInspector,
+                                   tuneFileIndexing, _fileHeaderContext);
+        memory_index.dump(index_builder);
+    }
 
     _selector->extractSaveInfo(index_dir + "/selector")->save(tuneFileAttributes, _fileHeaderContext);
 }
diff --git a/searchcore/src/vespa/searchcore/proton/index/memoryindexwrapper.cpp b/searchcore/src/vespa/searchcore/proton/index/memoryindexwrapper.cpp
index fb14534c47c..b4d32cb4376 100644
--- a/searchcore/src/vespa/searchcore/proton/index/memoryindexwrapper.cpp
+++ b/searchcore/src/vespa/searchcore/proton/index/memoryindexwrapper.cpp
@@ -36,11 +36,10 @@ MemoryIndexWrapper::flushToDisk(const vespalib::string &flushDir, uint32_t docId
 {
     const uint64_t numWords = _index.getNumWords();
     _index.freeze(); // TODO(geirst): is this needed anymore?
-    IndexBuilder indexBuilder(_index.getSchema(), flushDir, docIdLimit);
     SerialNumFileHeaderContext fileHeaderContext(_fileHeaderContext, serialNum);
-    indexBuilder.open(numWords, *this, _tuneFileIndexing, fileHeaderContext);
+    IndexBuilder indexBuilder(_index.getSchema(), flushDir, docIdLimit,
+                              numWords, *this, _tuneFileIndexing, fileHeaderContext);
     _index.dump(indexBuilder);
-    indexBuilder.close();
 }
 
 search::SerialNum
-- 
cgit v1.2.3