- Provide an executor and use it for fusion.

- Use per field temporary directories.
- Keep WordNumMapping on the stack to make it thread safe.

4 files changed, 38 insertions, 31 deletions

diff --git a/searchcore/src/tests/proton/feed_and_search/feed_and_search.cpp b/searchcore/src/tests/proton/feed_and_search/feed_and_search.cpp
index b2334ed025e..d0b19d77181 100644
--- a/searchcore/src/tests/proton/feed_and_search/feed_and_search.cpp
+++ b/searchcore/src/tests/proton/feed_and_search/feed_and_search.cpp
@@ -17,6 +17,7 @@ LOG_SETUP("feed_and_search_test");
 #include <vespa/searchlib/queryeval/searchiterator.h>
 #include <vespa/searchlib/queryeval/fake_requestcontext.h>
 #include <vespa/vespalib/testkit/testapp.h>
+#include <vespa/vespalib/util/threadstackexecutor.h>
 #include <sstream>
 #include <vespa/searchlib/diskindex/fusion.h>
 #include <vespa/searchlib/common/documentsummary.h>
@@ -144,6 +145,7 @@ void Test::testSearch(Searchable &source,
 // again.
 void Test::requireThatMemoryIndexCanBeDumpedAndSearched() {
     Schema schema = getSchema();
+    vespalib::ThreadStackExecutor sharedExecutor(2, 0x10000);
     search::SequencedTaskExecutor indexFieldInverter(2);
     search::SequencedTaskExecutor indexFieldWriter(2);
     MemoryIndex memory_index(schema, indexFieldInverter, indexFieldWriter);
@@ -167,8 +169,7 @@ void Test::requireThatMemoryIndexCanBeDumpedAndSearched() {
     const uint64_t num_words = memory_index.getNumWords();
     search::TuneFileIndexing tuneFileIndexing;
     DummyFileHeaderContext fileHeaderContext;
-    index_builder.open(docIdLimit, num_words, tuneFileIndexing,
-                       fileHeaderContext);
+    index_builder.open(docIdLimit, num_words, tuneFileIndexing, fileHeaderContext);
     memory_index.dump(index_builder);
     index_builder.close();
 
@@ -177,17 +178,18 @@ void Test::requireThatMemoryIndexCanBeDumpedAndSearched() {
     std::vector<string> fusionInputs;
     fusionInputs.push_back(index_dir);
     uint32_t fusionDocIdLimit = 0;
-    typedef search::diskindex::Fusion FastS_Fusion;
+    using Fusion = search::diskindex::Fusion;
     bool fret1 = DocumentSummary::readDocIdLimit(index_dir, fusionDocIdLimit);
     ASSERT_TRUE(fret1);
     SelectorArray selector(fusionDocIdLimit, 0);
-    bool fret2 = FastS_Fusion::merge(schema,
-                                    index_dir2,
-                                    fusionInputs,
-                                    selector,
-                                    false /* dynamicKPosOccFormat */,
-                                     tuneFileIndexing,
-                                     fileHeaderContext);
+    bool fret2 = Fusion::merge(schema,
+                              index_dir2,
+                              fusionInputs,
+                              selector,
+                              false /* dynamicKPosOccFormat */,
+                              tuneFileIndexing,
+                              fileHeaderContext,
+                              sharedExecutor);
     ASSERT_TRUE(fret2);
 
     // Fusion test with all docs removed in output (doesn't affect word list)
@@ -198,13 +200,14 @@ void Test::requireThatMemoryIndexCanBeDumpedAndSearched() {
     bool fret3 = DocumentSummary::readDocIdLimit(index_dir, fusionDocIdLimit);
     ASSERT_TRUE(fret3);
     SelectorArray selector2(fusionDocIdLimit, 1);
-    bool fret4 = FastS_Fusion::merge(schema,
-                                    index_dir3,
-                                    fusionInputs,
-                                    selector2,
-                                    false /* dynamicKPosOccFormat */,
-                                     tuneFileIndexing,
-                                     fileHeaderContext);
+    bool fret4 = Fusion::merge(schema,
+                              index_dir3,
+                              fusionInputs,
+                              selector2,
+                              false /* dynamicKPosOccFormat */,
+                              tuneFileIndexing,
+                              fileHeaderContext,
+                              sharedExecutor);
     ASSERT_TRUE(fret4);
 
     // Fusion test with all docs removed in input (affects word list)
@@ -215,13 +218,14 @@ void Test::requireThatMemoryIndexCanBeDumpedAndSearched() {
     bool fret5 = DocumentSummary::readDocIdLimit(index_dir3, fusionDocIdLimit);
     ASSERT_TRUE(fret5);
     SelectorArray selector3(fusionDocIdLimit, 0);
-    bool fret6 = FastS_Fusion::merge(schema,
-                                    index_dir4,
-                                    fusionInputs,
-                                    selector3,
-                                    false /* dynamicKPosOccFormat */,
-                                     tuneFileIndexing,
-                                     fileHeaderContext);
+    bool fret6 = Fusion::merge(schema,
+                              index_dir4,
+                              fusionInputs,
+                              selector3,
+                              false /* dynamicKPosOccFormat */,
+                              tuneFileIndexing,
+                              fileHeaderContext,
+                              sharedExecutor);
     ASSERT_TRUE(fret6);
 
     DiskIndex disk_index(index_dir);
diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.cpp b/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.cpp
index 1abe9540859..113855d7e2b 100644
--- a/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.cpp
+++ b/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.cpp
@@ -1,9 +1,9 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
+#include "summarymanager.h"
 #include "documentstoreadapter.h"
 #include "summarycompacttarget.h"
 #include "summaryflushtarget.h"
-#include "summarymanager.h"
 #include <vespa/config/print/ostreamconfigwriter.h>
 #include <vespa/document/repo/documenttyperepo.h>
 #include <vespa/juniper/rpinterface.h>
@@ -12,6 +12,8 @@
 #include <vespa/vespalib/util/lambdatask.h>
 #include <vespa/searchsummary/docsummary/docsumconfig.h>
 #include <vespa/vespalib/util/exceptions.h>
+#include <vespa/fastlib/text/normwordfolder.h>
+
 #include <sstream>
 
 #include <vespa/log/log.h>
@@ -84,7 +86,7 @@ SummarySetup(const vespalib::string & baseDir, const DocTypeName & docTypeName,
              const search::IAttributeManager::SP &attributeMgr, const search::IDocumentStore::SP & docStore,
              const std::shared_ptr<const DocumentTypeRepo> &repo)
     : _docsumWriter(),
-      _wordFolder(),
+      _wordFolder(std::make_unique<Fast_NormalizeWordFolder>()),
       _juniperProps(juniperCfg),
       _juniperConfig(),
       _attributeMgr(attributeMgr),
@@ -103,7 +105,7 @@ SummarySetup(const vespalib::string & baseDir, const DocTypeName & docTypeName,
                          baseDir.c_str(), oss.str().c_str()));
     }
 
-    _juniperConfig = std::make_unique<juniper::Juniper>(&_juniperProps, &_wordFolder);
+    _juniperConfig = std::make_unique<juniper::Juniper>(&_juniperProps, _wordFolder.get());
     _docsumWriter = std::make_unique<DynamicDocsumWriter>(resultConfig.release(), nullptr);
     DynamicDocsumConfig dynCfg(this, _docsumWriter.get());
     dynCfg.configure(summarymapCfg);
diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.h b/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.h
index 8df65e3e2ff..4fde28e69de 100644
--- a/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.h
+++ b/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.h
@@ -10,13 +10,14 @@
 #include <vespa/searchlib/docstore/logdocumentstore.h>
 #include <vespa/searchlib/transactionlog/syncproxy.h>
 #include <vespa/document/fieldvalue/document.h>
-#include <vespa/vespalib/util/threadstackexecutor.h>
-#include <vespa/fastlib/text/normwordfolder.h>
+#include <vespa/vespalib/util/threadexecutor.h>
 
 namespace searchcorespi::index { struct IThreadService; }
 namespace search { class IBucketizer; }
 namespace search::common { class FileHeaderContext; }
 
+class Fast_NormalizeWordFolder;
+
 namespace proton {
 
 class SummaryManager : public ISummaryManager
@@ -25,7 +26,7 @@ public:
     class SummarySetup : public ISummarySetup {
     private:
         std::unique_ptr<search::docsummary::DynamicDocsumWriter> _docsumWriter;
-        Fast_NormalizeWordFolder              _wordFolder;
+        std::unique_ptr<Fast_NormalizeWordFolder>                _wordFolder;
         search::docsummary::JuniperProperties _juniperProps;
         std::unique_ptr<juniper::Juniper>     _juniperConfig;
         search::IAttributeManager::SP         _attributeMgr;
diff --git a/searchcore/src/vespa/searchcore/proton/index/indexmanager.cpp b/searchcore/src/vespa/searchcore/proton/index/indexmanager.cpp
index 2139e34818f..b74cc2c603f 100644
--- a/searchcore/src/vespa/searchcore/proton/index/indexmanager.cpp
+++ b/searchcore/src/vespa/searchcore/proton/index/indexmanager.cpp
@@ -66,7 +66,7 @@ IndexManager::MaintainerOperations::runFusion(const Schema &schema,
     SerialNumFileHeaderContext fileHeaderContext(_fileHeaderContext, serialNum);
     const bool dynamic_k_doc_pos_occ_format = false;
     return Fusion::merge(schema, outputDir, sources, selectorArray, dynamic_k_doc_pos_occ_format,
-                         _tuneFileIndexing, fileHeaderContext);
+                         _tuneFileIndexing, fileHeaderContext, _threadingService.shared());
 }