diff options
author | Henning Baldersheim <balder@yahoo-inc.com> | 2019-05-16 13:32:27 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2019-05-16 13:32:27 +0200 |
commit | fea2b9e8e00789feae3757fe7321b3b73dd5d257 (patch) | |
tree | 8b03d7cf0415c76fbc57bb676744703bbf76a7f5 /searchcore/src | |
parent | 05b5605b8bf551994ccd56b6a3070c496962fe2d (diff) | |
parent | c7ed78848c064c81a8b1e778017f889e5e1aefc8 (diff) |
Merge pull request #9425 from vespa-engine/balder/multiple-threads-for-fusion
- Provide an executor and use it for fusion.
Diffstat (limited to 'searchcore/src')
4 files changed, 38 insertions, 31 deletions
diff --git a/searchcore/src/tests/proton/feed_and_search/feed_and_search.cpp b/searchcore/src/tests/proton/feed_and_search/feed_and_search.cpp index b2334ed025e..d0b19d77181 100644 --- a/searchcore/src/tests/proton/feed_and_search/feed_and_search.cpp +++ b/searchcore/src/tests/proton/feed_and_search/feed_and_search.cpp @@ -17,6 +17,7 @@ LOG_SETUP("feed_and_search_test"); #include <vespa/searchlib/queryeval/searchiterator.h> #include <vespa/searchlib/queryeval/fake_requestcontext.h> #include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/util/threadstackexecutor.h> #include <sstream> #include <vespa/searchlib/diskindex/fusion.h> #include <vespa/searchlib/common/documentsummary.h> @@ -144,6 +145,7 @@ void Test::testSearch(Searchable &source, // again. void Test::requireThatMemoryIndexCanBeDumpedAndSearched() { Schema schema = getSchema(); + vespalib::ThreadStackExecutor sharedExecutor(2, 0x10000); search::SequencedTaskExecutor indexFieldInverter(2); search::SequencedTaskExecutor indexFieldWriter(2); MemoryIndex memory_index(schema, indexFieldInverter, indexFieldWriter); @@ -167,8 +169,7 @@ void Test::requireThatMemoryIndexCanBeDumpedAndSearched() { const uint64_t num_words = memory_index.getNumWords(); search::TuneFileIndexing tuneFileIndexing; DummyFileHeaderContext fileHeaderContext; - index_builder.open(docIdLimit, num_words, tuneFileIndexing, - fileHeaderContext); + index_builder.open(docIdLimit, num_words, tuneFileIndexing, fileHeaderContext); memory_index.dump(index_builder); index_builder.close(); @@ -177,17 +178,18 @@ void Test::requireThatMemoryIndexCanBeDumpedAndSearched() { std::vector<string> fusionInputs; fusionInputs.push_back(index_dir); uint32_t fusionDocIdLimit = 0; - typedef search::diskindex::Fusion FastS_Fusion; + using Fusion = search::diskindex::Fusion; bool fret1 = DocumentSummary::readDocIdLimit(index_dir, fusionDocIdLimit); ASSERT_TRUE(fret1); SelectorArray selector(fusionDocIdLimit, 0); - bool fret2 = FastS_Fusion::merge(schema, - index_dir2, - fusionInputs, - selector, - false /* dynamicKPosOccFormat */, - tuneFileIndexing, - fileHeaderContext); + bool fret2 = Fusion::merge(schema, + index_dir2, + fusionInputs, + selector, + false /* dynamicKPosOccFormat */, + tuneFileIndexing, + fileHeaderContext, + sharedExecutor); ASSERT_TRUE(fret2); // Fusion test with all docs removed in output (doesn't affect word list) @@ -198,13 +200,14 @@ void Test::requireThatMemoryIndexCanBeDumpedAndSearched() { bool fret3 = DocumentSummary::readDocIdLimit(index_dir, fusionDocIdLimit); ASSERT_TRUE(fret3); SelectorArray selector2(fusionDocIdLimit, 1); - bool fret4 = FastS_Fusion::merge(schema, - index_dir3, - fusionInputs, - selector2, - false /* dynamicKPosOccFormat */, - tuneFileIndexing, - fileHeaderContext); + bool fret4 = Fusion::merge(schema, + index_dir3, + fusionInputs, + selector2, + false /* dynamicKPosOccFormat */, + tuneFileIndexing, + fileHeaderContext, + sharedExecutor); ASSERT_TRUE(fret4); // Fusion test with all docs removed in input (affects word list) @@ -215,13 +218,14 @@ void Test::requireThatMemoryIndexCanBeDumpedAndSearched() { bool fret5 = DocumentSummary::readDocIdLimit(index_dir3, fusionDocIdLimit); ASSERT_TRUE(fret5); SelectorArray selector3(fusionDocIdLimit, 0); - bool fret6 = FastS_Fusion::merge(schema, - index_dir4, - fusionInputs, - selector3, - false /* dynamicKPosOccFormat */, - tuneFileIndexing, - fileHeaderContext); + bool fret6 = Fusion::merge(schema, + index_dir4, + fusionInputs, + selector3, + false /* dynamicKPosOccFormat */, + tuneFileIndexing, + fileHeaderContext, + sharedExecutor); ASSERT_TRUE(fret6); DiskIndex disk_index(index_dir); diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.cpp b/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.cpp index 1abe9540859..113855d7e2b 100644 --- a/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.cpp @@ -1,9 +1,9 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "summarymanager.h" #include "documentstoreadapter.h" #include "summarycompacttarget.h" #include "summaryflushtarget.h" -#include "summarymanager.h" #include <vespa/config/print/ostreamconfigwriter.h> #include <vespa/document/repo/documenttyperepo.h> #include <vespa/juniper/rpinterface.h> @@ -12,6 +12,8 @@ #include <vespa/vespalib/util/lambdatask.h> #include <vespa/searchsummary/docsummary/docsumconfig.h> #include <vespa/vespalib/util/exceptions.h> +#include <vespa/fastlib/text/normwordfolder.h> + #include <sstream> #include <vespa/log/log.h> @@ -84,7 +86,7 @@ SummarySetup(const vespalib::string & baseDir, const DocTypeName & docTypeName, const search::IAttributeManager::SP &attributeMgr, const search::IDocumentStore::SP & docStore, const std::shared_ptr<const DocumentTypeRepo> &repo) : _docsumWriter(), - _wordFolder(), + _wordFolder(std::make_unique<Fast_NormalizeWordFolder>()), _juniperProps(juniperCfg), _juniperConfig(), _attributeMgr(attributeMgr), @@ -103,7 +105,7 @@ SummarySetup(const vespalib::string & baseDir, const DocTypeName & docTypeName, baseDir.c_str(), oss.str().c_str())); } - _juniperConfig = std::make_unique<juniper::Juniper>(&_juniperProps, &_wordFolder); + _juniperConfig = std::make_unique<juniper::Juniper>(&_juniperProps, _wordFolder.get()); _docsumWriter = std::make_unique<DynamicDocsumWriter>(resultConfig.release(), nullptr); DynamicDocsumConfig dynCfg(this, _docsumWriter.get()); dynCfg.configure(summarymapCfg); diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.h b/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.h index 8df65e3e2ff..4fde28e69de 100644 --- a/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.h +++ b/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.h @@ -10,13 +10,14 @@ #include <vespa/searchlib/docstore/logdocumentstore.h> #include <vespa/searchlib/transactionlog/syncproxy.h> #include <vespa/document/fieldvalue/document.h> -#include <vespa/vespalib/util/threadstackexecutor.h> -#include <vespa/fastlib/text/normwordfolder.h> +#include <vespa/vespalib/util/threadexecutor.h> namespace searchcorespi::index { struct IThreadService; } namespace search { class IBucketizer; } namespace search::common { class FileHeaderContext; } +class Fast_NormalizeWordFolder; + namespace proton { class SummaryManager : public ISummaryManager @@ -25,7 +26,7 @@ public: class SummarySetup : public ISummarySetup { private: std::unique_ptr<search::docsummary::DynamicDocsumWriter> _docsumWriter; - Fast_NormalizeWordFolder _wordFolder; + std::unique_ptr<Fast_NormalizeWordFolder> _wordFolder; search::docsummary::JuniperProperties _juniperProps; std::unique_ptr<juniper::Juniper> _juniperConfig; search::IAttributeManager::SP _attributeMgr; diff --git a/searchcore/src/vespa/searchcore/proton/index/indexmanager.cpp b/searchcore/src/vespa/searchcore/proton/index/indexmanager.cpp index 2139e34818f..b74cc2c603f 100644 --- a/searchcore/src/vespa/searchcore/proton/index/indexmanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/index/indexmanager.cpp @@ -66,7 +66,7 @@ IndexManager::MaintainerOperations::runFusion(const Schema &schema, SerialNumFileHeaderContext fileHeaderContext(_fileHeaderContext, serialNum); const bool dynamic_k_doc_pos_occ_format = false; return Fusion::merge(schema, outputDir, sources, selectorArray, dynamic_k_doc_pos_occ_format, - _tuneFileIndexing, fileHeaderContext); + _tuneFileIndexing, fileHeaderContext, _threadingService.shared()); } |