diff options
15 files changed, 58 insertions, 46 deletions
diff --git a/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp b/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp index a50a112aecc..a89f54f7800 100644 --- a/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp +++ b/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp @@ -169,8 +169,8 @@ void Fixture::initViewSet(ViewSet &views) { Matchers::SP matchers(new Matchers(_clock, _queryLimiter, _constantValueRepo)); - IndexManager::SP indexMgr(new IndexManager(BASE_DIR, - 0.0, 2, 0, Schema(), Schema(), views._reconfigurer, + IndexManager::SP indexMgr(new IndexManager(BASE_DIR, searchcorespi::index::WarmupConfig(), + 2, 0, Schema(), Schema(), views._reconfigurer, views._writeService, _summaryExecutor, TuneFileIndexManager(), TuneFileAttributes(), views._fileHeaderContext)); AttributeManager::SP attrMgr(new AttributeManager(BASE_DIR, diff --git a/searchcore/src/tests/proton/index/indexcollection_test.cpp b/searchcore/src/tests/proton/index/indexcollection_test.cpp index f27b9c86260..93f336b4932 100644 --- a/searchcore/src/tests/proton/index/indexcollection_test.cpp +++ b/searchcore/src/tests/proton/index/indexcollection_test.cpp @@ -53,7 +53,7 @@ public: IndexCollection::UP Test::createWarmup(const IndexCollection::SP & prev, const IndexCollection::SP & next) { - return IndexCollection::UP(new WarmupIndexCollection(1.0, prev, next, *_warmup, _executor, *this)); + return IndexCollection::UP(new WarmupIndexCollection(1.0, prev, next, *_warmup, _executor, *this, false)); } int diff --git a/searchcore/src/tests/proton/index/indexmanager_test.cpp b/searchcore/src/tests/proton/index/indexmanager_test.cpp index 95f5a3b50ce..97558fa262c 100644 --- a/searchcore/src/tests/proton/index/indexmanager_test.cpp +++ b/searchcore/src/tests/proton/index/indexmanager_test.cpp @@ -169,7 +169,7 @@ Document::UP Fixture::addDocument(uint32_t id) { void Fixture::resetIndexManager() { _index_manager.reset(0); _index_manager.reset( - new IndexManager(index_dir, 0.0, 2, 0, getSchema(), getSchema(), + new IndexManager(index_dir, searchcorespi::index::WarmupConfig(), 2, 0, getSchema(), getSchema(), _reconfigurer, _writeService, _writeService.getMasterExecutor(), TuneFileIndexManager(), TuneFileAttributes(), _fileHeaderContext)); diff --git a/searchcore/src/vespa/searchcore/config/proton.def b/searchcore/src/vespa/searchcore/config/proton.def index bd89c7ec460..5419eb39be3 100644 --- a/searchcore/src/vespa/searchcore/config/proton.def +++ b/searchcore/src/vespa/searchcore/config/proton.def @@ -129,6 +129,7 @@ indexing.tasklimit int default=1000 restart ## How long a freshly loaded index shall be warmed up ## before being used for serving index.warmup.time double default=0.0 restart +index.warmup.unpack bool default=false restart ## How many flushed indexes there can be befor fusion is forced. ## Setting to 1 will force an immediate fusion. diff --git a/searchcore/src/vespa/searchcore/proton/index/index_manager_initializer.cpp b/searchcore/src/vespa/searchcore/proton/index/index_manager_initializer.cpp index a183a77dafe..706d30dddb2 100644 --- a/searchcore/src/vespa/searchcore/proton/index/index_manager_initializer.cpp +++ b/searchcore/src/vespa/searchcore/proton/index/index_manager_initializer.cpp @@ -11,24 +11,20 @@ namespace proton IndexManagerInitializer:: IndexManagerInitializer(const vespalib::string &baseDir, - double diskIndexWarmupTime, + const searchcorespi::index::WarmupConfig & warmup, size_t maxFlushed, size_t cacheSize, const search::index::Schema &schema, const search::index::Schema &fusionSchema, - searchcorespi::IIndexManager::Reconfigurer & - reconfigurer, - searchcorespi::index::IThreadingService & - threadingService, + searchcorespi::IIndexManager::Reconfigurer & reconfigurer, + searchcorespi::index::IThreadingService & threadingService, vespalib::ThreadExecutor & warmupExecutor, - const search::TuneFileIndexManager & - tuneFileIndexManager, + const search::TuneFileIndexManager & tuneFileIndexManager, const search::TuneFileAttributes &tuneFileAttributes, - const search::common::FileHeaderContext & - fileHeaderContext, + const search::common::FileHeaderContext & fileHeaderContext, std::shared_ptr<searchcorespi::IIndexManager::SP> indexManager) : _baseDir(baseDir), - _diskIndexWarmupTime(diskIndexWarmupTime), + _warmup(warmup), _maxFlushed(maxFlushed), _cacheSize(cacheSize), _schema(schema), @@ -52,7 +48,7 @@ IndexManagerInitializer::run() vespalib::mkdir(_baseDir, false); *_indexManager = std::make_shared<proton::IndexManager> (_baseDir, - _diskIndexWarmupTime, + _warmup, _maxFlushed, _cacheSize, _schema, diff --git a/searchcore/src/vespa/searchcore/proton/index/index_manager_initializer.h b/searchcore/src/vespa/searchcore/proton/index/index_manager_initializer.h index 45629860229..72967914388 100644 --- a/searchcore/src/vespa/searchcore/proton/index/index_manager_initializer.h +++ b/searchcore/src/vespa/searchcore/proton/index/index_manager_initializer.h @@ -16,7 +16,7 @@ namespace proton class IndexManagerInitializer : public initializer::InitializerTask { const vespalib::string _baseDir; - double _diskIndexWarmupTime; + const searchcorespi::index::WarmupConfig _warmup; size_t _maxFlushed; size_t _cacheSize; const search::index::Schema _schema; @@ -31,22 +31,17 @@ class IndexManagerInitializer : public initializer::InitializerTask public: // Note: lifetime of indexManager must be handled by caller. IndexManagerInitializer(const vespalib::string &baseDir, - double diskIndexWarmupTime, + const searchcorespi::index::WarmupConfig & warmup, size_t maxFlushed, size_t cacheSize, const search::index::Schema &schema, const search::index::Schema &fusionSchema, - searchcorespi::IIndexManager::Reconfigurer & - reconfigurer, - searchcorespi::index::IThreadingService & - threadingService, + searchcorespi::IIndexManager::Reconfigurer & reconfigurer, + searchcorespi::index::IThreadingService & threadingService, vespalib::ThreadExecutor & warmupExecutor, - const search::TuneFileIndexManager & - tuneFileIndexManager, - const search::TuneFileAttributes & - tuneFileAttributes, - const search::common::FileHeaderContext & - fileHeaderContext, + const search::TuneFileIndexManager & tuneFileIndexManager, + const search::TuneFileAttributes & tuneFileAttributes, + const search::common::FileHeaderContext & fileHeaderContext, std::shared_ptr<searchcorespi::IIndexManager::SP> indexManager); virtual void run() override; }; diff --git a/searchcore/src/vespa/searchcore/proton/index/indexmanager.cpp b/searchcore/src/vespa/searchcore/proton/index/indexmanager.cpp index acb5415a129..ad2e3f29853 100644 --- a/searchcore/src/vespa/searchcore/proton/index/indexmanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/index/indexmanager.cpp @@ -20,6 +20,7 @@ using search::TuneFileSearch; using searchcorespi::index::IDiskIndex; using search::diskindex::SelectorArray; using searchcorespi::index::IndexMaintainerConfig; +using searchcorespi::index::WarmupConfig; using searchcorespi::index::IndexMaintainerContext; using searchcorespi::index::IMemoryIndex; using searchcorespi::index::IThreadingService; @@ -79,7 +80,7 @@ IndexManager::MaintainerOperations::runFusion(const Schema &schema, IndexManager::IndexManager(const vespalib::string &baseDir, - const double warmupTime, + const WarmupConfig & warmup, const size_t maxFlushed, const size_t cacheSize, const Schema &schema, @@ -93,7 +94,7 @@ IndexManager::IndexManager(const vespalib::string &baseDir, _operations(fileHeaderContext, tuneFileIndexManager, cacheSize, threadingService), _maintainer(IndexMaintainerConfig(baseDir, - warmupTime, + warmup, maxFlushed, schema, fusionSchema, diff --git a/searchcore/src/vespa/searchcore/proton/index/indexmanager.h b/searchcore/src/vespa/searchcore/proton/index/indexmanager.h index 78e74bf0b80..a7d8cf77649 100644 --- a/searchcore/src/vespa/searchcore/proton/index/indexmanager.h +++ b/searchcore/src/vespa/searchcore/proton/index/indexmanager.h @@ -54,7 +54,7 @@ private: public: IndexManager(const vespalib::string &baseDir, - double diskIndexWarmupTime, + const searchcorespi::index::WarmupConfig & warmup, size_t maxFlushed, size_t cacheSize, const Schema &schema, diff --git a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp index 081a89c944a..a165d75e140 100644 --- a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp @@ -99,7 +99,7 @@ createIndexManagerInitializer(const DocumentDBConfig &configSnapshot, // Note: const_cast for reconfigurer role return std::make_shared<IndexManagerInitializer> (vespaIndexDir, - indexCfg.warmup.time, + searchcorespi::index::WarmupConfig(indexCfg.warmup.time, indexCfg.warmup.unpack), indexCfg.maxflushed, indexCfg.cache.size, *schema, diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp index c8c7fd180a9..1d726a61807 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp +++ b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp @@ -374,7 +374,8 @@ IndexMaintainer::swapInNewIndex(LockGuard & guard, indexes, static_cast<IDiskIndex &>(source), _ctx.getWarmupExecutor(), - *this); + *this, + _unpackAtWarmup); } else { LOG(debug, "No warmup needed as it is a memory index that is mapped in."); } @@ -790,7 +791,8 @@ IndexMaintainer::IndexMaintainer(const IndexMaintainerConfig &config, const IndexMaintainerContext &ctx, IIndexMaintainerOperations &operations) : _base_dir(config.getBaseDir()), - _diskIndexWarmupTime(config.getDiskIndexWarmupTime()), + _diskIndexWarmupTime(config.getWarmup().getDuration()), + _unpackAtWarmup(config.getWarmup().getUnpack()), _active_indexes(new ActiveDiskIndexes()), _layout(config.getBaseDir()), _schema(config.getSchema()), diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.h b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.h index 6e3b6fa9bdb..cdd87096287 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.h +++ b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.h @@ -88,6 +88,7 @@ class IndexMaintainer : public boost::noncopyable, typedef search::queryeval::ISourceSelector ISourceSelector; const vespalib::string _base_dir; const double _diskIndexWarmupTime; + const bool _unpackAtWarmup; ActiveDiskIndexes::SP _active_indexes; IndexDiskLayout _layout; Schema _schema; // Protected by SL + IUL diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainerconfig.cpp b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainerconfig.cpp index e2838434a1b..e1e88652bd4 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainerconfig.cpp +++ b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainerconfig.cpp @@ -12,13 +12,13 @@ namespace searchcorespi { namespace index { IndexMaintainerConfig::IndexMaintainerConfig(const vespalib::string &baseDir, - double diskIndexWarmupTime, + WarmupConfig warmup, size_t maxFlushed, const Schema &schema, const Schema &fusionSchema, const TuneFileAttributes &tuneFileAttributes) : _baseDir(baseDir), - _diskIndexWarmupTime(diskIndexWarmupTime), + _warmup(warmup), _maxFlushed(maxFlushed), _schema(schema), _fusionSchema(fusionSchema), diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainerconfig.h b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainerconfig.h index 7ee3e0337cc..4f7411b0e44 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainerconfig.h +++ b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainerconfig.h @@ -8,13 +8,24 @@ namespace searchcorespi { namespace index { +class WarmupConfig { +public: + WarmupConfig() : _duration(0.0), _unpack(false) { } + WarmupConfig(double duration, bool unpack) : _duration(duration), _unpack(unpack) { } + double getDuration() const { return _duration; } + bool getUnpack() const { return _unpack; } +private: + double _duration; + bool _unpack; +}; + /** * Class that keeps the config used when constructing an index maintainer. */ class IndexMaintainerConfig { private: const vespalib::string _baseDir; - const double _diskIndexWarmupTime; + const WarmupConfig _warmup; const size_t _maxFlushed; const search::index::Schema _schema; const search::index::Schema _fusionSchema; @@ -22,7 +33,7 @@ private: public: IndexMaintainerConfig(const vespalib::string &baseDir, - double diskIndexWarmupTime, + WarmupConfig warmup, size_t maxFlushed, const search::index::Schema &schema, const search::index::Schema &fusionSchema, @@ -35,9 +46,7 @@ public: return _baseDir; } - double getDiskIndexWarmupTime() const { - return _diskIndexWarmupTime; - } + WarmupConfig getWarmup() const { return _warmup; } /** * Returns the initial schema containing all current index fields. diff --git a/searchcorespi/src/vespa/searchcorespi/index/warmupindexcollection.cpp b/searchcorespi/src/vespa/searchcorespi/index/warmupindexcollection.cpp index 7d630a2c668..083db213a46 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/warmupindexcollection.cpp +++ b/searchcorespi/src/vespa/searchcorespi/index/warmupindexcollection.cpp @@ -26,14 +26,16 @@ WarmupIndexCollection::WarmupIndexCollection(double warmupSeconds, ISearchableIndexCollection::SP next, IndexSearchable & warmup, vespalib::ThreadExecutor & executor, - IWarmupDone & warmupDone) : + IWarmupDone & warmupDone, + bool doUnpack) : _prev(prev), _next(next), _warmup(warmup), _executor(executor), _warmupDone(warmupDone), _warmupEndTime(ClockSystem::now() + TimeStamp::Seconds(warmupSeconds)), - _handledTerms() + _handledTerms(), + _doUnpack(doUnpack) { if (next->valid()) { setCurrentIndex(next->getCurrentIndex()); @@ -72,8 +74,7 @@ WarmupIndexCollection::toString() const WarmupIndexCollection::~WarmupIndexCollection() { if (_warmupEndTime != 0) { - LOG(info, - "Warmup aborted due to new state change or application shutdown"); + LOG(info, "Warmup aborted due to new state change or application shutdown"); } _executor.sync(); } @@ -210,7 +211,10 @@ WarmupIndexCollection::WarmupTask::run() _bluePrint->fetchPostings(true); SearchIterator::UP it(_bluePrint->createSearch(*_matchData, true)); it->initFullRange(); - for (it->seek(0); !it->isAtEnd(); it->seek(it->getDocId()+1)) { + for (uint32_t docId = it->seekFirst(1); !it->isAtEnd(); docId = it->seekNext(docId+1)) { + if (_warmup.doUnpack()) { + it->unpack(docId); + } } } else { LOG(debug, "Warmup has finished, ignoring task."); diff --git a/searchcorespi/src/vespa/searchcorespi/index/warmupindexcollection.h b/searchcorespi/src/vespa/searchcorespi/index/warmupindexcollection.h index 7457b676f31..426d0a778ce 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/warmupindexcollection.h +++ b/searchcorespi/src/vespa/searchcorespi/index/warmupindexcollection.h @@ -27,7 +27,8 @@ public: ISearchableIndexCollection::SP next, IndexSearchable & warmup, vespalib::ThreadExecutor & executor, - IWarmupDone & warmupDone); + IWarmupDone & warmupDone, + bool doUnpack); ~WarmupIndexCollection(); // Implements IIndexCollection const ISourceSelector &getSourceSelector() const override; @@ -56,6 +57,7 @@ public: const ISearchableIndexCollection::SP & getNextIndexCollection() const { return _next; } vespalib::string toString() const override; + bool doUnpack() const { return _doUnpack; } private: typedef search::fef::MatchData MatchData; typedef search::queryeval::FakeRequestContext FakeRequestContext; @@ -105,6 +107,7 @@ private: typedef vespalib::hash_set<vespalib::string> TermMap; typedef vespalib::hash_map<uint32_t, TermMap> FieldTermMap; FieldTermMap _handledTerms; + bool _doUnpack; }; } // namespace searchcorespi |