diff options
20 files changed, 173 insertions, 134 deletions
diff --git a/searchcore/src/tests/proton/index/indexmanager_test.cpp b/searchcore/src/tests/proton/index/indexmanager_test.cpp index 9d8f4d8b7f6..27a2706aae8 100644 --- a/searchcore/src/tests/proton/index/indexmanager_test.cpp +++ b/searchcore/src/tests/proton/index/indexmanager_test.cpp @@ -690,36 +690,23 @@ TEST_F("requireThatFailedFusionIsRetried", Fixture) { EXPECT_EQUAL(2u, spec.flush_ids[1]); } -namespace { - -void expectSchemaIndexFields(uint32_t expIndexFields) { - Schema s; - s.loadFromFile("test_data/index.flush.1/schema.txt"); - EXPECT_EQUAL(expIndexFields, s.getNumIndexFields()); -} - -} - -TEST_F("require that setSchema updates schema on disk, wiping removed fields", Fixture) -{ +TEST_F("require that wipeHistory updates schema on disk", Fixture) { Schema empty_schema; f.addDocument(docid); f.flushIndexManager(); - TEST_DO(expectSchemaIndexFields(1)); - f.runAsMaster([&]() { f._index_manager->setSchema(empty_schema, ++f._serial_num); }); - TEST_DO(expectSchemaIndexFields(0)); -} - -TEST_F("require that wipeHistory updates schema on disk", Fixture) -{ - Schema empty_schema; + f.runAsMaster([&]() { f._index_manager->setSchema(empty_schema); }); f.addDocument(docid); f.flushIndexManager(); - TEST_DO(expectSchemaIndexFields(1)); - f.runAsMaster([&]() { f._index_manager->setSchema(empty_schema, f._serial_num); }); - TEST_DO(expectSchemaIndexFields(1)); - f.runAsMaster([&]() { f._index_manager->wipeHistory(++f._serial_num); }); - TEST_DO(expectSchemaIndexFields(0)); + + Schema s; + s.loadFromFile("test_data/index.flush.1/schema.txt"); + EXPECT_EQUAL(1u, s.getNumIndexFields()); + + f.runAsMaster([&]() { f._index_manager->wipeHistory(f._serial_num, + empty_schema); }); + + s.loadFromFile("test_data/index.flush.1/schema.txt"); + EXPECT_EQUAL(0u, s.getNumIndexFields()); } TEST_F("require that indexes manager stats can be generated", Fixture) diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attributedisklayout.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attributedisklayout.cpp index a4fb2e03c44..b5fa7db0983 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attributedisklayout.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attributedisklayout.cpp @@ -14,45 +14,6 @@ using search::AttributeVector; namespace proton { -vespalib::string -AttributeDiskLayout::getSnapshotDir(uint64_t syncToken) -{ - return vespalib::make_string("snapshot-%" PRIu64, syncToken); -} - -vespalib::string -AttributeDiskLayout::getSnapshotRemoveDir(const vespalib::string &baseDir, - const vespalib::string &snapDir) -{ - if (baseDir.empty()) { - return snapDir; - } - return vespalib::make_string("%s/%s", - baseDir.c_str(), - snapDir.c_str()); -} - -vespalib::string -AttributeDiskLayout::getAttributeBaseDir(const vespalib::string &baseDir, - const vespalib::string &attrName) -{ - if (baseDir.empty()) { - return attrName; - } - return vespalib::make_string("%s/%s", - baseDir.c_str(), - attrName.c_str()); -} - -AttributeVector::BaseName -AttributeDiskLayout::getAttributeFileName(const vespalib::string &baseDir, - const vespalib::string &attrName, - uint64_t syncToken) -{ - return AttributeVector::BaseName(getAttributeBaseDir(baseDir, attrName), - getSnapshotDir(syncToken), - attrName); -} bool AttributeDiskLayout::removeOldSnapshots(IndexMetaInfo &snapInfo, diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attributedisklayout.h b/searchcore/src/vespa/searchcore/proton/attribute/attributedisklayout.h index f085d9c6f61..1fbcf7ab8b3 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attributedisklayout.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/attributedisklayout.h @@ -15,14 +15,54 @@ namespace proton { class AttributeDiskLayout { private: - static vespalib::string getSnapshotDir(uint64_t syncToken); - static vespalib::string getSnapshotRemoveDir(const vespalib::string &baseDir, const vespalib::string &snapDir); + static vespalib::string + getSnapshotDir(uint64_t syncToken) + { + return vespalib::make_string("snapshot-%" PRIu64, syncToken); + } + static vespalib::string + getSnapshotRemoveDir(const vespalib::string &baseDir, + const vespalib::string &snapDir) + { + if (baseDir.empty()) { + return snapDir; + } + return vespalib::make_string("%s/%s", + baseDir.c_str(), + snapDir.c_str()); + } public: - static vespalib::string getAttributeBaseDir(const vespalib::string &baseDir, const vespalib::string &attrName); - static search::AttributeVector::BaseName getAttributeFileName(const vespalib::string &baseDir, const vespalib::string &attrName, uint64_t syncToken); - static bool removeOldSnapshots(search::IndexMetaInfo &snapInfo, vespalib::Lock &snapInfoLock); - static bool removeAttribute(const vespalib::string &baseDir, const vespalib::string &attrName); + static vespalib::string + getAttributeBaseDir(const vespalib::string &baseDir, + const vespalib::string &attrName) + { + if (baseDir.empty()) { + return attrName; + } + return vespalib::make_string("%s/%s", + baseDir.c_str(), + attrName.c_str()); + } + + static search::AttributeVector::BaseName + getAttributeFileName(const vespalib::string &baseDir, + const vespalib::string &attrName, + uint64_t syncToken) + { + return search::AttributeVector::BaseName(getAttributeBaseDir(baseDir, + attrName), + getSnapshotDir(syncToken), + attrName); + } + + static bool + removeOldSnapshots(search::IndexMetaInfo &snapInfo, + vespalib::Lock &snapInfoLock); + + static bool + removeAttribute(const vespalib::string &baseDir, + const vespalib::string &attrName); }; diff --git a/searchcore/src/vespa/searchcore/proton/index/indexmanager.h b/searchcore/src/vespa/searchcore/proton/index/indexmanager.h index 55f16047183..face9aefac4 100644 --- a/searchcore/src/vespa/searchcore/proton/index/indexmanager.h +++ b/searchcore/src/vespa/searchcore/proton/index/indexmanager.h @@ -114,12 +114,12 @@ public: return _maintainer.getFlushTargets(); } - virtual void setSchema(const Schema &schema, SerialNum serialNum) { - _maintainer.setSchema(schema, serialNum); + virtual void setSchema(const Schema &schema) { + _maintainer.setSchema(schema); } - virtual void wipeHistory(SerialNum wipeSerial) { - _maintainer.wipeHistory(wipeSerial); + virtual void wipeHistory(SerialNum wipeSerial, const Schema &historyFields) { + _maintainer.wipeHistory(wipeSerial, historyFields); } }; diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp index 61799861171..79529285706 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp @@ -531,7 +531,7 @@ DocumentDB::applyConfig(DocumentDBConfig::SP configSnapshot, } } if (params.shouldIndexManagerChange()) { - setIndexSchema(*configSnapshot, serialNum); + setIndexSchema(*configSnapshot); } if (!fallbackConfig) { if (_state.getRejectedConfig()) { @@ -873,10 +873,10 @@ DocumentDB::flushDone(SerialNum flushedSerial) } void -DocumentDB::setIndexSchema(const DocumentDBConfig &configSnapshot, SerialNum serialNum) +DocumentDB::setIndexSchema(const DocumentDBConfig &configSnapshot) { // Called by executor thread - _subDBs.getReadySubDB()->setIndexSchema(configSnapshot.getSchemaSP(), serialNum); + _subDBs.getReadySubDB()->setIndexSchema(configSnapshot.getSchemaSP()); // TODO: Adjust tune. } @@ -1017,7 +1017,7 @@ DocumentDB::internalWipeHistory(SerialNum wipeSerial, const Schema &wipeSchema) { // Called by executor thread - _subDBs.wipeHistory(wipeSerial, wipeSchema); + _subDBs.wipeHistory(wipeSerial, *newHistorySchema, wipeSchema); _historySchema.reset(newHistorySchema.release()); } diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.h b/searchcore/src/vespa/searchcore/proton/server/documentdb.h index ec1c7963dd8..43509cde8e3 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdb.h +++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.h @@ -186,7 +186,7 @@ private: reconfigureSchema(const DocumentDBConfig &configSnapshot, const DocumentDBConfig &oldConfigSnapshot); - void setIndexSchema(const DocumentDBConfig &configSnapshot, SerialNum serialNum); + void setIndexSchema(const DocumentDBConfig &configSnapshot); /** * Redo interrupted reprocessing if last entry in transaction log diff --git a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp index 5582571ba74..42f23fa7cf5 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp @@ -247,10 +247,11 @@ DocumentSubDBCollection::getNewestFlushedSerial() void DocumentSubDBCollection::wipeHistory(SerialNum wipeSerial, + const Schema &newHistorySchema, const Schema &wipeSchema) { for (auto subDb : _subDBs) { - subDb->wipeHistory(wipeSerial, wipeSchema); + subDb->wipeHistory(wipeSerial, newHistorySchema, wipeSchema); } } diff --git a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h index cc856fbc4aa..18d073f60e2 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h +++ b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h @@ -111,6 +111,7 @@ public: void wipeHistory(SerialNum wipeSerial, + const search::index::Schema &newHistorySchema, const search::index::Schema &wipeSchema); void diff --git a/searchcore/src/vespa/searchcore/proton/server/idocumentsubdb.h b/searchcore/src/vespa/searchcore/proton/server/idocumentsubdb.h index b0e73bc5c72..30887691900 100644 --- a/searchcore/src/vespa/searchcore/proton/server/idocumentsubdb.h +++ b/searchcore/src/vespa/searchcore/proton/server/idocumentsubdb.h @@ -122,8 +122,8 @@ public: * last part of transaction log. */ virtual SerialNum getNewestFlushedSerial() = 0; - virtual void wipeHistory(SerialNum wipeSerial, const Schema &wipeSchema) = 0; - virtual void setIndexSchema(const SchemaSP &schema, SerialNum serialNum) = 0; + virtual void wipeHistory(SerialNum wipeSerial, const Schema &newHistorySchema, const Schema &wipeSchema) = 0; + virtual void setIndexSchema(const SchemaSP &schema) = 0; virtual search::SearchableStats getSearchableStats() const = 0; virtual std::unique_ptr<IDocumentRetriever> getDocumentRetriever() = 0; diff --git a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp index 2713b61880b..8fbbc7d3031 100644 --- a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp @@ -298,25 +298,26 @@ SearchableDocSubDB::getFlushTargetsInternal() void SearchableDocSubDB::wipeHistory(SerialNum wipeSerial, + const Schema &newHistorySchema, const Schema &wipeSchema) { assert(_writeService.master().isCurrentThread()); SearchView::SP oldSearchView = _rSearchView.get(); IFeedView::SP oldFeedView = _iFeedView.get(); - _indexMgr->wipeHistory(wipeSerial); + _indexMgr->wipeHistory(wipeSerial, newHistorySchema); reconfigureIndexSearchable(); getAttributeManager()->wipeHistory(wipeSchema); } void -SearchableDocSubDB::setIndexSchema(const Schema::SP &schema, SerialNum serialNum) +SearchableDocSubDB::setIndexSchema(const Schema::SP &schema) { assert(_writeService.master().isCurrentThread()); SearchView::SP oldSearchView = _rSearchView.get(); IFeedView::SP oldFeedView = _iFeedView.get(); - _indexMgr->setSchema(*schema, serialNum); + _indexMgr->setSchema(*schema); reconfigureIndexSearchable(); } diff --git a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h index 9d2b45fa3ac..5462a25f75e 100644 --- a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h +++ b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h @@ -154,8 +154,8 @@ public: SerialNum getOldestFlushedSerial() override; SerialNum getNewestFlushedSerial() override; - void wipeHistory(SerialNum wipeSerial, const Schema &wipeSchema) override; - void setIndexSchema(const Schema::SP &schema, SerialNum serialNum) override; + void wipeHistory(SerialNum wipeSerial, const Schema &newHistorySchema, const Schema &wipeSchema) override; + void setIndexSchema(const Schema::SP &schema) override; size_t getNumActiveDocs() const override; search::SearchableStats getSearchableStats() const override ; IDocumentRetriever::UP getDocumentRetriever() override; diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp index 06bcd509522..22f4ee0b6f2 100644 --- a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp @@ -458,16 +458,15 @@ StoreOnlyDocSubDB::getIndexWriter() const } void -StoreOnlyDocSubDB::wipeHistory(SerialNum, const Schema &) +StoreOnlyDocSubDB::wipeHistory(SerialNum, const Schema &, const Schema &) { } void -StoreOnlyDocSubDB::setIndexSchema(const Schema::SP &schema, SerialNum serialNum) +StoreOnlyDocSubDB::setIndexSchema(const Schema::SP &schema) { assert(_writeService.master().isCurrentThread()); (void) schema; - (void) serialNum; } search::SearchableStats diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h index 11b0db61576..93cc0e181e3 100644 --- a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h +++ b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h @@ -249,8 +249,8 @@ public: SerialNum getOldestFlushedSerial() override; SerialNum getNewestFlushedSerial() override; - void wipeHistory(SerialNum wipeSerial, const Schema &wipeSchema) override; - void setIndexSchema(const Schema::SP &schema, SerialNum serialNum) override; + void wipeHistory(SerialNum wipeSerial, const Schema &newHistorySchema, const Schema &wipeSchema) override; + void setIndexSchema(const Schema::SP &schema) override; search::SearchableStats getSearchableStats() const override; IDocumentRetriever::UP getDocumentRetriever() override; matching::MatchingStats getMatcherStats(const vespalib::string &rankProfile) const override; diff --git a/searchcore/src/vespa/searchcore/proton/test/dummy_document_sub_db.h b/searchcore/src/vespa/searchcore/proton/test/dummy_document_sub_db.h index e2638b617b3..eeef8f46d11 100644 --- a/searchcore/src/vespa/searchcore/proton/test/dummy_document_sub_db.h +++ b/searchcore/src/vespa/searchcore/proton/test/dummy_document_sub_db.h @@ -81,8 +81,8 @@ struct DummyDocumentSubDb : public IDocumentSubDB void onReprocessDone(SerialNum) override { } SerialNum getOldestFlushedSerial() override { return 0; } SerialNum getNewestFlushedSerial() override { return 0; } - void wipeHistory(SerialNum, const Schema &) override { } - void setIndexSchema(const Schema::SP &, SerialNum) override { } + void wipeHistory(SerialNum, const Schema &, const Schema &) override { } + void setIndexSchema(const Schema::SP &) override { } search::SearchableStats getSearchableStats() const override { return search::SearchableStats(); } diff --git a/searchcore/src/vespa/searchcore/proton/test/mock_index_manager.h b/searchcore/src/vespa/searchcore/proton/test/mock_index_manager.h index 34b246a1d1b..242cca7c974 100644 --- a/searchcore/src/vespa/searchcore/proton/test/mock_index_manager.h +++ b/searchcore/src/vespa/searchcore/proton/test/mock_index_manager.h @@ -26,8 +26,8 @@ struct MockIndexManager : public IIndexManager virtual searchcorespi::IFlushTarget::List getFlushTargets() override { return searchcorespi::IFlushTarget::List(); } - virtual void setSchema(const Schema &, SerialNum) override {} - virtual void wipeHistory(SerialNum) override {} + virtual void setSchema(const Schema &) override {} + virtual void wipeHistory(SerialNum, const Schema &) override {} virtual void heartBeat(SerialNum) override {} }; diff --git a/searchcorespi/src/tests/plugin/plugin.cpp b/searchcorespi/src/tests/plugin/plugin.cpp index eff33e69464..0ebd22495f8 100644 --- a/searchcorespi/src/tests/plugin/plugin.cpp +++ b/searchcorespi/src/tests/plugin/plugin.cpp @@ -34,8 +34,8 @@ public: searchcorespi::IFlushTarget::List l; return l; } - virtual void setSchema(const Schema &, SerialNum) override { } - virtual void wipeHistory(SerialNum) override { } + virtual void setSchema(const Schema &) { } + virtual void wipeHistory(SerialNum , const Schema &) { } }; class IndexManagerFactory : public searchcorespi::IIndexManagerFactory diff --git a/searchcorespi/src/vespa/searchcorespi/index/iindexmanager.cpp b/searchcorespi/src/vespa/searchcorespi/index/iindexmanager.cpp index eb47aaf043e..b6d350ad52e 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/iindexmanager.cpp +++ b/searchcorespi/src/vespa/searchcorespi/index/iindexmanager.cpp @@ -4,9 +4,10 @@ namespace searchcorespi { void -IIndexManager::wipeHistory(SerialNum wipeSerial) +IIndexManager::wipeHistory(SerialNum wipeSerial, const Schema &historyFields) { (void) wipeSerial; + (void) historyFields; } IIndexManager::Reconfigurer::~Reconfigurer() diff --git a/searchcorespi/src/vespa/searchcorespi/index/iindexmanager.h b/searchcorespi/src/vespa/searchcorespi/index/iindexmanager.h index c2976699edb..b1e3e5515b5 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/iindexmanager.h +++ b/searchcorespi/src/vespa/searchcorespi/index/iindexmanager.h @@ -156,14 +156,17 @@ public: * * @param schema The new schema to start using. **/ - virtual void setSchema(const Schema &schema, SerialNum serialNum) = 0; + virtual void setSchema(const Schema &schema) = 0; /** - * Wipes remains of removed fields from this index manager. + * Wipes remains of removed fields from this index manager as specified in the history schema. + * This can for instance be removing these fields from disk indexes. + * The default implementation does nothing. * * @param wipeSerial The serial number of this wipe operation. + * @param historyFields The schema specifying which fields we should wipe away. **/ - virtual void wipeHistory(SerialNum wipeSerial); + virtual void wipeHistory(SerialNum wipeSerial, const Schema &historyFields); }; } // namespace searchcorespi diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp index c075e98e1fc..ce88cc1188b 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp +++ b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp @@ -173,11 +173,11 @@ IndexMaintainer::reopenDiskIndexes(ISearchableIndexCollection &coll) } const string indexDir = d->getIndexDir(); vespalib::string schemaName = IndexDiskLayout::getSchemaFileName(indexDir); - Schema trimmedSchema; - if (!trimmedSchema.loadFromFile(schemaName)) { + Schema oldSchema; + if (!oldSchema.loadFromFile(schemaName)) { LOG(error, "Could not open schema '%s'", schemaName.c_str()); } - if (trimmedSchema != d->getSchema()) { + if (oldSchema != d->getSchema()) { IDiskIndex::SP newIndex(reloadDiskIndex(*d)); coll.replace(coll.getSourceId(i), newIndex); hasReopenedAnything = true; @@ -435,6 +435,16 @@ IndexMaintainer::FlushArgs::~FlushArgs() { } IndexMaintainer::FlushArgs::FlushArgs(FlushArgs &&) = default; IndexMaintainer::FlushArgs & IndexMaintainer::FlushArgs::operator=(FlushArgs &&) = default; +IndexMaintainer::WipeHistoryArgs::WipeHistoryArgs() + : _old_source_list(), + _new_source_list() +{ } + +IndexMaintainer::WipeHistoryArgs::WipeHistoryArgs(WipeHistoryArgs &&) = default; +IndexMaintainer::WipeHistoryArgs & IndexMaintainer::WipeHistoryArgs::operator=(WipeHistoryArgs &&) = default; + +IndexMaintainer::WipeHistoryArgs::~WipeHistoryArgs() { } + bool IndexMaintainer::doneInitFlush(FlushArgs *args, IMemoryIndex::SP *new_index) { @@ -758,6 +768,20 @@ IndexMaintainer::doneSetSchema(SetSchemaArgs &args, IMemoryIndex::SP &newIndex) } +bool +IndexMaintainer::doneWipeHistory(WipeHistoryArgs &args) +{ + assert(_ctx.getThreadingService().master().isCurrentThread()); // with idle index executor + LockGuard state_lock(_state_lock); + LockGuard lock(_new_search_lock); + if (args._old_source_list.get() != _source_list.get()) { + return false; // Flush or fusion had started/completed, must retry + } + _source_list = args._new_source_list; + return true; +} + + Schema IndexMaintainer::getSchema(void) const { @@ -867,7 +891,7 @@ IndexMaintainer::IndexMaintainer(const IndexMaintainerConfig &config, LOG(debug, "Index manager created with flushed serial num %" PRIu64, _flush_serial_num); sourceList->append(_current_index_id, _current_index); sourceList->setCurrentIndex(_current_index_id); - _source_list = std::move(sourceList); + _source_list.reset(sourceList.release()); _fusion_spec = spec; } @@ -1180,10 +1204,9 @@ IndexMaintainer::getFlushTargets(void) } void -IndexMaintainer::setSchema(const Schema & schema, SerialNum serialNum) +IndexMaintainer::setSchema(const Schema & schema) { assert(_ctx.getThreadingService().master().isCurrentThread()); - internalWipeHistory(schema, serialNum); IMemoryIndex::SP new_index(_operations.createMemoryIndex(schema, _current_serial_num)); SetSchemaArgs args; @@ -1198,37 +1221,46 @@ IndexMaintainer::setSchema(const Schema & schema, SerialNum serialNum) } void -IndexMaintainer::internalWipeHistory(const Schema &schema, SerialNum wipeSerial) +IndexMaintainer::wipeHistory(SerialNum wipeSerial, const Schema &historyFields) { assert(_ctx.getThreadingService().master().isCurrentThread()); - ISearchableIndexCollection::SP new_source_list; - IIndexCollection::SP coll = getSourceCollection(); - updateIndexSchemas(*coll, schema, wipeSerial); - updateActiveFusionWipeTimeSchema(schema); + for (;;) { + const Schema before_schema = getSchema(); + IIndexCollection::SP before_coll = getSourceCollection(); + + Schema::UP schema = Schema::make_union(before_schema, historyFields); + updateIndexSchemas(*before_coll, *schema, wipeSerial); + updateActiveFusionWipeTimeSchema(*schema); + + const Schema after_schema(getSchema()); + IIndexCollection::SP after_coll = getSourceCollection(); + if (before_schema == after_schema && + before_coll.get() == after_coll.get()) + { + break; + } + } { LockGuard state_lock(_state_lock); LockGuard lock(_index_update_lock); _changeGens.bumpWipeGen(); } - { - LockGuard state_lock(_state_lock); - new_source_list = std::make_shared<IndexCollection>(_selector, *_source_list); - } - if (reopenDiskIndexes(*new_source_list)) { - scheduleCommit(); - _ctx.getThreadingService().sync(); - // Everything should be quiet now. - LockGuard state_lock(_state_lock); - LockGuard lock(_new_search_lock); - _source_list = new_source_list; + for (bool success(false); !success;) { + WipeHistoryArgs args; + { + LockGuard state_lock(_state_lock); + args._old_source_list = _source_list; + args._new_source_list.reset(new IndexCollection(_selector, *args._old_source_list)); + } + if (reopenDiskIndexes(*args._new_source_list)) { + _ctx.getThreadingService().sync(); + // Everything should be quiet now. + success = doneWipeHistory(args); + } else { + success = true; + } } } -void -IndexMaintainer::wipeHistory(SerialNum wipeSerial) -{ - internalWipeHistory(getSchema(), wipeSerial); -} - } // namespace index } // namespace searchcorespi diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.h b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.h index 34c25632012..c3cacd2cb25 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.h +++ b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.h @@ -95,7 +95,7 @@ class IndexMaintainer : public IIndexManager, uint32_t _source_selector_changes; // Protected by IUL // _selector is protected by SL + IUL ISourceSelector::SP _selector; - ISearchableIndexCollection::SP _source_list; // Protected by SL + NSL, only set by master thread + ISearchableIndexCollection::SP _source_list; // Protected by SL + NSL uint32_t _last_fusion_id; // Protected by SL + IUL uint32_t _next_id; // Protected by SL + IUL uint32_t _current_index_id; // Protected by SL + IUL @@ -272,6 +272,20 @@ class IndexMaintainer : public IIndexManager, void doneSetSchema(SetSchemaArgs &args, IMemoryIndex::SP &newIndex); + class WipeHistoryArgs + { + public: + ISearchableIndexCollection::SP _old_source_list; + ISearchableIndexCollection::SP _new_source_list; + + WipeHistoryArgs(); + WipeHistoryArgs(WipeHistoryArgs &&); + WipeHistoryArgs & operator=(WipeHistoryArgs &&); + + ~WipeHistoryArgs(); + }; + + bool doneWipeHistory(WipeHistoryArgs &args); Schema getSchema(void) const; Schema::SP getActiveFusionWipeTimeSchema() const; search::TuneFileAttributes getAttrTune(); @@ -287,7 +301,6 @@ class IndexMaintainer : public IIndexManager, bool makeSureAllRemainingWarmupIsDone(ISearchableIndexCollection::SP keepAlive); void scheduleCommit(); void commit(); - void internalWipeHistory(const Schema &schema, SerialNum wipeSerial); public: IndexMaintainer(const IndexMaintainer &) = delete; @@ -382,8 +395,8 @@ public: } IFlushTarget::List getFlushTargets() override; - void setSchema(const Schema & schema, SerialNum serialNum) override ; - void wipeHistory(SerialNum wipeSerial) override; + void setSchema(const Schema & schema) override ; + void wipeHistory(SerialNum wipeSerial, const Schema &historyFields) override; }; } // namespace index |