diff options
30 files changed, 111 insertions, 86 deletions
diff --git a/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp b/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp index 1b4848f9c8b..97faa81b48a 100644 --- a/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp +++ b/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp @@ -144,7 +144,7 @@ struct MyIndexWriter : public test::MockIndexWriter _wantedLidLimit(0), _tracer(tracer) {} - void put(SerialNum serialNum, const document::Document &doc, const DocumentIdT lid) override { + void put(SerialNum serialNum, const document::Document &doc, const DocumentIdT lid, OnWriteDoneType) override { (void) doc; _tracer.tracePut(indexAdapterTypeName, serialNum, lid); } diff --git a/searchcore/src/tests/proton/feed_and_search/feed_and_search.cpp b/searchcore/src/tests/proton/feed_and_search/feed_and_search.cpp index 8314fa6bfb8..855b31310a3 100644 --- a/searchcore/src/tests/proton/feed_and_search/feed_and_search.cpp +++ b/searchcore/src/tests/proton/feed_and_search/feed_and_search.cpp @@ -168,10 +168,10 @@ void Test::requireThatMemoryIndexCanBeDumpedAndSearched() { DocBuilder doc_builder(schema); Document::UP doc = buildDocument(doc_builder, doc_id1, word1); - memory_index.insertDocument(doc_id1, *doc.get()); + memory_index.insertDocument(doc_id1, *doc, {}); - doc = buildDocument(doc_builder, doc_id2, word2); - memory_index.insertDocument(doc_id2, *doc.get()); + auto doc2 = buildDocument(doc_builder, doc_id2, word2); + memory_index.insertDocument(doc_id2, *doc2, {}); commit_memory_index_and_wait(memory_index); testSearch(memory_index, word1, doc_id1); diff --git a/searchcore/src/tests/proton/index/fusionrunner_test.cpp b/searchcore/src/tests/proton/index/fusionrunner_test.cpp index 736bc4bae96..ae85211fe24 100644 --- a/searchcore/src/tests/proton/index/fusionrunner_test.cpp +++ b/searchcore/src/tests/proton/index/fusionrunner_test.cpp @@ -157,7 +157,7 @@ Document::UP buildDocument(DocBuilder & doc_builder, int id, const string &word) void addDocument(DocBuilder & doc_builder, MemoryIndex &index, ISourceSelector &selector, uint8_t index_id, uint32_t docid, const string &word) { Document::UP doc = buildDocument(doc_builder, docid, word); - index.insertDocument(docid, *doc); + index.insertDocument(docid, *doc, {}); vespalib::Gate gate; index.commit(std::make_shared<vespalib::GateCallback>(gate)); selector.setSource(docid, index_id); diff --git a/searchcore/src/tests/proton/index/index_writer/index_writer_test.cpp b/searchcore/src/tests/proton/index/index_writer/index_writer_test.cpp index ac17c17892c..62a691d72e6 100644 --- a/searchcore/src/tests/proton/index/index_writer/index_writer_test.cpp +++ b/searchcore/src/tests/proton/index/index_writer/index_writer_test.cpp @@ -49,7 +49,7 @@ struct MyIndexManager : public test::MockIndexManager return toString(removes[lid]); } // Implements IIndexManager - void putDocument(uint32_t lid, const Document &, SerialNum serialNum) override { + void putDocument(uint32_t lid, const Document &, SerialNum serialNum, OnWriteDoneType) override { puts[lid].push_back(serialNum); } void removeDocuments(LidVector lids, SerialNum serialNum) override { @@ -94,7 +94,7 @@ struct Fixture return builder.endDocument(); } void put(SerialNum serialNum, const search::DocumentIdT lid) { - iw.put(serialNum, *dummyDoc, lid); + iw.put(serialNum, *dummyDoc, lid, {}); iw.commit(serialNum, std::shared_ptr<IDestructorCallback>()); } void remove(SerialNum serialNum, const search::DocumentIdT lid) { diff --git a/searchcore/src/tests/proton/index/indexmanager_test.cpp b/searchcore/src/tests/proton/index/indexmanager_test.cpp index 8e41323e461..4c442d38443 100644 --- a/searchcore/src/tests/proton/index/indexmanager_test.cpp +++ b/searchcore/src/tests/proton/index/indexmanager_test.cpp @@ -192,7 +192,7 @@ IndexManagerTest::addDocument(uint32_t id) Document::UP doc = buildDocument(_builder, id, "foo"); SerialNum serialNum = ++_serial_num; vespalib::Gate gate; - runAsIndex([&]() { _index_manager->putDocument(id, *doc, serialNum); + runAsIndex([&]() { _index_manager->putDocument(id, *doc, serialNum, {}); _index_manager->commit(serialNum, std::make_shared<vespalib::GateCallback>(gate)); }); gate.await(); @@ -416,7 +416,7 @@ TEST_F(IndexManagerTest, require_that_flush_stats_are_calculated) EXPECT_EQ(0u, _index_manager->getMaintainer().getFlushStats().cpu_time_required); Document::UP doc = addDocument(docid); - inverter.invertDocument(docid, *doc); + inverter.invertDocument(docid, *doc, {}); push_documents_and_wait(inverter); index_size = fic.getMemoryUsage().allocatedBytes() - fixed_index_size; @@ -431,9 +431,9 @@ TEST_F(IndexManagerTest, require_that_flush_stats_are_calculated) _index_manager->getMaintainer().getFlushStats().cpu_time_required); doc = addDocument(docid + 10); - inverter.invertDocument(docid + 10, *doc); - doc = addDocument(docid + 100); - inverter.invertDocument(docid + 100, *doc); + inverter.invertDocument(docid + 10, *doc, {}); + auto doc100 = addDocument(docid + 100); + inverter.invertDocument(docid + 100, *doc100, {}); push_documents_and_wait(inverter); index_size = fic.getMemoryUsage().allocatedBytes() - fixed_index_size; /// Must account for both docid 0 being reserved and the extra after. diff --git a/searchcore/src/vespa/searchcore/proton/index/i_index_writer.h b/searchcore/src/vespa/searchcore/proton/index/i_index_writer.h index 8a920d3d580..a96e344979e 100644 --- a/searchcore/src/vespa/searchcore/proton/index/i_index_writer.h +++ b/searchcore/src/vespa/searchcore/proton/index/i_index_writer.h @@ -24,7 +24,7 @@ public: virtual const std::shared_ptr<IIndexManager> &getIndexManager() const = 0; // feed interface - virtual void put(search::SerialNum serialNum, const document::Document &doc, const search::DocumentIdT lid) = 0; + virtual void put(search::SerialNum serialNum, const document::Document &doc, const search::DocumentIdT lid, OnWriteDoneType on_write_done) = 0; void remove(search::SerialNum serialNum, search::DocumentIdT lid) { LidVector lids; lids.push_back(lid); diff --git a/searchcore/src/vespa/searchcore/proton/index/index_writer.cpp b/searchcore/src/vespa/searchcore/proton/index/index_writer.cpp index f86ada8c45c..3512d2eebad 100644 --- a/searchcore/src/vespa/searchcore/proton/index/index_writer.cpp +++ b/searchcore/src/vespa/searchcore/proton/index/index_writer.cpp @@ -23,7 +23,7 @@ IndexWriter::ignoreOperation(search::SerialNum serialNum) const { } void -IndexWriter::put(search::SerialNum serialNum, const document::Document &doc, const search::DocumentIdT lid) +IndexWriter::put(search::SerialNum serialNum, const document::Document &doc, const search::DocumentIdT lid, OnWriteDoneType on_write_done) { if (ignoreOperation(serialNum)) { return; @@ -39,7 +39,7 @@ IndexWriter::put(search::SerialNum serialNum, const document::Document &doc, con serialNum, doc.getId().toString().c_str()+accum, lid, s1.size(), accum, std::min(accum+chunksize, s1.size()), s1.c_str()); } } - _mgr->putDocument(lid, doc, serialNum); + _mgr->putDocument(lid, doc, serialNum, on_write_done); } void diff --git a/searchcore/src/vespa/searchcore/proton/index/index_writer.h b/searchcore/src/vespa/searchcore/proton/index/index_writer.h index 28357d6fd55..3e0822205bc 100644 --- a/searchcore/src/vespa/searchcore/proton/index/index_writer.h +++ b/searchcore/src/vespa/searchcore/proton/index/index_writer.h @@ -18,7 +18,7 @@ public: ~IndexWriter() override; const IIndexManager::SP & getIndexManager() const override { return _mgr; } - void put(search::SerialNum serialNum, const document::Document &doc, const search::DocumentIdT lid) override; + void put(search::SerialNum serialNum, const document::Document &doc, const search::DocumentIdT lid, OnWriteDoneType on_write_done) override; void removeDocs(search::SerialNum serialNum, LidVector lids) override; void commit(search::SerialNum serialNum, OnWriteDoneType onWriteDone) override; diff --git a/searchcore/src/vespa/searchcore/proton/index/indexmanager.h b/searchcore/src/vespa/searchcore/proton/index/indexmanager.h index 8212978527a..4113af30b0d 100644 --- a/searchcore/src/vespa/searchcore/proton/index/indexmanager.h +++ b/searchcore/src/vespa/searchcore/proton/index/indexmanager.h @@ -86,8 +86,8 @@ public: /** * Implements searchcorespi::IIndexManager **/ - void putDocument(uint32_t lid, const Document &doc, SerialNum serialNum) override { - _maintainer.putDocument(lid, doc, serialNum); + void putDocument(uint32_t lid, const Document &doc, SerialNum serialNum, OnWriteDoneType on_write_done) override { + _maintainer.putDocument(lid, doc, serialNum, on_write_done); } void removeDocuments(LidVector lids, SerialNum serialNum) override { diff --git a/searchcore/src/vespa/searchcore/proton/index/memoryindexwrapper.h b/searchcore/src/vespa/searchcore/proton/index/memoryindexwrapper.h index e4eaad538cb..432262a0322 100644 --- a/searchcore/src/vespa/searchcore/proton/index/memoryindexwrapper.h +++ b/searchcore/src/vespa/searchcore/proton/index/memoryindexwrapper.h @@ -76,8 +76,8 @@ public: vespalib::MemoryUsage getMemoryUsage() const override { return _index.getMemoryUsage(); } - void insertDocument(uint32_t lid, const document::Document &doc) override { - _index.insertDocument(lid, doc); + void insertDocument(uint32_t lid, const document::Document &doc, OnWriteDoneType on_write_done) override { + _index.insertDocument(lid, doc, on_write_done); } void removeDocuments(LidVector lids) override { _index.removeDocuments(std::move(lids)); diff --git a/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.cpp b/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.cpp index d973777020d..323ca9add17 100644 --- a/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.cpp @@ -61,7 +61,7 @@ SearchableFeedView::performIndexPut(SerialNum serialNum, search::DocumentIdT lid "database(%s): performIndexPut: serialNum(%" PRIu64 "), docId(%s), lid(%d)", _params._docTypeName.toString().c_str(), serialNum, doc.getId().toString().c_str(), lid); - _indexWriter->put(serialNum, doc, lid); + _indexWriter->put(serialNum, doc, lid, onWriteDone); } void diff --git a/searchcore/src/vespa/searchcore/proton/test/mock_index_manager.h b/searchcore/src/vespa/searchcore/proton/test/mock_index_manager.h index 7c4de5be2db..bf76dc3d59b 100644 --- a/searchcore/src/vespa/searchcore/proton/test/mock_index_manager.h +++ b/searchcore/src/vespa/searchcore/proton/test/mock_index_manager.h @@ -10,7 +10,7 @@ namespace proton::test { */ struct MockIndexManager : public searchcorespi::IIndexManager { - void putDocument(uint32_t, const Document &, SerialNum) override {} + void putDocument(uint32_t, const Document &, SerialNum, OnWriteDoneType) override {} void removeDocuments(LidVector, SerialNum) override {} void commit(SerialNum, OnWriteDoneType) override {} SerialNum getCurrentSerialNum() const override { return 0; } diff --git a/searchcore/src/vespa/searchcore/proton/test/mock_index_writer.h b/searchcore/src/vespa/searchcore/proton/test/mock_index_writer.h index 96af605b27c..bf571a74941 100644 --- a/searchcore/src/vespa/searchcore/proton/test/mock_index_writer.h +++ b/searchcore/src/vespa/searchcore/proton/test/mock_index_writer.h @@ -14,7 +14,7 @@ struct MockIndexWriter : public IIndexWriter MockIndexWriter() : _idxMgr() {} MockIndexWriter(const IIndexManager::SP &idxMgr) : _idxMgr(idxMgr) {} const IIndexManager::SP &getIndexManager() const override { return _idxMgr; } - void put(search::SerialNum, const document::Document &, const search::DocumentIdT) override {} + void put(search::SerialNum, const document::Document &, const search::DocumentIdT, OnWriteDoneType) override {} void removeDocs(search::SerialNum, LidVector) override {} void commit(search::SerialNum, OnWriteDoneType) override {} void heartBeat(search::SerialNum) override {} diff --git a/searchcorespi/src/vespa/searchcorespi/index/iindexmanager.h b/searchcorespi/src/vespa/searchcorespi/index/iindexmanager.h index 02adcbc11ce..5bd618d16a6 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/iindexmanager.h +++ b/searchcorespi/src/vespa/searchcorespi/index/iindexmanager.h @@ -94,7 +94,7 @@ public: * @param serialNum The unique monotoninc increasing serial number * for this operation. **/ - virtual void putDocument(uint32_t lid, const Document &doc, SerialNum serialNum) = 0; + virtual void putDocument(uint32_t lid, const Document &doc, SerialNum serialNum, OnWriteDoneType on_write_done) = 0; /** * Removes the given document from the index. This method is diff --git a/searchcorespi/src/vespa/searchcorespi/index/imemoryindex.h b/searchcorespi/src/vespa/searchcorespi/index/imemoryindex.h index bff929206a0..d502d290ea4 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/imemoryindex.h +++ b/searchcorespi/src/vespa/searchcorespi/index/imemoryindex.h @@ -43,7 +43,7 @@ struct IMemoryIndex : public searchcorespi::IndexSearchable { * @param lid the local document id. * @param doc the document to insert. */ - virtual void insertDocument(uint32_t lid, const document::Document &doc) = 0; + virtual void insertDocument(uint32_t lid, const document::Document &doc, OnWriteDoneType on_write_done) = 0; /** * Removes the given document from this memory index. diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp index 74848e93411..839618afb4d 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp +++ b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp @@ -1165,12 +1165,12 @@ IndexMaintainer::getNumFrozenMemoryIndexes(void) const } void -IndexMaintainer::putDocument(uint32_t lid, const Document &doc, SerialNum serialNum) +IndexMaintainer::putDocument(uint32_t lid, const Document &doc, SerialNum serialNum, OnWriteDoneType on_write_done) { assert(_ctx.getThreadingService().index().isCurrentThread()); LockGuard lock(_index_update_lock); try { - _current_index->insertDocument(lid, doc); + _current_index->insertDocument(lid, doc, on_write_done); } catch (const vespalib::IllegalStateException & e) { vespalib::string s = "Failed inserting document :\n" + doc.toXml(" ") + "\n"; LOG(error, "%s", s.c_str()); diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.h b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.h index 816ec76cd1f..55f05410108 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.h +++ b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.h @@ -324,7 +324,7 @@ public: vespalib::system_time getLastFlushTime() const { return _lastFlushTime; } // Implements IIndexManager - void putDocument(uint32_t lid, const Document &doc, SerialNum serialNum) override; + void putDocument(uint32_t lid, const Document &doc, SerialNum serialNum, OnWriteDoneType on_write_done) override; void removeDocuments(LidVector lids, SerialNum serialNum) override; void commit(SerialNum serialNum, OnWriteDoneType onWriteDone) override; void heartBeat(search::SerialNum serialNum) override; diff --git a/searchlib/src/apps/tests/memoryindexstress_test.cpp b/searchlib/src/apps/tests/memoryindexstress_test.cpp index dd445745f18..54864702a47 100644 --- a/searchlib/src/apps/tests/memoryindexstress_test.cpp +++ b/searchlib/src/apps/tests/memoryindexstress_test.cpp @@ -48,6 +48,7 @@ using search::query::Node; using search::query::SimplePhrase; using search::query::SimpleStringTerm; using search::index::test::MockFieldLengthInspector; +using vespalib::IDestructorCallback; using vespalib::asciistream; using vespalib::makeLambdaTask; @@ -190,6 +191,16 @@ Node::UP makePhrase(const std::string &term1, const std::string &term2) { return node; } +class HoldDoc : public IDestructorCallback { + std::unique_ptr<Document> _doc; +public: + HoldDoc(std::unique_ptr<Document> doc) noexcept + : _doc(std::move(doc)) + { + } + ~HoldDoc() override = default; +}; + } // namespace struct Fixture { @@ -224,7 +235,8 @@ struct Fixture { gate.await(); } void put(uint32_t id, Document::UP doc) { - index.insertDocument(id, *doc); + auto& docref = *doc; + index.insertDocument(id, docref, std::make_shared<HoldDoc>(std::move(doc))); } void remove(uint32_t id) { std::vector<uint32_t> lids; diff --git a/searchlib/src/tests/diskindex/fusion/fusion_test.cpp b/searchlib/src/tests/diskindex/fusion/fusion_test.cpp index 3889de5b4c4..1c86981372d 100644 --- a/searchlib/src/tests/diskindex/fusion/fusion_test.cpp +++ b/searchlib/src/tests/diskindex/fusion/fusion_test.cpp @@ -332,7 +332,7 @@ FusionTest::requireThatFusionIsWorking(const vespalib::string &prefix, bool dire Document::UP doc; doc = make_doc10(b); - inv.invertDocument(10, *doc); + inv.invertDocument(10, *doc, {}); myPushDocument(inv); b.startDocument("id:ns:searchdocument::11"). @@ -340,7 +340,7 @@ FusionTest::requireThatFusionIsWorking(const vespalib::string &prefix, bool dire startElement(-27).addStr("zz").endElement(). endField(); doc = b.endDocument(); - inv.invertDocument(11, *doc); + inv.invertDocument(11, *doc, {}); myPushDocument(inv); b.startDocument("id:ns:searchdocument::12"). @@ -348,7 +348,7 @@ FusionTest::requireThatFusionIsWorking(const vespalib::string &prefix, bool dire startElement(0).addStr("zz0").endElement(). endField(); doc = b.endDocument(); - inv.invertDocument(12, *doc); + inv.invertDocument(12, *doc, {}); myPushDocument(inv); IndexBuilder ib(schema); @@ -466,7 +466,8 @@ FusionTest::make_simple_index(const vespalib::string &dump_dir, const IFieldLeng DocumentInverterContext inv_context(_schema, *invertThreads, *pushThreads, fic); DocumentInverter inv(inv_context); - inv.invertDocument(10, *make_doc10(b)); + auto doc10 = make_doc10(b); + inv.invertDocument(10, *doc10, {}); myPushDocument(inv); IndexBuilder ib(_schema); diff --git a/searchlib/src/tests/memoryindex/document_inverter/document_inverter_test.cpp b/searchlib/src/tests/memoryindex/document_inverter/document_inverter_test.cpp index dec21c64456..3f8a04d9460 100644 --- a/searchlib/src/tests/memoryindex/document_inverter/document_inverter_test.cpp +++ b/searchlib/src/tests/memoryindex/document_inverter/document_inverter_test.cpp @@ -140,7 +140,8 @@ struct DocumentInverterTest : public ::testing::Test { TEST_F(DocumentInverterTest, require_that_fresh_insert_works) { - _inv.invertDocument(10, *makeDoc10(_b)); + auto doc10 = makeDoc10(_b); + _inv.invertDocument(10, *doc10, {}); pushDocuments(); EXPECT_EQ("f=0,w=a,a=10," "w=b,a=10," @@ -151,8 +152,10 @@ TEST_F(DocumentInverterTest, require_that_fresh_insert_works) TEST_F(DocumentInverterTest, require_that_multiple_docs_work) { - _inv.invertDocument(10, *makeDoc10(_b)); - _inv.invertDocument(11, *makeDoc11(_b)); + auto doc10 = makeDoc10(_b); + auto doc11 = makeDoc11(_b); + _inv.invertDocument(10, *doc10, {}); + _inv.invertDocument(11, *doc11, {}); pushDocuments(); EXPECT_EQ("f=0,w=a,a=10,a=11," "w=b,a=10,a=11," @@ -181,8 +184,10 @@ TEST_F(DocumentInverterTest, require_that_remove_works) TEST_F(DocumentInverterTest, require_that_reput_works) { - _inv.invertDocument(10, *makeDoc10(_b)); - _inv.invertDocument(10, *makeDoc11(_b)); + auto doc10 = makeDoc10(_b); + auto doc11 = makeDoc11(_b); + _inv.invertDocument(10, *doc10, {}); + _inv.invertDocument(10, *doc11, {}); pushDocuments(); EXPECT_EQ("f=0,w=a,a=10," "w=b,a=10," @@ -201,8 +206,8 @@ TEST_F(DocumentInverterTest, require_that_abort_pending_doc_works) auto doc13 = makeDoc13(_b); auto doc14 = makeDoc14(_b); - _inv.invertDocument(10, *doc10); - _inv.invertDocument(11, *doc11); + _inv.invertDocument(10, *doc10, {}); + _inv.invertDocument(11, *doc11, {}); _inv.removeDocument(10); pushDocuments(); EXPECT_EQ("f=0,w=a,a=11," @@ -213,11 +218,11 @@ TEST_F(DocumentInverterTest, require_that_abort_pending_doc_works) "w=g,a=11", _inserter_backend.toStr()); - _inv.invertDocument(10, *doc10); - _inv.invertDocument(11, *doc11); - _inv.invertDocument(12, *doc12); - _inv.invertDocument(13, *doc13); - _inv.invertDocument(14, *doc14); + _inv.invertDocument(10, *doc10, {}); + _inv.invertDocument(11, *doc11, {}); + _inv.invertDocument(12, *doc12, {}); + _inv.invertDocument(13, *doc13, {}); + _inv.invertDocument(14, *doc14, {}); _inv.removeDocument(11); _inv.removeDocument(13); _inserter_backend.reset(); @@ -232,11 +237,11 @@ TEST_F(DocumentInverterTest, require_that_abort_pending_doc_works) "w=j,a=14", _inserter_backend.toStr()); - _inv.invertDocument(10, *doc10); - _inv.invertDocument(11, *doc11); - _inv.invertDocument(12, *doc12); - _inv.invertDocument(13, *doc13); - _inv.invertDocument(14, *doc14); + _inv.invertDocument(10, *doc10, {}); + _inv.invertDocument(11, *doc11, {}); + _inv.invertDocument(12, *doc12, {}); + _inv.invertDocument(13, *doc13, {}); + _inv.invertDocument(14, *doc14, {}); _inv.removeDocument(11); _inv.removeDocument(12); _inv.removeDocument(13); @@ -256,7 +261,8 @@ TEST_F(DocumentInverterTest, require_that_mix_of_add_and_remove_works) _inv.getInverter(0)->remove("c", 9); _inv.getInverter(0)->remove("d", 10); _inv.getInverter(0)->remove("z", 12); - _inv.invertDocument(10, *makeDoc10(_b)); + auto doc10 = makeDoc10(_b); + _inv.invertDocument(10, *doc10, {}); pushDocuments(); EXPECT_EQ("f=0,w=a,a=10,r=11," "w=b,a=10," @@ -268,7 +274,8 @@ TEST_F(DocumentInverterTest, require_that_mix_of_add_and_remove_works) TEST_F(DocumentInverterTest, require_that_empty_document_can_be_inverted) { - _inv.invertDocument(15, *makeDoc15(_b)); + auto doc15 = makeDoc15(_b); + _inv.invertDocument(15, *doc15, {}); pushDocuments(); EXPECT_EQ("", _inserter_backend.toStr()); diff --git a/searchlib/src/tests/memoryindex/field_index/field_index_test.cpp b/searchlib/src/tests/memoryindex/field_index/field_index_test.cpp index 7b52eec78a6..ca30fe2d35e 100644 --- a/searchlib/src/tests/memoryindex/field_index/field_index_test.cpp +++ b/searchlib/src/tests/memoryindex/field_index/field_index_test.cpp @@ -956,7 +956,7 @@ TEST_F(BasicInverterTest, require_that_inversion_is_working) addStr("a").addStr("b").addStr("c").addStr("d"). endField(); doc = _b.endDocument(); - _inv.invertDocument(10, *doc); + _inv.invertDocument(10, *doc, {}); myPushDocument(_inv); _b.startDocument("id:ns:searchdocument::20"); @@ -964,7 +964,7 @@ TEST_F(BasicInverterTest, require_that_inversion_is_working) addStr("a").addStr("a").addStr("b").addStr("c").addStr("d"). endField(); doc = _b.endDocument(); - _inv.invertDocument(20, *doc); + _inv.invertDocument(20, *doc, {}); myPushDocument(_inv); _b.startDocument("id:ns:searchdocument::30"); @@ -993,7 +993,7 @@ TEST_F(BasicInverterTest, require_that_inversion_is_working) endElement(). endField(); doc = _b.endDocument(); - _inv.invertDocument(30, *doc); + _inv.invertDocument(30, *doc, {}); myPushDocument(_inv); _b.startDocument("id:ns:searchdocument::40"); @@ -1002,7 +1002,7 @@ TEST_F(BasicInverterTest, require_that_inversion_is_working) addStr("e").addStr("f"). endField(); doc = _b.endDocument(); - _inv.invertDocument(40, *doc); + _inv.invertDocument(40, *doc, {}); myPushDocument(_inv); _b.startDocument("id:ns:searchdocument::999"); @@ -1030,7 +1030,7 @@ TEST_F(BasicInverterTest, require_that_inversion_is_working) endField(); doc = _b.endDocument(); for (uint32_t docId = 10000; docId < 20000; ++docId) { - _inv.invertDocument(docId, *doc); + _inv.invertDocument(docId, *doc, {}); myPushDocument(_inv); } @@ -1144,13 +1144,13 @@ TEST_F(BasicInverterTest, require_that_inverter_handles_remove_via_document_remo _b.startIndexField("f0").addStr("a").addStr("b").endField(); _b.startIndexField("f1").addStr("a").addStr("c").endField(); Document::UP doc1 = _b.endDocument(); - _inv.invertDocument(1, *doc1.get()); + _inv.invertDocument(1, *doc1.get(), {}); myPushDocument(_inv); _b.startDocument("id:ns:searchdocument::2"); _b.startIndexField("f0").addStr("b").addStr("c").endField(); Document::UP doc2 = _b.endDocument(); - _inv.invertDocument(2, *doc2.get()); + _inv.invertDocument(2, *doc2.get(), {}); myPushDocument(_inv); EXPECT_TRUE(assertPostingList("[1]", find("a", 0))); @@ -1308,7 +1308,7 @@ TEST_F(UriInverterTest, require_that_uri_indexing_is_working) endElement(). endField(); doc = _b.endDocument(); - _inv.invertDocument(10, *doc); + _inv.invertDocument(10, *doc, {}); myPushDocument(_inv); SimpleMatchData match_data; @@ -1381,7 +1381,7 @@ TEST_F(CjkInverterTest, require_that_cjk_indexing_is_working) setAutoSpace(true). endField(); doc = _b.endDocument(); - _inv.invertDocument(10, *doc); + _inv.invertDocument(10, *doc, {}); myPushDocument(_inv); SimpleMatchData match_data; diff --git a/searchlib/src/tests/memoryindex/memory_index/memory_index_test.cpp b/searchlib/src/tests/memoryindex/memory_index/memory_index_test.cpp index 20cfb045081..c806a1b02ac 100644 --- a/searchlib/src/tests/memoryindex/memory_index/memory_index_test.cpp +++ b/searchlib/src/tests/memoryindex/memory_index/memory_index_test.cpp @@ -107,7 +107,7 @@ struct Index { Document::UP commit() { closeField(); Document::UP d = builder.endDocument(); - index.insertDocument(docid, *d); + index.insertDocument(docid, *d, {}); internalSyncCommit(); return d; } diff --git a/searchlib/src/vespa/searchlib/memoryindex/document_inverter.cpp b/searchlib/src/vespa/searchlib/memoryindex/document_inverter.cpp index a7f0325d363..c0623aec9df 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/document_inverter.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/document_inverter.cpp @@ -59,13 +59,13 @@ DocumentInverter::~DocumentInverter() } void -DocumentInverter::invertDocument(uint32_t docId, const Document &doc) +DocumentInverter::invertDocument(uint32_t docId, const Document &doc, OnWriteDoneType on_write_done) { auto& invert_threads = _context.get_invert_threads(); auto& invert_contexts = _context.get_invert_contexts(); for (auto& invert_context : invert_contexts) { auto id = invert_context.get_id(); - auto task = std::make_unique<InvertTask>(_context, invert_context, _inverters, _urlInverters, docId, doc); + auto task = std::make_unique<InvertTask>(_context, invert_context, _inverters, _urlInverters, docId, doc, on_write_done); invert_threads.executeTask(id, std::move(task)); } } @@ -89,7 +89,7 @@ DocumentInverter::removeDocuments(LidVector lids) } void -DocumentInverter::pushDocuments(const std::shared_ptr<vespalib::IDestructorCallback> &onWriteDone) +DocumentInverter::pushDocuments(OnWriteDoneType on_write_done) { auto retain = std::make_shared<RetainGuard>(_ref_count); using PushTasks = std::vector<std::shared_ptr<ScheduleSequencedTaskCallback>>; @@ -97,7 +97,7 @@ DocumentInverter::pushDocuments(const std::shared_ptr<vespalib::IDestructorCallb auto& push_threads = _context.get_push_threads(); auto& push_contexts = _context.get_push_contexts(); for (auto& push_context : push_contexts) { - auto task = std::make_unique<PushTask>(push_context, _inverters, _urlInverters, onWriteDone, retain); + auto task = std::make_unique<PushTask>(push_context, _inverters, _urlInverters, on_write_done, retain); all_push_tasks.emplace_back(std::make_shared<ScheduleSequencedTaskCallback>(push_threads, push_context.get_id(), std::move(task))); } auto& invert_threads = _context.get_invert_threads(); diff --git a/searchlib/src/vespa/searchlib/memoryindex/document_inverter.h b/searchlib/src/vespa/searchlib/memoryindex/document_inverter.h index dc4fba5d6b0..d89bdad5bb8 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/document_inverter.h +++ b/searchlib/src/vespa/searchlib/memoryindex/document_inverter.h @@ -16,9 +16,7 @@ namespace document { class FieldValue; } -namespace vespalib { - class IDestructorCallback; -} +namespace vespalib { class IDestructorCallback; } namespace search::memoryindex { @@ -40,6 +38,7 @@ private: DocumentInverterContext& _context; using LidVector = std::vector<uint32_t>; + using OnWriteDoneType = const std::shared_ptr<vespalib::IDestructorCallback> &; std::vector<std::unique_ptr<FieldInverter>> _inverters; std::vector<std::unique_ptr<UrlFieldInverter>> _urlInverters; @@ -61,13 +60,13 @@ public: * This function is async: * For each field inverter a task for pushing the inverted documents to the corresponding field index * is added to the 'push threads' executor, then this function returns. - * All tasks hold a reference to the 'onWriteDone' callback, so when the last task is completed, + * All tasks hold a reference to the 'on_write_done' callback, so when the last task is completed, * the callback is destructed. * * NOTE: The caller of this function should sync the 'invert threads' executor first, * to ensure that inverting is completed before pushing starts. */ - void pushDocuments(const std::shared_ptr<vespalib::IDestructorCallback> &onWriteDone); + void pushDocuments(OnWriteDoneType on_write_done); /** * Invert (add) the given document. @@ -76,7 +75,7 @@ public: * For each text and uri field in the document a task for inverting and adding that * field (using a field inverter) is added to the 'invert threads' executor, then this function returns. **/ - void invertDocument(uint32_t docId, const document::Document &doc); + void invertDocument(uint32_t docId, const document::Document &doc, OnWriteDoneType on_write_done); /** * Remove the given document. diff --git a/searchlib/src/vespa/searchlib/memoryindex/invert_task.cpp b/searchlib/src/vespa/searchlib/memoryindex/invert_task.cpp index 223017ed149..f120569f615 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/invert_task.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/invert_task.cpp @@ -24,14 +24,15 @@ get_field_value(const Document& doc, const std::unique_ptr<const Field>& field) } -InvertTask::InvertTask(const DocumentInverterContext& inv_context, const InvertContext& context, const std::vector<std::unique_ptr<FieldInverter>>& inverters, const std::vector<std::unique_ptr<UrlFieldInverter>>& uri_inverters, uint32_t lid, const document::Document& doc) +InvertTask::InvertTask(const DocumentInverterContext& inv_context, const InvertContext& context, const std::vector<std::unique_ptr<FieldInverter>>& inverters, const std::vector<std::unique_ptr<UrlFieldInverter>>& uri_inverters, uint32_t lid, const document::Document& doc, OnWriteDoneType on_write_done) : _inv_context(inv_context), _context(context), _inverters(inverters), _uri_inverters(uri_inverters), _field_values(), _uri_field_values(), - _lid(lid) + _lid(lid), + _on_write_done(on_write_done) { _context.set_data_type(_inv_context, doc); _field_values.reserve(_context.get_fields().size()); diff --git a/searchlib/src/vespa/searchlib/memoryindex/invert_task.h b/searchlib/src/vespa/searchlib/memoryindex/invert_task.h index 48f999a963d..b97e6739212 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/invert_task.h +++ b/searchlib/src/vespa/searchlib/memoryindex/invert_task.h @@ -9,6 +9,7 @@ namespace document { class Document; class FieldValue; } +namespace vespalib { class IDestructorCallback; } namespace search::memoryindex { @@ -23,6 +24,7 @@ class UrlFieldInverter; */ class InvertTask : public vespalib::Executor::Task { + using OnWriteDoneType = const std::shared_ptr<vespalib::IDestructorCallback> &; const DocumentInverterContext& _inv_context; const InvertContext& _context; const std::vector<std::unique_ptr<FieldInverter>>& _inverters; @@ -30,8 +32,9 @@ class InvertTask : public vespalib::Executor::Task std::vector<std::unique_ptr<document::FieldValue>> _field_values; std::vector<std::unique_ptr<document::FieldValue>> _uri_field_values; uint32_t _lid; + std::remove_reference_t<OnWriteDoneType> _on_write_done; public: - InvertTask(const DocumentInverterContext& inv_context, const InvertContext& context, const std::vector<std::unique_ptr<FieldInverter>>& inverters, const std::vector<std::unique_ptr<UrlFieldInverter>>& uri_inverters, uint32_t lid, const document::Document& doc); + InvertTask(const DocumentInverterContext& inv_context, const InvertContext& context, const std::vector<std::unique_ptr<FieldInverter>>& inverters, const std::vector<std::unique_ptr<UrlFieldInverter>>& uri_inverters, uint32_t lid, const document::Document& doc, OnWriteDoneType on_write_done); ~InvertTask() override; void run() override; }; diff --git a/searchlib/src/vespa/searchlib/memoryindex/memory_index.cpp b/searchlib/src/vespa/searchlib/memoryindex/memory_index.cpp index 15cdef2f664..a9d935aee3b 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/memory_index.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/memory_index.cpp @@ -79,7 +79,7 @@ MemoryIndex::~MemoryIndex() } void -MemoryIndex::insertDocument(uint32_t docId, const document::Document &doc) +MemoryIndex::insertDocument(uint32_t docId, const document::Document &doc, OnWriteDoneType on_write_done) { if (_frozen) { LOG(warning, "Memory index frozen: ignoring insert of document '%s'(%u): '%s'", @@ -88,7 +88,7 @@ MemoryIndex::insertDocument(uint32_t docId, const document::Document &doc) } updateMaxDocId(docId); auto& inverter = _inverters->get_active_inverter(); - inverter.invertDocument(docId, doc); + inverter.invertDocument(docId, doc, on_write_done); if (_indexedDocs.insert(docId).second) { incNumDocs(); } @@ -113,10 +113,10 @@ MemoryIndex::removeDocuments(LidVector lids) } void -MemoryIndex::commit(const std::shared_ptr<vespalib::IDestructorCallback> &onWriteDone) +MemoryIndex::commit(OnWriteDoneType on_write_done) { auto& inverter = _inverters->get_active_inverter(); - inverter.pushDocuments(onWriteDone); + inverter.pushDocuments(on_write_done); _inverters->switch_active_inverter(); } diff --git a/searchlib/src/vespa/searchlib/memoryindex/memory_index.h b/searchlib/src/vespa/searchlib/memoryindex/memory_index.h index 760a4ecfb0f..6953aee7875 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/memory_index.h +++ b/searchlib/src/vespa/searchlib/memoryindex/memory_index.h @@ -43,6 +43,7 @@ class MemoryIndex : public queryeval::Searchable { private: using ISequencedTaskExecutor = vespalib::ISequencedTaskExecutor; using LidVector = std::vector<uint32_t>; + using OnWriteDoneType = const std::shared_ptr<vespalib::IDestructorCallback> &; index::Schema _schema; ISequencedTaskExecutor &_invertThreads; ISequencedTaskExecutor &_pushThreads; @@ -107,7 +108,7 @@ public: * If the document is already in the index, the old version will be removed first. * This function is async. commit() must be called for changes to take effect. */ - void insertDocument(uint32_t docId, const document::Document &doc); + void insertDocument(uint32_t docId, const document::Document &doc, OnWriteDoneType on_write_done); /** * Remove a document from the underlying field indexes. @@ -119,11 +120,11 @@ public: /** * Commits the inserts and removes since the last commit, making them searchable. * - * When commit is completed, 'onWriteDone' goes out of scope, scheduling completion callback. + * When commit is completed, 'on_write_done' goes out of scope, scheduling completion callback. * * Callers can call pushThreads.sync() to wait for push completion. */ - void commit(const std::shared_ptr<vespalib::IDestructorCallback> &onWriteDone); + void commit(OnWriteDoneType on_write_done); /** * Freeze this index. diff --git a/searchlib/src/vespa/searchlib/memoryindex/push_task.cpp b/searchlib/src/vespa/searchlib/memoryindex/push_task.cpp index 0eca882836b..b68e23bfe02 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/push_task.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/push_task.cpp @@ -19,11 +19,11 @@ void push_inverter(Inverter& inverter) } -PushTask::PushTask(const PushContext& context, const std::vector<std::unique_ptr<FieldInverter>>& inverters, const std::vector<std::unique_ptr<UrlFieldInverter>>& uri_inverters, std::shared_ptr<vespalib::IDestructorCallback> on_write_done, std::shared_ptr<vespalib::RetainGuard> retain) +PushTask::PushTask(const PushContext& context, const std::vector<std::unique_ptr<FieldInverter>>& inverters, const std::vector<std::unique_ptr<UrlFieldInverter>>& uri_inverters, OnWriteDoneType on_write_done, std::shared_ptr<vespalib::RetainGuard> retain) : _context(context), _inverters(inverters), _uri_inverters(uri_inverters), - _on_write_done(std::move(on_write_done)), + _on_write_done(on_write_done), _retain(std::move(retain)) { } diff --git a/searchlib/src/vespa/searchlib/memoryindex/push_task.h b/searchlib/src/vespa/searchlib/memoryindex/push_task.h index 24474fb2003..002b9334b78 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/push_task.h +++ b/searchlib/src/vespa/searchlib/memoryindex/push_task.h @@ -22,13 +22,14 @@ class UrlFieldInverter; */ class PushTask : public vespalib::Executor::Task { + using OnWriteDoneType = const std::shared_ptr<vespalib::IDestructorCallback> &; const PushContext& _context; const std::vector<std::unique_ptr<FieldInverter>>& _inverters; const std::vector<std::unique_ptr<UrlFieldInverter>>& _uri_inverters; - std::shared_ptr<vespalib::IDestructorCallback> _on_write_done; + std::remove_reference_t<OnWriteDoneType> _on_write_done; std::shared_ptr<vespalib::RetainGuard> _retain; public: - PushTask(const PushContext& context, const std::vector<std::unique_ptr<FieldInverter>>& inverters, const std::vector<std::unique_ptr<UrlFieldInverter>>& uri_inverters, std::shared_ptr<vespalib::IDestructorCallback> on_write_done, std::shared_ptr<vespalib::RetainGuard> retain); + PushTask(const PushContext& context, const std::vector<std::unique_ptr<FieldInverter>>& inverters, const std::vector<std::unique_ptr<UrlFieldInverter>>& uri_inverters, OnWriteDoneType on_write_done, std::shared_ptr<vespalib::RetainGuard> retain); ~PushTask() override; void run() override; }; |