diff options
20 files changed, 128 insertions, 10 deletions
diff --git a/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp b/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp index c11bea14863..0f9df1fc594 100644 --- a/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp +++ b/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp @@ -140,12 +140,14 @@ struct MyIndexWriter : public test::MockIndexWriter MyLidVector _removes; int _heartBeatCount; uint32_t _commitCount; + uint32_t _wantedLidLimit; MyTracer &_tracer; MyIndexWriter(MyTracer &tracer) : test::MockIndexWriter(IIndexManager::SP(new test::MockIndexManager())), _removes(), _heartBeatCount(0), _commitCount(0), + _wantedLidLimit(0), _tracer(tracer) {} virtual void put(SerialNum serialNum, const document::Document &doc, @@ -164,6 +166,9 @@ struct MyIndexWriter : public test::MockIndexWriter _tracer.traceCommit(indexAdapterTypeName, serialNum); } virtual void heartBeat(SerialNum) override { ++_heartBeatCount; } + void compactLidSpace(SerialNum, uint32_t lidLimit) override { + _wantedLidLimit = lidLimit; + } }; struct MyGidToLidChangeHandler : public MockGidToLidChangeHandler @@ -1144,7 +1149,7 @@ TEST_F("require that compactLidSpace() propagates to document meta store and doc f.compactLidSpaceAndWait(2); // performIndexForceCommit in index thread, then completion callback // in master thread. - EXPECT_TRUE(assertThreadObserver(7, 4, 3, f.writeServiceObserver())); + EXPECT_TRUE(assertThreadObserver(7, 5, 3, f.writeServiceObserver())); EXPECT_EQUAL(2u, f.metaStoreObserver()._compactLidSpaceLidLimit); EXPECT_EQUAL(2u, f.getDocumentStore()._compactLidSpaceLidLimit); EXPECT_EQUAL(1u, f.metaStoreObserver()._holdUnblockShrinkLidSpaceCnt); @@ -1162,7 +1167,7 @@ TEST_F("require that compactLidSpace() doesn't propagate to " op.setSerialNum(0); f.runInMaster([&] () { f.fv.handleCompactLidSpace(op); }); // Delayed holdUnblockShrinkLidSpace() in index thread, then master thread - EXPECT_TRUE(assertThreadObserver(6, 3, 3, f.writeServiceObserver())); + EXPECT_TRUE(assertThreadObserver(6, 4, 3, f.writeServiceObserver())); EXPECT_EQUAL(0u, f.metaStoreObserver()._compactLidSpaceLidLimit); EXPECT_EQUAL(0u, f.getDocumentStore()._compactLidSpaceLidLimit); EXPECT_EQUAL(0u, f.metaStoreObserver()._holdUnblockShrinkLidSpaceCnt); @@ -1176,6 +1181,13 @@ TEST_F("require that compactLidSpace() propagates to attributeadapter", EXPECT_EQUAL(2u, f.maw._wantedLidLimit); } +TEST_F("require that compactLidSpace() propagates to index writer", + SearchableFeedViewFixture) +{ + f.populateBeforeCompactLidSpace(); + f.compactLidSpaceAndWait(2); + EXPECT_EQUAL(2u, f.miw._wantedLidLimit); +} TEST_F("require that commit is called if visibility delay is 0", SearchableFeedViewFixture) diff --git a/searchcore/src/tests/proton/index/index_writer/index_writer_test.cpp b/searchcore/src/tests/proton/index/index_writer/index_writer_test.cpp index 0d9fe42b978..8f5d998b77a 100644 --- a/searchcore/src/tests/proton/index/index_writer/index_writer_test.cpp +++ b/searchcore/src/tests/proton/index/index_writer/index_writer_test.cpp @@ -33,8 +33,11 @@ struct MyIndexManager : public test::MockIndexManager SerialNum current; SerialNum flushed; SerialNum commitSerial; + uint32_t wantedLidLimit; + SerialNum compactSerial; MyIndexManager() : puts(), removes(), current(0), flushed(0), - commitSerial(0) + commitSerial(0), + wantedLidLimit(0), compactSerial(0) { } std::string getPut(uint32_t lid) { @@ -62,6 +65,10 @@ struct MyIndexManager : public test::MockIndexManager virtual SerialNum getFlushedSerialNum() const override { return flushed; } + void compactLidSpace(uint32_t lidLimit, SerialNum serialNum) override { + wantedLidLimit = lidLimit; + compactSerial = serialNum; + } }; struct Fixture @@ -95,7 +102,7 @@ struct Fixture } }; -TEST_F("require that index adapter ignores old operations", Fixture) +TEST_F("require that index writer ignores old operations", Fixture) { f.mim.flushed = 10; f.put(8, 1); @@ -110,6 +117,13 @@ TEST_F("require that commit is forwarded to index manager", Fixture) EXPECT_EQUAL(10u, f.mim.commitSerial); } +TEST_F("require that compactLidSpace is forwarded to index manager", Fixture) +{ + f.iw.compactLidSpace(4, 2); + EXPECT_EQUAL(2u, f.mim.wantedLidLimit); + EXPECT_EQUAL(4u, f.mim.compactSerial); +} + TEST_MAIN() { TEST_RUN_ALL(); diff --git a/searchcore/src/tests/proton/index/indexmanager_test.cpp b/searchcore/src/tests/proton/index/indexmanager_test.cpp index c025bd257a2..d9f0a256faf 100644 --- a/searchcore/src/tests/proton/index/indexmanager_test.cpp +++ b/searchcore/src/tests/proton/index/indexmanager_test.cpp @@ -145,6 +145,14 @@ struct Fixture { }); _writeService.indexFieldWriter().sync(); } + void removeDocument(uint32_t docId) { + SerialNum serialNum = ++_serial_num; + removeDocument(docId, serialNum); + } + void compactLidSpace(uint32_t lidLimit) { + SerialNum serialNum = ++_serial_num; + runAsIndex([&]() { _index_manager->compactLidSpace(lidLimit, serialNum); }); + } void assertStats(uint32_t expNumDiskIndexes, uint32_t expNumMemoryIndexes, SerialNum expLastiskIndexSerialNum, @@ -715,6 +723,18 @@ TEST_F("require that indexes manager stats can be generated", Fixture) TEST_DO(f.assertStats(1, 1, 1, 2)); } +TEST_F("require that compactLidSpace works", Fixture) +{ + Schema empty_schema; + f.addDocument(1); + f.addDocument(2); + f.removeDocument(2); + auto fsc = f._index_manager->getMaintainer().getSourceCollection(); + EXPECT_EQUAL(3u, fsc->getSourceSelector().getDocIdLimit()); + f.compactLidSpace(2); + EXPECT_EQUAL(2u, fsc->getSourceSelector().getDocIdLimit()); +} + } // namespace TEST_MAIN() { diff --git a/searchcore/src/vespa/searchcore/proton/index/i_index_writer.h b/searchcore/src/vespa/searchcore/proton/index/i_index_writer.h index e42b8282941..9dc437e6460 100644 --- a/searchcore/src/vespa/searchcore/proton/index/i_index_writer.h +++ b/searchcore/src/vespa/searchcore/proton/index/i_index_writer.h @@ -27,6 +27,7 @@ public: virtual void remove(search::SerialNum serialNum, const search::DocumentIdT lid) = 0; virtual void commit(search::SerialNum serialNum, OnWriteDoneType onWriteDone) = 0; virtual void heartBeat(search::SerialNum serialNum) = 0; + virtual void compactLidSpace(search::SerialNum serialNum, const search::DocumentIdT lid) = 0; }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/index/index_writer.cpp b/searchcore/src/vespa/searchcore/proton/index/index_writer.cpp index 42480826d2b..ea7fcd99a3b 100644 --- a/searchcore/src/vespa/searchcore/proton/index/index_writer.cpp +++ b/searchcore/src/vespa/searchcore/proton/index/index_writer.cpp @@ -67,4 +67,10 @@ IndexWriter::heartBeat(search::SerialNum serialNum) _mgr->heartBeat(serialNum); } +void +IndexWriter::compactLidSpace(search::SerialNum serialNum, const search::DocumentIdT lid) +{ + _mgr->compactLidSpace(lid, serialNum); +} + } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/index/index_writer.h b/searchcore/src/vespa/searchcore/proton/index/index_writer.h index ca8c120ecb5..c231a43e06e 100644 --- a/searchcore/src/vespa/searchcore/proton/index/index_writer.h +++ b/searchcore/src/vespa/searchcore/proton/index/index_writer.h @@ -31,6 +31,7 @@ public: virtual void heartBeat(search::SerialNum serialNum) override; + void compactLidSpace(search::SerialNum serialNum, const search::DocumentIdT lid) override; }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/index/indexmanager.cpp b/searchcore/src/vespa/searchcore/proton/index/indexmanager.cpp index 82e32b0c26b..2139e34818f 100644 --- a/searchcore/src/vespa/searchcore/proton/index/indexmanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/index/indexmanager.cpp @@ -89,5 +89,11 @@ IndexManager::IndexManager(const vespalib::string &baseDir, IndexManager::~IndexManager() = default; +void +IndexManager::compactLidSpace(uint32_t lidLimit, SerialNum serialNum) +{ + _maintainer.compactLidSpace(lidLimit, serialNum); +} + } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/index/indexmanager.h b/searchcore/src/vespa/searchcore/proton/index/indexmanager.h index 15872583135..4f2d03c81e6 100644 --- a/searchcore/src/vespa/searchcore/proton/index/indexmanager.h +++ b/searchcore/src/vespa/searchcore/proton/index/indexmanager.h @@ -98,6 +98,7 @@ public: void heartBeat(SerialNum serialNum) override { _maintainer.heartBeat(serialNum); } + void compactLidSpace(uint32_t lidLimit, SerialNum serialNum) override; SerialNum getCurrentSerialNum() const override { return _maintainer.getCurrentSerialNum(); diff --git a/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.cpp b/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.cpp index 28b1c407429..eab118c1188 100644 --- a/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.cpp @@ -6,6 +6,7 @@ #include "removedonecontext.h" #include <vespa/searchcore/proton/common/feedtoken.h> #include <vespa/searchcore/proton/documentmetastore/ilidreusedelayer.h> +#include <vespa/searchcore/proton/feedoperation/compact_lid_space_operation.h> #include <vespa/searchlib/common/isequencedtaskexecutor.h> #include <vespa/vespalib/text/stringtokenizer.h> #include <vespa/vespalib/util/closuretask.h> @@ -204,6 +205,17 @@ SearchableFeedView::performIndexForceCommit(SerialNum serialNum, OnForceCommitDo } void +SearchableFeedView::handleCompactLidSpace(const CompactLidSpaceOperation &op) +{ + Parent::handleCompactLidSpace(op); + _writeService.index().execute( + makeLambdaTask([this, &op]() { + _indexWriter->compactLidSpace(op.getSerialNum(), op.getLidLimit()); + })); + _writeService.index().sync(); +} + +void SearchableFeedView::forceCommit(SerialNum serialNum, OnForceCommitDoneType onCommitDone) { Parent::forceCommit(serialNum, onCommitDone); diff --git a/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.h b/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.h index ed3ba6740b1..84de5d8c853 100644 --- a/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.h +++ b/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.h @@ -76,6 +76,7 @@ public: ~SearchableFeedView() override; const IIndexWriter::SP &getIndexWriter() const { return _indexWriter; } + void handleCompactLidSpace(const CompactLidSpaceOperation &op) override; void sync() override; }; diff --git a/searchcore/src/vespa/searchcore/proton/test/mock_index_manager.h b/searchcore/src/vespa/searchcore/proton/test/mock_index_manager.h index 9e105ef8a7f..6395ec05671 100644 --- a/searchcore/src/vespa/searchcore/proton/test/mock_index_manager.h +++ b/searchcore/src/vespa/searchcore/proton/test/mock_index_manager.h @@ -28,6 +28,7 @@ struct MockIndexManager : public searchcorespi::IIndexManager } virtual void setSchema(const Schema &, SerialNum) override {} virtual void heartBeat(SerialNum) override {} + void compactLidSpace(uint32_t, SerialNum) override {} virtual void setMaxFlushed(uint32_t) override { } }; diff --git a/searchcore/src/vespa/searchcore/proton/test/mock_index_writer.h b/searchcore/src/vespa/searchcore/proton/test/mock_index_writer.h index 6e2df177dd7..8fe180533dc 100644 --- a/searchcore/src/vespa/searchcore/proton/test/mock_index_writer.h +++ b/searchcore/src/vespa/searchcore/proton/test/mock_index_writer.h @@ -20,6 +20,7 @@ struct MockIndexWriter : public IIndexWriter virtual void remove(search::SerialNum, const search::DocumentIdT) override {} virtual void commit(search::SerialNum, OnWriteDoneType) override {} virtual void heartBeat(search::SerialNum) override {} + void compactLidSpace(search::SerialNum, const search::DocumentIdT) override {} }; } // namespace test diff --git a/searchcorespi/src/tests/plugin/plugin.cpp b/searchcorespi/src/tests/plugin/plugin.cpp index 5614ea1da3a..d32b02f45fd 100644 --- a/searchcorespi/src/tests/plugin/plugin.cpp +++ b/searchcorespi/src/tests/plugin/plugin.cpp @@ -20,6 +20,7 @@ public: virtual void removeDocument(uint32_t, SerialNum) override { } virtual void commit(SerialNum, OnWriteDoneType) override { } virtual void heartBeat(SerialNum ) override {} + void compactLidSpace(uint32_t, SerialNum) override {} virtual SerialNum getCurrentSerialNum() const override { return 0; } virtual SerialNum getFlushedSerialNum() const override { return 0; } virtual IndexSearchable::SP getSearchable() const override { diff --git a/searchcorespi/src/vespa/searchcorespi/index/iindexmanager.h b/searchcorespi/src/vespa/searchcorespi/index/iindexmanager.h index be439d499dd..cc07a940ef1 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/iindexmanager.h +++ b/searchcorespi/src/vespa/searchcorespi/index/iindexmanager.h @@ -108,6 +108,14 @@ public: virtual void heartBeat(SerialNum serialNum) = 0; /** + * This method is called when lid space is compacted. + * + * @param lidLimit The new lid limit. + * @param serialNum The serial number of the lid space compaction operation. + */ + virtual void compactLidSpace(uint32_t lidLimit, SerialNum serialNum) = 0; + + /** * Returns the current serial number of the index. * This should also reflect any heart beats. * diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp index 1e63741084d..6a7f4a14f2a 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp +++ b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp @@ -1173,6 +1173,16 @@ IndexMaintainer::heartBeat(SerialNum serialNum) _current_serial_num = serialNum; } +void +IndexMaintainer::compactLidSpace(uint32_t lidLimit, SerialNum serialNum) +{ + assert(_ctx.getThreadingService().index().isCurrentThread()); + LOG(info, "compactLidSpace(%u, %lu)", lidLimit, serialNum); + LockGuard lock(_index_update_lock); + _current_serial_num = serialNum; + _selector->compactLidSpace(lidLimit); +} + IFlushTarget::List IndexMaintainer::getFlushTargets(void) { diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.h b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.h index fb0bc2e5e78..06f02c5b9ef 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.h +++ b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.h @@ -357,6 +357,7 @@ public: void removeDocument(uint32_t lid, SerialNum serialNum) override; void commit(SerialNum serialNum, OnWriteDoneType onWriteDone) override; void heartBeat(search::SerialNum serialNum) override; + void compactLidSpace(uint32_t lidLimit, SerialNum serialNum) override; SerialNum getCurrentSerialNum() const override { return _current_serial_num; diff --git a/searchlib/src/tests/attribute/sourceselector/sourceselector_test.cpp b/searchlib/src/tests/attribute/sourceselector/sourceselector_test.cpp index a223b87a73f..1eda0b7cb67 100644 --- a/searchlib/src/tests/attribute/sourceselector/sourceselector_test.cpp +++ b/searchlib/src/tests/attribute/sourceselector/sourceselector_test.cpp @@ -41,7 +41,7 @@ private: void requireThatSelectorCanCloneAndSubtract(); void requireThatSelectorCanCloneAndSubtract(); template <typename SelectorType> - void requireThatSelectorCanSaveAndLoad(); + void requireThatSelectorCanSaveAndLoad(bool compactLidSpace); void requireThatSelectorCanSaveAndLoad(); template <typename SelectorType> void requireThatCompleteSourceRangeIsHandled(); @@ -140,12 +140,15 @@ Test::requireThatSelectorCanCloneAndSubtract() template <typename SelectorType> void -Test::requireThatSelectorCanSaveAndLoad() +Test::requireThatSelectorCanSaveAndLoad(bool compactLidSpace) { SelectorType selector(default_source, base_file_name2); setSources(selector); selector.setBaseId(base_id); selector.setSource(maxDocId + 1, default_source); + if (compactLidSpace) { + selector.compactLidSpace(maxDocId - 4); + } FastOS_FileInterface::EmptyAndRemoveDirectory(index_dir.c_str()); FastOS_FileInterface::MakeDirIfNotPresentOrExit(index_dir.c_str()); @@ -155,9 +158,13 @@ Test::requireThatSelectorCanSaveAndLoad() save_info->save(TuneFileAttributes(), DummyFileHeaderContext()); typename SelectorType::UP selector2(SelectorType::load(base_file_name)); - testSourceSelector(docs, arraysize(docs), default_source, *selector2); + testSourceSelector(docs, arraysize(docs) - compactLidSpace, default_source, *selector2); EXPECT_EQUAL(base_id, selector2->getBaseId()); - EXPECT_EQUAL(maxDocId + 2, selector2->getDocIdLimit()); + if (compactLidSpace) { + EXPECT_EQUAL(maxDocId - 4, selector2->getDocIdLimit()); + } else { + EXPECT_EQUAL(maxDocId + 2, selector2->getDocIdLimit()); + } FastOS_FileInterface::EmptyAndRemoveDirectory(index_dir.c_str()); } @@ -165,7 +172,8 @@ Test::requireThatSelectorCanSaveAndLoad() void Test::requireThatSelectorCanSaveAndLoad() { - requireThatSelectorCanSaveAndLoad<FixedSourceSelector>(); + requireThatSelectorCanSaveAndLoad<FixedSourceSelector>(false); + requireThatSelectorCanSaveAndLoad<FixedSourceSelector>(true); } template <typename SelectorType> diff --git a/searchlib/src/vespa/searchlib/attribute/fixedsourceselector.cpp b/searchlib/src/vespa/searchlib/attribute/fixedsourceselector.cpp index ae2f0234e60..385ffc95c40 100644 --- a/searchlib/src/vespa/searchlib/attribute/fixedsourceselector.cpp +++ b/searchlib/src/vespa/searchlib/attribute/fixedsourceselector.cpp @@ -88,4 +88,10 @@ FixedSourceSelector::setSource(uint32_t docId, queryeval::Source source) _source.commit(); } +void +FixedSourceSelector::compactLidSpace(uint32_t lidLimit) +{ + _source.compactLidSpace(lidLimit + 1); +} + } // namespace search diff --git a/searchlib/src/vespa/searchlib/attribute/fixedsourceselector.h b/searchlib/src/vespa/searchlib/attribute/fixedsourceselector.h index 4dbfc22c270..f92bb1e0eb2 100644 --- a/searchlib/src/vespa/searchlib/attribute/fixedsourceselector.h +++ b/searchlib/src/vespa/searchlib/attribute/fixedsourceselector.h @@ -38,8 +38,9 @@ public: // Inherit doc from ISourceSelector void setSource(uint32_t docId, queryeval::Source source) final override; uint32_t getDocIdLimit() const final override { - return _source.getNumDocs() - 1; + return _source.getCommittedDocIdLimit() - 1; } + void compactLidSpace(uint32_t lidLimit) override; std::unique_ptr<IIterator> createIterator() const final override { return std::make_unique<Iterator>(*this); } diff --git a/searchlib/src/vespa/searchlib/queryeval/isourceselector.h b/searchlib/src/vespa/searchlib/queryeval/isourceselector.h index 88a3cb57a8a..7151a785c26 100644 --- a/searchlib/src/vespa/searchlib/queryeval/isourceselector.h +++ b/searchlib/src/vespa/searchlib/queryeval/isourceselector.h @@ -73,6 +73,13 @@ public: virtual uint32_t getDocIdLimit() const = 0; /** + * Sets the lid limit in this selector. + * + * @param lidLimit the new lid limit (one above highest valid doc id). + */ + virtual void compactLidSpace(uint32_t lidLimit) = 0; + + /** * Create a new iterator over the data held by this source * selector. * |