diff options
author | Tor Egge <Tor.Egge@broadpark.no> | 2018-10-31 20:48:43 +0100 |
---|---|---|
committer | Tor Egge <tegge@oath.com> | 2019-01-08 18:30:29 +0000 |
commit | d04167de8b9cba666c2f03a5e53f5d167c8b4b50 (patch) | |
tree | 28c1601c4fa5212beafe30bcd8ba296525603e77 /searchcore | |
parent | 21809401b2244eaf1caeae54e501d97990abe43d (diff) |
Compact lid space on source selector.
Diffstat (limited to 'searchcore')
12 files changed, 80 insertions, 4 deletions
diff --git a/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp b/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp index c11bea14863..0f9df1fc594 100644 --- a/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp +++ b/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp @@ -140,12 +140,14 @@ struct MyIndexWriter : public test::MockIndexWriter MyLidVector _removes; int _heartBeatCount; uint32_t _commitCount; + uint32_t _wantedLidLimit; MyTracer &_tracer; MyIndexWriter(MyTracer &tracer) : test::MockIndexWriter(IIndexManager::SP(new test::MockIndexManager())), _removes(), _heartBeatCount(0), _commitCount(0), + _wantedLidLimit(0), _tracer(tracer) {} virtual void put(SerialNum serialNum, const document::Document &doc, @@ -164,6 +166,9 @@ struct MyIndexWriter : public test::MockIndexWriter _tracer.traceCommit(indexAdapterTypeName, serialNum); } virtual void heartBeat(SerialNum) override { ++_heartBeatCount; } + void compactLidSpace(SerialNum, uint32_t lidLimit) override { + _wantedLidLimit = lidLimit; + } }; struct MyGidToLidChangeHandler : public MockGidToLidChangeHandler @@ -1144,7 +1149,7 @@ TEST_F("require that compactLidSpace() propagates to document meta store and doc f.compactLidSpaceAndWait(2); // performIndexForceCommit in index thread, then completion callback // in master thread. - EXPECT_TRUE(assertThreadObserver(7, 4, 3, f.writeServiceObserver())); + EXPECT_TRUE(assertThreadObserver(7, 5, 3, f.writeServiceObserver())); EXPECT_EQUAL(2u, f.metaStoreObserver()._compactLidSpaceLidLimit); EXPECT_EQUAL(2u, f.getDocumentStore()._compactLidSpaceLidLimit); EXPECT_EQUAL(1u, f.metaStoreObserver()._holdUnblockShrinkLidSpaceCnt); @@ -1162,7 +1167,7 @@ TEST_F("require that compactLidSpace() doesn't propagate to " op.setSerialNum(0); f.runInMaster([&] () { f.fv.handleCompactLidSpace(op); }); // Delayed holdUnblockShrinkLidSpace() in index thread, then master thread - EXPECT_TRUE(assertThreadObserver(6, 3, 3, f.writeServiceObserver())); + EXPECT_TRUE(assertThreadObserver(6, 4, 3, f.writeServiceObserver())); EXPECT_EQUAL(0u, f.metaStoreObserver()._compactLidSpaceLidLimit); EXPECT_EQUAL(0u, f.getDocumentStore()._compactLidSpaceLidLimit); EXPECT_EQUAL(0u, f.metaStoreObserver()._holdUnblockShrinkLidSpaceCnt); @@ -1176,6 +1181,13 @@ TEST_F("require that compactLidSpace() propagates to attributeadapter", EXPECT_EQUAL(2u, f.maw._wantedLidLimit); } +TEST_F("require that compactLidSpace() propagates to index writer", + SearchableFeedViewFixture) +{ + f.populateBeforeCompactLidSpace(); + f.compactLidSpaceAndWait(2); + EXPECT_EQUAL(2u, f.miw._wantedLidLimit); +} TEST_F("require that commit is called if visibility delay is 0", SearchableFeedViewFixture) diff --git a/searchcore/src/tests/proton/index/index_writer/index_writer_test.cpp b/searchcore/src/tests/proton/index/index_writer/index_writer_test.cpp index 0d9fe42b978..8f5d998b77a 100644 --- a/searchcore/src/tests/proton/index/index_writer/index_writer_test.cpp +++ b/searchcore/src/tests/proton/index/index_writer/index_writer_test.cpp @@ -33,8 +33,11 @@ struct MyIndexManager : public test::MockIndexManager SerialNum current; SerialNum flushed; SerialNum commitSerial; + uint32_t wantedLidLimit; + SerialNum compactSerial; MyIndexManager() : puts(), removes(), current(0), flushed(0), - commitSerial(0) + commitSerial(0), + wantedLidLimit(0), compactSerial(0) { } std::string getPut(uint32_t lid) { @@ -62,6 +65,10 @@ struct MyIndexManager : public test::MockIndexManager virtual SerialNum getFlushedSerialNum() const override { return flushed; } + void compactLidSpace(uint32_t lidLimit, SerialNum serialNum) override { + wantedLidLimit = lidLimit; + compactSerial = serialNum; + } }; struct Fixture @@ -95,7 +102,7 @@ struct Fixture } }; -TEST_F("require that index adapter ignores old operations", Fixture) +TEST_F("require that index writer ignores old operations", Fixture) { f.mim.flushed = 10; f.put(8, 1); @@ -110,6 +117,13 @@ TEST_F("require that commit is forwarded to index manager", Fixture) EXPECT_EQUAL(10u, f.mim.commitSerial); } +TEST_F("require that compactLidSpace is forwarded to index manager", Fixture) +{ + f.iw.compactLidSpace(4, 2); + EXPECT_EQUAL(2u, f.mim.wantedLidLimit); + EXPECT_EQUAL(4u, f.mim.compactSerial); +} + TEST_MAIN() { TEST_RUN_ALL(); diff --git a/searchcore/src/tests/proton/index/indexmanager_test.cpp b/searchcore/src/tests/proton/index/indexmanager_test.cpp index c025bd257a2..d9f0a256faf 100644 --- a/searchcore/src/tests/proton/index/indexmanager_test.cpp +++ b/searchcore/src/tests/proton/index/indexmanager_test.cpp @@ -145,6 +145,14 @@ struct Fixture { }); _writeService.indexFieldWriter().sync(); } + void removeDocument(uint32_t docId) { + SerialNum serialNum = ++_serial_num; + removeDocument(docId, serialNum); + } + void compactLidSpace(uint32_t lidLimit) { + SerialNum serialNum = ++_serial_num; + runAsIndex([&]() { _index_manager->compactLidSpace(lidLimit, serialNum); }); + } void assertStats(uint32_t expNumDiskIndexes, uint32_t expNumMemoryIndexes, SerialNum expLastiskIndexSerialNum, @@ -715,6 +723,18 @@ TEST_F("require that indexes manager stats can be generated", Fixture) TEST_DO(f.assertStats(1, 1, 1, 2)); } +TEST_F("require that compactLidSpace works", Fixture) +{ + Schema empty_schema; + f.addDocument(1); + f.addDocument(2); + f.removeDocument(2); + auto fsc = f._index_manager->getMaintainer().getSourceCollection(); + EXPECT_EQUAL(3u, fsc->getSourceSelector().getDocIdLimit()); + f.compactLidSpace(2); + EXPECT_EQUAL(2u, fsc->getSourceSelector().getDocIdLimit()); +} + } // namespace TEST_MAIN() { diff --git a/searchcore/src/vespa/searchcore/proton/index/i_index_writer.h b/searchcore/src/vespa/searchcore/proton/index/i_index_writer.h index e42b8282941..9dc437e6460 100644 --- a/searchcore/src/vespa/searchcore/proton/index/i_index_writer.h +++ b/searchcore/src/vespa/searchcore/proton/index/i_index_writer.h @@ -27,6 +27,7 @@ public: virtual void remove(search::SerialNum serialNum, const search::DocumentIdT lid) = 0; virtual void commit(search::SerialNum serialNum, OnWriteDoneType onWriteDone) = 0; virtual void heartBeat(search::SerialNum serialNum) = 0; + virtual void compactLidSpace(search::SerialNum serialNum, const search::DocumentIdT lid) = 0; }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/index/index_writer.cpp b/searchcore/src/vespa/searchcore/proton/index/index_writer.cpp index 42480826d2b..ea7fcd99a3b 100644 --- a/searchcore/src/vespa/searchcore/proton/index/index_writer.cpp +++ b/searchcore/src/vespa/searchcore/proton/index/index_writer.cpp @@ -67,4 +67,10 @@ IndexWriter::heartBeat(search::SerialNum serialNum) _mgr->heartBeat(serialNum); } +void +IndexWriter::compactLidSpace(search::SerialNum serialNum, const search::DocumentIdT lid) +{ + _mgr->compactLidSpace(lid, serialNum); +} + } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/index/index_writer.h b/searchcore/src/vespa/searchcore/proton/index/index_writer.h index ca8c120ecb5..c231a43e06e 100644 --- a/searchcore/src/vespa/searchcore/proton/index/index_writer.h +++ b/searchcore/src/vespa/searchcore/proton/index/index_writer.h @@ -31,6 +31,7 @@ public: virtual void heartBeat(search::SerialNum serialNum) override; + void compactLidSpace(search::SerialNum serialNum, const search::DocumentIdT lid) override; }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/index/indexmanager.cpp b/searchcore/src/vespa/searchcore/proton/index/indexmanager.cpp index 82e32b0c26b..2139e34818f 100644 --- a/searchcore/src/vespa/searchcore/proton/index/indexmanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/index/indexmanager.cpp @@ -89,5 +89,11 @@ IndexManager::IndexManager(const vespalib::string &baseDir, IndexManager::~IndexManager() = default; +void +IndexManager::compactLidSpace(uint32_t lidLimit, SerialNum serialNum) +{ + _maintainer.compactLidSpace(lidLimit, serialNum); +} + } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/index/indexmanager.h b/searchcore/src/vespa/searchcore/proton/index/indexmanager.h index 15872583135..4f2d03c81e6 100644 --- a/searchcore/src/vespa/searchcore/proton/index/indexmanager.h +++ b/searchcore/src/vespa/searchcore/proton/index/indexmanager.h @@ -98,6 +98,7 @@ public: void heartBeat(SerialNum serialNum) override { _maintainer.heartBeat(serialNum); } + void compactLidSpace(uint32_t lidLimit, SerialNum serialNum) override; SerialNum getCurrentSerialNum() const override { return _maintainer.getCurrentSerialNum(); diff --git a/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.cpp b/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.cpp index 28b1c407429..eab118c1188 100644 --- a/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.cpp @@ -6,6 +6,7 @@ #include "removedonecontext.h" #include <vespa/searchcore/proton/common/feedtoken.h> #include <vespa/searchcore/proton/documentmetastore/ilidreusedelayer.h> +#include <vespa/searchcore/proton/feedoperation/compact_lid_space_operation.h> #include <vespa/searchlib/common/isequencedtaskexecutor.h> #include <vespa/vespalib/text/stringtokenizer.h> #include <vespa/vespalib/util/closuretask.h> @@ -204,6 +205,17 @@ SearchableFeedView::performIndexForceCommit(SerialNum serialNum, OnForceCommitDo } void +SearchableFeedView::handleCompactLidSpace(const CompactLidSpaceOperation &op) +{ + Parent::handleCompactLidSpace(op); + _writeService.index().execute( + makeLambdaTask([this, &op]() { + _indexWriter->compactLidSpace(op.getSerialNum(), op.getLidLimit()); + })); + _writeService.index().sync(); +} + +void SearchableFeedView::forceCommit(SerialNum serialNum, OnForceCommitDoneType onCommitDone) { Parent::forceCommit(serialNum, onCommitDone); diff --git a/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.h b/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.h index ed3ba6740b1..84de5d8c853 100644 --- a/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.h +++ b/searchcore/src/vespa/searchcore/proton/server/searchable_feed_view.h @@ -76,6 +76,7 @@ public: ~SearchableFeedView() override; const IIndexWriter::SP &getIndexWriter() const { return _indexWriter; } + void handleCompactLidSpace(const CompactLidSpaceOperation &op) override; void sync() override; }; diff --git a/searchcore/src/vespa/searchcore/proton/test/mock_index_manager.h b/searchcore/src/vespa/searchcore/proton/test/mock_index_manager.h index 9e105ef8a7f..6395ec05671 100644 --- a/searchcore/src/vespa/searchcore/proton/test/mock_index_manager.h +++ b/searchcore/src/vespa/searchcore/proton/test/mock_index_manager.h @@ -28,6 +28,7 @@ struct MockIndexManager : public searchcorespi::IIndexManager } virtual void setSchema(const Schema &, SerialNum) override {} virtual void heartBeat(SerialNum) override {} + void compactLidSpace(uint32_t, SerialNum) override {} virtual void setMaxFlushed(uint32_t) override { } }; diff --git a/searchcore/src/vespa/searchcore/proton/test/mock_index_writer.h b/searchcore/src/vespa/searchcore/proton/test/mock_index_writer.h index 6e2df177dd7..8fe180533dc 100644 --- a/searchcore/src/vespa/searchcore/proton/test/mock_index_writer.h +++ b/searchcore/src/vespa/searchcore/proton/test/mock_index_writer.h @@ -20,6 +20,7 @@ struct MockIndexWriter : public IIndexWriter virtual void remove(search::SerialNum, const search::DocumentIdT) override {} virtual void commit(search::SerialNum, OnWriteDoneType) override {} virtual void heartBeat(search::SerialNum) override {} + void compactLidSpace(search::SerialNum, const search::DocumentIdT) override {} }; } // namespace test |