diff options
17 files changed, 183 insertions, 63 deletions
diff --git a/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp b/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp index e948bb518d3..ba5d22dca28 100644 --- a/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp +++ b/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp @@ -20,6 +20,7 @@ #include <vespa/searchcore/proton/server/summaryadapter.h> #include <vespa/searchcore/proton/server/reconfig_params.h> #include <vespa/searchcore/proton/test/documentdb_config_builder.h> +#include <vespa/searchcore/proton/test/mock_summary_adapter.h> #include <vespa/searchlib/index/dummyfileheadercontext.h> #include <vespa/searchlib/transactionlog/nosyncproxy.h> #include <vespa/vespalib/io/fileutil.h> @@ -228,20 +229,7 @@ Fixture::initViewSet(ViewSet &views) } -struct MySummaryAdapter : public ISummaryAdapter -{ - virtual void put(search::SerialNum, const document::Document &, const search::DocumentIdT) {} - virtual void remove(search::SerialNum, const search::DocumentIdT) {} - virtual void heartBeat(search::SerialNum) {} - virtual const search::IDocumentStore &getDocumentStore() const { - const search::IDocumentStore *store = NULL; - return *store; - } - virtual std::unique_ptr<document::Document> get(const search::DocumentIdT, - const document::DocumentTypeRepo &) { - return std::unique_ptr<document::Document>(); - } -}; +using MySummaryAdapter = test::MockSummaryAdapter; struct MyFastAccessFeedView { diff --git a/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp b/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp index 5d932cb7dfc..60fc54cd388 100644 --- a/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp +++ b/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp @@ -22,6 +22,7 @@ LOG_SETUP("feedview_test"); #include <vespa/searchcore/proton/test/dummy_summary_manager.h> #include <vespa/searchcore/proton/test/mock_index_writer.h> #include <vespa/searchcore/proton/test/mock_index_manager.h> +#include <vespa/searchcore/proton/test/mock_summary_adapter.h> #include <vespa/searchcore/proton/test/thread_utils.h> #include <vespa/searchcore/proton/test/threading_service_observer.h> #include <vespa/searchlib/docstore/cachestats.h> @@ -181,10 +182,12 @@ struct MyDocumentStore : public test::DummyDocumentStore typedef std::map<DocumentIdT, document::Document::SP> DocMap; DocMap _docs; uint64_t _lastSyncToken; + uint32_t _compactLidSpaceLidLimit; MyDocumentStore() : test::DummyDocumentStore("."), _docs(), - _lastSyncToken(0) + _lastSyncToken(0), + _compactLidSpaceLidLimit(0) {} virtual Document::UP read(DocumentIdT lid, const document::DocumentTypeRepo &) const { DocMap::const_iterator itr = _docs.find(lid); @@ -206,6 +209,9 @@ struct MyDocumentStore : public test::DummyDocumentStore return syncToken; } virtual uint64_t lastSyncToken() const { return _lastSyncToken; } + virtual void compactLidSpace(uint32_t wantedDocLidLimit) override { + _compactLidSpaceLidLimit = wantedDocLidLimit; + } }; struct MySummaryManager : public test::DummySummaryManager @@ -215,11 +221,11 @@ struct MySummaryManager : public test::DummySummaryManager virtual search::IDocumentStore &getBackingStore() { return _store; } }; -struct MySummaryAdapter : public ISummaryAdapter +struct MySummaryAdapter : public test::MockSummaryAdapter { ISummaryManager::SP _sumMgr; MyDocumentStore &_store; - MyLidVector _removes; + MyLidVector _removes; MySummaryAdapter() : _sumMgr(new MySummaryManager()), @@ -227,26 +233,27 @@ struct MySummaryAdapter : public ISummaryAdapter _removes() { } - virtual void put(SerialNum serialNum, const document::Document &doc, - const DocumentIdT lid) { + virtual void put(SerialNum serialNum, const document::Document &doc, const DocumentIdT lid) override { (void) serialNum; _store.write(serialNum, doc, lid); } - virtual void remove(SerialNum serialNum, const DocumentIdT lid) { + virtual void remove(SerialNum serialNum, const DocumentIdT lid) override { LOG(info, "MySummaryAdapter::remove(): serialNum(%" PRIu64 "), docId(%u)", serialNum, lid); _store.remove(serialNum, lid); _removes.push_back(lid); } - virtual void heartBeat(SerialNum) {} - virtual const search::IDocumentStore &getDocumentStore() const { + virtual const search::IDocumentStore &getDocumentStore() const override { return _store; } virtual std::unique_ptr<document::Document> get(const search::DocumentIdT lid, - const document::DocumentTypeRepo &repo) { + const document::DocumentTypeRepo &repo) override { return _store.read(lid, repo); } + virtual void compactLidSpace(uint32_t wantedDocIdLimit) override { + _store.compactLidSpace(wantedDocIdLimit); + } }; struct MyAttributeWriter : public IAttributeWriter @@ -538,6 +545,9 @@ struct FixtureBase const IDocumentMetaStore &getMetaStore() const { return _dmsc->get(); } + const MyDocumentStore &getDocumentStore() const { + return msa._store; + } BucketDBOwner::Guard getBucketDB() const { return getMetaStore().getBucketDB().takeGuard(); @@ -1028,7 +1038,7 @@ TEST_F("require that update() to tensor attribute updates attribute and document requireThatUpdateUpdatesAttributeAndDocumentStore(f, "a3"); } -TEST_F("require that compactLidSpace() propagates to document meta store and " +TEST_F("require that compactLidSpace() propagates to document meta store and document store and " "blocks lid space shrinkage until generation is no longer used", SearchableFeedViewFixture) { @@ -1040,12 +1050,13 @@ TEST_F("require that compactLidSpace() propagates to document meta store and " // in master thread. EXPECT_TRUE(assertThreadObserver(3, 1, f.writeServiceObserver())); EXPECT_EQUAL(99u, f.metaStoreObserver()._compactLidSpaceLidLimit); + EXPECT_EQUAL(99u, f.getDocumentStore()._compactLidSpaceLidLimit); EXPECT_EQUAL(1u, f.metaStoreObserver()._holdUnblockShrinkLidSpaceCnt); EXPECT_EQUAL(99u, f._docIdLimit.get()); } TEST_F("require that compactLidSpace() doesn't propagate to " - "document meta store and " + "document meta store and document store and " "blocks lid space shrinkage until generation is no longer used", SearchableFeedViewFixture) { @@ -1056,6 +1067,7 @@ TEST_F("require that compactLidSpace() doesn't propagate to " // Delayed holdUnblockShrinkLidSpace() in index thread, then master thread EXPECT_TRUE(assertThreadObserver(2, 0, f.writeServiceObserver())); EXPECT_EQUAL(0u, f.metaStoreObserver()._compactLidSpaceLidLimit); + EXPECT_EQUAL(0u, f.getDocumentStore()._compactLidSpaceLidLimit); EXPECT_EQUAL(0u, f.metaStoreObserver()._holdUnblockShrinkLidSpaceCnt); } diff --git a/searchcore/src/tests/proton/documentdb/storeonlyfeedview/storeonlyfeedview_test.cpp b/searchcore/src/tests/proton/documentdb/storeonlyfeedview/storeonlyfeedview_test.cpp index 672bcd02529..eadc7566162 100644 --- a/searchcore/src/tests/proton/documentdb/storeonlyfeedview/storeonlyfeedview_test.cpp +++ b/searchcore/src/tests/proton/documentdb/storeonlyfeedview/storeonlyfeedview_test.cpp @@ -11,6 +11,7 @@ #include <vespa/searchcore/proton/server/executorthreadingservice.h> #include <vespa/searchcore/proton/server/storeonlyfeedview.h> #include <vespa/searchcore/proton/documentmetastore/lidreusedelayer.h> +#include <vespa/searchcore/proton/test/mock_summary_adapter.h> #include <vespa/searchcore/proton/test/thread_utils.h> #include <vespa/searchcore/proton/common/commit_time_tracker.h> #include <vespa/searchlib/index/docbuilder.h> @@ -37,7 +38,7 @@ using namespace proton; namespace { -class MySummaryAdapter : public ISummaryAdapter { +class MySummaryAdapter : public test::MockSummaryAdapter { int &_rm_count; int &_put_count; int &_heartbeat_count; @@ -48,15 +49,9 @@ public: _put_count(put_count), _heartbeat_count(heartbeat_count) { } - virtual void put(SerialNum, const Document &, DocumentIdT) - { ++ _put_count; } - virtual void remove(SerialNum, DocumentIdT) { ++_rm_count; } - virtual void heartBeat(SerialNum) { ++_heartbeat_count; } - virtual const search::IDocumentStore &getDocumentStore() const - { return *reinterpret_cast<const search::IDocumentStore *>(0); } - - virtual Document::UP get(const DocumentIdT, const DocumentTypeRepo &) - { return Document::UP(); } + virtual void put(SerialNum, const Document &, DocumentIdT) override { ++ _put_count; } + virtual void remove(SerialNum, DocumentIdT) override { ++_rm_count; } + virtual void heartBeat(SerialNum) override { ++_heartbeat_count; } }; DocumentTypeRepo::SP myGetDocumentTypeRepo() { diff --git a/searchcore/src/vespa/searchcore/proton/server/isummaryadapter.h b/searchcore/src/vespa/searchcore/proton/server/isummaryadapter.h index 4090cb970f2..165af7c3eee 100644 --- a/searchcore/src/vespa/searchcore/proton/server/isummaryadapter.h +++ b/searchcore/src/vespa/searchcore/proton/server/isummaryadapter.h @@ -35,6 +35,8 @@ public: virtual std::unique_ptr<document::Document> get(const search::DocumentIdT lid, const document::DocumentTypeRepo &repo) = 0; + + virtual void compactLidSpace(uint32_t wantedDocIdLimit) = 0; }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp index 31bcaca5014..caedb17fee3 100644 --- a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.cpp @@ -854,6 +854,9 @@ StoreOnlyFeedView::handleCompactLidSpace(const CompactLidSpaceOperation &op) commitContext->holdUnblockShrinkLidSpace(); forceCommit(serialNum, commitContext); } + if (useDocumentStore(serialNum)) { + _summaryAdapter->compactLidSpace(op.getLidLimit()); + } } const ISimpleDocumentMetaStore * diff --git a/searchcore/src/vespa/searchcore/proton/server/summaryadapter.h b/searchcore/src/vespa/searchcore/proton/server/summaryadapter.h index 3756ab04179..2e874ad4d25 100644 --- a/searchcore/src/vespa/searchcore/proton/server/summaryadapter.h +++ b/searchcore/src/vespa/searchcore/proton/server/summaryadapter.h @@ -22,25 +22,24 @@ public: */ virtual void put(search::SerialNum serialNum, const document::Document &doc, - const search::DocumentIdT lid); + const search::DocumentIdT lid) override; virtual void remove(search::SerialNum serialNum, - const search::DocumentIdT lid); + const search::DocumentIdT lid) override; - virtual void - heartBeat(search::SerialNum serialNum); + virtual void heartBeat(search::SerialNum serialNum) override; - virtual const search::IDocumentStore & - getDocumentStore(void) const - { + virtual const search::IDocumentStore &getDocumentStore() const { return _imgr->getBackingStore(); } - virtual std::unique_ptr<document::Document> - get(const search::DocumentIdT lid, - const document::DocumentTypeRepo &repo) - { + virtual std::unique_ptr<document::Document> get(const search::DocumentIdT lid, + const document::DocumentTypeRepo &repo) override { return _imgr->getBackingStore().read(lid, repo); } + + virtual void compactLidSpace(uint32_t wantedDocIdLimit) override { + _mgr->getBackingStore().compactLidSpace(wantedDocIdLimit); + } }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/test/dummy_document_store.h b/searchcore/src/vespa/searchcore/proton/test/dummy_document_store.h index 2b43546be86..94c644778e5 100644 --- a/searchcore/src/vespa/searchcore/proton/test/dummy_document_store.h +++ b/searchcore/src/vespa/searchcore/proton/test/dummy_document_store.h @@ -46,20 +46,19 @@ struct DummyDocumentStore : public search::IDocumentStore search::IDocumentStoreVisitorProgress &, const document::DocumentTypeRepo &) {} - virtual double - getVisitCost() const - { - return 1.0; - } + virtual double getVisitCost() const { return 1.0; } virtual search::DataStoreStorageStats getStorageStats() const override { return search::DataStoreStorageStats(0, 0, 0.0, 0, 0); } virtual search::MemoryUsage getMemoryUsage() const override { return search::MemoryUsage(); } - virtual std::vector<search::DataStoreFileChunkStats> - getFileChunkStats() const override { + virtual std::vector<search::DataStoreFileChunkStats> getFileChunkStats() const override { std::vector<search::DataStoreFileChunkStats> result; return result; } + + virtual void compactLidSpace(uint32_t wantedDocLidLimit) override { (void) wantedDocLidLimit; } + virtual bool canShrinkLidSpace() const override { return false; } + virtual void shrinkLidSpace() override {} }; } // namespace test diff --git a/searchcore/src/vespa/searchcore/proton/test/mock_summary_adapter.h b/searchcore/src/vespa/searchcore/proton/test/mock_summary_adapter.h new file mode 100644 index 00000000000..7fa750a9dde --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/test/mock_summary_adapter.h @@ -0,0 +1,33 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/searchcore/proton/server/isummaryadapter.h> + +namespace proton { + +namespace test { + +/** + * Mock of the ISummaryAdapter interface used for unit testing. + */ +struct MockSummaryAdapter : public ISummaryAdapter +{ + virtual void put(search::SerialNum, const document::Document &, const search::DocumentIdT) override {} + virtual void remove(search::SerialNum, const search::DocumentIdT) override {} + virtual void heartBeat(search::SerialNum) override {} + virtual const search::IDocumentStore &getDocumentStore() const override { + const search::IDocumentStore *store = NULL; + return *store; + } + virtual std::unique_ptr<document::Document> get(const search::DocumentIdT, + const document::DocumentTypeRepo &) override { + return std::unique_ptr<document::Document>(); + } + virtual void compactLidSpace(uint32_t wantedDocIdLimit) override { + (void) wantedDocIdLimit; + } +}; + +} // namespace test + +} // namespace proton diff --git a/searchlib/src/tests/docstore/document_store/document_store_test.cpp b/searchlib/src/tests/docstore/document_store/document_store_test.cpp index ef7f544e93e..45637828ab1 100644 --- a/searchlib/src/tests/docstore/document_store/document_store_test.cpp +++ b/searchlib/src/tests/docstore/document_store/document_store_test.cpp @@ -36,6 +36,9 @@ struct NullDataStore : IDataStore { std::vector<DataStoreFileChunkStats> result; return result; } + virtual void compactLidSpace(uint32_t wantedDocLidLimit) override { (void) wantedDocLidLimit; } + virtual bool canShrinkLidSpace() const override { return false; } + virtual void shrinkLidSpace() override {} }; TEST_FFF("require that uncache docstore lookups are counted", diff --git a/searchlib/src/vespa/searchlib/attribute/attributevector.h b/searchlib/src/vespa/searchlib/attribute/attributevector.h index c3d0e36184e..5011670bd41 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributevector.h +++ b/searchlib/src/vespa/searchlib/attribute/attributevector.h @@ -13,8 +13,9 @@ #include <vespa/searchcommon/common/undefinedvalues.h> #include <vespa/searchlib/common/address_space.h> #include <vespa/searchlib/common/bitvector.h> -#include <vespa/searchlib/common/range.h> +#include <vespa/searchlib/common/i_compactable_lid_space.h> #include <vespa/searchlib/common/identifiable.h> +#include <vespa/searchlib/common/range.h> #include <vespa/searchlib/common/rcuvector.h> #include <vespa/searchlib/queryeval/searchiterator.h> #include <vespa/vespalib/objects/identifiable.h> @@ -106,7 +107,8 @@ public: }; class AttributeVector : public vespalib::Identifiable, - public attribute::IAttributeVector + public attribute::IAttributeVector, + public common::ICompactableLidSpace { protected: using Config = search::attribute::Config; @@ -691,11 +693,11 @@ public: bool hasPostings(); virtual uint64_t getUniqueValueCount() const; virtual uint64_t getTotalValueCount() const; - virtual void compactLidSpace(uint32_t wantedLidLimit); + virtual void compactLidSpace(uint32_t wantedLidLimit) override; virtual void clearDocs(DocId lidLow, DocId lidLimit); bool wantShrinkLidSpace(void) const { return _committedDocIdLimit < getNumDocs(); } - virtual bool canShrinkLidSpace() const; - void shrinkLidSpace(); + virtual bool canShrinkLidSpace() const override; + virtual void shrinkLidSpace() override; virtual void onShrinkLidSpace(); void setInterlock(const std::shared_ptr<attribute::Interlock> &interlock); diff --git a/searchlib/src/vespa/searchlib/common/i_compactable_lid_space.h b/searchlib/src/vespa/searchlib/common/i_compactable_lid_space.h new file mode 100644 index 00000000000..65b04570dcb --- /dev/null +++ b/searchlib/src/vespa/searchlib/common/i_compactable_lid_space.h @@ -0,0 +1,33 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +namespace search { +namespace common { + +/** + * Interface for a component that has a lid space that can be compacted and shrinked. + */ +struct ICompactableLidSpace { + virtual ~ICompactableLidSpace() {} + + /** + * Compacts the lid space down to the wanted given doc id limit. + * After this, the remaining lid space is a candidate for shrinking (freeing of memory resources). + */ + virtual void compactLidSpace(uint32_t wantedDocLidLimit) = 0; + + /** + * Returns whether this lid space can be shrinked down to the wanted doc id limit. + */ + virtual bool canShrinkLidSpace() const = 0; + + /** + * Shrinks this lid space down to the wanted doc id limit (frees memory resources). + */ + virtual void shrinkLidSpace() = 0; +}; + +} +} + diff --git a/searchlib/src/vespa/searchlib/docstore/documentstore.cpp b/searchlib/src/vespa/searchlib/docstore/documentstore.cpp index 230da42244c..bf360784a56 100644 --- a/searchlib/src/vespa/searchlib/docstore/documentstore.cpp +++ b/searchlib/src/vespa/searchlib/docstore/documentstore.cpp @@ -508,5 +508,23 @@ CacheStats DocumentStore::getCacheStats() const { return singleStats; } +void +DocumentStore::compactLidSpace(uint32_t wantedDocLidLimit) +{ + _backingStore.compactLidSpace(wantedDocLidLimit); +} + +bool +DocumentStore::canShrinkLidSpace() const +{ + return _backingStore.canShrinkLidSpace(); +} + +void +DocumentStore::shrinkLidSpace() +{ + _backingStore.shrinkLidSpace(); +} + } // namespace search diff --git a/searchlib/src/vespa/searchlib/docstore/documentstore.h b/searchlib/src/vespa/searchlib/docstore/documentstore.h index 5d1b1f8137d..893e7e7c3d3 100644 --- a/searchlib/src/vespa/searchlib/docstore/documentstore.h +++ b/searchlib/src/vespa/searchlib/docstore/documentstore.h @@ -164,6 +164,13 @@ public: virtual std::vector<DataStoreFileChunkStats> getFileChunkStats() const override; + /** + * Implements common::ICompactableLidSpace + */ + virtual void compactLidSpace(uint32_t wantedDocLidLimit) override; + virtual bool canShrinkLidSpace() const override; + virtual void shrinkLidSpace() override; + private: bool useCache() const; diff --git a/searchlib/src/vespa/searchlib/docstore/idatastore.h b/searchlib/src/vespa/searchlib/docstore/idatastore.h index e09b50db08f..9351a0f7802 100644 --- a/searchlib/src/vespa/searchlib/docstore/idatastore.h +++ b/searchlib/src/vespa/searchlib/docstore/idatastore.h @@ -3,9 +3,10 @@ #pragma once #include "data_store_file_chunk_stats.h" -#include <vespa/vespalib/stllike/string.h> -#include <vespa/searchlib/util/memoryusage.h> #include <vespa/fastos/timestamp.h> +#include <vespa/searchlib/common/i_compactable_lid_space.h> +#include <vespa/searchlib/util/memoryusage.h> +#include <vespa/vespalib/stllike/string.h> #include <vector> namespace vespalib { class DataBuffer; } @@ -34,7 +35,7 @@ public: * Changes are held in memory until flush() is called. * A sync token is associated with each flush(). **/ -class IDataStore +class IDataStore : public common::ICompactableLidSpace { public: typedef std::vector<uint32_t> LidVector; diff --git a/searchlib/src/vespa/searchlib/docstore/idocumentstore.h b/searchlib/src/vespa/searchlib/docstore/idocumentstore.h index 2ce8c64a694..0d0e18a0d82 100644 --- a/searchlib/src/vespa/searchlib/docstore/idocumentstore.h +++ b/searchlib/src/vespa/searchlib/docstore/idocumentstore.h @@ -3,8 +3,9 @@ #pragma once #include <vespa/document/fieldvalue/document.h> -#include <vespa/searchlib/query/base.h> +#include <vespa/searchlib/common/i_compactable_lid_space.h> #include <vespa/searchlib/docstore/idatastore.h> +#include <vespa/searchlib/query/base.h> namespace search { @@ -47,7 +48,7 @@ private: * updates will be held in memory until flush() is called. * Uses a Local ID as key. **/ -class IDocumentStore +class IDocumentStore : public common::ICompactableLidSpace { public: /** diff --git a/searchlib/src/vespa/searchlib/docstore/logdatastore.cpp b/searchlib/src/vespa/searchlib/docstore/logdatastore.cpp index a3eb5ee03f7..703982472e9 100644 --- a/searchlib/src/vespa/searchlib/docstore/logdatastore.cpp +++ b/searchlib/src/vespa/searchlib/docstore/logdatastore.cpp @@ -1102,4 +1102,21 @@ LogDataStore::getFileChunkStats() const return std::move(result); } +void +LogDataStore::compactLidSpace(uint32_t wantedDocLidLimit) +{ + (void) wantedDocLidLimit; +} + +bool +LogDataStore::canShrinkLidSpace() const +{ + return false; +} + +void +LogDataStore::shrinkLidSpace() +{ +} + } // namespace search diff --git a/searchlib/src/vespa/searchlib/docstore/logdatastore.h b/searchlib/src/vespa/searchlib/docstore/logdatastore.h index 0fb5a582f5a..614d1103830 100644 --- a/searchlib/src/vespa/searchlib/docstore/logdatastore.h +++ b/searchlib/src/vespa/searchlib/docstore/logdatastore.h @@ -195,6 +195,13 @@ public: virtual std::vector<DataStoreFileChunkStats> getFileChunkStats() const override; + /** + * Implements common::ICompactableLidSpace + */ + virtual void compactLidSpace(uint32_t wantedDocLidLimit) override; + virtual bool canShrinkLidSpace() const override; + virtual void shrinkLidSpace() override; + private: class WrapVisitor; class WrapVisitorProgress; |