diff options
32 files changed, 83 insertions, 29 deletions
diff --git a/searchcore/src/tests/proton/docsummary/CMakeLists.txt b/searchcore/src/tests/proton/docsummary/CMakeLists.txt index e1b0da71c34..906a1e642f5 100644 --- a/searchcore/src/tests/proton/docsummary/CMakeLists.txt +++ b/searchcore/src/tests/proton/docsummary/CMakeLists.txt @@ -3,6 +3,7 @@ vespa_add_executable(searchcore_docsummary_test_app TEST SOURCES docsummary.cpp DEPENDS + searchcore_test searchcore_server searchcore_initializer searchcore_reprocessing diff --git a/searchcore/src/tests/proton/docsummary/docsummary.cpp b/searchcore/src/tests/proton/docsummary/docsummary.cpp index 74cfa89c44a..3b199d266a8 100644 --- a/searchcore/src/tests/proton/docsummary/docsummary.cpp +++ b/searchcore/src/tests/proton/docsummary/docsummary.cpp @@ -7,7 +7,7 @@ #include <vespa/eval/tensor/tensor_factory.h> #include <vespa/persistence/spi/test.h> #include <vespa/searchcore/proton/attribute/attribute_writer.h> -#include <vespa/searchcore/proton/common/bucketfactory.h> +#include <vespa/searchcore/proton/test/bucketfactory.h> #include <vespa/searchcore/proton/docsummary/docsumcontext.h> #include <vespa/searchcore/proton/docsummary/documentstoreadapter.h> #include <vespa/searchcore/proton/docsummary/summarymanager.h> diff --git a/searchcore/src/tests/proton/documentdb/combiningfeedview/combiningfeedview_test.cpp b/searchcore/src/tests/proton/documentdb/combiningfeedview/combiningfeedview_test.cpp index 123002ecfd2..d4af7b214b6 100644 --- a/searchcore/src/tests/proton/documentdb/combiningfeedview/combiningfeedview_test.cpp +++ b/searchcore/src/tests/proton/documentdb/combiningfeedview/combiningfeedview_test.cpp @@ -2,6 +2,7 @@ #include <vespa/log/log.h> LOG_SETUP("combiningfeedview_test"); +#include <vespa/persistence/spi/test.h> #include <vespa/searchcore/proton/feedoperation/moveoperation.h> #include <vespa/searchcore/proton/server/combiningfeedview.h> #include <vespa/searchcore/proton/test/test.h> @@ -13,6 +14,7 @@ using document::DocumentUpdate; using search::IDestructorCallback; using search::SerialNum; using storage::spi::Timestamp; +using storage::spi::test::makeBucketSpace; using namespace proton; typedef std::vector<IFeedView::SP> FeedViewVector; @@ -142,7 +144,7 @@ struct Fixture _removed(_builder.getRepo(), _bucketDB, SubDbType::REMOVED), _notReady(_builder.getRepo(), _bucketDB, SubDbType::NOTREADY), _calc(new test::BucketStateCalculator()), - _view(getVector(_ready, _removed, _notReady), _calc) + _view(getVector(_ready, _removed, _notReady), makeBucketSpace(), _calc) { _builder.createDoc(1, 1); _builder.createDoc(2, 2); diff --git a/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp b/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp index 60b1216dbe2..547e400cd76 100644 --- a/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp +++ b/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp @@ -1,5 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/persistence/spi/test.h> #include <vespa/searchcore/proton/attribute/imported_attributes_repo.h> #include <vespa/searchcore/proton/bucketdb/bucketdbhandler.h> #include <vespa/searchcore/proton/common/hw_info.h> @@ -51,6 +52,7 @@ using search::test::DirectoryHandler; using searchcorespi::IFlushTarget; using searchcorespi::index::IThreadingService; using storage::spi::Timestamp; +using storage::spi::test::makeBucketSpace; using vespa::config::search::core::ProtonConfig; using vespalib::mkdir; @@ -78,6 +80,7 @@ struct MySubDBOwner : public IDocumentSubDBOwner uint32_t _syncCnt; MySubDBOwner() : _syncCnt(0) {} void syncFeedView() override { ++_syncCnt; } + document::BucketSpace getBucketSpace() const override { return makeBucketSpace(); } vespalib::string getName() const override { return "owner"; } uint32_t getDistributionKey() const override { return -1; } }; diff --git a/searchcore/src/tests/proton/documentdb/documentbucketmover/documentbucketmover_test.cpp b/searchcore/src/tests/proton/documentdb/documentbucketmover/documentbucketmover_test.cpp index 5dbf5d523bc..4419e982abf 100644 --- a/searchcore/src/tests/proton/documentdb/documentbucketmover/documentbucketmover_test.cpp +++ b/searchcore/src/tests/proton/documentdb/documentbucketmover/documentbucketmover_test.cpp @@ -2,7 +2,7 @@ #include <vespa/searchcore/proton/bucketdb/bucketdbhandler.h> #include <vespa/searchcore/proton/bucketdb/bucket_create_notifier.h> -#include <vespa/searchcore/proton/common/bucketfactory.h> +#include <vespa/searchcore/proton/test/bucketfactory.h> #include <vespa/searchcore/proton/feedoperation/moveoperation.h> #include <vespa/searchcore/proton/server/bucketmovejob.h> #include <vespa/searchcore/proton/server/documentbucketmover.h> @@ -15,6 +15,7 @@ #include <vespa/searchcore/proton/test/disk_mem_usage_notifier.h> #include <vespa/searchcore/proton/test/test.h> #include <vespa/searchlib/index/docbuilder.h> +#include <vespa/persistence/spi/test.h> #include <vespa/vespalib/testkit/testapp.h> using namespace proton; @@ -31,6 +32,7 @@ using search::index::DocBuilder; using search::index::Schema; using storage::spi::BucketInfo; using storage::spi::Timestamp; +using storage::spi::test::makeBucketSpace; using vespalib::make_string; using BlockedReason = IBlockableMaintenanceJob::BlockedReason; @@ -622,7 +624,7 @@ ControllerFixtureBase::ControllerFixtureBase(const BlockableMaintenanceJobConfig _bmj(_calc, _moveHandler, _modifiedHandler, _ready._subDb, _notReady._subDb, _fbh, _bucketCreateNotifier, _clusterStateHandler, _bucketHandler, _diskMemUsageNotifier, blockableConfig, - "test"), + "test", makeBucketSpace()), _runner(_bmj) { } diff --git a/searchcore/src/tests/proton/documentdb/feedhandler/CMakeLists.txt b/searchcore/src/tests/proton/documentdb/feedhandler/CMakeLists.txt index 6215929faa0..8f93629b6a6 100644 --- a/searchcore/src/tests/proton/documentdb/feedhandler/CMakeLists.txt +++ b/searchcore/src/tests/proton/documentdb/feedhandler/CMakeLists.txt @@ -3,6 +3,7 @@ vespa_add_executable(searchcore_feedhandler_test_app TEST SOURCES feedhandler_test.cpp DEPENDS + searchcore_test searchcore_server searchcore_bucketdb searchcore_persistenceengine diff --git a/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp b/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp index 39a588f804d..cccbbededd1 100644 --- a/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp +++ b/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp @@ -6,7 +6,7 @@ #include <vespa/documentapi/messagebus/messages/updatedocumentreply.h> #include <vespa/persistence/spi/result.h> #include <vespa/searchcore/proton/bucketdb/bucketdbhandler.h> -#include <vespa/searchcore/proton/common/bucketfactory.h> +#include <vespa/searchcore/proton/test/bucketfactory.h> #include <vespa/searchcore/proton/common/feedtoken.h> #include <vespa/searchcore/proton/feedoperation/moveoperation.h> #include <vespa/searchcore/proton/feedoperation/pruneremoveddocumentsoperation.h> diff --git a/searchcore/src/tests/proton/documentdb/feedview/CMakeLists.txt b/searchcore/src/tests/proton/documentdb/feedview/CMakeLists.txt index 6d2dfb59172..ebf28f3bf16 100644 --- a/searchcore/src/tests/proton/documentdb/feedview/CMakeLists.txt +++ b/searchcore/src/tests/proton/documentdb/feedview/CMakeLists.txt @@ -3,6 +3,7 @@ vespa_add_executable(searchcore_feedview_test_app TEST SOURCES feedview_test.cpp DEPENDS + searchcore_test searchcore_server searchcore_index searchcore_feedoperation diff --git a/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp b/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp index eeec8122703..c820a9f392c 100644 --- a/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp +++ b/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp @@ -5,7 +5,7 @@ #include <vespa/documentapi/messagebus/messages/removedocumentreply.h> #include <vespa/documentapi/messagebus/messages/updatedocumentreply.h> #include <vespa/searchcore/proton/attribute/i_attribute_writer.h> -#include <vespa/searchcore/proton/common/bucketfactory.h> +#include <vespa/searchcore/proton/test/bucketfactory.h> #include <vespa/searchcore/proton/common/commit_time_tracker.h> #include <vespa/searchcore/proton/common/feedtoken.h> #include <vespa/searchcore/proton/documentmetastore/lidreusedelayer.h> diff --git a/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp b/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp index 0b48686bbdd..559dbb240a8 100644 --- a/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp +++ b/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp @@ -1,5 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/persistence/spi/test.h> #include <vespa/searchcore/proton/attribute/attribute_usage_filter.h> #include <vespa/searchcore/proton/attribute/i_attribute_manager.h> #include <vespa/searchcore/proton/common/doctypename.h> @@ -53,6 +54,7 @@ using search::IDestructorCallback; using search::SerialNum; using storage::spi::BucketInfo; using storage::spi::Timestamp; +using storage::spi::test::makeBucketSpace; using vespalib::Slime; using vespalib::makeClosure; using vespalib::makeTask; @@ -962,7 +964,7 @@ MaintenanceControllerFixture::injectMaintenanceJobs() { if (_injectDefaultJobs) { MaintenanceJobsInjector::injectJobs(_mc, *_mcCfg, _fh, _gsp, - _lscHandlers, _fh, _mc, _bucketCreateNotifier, _docTypeName.getName(), + _lscHandlers, _fh, _mc, _bucketCreateNotifier, _docTypeName.getName(), makeBucketSpace(), _fh, _fh, _bmc, _clusterStateHandler, _bucketHandler, _calc, _diskMemUsageNotifier, diff --git a/searchcore/src/tests/proton/matching/CMakeLists.txt b/searchcore/src/tests/proton/matching/CMakeLists.txt index f3ed2bd4833..14f3960c43e 100644 --- a/searchcore/src/tests/proton/matching/CMakeLists.txt +++ b/searchcore/src/tests/proton/matching/CMakeLists.txt @@ -3,6 +3,7 @@ vespa_add_executable(searchcore_matching_test_app TEST SOURCES matching_test.cpp DEPENDS + searchcore_test searchcore_server searchcore_fconfig searchcore_matching diff --git a/searchcore/src/tests/proton/matching/matching_test.cpp b/searchcore/src/tests/proton/matching/matching_test.cpp index 261827a1e06..27e677bb1dc 100644 --- a/searchcore/src/tests/proton/matching/matching_test.cpp +++ b/searchcore/src/tests/proton/matching/matching_test.cpp @@ -5,7 +5,7 @@ #include <vespa/document/base/globalid.h> #include <initializer_list> #include <vespa/searchcommon/attribute/iattributecontext.h> -#include <vespa/searchcore/proton/common/bucketfactory.h> +#include <vespa/searchcore/proton/test/bucketfactory.h> #include <vespa/searchcore/proton/documentmetastore/documentmetastore.h> #include <vespa/searchcore/proton/matching/error_constant_value.h> #include <vespa/searchcore/proton/matching/fakesearchcontext.h> diff --git a/searchcore/src/tests/proton/server/CMakeLists.txt b/searchcore/src/tests/proton/server/CMakeLists.txt index 7f97c6dcf92..31f67bdd0ac 100644 --- a/searchcore/src/tests/proton/server/CMakeLists.txt +++ b/searchcore/src/tests/proton/server/CMakeLists.txt @@ -33,6 +33,7 @@ vespa_add_executable(searchcore_feedstates_test_app TEST SOURCES feedstates_test.cpp DEPENDS + searchcore_test searchcore_server searchcore_bucketdb searchcore_persistenceengine diff --git a/searchcore/src/tests/proton/server/feedstates_test.cpp b/searchcore/src/tests/proton/server/feedstates_test.cpp index 7d409db690e..15c2fbe5a84 100644 --- a/searchcore/src/tests/proton/server/feedstates_test.cpp +++ b/searchcore/src/tests/proton/server/feedstates_test.cpp @@ -8,7 +8,7 @@ LOG_SETUP("feedstates_test"); #include <vespa/document/base/testdocrepo.h> #include <vespa/document/bucket/bucketid.h> #include <vespa/document/repo/documenttyperepo.h> -#include <vespa/searchcore/proton/common/bucketfactory.h> +#include <vespa/searchcore/proton/test/bucketfactory.h> #include <vespa/searchcore/proton/server/feedstates.h> #include <vespa/searchcore/proton/server/ireplayconfig.h> #include <vespa/searchcore/proton/server/memoryconfigstore.h> diff --git a/searchcore/src/vespa/searchcore/proton/common/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/common/CMakeLists.txt index cdca446f114..9aa6a87ea6c 100644 --- a/searchcore/src/vespa/searchcore/proton/common/CMakeLists.txt +++ b/searchcore/src/vespa/searchcore/proton/common/CMakeLists.txt @@ -3,7 +3,6 @@ vespa_add_library(searchcore_pcommon STATIC SOURCES attributefieldvaluenode.cpp attrupdate.cpp - bucketfactory.cpp cachedselect.cpp commit_time_tracker.cpp dbdocumentid.cpp diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp index 72f2ea8c34d..69a3a902af8 100644 --- a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp @@ -585,7 +585,7 @@ PersistenceEngine::getModifiedBuckets(BucketSpace bucketSpace) const MBV extraModifiedBuckets; { LockGuard guard(_lock); - extraModifiedBuckets.swap(_extraModifiedBuckets); + extraModifiedBuckets.swap(_extraModifiedBuckets[bucketSpace]); } HandlerSnapshot::UP snap = getHandlerSnapshot(bucketSpace); SynchronizedBucketIdListResultHandler resultHandler(snap->size() + extraModifiedBuckets.size()); @@ -699,13 +699,13 @@ PersistenceEngine::propagateSavedClusterState(IPersistenceHandler &handler) } void -PersistenceEngine::grabExtraModifiedBuckets(IPersistenceHandler &handler) +PersistenceEngine::grabExtraModifiedBuckets(BucketSpace bucketSpace, IPersistenceHandler &handler) { BucketIdListResultHandler resultHandler; handler.handleListBuckets(resultHandler); auto result = std::make_shared<BucketIdListResult>(resultHandler.getResult()); LockGuard guard(_lock); - _extraModifiedBuckets.push_back(result); + _extraModifiedBuckets[bucketSpace].push_back(result); } diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.h b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.h index d9a27f9e460..b2abc7911d7 100644 --- a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.h +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.h @@ -59,19 +59,25 @@ private: in_use(false), bucket_guards() {} }; + struct BucketSpaceHash { + std::size_t operator() (const document::BucketSpace &bucketSpace) const { return bucketSpace.getId(); } + }; + typedef std::map<IteratorId, IteratorEntry *> Iterators; typedef std::vector<std::shared_ptr<BucketIdListResult> > BucketIdListResultV; + using ExtraModifiedBuckets = std::unordered_map<BucketSpace, BucketIdListResultV, BucketSpaceHash>; + const ssize_t _defaultSerializedSize; const bool _ignoreMaxBytes; - mutable PersistenceHandlerMap _handlers; + PersistenceHandlerMap _handlers; vespalib::Lock _lock; Iterators _iterators; vespalib::Lock _iterators_lock; IPersistenceEngineOwner &_owner; const IResourceWriteFilter &_writeFilter; ClusterState::SP _clusterState; - mutable BucketIdListResultV _extraModifiedBuckets; + mutable ExtraModifiedBuckets _extraModifiedBuckets; mutable std::shared_timed_mutex _rwMutex; IPersistenceHandler::SP getHandler(document::BucketSpace bucketSpace, @@ -124,7 +130,7 @@ public: void destroyIterators(); void propagateSavedClusterState(IPersistenceHandler &handler); - void grabExtraModifiedBuckets(IPersistenceHandler &handler); + void grabExtraModifiedBuckets(BucketSpace bucketSpace, IPersistenceHandler &handler); void populateInitialBucketDB(BucketSpace bucketSpace, IPersistenceHandler &targetHandler); std::unique_lock<std::shared_timed_mutex> getWLock() const; }; diff --git a/searchcore/src/vespa/searchcore/proton/server/bucketmovejob.cpp b/searchcore/src/vespa/searchcore/proton/server/bucketmovejob.cpp index 7320b48140e..cb929e0a6c7 100644 --- a/searchcore/src/vespa/searchcore/proton/server/bucketmovejob.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/bucketmovejob.cpp @@ -67,7 +67,7 @@ BucketMoveJob::checkBucket(const BucketId &bucket, if (_calc->nodeRetired() && !isActive) { return; } - const bool shouldBeReady = _calc->shouldBeReady(document::Bucket(document::BucketSpace::placeHolder(), bucket)); + const bool shouldBeReady = _calc->shouldBeReady(document::Bucket(_bucketSpace, bucket)); const bool wantReady = shouldBeReady || isActive; LOG(spam, "checkBucket(): bucket(%s), shouldBeReady(%s), active(%s)", bucket.toString().c_str(), bool2str(shouldBeReady), bool2str(isActive)); @@ -160,7 +160,8 @@ BucketMoveJob(const IBucketStateCalculator::SP &calc, IBucketStateChangedNotifier &bucketStateChangedNotifier, IDiskMemUsageNotifier &diskMemUsageNotifier, const BlockableMaintenanceJobConfig &blockableConfig, - const vespalib::string &docTypeName) + const vespalib::string &docTypeName, + document::BucketSpace bucketSpace) : BlockableMaintenanceJob("move_buckets." + docTypeName, 0.0, 0.0, blockableConfig), IClusterStateChangedHandler(), IBucketFreezeListener(), @@ -177,6 +178,7 @@ BucketMoveJob(const IBucketStateCalculator::SP &calc, _scanPos(), _scanPass(FIRST_SCAN_PASS), _endPos(), + _bucketSpace(bucketSpace), _delayedBuckets(), _delayedBucketsFrozen(), _frozenBuckets(frozenBuckets), @@ -213,7 +215,7 @@ BucketMoveJob::maybeCancelMover(DocumentBucketMover &mover) if (!mover.bucketDone()) { bool ready = mover.getSource() == &_ready; if (isBlocked() || - _calc->shouldBeReady(document::Bucket(document::BucketSpace::placeHolder(), mover.getBucket())) == ready) { + _calc->shouldBeReady(document::Bucket(_bucketSpace, mover.getBucket())) == ready) { mover.cancel(); } } diff --git a/searchcore/src/vespa/searchcore/proton/server/bucketmovejob.h b/searchcore/src/vespa/searchcore/proton/server/bucketmovejob.h index 6b43af9d4c0..7147613caee 100644 --- a/searchcore/src/vespa/searchcore/proton/server/bucketmovejob.h +++ b/searchcore/src/vespa/searchcore/proton/server/bucketmovejob.h @@ -91,6 +91,7 @@ private: ScanPosition _scanPos; uint32_t _scanPass; ScanPosition _endPos; + document::BucketSpace _bucketSpace; typedef std::set<document::BucketId> DelayedBucketSet; @@ -147,7 +148,8 @@ public: IBucketStateChangedNotifier &bucketStateChangedNotifier, IDiskMemUsageNotifier &diskMemUsageNotifier, const BlockableMaintenanceJobConfig &blockableConfig, - const vespalib::string &docTypeName); + const vespalib::string &docTypeName, + document::BucketSpace bucketSpace); virtual ~BucketMoveJob(); diff --git a/searchcore/src/vespa/searchcore/proton/server/combiningfeedview.cpp b/searchcore/src/vespa/searchcore/proton/server/combiningfeedview.cpp index a9dfeb12ed7..30ee6e1ba75 100644 --- a/searchcore/src/vespa/searchcore/proton/server/combiningfeedview.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/combiningfeedview.cpp @@ -34,13 +34,15 @@ getRepo(const std::vector<IFeedView::SP> &views) }; CombiningFeedView::CombiningFeedView(const std::vector<IFeedView::SP> &views, + document::BucketSpace bucketSpace, const IBucketStateCalculator::SP &calc) : _repo(getRepo(views)), _views(views), _metaStores(), _calc(calc), _clusterUp(calc.get() != NULL && calc->clusterUp()), - _forceReady(!_clusterUp || !hasNotReadyFeedView()) + _forceReady(!_clusterUp || !hasNotReadyFeedView()), + _bucketSpace(bucketSpace) { _metaStores.reserve(views.size()); for (const auto &view : views) { @@ -273,7 +275,7 @@ CombiningFeedView::setCalculator(const IBucketStateCalculator::SP &newCalc) bool CombiningFeedView::shouldBeReady(const document::BucketId &bucket) const { - document::Bucket dbucket(document::BucketSpace::placeHolder(), bucket); + document::Bucket dbucket(_bucketSpace, bucket); LOG(debug, "shouldBeReady(%s): forceReady(%s), clusterUp(%s), calcReady(%s)", bucket.toString().c_str(), diff --git a/searchcore/src/vespa/searchcore/proton/server/combiningfeedview.h b/searchcore/src/vespa/searchcore/proton/server/combiningfeedview.h index ef8616a25eb..ea4ac64176a 100644 --- a/searchcore/src/vespa/searchcore/proton/server/combiningfeedview.h +++ b/searchcore/src/vespa/searchcore/proton/server/combiningfeedview.h @@ -29,6 +29,7 @@ private: IBucketStateCalculator::SP _calc; bool _clusterUp; bool _forceReady; + document::BucketSpace _bucketSpace; const ISimpleDocumentMetaStore * getDocumentMetaStorePtr() const override; @@ -59,6 +60,7 @@ public: typedef std::shared_ptr<CombiningFeedView> SP; CombiningFeedView(const std::vector<IFeedView::SP> &views, + document::BucketSpace bucketSpace, const IBucketStateCalculator::SP &calc); virtual ~CombiningFeedView(); diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp index e02a6eb7ee3..ada2c1b1069 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp @@ -914,6 +914,7 @@ DocumentDB::injectMaintenanceJobs(const DocumentDBMaintenanceConfig &config) _maintenanceController, // IFrozenBucketHandler _subDBs.getBucketCreateNotifier(), _docTypeName.getName(), + _bucketSpace, _feedHandler, // IPruneRemovedDocumentsHandler _feedHandler, // IDocumentMoveHandler _clusterStateHandler, // IBucketModifiedHandler @@ -1310,6 +1311,18 @@ DocumentDB::waitForOnlineState() _state.waitForOnlineState(); } +vespalib::string +DocumentDB::getName() const +{ + return _docTypeName.getName(); +} + +document::BucketSpace +DocumentDB::getBucketSpace() const +{ + return _bucketSpace; +} + uint32_t DocumentDB::getDistributionKey() const { diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.h b/searchcore/src/vespa/searchcore/proton/server/documentdb.h index b97b5e310ea..cd0335b7b75 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdb.h +++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.h @@ -394,12 +394,15 @@ public: bool getDelayedConfig() const { return _state.getDelayedConfig(); } void replayConfig(SerialNum serialNum) override; const DocTypeName & getDocTypeName() const { return _docTypeName; } - document::BucketSpace getBucketSpace() const { return _bucketSpace; } void newConfigSnapshot(DocumentDBConfig::SP snapshot); void reconfigure(const DocumentDBConfig::SP & snapshot) override; int64_t getActiveGeneration() const; + /* + * Implements IDocumentSubDBOwner + */ void syncFeedView() override; - vespalib::string getName() const override { return _docTypeName.getName(); } + document::BucketSpace getBucketSpace() const override; + vespalib::string getName() const override; uint32_t getDistributionKey() const override; /** diff --git a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp index a92c21576a2..c766cc89bb3 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp @@ -4,6 +4,7 @@ #include "commit_and_wait_document_retriever.h" #include "document_subdb_collection_initializer.h" #include "documentsubdbcollection.h" +#include "i_document_subdb_owner.h" #include "maintenancecontroller.h" #include "searchabledocsubdb.h" @@ -34,6 +35,7 @@ DocumentSubDBCollection::DocumentSubDBCollection( const ProtonConfig &protonCfg, const HwInfo &hwInfo) : _subDBs(), + _owner(owner), _calc(), _readySubDbId(0), _remSubDbId(1), @@ -274,7 +276,7 @@ DocumentSubDBCollection::getFeedView() IFeedView::SP newFeedView; assert(views.size() >= 1); if (views.size() > 1) { - return IFeedView::SP(new CombiningFeedView(views, _calc)); + return IFeedView::SP(new CombiningFeedView(views, _owner.getBucketSpace(), _calc)); } else { assert(views.front() != NULL); return views.front(); diff --git a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h index 3146a65d3a3..7290250c59e 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h +++ b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h @@ -62,6 +62,7 @@ private: using SessionManagerSP = std::shared_ptr<matching::SessionManager>; using IFlushTargetList = std::vector<std::shared_ptr<searchcorespi::IFlushTarget>>; SubDBVector _subDBs; + IDocumentSubDBOwner &_owner; IBucketStateCalculatorSP _calc; const uint32_t _readySubDbId; const uint32_t _remSubDbId; diff --git a/searchcore/src/vespa/searchcore/proton/server/i_document_subdb_owner.h b/searchcore/src/vespa/searchcore/proton/server/i_document_subdb_owner.h index 3fa205bfbfe..fa19e5c77c4 100644 --- a/searchcore/src/vespa/searchcore/proton/server/i_document_subdb_owner.h +++ b/searchcore/src/vespa/searchcore/proton/server/i_document_subdb_owner.h @@ -2,6 +2,7 @@ #pragma once #include <vespa/vespalib/stllike/string.h> +#include <vespa/document/bucket/bucketspace.h> #include <memory> namespace proton { @@ -15,6 +16,7 @@ class IDocumentSubDBOwner public: virtual ~IDocumentSubDBOwner() {} virtual void syncFeedView() = 0; + virtual document::BucketSpace getBucketSpace() const = 0; virtual vespalib::string getName() const = 0; virtual uint32_t getDistributionKey() const = 0; }; diff --git a/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.cpp b/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.cpp index d19e702c574..c68d794a5e0 100644 --- a/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.cpp @@ -53,6 +53,7 @@ injectBucketMoveJob(MaintenanceController &controller, IFrozenBucketHandler &fbHandler, bucketdb::IBucketCreateNotifier &bucketCreateNotifier, const vespalib::string &docTypeName, + document::BucketSpace bucketSpace, IDocumentMoveHandler &moveHandler, IBucketModifiedHandler &bucketModifiedHandler, IClusterStateChangedNotifier &clusterStateChangedNotifier, @@ -74,7 +75,7 @@ injectBucketMoveJob(MaintenanceController &controller, bucketStateChangedNotifier, diskMemUsageNotifier, blockableConfig, - docTypeName)); + docTypeName, bucketSpace)); controller.registerJobInMasterThread(std::move(trackJob(jobTrackers.getBucketMove(), std::move(bmj)))); } @@ -91,6 +92,7 @@ MaintenanceJobsInjector::injectJobs(MaintenanceController &controller, IFrozenBucketHandler &fbHandler, bucketdb::IBucketCreateNotifier &bucketCreateNotifier, const vespalib::string &docTypeName, + document::BucketSpace bucketSpace, IPruneRemovedDocumentsHandler &prdHandler, IDocumentMoveHandler &moveHandler, IBucketModifiedHandler &bucketModifiedHandler, @@ -119,7 +121,7 @@ MaintenanceJobsInjector::injectJobs(MaintenanceController &controller, fbHandler, jobTrackers.getLidSpaceCompact(), diskMemUsageNotifier, clusterStateChangedNotifier, calc); } - injectBucketMoveJob(controller, fbHandler, bucketCreateNotifier, docTypeName, moveHandler, bucketModifiedHandler, + injectBucketMoveJob(controller, fbHandler, bucketCreateNotifier, docTypeName, bucketSpace, moveHandler, bucketModifiedHandler, clusterStateChangedNotifier, bucketStateChangedNotifier, calc, jobTrackers, diskMemUsageNotifier, config.getBlockableJobConfig()); controller.registerJobInMasterThread(std::make_unique<SampleAttributeUsageJob> diff --git a/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.h b/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.h index 85fed392ab6..55c218cca6d 100644 --- a/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.h +++ b/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.h @@ -39,6 +39,7 @@ struct MaintenanceJobsInjector IFrozenBucketHandler &fbHandler, bucketdb::IBucketCreateNotifier &bucketCreateNotifier, const vespalib::string &docTypeName, + document::BucketSpace bucketSpace, IPruneRemovedDocumentsHandler &prdHandler, IDocumentMoveHandler &moveHandler, IBucketModifiedHandler &bucketModifiedHandler, diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.cpp b/searchcore/src/vespa/searchcore/proton/server/proton.cpp index 34cfaaeddb2..7caed5198bd 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/proton.cpp @@ -623,7 +623,7 @@ Proton::removeDocumentDB(const DocTypeName &docTypeName) oldHandler = _persistenceEngine->removeHandler(old->getBucketSpace(), docTypeName); if (_initComplete && oldHandler) { // TODO: Fix race with bucket db modifying ops. - _persistenceEngine->grabExtraModifiedBuckets(*oldHandler); + _persistenceEngine->grabExtraModifiedBuckets(old->getBucketSpace(), *oldHandler); } } _persistenceEngine->destroyIterators(); diff --git a/searchcore/src/vespa/searchcore/proton/test/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/test/CMakeLists.txt index e1fb848b1d4..10fd4b9c518 100644 --- a/searchcore/src/vespa/searchcore/proton/test/CMakeLists.txt +++ b/searchcore/src/vespa/searchcore/proton/test/CMakeLists.txt @@ -1,6 +1,7 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_add_library(searchcore_test STATIC SOURCES + bucketfactory.cpp buckethandler.cpp clusterstatehandler.cpp documentdb_config_builder.cpp diff --git a/searchcore/src/vespa/searchcore/proton/common/bucketfactory.cpp b/searchcore/src/vespa/searchcore/proton/test/bucketfactory.cpp index e7048b74b10..37c0a965f48 100644 --- a/searchcore/src/vespa/searchcore/proton/common/bucketfactory.cpp +++ b/searchcore/src/vespa/searchcore/proton/test/bucketfactory.cpp @@ -1,11 +1,13 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "bucketfactory.h" +#include <vespa/persistence/spi/test.h> using document::BucketId; using document::DocumentId; using storage::spi::Bucket; using storage::spi::PartitionId; +using storage::spi::test::makeBucket; namespace proton { @@ -21,7 +23,7 @@ BucketFactory::getBucketId(const DocumentId &docId) Bucket BucketFactory::getBucket(const DocumentId &docId) { - return Bucket(document::Bucket(document::BucketSpace::placeHolder(), getBucketId(docId)), PartitionId(0)); + return makeBucket(getBucketId(docId)); } } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/common/bucketfactory.h b/searchcore/src/vespa/searchcore/proton/test/bucketfactory.h index 0d8f33b436e..0d8f33b436e 100644 --- a/searchcore/src/vespa/searchcore/proton/common/bucketfactory.h +++ b/searchcore/src/vespa/searchcore/proton/test/bucketfactory.h |