diff options
17 files changed, 43 insertions, 36 deletions
diff --git a/persistence/src/tests/dummyimpl/dummypersistence_test.cpp b/persistence/src/tests/dummyimpl/dummypersistence_test.cpp index 30e6e2236de..c9e6b44508e 100644 --- a/persistence/src/tests/dummyimpl/dummypersistence_test.cpp +++ b/persistence/src/tests/dummyimpl/dummypersistence_test.cpp @@ -4,6 +4,7 @@ #include <vespa/persistence/dummyimpl/dummypersistence.h> #include <vespa/vespalib/testkit/testapp.h> #include <vespa/document/base/documentid.h> +#include <vespa/document/test/make_bucket_space.h> #include <vespa/vdslib/distribution/distribution.h> #include <vespa/vdslib/state/clusterstate.h> #include <vespa/config-stor-distribution.h> @@ -11,6 +12,7 @@ using namespace storage::spi; using namespace storage; +using document::test::makeBucketSpace; using dummy::BucketContent; namespace { @@ -79,7 +81,7 @@ TEST_F("require that setClusterState sets the cluster state", Fixture) { document::DocumentTypeRepo::SP repo; dummy::DummyPersistence provider(repo); - provider.setClusterState(state); + provider.setClusterState(makeBucketSpace(), state); EXPECT_EQUAL(false, provider.getClusterState().nodeUp()); } diff --git a/persistence/src/vespa/persistence/conformancetest/conformancetest.cpp b/persistence/src/vespa/persistence/conformancetest/conformancetest.cpp index 57ebd36ce00..885d3e9aad7 100644 --- a/persistence/src/vespa/persistence/conformancetest/conformancetest.cpp +++ b/persistence/src/vespa/persistence/conformancetest/conformancetest.cpp @@ -2137,7 +2137,7 @@ void ConformanceTest::testBucketActivation() Context context(defaultLoadType, Priority(0), Trace::TraceLevel(0)); Bucket bucket(makeSpiBucket(BucketId(8, 0x01))); - spi->setClusterState(createClusterState()); + spi->setClusterState(makeBucketSpace(), createClusterState()); spi->createBucket(bucket, context); CPPUNIT_ASSERT(!spi->getBucketInfo(bucket).getBucketInfo().isActive()); @@ -2156,9 +2156,9 @@ void ConformanceTest::testBucketActivation() CPPUNIT_ASSERT(spi->getBucketInfo(bucket).getBucketInfo().isActive()); // Setting node down should clear active flag. - spi->setClusterState(createClusterState(lib::State::DOWN)); + spi->setClusterState(makeBucketSpace(), createClusterState(lib::State::DOWN)); CPPUNIT_ASSERT(!spi->getBucketInfo(bucket).getBucketInfo().isActive()); - spi->setClusterState(createClusterState(lib::State::UP)); + spi->setClusterState(makeBucketSpace(), createClusterState(lib::State::UP)); CPPUNIT_ASSERT(!spi->getBucketInfo(bucket).getBucketInfo().isActive()); // Actively clearing it should of course also clear it @@ -2185,7 +2185,7 @@ void ConformanceTest::testBucketActivationSplitAndJoin() Document::SP doc1 = testDocMan.createRandomDocumentAtLocation(0x02, 1); Document::SP doc2 = testDocMan.createRandomDocumentAtLocation(0x06, 2); - spi->setClusterState(createClusterState()); + spi->setClusterState(makeBucketSpace(), createClusterState()); spi->createBucket(bucketC, context); spi->put(bucketC, Timestamp(1), doc1, context); spi->put(bucketC, Timestamp(2), doc2, context); diff --git a/persistence/src/vespa/persistence/dummyimpl/dummypersistence.cpp b/persistence/src/vespa/persistence/dummyimpl/dummypersistence.cpp index 5c38679f52c..9a4f094d32c 100644 --- a/persistence/src/vespa/persistence/dummyimpl/dummypersistence.cpp +++ b/persistence/src/vespa/persistence/dummyimpl/dummypersistence.cpp @@ -370,7 +370,7 @@ DummyPersistence::getModifiedBuckets(BucketSpace) const } Result -DummyPersistence::setClusterState(const ClusterState& c) +DummyPersistence::setClusterState(BucketSpace, const ClusterState& c) { vespalib::MonitorGuard lock(_monitor); _clusterState.reset(new ClusterState(c)); diff --git a/persistence/src/vespa/persistence/dummyimpl/dummypersistence.h b/persistence/src/vespa/persistence/dummyimpl/dummypersistence.h index 50a4562ea3b..f89afbcac5d 100644 --- a/persistence/src/vespa/persistence/dummyimpl/dummypersistence.h +++ b/persistence/src/vespa/persistence/dummyimpl/dummypersistence.h @@ -142,7 +142,7 @@ public: */ BucketIdListResult getModifiedBuckets(BucketSpace bucketSpace) const override; - Result setClusterState(const ClusterState& newState) override; + Result setClusterState(BucketSpace bucketSpace, const ClusterState& newState) override; Result setActiveState(const Bucket& bucket, BucketInfo::ActiveState newState) override; BucketInfoResult getBucketInfo(const Bucket&) const override; Result put(const Bucket&, Timestamp, const DocumentSP&, Context&) override; diff --git a/persistence/src/vespa/persistence/spi/abstractpersistenceprovider.h b/persistence/src/vespa/persistence/spi/abstractpersistenceprovider.h index 2460259cfe7..07482873467 100644 --- a/persistence/src/vespa/persistence/spi/abstractpersistenceprovider.h +++ b/persistence/src/vespa/persistence/spi/abstractpersistenceprovider.h @@ -55,7 +55,7 @@ public: /** * Default impl empty. */ - Result setClusterState(const ClusterState&) override { return Result(); } + Result setClusterState(BucketSpace, const ClusterState&) override { return Result(); } /** * Default impl empty. diff --git a/persistence/src/vespa/persistence/spi/metricpersistenceprovider.cpp b/persistence/src/vespa/persistence/spi/metricpersistenceprovider.cpp index e338c76fb88..76b0a3c4686 100644 --- a/persistence/src/vespa/persistence/spi/metricpersistenceprovider.cpp +++ b/persistence/src/vespa/persistence/spi/metricpersistenceprovider.cpp @@ -113,10 +113,10 @@ Impl::listBuckets(BucketSpace bucketSpace, PartitionId v1) const } Result -Impl::setClusterState(const ClusterState& v1) +Impl::setClusterState(BucketSpace bucketSpace, const ClusterState& v1) { PRE_PROCESS(3); - Result r(_next->setClusterState(v1)); + Result r(_next->setClusterState(bucketSpace, v1)); POST_PROCESS(3, r); return r; } diff --git a/persistence/src/vespa/persistence/spi/metricpersistenceprovider.h b/persistence/src/vespa/persistence/spi/metricpersistenceprovider.h index 8ec2e2dd1bc..e169ad098c7 100644 --- a/persistence/src/vespa/persistence/spi/metricpersistenceprovider.h +++ b/persistence/src/vespa/persistence/spi/metricpersistenceprovider.h @@ -35,7 +35,7 @@ public: Result initialize() override; PartitionStateListResult getPartitionStates() const override; BucketIdListResult listBuckets(BucketSpace bucketSpace, PartitionId) const override; - Result setClusterState(const ClusterState&) override; + Result setClusterState(BucketSpace bucketSpace, const ClusterState&) override; Result setActiveState(const Bucket&, BucketInfo::ActiveState) override; BucketInfoResult getBucketInfo(const Bucket&) const override; Result put(const Bucket&, Timestamp, const DocumentSP&, Context&) override; diff --git a/persistence/src/vespa/persistence/spi/persistenceprovider.h b/persistence/src/vespa/persistence/spi/persistenceprovider.h index b10ed618e88..d7a5fa2b4a4 100644 --- a/persistence/src/vespa/persistence/spi/persistenceprovider.h +++ b/persistence/src/vespa/persistence/spi/persistenceprovider.h @@ -93,7 +93,7 @@ struct PersistenceProvider * supplied (changes that relate to the distributor will not cause an * update here). */ - virtual Result setClusterState(const ClusterState&) = 0; + virtual Result setClusterState(BucketSpace bucketSpace, const ClusterState&) = 0; /** * Sets the bucket state to active or inactive. After this returns, diff --git a/searchcore/src/apps/proton/downpersistence.cpp b/searchcore/src/apps/proton/downpersistence.cpp index 4ee9a0635b0..33ad4bc5024 100644 --- a/searchcore/src/apps/proton/downpersistence.cpp +++ b/searchcore/src/apps/proton/downpersistence.cpp @@ -45,7 +45,7 @@ DownPersistence::listBuckets(BucketSpace, PartitionId) const } Result -DownPersistence:: setClusterState(const ClusterState&) +DownPersistence:: setClusterState(BucketSpace, const ClusterState&) { return Result(); } diff --git a/searchcore/src/apps/proton/downpersistence.h b/searchcore/src/apps/proton/downpersistence.h index 9e64b89f065..0a602c4467e 100644 --- a/searchcore/src/apps/proton/downpersistence.h +++ b/searchcore/src/apps/proton/downpersistence.h @@ -29,7 +29,7 @@ public: Result initialize() override; PartitionStateListResult getPartitionStates() const override; BucketIdListResult listBuckets(BucketSpace bucketSpace, PartitionId) const override; - Result setClusterState(const ClusterState&) override; + Result setClusterState(BucketSpace, const ClusterState&) override; Result setActiveState(const Bucket&, BucketInfo::ActiveState) override; BucketInfoResult getBucketInfo(const Bucket&) const override; Result put(const Bucket&, Timestamp, const DocumentSP&, Context&) override; diff --git a/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp b/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp index aa7c5a4eda9..4a195514db1 100644 --- a/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp +++ b/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp @@ -611,7 +611,7 @@ TEST_F("require that setClusterState() is routed to handlers", SimpleFixture) { ClusterState state(createClusterState()); - f.engine.setClusterState(state); + f.engine.setClusterState(makeBucketSpace(), state); EXPECT_EQUAL(&state, f.hset.handler1.lastCalc); EXPECT_EQUAL(&state, f.hset.handler2.lastCalc); } diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp index 6f2fbabfb1c..c7b01d209ee 100644 --- a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp @@ -189,7 +189,7 @@ PersistenceEngine::PersistenceEngine(IPersistenceEngineOwner &owner, const IReso _iterators_lock(), _owner(owner), _writeFilter(writeFilter), - _clusterState(), + _clusterStates(), _extraModifiedBuckets(), _rwMutex() { @@ -272,11 +272,11 @@ PersistenceEngine::listBuckets(BucketSpace bucketSpace, PartitionId id) const Result -PersistenceEngine::setClusterState(const ClusterState &calc) +PersistenceEngine::setClusterState(BucketSpace bucketSpace, const ClusterState &calc) { std::shared_lock<std::shared_timed_mutex> rguard(_rwMutex); - saveClusterState(calc); - HandlerSnapshot::UP snap = getHandlerSnapshot(); + saveClusterState(bucketSpace, calc); + HandlerSnapshot::UP snap = getHandlerSnapshot(bucketSpace); GenericResultHandler resultHandler(snap->size()); for (; snap->handlers().valid(); snap->handlers().next()) { IPersistenceHandler *handler = snap->handlers().get(); @@ -630,26 +630,27 @@ PersistenceEngine::destroyIterators() void -PersistenceEngine::saveClusterState(const ClusterState &calc) +PersistenceEngine::saveClusterState(BucketSpace bucketSpace, const ClusterState &calc) { auto clusterState = std::make_shared<ClusterState>(calc); { std::lock_guard<std::mutex> guard(_lock); - clusterState.swap(_clusterState); + clusterState.swap(_clusterStates[bucketSpace]); } } PersistenceEngine::ClusterState::SP -PersistenceEngine::savedClusterState() const +PersistenceEngine::savedClusterState(BucketSpace bucketSpace) const { std::lock_guard<std::mutex> guard(_lock); - return _clusterState; + auto itr(_clusterStates.find(bucketSpace)); + return ((itr != _clusterStates.end()) ? itr->second : ClusterState::SP()); } void -PersistenceEngine::propagateSavedClusterState(IPersistenceHandler &handler) +PersistenceEngine::propagateSavedClusterState(BucketSpace bucketSpace, IPersistenceHandler &handler) { - ClusterState::SP clusterState(savedClusterState()); + ClusterState::SP clusterState(savedClusterState(bucketSpace)); if (!clusterState) return; // Propagate saved cluster state. diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.h b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.h index b3e70e9f433..a9c15e6f2b9 100644 --- a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.h +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.h @@ -10,6 +10,7 @@ #include <vespa/searchcore/proton/persistenceengine/ipersistencehandler.h> #include <mutex> #include <shared_mutex> +#include <unordered_map> namespace proton { @@ -75,7 +76,7 @@ private: mutable std::mutex _iterators_lock; IPersistenceEngineOwner &_owner; const IResourceWriteFilter &_writeFilter; - ClusterState::SP _clusterState; + std::unordered_map<BucketSpace, ClusterState::SP, BucketSpace::hash> _clusterStates; mutable ExtraModifiedBuckets _extraModifiedBuckets; mutable std::shared_timed_mutex _rwMutex; @@ -86,8 +87,8 @@ private: HandlerSnapshot::UP getHandlerSnapshot(document::BucketSpace bucketSpace, const document::DocumentId &docId) const; - void saveClusterState(const ClusterState &calc); - ClusterState::SP savedClusterState() const; + void saveClusterState(BucketSpace bucketSpace, const ClusterState &calc); + ClusterState::SP savedClusterState(BucketSpace bucketSpace) const; public: typedef std::unique_ptr<PersistenceEngine> UP; @@ -107,7 +108,7 @@ public: virtual Result initialize() override; virtual PartitionStateListResult getPartitionStates() const override; virtual BucketIdListResult listBuckets(BucketSpace bucketSpace, PartitionId) const override; - virtual Result setClusterState(const ClusterState& calc) override; + virtual Result setClusterState(BucketSpace bucketSpace, const ClusterState& calc) override; virtual Result setActiveState(const Bucket& bucket, BucketInfo::ActiveState newState) override; virtual BucketInfoResult getBucketInfo(const Bucket&) const override; virtual Result put(const Bucket&, Timestamp, const document::Document::SP&, Context&) override; @@ -128,7 +129,7 @@ public: virtual Result maintain(const Bucket&, MaintenanceLevel) override; void destroyIterators(); - void propagateSavedClusterState(IPersistenceHandler &handler); + void propagateSavedClusterState(BucketSpace bucketSpace, IPersistenceHandler &handler); void grabExtraModifiedBuckets(BucketSpace bucketSpace, IPersistenceHandler &handler); void populateInitialBucketDB(BucketSpace bucketSpace, IPersistenceHandler &targetHandler); std::unique_lock<std::shared_timed_mutex> getWLock() const; diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.cpp b/searchcore/src/vespa/searchcore/proton/server/proton.cpp index 2794619273c..6de5410470f 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/proton.cpp @@ -568,7 +568,7 @@ Proton::addDocumentDB(const document::DocumentType &docType, std::unique_lock<std::shared_timed_mutex> persistenceWGuard(_persistenceEngine->getWLock()); auto persistenceHandler = std::make_shared<PersistenceHandlerProxy>(ret); if (!_isInitializing) { - _persistenceEngine->propagateSavedClusterState(*persistenceHandler); + _persistenceEngine->propagateSavedClusterState(bucketSpace, *persistenceHandler); _persistenceEngine->populateInitialBucketDB(bucketSpace, *persistenceHandler); } // TODO: Fix race with new cluster state setting. diff --git a/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp b/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp index 88a7343f8c8..dfd04b271b2 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp +++ b/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp @@ -966,7 +966,6 @@ void FileStorManager::updateState() { lib::ClusterState::CSP state(_component.getStateUpdater().getSystemState()); - spi::ClusterState spiState(*state, _component.getIndex(), *_component.getDistribution()); lib::Node node(_component.getNodeType(), _component.getIndex()); bool nodeUp = state->getNodeState(node).getState().oneOf("uir"); @@ -977,7 +976,11 @@ FileStorManager::updateState() Deactivator deactivator; _component.getBucketSpaceRepo().forEachBucket(deactivator, "FileStorManager::updateState"); } - _provider->setClusterState(spiState); + for (const auto &elem : _component.getBucketSpaceRepo()) { + BucketSpace bucketSpace(elem.first); + spi::ClusterState spiState(*state, _component.getIndex(), *_component.getDistribution()); + _provider->setClusterState(bucketSpace, spiState); + } _nodeUpInLastNodeStateSeenByProvider = nodeUp; } diff --git a/storage/src/vespa/storage/persistence/provider_error_wrapper.cpp b/storage/src/vespa/storage/persistence/provider_error_wrapper.cpp index 15b0a469b35..056561e8e21 100644 --- a/storage/src/vespa/storage/persistence/provider_error_wrapper.cpp +++ b/storage/src/vespa/storage/persistence/provider_error_wrapper.cpp @@ -55,9 +55,9 @@ ProviderErrorWrapper::listBuckets(BucketSpace bucketSpace, spi::PartitionId part } spi::Result -ProviderErrorWrapper::setClusterState(const spi::ClusterState& state) +ProviderErrorWrapper::setClusterState(BucketSpace bucketSpace, const spi::ClusterState& state) { - return checkResult(_impl.setClusterState(state)); + return checkResult(_impl.setClusterState(bucketSpace, state)); } spi::Result diff --git a/storage/src/vespa/storage/persistence/provider_error_wrapper.h b/storage/src/vespa/storage/persistence/provider_error_wrapper.h index 122837e75ed..3b5ace90d13 100644 --- a/storage/src/vespa/storage/persistence/provider_error_wrapper.h +++ b/storage/src/vespa/storage/persistence/provider_error_wrapper.h @@ -44,7 +44,7 @@ public: spi::Result initialize() override; spi::PartitionStateListResult getPartitionStates() const override; spi::BucketIdListResult listBuckets(BucketSpace bucketSpace, spi::PartitionId) const override; - spi::Result setClusterState(const spi::ClusterState&) override; + spi::Result setClusterState(BucketSpace bucketSpace, const spi::ClusterState&) override; spi::Result setActiveState(const spi::Bucket& bucket, spi::BucketInfo::ActiveState newState) override; spi::BucketInfoResult getBucketInfo(const spi::Bucket&) const override; spi::Result put(const spi::Bucket&, spi::Timestamp, const spi::DocumentSP&, spi::Context&) override; |