diff options
author | Tor Brede Vekterli <vekterli@verizonmedia.com> | 2021-09-20 14:24:41 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2021-09-20 14:24:41 +0200 |
commit | a26d83aab1acbe84ce31a7f886374e33202ec0d0 (patch) | |
tree | a09b952441bc566393b65ffc58a18d475d806f41 /storage/src/tests/distributor | |
parent | 23a050f3ae7755670a1b13e9f26fe8b19fb81042 (diff) | |
parent | 830bd44f5fa5cb5a0a2ee7ca34e8afaff40c474d (diff) |
Merge pull request #19202 from vespa-engine/vekterli/remove-most-of-distributor-legacy-mode
Remove most traces of distributor legacy mode [run-systemtest]
Diffstat (limited to 'storage/src/tests/distributor')
11 files changed, 13 insertions, 5007 deletions
diff --git a/storage/src/tests/distributor/CMakeLists.txt b/storage/src/tests/distributor/CMakeLists.txt index 67a7fed8d0b..5a5422156d1 100644 --- a/storage/src/tests/distributor/CMakeLists.txt +++ b/storage/src/tests/distributor/CMakeLists.txt @@ -15,15 +15,12 @@ vespa_add_executable(storage_distributor_gtest_runner_app TEST distributor_stripe_pool_test.cpp distributor_stripe_test.cpp distributor_stripe_test_util.cpp - distributortestutil.cpp externaloperationhandlertest.cpp garbagecollectiontest.cpp getoperationtest.cpp gtest_runner.cpp idealstatemanagertest.cpp joinbuckettest.cpp - legacy_bucket_db_updater_test.cpp - legacy_distributor_test.cpp maintenanceschedulertest.cpp mergelimitertest.cpp mergeoperationtest.cpp diff --git a/storage/src/tests/distributor/distributor_stripe_test_util.cpp b/storage/src/tests/distributor/distributor_stripe_test_util.cpp index 043f996a4a1..4b7e73d3e43 100644 --- a/storage/src/tests/distributor/distributor_stripe_test_util.cpp +++ b/storage/src/tests/distributor/distributor_stripe_test_util.cpp @@ -20,7 +20,6 @@ namespace storage::distributor { DistributorStripeTestUtil::DistributorStripeTestUtil() : _config(), _node(), - _threadPool(), _stripe(), _sender(), _senderDown(), @@ -37,18 +36,14 @@ void DistributorStripeTestUtil::createLinks() { _node.reset(new TestDistributorApp(_config.getConfigId())); - _threadPool = framework::TickingThreadPool::createDefault("distributor"); _metrics = std::make_shared<DistributorMetricSet>(); _ideal_state_metrics = std::make_shared<IdealStateMetricSet>(); _stripe = std::make_unique<DistributorStripe>(_node->getComponentRegister(), *_metrics, *_ideal_state_metrics, _node->node_identity(), - *_threadPool, - *this, _messageSender, *this, - false, _done_initializing); } diff --git a/storage/src/tests/distributor/distributor_stripe_test_util.h b/storage/src/tests/distributor/distributor_stripe_test_util.h index ccade98fd01..e73d1a3baa1 100644 --- a/storage/src/tests/distributor/distributor_stripe_test_util.h +++ b/storage/src/tests/distributor/distributor_stripe_test_util.h @@ -212,7 +212,6 @@ public: protected: vdstestlib::DirConfig _config; std::unique_ptr<TestDistributorApp> _node; - std::unique_ptr<framework::TickingThreadPool> _threadPool; std::shared_ptr<DistributorMetricSet> _metrics; std::shared_ptr<IdealStateMetricSet> _ideal_state_metrics; std::unique_ptr<DistributorStripe> _stripe; diff --git a/storage/src/tests/distributor/distributortestutil.cpp b/storage/src/tests/distributor/distributortestutil.cpp deleted file mode 100644 index 1e24bd72c9b..00000000000 --- a/storage/src/tests/distributor/distributortestutil.cpp +++ /dev/null @@ -1,512 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "distributortestutil.h" -#include <vespa/config-stor-distribution.h> -#include <vespa/document/test/make_bucket_space.h> -#include <vespa/document/test/make_document_bucket.h> -#include <vespa/storage/distributor/top_level_distributor.h> -#include <vespa/storage/distributor/distributor_bucket_space.h> -#include <vespa/storage/distributor/distributor_stripe.h> -#include <vespa/storage/distributor/distributor_stripe_component.h> -#include <vespa/storage/distributor/distributor_stripe_pool.h> -#include <vespa/vdslib/distribution/distribution.h> -#include <vespa/vespalib/text/stringtokenizer.h> - -using document::test::makeBucketSpace; -using document::test::makeDocumentBucket; - -namespace storage::distributor { - -DistributorTestUtil::DistributorTestUtil() - : _messageSender(_sender, _senderDown), - _num_distributor_stripes(0) // TODO STRIPE change default -{ - _config = getStandardConfig(false); -} -DistributorTestUtil::~DistributorTestUtil() { } - -void -DistributorTestUtil::createLinks() -{ - _node.reset(new TestDistributorApp(_config.getConfigId())); - _threadPool = framework::TickingThreadPool::createDefault("distributor"); - _stripe_pool = std::make_unique<DistributorStripePool>(); - _distributor.reset(new TopLevelDistributor( - _node->getComponentRegister(), - _node->node_identity(), - *_threadPool, - *_stripe_pool, - *this, - _num_distributor_stripes, - _hostInfo, - &_messageSender)); - _component.reset(new storage::DistributorComponent(_node->getComponentRegister(), "distrtestutil")); -}; - -void -DistributorTestUtil::setupDistributor(int redundancy, - int nodeCount, - const std::string& systemState, - uint32_t earlyReturn, - bool requirePrimaryToBeWritten) -{ - setup_distributor(redundancy, nodeCount, lib::ClusterStateBundle(lib::ClusterState(systemState)), earlyReturn, requirePrimaryToBeWritten); -} - -void -DistributorTestUtil::setup_distributor(int redundancy, - int node_count, - const lib::ClusterStateBundle& state, - uint32_t early_return, - bool require_primary_to_be_written) -{ - lib::Distribution::DistributionConfigBuilder config( - lib::Distribution::getDefaultDistributionConfig(redundancy, node_count).get()); - config.redundancy = redundancy; - config.initialRedundancy = early_return; - config.ensurePrimaryPersisted = require_primary_to_be_written; - auto distribution = std::make_shared<lib::Distribution>(config); - _node->getComponentRegister().setDistribution(distribution); - enable_distributor_cluster_state(state); - // This is for all intents and purposes a hack to avoid having the - // distributor treat setting the distribution explicitly as a signal that - // it should send RequestBucketInfo to all configured nodes. - // If we called storage_distribution_changed followed by enableDistribution - // explicitly (which is what happens in "real life"), that is what would - // take place. - // The inverse case of this can be explicitly accomplished by calling - // triggerDistributionChange(). - // This isn't pretty, folks, but it avoids breaking the world for now, - // as many tests have implicit assumptions about this being the behavior. - _distributor->propagateDefaultDistribution(distribution); -} - -void -DistributorTestUtil::setRedundancy(uint32_t redundancy) -{ - auto distribution = std::make_shared<lib::Distribution>( - lib::Distribution::getDefaultDistributionConfig( - redundancy, 100)); - // Same rationale for not triggering a full distribution change as - // in setupDistributor() - _node->getComponentRegister().setDistribution(distribution); - _distributor->propagateDefaultDistribution(std::move(distribution)); -} - -void -DistributorTestUtil::triggerDistributionChange(lib::Distribution::SP distr) -{ - _node->getComponentRegister().setDistribution(std::move(distr)); - _distributor->storageDistributionChanged(); - _distributor->enableNextDistribution(); -} - -void -DistributorTestUtil::receive_set_system_state_command(const vespalib::string& state_str) -{ - auto state_cmd = std::make_shared<api::SetSystemStateCommand>(lib::ClusterState(state_str)); - _distributor->handleMessage(state_cmd); // TODO move semantics -} - -void -DistributorTestUtil::handle_top_level_message(const std::shared_ptr<api::StorageMessage>& msg) -{ - _distributor->handleMessage(msg); -} - -void -DistributorTestUtil::setTypeRepo(const std::shared_ptr<const document::DocumentTypeRepo> &repo) -{ - _node->getComponentRegister().setDocumentTypeRepo(repo); -} - -void -DistributorTestUtil::close() -{ - _component.reset(0); - if (_distributor.get()) { - _distributor->onClose(); - } - _sender.clear(); - _node.reset(0); - _config = getStandardConfig(false); -} - -namespace { - std::string dumpVector(const std::vector<uint16_t>& vec) { - std::ostringstream ost; - for (uint32_t i = 0; i < vec.size(); ++i) { - if (i != 0) { - ost << ","; - } - ost << vec[i]; - } - return ost.str(); - } -} - -std::string -DistributorTestUtil::getNodes(document::BucketId id) -{ - BucketDatabase::Entry entry = getBucket(id); - - if (!entry.valid()) { - return id.toString(); - } else { - std::vector<uint16_t> nodes = entry->getNodes(); - std::sort(nodes.begin(), nodes.end()); - - std::ostringstream ost; - ost << id << ": " << dumpVector(nodes); - return ost.str(); - } -} - -std::string -DistributorTestUtil::getIdealStr(document::BucketId id, const lib::ClusterState& state) -{ - if (!getDistributorBucketSpace().owns_bucket_in_state(state, id)) { - return id.toString(); - } - - std::vector<uint16_t> nodes; - getDistribution().getIdealNodes( - lib::NodeType::STORAGE, state, id, nodes); - std::sort(nodes.begin(), nodes.end()); - std::ostringstream ost; - ost << id << ": " << dumpVector(nodes); - return ost.str(); -} - -void -DistributorTestUtil::addIdealNodes(const lib::ClusterState& state, - const document::BucketId& id) -{ - BucketDatabase::Entry entry = getBucket(id); - - if (!entry.valid()) { - entry = BucketDatabase::Entry(id); - } - - std::vector<uint16_t> res; - assert(_component.get()); - getDistribution().getIdealNodes( - lib::NodeType::STORAGE, state, id, res); - - for (uint32_t i = 0; i < res.size(); ++i) { - if (state.getNodeState(lib::Node(lib::NodeType::STORAGE, res[i])).getState() != - lib::State::MAINTENANCE) - { - entry->addNode(BucketCopy(0, res[i], api::BucketInfo(1,1,1)), - toVector<uint16_t>(0)); - } - } - - getBucketDatabase().update(entry); -} - -void DistributorTestUtil::addNodesToBucketDB(const document::Bucket& bucket, const std::string& nodeStr) { - BucketDatabase::Entry entry = getBucket(bucket); - - if (!entry.valid()) { - entry = BucketDatabase::Entry(bucket.getBucketId()); - } - - entry->clear(); - - vespalib::StringTokenizer tokenizer(nodeStr, ","); - for (uint32_t i = 0; i < tokenizer.size(); ++i) { - vespalib::StringTokenizer tok2(tokenizer[i], "="); - vespalib::StringTokenizer tok3(tok2[1], "/"); - - api::BucketInfo info(atoi(tok3[0].data()), - atoi(tok3.size() > 1 ? tok3[1].data() : tok3[0].data()), - atoi(tok3.size() > 2 ? tok3[2].data() : tok3[0].data())); - - size_t flagsIdx = 3; - - // Meta info override? For simplicity, require both meta count and size - if (tok3.size() > 4 && (!tok3[3].empty() && isdigit(tok3[3][0]))) { - info.setMetaCount(atoi(tok3[3].data())); - info.setUsedFileSize(atoi(tok3[4].data())); - flagsIdx = 5; - } - - if ((tok3.size() > flagsIdx + 1) && tok3[flagsIdx + 1] == "a") { - info.setActive(); - } else { - info.setActive(false); - } - if ((tok3.size() > flagsIdx + 2) && tok3[flagsIdx + 2] == "r") { - info.setReady(); - } else { - info.setReady(false); - } - - uint16_t idx = atoi(tok2[0].data()); - BucketCopy node( - 0, - idx, - info); - - // Allow user to manually override trusted and active. - if (tok3.size() > flagsIdx && tok3[flagsIdx] == "t") { - node.setTrusted(); - } - - entry->addNodeManual(node); - } - - getBucketDatabase(bucket.getBucketSpace()).update(entry); -} - -void -DistributorTestUtil::addNodesToBucketDB(const document::BucketId& id, - const std::string& nodeStr) -{ - addNodesToBucketDB(document::Bucket(makeBucketSpace(), id), nodeStr); -} - -void -DistributorTestUtil::removeFromBucketDB(const document::BucketId& id) -{ - getBucketDatabase().remove(id); -} - -void -DistributorTestUtil::addIdealNodes(const document::BucketId& id) -{ - // TODO STRIPE roundabout way of getting state bundle..! - addIdealNodes(*operation_context().cluster_state_bundle().getBaselineClusterState(), id); -} - -void -DistributorTestUtil::insertBucketInfo(document::BucketId id, - uint16_t node, - uint32_t checksum, - uint32_t count, - uint32_t size, - bool trusted, - bool active) -{ - api::BucketInfo info(checksum, count, size); - insertBucketInfo(id, node, info, trusted, active); -} - -void -DistributorTestUtil::insertBucketInfo(document::BucketId id, - uint16_t node, - const api::BucketInfo& info, - bool trusted, - bool active) -{ - BucketDatabase::Entry entry = getBucketDatabase().get(id); - if (!entry.valid()) { - entry = BucketDatabase::Entry(id, BucketInfo()); - } - - api::BucketInfo info2(info); - if (active) { - info2.setActive(); - } - BucketCopy copy(operation_context().generate_unique_timestamp(), node, info2); - - entry->addNode(copy.setTrusted(trusted), toVector<uint16_t>(0)); - - getBucketDatabase().update(entry); -} - -std::string -DistributorTestUtil::dumpBucket(const document::BucketId& bid) -{ - return getBucketDatabase().get(bid).toString(); -} - -void -DistributorTestUtil::sendReply(Operation& op, - int idx, - api::ReturnCode::Result result) -{ - if (idx == -1) { - idx = _sender.commands().size() - 1; - } - assert(idx >= 0 && idx < static_cast<int>(_sender.commands().size())); - - std::shared_ptr<api::StorageCommand> cmd = _sender.command(idx); - api::StorageReply::SP reply(cmd->makeReply().release()); - reply->setResult(result); - op.receive(_sender, reply); -} - -BucketDatabase::Entry DistributorTestUtil::getBucket(const document::Bucket& bucket) const { - return getBucketDatabase(bucket.getBucketSpace()).get(bucket.getBucketId()); -} - -BucketDatabase::Entry -DistributorTestUtil::getBucket(const document::BucketId& bId) const -{ - return getBucketDatabase().get(bId); -} - -void -DistributorTestUtil::disableBucketActivationInConfig(bool disable) -{ - vespa::config::content::core::StorDistributormanagerConfigBuilder config; - config.disableBucketActivation = disable; - getConfig().configure(config); -} - -StripeBucketDBUpdater& -DistributorTestUtil::getBucketDBUpdater() { - return _distributor->bucket_db_updater(); -} -IdealStateManager& -DistributorTestUtil::getIdealStateManager() { - return _distributor->ideal_state_manager(); -} -ExternalOperationHandler& -DistributorTestUtil::getExternalOperationHandler() { - return _distributor->external_operation_handler(); -} - -const storage::distributor::DistributorNodeContext& -DistributorTestUtil::node_context() const { - return _distributor->distributor_component(); -} - -storage::distributor::DistributorStripeOperationContext& -DistributorTestUtil::operation_context() { - return _distributor->distributor_component(); -} - -const DocumentSelectionParser& -DistributorTestUtil::doc_selection_parser() const { - return _distributor->distributor_component(); -} - -bool -DistributorTestUtil::tick() { - framework::ThreadWaitInfo res( - framework::ThreadWaitInfo::NO_MORE_CRITICAL_WORK_KNOWN); - { - framework::TickingLockGuard lock( - _distributor->_threadPool.freezeCriticalTicks()); - res.merge(_distributor->doCriticalTick(0)); - } - res.merge(_distributor->doNonCriticalTick(0)); - return !res.waitWanted(); -} - -DistributorConfiguration& -DistributorTestUtil::getConfig() { - // TODO STRIPE avoid const cast - return const_cast<DistributorConfiguration&>(_distributor->getConfig()); -} - -DistributorBucketSpace & -DistributorTestUtil::getDistributorBucketSpace() -{ - return getBucketSpaceRepo().get(makeBucketSpace()); -} - -BucketDatabase& -DistributorTestUtil::getBucketDatabase() { - return getDistributorBucketSpace().getBucketDatabase(); -} - -BucketDatabase& DistributorTestUtil::getBucketDatabase(document::BucketSpace space) { - return getBucketSpaceRepo().get(space).getBucketDatabase(); -} - -const BucketDatabase& -DistributorTestUtil::getBucketDatabase() const { - return getBucketSpaceRepo().get(makeBucketSpace()).getBucketDatabase(); -} - -const BucketDatabase& DistributorTestUtil::getBucketDatabase(document::BucketSpace space) const { - return getBucketSpaceRepo().get(space).getBucketDatabase(); -} - -DistributorBucketSpaceRepo & -DistributorTestUtil::getBucketSpaceRepo() { - return _distributor->getBucketSpaceRepo(); -} - -const DistributorBucketSpaceRepo & -DistributorTestUtil::getBucketSpaceRepo() const { - return _distributor->getBucketSpaceRepo(); -} - -DistributorBucketSpaceRepo & -DistributorTestUtil::getReadOnlyBucketSpaceRepo() { - return _distributor->getReadOnlyBucketSpaceRepo(); -} - -const DistributorBucketSpaceRepo & -DistributorTestUtil::getReadOnlyBucketSpaceRepo() const { - return _distributor->getReadOnlyBucketSpaceRepo(); -} - -bool -DistributorTestUtil::distributor_is_in_recovery_mode() const noexcept { - return _distributor->isInRecoveryMode(); -} - -const lib::ClusterStateBundle& -DistributorTestUtil::current_distributor_cluster_state_bundle() const noexcept { - return getDistributor().getClusterStateBundle(); -} - -std::string -DistributorTestUtil::active_ideal_state_operations() const { - return _distributor->getActiveIdealStateOperations(); -} - -const PendingMessageTracker& -DistributorTestUtil::pending_message_tracker() const noexcept { - return _distributor->getPendingMessageTracker(); -} - -PendingMessageTracker& -DistributorTestUtil::pending_message_tracker() noexcept { - return _distributor->getPendingMessageTracker(); -} - -std::chrono::steady_clock::duration -DistributorTestUtil::db_memory_sample_interval() const noexcept { - return _distributor->db_memory_sample_interval(); -} - -const lib::Distribution& -DistributorTestUtil::getDistribution() const { - return getBucketSpaceRepo().get(makeBucketSpace()).getDistribution(); -} - -std::vector<document::BucketSpace> -DistributorTestUtil::getBucketSpaces() const -{ - std::vector<document::BucketSpace> res; - for (const auto &repo : getBucketSpaceRepo()) { - res.push_back(repo.first); - } - return res; -} - -void -DistributorTestUtil::enableDistributorClusterState(vespalib::stringref state) -{ - getBucketDBUpdater().simulate_cluster_state_bundle_activation( - lib::ClusterStateBundle(lib::ClusterState(state))); -} - -void -DistributorTestUtil::enable_distributor_cluster_state(const lib::ClusterStateBundle& state) -{ - getBucketDBUpdater().simulate_cluster_state_bundle_activation(state); -} - -void -DistributorTestUtil::setSystemState(const lib::ClusterState& systemState) { - _distributor->enableClusterStateBundle(lib::ClusterStateBundle(systemState)); -} - -} diff --git a/storage/src/tests/distributor/distributortestutil.h b/storage/src/tests/distributor/distributortestutil.h deleted file mode 100644 index 9457bfeba83..00000000000 --- a/storage/src/tests/distributor/distributortestutil.h +++ /dev/null @@ -1,248 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include "distributor_message_sender_stub.h" -#include <tests/common/dummystoragelink.h> -#include <tests/common/testhelper.h> -#include <tests/common/teststorageapp.h> -#include <vespa/storage/common/hostreporter/hostinfo.h> -#include <vespa/storage/frameworkimpl/component/distributorcomponentregisterimpl.h> -#include <vespa/storage/storageutil/utils.h> -#include <vespa/storageapi/message/state.h> -#include <vespa/storageframework/defaultimplementation/clock/fakeclock.h> - -namespace storage { - -namespace framework { struct TickingThreadPool; } - -namespace distributor { - -class TopLevelDistributor; -class DistributorBucketSpace; -class DistributorBucketSpaceRepo; -class DistributorNodeContext; -class DistributorStripe; -class DistributorStripeComponent; -class DistributorStripeOperationContext; -class DistributorStripePool; -class DocumentSelectionParser; -class ExternalOperationHandler; -class IdealStateManager; -class Operation; -class StripeBucketDBUpdater; - -// TODO STRIPE rename to DistributorStripeTestUtil? -class DistributorTestUtil : private DoneInitializeHandler -{ -public: - DistributorTestUtil(); - ~DistributorTestUtil(); - - /** - * Sets up the storage link chain. - */ - void createLinks(); - void setTypeRepo(const std::shared_ptr<const document::DocumentTypeRepo> &repo); - - void close(); - - /** - * Returns a string with the nodes currently stored in the bucket - * database for the given bucket. - */ - std::string getNodes(document::BucketId id); - - /** - * Returns a string with the ideal state nodes for the given bucket. - */ - std::string getIdealStr(document::BucketId id, const lib::ClusterState& state); - - /** - * Adds the ideal nodes for the given bucket and the given cluster state - * to the bucket database. - */ - void addIdealNodes(const lib::ClusterState& state, const document::BucketId& id); - - /** - * Adds all the ideal nodes for the given bucket to the bucket database. - */ - void addIdealNodes(const document::BucketId& id); - - /** - * Parses the given string to a set of node => bucket info data, - * and inserts them as nodes in the given bucket. - * Format: - * "node1=checksum/docs/size,node2=checksum/docs/size" - */ - void addNodesToBucketDB(const document::Bucket& bucket, const std::string& nodeStr); - // As the above, but always inserts into default bucket space - void addNodesToBucketDB(const document::BucketId& id, const std::string& nodeStr); - - /** - * Removes the given bucket from the bucket database. - */ - void removeFromBucketDB(const document::BucketId& id); - - /** - * Inserts the given bucket information for the given bucket and node in - * the bucket database. - */ - void insertBucketInfo(document::BucketId id, - uint16_t node, - uint32_t checksum, - uint32_t count, - uint32_t size, - bool trusted = false, - bool active = false); - - /** - * Inserts the given bucket information for the given bucket and node in - * the bucket database. - */ - void insertBucketInfo(document::BucketId id, - uint16_t node, - const api::BucketInfo& info, - bool trusted = false, - bool active = false); - - std::string dumpBucket(const document::BucketId& bucket); - - /** - * Replies to message idx sent upwards with the given result code. - * If idx = -1, replies to the last command received upwards. - */ - void sendReply(Operation& op, - int idx = -1, - api::ReturnCode::Result result = api::ReturnCode::OK); - - StripeBucketDBUpdater& getBucketDBUpdater(); - IdealStateManager& getIdealStateManager(); - ExternalOperationHandler& getExternalOperationHandler(); - const storage::distributor::DistributorNodeContext& node_context() const; - storage::distributor::DistributorStripeOperationContext& operation_context(); - const DocumentSelectionParser& doc_selection_parser() const; - - TopLevelDistributor& getDistributor() noexcept { return *_distributor; } - const TopLevelDistributor& getDistributor() const noexcept { return *_distributor; } - - bool tick(); - - DistributorConfiguration& getConfig(); - - vdstestlib::DirConfig& getDirConfig() { - return _config; - } - - // TODO explicit notion of bucket spaces for tests - DistributorBucketSpace &getDistributorBucketSpace(); - BucketDatabase& getBucketDatabase(); // Implicit default space only - BucketDatabase& getBucketDatabase(document::BucketSpace space); - const BucketDatabase& getBucketDatabase() const; // Implicit default space only - const BucketDatabase& getBucketDatabase(document::BucketSpace space) const; - DistributorBucketSpaceRepo &getBucketSpaceRepo(); - const DistributorBucketSpaceRepo &getBucketSpaceRepo() const; - DistributorBucketSpaceRepo& getReadOnlyBucketSpaceRepo(); - const DistributorBucketSpaceRepo& getReadOnlyBucketSpaceRepo() const; - [[nodiscard]] bool distributor_is_in_recovery_mode() const noexcept; - [[nodiscard]] const lib::ClusterStateBundle& current_distributor_cluster_state_bundle() const noexcept; - [[nodiscard]] std::string active_ideal_state_operations() const; - [[nodiscard]] const PendingMessageTracker& pending_message_tracker() const noexcept; - [[nodiscard]] PendingMessageTracker& pending_message_tracker() noexcept; - [[nodiscard]] std::chrono::steady_clock::duration db_memory_sample_interval() const noexcept; - - const lib::Distribution& getDistribution() const; - // "End to end" distribution change trigger, which will invoke the bucket - // DB updater as expected based on the previous and new cluster state - // and config. - void triggerDistributionChange(std::shared_ptr<lib::Distribution> distr); - - framework::defaultimplementation::FakeClock& getClock() { return _node->getClock(); } - DistributorComponentRegister& getComponentRegister() { return _node->getComponentRegister(); } - DistributorComponentRegisterImpl& getComponentRegisterImpl() { return _node->getComponentRegister(); } - - StorageComponent& getComponent() { - if (_component.get() == 0) { - _component.reset(new storage::DistributorComponent( - _node->getComponentRegister(), "distributor_test_utils")); - } - return *_component; - } - - void setupDistributor(int redundancy, - int nodeCount, - const std::string& systemState, - uint32_t earlyReturn = false, - bool requirePrimaryToBeWritten = true); - - void setup_distributor(int redundancy, - int node_count, - const lib::ClusterStateBundle& state, - uint32_t early_return = false, - bool require_primary_to_be_written = true); - - void setRedundancy(uint32_t redundancy); - - void notifyDoneInitializing() override {} - - // Must implement this for storage server interface for now - virtual api::Timestamp getUniqueTimestamp() { - return _component->getUniqueTimestamp(); - } - - void disableBucketActivationInConfig(bool disable); - - BucketDatabase::Entry getBucket(const document::Bucket& bucket) const; - // Gets bucket entry from default space only - BucketDatabase::Entry getBucket(const document::BucketId& bId) const; - - std::vector<document::BucketSpace> getBucketSpaces() const; - - DistributorMessageSenderStub& sender() noexcept { return _sender; } - const DistributorMessageSenderStub& sender() const noexcept { return _sender; } - - void setSystemState(const lib::ClusterState& systemState); - - // Invokes full cluster state transition pipeline rather than directly applying - // the state and just pretending everything has been completed. - void receive_set_system_state_command(const vespalib::string& state_str); - - void handle_top_level_message(const std::shared_ptr<api::StorageMessage>& msg); - - // Must be called prior to createLinks() to have any effect - void set_num_distributor_stripes(uint32_t n_stripes) noexcept { - _num_distributor_stripes = n_stripes; - } -protected: - vdstestlib::DirConfig _config; - std::unique_ptr<TestDistributorApp> _node; - std::unique_ptr<framework::TickingThreadPool> _threadPool; - std::unique_ptr<DistributorStripePool> _stripe_pool; - std::unique_ptr<TopLevelDistributor> _distributor; - std::unique_ptr<storage::DistributorComponent> _component; - DistributorMessageSenderStub _sender; - DistributorMessageSenderStub _senderDown; - HostInfo _hostInfo; - - struct MessageSenderImpl : public ChainedMessageSender { - DistributorMessageSenderStub& _sender; - DistributorMessageSenderStub& _senderDown; - MessageSenderImpl(DistributorMessageSenderStub& up, DistributorMessageSenderStub& down) - : _sender(up), _senderDown(down) {} - - void sendUp(const std::shared_ptr<api::StorageMessage>& msg) override { - _sender.send(msg); - } - void sendDown(const std::shared_ptr<api::StorageMessage>& msg) override { - _senderDown.send(msg); - } - }; - MessageSenderImpl _messageSender; - uint32_t _num_distributor_stripes; - - void enableDistributorClusterState(vespalib::stringref state); - void enable_distributor_cluster_state(const lib::ClusterStateBundle& state); -}; - -} - -} diff --git a/storage/src/tests/distributor/legacy_bucket_db_updater_test.cpp b/storage/src/tests/distributor/legacy_bucket_db_updater_test.cpp deleted file mode 100644 index 5fa3ae5840b..00000000000 --- a/storage/src/tests/distributor/legacy_bucket_db_updater_test.cpp +++ /dev/null @@ -1,2893 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/storageapi/message/persistence.h> -#include <vespa/storage/distributor/top_level_bucket_db_updater.h> -#include <vespa/storage/distributor/bucket_space_distribution_context.h> -#include <vespa/storage/distributor/distributormetricsset.h> -#include <vespa/storage/distributor/pending_bucket_space_db_transition.h> -#include <vespa/storage/distributor/outdated_nodes_map.h> -#include <vespa/storage/storageutil/distributorstatecache.h> -#include <tests/distributor/distributortestutil.h> -#include <vespa/document/test/make_document_bucket.h> -#include <vespa/document/test/make_bucket_space.h> -#include <vespa/document/bucket/fixed_bucket_spaces.h> -#include <vespa/storage/distributor/simpleclusterinformation.h> -#include <vespa/storage/distributor/top_level_distributor.h> -#include <vespa/storage/distributor/distributor_stripe.h> -#include <vespa/storage/distributor/distributor_bucket_space.h> -#include <vespa/vespalib/gtest/gtest.h> -#include <vespa/vespalib/text/stringtokenizer.h> -#include <vespa/vespalib/util/benchmark_timer.h> -#include <sstream> -#include <iomanip> - -using namespace storage::api; -using namespace storage::lib; -using document::test::makeDocumentBucket; -using document::test::makeBucketSpace; -using document::BucketSpace; -using document::FixedBucketSpaces; -using document::BucketId; -using document::Bucket; - -using namespace ::testing; - -namespace storage::distributor { - -namespace { - -std::string -getStringList(std::string s, uint32_t count) -{ - std::ostringstream ost; - for (uint32_t i = 0; i < count; ++i) { - if (i > 0) { - ost << ","; - } - ost << s; - } - return ost.str(); -} - -std::string -getRequestBucketInfoStrings(uint32_t count) -{ - return getStringList("Request bucket info", count); -} - -} - -// TODO STRIPE: Remove this test when legacy mode is gone. -class LegacyBucketDBUpdaterTest : public Test, - public DistributorTestUtil -{ -public: - LegacyBucketDBUpdaterTest(); - ~LegacyBucketDBUpdaterTest() override; - - auto &defaultDistributorBucketSpace() { return getBucketSpaceRepo().get(makeBucketSpace()); } - - bool bucketExistsThatHasNode(int bucketCount, uint16_t node) const; - - ClusterInformation::CSP createClusterInfo(const std::string& clusterStateString) { - lib::ClusterState baselineClusterState(clusterStateString); - lib::ClusterStateBundle clusterStateBundle(baselineClusterState); - ClusterInformation::CSP clusterInfo( - new SimpleClusterInformation( - getBucketDBUpdater().node_context().node_index(), - clusterStateBundle, - "ui")); - for (auto* repo : {&mutable_repo(), &read_only_repo()}) { - for (auto& space : *repo) { - space.second->setClusterState(clusterStateBundle.getDerivedClusterState(space.first)); - } - } - return clusterInfo; - } - - DistributorBucketSpaceRepo& mutable_repo() noexcept { return getBucketSpaceRepo(); } - // Note: not calling this "immutable_repo" since it may actually be modified by the pending - // cluster state component (just not by operations), so it would not have the expected semantics. - DistributorBucketSpaceRepo& read_only_repo() noexcept { return getReadOnlyBucketSpaceRepo(); } - - BucketDatabase& mutable_default_db() noexcept { - return mutable_repo().get(FixedBucketSpaces::default_space()).getBucketDatabase(); - } - BucketDatabase& mutable_global_db() noexcept { - return mutable_repo().get(FixedBucketSpaces::global_space()).getBucketDatabase(); - } - BucketDatabase& read_only_default_db() noexcept { - return read_only_repo().get(FixedBucketSpaces::default_space()).getBucketDatabase(); - } - BucketDatabase& read_only_global_db() noexcept { - return read_only_repo().get(FixedBucketSpaces::global_space()).getBucketDatabase(); - } - - static std::string getNodeList(std::vector<uint16_t> nodes, size_t count); - - std::string getNodeList(std::vector<uint16_t> nodes); - - std::vector<uint16_t> - expandNodeVec(const std::vector<uint16_t> &nodes); - - std::vector<document::BucketSpace> _bucketSpaces; - - size_t messageCount(size_t messagesPerBucketSpace) const { - return messagesPerBucketSpace * _bucketSpaces.size(); - } - - void trigger_completed_but_not_yet_activated_transition( - vespalib::stringref initial_state, uint32_t initial_buckets, uint32_t initial_expected_msgs, - vespalib::stringref pending_state, uint32_t pending_buckets, uint32_t pending_expected_msgs); - -public: - using OutdatedNodesMap = dbtransition::OutdatedNodesMap; - void SetUp() override { - createLinks(); - _bucketSpaces = getBucketSpaces(); - // Disable deferred activation by default (at least for now) to avoid breaking the entire world. - getBucketDBUpdater().set_stale_reads_enabled(false); - }; - - void TearDown() override { - close(); - } - - std::shared_ptr<RequestBucketInfoReply> getFakeBucketReply( - const lib::ClusterState& state, - const RequestBucketInfoCommand& cmd, - int storageIndex, - uint32_t bucketCount, - uint32_t invalidBucketCount = 0) - { - auto sreply = std::make_shared<RequestBucketInfoReply>(cmd); - sreply->setAddress(storageAddress(storageIndex)); - - api::RequestBucketInfoReply::EntryVector &vec = sreply->getBucketInfo(); - - for (uint32_t i=0; i<bucketCount + invalidBucketCount; i++) { - if (!getDistributorBucketSpace().owns_bucket_in_state(state, document::BucketId(16, i))) { - continue; - } - - std::vector<uint16_t> nodes; - defaultDistributorBucketSpace().getDistribution().getIdealNodes( - lib::NodeType::STORAGE, - state, - document::BucketId(16, i), - nodes); - - for (uint32_t j=0; j<nodes.size(); j++) { - if (nodes[j] == storageIndex) { - if (i >= bucketCount) { - vec.push_back(api::RequestBucketInfoReply::Entry( - document::BucketId(16, i), - api::BucketInfo())); - } else { - vec.push_back(api::RequestBucketInfoReply::Entry( - document::BucketId(16, i), - api::BucketInfo(10,1,1))); - } - } - } - } - - return sreply; - } - - void fakeBucketReply(const lib::ClusterState &state, - const api::StorageCommand &cmd, - uint32_t bucketCount, - uint32_t invalidBucketCount = 0) - { - ASSERT_EQ(cmd.getType(), MessageType::REQUESTBUCKETINFO); - const api::StorageMessageAddress &address(*cmd.getAddress()); - getBucketDBUpdater().onRequestBucketInfoReply( - getFakeBucketReply(state, - dynamic_cast<const RequestBucketInfoCommand &>(cmd), - address.getIndex(), - bucketCount, - invalidBucketCount)); - } - - void sendFakeReplyForSingleBucketRequest( - const api::RequestBucketInfoCommand& rbi) - { - ASSERT_EQ(size_t(1), rbi.getBuckets().size()); - const document::BucketId& bucket(rbi.getBuckets()[0]); - - std::shared_ptr<api::RequestBucketInfoReply> reply( - new api::RequestBucketInfoReply(rbi)); - reply->getBucketInfo().push_back( - api::RequestBucketInfoReply::Entry(bucket, - api::BucketInfo(20, 10, 12, 50, 60, true, true))); - getBucketDBUpdater().onRequestBucketInfoReply(reply); - } - - std::string verifyBucket(document::BucketId id, const lib::ClusterState& state) { - BucketDatabase::Entry entry = getBucketDatabase().get(id); - if (!entry.valid()) { - return vespalib::make_string("%s doesn't exist in DB", - id.toString().c_str()); - } - - std::vector<uint16_t> nodes; - defaultDistributorBucketSpace().getDistribution().getIdealNodes( - lib::NodeType::STORAGE, - state, - document::BucketId(id), - nodes); - - if (nodes.size() != entry->getNodeCount()) { - return vespalib::make_string("Bucket Id %s has %d nodes in " - "ideal state, but has only %d in DB", - id.toString().c_str(), - (int)nodes.size(), - (int)entry->getNodeCount()); - } - - for (uint32_t i = 0; i<nodes.size(); i++) { - bool found = false; - - for (uint32_t j = 0; j<entry->getNodeCount(); j++) { - if (nodes[i] == entry->getNodeRef(j).getNode()) { - found = true; - } - } - - if (!found) { - return vespalib::make_string( - "Bucket Id %s has no copy from node %d", - id.toString().c_str(), - nodes[i]); - } - } - - return ""; - } - - - void verifyInvalid(document::BucketId id, int storageNode) { - BucketDatabase::Entry entry = getBucketDatabase().get(id); - - ASSERT_TRUE(entry.valid()); - - bool found = false; - for (uint32_t j = 0; j<entry->getNodeCount(); j++) { - if (entry->getNodeRef(j).getNode() == storageNode) { - ASSERT_FALSE(entry->getNodeRef(j).valid()); - found = true; - } - } - - ASSERT_TRUE(found); - } - - struct OrderByIncreasingNodeIndex { - template <typename T> - bool operator()(const T& lhs, const T& rhs) { - return (lhs->getAddress()->getIndex() - < rhs->getAddress()->getIndex()); - } - }; - - void sortSentMessagesByIndex(DistributorMessageSenderStub& sender, - size_t sortFromOffset = 0) - { - std::sort(sender.commands().begin() + sortFromOffset, - sender.commands().end(), - OrderByIncreasingNodeIndex()); - } - - void setSystemState(const lib::ClusterState& state) { - const size_t sizeBeforeState = _sender.commands().size(); - getBucketDBUpdater().onSetSystemState( - std::make_shared<api::SetSystemStateCommand>(state)); - // A lot of test logic has the assumption that all messages sent as a - // result of cluster state changes will be in increasing index order - // (for simplicity, not because this is required for correctness). - // Only sort the messages that arrived as a result of the state, don't - // jumble the sorting with any existing messages. - sortSentMessagesByIndex(_sender, sizeBeforeState); - } - - void set_cluster_state_bundle(const lib::ClusterStateBundle& state) { - const size_t sizeBeforeState = _sender.commands().size(); - getBucketDBUpdater().onSetSystemState( - std::make_shared<api::SetSystemStateCommand>(state)); - sortSentMessagesByIndex(_sender, sizeBeforeState); - } - - bool activate_cluster_state_version(uint32_t version) { - return getBucketDBUpdater().onActivateClusterStateVersion( - std::make_shared<api::ActivateClusterStateVersionCommand>(version)); - } - - void assert_has_activate_cluster_state_reply_with_actual_version(uint32_t version) { - ASSERT_EQ(size_t(1), _sender.replies().size()); - auto* response = dynamic_cast<api::ActivateClusterStateVersionReply*>(_sender.replies().back().get()); - ASSERT_TRUE(response != nullptr); - ASSERT_EQ(version, response->actualVersion()); - _sender.clear(); - } - - void completeBucketInfoGathering(const lib::ClusterState& state, - size_t expectedMsgs, - uint32_t bucketCount = 1, - uint32_t invalidBucketCount = 0) - { - ASSERT_EQ(expectedMsgs, _sender.commands().size()); - - for (uint32_t i = 0; i < _sender.commands().size(); i++) { - ASSERT_NO_FATAL_FAILURE(fakeBucketReply(state, *_sender.command(i), - bucketCount, invalidBucketCount)); - } - } - - void setAndEnableClusterState(const lib::ClusterState& state, - uint32_t expectedMsgs, - uint32_t nBuckets) - { - _sender.clear(); - setSystemState(state); - ASSERT_NO_FATAL_FAILURE(completeBucketInfoGathering(state, expectedMsgs, nBuckets)); - } - - void completeStateTransitionInSeconds(const std::string& stateStr, - uint32_t seconds, - uint32_t expectedMsgs) - { - _sender.clear(); - lib::ClusterState state(stateStr); - setSystemState(state); - getClock().addSecondsToTime(seconds); - ASSERT_NO_FATAL_FAILURE(completeBucketInfoGathering(state, expectedMsgs)); - } - - uint64_t lastTransitionTimeInMillis() { - return uint64_t(getDistributor().getMetrics().stateTransitionTime.getLast()); - } - - void setStorageNodes(uint32_t numStorageNodes) { - _sender.clear(); - - lib::ClusterState newState( - vespalib::make_string("distributor:1 storage:%d", numStorageNodes)); - - setSystemState(newState); - - for (uint32_t i=0; i< messageCount(numStorageNodes); i++) { - ASSERT_EQ(_sender.command(i)->getType(), MessageType::REQUESTBUCKETINFO); - - const api::StorageMessageAddress *address = _sender.command(i)->getAddress(); - ASSERT_EQ((uint32_t)(i / _bucketSpaces.size()), (uint32_t)address->getIndex()); - } - } - - void initializeNodesAndBuckets(uint32_t numStorageNodes, - uint32_t numBuckets) - { - ASSERT_NO_FATAL_FAILURE(setStorageNodes(numStorageNodes)); - - vespalib::string state(vespalib::make_string( - "distributor:1 storage:%d", numStorageNodes)); - lib::ClusterState newState(state); - - for (uint32_t i=0; i< messageCount(numStorageNodes); i++) { - ASSERT_NO_FATAL_FAILURE(fakeBucketReply(newState, *_sender.command(i), numBuckets)); - } - ASSERT_NO_FATAL_FAILURE(assertCorrectBuckets(numBuckets, state)); - } - - bool bucketHasNode(document::BucketId id, uint16_t node) const { - BucketDatabase::Entry entry = getBucket(id); - assert(entry.valid()); - - for (uint32_t j=0; j<entry->getNodeCount(); j++) { - if (entry->getNodeRef(j).getNode() == node) { - return true; - } - } - - return false; - } - - api::StorageMessageAddress storageAddress(uint16_t node) { - static vespalib::string _storage("storage"); - return api::StorageMessageAddress(&_storage, lib::NodeType::STORAGE, node); - } - - std::string getSentNodes(const std::string& oldClusterState, - const std::string& newClusterState); - - std::string getSentNodesDistributionChanged( - const std::string& oldClusterState); - - std::vector<uint16_t> getSentNodesWithPreemption( - const std::string& oldClusterState, - uint32_t expectedOldStateMessages, - const std::string& preemptedClusterState, - const std::string& newClusterState); - - std::vector<uint16_t> getSendSet() const; - - std::string mergeBucketLists( - const lib::ClusterState& oldState, - const std::string& existingData, - const lib::ClusterState& newState, - const std::string& newData, - bool includeBucketInfo = false); - - std::string mergeBucketLists( - const std::string& existingData, - const std::string& newData, - bool includeBucketInfo = false); - - void assertCorrectBuckets(int numBuckets, const std::string& stateStr) { - lib::ClusterState state(stateStr); - for (int i=0; i<numBuckets; i++) { - ASSERT_EQ(getIdealStr(document::BucketId(16, i), state), - getNodes(document::BucketId(16, i))); - } - } - - void setDistribution(const std::string& distConfig) { - triggerDistributionChange( - std::make_shared<lib::Distribution>(distConfig)); - } - - std::string getDistConfig6Nodes2Groups() const { - return ("redundancy 2\n" - "group[3]\n" - "group[0].name \"invalid\"\n" - "group[0].index \"invalid\"\n" - "group[0].partitions 1|*\n" - "group[0].nodes[0]\n" - "group[1].name rack0\n" - "group[1].index 0\n" - "group[1].nodes[3]\n" - "group[1].nodes[0].index 0\n" - "group[1].nodes[1].index 1\n" - "group[1].nodes[2].index 2\n" - "group[2].name rack1\n" - "group[2].index 1\n" - "group[2].nodes[3]\n" - "group[2].nodes[0].index 3\n" - "group[2].nodes[1].index 4\n" - "group[2].nodes[2].index 5\n"); - } - - std::string getDistConfig6Nodes4Groups() const { - return ("redundancy 2\n" - "group[4]\n" - "group[0].name \"invalid\"\n" - "group[0].index \"invalid\"\n" - "group[0].partitions 1|*\n" - "group[0].nodes[0]\n" - "group[1].name rack0\n" - "group[1].index 0\n" - "group[1].nodes[2]\n" - "group[1].nodes[0].index 0\n" - "group[1].nodes[1].index 1\n" - "group[2].name rack1\n" - "group[2].index 1\n" - "group[2].nodes[2]\n" - "group[2].nodes[0].index 2\n" - "group[2].nodes[1].index 3\n" - "group[3].name rack2\n" - "group[3].index 2\n" - "group[3].nodes[2]\n" - "group[3].nodes[0].index 4\n" - "group[3].nodes[1].index 5\n"); - } - - std::string getDistConfig3Nodes1Group() const { - return ("redundancy 2\n" - "group[2]\n" - "group[0].name \"invalid\"\n" - "group[0].index \"invalid\"\n" - "group[0].partitions 1|*\n" - "group[0].nodes[0]\n" - "group[1].name rack0\n" - "group[1].index 0\n" - "group[1].nodes[3]\n" - "group[1].nodes[0].index 0\n" - "group[1].nodes[1].index 1\n" - "group[1].nodes[2].index 2\n"); - } - - struct PendingClusterStateFixture { - DistributorMessageSenderStub sender; - framework::defaultimplementation::FakeClock clock; - std::unique_ptr<PendingClusterState> state; - - PendingClusterStateFixture( - LegacyBucketDBUpdaterTest& owner, - const std::string& oldClusterState, - const std::string& newClusterState) - { - std::shared_ptr<api::SetSystemStateCommand> cmd( - new api::SetSystemStateCommand( - lib::ClusterState(newClusterState))); - - ClusterInformation::CSP clusterInfo( - owner.createClusterInfo(oldClusterState)); - - OutdatedNodesMap outdatedNodesMap; - state = PendingClusterState::createForClusterStateChange( - clock, clusterInfo, sender, - owner.getBucketSpaceRepo(), - cmd, outdatedNodesMap, api::Timestamp(1)); - } - - PendingClusterStateFixture( - LegacyBucketDBUpdaterTest& owner, - const std::string& oldClusterState) - { - ClusterInformation::CSP clusterInfo( - owner.createClusterInfo(oldClusterState)); - - state = PendingClusterState::createForDistributionChange( - clock, clusterInfo, sender, owner.getBucketSpaceRepo(), api::Timestamp(1)); - } - }; - - std::unique_ptr<PendingClusterStateFixture> createPendingStateFixtureForStateChange( - const std::string& oldClusterState, - const std::string& newClusterState) - { - return std::make_unique<PendingClusterStateFixture>(*this, oldClusterState, newClusterState); - } - - std::unique_ptr<PendingClusterStateFixture> createPendingStateFixtureForDistributionChange( - const std::string& oldClusterState) - { - return std::make_unique<PendingClusterStateFixture>(*this, oldClusterState); - } - - uint32_t populate_bucket_db_via_request_bucket_info_for_benchmarking(); - - void complete_recovery_mode() { - _distributor->scanAllBuckets(); - } -}; - -LegacyBucketDBUpdaterTest::LegacyBucketDBUpdaterTest() - : DistributorTestUtil(), - _bucketSpaces() -{ -} - -LegacyBucketDBUpdaterTest::~LegacyBucketDBUpdaterTest() = default; - -// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest -TEST_F(LegacyBucketDBUpdaterTest, normal_usage) { - setSystemState(lib::ClusterState("distributor:2 .0.s:i .1.s:i storage:3")); - - ASSERT_EQ(messageCount(3), _sender.commands().size()); - - // Ensure distribution hash is set correctly - ASSERT_EQ( - defaultDistributorBucketSpace().getDistribution() - .getNodeGraph().getDistributionConfigHash(), - dynamic_cast<const RequestBucketInfoCommand&>( - *_sender.command(0)).getDistributionHash()); - - ASSERT_NO_FATAL_FAILURE(fakeBucketReply(lib::ClusterState("distributor:2 .0.s:i .1.s:i storage:3"), - *_sender.command(0), 10)); - - _sender.clear(); - - // Optimization for not refetching unneeded data after cluster state - // change is only implemented after completion of previous cluster state - setSystemState(lib::ClusterState("distributor:2 .0.s:i storage:3")); - - ASSERT_EQ(messageCount(3), _sender.commands().size()); - // Expect reply of first set SystemState request. - ASSERT_EQ(size_t(1), _sender.replies().size()); - - ASSERT_NO_FATAL_FAILURE(completeBucketInfoGathering( - lib::ClusterState("distributor:2 .0.s:i .1.s:i storage:3"), - messageCount(3), 10)); - ASSERT_NO_FATAL_FAILURE(assertCorrectBuckets(10, "distributor:2 storage:3")); -} - -// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest -TEST_F(LegacyBucketDBUpdaterTest, distributor_change) { - int numBuckets = 100; - - // First sends request - setSystemState(lib::ClusterState("distributor:2 .0.s:i .1.s:i storage:3")); - ASSERT_EQ(messageCount(3), _sender.commands().size()); - ASSERT_NO_FATAL_FAILURE(completeBucketInfoGathering(lib::ClusterState("distributor:2 .0.s:i .1.s:i storage:3"), - messageCount(3), numBuckets)); - _sender.clear(); - - // No change from initializing to up (when done with last job) - setSystemState(lib::ClusterState("distributor:2 storage:3")); - ASSERT_EQ(size_t(0), _sender.commands().size()); - _sender.clear(); - - // Adding node. No new read requests, but buckets thrown - setSystemState(lib::ClusterState("distributor:3 storage:3")); - ASSERT_EQ(size_t(0), _sender.commands().size()); - ASSERT_NO_FATAL_FAILURE(assertCorrectBuckets(numBuckets, "distributor:3 storage:3")); - _sender.clear(); - - // Removing distributor. Need to refetch new data from all nodes. - setSystemState(lib::ClusterState("distributor:2 storage:3")); - ASSERT_EQ(messageCount(3), _sender.commands().size()); - ASSERT_NO_FATAL_FAILURE(completeBucketInfoGathering(lib::ClusterState("distributor:2 storage:3"), - messageCount(3), numBuckets)); - _sender.clear(); - ASSERT_NO_FATAL_FAILURE(assertCorrectBuckets(numBuckets, "distributor:2 storage:3")); -} - -// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest -TEST_F(LegacyBucketDBUpdaterTest, distributor_change_with_grouping) { - std::string distConfig(getDistConfig6Nodes2Groups()); - setDistribution(distConfig); - int numBuckets = 100; - - setSystemState(lib::ClusterState("distributor:6 storage:6")); - ASSERT_EQ(messageCount(6), _sender.commands().size()); - ASSERT_NO_FATAL_FAILURE(completeBucketInfoGathering(lib::ClusterState("distributor:6 storage:6"), - messageCount(6), numBuckets)); - _sender.clear(); - - // Distributor going down in other group, no change - setSystemState(lib::ClusterState("distributor:6 .5.s:d storage:6")); - ASSERT_EQ(size_t(0), _sender.commands().size()); - _sender.clear(); - - setSystemState(lib::ClusterState("distributor:6 storage:6")); - ASSERT_EQ(size_t(0), _sender.commands().size()); - ASSERT_NO_FATAL_FAILURE(assertCorrectBuckets(numBuckets, "distributor:6 storage:6")); - _sender.clear(); - - // Unchanged grouping cause no change. - setDistribution(distConfig); - ASSERT_EQ(size_t(0), _sender.commands().size()); - - // Changed grouping cause change - setDistribution(getDistConfig6Nodes4Groups()); - - ASSERT_EQ(messageCount(6), _sender.commands().size()); -} - -// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest -TEST_F(LegacyBucketDBUpdaterTest, normal_usage_initializing) { - setSystemState(lib::ClusterState("distributor:1 .0.s:i storage:1 .0.s:i")); - - ASSERT_EQ(_bucketSpaces.size(), _sender.commands().size()); - - // Not yet passing on system state. - ASSERT_EQ(size_t(0), _senderDown.commands().size()); - - ASSERT_NO_FATAL_FAILURE(completeBucketInfoGathering(lib::ClusterState("distributor:1 .0.s:i storage:1"), - _bucketSpaces.size(), 10, 10)); - - ASSERT_NO_FATAL_FAILURE(assertCorrectBuckets(10, "distributor:1 storage:1")); - - for (int i=10; i<20; i++) { - ASSERT_NO_FATAL_FAILURE(verifyInvalid(document::BucketId(16, i), 0)); - } - - // Pass on cluster state and recheck buckets now. - ASSERT_EQ(size_t(1), _senderDown.commands().size()); - - _sender.clear(); - _senderDown.clear(); - - setSystemState(lib::ClusterState("distributor:1 .0.s:i storage:1")); - - // Send a new request bucket info up. - ASSERT_EQ(_bucketSpaces.size(), _sender.commands().size()); - - ASSERT_NO_FATAL_FAILURE(completeBucketInfoGathering(lib::ClusterState("distributor:1 .0.s:i storage:1"), - _bucketSpaces.size(), 20)); - - // Pass on cluster state and recheck buckets now. - ASSERT_EQ(size_t(1), _senderDown.commands().size()); - - ASSERT_NO_FATAL_FAILURE(assertCorrectBuckets(20, "distributor:1 storage:1")); -} - -// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest -TEST_F(LegacyBucketDBUpdaterTest, failed_request_bucket_info) { - setSystemState(lib::ClusterState("distributor:1 .0.s:i storage:1")); - - // 2 messages sent up: 1 to the nodes, and one reply to the setsystemstate. - ASSERT_EQ(_bucketSpaces.size(), _sender.commands().size()); - - { - for (uint32_t i = 0; i < _bucketSpaces.size(); ++i) { - std::shared_ptr<api::RequestBucketInfoReply> reply = - getFakeBucketReply(lib::ClusterState("distributor:1 .0.s:i storage:1"), - *((RequestBucketInfoCommand*)_sender.command(i).get()), - 0, - 10); - reply->setResult(api::ReturnCode::NOT_CONNECTED); - getBucketDBUpdater().onRequestBucketInfoReply(reply); - } - - // Trigger that delayed message is sent - getClock().addSecondsToTime(10); - getBucketDBUpdater().resendDelayedMessages(); - } - - // Should be resent. - ASSERT_EQ(getRequestBucketInfoStrings(messageCount(2)), _sender.getCommands()); - - ASSERT_EQ(size_t(0), _senderDown.commands().size()); - - for (uint32_t i = 0; i < _bucketSpaces.size(); ++i) { - ASSERT_NO_FATAL_FAILURE(fakeBucketReply(lib::ClusterState("distributor:1 .0.s:i storage:1"), - *_sender.command(_bucketSpaces.size() + i), 10)); - } - - for (int i=0; i<10; i++) { - EXPECT_EQ(std::string(""), - verifyBucket(document::BucketId(16, i), - lib::ClusterState("distributor:1 storage:1"))); - } - - // Set system state should now be passed on - EXPECT_EQ(std::string("Set system state"), _senderDown.getCommands()); -} - -// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest -TEST_F(LegacyBucketDBUpdaterTest, down_while_init) { - ASSERT_NO_FATAL_FAILURE(setStorageNodes(3)); - - ASSERT_NO_FATAL_FAILURE(fakeBucketReply(lib::ClusterState("distributor:1 storage:3"), - *_sender.command(0), 5)); - - setSystemState(lib::ClusterState("distributor:1 storage:3 .1.s:d")); - - ASSERT_NO_FATAL_FAILURE(fakeBucketReply(lib::ClusterState("distributor:1 storage:3"), - *_sender.command(2), 5)); - - ASSERT_NO_FATAL_FAILURE(fakeBucketReply(lib::ClusterState("distributor:1 storage:3"), - *_sender.command(1), 5)); -} - -bool -LegacyBucketDBUpdaterTest::bucketExistsThatHasNode(int bucketCount, uint16_t node) const -{ - for (int i=1; i<bucketCount; i++) { - if (bucketHasNode(document::BucketId(16, i), node)) { - return true; - } - } - - return false; -} - -std::string -LegacyBucketDBUpdaterTest::getNodeList(std::vector<uint16_t> nodes, size_t count) -{ - std::ostringstream ost; - bool first = true; - for (const auto &node : nodes) { - for (uint32_t i = 0; i < count; ++i) { - if (!first) { - ost << ","; - } - ost << node; - first = false; - } - } - return ost.str(); -} - -std::string -LegacyBucketDBUpdaterTest::getNodeList(std::vector<uint16_t> nodes) -{ - return getNodeList(std::move(nodes), _bucketSpaces.size()); -} - -std::vector<uint16_t> -LegacyBucketDBUpdaterTest::expandNodeVec(const std::vector<uint16_t> &nodes) -{ - std::vector<uint16_t> res; - size_t count = _bucketSpaces.size(); - for (const auto &node : nodes) { - for (uint32_t i = 0; i < count; ++i) { - res.push_back(node); - } - } - return res; -} - -// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest -TEST_F(LegacyBucketDBUpdaterTest, node_down) { - ASSERT_NO_FATAL_FAILURE(setStorageNodes(3)); - enableDistributorClusterState("distributor:1 storage:3"); - - for (int i=1; i<100; i++) { - addIdealNodes(document::BucketId(16, i)); - } - - EXPECT_TRUE(bucketExistsThatHasNode(100, 1)); - - setSystemState(lib::ClusterState("distributor:1 storage:3 .1.s:d")); - - EXPECT_FALSE(bucketExistsThatHasNode(100, 1)); -} - -// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest -TEST_F(LegacyBucketDBUpdaterTest, storage_node_in_maintenance_clears_buckets_for_node) { - ASSERT_NO_FATAL_FAILURE(setStorageNodes(3)); - enableDistributorClusterState("distributor:1 storage:3"); - - for (int i=1; i<100; i++) { - addIdealNodes(document::BucketId(16, i)); - } - - EXPECT_TRUE(bucketExistsThatHasNode(100, 1)); - - setSystemState(lib::ClusterState("distributor:1 storage:3 .1.s:m")); - - EXPECT_FALSE(bucketExistsThatHasNode(100, 1)); -} - -// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest -TEST_F(LegacyBucketDBUpdaterTest, node_down_copies_get_in_sync) { - ASSERT_NO_FATAL_FAILURE(setStorageNodes(3)); - - lib::ClusterState systemState("distributor:1 storage:3"); - document::BucketId bid(16, 1); - - addNodesToBucketDB(bid, "0=3,1=2,2=3"); - - setSystemState(lib::ClusterState("distributor:1 storage:3 .1.s:d")); - - EXPECT_EQ( - std::string("BucketId(0x4000000000000001) : " - "node(idx=0,crc=0x3,docs=3/3,bytes=3/3,trusted=true,active=false,ready=false), " - "node(idx=2,crc=0x3,docs=3/3,bytes=3/3,trusted=true,active=false,ready=false)"), - dumpBucket(bid)); -} - -// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest -TEST_F(LegacyBucketDBUpdaterTest, initializing_while_recheck) { - lib::ClusterState systemState("distributor:1 storage:2 .0.s:i .0.i:0.1"); - setSystemState(systemState); - - ASSERT_EQ(messageCount(2), _sender.commands().size()); - ASSERT_EQ(size_t(0), _senderDown.commands().size()); - - getBucketDBUpdater().recheckBucketInfo(1, makeDocumentBucket(document::BucketId(16, 3))); - - for (uint32_t i = 0; i < messageCount(2); ++i) { - ASSERT_NO_FATAL_FAILURE(fakeBucketReply(systemState, *_sender.command(i), 100)); - } - - // Now we can pass on system state. - ASSERT_EQ(size_t(1), _senderDown.commands().size()); - EXPECT_EQ(MessageType::SETSYSTEMSTATE, _senderDown.command(0)->getType()); -} - -// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest -TEST_F(LegacyBucketDBUpdaterTest, bit_change) { - std::vector<document::BucketId> bucketlist; - - { - setSystemState(lib::ClusterState("bits:14 storage:1 distributor:2")); - - ASSERT_EQ(_bucketSpaces.size(), _sender.commands().size()); - - for (uint32_t bsi = 0; bsi < _bucketSpaces.size(); ++bsi) { - ASSERT_EQ(_sender.command(bsi)->getType(), MessageType::REQUESTBUCKETINFO); - const auto &req = dynamic_cast<const RequestBucketInfoCommand &>(*_sender.command(bsi)); - auto sreply = std::make_shared<RequestBucketInfoReply>(req); - sreply->setAddress(storageAddress(0)); - auto& vec = sreply->getBucketInfo(); - if (req.getBucketSpace() == FixedBucketSpaces::default_space()) { - int cnt=0; - for (int i=0; cnt < 2; i++) { - lib::Distribution distribution = defaultDistributorBucketSpace().getDistribution(); - std::vector<uint16_t> distributors; - if (distribution.getIdealDistributorNode( - lib::ClusterState("bits:14 storage:1 distributor:2"), - document::BucketId(16, i)) - == 0) - { - vec.push_back(api::RequestBucketInfoReply::Entry( - document::BucketId(16, i), - api::BucketInfo(10,1,1))); - - bucketlist.push_back(document::BucketId(16, i)); - cnt++; - } - } - } - - getBucketDBUpdater().onRequestBucketInfoReply(sreply); - } - } - - EXPECT_EQ(std::string("BucketId(0x4000000000000001) : " - "node(idx=0,crc=0xa,docs=1/1,bytes=1/1,trusted=true,active=false,ready=false)"), - dumpBucket(bucketlist[0])); - EXPECT_EQ(std::string("BucketId(0x4000000000000002) : " - "node(idx=0,crc=0xa,docs=1/1,bytes=1/1,trusted=true,active=false,ready=false)"), - dumpBucket(bucketlist[1])); - - { - _sender.clear(); - setSystemState(lib::ClusterState("bits:16 storage:1 distributor:2")); - - ASSERT_EQ(_bucketSpaces.size(), _sender.commands().size()); - for (uint32_t bsi = 0; bsi < _bucketSpaces.size(); ++bsi) { - - ASSERT_EQ(_sender.command(bsi)->getType(), MessageType::REQUESTBUCKETINFO); - const auto &req = dynamic_cast<const RequestBucketInfoCommand &>(*_sender.command(bsi)); - auto sreply = std::make_shared<RequestBucketInfoReply>(req); - sreply->setAddress(storageAddress(0)); - sreply->setResult(api::ReturnCode::OK); - if (req.getBucketSpace() == FixedBucketSpaces::default_space()) { - api::RequestBucketInfoReply::EntryVector &vec = sreply->getBucketInfo(); - - for (uint32_t i = 0; i < 3; ++i) { - vec.push_back(api::RequestBucketInfoReply::Entry( - document::BucketId(16, i), - api::BucketInfo(10,1,1))); - } - - vec.push_back(api::RequestBucketInfoReply::Entry( - document::BucketId(16, 4), - api::BucketInfo(10,1,1))); - } - - getBucketDBUpdater().onRequestBucketInfoReply(sreply); - } - } - - EXPECT_EQ(std::string("BucketId(0x4000000000000000) : " - "node(idx=0,crc=0xa,docs=1/1,bytes=1/1,trusted=true,active=false,ready=false)"), - dumpBucket(document::BucketId(16, 0))); - EXPECT_EQ(std::string("BucketId(0x4000000000000001) : " - "node(idx=0,crc=0xa,docs=1/1,bytes=1/1,trusted=true,active=false,ready=false)"), - dumpBucket(document::BucketId(16, 1))); - EXPECT_EQ(std::string("BucketId(0x4000000000000002) : " - "node(idx=0,crc=0xa,docs=1/1,bytes=1/1,trusted=true,active=false,ready=false)"), - dumpBucket(document::BucketId(16, 2))); - EXPECT_EQ(std::string("BucketId(0x4000000000000004) : " - "node(idx=0,crc=0xa,docs=1/1,bytes=1/1,trusted=true,active=false,ready=false)"), - dumpBucket(document::BucketId(16, 4))); - - { - _sender.clear(); - setSystemState(lib::ClusterState("storage:1 distributor:2 .1.s:i")); - } - - { - _sender.clear(); - setSystemState(lib::ClusterState("storage:1 distributor:2")); - } -}; - -// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest -TEST_F(LegacyBucketDBUpdaterTest, recheck_node_with_failure) { - ASSERT_NO_FATAL_FAILURE(initializeNodesAndBuckets(3, 5)); - - _sender.clear(); - - getBucketDBUpdater().recheckBucketInfo(1, makeDocumentBucket(document::BucketId(16, 3))); - - ASSERT_EQ(size_t(1), _sender.commands().size()); - - uint16_t index = 0; - { - auto& rbi = dynamic_cast<RequestBucketInfoCommand&>(*_sender.command(0)); - ASSERT_EQ(size_t(1), rbi.getBuckets().size()); - EXPECT_EQ(document::BucketId(16, 3), rbi.getBuckets()[0]); - auto reply = std::make_shared<api::RequestBucketInfoReply>(rbi); - const api::StorageMessageAddress *address = _sender.command(0)->getAddress(); - index = address->getIndex(); - reply->setResult(api::ReturnCode::NOT_CONNECTED); - getBucketDBUpdater().onRequestBucketInfoReply(reply); - // Trigger that delayed message is sent - getClock().addSecondsToTime(10); - getBucketDBUpdater().resendDelayedMessages(); - } - - ASSERT_EQ(size_t(2), _sender.commands().size()); - - setSystemState( - lib::ClusterState(vespalib::make_string("distributor:1 storage:3 .%d.s:d", index))); - - // Recheck bucket. - { - auto& rbi = dynamic_cast<RequestBucketInfoCommand&>(*_sender.command(1)); - ASSERT_EQ(size_t(1), rbi.getBuckets().size()); - EXPECT_EQ(document::BucketId(16, 3), rbi.getBuckets()[0]); - auto reply = std::make_shared<api::RequestBucketInfoReply>(rbi); - reply->setResult(api::ReturnCode::NOT_CONNECTED); - getBucketDBUpdater().onRequestBucketInfoReply(reply); - } - - // Should not retry since node is down. - EXPECT_EQ(size_t(2), _sender.commands().size()); -} - -// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest -TEST_F(LegacyBucketDBUpdaterTest, recheck_node) { - ASSERT_NO_FATAL_FAILURE(initializeNodesAndBuckets(3, 5)); - - _sender.clear(); - - getBucketDBUpdater().recheckBucketInfo(1, makeDocumentBucket(document::BucketId(16, 3))); - - ASSERT_EQ(size_t(1), _sender.commands().size()); - - auto& rbi = dynamic_cast<RequestBucketInfoCommand&>(*_sender.command(0)); - ASSERT_EQ(size_t(1), rbi.getBuckets().size()); - EXPECT_EQ(document::BucketId(16, 3), rbi.getBuckets()[0]); - - auto reply = std::make_shared<api::RequestBucketInfoReply>(rbi); - reply->getBucketInfo().push_back( - api::RequestBucketInfoReply::Entry(document::BucketId(16, 3), - api::BucketInfo(20, 10, 12, 50, 60, true, true))); - getBucketDBUpdater().onRequestBucketInfoReply(reply); - - lib::ClusterState state("distributor:1 storage:3"); - for (uint32_t i = 0; i < 3; i++) { - EXPECT_EQ(getIdealStr(document::BucketId(16, i), state), - getNodes(document::BucketId(16, i))); - } - - for (uint32_t i = 4; i < 5; i++) { - EXPECT_EQ(getIdealStr(document::BucketId(16, i), state), - getNodes(document::BucketId(16, i))); - } - - BucketDatabase::Entry entry = getBucketDatabase().get(document::BucketId(16, 3)); - ASSERT_TRUE(entry.valid()); - - const BucketCopy* copy = entry->getNode(1); - ASSERT_TRUE(copy != nullptr); - EXPECT_EQ(api::BucketInfo(20,10,12, 50, 60, true, true), copy->getBucketInfo()); -} - -// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest -TEST_F(LegacyBucketDBUpdaterTest, notify_bucket_change) { - enableDistributorClusterState("distributor:1 storage:1"); - - addNodesToBucketDB(document::BucketId(16, 1), "0=1234"); - _sender.replies().clear(); - - { - api::BucketInfo info(1, 2, 3, 4, 5, true, true); - auto cmd(std::make_shared<api::NotifyBucketChangeCommand>( - makeDocumentBucket(document::BucketId(16, 1)), info)); - cmd->setSourceIndex(0); - getBucketDBUpdater().onNotifyBucketChange(cmd); - } - - { - api::BucketInfo info(10, 11, 12, 13, 14, false, false); - auto cmd(std::make_shared<api::NotifyBucketChangeCommand>( - makeDocumentBucket(document::BucketId(16, 2)), info)); - cmd->setSourceIndex(0); - getBucketDBUpdater().onNotifyBucketChange(cmd); - } - - // Must receive reply - ASSERT_EQ(size_t(2), _sender.replies().size()); - - for (int i = 0; i < 2; ++i) { - ASSERT_EQ(MessageType::NOTIFYBUCKETCHANGE_REPLY, - _sender.reply(i)->getType()); - } - - // No database update until request bucket info replies have been received. - EXPECT_EQ(std::string("BucketId(0x4000000000000001) : " - "node(idx=0,crc=0x4d2,docs=1234/1234,bytes=1234/1234," - "trusted=false,active=false,ready=false)"), - dumpBucket(document::BucketId(16, 1))); - EXPECT_EQ(std::string("NONEXISTING"), dumpBucket(document::BucketId(16, 2))); - - ASSERT_EQ(size_t(2), _sender.commands().size()); - - std::vector<api::BucketInfo> infos; - infos.push_back(api::BucketInfo(4567, 200, 2000, 400, 4000, true, true)); - infos.push_back(api::BucketInfo(8999, 300, 3000, 500, 5000, false, false)); - - for (int i = 0; i < 2; ++i) { - auto& rbi = dynamic_cast<RequestBucketInfoCommand&>(*_sender.command(i)); - ASSERT_EQ(size_t(1), rbi.getBuckets().size()); - EXPECT_EQ(document::BucketId(16, i + 1), rbi.getBuckets()[0]); - - auto reply = std::make_shared<api::RequestBucketInfoReply>(rbi); - reply->getBucketInfo().push_back( - api::RequestBucketInfoReply::Entry(document::BucketId(16, i + 1), - infos[i])); - getBucketDBUpdater().onRequestBucketInfoReply(reply); - } - - EXPECT_EQ(std::string("BucketId(0x4000000000000001) : " - "node(idx=0,crc=0x11d7,docs=200/400,bytes=2000/4000,trusted=true,active=true,ready=true)"), - dumpBucket(document::BucketId(16, 1))); - EXPECT_EQ(std::string("BucketId(0x4000000000000002) : " - "node(idx=0,crc=0x2327,docs=300/500,bytes=3000/5000,trusted=true,active=false,ready=false)"), - dumpBucket(document::BucketId(16, 2))); -} - -// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest -TEST_F(LegacyBucketDBUpdaterTest, notify_bucket_change_from_node_down) { - enableDistributorClusterState("distributor:1 storage:2"); - - addNodesToBucketDB(document::BucketId(16, 1), "1=1234"); - - _sender.replies().clear(); - - { - api::BucketInfo info(8999, 300, 3000, 500, 5000, false, false); - auto cmd(std::make_shared<api::NotifyBucketChangeCommand>( - makeDocumentBucket(document::BucketId(16, 1)), info)); - cmd->setSourceIndex(0); - getBucketDBUpdater().onNotifyBucketChange(cmd); - } - // Enable here to avoid having request bucket info be silently swallowed - // (send_request_bucket_info drops message if node is down). - enableDistributorClusterState("distributor:1 storage:2 .0.s:d"); - - ASSERT_EQ(std::string("BucketId(0x4000000000000001) : " - "node(idx=1,crc=0x4d2,docs=1234/1234,bytes=1234/1234,trusted=false,active=false,ready=false)"), - dumpBucket(document::BucketId(16, 1))); - - ASSERT_EQ(size_t(1), _sender.replies().size()); - ASSERT_EQ(MessageType::NOTIFYBUCKETCHANGE_REPLY, _sender.reply(0)->getType()); - - // Currently, this pending operation will be auto-flushed when the cluster state - // changes so the behavior is still correct. Keep this test around to prevent - // regressions here. - ASSERT_EQ(size_t(1), _sender.commands().size()); - auto& rbi = dynamic_cast<RequestBucketInfoCommand&>(*_sender.command(0)); - ASSERT_EQ(size_t(1), rbi.getBuckets().size()); - EXPECT_EQ(document::BucketId(16, 1), rbi.getBuckets()[0]); - - auto reply = std::make_shared<api::RequestBucketInfoReply>(rbi); - reply->getBucketInfo().push_back( - api::RequestBucketInfoReply::Entry( - document::BucketId(16, 1), - api::BucketInfo(8999, 300, 3000, 500, 5000, false, false))); - getBucketDBUpdater().onRequestBucketInfoReply(reply); - - // No change - EXPECT_EQ(std::string("BucketId(0x4000000000000001) : " - "node(idx=1,crc=0x4d2,docs=1234/1234,bytes=1234/1234,trusted=false,active=false,ready=false)"), - dumpBucket(document::BucketId(16, 1))); -} - -// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest -/** - * Test that NotifyBucketChange received while there's a pending cluster state - * waits until the cluster state has been enabled as current before it sends off - * the single bucket info requests. This is to prevent a race condition where - * the replies to bucket info requests for buckets that would be owned by the - * distributor in the pending state but not by the current state would be - * discarded when attempted inserted into the bucket database. - */ -TEST_F(LegacyBucketDBUpdaterTest, notify_change_with_pending_state_queues_bucket_info_requests) { - setSystemState(lib::ClusterState("distributor:1 storage:1")); - ASSERT_EQ(_bucketSpaces.size(), _sender.commands().size()); - - { - api::BucketInfo info(8999, 300, 3000, 500, 5000, false, false); - auto cmd(std::make_shared<api::NotifyBucketChangeCommand>( - makeDocumentBucket(document::BucketId(16, 1)), info)); - cmd->setSourceIndex(0); - getBucketDBUpdater().onNotifyBucketChange(cmd); - } - - ASSERT_EQ(_bucketSpaces.size(), _sender.commands().size()); - - ASSERT_NO_FATAL_FAILURE(completeBucketInfoGathering(lib::ClusterState("distributor:1 storage:1"), - _bucketSpaces.size(), 10)); - - ASSERT_EQ(_bucketSpaces.size() + 1, _sender.commands().size()); - - { - auto& rbi = dynamic_cast<RequestBucketInfoCommand&>(*_sender.command(_bucketSpaces.size())); - ASSERT_EQ(size_t(1), rbi.getBuckets().size()); - EXPECT_EQ(document::BucketId(16, 1), rbi.getBuckets()[0]); - } - _sender.clear(); - - // Queue must be cleared once pending state is enabled. - { - lib::ClusterState state("distributor:1 storage:2"); - uint32_t expectedMsgs = _bucketSpaces.size(), dummyBucketsToReturn = 1; - ASSERT_NO_FATAL_FAILURE(setAndEnableClusterState(state, expectedMsgs, dummyBucketsToReturn)); - } - ASSERT_EQ(_bucketSpaces.size(), _sender.commands().size()); - { - auto& rbi = dynamic_cast<RequestBucketInfoCommand&>(*_sender.command(0)); - EXPECT_EQ(size_t(0), rbi.getBuckets().size()); - } -} - -// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest -TEST_F(LegacyBucketDBUpdaterTest, merge_reply) { - enableDistributorClusterState("distributor:1 storage:3"); - - addNodesToBucketDB(document::BucketId(16, 1234), - "0=1234,1=1234,2=1234"); - - std::vector<api::MergeBucketCommand::Node> nodes; - nodes.push_back(api::MergeBucketCommand::Node(0)); - nodes.push_back(api::MergeBucketCommand::Node(1)); - nodes.push_back(api::MergeBucketCommand::Node(2)); - - api::MergeBucketCommand cmd(makeDocumentBucket(document::BucketId(16, 1234)), nodes, 0); - - auto reply = std::make_shared<api::MergeBucketReply>(cmd); - - _sender.clear(); - getBucketDBUpdater().onMergeBucketReply(reply); - - ASSERT_EQ(size_t(3), _sender.commands().size()); - - for (uint32_t i = 0; i < 3; i++) { - auto req = std::dynamic_pointer_cast<api::RequestBucketInfoCommand>(_sender.command(i)); - - ASSERT_TRUE(req.get() != nullptr); - ASSERT_EQ(size_t(1), req->getBuckets().size()); - EXPECT_EQ(document::BucketId(16, 1234), req->getBuckets()[0]); - - auto reqreply = std::make_shared<api::RequestBucketInfoReply>(*req); - reqreply->getBucketInfo().push_back( - api::RequestBucketInfoReply::Entry(document::BucketId(16, 1234), - api::BucketInfo(10 * (i + 1), 100 * (i +1), 1000 * (i+1)))); - - getBucketDBUpdater().onRequestBucketInfoReply(reqreply); - } - - EXPECT_EQ(std::string("BucketId(0x40000000000004d2) : " - "node(idx=0,crc=0xa,docs=100/100,bytes=1000/1000,trusted=false,active=false,ready=false), " - "node(idx=1,crc=0x14,docs=200/200,bytes=2000/2000,trusted=false,active=false,ready=false), " - "node(idx=2,crc=0x1e,docs=300/300,bytes=3000/3000,trusted=false,active=false,ready=false)"), - dumpBucket(document::BucketId(16, 1234))); -}; - -// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest -TEST_F(LegacyBucketDBUpdaterTest, merge_reply_node_down) { - enableDistributorClusterState("distributor:1 storage:3"); - std::vector<api::MergeBucketCommand::Node> nodes; - - addNodesToBucketDB(document::BucketId(16, 1234), "0=1234,1=1234,2=1234"); - - for (uint32_t i = 0; i < 3; ++i) { - nodes.push_back(api::MergeBucketCommand::Node(i)); - } - - api::MergeBucketCommand cmd(makeDocumentBucket(document::BucketId(16, 1234)), nodes, 0); - - auto reply = std::make_shared<api::MergeBucketReply>(cmd); - - setSystemState(lib::ClusterState("distributor:1 storage:2")); - - _sender.clear(); - getBucketDBUpdater().onMergeBucketReply(reply); - - ASSERT_EQ(size_t(2), _sender.commands().size()); - - for (uint32_t i = 0; i < 2; i++) { - auto req = std::dynamic_pointer_cast<api::RequestBucketInfoCommand>(_sender.command(i)); - - ASSERT_TRUE(req.get() != nullptr); - ASSERT_EQ(size_t(1), req->getBuckets().size()); - EXPECT_EQ(document::BucketId(16, 1234), req->getBuckets()[0]); - - auto reqreply = std::make_shared<api::RequestBucketInfoReply>(*req); - reqreply->getBucketInfo().push_back( - api::RequestBucketInfoReply::Entry( - document::BucketId(16, 1234), - api::BucketInfo(10 * (i + 1), 100 * (i +1), 1000 * (i+1)))); - getBucketDBUpdater().onRequestBucketInfoReply(reqreply); - } - - EXPECT_EQ(std::string("BucketId(0x40000000000004d2) : " - "node(idx=0,crc=0xa,docs=100/100,bytes=1000/1000,trusted=false,active=false,ready=false), " - "node(idx=1,crc=0x14,docs=200/200,bytes=2000/2000,trusted=false,active=false,ready=false)"), - dumpBucket(document::BucketId(16, 1234))); -}; - -// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest -TEST_F(LegacyBucketDBUpdaterTest, merge_reply_node_down_after_request_sent) { - enableDistributorClusterState("distributor:1 storage:3"); - std::vector<api::MergeBucketCommand::Node> nodes; - - addNodesToBucketDB(document::BucketId(16, 1234), "0=1234,1=1234,2=1234"); - - for (uint32_t i = 0; i < 3; ++i) { - nodes.push_back(api::MergeBucketCommand::Node(i)); - } - - api::MergeBucketCommand cmd(makeDocumentBucket(document::BucketId(16, 1234)), nodes, 0); - - auto reply(std::make_shared<api::MergeBucketReply>(cmd)); - - _sender.clear(); - getBucketDBUpdater().onMergeBucketReply(reply); - - ASSERT_EQ(size_t(3), _sender.commands().size()); - - setSystemState(lib::ClusterState("distributor:1 storage:2")); - - for (uint32_t i = 0; i < 3; i++) { - auto req = std::dynamic_pointer_cast<api::RequestBucketInfoCommand>(_sender.command(i)); - - ASSERT_TRUE(req.get() != nullptr); - ASSERT_EQ(size_t(1), req->getBuckets().size()); - EXPECT_EQ(document::BucketId(16, 1234), req->getBuckets()[0]); - - auto reqreply = std::make_shared<api::RequestBucketInfoReply>(*req); - reqreply->getBucketInfo().push_back( - api::RequestBucketInfoReply::Entry( - document::BucketId(16, 1234), - api::BucketInfo(10 * (i + 1), 100 * (i +1), 1000 * (i+1)))); - getBucketDBUpdater().onRequestBucketInfoReply(reqreply); - } - - EXPECT_EQ(std::string("BucketId(0x40000000000004d2) : " - "node(idx=0,crc=0xa,docs=100/100,bytes=1000/1000,trusted=false,active=false,ready=false), " - "node(idx=1,crc=0x14,docs=200/200,bytes=2000/2000,trusted=false,active=false,ready=false)"), - dumpBucket(document::BucketId(16, 1234))); -}; - -// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest -TEST_F(LegacyBucketDBUpdaterTest, flush) { - enableDistributorClusterState("distributor:1 storage:3"); - _sender.clear(); - - addNodesToBucketDB(document::BucketId(16, 1234), "0=1234,1=1234,2=1234"); - - std::vector<api::MergeBucketCommand::Node> nodes; - for (uint32_t i = 0; i < 3; ++i) { - nodes.push_back(api::MergeBucketCommand::Node(i)); - } - - api::MergeBucketCommand cmd(makeDocumentBucket(document::BucketId(16, 1234)), nodes, 0); - - auto reply(std::make_shared<api::MergeBucketReply>(cmd)); - - _sender.clear(); - getBucketDBUpdater().onMergeBucketReply(reply); - - ASSERT_EQ(size_t(3), _sender.commands().size()); - ASSERT_EQ(size_t(0), _senderDown.replies().size()); - - getBucketDBUpdater().flush(); - // Flushing should drop all merge bucket replies - EXPECT_EQ(size_t(0), _senderDown.commands().size()); -} - -std::string -LegacyBucketDBUpdaterTest::getSentNodes( - const std::string& oldClusterState, - const std::string& newClusterState) -{ - auto fixture = createPendingStateFixtureForStateChange( - oldClusterState, newClusterState); - - sortSentMessagesByIndex(fixture->sender); - - std::ostringstream ost; - for (uint32_t i = 0; i < fixture->sender.commands().size(); i++) { - auto& req = dynamic_cast<RequestBucketInfoCommand&>(*fixture->sender.command(i)); - - if (i > 0) { - ost << ","; - } - - ost << req.getAddress()->getIndex(); - } - - return ost.str(); -} - -std::string -LegacyBucketDBUpdaterTest::getSentNodesDistributionChanged( - const std::string& oldClusterState) -{ - DistributorMessageSenderStub sender; - - framework::defaultimplementation::FakeClock clock; - ClusterInformation::CSP clusterInfo(createClusterInfo(oldClusterState)); - std::unique_ptr<PendingClusterState> state( - PendingClusterState::createForDistributionChange( - clock, clusterInfo, sender, getBucketSpaceRepo(), api::Timestamp(1))); - - sortSentMessagesByIndex(sender); - - std::ostringstream ost; - for (uint32_t i = 0; i < sender.commands().size(); i++) { - auto& req = dynamic_cast<RequestBucketInfoCommand&>(*sender.command(i)); - - if (i > 0) { - ost << ","; - } - - ost << req.getAddress()->getIndex(); - } - - return ost.str(); -} - -// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest -TEST_F(LegacyBucketDBUpdaterTest, pending_cluster_state_send_messages) { - EXPECT_EQ(getNodeList({0, 1, 2}), - getSentNodes("cluster:d", - "distributor:1 storage:3")); - - EXPECT_EQ(getNodeList({0, 1}), - getSentNodes("cluster:d", - "distributor:1 storage:3 .2.s:m")); - - EXPECT_EQ(getNodeList({2}), - getSentNodes("distributor:1 storage:2", - "distributor:1 storage:3")); - - EXPECT_EQ(getNodeList({2, 3, 4, 5}), - getSentNodes("distributor:1 storage:2", - "distributor:1 storage:6")); - - EXPECT_EQ(getNodeList({0, 1, 2}), - getSentNodes("distributor:4 storage:3", - "distributor:3 storage:3")); - - EXPECT_EQ(getNodeList({0, 1, 2, 3}), - getSentNodes("distributor:4 storage:3", - "distributor:4 .2.s:d storage:4")); - - EXPECT_EQ(std::string(""), - getSentNodes("distributor:4 storage:3", - "distributor:4 .0.s:d storage:4")); - - EXPECT_EQ(std::string(""), - getSentNodes("distributor:3 storage:3", - "distributor:4 storage:3")); - - EXPECT_EQ(getNodeList({2}), - getSentNodes("distributor:3 storage:3 .2.s:i", - "distributor:3 storage:3")); - - EXPECT_EQ(getNodeList({1}), - getSentNodes("distributor:3 storage:3 .1.s:d", - "distributor:3 storage:3")); - - EXPECT_EQ(getNodeList({1, 2, 4}), - getSentNodes("distributor:3 storage:4 .1.s:d .2.s:i", - "distributor:3 storage:5")); - - EXPECT_EQ(std::string(""), - getSentNodes("distributor:1 storage:3", - "cluster:d")); - - EXPECT_EQ(std::string(""), - getSentNodes("distributor:1 storage:3", - "distributor:1 storage:3")); - - EXPECT_EQ(std::string(""), - getSentNodes("distributor:1 storage:3", - "cluster:d distributor:1 storage:6")); - - EXPECT_EQ(std::string(""), - getSentNodes("distributor:3 storage:3", - "distributor:3 .2.s:m storage:3")); - - EXPECT_EQ(getNodeList({0, 1, 2}), - getSentNodes("distributor:3 .2.s:m storage:3", - "distributor:3 .2.s:d storage:3")); - - EXPECT_EQ(std::string(""), - getSentNodes("distributor:3 .2.s:m storage:3", - "distributor:3 storage:3")); - - EXPECT_EQ(getNodeList({0, 1, 2}), - getSentNodesDistributionChanged("distributor:3 storage:3")); - - EXPECT_EQ(getNodeList({0, 1}), - getSentNodes("distributor:10 storage:2", - "distributor:10 .1.s:d storage:2")); - - EXPECT_EQ(std::string(""), - getSentNodes("distributor:2 storage:2", - "distributor:3 .2.s:i storage:2")); - - EXPECT_EQ(getNodeList({0, 1, 2}), - getSentNodes("distributor:3 storage:3", - "distributor:3 .2.s:s storage:3")); - - EXPECT_EQ(std::string(""), - getSentNodes("distributor:3 .2.s:s storage:3", - "distributor:3 .2.s:d storage:3")); - - EXPECT_EQ(getNodeList({1}), - getSentNodes("distributor:3 storage:3 .1.s:m", - "distributor:3 storage:3")); - - EXPECT_EQ(std::string(""), - getSentNodes("distributor:3 storage:3", - "distributor:3 storage:3 .1.s:m")); -}; - -// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest -TEST_F(LegacyBucketDBUpdaterTest, pending_cluster_state_receive) { - DistributorMessageSenderStub sender; - - auto cmd(std::make_shared<api::SetSystemStateCommand>( - lib::ClusterState("distributor:1 storage:3"))); - - framework::defaultimplementation::FakeClock clock; - ClusterInformation::CSP clusterInfo(createClusterInfo("cluster:d")); - OutdatedNodesMap outdatedNodesMap; - std::unique_ptr<PendingClusterState> state( - PendingClusterState::createForClusterStateChange( - clock, clusterInfo, sender, getBucketSpaceRepo(), - cmd, outdatedNodesMap, api::Timestamp(1))); - - ASSERT_EQ(messageCount(3), sender.commands().size()); - - sortSentMessagesByIndex(sender); - - std::ostringstream ost; - for (uint32_t i = 0; i < sender.commands().size(); i++) { - auto* req = dynamic_cast<RequestBucketInfoCommand*>(sender.command(i).get()); - ASSERT_TRUE(req != nullptr); - - auto rep = std::make_shared<RequestBucketInfoReply>(*req); - - rep->getBucketInfo().push_back( - RequestBucketInfoReply::Entry( - document::BucketId(16, i), - api::BucketInfo(i, i, i, i, i))); - - ASSERT_TRUE(state->onRequestBucketInfoReply(rep)); - ASSERT_EQ((i == (sender.commands().size() - 1)), state->done()); - } - - auto& pendingTransition = state->getPendingBucketSpaceDbTransition(makeBucketSpace()); - EXPECT_EQ(3, (int)pendingTransition.results().size()); -} - -// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest -TEST_F(LegacyBucketDBUpdaterTest, pending_cluster_state_with_group_down) { - std::string config(getDistConfig6Nodes4Groups()); - config += "distributor_auto_ownership_transfer_on_whole_group_down true\n"; - setDistribution(config); - - // Group config has nodes {0, 1}, {2, 3}, {4, 5} - // We're node index 0. - - // Entire group 1 goes down. Must refetch from all nodes. - EXPECT_EQ(getNodeList({0, 1, 2, 3, 4, 5}), - getSentNodes("distributor:6 storage:6", - "distributor:6 .2.s:d .3.s:d storage:6")); - - // But don't fetch if not the entire group is down. - EXPECT_EQ(std::string(""), - getSentNodes("distributor:6 storage:6", - "distributor:6 .2.s:d storage:6")); -} - -// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest -TEST_F(LegacyBucketDBUpdaterTest, pending_cluster_state_with_group_down_and_no_handover) { - std::string config(getDistConfig6Nodes4Groups()); - config += "distributor_auto_ownership_transfer_on_whole_group_down false\n"; - setDistribution(config); - - // Group is down, but config says to not do anything about it. - EXPECT_EQ(getNodeList({0, 1, 2, 3, 4, 5}, _bucketSpaces.size() - 1), - getSentNodes("distributor:6 storage:6", - "distributor:6 .2.s:d .3.s:d storage:6")); -} - -namespace { - -void -parseInputData(const std::string& data, - uint64_t timestamp, - PendingClusterState& state, - bool includeBucketInfo) -{ - vespalib::StringTokenizer tokenizer(data, "|"); - for (uint32_t i = 0; i < tokenizer.size(); i++) { - vespalib::StringTokenizer tok2(tokenizer[i], ":"); - - uint16_t node = atoi(tok2[0].data()); - - state.setNodeReplied(node); - auto& pendingTransition = state.getPendingBucketSpaceDbTransition(makeBucketSpace()); - - vespalib::StringTokenizer tok3(tok2[1], ","); - for (uint32_t j = 0; j < tok3.size(); j++) { - if (includeBucketInfo) { - vespalib::StringTokenizer tok4(tok3[j], "/"); - - pendingTransition.addNodeInfo( - document::BucketId(16, atoi(tok4[0].data())), - BucketCopy( - timestamp, - node, - api::BucketInfo( - atoi(tok4[1].data()), - atoi(tok4[2].data()), - atoi(tok4[3].data()), - atoi(tok4[2].data()), - atoi(tok4[3].data())))); - } else { - pendingTransition.addNodeInfo( - document::BucketId(16, atoi(tok3[j].data())), - BucketCopy(timestamp, - node, - api::BucketInfo(3, 3, 3, 3, 3))); - } - } - } -} - -struct BucketDumper : public BucketDatabase::EntryProcessor -{ - std::ostringstream ost; - bool _includeBucketInfo; - - explicit BucketDumper(bool includeBucketInfo) - : _includeBucketInfo(includeBucketInfo) - { - } - - bool process(const BucketDatabase::ConstEntryRef& e) override { - document::BucketId bucketId(e.getBucketId()); - - ost << (uint32_t)bucketId.getRawId() << ":"; - for (uint32_t i = 0; i < e->getNodeCount(); ++i) { - if (i > 0) { - ost << ","; - } - const BucketCopy& copy(e->getNodeRef(i)); - ost << copy.getNode(); - if (_includeBucketInfo) { - ost << "/" << copy.getChecksum() - << "/" << copy.getDocumentCount() - << "/" << copy.getTotalDocumentSize() - << "/" << (copy.trusted() ? "t" : "u"); - } - } - ost << "|"; - return true; - } -}; - -} - -std::string -LegacyBucketDBUpdaterTest::mergeBucketLists( - const lib::ClusterState& oldState, - const std::string& existingData, - const lib::ClusterState& newState, - const std::string& newData, - bool includeBucketInfo) -{ - framework::defaultimplementation::FakeClock clock; - framework::MilliSecTimer timer(clock); - - DistributorMessageSenderStub sender; - OutdatedNodesMap outdatedNodesMap; - - { - auto cmd(std::make_shared<api::SetSystemStateCommand>(oldState)); - - api::Timestamp beforeTime(1); - - ClusterInformation::CSP clusterInfo(createClusterInfo("cluster:d")); - std::unique_ptr<PendingClusterState> state( - PendingClusterState::createForClusterStateChange( - clock, clusterInfo, sender, getBucketSpaceRepo(), - cmd, outdatedNodesMap, beforeTime)); - - parseInputData(existingData, beforeTime, *state, includeBucketInfo); - state->mergeIntoBucketDatabases(); - } - - BucketDumper dumper_tmp(true); - getBucketDatabase().forEach(dumper_tmp); - - { - auto cmd(std::make_shared<api::SetSystemStateCommand>( - lib::ClusterState(newState))); - - api::Timestamp afterTime(2); - - ClusterInformation::CSP clusterInfo(createClusterInfo(oldState.toString())); - std::unique_ptr<PendingClusterState> state( - PendingClusterState::createForClusterStateChange( - clock, clusterInfo, sender, getBucketSpaceRepo(), - cmd, outdatedNodesMap, afterTime)); - - parseInputData(newData, afterTime, *state, includeBucketInfo); - state->mergeIntoBucketDatabases(); - } - - BucketDumper dumper(includeBucketInfo); - auto &bucketDb(defaultDistributorBucketSpace().getBucketDatabase()); - bucketDb.forEach(dumper); - bucketDb.clear(); - return dumper.ost.str(); -} - -std::string -LegacyBucketDBUpdaterTest::mergeBucketLists(const std::string& existingData, - const std::string& newData, - bool includeBucketInfo) -{ - return mergeBucketLists( - lib::ClusterState("distributor:1 storage:3"), - existingData, - lib::ClusterState("distributor:1 storage:3"), - newData, - includeBucketInfo); -} - -// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest -TEST_F(LegacyBucketDBUpdaterTest, pending_cluster_state_merge) { - // Simple initializing case - ask all nodes for info - EXPECT_EQ( - // Result is on the form: [bucket w/o count bits]:[node indexes]|.. - std::string("4:0,1|2:0,1|6:1,2|1:0,2|5:2,0|3:2,1|"), - // Input is on the form: [node]:[bucket w/o count bits]|... - mergeBucketLists( - "", - "0:1,2,4,5|1:2,3,4,6|2:1,3,5,6")); - - // New node came up - EXPECT_EQ( - std::string("4:0,1|2:0,1|6:1,2,3|1:0,2,3|5:2,0,3|3:2,1,3|"), - mergeBucketLists( - "0:1,2,4,5|1:2,3,4,6|2:1,3,5,6", - "3:1,3,5,6")); - - // Node came up with some buckets removed and some added - // Buckets that were removed should not be removed as the node - // didn't lose a disk. - EXPECT_EQ( - std::string("8:0|4:0,1|2:0,1|6:1,0,2|1:0,2|5:2,0|3:2,1|"), - mergeBucketLists( - "0:1,2,4,5|1:2,3,4,6|2:1,3,5,6", - "0:1,2,6,8")); - - // Bucket info format is "bucketid/checksum/count/size" - // Node went from initializing to up and invalid bucket went to empty. - EXPECT_EQ( - std::string("2:0/0/0/0/t|"), - mergeBucketLists( - "0:2/0/0/1", - "0:2/0/0/0", - true)); - - EXPECT_EQ(std::string("5:1/2/3/4/u,0/0/0/0/u|"), - mergeBucketLists("", "0:5/0/0/0|1:5/2/3/4", true)); -} - -// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest -TEST_F(LegacyBucketDBUpdaterTest, pending_cluster_state_merge_replica_changed) { - // Node went from initializing to up and non-invalid bucket changed. - EXPECT_EQ( - std::string("2:0/2/3/4/t|3:0/2/4/6/t|"), - mergeBucketLists( - lib::ClusterState("distributor:1 storage:1 .0.s:i"), - "0:2/1/2/3,3/2/4/6", - lib::ClusterState("distributor:1 storage:1"), - "0:2/2/3/4,3/2/4/6", - true)); -} - -// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest -TEST_F(LegacyBucketDBUpdaterTest, no_db_resurrection_for_bucket_not_owned_in_current_state) { - document::BucketId bucket(16, 3); - lib::ClusterState stateBefore("distributor:1 storage:1"); - { - uint32_t expectedMsgs = _bucketSpaces.size(), dummyBucketsToReturn = 1; - ASSERT_NO_FATAL_FAILURE(setAndEnableClusterState(stateBefore, expectedMsgs, dummyBucketsToReturn)); - } - _sender.clear(); - - getBucketDBUpdater().recheckBucketInfo(0, makeDocumentBucket(bucket)); - - ASSERT_EQ(size_t(1), _sender.commands().size()); - std::shared_ptr<api::RequestBucketInfoCommand> rbi( - std::dynamic_pointer_cast<RequestBucketInfoCommand>( - _sender.command(0))); - - lib::ClusterState stateAfter("distributor:3 storage:3"); - - { - uint32_t expectedMsgs = messageCount(2), dummyBucketsToReturn = 1; - ASSERT_NO_FATAL_FAILURE(setAndEnableClusterState(stateAfter, expectedMsgs, dummyBucketsToReturn)); - } - EXPECT_FALSE(getDistributorBucketSpace().get_bucket_ownership_flags(bucket).owned_in_current_state()); - - ASSERT_NO_FATAL_FAILURE(sendFakeReplyForSingleBucketRequest(*rbi)); - - EXPECT_EQ(std::string("NONEXISTING"), dumpBucket(bucket)); -} - -// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest -TEST_F(LegacyBucketDBUpdaterTest, no_db_resurrection_for_bucket_not_owned_in_pending_state) { - document::BucketId bucket(16, 3); - lib::ClusterState stateBefore("distributor:1 storage:1"); - { - uint32_t expectedMsgs = _bucketSpaces.size(), dummyBucketsToReturn = 1; - ASSERT_NO_FATAL_FAILURE(setAndEnableClusterState(stateBefore, expectedMsgs, dummyBucketsToReturn)); - } - _sender.clear(); - - getBucketDBUpdater().recheckBucketInfo(0, makeDocumentBucket(bucket)); - - ASSERT_EQ(size_t(1), _sender.commands().size()); - std::shared_ptr<api::RequestBucketInfoCommand> rbi( - std::dynamic_pointer_cast<RequestBucketInfoCommand>( - _sender.command(0))); - - lib::ClusterState stateAfter("distributor:3 storage:3"); - // Set, but _don't_ enable cluster state. We want it to be pending. - setSystemState(stateAfter); - EXPECT_TRUE(getDistributorBucketSpace().get_bucket_ownership_flags(bucket).owned_in_current_state()); - EXPECT_FALSE(getDistributorBucketSpace().get_bucket_ownership_flags(bucket).owned_in_pending_state()); - - ASSERT_NO_FATAL_FAILURE(sendFakeReplyForSingleBucketRequest(*rbi)); - - EXPECT_EQ(std::string("NONEXISTING"), dumpBucket(bucket)); -} - -// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest -/* - * If we get a distribution config change, it's important that cluster states that - * arrive after this--but _before_ the pending cluster state has finished--must trigger - * a full bucket info fetch no matter what the cluster state change was! Otherwise, we - * will with a high likelihood end up not getting the complete view of the buckets in - * the cluster. - */ -TEST_F(LegacyBucketDBUpdaterTest, cluster_state_always_sends_full_fetch_when_distribution_change_pending) { - lib::ClusterState stateBefore("distributor:6 storage:6"); - { - uint32_t expectedMsgs = messageCount(6), dummyBucketsToReturn = 1; - ASSERT_NO_FATAL_FAILURE(setAndEnableClusterState(stateBefore, expectedMsgs, dummyBucketsToReturn)); - } - _sender.clear(); - std::string distConfig(getDistConfig6Nodes2Groups()); - setDistribution(distConfig); - - sortSentMessagesByIndex(_sender); - ASSERT_EQ(messageCount(6), _sender.commands().size()); - // Suddenly, a wild cluster state change appears! Even though this state - // does not in itself imply any bucket changes, it will still overwrite the - // pending cluster state and thus its state of pending bucket info requests. - setSystemState(lib::ClusterState("distributor:6 .2.t:12345 storage:6")); - - ASSERT_EQ(messageCount(12), _sender.commands().size()); - - // Send replies for first messageCount(6) (outdated requests). - int numBuckets = 10; - for (uint32_t i = 0; i < messageCount(6); ++i) { - ASSERT_NO_FATAL_FAILURE(fakeBucketReply(lib::ClusterState("distributor:6 storage:6"), - *_sender.command(i), numBuckets)); - } - // No change from these. - ASSERT_NO_FATAL_FAILURE(assertCorrectBuckets(1, "distributor:6 storage:6")); - - // Send for current pending. - for (uint32_t i = 0; i < messageCount(6); ++i) { - ASSERT_NO_FATAL_FAILURE(fakeBucketReply(lib::ClusterState("distributor:6 .2.t:12345 storage:6"), - *_sender.command(i + messageCount(6)), - numBuckets)); - } - ASSERT_NO_FATAL_FAILURE(assertCorrectBuckets(numBuckets, "distributor:6 storage:6")); - _sender.clear(); - - // No more pending global fetch; this should be a no-op state. - setSystemState(lib::ClusterState("distributor:6 .3.t:12345 storage:6")); - EXPECT_EQ(size_t(0), _sender.commands().size()); -} - -// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest -TEST_F(LegacyBucketDBUpdaterTest, changed_distribution_config_triggers_recovery_mode) { - ASSERT_NO_FATAL_FAILURE(setAndEnableClusterState(lib::ClusterState("distributor:6 storage:6"), messageCount(6), 20)); - _sender.clear(); - EXPECT_TRUE(distributor_is_in_recovery_mode()); - complete_recovery_mode(); - EXPECT_FALSE(distributor_is_in_recovery_mode()); - - std::string distConfig(getDistConfig6Nodes4Groups()); - setDistribution(distConfig); - sortSentMessagesByIndex(_sender); - // No replies received yet, still no recovery mode. - EXPECT_FALSE(distributor_is_in_recovery_mode()); - - ASSERT_EQ(messageCount(6), _sender.commands().size()); - uint32_t numBuckets = 10; - for (uint32_t i = 0; i < messageCount(6); ++i) { - ASSERT_NO_FATAL_FAILURE(fakeBucketReply(lib::ClusterState("distributor:6 storage:6"), - *_sender.command(i), numBuckets)); - } - - // Pending cluster state (i.e. distribution) has been enabled, which should - // cause recovery mode to be entered. - EXPECT_TRUE(distributor_is_in_recovery_mode()); - complete_recovery_mode(); - EXPECT_FALSE(distributor_is_in_recovery_mode()); -} - -namespace { - -template <typename Func> -struct FunctorProcessor : BucketDatabase::EntryProcessor { - Func _f; - - template <typename F> - explicit FunctorProcessor(F&& f) : _f(std::forward<F>(f)) {} - - bool process(const BucketDatabase::ConstEntryRef& e) override { - _f(e); - return true; - } -}; - -template <typename Func> -std::unique_ptr<BucketDatabase::EntryProcessor> func_processor(Func&& f) { - return std::make_unique<FunctorProcessor<Func>>(std::forward<Func>(f)); -} - -} - -// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest -TEST_F(LegacyBucketDBUpdaterTest, changed_distribution_config_does_not_elide_bucket_db_pruning) { - setDistribution(getDistConfig3Nodes1Group()); - - constexpr uint32_t n_buckets = 100; - ASSERT_NO_FATAL_FAILURE( - setAndEnableClusterState(lib::ClusterState("distributor:6 storage:6"), messageCount(6), n_buckets)); - _sender.clear(); - - // Config implies a different node set than the current cluster state, so it's crucial that - // DB pruning is _not_ elided. Yes, this is inherently racing with cluster state changes and - // should be changed to be atomic and controlled by the cluster controller instead of config. - // But this is where we currently are. - setDistribution(getDistConfig6Nodes2Groups()); - - getBucketDatabase().forEach(*func_processor([&](const auto& e) { - EXPECT_TRUE(getDistributorBucketSpace().get_bucket_ownership_flags(e.getBucketId()).owned_in_pending_state()); - })); -} - -// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest -TEST_F(LegacyBucketDBUpdaterTest, newly_added_buckets_have_current_time_as_gc_timestamp) { - getClock().setAbsoluteTimeInSeconds(101234); - lib::ClusterState stateBefore("distributor:1 storage:1"); - { - uint32_t expectedMsgs = _bucketSpaces.size(), dummyBucketsToReturn = 1; - ASSERT_NO_FATAL_FAILURE(setAndEnableClusterState(stateBefore, expectedMsgs, dummyBucketsToReturn)); - } - - // setAndEnableClusterState adds n buckets with id (16, i) - document::BucketId bucket(16, 0); - BucketDatabase::Entry e(getBucket(bucket)); - ASSERT_TRUE(e.valid()); - EXPECT_EQ(uint32_t(101234), e->getLastGarbageCollectionTime()); -} - -// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest -TEST_F(LegacyBucketDBUpdaterTest, newer_mutations_not_overwritten_by_earlier_bucket_fetch) { - { - lib::ClusterState stateBefore("distributor:1 storage:1 .0.s:i"); - uint32_t expectedMsgs = _bucketSpaces.size(), dummyBucketsToReturn = 0; - // This step is required to make the distributor ready for accepting - // the below explicit database insertion towards node 0. - ASSERT_NO_FATAL_FAILURE(setAndEnableClusterState(stateBefore, expectedMsgs, - dummyBucketsToReturn)); - } - _sender.clear(); - getClock().setAbsoluteTimeInSeconds(1000); - lib::ClusterState state("distributor:1 storage:1"); - setSystemState(state); - ASSERT_EQ(_bucketSpaces.size(), _sender.commands().size()); - - // Before replying with the bucket info, simulate the arrival of a mutation - // reply that alters the state of the bucket with information that will be - // more recent that what is returned by the bucket info. This information - // must not be lost when the bucket info is later merged into the database. - document::BucketId bucket(16, 1); - constexpr uint64_t insertionTimestamp = 1001ULL * 1000000; - api::BucketInfo wantedInfo(5, 6, 7); - getBucketDBUpdater().operation_context().update_bucket_database( - makeDocumentBucket(bucket), - BucketCopy(insertionTimestamp, 0, wantedInfo), - DatabaseUpdate::CREATE_IF_NONEXISTING); - - getClock().setAbsoluteTimeInSeconds(1002); - constexpr uint32_t bucketsReturned = 10; // Buckets (16, 0) ... (16, 9) - // Return bucket information which on the timeline might originate from - // anywhere between [1000, 1002]. Our assumption is that any mutations - // taking place after t=1000 must have its reply received and processed - // by this distributor and timestamped strictly higher than t=1000 (modulo - // clock skew, of course, but that is outside the scope of this). A mutation - // happening before t=1000 but receiving a reply at t>1000 does not affect - // correctness, as this should contain the same bucket info as that - // contained in the full bucket reply and the DB update is thus idempotent. - for (uint32_t i = 0; i < _bucketSpaces.size(); ++i) { - ASSERT_NO_FATAL_FAILURE(fakeBucketReply(state, *_sender.command(i), bucketsReturned)); - } - - BucketDatabase::Entry e(getBucket(bucket)); - ASSERT_EQ(uint32_t(1), e->getNodeCount()); - EXPECT_EQ(wantedInfo, e->getNodeRef(0).getBucketInfo()); -} - -std::vector<uint16_t> -LegacyBucketDBUpdaterTest::getSendSet() const -{ - std::vector<uint16_t> nodes; - std::transform(_sender.commands().begin(), - _sender.commands().end(), - std::back_inserter(nodes), - [](auto& cmd) - { - auto& req(dynamic_cast<const api::RequestBucketInfoCommand&>(*cmd)); - return req.getAddress()->getIndex(); - }); - return nodes; -} - -std::vector<uint16_t> -LegacyBucketDBUpdaterTest::getSentNodesWithPreemption( - const std::string& oldClusterState, - uint32_t expectedOldStateMessages, - const std::string& preemptedClusterState, - const std::string& newClusterState) -{ - lib::ClusterState stateBefore(oldClusterState); - uint32_t dummyBucketsToReturn = 10; - // FIXME cannot chain assertion checks in non-void function - setAndEnableClusterState(lib::ClusterState(oldClusterState), - expectedOldStateMessages, - dummyBucketsToReturn); - - _sender.clear(); - - setSystemState(lib::ClusterState(preemptedClusterState)); - _sender.clear(); - // Do not allow the pending state to become the active state; trigger a - // new transition without ACKing the info requests first. This will - // overwrite the pending state entirely. - setSystemState(lib::ClusterState(newClusterState)); - return getSendSet(); -} - -using nodeVec = std::vector<uint16_t>; - -// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest -/* - * If we don't carry over the set of nodes that we need to fetch from, - * a naive comparison between the active state and the new state will - * make it appear to the distributor that nothing has changed, as any - * database modifications caused by intermediate states will not be - * accounted for (basically the ABA problem in a distributed setting). - */ -TEST_F(LegacyBucketDBUpdaterTest, preempted_distributor_change_carries_node_set_over_to_next_state_fetch) { - EXPECT_EQ( - expandNodeVec({0, 1, 2, 3, 4, 5}), - getSentNodesWithPreemption("version:1 distributor:6 storage:6", - messageCount(6), - "version:2 distributor:6 .5.s:d storage:6", - "version:3 distributor:6 storage:6")); -} - -// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest -TEST_F(LegacyBucketDBUpdaterTest, preempted_storage_change_carries_node_set_over_to_next_state_fetch) { - EXPECT_EQ( - expandNodeVec({2, 3}), - getSentNodesWithPreemption( - "version:1 distributor:6 storage:6 .2.s:d", - messageCount(5), - "version:2 distributor:6 storage:6 .2.s:d .3.s:d", - "version:3 distributor:6 storage:6")); -} - -// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest -TEST_F(LegacyBucketDBUpdaterTest, preempted_storage_node_down_must_be_re_fetched) { - EXPECT_EQ( - expandNodeVec({2}), - getSentNodesWithPreemption( - "version:1 distributor:6 storage:6", - messageCount(6), - "version:2 distributor:6 storage:6 .2.s:d", - "version:3 distributor:6 storage:6")); -} - -// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest -TEST_F(LegacyBucketDBUpdaterTest, do_not_send_to_preempted_node_now_in_down_state) { - EXPECT_EQ( - nodeVec{}, - getSentNodesWithPreemption( - "version:1 distributor:6 storage:6 .2.s:d", - messageCount(5), - "version:2 distributor:6 storage:6", // Sends to 2. - "version:3 distributor:6 storage:6 .2.s:d")); // 2 down again. -} - -// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest -TEST_F(LegacyBucketDBUpdaterTest, doNotSendToPreemptedNodeNotPartOfNewState) { - // Even though 100 nodes are preempted, not all of these should be part - // of the request afterwards when only 6 are part of the state. - EXPECT_EQ( - expandNodeVec({0, 1, 2, 3, 4, 5}), - getSentNodesWithPreemption( - "version:1 distributor:6 storage:100", - messageCount(100), - "version:2 distributor:5 .4.s:d storage:100", - "version:3 distributor:6 storage:6")); -} - -// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest -TEST_F(LegacyBucketDBUpdaterTest, outdated_node_set_cleared_after_successful_state_completion) { - lib::ClusterState stateBefore( - "version:1 distributor:6 storage:6 .1.t:1234"); - uint32_t expectedMsgs = messageCount(6), dummyBucketsToReturn = 10; - ASSERT_NO_FATAL_FAILURE(setAndEnableClusterState(stateBefore, expectedMsgs, dummyBucketsToReturn)); - _sender.clear(); - // New cluster state that should not by itself trigger any new fetches, - // unless outdated node set is somehow not cleared after an enabled - // (completed) cluster state has been set. - lib::ClusterState stateAfter("version:3 distributor:6 storage:6"); - setSystemState(stateAfter); - EXPECT_EQ(size_t(0), _sender.commands().size()); -} - -// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest (despite being disabled) -// XXX test currently disabled since distribution config currently isn't used -// at all in order to deduce the set of nodes to send to. This might not matter -// in practice since it is assumed that the cluster state matching the new -// distribution config will follow very shortly after the config has been -// applied to the node. The new cluster state will then send out requests to -// the correct node set. -TEST_F(LegacyBucketDBUpdaterTest, DISABLED_cluster_config_downsize_only_sends_to_available_nodes) { - uint32_t expectedMsgs = 6, dummyBucketsToReturn = 20; - ASSERT_NO_FATAL_FAILURE(setAndEnableClusterState(lib::ClusterState("distributor:6 storage:6"), - expectedMsgs, dummyBucketsToReturn)); - _sender.clear(); - - // Intentionally trigger a racing config change which arrives before the - // new cluster state representing it. - std::string distConfig(getDistConfig3Nodes1Group()); - setDistribution(distConfig); - sortSentMessagesByIndex(_sender); - - EXPECT_EQ((nodeVec{0, 1, 2}), getSendSet()); -} - -// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest -/** - * Test scenario where a cluster is downsized by removing a subset of the nodes - * from the distribution configuration. The system must be able to deal with - * a scenario where the set of nodes between two cluster states across a config - * change may differ. - * - * See VESPA-790 for details. - */ -TEST_F(LegacyBucketDBUpdaterTest, node_missing_from_config_is_treated_as_needing_ownership_transfer) { - uint32_t expectedMsgs = messageCount(3), dummyBucketsToReturn = 1; - ASSERT_NO_FATAL_FAILURE(setAndEnableClusterState(lib::ClusterState("distributor:3 storage:3"), - expectedMsgs, dummyBucketsToReturn)); - _sender.clear(); - - // Cluster goes from {0, 1, 2} -> {0, 1}. This leaves us with a config - // that does not contain node 2 while the _active_ cluster state still - // contains this node. - const char* downsizeCfg = - "redundancy 2\n" - "distributor_auto_ownership_transfer_on_whole_group_down true\n" - "group[2]\n" - "group[0].name \"invalid\"\n" - "group[0].index \"invalid\"\n" - "group[0].partitions 1|*\n" - "group[0].nodes[0]\n" - "group[1].name rack0\n" - "group[1].index 0\n" - "group[1].nodes[2]\n" - "group[1].nodes[0].index 0\n" - "group[1].nodes[1].index 1\n"; - - setDistribution(downsizeCfg); - sortSentMessagesByIndex(_sender); - _sender.clear(); - - // Attempt to apply state with {0, 1} set. This will compare the new state - // with the previous state, which still has node 2. - expectedMsgs = messageCount(2); - ASSERT_NO_FATAL_FAILURE(setAndEnableClusterState(lib::ClusterState("distributor:2 storage:2"), - expectedMsgs, dummyBucketsToReturn)); - - EXPECT_EQ(expandNodeVec({0, 1}), getSendSet()); -} - -// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest -TEST_F(LegacyBucketDBUpdaterTest, changed_distributor_set_implies_ownership_transfer) { - auto fixture = createPendingStateFixtureForStateChange( - "distributor:2 storage:2", "distributor:1 storage:2"); - EXPECT_TRUE(fixture->state->hasBucketOwnershipTransfer()); - - fixture = createPendingStateFixtureForStateChange( - "distributor:2 storage:2", "distributor:2 .1.s:d storage:2"); - EXPECT_TRUE(fixture->state->hasBucketOwnershipTransfer()); -} - -// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest -TEST_F(LegacyBucketDBUpdaterTest, unchanged_distributor_set_implies_no_ownership_transfer) { - auto fixture = createPendingStateFixtureForStateChange( - "distributor:2 storage:2", "distributor:2 storage:1"); - EXPECT_FALSE(fixture->state->hasBucketOwnershipTransfer()); - - fixture = createPendingStateFixtureForStateChange( - "distributor:2 storage:2", "distributor:2 storage:2 .1.s:d"); - EXPECT_FALSE(fixture->state->hasBucketOwnershipTransfer()); -} - -// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest -TEST_F(LegacyBucketDBUpdaterTest, changed_distribution_config_implies_ownership_transfer) { - auto fixture = createPendingStateFixtureForDistributionChange( - "distributor:2 storage:2"); - EXPECT_TRUE(fixture->state->hasBucketOwnershipTransfer()); -} - -// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest -TEST_F(LegacyBucketDBUpdaterTest, transition_time_tracked_for_single_state_change) { - ASSERT_NO_FATAL_FAILURE(completeStateTransitionInSeconds("distributor:2 storage:2", 5, messageCount(2))); - - EXPECT_EQ(uint64_t(5000), lastTransitionTimeInMillis()); -} - -// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest -TEST_F(LegacyBucketDBUpdaterTest, transition_time_reset_across_non_preempting_state_changes) { - ASSERT_NO_FATAL_FAILURE(completeStateTransitionInSeconds("distributor:2 storage:2", 5, messageCount(2))); - ASSERT_NO_FATAL_FAILURE(completeStateTransitionInSeconds("distributor:2 storage:3", 3, messageCount(1))); - - EXPECT_EQ(uint64_t(3000), lastTransitionTimeInMillis()); -} - -// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest -TEST_F(LegacyBucketDBUpdaterTest, transition_time_tracked_for_distribution_config_change) { - lib::ClusterState state("distributor:2 storage:2"); - ASSERT_NO_FATAL_FAILURE(setAndEnableClusterState(state, messageCount(2), 1)); - - _sender.clear(); - std::string distConfig(getDistConfig3Nodes1Group()); - setDistribution(distConfig); - getClock().addSecondsToTime(4); - ASSERT_NO_FATAL_FAILURE(completeBucketInfoGathering(state, messageCount(2))); - EXPECT_EQ(uint64_t(4000), lastTransitionTimeInMillis()); -} - -// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest -TEST_F(LegacyBucketDBUpdaterTest, transition_time_tracked_across_preempted_transitions) { - _sender.clear(); - lib::ClusterState state("distributor:2 storage:2"); - setSystemState(state); - getClock().addSecondsToTime(5); - // Pre-empted with new state here, which will push out the old pending - // state and replace it with a new one. We should still count the time - // used processing the old state. - ASSERT_NO_FATAL_FAILURE(completeStateTransitionInSeconds("distributor:2 storage:3", 3, messageCount(3))); - - EXPECT_EQ(uint64_t(8000), lastTransitionTimeInMillis()); -} - -// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest -/* - * Brief reminder on test DSL for checking bucket merge operations: - * - * mergeBucketLists() takes as input strings of the format - * <node>:<raw bucket id>/<checksum>/<num docs>/<doc size>|<node>: - * and returns a string describing the bucket DB post-merge with the format - * <raw bucket id>:<node>/<checksum>/<num docs>/<doc size>,<node>:....|<raw bucket id>:.... - * - * Yes, the order of node<->bucket id is reversed between the two, perhaps to make sure you're awake. - */ - -TEST_F(LegacyBucketDBUpdaterTest, batch_update_of_existing_diverging_replicas_does_not_mark_any_as_trusted) { - // Replacing bucket information for content node 0 should not mark existing - // untrusted replica as trusted as a side effect. - EXPECT_EQ( - std::string("5:1/7/8/9/u,0/1/2/3/u|"), - mergeBucketLists( - lib::ClusterState("distributor:1 storage:3 .0.s:i"), - "0:5/0/0/0|1:5/7/8/9", - lib::ClusterState("distributor:1 storage:3 .0.s:u"), - "0:5/1/2/3|1:5/7/8/9", true)); -} - -// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest -TEST_F(LegacyBucketDBUpdaterTest, batch_add_of_new_diverging_replicas_does_not_mark_any_as_trusted) { - EXPECT_EQ(std::string("5:1/7/8/9/u,0/1/2/3/u|"), - mergeBucketLists("", "0:5/1/2/3|1:5/7/8/9", true)); -} - -// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest -TEST_F(LegacyBucketDBUpdaterTest, batch_add_with_single_resulting_replica_implicitly_marks_as_trusted) { - EXPECT_EQ(std::string("5:0/1/2/3/t|"), - mergeBucketLists("", "0:5/1/2/3", true)); -} - -// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest -TEST_F(LegacyBucketDBUpdaterTest, identity_update_of_single_replica_does_not_clear_trusted) { - EXPECT_EQ(std::string("5:0/1/2/3/t|"), - mergeBucketLists("0:5/1/2/3", "0:5/1/2/3", true)); -} - -// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest -TEST_F(LegacyBucketDBUpdaterTest, identity_update_of_diverging_untrusted_replicas_does_not_mark_any_as_trusted) { - EXPECT_EQ(std::string("5:1/7/8/9/u,0/1/2/3/u|"), - mergeBucketLists("0:5/1/2/3|1:5/7/8/9", "0:5/1/2/3|1:5/7/8/9", true)); -} - -// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest -TEST_F(LegacyBucketDBUpdaterTest, adding_diverging_replica_to_existing_trusted_does_not_remove_trusted) { - EXPECT_EQ(std::string("5:1/2/3/4/u,0/1/2/3/t|"), - mergeBucketLists("0:5/1/2/3", "0:5/1/2/3|1:5/2/3/4", true)); -} - -// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest -TEST_F(LegacyBucketDBUpdaterTest, batch_update_from_distributor_change_does_not_mark_diverging_replicas_as_trusted) { - // This differs from batch_update_of_existing_diverging_replicas_does_not_mark_any_as_trusted - // in that _all_ content nodes are considered outdated when distributor changes take place, - // and therefore a slightly different code path is taken. In particular, bucket info for - // outdated nodes gets removed before possibly being re-added (if present in the bucket info - // response). - EXPECT_EQ( - std::string("5:1/7/8/9/u,0/1/2/3/u|"), - mergeBucketLists( - lib::ClusterState("distributor:2 storage:3"), - "0:5/1/2/3|1:5/7/8/9", - lib::ClusterState("distributor:1 storage:3"), - "0:5/1/2/3|1:5/7/8/9", true)); -} - -// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest -// TODO remove on Vespa 8 - this is a workaround for https://github.com/vespa-engine/vespa/issues/8475 -TEST_F(LegacyBucketDBUpdaterTest, global_distribution_hash_falls_back_to_legacy_format_upon_request_rejection) { - std::string distConfig(getDistConfig6Nodes2Groups()); - setDistribution(distConfig); - - const vespalib::string current_hash = "(0d*|*(0;0;1;2)(1;3;4;5))"; - const vespalib::string legacy_hash = "(0d3|3|*(0;0;1;2)(1;3;4;5))"; - - setSystemState(lib::ClusterState("distributor:6 storage:6")); - ASSERT_EQ(messageCount(6), _sender.commands().size()); - - api::RequestBucketInfoCommand* global_req = nullptr; - for (auto& cmd : _sender.commands()) { - auto& req_cmd = dynamic_cast<api::RequestBucketInfoCommand&>(*cmd); - if (req_cmd.getBucketSpace() == document::FixedBucketSpaces::global_space()) { - global_req = &req_cmd; - break; - } - } - ASSERT_TRUE(global_req != nullptr); - ASSERT_EQ(current_hash, global_req->getDistributionHash()); - - auto reply = std::make_shared<api::RequestBucketInfoReply>(*global_req); - reply->setResult(api::ReturnCode::REJECTED); - getBucketDBUpdater().onRequestBucketInfoReply(reply); - - getClock().addSecondsToTime(10); - getBucketDBUpdater().resendDelayedMessages(); - - // Should now be a resent request with legacy distribution hash - ASSERT_EQ(messageCount(6) + 1, _sender.commands().size()); - auto& legacy_req = dynamic_cast<api::RequestBucketInfoCommand&>(*_sender.commands().back()); - ASSERT_EQ(legacy_hash, legacy_req.getDistributionHash()); - - // Now if we reject it _again_ we should cycle back to the current hash - // in case it wasn't a hash-based rejection after all. And the circle of life continues. - reply = std::make_shared<api::RequestBucketInfoReply>(legacy_req); - reply->setResult(api::ReturnCode::REJECTED); - getBucketDBUpdater().onRequestBucketInfoReply(reply); - - getClock().addSecondsToTime(10); - getBucketDBUpdater().resendDelayedMessages(); - - ASSERT_EQ(messageCount(6) + 2, _sender.commands().size()); - auto& new_current_req = dynamic_cast<api::RequestBucketInfoCommand&>(*_sender.commands().back()); - ASSERT_EQ(current_hash, new_current_req.getDistributionHash()); -} - -namespace { - -template <typename Func> -void for_each_bucket(const BucketDatabase& db, const document::BucketSpace& space, Func&& f) { - BucketId last(0); - auto e = db.getNext(last); - while (e.valid()) { - f(space, e); - e = db.getNext(e.getBucketId()); - } -} - -template <typename Func> -void for_each_bucket(const DistributorBucketSpaceRepo& repo, Func&& f) { - for (const auto& space : repo) { - for_each_bucket(space.second->getBucketDatabase(), space.first, f); - } -} - -} - -// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest -TEST_F(LegacyBucketDBUpdaterTest, non_owned_buckets_moved_to_read_only_db_on_ownership_change) { - getBucketDBUpdater().set_stale_reads_enabled(true); - - lib::ClusterState initial_state("distributor:1 storage:4"); // All buckets owned by us by definition - set_cluster_state_bundle(lib::ClusterStateBundle(initial_state, {}, false)); // Skip activation step for simplicity - - ASSERT_EQ(messageCount(4), _sender.commands().size()); - constexpr uint32_t n_buckets = 10; - ASSERT_NO_FATAL_FAILURE(completeBucketInfoGathering(initial_state, messageCount(4), n_buckets)); - _sender.clear(); - - EXPECT_EQ(size_t(n_buckets), mutable_default_db().size()); - EXPECT_EQ(size_t(n_buckets), mutable_global_db().size()); - EXPECT_EQ(size_t(0), read_only_default_db().size()); - EXPECT_EQ(size_t(0), read_only_global_db().size()); - - lib::ClusterState pending_state("distributor:2 storage:4"); - - std::unordered_set<Bucket, Bucket::hash> buckets_not_owned_in_pending_state; - for_each_bucket(mutable_repo(), [&](const auto& space, const auto& entry) { - if (!getDistributorBucketSpace().owns_bucket_in_state(pending_state, entry.getBucketId())) { - buckets_not_owned_in_pending_state.insert(Bucket(space, entry.getBucketId())); - } - }); - EXPECT_FALSE(buckets_not_owned_in_pending_state.empty()); - - set_cluster_state_bundle(lib::ClusterStateBundle(pending_state, {}, true)); // Now requires activation - - const auto buckets_not_owned_per_space = (buckets_not_owned_in_pending_state.size() / 2); // 2 spaces - const auto expected_mutable_buckets = n_buckets - buckets_not_owned_per_space; - EXPECT_EQ(expected_mutable_buckets, mutable_default_db().size()); - EXPECT_EQ(expected_mutable_buckets, mutable_global_db().size()); - EXPECT_EQ(buckets_not_owned_per_space, read_only_default_db().size()); - EXPECT_EQ(buckets_not_owned_per_space, read_only_global_db().size()); - - for_each_bucket(read_only_repo(), [&](const auto& space, const auto& entry) { - EXPECT_TRUE(buckets_not_owned_in_pending_state.find(Bucket(space, entry.getBucketId())) - != buckets_not_owned_in_pending_state.end()); - }); -} - -// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest -TEST_F(LegacyBucketDBUpdaterTest, buckets_no_longer_available_are_not_moved_to_read_only_database) { - constexpr uint32_t n_buckets = 10; - // No ownership change, just node down. Test redundancy is 2, so removing 2 nodes will - // cause some buckets to be entirely unavailable. - trigger_completed_but_not_yet_activated_transition("version:1 distributor:1 storage:4", n_buckets, 4, - "version:2 distributor:1 storage:4 .0.s:d .1.s:m", n_buckets, 0); - - EXPECT_EQ(size_t(0), read_only_default_db().size()); - EXPECT_EQ(size_t(0), read_only_global_db().size()); -} - -// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest -TEST_F(LegacyBucketDBUpdaterTest, non_owned_buckets_purged_when_read_only_support_is_config_disabled) { - getBucketDBUpdater().set_stale_reads_enabled(false); - - lib::ClusterState initial_state("distributor:1 storage:4"); // All buckets owned by us by definition - set_cluster_state_bundle(lib::ClusterStateBundle(initial_state, {}, false)); // Skip activation step for simplicity - - ASSERT_EQ(messageCount(4), _sender.commands().size()); - constexpr uint32_t n_buckets = 10; - ASSERT_NO_FATAL_FAILURE(completeBucketInfoGathering(initial_state, messageCount(4), n_buckets)); - _sender.clear(); - - // Nothing in read-only DB after first bulk load of buckets. - EXPECT_EQ(size_t(0), read_only_default_db().size()); - EXPECT_EQ(size_t(0), read_only_global_db().size()); - - lib::ClusterState pending_state("distributor:2 storage:4"); - setSystemState(pending_state); - // No buckets should be moved into read only db after ownership changes. - EXPECT_EQ(size_t(0), read_only_default_db().size()); - EXPECT_EQ(size_t(0), read_only_global_db().size()); -} - -void LegacyBucketDBUpdaterTest::trigger_completed_but_not_yet_activated_transition( - vespalib::stringref initial_state_str, - uint32_t initial_buckets, - uint32_t initial_expected_msgs, - vespalib::stringref pending_state_str, - uint32_t pending_buckets, - uint32_t pending_expected_msgs) -{ - lib::ClusterState initial_state(initial_state_str); - setSystemState(initial_state); - ASSERT_EQ(messageCount(initial_expected_msgs), _sender.commands().size()); - ASSERT_NO_FATAL_FAILURE(completeBucketInfoGathering( - initial_state, messageCount(initial_expected_msgs), initial_buckets)); - _sender.clear(); - - lib::ClusterState pending_state(pending_state_str); // Ownership change - set_cluster_state_bundle(lib::ClusterStateBundle(pending_state, {}, true)); - ASSERT_EQ(messageCount(pending_expected_msgs), _sender.commands().size()); - ASSERT_NO_FATAL_FAILURE(completeBucketInfoGathering( - pending_state, messageCount(pending_expected_msgs), pending_buckets)); - _sender.clear(); -} - -// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest -TEST_F(LegacyBucketDBUpdaterTest, deferred_activated_state_does_not_enable_state_until_activation_received) { - getBucketDBUpdater().set_stale_reads_enabled(true); - constexpr uint32_t n_buckets = 10; - ASSERT_NO_FATAL_FAILURE( - trigger_completed_but_not_yet_activated_transition("version:1 distributor:2 storage:4", 0, 4, - "version:2 distributor:1 storage:4", n_buckets, 4)); - - // Version should not be switched over yet - EXPECT_EQ(uint32_t(1), current_distributor_cluster_state_bundle().getVersion()); - - EXPECT_EQ(uint64_t(0), mutable_default_db().size()); - EXPECT_EQ(uint64_t(0), mutable_global_db().size()); - - EXPECT_FALSE(activate_cluster_state_version(2)); - - EXPECT_EQ(uint32_t(2), current_distributor_cluster_state_bundle().getVersion()); - EXPECT_EQ(uint64_t(n_buckets), mutable_default_db().size()); - EXPECT_EQ(uint64_t(n_buckets), mutable_global_db().size()); -} - -// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest -TEST_F(LegacyBucketDBUpdaterTest, read_only_db_cleared_once_pending_state_is_activated) { - getBucketDBUpdater().set_stale_reads_enabled(true); - constexpr uint32_t n_buckets = 10; - ASSERT_NO_FATAL_FAILURE( - trigger_completed_but_not_yet_activated_transition("version:1 distributor:1 storage:4", n_buckets, 4, - "version:2 distributor:2 storage:4", n_buckets, 0)); - EXPECT_FALSE(activate_cluster_state_version(2)); - - EXPECT_EQ(uint64_t(0), read_only_default_db().size()); - EXPECT_EQ(uint64_t(0), read_only_global_db().size()); -} - -// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest -TEST_F(LegacyBucketDBUpdaterTest, read_only_db_is_populated_even_when_self_is_marked_down) { - getBucketDBUpdater().set_stale_reads_enabled(true); - constexpr uint32_t n_buckets = 10; - ASSERT_NO_FATAL_FAILURE( - trigger_completed_but_not_yet_activated_transition("version:1 distributor:1 storage:4", n_buckets, 4, - "version:2 distributor:1 .0.s:d storage:4", n_buckets, 0)); - - // State not yet activated, so read-only DBs have got all the buckets we used to have. - EXPECT_EQ(uint64_t(0), mutable_default_db().size()); - EXPECT_EQ(uint64_t(0), mutable_global_db().size()); - EXPECT_EQ(uint64_t(n_buckets), read_only_default_db().size()); - EXPECT_EQ(uint64_t(n_buckets), read_only_global_db().size()); -} - -// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest -TEST_F(LegacyBucketDBUpdaterTest, activate_cluster_state_request_with_mismatching_version_returns_actual_version) { - getBucketDBUpdater().set_stale_reads_enabled(true); - constexpr uint32_t n_buckets = 10; - ASSERT_NO_FATAL_FAILURE( - trigger_completed_but_not_yet_activated_transition("version:4 distributor:1 storage:4", n_buckets, 4, - "version:5 distributor:2 storage:4", n_buckets, 0)); - - EXPECT_TRUE(activate_cluster_state_version(4)); // Too old version - ASSERT_NO_FATAL_FAILURE(assert_has_activate_cluster_state_reply_with_actual_version(5)); - - EXPECT_TRUE(activate_cluster_state_version(6)); // More recent version than what has been observed - ASSERT_NO_FATAL_FAILURE(assert_has_activate_cluster_state_reply_with_actual_version(5)); -} - -// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest -TEST_F(LegacyBucketDBUpdaterTest, activate_cluster_state_request_without_pending_transition_passes_message_through) { - getBucketDBUpdater().set_stale_reads_enabled(true); - constexpr uint32_t n_buckets = 10; - ASSERT_NO_FATAL_FAILURE( - trigger_completed_but_not_yet_activated_transition("version:1 distributor:2 storage:4", 0, 4, - "version:2 distributor:1 storage:4", n_buckets, 4)); - // Activate version 2; no pending cluster state after this. - EXPECT_FALSE(activate_cluster_state_version(2)); - - // No pending cluster state for version 3, just passed through to be implicitly bounced by state manager. - // Note: state manager is not modelled in this test, so we just check that the message handler returns - // false (meaning "didn't take message ownership") and there's no auto-generated reply. - EXPECT_FALSE(activate_cluster_state_version(3)); - EXPECT_EQ(size_t(0), _sender.replies().size()); -} - -// TODO STRIPE disabled benchmark tests are NOT migrated to new test suite -TEST_F(LegacyBucketDBUpdaterTest, DISABLED_benchmark_bulk_loading_into_empty_db) { - // Need to trigger an initial edge to complete first bucket scan - ASSERT_NO_FATAL_FAILURE(setAndEnableClusterState(lib::ClusterState("distributor:2 storage:1"), - messageCount(1), 0)); - _sender.clear(); - - lib::ClusterState state("distributor:1 storage:1"); - setSystemState(state); - - constexpr uint32_t superbuckets = 1u << 16u; - constexpr uint32_t sub_buckets = 14; - constexpr uint32_t n_buckets = superbuckets * sub_buckets; - - ASSERT_EQ(_bucketSpaces.size(), _sender.commands().size()); - for (uint32_t bsi = 0; bsi < _bucketSpaces.size(); ++bsi) { - ASSERT_EQ(_sender.command(bsi)->getType(), MessageType::REQUESTBUCKETINFO); - const auto& req = dynamic_cast<const RequestBucketInfoCommand&>(*_sender.command(bsi)); - - auto sreply = std::make_shared<RequestBucketInfoReply>(req); - sreply->setAddress(storageAddress(0)); - auto& vec = sreply->getBucketInfo(); - if (req.getBucketSpace() == FixedBucketSpaces::default_space()) { - for (uint32_t sb = 0; sb < superbuckets; ++sb) { - for (uint64_t i = 0; i < sub_buckets; ++i) { - document::BucketId bucket(48, (i << 32ULL) | sb); - vec.push_back(api::RequestBucketInfoReply::Entry(bucket, api::BucketInfo(10, 1, 1))); - } - } - } - - vespalib::BenchmarkTimer timer(1.0); - // Global space has no buckets but will serve as a trigger for merging - // buckets into the DB. This lets us measure the overhead of just this part. - if (req.getBucketSpace() == FixedBucketSpaces::global_space()) { - timer.before(); - } - getBucketDBUpdater().onRequestBucketInfoReply(sreply); - if (req.getBucketSpace() == FixedBucketSpaces::global_space()) { - timer.after(); - fprintf(stderr, "Took %g seconds to merge %u buckets into DB\n", timer.min_time(), n_buckets); - } - } - - EXPECT_EQ(size_t(n_buckets), mutable_default_db().size()); - EXPECT_EQ(size_t(0), mutable_global_db().size()); -} - -uint32_t LegacyBucketDBUpdaterTest::populate_bucket_db_via_request_bucket_info_for_benchmarking() { - // Need to trigger an initial edge to complete first bucket scan - setAndEnableClusterState(lib::ClusterState("distributor:2 storage:1"), messageCount(1), 0); - _sender.clear(); - - lib::ClusterState state("distributor:1 storage:1"); - setSystemState(state); - - constexpr uint32_t superbuckets = 1u << 16u; - constexpr uint32_t sub_buckets = 14; - constexpr uint32_t n_buckets = superbuckets * sub_buckets; - - assert(_bucketSpaces.size() == _sender.commands().size()); - for (uint32_t bsi = 0; bsi < _bucketSpaces.size(); ++bsi) { - assert(_sender.command(bsi)->getType() == MessageType::REQUESTBUCKETINFO); - const auto& req = dynamic_cast<const RequestBucketInfoCommand&>(*_sender.command(bsi)); - - auto sreply = std::make_shared<RequestBucketInfoReply>(req); - sreply->setAddress(storageAddress(0)); - auto& vec = sreply->getBucketInfo(); - if (req.getBucketSpace() == FixedBucketSpaces::default_space()) { - for (uint32_t sb = 0; sb < superbuckets; ++sb) { - for (uint64_t i = 0; i < sub_buckets; ++i) { - document::BucketId bucket(48, (i << 32ULL) | sb); - vec.push_back(api::RequestBucketInfoReply::Entry(bucket, api::BucketInfo(10, 1, 1))); - } - } - } - getBucketDBUpdater().onRequestBucketInfoReply(sreply); - } - - assert(mutable_default_db().size() == n_buckets); - assert(mutable_global_db().size() == 0); - return n_buckets; -} - -TEST_F(LegacyBucketDBUpdaterTest, DISABLED_benchmark_removing_buckets_for_unavailable_storage_nodes) { - const uint32_t n_buckets = populate_bucket_db_via_request_bucket_info_for_benchmarking(); - - lib::ClusterState no_op_state("distributor:1 storage:1 .0.s:m"); // Removing all buckets via ownership - vespalib::BenchmarkTimer timer(1.0); - timer.before(); - setSystemState(no_op_state); - timer.after(); - fprintf(stderr, "Took %g seconds to scan and remove %u buckets\n", timer.min_time(), n_buckets); -} - -TEST_F(LegacyBucketDBUpdaterTest, DISABLED_benchmark_no_buckets_removed_during_node_remover_db_pass) { - const uint32_t n_buckets = populate_bucket_db_via_request_bucket_info_for_benchmarking(); - - // TODO this benchmark is void if we further restrict the pruning elision logic to allow - // elision when storage nodes come online. - lib::ClusterState no_op_state("distributor:1 storage:2"); // Not removing any buckets - vespalib::BenchmarkTimer timer(1.0); - timer.before(); - setSystemState(no_op_state); - timer.after(); - fprintf(stderr, "Took %g seconds to scan %u buckets with no-op action\n", timer.min_time(), n_buckets); -} - -TEST_F(LegacyBucketDBUpdaterTest, DISABLED_benchmark_all_buckets_removed_during_node_remover_db_pass) { - const uint32_t n_buckets = populate_bucket_db_via_request_bucket_info_for_benchmarking(); - - lib::ClusterState no_op_state("distributor:1 storage:1 .0.s:m"); // Removing all buckets via all replicas gone - vespalib::BenchmarkTimer timer(1.0); - timer.before(); - setSystemState(no_op_state); - timer.after(); - fprintf(stderr, "Took %g seconds to scan and remove %u buckets\n", timer.min_time(), n_buckets); -} - -// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest -TEST_F(LegacyBucketDBUpdaterTest, pending_cluster_state_getter_is_non_null_only_when_state_is_pending) { - auto initial_baseline = std::make_shared<lib::ClusterState>("distributor:1 storage:2 .0.s:d"); - auto initial_default = std::make_shared<lib::ClusterState>("distributor:1 storage:2 .0.s:m"); - - lib::ClusterStateBundle initial_bundle(*initial_baseline, {{FixedBucketSpaces::default_space(), initial_default}, - {FixedBucketSpaces::global_space(), initial_baseline}}); - set_cluster_state_bundle(initial_bundle); - - auto* state = getBucketDBUpdater().pendingClusterStateOrNull(FixedBucketSpaces::default_space()); - ASSERT_TRUE(state != nullptr); - EXPECT_EQ(*initial_default, *state); - - state = getBucketDBUpdater().pendingClusterStateOrNull(FixedBucketSpaces::global_space()); - ASSERT_TRUE(state != nullptr); - EXPECT_EQ(*initial_baseline, *state); - - ASSERT_NO_FATAL_FAILURE(completeBucketInfoGathering(*initial_baseline, messageCount(1), 0)); - - state = getBucketDBUpdater().pendingClusterStateOrNull(FixedBucketSpaces::default_space()); - EXPECT_TRUE(state == nullptr); - - state = getBucketDBUpdater().pendingClusterStateOrNull(FixedBucketSpaces::global_space()); - EXPECT_TRUE(state == nullptr); -} - -struct LegacyBucketDBUpdaterSnapshotTest : LegacyBucketDBUpdaterTest { - lib::ClusterState empty_state; - std::shared_ptr<lib::ClusterState> initial_baseline; - std::shared_ptr<lib::ClusterState> initial_default; - lib::ClusterStateBundle initial_bundle; - Bucket default_bucket; - Bucket global_bucket; - - LegacyBucketDBUpdaterSnapshotTest() - : LegacyBucketDBUpdaterTest(), - empty_state(), - initial_baseline(std::make_shared<lib::ClusterState>("distributor:1 storage:2 .0.s:d")), - initial_default(std::make_shared<lib::ClusterState>("distributor:1 storage:2 .0.s:m")), - initial_bundle(*initial_baseline, {{FixedBucketSpaces::default_space(), initial_default}, - {FixedBucketSpaces::global_space(), initial_baseline}}), - default_bucket(FixedBucketSpaces::default_space(), BucketId(16, 1234)), - global_bucket(FixedBucketSpaces::global_space(), BucketId(16, 1234)) - { - } - ~LegacyBucketDBUpdaterSnapshotTest() override; - - void SetUp() override { - LegacyBucketDBUpdaterTest::SetUp(); - getBucketDBUpdater().set_stale_reads_enabled(true); - }; - - // Assumes that the distributor owns all buckets, so it may choose any arbitrary bucket in the bucket space - uint32_t buckets_in_snapshot_matching_current_db(DistributorBucketSpaceRepo& repo, BucketSpace bucket_space) { - auto rs = getBucketDBUpdater().read_snapshot_for_bucket(Bucket(bucket_space, BucketId(16, 1234))); - if (!rs.is_routable()) { - return 0; - } - auto guard = rs.steal_read_guard(); - uint32_t found_buckets = 0; - for_each_bucket(repo, [&](const auto& space, const auto& entry) { - if (space == bucket_space) { - auto entries = guard->find_parents_and_self(entry.getBucketId()); - if (entries.size() == 1) { - ++found_buckets; - } - } - }); - return found_buckets; - } -}; - -LegacyBucketDBUpdaterSnapshotTest::~LegacyBucketDBUpdaterSnapshotTest() = default; - -// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest -TEST_F(LegacyBucketDBUpdaterSnapshotTest, default_space_snapshot_prior_to_activated_state_is_non_routable) { - auto rs = getBucketDBUpdater().read_snapshot_for_bucket(default_bucket); - EXPECT_FALSE(rs.is_routable()); -} - -// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest -TEST_F(LegacyBucketDBUpdaterSnapshotTest, global_space_snapshot_prior_to_activated_state_is_non_routable) { - auto rs = getBucketDBUpdater().read_snapshot_for_bucket(global_bucket); - EXPECT_FALSE(rs.is_routable()); -} - -// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest -TEST_F(LegacyBucketDBUpdaterSnapshotTest, read_snapshot_returns_appropriate_cluster_states) { - set_cluster_state_bundle(initial_bundle); - // State currently pending, empty initial state is active - - auto def_rs = getBucketDBUpdater().read_snapshot_for_bucket(default_bucket); - EXPECT_EQ(def_rs.context().active_cluster_state()->toString(), empty_state.toString()); - EXPECT_EQ(def_rs.context().default_active_cluster_state()->toString(), empty_state.toString()); - ASSERT_TRUE(def_rs.context().has_pending_state_transition()); - EXPECT_EQ(def_rs.context().pending_cluster_state()->toString(), initial_default->toString()); - - auto global_rs = getBucketDBUpdater().read_snapshot_for_bucket(global_bucket); - EXPECT_EQ(global_rs.context().active_cluster_state()->toString(), empty_state.toString()); - EXPECT_EQ(global_rs.context().default_active_cluster_state()->toString(), empty_state.toString()); - ASSERT_TRUE(global_rs.context().has_pending_state_transition()); - EXPECT_EQ(global_rs.context().pending_cluster_state()->toString(), initial_baseline->toString()); - - ASSERT_NO_FATAL_FAILURE(completeBucketInfoGathering(*initial_baseline, messageCount(1), 0)); - // State now activated, no pending - - def_rs = getBucketDBUpdater().read_snapshot_for_bucket(default_bucket); - EXPECT_EQ(def_rs.context().active_cluster_state()->toString(), initial_default->toString()); - EXPECT_EQ(def_rs.context().default_active_cluster_state()->toString(), initial_default->toString()); - EXPECT_FALSE(def_rs.context().has_pending_state_transition()); - - global_rs = getBucketDBUpdater().read_snapshot_for_bucket(global_bucket); - EXPECT_EQ(global_rs.context().active_cluster_state()->toString(), initial_baseline->toString()); - EXPECT_EQ(global_rs.context().default_active_cluster_state()->toString(), initial_default->toString()); - EXPECT_FALSE(global_rs.context().has_pending_state_transition()); -} - -// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest -TEST_F(LegacyBucketDBUpdaterSnapshotTest, snapshot_with_no_pending_state_transition_returns_mutable_db_guard) { - constexpr uint32_t n_buckets = 10; - ASSERT_NO_FATAL_FAILURE( - trigger_completed_but_not_yet_activated_transition("version:1 distributor:2 storage:4", 0, 4, - "version:2 distributor:1 storage:4", n_buckets, 4)); - EXPECT_FALSE(activate_cluster_state_version(2)); - EXPECT_EQ(buckets_in_snapshot_matching_current_db(mutable_repo(), FixedBucketSpaces::default_space()), - n_buckets); - EXPECT_EQ(buckets_in_snapshot_matching_current_db(mutable_repo(), FixedBucketSpaces::global_space()), - n_buckets); -} - -// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest -TEST_F(LegacyBucketDBUpdaterSnapshotTest, snapshot_returns_unroutable_for_non_owned_bucket_in_current_state) { - ASSERT_NO_FATAL_FAILURE( - trigger_completed_but_not_yet_activated_transition("version:1 distributor:2 storage:4", 0, 4, - "version:2 distributor:2 .0.s:d storage:4", 0, 0)); - EXPECT_FALSE(activate_cluster_state_version(2)); - // We're down in state 2 and therefore do not own any buckets - auto def_rs = getBucketDBUpdater().read_snapshot_for_bucket(default_bucket); - EXPECT_FALSE(def_rs.is_routable()); -} - -// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest -TEST_F(LegacyBucketDBUpdaterSnapshotTest, snapshot_with_pending_state_returns_read_only_guard_for_bucket_only_owned_in_current_state) { - constexpr uint32_t n_buckets = 10; - ASSERT_NO_FATAL_FAILURE( - trigger_completed_but_not_yet_activated_transition("version:1 distributor:1 storage:4", n_buckets, 4, - "version:2 distributor:2 .0.s:d storage:4", 0, 0)); - EXPECT_EQ(buckets_in_snapshot_matching_current_db(read_only_repo(), FixedBucketSpaces::default_space()), - n_buckets); - EXPECT_EQ(buckets_in_snapshot_matching_current_db(read_only_repo(), FixedBucketSpaces::global_space()), - n_buckets); -} - -// TODO STRIPE migrated to TopLevelBucketDBUpdaterTest -TEST_F(LegacyBucketDBUpdaterSnapshotTest, snapshot_is_unroutable_if_stale_reads_disabled_and_bucket_not_owned_in_pending_state) { - getBucketDBUpdater().set_stale_reads_enabled(false); - constexpr uint32_t n_buckets = 10; - ASSERT_NO_FATAL_FAILURE( - trigger_completed_but_not_yet_activated_transition("version:1 distributor:1 storage:4", n_buckets, 4, - "version:2 distributor:2 .0.s:d storage:4", 0, 0)); - auto def_rs = getBucketDBUpdater().read_snapshot_for_bucket(default_bucket); - EXPECT_FALSE(def_rs.is_routable()); -} - -} diff --git a/storage/src/tests/distributor/legacy_distributor_test.cpp b/storage/src/tests/distributor/legacy_distributor_test.cpp deleted file mode 100644 index 90d64ddb130..00000000000 --- a/storage/src/tests/distributor/legacy_distributor_test.cpp +++ /dev/null @@ -1,1326 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/storage/distributor/idealstatemetricsset.h> -#include <vespa/storageapi/message/persistence.h> -#include <vespa/storageapi/message/bucketsplitting.h> -#include <vespa/storageapi/message/visitor.h> -#include <vespa/storageapi/message/removelocation.h> -#include <vespa/storageframework/defaultimplementation/thread/threadpoolimpl.h> -#include <tests/distributor/distributortestutil.h> -#include <vespa/document/bucket/fixed_bucket_spaces.h> -#include <vespa/document/fieldset/fieldsets.h> -#include <vespa/document/test/make_document_bucket.h> -#include <vespa/document/test/make_bucket_space.h> -#include <vespa/storage/config/config-stor-distributormanager.h> -#include <vespa/storage/distributor/top_level_distributor.h> -#include <vespa/storage/distributor/distributor_stripe.h> -#include <vespa/storage/distributor/distributor_status.h> -#include <vespa/storage/distributor/distributor_bucket_space.h> -#include <vespa/storage/distributor/distributormetricsset.h> -#include <vespa/vespalib/text/stringtokenizer.h> -#include <vespa/metrics/updatehook.h> -#include <thread> -#include <vespa/vespalib/gtest/gtest.h> -#include <gmock/gmock.h> - -using document::test::makeDocumentBucket; -using document::test::makeBucketSpace; -using document::FixedBucketSpaces; -using document::BucketSpace; -using document::Bucket; -using document::BucketId; -using namespace ::testing; - -namespace storage::distributor { - -// TODO STRIPE: Remove this test when legacy mode is gone. -struct LegacyDistributorTest : Test, DistributorTestUtil { - LegacyDistributorTest(); - ~LegacyDistributorTest() override; - - // TODO handle edge case for window between getnodestate reply already - // sent and new request not yet received - - void assertBucketSpaceStats(size_t expBucketPending, size_t expBucketTotal, uint16_t node, const vespalib::string &bucketSpace, - const BucketSpacesStatsProvider::PerNodeBucketSpacesStats &stats); - std::vector<document::BucketSpace> _bucketSpaces; - - void SetUp() override { - createLinks(); - _bucketSpaces = getBucketSpaces(); - }; - - void TearDown() override { - close(); - } - - // Simple type aliases to make interfacing with certain utility functions - // easier. Note that this is only for readability and does not provide any - // added type safety. - using NodeCount = int; - using Redundancy = int; - - using ConfigBuilder = vespa::config::content::core::StorDistributormanagerConfigBuilder; - - void configureDistributor(const ConfigBuilder& config) { - getConfig().configure(config); - _distributor->enable_next_config_if_changed(); - } - - auto currentReplicaCountingMode() const noexcept { - return _distributor->bucket_db_metric_updater().getMinimumReplicaCountingMode(); - } - - std::string testOp(std::shared_ptr<api::StorageMessage> msg) - { - _distributor->handleMessage(msg); - - std::string tmp = _sender.getCommands(); - _sender.clear(); - return tmp; - } - - void tickDistributorNTimes(uint32_t n) { - for (uint32_t i = 0; i < n; ++i) { - tick(); - } - } - - typedef bool ResetTrusted; - - std::string updateBucketDB(const std::string& firstState, - const std::string& secondState, - bool resetTrusted = false) - { - std::vector<std::string> states(toVector<std::string>(firstState, secondState)); - - for (uint32_t i = 0; i < states.size(); ++i) { - std::vector<uint16_t> removedNodes; - std::vector<BucketCopy> changedNodes; - - vespalib::StringTokenizer tokenizer(states[i], ","); - for (uint32_t j = 0; j < tokenizer.size(); ++j) { - vespalib::StringTokenizer tokenizer2(tokenizer[j], ":"); - - bool trusted = false; - if (tokenizer2.size() > 2) { - trusted = true; - } - - uint16_t node = atoi(tokenizer2[0].data()); - if (tokenizer2[1] == "r") { - removedNodes.push_back(node); - } else { - uint32_t checksum = atoi(tokenizer2[1].data()); - changedNodes.push_back( - BucketCopy( - i + 1, - node, - api::BucketInfo( - checksum, - checksum / 2, - checksum / 4)).setTrusted(trusted)); - } - } - - operation_context().remove_nodes_from_bucket_database(makeDocumentBucket(document::BucketId(16, 1)), removedNodes); - - uint32_t flags(DatabaseUpdate::CREATE_IF_NONEXISTING - | (resetTrusted ? DatabaseUpdate::RESET_TRUSTED : 0)); - - operation_context().update_bucket_database(makeDocumentBucket(document::BucketId(16, 1)), - changedNodes, - flags); - } - - std::string retVal = dumpBucket(document::BucketId(16, 1)); - getBucketDatabase().clear(); - return retVal; - } - - size_t explicit_node_state_reply_send_invocations() const noexcept { - return _node->getNodeStateUpdater().explicit_node_state_reply_send_invocations(); - } - - StatusReporterDelegate& distributor_status_delegate() { - // TODO STRIPE - return _distributor->_stripe->_distributorStatusDelegate; - } - - framework::TickingThreadPool& distributor_thread_pool() { - return _distributor->_threadPool; - } - - const std::vector<std::shared_ptr<DistributorStatus>>& distributor_status_todos() { - // TODO STRIPE - return _distributor->_stripe->_statusToDo; - } - - TopLevelDistributor::MetricUpdateHook distributor_metric_update_hook() { - return _distributor->_metricUpdateHook; - } - - SimpleMaintenanceScanner::PendingMaintenanceStats distributor_maintenance_stats() { - return _distributor->pending_maintenance_stats(); - } - - BucketSpacesStatsProvider::PerNodeBucketSpacesStats distributor_bucket_spaces_stats() { - return _distributor->getBucketSpacesStats(); - } - - DistributorHostInfoReporter& distributor_host_info_reporter() { - return _distributor->_hostInfoReporter; - } - - bool distributor_handle_message(const std::shared_ptr<api::StorageMessage>& msg) { - return _distributor->handleMessage(msg); - } - - uint64_t db_sample_interval_sec() const noexcept { - return std::chrono::duration_cast<std::chrono::seconds>(_distributor->db_memory_sample_interval()).count(); - } - - void configure_stale_reads_enabled(bool enabled) { - ConfigBuilder builder; - builder.allowStaleReadsDuringClusterStateTransitions = enabled; - configureDistributor(builder); - } - - void configure_update_fast_path_restart_enabled(bool enabled) { - ConfigBuilder builder; - builder.restartWithFastUpdatePathIfAllGetTimestampsAreConsistent = enabled; - configureDistributor(builder); - } - - void configure_merge_operations_disabled(bool disabled) { - ConfigBuilder builder; - builder.mergeOperationsDisabled = disabled; - configureDistributor(builder); - } - - void configure_use_weak_internal_read_consistency(bool use_weak) { - ConfigBuilder builder; - builder.useWeakInternalReadConsistencyForClientGets = use_weak; - configureDistributor(builder); - } - - void configure_metadata_update_phase_enabled(bool enabled) { - ConfigBuilder builder; - builder.enableMetadataOnlyFetchPhaseForInconsistentUpdates = enabled; - configureDistributor(builder); - } - - void configure_prioritize_global_bucket_merges(bool enabled) { - ConfigBuilder builder; - builder.prioritizeGlobalBucketMerges = enabled; - configureDistributor(builder); - } - - void configure_max_activation_inhibited_out_of_sync_groups(uint32_t n_groups) { - ConfigBuilder builder; - builder.maxActivationInhibitedOutOfSyncGroups = n_groups; - configureDistributor(builder); - } - - void configureMaxClusterClockSkew(int seconds); - void replyToSingleRequestBucketInfoCommandWith1Bucket(); - void sendDownDummyRemoveCommand(); - void assertSingleBouncedRemoveReplyPresent(); - void assertNoMessageBounced(); - void configure_mutation_sequencing(bool enabled); - void configure_merge_busy_inhibit_duration(int seconds); - void do_test_pending_merge_getnodestate_reply_edge(BucketSpace space); - - void set_up_and_start_get_op_with_stale_reads_enabled(bool enabled); -}; - -LegacyDistributorTest::LegacyDistributorTest() - : Test(), - DistributorTestUtil(), - _bucketSpaces() -{ -} - -LegacyDistributorTest::~LegacyDistributorTest() = default; - -// Migrated to DistributorStripeTest -TEST_F(LegacyDistributorTest, operation_generation) { - setupDistributor(Redundancy(1), NodeCount(1), "storage:1 distributor:1"); - - document::BucketId bid; - addNodesToBucketDB(document::BucketId(16, 1), "0=1/1/1/t"); - - EXPECT_EQ("Remove", testOp(std::make_shared<api::RemoveCommand>( - makeDocumentBucket(bid), - document::DocumentId("id:m:test:n=1:foo"), - api::Timestamp(1234)))); - - auto cmd = std::make_shared<api::CreateVisitorCommand>(makeBucketSpace(), "foo", "bar", ""); - cmd->addBucketToBeVisited(document::BucketId(16, 1)); - cmd->addBucketToBeVisited(document::BucketId()); - - EXPECT_EQ("Visitor Create", testOp(cmd)); -} - -// Migrated to DistributorStripeTest -TEST_F(LegacyDistributorTest, operations_generated_and_started_without_duplicates) { - setupDistributor(Redundancy(1), NodeCount(1), "storage:1 distributor:1"); - - for (uint32_t i = 0; i < 6; ++i) { - addNodesToBucketDB(document::BucketId(16, i), "0=1"); - } - - tickDistributorNTimes(20); - - ASSERT_FALSE(tick()); - - ASSERT_EQ(6, _sender.commands().size()); -} - - -// Migrated to DistributorStripeTest -// Migrated to TopLevelDistributorTest -TEST_F(LegacyDistributorTest, recovery_mode_on_cluster_state_change) { - setupDistributor(Redundancy(1), NodeCount(2), - "storage:1 .0.s:d distributor:1"); - enableDistributorClusterState("storage:1 distributor:1"); - - EXPECT_TRUE(distributor_is_in_recovery_mode()); - for (uint32_t i = 0; i < 3; ++i) { - addNodesToBucketDB(document::BucketId(16, i), "0=1"); - } - for (int i = 0; i < 3; ++i) { - tick(); - EXPECT_TRUE(distributor_is_in_recovery_mode()); - } - tick(); - EXPECT_FALSE(distributor_is_in_recovery_mode()); - - enableDistributorClusterState("storage:2 distributor:1"); - EXPECT_TRUE(distributor_is_in_recovery_mode()); -} - -TEST_F(LegacyDistributorTest, distributor_considered_initialized_once_self_observed_up) { - setupDistributor(Redundancy(1), NodeCount(2), "distributor:1 .0.s:d storage:1"); // We're down D: - EXPECT_FALSE(_distributor->done_initializing()); - enableDistributorClusterState("distributor:1 storage:1"); // We're up :D - EXPECT_TRUE(_distributor->done_initializing()); - enableDistributorClusterState("distributor:1 .0.s:d storage:1"); // And down again :I but that does not change init state - EXPECT_TRUE(_distributor->done_initializing()); -} - -// Migrated to DistributorStripeTest -TEST_F(LegacyDistributorTest, operations_are_throttled) { - setupDistributor(Redundancy(1), NodeCount(1), "storage:1 distributor:1"); - getConfig().setMinPendingMaintenanceOps(1); - getConfig().setMaxPendingMaintenanceOps(1); - - for (uint32_t i = 0; i < 6; ++i) { - addNodesToBucketDB(document::BucketId(16, i), "0=1"); - } - tickDistributorNTimes(20); - ASSERT_EQ(1, _sender.commands().size()); -} - -// Migrated to DistributorStripeTest -TEST_F(LegacyDistributorTest, handle_unknown_maintenance_reply) { - setupDistributor(Redundancy(1), NodeCount(1), "storage:1 distributor:1"); - - { - auto cmd = std::make_shared<api::SplitBucketCommand>(makeDocumentBucket(document::BucketId(16, 1234))); - auto reply = std::make_shared<api::SplitBucketReply>(*cmd); - ASSERT_TRUE(_distributor->handleReply(reply)); - } - - { - // RemoveLocationReply must be treated as a maintenance reply since - // it's what GC is currently built around. - auto cmd = std::make_shared<api::RemoveLocationCommand>( - "false", makeDocumentBucket(document::BucketId(30, 1234))); - auto reply = std::shared_ptr<api::StorageReply>(cmd->makeReply()); - ASSERT_TRUE(_distributor->handleReply(reply)); - } -} - -// Migrated to TopLevelDistributorTest -TEST_F(LegacyDistributorTest, contains_time_statement) { - setupDistributor(Redundancy(1), NodeCount(1), "storage:1 distributor:1"); - - EXPECT_FALSE(getConfig().containsTimeStatement("")); - EXPECT_FALSE(getConfig().containsTimeStatement("testdoctype1")); - EXPECT_FALSE(getConfig().containsTimeStatement("testdoctype1.headerfield > 42")); - EXPECT_TRUE(getConfig().containsTimeStatement("testdoctype1.headerfield > now()")); - EXPECT_TRUE(getConfig().containsTimeStatement("testdoctype1.headerfield > now() - 3600")); - EXPECT_TRUE(getConfig().containsTimeStatement("testdoctype1.headerfield == now() - 3600")); -} - -// Migrated to DistributorStripeTest -TEST_F(LegacyDistributorTest, update_bucket_database) { - enableDistributorClusterState("distributor:1 storage:3"); - - EXPECT_EQ("BucketId(0x4000000000000001) : " - "node(idx=0,crc=0x1c8,docs=228/228,bytes=114/114,trusted=true,active=false,ready=false), " - "node(idx=1,crc=0x1c8,docs=228/228,bytes=114/114,trusted=true,active=false,ready=false)", - updateBucketDB("0:456,1:456,2:789", "2:r")); - - EXPECT_EQ("BucketId(0x4000000000000001) : " - "node(idx=0,crc=0x1c8,docs=228/228,bytes=114/114,trusted=true,active=false,ready=false), " - "node(idx=2,crc=0x1c8,docs=228/228,bytes=114/114,trusted=true,active=false,ready=false), " - "node(idx=1,crc=0x1c8,docs=228/228,bytes=114/114,trusted=true,active=false,ready=false)", - updateBucketDB("0:456,1:456", "2:456")); - - EXPECT_EQ("BucketId(0x4000000000000001) : " - "node(idx=0,crc=0x315,docs=394/394,bytes=197/197,trusted=false,active=false,ready=false), " - "node(idx=2,crc=0x14d,docs=166/166,bytes=83/83,trusted=false,active=false,ready=false), " - "node(idx=1,crc=0x34a,docs=421/421,bytes=210/210,trusted=false,active=false,ready=false)", - updateBucketDB("0:456:t,1:456:t,2:123", "0:789,1:842,2:333")); - - EXPECT_EQ("BucketId(0x4000000000000001) : " - "node(idx=0,crc=0x315,docs=394/394,bytes=197/197,trusted=true,active=false,ready=false), " - "node(idx=2,crc=0x14d,docs=166/166,bytes=83/83,trusted=false,active=false,ready=false), " - "node(idx=1,crc=0x315,docs=394/394,bytes=197/197,trusted=true,active=false,ready=false)", - updateBucketDB("0:456:t,1:456:t,2:123", "0:789,1:789,2:333")); - - EXPECT_EQ("BucketId(0x4000000000000001) : " - "node(idx=2,crc=0x14d,docs=166/166,bytes=83/83,trusted=true,active=false,ready=false)", - updateBucketDB("0:456:t,1:456:t", "0:r,1:r,2:333")); - - // Copies are in sync so should still be trusted even if explicitly reset. - EXPECT_EQ("BucketId(0x4000000000000001) : " - "node(idx=0,crc=0x1c8,docs=228/228,bytes=114/114,trusted=true,active=false,ready=false), " - "node(idx=2,crc=0x1c8,docs=228/228,bytes=114/114,trusted=true,active=false,ready=false), " - "node(idx=1,crc=0x1c8,docs=228/228,bytes=114/114,trusted=true,active=false,ready=false)", - updateBucketDB("0:456,1:456", "2:456", ResetTrusted(true))); - - // When resetting, first inserted copy should not end up as implicitly trusted. - EXPECT_EQ("BucketId(0x4000000000000001) : " - "node(idx=0,crc=0x1c8,docs=228/228,bytes=114/114,trusted=false,active=false,ready=false), " - "node(idx=2,crc=0x14d,docs=166/166,bytes=83/83,trusted=false,active=false,ready=false)", - updateBucketDB("0:456", "2:333", ResetTrusted(true))); -} - -namespace { - -using namespace framework::defaultimplementation; - -class StatusRequestThread : public framework::Runnable { - StatusReporterDelegate& _reporter; - std::string _result; -public: - explicit StatusRequestThread(StatusReporterDelegate& reporter) - : _reporter(reporter) - {} - void run(framework::ThreadHandle&) override { - framework::HttpUrlPath path("/distributor?page=buckets"); - std::ostringstream stream; - _reporter.reportStatus(stream, path); - _result = stream.str(); - } - - std::string getResult() const { - return _result; - } -}; - -} - -// Migrated to TopLevelDistributorTest -TEST_F(LegacyDistributorTest, tick_processes_status_requests) { - setupDistributor(Redundancy(1), NodeCount(1), "storage:1 distributor:1"); - - addNodesToBucketDB(document::BucketId(16, 1), "0=1/1/1/t"); - - // Must go via delegate since reportStatus is now just a rendering - // function and not a request enqueuer (see Distributor::handleStatusRequest). - StatusRequestThread thread(distributor_status_delegate()); - FakeClock clock; - ThreadPoolImpl pool(clock); - int ticksBeforeWait = 1; - framework::Thread::UP tp(pool.startThread(thread, "statustest", 5ms, 5s, ticksBeforeWait)); - - while (true) { - std::this_thread::sleep_for(1ms); - framework::TickingLockGuard guard(distributor_thread_pool().freezeCriticalTicks()); - if (!distributor_status_todos().empty()) { - break; - } - - } - ASSERT_TRUE(tick()); - - tp->interruptAndJoin(); - - EXPECT_THAT(thread.getResult(), HasSubstr("BucketId(0x4000000000000001)")); -} - -// Migrated to TopLevelDistributorTest -TEST_F(LegacyDistributorTest, metric_update_hook_updates_pending_maintenance_metrics) { - setupDistributor(Redundancy(2), NodeCount(2), "storage:2 distributor:1"); - // To ensure we count all operations, not just those fitting within the - // pending window. - getConfig().setMinPendingMaintenanceOps(1); - getConfig().setMaxPendingMaintenanceOps(1); - - // 1 bucket must be merged, 1 must be split, 1 should be activated. - addNodesToBucketDB(document::BucketId(16, 1), "0=2/2/2/t/a,1=1/1/1"); - addNodesToBucketDB(document::BucketId(16, 2), - "0=100/10000000/200000/t/a,1=100/10000000/200000/t"); - addNodesToBucketDB(document::BucketId(16, 3), - "0=200/300/400/t,1=200/300/400/t"); - - // Go many full scanner rounds to check that metrics are set, not - // added to existing. - tickDistributorNTimes(50); - - // By this point, no hook has been called so the metrics have not been - // set. - using MO = MaintenanceOperation; - { - const IdealStateMetricSet& metrics(getIdealStateManager().getMetrics()); - EXPECT_EQ(0, metrics.operations[MO::MERGE_BUCKET]->pending.getLast()); - EXPECT_EQ(0, metrics.operations[MO::SPLIT_BUCKET]->pending.getLast()); - EXPECT_EQ(0, metrics.operations[MO::SET_BUCKET_STATE]->pending.getLast()); - EXPECT_EQ(0, metrics.operations[MO::DELETE_BUCKET]->pending.getLast()); - EXPECT_EQ(0, metrics.operations[MO::JOIN_BUCKET]->pending.getLast()); - EXPECT_EQ(0, metrics.operations[MO::GARBAGE_COLLECTION]->pending.getLast()); - } - - // Force trigger update hook - std::mutex l; - distributor_metric_update_hook().updateMetrics(metrics::MetricLockGuard(l)); - // Metrics should now be updated to the last complete working state - { - const IdealStateMetricSet& metrics(getIdealStateManager().getMetrics()); - EXPECT_EQ(1, metrics.operations[MO::MERGE_BUCKET]->pending.getLast()); - EXPECT_EQ(1, metrics.operations[MO::SPLIT_BUCKET]->pending.getLast()); - EXPECT_EQ(1, metrics.operations[MO::SET_BUCKET_STATE]->pending.getLast()); - EXPECT_EQ(0, metrics.operations[MO::DELETE_BUCKET]->pending.getLast()); - EXPECT_EQ(0, metrics.operations[MO::JOIN_BUCKET]->pending.getLast()); - EXPECT_EQ(0, metrics.operations[MO::GARBAGE_COLLECTION]->pending.getLast()); - } -} - -// Migrated to TopLevelDistributorTest -TEST_F(LegacyDistributorTest, bucket_db_memory_usage_metrics_only_updated_at_fixed_time_intervals) { - getClock().setAbsoluteTimeInSeconds(1000); - - setupDistributor(Redundancy(2), NodeCount(2), "storage:2 distributor:1"); - addNodesToBucketDB(document::BucketId(16, 1), "0=1/1/1/t/a,1=2/2/2"); - tickDistributorNTimes(10); - - std::mutex l; - distributor_metric_update_hook().updateMetrics(metrics::MetricLockGuard(l)); - auto* m = getDistributor().getMetrics().mutable_dbs.memory_usage.getMetric("used_bytes"); - ASSERT_TRUE(m != nullptr); - auto last_used = m->getLongValue("last"); - EXPECT_GT(last_used, 0); - - // Add another bucket to the DB. This should increase the underlying used number of - // bytes, but this should not be aggregated into the metrics until the sampling time - // interval has passed. Instead, old metric gauge values should be preserved. - addNodesToBucketDB(document::BucketId(16, 2), "0=1/1/1/t/a,1=2/2/2"); - - const auto sample_interval_sec = db_sample_interval_sec(); - getClock().setAbsoluteTimeInSeconds(1000 + sample_interval_sec - 1); // Not there yet. - tickDistributorNTimes(50); - distributor_metric_update_hook().updateMetrics(metrics::MetricLockGuard(l)); - - m = getDistributor().getMetrics().mutable_dbs.memory_usage.getMetric("used_bytes"); - auto now_used = m->getLongValue("last"); - EXPECT_EQ(now_used, last_used); - - getClock().setAbsoluteTimeInSeconds(1000 + sample_interval_sec + 1); - tickDistributorNTimes(10); - distributor_metric_update_hook().updateMetrics(metrics::MetricLockGuard(l)); - - m = getDistributor().getMetrics().mutable_dbs.memory_usage.getMetric("used_bytes"); - now_used = m->getLongValue("last"); - EXPECT_GT(now_used, last_used); -} - -// Migrated to DistributorStripeTest -TEST_F(LegacyDistributorTest, priority_config_is_propagated_to_distributor_configuration) { - using namespace vespa::config::content::core; - - setupDistributor(Redundancy(2), NodeCount(2), "storage:2 distributor:1"); - - ConfigBuilder builder; - builder.priorityMergeMoveToIdealNode = 1; - builder.priorityMergeOutOfSyncCopies = 2; - builder.priorityMergeTooFewCopies = 3; - builder.priorityActivateNoExistingActive = 4; - builder.priorityActivateWithExistingActive = 5; - builder.priorityDeleteBucketCopy = 6; - builder.priorityJoinBuckets = 7; - builder.prioritySplitDistributionBits = 8; - builder.prioritySplitLargeBucket = 9; - builder.prioritySplitInconsistentBucket = 10; - builder.priorityGarbageCollection = 11; - builder.priorityMergeGlobalBuckets = 12; - - getConfig().configure(builder); - - const auto& mp = getConfig().getMaintenancePriorities(); - EXPECT_EQ(1, static_cast<int>(mp.mergeMoveToIdealNode)); - EXPECT_EQ(2, static_cast<int>(mp.mergeOutOfSyncCopies)); - EXPECT_EQ(3, static_cast<int>(mp.mergeTooFewCopies)); - EXPECT_EQ(4, static_cast<int>(mp.activateNoExistingActive)); - EXPECT_EQ(5, static_cast<int>(mp.activateWithExistingActive)); - EXPECT_EQ(6, static_cast<int>(mp.deleteBucketCopy)); - EXPECT_EQ(7, static_cast<int>(mp.joinBuckets)); - EXPECT_EQ(8, static_cast<int>(mp.splitDistributionBits)); - EXPECT_EQ(9, static_cast<int>(mp.splitLargeBucket)); - EXPECT_EQ(10, static_cast<int>(mp.splitInconsistentBucket)); - EXPECT_EQ(11, static_cast<int>(mp.garbageCollection)); - EXPECT_EQ(12, static_cast<int>(mp.mergeGlobalBuckets)); -} - -// Migrated to DistributorStripeTest -// Explicit cluster state edge test added in TopLevelDistributorTest::cluster_state_lifecycle_is_propagated_to_stripes -TEST_F(LegacyDistributorTest, no_db_resurrection_for_bucket_not_owned_in_pending_state) { - setupDistributor(Redundancy(1), NodeCount(10), "storage:2 distributor:2"); - lib::ClusterState newState("storage:10 distributor:10"); - auto stateCmd = std::make_shared<api::SetSystemStateCommand>(newState); - // Force newState into being the pending state. According to the initial - // state we own the bucket, but according to the pending state, we do - // not. This must be handled correctly by the database update code. - getBucketDBUpdater().onSetSystemState(stateCmd); - - document::BucketId nonOwnedBucket(16, 3); - EXPECT_FALSE(getDistributorBucketSpace().get_bucket_ownership_flags(nonOwnedBucket).owned_in_pending_state()); - EXPECT_FALSE(getDistributorBucketSpace().check_ownership_in_pending_and_current_state(nonOwnedBucket).isOwned()); - - std::vector<BucketCopy> copies; - copies.emplace_back(1234, 0, api::BucketInfo(0x567, 1, 2)); - operation_context().update_bucket_database(makeDocumentBucket(nonOwnedBucket), copies, - DatabaseUpdate::CREATE_IF_NONEXISTING); - - EXPECT_EQ("NONEXISTING", dumpBucket(nonOwnedBucket)); -} - -// Migrated to DistributorStripeTest -TEST_F(LegacyDistributorTest, added_db_buckets_without_gc_timestamp_implicitly_get_current_time) { - setupDistributor(Redundancy(1), NodeCount(10), "storage:2 distributor:2"); - getClock().setAbsoluteTimeInSeconds(101234); - document::BucketId bucket(16, 7654); - - std::vector<BucketCopy> copies; - copies.emplace_back(1234, 0, api::BucketInfo(0x567, 1, 2)); - operation_context().update_bucket_database(makeDocumentBucket(bucket), copies, - DatabaseUpdate::CREATE_IF_NONEXISTING); - BucketDatabase::Entry e(getBucket(bucket)); - EXPECT_EQ(101234, e->getLastGarbageCollectionTime()); -} - -// Migrated to DistributorStripeTest -TEST_F(LegacyDistributorTest, merge_stats_are_accumulated_during_database_iteration) { - setupDistributor(Redundancy(2), NodeCount(3), "storage:3 distributor:1"); - // Copies out of sync. Not possible for distributor to _reliably_ tell - // which direction(s) data will flow, so for simplicity assume that we - // must sync both copies. - // Note that we mark certain copies as active to prevent the bucketstate - // checker from pre-empting the merges. - // -> syncing[0] += 1, syncing[2] += 1 - addNodesToBucketDB(document::BucketId(16, 1), "0=1/1/1/t/a,2=2/2/2"); - // Must add missing node 2 for bucket - // -> copyingOut[0] += 1, copyingIn[2] += 1 - addNodesToBucketDB(document::BucketId(16, 2), "0=1/1/1/t/a"); - // Moving from non-ideal node 1 to ideal node 2. Both nodes 0 and 1 will - // be involved in this merge, but only node 1 will be tagged as source only - // (i.e. to be deleted after the merge is completed). - // -> copyingOut[0] += 1, movingOut[1] += 1, copyingIn[2] += 1 - addNodesToBucketDB(document::BucketId(16, 3), "0=2/2/2/t/a,1=2/2/2/t"); - - // Go many full scanner rounds to check that stats are set, not - // added to existing. - tickDistributorNTimes(50); - - const auto& stats = distributor_maintenance_stats(); - { - NodeMaintenanceStats wanted; - wanted.syncing = 1; - wanted.copyingOut = 2; - wanted.total = 3; - EXPECT_EQ(wanted, stats.perNodeStats.forNode(0, makeBucketSpace())); - } - { - NodeMaintenanceStats wanted; - wanted.movingOut = 1; - wanted.total = 1; - EXPECT_EQ(wanted, stats.perNodeStats.forNode(1, makeBucketSpace())); - } - { - NodeMaintenanceStats wanted; - wanted.syncing = 1; - wanted.copyingIn = 2; - wanted.total = 1; - EXPECT_EQ(wanted, stats.perNodeStats.forNode(2, makeBucketSpace())); - } - auto bucketStats = distributor_bucket_spaces_stats(); - ASSERT_EQ(3, bucketStats.size()); - assertBucketSpaceStats(1, 3, 0, "default", bucketStats); - assertBucketSpaceStats(0, 1, 1, "default", bucketStats); - assertBucketSpaceStats(3, 1, 2, "default", bucketStats); -} - -void -LegacyDistributorTest::assertBucketSpaceStats(size_t expBucketPending, size_t expBucketTotal, uint16_t node, - const vespalib::string& bucketSpace, - const BucketSpacesStatsProvider::PerNodeBucketSpacesStats& stats) -{ - auto nodeItr = stats.find(node); - ASSERT_TRUE(nodeItr != stats.end()); - ASSERT_EQ(1, nodeItr->second.size()); - auto bucketSpaceItr = nodeItr->second.find(bucketSpace); - ASSERT_TRUE(bucketSpaceItr != nodeItr->second.end()); - ASSERT_TRUE(bucketSpaceItr->second.valid()); - ASSERT_EQ(expBucketTotal, bucketSpaceItr->second.bucketsTotal()); - ASSERT_EQ(expBucketPending, bucketSpaceItr->second.bucketsPending()); -} - -/** - * Since maintenance operations are prioritized differently, activation - * pre-empts merging and other ops. If this also implies pre-empting running - * their state checkers at all, we won't get any statistics from any other - * operations for the bucket. - */ -// Migrated to DistributorStripeTest -TEST_F(LegacyDistributorTest, stats_generated_for_preempted_operations) { - setupDistributor(Redundancy(2), NodeCount(2), "storage:2 distributor:1"); - // For this test it suffices to have a single bucket with multiple aspects - // wrong about it. In this case, let a bucket be both out of sync _and_ - // missing an active copy. This _should_ give a statistic with both nodes 0 - // and 1 requiring a sync. If instead merge stats generation is preempted - // by activation, we'll see no merge stats at all. - addNodesToBucketDB(document::BucketId(16, 1), "0=1/1/1,1=2/2/2"); - tickDistributorNTimes(50); - const auto& stats = distributor_maintenance_stats(); - { - NodeMaintenanceStats wanted; - wanted.syncing = 1; - wanted.total = 1; - EXPECT_EQ(wanted, stats.perNodeStats.forNode(0, makeBucketSpace())); - } - { - NodeMaintenanceStats wanted; - wanted.syncing = 1; - wanted.total = 1; - EXPECT_EQ(wanted, stats.perNodeStats.forNode(1, makeBucketSpace())); - } -} - -// Migrated to TopLevelDistributorTest -TEST_F(LegacyDistributorTest, host_info_reporter_config_is_propagated_to_reporter) { - setupDistributor(Redundancy(2), NodeCount(2), "storage:2 distributor:1"); - - // Default is enabled=true. - EXPECT_TRUE(distributor_host_info_reporter().isReportingEnabled()); - - ConfigBuilder builder; - builder.enableHostInfoReporting = false; - configureDistributor(builder); - - EXPECT_FALSE(distributor_host_info_reporter().isReportingEnabled()); -} - -// Migrated to DistributorStripeTest -TEST_F(LegacyDistributorTest, replica_counting_mode_is_configured_to_trusted_by_default) { - setupDistributor(Redundancy(2), NodeCount(2), "storage:2 distributor:1"); - EXPECT_EQ(ConfigBuilder::MinimumReplicaCountingMode::TRUSTED, currentReplicaCountingMode()); -} - -// Migrated to DistributorStripeTest -TEST_F(LegacyDistributorTest, replica_counting_mode_config_is_propagated_to_metric_updater) { - setupDistributor(Redundancy(2), NodeCount(2), "storage:2 distributor:1"); - ConfigBuilder builder; - builder.minimumReplicaCountingMode = ConfigBuilder::MinimumReplicaCountingMode::ANY; - configureDistributor(builder); - EXPECT_EQ(ConfigBuilder::MinimumReplicaCountingMode::ANY, currentReplicaCountingMode()); -} - -// Migrated to DistributorStripeTest -TEST_F(LegacyDistributorTest, max_consecutively_inhibited_maintenance_ticks_config_is_propagated_to_internal_config) { - setupDistributor(Redundancy(2), NodeCount(2), "storage:2 distributor:1"); - ConfigBuilder builder; - builder.maxConsecutivelyInhibitedMaintenanceTicks = 123; - getConfig().configure(builder); - EXPECT_EQ(getConfig().max_consecutively_inhibited_maintenance_ticks(), 123); -} - -// Migrated to DistributorStripeTest -TEST_F(LegacyDistributorTest, bucket_activation_is_enabled_by_default) { - setupDistributor(Redundancy(2), NodeCount(2), "storage:2 distributor:1"); - EXPECT_FALSE(getConfig().isBucketActivationDisabled()); -} - -// Migrated to DistributorStripeTest -TEST_F(LegacyDistributorTest, bucket_activation_config_is_propagated_to_distributor_configuration) { - using namespace vespa::config::content::core; - - setupDistributor(Redundancy(2), NodeCount(2), "storage:2 distributor:1"); - - ConfigBuilder builder; - builder.disableBucketActivation = true; - getConfig().configure(builder); - - EXPECT_TRUE(getConfig().isBucketActivationDisabled()); -} - -void -LegacyDistributorTest::configureMaxClusterClockSkew(int seconds) { - using namespace vespa::config::content::core; - - ConfigBuilder builder; - builder.maxClusterClockSkewSec = seconds; - getConfig().configure(builder); - _distributor->enable_next_config_if_changed(); -} - -// Migrated to DistributorStripeTest -TEST_F(LegacyDistributorTest, max_clock_skew_config_is_propagated_to_distributor_config) { - setupDistributor(Redundancy(2), NodeCount(2), "storage:2 distributor:1"); - - configureMaxClusterClockSkew(5); - EXPECT_EQ(getConfig().getMaxClusterClockSkew(), std::chrono::seconds(5)); -} - -namespace { - -auto makeDummyRemoveCommand() { - return std::make_shared<api::RemoveCommand>( - makeDocumentBucket(document::BucketId(0)), - document::DocumentId("id:foo:testdoctype1:n=1:foo"), - api::Timestamp(0)); -} - -auto make_dummy_get_command_for_bucket_1() { - return std::make_shared<api::GetCommand>( - makeDocumentBucket(document::BucketId(0)), - document::DocumentId("id:foo:testdoctype1:n=1:foo"), - document::AllFields::NAME); -} - -} - -void LegacyDistributorTest::replyToSingleRequestBucketInfoCommandWith1Bucket() { - ASSERT_EQ(_bucketSpaces.size(), _sender.commands().size()); - for (uint32_t i = 0; i < _sender.commands().size(); ++i) { - ASSERT_EQ(api::MessageType::REQUESTBUCKETINFO, _sender.command(i)->getType()); - auto& bucketReq(static_cast<api::RequestBucketInfoCommand&> - (*_sender.command(i))); - auto bucketReply = bucketReq.makeReply(); - if (bucketReq.getBucketSpace() == FixedBucketSpaces::default_space()) { - // Make sure we have a bucket to route our remove op to, or we'd get - // an immediate reply anyway. - dynamic_cast<api::RequestBucketInfoReply&>(*bucketReply) - .getBucketInfo().push_back( - api::RequestBucketInfoReply::Entry(document::BucketId(1, 1), - api::BucketInfo(20, 10, 12, 50, 60, true, true))); - } - _distributor->handleMessage(std::move(bucketReply)); - } - _sender.commands().clear(); -} - -void LegacyDistributorTest::sendDownDummyRemoveCommand() { - _distributor->handleMessage(makeDummyRemoveCommand()); -} - -void LegacyDistributorTest::assertSingleBouncedRemoveReplyPresent() { - ASSERT_EQ(1, _sender.replies().size()); // Rejected remove - ASSERT_EQ(api::MessageType::REMOVE_REPLY, _sender.reply(0)->getType()); - auto& reply(static_cast<api::RemoveReply&>(*_sender.reply(0))); - ASSERT_EQ(api::ReturnCode::STALE_TIMESTAMP, reply.getResult().getResult()); - _sender.replies().clear(); -} - -void LegacyDistributorTest::assertNoMessageBounced() { - ASSERT_EQ(0, _sender.replies().size()); -} - -// Migrated to TopLevelDistributorTest -TEST_F(LegacyDistributorTest, configured_safe_time_point_rejection_works_end_to_end) { - setupDistributor(Redundancy(2), NodeCount(2), - "bits:1 storage:1 distributor:2"); - getClock().setAbsoluteTimeInSeconds(1000); - configureMaxClusterClockSkew(10); - - receive_set_system_state_command("bits:1 storage:1 distributor:1"); - ASSERT_NO_FATAL_FAILURE(replyToSingleRequestBucketInfoCommandWith1Bucket()); - // SetSystemStateCommand sent down chain at this point. - sendDownDummyRemoveCommand(); - ASSERT_NO_FATAL_FAILURE(assertSingleBouncedRemoveReplyPresent()); - - // Increment time to first whole second of clock + 10 seconds of skew. - // Should now not get any feed rejections. - getClock().setAbsoluteTimeInSeconds(1011); - - sendDownDummyRemoveCommand(); - ASSERT_NO_FATAL_FAILURE(assertNoMessageBounced()); -} - -void LegacyDistributorTest::configure_mutation_sequencing(bool enabled) { - using namespace vespa::config::content::core; - - ConfigBuilder builder; - builder.sequenceMutatingOperations = enabled; - getConfig().configure(builder); - _distributor->enable_next_config_if_changed(); -} - -// Migrated to DistributorStripeTest -TEST_F(LegacyDistributorTest, sequencing_config_is_propagated_to_distributor_config) { - setupDistributor(Redundancy(2), NodeCount(2), "storage:2 distributor:1"); - - // Should be enabled by default - EXPECT_TRUE(getConfig().getSequenceMutatingOperations()); - - // Explicitly disabled. - configure_mutation_sequencing(false); - EXPECT_FALSE(getConfig().getSequenceMutatingOperations()); - - // Explicitly enabled. - configure_mutation_sequencing(true); - EXPECT_TRUE(getConfig().getSequenceMutatingOperations()); -} - -void -LegacyDistributorTest::configure_merge_busy_inhibit_duration(int seconds) { - using namespace vespa::config::content::core; - - ConfigBuilder builder; - builder.inhibitMergeSendingOnBusyNodeDurationSec = seconds; - getConfig().configure(builder); - _distributor->enable_next_config_if_changed(); -} - -// Migrated to DistributorStripeTest -TEST_F(LegacyDistributorTest, merge_busy_inhibit_duration_config_is_propagated_to_distributor_config) { - setupDistributor(Redundancy(2), NodeCount(2), "storage:2 distributor:1"); - - configure_merge_busy_inhibit_duration(7); - EXPECT_EQ(getConfig().getInhibitMergesOnBusyNodeDuration(), std::chrono::seconds(7)); -} - -// Migrated to DistributorStripeTest -TEST_F(LegacyDistributorTest, merge_busy_inhibit_duration_is_propagated_to_pending_message_tracker) { - setupDistributor(Redundancy(2), NodeCount(2), "storage:1 distributor:1"); - addNodesToBucketDB(document::BucketId(16, 1), "0=1/1/1/t"); - - configure_merge_busy_inhibit_duration(100); - auto cmd = makeDummyRemoveCommand(); // Remove is for bucket 1 - distributor_handle_message(cmd); - - // Should send to content node 0 - ASSERT_EQ(1, _sender.commands().size()); - ASSERT_EQ(api::MessageType::REMOVE, _sender.command(0)->getType()); - auto& fwd_cmd = dynamic_cast<api::RemoveCommand&>(*_sender.command(0)); - auto reply = fwd_cmd.makeReply(); - reply->setResult(api::ReturnCode(api::ReturnCode::BUSY)); - _distributor->handleReply(std::shared_ptr<api::StorageReply>(std::move(reply))); - - auto& node_info = pending_message_tracker().getNodeInfo(); - - EXPECT_TRUE(node_info.isBusy(0)); - getClock().addSecondsToTime(99); - EXPECT_TRUE(node_info.isBusy(0)); - getClock().addSecondsToTime(2); - EXPECT_FALSE(node_info.isBusy(0)); -} - -// Migrated to DistributorStripeTest -TEST_F(LegacyDistributorTest, external_client_requests_are_handled_individually_in_priority_order) { - setupDistributor(Redundancy(1), NodeCount(1), "storage:1 distributor:1"); - addNodesToBucketDB(document::BucketId(16, 1), "0=1/1/1/t/a"); - - std::vector<api::StorageMessage::Priority> priorities({50, 255, 10, 40, 0}); - document::DocumentId id("id:foo:testdoctype1:n=1:foo"); - vespalib::stringref field_set = ""; - for (auto pri : priorities) { - auto cmd = std::make_shared<api::GetCommand>(makeDocumentBucket(document::BucketId()), id, field_set); - cmd->setPriority(pri); - // onDown appends to internal message FIFO queue, awaiting hand-off. - _distributor->onDown(cmd); - } - // At the hand-off point we expect client requests to be prioritized. - // For each tick, a priority-order client request is processed and sent off. - for (size_t i = 1; i <= priorities.size(); ++i) { - tickDistributorNTimes(1); - ASSERT_EQ(i, _sender.commands().size()); - } - - std::vector<int> expected({0, 10, 40, 50, 255}); - std::vector<int> actual; - for (auto& msg : _sender.commands()) { - actual.emplace_back(static_cast<int>(msg->getPriority())); - } - EXPECT_THAT(actual, ContainerEq(expected)); -} - -// Migrated to DistributorStripeTest -TEST_F(LegacyDistributorTest, internal_messages_are_started_in_fifo_order_batch) { - // To test internal request ordering, we use NotifyBucketChangeCommand - // for the reason that it explicitly updates the bucket database for - // each individual invocation. - setupDistributor(Redundancy(1), NodeCount(1), "storage:1 distributor:1"); - document::BucketId bucket(16, 1); - addNodesToBucketDB(bucket, "0=1/1/1/t"); - - std::vector<api::StorageMessage::Priority> priorities({50, 255, 10, 40, 1}); - for (auto pri : priorities) { - api::BucketInfo fake_info(pri, pri, pri); - auto cmd = std::make_shared<api::NotifyBucketChangeCommand>(makeDocumentBucket(bucket), fake_info); - cmd->setSourceIndex(0); - cmd->setPriority(pri); - _distributor->onDown(cmd); - } - - // Doing a single tick should process all internal requests in one batch - tickDistributorNTimes(1); - ASSERT_EQ(5, _sender.replies().size()); - - // The bucket info for priority 1 (last FIFO-order change command received, but - // highest priority) should be the end-state of the bucket database, _not_ that - // of lowest priority 255. - BucketDatabase::Entry e(getBucket(bucket)); - EXPECT_EQ(api::BucketInfo(1, 1, 1), e.getBucketInfo().getNode(0)->getBucketInfo()); -} - -// Migrated to DistributorStripeTest -TEST_F(LegacyDistributorTest, closing_aborts_priority_queued_client_requests) { - setupDistributor(Redundancy(1), NodeCount(1), "storage:1 distributor:1"); - document::BucketId bucket(16, 1); - addNodesToBucketDB(bucket, "0=1/1/1/t"); - - document::DocumentId id("id:foo:testdoctype1:n=1:foo"); - vespalib::stringref field_set = ""; - for (int i = 0; i < 10; ++i) { - auto cmd = std::make_shared<api::GetCommand>(makeDocumentBucket(document::BucketId()), id, field_set); - _distributor->onDown(cmd); - } - tickDistributorNTimes(1); - // Closing should trigger 1 abort via startet GetOperation and 9 aborts from pri queue - _distributor->close(); - ASSERT_EQ(10, _sender.replies().size()); - for (auto& msg : _sender.replies()) { - EXPECT_EQ(api::ReturnCode::ABORTED, dynamic_cast<api::StorageReply&>(*msg).getResult().getResult()); - } -} - -namespace { - -void assert_invalid_stats_for_all_spaces( - const BucketSpacesStatsProvider::PerNodeBucketSpacesStats& stats, - uint16_t node_index) { - auto stats_iter = stats.find(node_index); - ASSERT_TRUE(stats_iter != stats.cend()); - ASSERT_EQ(2, stats_iter->second.size()); - auto space_iter = stats_iter->second.find(document::FixedBucketSpaces::default_space_name()); - ASSERT_TRUE(space_iter != stats_iter->second.cend()); - ASSERT_FALSE(space_iter->second.valid()); - space_iter = stats_iter->second.find(document::FixedBucketSpaces::global_space_name()); - ASSERT_TRUE(space_iter != stats_iter->second.cend()); - ASSERT_FALSE(space_iter->second.valid()); -} - -} - -// Migrated to DistributorStripeTest -// Cross-stripe bucket stats test added in TopLevelDistributorTest::entering_recovery_mode_resets_bucket_space_stats_across_all_stripes -TEST_F(LegacyDistributorTest, entering_recovery_mode_resets_bucket_space_stats) { - // Set up a cluster state + DB contents which implies merge maintenance ops - setupDistributor(Redundancy(2), NodeCount(2), "version:1 distributor:1 storage:2"); - addNodesToBucketDB(document::BucketId(16, 1), "0=1/1/1/t/a"); - addNodesToBucketDB(document::BucketId(16, 2), "0=1/1/1/t/a"); - addNodesToBucketDB(document::BucketId(16, 3), "0=2/2/2/t/a"); - - tickDistributorNTimes(5); // 1/3rds into second round through database - - enableDistributorClusterState("version:2 distributor:1 storage:3 .1.s:d"); - EXPECT_TRUE(distributor_is_in_recovery_mode()); - // Bucket space stats should now be invalid per space per node, pending stats - // from state version 2. Exposing stats from version 1 risks reporting stale - // information back to the cluster controller. - const auto stats = distributor_bucket_spaces_stats(); - ASSERT_EQ(2, stats.size()); - - assert_invalid_stats_for_all_spaces(stats, 0); - assert_invalid_stats_for_all_spaces(stats, 2); -} - -// Migrated to TopLevelDistributorTest -TEST_F(LegacyDistributorTest, leaving_recovery_mode_immediately_sends_getnodestate_replies) { - setupDistributor(Redundancy(2), NodeCount(2), "version:1 distributor:1 storage:2"); - // Should not send explicit replies during init stage - ASSERT_EQ(0, explicit_node_state_reply_send_invocations()); - // Add a couple of buckets so we have something to iterate over - addNodesToBucketDB(document::BucketId(16, 1), "0=1/1/1/t/a"); - addNodesToBucketDB(document::BucketId(16, 2), "0=1/1/1/t/a"); - - enableDistributorClusterState("version:2 distributor:1 storage:3 .1.s:d"); - EXPECT_TRUE(distributor_is_in_recovery_mode()); - EXPECT_EQ(0, explicit_node_state_reply_send_invocations()); - tickDistributorNTimes(1); // DB round not yet complete - EXPECT_EQ(0, explicit_node_state_reply_send_invocations()); - tickDistributorNTimes(2); // DB round complete after 2nd bucket + "scan done" discovery tick - EXPECT_EQ(1, explicit_node_state_reply_send_invocations()); - EXPECT_FALSE(distributor_is_in_recovery_mode()); - // Now out of recovery mode, subsequent round completions should not send replies - tickDistributorNTimes(10); - EXPECT_EQ(1, explicit_node_state_reply_send_invocations()); -} - -void LegacyDistributorTest::do_test_pending_merge_getnodestate_reply_edge(BucketSpace space) { - setupDistributor(Redundancy(2), NodeCount(2), "version:1 distributor:1 storage:2"); - EXPECT_TRUE(distributor_is_in_recovery_mode()); - // 2 buckets with missing replicas triggering merge pending stats - addNodesToBucketDB(Bucket(space, BucketId(16, 1)), "0=1/1/1/t/a"); - addNodesToBucketDB(Bucket(space, BucketId(16, 2)), "0=1/1/1/t/a"); - tickDistributorNTimes(3); - EXPECT_FALSE(distributor_is_in_recovery_mode()); - const auto space_name = FixedBucketSpaces::to_string(space); - assertBucketSpaceStats(2, 0, 1, space_name, _distributor->getBucketSpacesStats()); - // First completed scan sends off merge stats et al to cluster controller - EXPECT_EQ(1, explicit_node_state_reply_send_invocations()); - - // Edge not triggered when 1 bucket with missing replica left - addNodesToBucketDB(Bucket(space, BucketId(16, 1)), "0=1/1/1/t/a,1=1/1/1/t"); - tickDistributorNTimes(3); - assertBucketSpaceStats(1, 1, 1, space_name, _distributor->getBucketSpacesStats()); - EXPECT_EQ(1, explicit_node_state_reply_send_invocations()); - - // Edge triggered when no more buckets with requiring merge - addNodesToBucketDB(Bucket(space, BucketId(16, 2)), "0=1/1/1/t/a,1=1/1/1/t"); - tickDistributorNTimes(3); - assertBucketSpaceStats(0, 2, 1, space_name, _distributor->getBucketSpacesStats()); - EXPECT_EQ(2, explicit_node_state_reply_send_invocations()); - - // Should only send when edge happens, not in subsequent DB iterations - tickDistributorNTimes(10); - EXPECT_EQ(2, explicit_node_state_reply_send_invocations()); - - // Going back to merges pending should _not_ send a getnodestate reply (at least for now) - addNodesToBucketDB(Bucket(space, BucketId(16, 1)), "0=1/1/1/t/a"); - tickDistributorNTimes(3); - assertBucketSpaceStats(1, 1, 1, space_name, _distributor->getBucketSpacesStats()); - EXPECT_EQ(2, explicit_node_state_reply_send_invocations()); -} - -// TODO: rewrite into DistributorStripeTest -TEST_F(LegacyDistributorTest, pending_to_no_pending_default_merges_edge_immediately_sends_getnodestate_replies) { - do_test_pending_merge_getnodestate_reply_edge(FixedBucketSpaces::default_space()); -} - -// TODO: rewrite into DistributorStripeTest -TEST_F(LegacyDistributorTest, pending_to_no_pending_global_merges_edge_immediately_sends_getnodestate_replies) { - do_test_pending_merge_getnodestate_reply_edge(FixedBucketSpaces::global_space()); -} - -// Migrated to DistributorStripeTest -TEST_F(LegacyDistributorTest, stale_reads_config_is_propagated_to_external_operation_handler) { - createLinks(); - setupDistributor(Redundancy(1), NodeCount(1), "distributor:1 storage:1"); - - configure_stale_reads_enabled(true); - EXPECT_TRUE(getExternalOperationHandler().concurrent_gets_enabled()); - - configure_stale_reads_enabled(false); - EXPECT_FALSE(getExternalOperationHandler().concurrent_gets_enabled()); -} - -// Migrated to DistributorStripeTest -TEST_F(LegacyDistributorTest, fast_path_on_consistent_gets_config_is_propagated_to_internal_config) { - createLinks(); - setupDistributor(Redundancy(1), NodeCount(1), "distributor:1 storage:1"); - - configure_update_fast_path_restart_enabled(true); - EXPECT_TRUE(getConfig().update_fast_path_restart_enabled()); - - configure_update_fast_path_restart_enabled(false); - EXPECT_FALSE(getConfig().update_fast_path_restart_enabled()); -} - -// Migrated to DistributorStripeTest -TEST_F(LegacyDistributorTest, merge_disabling_config_is_propagated_to_internal_config) { - createLinks(); - setupDistributor(Redundancy(1), NodeCount(1), "distributor:1 storage:1"); - - configure_merge_operations_disabled(true); - EXPECT_TRUE(getConfig().merge_operations_disabled()); - - configure_merge_operations_disabled(false); - EXPECT_FALSE(getConfig().merge_operations_disabled()); -} - -// Migrated to DistributorStripeTest -TEST_F(LegacyDistributorTest, metadata_update_phase_config_is_propagated_to_internal_config) { - createLinks(); - setupDistributor(Redundancy(1), NodeCount(1), "distributor:1 storage:1"); - - configure_metadata_update_phase_enabled(true); - EXPECT_TRUE(getConfig().enable_metadata_only_fetch_phase_for_inconsistent_updates()); - - configure_metadata_update_phase_enabled(false); - EXPECT_FALSE(getConfig().enable_metadata_only_fetch_phase_for_inconsistent_updates()); -} - -// Migrated to DistributorStripeTest -TEST_F(LegacyDistributorTest, weak_internal_read_consistency_config_is_propagated_to_internal_configs) { - createLinks(); - setupDistributor(Redundancy(1), NodeCount(1), "distributor:1 storage:1"); - - configure_use_weak_internal_read_consistency(true); - EXPECT_TRUE(getConfig().use_weak_internal_read_consistency_for_client_gets()); - EXPECT_TRUE(getExternalOperationHandler().use_weak_internal_read_consistency_for_gets()); - - configure_use_weak_internal_read_consistency(false); - EXPECT_FALSE(getConfig().use_weak_internal_read_consistency_for_client_gets()); - EXPECT_FALSE(getExternalOperationHandler().use_weak_internal_read_consistency_for_gets()); -} - -void LegacyDistributorTest::set_up_and_start_get_op_with_stale_reads_enabled(bool enabled) { - createLinks(); - setupDistributor(Redundancy(1), NodeCount(1), "distributor:1 storage:1"); - configure_stale_reads_enabled(enabled); - - document::BucketId bucket(16, 1); - addNodesToBucketDB(bucket, "0=1/1/1/t"); - _distributor->onDown(make_dummy_get_command_for_bucket_1()); -} - -// Migrated to DistributorStripeTest -TEST_F(LegacyDistributorTest, gets_are_started_outside_main_distributor_logic_if_stale_reads_enabled) { - set_up_and_start_get_op_with_stale_reads_enabled(true); - ASSERT_THAT(_sender.commands(), SizeIs(1)); - EXPECT_THAT(_sender.replies(), SizeIs(0)); - - // Reply is routed to the correct owner - auto reply = std::shared_ptr<api::StorageReply>(_sender.command(0)->makeReply()); - _distributor->onDown(reply); - ASSERT_THAT(_sender.commands(), SizeIs(1)); - EXPECT_THAT(_sender.replies(), SizeIs(1)); -} - -// Migrated to DistributorStripeTest -TEST_F(LegacyDistributorTest, gets_are_not_started_outside_main_distributor_logic_if_stale_reads_disabled) { - set_up_and_start_get_op_with_stale_reads_enabled(false); - // Get has been placed into distributor queue, so no external messages are produced. - EXPECT_THAT(_sender.commands(), SizeIs(0)); - EXPECT_THAT(_sender.replies(), SizeIs(0)); -} - -// Migrated to DistributorStripeTest -// There's no need or desire to track "lockfree" Gets in the main pending message tracker, -// as we only have to track mutations to inhibit maintenance ops safely. Furthermore, -// the message tracker is a multi-index and therefore has some runtime cost. -TEST_F(LegacyDistributorTest, gets_started_outside_main_thread_are_not_tracked_by_main_pending_message_tracker) { - set_up_and_start_get_op_with_stale_reads_enabled(true); - Bucket bucket(FixedBucketSpaces::default_space(), BucketId(16, 1)); - EXPECT_FALSE(pending_message_tracker().hasPendingMessage( - 0, bucket, api::MessageType::GET_ID)); -} - -// Migrated to DistributorStripeTest -TEST_F(LegacyDistributorTest, closing_aborts_gets_started_outside_main_distributor_thread) { - set_up_and_start_get_op_with_stale_reads_enabled(true); - _distributor->close(); - ASSERT_EQ(1, _sender.replies().size()); - EXPECT_EQ(api::ReturnCode::ABORTED, _sender.reply(0)->getResult().getResult()); -} - -// Migrated to DistributorStripeTest -TEST_F(LegacyDistributorTest, prioritize_global_bucket_merges_config_is_propagated_to_internal_config) { - createLinks(); - setupDistributor(Redundancy(1), NodeCount(1), "distributor:1 storage:1"); - - configure_prioritize_global_bucket_merges(true); - EXPECT_TRUE(getConfig().prioritize_global_bucket_merges()); - - configure_prioritize_global_bucket_merges(false); - EXPECT_FALSE(getConfig().prioritize_global_bucket_merges()); -} - -// Migrated to DistributorStripeTest -TEST_F(LegacyDistributorTest, max_activation_inhibited_out_of_sync_groups_config_is_propagated_to_internal_config) { - createLinks(); - setupDistributor(Redundancy(1), NodeCount(1), "distributor:1 storage:1"); - - configure_max_activation_inhibited_out_of_sync_groups(3); - EXPECT_EQ(getConfig().max_activation_inhibited_out_of_sync_groups(), 3); - - configure_max_activation_inhibited_out_of_sync_groups(0); - EXPECT_EQ(getConfig().max_activation_inhibited_out_of_sync_groups(), 0); -} - -// Migrated to DistributorStripeTest -TEST_F(LegacyDistributorTest, wanted_split_bit_count_is_lower_bounded) { - createLinks(); - setupDistributor(Redundancy(1), NodeCount(1), "distributor:1 storage:1"); - - ConfigBuilder builder; - builder.minsplitcount = 7; - configureDistributor(builder); - - EXPECT_EQ(getConfig().getMinimalBucketSplit(), 8); -} - -// Migrated to TopLevelDistributorTest -TEST_F(LegacyDistributorTest, host_info_sent_immediately_once_all_stripes_first_reported) { - set_num_distributor_stripes(4); - createLinks(); - getClock().setAbsoluteTimeInSeconds(1000); - // TODO STRIPE can't call this currently since it touches the bucket DB updater directly: - // setupDistributor(Redundancy(2), NodeCount(2), "version:1 distributor:1 storage:2"); - - tickDistributorNTimes(1); - EXPECT_EQ(0, explicit_node_state_reply_send_invocations()); // Nothing yet - getDistributor().notify_stripe_wants_to_send_host_info(1); - getDistributor().notify_stripe_wants_to_send_host_info(2); - getDistributor().notify_stripe_wants_to_send_host_info(3); - - tickDistributorNTimes(1); - // Still nothing. Missing initial report from stripe 0 - EXPECT_EQ(0, explicit_node_state_reply_send_invocations()); - - getDistributor().notify_stripe_wants_to_send_host_info(0); - tickDistributorNTimes(1); - // All stripes have reported in, it's time to party! - EXPECT_EQ(1, explicit_node_state_reply_send_invocations()); - - // No further sends if stripes haven't requested it yet. - getClock().setAbsoluteTimeInSeconds(2000); - tickDistributorNTimes(10); - EXPECT_EQ(1, explicit_node_state_reply_send_invocations()); -} - -// Migrated to TopLevelDistributorTest -TEST_F(LegacyDistributorTest, non_bootstrap_host_info_send_request_delays_sending) { - set_num_distributor_stripes(4); - createLinks(); - getClock().setAbsoluteTimeInSeconds(1000); - - for (uint16_t i = 0; i < 4; ++i) { - getDistributor().notify_stripe_wants_to_send_host_info(i); - } - tickDistributorNTimes(1); - // Bootstrap case - EXPECT_EQ(1, explicit_node_state_reply_send_invocations()); - - // Stripe 1 suddenly really wants to tell the cluster controller something again - getDistributor().notify_stripe_wants_to_send_host_info(1); - tickDistributorNTimes(1); - // But its cry for attention is not yet honored since the delay hasn't passed. - EXPECT_EQ(1, explicit_node_state_reply_send_invocations()); - - getClock().addMilliSecondsToTime(999); - tickDistributorNTimes(1); - // 1 sec delay has still not passed - EXPECT_EQ(1, explicit_node_state_reply_send_invocations()); - - getClock().addMilliSecondsToTime(1); - tickDistributorNTimes(1); - // But now it has - EXPECT_EQ(2, explicit_node_state_reply_send_invocations()); -} - -} diff --git a/storage/src/tests/distributor/statusreporterdelegatetest.cpp b/storage/src/tests/distributor/statusreporterdelegatetest.cpp index 9e66f1920e2..3cac901619e 100644 --- a/storage/src/tests/distributor/statusreporterdelegatetest.cpp +++ b/storage/src/tests/distributor/statusreporterdelegatetest.cpp @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <tests/common/testhelper.h> -#include <tests/distributor/distributortestutil.h> +#include <tests/common/teststorageapp.h> #include <vespa/storage/distributor/statusreporterdelegate.h> #include <vespa/vespalib/gtest/gtest.h> diff --git a/storage/src/tests/distributor/top_level_bucket_db_updater_test.cpp b/storage/src/tests/distributor/top_level_bucket_db_updater_test.cpp index 01f7d5a4f0a..d57ce228908 100644 --- a/storage/src/tests/distributor/top_level_bucket_db_updater_test.cpp +++ b/storage/src/tests/distributor/top_level_bucket_db_updater_test.cpp @@ -630,7 +630,7 @@ TopLevelBucketDBUpdaterTest::trigger_completed_but_not_yet_activated_transition( } TEST_F(TopLevelBucketDBUpdaterTest, normal_usage) { - set_cluster_state(lib::ClusterState("distributor:2 .0.s:i .1.s:i storage:3")); + set_cluster_state(lib::ClusterState("distributor:2 .0.s:i .1.s:i storage:3")); // FIXME init mode why? ASSERT_EQ(message_count(3), _sender.commands().size()); @@ -638,21 +638,21 @@ TEST_F(TopLevelBucketDBUpdaterTest, normal_usage) { ASSERT_EQ(_component->getDistribution()->getNodeGraph().getDistributionConfigHash(), dynamic_cast<const RequestBucketInfoCommand&>(*_sender.command(0)).getDistributionHash()); - ASSERT_NO_FATAL_FAILURE(fake_bucket_reply(lib::ClusterState("distributor:2 .0.s:i .1.s:i storage:3"), + ASSERT_NO_FATAL_FAILURE(fake_bucket_reply(lib::ClusterState("distributor:2 .0.s:i .1.s:i storage:3"), // FIXME init mode why? *_sender.command(0), 10)); _sender.clear(); // Optimization for not refetching unneeded data after cluster state // change is only implemented after completion of previous cluster state - set_cluster_state("distributor:2 .0.s:i storage:3"); + set_cluster_state("distributor:2 .0.s:i storage:3"); // FIXME init mode why? ASSERT_EQ(message_count(3), _sender.commands().size()); // Expect reply of first set SystemState request. ASSERT_EQ(size_t(1), _sender.replies().size()); ASSERT_NO_FATAL_FAILURE(complete_bucket_info_gathering( - lib::ClusterState("distributor:2 .0.s:i .1.s:i storage:3"), + lib::ClusterState("distributor:2 .0.s:i .1.s:i storage:3"), // FIXME init mode why? message_count(3), 10)); ASSERT_NO_FATAL_FAILURE(assert_correct_buckets(10, "distributor:2 storage:3")); } @@ -661,9 +661,9 @@ TEST_F(TopLevelBucketDBUpdaterTest, distributor_change) { int num_buckets = 100; // First sends request - set_cluster_state("distributor:2 .0.s:i .1.s:i storage:3"); + set_cluster_state("distributor:2 .0.s:i .1.s:i storage:3"); // FIXME init mode why? ASSERT_EQ(message_count(3), _sender.commands().size()); - ASSERT_NO_FATAL_FAILURE(complete_bucket_info_gathering(lib::ClusterState("distributor:2 .0.s:i .1.s:i storage:3"), + ASSERT_NO_FATAL_FAILURE(complete_bucket_info_gathering(lib::ClusterState("distributor:2 .0.s:i .1.s:i storage:3"), // FIXME init mode why? message_count(3), num_buckets)); _sender.clear(); @@ -718,14 +718,14 @@ TEST_F(TopLevelBucketDBUpdaterTest, distributor_change_with_grouping) { } TEST_F(TopLevelBucketDBUpdaterTest, normal_usage_initializing) { - set_cluster_state("distributor:1 .0.s:i storage:1 .0.s:i"); + set_cluster_state("distributor:1 .0.s:i storage:1 .0.s:i"); // FIXME init mode why? ASSERT_EQ(_bucket_spaces.size(), _sender.commands().size()); // Not yet passing on system state. ASSERT_EQ(size_t(0), _sender_down.commands().size()); - ASSERT_NO_FATAL_FAILURE(complete_bucket_info_gathering(lib::ClusterState("distributor:1 .0.s:i storage:1"), + ASSERT_NO_FATAL_FAILURE(complete_bucket_info_gathering(lib::ClusterState("distributor:1 .0.s:i storage:1"), // FIXME init mode why? _bucket_spaces.size(), 10, 10)); ASSERT_NO_FATAL_FAILURE(assert_correct_buckets(10, "distributor:1 storage:1")); @@ -740,12 +740,12 @@ TEST_F(TopLevelBucketDBUpdaterTest, normal_usage_initializing) { _sender.clear(); _sender_down.clear(); - set_cluster_state("distributor:1 .0.s:i storage:1"); + set_cluster_state("distributor:1 .0.s:i storage:1"); // FIXME init mode why? // Send a new request bucket info up. ASSERT_EQ(_bucket_spaces.size(), _sender.commands().size()); - ASSERT_NO_FATAL_FAILURE(complete_bucket_info_gathering(lib::ClusterState("distributor:1 .0.s:i storage:1"), + ASSERT_NO_FATAL_FAILURE(complete_bucket_info_gathering(lib::ClusterState("distributor:1 .0.s:i storage:1"), // FIXME init mode why? _bucket_spaces.size(), 20)); // Pass on cluster state and recheck buckets now. @@ -755,7 +755,7 @@ TEST_F(TopLevelBucketDBUpdaterTest, normal_usage_initializing) { } TEST_F(TopLevelBucketDBUpdaterTest, failed_request_bucket_info) { - set_cluster_state("distributor:1 .0.s:i storage:1"); + set_cluster_state("distributor:1 .0.s:i storage:1"); // FIXME init mode why? // 2 messages sent up: 1 to the nodes, and one reply to the setsystemstate. ASSERT_EQ(_bucket_spaces.size(), _sender.commands().size()); @@ -781,7 +781,7 @@ TEST_F(TopLevelBucketDBUpdaterTest, failed_request_bucket_info) { ASSERT_EQ(size_t(0), _sender_down.commands().size()); for (uint32_t i = 0; i < _bucket_spaces.size(); ++i) { - ASSERT_NO_FATAL_FAILURE(fake_bucket_reply(lib::ClusterState("distributor:1 .0.s:i storage:1"), + ASSERT_NO_FATAL_FAILURE(fake_bucket_reply(lib::ClusterState("distributor:1 .0.s:i storage:1"), // FIXME init mode why? *_sender.command(_bucket_spaces.size() + i), 10)); } diff --git a/storage/src/tests/distributor/top_level_distributor_test_util.cpp b/storage/src/tests/distributor/top_level_distributor_test_util.cpp index b6e9beb38ae..e4002b3d1cb 100644 --- a/storage/src/tests/distributor/top_level_distributor_test_util.cpp +++ b/storage/src/tests/distributor/top_level_distributor_test_util.cpp @@ -303,11 +303,6 @@ TopLevelDistributorTestUtil::total_distributor_metrics() const return *_distributor->_total_metrics; } -const storage::distributor::DistributorNodeContext& -TopLevelDistributorTestUtil::node_context() const { - return _distributor->distributor_component(); -} - DistributorBucketSpace& TopLevelDistributorTestUtil::distributor_bucket_space(const document::BucketId& id) { diff --git a/storage/src/tests/distributor/top_level_distributor_test_util.h b/storage/src/tests/distributor/top_level_distributor_test_util.h index 8832f8ada6e..e8794eb4199 100644 --- a/storage/src/tests/distributor/top_level_distributor_test_util.h +++ b/storage/src/tests/distributor/top_level_distributor_test_util.h @@ -69,7 +69,6 @@ public: TopLevelBucketDBUpdater& bucket_db_updater(); const IdealStateMetricSet& total_ideal_state_metrics() const; const DistributorMetricSet& total_distributor_metrics() const; - const storage::distributor::DistributorNodeContext& node_context() const; DistributorBucketSpace& distributor_bucket_space(const document::BucketId& id); const DistributorBucketSpace& distributor_bucket_space(const document::BucketId& id) const; |